diff --git a/core/lib/Drupal/Core/Entity/Plugin/Validation/Constraint/ValidReferenceConstraintValidator.php b/core/lib/Drupal/Core/Entity/Plugin/Validation/Constraint/ValidReferenceConstraintValidator.php
index eab3f4e72b4b0c1a9758f1d1832a9c5ba892bdcc..f5699e737de182b2f00e5074527105baa42ee0f7 100644
--- a/core/lib/Drupal/Core/Entity/Plugin/Validation/Constraint/ValidReferenceConstraintValidator.php
+++ b/core/lib/Drupal/Core/Entity/Plugin/Validation/Constraint/ValidReferenceConstraintValidator.php
@@ -121,17 +121,17 @@ public function validate($value, Constraint $constraint): void {
 
     // Add violations on deltas with a target_id that is not valid.
     if ($target_ids) {
-      // Get a list of pre-existing references.
-      $previously_referenced_ids = [];
-      if ($entity && !$entity->isNew()) {
-        $existing_entity = $this->entityTypeManager->getStorage($entity->getEntityTypeId())->loadUnchanged($entity->id());
-        foreach ($existing_entity->{$value->getFieldDefinition()->getName()}->getValue() as $item) {
-          $previously_referenced_ids[$item['target_id']] = $item['target_id'];
-        }
-      }
-
       $valid_target_ids = $handler->validateReferenceableEntities($target_ids);
       if ($invalid_target_ids = array_diff($target_ids, $valid_target_ids)) {
+        // Get a list of pre-existing references.
+        $previously_referenced_ids = [];
+        if ($entity && !$entity->isNew()) {
+          $existing_entity = $this->entityTypeManager->getStorage($entity->getEntityTypeId())->loadUnchanged($entity->id());
+          foreach ($existing_entity->{$value->getFieldDefinition()->getName()}->getValue() as $item) {
+            $previously_referenced_ids[$item['target_id']] = $item['target_id'];
+          }
+        }
+
         // For accuracy of the error message, differentiate non-referenceable
         // and non-existent entities.
         $existing_entities = $this->entityTypeManager->getStorage($target_type_id)->loadMultiple($invalid_target_ids);
diff --git a/core/tests/Drupal/KernelTests/Core/Entity/ValidReferenceConstraintValidatorTest.php b/core/tests/Drupal/KernelTests/Core/Entity/ValidReferenceConstraintValidatorTest.php
index 7ab8514026cf47fcd30a1a466448e87b70385c86..ed24f9067f88e1c6aef97f9f4db7eb8dcf38e595 100644
--- a/core/tests/Drupal/KernelTests/Core/Entity/ValidReferenceConstraintValidatorTest.php
+++ b/core/tests/Drupal/KernelTests/Core/Entity/ValidReferenceConstraintValidatorTest.php
@@ -175,9 +175,16 @@ public function testPreExistingItemsValidation(): void {
 
     $this->container->get('account_switcher')->switchTo($user_with_access);
 
+    // We check if the referencing entity was loaded by checking
+    // if it was added to the memory cache.
+    // This requires an empty memory cache for the test to be reliable.
+    $this->container->get('entity.memory_cache')->reset();
     $violations = $referencing_entity->field_test->validate();
     $this->assertCount(0, $violations);
 
+    // No invalid target was found, so the referencing entity was not reloaded.
+    $this->assertFalse($this->container->get('entity.memory_cache')->get('values:' . $referencing_entity->getEntityTypeId() . ':' . $referencing_entity->id()));
+
     // Check that users without access are able pass the validation for fields
     // with pre-existing content.
     $this->container->get('account_switcher')->switchTo($user_without_access);
@@ -204,10 +211,14 @@ public function testPreExistingItemsValidation(): void {
     $field->save();
     $referencing_entity = $this->reloadEntity($referencing_entity);
 
+    $this->container->get('entity.memory_cache')->reset();
     $violations = $referencing_entity->field_test->validate();
     $this->assertCount(1, $violations);
     $this->assertEquals(sprintf('This entity (node: %s) cannot be referenced.', $different_bundle_node->id()), $violations[0]->getMessage());
 
+    // An invalid target was found, so the referencing entity had to be reloaded.
+    $this->assertNotFalse($this->container->get('entity.memory_cache')->get('values:' . $referencing_entity->getEntityTypeId() . ':' . $referencing_entity->id()));
+
     // Delete the last node and check that the pre-existing reference is not
     // valid anymore.
     $deleted_node->delete();