Unverified Commit a0378efd authored by Alex Pott's avatar Alex Pott
Browse files

Issue #1237636 by catch, berdir, godotislate: Lazy load multiple entities at a time using fibers

parent 3ce5c201

core/lib/Drupal/Core/Entity/EntityStorageBase.php

+50 −14
Original line number Diff line number Diff line
@@ -89,6 +89,11 @@ abstract class EntityStorageBase extends EntityHandlerBase implements EntityStor 
   */

  protected $memoryCacheTag;



  /**

   * Entity IDs awaiting loading.

   */

  protected array $entityIdsToLoad = [];



  /**

   * Constructs an EntityStorageBase instance.

   *
@@ -177,13 +182,11 @@ public function resetCache(?array $ids = NULL) { 
  protected function getFromStaticCache(array $ids) {

    $entities = [];

    // Load any available entities from the internal cache.

    if ($this->entityType->isStaticallyCacheable()) {

    foreach ($ids as $id) {

      if ($cached = $this->memoryCache->get($this->buildCacheId($id))) {

        $entities[$id] = $cached->data;

      }

    }

    }

    return $entities;

  }
@@ -279,10 +282,36 @@ public function loadMultiple(?array $ids = NULL) { 
    $flipped_ids = $ids ? array_flip($ids) : FALSE;

    // Try to load entities from the static cache, if the entity type supports

    // static caching.

    if ($ids) {

    if ($ids && $this->entityType->isStaticallyCacheable()) {

      $entities += $this->getFromStaticCache($ids);

      // If any entities were loaded, remove them from the IDs still to load.

      $ids = array_keys(array_diff_key($flipped_ids, $entities));

      // If any entities were in the static cache remove them from the

      // remaining IDs.

      $ids = array_diff($ids, array_keys($entities));



      $fiber = \Fiber::getCurrent();

      if ($ids && $fiber !== NULL) {

        // Before suspending the fiber, add the IDs passed in to the full list

        // of entities to load, so that another call can load everything at

        // once.

        $this->entityIdsToLoad = array_unique(array_merge($this->entityIdsToLoad, $ids));

        $fiber->suspend();



        // At this point the entityIdsToLoad property will either have been

        // reset within another Fiber, or will contain the IDs passed in as

        // well as any others waiting to be loaded.

        if ($this->entityIdsToLoad) {

          $ids = $this->entityIdsToLoad;



          // Reset the entityIdsToLoad property so that any further calls start

          // with a blank slate (apart from the entity static cache).

          $this->entityIdsToLoad = [];

        }

        $entities += $this->getFromStaticCache($ids);



        // If any entities were in the static cache remove them from the

        // remaining IDs.

        $ids = array_diff($ids, array_keys($entities));

      }

    }



    // Try to gather any remaining entities from a 'preload' method. This method
@@ -300,7 +329,7 @@ public function loadMultiple(?array $ids = NULL) { 


      // If any entities were pre-loaded, remove them from the IDs still to

      // load.

      $ids = array_keys(array_diff_key($flipped_ids, $entities));

      $ids = array_diff($ids, array_keys($entities));



      // Add pre-loaded entities to the cache.

      $this->setStaticCache($preloaded_entities);
@@ -324,12 +353,19 @@ public function loadMultiple(?array $ids = NULL) { 
      $this->setStaticCache($queried_entities);

    }



    // Ensure that the returned array is ordered the same as the original

    // $ids array if this was passed in and remove any invalid IDs.

    if ($flipped_ids) {

      // Remove any invalid IDs from the array and preserve the order passed in.

      $flipped_ids = array_intersect_key($flipped_ids, $entities);

      $entities = array_replace($flipped_ids, $entities);

      // When IDs were passed in, ensure only entities that were loaded by this

      // specific method call (e.g. not for other Fibers) are returned, and that

      // any entities that could not be loaded are removed.

      foreach ($flipped_ids as $entity_id => $value) {

        if (isset($entities[$entity_id])) {

          $flipped_ids[$entity_id] = $entities[$entity_id];

        }

        else {

          unset($flipped_ids[$entity_id]);

        }

      }

      $entities = $flipped_ids;

    }



    return $entities;

core/profiles/demo_umami/tests/src/FunctionalJavascript/OpenTelemetryFrontPagePerformanceTest.php

+6 −6
Original line number Diff line number Diff line
@@ -52,9 +52,9 @@ protected function testFrontPageColdCache(): void { 
    $this->assertSession()->pageTextContains('Umami');



    $expected = [

      'QueryCount' => 381,

      'CacheGetCount' => 472,

      'CacheSetCount' => 467,

      'QueryCount' => 364,

      'CacheGetCount' => 466,

      'CacheSetCount' => 461,

      'CacheDeleteCount' => 0,

      'CacheTagLookupQueryCount' => 49,

      'CacheTagInvalidationCount' => 0,
@@ -122,9 +122,9 @@ protected function testFrontPageCoolCache(): void { 
    }, 'umamiFrontPageCoolCache');



    $expected = [

      'QueryCount' => 112,

      'CacheGetCount' => 239,

      'CacheSetCount' => 93,

      'QueryCount' => 103,

      'CacheGetCount' => 236,

      'CacheSetCount' => 90,

      'CacheDeleteCount' => 0,

      'CacheTagInvalidationCount' => 0,

      'CacheTagLookupQueryCount' => 31,

core/tests/Drupal/KernelTests/Core/Entity/EntityApiTest.php

+97 −0
Original line number Diff line number Diff line
@@ -143,6 +143,103 @@ protected function assertCRUD(string $entity_type, UserInterface $user1): void { 
    }

  }



  /**

   * Test lazy preloading.

   */

  public function testLazyPreLoading(): void {

    $storage = $this->container->get('entity_type.manager')->getStorage('entity_test');

    $ids = [];

    $entity = $storage->create(['name' => 'test']);

    $entity->save();

    $ids[] = $entity->id();



    $entity = $storage->create(['name' => 'test2']);

    $entity->save();

    $ids[] = $entity->id();



    $fiber1 = new \Fiber(fn () => $storage->load($ids[0]));

    $fiber2 = new \Fiber(fn () => $storage->load($ids[1]));



    // Make sure the entity cache is empty.

    $this->container->get('entity.memory_cache')->reset();



    // Start Fiber 1, this should set the first entity to be loaded, without

    // actually loading it, and then suspend.

    $fiber1->start();

    $this->assertTrue($fiber1->isSuspended());

    $this->assertFalse($this->container->get('entity.memory_cache')->get('values:entity_test:' . $ids[0]));



    // Start Fiber 2, this should set the first entity to be loaded, without

    // actually loading it, and then suspend.

    $fiber2->start();

    $this->assertTrue($fiber2->isSuspended());

    $this->assertFalse($this->container->get('entity.memory_cache')->get('values:entity_test:' . $ids[1]));



    $fiber2->resume();



    $this->assertTrue($fiber2->isTerminated());



    $this->assertSame($fiber2->getReturn()->id(), $ids[1]);



    // Now both entities should be loaded.

    $this->assertNotFalse($this->container->get('entity.memory_cache')->get('values:entity_test:' . $ids[0]));

    $this->assertNotFalse($this->container->get('entity.memory_cache')->get('values:entity_test:' . $ids[1]));

    $fiber1->resume();

    $this->assertTrue($fiber1->isTerminated());

    $this->assertSame($fiber1->getReturn()->id(), $ids[0]);

  }



  /**

   * Test lazy preloading.

   */

  public function testLazyPreLoadingMultiple(): void {

    $storage = $this->container->get('entity_type.manager')->getStorage('entity_test');

    $ids = [];

    $entity = $storage->create(['name' => 'test']);

    $entity->save();

    $ids[] = $entity->id();



    $entity = $storage->create(['name' => 'test2']);

    $entity->save();

    $ids[] = $entity->id();



    $fiber1 = new \Fiber(fn () => $storage->loadMultiple([$ids[0]]));

    $fiber2 = new \Fiber(fn () => $storage->loadMultiple([$ids[1]]));



    // Make sure the entity cache is empty.

    $this->container->get('entity.memory_cache')->reset();



    // Start Fiber 1, this should set the first entity to be loaded, without

    // actually loading it, and then suspend.

    $fiber1->start();

    $this->assertTrue($fiber1->isSuspended());

    $this->assertFalse($this->container->get('entity.memory_cache')->get('values:entity_test:' . $ids[0]));



    // Start Fiber 2, this should set the first entity to be loaded, without

    // actually loading it, and then suspend.

    $fiber2->start();

    $this->assertTrue($fiber2->isSuspended());

    $this->assertFalse($this->container->get('entity.memory_cache')->get('values:entity_test:' . $ids[1]));



    $fiber2->resume();



    $this->assertTrue($fiber2->isTerminated());



    $return2 = $fiber2->getReturn();



    $this->assertSame($return2[2]->id(), $ids[1]);

    $this->assertSame(\count($return2), 1);



    // Now both entities should be loaded.

    $this->assertNotFalse($this->container->get('entity.memory_cache')->get('values:entity_test:' . $ids[0]));

    $this->assertNotFalse($this->container->get('entity.memory_cache')->get('values:entity_test:' . $ids[1]));

    $fiber1->resume();

    $this->assertTrue($fiber1->isTerminated());

    $return1 = $fiber1->getReturn();

    $this->assertSame($return1[1]->id(), $ids[0]);

    $this->assertSame(\count($return1), 1);

  }



  /**

   * Tests that the Entity storage loads the entities in the correct order.

   *

core/tests/Drupal/Tests/Core/Entity/EntityStorageBaseTest.php

+2 −2
Original line number Diff line number Diff line
@@ -91,10 +91,10 @@ public static function providerLoadMultiple(): \Generator { 


    // Data set for results for all IDs.

    $ids = ['1', '2', '3'];

    yield 'results-for-all-ids' => [$ids, $ids, $ids];

    yield 'results-for-all-ids' => [array_combine($ids, $ids), array_combine($ids, $ids), $ids];



    // Data set for partial results for multiple IDs.

    yield 'partial-results-for-multiple-ids' => [$ids, $ids, array_merge($ids, ['11', '12'])];

    yield 'partial-results-for-multiple-ids' => [array_combine($ids, $ids), array_combine($ids, $ids), array_merge($ids, ['11', '12'])];

  }



  /**