Verified Commit 35c7c095 authored by godotislate's avatar godotislate
Browse files

task: #3608184 Load multiple cardinality fields with a smaller result set 

By: catch
By: alexpott
By: godotislate
By: nicxvan
By: scott_euser
parent 7634c658

core/lib/Drupal/Core/Entity/Sql/SqlContentEntityStorage.php

+81 −38
Original line number Diff line number Diff line
@@ -1333,20 +1333,20 @@ protected function loadFromDedicatedTables(array &$values, $load_from_revision, 
    }



    if ($multiple_cardinality_fields) {

      foreach ($multiple_cardinality_fields as $field_name => $storage_definition) {

        $fields = [$field_name => $storage_definition];

        $this->loadMultipleCardinalityFields($values, $base_query, $base_table, $id_key, $base_id_key, $base_langcode_alias, $load_from_revision, $fields, $definitions, $field_columns, $field_definition_columns, $default_langcodes);

      }

      // Ensure that all of the deltas from all of the multiple cardinality

      // fields are returned in the correct order.

      foreach ($values as &$fields) {

        foreach ($fields as $field_name => &$field_data) {

          if (isset($multiple_cardinality_fields[$field_name])) {

            foreach ($field_data as &$language_data) {

              ksort($language_data);

            }

      // Load several multiple cardinality fields in a single query, using the

      // same chunking as single cardinality fields to stay within the SQL join

      // limit.

      if (count($multiple_cardinality_fields) > static::FIELD_MINIMUM_CHUNK_SIZE) {

        $chunks = array_chunk($multiple_cardinality_fields, static::FIELD_MINIMUM_CHUNK_SIZE, TRUE);

        $last_chunk = array_pop($chunks);

        $last_key = array_key_last($chunks);

        $chunks[$last_key] = array_merge($chunks[$last_key], $last_chunk);

      }

      else {

        $chunks = [$multiple_cardinality_fields];

      }

      foreach ($chunks as $fields) {

        $this->loadMultipleCardinalityFields($values, $id_key, $load_from_revision, $fields, $definitions, $field_columns, $field_definition_columns, $default_langcodes, $ids);

      }

    }

  }
@@ -1536,18 +1536,9 @@ private function loadSingleCardinalityFields( 
   *

   * @param array &$values

   *   The entity values populated so far.

   * @param \Drupal\Core\Database\Query\SelectInterface $base_query

   *   The base database query.

   * @param string $base_table

   *   The base table used in the query.

   * @param string $id_key

   *   The ID key depending on whether regular entities or revisions are being

   *   loaded.

   * @param string $base_id_key

   *   The base ID key depending on whether regular entities or revisions are

   *   being loaded.

   * @param string $base_langcode_alias

   *   The base langcode alias.

   * @param bool $load_from_revision

   *   Whether we're loading from revisions.

   * @param array $multiple_cardinality_fields
@@ -1560,48 +1551,100 @@ private function loadSingleCardinalityFields( 
   *   The field definition columns.

   * @param array $default_langcodes

   *   The default langcodes.

   * @param array $ids

   *   The entity or revision IDs being loaded.

   */

  private function loadMultipleCardinalityFields(

    array &$values,

    SelectInterface $base_query,

    string $base_table,

    string $id_key,

    string $base_id_key,

    string $base_langcode_alias,

    bool $load_from_revision,

    array $multiple_cardinality_fields,

    array $definitions,

    array $field_columns,

    array $field_definition_columns,

    array $default_langcodes,

    array $ids,

  ): void {

    $table_mapping = $this->getTableMapping();

    $query = clone $base_query;

    $delta_keys = [];



    if (count($multiple_cardinality_fields) > 1) {

      // Each field table holds one row per delta, and different fields can have

      // different deltas. Left joining several field tables directly would

      // multiply their rows together (a cartesian product). To avoid this,

      // build a "delta spine": the set of (ID, langcode, delta) combinations

      // that exist across the fields in this chunk. Every field table is then

      // left joined on that delta, so each delta produces one row and the field

      // values line up.

      $query = $delta_query = NULL;

      foreach ($multiple_cardinality_fields as $field_name => $storage_definition) {

        $table = !$load_from_revision ? $table_mapping->getDedicatedDataTableName($storage_definition) : $table_mapping->getDedicatedRevisionTableName($storage_definition);

      // If the entity is translatable, add the langcode to the join and

      // a condition on valid langcodes.

        $select = $this->database->select($table, $table);

        $select->addField($table, $id_key, 'id');

        if ($this->langcodeKey) {

        $query->join($table, $table, "[$table].[$id_key] = [$base_table].[$base_id_key] AND [$table].[langcode] = [$base_table].[$this->langcodeKey] AND [$table].[deleted] = 0");

          $select->addField($table, 'langcode', 'langcode');

        }

        $select->addField($table, 'delta', 'delta');

        $select->condition("[$table].[$id_key]", $ids, 'IN');

        $select->condition("[$table].[deleted]", 0);

        if ($delta_query === NULL) {

          $delta_query = $select;

        }

        else {

        $query->join($table, $table, "[$table].[$id_key] = [$base_table].[$base_id_key] AND [$table].[deleted] = 0");

          $delta_query->union($select);

        }



        if (!$query) {

          $query = $this->database->select($delta_query, 'delta_join');

          $query->addField('delta_join', 'id', 'id');

          if ($this->langcodeKey) {

            $query->addField('delta_join', 'langcode', 'langcode');

          }

          // Sort by the delta provided by the delta spine.

          $query->orderBy('[delta_join].[delta]');

        }



        $table = !$load_from_revision ? $table_mapping->getDedicatedDataTableName($storage_definition) : $table_mapping->getDedicatedRevisionTableName($storage_definition);

        // Left join the field table to the delta spine on ID, langcode and

        // delta.

        if ($this->langcodeKey) {

          $query->leftJoin($table, $table, "[$table].[$id_key] = [delta_join].[id] AND [$table].[langcode] = [delta_join].[langcode] AND [$table].[delta] = [delta_join].[delta] AND [$table].[deleted] = 0");

        }

        else {

          $query->leftJoin($table, $table, "[$table].[$id_key] = [delta_join].[id] AND [$table].[delta] = [delta_join].[delta] AND [$table].[deleted] = 0");

        }

        $query->fields($table, $field_columns[$field_name]);

        $delta_keys[$field_name] = $query->addField($table, 'delta', $field_name . '_delta');

      }

    }

    else {

      // A single field table cannot multiply, so select from it directly.

      $field_name = key($multiple_cardinality_fields);

      $table = !$load_from_revision ? $table_mapping->getDedicatedDataTableName($multiple_cardinality_fields[$field_name]) : $table_mapping->getDedicatedRevisionTableName($multiple_cardinality_fields[$field_name]);



      $query = $this->database->select($table, $table);

      $query->addField($table, $id_key, 'id');

      if ($this->langcodeKey) {

        $query->addField($table, 'langcode', 'langcode');

      }

      $query->condition("[$table].[$id_key]", $ids, 'IN');

      $query->condition("[$table].[deleted]", 0);

      $query->orderBy("[$table].[delta]");



      $query->fields($table, $field_columns[$field_name]);

      $delta_keys[$field_name] = $query->addField($table, 'delta', $field_name . '_delta');

    }



    $results = $query->execute();



    foreach ($results as $row) {

      $row = (array) $row;

      $value_key = $row[$base_id_key];

      $value_key = $row['id'];

      // Field values in default language are stored with

      // LanguageInterface::LANGCODE_DEFAULT as key.

      $langcode = LanguageInterface::LANGCODE_DEFAULT;

      if ($this->langcodeKey && isset($default_langcodes[$value_key]) && $row[$base_langcode_alias] != $default_langcodes[$value_key]) {

        $langcode = $row[$base_langcode_alias];

      if ($this->langcodeKey && isset($default_langcodes[$value_key]) && $row['langcode'] != $default_langcodes[$value_key]) {

        $langcode = $row['langcode'];

      }



      foreach ($multiple_cardinality_fields as $field_name => $storage_definition) {

core/profiles/demo_umami/tests/src/FunctionalJavascript/OpenTelemetryAuthenticatedPerformanceTest.php

+1 −1
Original line number Diff line number Diff line
@@ -130,7 +130,7 @@ protected function doTestNodePageAdministrator(): void { 
    }, 'administratorNodePage');



    $expected = [

      'QueryCount' => 273,

      'QueryCount' => 264,

      'CacheGetCount' => 268,

      'CacheGetCountByBin' => [

        'config' => 60,

core/profiles/demo_umami/tests/src/FunctionalJavascript/OpenTelemetryFrontPagePerformanceTest.php

+2 −2
Original line number Diff line number Diff line
@@ -52,7 +52,7 @@ protected function testFrontPageColdCache(): void { 
    $this->assertSession()->pageTextContains('Umami');



    $expected = [

      'QueryCount' => 184,

      'QueryCount' => 178,

      'CacheGetCount' => 237,

      'CacheSetCount' => 241,

      'CacheDeleteCount' => 0,
@@ -122,7 +122,7 @@ protected function testFrontPageCoolCache(): void { 
    }, 'umamiFrontPageCoolCache');



    $expected = [

      'QueryCount' => 60,

      'QueryCount' => 54,

      'CacheGetCount' => 174,

      'CacheSetCount' => 75,

      'CacheDeleteCount' => 0,

core/profiles/standard/tests/src/FunctionalJavascript/StandardPerformanceTest.php

+4 −4
Original line number Diff line number Diff line
@@ -105,7 +105,7 @@ protected function testAnonymous(): void { 
      'SELECT "config"."name" AS "name" FROM "config" "config" WHERE ("collection" = "") AND ("name" LIKE "node.type.%" ESCAPE ' . "'\\\\'" . ') ORDER BY "collection" ASC, "name" ASC',

      'SELECT "base"."uid" AS "uid", "base"."uuid" AS "uuid", "base"."langcode" AS "langcode" FROM "users" "base" WHERE "base"."uid" IN (0)',

      'SELECT "users_field_data".*, "users_field_data"."langcode" AS "users_field_data__langcode", "user__user_picture"."user_picture_target_id" AS "user_picture_target_id", "user__user_picture"."user_picture_alt" AS "user_picture_alt", "user__user_picture"."user_picture_title" AS "user_picture_title", "user__user_picture"."user_picture_width" AS "user_picture_width", "user__user_picture"."user_picture_height" AS "user_picture_height" FROM "users_field_data" "users_field_data" LEFT OUTER JOIN "user__user_picture" "user__user_picture" ON "user__user_picture"."entity_id" = "users_field_data"."uid" AND "user__user_picture"."langcode" = "users_field_data"."langcode" AND "user__user_picture"."deleted" = 0 WHERE "users_field_data"."uid" IN (0)',

      'SELECT "users_field_data".*, "users_field_data"."langcode" AS "users_field_data__langcode", "user__roles"."roles_target_id" AS "roles_target_id", "user__roles"."delta" AS "roles_delta" FROM "users_field_data" "users_field_data" INNER JOIN "user__roles" "user__roles" ON "user__roles"."entity_id" = "users_field_data"."uid" AND "user__roles"."langcode" = "users_field_data"."langcode" AND "user__roles"."deleted" = 0 WHERE "users_field_data"."uid" IN (0)',

      'SELECT "user__roles"."entity_id" AS "id", "user__roles"."langcode" AS "langcode", "user__roles"."roles_target_id" AS "roles_target_id", "user__roles"."delta" AS "roles_delta" FROM "user__roles" "user__roles" WHERE ("user__roles"."entity_id" IN (0)) AND ("user__roles"."deleted" = 0) ORDER BY "user__roles"."delta" ASC',

      'SELECT "name", "data" FROM "config" WHERE "collection" = "" AND "name" IN ( "core.date_format.medium" )',

      'SELECT "name", "data" FROM "config" WHERE "collection" = "" AND "name" IN ( "core.date_format.long" )',

      'SELECT "name", "route" FROM "router" WHERE "name" IN ( "entity.node.canonical" )',
@@ -253,7 +253,7 @@ protected function testAnonymous(): void { 
      'SELECT "name", "route", "fit" FROM "router" WHERE "pattern_outline" IN ( "/user/2", "/user/%", "/user" ) AND "number_parts" >= 2',

      'SELECT "base"."uid" AS "uid", "base"."uuid" AS "uuid", "base"."langcode" AS "langcode" FROM "users" "base" WHERE "base"."uid" IN (2)',

      'SELECT "users_field_data".*, "users_field_data"."langcode" AS "users_field_data__langcode", "user__user_picture"."user_picture_target_id" AS "user_picture_target_id", "user__user_picture"."user_picture_alt" AS "user_picture_alt", "user__user_picture"."user_picture_title" AS "user_picture_title", "user__user_picture"."user_picture_width" AS "user_picture_width", "user__user_picture"."user_picture_height" AS "user_picture_height" FROM "users_field_data" "users_field_data" LEFT OUTER JOIN "user__user_picture" "user__user_picture" ON "user__user_picture"."entity_id" = "users_field_data"."uid" AND "user__user_picture"."langcode" = "users_field_data"."langcode" AND "user__user_picture"."deleted" = 0 WHERE "users_field_data"."uid" IN (2)',

      'SELECT "users_field_data".*, "users_field_data"."langcode" AS "users_field_data__langcode", "user__roles"."roles_target_id" AS "roles_target_id", "user__roles"."delta" AS "roles_delta" FROM "users_field_data" "users_field_data" INNER JOIN "user__roles" "user__roles" ON "user__roles"."entity_id" = "users_field_data"."uid" AND "user__roles"."langcode" = "users_field_data"."langcode" AND "user__roles"."deleted" = 0 WHERE "users_field_data"."uid" IN (2)',

      'SELECT "user__roles"."entity_id" AS "id", "user__roles"."langcode" AS "langcode", "user__roles"."roles_target_id" AS "roles_target_id", "user__roles"."delta" AS "roles_delta" FROM "user__roles" "user__roles" WHERE ("user__roles"."entity_id" IN (2)) AND ("user__roles"."deleted" = 0) ORDER BY "user__roles"."delta" ASC',

      'SELECT "name", "route" FROM "router" WHERE "name" IN ( "entity.user.canonical" )',

      'SELECT "name", "value" FROM "key_value" WHERE "name" IN ( "theme:stark" ) AND "collection" = "config.entity.key_store.block"',

      'SELECT "menu_tree"."menu_name" AS "menu_name", "menu_tree"."route_name" AS "route_name", "menu_tree"."route_parameters" AS "route_parameters", "menu_tree"."url" AS "url", "menu_tree"."title" AS "title", "menu_tree"."description" AS "description", "menu_tree"."parent" AS "parent", "menu_tree"."weight" AS "weight", "menu_tree"."options" AS "options", "menu_tree"."expanded" AS "expanded", "menu_tree"."enabled" AS "enabled", "menu_tree"."provider" AS "provider", "menu_tree"."metadata" AS "metadata", "menu_tree"."class" AS "class", "menu_tree"."form_class" AS "form_class", "menu_tree"."id" AS "id" FROM "menu_tree" "menu_tree" WHERE ("route_name" = "entity.user.canonical") AND ("route_param_key" = "user=2") AND ("menu_name" = "main") ORDER BY "depth" ASC, "weight" ASC, "id" ASC',
@@ -410,7 +410,7 @@ protected function testLogin(): void { 
      'SELECT "roles_target_id" FROM "user__roles" WHERE "entity_id" = "2"',

      'SELECT "base"."uid" AS "uid", "base"."uuid" AS "uuid", "base"."langcode" AS "langcode" FROM "users" "base" WHERE "base"."uid" IN (2)',

      'SELECT "users_field_data".*, "users_field_data"."langcode" AS "users_field_data__langcode", "user__user_picture"."user_picture_target_id" AS "user_picture_target_id", "user__user_picture"."user_picture_alt" AS "user_picture_alt", "user__user_picture"."user_picture_title" AS "user_picture_title", "user__user_picture"."user_picture_width" AS "user_picture_width", "user__user_picture"."user_picture_height" AS "user_picture_height" FROM "users_field_data" "users_field_data" LEFT OUTER JOIN "user__user_picture" "user__user_picture" ON "user__user_picture"."entity_id" = "users_field_data"."uid" AND "user__user_picture"."langcode" = "users_field_data"."langcode" AND "user__user_picture"."deleted" = 0 WHERE "users_field_data"."uid" IN (2)',

      'SELECT "users_field_data".*, "users_field_data"."langcode" AS "users_field_data__langcode", "user__roles"."roles_target_id" AS "roles_target_id", "user__roles"."delta" AS "roles_delta" FROM "users_field_data" "users_field_data" INNER JOIN "user__roles" "user__roles" ON "user__roles"."entity_id" = "users_field_data"."uid" AND "user__roles"."langcode" = "users_field_data"."langcode" AND "user__roles"."deleted" = 0 WHERE "users_field_data"."uid" IN (2)',

      'SELECT "user__roles"."entity_id" AS "id", "user__roles"."langcode" AS "langcode", "user__roles"."roles_target_id" AS "roles_target_id", "user__roles"."delta" AS "roles_delta" FROM "user__roles" "user__roles" WHERE ("user__roles"."entity_id" IN (2)) AND ("user__roles"."deleted" = 0) ORDER BY "user__roles"."delta" ASC',

      'SELECT "name", "value" FROM "key_value" WHERE "name" IN ( "theme:stark" ) AND "collection" = "config.entity.key_store.block"',

    ];

    $recorded_queries = $performance_data->getQueries();
@@ -497,7 +497,7 @@ protected function testLoginBlock(): void { 
      'SELECT "base_table"."uid" AS "uid", "base_table"."uid" AS "base_table_uid" FROM "users" "base_table" INNER JOIN "users_field_data" "users_field_data" ON "users_field_data"."uid" = "base_table"."uid" WHERE ("users_field_data"."name" IN ("ACCOUNT_NAME")) AND ("users_field_data"."default_langcode" IN (1))',

      'SELECT "base"."uid" AS "uid", "base"."uuid" AS "uuid", "base"."langcode" AS "langcode" FROM "users" "base" WHERE "base"."uid" IN (2)',

      'SELECT "users_field_data".*, "users_field_data"."langcode" AS "users_field_data__langcode", "user__user_picture"."user_picture_target_id" AS "user_picture_target_id", "user__user_picture"."user_picture_alt" AS "user_picture_alt", "user__user_picture"."user_picture_title" AS "user_picture_title", "user__user_picture"."user_picture_width" AS "user_picture_width", "user__user_picture"."user_picture_height" AS "user_picture_height" FROM "users_field_data" "users_field_data" LEFT OUTER JOIN "user__user_picture" "user__user_picture" ON "user__user_picture"."entity_id" = "users_field_data"."uid" AND "user__user_picture"."langcode" = "users_field_data"."langcode" AND "user__user_picture"."deleted" = 0 WHERE "users_field_data"."uid" IN (2)',

      'SELECT "users_field_data".*, "users_field_data"."langcode" AS "users_field_data__langcode", "user__roles"."roles_target_id" AS "roles_target_id", "user__roles"."delta" AS "roles_delta" FROM "users_field_data" "users_field_data" INNER JOIN "user__roles" "user__roles" ON "user__roles"."entity_id" = "users_field_data"."uid" AND "user__roles"."langcode" = "users_field_data"."langcode" AND "user__roles"."deleted" = 0 WHERE "users_field_data"."uid" IN (2)',

      'SELECT "user__roles"."entity_id" AS "id", "user__roles"."langcode" AS "langcode", "user__roles"."roles_target_id" AS "roles_target_id", "user__roles"."delta" AS "roles_delta" FROM "user__roles" "user__roles" WHERE ("user__roles"."entity_id" IN (2)) AND ("user__roles"."deleted" = 0) ORDER BY "user__roles"."delta" ASC',

      'SELECT COUNT(*) AS "expression" FROM (SELECT 1 AS "expression" FROM "flood" "f" WHERE ("event" = "user.failed_login_user") AND ("identifier" = "CLIENT_IP") AND ("timestamp" > "TIMESTAMP")) "subquery"',

      'INSERT INTO "watchdog" ("uid", "type", "message", "variables", "severity", "link", "location", "referer", "hostname", "timestamp") VALUES ("2", "user", "Session opened for %name.", "WATCHDOG_DATA", 6, "", "LOCATION", "REFERER", "CLIENT_IP", "TIMESTAMP")',

      'UPDATE "users_field_data" SET "login"="TIMESTAMP" WHERE "uid" = "2"',