Unverified Commit c840c458 authored by Alex Pott's avatar Alex Pott
Browse files

perf: #3564689 Combine multiple cardinality field loading into a single database query 

By: catch
By: godotislate
By: alexpott
(cherry picked from commit b68de1a5f60ab847d34027e66a5cb615c75f9898)
parent 6f12b759

core/lib/Drupal/Core/Entity/Sql/SqlContentEntityStorage.php

+81 −51
Original line number Diff line number Diff line
@@ -1220,6 +1220,10 @@ protected function loadFromDedicatedTables(array &$values, $load_from_revision) 
      }

    }



    if (!$storage_definitions) {

      return;

    }



    // Load field data.

    $langcodes = array_keys($this->languageManager->getLanguages(LanguageInterface::STATE_ALL));
@@ -1234,25 +1238,27 @@ protected function loadFromDedicatedTables(array &$values, $load_from_revision) 
        $multiple_cardinality_fields[$field_name] = $storage_definition;

      }

    }

    if ($single_cardinality_fields) {

      $id_key = !$load_from_revision ? 'entity_id' : 'revision_id';



    $id_key = !$load_from_revision ? 'entity_id' : 'revision_id';

    // Because any field could potentially have no data, we need to begin

    // the query from a table that will reliably exist, which means the base,

    // data, or revision table.

    $base_table = !$load_from_revision ? ($this->dataTable ?? $this->baseTable) : $this->revisionDataTable ?? $this->revisionTable;

    $base_id_key = !$load_from_revision ? $this->idKey : $this->revisionKey;

      $query = $this->database->select($base_table, $base_table)

    $base_query = $this->database->select($base_table, $base_table)

      ->fields($base_table, [$base_id_key])

      ->condition("[$base_table].[$base_id_key]", $ids, 'IN');



    // If the entity is translatable, ensure only rows with valid langcodes

    // are loaded.

    if ($this->langcodeKey) {

        $query->condition("[$base_table].[$this->langcodeKey]", $langcodes, 'IN');

        $query->addField($base_table, $this->langcodeKey);

      $base_query->condition("[$base_table].[$this->langcodeKey]", $langcodes, 'IN');

      $base_query->addField($base_table, $this->langcodeKey);

    }



    if ($single_cardinality_fields) {

      $query = clone $base_query;



      // Add a left join for each single cardinality field.

      foreach ($single_cardinality_fields as $field_name => $storage_definition) {

        $table = !$load_from_revision ? $table_mapping->getDedicatedDataTableName($storage_definition) : $table_mapping->getDedicatedRevisionTableName($storage_definition);
@@ -1313,31 +1319,44 @@ protected function loadFromDedicatedTables(array &$values, $load_from_revision) 
        }

      }

    }

    if ($multiple_cardinality_fields) {

      $query = clone $base_query;

      $delta_keys = [];

      foreach ($multiple_cardinality_fields as $field_name => $storage_definition) {

        $table = !$load_from_revision ? $table_mapping->getDedicatedDataTableName($storage_definition) : $table_mapping->getDedicatedRevisionTableName($storage_definition);

        // If the entity is translatable, add the langcode to the join and

        // a condition on valid langcodes.

        if ($this->langcodeKey) {

          $query->leftJoin($table, $table, "[$table].[$id_key] = [$base_table].[$base_id_key] AND [$table].[langcode] = [$base_table].[$this->langcodeKey] AND [$table].[deleted] = 0");

        }

        else {

          $query->leftJoin($table, $table, "[$table].[$id_key] = [$base_table].[$base_id_key] AND [$table].[deleted] = 0");

        }

        $query->fields($table, $this->tableMapping->getColumnNames($field_name));

        $delta_keys[$field_name] = $query->addField($table, 'delta', $field_name . '_delta');

      }



      // Ensure that only values having valid languages are retrieved. Since we

      // are loading values for multiple entities, we cannot limit the query to

      // the available translations.

      $results = $this->database->select($table, 't')

        ->fields('t')

        ->condition(!$load_from_revision ? 'entity_id' : 'revision_id', $ids, 'IN')

        ->condition('deleted', 0)

        ->condition('langcode', $langcodes, 'IN')

        ->orderBy('delta')

        ->execute();

      $results = $query->execute();



      foreach ($results as $row) {

        $bundle = $row->bundle;



        $value_key = !$load_from_revision ? $row->entity_id : $row->revision_id;

        $row = (array) $row;

        $value_key = $row[$base_id_key];

        // Field values in default language are stored with

        // LanguageInterface::LANGCODE_DEFAULT as key.

        $langcode = LanguageInterface::LANGCODE_DEFAULT;

        if ($this->langcodeKey && isset($default_langcodes[$value_key]) && $row->langcode != $default_langcodes[$value_key]) {

          $langcode = $row->langcode;

        if ($this->langcodeKey && isset($default_langcodes[$value_key]) && $row[$this->langcodeKey] != $default_langcodes[$value_key]) {

          $langcode = $row[$this->langcodeKey];

        }



        foreach ($multiple_cardinality_fields as $field_name => $storage_definition) {

          $delta_key = $delta_keys[$field_name];

          $bundle = $this->bundleKey ? $values[$value_key][$this->bundleKey][LanguageInterface::LANGCODE_DEFAULT] : $this->entityTypeId;



          // If the delta is null then there are no values at this delta for

          // this field.

          if (!isset($row[$delta_key])) {

            continue;

          }

          if (!isset($values[$value_key][$field_name][$langcode])) {

            $values[$value_key][$field_name][$langcode] = [];

          }
@@ -1345,18 +1364,29 @@ protected function loadFromDedicatedTables(array &$values, $load_from_revision) 
          // Ensure that records for non-translatable fields having invalid

          // languages are skipped.

          if ($langcode == LanguageInterface::LANGCODE_DEFAULT || $definitions[$bundle][$field_name]->isTranslatable()) {

          if ($storage_definition->getCardinality() == FieldStorageDefinitionInterface::CARDINALITY_UNLIMITED || count($values[$value_key][$field_name][$langcode]) < $storage_definition->getCardinality()) {

            if (($storage_definition->getCardinality() === FieldStorageDefinitionInterface::CARDINALITY_UNLIMITED || $row[$delta_key] < $storage_definition->getCardinality())) {

              $item = [];

            // For each column declared by the field, populate the item from the

            // prefixed database column.

              // For each column declared by the field, populate the item from

              // the prefixed database column.

              foreach ($storage_definition->getColumns() as $column => $attributes) {

                $column_name = $table_mapping->getFieldColumnName($storage_definition, $column);

                // Unserialize the value if specified in the column schema.

              $item[$column] = (!empty($attributes['serialize'])) ? $this->handleNullableFieldUnserialize($row->$column_name) : $row->$column_name;

                $item[$column] = (!empty($attributes['serialize'])) ? $this->handleNullableFieldUnserialize($row[$column_name]) : $row[$column_name];

              }



              // Add the item to the field values for the entity.

            $values[$value_key][$field_name][$langcode][] = $item;

              $values[$value_key][$field_name][$langcode][(int) $row[$delta_key]] = $item;

            }

          }

        }

      }

      // Ensure that all of the deltas from all of the multiple cardinality

      // fields are returned in the correct order.

      foreach ($values as &$fields) {

        foreach ($fields as $field_name => &$field_data) {

          if (isset($multiple_cardinality_fields[$field_name])) {

            foreach ($field_data as &$language_data) {

              ksort($language_data);

            }

          }

        }

      }

core/profiles/demo_umami/tests/src/FunctionalJavascript/OpenTelemetryAuthenticatedPerformanceTest.php

+1 −1
Original line number Diff line number Diff line
@@ -128,7 +128,7 @@ protected function doTestNodePageAdministrator(): void { 
    }, 'administratorNodePage');



    $expected = [

      'QueryCount' => 331,

      'QueryCount' => 322,

      'CacheGetCount' => 349,

      'CacheGetCountByBin' => [

        'config' => 91,

core/profiles/demo_umami/tests/src/FunctionalJavascript/OpenTelemetryFrontPagePerformanceTest.php

+2 −2
Original line number Diff line number Diff line
@@ -52,7 +52,7 @@ protected function testFrontPageColdCache(): void { 
    $this->assertSession()->pageTextContains('Umami');



    $expected = [

      'QueryCount' => 245,

      'QueryCount' => 239,

      'CacheGetCount' => 316,

      'CacheSetCount' => 315,

      'CacheDeleteCount' => 0,
@@ -122,7 +122,7 @@ protected function testFrontPageCoolCache(): void { 
    }, 'umamiFrontPageCoolCache');



    $expected = [

      'QueryCount' => 80,

      'QueryCount' => 74,

      'CacheGetCount' => 181,

      'CacheSetCount' => 79,

      'CacheDeleteCount' => 0,

core/profiles/standard/tests/src/FunctionalJavascript/StandardPerformanceTest.php

+7 −7
Original line number Diff line number Diff line
@@ -99,7 +99,7 @@ protected function testAnonymous(): void { 
      'SELECT "revision"."vid" AS "vid", "revision"."langcode" AS "langcode", "revision"."revision_uid" AS "revision_uid", "revision"."revision_timestamp" AS "revision_timestamp", "revision"."revision_log" AS "revision_log", "revision"."revision_default" AS "revision_default", "base"."nid" AS "nid", "base"."type" AS "type", "base"."uuid" AS "uuid", CASE "base"."vid" WHEN "revision"."vid" THEN 1 ELSE 0 END AS "isDefaultRevision" FROM "node" "base" INNER JOIN "node_revision" "revision" ON "revision"."vid" = "base"."vid" WHERE "base"."nid" IN (1)',

      'SELECT "revision".* FROM "node_field_revision" "revision" WHERE ("revision"."nid" IN (1)) AND ("revision"."vid" IN ("1")) ORDER BY "revision"."nid" ASC',

      'SELECT "node_field_data"."nid" AS "nid", "node_field_data"."langcode" AS "langcode", "node__body"."body_value" AS "body_value", "node__body"."body_summary" AS "body_summary", "node__body"."body_format" AS "body_format", "node__comment"."comment_status" AS "comment_status", "node__field_image"."field_image_target_id" AS "field_image_target_id", "node__field_image"."field_image_alt" AS "field_image_alt", "node__field_image"."field_image_title" AS "field_image_title", "node__field_image"."field_image_width" AS "field_image_width", "node__field_image"."field_image_height" AS "field_image_height" FROM "node_field_data" "node_field_data" LEFT OUTER JOIN "node__body" "node__body" ON "node__body"."entity_id" = "node_field_data"."nid" AND "node__body"."langcode" = "node_field_data"."langcode" AND "node__body"."deleted" = 0 LEFT OUTER JOIN "node__comment" "node__comment" ON "node__comment"."entity_id" = "node_field_data"."nid" AND "node__comment"."langcode" = "node_field_data"."langcode" AND "node__comment"."deleted" = 0 LEFT OUTER JOIN "node__field_image" "node__field_image" ON "node__field_image"."entity_id" = "node_field_data"."nid" AND "node__field_image"."langcode" = "node_field_data"."langcode" AND "node__field_image"."deleted" = 0 WHERE ("node_field_data"."nid" IN (1)) AND ("node_field_data"."langcode" IN ("en", "und", "zxx"))',

      'SELECT "t".* FROM "node__field_tags" "t" WHERE ("entity_id" IN (1)) AND ("deleted" = 0) AND ("langcode" IN ("en", "und", "zxx")) ORDER BY "delta" ASC',

      'SELECT "node_field_data"."nid" AS "nid", "node_field_data"."langcode" AS "langcode", "node__field_tags"."field_tags_target_id" AS "field_tags_target_id", "node__field_tags"."delta" AS "field_tags_delta" FROM "node_field_data" "node_field_data" LEFT OUTER JOIN "node__field_tags" "node__field_tags" ON "node__field_tags"."entity_id" = "node_field_data"."nid" AND "node__field_tags"."langcode" = "node_field_data"."langcode" AND "node__field_tags"."deleted" = 0 WHERE ("node_field_data"."nid" IN (1)) AND ("node_field_data"."langcode" IN ("en", "und", "zxx"))',

      'SELECT "ces".* FROM "comment_entity_statistics" "ces" WHERE ("ces"."entity_id" IN (1)) AND ("ces"."entity_type" = "node")',

      'SELECT "name", "data" FROM "config" WHERE "collection" = "" AND "name" IN ( "core.entity_view_display.node.article.teaser", "core.entity_view_display.node.article.default" )',

      'SELECT "config"."name" AS "name" FROM "config" "config" WHERE ("collection" = "") AND ("name" LIKE "comment.type.%" ESCAPE ' . "'\\\\'" . ') ORDER BY "collection" ASC, "name" ASC',
@@ -107,7 +107,7 @@ protected function testAnonymous(): void { 
      'SELECT "base"."uid" AS "uid", "base"."uuid" AS "uuid", "base"."langcode" AS "langcode" FROM "users" "base" WHERE "base"."uid" IN (0)',

      'SELECT "data".* FROM "users_field_data" "data" WHERE "data"."uid" IN (0) ORDER BY "data"."uid" ASC',

      'SELECT "users_field_data"."uid" AS "uid", "users_field_data"."langcode" AS "langcode", "user__user_picture"."user_picture_target_id" AS "user_picture_target_id", "user__user_picture"."user_picture_alt" AS "user_picture_alt", "user__user_picture"."user_picture_title" AS "user_picture_title", "user__user_picture"."user_picture_width" AS "user_picture_width", "user__user_picture"."user_picture_height" AS "user_picture_height" FROM "users_field_data" "users_field_data" LEFT OUTER JOIN "user__user_picture" "user__user_picture" ON "user__user_picture"."entity_id" = "users_field_data"."uid" AND "user__user_picture"."langcode" = "users_field_data"."langcode" AND "user__user_picture"."deleted" = 0 WHERE ("users_field_data"."uid" IN (0)) AND ("users_field_data"."langcode" IN ("en", "und", "zxx"))',

      'SELECT "t".* FROM "user__roles" "t" WHERE ("entity_id" IN (0)) AND ("deleted" = 0) AND ("langcode" IN ("en", "und", "zxx")) ORDER BY "delta" ASC',

      'SELECT "users_field_data"."uid" AS "uid", "users_field_data"."langcode" AS "langcode", "user__roles"."roles_target_id" AS "roles_target_id", "user__roles"."delta" AS "roles_delta" FROM "users_field_data" "users_field_data" LEFT OUTER JOIN "user__roles" "user__roles" ON "user__roles"."entity_id" = "users_field_data"."uid" AND "user__roles"."langcode" = "users_field_data"."langcode" AND "user__roles"."deleted" = 0 WHERE ("users_field_data"."uid" IN (0)) AND ("users_field_data"."langcode" IN ("en", "und", "zxx"))',

      'SELECT "name", "data" FROM "config" WHERE "collection" = "" AND "name" IN ( "core.date_format.medium" )',

      'SELECT "name", "data" FROM "config" WHERE "collection" = "" AND "name" IN ( "core.date_format.long" )',

      'SELECT 1 AS "expression" FROM "path_alias" "base_table" WHERE ("base_table"."status" = 1) AND ("base_table"."path" LIKE "/node%" ESCAPE ' . "'\\\\'" . ') LIMIT 1 OFFSET 0',
@@ -135,12 +135,12 @@ protected function testAnonymous(): void { 
    $this->assertSame($expected_queries, $recorded_queries);

    $expected = [

      'QueryCount' => 39,

      'CacheGetCount' => 97,

      'CacheGetCount' => 98,

      'CacheGetCountByBin' => [

        'page' => 1,

        'config' => 20,

        'data' => 5,

        'discovery' => 37,

        'discovery' => 38,

        'bootstrap' => 10,

        'dynamic_page_cache' => 1,

        'render' => 13,
@@ -266,7 +266,7 @@ protected function testAnonymous(): void { 
      'SELECT "base"."uid" AS "uid", "base"."uuid" AS "uuid", "base"."langcode" AS "langcode" FROM "users" "base" WHERE "base"."uid" IN (2)',

      'SELECT "data".* FROM "users_field_data" "data" WHERE "data"."uid" IN (2) ORDER BY "data"."uid" ASC',

      'SELECT "users_field_data"."uid" AS "uid", "users_field_data"."langcode" AS "langcode", "user__user_picture"."user_picture_target_id" AS "user_picture_target_id", "user__user_picture"."user_picture_alt" AS "user_picture_alt", "user__user_picture"."user_picture_title" AS "user_picture_title", "user__user_picture"."user_picture_width" AS "user_picture_width", "user__user_picture"."user_picture_height" AS "user_picture_height" FROM "users_field_data" "users_field_data" LEFT OUTER JOIN "user__user_picture" "user__user_picture" ON "user__user_picture"."entity_id" = "users_field_data"."uid" AND "user__user_picture"."langcode" = "users_field_data"."langcode" AND "user__user_picture"."deleted" = 0 WHERE ("users_field_data"."uid" IN (2)) AND ("users_field_data"."langcode" IN ("en", "und", "zxx"))',

      'SELECT "t".* FROM "user__roles" "t" WHERE ("entity_id" IN (2)) AND ("deleted" = 0) AND ("langcode" IN ("en", "und", "zxx")) ORDER BY "delta" ASC',

      'SELECT "users_field_data"."uid" AS "uid", "users_field_data"."langcode" AS "langcode", "user__roles"."roles_target_id" AS "roles_target_id", "user__roles"."delta" AS "roles_delta" FROM "users_field_data" "users_field_data" LEFT OUTER JOIN "user__roles" "user__roles" ON "user__roles"."entity_id" = "users_field_data"."uid" AND "user__roles"."langcode" = "users_field_data"."langcode" AND "user__roles"."deleted" = 0 WHERE ("users_field_data"."uid" IN (2)) AND ("users_field_data"."langcode" IN ("en", "und", "zxx"))',

      'SELECT "name", "data" FROM "config" WHERE "collection" = "" AND "name" IN ( "core.entity_view_display.user.user.full" )',

      'SELECT "name", "value" FROM "key_value" WHERE "name" IN ( "theme:stark" ) AND "collection" = "config.entity.key_store.block"',

      'SELECT "menu_tree"."menu_name" AS "menu_name", "menu_tree"."route_name" AS "route_name", "menu_tree"."route_parameters" AS "route_parameters", "menu_tree"."url" AS "url", "menu_tree"."title" AS "title", "menu_tree"."description" AS "description", "menu_tree"."parent" AS "parent", "menu_tree"."weight" AS "weight", "menu_tree"."options" AS "options", "menu_tree"."expanded" AS "expanded", "menu_tree"."enabled" AS "enabled", "menu_tree"."provider" AS "provider", "menu_tree"."metadata" AS "metadata", "menu_tree"."class" AS "class", "menu_tree"."form_class" AS "form_class", "menu_tree"."id" AS "id" FROM "menu_tree" "menu_tree" WHERE ("route_name" = "entity.user.canonical") AND ("route_param_key" = "user=2") AND ("menu_name" = "main") ORDER BY "depth" ASC, "weight" ASC, "id" ASC',
@@ -427,7 +427,7 @@ protected function testLogin(): void { 
      'SELECT "base"."uid" AS "uid", "base"."uuid" AS "uuid", "base"."langcode" AS "langcode" FROM "users" "base" WHERE "base"."uid" IN (2)',

      'SELECT "data".* FROM "users_field_data" "data" WHERE "data"."uid" IN (2) ORDER BY "data"."uid" ASC',

      'SELECT "users_field_data"."uid" AS "uid", "users_field_data"."langcode" AS "langcode", "user__user_picture"."user_picture_target_id" AS "user_picture_target_id", "user__user_picture"."user_picture_alt" AS "user_picture_alt", "user__user_picture"."user_picture_title" AS "user_picture_title", "user__user_picture"."user_picture_width" AS "user_picture_width", "user__user_picture"."user_picture_height" AS "user_picture_height" FROM "users_field_data" "users_field_data" LEFT OUTER JOIN "user__user_picture" "user__user_picture" ON "user__user_picture"."entity_id" = "users_field_data"."uid" AND "user__user_picture"."langcode" = "users_field_data"."langcode" AND "user__user_picture"."deleted" = 0 WHERE ("users_field_data"."uid" IN (2)) AND ("users_field_data"."langcode" IN ("en", "und", "zxx"))',

      'SELECT "t".* FROM "user__roles" "t" WHERE ("entity_id" IN (2)) AND ("deleted" = 0) AND ("langcode" IN ("en", "und", "zxx")) ORDER BY "delta" ASC',

      'SELECT "users_field_data"."uid" AS "uid", "users_field_data"."langcode" AS "langcode", "user__roles"."roles_target_id" AS "roles_target_id", "user__roles"."delta" AS "roles_delta" FROM "users_field_data" "users_field_data" LEFT OUTER JOIN "user__roles" "user__roles" ON "user__roles"."entity_id" = "users_field_data"."uid" AND "user__roles"."langcode" = "users_field_data"."langcode" AND "user__roles"."deleted" = 0 WHERE ("users_field_data"."uid" IN (2)) AND ("users_field_data"."langcode" IN ("en", "und", "zxx"))',

      'SELECT "name", "value" FROM "key_value" WHERE "name" IN ( "theme:stark" ) AND "collection" = "config.entity.key_store.block"',

    ];

    $recorded_queries = $performance_data->getQueries();
@@ -520,7 +520,7 @@ protected function testLoginBlock(): void { 
      'SELECT "base"."uid" AS "uid", "base"."uuid" AS "uuid", "base"."langcode" AS "langcode" FROM "users" "base" WHERE "base"."uid" IN (2)',

      'SELECT "data".* FROM "users_field_data" "data" WHERE "data"."uid" IN (2) ORDER BY "data"."uid" ASC',

      'SELECT "users_field_data"."uid" AS "uid", "users_field_data"."langcode" AS "langcode", "user__user_picture"."user_picture_target_id" AS "user_picture_target_id", "user__user_picture"."user_picture_alt" AS "user_picture_alt", "user__user_picture"."user_picture_title" AS "user_picture_title", "user__user_picture"."user_picture_width" AS "user_picture_width", "user__user_picture"."user_picture_height" AS "user_picture_height" FROM "users_field_data" "users_field_data" LEFT OUTER JOIN "user__user_picture" "user__user_picture" ON "user__user_picture"."entity_id" = "users_field_data"."uid" AND "user__user_picture"."langcode" = "users_field_data"."langcode" AND "user__user_picture"."deleted" = 0 WHERE ("users_field_data"."uid" IN (2)) AND ("users_field_data"."langcode" IN ("en", "und", "zxx"))',

      'SELECT "t".* FROM "user__roles" "t" WHERE ("entity_id" IN (2)) AND ("deleted" = 0) AND ("langcode" IN ("en", "und", "zxx")) ORDER BY "delta" ASC',

      'SELECT "users_field_data"."uid" AS "uid", "users_field_data"."langcode" AS "langcode", "user__roles"."roles_target_id" AS "roles_target_id", "user__roles"."delta" AS "roles_delta" FROM "users_field_data" "users_field_data" LEFT OUTER JOIN "user__roles" "user__roles" ON "user__roles"."entity_id" = "users_field_data"."uid" AND "user__roles"."langcode" = "users_field_data"."langcode" AND "user__roles"."deleted" = 0 WHERE ("users_field_data"."uid" IN (2)) AND ("users_field_data"."langcode" IN ("en", "und", "zxx"))',

      'SELECT COUNT(*) AS "expression" FROM (SELECT 1 AS "expression" FROM "flood" "f" WHERE ("event" = "user.failed_login_user") AND ("identifier" = "CLIENT_IP") AND ("timestamp" > "TIMESTAMP")) "subquery"',

      'INSERT INTO "watchdog" ("uid", "type", "message", "variables", "severity", "link", "location", "referer", "hostname", "timestamp") VALUES ("2", "user", "Session opened for %name.", "WATCHDOG_DATA", 6, "", "LOCATION", "REFERER", "CLIENT_IP", "TIMESTAMP")',

      'UPDATE "users_field_data" SET "login"="TIMESTAMP" WHERE "uid" = "2"',