Unverified Commit 4c021eac authored by Alex Pott's avatar Alex Pott
Browse files

perf: #3561960 Combine the database queries in ::loadFromSharedTables() and... 

perf: #3561960 Combine the database queries in ::loadFromSharedTables() and ::loadFromDedicatedTables()

By: catch
By: alexpott
By: heddn
parent 117fee74

core/lib/Drupal/Core/Entity/Sql/SqlContentEntityStorage.php

+128 −69
Original line number Diff line number Diff line
@@ -453,7 +453,7 @@ protected function mapFromStorageRecords(array $records, $load_from_revision = F 


    // Get the names of the fields that are stored in the base table and, if

    // applicable, the revision table. Other entity data will be loaded in

    // loadFromSharedTables() and loadFromDedicatedTables().

    // loadFromDedicatedTables().

    $field_names = $this->tableMapping->getFieldNames($this->baseTable);

    if ($this->revisionTable) {

      $field_names = array_unique(array_merge($field_names, $this->tableMapping->getFieldNames($this->revisionTable)));
@@ -505,8 +505,7 @@ protected function mapFromStorageRecords(array $records, $load_from_revision = F 
    $translations = array_fill_keys(array_keys($values), []);



    // Load values from shared and dedicated tables.

    $this->loadFromSharedTables($values, $translations, $load_from_revision);

    $this->loadFromDedicatedTables($values, $load_from_revision);

    $this->loadFromDedicatedTables($values, $load_from_revision, $translations);



    $entities = [];

    foreach ($values as $id => $entity_values) {
@@ -529,8 +528,13 @@ protected function mapFromStorageRecords(array $records, $load_from_revision = F 
   *   List of translations, keyed on the entity ID.

   * @param bool $load_from_revision

   *   Flag to indicate whether revisions should be loaded or not.

   *

   * @deprecated in drupal:11.4.0 and is removed from drupal:12.0.0. There is no

   * replacement.

   * @see https://www.drupal.org/node/3586362

   */

  protected function loadFromSharedTables(array &$values, array &$translations, $load_from_revision) {

    @trigger_error(__METHOD__ . ' is deprecated in drupal:11.4.0 and is removed from drupal:12.0.0. There is no replacement. See https://www.drupal.org/node/3586362');

    $record_key = !$load_from_revision ? $this->idKey : $this->revisionKey;

    if ($this->dataTable) {

      // If a revision table is available, we need all the properties of the
@@ -1221,8 +1225,10 @@ protected function getQueryServiceName() { 
   *   An array of values keyed by entity ID.

   * @param bool $load_from_revision

   *   Flag to indicate whether revisions should be loaded or not.

   * @param array &$translations

   *   List of translations, keyed on the entity ID.

   */

  protected function loadFromDedicatedTables(array &$values, $load_from_revision) {

  protected function loadFromDedicatedTables(array &$values, $load_from_revision, &$translations) {

    if (empty($values)) {

      return;

    }
@@ -1253,13 +1259,6 @@ protected function loadFromDedicatedTables(array &$values, $load_from_revision) 
      }

    }



    if (!$storage_definitions) {

      return;

    }



    // Load field data.

    $langcodes = array_keys($this->languageManager->getLanguages(LanguageInterface::STATE_ALL));



    $single_cardinality_fields = [];

    $multiple_cardinality_fields = [];

    $field_definition_columns = [];
@@ -1280,22 +1279,63 @@ protected function loadFromDedicatedTables(array &$values, $load_from_revision) 
    // Because any field could potentially have no data, we need to begin

    // the query from a table that will reliably exist, which means the base,

    // data, or revision table.

    $base_table = !$load_from_revision ? ($this->dataTable ?? $this->baseTable) : $this->revisionDataTable ?? $this->revisionTable;

    $base_table = $load_from_revision ? ($this->revisionDataTable ?? $this->revisionTable) : $this->dataTable ?? $this->baseTable;

    $base_id_key = !$load_from_revision ? $this->idKey : $this->revisionKey;

    $base_query = $this->database->select($base_table, $base_table)

      ->fields($base_table, [$base_id_key])

      ->fields($base_table)

      ->condition("[$base_table].[$base_id_key]", $ids, 'IN');



    // If the entity is translatable, ensure only rows with valid langcodes

    // are loaded.

    // Alias the base langcode field so that it does not get overwritten by

    // results from field tables.

    $base_langcode_alias = '';

    if ($this->langcodeKey) {

      $base_query->condition("[$base_table].[$this->langcodeKey]", $langcodes, 'IN');

      $base_query->addField($base_table, $this->langcodeKey);

      $base_langcode_alias = $base_table . '__' . $this->langcodeKey;

      $base_query->addField($base_table, $this->langcodeKey, $base_langcode_alias);

    }



    if ($single_cardinality_fields) {

    $query = clone $base_query;



    if ($this->revisionDataTable) {

      // Find revisioned fields that are not entity keys. Exclude the langcode

      // key as the base table holds only the default language.

      $base_fields = array_diff($table_mapping->getFieldNames($this->baseTable), [$this->langcodeKey]);

      $revisioned_fields = array_diff($table_mapping->getFieldNames($this->revisionDataTable), $base_fields);



      // Find fields that are not revisioned or entity keys. Data fields have

      // the same value regardless of entity revision.

      $data_fields = array_diff($table_mapping->getFieldNames($this->dataTable), $revisioned_fields, $base_fields);

      // If there are no data fields then only revisioned fields are needed

      // else both data fields and revisioned fields are needed to map the

      // entity values.

      $shared_fields = $revisioned_fields;

      if ($data_fields) {

        $shared_fields = array_merge($revisioned_fields, $data_fields);

        if ($load_from_revision) {

          $query->leftJoin($this->dataTable, 'data', "([data].[$this->idKey] = [$base_table].[$this->idKey] AND [$base_table].[$this->langcodeKey] = [data].[$this->langcodeKey])");

          $column_names = [];

          // Some fields can have more then one columns in the data table so

          // column names are needed.

          foreach ($data_fields as $data_field) {

            // \Drupal\Core\Entity\Sql\TableMappingInterface::getColumnNames()

            // returns an array keyed by property names so remove the keys

            // before array_merge() to avoid losing data with fields having the

            // same columns i.e. value.

            $column_names[] = array_values($table_mapping->getColumnNames($data_field));

          }

          $column_names = array_merge(...$column_names);

          $query->fields('data', $column_names);

        }

      }

    }

    else {

      $shared_fields = $table_mapping->getFieldNames($base_table);

    }

    foreach ($shared_fields as $field_name) {

      $storage_definition = $this->fieldStorageDefinitions[$field_name];

      $field_definition_columns[$field_name] = $storage_definition->getColumns();

      $field_columns[$field_name] = $table_mapping->getColumnNames($field_name);

    }



    // Add a left join for each single cardinality field.

    foreach ($single_cardinality_fields as $field_name => $storage_definition) {

      $table = !$load_from_revision ? $table_mapping->getDedicatedDataTableName($storage_definition) : $table_mapping->getDedicatedRevisionTableName($storage_definition);
@@ -1317,15 +1357,35 @@ protected function loadFromDedicatedTables(array &$values, $load_from_revision) 
    foreach ($results as $row) {

      $row = (array) $row;

      $value_key = $row[$base_id_key];

      $bundle = $this->bundleKey ? $values[$value_key][$this->bundleKey][LanguageInterface::LANGCODE_DEFAULT] : $this->entityTypeId;

      // Field values in default language are stored with

      // LanguageInterface::LANGCODE_DEFAULT as key.

        $langcode = LanguageInterface::LANGCODE_DEFAULT;

        if ($this->langcodeKey && isset($default_langcodes[$value_key]) && $row[$this->langcodeKey] != $default_langcodes[$value_key]) {

          $langcode = $row[$this->langcodeKey];

      $langcode = $this->langcodeKey && empty($row[$this->defaultLangcodeKey]) ? $row[$base_langcode_alias] : LanguageInterface::LANGCODE_DEFAULT;

      $translations[$value_key][$langcode] = TRUE;

      foreach ($shared_fields as $field_name) {

        $definition_columns = $field_definition_columns[$field_name];

        $columns = $field_columns[$field_name];

        // Do not key single-column fields by property name.

        if (count($columns) == 1) {

          $column_name = reset($columns);

          // Special case for the langcode key which is aliased in the query

          // results.

          if ($column_name === $this->langcodeKey) {

            $column_name = $base_langcode_alias;

          }

          $column_attributes = $definition_columns[key($columns)];

          $values[$value_key][$field_name][$langcode] = (!empty($column_attributes['serialize'])) ? $this->handleNullableFieldUnserialize($row[$column_name]) : $row[$column_name];

        }

        else {

          foreach ($columns as $property_name => $column_name) {

            $column_attributes = $definition_columns[$property_name];

            $values[$value_key][$field_name][$langcode][$property_name] = (!empty($column_attributes['serialize'])) ? $this->handleNullableFieldUnserialize($row[$column_name]) : $row[$column_name];

          }

        }

      }

      $langcode = $this->langcodeKey && empty($row[$this->defaultLangcodeKey]) && isset($default_langcodes[$value_key]) && $row[$base_langcode_alias] != $default_langcodes[$value_key] ? $row[$base_langcode_alias] : LanguageInterface::LANGCODE_DEFAULT;



      foreach ($single_cardinality_fields as $field_name => $storage_definition) {

          $bundle = $this->bundleKey ? $values[$value_key][$this->bundleKey][LanguageInterface::LANGCODE_DEFAULT] : $this->entityTypeId;

        $field_values = array_intersect_key($row, array_flip($field_columns[$field_name]));



        // If all the field values are null, then there was no result for this
@@ -1356,7 +1416,6 @@ protected function loadFromDedicatedTables(array &$values, $load_from_revision) 
        }

      }

    }

    }

    if ($multiple_cardinality_fields) {

      $query = clone $base_query;

      $delta_keys = [];
@@ -1382,8 +1441,8 @@ protected function loadFromDedicatedTables(array &$values, $load_from_revision) 
        // Field values in default language are stored with

        // LanguageInterface::LANGCODE_DEFAULT as key.

        $langcode = LanguageInterface::LANGCODE_DEFAULT;

        if ($this->langcodeKey && isset($default_langcodes[$value_key]) && $row[$this->langcodeKey] != $default_langcodes[$value_key]) {

          $langcode = $row[$this->langcodeKey];

        if ($this->langcodeKey && isset($default_langcodes[$value_key]) && $row[$base_langcode_alias] != $default_langcodes[$value_key]) {

          $langcode = $row[$base_langcode_alias];

        }



        foreach ($multiple_cardinality_fields as $field_name => $storage_definition) {

core/modules/jsonapi/tests/src/FunctionalJavascript/JsonApiPerformanceTest.php

+6 −7
Original line number Diff line number Diff line
@@ -65,9 +65,8 @@ public function testGetIndividual(): void { 
      'SELECT "name", "route", "fit" FROM "router" WHERE "pattern_outline" IN ( "/jsonapi/node/article/677f9911-f002-4639-9891-5c39e8b00d9d", "/jsonapi/node/article/%", "/jsonapi/node/%/%", "/jsonapi/%/article/677f9911-f002-4639-9891-5c39e8b00d9d", "/jsonapi/%/%/%", "/jsonapi/node/article", "/jsonapi/node/%", "/jsonapi/%/article", "/jsonapi/node", "/jsonapi/%", "/jsonapi" ) AND "number_parts" >= 4',

      'SELECT "base_table"."vid" AS "vid", "base_table"."nid" AS "nid" FROM "node" "base_table" INNER JOIN "node" "node" ON "node"."nid" = "base_table"."nid" INNER JOIN "node_field_data" "node_field_data" ON "node_field_data"."nid" = "base_table"."nid" WHERE ("node"."uuid" IN ("677f9911-f002-4639-9891-5c39e8b00d9d")) AND ("node_field_data"."default_langcode" IN (1))',

      'SELECT "revision"."vid" AS "vid", "revision"."langcode" AS "langcode", "revision"."revision_uid" AS "revision_uid", "revision"."revision_timestamp" AS "revision_timestamp", "revision"."revision_log" AS "revision_log", "revision"."revision_default" AS "revision_default", "base"."nid" AS "nid", "base"."type" AS "type", "base"."uuid" AS "uuid", CASE "base"."vid" WHEN "revision"."vid" THEN 1 ELSE 0 END AS "isDefaultRevision" FROM "node" "base" INNER JOIN "node_revision" "revision" ON "revision"."vid" = "base"."vid" WHERE "base"."nid" IN (1)',

      'SELECT "revision".* FROM "node_field_revision" "revision" WHERE ("revision"."nid" IN (1)) AND ("revision"."vid" IN ("1")) ORDER BY "revision"."nid" ASC',

      'SELECT "node_field_data"."nid" AS "nid", "node_field_data"."langcode" AS "langcode", "node__body"."body_value" AS "body_value", "node__body"."body_format" AS "body_format" FROM "node_field_data" "node_field_data" LEFT OUTER JOIN "node__body" "node__body" ON "node__body"."entity_id" = "node_field_data"."nid" AND "node__body"."langcode" = "node_field_data"."langcode" AND "node__body"."deleted" = 0 WHERE ("node_field_data"."nid" IN (1)) AND ("node_field_data"."langcode" IN ("en", "und", "zxx"))',

      'SELECT 1 AS "expression" FROM "path_alias" "base_table" WHERE ("base_table"."status" = 1) AND ("base_table"."path" LIKE "/jsonapi%" ESCAPE ' . "'\\\\'" . ') LIMIT 1 OFFSET 0',

      'SELECT "node_field_data".*, "node_field_data"."langcode" AS "node_field_data__langcode", "node__body"."body_value" AS "body_value", "node__body"."body_format" AS "body_format" FROM "node_field_data" "node_field_data" LEFT OUTER JOIN "node__body" "node__body" ON "node__body"."entity_id" = "node_field_data"."nid" AND "node__body"."langcode" = "node_field_data"."langcode" AND "node__body"."deleted" = 0 WHERE "node_field_data"."nid" IN (1)',

      'SELECT 1 AS "expression" FROM "path_alias" "base_table" WHERE ("base_table"."status" = 1) AND ("base_table"."path" LIKE "/jsonapi%" ESCAPE \'\\\\\') LIMIT 1 OFFSET 0',

      'SELECT "name", "route" FROM "router" WHERE "name" IN ( "jsonapi.node--article.node_type.relationship.get" )',

      'SELECT "name", "route" FROM "router" WHERE "name" IN ( "jsonapi.node--article.node_type.related" )',

      'SELECT "base_table"."vid" AS "vid", "base_table"."nid" AS "nid" FROM "node" "base_table" INNER JOIN "node" "node" ON "node"."nid" = "base_table"."nid" INNER JOIN "node_field_data" "node_field_data" ON "node_field_data"."nid" = "base_table"."nid" WHERE ("node"."uuid" IN ("677f9911-f002-4639-9891-5c39e8b00d9d")) AND ("node_field_data"."default_langcode" IN (1))',
@@ -84,12 +83,13 @@ public function testGetIndividual(): void { 
      'SELECT "base_table"."vid" AS "vid", "base_table"."nid" AS "nid" FROM "node" "base_table" INNER JOIN "node" "node" ON "node"."nid" = "base_table"."nid" INNER JOIN "node_field_data" "node_field_data" ON "node_field_data"."nid" = "base_table"."nid" WHERE ("node"."uuid" IN ("677f9911-f002-4639-9891-5c39e8b00d9d")) AND ("node_field_data"."default_langcode" IN (1))',

      'INSERT INTO "semaphore" ("name", "value", "expire") VALUES ("path_alias_prefix_list:Drupal\Core\Cache\CacheCollector", "LOCK_ID", "EXPIRE")',

      'DELETE FROM "semaphore"  WHERE ("name" = "path_alias_prefix_list:Drupal\Core\Cache\CacheCollector") AND ("value" = "LOCK_ID")',



    ];

    $recorded_queries = $performance_data->getQueries();

    $this->assertSame($expected_queries, $recorded_queries);



    $expected = [

      'QueryCount' => 26,

      'QueryCount' => 25,

      'CacheGetCount' => 41,

      'CacheGetCountByBin' => [

        'config' => 7,
@@ -195,8 +195,7 @@ public function testGetIndividual(): void { 
      'SELECT "roles_target_id" FROM "user__roles" WHERE "entity_id" = "2"',

      'SELECT "base_table"."vid" AS "vid", "base_table"."nid" AS "nid" FROM "node" "base_table" INNER JOIN "node" "node" ON "node"."nid" = "base_table"."nid" INNER JOIN "node_field_data" "node_field_data" ON "node_field_data"."nid" = "base_table"."nid" WHERE ("node"."uuid" IN ("677f9911-f002-4639-9891-5c39e8b00d9d")) AND ("node_field_data"."default_langcode" IN (1))',

      'SELECT "revision"."vid" AS "vid", "revision"."langcode" AS "langcode", "revision"."revision_uid" AS "revision_uid", "revision"."revision_timestamp" AS "revision_timestamp", "revision"."revision_log" AS "revision_log", "revision"."revision_default" AS "revision_default", "base"."nid" AS "nid", "base"."type" AS "type", "base"."uuid" AS "uuid", CASE "base"."vid" WHEN "revision"."vid" THEN 1 ELSE 0 END AS "isDefaultRevision" FROM "node" "base" INNER JOIN "node_revision" "revision" ON "revision"."vid" = "base"."vid" WHERE "base"."nid" IN (1)',

      'SELECT "revision".* FROM "node_field_revision" "revision" WHERE ("revision"."nid" IN (1)) AND ("revision"."vid" IN ("1")) ORDER BY "revision"."nid" ASC',

      'SELECT "node_field_data"."nid" AS "nid", "node_field_data"."langcode" AS "langcode", "node__body"."body_value" AS "body_value", "node__body"."body_format" AS "body_format" FROM "node_field_data" "node_field_data" LEFT OUTER JOIN "node__body" "node__body" ON "node__body"."entity_id" = "node_field_data"."nid" AND "node__body"."langcode" = "node_field_data"."langcode" AND "node__body"."deleted" = 0 WHERE ("node_field_data"."nid" IN (1)) AND ("node_field_data"."langcode" IN ("en", "und", "zxx"))',

      'SELECT "node_field_data".*, "node_field_data"."langcode" AS "node_field_data__langcode", "node__body"."body_value" AS "body_value", "node__body"."body_format" AS "body_format" FROM "node_field_data" "node_field_data" LEFT OUTER JOIN "node__body" "node__body" ON "node__body"."entity_id" = "node_field_data"."nid" AND "node__body"."langcode" = "node_field_data"."langcode" AND "node__body"."deleted" = 0 WHERE "node_field_data"."nid" IN (1)',

      'SELECT "base_table"."vid" AS "vid", "base_table"."nid" AS "nid" FROM "node" "base_table" INNER JOIN "node" "node" ON "node"."nid" = "base_table"."nid" INNER JOIN "node_field_data" "node_field_data" ON "node_field_data"."nid" = "base_table"."nid" WHERE ("node"."uuid" IN ("677f9911-f002-4639-9891-5c39e8b00d9d")) AND ("node_field_data"."default_langcode" IN (1))',

      'SELECT "base_table"."vid" AS "vid", "base_table"."nid" AS "nid" FROM "node" "base_table" INNER JOIN "node" "node" ON "node"."nid" = "base_table"."nid" INNER JOIN "node_field_data" "node_field_data" ON "node_field_data"."nid" = "base_table"."nid" WHERE ("node"."uuid" IN ("677f9911-f002-4639-9891-5c39e8b00d9d")) AND ("node_field_data"."default_langcode" IN (1))',

      'SELECT "base_table"."vid" AS "vid", "base_table"."nid" AS "nid" FROM "node" "base_table" INNER JOIN "node" "node" ON "node"."nid" = "base_table"."nid" INNER JOIN "node_field_data" "node_field_data" ON "node_field_data"."nid" = "base_table"."nid" WHERE ("node"."uuid" IN ("677f9911-f002-4639-9891-5c39e8b00d9d")) AND ("node_field_data"."default_langcode" IN (1))',
@@ -210,7 +209,7 @@ public function testGetIndividual(): void { 
    $this->assertSame($expected_queries, $recorded_queries);



    $expected = [

      'QueryCount' => 15,

      'QueryCount' => 14,

      'CacheGetCount' => 42,

      'CacheGetCountByBin' => [

        'config' => 7,

core/profiles/demo_umami/tests/src/FunctionalJavascript/OpenTelemetryAuthenticatedPerformanceTest.php

+1 −1
Original line number Diff line number Diff line
@@ -126,7 +126,7 @@ protected function doTestNodePageAdministrator(): void { 
    }, 'administratorNodePage');



    $expected = [

      'QueryCount' => 279,

      'QueryCount' => 272,

      'CacheGetCount' => 266,

      'CacheGetCountByBin' => [

        'config' => 62,

core/profiles/demo_umami/tests/src/FunctionalJavascript/OpenTelemetryFrontPagePerformanceTest.php

+2 −2
Original line number Diff line number Diff line
@@ -52,7 +52,7 @@ protected function testFrontPageColdCache(): void { 
    $this->assertSession()->pageTextContains('Umami');



    $expected = [

      'QueryCount' => 195,

      'QueryCount' => 192,

      'CacheGetCount' => 241,

      'CacheSetCount' => 246,

      'CacheDeleteCount' => 0,
@@ -122,7 +122,7 @@ protected function testFrontPageCoolCache(): void { 
    }, 'umamiFrontPageCoolCache');



    $expected = [

      'QueryCount' => 61,

      'QueryCount' => 58,

      'CacheGetCount' => 172,

      'CacheSetCount' => 74,

      'CacheDeleteCount' => 0,