fix: #3593963 Field loading can hit the MySQL 61 table join limit if there are ~60 fields (364e29e1) · Commits · project / drupal

core/lib/Drupal/Core/Entity/Sql/SqlContentEntityStorage.php

+249 −98

Original line number	Diff line number	Diff line
		@@ -6,6 +6,7 @@
		use Drupal\Core\Cache\MemoryCache\MemoryCacheInterface;
		use Drupal\Core\Database\Connection;
		use Drupal\Core\Database\DatabaseExceptionWrapper;
		use Drupal\Core\Database\Query\SelectInterface;
		use Drupal\Core\Database\SchemaException;
		use Drupal\Core\Database\Statement\FetchAs;
		use Drupal\Core\Database\TransactionOutOfOrderException;
		@@ -43,6 +44,21 @@
		*/
		class SqlContentEntityStorage extends ContentEntityStorageBase implements SqlEntityStorageInterface, DynamicallyFieldableEntityStorageSchemaInterface, EntityBundleListenerInterface {

		/**
		* The minimum chunk size for field loading.
		*
		* This avoids running into database join limits.
		*
		* @see https://dev.mysql.com/doc/refman/9.7/en/join.html
		* @see https://sqlite.org/limits.html#max_sql_length
		*
		* When calculating the chunks, the last chunk is appended to the previous one
		* so that the maximum fields to load at once is double this number. This
		* ensures that 26 fields are loaded as a single group of 26, instead of 26
		* and 1, or that 51 fields are loaded in groups of 25 and 26.
		*/
		protected const int FIELD_MINIMUM_CHUNK_SIZE = 25;

		/**
		* The entity type's field storage definitions.
		*
		@@ -1293,8 +1309,108 @@ protected function loadFromDedicatedTables(array &$values, $load_from_revision,
		$base_query->addField($base_table, $this->langcodeKey, $base_langcode_alias);
		}

		$query = clone $base_query;
		// When the number of fields exceeds the chunk size, split the fields to
		// load into chunks. The SQL limits for table joins are over 60, so always
		// combine the last two chunks. This means that 26 fields end up in a single
		// chunk of 26, instead of chunks of 25 and 1.
		$load_shared_table_fields = TRUE;
		if (count($single_cardinality_fields) > static::FIELD_MINIMUM_CHUNK_SIZE) {
		$chunks = array_chunk($single_cardinality_fields, static::FIELD_MINIMUM_CHUNK_SIZE, TRUE);
		$last_chunk = array_pop($chunks);
		$last_key = array_key_last($chunks);
		$chunks[$last_key] = array_merge($chunks[$last_key], $last_chunk);
		}
		else {
		$chunks = [$single_cardinality_fields];
		}
		foreach ($chunks as $fields) {
		$this->loadSingleCardinalityFields($values, $base_query, $base_table, $id_key, $base_id_key, $base_langcode_alias, $load_from_revision, $fields, $definitions, $field_columns, $field_definition_columns, $default_langcodes, $load_shared_table_fields, $translations);
		$load_shared_table_fields = FALSE;
		}

		if ($multiple_cardinality_fields) {
		if (count($multiple_cardinality_fields) > static::FIELD_MINIMUM_CHUNK_SIZE) {
		$chunks = array_chunk($multiple_cardinality_fields, static::FIELD_MINIMUM_CHUNK_SIZE, TRUE);
		$last_chunk = array_pop($chunks);
		$last_key = array_key_last($chunks);
		$chunks[$last_key] = array_merge($chunks[$last_key], $last_chunk);
		}
		else {
		$chunks = [$multiple_cardinality_fields];
		}
		foreach ($chunks as $fields) {
		$this->loadMultipleCardinalityFields($values, $base_query, $base_table, $id_key, $base_id_key, $base_langcode_alias, $load_from_revision, $fields, $definitions, $field_columns, $field_definition_columns, $default_langcodes);
		}
		// Ensure that all of the deltas from all of the multiple cardinality
		// fields are returned in the correct order.
		foreach ($values as &$fields) {
		foreach ($fields as $field_name => &$field_data) {
		if (isset($multiple_cardinality_fields[$field_name])) {
		foreach ($field_data as &$language_data) {
		ksort($language_data);
		}
		}
		}
		}
		}
		}

		/**
		* Load single cardinality fields.
		*
		* @param array &$values
		* The entity values populated so far.
		* @param \Drupal\Core\Database\Query\SelectInterface $base_query
		* The base database query.
		* @param string $base_table
		* The base table used in the query.
		* @param string $id_key
		* The ID key depending on whether regular entities or revisions are being
		* loaded.
		* @param string $base_id_key
		* The base ID key depending on whether regular entities or revisions are
		* being loaded.
		* @param string $base_langcode_alias
		* The base langcode alias.
		* @param bool $load_from_revision
		* Whether we're loading from revisions.
		* @param array $single_cardinality_fields
		* The single cardinality fields to load.
		* @param array $definitions
		* The field definitions.
		* @param array $field_columns
		* The field columns.
		* @param array $field_definition_columns
		* The field definition columns.
		* @param array $default_langcodes
		* The default langcodes.
		* @param bool $load_shared_table_fields
		* Whether to also load fields from the shared data/revision data tables.
		* Should be TRUE only on the first chunk iteration to avoid overwriting
		* shared field values.
		* @param array $translations
		* The translations array, keyed by entity ID and langcode.
		*/
		private function loadSingleCardinalityFields(
		array &$values,
		SelectInterface $base_query,
		string $base_table,
		string $id_key,
		string $base_id_key,
		string $base_langcode_alias,
		bool $load_from_revision,
		array $single_cardinality_fields,
		array $definitions,
		array $field_columns,
		array $field_definition_columns,
		array $default_langcodes,
		bool $load_shared_table_fields = TRUE,
		array &$translations = [],
		): void {
		$query = clone $base_query;
		$shared_fields = [];
		$table_mapping = $this->getTableMapping();
		if ($load_shared_table_fields) {
		if ($this->revisionDataTable) {
		// Find revisioned fields that are not entity keys. Exclude the langcode
		// key as the base table holds only the default language.
		@@ -1318,8 +1434,8 @@ protected function loadFromDedicatedTables(array &$values, $load_from_revision,
		foreach ($data_fields as $data_field) {
		// \Drupal\Core\Entity\Sql\TableMappingInterface::getColumnNames()
		// returns an array keyed by property names so remove the keys
		// before array_merge() to avoid losing data with fields having the
		// same columns i.e. value.
		// before array_merge() to avoid losing data with fields having
		// the same columns i.e. value.
		$column_names[] = array_values($table_mapping->getColumnNames($data_field));
		}
		$column_names = array_merge(...$column_names);
		@@ -1335,6 +1451,7 @@ protected function loadFromDedicatedTables(array &$values, $load_from_revision,
		$field_definition_columns[$field_name] = $storage_definition->getColumns();
		$field_columns[$field_name] = $table_mapping->getColumnNames($field_name);
		}
		}

		// Add a left join for each single cardinality field.
		foreach ($single_cardinality_fields as $field_name => $storage_definition) {
		@@ -1347,7 +1464,7 @@ protected function loadFromDedicatedTables(array &$values, $load_from_revision,
		else {
		$query->leftJoin($table, $table, "[$table].[$id_key] = [$base_table].[$base_id_key] AND [$table].[deleted] = 0");
		}
		$query->fields($table, $this->tableMapping->getColumnNames($field_name));
		$query->fields($table, $table_mapping->getColumnNames($field_name));
		}

		$results = $query->execute();
		@@ -1416,7 +1533,53 @@ protected function loadFromDedicatedTables(array &$values, $load_from_revision,
		}
		}
		}
		if ($multiple_cardinality_fields) {
		}

		/**
		* Load multiple cardinality fields.
		*
		* @param array &$values
		* The entity values populated so far.
		* @param \Drupal\Core\Database\Query\SelectInterface $base_query
		* The base database query.
		* @param string $base_table
		* The base table used in the query.
		* @param string $id_key
		* The ID key depending on whether regular entities or revisions are being
		* loaded.
		* @param string $base_id_key
		* The base ID key depending on whether regular entities or revisions are
		* being loaded.
		* @param string $base_langcode_alias
		* The base langcode alias.
		* @param bool $load_from_revision
		* Whether we're loading from revisions.
		* @param array $multiple_cardinality_fields
		* The multiple cardinality fields to load.
		* @param array $definitions
		* The field definitions.
		* @param array $field_columns
		* The field columns.
		* @param array $field_definition_columns
		* The field definition columns.
		* @param array $default_langcodes
		* The default langcodes.
		*/
		private function loadMultipleCardinalityFields(
		array &$values,
		SelectInterface $base_query,
		string $base_table,
		string $id_key,
		string $base_id_key,
		string $base_langcode_alias,
		bool $load_from_revision,
		array $multiple_cardinality_fields,
		array $definitions,
		array $field_columns,
		array $field_definition_columns,
		array $default_langcodes,
		): void {
		$table_mapping = $this->getTableMapping();
		$query = clone $base_query;
		$delta_keys = [];
		foreach ($multiple_cardinality_fields as $field_name => $storage_definition) {
		@@ -1476,18 +1639,6 @@ protected function loadFromDedicatedTables(array &$values, $load_from_revision,
		}
		}
		}
		// Ensure that all of the deltas from all of the multiple cardinality
		// fields are returned in the correct order.
		foreach ($values as &$fields) {
		foreach ($fields as $field_name => &$field_data) {
		if (isset($multiple_cardinality_fields[$field_name])) {
		foreach ($field_data as &$language_data) {
		ksort($language_data);
		}
		}
		}
		}
		}
		}

		/**

core/tests/Drupal/KernelTests/Core/Entity/Sql/SqlContentEntityStorageTest.php

+71 −2

Original line number	Diff line number	Diff line
		@@ -9,7 +9,10 @@
		use Drupal\Core\Entity\EntityTypeInterface;
		use Drupal\Core\Entity\Sql\DefaultTableMapping;
		use Drupal\Core\Field\FieldStorageDefinitionInterface;
		use Drupal\KernelTests\KernelTestBase;
		use Drupal\field\Entity\FieldConfig;
		use Drupal\field\Entity\FieldStorageConfig;
		use Drupal\KernelTests\Core\Entity\EntityKernelTestBase;
		use PHPUnit\Framework\Attributes\DataProvider;
		use PHPUnit\Framework\Attributes\Group;
		use PHPUnit\Framework\Attributes\RunTestsInSeparateProcesses;

		@@ -18,7 +21,15 @@
		*/
		#[Group('Entity')]
		#[RunTestsInSeparateProcesses]
		class SqlContentEntityStorageTest extends KernelTestBase {
		class SqlContentEntityStorageTest extends EntityKernelTestBase {

		/**
		* {@inheritdoc}
		*/
		public function setUp(): void {
		parent::setUp();
		$this->installEntitySchema('entity_test_rev');
		}

		/**
		* Tests that only one SQL DELETE is executed on dedicated data tables.
		@@ -111,4 +122,62 @@ public function testDeleteFromDedicatedTablesExecutesOneDelete(): void {
		$this->assertCount(1, $dedicatedRevisionTableDeleteQueries, 'Only one DELETE query on the dedicated revision data table was executed.');
		}

		/**
		* Tests that entities with a large number (65+) of fields load successfully.
		*/
		#[DataProvider('providerCardinality')]
		public function testEntityWithManyFieldsLoad(int $cardinality): void {
		$fieldCount = 71;
		for ($i = 1; $i <= $fieldCount; $i++) {
		$fieldStorage = FieldStorageConfig::create([
		'field_name' => 'field_test_' . $i,
		'entity_type' => 'entity_test_rev',
		'type' => 'string',
		'cardinality' => $cardinality,
		]);
		$fieldStorage->save();

		FieldConfig::create([
		'field_storage' => $fieldStorage,
		'bundle' => 'entity_test_rev',
		])->save();
		}

		$values = [
		'name' => 'Test entity',
		'bundle' => 'entity_test_rev',
		];
		for ($i = 1; $i <= $fieldCount; $i++) {
		$values['field_test_' . $i] = 'value_' . $i;
		}
		$storage = $this->container->get('entity_type.manager')->getStorage('entity_test_rev');
		$entity = $storage->create($values);
		$entity->save();
		$id = $entity->id();
		$storage->resetCache();
		$entity = $storage->load($id);
		for ($i = 1; $i <= $fieldCount; $i++) {
		$this->assertSame("value_$i", $entity->get("field_test_$i")->value);
		}
		$revision_id = $entity->getRevisionId();
		$storage->resetCache();
		$revision = $storage->loadRevision($revision_id);
		for ($i = 1; $i <= $fieldCount; $i++) {
		$this->assertSame("value_$i", $revision->get("field_test_$i")->value);
		}
		}

		/**
		* Data provider for testEntityWithManyFieldsLoad().
		*
		* @return array
		* Test cases.
		*/
		public static function providerCardinality(): array {
		return [
		'single cardinality' => [1],
		'unlimited cardinality' => [-1],
		];
		}

		}