Issue #3098282 by quietone, raman.b, Vidushi Mehta, mikelutz, ankithashetty,...

Issue #3098282 by quietone, raman.b, Vidushi Mehta, mikelutz, ankithashetty, alexpott, gapple, larowlan, xjm: SQL error if migration has too many ID fields

(cherry picked from commit 9f3a8615)

core/modules/migrate/src/Plugin/migrate/id_map/Sql.php

@@ -312,6 +312,8 @@ public function setMessage(MigrateMessageInterface $message) {
 
   /**
    * Create the map and message tables if they don't already exist.
+   *
+   * @throws \Drupal\Core\Database\DatabaseException
    */
   protected function ensureTables() {
     if (!$this->getDatabase()->schema()->tableExists($this->mapTableName)) {
@@ -373,13 +375,46 @@ protected function ensureTables() {
         'not null' => FALSE,
         'description' => 'Hash of source row data, for detecting changes',
       ];
-      $schema = [
-        'description' => 'Mappings from source identifier value(s) to destination identifier value(s).',
-        'fields' => $fields,
-        'primary key' => [$this::SOURCE_IDS_HASH],
-        'indexes' => $indexes,
-      ];
-      $this->getDatabase()->schema()->createTable($this->mapTableName, $schema);
+
+      // To keep within the MySQL maximum key length of 3072 bytes we try
+      // different groupings of the source IDs. Groups are created in chunks
+      // starting at a chunk size equivalent to the number of the source IDs.
+      // On each loop the chunk size is reduced by one until either the map
+      // table is successfully created or the chunk_size is less than zero. If
+      // there are no source IDs the table is created.
+      $chunk_size = count($source_id_schema);
+      while ($chunk_size >= 0) {
+        $indexes = [];
+        if ($chunk_size > 0) {
+          foreach (array_chunk(array_keys($source_id_schema), $chunk_size) as $key => $index_columns) {
+            $index_name = ($key === 0) ? 'source' : "source$key";
+            $indexes[$index_name] = $index_columns;
+          }
+        }
+        $schema = [
+          'description' => 'Mappings from source identifier value(s) to destination identifier value(s).',
+          'fields' => $fields,
+          'primary key' => [$this::SOURCE_IDS_HASH],
+          'indexes' => $indexes,
+        ];
+
+        try {
+          $this->getDatabase()
+            ->schema()
+            ->createTable($this->mapTableName, $schema);
+          break;
+        }
+        catch (DatabaseException $e) {
+          $pdo_exception = $e->getPrevious();
+          $mysql_index_error = $pdo_exception instanceof \PDOException && $pdo_exception->getCode() === '42000' && $pdo_exception->errorInfo[1] === 1071;
+          $chunk_size--;
+          // Rethrow the exception if the source IDs can not be in smaller
+          // groups.
+          if (!$mysql_index_error || $chunk_size <= 0) {
+            throw $e;
+          }
+        }
+      }
 
       // Now do the message table.
       if (!$this->getDatabase()->schema()->tableExists($this->messageTableName())) {

core/modules/migrate/tests/src/Kernel/Plugin/id_map/SqlTest.php

+<?php
+
+namespace Drupal\Tests\migrate\Kernel\Plugin\id_map;
+
+use Drupal\Core\Database\Database;
+use Drupal\Core\Database\DatabaseExceptionWrapper;
+use Drupal\Tests\migrate\Kernel\MigrateTestBase;
+use Drupal\Tests\migrate\Unit\TestSqlIdMap;
+use Drupal\migrate\MigrateException;
+use Symfony\Component\EventDispatcher\EventDispatcherInterface;
+
+/**
+ * Tests that the migrate map table is created.
+ *
+ * @group migrate
+ */
+class SqlTest extends MigrateTestBase {
+
+  /**
+   * Database connection.
+   *
+   * @var \Drupal\Core\Database\Connection
+   */
+  protected $database;
+
+  /**
+   * Prophesized event dispatcher.
+   *
+   * @var object|\Prophecy\Prophecy\ProphecySubjectInterface|\Symfony\Component\EventDispatcher\EventDispatcherInterface
+   */
+  protected $eventDispatcher;
+
+  /**
+   * Definition of a test migration.
+   *
+   * @var array
+   */
+  protected $migrationDefinition;
+
+  /**
+   * The migration plugin manager.
+   *
+   * @var \Drupal\migrate\Plugin\MigrationPluginManager
+   */
+  protected $migrationPluginManager;
+
+  /**
+   * {@inheritdoc}
+   */
+  public function setUp(): void {
+    parent::setUp();
+    $this->database = \Drupal::database();
+    $this->eventDispatcher = $this->prophesize(EventDispatcherInterface::class)
+      ->reveal();
+    $this->migrationPluginManager = \Drupal::service('plugin.manager.migration');
+
+    $this->migrationDefinition = [
+      'id' => 'test',
+      'source' => [
+        'plugin' => 'embedded_data',
+        'data_rows' => [
+          [
+            'alpha' => '1',
+            'bravo' => '2',
+            'charlie' => '3',
+            'delta' => '4',
+            'echo' => '5',
+          ],
+        ],
+        'ids' => [],
+      ],
+      'process' => [],
+      'destination' => [
+        'plugin' => 'null',
+      ],
+    ];
+  }
+
+  /**
+   * Tests that ensureTables creates the migrate map table.
+   *
+   * @dataProvider providerTestEnsureTables
+   */
+  public function testEnsureTables($ids) {
+    $this->migrationDefinition['source']['ids'] = $ids;
+    $migration = $this->migrationPluginManager->createStubMigration($this->migrationDefinition);
+
+    $map = new TestSqlIdMap($this->database, [], 'test', [], $migration, $this->eventDispatcher);
+    $map->ensureTables();
+
+    // Checks that the map table was created.
+    $exists = $this->database->schema()->tableExists('migrate_map_test');
+    $this->assertTrue($exists);
+  }
+
+  /**
+   * Provides data for testEnsureTables.
+   */
+  public function providerTestEnsureTables() {
+    return [
+      'no ids' => [
+        [],
+      ],
+      'one id' => [
+        [
+          'alpha' => [
+            'type' => 'string',
+          ],
+        ],
+      ],
+      'too many' => [
+        [
+          'alpha' => [
+            'type' => 'string',
+          ],
+          'bravo' => [
+            'type' => 'string',
+          ],
+          'charlie' => [
+            'type' => 'string',
+          ],
+          'delta' => [
+            'type' => 'string',
+          ],
+          'echo ' => [
+            'type' => 'string',
+          ],
+        ],
+      ],
+    ];
+  }
+
+  /**
+   * Tests exception is thrown in ensureTables fails.
+   *
+   * @dataProvider providerTestFailEnsureTables
+   */
+  public function testFailEnsureTables($ids) {
+    // This just tests mysql, as other PDO integrations allow longer indexes.
+    if (Database::getConnection()->databaseType() !== 'mysql') {
+      $this->markTestSkipped("This test only runs for MySQL");
+    }
+
+    $this->migrationDefinition['source']['ids'] = $ids;
+    $migration = $this->container
+      ->get('plugin.manager.migration')
+      ->createStubMigration($this->migrationDefinition);
+
+    // Use local id map plugin to force an error.
+    $map = new SqlIdMapTest($this->database, [], 'test', [], $migration, $this->eventDispatcher);
+
+    $this->expectException(DatabaseExceptionWrapper::class);
+    $this->expectExceptionMessage("Syntax error or access violation: 1074 Column length too big for column 'sourceid1' (max = 16383); use BLOB or TEXT instead:");
+    $map->ensureTables();
+  }
+
+  /**
+   * Provides data for testFailEnsureTables.
+   */
+  public function providerTestFailEnsureTables() {
+    return [
+      'one id' => [
+        [
+          'alpha' => [
+            'type' => 'string',
+          ],
+        ],
+      ],
+    ];
+  }
+
+}
+
+/**
+ * Defines a test SQL ID map for use in tests.
+ */
+class SqlIdMapTest extends TestSqlIdMap implements \Iterator {
+
+  /**
+   * {@inheritdoc}
+   */
+  protected function getFieldSchema(array $id_definition) {
+    if (!isset($id_definition['type'])) {
+      return [];
+    }
+    switch ($id_definition['type']) {
+      case 'integer':
+        return [
+          'type' => 'int',
+          'not null' => TRUE,
+        ];
+
+      case 'string':
+        return [
+          'type' => 'varchar',
+          'length' => 65536,
+          'not null' => FALSE,
+        ];
+
+      default:
+        throw new MigrateException($id_definition['type'] . ' not supported');
+    }
+  }
+
+}

core/modules/migrate/tests/src/Unit/TestSqlIdMap.php

@@ -81,4 +81,11 @@ protected function getFieldSchema(array $id_definition) {
     }
   }
 
+  /**
+   * {@inheritdoc}
+   */
+  public function ensureTables() {
+    parent::ensureTables();
+  }
+
 }