Unverified Commit 1b4e034d authored by alexpott's avatar alexpott

Issue #3098282 by quietone, raman.b, Vidushi Mehta, mikelutz, ankithashetty,...

Issue #3098282 by quietone, raman.b, Vidushi Mehta, mikelutz, ankithashetty, alexpott, gapple, larowlan, xjm: SQL error if migration has too many ID fields

(cherry picked from commit 9f3a8615)
parent 4ce4ae9f
......@@ -312,6 +312,8 @@ public function setMessage(MigrateMessageInterface $message) {
/**
* Create the map and message tables if they don't already exist.
*
* @throws \Drupal\Core\Database\DatabaseException
*/
protected function ensureTables() {
if (!$this->getDatabase()->schema()->tableExists($this->mapTableName)) {
......@@ -373,13 +375,46 @@ protected function ensureTables() {
'not null' => FALSE,
'description' => 'Hash of source row data, for detecting changes',
];
$schema = [
'description' => 'Mappings from source identifier value(s) to destination identifier value(s).',
'fields' => $fields,
'primary key' => [$this::SOURCE_IDS_HASH],
'indexes' => $indexes,
];
$this->getDatabase()->schema()->createTable($this->mapTableName, $schema);
// To keep within the MySQL maximum key length of 3072 bytes we try
// different groupings of the source IDs. Groups are created in chunks
// starting at a chunk size equivalent to the number of the source IDs.
// On each loop the chunk size is reduced by one until either the map
// table is successfully created or the chunk_size is less than zero. If
// there are no source IDs the table is created.
$chunk_size = count($source_id_schema);
while ($chunk_size >= 0) {
$indexes = [];
if ($chunk_size > 0) {
foreach (array_chunk(array_keys($source_id_schema), $chunk_size) as $key => $index_columns) {
$index_name = ($key === 0) ? 'source' : "source$key";
$indexes[$index_name] = $index_columns;
}
}
$schema = [
'description' => 'Mappings from source identifier value(s) to destination identifier value(s).',
'fields' => $fields,
'primary key' => [$this::SOURCE_IDS_HASH],
'indexes' => $indexes,
];
try {
$this->getDatabase()
->schema()
->createTable($this->mapTableName, $schema);
break;
}
catch (DatabaseException $e) {
$pdo_exception = $e->getPrevious();
$mysql_index_error = $pdo_exception instanceof \PDOException && $pdo_exception->getCode() === '42000' && $pdo_exception->errorInfo[1] === 1071;
$chunk_size--;
// Rethrow the exception if the source IDs can not be in smaller
// groups.
if (!$mysql_index_error || $chunk_size <= 0) {
throw $e;
}
}
}
// Now do the message table.
if (!$this->getDatabase()->schema()->tableExists($this->messageTableName())) {
......
<?php
namespace Drupal\Tests\migrate\Kernel\Plugin\id_map;
use Drupal\Core\Database\Database;
use Drupal\Core\Database\DatabaseExceptionWrapper;
use Drupal\Tests\migrate\Kernel\MigrateTestBase;
use Drupal\Tests\migrate\Unit\TestSqlIdMap;
use Drupal\migrate\MigrateException;
use Symfony\Component\EventDispatcher\EventDispatcherInterface;
/**
* Tests that the migrate map table is created.
*
* @group migrate
*/
class SqlTest extends MigrateTestBase {
/**
* Database connection.
*
* @var \Drupal\Core\Database\Connection
*/
protected $database;
/**
* Prophesized event dispatcher.
*
* @var object|\Prophecy\Prophecy\ProphecySubjectInterface|\Symfony\Component\EventDispatcher\EventDispatcherInterface
*/
protected $eventDispatcher;
/**
* Definition of a test migration.
*
* @var array
*/
protected $migrationDefinition;
/**
* The migration plugin manager.
*
* @var \Drupal\migrate\Plugin\MigrationPluginManager
*/
protected $migrationPluginManager;
/**
* {@inheritdoc}
*/
public function setUp(): void {
parent::setUp();
$this->database = \Drupal::database();
$this->eventDispatcher = $this->prophesize(EventDispatcherInterface::class)
->reveal();
$this->migrationPluginManager = \Drupal::service('plugin.manager.migration');
$this->migrationDefinition = [
'id' => 'test',
'source' => [
'plugin' => 'embedded_data',
'data_rows' => [
[
'alpha' => '1',
'bravo' => '2',
'charlie' => '3',
'delta' => '4',
'echo' => '5',
],
],
'ids' => [],
],
'process' => [],
'destination' => [
'plugin' => 'null',
],
];
}
/**
* Tests that ensureTables creates the migrate map table.
*
* @dataProvider providerTestEnsureTables
*/
public function testEnsureTables($ids) {
$this->migrationDefinition['source']['ids'] = $ids;
$migration = $this->migrationPluginManager->createStubMigration($this->migrationDefinition);
$map = new TestSqlIdMap($this->database, [], 'test', [], $migration, $this->eventDispatcher);
$map->ensureTables();
// Checks that the map table was created.
$exists = $this->database->schema()->tableExists('migrate_map_test');
$this->assertTrue($exists);
}
/**
* Provides data for testEnsureTables.
*/
public function providerTestEnsureTables() {
return [
'no ids' => [
[],
],
'one id' => [
[
'alpha' => [
'type' => 'string',
],
],
],
'too many' => [
[
'alpha' => [
'type' => 'string',
],
'bravo' => [
'type' => 'string',
],
'charlie' => [
'type' => 'string',
],
'delta' => [
'type' => 'string',
],
'echo ' => [
'type' => 'string',
],
],
],
];
}
/**
* Tests exception is thrown in ensureTables fails.
*
* @dataProvider providerTestFailEnsureTables
*/
public function testFailEnsureTables($ids) {
// This just tests mysql, as other PDO integrations allow longer indexes.
if (Database::getConnection()->databaseType() !== 'mysql') {
$this->markTestSkipped("This test only runs for MySQL");
}
$this->migrationDefinition['source']['ids'] = $ids;
$migration = $this->container
->get('plugin.manager.migration')
->createStubMigration($this->migrationDefinition);
// Use local id map plugin to force an error.
$map = new SqlIdMapTest($this->database, [], 'test', [], $migration, $this->eventDispatcher);
$this->expectException(DatabaseExceptionWrapper::class);
$this->expectExceptionMessage("Syntax error or access violation: 1074 Column length too big for column 'sourceid1' (max = 16383); use BLOB or TEXT instead:");
$map->ensureTables();
}
/**
* Provides data for testFailEnsureTables.
*/
public function providerTestFailEnsureTables() {
return [
'one id' => [
[
'alpha' => [
'type' => 'string',
],
],
],
];
}
}
/**
* Defines a test SQL ID map for use in tests.
*/
class SqlIdMapTest extends TestSqlIdMap implements \Iterator {
/**
* {@inheritdoc}
*/
protected function getFieldSchema(array $id_definition) {
if (!isset($id_definition['type'])) {
return [];
}
switch ($id_definition['type']) {
case 'integer':
return [
'type' => 'int',
'not null' => TRUE,
];
case 'string':
return [
'type' => 'varchar',
'length' => 65536,
'not null' => FALSE,
];
default:
throw new MigrateException($id_definition['type'] . ' not supported');
}
}
}
......@@ -81,4 +81,11 @@ protected function getFieldSchema(array $id_definition) {
}
}
/**
* {@inheritdoc}
*/
public function ensureTables() {
parent::ensureTables();
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment