Unverified Commit dcd666c6 authored by alexpott's avatar alexpott
Browse files

Issue #2746541 by quietone, alexpott, maxocub, Wim Leers, mikelutz, jofitz,...

Issue #2746541 by quietone, alexpott, maxocub, Wim Leers, mikelutz, jofitz, masipila, firewaller, Madhura BK, Gábor Hojtsy, catch, heddn, plach, hchonov: Migrate D6 and D7 node revision translations to D8
parent ae2efb40
......@@ -719,6 +719,19 @@
*/
$settings['entity_update_backup'] = TRUE;
/**
* Node migration type.
*
* This is used to force the migration system to use the classic node migrations
* instead of the default complete node migrations. The migration system will
* use the classic node migration only if there are existing migrate_map tables
* for the classic node migrations and they contain data. These tables may not
* exist if you are developing custom migrations and do not want to use the
* complete node migrations. Set this to TRUE to force the use of the classic
* node migrations.
*/
$settings['migrate_node_migrate_type_classic'] = FALSE;
/**
* Load local development override configuration, if available.
*
......
......@@ -24,9 +24,12 @@ process:
-
plugin: migration_lookup
migration:
- d6_node_complete
- d6_node
- d6_node_translation
source: nid
-
plugin: node_complete_node_lookup
-
plugin: skip_on_empty
method: row
......
......@@ -25,9 +25,12 @@ process:
-
plugin: migration_lookup
migration:
- d7_node_complete
- d7_node
- d7_node_translation
source: nid
-
plugin: node_complete_node_lookup
-
plugin: skip_on_empty
method: row
......
......@@ -14,6 +14,7 @@ provider:
target_types:
node:
- d6_node_translation
- d6_node_complete
# The source plugin will be set by the deriver.
source:
plugin: empty
......
......@@ -11,8 +11,12 @@ process:
dest_nid:
-
plugin: migration_lookup
migration: d6_node_translation
migration:
- d6_node_complete
- d6_node_translation
source: nid
-
plugin: node_complete_node_translation_lookup
-
plugin: skip_on_empty
method: row
......
......@@ -14,6 +14,7 @@ provider:
target_types:
node:
- d7_node_translation
- d7_node_complete
# The source plugin will be set by the deriver.
source:
plugin: empty
......
......@@ -61,6 +61,8 @@ process:
# d7_node_translation mapping tables to find the new node ID.
plugin: migration_lookup
migration:
- d6_node_complete
- d7_node_complete
- d6_node_translation
- d7_node_translation
no_stub: true
......
......@@ -9,6 +9,8 @@ finished:
- menu_link_content
locale: content_translation
menu: content_translation
# Node revision translations.
node: content_translation
statistics: statistics
taxonomy: content_translation
7:
......@@ -20,7 +22,8 @@ finished:
locale: content_translation
menu: content_translation
statistics: statistics
# Node revision translations.
node: content_translation
not_finished:
# Also D6 and D7 node revision translations.
6:
......@@ -29,9 +32,6 @@ not_finished:
i18n: content_translation
# Taxonomy term references.
i18ntaxonomy: content_translation
# Node revision translations.
# https://www.drupal.org/project/drupal/issues/2746541
node: content_translation
7:
# @TODO: Move to finished when remaining Drupal 7 i18n issues are resolved.
# See https://www.drupal.org/project/drupal/issues/2208401
......@@ -42,6 +42,3 @@ not_finished:
# @TODO: Remove when taxonomy term field translations are migrated.
# See https://www.drupal.org/project/drupal/issues/3073050
i18n_taxonomy: content_translation
# Node revision translations.
# https://www.drupal.org/project/drupal/issues/2746541
node: content_translation
......@@ -3,6 +3,7 @@
namespace Drupal\migrate\Audit;
use Drupal\Core\StringTranslation\StringTranslationTrait;
use Drupal\migrate\Plugin\migrate\destination\EntityContentComplete;
use Drupal\migrate\Plugin\MigrationInterface;
/**
......@@ -33,11 +34,12 @@ public function audit(MigrationInterface $migration) {
throw new AuditException($migration, "ID map does not implement $interface");
}
if ($destination->getHighestId() > $id_map->getHighestId()) {
if ($destination->getHighestId() > $id_map->getHighestId() || ($destination instanceof EntityContentComplete && !$this->auditEntityComplete($migration))) {
return AuditResult::fail($migration, [
$this->t('The destination system contains data which was not created by a migration.'),
]);
}
return AuditResult::pass($migration);
}
......@@ -55,4 +57,44 @@ public function auditMultiple(array $migrations) {
return $conflicts;
}
/**
* Audits an EntityComplete migration.
*
* @param \Drupal\migrate\Plugin\MigrationInterface $migration
* The migration to audit.
*
* @return bool
* TRUE if the audit passes and FALSE if not.
*
* @todo Refactor in https://www.drupal.org/project/drupal/issues/3061676 or
* https://www.drupal.org/project/drupal/issues/3091004
*/
private function auditEntityComplete(MigrationInterface $migration) {
$map_table = $migration->getIdMap()->mapTableName();
$database = \Drupal::database();
if (!$database->schema()->tableExists($map_table)) {
throw new \InvalidArgumentException();
}
$query = $database->select($map_table, 'map')
->fields('map', ['destid2'])
->range(0, 1)
->orderBy('destid2', 'DESC');
$max = (int) $query->execute()->fetchField();
// Make a migration based on node_complete but with an entity_revision
// destination.
$revision_migration = $migration->getPluginDefinition();
$revision_migration['id'] = $migration->getPluginId() . '-revision';
$revision_migration['destination']['plugin'] = 'entity_revision:node';
$revision_migration = \Drupal::service('plugin.manager.migration')->createStubMigration($revision_migration);
// Get the highest node revision ID.
$destination = $revision_migration->getDestinationPlugin();
$highest = $destination->getHighestId();
return $max <= $highest;
}
}
<?php
namespace Drupal\migrate\Plugin\Derivative;
use Drupal\migrate\Plugin\migrate\destination\EntityContentComplete;
/**
* MigrateEntityComplete deriver.
*/
class MigrateEntityComplete extends MigrateEntity {
/**
* {@inheritdoc}
*/
public function getDerivativeDefinitions($base_plugin_definition) {
foreach ($this->entityDefinitions as $entity_type => $entity_info) {
$this->derivatives[$entity_type] = [
'id' => "entity_complete:$entity_type",
'class' => EntityContentComplete::class,
'requirements_met' => 1,
'provider' => $entity_info->getProvider(),
];
}
return $this->derivatives;
}
}
<?php
namespace Drupal\migrate\Plugin\migrate\destination;
use Drupal\Core\Entity\ContentEntityInterface;
use Drupal\Core\Entity\EntityChangedInterface;
use Drupal\migrate\EntityFieldDefinitionTrait;
use Drupal\migrate\Row;
/**
* Provides a destination for migrating the entire entity revision table.
*
* @MigrateDestination(
* id = "entity_complete",
* deriver = "Drupal\migrate\Plugin\Derivative\MigrateEntityComplete"
* )
*/
class EntityContentComplete extends EntityContentBase {
use EntityFieldDefinitionTrait;
/**
* {@inheritdoc}
*/
public function getIds() {
$ids = [];
$id_key = $this->getKey('id');
$ids[$id_key] = $this->getDefinitionFromEntity($id_key);
$revision_key = $this->getKey('revision');
if ($revision_key) {
$ids[$revision_key] = $this->getDefinitionFromEntity($revision_key);
}
$langcode_key = $this->getKey('langcode');
if ($langcode_key) {
$ids[$langcode_key] = $this->getDefinitionFromEntity($langcode_key);
}
return $ids;
}
/**
* Gets the entity.
*
* @param \Drupal\migrate\Row $row
* The row object.
* @param array $old_destination_id_values
* The old destination IDs.
*
* @return \Drupal\Core\Entity\EntityInterface
* The entity.
*/
protected function getEntity(Row $row, array $old_destination_id_values) {
$revision_id = $old_destination_id_values
? $old_destination_id_values[1]
: $row->getDestinationProperty($this->getKey('revision'));
// If we are re-running a migration with set revision IDs and the
// destination revision ID already exists then do not create a new revision.
if (!empty($revision_id) && ($entity = $this->storage->loadRevision($revision_id))) {
$entity->setNewRevision(FALSE);
}
elseif (($entity_id = $row->getDestinationProperty($this->getKey('id'))) && ($entity = $this->storage->load($entity_id))) {
// We want to create a new entity. Set enforceIsNew() FALSE is necessary
// to properly save a new entity while setting the ID. Without it, the
// system would see that the ID is already set and assume it is an update.
$entity->enforceIsNew(FALSE);
// Intentionally create a new revision. Setting new revision TRUE here may
// not be necessary, it is done for clarity.
$entity->setNewRevision(TRUE);
}
else {
// Attempt to set the bundle.
if ($bundle = $this->getBundle($row)) {
$row->setDestinationProperty($this->getKey('bundle'), $bundle);
}
// Stubs might need some required fields filled in.
if ($row->isStub()) {
$this->processStubRow($row);
}
$entity = $this->storage->create($row->getDestination());
$entity->enforceIsNew();
}
// We need to update the entity, so that the destination row IDs are
// correct.
$entity = $this->updateEntity($entity, $row);
$entity->isDefaultRevision(TRUE);
if ($entity instanceof EntityChangedInterface && $entity instanceof ContentEntityInterface) {
// If we updated any untranslatable fields, update the timestamp for the
// other translations.
/** @var \Drupal\Core\Entity\ContentEntityInterface|\Drupal\Core\Entity\EntityChangedInterface $entity */
foreach ($entity->getTranslationLanguages() as $langcode => $language) {
// If we updated an untranslated field, then set the changed time for
// for all translations to match the current row that we are saving.
// In this context, getChangedTime() should return the value we just
// set in the updateEntity() call above.
if ($entity->getTranslation($langcode)->hasTranslationChanges()) {
$entity->getTranslation($langcode)->setChangedTime($entity->getChangedTime());
}
}
}
return $entity;
}
/**
* {@inheritdoc}
*/
protected function save(ContentEntityInterface $entity, array $old_destination_id_values = []) {
parent::save($entity, $old_destination_id_values);
return [
$entity->id(),
$entity->getRevisionId(),
];
}
}
......@@ -3,6 +3,7 @@
namespace Drupal\migrate\Plugin\migrate\id_map;
use Drupal\Core\Database\DatabaseException;
use Drupal\Core\Database\DatabaseExceptionWrapper;
use Drupal\Core\Field\BaseFieldDefinition;
use Drupal\Core\Plugin\ContainerFactoryPluginInterface;
use Drupal\Core\Plugin\PluginBase;
......@@ -591,7 +592,16 @@ public function lookupDestinationIds(array $source_id_values) {
}
}
return $query->execute()->fetchAll(\PDO::FETCH_NUM);
try {
return $query->execute()->fetchAll(\PDO::FETCH_NUM);
}
catch (DatabaseExceptionWrapper $e) {
// It's possible that the query will cause an exception to be thrown. For
// example, the URL alias migration uses a dummy node ID of 'INVALID_NID'
// to cause the lookup to return no results. On PostgreSQL this causes an
// exception because 'INVALID_NID' is not the expected type.
return [];
}
}
/**
......
......@@ -5,9 +5,64 @@
* Contains install and update functions for Migrate Drupal
*/
use Drupal\Core\Database\Database;
/**
* Implements hook_update_last_removed().
*/
function migrate_drupal_update_last_removed() {
return 8601;
}
/**
* Add revision ID to entity reference translation migrate_map tables..
*/
function migrate_drupal_update_8901(&$sandbox) {
$schema = Database::getConnection()->schema();
$table_expression = 'migrate_map%entity_reference_translation%node%';
$tables = $schema->findTables($table_expression);
foreach ($tables as $table) {
// Move language code to sourceid3.
$spec = [
'type' => 'varchar',
'length' => 12,
'not null' => TRUE,
];
$schema->changeField($table, 'sourceid2', 'sourceid3', $spec);
// Add revision ID.
$spec = [
'type' => 'int',
'unsigned' => TRUE,
'not null' => TRUE,
'default' => 0,
];
$schema->addField($table, 'sourceid2', $spec);
// Add sourceid2 to index.
$spec = [
'fields' => [
'sourceid1' => [
'type' => 'int',
'not_null' => TRUE,
],
'sourceid2' => [
'type' => 'int',
'not_null' => TRUE,
],
'sourceid3' => [
'type' => 'varchar',
'length' => 12,
'not null' => TRUE,
],
],
];
$fields = [
'sourceid1',
'sourceid2',
'sourceid3',
];
$schema->dropIndex($table, 'source');
$schema->addIndex($table, 'source', $fields, $spec);
}
}
......@@ -46,7 +46,8 @@ function migrate_drupal_migration_plugins_alter(&$definitions) {
],
];
$vocabulary_migration = \Drupal::service('plugin.manager.migration')->createStubMigration($vocabulary_migration_definition);
$translation_active = \Drupal::service('module_handler')->moduleExists('content_translation');
$module_handler = \Drupal::service('module_handler');
$translation_active = $module_handler->moduleExists('content_translation');
try {
$source_plugin = $vocabulary_migration->getSourcePlugin();
......@@ -89,6 +90,5 @@ function migrate_drupal_migration_plugins_alter(&$definitions) {
// When the definitions are loaded it is possible the tables will not
// exist.
}
}
}
......@@ -121,8 +121,29 @@ protected function createDatabaseStateSettings(array $database, $drupal_version)
*/
protected function getMigrations($database_state_key, $drupal_version) {
$version_tag = 'Drupal ' . $drupal_version;
/** @var \Drupal\migrate\Plugin\Migration[] $all_migrations */
/** @var \Drupal\migrate\Plugin\MigrationInterface[] $all_migrations */
$all_migrations = $this->getMigrationPluginManager()->createInstancesByTag($version_tag);
// Unset the node migrations that should not run based on the type of node
// migration. That is, if this is a complete node migration then unset the
// classic node migrations and if this is a classic node migration then
// unset the complete node migrations.
$type = NodeMigrateType::getNodeMigrateType(\Drupal::database(), $drupal_version);
switch ($type) {
case NodeMigrateType::NODE_MIGRATE_TYPE_COMPLETE:
$patterns = '/(d' . $drupal_version . '_node:)|(d' . $drupal_version . '_node_translation:)|(d' . $drupal_version . '_node_revision:)|(d7_node_entity_translation:)/';
break;
case NodeMigrateType::NODE_MIGRATE_TYPE_CLASSIC:
$patterns = '/(d' . $drupal_version . '_node_complete:)/';
break;
}
foreach ($all_migrations as $key => $migrations) {
if (preg_match($patterns, $key)) {
unset($all_migrations[$key]);
}
}
$migrations = [];
foreach ($all_migrations as $migration) {
// Skip migrations tagged with any of the follow-up migration tags. They
......@@ -185,7 +206,7 @@ protected function getFollowUpMigrationTags() {
* A string representing the major branch of Drupal core (e.g. '6' for
* Drupal 6.x), or FALSE if no valid version is matched.
*/
protected function getLegacyDrupalVersion(Connection $connection) {
public static function getLegacyDrupalVersion(Connection $connection) {
// Don't assume because a table of that name exists, that it has the columns
// we're querying. Catch exceptions and report that the source database is
// not Drupal.
......
<?php
namespace Drupal\migrate_drupal;
use Drupal\Core\Database\Connection;
use Drupal\Core\Site\Settings;
/**
* Provides a class to determine the type of migration.
*/
final class NodeMigrateType {
use MigrationConfigurationTrait;
/**
* Only the complete node migration map tables are in use.
*/
const NODE_MIGRATE_TYPE_COMPLETE = 'COMPLETE';
/**
* Only the classic node migration map tables are in use.
*/
const NODE_MIGRATE_TYPE_CLASSIC = 'CLASSIC';
/**
* Determines the type of node migration to be used.
*
* The node complete migration is the default. It is not used when there
* are existing tables for dN_node.
*
* @param \Drupal\Core\Database\Connection $connection
* The connection to the target database.
* @param string|false $version
* The Drupal version of the source database, FALSE if it cannot be
* determined.
*
* @return string
* The migrate type.
*
* @internal
*/
public static function getNodeMigrateType(Connection $connection, $version) {
$migrate_node_migrate_type_classic = Settings::get('migrate_node_migrate_type_classic', FALSE);
if ($migrate_node_migrate_type_classic) {
return static::NODE_MIGRATE_TYPE_CLASSIC;
}
$migrate_type = static::NODE_MIGRATE_TYPE_COMPLETE;
if ($version) {
// Create the variable name, 'node_has_rows' or 'node_complete_exists' and
// set it the default value, FALSE.
$node_has_rows = FALSE;
$node_complete_has_rows = FALSE;
// Find out what migrate map tables have rows for the node migrations.
// It is either the classic, 'dN_node', or the complete,
// 'dN_node_complete', or both. This is used to determine which migrations
// are run and if migrations using the node migrations in a
// migration_lookup are altered.
$bases = ['node', 'node_complete'];
$tables = $connection->schema()
->findTables('migrate_map_d' . $version . '_node%');
foreach ($bases as $base) {
$has_rows = $base . '_has_rows';
$base_tables = preg_grep('/^migrate_map_d' . $version . '_' . $base . '_{2}.*$/', $tables);
// Set the has_rows True when a map table has rows with a positive
// count for the matched migration.
foreach ($base_tables as $base_table) {
if ($connection->schema()->tableExists($base_table)) {
$count = $connection->select($base_table)->countQuery()
->execute()->fetchField();
if ($count > 0) {
$$has_rows = TRUE;
break;
}
}
}
}
// Set the node migration type to use.
if ($node_has_rows && !$node_complete_has_rows) {
$migrate_type = static::NODE_MIGRATE_TYPE_CLASSIC;
}
}
return $migrate_type;
}
}
<
<?php
namespace Drupal\migrate_drupal\Plugin\migrate\process;
use Drupal\migrate\MigrateExecutableInterface;
use Drupal\migrate\ProcessPluginBase;
use Drupal\migrate\Row;
/**
* Returns only the nid from migration_lookup on node_complete migration.
*
* It is possible that migration_lookups that use the classic node migrations
* in the migration key have been altered to include the complete node
* migration. The classic node migration and complete node migration have a
* different number of destination keys. This process plugin will ensure that