Commit 5ea62f1d authored by larowlan's avatar larowlan

Issue #2876085 by heddn, maxocub, phenaproxima, Jo Fitzgerald, vasi, quietone,...

Issue #2876085 by heddn, maxocub, phenaproxima, Jo Fitzgerald, vasi, quietone, yoroy, masipila, larowlan, neclimdul, krystalcode, catch: Before upgrading, audit for potential ID conflicts
parent 66d19eaa
id: d6_aggregator_feed
label: Aggregator feeds
audit: true
migration_tags:
- Drupal 6
source:
......
id: d6_aggregator_item
label: Aggregator items
audit: true
migration_tags:
- Drupal 6
source:
......
id: d7_aggregator_feed
label: Aggregator feeds
audit: true
migration_tags:
- Drupal 7
source:
......
id: d7_aggregator_item
label: Aggregator items
audit: true
migration_tags:
- Drupal 7
source:
......
id: d6_custom_block
label: Custom blocks
audit: true
migration_tags:
- Drupal 6
source:
......
id: d7_custom_block
label: Custom blocks
audit: true
migration_tags:
- Drupal 7
source:
......
id: d6_comment
label: Comments
audit: true
migration_tags:
- Drupal 6
source:
......
id: d7_comment
label: Comments
audit: true
migration_tags:
- Drupal 7
source:
......
......@@ -2,6 +2,7 @@
# migration as an optional dependency.
id: d6_file
label: Public files
audit: true
migration_tags:
- Drupal 6
source:
......
......@@ -2,6 +2,7 @@
# migration as an optional dependency.
id: d7_file
label: Public files
audit: true
migration_tags:
- Drupal 7
source:
......
id: d7_file_private
label: Private files
audit: true
migration_tags:
- Drupal 7
source:
......
id: d6_menu_links
label: Menu links
audit: true
migration_tags:
- Drupal 6
source:
......
id: d7_menu_links
label: Menu links
audit: true
migration_tags:
- Drupal 7
source:
......
<?php
namespace Drupal\migrate\Audit;
use Drupal\migrate\Plugin\MigrationInterface;
/**
* Defines an exception to throw if an error occurs during a migration audit.
*/
class AuditException extends \RuntimeException {
/**
* AuditException constructor.
*
* @param \Drupal\migrate\Plugin\MigrationInterface $migration
* The migration that caused the exception.
* @param string $message
* The reason the audit failed.
* @param \Exception $previous
* (optional) The previous exception.
*/
public function __construct(MigrationInterface $migration, $message, \Exception $previous = NULL) {
$message = sprintf('Cannot audit migration %s: %s', $migration->id(), $message);
parent::__construct($message, 0, $previous);
}
}
<?php
namespace Drupal\migrate\Audit;
use Drupal\Component\Render\MarkupInterface;
use Drupal\migrate\Plugin\MigrationInterface;
/**
* Encapsulates the result of a migration audit.
*/
class AuditResult implements MarkupInterface, \Countable {
/**
* The audited migration.
*
* @var \Drupal\migrate\Plugin\MigrationInterface
*/
protected $migration;
/**
* The result of the audit (TRUE if passed, FALSE otherwise).
*
* @var bool
*/
protected $status;
/**
* The reasons why the migration passed or failed the audit.
*
* @var string[]
*/
protected $reasons = [];
/**
* AuditResult constructor.
*
* @param \Drupal\migrate\Plugin\MigrationInterface $migration
* The audited migration.
* @param bool $status
* The result of the audit (TRUE if passed, FALSE otherwise).
* @param string[] $reasons
* (optional) The reasons why the migration passed or failed the audit.
*/
public function __construct(MigrationInterface $migration, $status, array $reasons = []) {
if (!is_bool($status)) {
throw new \InvalidArgumentException('Audit results must have a boolean status.');
}
$this->migration = $migration;
$this->status = $status;
array_walk($reasons, [$this, 'addReason']);
}
/**
* Returns the audited migration.
*
* @return \Drupal\migrate\Plugin\MigrationInterface
* The audited migration.
*/
public function getMigration() {
return $this->migration;
}
/**
* Returns the boolean result of the audit.
*
* @return bool
* The result of the audit. TRUE if the migration passed the audit, FALSE
* otherwise.
*/
public function passed() {
return $this->status;
}
/**
* Adds a reason why the migration passed or failed the audit.
*
* @param string|object $reason
* The reason to add. Can be a string or a string-castable object.
*
* @return $this
*/
public function addReason($reason) {
array_push($this->reasons, (string) $reason);
return $this;
}
/**
* Creates a passing audit result for a migration.
*
* @param \Drupal\migrate\Plugin\MigrationInterface $migration
* The audited migration.
* @param string[] $reasons
* (optional) The reasons why the migration passed the audit.
*
* @return static
*/
public static function pass(MigrationInterface $migration, array $reasons = []) {
return new static($migration, TRUE, $reasons);
}
/**
* Creates a failing audit result for a migration.
*
* @param \Drupal\migrate\Plugin\MigrationInterface $migration
* The audited migration.
* @param array $reasons
* (optional) The reasons why the migration failed the audit.
*
* @return static
*/
public static function fail(MigrationInterface $migration, array $reasons = []) {
return new static($migration, FALSE, $reasons);
}
/**
* Implements \Countable::count() for Twig template compatibility.
*
* @return int
*
* @see \Drupal\Component\Render\MarkupInterface
*/
public function count() {
return count($this->reasons);
}
/**
* Returns the reasons the migration passed or failed, as a string.
*
* @return string
*
* @see \Drupal\Component\Render\MarkupInterface
*/
public function __toString() {
return implode("\n", $this->reasons);
}
/**
* Returns the reasons the migration passed or failed, for JSON serialization.
*
* @return string[]
*/
public function jsonSerialize() {
return $this->reasons;
}
}
<?php
namespace Drupal\migrate\Audit;
use Drupal\migrate\Plugin\MigrationInterface;
/**
* Defines an interface for migration auditors.
*
* A migration auditor is a class which can examine a migration to determine if
* it will cause conflicts with data already existing in the destination system.
* What kind of auditing it does, and how it does it, is up to the implementing
* class.
*/
interface AuditorInterface {
/**
* Audits a migration.
*
* @param \Drupal\migrate\Plugin\MigrationInterface $migration
* The migration to audit.
*
* @throws \Drupal\migrate\Audit\AuditException
* If the audit fails.
*
* @return \Drupal\migrate\Audit\AuditResult
* The result of the audit.
*/
public function audit(MigrationInterface $migration);
/**
* Audits a set of migrations.
*
* @param \Drupal\migrate\Plugin\MigrationInterface[] $migrations
* The migrations to audit.
*
* @return \Drupal\migrate\Audit\AuditResult[]
* The audit results, keyed by migration ID.
*/
public function auditMultiple(array $migrations);
}
<?php
namespace Drupal\migrate\Audit;
/**
* Defines an interface for destination and ID maps which track a highest ID.
*
* When implemented by destination plugins, getHighestId() should return the
* highest ID of the destination entity type that exists in the system. So, for
* example, the entity:node plugin should return the highest node ID that
* exists, regardless of whether it was created by a migration.
*
* When implemented by an ID map, getHighestId() should return the highest
* migrated ID of the destination entity type.
*/
interface HighestIdInterface {
/**
* Returns the highest ID tracked by the implementing plugin.
*
* @return int
* The highest ID.
*/
public function getHighestId();
}
<?php
namespace Drupal\migrate\Audit;
use Drupal\Core\StringTranslation\StringTranslationTrait;
use Drupal\migrate\Plugin\MigrationInterface;
/**
* Audits migrations that create content entities in the destination system.
*/
class IdAuditor implements AuditorInterface {
use StringTranslationTrait;
/**
* {@inheritdoc}
*/
public function audit(MigrationInterface $migration) {
$plugin_definition = $migration->getPluginDefinition();
// If the migration does not opt into auditing, it passes.
// @todo Use $migration->isAuditable() when
// https://www.drupal.org/project/drupal/issues/2930832 is in.
if (empty($plugin_definition['audit'])) {
return AuditResult::pass($migration);
}
$interface = HighestIdInterface::class;
$destination = $migration->getDestinationPlugin();
if (!$destination instanceof HighestIdInterface) {
throw new AuditException($migration, "Destination does not implement $interface");
}
$id_map = $migration->getIdMap();
if (!$id_map instanceof HighestIdInterface) {
throw new AuditException($migration, "ID map does not implement $interface");
}
if ($destination->getHighestId() > $id_map->getHighestId()) {
return AuditResult::fail($migration, [
$this->t('The destination system contains data which was not created by a migration.'),
]);
}
return AuditResult::pass($migration);
}
/**
* {@inheritdoc}
*/
public function auditMultiple(array $migrations) {
$conflicts = [];
foreach ($migrations as $migration) {
$migration_id = $migration->getPluginId();
$conflicts[$migration_id] = $this->audit($migration);
}
ksort($conflicts);
return $conflicts;
}
}
......@@ -154,6 +154,17 @@ class Migration extends PluginBase implements MigrationInterface, RequirementsIn
*/
protected $migration_tags = [];
/**
* Whether the migration is auditable.
*
* If set to TRUE, the migration's IDs will be audited. This means that, if
* the highest destination ID is greater than the highest source ID, a warning
* will be displayed that entities might be overwritten.
*
* @var bool
*/
protected $audit = FALSE;
/**
* These migrations, if run, must be executed before this migration.
*
......
......@@ -94,6 +94,10 @@ abstract class Entity extends DestinationBase implements ContainerFactoryPluginI
* The list of bundles this entity type has.
*/
public function __construct(array $configuration, $plugin_id, $plugin_definition, MigrationInterface $migration, EntityStorageInterface $storage, array $bundles) {
$plugin_definition += [
'label' => $storage->getEntityType()->getPluralLabel(),
];
parent::__construct($configuration, $plugin_id, $plugin_definition, $migration);
$this->storage = $storage;
$this->bundles = $bundles;
......
......@@ -9,6 +9,7 @@
use Drupal\Core\Field\FieldTypePluginManagerInterface;
use Drupal\Core\TypedData\TranslatableInterface;
use Drupal\Core\TypedData\TypedDataInterface;
use Drupal\migrate\Audit\HighestIdInterface;
use Drupal\migrate\Plugin\MigrationInterface;
use Drupal\migrate\MigrateException;
use Drupal\migrate\Plugin\MigrateIdMapInterface;
......@@ -18,7 +19,7 @@
/**
* The destination class for all content entities lacking a specific class.
*/
class EntityContentBase extends Entity {
class EntityContentBase extends Entity implements HighestIdInterface {
/**
* Entity manager.
......@@ -111,12 +112,9 @@ protected function save(ContentEntityInterface $entity, array $old_destination_i
}
/**
* Get whether this destination is for translations.
*
* @return bool
* Whether this destination is for translations.
* {@inheritdoc}
*/
protected function isTranslationDestination() {
public function isTranslationDestination() {
return !empty($this->configuration['translations']);
}
......@@ -294,4 +292,16 @@ protected function getDefinitionFromEntity($key) {
] + $field_definition->getSettings();
}
/**
* {@inheritdoc}
*/
public function getHighestId() {
$values = $this->storage->getQuery()
->accessCheck(FALSE)
->sort($this->getKey('id'), 'DESC')
->range(0, 1)
->execute();
return (int) current($values);
}
}
......@@ -3,7 +3,12 @@
namespace Drupal\migrate\Plugin\migrate\destination;
use Drupal\Core\Entity\ContentEntityInterface;
use Drupal\Core\Entity\EntityManagerInterface;
use Drupal\Core\Entity\EntityStorageInterface;
use Drupal\Core\Field\FieldTypePluginManagerInterface;
use Drupal\Core\StringTranslation\TranslatableMarkup;
use Drupal\migrate\MigrateException;
use Drupal\migrate\Plugin\MigrationInterface;
use Drupal\migrate\Row;
/**
......@@ -16,6 +21,16 @@
*/
class EntityRevision extends EntityContentBase {
/**
* {@inheritdoc}
*/
public function __construct(array $configuration, $plugin_id, $plugin_definition, MigrationInterface $migration, EntityStorageInterface $storage, array $bundles, EntityManagerInterface $entity_manager, FieldTypePluginManagerInterface $field_type_manager) {
$plugin_definition += [
'label' => new TranslatableMarkup('@entity_type revisions', ['@entity_type' => $storage->getEntityType()->getSingularLabel()]),
];
parent::__construct($configuration, $plugin_id, $plugin_definition, $migration, $storage, $bundles, $entity_manager, $field_type_manager);
}
/**
* {@inheritdoc}
*/
......@@ -78,4 +93,19 @@ public function getIds() {
throw new MigrateException('This entity type does not support revisions.');
}
/**
* {@inheritdoc}
*/
public function getHighestId() {
$values = $this->storage->getQuery()
->accessCheck(FALSE)
->allRevisions()
->sort($this->getKey('revision'), 'DESC')
->range(0, 1)
->execute();
// The array keys are the revision IDs.
// The array contains only one entry, so we can use key().
return (int) key($values);
}
}
......@@ -7,6 +7,7 @@
use Drupal\Core\Plugin\ContainerFactoryPluginInterface;
use Drupal\Core\Plugin\PluginBase;
use Drupal\migrate\MigrateMessage;
use Drupal\migrate\Audit\HighestIdInterface;
use Drupal\migrate\Plugin\MigrationInterface;
use Drupal\migrate\Event\MigrateIdMapMessageEvent;
use Drupal\migrate\MigrateException;
......@@ -27,7 +28,7 @@
*
* @PluginID("sql")
*/
class Sql extends PluginBase implements MigrateIdMapInterface, ContainerFactoryPluginInterface {
class Sql extends PluginBase implements MigrateIdMapInterface, ContainerFactoryPluginInterface, HighestIdInterface {
/**
* Column name of hashed source id values.
......@@ -152,6 +153,8 @@ class Sql extends PluginBase implements MigrateIdMapInterface, ContainerFactoryP
* The configuration for the plugin.
* @param \Drupal\migrate\Plugin\MigrationInterface $migration
* The migration to do.
* @param \Symfony\Component\EventDispatcher\EventDispatcherInterface $event_dispatcher
* The event dispatcher.
*/
public function __construct(array $configuration, $plugin_id, $plugin_definition, MigrationInterface $migration, EventDispatcherInterface $event_dispatcher) {
parent::__construct($configuration, $plugin_id, $plugin_definition);
......@@ -925,4 +928,69 @@ public function valid() {
return $this->currentRow !== FALSE;
}
/**
* Returns the migration plugin manager.
*
* @todo Inject as a dependency in https://www.drupal.org/node/2919158.
*
* @return \Drupal\migrate\Plugin\MigrationPluginManagerInterface
* The migration plugin manager.
*/
protected function getMigrationPluginManager() {
return \Drupal::service('plugin.manager.migration');
}
/**
* {@inheritdoc}
*/
public function getHighestId() {
array_filter(
$this->migration->getDestinationPlugin()->getIds(),
function (array $id) {
if ($id['type'] !== 'integer') {
throw new \LogicException('Cannot determine the highest migrated ID without an integer ID column');
}
}
);
// List of mapping tables to look in for the highest ID.
$map_tables = [
$this->migration->id() => $this->mapTableName(),
];
// If there's a bundle, it means we have a derived migration and we need to
// find all the mapping tables from the related derived migrations.
if ($base_id = substr($this->migration->id(), 0, strpos($this->migration->id(), static::DERIVATIVE_SEPARATOR))) {
$migration_manager = $this->getMigrationPluginManager();
$migrations = $migration_manager->getDefinitions();
foreach ($migrations as $migration_id => $migration) {
if ($migration['id'] === $base_id) {
// Get this derived migration's mapping table and add it to the list
// of mapping tables to look in for the highest ID.
$stub = $migration_manager->createInstance($migration_id);
$map_tables[$migration_id] = $stub->getIdMap()->mapTableName();
}
}
}
// Get the highest id from the list of map tables.
$ids = [0];
foreach ($map_tables as $map_table) {
if (!$this->getDatabase()->schema()->tableExists($map_table)) {
break;
}
$query = $this->getDatabase()->select($map_table, 'map')
->fields('map', $this->destinationIdFields())
->range(0, 1);
foreach (array_values($this->destinationIdFields()) as $order_field) {
$query->orderBy($order_field, 'DESC');
}
$ids[] = $query->execute()->fetchField();
}
// Return the highest of all the mapped IDs.
return (int) max($ids);
}
}
......@@ -8,9 +8,9 @@
namespace Drupal\Tests\migrate\Unit\Plugin\migrate\destination;
use Drupal\Core\Entity\ContentEntityInterface;
use Drupal\Core\Entity\ContentEntityType;
use Drupal\Core\Entity\EntityManagerInterface;
use Drupal\Core\Entity\EntityStorageInterface;
use Drupal\Core\Entity\EntityTypeInterface;
use Drupal\Core\Field\BaseFieldDefinition;
use Drupal\Core\Field\FieldTypePluginManagerInterface;
use Drupal\migrate\MigrateException;
......@@ -38,6 +38,11 @@ class EntityContentBaseTest extends UnitTestCase {
*/
protected $storage;
/**
* @var \Drupal\Core\Entity\EntityTypeInterface
*/
protected $entityType;
/**
* @var \Drupal\Core\Entity\EntityManagerInterface
*/
......@@ -51,6 +56,11 @@ protected function setUp() {
$this->migration = $this->prophesize(MigrationInterface::class);
$this->storage = $this->prophesize(EntityStorageInterface::class);
$this->entityType = $this->prophesize(EntityTypeInterface::class);
$this->entityType->getPluralLabel()->willReturn('wonkiness');
$this->storage->getEntityType()->willReturn($this->entityType->reveal());
$this->entityManager = $this->prophesize(EntityManagerInterface::class);
}
......@@ -104,14 +114,11 @@ public function testImportEntityLoadFailure() {
*/
public function testUntranslatable() {
// An entity type without a language.
$entity_type = $this->prophesize(ContentEntityType::class);
$entity_type->getKey('langcode')->willReturn('');
$entity_type->getKey('id')->willReturn('id');
$this->entityType->getKey('langcode')->willReturn('');
$this->entityType->getKey('id')->willReturn('id');
$this->entityManager->getBaseFieldDefinitions('foo')
->willReturn(['id' => BaseFieldDefinitionTest::create('integer')]);
$this->storage->getEntityType()->willReturn($entity_type->reveal());
$destination = new EntityTestDestination(
['translations' => TRUE],
'',
......
......@@ -9,6 +9,7 @@
use Drupal\Core\Entity\ContentEntityInterface;
use Drupal\Core\Entity\EntityInterface;
use Drupal\Core\Entity\EntityTypeInterface;
use Drupal\migrate\Plugin\MigrationInterface;
use Drupal\migrate\Plugin\migrate\destination\EntityRevision as RealEntityRevision;
use Drupal\migrate\Row;
......@@ -48,6 +49,12 @@ protected function setUp() {
// Setup mocks to be used when creating a revision destination.
$this->migration = $this->prophesize(MigrationInterface::class);
$this->storage = $this->prophesize('\Drupal\Core\Entity\EntityStorageInterface');
$entity_type = $this->prophesize(EntityTypeInterface::class);
$entity_type->getSingularLabel()->willReturn('crazy');
$entity_type->getPluralLabel()->willReturn('craziness');
$this->storage->getEntityType()->willReturn($entity_type->reveal());
$this->entityManager = $this->prophesize('\Drupal\Core\Entity\EntityManagerInterface');
$this->fieldTypeManager = $this->prophesize('\Drupal\Core\Field\FieldTypePluginManagerInterface');
}
......