From 07c8d58cbed415f0a569fc414efaaafcd5be3284 Mon Sep 17 00:00:00 2001 From: webchick <drupal@webchick.net> Date: Mon, 28 Sep 2015 10:04:29 -0700 Subject: [PATCH] Issue #2361093 by mikeryan, dawehner, Devin Carlson, benjy, phenaproxima: Add a rollback functionality to migrate --- .../migrate/src/Event/MigrateEvents.php | 62 +++++++ .../src/Event/MigrateRollbackEvent.php | 45 +++++ .../src/Event/MigrateRowDeleteEvent.php | 65 +++++++ .../modules/migrate/src/MigrateExecutable.php | 69 +++++++- .../src/MigrateExecutableInterface.php | 5 + .../Plugin/MigrateDestinationInterface.php | 14 +- .../src/Plugin/MigrateIdMapInterface.php | 10 +- .../src/Plugin/migrate/destination/Config.php | 14 -- .../migrate/destination/DestinationBase.php | 15 +- .../src/Plugin/migrate/destination/Entity.php | 12 ++ .../migrate/src/Plugin/migrate/id_map/Sql.php | 16 ++ .../migrate/src/Tests/MigrateRollbackTest.php | 167 ++++++++++++++++++ 12 files changed, 464 insertions(+), 30 deletions(-) create mode 100644 core/modules/migrate/src/Event/MigrateRollbackEvent.php create mode 100644 core/modules/migrate/src/Event/MigrateRowDeleteEvent.php create mode 100644 core/modules/migrate/src/Tests/MigrateRollbackTest.php diff --git a/core/modules/migrate/src/Event/MigrateEvents.php b/core/modules/migrate/src/Event/MigrateEvents.php index 2370c7638443..a6cde25030f8 100644 --- a/core/modules/migrate/src/Event/MigrateEvents.php +++ b/core/modules/migrate/src/Event/MigrateEvents.php @@ -15,6 +15,8 @@ * @see \Drupal\migrate\Event\MigrateImportEvent * @see \Drupal\migrate\Event\MigratePreRowSaveEvent * @see \Drupal\migrate\Event\MigratePostRowSaveEvent + * @see \Drupal\migrate\Event\MigrateRollbackEvent + * @see \Drupal\migrate\Event\MigrateRowDeleteEvent */ final class MigrateEvents { @@ -108,4 +110,64 @@ final class MigrateEvents { */ const POST_ROW_SAVE = 'migrate.post_row_save'; + /** + * Name of the event fired when beginning a migration rollback operation. + * + * This event allows modules to perform an action whenever a migration + * rollback operation is about to begin. The event listener method receives a + * \Drupal\migrate\Event\MigrateRollbackEvent instance. + * + * @Event + * + * @see \Drupal\migrate\Event\MigrateRollbackEvent + * + * @var string + */ + const PRE_ROLLBACK = 'migrate.pre_rollback'; + + /** + * Name of the event fired when finishing a migration rollback operation. + * + * This event allows modules to perform an action whenever a migration + * rollback operation is completing. The event listener method receives a + * \Drupal\migrate\Event\MigrateRollbackEvent instance. + * + * @Event + * + * @see \Drupal\migrate\Event\MigrateRollbackEvent + * + * @var string + */ + const POST_ROLLBACK = 'migrate.post_rollback'; + + /** + * Name of the event fired when about to delete a single item. + * + * This event allows modules to perform an action whenever a specific item + * is about to be deleted by the destination plugin. The event listener method + * receives a \Drupal\migrate\Event\MigrateRowDeleteEvent instance. + * + * @Event + * + * @see \Drupal\migrate\Event\MigrateRowDeleteEvent + * + * @var string + */ + const PRE_ROW_DELETE = 'migrate.pre_row_delete'; + + /** + * Name of the event fired just after a single item has been deleted. + * + * This event allows modules to perform an action whenever a specific item + * has been deleted by the destination plugin. The event listener method + * receives a \Drupal\migrate\Event\MigrateRowDeleteEvent instance. + * + * @Event + * + * @see \Drupal\migrate\Event\MigrateRowDeleteEvent + * + * @var string + */ + const POST_ROW_DELETE = 'migrate.post_row_delete'; + } diff --git a/core/modules/migrate/src/Event/MigrateRollbackEvent.php b/core/modules/migrate/src/Event/MigrateRollbackEvent.php new file mode 100644 index 000000000000..1a6ac314b712 --- /dev/null +++ b/core/modules/migrate/src/Event/MigrateRollbackEvent.php @@ -0,0 +1,45 @@ +<?php + +/** + * @file + * Contains \Drupal\migrate\Event\MigrateRollbackEvent. + */ + +namespace Drupal\migrate\Event; + +use Drupal\migrate\Entity\MigrationInterface; +use Symfony\Component\EventDispatcher\Event; + +/** + * Wraps a pre- or post-rollback event for event listeners. + */ +class MigrateRollbackEvent extends Event { + + /** + * Migration entity. + * + * @var \Drupal\migrate\Entity\MigrationInterface + */ + protected $migration; + + /** + * Constructs an rollback event object. + * + * @param \Drupal\migrate\Entity\MigrationInterface $migration + * Migration entity. + */ + public function __construct(MigrationInterface $migration) { + $this->migration = $migration; + } + + /** + * Gets the migration entity. + * + * @return \Drupal\migrate\Entity\MigrationInterface + * The migration entity involved. + */ + public function getMigration() { + return $this->migration; + } + +} diff --git a/core/modules/migrate/src/Event/MigrateRowDeleteEvent.php b/core/modules/migrate/src/Event/MigrateRowDeleteEvent.php new file mode 100644 index 000000000000..7d3766af5b1f --- /dev/null +++ b/core/modules/migrate/src/Event/MigrateRowDeleteEvent.php @@ -0,0 +1,65 @@ +<?php + +/** + * @file + * Contains \Drupal\migrate\Event\MigrateRowDeleteEvent. + */ + +namespace Drupal\migrate\Event; + +use Drupal\migrate\Entity\MigrationInterface; +use Symfony\Component\EventDispatcher\Event; + +/** + * Wraps a row deletion event for event listeners. + */ +class MigrateRowDeleteEvent extends Event { + + /** + * Migration entity. + * + * @var \Drupal\migrate\Entity\MigrationInterface + */ + protected $migration; + + /** + * Values representing the destination ID. + * + * @var array + */ + protected $destinationIdValues; + + /** + * Constructs a row deletion event object. + * + * @param \Drupal\migrate\Entity\MigrationInterface $migration + * Migration entity. + * @param array $destination_id_values + * Values represent the destination ID. + */ + public function __construct(MigrationInterface $migration, $destination_id_values) { + $this->migration = $migration; + $this->destinationIdValues = $destination_id_values; + } + + /** + * Gets the migration entity. + * + * @return \Drupal\migrate\Entity\MigrationInterface + * The migration being rolled back. + */ + public function getMigration() { + return $this->migration; + } + + /** + * Gets the destination ID values. + * + * @return array + * The destination ID as an array. + */ + public function getDestinationIdValues() { + return $this->destinationIdValues; + } + +} diff --git a/core/modules/migrate/src/MigrateExecutable.php b/core/modules/migrate/src/MigrateExecutable.php index 8a42bc9036b5..b037c790cf11 100644 --- a/core/modules/migrate/src/MigrateExecutable.php +++ b/core/modules/migrate/src/MigrateExecutable.php @@ -14,6 +14,8 @@ use Drupal\migrate\Event\MigrateImportEvent; use Drupal\migrate\Event\MigratePostRowSaveEvent; use Drupal\migrate\Event\MigratePreRowSaveEvent; +use Drupal\migrate\Event\MigrateRollbackEvent; +use Drupal\migrate\Event\MigrateRowDeleteEvent; use Drupal\migrate\Exception\RequirementsException; use Drupal\migrate\Plugin\MigrateIdMapInterface; use Symfony\Component\EventDispatcher\EventDispatcherInterface; @@ -76,15 +78,6 @@ class MigrateExecutable implements MigrateExecutableInterface { */ protected $counts = array(); - /** - * The maximum number of items to pass in a single call during a rollback. - * - * For use in bulkRollback(). Can be overridden in derived class constructor. - * - * @var int - */ - protected $rollbackBatchSize = 50; - /** * The object currently being constructed. * @@ -312,6 +305,64 @@ public function import() { return $return; } + /** + * {@inheritdoc} + */ + public function rollback() { + // Only begin the rollback operation if the migration is currently idle. + if ($this->migration->getStatus() !== MigrationInterface::STATUS_IDLE) { + $this->message->display($this->t('Migration @id is busy with another operation: @status', ['@id' => $this->migration->id(), '@status' => $this->t($this->migration->getStatusLabel())]), 'error'); + return MigrationInterface::RESULT_FAILED; + } + + // Announce that rollback is about to happen. + $this->getEventDispatcher()->dispatch(MigrateEvents::PRE_ROLLBACK, new MigrateRollbackEvent($this->migration)); + + // Optimistically assume things are going to work out; if not, $return will be + // updated to some other status. + $return = MigrationInterface::RESULT_COMPLETED; + + $this->migration->setStatus(MigrationInterface::STATUS_ROLLING_BACK); + $id_map = $this->migration->getIdMap(); + $destination = $this->migration->getDestinationPlugin(); + + // Loop through each row in the map, and try to roll it back. + foreach ($id_map as $serialized_key => $map_row) { + $destination_key = $id_map->currentDestination(); + if ($destination_key) { + $this->getEventDispatcher() + ->dispatch(MigrateEvents::PRE_ROW_DELETE, new MigrateRowDeleteEvent($this->migration, $destination_key)); + $destination->rollback($destination_key); + $this->getEventDispatcher() + ->dispatch(MigrateEvents::POST_ROW_DELETE, new MigrateRowDeleteEvent($this->migration, $destination_key)); + // We're now done with this row, so remove it from the map. + $id_map->delete(unserialize($serialized_key)); + } + + // Check for memory exhaustion. + if (($return = $this->checkStatus()) != MigrationInterface::RESULT_COMPLETED) { + break; + } + + // If anyone has requested we stop, return the requested result. + if ($this->migration->getStatus() == MigrationInterface::STATUS_STOPPING) { + $return = $this->migration->getMigrationResult(); + break; + } + } + // If rollback completed successfully, reset the high water mark. + if ($return == MigrationInterface::RESULT_COMPLETED) { + $this->migration->saveHighWater(NULL); + } + + // Notify modules that rollback attempt was complete. + $this->migration->setMigrationResult($return); + $this->getEventDispatcher()->dispatch(MigrateEvents::POST_ROLLBACK, new MigrateRollbackEvent($this->migration)); + $this->migration->setStatus(MigrationInterface::STATUS_IDLE); + + return $return; + } + /** * {@inheritdoc} */ diff --git a/core/modules/migrate/src/MigrateExecutableInterface.php b/core/modules/migrate/src/MigrateExecutableInterface.php index 3edf6200559e..71fa01769466 100644 --- a/core/modules/migrate/src/MigrateExecutableInterface.php +++ b/core/modules/migrate/src/MigrateExecutableInterface.php @@ -16,6 +16,11 @@ interface MigrateExecutableInterface { */ public function import(); + /** + * Performs a rollback operation - remove previously-imported items. + */ + public function rollback(); + /** * Processes a row. * diff --git a/core/modules/migrate/src/Plugin/MigrateDestinationInterface.php b/core/modules/migrate/src/Plugin/MigrateDestinationInterface.php index efa873c958d1..e19e24c5325c 100644 --- a/core/modules/migrate/src/Plugin/MigrateDestinationInterface.php +++ b/core/modules/migrate/src/Plugin/MigrateDestinationInterface.php @@ -59,7 +59,7 @@ public function fields(MigrationInterface $migration = NULL); * Import the row. * * Derived classes must implement import(), to construct one new object - * (pre-populated) using ID mappings in the Migration). + * (pre-populated) using ID mappings in the Migration. * * @param \Drupal\migrate\Row $row * The row object. @@ -72,11 +72,15 @@ public function fields(MigrationInterface $migration = NULL); public function import(Row $row, array $old_destination_id_values = array()); /** - * Delete the specified IDs from the target Drupal. + * Delete the specified destination object from the target Drupal. * - * @param array $destination_identifiers - * The destination ids to delete. + * @param array $destination_identifier + * The ID of the destination object to delete. */ - public function rollbackMultiple(array $destination_identifiers); + public function rollback(array $destination_identifier); + /** + * @return bool + */ + public function supportsRollback(); } diff --git a/core/modules/migrate/src/Plugin/MigrateIdMapInterface.php b/core/modules/migrate/src/Plugin/MigrateIdMapInterface.php index d6f17df34421..144c8999f8d5 100644 --- a/core/modules/migrate/src/Plugin/MigrateIdMapInterface.php +++ b/core/modules/migrate/src/Plugin/MigrateIdMapInterface.php @@ -210,7 +210,7 @@ public function getRowsNeedingUpdate($count); public function lookupSourceID(array $destination_id_values); /** - * Looks up the destination identifier. + * Looks up the destination identifier corresponding to a source key. * * Given a (possibly multi-field) source identifier value, return the * (possibly multi-field) destination identifier value it is mapped to. @@ -223,6 +223,14 @@ public function lookupSourceID(array $destination_id_values); */ public function lookupDestinationId(array $source_id_values); + /** + * Looks up the destination identifier currently being iterated. + * + * @return array + * The destination identifier values of the record, or NULL on failure. + */ + public function currentDestination(); + /** * Removes any persistent storage used by this map. * diff --git a/core/modules/migrate/src/Plugin/migrate/destination/Config.php b/core/modules/migrate/src/Plugin/migrate/destination/Config.php index ac05db422964..7cc737e0d741 100644 --- a/core/modules/migrate/src/Plugin/migrate/destination/Config.php +++ b/core/modules/migrate/src/Plugin/migrate/destination/Config.php @@ -13,10 +13,8 @@ use Drupal\Core\Entity\DependencyTrait; use Drupal\Core\Plugin\ContainerFactoryPluginInterface; use Drupal\migrate\Entity\MigrationInterface; -use Drupal\migrate\MigrateException; use Drupal\migrate\Row; use Symfony\Component\DependencyInjection\ContainerInterface; -use Drupal\Core\Config\Config as ConfigObject; /** * Persist data to the config system. @@ -83,18 +81,6 @@ public function import(Row $row, array $old_destination_id_values = array()) { return TRUE; } - /** - * Throw an exception because config can not be rolled back. - * - * @param array $destination_keys - * The array of destination ids to roll back. - * - * @throws \Drupal\migrate\MigrateException - */ - public function rollbackMultiple(array $destination_keys) { - throw new MigrateException('Configuration can not be rolled back'); - } - /** * {@inheritdoc} */ diff --git a/core/modules/migrate/src/Plugin/migrate/destination/DestinationBase.php b/core/modules/migrate/src/Plugin/migrate/destination/DestinationBase.php index 663a4290c3a6..4f1d596dc999 100644 --- a/core/modules/migrate/src/Plugin/migrate/destination/DestinationBase.php +++ b/core/modules/migrate/src/Plugin/migrate/destination/DestinationBase.php @@ -26,6 +26,13 @@ */ abstract class DestinationBase extends PluginBase implements MigrateDestinationInterface, RequirementsInterface { + /** + * Indicates whether the destination can be rolled back. + * + * @var bool + */ + protected $supportsRollback = FALSE; + /** * The migration. * @@ -62,8 +69,14 @@ public function checkRequirements() { /** * {@inheritdoc} */ - public function rollbackMultiple(array $destination_identifiers) { + public function rollback(array $destination_identifier) { // By default we do nothing. } + /** + * {@inheritdoc} + */ + public function supportsRollback() { + return $this->supportsRollback; + } } diff --git a/core/modules/migrate/src/Plugin/migrate/destination/Entity.php b/core/modules/migrate/src/Plugin/migrate/destination/Entity.php index c3cc71b5bc4d..64123d6d448b 100644 --- a/core/modules/migrate/src/Plugin/migrate/destination/Entity.php +++ b/core/modules/migrate/src/Plugin/migrate/destination/Entity.php @@ -58,6 +58,7 @@ public function __construct(array $configuration, $plugin_id, $plugin_definition parent::__construct($configuration, $plugin_id, $plugin_definition, $migration); $this->storage = $storage; $this->bundles = $bundles; + $this->supportsRollback = TRUE; } /** @@ -163,6 +164,17 @@ protected function getKey($key) { return $this->storage->getEntityType()->getKey($key); } + /** + * {@inheritdoc} + */ + public function rollback(array $destination_identifier) { + // Delete the specified entity from Drupal if it exists. + $entity = $this->storage->load(reset($destination_identifier)); + if ($entity) { + $entity->delete(); + } + } + /** * {@inheritdoc} */ diff --git a/core/modules/migrate/src/Plugin/migrate/id_map/Sql.php b/core/modules/migrate/src/Plugin/migrate/id_map/Sql.php index 12bb419fe856..ff832ef52b9f 100644 --- a/core/modules/migrate/src/Plugin/migrate/id_map/Sql.php +++ b/core/modules/migrate/src/Plugin/migrate/id_map/Sql.php @@ -805,6 +805,22 @@ public function key() { return serialize($this->currentKey); } + /** + * @inheritdoc + */ + public function currentDestination() { + if ($this->valid()) { + $result = array(); + foreach ($this->destinationIdFields() as $field_name) { + $result[$field_name] = $this->currentRow[$field_name]; + } + return $result; + } + else { + return NULL; + } + } + /** * Implementation of Iterator::next(). * diff --git a/core/modules/migrate/src/Tests/MigrateRollbackTest.php b/core/modules/migrate/src/Tests/MigrateRollbackTest.php new file mode 100644 index 000000000000..ce6eb0051e42 --- /dev/null +++ b/core/modules/migrate/src/Tests/MigrateRollbackTest.php @@ -0,0 +1,167 @@ +<?php + +/** + * @file + * Contains \Drupal\migrate\Tests\MigrateRollbackTest. + */ + +namespace Drupal\migrate\Tests; + +use Drupal\migrate\Entity\Migration; +use Drupal\migrate\MigrateExecutable; +use Drupal\taxonomy\Entity\Term; +use Drupal\taxonomy\Entity\Vocabulary; + +/** + * Tests rolling back of imports. + * + * @group migrate + */ +class MigrateRollbackTest extends MigrateTestBase { + + /** + * Modules to enable. + * + * @var array + */ + public static $modules = ['field', 'taxonomy', 'text']; + + /** + * {@inheritdoc} + */ + protected function setUp() { + parent::setUp(); + $this->installEntitySchema('taxonomy_vocabulary'); + $this->installEntitySchema('taxonomy_term'); + $this->installConfig(['taxonomy']); + } + + /** + * Tests rolling back configuration and content entities. + */ + public function testRollback() { + // We use vocabularies to demonstrate importing and rolling back + // configuration entities. + $vocabulary_data_rows = [ + ['id' => '1', 'name' => 'categories', 'weight' => '2'], + ['id' => '2', 'name' => 'tags', 'weight' => '1'], + ]; + $ids = ['id' => ['type' => 'integer']]; + $config = [ + 'id' => 'vocabularies', + 'migration_tags' => ['Import and rollback test'], + 'source' => [ + 'plugin' => 'embedded_data', + 'data_rows' => $vocabulary_data_rows, + 'ids' => $ids, + ], + 'process' => [ + 'vid' => 'id', + 'name' => 'name', + 'weight' => 'weight', + ], + 'destination' => ['plugin' => 'entity:taxonomy_vocabulary'], + ]; + + $vocabulary_migration = Migration::create($config); + $vocabulary_id_map = $vocabulary_migration->getIdMap(); + + $this->assertTrue($vocabulary_migration->getDestinationPlugin()->supportsRollback()); + + // Import and validate vocabulary config entities were created. + $vocabulary_executable = new MigrateExecutable($vocabulary_migration, $this); + $vocabulary_executable->import(); + foreach ($vocabulary_data_rows as $row) { + /** @var Vocabulary $vocabulary */ + $vocabulary = Vocabulary::load($row['id']); + $this->assertTrue($vocabulary); + $map_row = $vocabulary_id_map->getRowBySource([$row['id']]); + $this->assertNotNull($map_row['destid1']); + } + + // We use taxonomy terms to demonstrate importing and rolling back + // content entities. + $term_data_rows = [ + ['id' => '1', 'vocab' => '1', 'name' => 'music'], + ['id' => '2', 'vocab' => '2', 'name' => 'Bach'], + ['id' => '3', 'vocab' => '2', 'name' => 'Beethoven'], + ]; + $ids = ['id' => ['type' => 'integer']]; + $config = [ + 'id' => 'terms', + 'migration_tags' => ['Import and rollback test'], + 'source' => [ + 'plugin' => 'embedded_data', + 'data_rows' => $term_data_rows, + 'ids' => $ids, + ], + 'process' => [ + 'tid' => 'id', + 'vid' => 'vocab', + 'name' => 'name', + ], + 'destination' => ['plugin' => 'entity:taxonomy_term'], + 'migration_dependencies' => ['required' => ['vocabularies']], + ]; + + $term_migration = Migration::create($config); + $term_id_map = $term_migration->getIdMap(); + + $this->assertTrue($term_migration->getDestinationPlugin()->supportsRollback()); + + // Import and validate term entities were created. + $term_executable = new MigrateExecutable($term_migration, $this); + $term_executable->import(); + foreach ($term_data_rows as $row) { + /** @var Term $term */ + $term = Term::load($row['id']); + $this->assertTrue($term); + $map_row = $term_id_map->getRowBySource([$row['id']]); + $this->assertNotNull($map_row['destid1']); + } + + // Rollback and verify the entities are gone. + $term_executable->rollback(); + foreach ($term_data_rows as $row) { + $term = Term::load($row['id']); + $this->assertNull($term); + $map_row = $term_id_map->getRowBySource([$row['id']]); + $this->assertFalse($map_row); + } + $vocabulary_executable->rollback(); + foreach ($vocabulary_data_rows as $row) { + $term = Vocabulary::load($row['id']); + $this->assertNull($term); + $map_row = $vocabulary_id_map->getRowBySource([$row['id']]); + $this->assertFalse($map_row); + } + + // Test that simple configuration is not rollbackable. + $term_setting_rows = [ + ['id' => 1, 'override_selector' => '0', 'terms_per_page_admin' => '10'], + ]; + $ids = ['id' => ['type' => 'integer']]; + $config = [ + 'id' => 'taxonomy_settings', + 'migration_tags' => ['Import and rollback test'], + 'source' => [ + 'plugin' => 'embedded_data', + 'data_rows' => $term_setting_rows, + 'ids' => $ids, + ], + 'process' => [ + 'override_selector' => 'override_selector', + 'terms_per_page_admin' => 'terms_per_page_admin', + ], + 'destination' => [ + 'plugin' => 'config', + 'config_name' => 'taxonomy.settings', + ], + 'migration_dependencies' => ['required' => ['vocabularies']], + ]; + + $settings_migration = Migration::create($config); + $this->assertFalse($settings_migration->getDestinationPlugin()->supportsRollback()); + } + +} -- GitLab