Issue #3024399 by heddn, arnested, daveiano, moshe weitzman, mpp, ThomWilhelm,... (09e28fd6) · Commits · project / migrate_tools

src/Commands/MigrateToolsCommands.php

+59 −86

Original line number	Diff line number	Diff line
		@@ -21,6 +21,11 @@ use Drush\Commands\DrushCommands;
		*/
		class MigrateToolsCommands extends DrushCommands {

		/**
		* Default ID list delimiter.
		*/
		public const DEFAULT_ID_LIST_DELIMITER = ':';

		/**
		* Migration plugin manager service.
		*
		@@ -121,12 +126,11 @@ class MigrateToolsCommands extends DrushCommands {
		* @return \Consolidation\OutputFormatters\StructuredData\RowsOfFields
		* Migrations status formatted as table.
		*/
		public function status($migration_names = '', array $options = []) {
		$options += [
		'group' => NULL,
		'tag' => NULL,
		'names-only' => NULL,
		];
		public function status($migration_names = '', array $options = [
		'group' => self::REQ,
		'tag' => self::REQ,
		'names-only' => FALSE,
		]) {
		$names_only = $options['names-only'];

		$migrations = $this->migrationsList($migration_names, $options);
		@@ -245,7 +249,7 @@ class MigrateToolsCommands extends DrushCommands {
		* @option limit Limit on the number of items to process in each migration
		* @option feedback Frequency of progress messages, in items processed
		* @option idlist Comma-separated list of IDs to import
		* @option idlist-delimiter The delimiter for records, defaults to ':'
		* @option idlist-delimiter The delimiter for records
		* @option update In addition to processing unprocessed items from the
		* source, update previously-imported items with the current data
		* @option force Force an operation to run, even if all dependencies are not
		@@ -278,42 +282,25 @@ class MigrateToolsCommands extends DrushCommands {
		* @throws \Exception
		* If there are not enough parameters to the command.
		*/
		public function import($migration_names = '', array $options = []) {
		$options += [
		'all' => NULL,
		'group' => NULL,
		'tag' => NULL,
		'limit' => NULL,
		'feedback' => NULL,
		'idlist' => NULL,
		'idlist-delimiter' => ':',
		'update' => NULL,
		'force' => NULL,
		'execute-dependencies' => NULL,
		];
		public function import($migration_names = '', array $options = [
		'all' => FALSE,
		'group' => self::REQ,
		'tag' => self::REQ,
		'limit' => self::REQ,
		'feedback' => self::REQ,
		'idlist' => self::REQ,
		'idlist-delimiter' => self::DEFAULT_ID_LIST_DELIMITER,
		'update' => FALSE,
		'force' => FALSE,
		'execute-dependencies' => FALSE,
		]) {
		$group_names = $options['group'];
		$tag_names = $options['tag'];
		$all = $options['all'];
		$additional_options = [];
		if (!$all && !$group_names && !$migration_names && !$tag_names) {
		throw new \Exception(dt('You must specify --all, --group, --tag or one or more migration names separated by commas'));
		}

		$possible_options = [
		'limit',
		'feedback',
		'idlist',
		'idlist-delimiter',
		'update',
		'force',
		'execute-dependencies',
		];
		foreach ($possible_options as $option) {
		if ($options[$option]) {
		$additional_options[$option] = $options[$option];
		}
		}

		$migrations = $this->migrationsList($migration_names, $options);
		if (empty($migrations)) {
		$this->logger->error(dt('No migrations found.'));
		@@ -324,7 +311,7 @@ class MigrateToolsCommands extends DrushCommands {
		array_walk(
		$migration_list,
		[$this, 'executeMigration'],
		$additional_options
		$options
		);
		}
		}
		@@ -343,9 +330,8 @@ class MigrateToolsCommands extends DrushCommands {
		* @option group A comma-separated list of migration groups to rollback
		* @option tag ID of the migration tag to rollback
		* @option feedback Frequency of progress messages, in items processed
		* @option feedback Frequency of progress messages, in items processed
		* @option idlist Comma-separated list of IDs to rollback
		* @option idlist-delimiter The delimiter for records, defaults to ':'
		* @option idlist-delimiter The delimiter for records
		*
		* @default $options []
		*
		@@ -368,29 +354,21 @@ class MigrateToolsCommands extends DrushCommands {
		* @throws \Exception
		* If there are not enough parameters to the command.
		*/
		public function rollback($migration_names = '', array $options = []) {
		$options += [
		'all' => NULL,
		'group' => NULL,
		'tag' => NULL,
		'feedback' => NULL,
		'idlist' => NULL,
		'idlist-delimiter' => ':',
		];
		public function rollback($migration_names = '', array $options = [
		'all' => FALSE,
		'group' => self::REQ,
		'tag' => self::REQ,
		'feedback' => self::REQ,
		'idlist' => self::REQ,
		'idlist-delimiter' => self::DEFAULT_ID_LIST_DELIMITER,
		]) {
		$group_names = $options['group'];
		$tag_names = $options['tag'];
		$all = $options['all'];
		$additional_options = [];
		if (!$all && !$group_names && !$migration_names && !$tag_names) {
		throw new \Exception(dt('You must specify --all, --group, --tag, or one or more migration names separated by commas'));
		}

		foreach (['feedback', 'idlist', 'idlist-delimiter'] as $option) {
		if ($options[$option]) {
		$additional_options[$option] = $options[$option];
		}
		}

		$migrations = $this->migrationsList($migration_names, $options);
		if (empty($migrations)) {
		$this->logger()->error(dt('No migrations found.'));
		@@ -399,14 +377,14 @@ class MigrateToolsCommands extends DrushCommands {
		// Take it one group at a time,
		// rolling back the migrations within each group.
		$has_failure = FALSE;
		foreach ($migrations as $group_id => $migration_list) {
		foreach ($migrations as $migration_list) {
		// Roll back in reverse order.
		$migration_list = array_reverse($migration_list);
		foreach ($migration_list as $migration_id => $migration) {
		$executable = new MigrateExecutable(
		$migration,
		$this->getMigrateMessage(),
		$additional_options
		$options
		);
		// drush_op() provides --simulate support.
		$result = drush_op([$executable, 'rollback']);
		@@ -524,7 +502,7 @@ class MigrateToolsCommands extends DrushCommands {
		*
		* @option csv Export messages as a CSV
		* @option idlist Comma-separated list of IDs to import
		* @option idlist-delimiter The delimiter for records, defaults to ':'
		* @option idlist-delimiter The delimiter for records
		*
		* @default $options []
		*
		@@ -544,12 +522,11 @@ class MigrateToolsCommands extends DrushCommands {
		* @return \Consolidation\OutputFormatters\StructuredData\RowsOfFields
		* Source fields of the given migration formatted as a table.
		*/
		public function messages($migration_id, array $options = []) {
		$options += [
		'csv' => NULL,
		'idlist' => NULL,
		'idlist-delimiter' => ':',
		];
		public function messages($migration_id, array $options = [
		'csv' => FALSE,
		'idlist' => self::REQ,
		'idlist-delimiter' => self::DEFAULT_ID_LIST_DELIMITER,
		]) {
		/** @var \Drupal\migrate\Plugin\MigrationInterface $migration */
		$migration = $this->migrationPluginManager->createInstance(
		$migration_id
		@@ -562,7 +539,9 @@ class MigrateToolsCommands extends DrushCommands {
		$id_list = MigrateTools::buildIdList($options);
		$map = new IdMapFilter($migration->getIdMap(), $id_list);
		$table = [];
		foreach ($map->getMessageIterator() as $row) {
		// TODO: Remove after 8.7 support goes away.
		$iterator_method = method_exists($map, 'getMessages') ? 'getMessages' : 'getMessageIterator';
		foreach ($map->{$iterator_method}() as $row) {
		unset($row->msgid);
		$table[] = (array) $row;
		}
		@@ -644,14 +623,8 @@ class MigrateToolsCommands extends DrushCommands {
		*/
		protected function migrationsList($migration_ids = '', array $options = []) {
		// Filter keys must match the migration configuration property name.
		$filter['migration_group'] = $options['group'] ? explode(
		',',
		$options['group']
		) : [];
		$filter['migration_tags'] = $options['tag'] ? explode(
		',',
		$options['tag']
		) : [];
		$filter['migration_group'] = explode(',', $options['group']);
		$filter['migration_tags'] = explode(',', $options['tag']);

		$manager = $this->migrationPluginManager;
		$plugins = $manager->createInstances([]);
		@@ -694,15 +667,14 @@ class MigrateToolsCommands extends DrushCommands {
		foreach ($values as $search_value) {
		foreach ($matched_migrations as $id => $migration) {
		// Cast to array because migration_tags can be an array.
		$configured_values = (array) $migration->get($property);
		$configured_id = (in_array(
		$search_value,
		$configured_values
		)) ? $search_value : 'default';
		if (empty($search_value) \|\| $search_value == $configured_id) {
		$definition = $migration->getPluginDefinition();
		$configured_values = (array) ($definition[$property] ?? NULL);
		$configured_id = in_array($search_value, $configured_values, TRUE) ? $search_value : 'default';
		if (empty($search_value) \|\| $search_value === $configured_id) {
		if (empty($migration_ids) \|\| in_array(
		mb_strtolower($id),
		$migration_ids
		$migration_ids,
		TRUE
		)) {
		$filtered_migrations[$id] = $migration;
		}
		@@ -721,7 +693,7 @@ class MigrateToolsCommands extends DrushCommands {
		$migrations[$configured_group_id][$id] = $migration;
		}
		}
		return isset($migrations) ? $migrations : [];
		return $migrations ?? [];
		}

		/**
		@@ -747,8 +719,9 @@ class MigrateToolsCommands extends DrushCommands {
		// migration is not run multiple times.
		static $executed_migrations = [];

		if (isset($options['execute-dependencies'])) {
		$required_migrations = $migration->get('requirements');
		if ($options['execute-dependencies']) {
		$definition = $migration->getPluginDefinition();
		$required_migrations = $definition['requirements'] ?? [];
		$required_migrations = array_filter($required_migrations, function ($value) use ($executed_migrations) {
		return !isset($executed_migrations[$value]);
		});
		@@ -761,11 +734,11 @@ class MigrateToolsCommands extends DrushCommands {
		$executed_migrations += $required_migrations;
		}
		}
		if (!empty($options['force'])) {
		if ($options['force']) {
		$migration->set('requirements', []);
		}
		if (!empty($options['update'])) {
		if (empty($options['idlist'])) {
		if ($options['update']) {
		if (!$options['idlist']) {
		$migration->getIdMap()->prepareUpdate();
		}
		else {

src/MigrateExecutable.php

+1 −1

Original line number	Diff line number	Diff line
		@@ -375,7 +375,7 @@ class MigrateExecutable extends MigrateExecutableBase {
		throw new MigrateSkipRowException('Skipped due to idlist.', FALSE);
		}
		}
		if ($this->feedback && ($this->counter) && $this->counter % $this->feedback == 0) {
		if ($this->feedback && $this->counter && $this->counter % $this->feedback == 0) {
		$this->progressMessage(FALSE);
		$this->resetCounters();
		}

src/MigrateTools.php

+3 −1

Original line number	Diff line number	Diff line
		@@ -2,6 +2,8 @@

		namespace Drupal\migrate_tools;

		use Drupal\migrate_tools\Commands\MigrateToolsCommands;

		/**
		* Utility functionality for use in migrate_tools.
		*/
		@@ -19,7 +21,7 @@ class MigrateTools {
		public static function buildIdList(array $options) {
		$options += [
		'idlist' => NULL,
		'idlist-delimiter' => ':',
		'idlist-delimiter' => MigrateToolsCommands::DEFAULT_ID_LIST_DELIMITER,
		];
		$id_list = [];
		if ($options['idlist']) {

tests/src/Functional/DrushCommandsTest.php

0 → 100644

+113 −0

Original line number	Diff line number	Diff line
		<?php

		namespace Drupal\Tests\migrate_tools\Functional;

		use Drupal\Tests\BrowserTestBase;
		use Drush\TestTraits\DrushTestTrait;

		/**
		* Execute drush on fully functional website.
		*
		* @group migrate_tools
		*/
		class DrushCommandsTest extends BrowserTestBase {
		use DrushTestTrait;

		/**
		* {@inheritdoc}
		*/
		protected static $modules = [
		'migrate_tools_test',
		'migrate_tools',
		'migrate_plus',
		'taxonomy',
		'text',
		'system',
		'user',
		];

		/**
		* Tests migrate:import with feedback.
		*
		* @throws \Drupal\Component\Plugin\Exception\PluginException
		*/
		public function testFeedback() {
		$this->drush('mim', ['fruit_terms'], ['feedback' => 2]);
		$output = $this->getErrorOutputAsList();
		$expected = [
		'[notice] Processed 2 items (2 created, 0 updated, 0 failed, 0 ignored) - continuing with \'fruit_terms\'',
		' [notice] Processed 1 item (1 created, 0 updated, 0 failed, 0 ignored) - done with \'fruit_terms\'',
		];
		$this->assertEquals($expected, $output);
		}

		/**
		* Tests migrate:import with limit.
		*/
		public function testLimit() {
		$this->drush('mim', ['fruit_terms'], ['limit' => 2]);
		$output = $this->getErrorOutputAsList();
		$expected = [
		'[notice] Processed 2 items (2 created, 0 updated, 0 failed, 0 ignored) - done with \'fruit_terms\'',
		];
		$this->assertEquals($expected, $output);
		}

		/**
		* Tests many of the migrate drush commands.
		*/
		public function testDrush() {
		$this->drush('ms', [], [], NULL, NULL, 1);
		$this->assertContains('The "does_not_exist" plugin does not exist.', $this->getErrorOutput());
		$this->drush('cdel', ['migrate_plus.migration.invalid_plugin']);
		// Flush cache so the recently removed invalid migration is cleared.
		$this->drush('cr');
		$this->drush('ms', [], ['format' => 'json']);
		$expected = [
		[
		'group' => 'Default (default)',
		'id' => 'fruit_terms',
		'imported' => 0,
		'status' => 'Idle',
		'total' => 3,
		'unprocessed' => 3,
		'last_imported' => '',
		],
		[
		'group' => 'Default (default)',
		'id' => 'source_exception',
		'imported' => 0,
		'status' => 'Idle',
		'total' => 0,
		'unprocessed' => 0,
		'last_imported' => '',
		],
		];
		$this->assertEquals($expected, $this->getOutputFromJSON());
		$this->drush('mim', ['fruit_terms']);
		$this->assertErrorOutputEquals('[notice] Processed 3 items (3 created, 0 updated, 0 failed, 0 ignored) - done with \'fruit_terms\'');
		$this->drush('mim', ['fruit_terms'], [
		'update' => NULL,
		'force' => NULL,
		'execute-dependencies' => NULL,
		]);
		$this->assertErrorOutputEquals('[notice] Processed 3 items (0 created, 3 updated, 0 failed, 0 ignored) - done with \'fruit_terms\'');
		$this->drush('mrs', ['fruit_terms']);
		$this->assertErrorOutputEquals('[warning] Migration fruit_terms is already Idle');
		$this->drush('mfs', ['fruit_terms'], ['format' => 'json']);
		$expected = [
		[
		'machine_name' => 'name',
		'description' => 'name',
		],
		];
		$this->assertEquals($expected, $this->getOutputFromJSON());
		$this->drush('mmsg', ['fruit_terms']);
		$this->assertErrorOutputEquals('[notice] No messages for this migration');
		$this->drush('mr', ['fruit_terms']);
		$this->assertErrorOutputEquals('[notice] Rolled back 3 items - done with \'fruit_terms\'');
		$this->drush('migrate:stop', ['fruit_terms']);
		$this->assertErrorOutputEquals('[warning] Migration fruit_terms is idle');
		}

		}

tests/src/Kernel/DrushTest.php

+41 −14

Original line number	Diff line number	Diff line
		@@ -38,7 +38,7 @@ class DrushTest extends MigrateTestBase {
		'limit' => NULL,
		'feedback' => NULL,
		'idlist' => NULL,
		'idlist-delimiter' => ':',
		'idlist-delimiter' => MigrateToolsCommands::DEFAULT_ID_LIST_DELIMITER,
		'update' => NULL,
		'force' => NULL,
		'execute-dependencies' => NULL,
		@@ -73,7 +73,9 @@ class DrushTest extends MigrateTestBase {
		$this->installConfig('migrate_plus');
		$this->installConfig('migrate_tools_test');
		$this->installEntitySchema('taxonomy_term');
		$this->installEntitySchema('user');
		$this->installSchema('system', ['key_value', 'key_value_expire']);
		$this->installSchema('user', ['users_data']);
		$this->migrationPluginManager = $this->container->get('plugin.manager.migration');
		$this->logger = $this->container->get('logger.channel.migrate_tools');
		$this->commands = new MigrateToolsCommands(
		@@ -90,14 +92,26 @@ class DrushTest extends MigrateTestBase {
		public function testStatus() {
		$this->executeMigration('fruit_terms');
		/** @var \Consolidation\OutputFormatters\StructuredData\RowsOfFields $result */
		$result = $this->commands->status('fruit_terms');
		$result = $this->commands->status('fruit_terms', [
		'group' => NULL,
		'tag' => NULL,
		'names-only' => FALSE,
		]);
		$rows = $result->getArrayCopy();
		$this->assertSame(1, count($rows));
		$this->assertCount(1, $rows);
		$row = reset($rows);
		$this->assertSame('fruit_terms', $row['id']);
		$this->assertSame(3, $row['total']);
		$this->assertSame(3, $row['imported']);
		$this->assertSame('Idle', $row['status']);

		// Migrate status should not display migrate_drupal migrations if no source
		// database is defined.
		\Drupal::service('module_installer')->uninstall(['migrate_tools_test']);
		$this->enableModules(['migrate_drupal']);
		\Drupal::configFactory()->getEditable('migrate_plus.migration.fruit_terms')->delete();
		$rows = $this->commands->status();
		$this->assertEmpty($rows);
		}

		/**
		@@ -118,12 +132,12 @@ class DrushTest extends MigrateTestBase {
		/** @var \Drupal\migrate\Plugin\MigrationInterface $migration */
		$migration = $this->migrationPluginManager->createInstance('fruit_terms');
		$id_map = $migration->getIdMap();
		$this->commands->import('fruit_terms', ['idlist' => 'Apple'] + $this->importBaseOptions);
		$this->commands->import('fruit_terms', array_merge($this->importBaseOptions, ['idlist' => 'Apple']));
		$this->assertSame(1, $id_map->importedCount());
		$this->commands->import('fruit_terms');
		$this->commands->import('fruit_terms', $this->importBaseOptions);
		$this->assertSame(3, $id_map->importedCount());
		$this->commands->import('fruit_terms', ['idlist' => 'Apple', 'update' => TRUE] + $this->importBaseOptions);
		$this->assertSame(0, count($id_map->getRowsNeedingUpdate(100)));
		$this->commands->import('fruit_terms', array_merge($this->importBaseOptions, ['idlist' => 'Apple', 'update' => TRUE]));
		$this->assertCount(0, $id_map->getRowsNeedingUpdate(100));
		}

		/**
		@@ -132,7 +146,7 @@ class DrushTest extends MigrateTestBase {
		public function testFailingImportThrowsException() {
		$this->expectException(\Exception::class);
		$this->expectExceptionMessage('source_exception migration failed.');
		$this->commands->import('source_exception');
		$this->commands->import('source_exception', $this->importBaseOptions);
		}

		/**
		@@ -148,7 +162,11 @@ class DrushTest extends MigrateTestBase {
		$id_map = $migration->getIdMap();
		$id_map->saveMessage(['name' => 'Apple'], $message);
		/** @var \Consolidation\OutputFormatters\StructuredData\RowsOfFields $result */
		$result = $this->commands->messages('fruit_terms');
		$result = $this->commands->messages('fruit_terms', [
		'csv' => FALSE,
		'idlist' => NULL,
		'idlist-delimiter' => MigrateToolsCommands::DEFAULT_ID_LIST_DELIMITER,
		]);
		$rows = $result->getArrayCopy();
		$this->assertSame($message, $rows[0]['message']);
		}
		@@ -159,7 +177,11 @@ class DrushTest extends MigrateTestBase {
		public function testFailingMessagesThrowsException() {
		$this->expectException(\Exception::class);
		$this->expectExceptionMessage('Migration does_not_exist does not exist');
		$this->commands->messages('does_not_exist');
		$this->commands->messages('does_not_exist', [
		'csv' => FALSE,
		'idlist' => NULL,
		'idlist-delimiter' => MigrateToolsCommands::DEFAULT_ID_LIST_DELIMITER,
		]);
		}

		/**
		@@ -171,7 +193,7 @@ class DrushTest extends MigrateTestBase {
		$migration = $this->migrationPluginManager->createInstance('fruit_terms');
		$id_map = $migration->getIdMap();
		$this->assertSame(3, $id_map->importedCount());
		$this->commands->rollback('fruit_terms');
		$this->commands->rollback('fruit_terms', $this->importBaseOptions);
		$this->assertSame(0, $id_map->importedCount());
		}

		@@ -184,7 +206,7 @@ class DrushTest extends MigrateTestBase {
		/** @var \Drupal\migrate\Plugin\MigrationInterface $migration */
		$migration = $this->migrationPluginManager->createInstance('source_exception');
		$migration->setStatus(MigrationInterface::STATUS_IMPORTING);
		$this->commands->rollback('source_exception');
		$this->commands->rollback('source_exception', $this->importBaseOptions);
		}

		/**
		@@ -196,7 +218,12 @@ class DrushTest extends MigrateTestBase {
		/** @var \Drupal\migrate\Plugin\MigrationInterface $migration */
		$migration = $this->migrationPluginManager->createInstance('fruit_terms');
		$migration->setStatus(MigrationInterface::STATUS_IMPORTING);
		$this->assertSame('Importing', $this->commands->status('fruit_terms')->getArrayCopy()[0]['status']);
		$status = $this->commands->status('fruit_terms', [
		'group' => NULL,
		'tag' => NULL,
		'names-only' => FALSE,
		])->getArrayCopy()[0]['status'];
		$this->assertSame('Importing', $status);
		$this->commands->resetStatus('fruit_terms');
		$this->assertSame(MigrationInterface::STATUS_IDLE, $migration->getStatus());

		@@ -240,7 +267,7 @@ class DrushTest extends MigrateTestBase {
		/** @var \Consolidation\OutputFormatters\StructuredData\RowsOfFields $result */
		$result = $this->commands->fieldsSource('fruit_terms');
		$rows = $result->getArrayCopy();
		$this->assertSame(1, count($rows));
		$this->assertCount(1, $rows);
		$this->assertSame('name', $rows[0]['machine_name']);
		$this->assertSame('name', $rows[0]['description']);
		}