Commit bf98b07d authored by heddn's avatar heddn Committed by heddn

Issue #2470882 by Zemelia, kriboogh, pguillard, heddn, edurenye, 8balldev,...

Issue #2470882 by Zemelia, kriboogh, pguillard, heddn, edurenye, 8balldev, edysmp: Implement running migration processes through the UI
parent 05f820df
......@@ -74,6 +74,13 @@ entity.migration.process:
_title: 'Process'
requirements:
_permission: 'administer migrations'
entity.migration.process.run:
path: '/admin/structure/migrate/manage/{migration_group}/migrations/{migration}/process/run'
defaults:
_controller: '\Drupal\migrate_tools\Controller\MigrationController::run'
_title: 'Run'
requirements:
_permission: 'administer migrations'
entity.migration.destination:
path: '/admin/structure/migrate/manage/{migration_group}/migrations/{migration}/destination'
defaults:
......@@ -107,3 +114,11 @@ migrate_tools.messages:
_title: 'Messages'
requirements:
_permission: 'administer migrations'
migrate_tools.execute:
path: '/admin/structure/migrate/manage/{migration_group}/migrations/{migration}/execute'
defaults:
_form: '\Drupal\migrate_tools\Form\MigrationExecuteForm'
_title: 'Execute migration'
requirements:
_permission: 'administer migrations'
......@@ -5,9 +5,13 @@ namespace Drupal\migrate_tools\Controller;
use Drupal\Core\Controller\ControllerBase;
use Drupal\Component\Utility\Xss;
use Drupal\Component\Utility\Html;
use Drupal\Core\Routing\CurrentRouteMatch;
use Drupal\migrate\Plugin\MigrationPluginManagerInterface;
use Drupal\Core\DependencyInjection\ContainerInjectionInterface;
use Drupal\migrate_tools\MigrateBatchExecutable;
use Symfony\Component\DependencyInjection\ContainerInterface;
use Drupal\Core\Url;
use Drupal\migrate\MigrateMessage;
/**
* Returns responses for migrate_tools migration view routes.
......@@ -21,14 +25,24 @@ class MigrationController extends ControllerBase implements ContainerInjectionIn
*/
protected $migrationPluginManager;
/**
* The current route match.
*
* @var \Drupal\Core\Routing\CurrentRouteMatch
*/
protected $currentRouteMatch;
/**
* Constructs a new MigrationController object.
*
* @param \Drupal\migrate\Plugin\MigrationPluginManagerInterface $migration_plugin_manager
* The plugin manager for config entity-based migrations.
* @param \Drupal\Core\Routing\CurrentRouteMatch $currentRouteMatch
* The current route match.
*/
public function __construct(MigrationPluginManagerInterface $migration_plugin_manager) {
public function __construct(MigrationPluginManagerInterface $migration_plugin_manager, CurrentRouteMatch $currentRouteMatch) {
$this->migrationPluginManager = $migration_plugin_manager;
$this->currentRouteMatch = $currentRouteMatch;
}
/**
......@@ -36,7 +50,8 @@ class MigrationController extends ControllerBase implements ContainerInjectionIn
*/
public static function create(ContainerInterface $container) {
return new static(
$container->get('plugin.manager.migration')
$container->get('plugin.manager.migration'),
$container->get('current_route_match')
);
}
......@@ -144,6 +159,35 @@ class MigrationController extends ControllerBase implements ContainerInjectionIn
return $build;
}
/**
* Run a migration.
*
* @param string $migration_group
* Machine name of the migration's group.
* @param string $migration
* Machine name of the migration.
*
* @return array
* A render array as expected by drupal_render().
*/
public function run($migration_group, $migration) {
/** @var \Drupal\migrate\Plugin\MigrationInterface $migration */
$migration = $this->migrationPluginManager->createInstance($migration);
$migrateMessage = new MigrateMessage();
$options = [];
$executable = new MigrateBatchExecutable($migration, $migrateMessage, $options);
$executable->batchImport();
$migration_group = $this->currentRouteMatch->getParameter('migration_group');
$route_parameters = [
'migration_group' => $migration_group,
'migration' => $migration->id(),
];
return batch_process(Url::fromRoute('entity.migration.process', $route_parameters));
}
/**
* Display process information of a migration entity.
*
......@@ -204,6 +248,12 @@ class MigrationController extends ControllerBase implements ContainerInjectionIn
'#empty' => $this->t('No process defined.'),
];
$build['process']['run'] = [
'#type' => 'link',
'#title' => $this->t('Run'),
'#url' => Url::fromRoute('entity.migration.process.run', ['migration_group' => $migration_group, 'migration' => $migration->id()]),
];
return $build;
}
......
......@@ -2,6 +2,7 @@
namespace Drupal\migrate_tools\Controller;
use Drupal\Component\Plugin\Exception\PluginException;
use Drupal\Core\Config\Entity\ConfigEntityListBuilder;
use Drupal\Core\Entity\EntityHandlerInterface;
use Drupal\Core\Entity\EntityInterface;
......@@ -99,7 +100,7 @@ class MigrationListBuilder extends ConfigEntityListBuilder implements EntityHand
* @return array
* A render array structure of header strings.
*
* @see Drupal\Core\Entity\EntityListController::render()
* @see \Drupal\Core\Entity\EntityListController::render()
*/
public function buildHeader() {
$header['label'] = $this->t('Migration');
......@@ -110,6 +111,7 @@ class MigrationListBuilder extends ConfigEntityListBuilder implements EntityHand
$header['unprocessed'] = $this->t('Unprocessed');
$header['messages'] = $this->t('Messages');
$header['last_imported'] = $this->t('Last Imported');
$header['operations'] = $this->t('Operations');
return $header;
}
......@@ -125,56 +127,86 @@ class MigrationListBuilder extends ConfigEntityListBuilder implements EntityHand
* @see \Drupal\Core\Entity\EntityListController::render()
*/
public function buildRow(EntityInterface $migration_entity) {
$migration = $this->migrationPluginManager->createInstance($migration_entity->id());
$migration_group = $migration->get('migration_group');
if (!$migration_group) {
$migration_group = 'default';
try {
/** @var \Drupal\migrate\Plugin\MigrationInterface $migration */
$migration = $this->migrationPluginManager->createInstance($migration_entity->id());
$migration_group = $migration->get('migration_group');
if (!$migration_group) {
$migration_group = 'default';
}
$route_parameters = [
'migration_group' => $migration_group,
'migration' => $migration->id(),
];
$row['label'] = [
'data' => [
'#type' => 'link',
'#title' => $migration->label(),
'#url' => Url::fromRoute("entity.migration.overview", $route_parameters),
],
];
$row['machine_name'] = $migration->id();
$row['status'] = $migration->getStatusLabel();
} catch (PluginException $e) {
return NULL;
}
$route_parameters = [
'migration_group' => $migration_group,
'migration' => $migration->id(),
];
$row['label'] = [
'data' => [
'#type' => 'link',
'#title' => $migration->label(),
'#url' => Url::fromRoute("entity.migration.overview", $route_parameters),
],
];
$row['machine_name'] = $migration->id();
$row['status'] = $migration->getStatusLabel();
// Derive the stats.
$source_plugin = $migration->getSourcePlugin();
$row['total'] = $source_plugin->count();
$map = $migration->getIdMap();
$row['imported'] = $map->importedCount();
// -1 indicates uncountable sources.
if ($row['total'] == -1) {
try {
// Derive the stats.
$source_plugin = $migration->getSourcePlugin();
$row['total'] = $source_plugin->count();
$map = $migration->getIdMap();
$row['imported'] = $map->importedCount();
// -1 indicates uncountable sources.
if ($row['total'] == -1) {
$row['total'] = $this->t('N/A');
$row['unprocessed'] = $this->t('N/A');
}
else {
$row['unprocessed'] = $row['total'] - $map->processedCount();
}
$row['messages'] = [
'data' => [
'#type' => 'link',
'#title' => $map->messageCount(),
'#url' => Url::fromRoute("migrate_tools.messages", $route_parameters),
],
];
$migrate_last_imported_store = \Drupal::keyValue('migrate_last_imported');
$last_imported = $migrate_last_imported_store->get($migration->id(), FALSE);
if ($last_imported) {
/** @var \Drupal\Core\Datetime\DateFormatter $date_formatter */
$date_formatter = \Drupal::service('date.formatter');
$row['last_imported'] = $date_formatter->format($last_imported / 1000,
'custom', 'Y-m-d H:i:s');
}
else {
$row['last_imported'] = '';
}
$row['operations']['data'] = [
'#type' => 'dropbutton',
'#links' => [
'simple_form' => [
'title' => $this->t('Execute'),
'url' => Url::fromRoute('migrate_tools.execute', [
'migration_group' => $migration_group,
'migration' => $migration->id(),
]),
],
],
];
} catch (PluginException $e) {
// Derive the stats.
$row['status'] = $this->t('No data found');
$row['total'] = $this->t('N/A');
$row['imported'] = $this->t('N/A');
$row['unprocessed'] = $this->t('N/A');
$row['messages'] = $this->t('N/A');
$row['last_imported'] = $this->t('N/A');
$row['operations'] = $this->t('N/A');
}
else {
$row['unprocessed'] = $row['total'] - $map->processedCount();
}
$row['messages'] = [
'data' => [
'#type' => 'link',
'#title' => $map->messageCount(),
'#url' => Url::fromRoute("migrate_tools.messages", $route_parameters),
],
];
$migrate_last_imported_store = \Drupal::keyValue('migrate_last_imported');
$last_imported = $migrate_last_imported_store->get($migration->id(), FALSE);
if ($last_imported) {
/** @var DateFormatter $date_formatter */
$date_formatter = \Drupal::service('date.formatter');
$row['last_imported'] = $date_formatter->format($last_imported / 1000,
'custom', 'Y-m-d H:i:s');
}
else {
$row['last_imported'] = '';
}
return $row;
}
......
<?php
/**
* @file
* Contains \Drupal\migrate_tools\Form\MigrationExecuteForm
*/
namespace Drupal\migrate_tools\Form;
use Drupal\Core\Form\FormStateInterface;
use Drupal\Core\Form\FormBase;
use Drupal\migrate\MigrateExecutableInterface;
use Drupal\migrate\MigrateMessage;
use Drupal\migrate\MigrateMessageInterface;
use Drupal\migrate\Plugin\Migration;
use Drupal\migrate\Plugin\MigrationInterface;
use Drupal\migrate\Plugin\MigrationPluginManagerInterface;
use Drupal\migrate_tools\MigrateBatchExecutable;
use Drupal\migrate_tools\MigrateExecutable;
use Symfony\Component\DependencyInjection\ContainerInterface;
/**
* This form is specifically for configuring process pipelines.
*/
class MigrationExecuteForm extends FormBase {
/**
* Plugin manager for migration plugins.
*
* @var \Drupal\migrate\Plugin\MigrationPluginManagerInterface
*/
protected $migrationPluginManager;
/**
* Constructs a new MigrationExecuteForm object.
*
* @param \Drupal\migrate\Plugin\MigrationPluginManagerInterface $migration_plugin_manager
* The plugin manager for config entity-based migrations.
*/
public function __construct(MigrationPluginManagerInterface $migration_plugin_manager) {
$this->migrationPluginManager = $migration_plugin_manager;
}
/**
* {@inheritdoc}
*/
public static function create(ContainerInterface $container) {
return new static(
$container->get('plugin.manager.migration')
);
}
/**
* {@inheritdoc}
*/
public function getFormId() {
return 'migration_execute_form';
}
/**
* {@inheritdoc}
*/
public function buildForm(array $form, FormStateInterface $form_state) {
$form = [];
$form['operations'] = $this->migrateMigrateOperations();
return $form;
}
/**
* Get Operations.
*/
private function migrateMigrateOperations() {
// Build the 'Update options' form.
$form = [
'#type' => 'fieldset',
'#title' => t('Operations'),
];
$options = [
'import' => t('Import'),
'rollback' => t('Rollback'),
'stop' => t('Stop'),
'reset' => t('Reset'),
];
$form['operation'] = [
'#type' => 'select',
'#title' => t('Choose an operation to run'),
'#options' => $options,
'#default_value' => 'import',
'#required' => TRUE,
];
$form['submit'] = [
'#type' => 'submit',
'#value' => t('Execute'),
];
$definitions = [];
$definitions[] = $this->t('Import: Imports all previously unprocessed records from the source, plus any records marked for update, into destination Drupal objects.');
$definitions[] = $this->t('Rollback: Deletes all Drupal objects created by the import.');
$definitions[] = $this->t('Stop: Cleanly interrupts any import or rollback processes that may currently be running.');
$definitions[] = $this->t('Reset: Sometimes a process may fail to stop cleanly, and be left stuck in an Importing or Rolling Back status. Choose Reset to clear the status and permit other operations to proceed.');
$form['definitions'] = [
'#theme' => 'item_list',
'#title' => $this->t('Definitions'),
'#list_type' => 'ul',
'#items' => $definitions,
];
$form['options'] = [
'#type' => 'fieldset',
'#title' => t('Options'),
'#collapsible' => TRUE,
'#collapsed' => TRUE,
];
$form['options']['update'] = [
'#type' => 'checkbox',
'#title' => t('Update'),
'#description' => t('Check this box to update all previously-imported content
in addition to importing new content. Leave unchecked to only import
new content'),
];
$form['options']['force'] = [
'#type' => 'checkbox',
'#title' => t('Ignore dependencies'),
'#description' => t('Check this box to ignore dependencies when running imports
- all tasks will run whether or not their dependent tasks have
completed.'),
];
// @TODO: Limit is not working. Perhaps because of batch? See
// https://www.drupal.org/project/migrate_tools/issues/2924298
// $form['options']['limit'] = [
// '#type' => 'textfield',
// '#title' => t('Limit to:'),
// '#size' => 10,
// '#description' => t('Set a limit of how many items to process for each migration task.'),
// ];
return $form;
}
/**
* {@inheritdoc}
*/
public function validateForm(array &$form, FormStateInterface $form_state) {
if (empty($form_state->getValue('operation'))) {
$form_state->setErrorByName('operation', $this->t('Please select an operation.'));
return;
}
}
/**
* {@inheritdoc}
*/
public function submitForm(array &$form, FormStateInterface $form_state) {
$operation = $form_state->getValue('operation');
if ($form_state->getValue('limit')) {
$limit = $form_state->getValue('limit');
}
else {
$limit = 0;
}
if ($form_state->getValue('update')) {
$update = $form_state->getValue('update');
}
else {
$update = 0;
}
if ($form_state->getValue('force')) {
$force = $form_state->getValue('force');
}
else {
$force = 0;
}
$migration_name = \Drupal::routeMatch()->getParameter('migration');
if ($migration_name) {
/** @var MigrationInterface $migration */
$migration = $this->migrationPluginManager->createInstance($migration_name);
$migrateMessage = new MigrateMessage();
switch ($operation) {
case 'import':
$options = [
'limit' => $limit,
'update' => $update,
'force' => $force,
];
$executable = new MigrateBatchExecutable($migration, $migrateMessage, $options);
$executable->batchImport();
break;
case 'rollback':
$options = [
'limit' => $limit,
'update' => $update,
'force' => $force
];
$executable = new MigrateBatchExecutable($migration, $migrateMessage, $options);
$executable->rollback();
break;
case 'stop':
$migration->interruptMigration(MigrationInterface::RESULT_STOPPED);
break;
case 'reset':
$migration->setStatus(MigrationInterface::STATUS_IDLE);
break;
}
}
}
}
<?php
namespace Drupal\migrate_tools;
use Drupal\migrate\MigrateMessage;
use Drupal\migrate\MigrateMessageInterface;
use Drupal\migrate\Plugin\Migration;
use Drupal\migrate\Plugin\MigrationInterface;
/**
* Defines a migrate executable class for batch migrations through UI.
*/
class MigrateBatchExecutable extends MigrateExecutable {
/**
* Representing a batch import operation.
*/
const BATCH_IMPORT = 1;
/**
* Indicates if we need to update existing rows or skip them.
*
* @var int
*/
protected $updateExistingRows = 0;
/**
* Indicates if we need import dependent migrations also.
*
* @var int
*/
protected $checkDependencies = 0;
/**
* The current batch context.
*
* @var array
*/
protected $batchContext = [];
/**
* Plugin manager for migration plugins.
*
* @var \Drupal\migrate\Plugin\MigrationPluginManagerInterface
*/
protected $migrationPluginManager;
/**
* {@inheritdoc}
*/
public function __construct(MigrationInterface $migration, MigrateMessageInterface $message, array $options = []) {
if (isset($options['update'])) {
$this->updateExistingRows = $options['update'];
}
if (isset($options['force'])) {
$this->checkDependencies = $options['force'];
}
parent::__construct($migration, $message, $options);
$this->migrationPluginManager = \Drupal::getContainer()->get('plugin.manager.migration');
}
/**
* Sets the current batch content so listeners can update the messages.
*
* @param array $context
*/
public function setBatchContext(&$context) {
$this->batchContext = &$context;
}
/**
* Gets a reference to the current batch context.
*
* @return array
*/
public function &getBatchContext() {
return $this->batchContext;
}
/**
* Setup batch operations for running the migration.
*/
public function batchImport() {
// Create the batch operations for each migration that needs to be executed.
// This includes the migration for this executable, but also the dependent
// migrations.
$operations = $this->batchOperations([$this->migration], 'import', [
'limit' => $this->itemLimit,
'update' => $this->updateExistingRows,
'force' => $this->checkDependencies
]);
if (count($operations) > 0) {
$batch = [
'operations' => $operations,
'title' => t('Migrating %migrate', ['%migrate' => $this->migration->label()]),
'init_message' => t('Start migrating %migrate', ['%migrate' => $this->migration->label()]),
'progress_message' => t('Migrating %migrate', ['%migrate' => $this->migration->label()]),
'error_message' => t('An error occurred while migrating %migrate.', ['%migrate' => $this->migration->label()]),
'finished' => '\Drupal\migrate_tools\MigrateBatchExecutable::batchFinishedImport',
];
batch_set($batch);
}
}
/**
* Helper to generate the batch operations for importing migrations.
*
* @param array $migrations
* @param array $operation
* @param array $options
*
* @return array
*/
protected function batchOperations($migrations, $operation, $options = []) {
$operations = [];
/**
* @var string $id
* @var Migration $migration
*/
foreach ($migrations as $id => $migration) {
if (!empty($options['update'])) {
$migration->getIdMap()->prepareUpdate();
}
if (!empty($options['force'])) {
$migration->set('requirements', []);
}
else {
$dependencies = $migration->getMigrationDependencies();
if (!empty($dependencies['required'])) {
$required_migrations = $this->migrationPluginManager->createInstances($dependencies['required']);
// For dependent migrations will need to be migrate all items.
$dependent_options = $options;
$dependent_options['limit'] = 0;
$operations += $this->batchOperations($required_migrations, $operation, [
'limit' => 0,
'update' => $options['update'],
'force' => $options['force']
]);
}
}
$operations[] = [
'\Drupal\migrate_tools\MigrateBatchExecutable::batchProcessImport',
[$migration->id(), $options]
];
}
return $operations;
}
/**
* Batch 'operation' callback
*
* @param string $migration_id
* @param array $options
* @param array $context
*
*/
static public function batchProcessImport($migration_id, $options, &$context) {
if (empty($context['sandbox'])) {
$context['finished'] = 0;
$context['sandbox'] = [];
$context['sandbox']['total'] = 0;
$context['sandbox']['counter'] = 0;
$context['sandbox']['batch_limit'] = 0;
$context['sandbox']['operation'] = MigrateBatchExecutable::BATCH_IMPORT;
}
// Prepare the migration executable.
$message = new MigrateMessage();
/** @var MigrationInterface $migration */
$migration = \Drupal::getContainer()->get('plugin.manager.migration')->createInstance($migration_id);
$executable = new MigrateBatchExecutable($migration, $message, $options);
if (empty($context['sandbox']['total'])) {
$context['sandbox']['total'] = $executable->getSource()->count();
$context['sandbox']['batch_limit'] = $executable->calculateBatchLimit($context);
$context['results'][$migration->id()] = [
'@numitems' => 0,
'@created' => 0,
'@updated' => 0,
'@failures' => 0,
'@ignored' => 0,
'@name' => $migration->id()
];
}
// Every iteration, we reset out batch counter.
$context['sandbox']['batch_counter'] = 0;
// Make sure we know our batch context.
$executable->setBatchContext($context);
// Do the import.
$result = $executable->import();
// Store the result, we will need to combine the results of all our iterations.
$context['results'][$migration->id()] = [
'@numitems' => $context['results'][$migration->id()]['@numitems'] + $executable->getProcessedCount(),
'@created' => $context['results'][$migration->id()]['@created'] + $executable->getCreatedCount(),
'@updated' => $context['results'][$migration->id()]['@updated'] + $executable->getUpdatedCount(),
'@failures' => $context['results'][$migration->id()]['@failures'] + $executable->getFailedCount(),
'@ignored' => $context['results'][$migration->id()]['@ignored'] + $executable->getIgnoredCount(),
'@name' => $migration->id()