Commit c136a851 authored by xjm's avatar xjm

Issue #2281691 by xjm, mikeryan, Gábor Hojtsy, alexpott, tim.plunkett,...

Issue #2281691 by xjm, mikeryan, Gábor Hojtsy, alexpott, tim.plunkett, webchick, abhishek-anand, quietone, yoroy, stevepurkiss, benjy, Bojhan, Ryan Weal, aleksip, anavarre, hussainweb, er.pushpinderrana, cilefen, Luukyb, Jo Fitzgerald: User interface for migration-based upgrades
parent 50ca1107
......@@ -101,6 +101,7 @@
"drupal/menu_ui": "self.version",
"drupal/migrate": "self.version",
"drupal/migrate_drupal": "self.version",
"drupal/migrate_drupal_ui": "self.version",
"drupal/node": "self.version",
"drupal/options": "self.version",
"drupal/page_cache": "self.version",
......
<?php
/**
* @file
* Contains \Drupal\migrate_drupal\MigrationCreationTrait.
*/
namespace Drupal\migrate_drupal;
use Drupal\Core\Database\Connection;
use Drupal\Core\Database\Database;
use Drupal\migrate\Entity\Migration;
use Drupal\migrate\Exception\RequirementsException;
use Drupal\migrate\Plugin\RequirementsInterface;
use Drupal\Component\Plugin\Exception\PluginNotFoundException;
/**
* Creates the appropriate migrations for a given source Drupal database.
*/
trait MigrationCreationTrait {
/**
* Gets the database connection for the source Drupal database.
*
* @param array $database
* Database array representing the source Drupal database.
*
* @return \Drupal\Core\Database\Connection
* The database connection for the source Drupal database.
*/
protected function getConnection(array $database) {
// Set up the connection.
Database::addConnectionInfo('upgrade', 'default', $database);
$connection = Database::getConnection('default', 'upgrade');
return $connection;
}
/**
* Gets the system data from the system table of the source Drupal database.
*
* @param array $database
* Database array representing the source Drupal database.
*
* @return array
* The system data from the system table of the source Drupal database.
*/
protected function getSystemData(array $database) {
$connection = $this->getConnection($database);
$system_data = [];
try {
$results = $connection->select('system', 's', [
'fetch' => \PDO::FETCH_ASSOC,
])
->fields('s')
->execute();
foreach ($results as $result) {
$system_data[$result['type']][$result['name']] = $result;
}
}
catch (\Exception $e) {
// The table might not exist for example in tests.
}
return $system_data;
}
/**
* Sets up the relevant migrations for import from a database connection.
*
* @param array $database
* Database array representing the source Drupal database.
* @param string $source_base_path
* (Optional) Address of the source Drupal site (e.g., http://example.com/).
*
* @return array
* An array of the migration templates (parsed YAML config arrays) that were
* tagged for the identified source Drupal version. The templates are
* populated with database state key and file source base path information
* for execution. The array is keyed by migration IDs.
*
* @throws \Exception
*/
protected function getMigrationTemplates(array $database, $source_base_path = '') {
// Set up the connection.
$connection = $this->getConnection($database);
if (!$drupal_version = $this->getLegacyDrupalVersion($connection)) {
throw new \Exception('Source database does not contain a recognizable Drupal version.');
}
$database_state['key'] = 'upgrade';
$database_state['database'] = $database;
$database_state_key = 'migrate_drupal_' . $drupal_version;
\Drupal::state()->set($database_state_key, $database_state);
$version_tag = 'Drupal ' . $drupal_version;
$template_storage = \Drupal::service('migrate.template_storage');
$migration_templates = $template_storage->findTemplatesByTag($version_tag);
foreach ($migration_templates as $id => $template) {
$migration_templates[$id]['source']['database_state_key'] = $database_state_key;
// Configure file migrations so they can find the files.
if ($template['destination']['plugin'] == 'entity:file') {
if ($source_base_path) {
// Make sure we have a single trailing slash.
$source_base_path = rtrim($source_base_path, '/') . '/';
$migration_templates[$id]['destination']['source_base_path'] = $source_base_path;
}
}
}
return $migration_templates;
}
/**
* Gets the migrations for import.
*
* Uses the migration template connection to ensure that only the relevant
* migrations are returned.
*
* @param array $migration_templates
* Migration templates (parsed YAML config arrays), keyed by the ID.
*
* @return \Drupal\migrate\Entity\MigrationInterface[]
* The migrations for import.
*/
protected function getMigrations(array $migration_templates) {
// Let the builder service create our migration configuration entities from
// the templates, expanding them to multiple entities where necessary.
/** @var \Drupal\migrate\MigrationBuilder $builder */
$builder = \Drupal::service('migrate.migration_builder');
$initial_migrations = $builder->createMigrations($migration_templates);
$migrations = [];
foreach ($initial_migrations as $migration) {
try {
// Any plugin that has specific requirements to check will implement
// RequirementsInterface.
$source_plugin = $migration->getSourcePlugin();
if ($source_plugin instanceof RequirementsInterface) {
$source_plugin->checkRequirements();
}
$destination_plugin = $migration->getDestinationPlugin();
if ($destination_plugin instanceof RequirementsInterface) {
$destination_plugin->checkRequirements();
}
$migrations[] = $migration;
}
// Migrations which are not applicable given the source and destination
// site configurations (e.g., what modules are enabled) will be silently
// ignored.
catch (RequirementsException $e) {
}
catch (PluginNotFoundException $e) {
}
}
return $migrations;
}
/**
* Saves the migrations for import from the provided template connection.
*
* @param array $migration_templates
* Migration template.
*
* @return array
* The migration IDs sorted in dependency order.
*/
protected function createMigrations(array $migration_templates) {
$migration_ids = [];
$migrations = $this->getMigrations($migration_templates);
foreach ($migrations as $migration) {
// Don't try to resave migrations that already exist.
if (!Migration::load($migration->id())) {
$migration->save();
}
$migration_ids[] = $migration->id();
}
// loadMultiple will sort the migrations in dependency order.
return array_keys(Migration::loadMultiple($migration_ids));
}
/**
* Determines what version of Drupal the source database contains.
*
* @param \Drupal\Core\Database\Connection $connection
* The database connection object.
*
* @return int|FALSE
* An integer representing the major branch of Drupal core (e.g. '6' for
* Drupal 6.x), or FALSE if no valid version is matched.
*/
protected function getLegacyDrupalVersion(Connection $connection) {
// Don't assume because a table of that name exists, that it has the columns
// we're querying. Catch exceptions and report that the source database is
// not Drupal.
// Drupal 5/6/7 can be detected by the schema_version in the system table.
if ($connection->schema()->tableExists('system')) {
try {
$version_string = $connection
->query('SELECT schema_version FROM {system} WHERE name = :module', [':module' => 'system'])
->fetchField();
if ($version_string && $version_string[0] == '1') {
if ((int) $version_string >= 1000) {
$version_string = '5';
}
else {
$version_string = FALSE;
}
}
}
catch (\PDOException $e) {
$version_string = FALSE;
}
}
// For Drupal 8 (and we're predicting beyond) the schema version is in the
// key_value store.
elseif ($connection->schema()->tableExists('key_value')) {
$result = $connection
->query("SELECT value FROM {key_value} WHERE collection = :system_schema and name = :module", [':system_schema' => 'system.schema', ':module' => 'system'])
->fetchField();
$version_string = unserialize($result);
}
else {
$version_string = FALSE;
}
return $version_string ? substr($version_string, 0, 1) : FALSE;
}
}
name: 'Drupal Upgrade UI'
type: module
description: 'UI for direct upgrades from older Drupal versions.'
package: 'Core (Experimental)'
version: VERSION
core: 8.x
configure: migrate_drupal_ui.upgrade
dependencies:
- migrate
- migrate_drupal
- dblog
<?php
/**
* @file
* Install, update, and uninstall functions for the migrate_drupal_ui module.
*/
use Drupal\Core\Url;
/**
* Implements hook_install().
*/
function migrate_drupal_ui_install() {
$url = Url::fromUri('base:upgrade')->toString();
drupal_set_message(t('The Drupal Upgrade UI module has been enabled. Proceed to the <a href=":url">upgrade form</a>.', [':url' => $url]));
}
<?php
/**
* @file
* Alert administrators before starting the import process.
*/
use Drupal\Core\Routing\RouteMatchInterface;
/**
* Implements hook_help().
*/
function migrate_drupal_ui_help($route_name, RouteMatchInterface $route_match) {
switch ($route_name) {
case 'help.page.migrate_drupal_ui':
$output = '<p>' . t('The Drupal Upgrade UI module provides a one-click upgrade from an earlier version of Drupal. For details, see the <a href=":migrate">online documentation for the Drupal Upgrade UI module</a> in the handbook on upgrading from previous versions.', [':migrate' => 'https://www.drupal.org/upgrade/migrate']) . '</p>';
return $output;
}
}
migrate_drupal_ui.upgrade:
path: '/upgrade'
defaults:
_form: '\Drupal\migrate_drupal_ui\Form\MigrateUpgradeForm'
_title: 'Upgrade'
requirements:
_permission: 'administer software updates'
options:
_admin_route: TRUE
migrate_drupal_ui.log:
path: '/upgrade/log'
defaults:
_controller: '\Drupal\migrate_drupal_ui\Controller\MigrateController::showLog'
requirements:
_permission: 'administer software updates'
options:
_admin_route: TRUE
<?php
/**
* @file
* Contains \Drupal\migrate_drupal_ui\Controller\MigrateController.
*/
namespace Drupal\migrate_drupal_ui\Controller;
use Drupal\Core\Controller\ControllerBase;
/**
* Provides controller methods for the migration.
*/
class MigrateController extends ControllerBase {
/**
* Sets a log filter and redirects to the log.
*
* @return \Symfony\Component\HttpFoundation\RedirectResponse
* A redirect response object that may be returned by the controller.
*/
public function showLog() {
$_SESSION['dblog_overview_filter'] = [];
$_SESSION['dblog_overview_filter']['type'] = ['migrate_drupal_ui' => 'migrate_drupal_ui'];
return $this->redirect('dblog.overview');
}
}
This diff is collapsed.
<?php
/**
* @file
* Contains \Drupal\migrate_drupal_ui\MigrateMessageCapture.
*/
namespace Drupal\migrate_drupal_ui;
use Drupal\migrate\MigrateMessageInterface;
/**
* Allows capturing messages rather than displaying them directly.
*/
class MigrateMessageCapture implements MigrateMessageInterface {
/**
* Array of recorded messages.
*
* @var array
*/
protected $messages = [];
/**
* {@inheritdoc}
*/
public function display($message, $type = 'status') {
$this->messages[] = $message;
}
/**
* Clears out any captured messages.
*/
public function clear() {
$this->messages = [];
}
/**
* Returns any captured messages.
*
* @return array
* The captured messages.
*/
public function getMessages() {
return $this->messages;
}
}
<?php
/**
* @file
* Contains \Drupal\migrate_drupal_ui\MigrateUpgradeRunBatch.
*/
namespace Drupal\migrate_drupal_ui;
use Drupal\Core\Link;
use Drupal\Core\Url;
use Drupal\migrate\Entity\Migration;
use Drupal\migrate\Entity\MigrationInterface;
use Drupal\migrate\Event\MigrateEvents;
use Drupal\migrate\Event\MigrateIdMapMessageEvent;
use Drupal\migrate\Event\MigrateMapDeleteEvent;
use Drupal\migrate\Event\MigrateMapSaveEvent;
use Drupal\migrate\Event\MigratePostRowSaveEvent;
use Drupal\migrate\Event\MigrateRowDeleteEvent;
use Drupal\migrate\MigrateExecutable;
/**
* Runs a single migration batch.
*/
class MigrateUpgradeRunBatch {
/**
* Maximum number of previous messages to display.
*/
const MESSAGE_LENGTH = 20;
/**
* The processed items for one batch of a given migration.
*
* @var int
*/
protected static $numProcessed = 0;
/**
* Ensure we only add the listeners once per request.
*
* @var bool
*/
protected static $listenersAdded = FALSE;
/**
* The maximum length in seconds to allow processing in a request.
*
* @see self::run()
*
* @var int
*/
protected static $maxExecTime;
/**
* MigrateMessage instance to capture messages during the migration process.
*
* @var \Drupal\migrate_drupal_ui\MigrateMessageCapture
*/
protected static $messages;
/**
* Runs a single migration batch.
*
* @param int[] $initial_ids
* The full set of migration IDs to import.
* @param string $operation
* The operation to perform, 'import' or 'rollback'.
* @param array $context
* The batch context.
*/
public static function run($initial_ids, $operation, &$context) {
if (!static::$listenersAdded) {
$event_dispatcher = \Drupal::service('event_dispatcher');
if ($operation == 'import') {
$event_dispatcher->addListener(MigrateEvents::POST_ROW_SAVE, [static::class, 'onPostRowSave']);
$event_dispatcher->addListener(MigrateEvents::MAP_SAVE, [static::class, 'onMapSave']);
$event_dispatcher->addListener(MigrateEvents::IDMAP_MESSAGE, [static::class, 'onIdMapMessage']);
}
else {
$event_dispatcher->addListener(MigrateEvents::POST_ROW_DELETE, [static::class, 'onPostRowDelete']);
$event_dispatcher->addListener(MigrateEvents::MAP_DELETE, [static::class, 'onMapDelete']);
}
static::$maxExecTime = ini_get('max_execution_time');
if (static::$maxExecTime <= 0) {
static::$maxExecTime = 60;
}
// Set an arbitrary threshold of 3 seconds (e.g., if max_execution_time is
// 45 seconds, we will quit at 42 seconds so a slow item or cleanup
// overhead don't put us over 45).
static::$maxExecTime -= 3;
static::$listenersAdded = TRUE;
}
if (!isset($context['sandbox']['migration_ids'])) {
$context['sandbox']['max'] = count($initial_ids);
$context['sandbox']['current'] = 1;
// Total number processed for this migration.
$context['sandbox']['num_processed'] = 0;
// migration_ids will be the list of IDs remaining to run.
$context['sandbox']['migration_ids'] = $initial_ids;
$context['sandbox']['messages'] = [];
$context['results']['failures'] = 0;
$context['results']['successes'] = 0;
$context['results']['operation'] = $operation;
}
// Number processed in this batch.
static::$numProcessed = 0;
$migration_id = reset($context['sandbox']['migration_ids']);
/** @var \Drupal\migrate\Entity\Migration $migration */
$migration = Migration::load($migration_id);
if ($migration) {
static::$messages = new MigrateMessageCapture();
$executable = new MigrateExecutable($migration, static::$messages);
$migration_name = $migration->label() ? $migration->label() : $migration_id;
try {
if ($operation == 'import') {
$migration_status = $executable->import();
}
else {
$migration_status = $executable->rollback();
}
}
catch (\Exception $e) {
static::logger()->error($e->getMessage());
$migration_status = MigrationInterface::RESULT_FAILED;
}
switch ($migration_status) {
case MigrationInterface::RESULT_COMPLETED:
// Store the number processed in the sandbox.
$context['sandbox']['num_processed'] += static::$numProcessed;
if ($operation == 'import') {
$message = static::getTranslation()->formatPlural(
$context['sandbox']['num_processed'], 'Upgraded @migration (processed 1 item total)', 'Upgraded @migration (processed @num_processed items total)',
['@migration' => $migration_name, '@num_processed' => $context['sandbox']['num_processed']]);
}
else {
$message = static::getTranslation()->formatPlural(
$context['sandbox']['num_processed'], 'Rolled back @migration (processed 1 item total)', 'Rolled back @migration (processed @num_processed items total)',
['@migration' => $migration_name, '@num_processed' => $context['sandbox']['num_processed']]);
$migration->delete();
}
$context['sandbox']['messages'][] = $message;
static::logger()->notice($message);
$context['sandbox']['num_processed'] = 0;
$context['results']['successes']++;
break;
case MigrationInterface::RESULT_INCOMPLETE:
$context['sandbox']['messages'][] = static::getTranslation()->formatPlural(
static::$numProcessed, 'Continuing with @migration (processed 1 item)', 'Continuing with @migration (processed @num_processed items)',
['@migration' => $migration_name, '@num_processed' => static::$numProcessed]);
$context['sandbox']['num_processed'] += static::$numProcessed;
break;
case MigrationInterface::RESULT_STOPPED:
$context['sandbox']['messages'][] = t('Operation stopped by request');
break;
case MigrationInterface::RESULT_FAILED:
$context['sandbox']['messages'][] = t('Operation on @migration failed', ['@migration' => $migration_name]);
$context['results']['failures']++;
static::logger()->error('Operation on @migration failed', ['@migration' => $migration_name]);
break;
case MigrationInterface::RESULT_SKIPPED:
$context['sandbox']['messages'][] = t('Operation on @migration skipped due to unfulfilled dependencies', ['@migration' => $migration_name]);
static::logger()->error('Operation on @migration skipped due to unfulfilled dependencies', ['@migration' => $migration_name]);
break;
case MigrationInterface::RESULT_DISABLED:
// Skip silently if disabled.
break;
}
// Unless we're continuing on with this migration, take it off the list.
if ($migration_status != MigrationInterface::RESULT_INCOMPLETE) {
array_shift($context['sandbox']['migration_ids']);
$context['sandbox']['current']++;
}
// Add and log any captured messages.
foreach (static::$messages->getMessages() as $message) {
$context['sandbox']['messages'][] = $message;
static::logger()->error($message);
}
// Only display the last MESSAGE_LENGTH messages, in reverse order.
$message_count = count($context['sandbox']['messages']);
$context['message'] = '';
for ($index = max(0, $message_count - self::MESSAGE_LENGTH); $index < $message_count; $index++) {
$context['message'] = $context['sandbox']['messages'][$index] . "<br />\n" . $context['message'];
}
if ($message_count > self::MESSAGE_LENGTH) {
// Indicate there are earlier messages not displayed.
$context['message'] .= '&hellip;';
}
// At the top of the list, display the next one (which will be the one
// that is running while this message is visible).
if (!empty($context['sandbox']['migration_ids'])) {
$migration_id = reset($context['sandbox']['migration_ids']);
$migration = Migration::load($migration_id);
$migration_name = $migration->label() ? $migration->label() : $migration_id;
if ($operation == 'import') {
$context['message'] = t('Currently upgrading @migration (@current of @max total tasks)', [
'@migration' => $migration_name,
'@current' => $context['sandbox']['current'],
'@max' => $context['sandbox']['max'],
]) . "<br />\n" . $context['message'];
}
else {
$context['message'] = t('Currently rolling back @migration (@current of @max total tasks)', [
'@migration' => $migration_name,
'@current' => $context['sandbox']['current'],
'@max' => $context['sandbox']['max'],
]) . "<br />\n" . $context['message'];
}
}
}
else {
array_shift($context['sandbox']['migration_ids']);
$context['sandbox']['current']++;