Commit 669c2ef2 authored by Gábor Hojtsy's avatar Gábor Hojtsy

Issue #2687843 by quietone, heddn, Jo Fitzgerald, alexpott, piggito, mallezie,...

Issue #2687843 by quietone, heddn, Jo Fitzgerald, alexpott, piggito, mallezie, mikeryan, xjm, phenaproxima, benjifisher, webchick, Gábor Hojtsy, masipila, benjy, abhishek-anand, catch: Add back incremental migrations through the UI
parent 97147960
......@@ -167,8 +167,10 @@ public function build() {
*/
public function getCacheTags() {
$cache_tags = parent::getCacheTags();
$feed = $this->feedStorage->load($this->configuration['feed']);
return Cache::mergeTags($cache_tags, $feed->getCacheTags());
if ($feed = $this->feedStorage->load($this->configuration['feed'])) {
$cache_tags = Cache::mergeTags($cache_tags, $feed->getCacheTags());
}
return $cache_tags;
}
}
......@@ -37,6 +37,7 @@ protected function setUp() {
$this->installSchema('forum', ['forum_index']);
$this->installSchema('node', ['node_access']);
$this->installSchema('search', ['search_dataset']);
$this->installSchema('system', ['sequences']);
$this->installSchema('tracker', ['tracker_node', 'tracker_user']);
// Enable content moderation for nodes of type page.
......
......@@ -37,6 +37,7 @@ protected function setUp() {
$this->installSchema('forum', ['forum_index']);
$this->installSchema('node', ['node_access']);
$this->installSchema('search', ['search_dataset']);
$this->installSchema('system', ['sequences']);
$this->installSchema('tracker', ['tracker_node', 'tracker_user']);
// Enable content moderation for nodes of type page.
......
......@@ -2,16 +2,6 @@
namespace Drupal\Tests\migrate_drupal\Traits;
use Drupal\aggregator\Entity\Feed;
use Drupal\aggregator\Entity\Item;
use Drupal\block_content\Entity\BlockContent;
use Drupal\comment\Entity\Comment;
use Drupal\file\Entity\File;
use Drupal\menu_link_content\Entity\MenuLinkContent;
use Drupal\node\Entity\Node;
use Drupal\taxonomy\Entity\Term;
use Drupal\user\Entity\User;
/**
* Provides helper methods for creating test content.
*/
......@@ -60,72 +50,161 @@ protected function installEntitySchemas() {
* Create several pieces of generic content.
*/
protected function createContent() {
$entity_type_manager = \Drupal::entityTypeManager();
// Create an aggregator feed.
$feed = Feed::create([
'title' => 'feed',
'url' => 'http://www.example.com',
]);
$feed->save();
// Create an aggregator feed item.
$item = Item::create([
'title' => 'feed item',
'fid' => $feed->id(),
'link' => 'http://www.example.com',
]);
$item->save();
if ($entity_type_manager->hasDefinition('aggregator_feed')) {
$feed = $entity_type_manager->getStorage('aggregator_feed')->create([
'title' => 'feed',
'url' => 'http://www.example.com',
]);
$feed->save();
// Create an aggregator feed item.
$item = $entity_type_manager->getStorage('aggregator_item')->create([
'title' => 'feed item',
'fid' => $feed->id(),
'link' => 'http://www.example.com',
]);
$item->save();
}
// Create a block content.
if ($entity_type_manager->hasDefinition('block_content')) {
$block = $entity_type_manager->getStorage('block_content')->create([
'info' => 'block',
'type' => 'block',
]);
$block->save();
}
// Create a node.
if ($entity_type_manager->hasDefinition('node')) {
$node = $entity_type_manager->getStorage('node')->create([
'type' => 'page',
'title' => 'page',
]);
$node->save();
// Create a comment.
if ($entity_type_manager->hasDefinition('comment')) {
$comment = $entity_type_manager->getStorage('comment')->create([
'comment_type' => 'comment',
'field_name' => 'comment',
'entity_type' => 'node',
'entity_id' => $node->id(),
]);
$comment->save();
}
}
// Create a file.
if ($entity_type_manager->hasDefinition('file')) {
$file = $entity_type_manager->getStorage('file')->create([
'uri' => 'public://example.txt',
]);
$file->save();
}
// Create a menu link.
if ($entity_type_manager->hasDefinition('menu_link_content')) {
$menu_link = $entity_type_manager->getStorage('menu_link_content')->create([
'title' => 'menu link',
'link' => ['uri' => 'http://www.example.com'],
'menu_name' => 'tools',
]);
$menu_link->save();
}
// Create a taxonomy term.
if ($entity_type_manager->hasDefinition('taxonomy_term')) {
$term = $entity_type_manager->getStorage('taxonomy_term')->create([
'name' => 'term',
'vid' => 'term',
]);
$term->save();
}
// Create a user.
if ($entity_type_manager->hasDefinition('user')) {
$user = $entity_type_manager->getStorage('user')->create([
'name' => 'user',
'mail' => 'user@example.com',
]);
$user->save();
}
}
/**
* Create several pieces of generic content.
*/
protected function createContentPostUpgrade() {
$entity_type_manager = \Drupal::entityTypeManager();
// Create a block content.
$block = BlockContent::create([
'info' => 'block',
'type' => 'block',
]);
$block->save();
if ($entity_type_manager->hasDefinition('block_content')) {
$block = $entity_type_manager->getStorage('block_content')->create([
'info' => 'Post upgrade block',
'type' => 'block',
]);
$block->save();
}
// Create a node.
$node = Node::create([
'type' => 'page',
'title' => 'page',
]);
$node->save();
// Create a comment.
$comment = Comment::create([
'comment_type' => 'comment',
'field_name' => 'comment',
'entity_type' => 'node',
'entity_id' => $node->id(),
]);
$comment->save();
if ($entity_type_manager->hasDefinition('node')) {
$node = $entity_type_manager->getStorage('node')->create([
'type' => 'page',
'title' => 'Post upgrade page',
]);
$node->save();
// Create a comment.
if ($entity_type_manager->hasDefinition('comment')) {
$comment = $entity_type_manager->getStorage('comment')->create([
'comment_type' => 'comment',
'field_name' => 'comment',
'entity_type' => 'node',
'entity_id' => $node->id(),
]);
$comment->save();
}
}
// Create a file.
$file = File::create([
'uri' => 'public://example.txt',
]);
$file->save();
if ($entity_type_manager->hasDefinition('file')) {
$file = $entity_type_manager->getStorage('file')->create([
'uri' => 'public://post_upgrade_example.txt',
]);
$file->save();
}
// Create a menu link.
$menu_link = MenuLinkContent::create([
'title' => 'menu link',
'link' => ['uri' => 'http://www.example.com'],
'menu_name' => 'tools',
]);
$menu_link->save();
if ($entity_type_manager->hasDefinition('menu_link_content')) {
$menu_link = $entity_type_manager->getStorage('menu_link_content')->create([
'title' => 'post upgrade menu link',
'link' => ['uri' => 'http://www.drupal.org'],
'menu_name' => 'tools',
]);
$menu_link->save();
}
// Create a taxonomy term.
$term = Term::create([
'name' => 'term',
'vid' => 'term',
]);
$term->save();
if ($entity_type_manager->hasDefinition('taxonomy_term')) {
$term = $entity_type_manager->getStorage('taxonomy_term')->create([
'name' => 'post upgrade term',
'vid' => 'term',
]);
$term->save();
}
// Create a user.
$user = User::create([
'uid' => 2,
'name' => 'user',
'mail' => 'user@example.com',
]);
$user->save();
if ($entity_type_manager->hasDefinition('user')) {
$user = $entity_type_manager->getStorage('user')->create([
'name' => 'universe',
'mail' => 'universe@example.com',
]);
$user->save();
}
}
}
......@@ -31,8 +31,6 @@ function migrate_drupal_ui_help($route_name, RouteMatchInterface $route_match) {
$output .= '<dt>' . t('Reviewing the upgrade log') . '</dt>';
$output .= '<dd>' . t('You can review a <a href=":log">log of upgrade messages</a> by clicking the link in the message provided after the upgrade or by filtering the messages for the type <em>migrate_drupal_ui</em> on the <a href=":messages">Recent log messages</a> page.',
[':log' => \Drupal::url('migrate_drupal_ui.log'), ':messages' => \Drupal::url('dblog.overview')]) . '</dd>';
$output .= '<dt>' . t('Incremental upgrades') . '</dt>';
$output .= '<dd>' . t('Incremental upgrades are not yet supported through the user interface.') . '</dd>';
$output .= '<dt>' . t('Rolling back an upgrade') . '</dt>';
$output .= '<dd>' . t('Rolling back an upgrade is not yet supported through the user interface.') . '</dd>';
$output .= '</dl>';
......
......@@ -2,6 +2,7 @@
namespace Drupal\migrate_drupal_ui\Form;
use Drupal\Core\Database\DatabaseExceptionWrapper;
use Drupal\Core\Datetime\DateFormatterInterface;
use Drupal\Core\Extension\ModuleHandlerInterface;
use Drupal\Core\Form\ConfirmFormBase;
......@@ -279,9 +280,16 @@ public function buildOverviewForm(array $form, FormStateInterface $form_state) {
// https://www.drupal.org/node/2687849
$form['upgrade_option_item'] = [
'#type' => 'item',
'#prefix' => $this->t('An upgrade has already been performed on this site. To perform a new migration, create a clean and empty new install of Drupal 8. Rollbacks and incremental migrations are not yet supported through the user interface. For more information, see the <a href=":url">upgrading handbook</a>.', [':url' => 'https://www.drupal.org/upgrade/migrate']),
'#prefix' => $this->t('An upgrade has already been performed on this site. To perform a new migration, create a clean and empty new install of Drupal 8. Rollbacks are not yet supported through the user interface. For more information, see the <a href=":url">upgrading handbook</a>.', [':url' => 'https://www.drupal.org/upgrade/migrate']),
'#description' => $this->t('Last upgrade: @date', ['@date' => $this->dateFormatter->format($date_performed)]),
];
$form['actions']['incremental'] = [
'#type' => 'submit',
'#value' => $this->t('Import new configuration and content from old site'),
'#button_type' => 'primary',
'#validate' => ['::validateIncrementalForm'],
'#submit' => ['::submitIncrementalForm'],
];
return $form;
}
else {
......@@ -344,8 +352,50 @@ public function buildOverviewForm(array $form, FormStateInterface $form_state) {
* The current state of the form.
*/
public function submitOverviewForm(array &$form, FormStateInterface $form_state) {
$form_state->set('step', 'credentials');
$form_state->setRebuild();
$form_state->set('step', 'credentials')->setRebuild();
}
/**
* Validation handler for the incremental overview form.
*
* @param array $form
* An associative array containing the structure of the form.
* @param \Drupal\Core\Form\FormStateInterface $form_state
* The current state of the form.
*/
public function validateIncrementalForm(array &$form, FormStateInterface $form_state) {
// Retrieve the database driver from state.
$database_state_key = $this->state->get('migrate.fallback_state_key', '');
if ($database_state_key) {
try {
$database = $this->state->get($database_state_key, [])['database'];
if ($connection = $this->getConnection($database)) {
if ($version = $this->getLegacyDrupalVersion($connection)) {
$this->setupMigrations($database, $form_state);
$valid_legacy_database = TRUE;
}
}
}
catch (DatabaseExceptionWrapper $exception) {
// Hide DB exceptions and forward to the DB credentials form. In that
// form we can more properly display errors and accept new credentials.
}
}
if (empty($valid_legacy_database)) {
$form_state->setValue('step', 'credentials')->setRebuild();
}
}
/**
* Form submission handler for the incremental overview form.
*
* @param array $form
* An associative array containing the structure of the form.
* @param \Drupal\Core\Form\FormStateInterface $form_state
* The current state of the form.
*/
public function submitIncrementalForm(array &$form, FormStateInterface $form_state) {
$form_state->set('step', 'confirm_id_conflicts')->setRebuild();
}
/**
......@@ -528,31 +578,7 @@ public function validateCredentialForm(array &$form, FormStateInterface $form_st
]));
}
else {
$this->createDatabaseStateSettings($database, $version);
$migrations = $this->getMigrations('migrate_drupal_' . $version, $version);
// Get the system data from source database.
$system_data = $this->getSystemData($connection);
// Convert the migration object into array
// so that it can be stored in form storage.
$migration_array = [];
foreach ($migrations as $migration) {
$migration_array[$migration->id()] = $migration->label();
}
// Store the retrieved migration IDs in form storage.
$form_state->set('version', $version);
$form_state->set('migrations', $migration_array);
if ($version === '6') {
$form_state->set('source_base_path', $form_state->getValue('d6_source_base_path'));
}
else {
$form_state->set('source_base_path', $form_state->getValue('source_base_path'));
}
$form_state->set('source_private_file_path', $form_state->getValue('source_private_file_path'));
// Store the retrived system data in form storage.
$form_state->set('system_data', $system_data);
$this->setupMigrations($database, $form_state);
}
}
catch (\Exception $e) {
......@@ -974,6 +1000,48 @@ protected function getDatabaseTypes() {
return drupal_get_database_types();
}
/**
* Puts migrations information in form state.
*
* Gets all the migrations, converts each to an array and stores it in the
* form state. The source base path for public and private files is also
* put into form state.
*
* @param array $database
* Database array representing the source Drupal database.
* @param \Drupal\Core\Form\FormStateInterface $form_state
* The current state of the form.
*/
protected function setupMigrations(array $database, FormStateInterface $form_state) {
$connection = $this->getConnection($database);
$version = $this->getLegacyDrupalVersion($connection);
$this->createDatabaseStateSettings($database, $version);
$migrations = $this->getMigrations('migrate_drupal_' . $version, $version);
// Get the system data from source database.
$system_data = $this->getSystemData($connection);
// Convert the migration object into array
// so that it can be stored in form storage.
$migration_array = [];
foreach ($migrations as $migration) {
$migration_array[$migration->id()] = $migration->label();
}
// Store the retrieved migration IDs in form storage.
$form_state->set('version', $version);
$form_state->set('migrations', $migration_array);
if ($version == 6) {
$form_state->set('source_base_path', $form_state->getValue('d6_source_base_path'));
}
else {
$form_state->set('source_base_path', $form_state->getValue('source_base_path'));
}
$form_state->set('source_private_file_path', $form_state->getValue('source_private_file_path'));
// Store the retrieved system data in form storage.
$form_state->set('system_data', $system_data);
}
/**
* {@inheritdoc}
*/
......
......@@ -2,7 +2,6 @@
namespace Drupal\Tests\migrate_drupal_ui\Functional;
use Drupal\migrate\Plugin\MigrateIdMapInterface;
use Drupal\migrate_drupal\MigrationConfigurationTrait;
use Drupal\Tests\migrate_drupal\Traits\CreateTestContentEntitiesTrait;
......@@ -26,6 +25,12 @@ protected function setUp() {
/**
* Executes all steps of migrations upgrade.
*
* The upgrade is started three times. The first time is to test that
* providing incorrect database credentials fails as expected. The second
* time is to run the migration and assert the results. The third time is
* to test an incremental migration, by installing the aggregator module,
* and assert the results.
*/
public function testMigrateUpgradeExecute() {
$connection_options = $this->sourceDatabase->getConnectionOptions();
......@@ -84,22 +89,11 @@ public function testMigrateUpgradeExecute() {
$session->fieldExists('mysql[host]');
$this->drupalPostForm(NULL, $edits, t('Review upgrade'));
$session->pageTextContains('WARNING: Content may be overwritten on your new site.');
$session->pageTextContains('There is conflicting content of these types:');
$session->pageTextContains('aggregator feed entities');
$session->pageTextContains('aggregator feed item entities');
$session->pageTextContains('custom block entities');
$session->pageTextContains('custom menu link entities');
$session->pageTextContains('file entities');
$session->pageTextContains('taxonomy term entities');
$session->pageTextContains('user entities');
$session->pageTextContains('comments');
$session->pageTextContains('content item revisions');
$session->pageTextContains('content items');
$session->pageTextContains('There is translated content of these types:');
$this->assertIdConflict($session);
$this->drupalPostForm(NULL, [], t('I acknowledge I may lose data. Continue anyway.'));
$session->statusCodeEquals(200);
$session->pageTextContains('What will be upgraded?');
// Ensure there are no errors about missing modules from the test module.
$session->pageTextNotContains(t('Source module not found for migration_provider_no_annotation.'));
$session->pageTextNotContains(t('Source module not found for migration_provider_test.'));
......@@ -109,49 +103,40 @@ public function testMigrateUpgradeExecute() {
// Test the upgrade paths.
$available_paths = $this->getAvailablePaths();
$missing_paths = $this->getMissingPaths();
$this->assertUpgradePaths($session, $available_paths, $missing_paths);
$this->assertReviewPage($session, $available_paths, $missing_paths);
$this->drupalPostForm(NULL, [], t('Perform upgrade'));
$this->assertText(t('Congratulations, you upgraded Drupal!'));
$this->assertMigrationResults($this->getEntityCounts(), $version);
// Have to reset all the statics after migration to ensure entities are
// loadable.
$this->resetAll();
$expected_counts = $this->getEntityCounts();
foreach (array_keys(\Drupal::entityTypeManager()
->getDefinitions()) as $entity_type) {
$real_count = \Drupal::entityQuery($entity_type)->count()->execute();
$expected_count = isset($expected_counts[$entity_type]) ? $expected_counts[$entity_type] : 0;
$this->assertEqual($expected_count, $real_count, "Found $real_count $entity_type entities, expected $expected_count.");
}
$plugin_manager = \Drupal::service('plugin.manager.migration');
/** @var \Drupal\migrate\Plugin\Migration[] $all_migrations */
$all_migrations = $plugin_manager->createInstancesByTag('Drupal ' . $version);
foreach ($all_migrations as $migration) {
$id_map = $migration->getIdMap();
foreach ($id_map as $source_id => $map) {
// Convert $source_id into a keyless array so that
// \Drupal\migrate\Plugin\migrate\id_map\Sql::getSourceHash() works as
// expected.
$source_id_values = array_values(unserialize($source_id));
$row = $id_map->getRowBySource($source_id_values);
$destination = serialize($id_map->currentDestination());
$message = "Migration of $source_id to $destination as part of the {$migration->id()} migration. The source row status is " . $row['source_row_status'];
// A completed migration should have maps with
// MigrateIdMapInterface::STATUS_IGNORED or
// MigrateIdMapInterface::STATUS_IMPORTED.
if ($row['source_row_status'] == MigrateIdMapInterface::STATUS_FAILED || $row['source_row_status'] == MigrateIdMapInterface::STATUS_NEEDS_UPDATE) {
$this->fail($message);
}
else {
$this->pass($message);
}
}
}
\Drupal::service('module_installer')->install(['forum']);
\Drupal::service('module_installer')->install(['book']);
// Test incremental migration.
$this->createContentPostUpgrade();
$this->drupalGet('/upgrade');
$session->pageTextContains('An upgrade has already been performed on this site. To perform a new migration, create a clean and empty new install of Drupal 8. Rollbacks are not yet supported through the user interface.');
$this->drupalPostForm(NULL, [], t('Import new configuration and content from old site'));
$session->pageTextContains('WARNING: Content may be overwritten on your new site.');
$session->pageTextContains('There is conflicting content of these types:');
$session->pageTextContains('file entities');
$session->pageTextContains('content item revisions');
$session->pageTextContains('There is translated content of these types:');
$session->pageTextContains('content items');
$this->drupalPostForm(NULL, [], t('I acknowledge I may lose data. Continue anyway.'));
$session->statusCodeEquals(200);
// Need to update available and missing path lists.
$all_available = $this->getAvailablePaths();
$all_available[] = 'aggregator';
$all_missing = $this->getMissingPaths();
$all_missing = array_diff($all_missing, ['aggregator']);
$this->assertReviewPage($session, $all_available, $all_missing);
$this->drupalPostForm(NULL, [], t('Perform upgrade'));
$session->pageTextContains(t('Congratulations, you upgraded Drupal!'));
$this->assertMigrationResults($this->getEntityCountsIncremental(), $version);
}
}
......@@ -121,4 +121,11 @@ protected function getEntityCounts() {
return [];
}
/**
* {@inheritdoc}
*/
protected function getEntityCountsIncremental() {
return [];
}
}
......@@ -3,6 +3,7 @@
namespace Drupal\Tests\migrate_drupal_ui\Functional;
use Drupal\Core\Database\Database;
use Drupal\migrate\Plugin\MigrateIdMapInterface;
use Drupal\migrate_drupal\MigrationConfigurationTrait;
use Drupal\Tests\BrowserTestBase;
use Drupal\Tests\migrate_drupal\Traits\CreateTestContentEntitiesTrait;
......@@ -178,4 +179,107 @@ protected function assertUpgradePaths(WebAssert $session, array $available_paths
*/
abstract protected function getMissingPaths();
/**
* Gets expected number of entities per entity after incremental migration.
*
* @return int[]
* An array of expected counts keyed by entity type ID.
*/
abstract protected function getEntityCountsIncremental();
/**
* Helper method to assert the text on the 'Upgrade analysis report' page.
*
* @param \Drupal\Tests\WebAssert $session
* The current session.
* @param array $all_available
* Array of modules that will be upgraded.
* @param array $all_missing
* Array of modules that will not be upgraded.
*/
protected function assertReviewPage(WebAssert $session, array $all_available, array $all_missing) {
$this->assertText('What will be upgraded?');
// Ensure there are no errors about the missing modules from the test module.
$session->pageTextNotContains(t('Source module not found for migration_provider_no_annotation.'));
$session->pageTextNotContains(t('Source module not found for migration_provider_test.'));
$session->pageTextNotContains(t('Destination module not found for migration_provider_test'));
// Ensure there are no errors about any other missing migration providers.
$session->pageTextNotContains(t('module not found'));
// Test the available migration paths.
foreach ($all_available as $available) {
$session->elementExists('xpath', "//span[contains(@class, 'checked') and text() = '$available']");
$session->elementNotExists('xpath', "//span[contains(@class, 'error') and text() = '$available']");
}
// Test the missing migration paths.
foreach ($all_missing as $missing) {
$session->elementExists('xpath', "//span[contains(@class, 'error') and text() = '$missing']");
$session->elementNotExists('xpath', "//span[contains(@class, 'checked') and text() = '$missing']");
}
}
/**
* Helper method that asserts text on the ID conflict form.
*
* @param \Drupal\Tests\WebAssert $session
* The current session.
* @param $session
* The current session.
*/
protected function assertIdConflict(WebAssert $session) {
$session->pageTextContains('WARNING: Content may be overwritten on your new site.');
$session->pageTextContains('There is conflicting content of these types:');
$session->pageTextContains('custom block entities');
$session->pageTextContains('custom menu link entities');
$session->pageTextContains('file entities');
$session->pageTextContains('taxonomy term entities');
$session->pageTextContains('user entities');
$session->pageTextContains('comments');
$session->pageTextContains('content item revisions');
$session->pageTextContains('content items');
$session->pageTextContains('There is translated content of these types:');
}
/**
* Checks that migrations have been performed successfully.
*
* @param array $expected_counts
* The expected counts of each entity type.
* @param int $version
* The Drupal version.
*/
protected function assertMigrationResults(array $expected_counts, $version) {
// Have to reset all the statics after migration to ensure entities are
// loadable.
$this->resetAll();
foreach (array_keys(\Drupal::entityTypeManager()->getDefinitions()) as $entity_type) {
$real_count = (int) \Drupal::entityQuery($entity_type)->count()->execute();
$expected_count = isset($expected_counts[$entity_type]) ? $expected_counts[$entity_type] : 0;
$this->assertSame($expected_count, $real_count, "Found $real_count $entity_type entities, expected $expected_count.");
}
$plugin_manager = \Drupal::service('plugin.manager.migration');
/** @var \Drupal\migrate\Plugin\Migration[] $all_migrations */
$all_migrations = $plugin_manager->createInstancesByTag('Drupal ' . $version);
foreach ($all_migrations as $migration) {
$id_map = $migration->getIdMap();
foreach ($id_map as $source_id => $map) {
// Convert $source_id into a keyless array so that
// \Drupal\migrate\Plugin\migrate\id_map\Sql::getSourceHash() works as
// expected.
$source_id_values = array_values(unserialize($source_id));
$row = $id_map->getRowBySource($source_id_values);
$destination = serialize($id_map->currentDestination());
$message = "Migration of $source_id to $destination as part of the {$migration->id()} migration. The source row status is " . $row['source_row_status'];
// A completed migration should have maps with
// MigrateIdMapInterface::STATUS_IGNORED or
// MigrateIdMapInterface::STATUS_IMPORTED.
$this->assertNotSame(MigrateIdMapInterface::STATUS_FAILED, $row['source_row_status'], $message);
$this->assertNotSame(MigrateIdMapInterface::STATUS_NEEDS_UPDATE, $row['source_row_status'], $message);