Commit db464d80 authored by catch's avatar catch

Issue #2676258 by benjy, alexpott: Nuke builder plugins and migration storage

parent 3a7869c3
......@@ -5,12 +5,6 @@ services:
- { name: cache.bin }
factory: cache_factory:get
arguments: [migrate]
migrate.template_storage:
class: Drupal\migrate\MigrateTemplateStorage
arguments: ['@module_handler']
migrate.migration_builder:
class: Drupal\migrate\MigrationBuilder
arguments: ['@plugin.manager.migrate.builder']
plugin.manager.migrate.source:
class: Drupal\migrate\Plugin\MigratePluginManager
arguments: [source, '@container.namespaces', '@cache.discovery', '@module_handler', 'Drupal\migrate\Annotation\MigrateSource']
......@@ -23,9 +17,6 @@ services:
plugin.manager.migrate.id_map:
class: Drupal\migrate\Plugin\MigratePluginManager
arguments: [id_map, '@container.namespaces', '@cache.discovery', '@module_handler']
plugin.manager.migrate.builder:
class: Drupal\migrate\Plugin\MigratePluginManager
arguments: [builder, '@container.namespaces', '@cache.discovery', '@module_handler']
plugin.manager.migration:
class: Drupal\migrate\Plugin\MigrationPluginManager
arguments: ['@module_handler', '@cache.discovery', '@language_manager']
<?php
/**
* @file
* Contains \Drupal\migrate\MigrateTemplateStorage.
*/
namespace Drupal\migrate;
use Drupal\Component\Serialization\Yaml;
use Drupal\Core\Extension\ModuleHandlerInterface;
/**
* Storage to access migration template configuration in enabled extensions.
*/
class MigrateTemplateStorage implements MigrateTemplateStorageInterface {
/**
* Extension sub-directory containing default configuration for migrations.
*/
const MIGRATION_TEMPLATE_DIRECTORY = 'migration_templates';
/**
* Template subdirectory.
*
* @var string
*/
protected $directory;
/**
* @var \Drupal\Core\Extension\ModuleHandlerInterface
*/
protected $moduleHandler;
/**
* {@inheritdoc}
*/
public function __construct(ModuleHandlerInterface $module_handler, $directory = self::MIGRATION_TEMPLATE_DIRECTORY) {
$this->moduleHandler = $module_handler;
$this->directory = $directory;
}
/**
* {@inheritdoc}
*/
public function findTemplatesByTag($tag) {
$templates = $this->getAllTemplates();
$matched_templates = [];
foreach ($templates as $template_name => $template) {
if (!empty($template['migration_tags'])) {
if (in_array($tag, $template['migration_tags'])) {
$matched_templates[$template_name] = $template;
}
}
}
return $matched_templates;
}
/**
* {@inheritdoc}
*/
public function getTemplateByName($name) {
$templates = $this->getAllTemplates();
return isset($templates[$name]) ? $templates[$name] : NULL;
}
/**
* {@inheritdoc}
*/
public function getAllTemplates() {
$templates = [];
foreach ($this->moduleHandler->getModuleDirectories() as $directory) {
$full_directory = $directory . '/' . $this->directory;
if (file_exists($full_directory)) {
$files = scandir($full_directory);
foreach ($files as $file) {
if ($file[0] !== '.' && preg_match('/\.yml$/', $file)) {
$templates[basename($file, '.yml')] = Yaml::decode(file_get_contents("$full_directory/$file"));
}
}
}
}
return $templates;
}
}
<?php
/**
* @file
* Contains \Drupal\migrate\MigrateTemplateStorageInterface.
*/
namespace Drupal\migrate;
/**
* The MigrateTemplateStorageInterface interface.
*/
interface MigrateTemplateStorageInterface {
/**
* Find all migration templates with the specified tag.
*
* @param $tag
* The tag to match.
*
* @return array
* Any templates (parsed YAML config) that matched, keyed by the ID.
*/
public function findTemplatesByTag($tag);
/**
* Retrieve a template given a specific name.
*
* @param string $name
* A migration template name.
*
* @return NULL|array
* A parsed migration template, or NULL if it doesn't exist.
*/
public function getTemplateByName($name);
/**
* Retrieves all migration templates belonging to enabled extensions.
*
* @return array
* Array of parsed templates, keyed by the fully-qualified id.
*/
public function getAllTemplates();
}
<?php
/**
* @file
* Contains \Drupal\migrate\MigrationBuilder.
*/
namespace Drupal\migrate;
use Drupal\migrate\Plugin\MigratePluginManager;
/**
* Builds migration entities from migration templates.
*/
class MigrationBuilder implements MigrationBuilderInterface {
/**
* The builder plugin manager.
*
* @var \Drupal\migrate\Plugin\MigratePluginManager
*/
protected $builderManager;
/**
* Constructs a MigrationBuilder.
*
* @param \Drupal\migrate\Plugin\MigratePluginManager $builder_manager
* The builder plugin manager.
*/
public function __construct(MigratePluginManager $builder_manager) {
$this->builderManager = $builder_manager;
}
/**
* {@inheritdoc}
*/
public function createMigrations(array $templates) {
/** @var \Drupal\migrate\Entity\MigrationInterface[] $migrations */
$migrations = [];
foreach ($templates as $template_id => $template) {
if (isset($template['builder'])) {
$variants = $this->builderManager
->createInstance($template['builder']['plugin'], $template['builder'])
->buildMigrations($template);
}
else {
$variants = array(Migration::create($template));
}
/** @var \Drupal\migrate\Entity\MigrationInterface[] $variants */
foreach ($variants as $variant) {
$variant->set('template', $template_id);
}
$migrations = array_merge($migrations, $variants);
}
return $migrations;
}
}
<?php
/**
* @file
* Contains \Drupal\migrate\MigrationBuilderInterface.
*/
namespace Drupal\migrate;
/**
* The migration builder interface.
*/
interface MigrationBuilderInterface {
/**
* Builds migration entities from templates.
*
* @param array $templates
* The parsed templates (each of which is an array parsed from YAML), keyed
* by ID.
*
* @return \Drupal\migrate\Entity\MigrationInterface[]
* The migration entities derived from the templates.
*/
public function createMigrations(array $templates);
}
<?php
/**
* @file
* Contains \Drupal\migrate\Plugin\MigrateBuilderInterface.
*/
namespace Drupal\migrate\Plugin;
/**
* Defines the builder plugin type.
*
* Builder plugins implement custom logic to generate migration entities from
* migration templates. For example, a migration may need to be customized based
* on data that's present in the source database; such customization is
* implemented by builders.
*/
interface MigrateBuilderInterface {
/**
* Builds migration entities based on a template.
*
* @param array $template
* The parsed template.
*
* @return \Drupal\migrate\Entity\MigrationInterface[]
* The unsaved migrations generated from the template.
*/
public function buildMigrations(array $template);
}
<?php
/**
* @file
* Contains \Drupal\migrate\Plugin\migrate\builder\BuilderBase.
*/
namespace Drupal\migrate\Plugin\migrate\builder;
use Drupal\Core\Plugin\PluginBase;
use Drupal\migrate\Plugin\MigrateBuilderInterface;
/**
* Provides abstract base class for builder plugins.
*/
abstract class BuilderBase extends PluginBase implements MigrateBuilderInterface {
/**
* Returns a fully initialized instance of a source plugin.
*
* @param string $plugin_id
* The plugin ID.
* @param array $configuration
* (optional) Additional configuration for the plugin. Defaults to an empty
* array.
*
* @return \Drupal\migrate\Plugin\MigrateSourceInterface|\Drupal\migrate\Plugin\RequirementsInterface
* The fully initialized source plugin.
*/
protected function getSourcePlugin($plugin_id, array $configuration = []) {
$configuration['plugin'] = $plugin_id;
// By default, SqlBase subclasses will try to join on a map table. But in
// this case we're trying to use the source plugin as a detached iterator
// over the source data, so we don't want to join on (or create) the map
// table.
// @see SqlBase::initializeIterator()
$configuration['ignore_map'] = TRUE;
// Source plugins are tightly coupled to migration entities, so we need
// to create a fake migration in order to properly initialize the plugin.
$values = [
'id' => uniqid(),
'source' => $configuration,
// Since this isn't a real migration, we don't want a real destination --
// the 'null' destination is perfect for this.
'destination' => [
'plugin' => 'null',
],
];
return Migration::create($values)->getSourcePlugin();
}
}
<?php
/**
* @file
* Contains \Drupal\migrate\Tests\TemplateTest.
*/
namespace Drupal\migrate\Tests;
/**
* Tests the migration template functionality.
*
* @group migrate
*/
class TemplateTest extends MigrateTestBase {
/**
* Modules to enable.
*
* @var array
*/
public static $modules = array('template_test');
/**
* Tests different connection types.
*/
public function testTemplates() {
$migration_templates = \Drupal::service('migrate.template_storage')->findTemplatesByTag("Template Test");
$expected_url = [
'id' => 'url_template',
'label' => 'Template test - URL',
'migration_tags' => ['Template Test'],
'source' => ['plugin' => 'empty'],
'process' => ['src' => 'foobar'],
'destination' => ['plugin' => 'url_alias'],
];
$expected_node = [
'id' => 'node_template',
'label' => 'Template test - node',
'migration_tags' => ['Template Test'],
'source' => ['plugin' => 'empty'],
'process' => ['src' => 'barfoo'],
'destination' => ['plugin' => 'entity:node'],
];
$this->assertIdentical($migration_templates['migrate.migration.url_template'], $expected_url);
$this->assertIdentical($migration_templates['migrate.migration.node_template'], $expected_node);
$this->assertFalse(isset($migration_templates['migrate.migration.other_template']));
}
/**
* Tests retrieving a template by name.
*/
public function testGetTemplateByName() {
/** @var \Drupal\migrate\MigrateTemplateStorageInterface $template_storage */
$template_storage = \Drupal::service('migrate.template_storage');
$expected_url = [
'id' => 'url_template',
'label' => 'Template test - URL',
'migration_tags' => ['Template Test'],
'source' => ['plugin' => 'empty'],
'process' => ['src' => 'foobar'],
'destination' => ['plugin' => 'url_alias'],
];
$expected_node = [
'id' => 'node_template',
'label' => 'Template test - node',
'migration_tags' => ['Template Test'],
'source' => ['plugin' => 'empty'],
'process' => ['src' => 'barfoo'],
'destination' => ['plugin' => 'entity:node'],
];
$this->assertIdentical($template_storage->getTemplateByName('migrate.migration.url_template'), $expected_url);
$this->assertIdentical($template_storage->getTemplateByName('migrate.migration.node_template'), $expected_node);
$this->assertNull($template_storage->getTemplateByName('migrate.migration.dne'));
}
}
<?php
/**
* @file
* Contains \Drupal\migrate_drupal\Plugin\migrate\builder\CckBuilder.
*/
namespace Drupal\migrate_drupal\Plugin\migrate\builder;
use Drupal\Core\Plugin\ContainerFactoryPluginInterface;
use Drupal\migrate\Entity\MigrationInterface;
use Drupal\migrate\Plugin\migrate\builder\BuilderBase;
use Drupal\migrate\Plugin\MigratePluginManager;
use Symfony\Component\DependencyInjection\ContainerInterface;
/**
* Base class for builders which leverage cckfield plugins.
*/
abstract class CckBuilder extends BuilderBase implements ContainerFactoryPluginInterface {
/**
* The cckfield plugin manager.
*
* @var \Drupal\migrate\Plugin\MigratePluginManager
*/
protected $cckPluginManager;
/**
* Already-instantiated cckfield plugins, keyed by ID.
*
* @var \Drupal\migrate_drupal\Plugin\MigrateCckFieldInterface[]
*/
protected $cckPluginCache = [];
/**
* Constructs a CckBuilder.
*
* @param array $configuration
* Plugin configuration.
* @param string $plugin_id
* The plugin ID.
* @param mixed $plugin_definition
* The plugin definition.
* @param \Drupal\migrate\Plugin\MigratePluginManager $cck_manager
* The cckfield plugin manager.
*/
public function __construct(array $configuration, $plugin_id, $plugin_definition, MigratePluginManager $cck_manager) {
parent::__construct($configuration, $plugin_id, $plugin_definition);
$this->cckPluginManager = $cck_manager;
}
/**
* {@inheritdoc}
*/
public static function create(ContainerInterface $container, array $configuration, $plugin_id, $plugin_definition) {
return new static(
$configuration,
$plugin_id,
$plugin_definition,
$container->get('plugin.manager.migrate.cckfield')
);
}
/**
* Gets a cckfield plugin instance.
*
* @param string $field_type
* The field type (plugin ID).
* @param \Drupal\migrate\Entity\MigrationInterface|NULL $migration
* The migration, if any.
*
* @return \Drupal\migrate_drupal\Plugin\MigrateCckFieldInterface
* The cckfield plugin instance.
*/
protected function getCckPlugin($field_type, MigrationInterface $migration = NULL) {
if (empty($this->cckPluginCache[$field_type])) {
$this->cckPluginCache[$field_type] = $this->cckPluginManager->createInstance($field_type, [], $migration);
}
return $this->cckPluginCache[$field_type];
}
}
<?php
/**
* @file
* Contains \Drupal\migrate_drupal\Plugin\migrate\builder\d6\CckMigration.
*/
namespace Drupal\migrate_drupal\Plugin\migrate\builder\d6;
use Drupal\migrate\Exception\RequirementsException;
use Drupal\migrate\Plugin\RequirementsInterface;
use Drupal\migrate_drupal\Plugin\migrate\builder\CckBuilder;
/**
* @PluginID("d6_cck_migration")
*/
class CckMigration extends CckBuilder {
/**
* List of cckfield plugin IDs which have already run.
*
* @var string[]
*/
protected $processedFieldTypes = [];
/**
* {@inheritdoc}
*/
public function buildMigrations(array $template) {
$migration = Migration::create($template);
$source_plugin = $migration->getSourcePlugin();
// The source plugin will throw RequirementsException if CCK is not enabled,
// in which case there is nothing else for us to do.
if ($source_plugin instanceof RequirementsInterface) {
try {
$source_plugin->checkRequirements();
}
catch (RequirementsException $e) {
return [$migration];
}
}
// Loop through every field that will be migrated.
foreach ($source_plugin as $field) {
$field_type = $field->getSourceProperty('type');
// Each field type should only be processed once.
if (in_array($field_type, $this->processedFieldTypes)) {
continue;
}
// Only process the current field type if a relevant cckfield plugin
// exists.
elseif ($this->cckPluginManager->hasDefinition($field_type)) {
$this->processedFieldTypes[] = $field_type;
// Allow the cckfield plugin to alter the migration as necessary so that
// it knows how to handle fields of this type.
$this->cckPluginManager
->createInstance($field_type, [], $migration)
->{$this->configuration['cck_plugin_method']}($migration);
}
}
return [$migration];
}
}
......@@ -50,29 +50,4 @@ protected function loadFixture($path) {
Database::setActiveConnection($default_db);
}
/**
* Turn all the migration templates for the specified drupal version into
* real migration entities so we can test them.
*
* @param string $version
* Drupal version as provided in migration_tags - e.g., 'Drupal 6'.
*/
protected function installMigrations($version) {
// @TODO https://www.drupal.org/node/2668436
return;
$migration_templates = \Drupal::service('migrate.template_storage')->findTemplatesByTag($version);
$migrations = \Drupal::service('migrate.migration_builder')->createMigrations($migration_templates);
foreach ($migrations as $migration) {
try {
$migration->save();
}
catch (PluginNotFoundException $e) {
// Migrations requiring modules not enabled will throw an exception.
// Ignoring this exception is equivalent to placing config in the
// optional subdirectory - the migrations we require for the test will
// be successfully saved.
}
}
}
}
......@@ -6,6 +6,7 @@
*/
namespace Drupal\migrate_drupal\Tests\d6;
use Drupal\migrate_drupal\Tests\MigrateDrupalTestBase;
/**
......@@ -33,7 +34,6 @@ abstract class MigrateDrupal6TestBase extends MigrateDrupalTestBase {
protected function setUp() {
parent::setUp();
$this->loadFixture( __DIR__ . '/../../../tests/fixtures/drupal6.php');
$this->installMigrations('Drupal 6');
}
/**
......
......@@ -20,7 +20,6 @@ abstract class MigrateDrupal7TestBase extends MigrateDrupalTestBase {
protected function setUp() {
parent::setUp();
$this->loadFixture(__DIR__ . '/../../../tests/fixtures/drupal7.php');
$this->installMigrations('Drupal 7');
}
}
......@@ -32,8 +32,8 @@ public function testMigrateDependenciesOrder() {
$expected_order = array('d6_filter_format', 'd6_node:page', 'd6_comment');
$this->assertIdentical(array_keys($migrations), $expected_order);
$expected_requirements = array(
// d6_comment depends on d6_node:*, which the storage controller expands
// into every variant of d6_node created by the MigrationBuilder.
// d6_comment depends on d6_node:*, which the deriver expands into every
// variant of d6_node.
'd6_node:article',
'd6_node:company',
'd6_node:employee',
......
<?php
/**
* @file
* Contains \Drupal\node\Plugin\migrate\builder\d6\Node.
*/
namespace Drupal\node\Plugin\migrate\builder\d6;
use Drupal\migrate\Exception\RequirementsException;
use Drupal\migrate_drupal\Plugin\migrate\builder\CckBuilder;
/**
* @PluginID("d6_node")
*/
class Node extends CckBuilder {
/**
* {@inheritdoc}
*/
public function buildMigrations(array $template) {
$migrations = [];
// Read all CCK field instance definitions in the source database.
$fields = array();
$source_plugin = $this->getSourcePlugin('d6_field_instance', $template['source']);
try {
$source_plugin->checkRequirements();
foreach ($source_plugin as $field) {
$info = $field->getSource();
$fields[$info['type_name']][$info['field_name']] = $info;
}
}
catch (RequirementsException $e) {
// Don't do anything; $fields will be empty.
}
foreach ($this->getSourcePlugin('d6_node_type', $template['source']) as $row) {
$node_type = $row->getSourceProperty('type');
$values = $template;
$values['id'] = $template['id'] . '__' . $node_type;
$label = $template['label'];
$values['label'] = $this->t("@label (@type)", ['@label' => $label, '@type' => $node_type]);
$values['source']['node_type'] = $node_type;
// If this migration is based on the d6_node_revision template, it should
// explicitly depend on the corresponding d6_node variant.
if ($template['id'] == 'd6_node_revision') {
$values['migration_dependencies']['required'][] = 'd6_node__' . $node_type;
}
$migration = Migration::create($values);
if (isset($fields[$node_type])) {
foreach ($fields[$node_type] as $field => $info) {
if ($this->cckPluginManager->hasDefinition($info['type'])) {
$this->getCckPlugin($info['type'])
->processCckFieldValues($migration, $field, $info);
}
else {
$migration->setProcessOfProperty($field, $field);
}
}
}
$migrations[] = $migration;
}
return $migrations;
}
}
<?php
/**
* @file
* Contains \Drupal\node\Plugin\migrate\builder\d7\Node.
*/
namespace Drupal\node\Plugin\migrate\builder\d7;