Commit 4bd0cdc0 authored by catch's avatar catch

Issue #2560795 by chx, alexpott, mikeryan, phenaproxima, penyaskito, willwh,...

Issue #2560795 by chx, alexpott, mikeryan, phenaproxima, penyaskito, willwh, generalredneck, benjy: Source plugins have a hidden dependency on migrate_drupal
parent 07c10807
......@@ -30,3 +30,6 @@ destination:
migration_dependencies:
required:
- block_content_type
provider:
- block_content
- migrate_drupal
......@@ -17,3 +17,6 @@ process:
label: label
destination:
plugin: entity:block_content_type
provider:
- block_content
- migrate_drupal
......@@ -40,7 +40,8 @@
* 'core/modules/action/migration_templates'. The plugin class is
* \Drupal\migrate\Plugin\Migration, with interface
* \Drupal\migrate\Plugin\MigrationInterface. Migration plugins are managed by
* the \Drupal\migrate\Plugin\MigrationPluginManager class.
* the \Drupal\migrate\Plugin\MigrationPluginManager class. Migration plugins
* are only available if the providers of their source plugins are installed.
*
* @section sec_source Source plugins
* Migration source plugins implement
......@@ -49,7 +50,8 @@
* with \Drupal\migrate\Annotation\MigrateSource annotation, and must be in
* namespace subdirectory Plugin\migrate\source under the namespace of the
* module that defines them. Migration source plugins are managed by the
* \Drupal\migrate\Plugin\MigratePluginManager class.
* \Drupal\migrate\Plugin\MigratePluginManager class. Source plugin providers
* are determined by their and their parents namespaces.
*
* @section sec_process Process plugins
* Migration process plugins implement
......
......@@ -6,8 +6,8 @@ services:
factory: cache_factory:get
arguments: [migrate]
plugin.manager.migrate.source:
class: Drupal\migrate\Plugin\MigratePluginManager
arguments: [source, '@container.namespaces', '@cache.discovery', '@module_handler', 'Drupal\migrate\Annotation\MigrateSource']
class: Drupal\migrate\Plugin\MigrateSourcePluginManager
arguments: [source, '@container.namespaces', '@cache.discovery', '@module_handler', '@class_loader']
plugin.manager.migrate.process:
class: Drupal\migrate\Plugin\MigratePluginManager
arguments: [process, '@container.namespaces', '@cache.discovery', '@module_handler', 'Drupal\migrate\Annotation\MigrateProcessPlugin']
......
......@@ -24,7 +24,7 @@
*
* @Annotation
*/
class MigrateSource extends Plugin {
class MigrateSource extends Plugin implements MultipleProviderAnnotationInterface {
/**
* A unique identifier for the process plugin.
......@@ -66,4 +66,34 @@ class MigrateSource extends Plugin {
*/
public $minimum_version;
/**
* {@inheritdoc}
*/
public function getProvider() {
if (isset($this->definition['provider'])) {
return is_array($this->definition['provider']) ? reset($this->definition['provider']) : $this->definition['provider'];
}
return FALSE;
}
/**
* {@inheritdoc}
*/
public function getProviders() {
if (isset($this->definition['provider'])) {
// Ensure that we return an array even if
// \Drupal\Component\Annotation\AnnotationInterface::setProvider() has
// been called.
return (array) $this->definition['provider'];
}
return [];
}
/**
* {@inheritdoc}
*/
public function setProviders(array $providers) {
$this->definition['provider'] = $providers;
}
}
<?php
namespace Drupal\migrate\Annotation;
use Drupal\Component\Annotation\AnnotationInterface;
/**
* Defines a common interface for classed annotations with multiple providers.
*
* @todo This is a temporary solution to the fact that migration source plugins
* have more than one provider. This functionality will be moved to core in
* https://www.drupal.org/node/2786355.
*/
interface MultipleProviderAnnotationInterface extends AnnotationInterface {
/**
* Gets the name of the provider of the annotated class.
*
* @return string
* The provider of the annotation. If there are multiple providers the first
* is returned.
*/
public function getProvider();
/**
* Gets the provider names of the annotated class.
*
* @return string[]
* The providers of the annotation.
*/
public function getProviders();
/**
* Sets the provider names of the annotated class.
*
* @param string[] $providers
* The providers of the annotation.
*/
public function setProviders(array $providers);
}
<?php
namespace Drupal\migrate\Plugin\Discovery;
use Doctrine\Common\Annotations\AnnotationRegistry;
use Doctrine\Common\Reflection\StaticReflectionParser as BaseStaticReflectionParser;
use Drupal\Component\Annotation\AnnotationInterface;
use Drupal\Component\Annotation\Reflection\MockFileFinder;
use Drupal\Core\Plugin\Discovery\AnnotatedClassDiscovery;
use Drupal\migrate\Annotation\MultipleProviderAnnotationInterface;
/**
* Determines providers based on a class's and its parent's namespaces.
*
* @internal
* This is a temporary solution to the fact that migration source plugins have
* more than one provider. This functionality will be moved to core in
* https://www.drupal.org/node/2786355.
*/
class AnnotatedClassDiscoveryAutomatedProviders extends AnnotatedClassDiscovery {
/**
* Any class loader with a findFile() method.
*
* @var \Composer\Autoload\ClassLoader
*/
protected $finder;
/**
* Constructs an AnnotatedClassDiscoveryAutomatedProviders object.
*
* @param string $subdir
* Either the plugin's subdirectory, for example 'Plugin/views/filter', or
* empty string if plugins are located at the top level of the namespace.
* @param \Traversable $root_namespaces
* An object that implements \Traversable which contains the root paths
* keyed by the corresponding namespace to look for plugin implementations.
* If $subdir is not an empty string, it will be appended to each namespace.
* @param string $plugin_definition_annotation_name
* The name of the annotation that contains the plugin definition.
* Defaults to 'Drupal\Component\Annotation\Plugin'.
* @param string[] $annotation_namespaces
* Additional namespaces to scan for annotation definitions.
* @param $class_loader
* The class loader already knows where to find the classes so it is reused
* as the class finder.
*/
public function __construct($subdir, \Traversable $root_namespaces, $plugin_definition_annotation_name = 'Drupal\Component\Annotation\Plugin', array $annotation_namespaces = [], $class_loader) {
parent::__construct($subdir, $root_namespaces, $plugin_definition_annotation_name, $annotation_namespaces);
if (!method_exists($class_loader, 'findFile')) {
throw new \LogicException(sprintf('Class loader (%s) must implement findFile() method', get_class($class_loader)));
}
$this->finder = $class_loader;
}
/**
* {@inheritdoc}
*/
protected function prepareAnnotationDefinition(AnnotationInterface $annotation, $class, BaseStaticReflectionParser $parser = NULL) {
if (!($annotation instanceof MultipleProviderAnnotationInterface)) {
throw new \LogicException('AnnotatedClassDiscoveryAutomatedProviders annotations must implement \Drupal\migrate\Annotation\MultipleProviderAnnotationInterface');
}
$annotation->setClass($class);
$providers = $annotation->getProviders();
// Loop through all the parent classes and add their providers (which we
// infer by parsing their namespaces) to the $providers array.
do {
$providers[] = $this->getProviderFromNamespace($parser->getNamespaceName());
} while ($parser = StaticReflectionParser::getParentParser($parser, $this->finder));
$providers = array_unique(array_filter($providers, function ($provider) {
return $provider && $provider !== 'component';
}));
$annotation->setProviders($providers);
}
/**
* {@inheritdoc}
*/
public function getDefinitions() {
$definitions = array();
$reader = $this->getAnnotationReader();
// Clear the annotation loaders of any previous annotation classes.
AnnotationRegistry::reset();
// Register the namespaces of classes that can be used for annotations.
AnnotationRegistry::registerLoader('class_exists');
// Search for classes within all PSR-0 namespace locations.
foreach ($this->getPluginNamespaces() as $namespace => $dirs) {
foreach ($dirs as $dir) {
if (file_exists($dir)) {
$iterator = new \RecursiveIteratorIterator(
new \RecursiveDirectoryIterator($dir, \RecursiveDirectoryIterator::SKIP_DOTS)
);
foreach ($iterator as $fileinfo) {
if ($fileinfo->getExtension() == 'php') {
if ($cached = $this->fileCache->get($fileinfo->getPathName())) {
if (isset($cached['id'])) {
// Explicitly unserialize this to create a new object instance.
$definitions[$cached['id']] = unserialize($cached['content']);
}
continue;
}
$sub_path = $iterator->getSubIterator()->getSubPath();
$sub_path = $sub_path ? str_replace(DIRECTORY_SEPARATOR, '\\', $sub_path) . '\\' : '';
$class = $namespace . '\\' . $sub_path . $fileinfo->getBasename('.php');
// The filename is already known, so there is no need to find the
// file. However, StaticReflectionParser needs a finder, so use a
// mock version.
$finder = MockFileFinder::create($fileinfo->getPathName());
$parser = new BaseStaticReflectionParser($class, $finder, FALSE);
/** @var $annotation \Drupal\Component\Annotation\AnnotationInterface */
if ($annotation = $reader->getClassAnnotation($parser->getReflectionClass(), $this->pluginDefinitionAnnotationName)) {
$this->prepareAnnotationDefinition($annotation, $class, $parser);
$id = $annotation->getId();
$content = $annotation->get();
$definitions[$id] = $content;
// Explicitly serialize this to create a new object instance.
$this->fileCache->set($fileinfo->getPathName(), ['id' => $id, 'content' => serialize($content)]);
}
else {
// Store a NULL object, so the file is not reparsed again.
$this->fileCache->set($fileinfo->getPathName(), [NULL]);
}
}
}
}
}
}
// Don't let annotation loaders pile up.
AnnotationRegistry::reset();
return $definitions;
}
}
<?php
namespace Drupal\migrate\Plugin\Discovery;
use Drupal\Component\Plugin\Discovery\DiscoveryInterface;
use Drupal\Component\Plugin\Discovery\DiscoveryTrait;
/**
* Remove plugin definitions with non-existing providers.
*
* @internal
* This is a temporary solution to the fact that migration source plugins have
* more than one provider. This functionality will be moved to core in
* https://www.drupal.org/node/2786355.
*/
class ProviderFilterDecorator implements DiscoveryInterface {
use DiscoveryTrait;
/**
* The Discovery object being decorated.
*
* @var \Drupal\Component\Plugin\Discovery\DiscoveryInterface
*/
protected $decorated;
/**
* A callable for testing if a provider exists.
*
* @var callable
*/
protected $providerExists;
/**
* Constructs a InheritProviderDecorator object.
*
* @param \Drupal\Component\Plugin\Discovery\DiscoveryInterface $decorated
* The object implementing DiscoveryInterface that is being decorated.
* @param callable $provider_exists
* A callable, gets passed a provider name, should return TRUE if the
* provider exists and FALSE if not.
*/
public function __construct(DiscoveryInterface $decorated, callable $provider_exists) {
$this->decorated = $decorated;
$this->providerExists = $provider_exists;
}
/**
* Removes plugin definitions with non-existing providers.
*
* @param mixed[] $definitions
* An array of plugin definitions (empty array if no definitions were
* found). Keys are plugin IDs.
* @param callable $provider_exists
* A callable, gets passed a provider name, should return TRUE if the
* provider exists and FALSE if not.
*
* @return array|\mixed[] $definitions
* An array of plugin definitions. If a definition is an array and has a
* provider key that provider is guaranteed to exist.
*/
public static function filterDefinitions(array $definitions, callable $provider_exists) {
// Besides what the caller accepts, we also accept core or component.
$provider_exists = function ($provider) use ($provider_exists) {
return in_array($provider, ['core', 'component']) || $provider_exists($provider);
};
return array_filter($definitions, function ($definition) use ($provider_exists) {
// Plugin definitions can be objects (for example, Typed Data) those will
// become empty array here and cause no problems.
$definition = (array) $definition + ['provider' => []];
// There can be one or many providers, handle them as multiple always.
$providers = (array) $definition['provider'];
return count($providers) == count(array_filter($providers, $provider_exists));
});
}
/**
* {@inheritdoc}
*/
public function getDefinitions() {
return static::filterDefinitions($this->decorated->getDefinitions(), $this->providerExists);
}
/**
* Passes through all unknown calls onto the decorated object.
*
* @param string $method
* The method to call on the decorated object.
* @param array $args
* Call arguments.
*
* @return mixed
* The return value from the method on the decorated object.
*/
public function __call($method, array $args) {
return call_user_func_array([$this->decorated, $method], $args);
}
}
<?php
namespace Drupal\migrate\Plugin\Discovery;
use Doctrine\Common\Reflection\StaticReflectionParser as BaseStaticReflectionParser;
/**
* Allows getting the reflection parser for the parent class.
*
* @internal
* This is a temporary solution to the fact that migration source plugins have
* more than one provider. This functionality will be moved to core in
* https://www.drupal.org/node/2786355.
*/
class StaticReflectionParser extends BaseStaticReflectionParser {
/**
* If the current class extends another, get the parser for the latter.
*
* @param \Doctrine\Common\Reflection\StaticReflectionParser $parser
* The current static parser.
* @param $finder
* The class finder. Must implement
* \Doctrine\Common\Reflection\ClassFinderInterface, but can do so
* implicitly (i.e., implements the interface's methods but not the actual
* interface).
*
* @return static|null
* The static parser for the parent if there's a parent class or NULL.
*/
public static function getParentParser(BaseStaticReflectionParser $parser, $finder) {
// Ensure the class has been parsed before accessing the parentClassName
// property.
$parser->parse();
if ($parser->parentClassName) {
return new static($parser->parentClassName, $finder, $parser->classAnnotationOptimize);
}
}
}
<?php
namespace Drupal\migrate\Plugin;
use Drupal\Core\Cache\CacheBackendInterface;
use Drupal\Core\Extension\ModuleHandlerInterface;
use Drupal\migrate\Plugin\Discovery\AnnotatedClassDiscoveryAutomatedProviders;
use Drupal\Core\Plugin\Discovery\ContainerDerivativeDiscoveryDecorator;
use Drupal\migrate\Plugin\Discovery\ProviderFilterDecorator;
/**
* Plugin manager for migrate source plugins.
*
* @see \Drupal\migrate\Plugin\MigrateSourceInterface
* @see \Drupal\migrate\Plugin\source\SourcePluginBase
* @see \Drupal\migrate\Annotation\MigrateSource
* @see plugin_api
*
* @ingroup migration
*/
class MigrateSourcePluginManager extends MigratePluginManager {
/**
* The class loader.
*
* @var object
*/
protected $classLoader;
/**
* MigrateSourcePluginManager constructor.
*
* @param string $type
* The type of the plugin: row, source, process, destination, entity_field,
* id_map.
* @param \Traversable $namespaces
* An object that implements \Traversable which contains the root paths
* keyed by the corresponding namespace to look for plugin implementations.
* @param \Drupal\Core\Cache\CacheBackendInterface $cache_backend
* Cache backend instance to use.
* @param \Drupal\Core\Extension\ModuleHandlerInterface $module_handler
* The module handler to invoke the alter hook with.
* @param object $class_loader
* The class loader.
*/
public function __construct($type, \Traversable $namespaces, CacheBackendInterface $cache_backend, ModuleHandlerInterface $module_handler, $class_loader) {
parent::__construct($type, $namespaces, $cache_backend, $module_handler, 'Drupal\migrate\Annotation\MigrateSource');
$this->classLoader = $class_loader;
}
/**
* {@inheritdoc}
*/
protected function getDiscovery() {
if (!$this->discovery) {
$discovery = new AnnotatedClassDiscoveryAutomatedProviders($this->subdir, $this->namespaces, $this->pluginDefinitionAnnotationName, $this->additionalAnnotationNamespaces, $this->classLoader);
$this->discovery = new ContainerDerivativeDiscoveryDecorator($discovery);
}
return $this->discovery;
}
/**
* Finds plugin definitions.
*
* @return array
* List of definitions to store in cache.
*
* @todo This is a temporary solution to the fact that migration source
* plugins have more than one provider. This functionality will be moved to
* core in https://www.drupal.org/node/2786355.
*/
protected function findDefinitions() {
$definitions = $this->getDiscovery()->getDefinitions();
foreach ($definitions as $plugin_id => &$definition) {
$this->processDefinition($definition, $plugin_id);
}
$this->alterDefinitions($definitions);
return ProviderFilterDecorator::filterDefinitions($definitions, function ($provider) {
return $this->providerExists($provider);
});
}
}
......@@ -9,6 +9,7 @@
use Drupal\Core\Language\LanguageManagerInterface;
use Drupal\Core\Plugin\DefaultPluginManager;
use Drupal\Core\Plugin\Discovery\ContainerDerivativeDiscoveryDecorator;
use Drupal\migrate\Plugin\Discovery\ProviderFilterDecorator;
use Drupal\Core\Plugin\Discovery\YamlDirectoryDiscovery;
use Drupal\Core\Plugin\Factory\ContainerFactory;
use Drupal\migrate\MigrateBuildDependencyInterface;
......@@ -68,7 +69,15 @@ protected function getDiscovery() {
}, $this->moduleHandler->getModuleDirectories());
$yaml_discovery = new YamlDirectoryDiscovery($directories, 'migrate');
$this->discovery = new ContainerDerivativeDiscoveryDecorator($yaml_discovery);
// This gets rid of migrations which try to use a non-existent source
// plugin. The common case for this is if the source plugin has, or
// specifies, a non-existent provider.
$only_with_source_discovery = new NoSourcePluginDecorator($yaml_discovery);
// This gets rid of migrations with explicit providers set if one of the
// providers do not exist before we try to use a potentially non-existing
// deriver. This is a rare case.
$filtered_discovery = new ProviderFilterDecorator($only_with_source_discovery, [$this->moduleHandler, 'moduleExists']);
$this->discovery = new ContainerDerivativeDiscoveryDecorator($filtered_discovery);
}
return $this->discovery;
}
......@@ -228,4 +237,25 @@ public function createStubMigration(array $definition) {
return Migration::create(\Drupal::getContainer(), [], $id, $definition);
}
/**
* Finds plugin definitions.
*
* @return array
* List of definitions to store in cache.
*
* @todo This is a temporary solution to the fact that migration source
* plugins have more than one provider. This functionality will be moved to
* core in https://www.drupal.org/node/2786355.
*/
protected function findDefinitions() {
$definitions = $this->getDiscovery()->getDefinitions();
foreach ($definitions as $plugin_id => &$definition) {
$this->processDefinition($definition, $plugin_id);
}
$this->alterDefinitions($definitions);
return ProviderFilterDecorator::filterDefinitions($definitions, function ($provider) {
return $this->providerExists($provider);
});
}
}
<?php
namespace Drupal\migrate\Plugin;
use Drupal\Component\Plugin\Discovery\DiscoveryInterface;
use Drupal\Component\Plugin\Discovery\DiscoveryTrait;
/**
* Remove definitions which refer to a non-existing source plugin.
*/
class NoSourcePluginDecorator implements DiscoveryInterface {
use DiscoveryTrait;
/**
* The Discovery object being decorated.
*
* @var \Drupal\Component\Plugin\Discovery\DiscoveryInterface
*/
protected $decorated;
/**
* Constructs a NoSourcePluginDecorator object.
*
* @param \Drupal\Component\Plugin\Discovery\DiscoveryInterface $decorated
* The object implementing DiscoveryInterface that is being decorated.
*/
public function __construct(DiscoveryInterface $decorated) {
$this->decorated = $decorated;
}
/**
* {@inheritdoc}
*/
public function getDefinitions() {
/** @var \Drupal\Component\Plugin\PluginManagerInterface $source_plugin_manager */
$source_plugin_manager = \Drupal::service('plugin.manager.migrate.source');
return array_filter($this->decorated->getDefinitions(), function (array $definition) use ($source_plugin_manager) {
return $source_plugin_manager->hasDefinition($definition['source']['plugin']);
});
}
/**
* Passes through all unknown calls onto the decorated object.
*
* @param string $method
* The method to call on the decorated object.
* @param array $args
* Call arguments.
*
* @return mixed
* The return value from the method on the decorated object.
*/
public function __call($method, array $args) {
return call_user_func_array([$this->decorated, $method], $args);
}
}
......@@ -34,8 +34,8 @@ public function testSetInvalidation() {
$this->assertEqual('entity:entity_view_mode', $migration->getDestinationPlugin()->getPluginId());
// Test the source plugin is invalidated.
$migration->set('source', ['plugin' => 'd6_field']);
$this->assertEqual('d6_field', $migration->getSourcePlugin()->getPluginId());
$migration->set('source', ['plugin' => 'embedded_data', 'data_rows' => [], 'ids' => []]);
$this->assertEqual('embedded_data', $migration->getSourcePlugin()->getPluginId());
// Test the destination plugin is invalidated.
$migration->set('destination', ['plugin' => 'null']);
......
<?php
namespace Drupal\Tests\migrate\Kernel\Plugin;
use Drupal\Core\Database\Database;
use Drupal\KernelTests\KernelTestBase;
/**
* Tests the migration plugin manager.
*
* @coversDefaultClass \Drupal\migrate\Plugin\MigratePluginManager
* @group migrate
*/
class MigrationPluginListTest extends KernelTestBase {
/**
* {@inheritdoc}
*/
public static $modules = [
'migrate',
// Test with all modules containing Drupal migrations.
'action',
'aggregator',
'ban',
'block',
'block_content',
'book',
'comment',
'contact',
'dblog',
'field',
'file',
'filter',
'forum',
'image',
'language',
'locale',
'menu_link_content',
'menu_ui',
'node',
'path',
'search',
'shortcut',
'simpletest',
'statistics',
'syslog',
'system',
'taxonomy',
'text',
'tracker',
'update',
'user',
];
/**
* @covers ::getDefinitions
*/
public function testGetDefinitions() {
// Make sure retrieving all the core migration plugins does not throw any
// errors.
$migration_plugins = $this->container->get('plugin.manager.migration')->getDefinitions();
// All the plugins provided by core depend on migrate_drupal.
$this->assertEmpty($migration_plugins);
// Enable a module that provides migrations that do not depend on
// migrate_drupal.
$this->enableModules(['migrate_external_translated_test']);
$migration_plugins = $this->container->get('plugin.manager.migration')->getDefinitions();
// All the plugins provided by migrate_external_translated_test do not
// depend on migrate_drupal.
$this::assertArrayHasKey('external_translated_test_node', $migration_plugins);
$this::assertArrayHasKey('external_translated_test_node_translation', $migration_plugins);
// Disable the test module and the list should be empty again.
$this->disableModules(['migrate_external_translated_test']);
$migration_plugins = $this->container->get('plugin.manager.migration')->getDefinitions();