Issue #2560795 by chx, alexpott, mikeryan, phenaproxima, penyaskito, willwh,...

Issue #2560795 by chx, alexpott, mikeryan, phenaproxima, penyaskito, willwh, generalredneck, benjy: Source plugins have a hidden dependency on migrate_drupal

core/modules/block_content/migration_templates/block_content_body_field.yml

@@ -30,3 +30,6 @@ destination:
 migration_dependencies:
   required:
     - block_content_type
+provider:
+  - block_content
+  - migrate_drupal

core/modules/block_content/migration_templates/block_content_type.yml

@@ -17,3 +17,6 @@ process:
   label: label
 destination:
   plugin: entity:block_content_type
+provider:
+  - block_content
+  - migrate_drupal

core/modules/migrate/migrate.api.php

@@ -40,7 +40,8 @@
  * 'core/modules/action/migration_templates'. The plugin class is
  * \Drupal\migrate\Plugin\Migration, with interface
  * \Drupal\migrate\Plugin\MigrationInterface. Migration plugins are managed by
- * the \Drupal\migrate\Plugin\MigrationPluginManager class.
+ * the \Drupal\migrate\Plugin\MigrationPluginManager class. Migration plugins
+ * are only available if the providers of their source plugins are installed.
  *
  * @section sec_source Source plugins
  * Migration source plugins implement
@@ -49,7 +50,8 @@
  * with \Drupal\migrate\Annotation\MigrateSource annotation, and must be in
  * namespace subdirectory Plugin\migrate\source under the namespace of the
  * module that defines them. Migration source plugins are managed by the
- * \Drupal\migrate\Plugin\MigratePluginManager class.
+ * \Drupal\migrate\Plugin\MigratePluginManager class. Source plugin providers
+ * are determined by their and their parents namespaces.
  *
  * @section sec_process Process plugins
  * Migration process plugins implement

core/modules/migrate/migrate.services.yml

@@ -6,8 +6,8 @@ services:
     factory: cache_factory:get
     arguments: [migrate]
   plugin.manager.migrate.source:
-    class: Drupal\migrate\Plugin\MigratePluginManager
-    arguments: [source, '@container.namespaces', '@cache.discovery', '@module_handler', 'Drupal\migrate\Annotation\MigrateSource']
+    class: Drupal\migrate\Plugin\MigrateSourcePluginManager
+    arguments: [source, '@container.namespaces', '@cache.discovery', '@module_handler', '@class_loader']
   plugin.manager.migrate.process:
     class: Drupal\migrate\Plugin\MigratePluginManager
     arguments: [process, '@container.namespaces', '@cache.discovery', '@module_handler', 'Drupal\migrate\Annotation\MigrateProcessPlugin']

core/modules/migrate/src/Annotation/MigrateSource.php

@@ -24,7 +24,7 @@
  *
  * @Annotation
  */
-class MigrateSource extends Plugin {
+class MigrateSource extends Plugin implements MultipleProviderAnnotationInterface {
 
   /**
    * A unique identifier for the process plugin.
@@ -66,4 +66,34 @@ class MigrateSource extends Plugin {
    */
   public $minimum_version;
 
+  /**
+   * {@inheritdoc}
+   */
+  public function getProvider() {
+    if (isset($this->definition['provider'])) {
+      return is_array($this->definition['provider']) ? reset($this->definition['provider']) : $this->definition['provider'];
+    }
+    return FALSE;
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public function getProviders() {
+    if (isset($this->definition['provider'])) {
+      // Ensure that we return an array even if
+      // \Drupal\Component\Annotation\AnnotationInterface::setProvider() has
+      // been called.
+      return (array) $this->definition['provider'];
+    }
+    return [];
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public function setProviders(array $providers) {
+    $this->definition['provider'] = $providers;
+  }
+
 }

core/modules/migrate/src/Annotation/MultipleProviderAnnotationInterface.php

+<?php
+
+namespace Drupal\migrate\Annotation;
+
+use Drupal\Component\Annotation\AnnotationInterface;
+
+/**
+ * Defines a common interface for classed annotations with multiple providers.
+ *
+ * @todo This is a temporary solution to the fact that migration source plugins
+ *   have more than one provider. This functionality will be moved to core in
+ *   https://www.drupal.org/node/2786355.
+ */
+interface MultipleProviderAnnotationInterface extends AnnotationInterface {
+
+  /**
+   * Gets the name of the provider of the annotated class.
+   *
+   * @return string
+   *   The provider of the annotation. If there are multiple providers the first
+   *   is returned.
+   */
+  public function getProvider();
+
+  /**
+   * Gets the provider names of the annotated class.
+   *
+   * @return string[]
+   *   The providers of the annotation.
+   */
+  public function getProviders();
+
+  /**
+   * Sets the provider names of the annotated class.
+   *
+   * @param string[] $providers
+   *   The providers of the annotation.
+   */
+  public function setProviders(array $providers);
+
+}

core/modules/migrate/src/Plugin/Discovery/AnnotatedClassDiscoveryAutomatedProviders.php

+<?php
+
+namespace Drupal\migrate\Plugin\Discovery;
+
+use Doctrine\Common\Annotations\AnnotationRegistry;
+use Doctrine\Common\Reflection\StaticReflectionParser as BaseStaticReflectionParser;
+use Drupal\Component\Annotation\AnnotationInterface;
+use Drupal\Component\Annotation\Reflection\MockFileFinder;
+use Drupal\Core\Plugin\Discovery\AnnotatedClassDiscovery;
+use Drupal\migrate\Annotation\MultipleProviderAnnotationInterface;
+
+/**
+ * Determines providers based on a class's and its parent's namespaces.
+ *
+ * @internal
+ *   This is a temporary solution to the fact that migration source plugins have
+ *   more than one provider. This functionality will be moved to core in
+ *   https://www.drupal.org/node/2786355.
+ */
+class AnnotatedClassDiscoveryAutomatedProviders extends AnnotatedClassDiscovery {
+
+  /**
+   * Any class loader with a findFile() method.
+   *
+   * @var \Composer\Autoload\ClassLoader
+   */
+  protected $finder;
+
+  /**
+   * Constructs an AnnotatedClassDiscoveryAutomatedProviders object.
+   *
+   * @param string $subdir
+   *   Either the plugin's subdirectory, for example 'Plugin/views/filter', or
+   *   empty string if plugins are located at the top level of the namespace.
+   * @param \Traversable $root_namespaces
+   *   An object that implements \Traversable which contains the root paths
+   *   keyed by the corresponding namespace to look for plugin implementations.
+   *   If $subdir is not an empty string, it will be appended to each namespace.
+   * @param string $plugin_definition_annotation_name
+   *   The name of the annotation that contains the plugin definition.
+   *   Defaults to 'Drupal\Component\Annotation\Plugin'.
+   * @param string[] $annotation_namespaces
+   *   Additional namespaces to scan for annotation definitions.
+   * @param $class_loader
+   *   The class loader already knows where to find the classes so it is reused
+   *   as the class finder.
+   */
+  public function __construct($subdir, \Traversable $root_namespaces, $plugin_definition_annotation_name = 'Drupal\Component\Annotation\Plugin', array $annotation_namespaces = [], $class_loader) {
+    parent::__construct($subdir, $root_namespaces, $plugin_definition_annotation_name, $annotation_namespaces);
+
+    if (!method_exists($class_loader, 'findFile')) {
+      throw new \LogicException(sprintf('Class loader (%s) must implement findFile() method', get_class($class_loader)));
+    }
+    $this->finder = $class_loader;
+  }
+
+
+  /**
+   * {@inheritdoc}
+   */
+  protected function prepareAnnotationDefinition(AnnotationInterface $annotation, $class, BaseStaticReflectionParser $parser = NULL) {
+    if (!($annotation instanceof MultipleProviderAnnotationInterface)) {
+      throw new \LogicException('AnnotatedClassDiscoveryAutomatedProviders annotations must implement \Drupal\migrate\Annotation\MultipleProviderAnnotationInterface');
+    }
+    $annotation->setClass($class);
+    $providers = $annotation->getProviders();
+    // Loop through all the parent classes and add their providers (which we
+    // infer by parsing their namespaces) to the $providers array.
+    do {
+      $providers[] = $this->getProviderFromNamespace($parser->getNamespaceName());
+    } while ($parser = StaticReflectionParser::getParentParser($parser, $this->finder));
+    $providers = array_unique(array_filter($providers, function ($provider) {
+      return $provider && $provider !== 'component';
+    }));
+    $annotation->setProviders($providers);
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public function getDefinitions() {
+    $definitions = array();
+
+    $reader = $this->getAnnotationReader();
+
+    // Clear the annotation loaders of any previous annotation classes.
+    AnnotationRegistry::reset();
+    // Register the namespaces of classes that can be used for annotations.
+    AnnotationRegistry::registerLoader('class_exists');
+
+    // Search for classes within all PSR-0 namespace locations.
+    foreach ($this->getPluginNamespaces() as $namespace => $dirs) {
+      foreach ($dirs as $dir) {
+        if (file_exists($dir)) {
+          $iterator = new \RecursiveIteratorIterator(
+            new \RecursiveDirectoryIterator($dir, \RecursiveDirectoryIterator::SKIP_DOTS)
+          );
+          foreach ($iterator as $fileinfo) {
+            if ($fileinfo->getExtension() == 'php') {
+              if ($cached = $this->fileCache->get($fileinfo->getPathName())) {
+                if (isset($cached['id'])) {
+                  // Explicitly unserialize this to create a new object instance.
+                  $definitions[$cached['id']] = unserialize($cached['content']);
+                }
+                continue;
+              }
+
+              $sub_path = $iterator->getSubIterator()->getSubPath();
+              $sub_path = $sub_path ? str_replace(DIRECTORY_SEPARATOR, '\\', $sub_path) . '\\' : '';
+              $class = $namespace . '\\' . $sub_path . $fileinfo->getBasename('.php');
+
+              // The filename is already known, so there is no need to find the
+              // file. However, StaticReflectionParser needs a finder, so use a
+              // mock version.
+              $finder = MockFileFinder::create($fileinfo->getPathName());
+              $parser = new BaseStaticReflectionParser($class, $finder, FALSE);
+
+              /** @var $annotation \Drupal\Component\Annotation\AnnotationInterface */
+              if ($annotation = $reader->getClassAnnotation($parser->getReflectionClass(), $this->pluginDefinitionAnnotationName)) {
+                $this->prepareAnnotationDefinition($annotation, $class, $parser);
+
+                $id = $annotation->getId();
+                $content = $annotation->get();
+                $definitions[$id] = $content;
+                // Explicitly serialize this to create a new object instance.
+                $this->fileCache->set($fileinfo->getPathName(), ['id' => $id, 'content' => serialize($content)]);
+              }
+              else {
+                // Store a NULL object, so the file is not reparsed again.
+                $this->fileCache->set($fileinfo->getPathName(), [NULL]);
+              }
+            }
+          }
+        }
+      }
+    }
+
+    // Don't let annotation loaders pile up.
+    AnnotationRegistry::reset();
+
+    return $definitions;
+  }
+
+}

core/modules/migrate/src/Plugin/Discovery/ProviderFilterDecorator.php

+<?php
+
+namespace Drupal\migrate\Plugin\Discovery;
+
+use Drupal\Component\Plugin\Discovery\DiscoveryInterface;
+use Drupal\Component\Plugin\Discovery\DiscoveryTrait;
+
+/**
+ * Remove plugin definitions with non-existing providers.
+ *
+ * @internal
+ *   This is a temporary solution to the fact that migration source plugins have
+ *   more than one provider. This functionality will be moved to core in
+ *   https://www.drupal.org/node/2786355.
+ */
+class ProviderFilterDecorator implements DiscoveryInterface {
+
+  use DiscoveryTrait;
+
+  /**
+   * The Discovery object being decorated.
+   *
+   * @var \Drupal\Component\Plugin\Discovery\DiscoveryInterface
+   */
+  protected $decorated;
+
+  /**
+   * A callable for testing if a provider exists.
+   *
+   * @var callable
+   */
+  protected $providerExists;
+
+  /**
+   * Constructs a InheritProviderDecorator object.
+   *
+   * @param \Drupal\Component\Plugin\Discovery\DiscoveryInterface $decorated
+   *   The object implementing DiscoveryInterface that is being decorated.
+   * @param callable $provider_exists
+   *   A callable, gets passed a provider name, should return TRUE if the
+   *   provider exists and FALSE if not.
+   */
+  public function __construct(DiscoveryInterface $decorated, callable $provider_exists) {
+    $this->decorated = $decorated;
+    $this->providerExists = $provider_exists;
+  }
+
+  /**
+   * Removes plugin definitions with non-existing providers.
+   *
+   * @param mixed[] $definitions
+   *   An array of plugin definitions (empty array if no definitions were
+   *   found). Keys are plugin IDs.
+   * @param callable $provider_exists
+   *   A callable, gets passed a provider name, should return TRUE if the
+   *   provider exists and FALSE if not.
+   *
+   * @return array|\mixed[] $definitions
+   *   An array of plugin definitions. If a definition is an array and has a
+   *   provider key that provider is guaranteed to exist.
+   */
+  public static function filterDefinitions(array $definitions, callable $provider_exists) {
+    // Besides what the caller accepts, we also accept core or component.
+    $provider_exists = function ($provider) use ($provider_exists) {
+      return in_array($provider, ['core', 'component']) || $provider_exists($provider);
+    };
+    return array_filter($definitions, function ($definition) use ($provider_exists) {
+      // Plugin definitions can be objects (for example, Typed Data) those will
+      // become empty array here and cause no problems.
+      $definition = (array) $definition + ['provider' => []];
+      // There can be one or many providers, handle them as multiple always.
+      $providers = (array) $definition['provider'];
+      return count($providers) == count(array_filter($providers, $provider_exists));
+    });
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public function getDefinitions() {
+    return static::filterDefinitions($this->decorated->getDefinitions(), $this->providerExists);
+  }
+
+  /**
+   * Passes through all unknown calls onto the decorated object.
+   *
+   * @param string $method
+   *   The method to call on the decorated object.
+   * @param array $args
+   *   Call arguments.
+   *
+   * @return mixed
+   *   The return value from the method on the decorated object.
+   */
+  public function __call($method, array $args) {
+    return call_user_func_array([$this->decorated, $method], $args);
+  }
+
+}

core/modules/migrate/src/Plugin/Discovery/StaticReflectionParser.php

+<?php
+
+namespace Drupal\migrate\Plugin\Discovery;
+
+use Doctrine\Common\Reflection\StaticReflectionParser as BaseStaticReflectionParser;
+
+/**
+ * Allows getting the reflection parser for the parent class.
+ *
+ * @internal
+ *   This is a temporary solution to the fact that migration source plugins have
+ *   more than one provider. This functionality will be moved to core in
+ *   https://www.drupal.org/node/2786355.
+ */
+class StaticReflectionParser extends BaseStaticReflectionParser {
+
+  /**
+   * If the current class extends another, get the parser for the latter.
+   *
+   * @param \Doctrine\Common\Reflection\StaticReflectionParser $parser
+   *   The current static parser.
+   * @param $finder
+   *   The class finder. Must implement
+   *   \Doctrine\Common\Reflection\ClassFinderInterface, but can do so
+   *   implicitly (i.e., implements the interface's methods but not the actual
+   *   interface).
+   *
+   * @return static|null
+   *   The static parser for the parent if there's a parent class or NULL.
+   */
+  public static function getParentParser(BaseStaticReflectionParser $parser, $finder) {
+    // Ensure the class has been parsed before accessing the parentClassName
+    // property.
+    $parser->parse();
+    if ($parser->parentClassName) {
+      return new static($parser->parentClassName, $finder, $parser->classAnnotationOptimize);
+    }
+  }
+
+}

core/modules/migrate/src/Plugin/MigrateSourcePluginManager.php

+<?php
+
+namespace Drupal\migrate\Plugin;
+
+use Drupal\Core\Cache\CacheBackendInterface;
+use Drupal\Core\Extension\ModuleHandlerInterface;
+use Drupal\migrate\Plugin\Discovery\AnnotatedClassDiscoveryAutomatedProviders;
+use Drupal\Core\Plugin\Discovery\ContainerDerivativeDiscoveryDecorator;
+use Drupal\migrate\Plugin\Discovery\ProviderFilterDecorator;
+
+/**
+ * Plugin manager for migrate source plugins.
+ *
+ * @see \Drupal\migrate\Plugin\MigrateSourceInterface
+ * @see \Drupal\migrate\Plugin\source\SourcePluginBase
+ * @see \Drupal\migrate\Annotation\MigrateSource
+ * @see plugin_api
+ *
+ * @ingroup migration
+ */
+class MigrateSourcePluginManager extends MigratePluginManager {
+
+  /**
+   * The class loader.
+   *
+   * @var object
+   */
+  protected $classLoader;
+
+  /**
+   * MigrateSourcePluginManager constructor.
+   *
+   * @param string $type
+   *   The type of the plugin: row, source, process, destination, entity_field,
+   *   id_map.
+   * @param \Traversable $namespaces
+   *   An object that implements \Traversable which contains the root paths
+   *   keyed by the corresponding namespace to look for plugin implementations.
+   * @param \Drupal\Core\Cache\CacheBackendInterface $cache_backend
+   *   Cache backend instance to use.
+   * @param \Drupal\Core\Extension\ModuleHandlerInterface $module_handler
+   *   The module handler to invoke the alter hook with.
+   * @param object $class_loader
+   *   The class loader.
+   */
+  public function __construct($type, \Traversable $namespaces, CacheBackendInterface $cache_backend, ModuleHandlerInterface $module_handler, $class_loader) {
+    parent::__construct($type, $namespaces, $cache_backend, $module_handler, 'Drupal\migrate\Annotation\MigrateSource');
+    $this->classLoader = $class_loader;
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  protected function getDiscovery() {
+    if (!$this->discovery) {
+      $discovery = new AnnotatedClassDiscoveryAutomatedProviders($this->subdir, $this->namespaces, $this->pluginDefinitionAnnotationName, $this->additionalAnnotationNamespaces, $this->classLoader);
+      $this->discovery = new ContainerDerivativeDiscoveryDecorator($discovery);
+    }
+    return $this->discovery;
+  }
+
+  /**
+   * Finds plugin definitions.
+   *
+   * @return array
+   *   List of definitions to store in cache.
+   *
+   * @todo This is a temporary solution to the fact that migration source
+   *   plugins have more than one provider. This functionality will be moved to
+   *   core in https://www.drupal.org/node/2786355.
+   */
+  protected function findDefinitions() {
+    $definitions = $this->getDiscovery()->getDefinitions();
+    foreach ($definitions as $plugin_id => &$definition) {
+      $this->processDefinition($definition, $plugin_id);
+    }
+    $this->alterDefinitions($definitions);
+    return ProviderFilterDecorator::filterDefinitions($definitions, function ($provider) {
+      return $this->providerExists($provider);
+    });
+  }
+
+}

core/modules/migrate/src/Plugin/MigrationPluginManager.php

@@ -9,6 +9,7 @@
 use Drupal\Core\Language\LanguageManagerInterface;
 use Drupal\Core\Plugin\DefaultPluginManager;
 use Drupal\Core\Plugin\Discovery\ContainerDerivativeDiscoveryDecorator;
+use Drupal\migrate\Plugin\Discovery\ProviderFilterDecorator;
 use Drupal\Core\Plugin\Discovery\YamlDirectoryDiscovery;
 use Drupal\Core\Plugin\Factory\ContainerFactory;
 use Drupal\migrate\MigrateBuildDependencyInterface;
@@ -68,7 +69,15 @@ protected function getDiscovery() {
       }, $this->moduleHandler->getModuleDirectories());
 
       $yaml_discovery = new YamlDirectoryDiscovery($directories, 'migrate');
-      $this->discovery = new ContainerDerivativeDiscoveryDecorator($yaml_discovery);
+      // This gets rid of migrations which try to use a non-existent source
+      // plugin. The common case for this is if the source plugin has, or
+      // specifies, a non-existent provider.
+      $only_with_source_discovery  = new NoSourcePluginDecorator($yaml_discovery);
+      // This gets rid of migrations with explicit providers set if one of the
+      // providers do not exist before we try to use a potentially non-existing
+      // deriver. This is a rare case.
+      $filtered_discovery = new ProviderFilterDecorator($only_with_source_discovery, [$this->moduleHandler, 'moduleExists']);
+      $this->discovery = new ContainerDerivativeDiscoveryDecorator($filtered_discovery);
     }
     return $this->discovery;
   }
@@ -228,4 +237,25 @@ public function createStubMigration(array $definition) {
     return Migration::create(\Drupal::getContainer(), [], $id, $definition);
   }
 
+  /**
+   * Finds plugin definitions.
+   *
+   * @return array
+   *   List of definitions to store in cache.
+   *
+   * @todo This is a temporary solution to the fact that migration source
+   *   plugins have more than one provider. This functionality will be moved to
+   *   core in https://www.drupal.org/node/2786355.
+   */
+  protected function findDefinitions() {
+    $definitions = $this->getDiscovery()->getDefinitions();
+    foreach ($definitions as $plugin_id => &$definition) {
+      $this->processDefinition($definition, $plugin_id);
+    }
+    $this->alterDefinitions($definitions);
+    return ProviderFilterDecorator::filterDefinitions($definitions, function ($provider) {
+      return $this->providerExists($provider);
+    });
+  }
+
 }

core/modules/migrate/src/Plugin/NoSourcePluginDecorator.php

+<?php
+
+namespace Drupal\migrate\Plugin;
+
+use Drupal\Component\Plugin\Discovery\DiscoveryInterface;
+use Drupal\Component\Plugin\Discovery\DiscoveryTrait;
+
+/**
+ * Remove definitions which refer to a non-existing source plugin.
+ */
+class NoSourcePluginDecorator implements DiscoveryInterface {
+
+  use DiscoveryTrait;
+
+  /**
+   * The Discovery object being decorated.
+   *
+   * @var \Drupal\Component\Plugin\Discovery\DiscoveryInterface
+   */
+  protected $decorated;
+
+  /**
+   * Constructs a NoSourcePluginDecorator object.
+   *
+   * @param \Drupal\Component\Plugin\Discovery\DiscoveryInterface $decorated
+   *   The object implementing DiscoveryInterface that is being decorated.
+   */
+  public function __construct(DiscoveryInterface $decorated) {
+    $this->decorated = $decorated;
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public function getDefinitions() {
+    /** @var \Drupal\Component\Plugin\PluginManagerInterface $source_plugin_manager */
+    $source_plugin_manager = \Drupal::service('plugin.manager.migrate.source');