Issue #2549013 by phenaproxima, mikeryan, benjy: Remove load plugins

core/modules/book/src/Tests/Migrate/d6/MigrateBookTest.php

@@ -29,7 +29,15 @@ protected function setUp() {
     $this->installSchema('book', array('book'));
     $this->installSchema('node', array('node_access'));
 
-    $id_mappings = array();
+    // Create a default bogus mapping for all variants of d6_node.
+    $id_mappings = array(
+      'd6_node:*' => array(
+        array(
+          array(0),
+          array(0),
+        ),
+      ),
+    );
     for ($i = 4; $i <= 8; $i++) {
       $entity = entity_create('node', array(
         'type' => 'story',
@@ -39,7 +47,7 @@ protected function setUp() {
       ));
       $entity->enforceIsNew();
       $entity->save();
-      $id_mappings['d6_node'][] = array(array($i), array($i));
+      $id_mappings['d6_node__story'][] = array(array($i), array($i));
     }
     $this->prepareMigrations($id_mappings);
     $this->executeMigration('d6_book');

core/modules/comment/migration_templates/d6_comment.yml

@@ -37,7 +37,7 @@ destination:
   plugin: entity:comment
 migration_dependencies:
   required:
-    - d6_node
+    - d6_node:*
     - d6_comment_type
     - d6_user
     - d6_comment_entity_display

core/modules/comment/src/Tests/Migrate/d6/MigrateCommentTest.php

@@ -51,7 +51,7 @@ protected function setUp() {
     $node->save();
     $id_mappings = array(
       'd6_filter_format' => array(array(array(1), array('filtered_html'))),
-      'd6_node' => array(array(array(1), array(1))),
+      'd6_node:*' => array(array(array(1), array(1))),
       'd6_user' => array(array(array(0), array(0))),
       'd6_comment_type' => array(array(array('comment'), array('comment_no_subject'))),
       'd6_comment_entity_display' => array(array(array('story'), array('node', 'story', 'default', 'comment'))),

core/modules/file/src/Plugin/migrate/cckfield/FileField.php

@@ -44,11 +44,7 @@ public function getFieldFormatterMap() {
   public function processCckFieldValues(MigrationInterface $migration, $field_name, $data) {
     $process = [
       'plugin' => 'd6_cck_file',
-      'source' => [
-        $field_name,
-        $field_name . '_list',
-        $field_name . '_data',
-      ],
+      'source' => $field_name,
     ];
     $migration->mergeProcessOfProperty($field_name, $process);
   }

core/modules/file/src/Plugin/migrate/process/d6/CckFile.php

@@ -8,40 +8,95 @@
 namespace Drupal\file\Plugin\migrate\process\d6;
 
 use Drupal\Core\Plugin\ContainerFactoryPluginInterface;
+use Drupal\migrate\Entity\MigrationInterface;
 use Drupal\migrate\MigrateExecutableInterface;
+use Drupal\migrate\MigrateSkipRowException;
+use Drupal\migrate\Plugin\MigrateProcessInterface;
+use Drupal\migrate\ProcessPluginBase;
 use Drupal\migrate\Row;
-use Drupal\migrate\Plugin\migrate\process\Route;
+use Symfony\Component\DependencyInjection\ContainerInterface;
 
 /**
  * @MigrateProcessPlugin(
  *   id = "d6_cck_file"
  * )
  */
-class CckFile extends Route implements ContainerFactoryPluginInterface {
+class CckFile extends ProcessPluginBase implements ContainerFactoryPluginInterface {
+
+  protected $migrationPlugin;
+
+  /**
+   * Constructs a CckFile plugin instance.
+   *
+   * @param array $configuration
+   *   The plugin configuration.
+   * @param string $plugin_id
+   *   The plugin ID.
+   * @param mixed $plugin_definition
+   *   The plugin definition.
+   * @param \Drupal\migrate\Entity\MigrationInterface $migration
+   *   The current migration.
+   * @param \Drupal\migrate\Plugin\MigrateProcessInterface $migration_plugin
+   *   An instance of the 'migration' process plugin.
+   */
+  public function __construct(array $configuration, $plugin_id, $plugin_definition, MigrationInterface $migration, MigrateProcessInterface $migration_plugin) {
+    parent::__construct($configuration, $plugin_id, $plugin_definition);
+    $this->migration = $migration;
+    $this->migrationPlugin = $migration_plugin;
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public static function create(ContainerInterface $container, array $configuration, $plugin_id, $plugin_definition, MigrationInterface $migration = NULL) {
+    // Configure the migration process plugin to look up migrated IDs from
+    // the d6_file migration.
+    $migration_plugin_configuration = [
+      'source' => ['fid'],
+      'migration' => 'd6_file',
+    ];
+
+    return new static(
+      $configuration,
+      $plugin_id,
+      $plugin_definition,
+      $migration,
+      $container->get('plugin.manager.migrate.process')->createInstance('migration', $migration_plugin_configuration, $migration)
+    );
+  }
 
   /**
    * {@inheritdoc}
    */
   public function transform($value, MigrateExecutableInterface $migrate_executable, Row $row, $destination_property) {
-    list($fid, $list, $data) = $value;
+    $options = unserialize($value['data']);
 
-    // If $fid is still an array at this point, that's because we have a file
-    // attachment as per D6 core. If not, then we have a filefield from contrib.
-    if (is_array($fid)) {
-      $list = $fid['list'];
-      $fid = $fid['fid'];
+    // Try to look up the ID of the migrated file. If one cannot be found, it
+    // means the file referenced by the current field item did not migrate for
+    // some reason -- file migration is notoriously brittle -- and we do NOT
+    // want to send invalid file references into the field system (it causes
+    // fatals), so return an empty item instead.
+    try {
+      $fid = $this->migrationPlugin->transform($value['fid'], $migrate_executable, $row, $destination_property);
     }
-    else {
-      $options = unserialize($data);
+    // If the migration plugin completely fails its lookup process, it will
+    // throw a MigrateSkipRowException. It shouldn't, but that is being dealt
+    // with at https://www.drupal.org/node/2487568. Until that lands, return
+    // an empty item.
+    catch (MigrateSkipRowException $e) {
+      return [];
     }
 
-    $file = [
-      'target_id' => $fid,
-      'display' => isset($list) ? $list : 0,
-      'description' => isset($options['description']) ? $options['description'] : '',
-    ];
-
-    return $file;
+    if ($fid) {
+      return [
+        'target_id' => $fid,
+        'display' => $value['list'],
+        'description' => isset($options['description']) ? $options['description'] : '',
+      ];
+    }
+    else {
+      return [];
+    }
   }
 
 }

core/modules/file/src/Tests/Migrate/d6/MigrateUploadTest.php

@@ -21,6 +21,15 @@ class MigrateUploadTest extends MigrateUploadBase {
    */
   protected function setUp() {
     parent::setUp();
+    $id_mappings = array(
+      'd6_node:*' => array(
+        array(
+          array(0),
+          array(0),
+        ),
+      ),
+    );
+    $this->prepareMigrations($id_mappings);
     $this->executeMigration('d6_upload');
   }
 

core/modules/link/src/Plugin/migrate/cckfield/LinkField.php

@@ -39,11 +39,7 @@ public function getFieldFormatterMap() {
   public function processCckFieldValues(MigrationInterface $migration, $field_name, $data) {
       $process = [
         'plugin' => 'd6_cck_link',
-        'source' => [
-          $field_name,
-          $field_name . '_title',
-          $field_name . '_attributes',
-        ],
+        'source' => $field_name,
       ];
       $migration->mergeProcessOfProperty($field_name, $process);
   }

core/modules/link/src/Plugin/migrate/process/d6/CckLink.php

@@ -45,15 +45,13 @@ public static function create(ContainerInterface $container, array $configuratio
    * {@inheritdoc}
    */
   public function transform($value, MigrateExecutableInterface $migrate_executable, Row $row, $destination_property) {
-    list($url, $title, $attributes) = $value;
-
     // Drupal 6 link attributes are double serialized.
-    $attributes = unserialize(unserialize($attributes));
+    $attributes = unserialize(unserialize($value['attributes']));
 
     // Massage the values into the correct form for the link.
-    $route['uri'] = $url;
+    $route['uri'] = $value['url'];
     $route['options']['attributes'] = $attributes;
-    $route['title'] = $title;
+    $route['title'] = $value['title'];
     return $route;
   }
 

core/modules/migrate/config/schema/migrate.load.schema.yml

-# Schema for the migrate load plugins.
-
-migrate.load.*:
-  type: migrate_load
-  label: 'Default load'
-
-migrate.load.drupal_entity:
-  type: migrate_load
-  label: 'Default source'
-  mapping:
-    bundle_migration:
-      type: string
-      label: 'Bundle migration'
-
-migrate.load.d6_term_node:
-  type: migrate_load
-  label: 'Default source'
-  mapping:
-    bundle_migration:
-      type: string
-      label: 'Bundle migration'

core/modules/migrate/config/schema/migrate.schema.yml

@@ -16,9 +16,6 @@ migrate.migration.*:
     label:
       type: label
       label: 'Label'
-    load:
-      type: migrate.load.[plugin]
-      label: 'Source'
     source:
       type: migrate.source.[plugin]
       label: 'Source'

core/modules/migrate/src/MigrationStorage.php

@@ -64,52 +64,54 @@ public static function createInstance(ContainerInterface $container, EntityTypeI
    * {@inheritdoc}
    */
   public function loadMultiple(array $ids = NULL) {
+    if ($ids) {
+      $ids = $this->getVariantIds($ids);
+    }
     /** @var \Drupal\migrate\Entity\MigrationInterface[] $migrations */
     $migrations = parent::loadMultiple($ids);
 
     foreach ($migrations as $migration) {
-      $migration->set('migration_dependencies', $this->expandDependencies($migration->getMigrationDependencies()));
+      $dependencies = array_map([$this, 'getVariantIds'], $migration->getMigrationDependencies());
+      $migration->set('migration_dependencies', $dependencies);
     }
-    return $migrations;
+
+    // Build an array of dependencies and set the order of the migrations.
+    return $this->buildDependencyMigration($migrations, []);
   }
 
   /**
-   * Expands template dependencies.
+   * Splices variant IDs into a list of migration IDs.
    *
-   * Migration dependencies which match the template_id:* pattern are a signal
-   * that the migration depends on every variant of template_id. This method
-   * queries for those variant IDs and splices them into the list of
-   * dependencies.
+   * IDs which match the template_id:* pattern are shorthand for every variant
+   * of template_id. This method queries for those variant IDs and splices them
+   * into the original list.
    *
-   * @param array $dependencies
-   *   The original migration dependencies (with template IDs), organized by
-   *   group (required, optional, etc.)
+   * @param string[] $ids
+   *   A set of migration IDs.
    *
-   * @return array
-   *   The expanded list of dependencies, organized by group.
+   * @return string[]
+   *   The expanded list of IDs.
    */
-  protected function expandDependencies(array $dependencies) {
-    $expanded_dependencies = [];
-
-    foreach (array_keys($dependencies) as $group) {
-      $expanded_dependencies[$group] = [];
-
-      foreach ($dependencies[$group] as $dependency_id) {
-        if (substr($dependency_id, -2) == ':*') {
-          $template_id = substr($dependency_id, 0, -2);
-          $variants = $this->queryFactory->get($this->entityType, 'OR')
-            ->condition('id', $template_id)
-            ->condition('template', $template_id)
-            ->execute();
-          $expanded_dependencies[$group] = array_merge($expanded_dependencies[$group], $variants);
-        }
-        else {
-          $expanded_dependencies[$group][] = $dependency_id;
-        }
+  protected function getVariantIds(array $ids) {
+    // Re-index the array numerically, since we need to limit the loop by size.
+    $ids = array_values($ids);
+
+    $index = 0;
+    while ($index < count($ids)) {
+      if (substr($ids[$index], -2) == ':*') {
+        $template_id = substr($ids[$index], 0, -2);
+        $variants = $this->queryFactory->get($this->entityType, 'OR')
+          ->condition('id', $template_id)
+          ->condition('template', $template_id)
+          ->execute();
+        array_splice($ids, $index, 1, $variants);
+        $index += count($variants);
+      }
+      else {
+        $index++;
       }
     }
-
-    return $expanded_dependencies;
+    return $ids;
   }
 
   /**

core/modules/migrate/src/Plugin/SourceEntityInterface.php

-<?php
-/**
- * @file
- * Contains \Drupal\migrate\Plugin\SourceEntityInterface.
- */
-
-namespace Drupal\migrate\Plugin;
-
-/**
- * Interface for sources providing an entity.
- */
-interface SourceEntityInterface {
-
-  /**
-   * Whether this migration has a bundle migration.
-   *
-   * @return bool
-   *   TRUE when the bundle_migration key is required.
-   */
-  public function bundleMigrationRequired();
-
-  /**
-   * The entity type id (user, node etc).
-   *
-   * This function is used when bundleMigrationRequired() is FALSE.
-   *
-   * @return string
-   *   The entity type id.
-   */
-  public function entityTypeId();
-
-}

core/modules/migrate/src/Plugin/migrate/process/Route.php

@@ -47,6 +47,7 @@ public static function create(ContainerInterface $container, array $configuratio
       $container->get('path.validator')
     );
   }
+
   /**
    * {@inheritdoc}
    *

core/modules/migrate/src/Tests/MigrateTestBase.php

@@ -116,20 +116,19 @@ protected function loadDumps(array $files, $method = 'load') {
    *   ids.
    */
   protected function prepareMigrations(array $id_mappings) {
-    /** @var \Drupal\migrate\Entity\MigrationInterface[] $migrations */
-    $migrations = entity_load_multiple('migration', array_keys($id_mappings));
     foreach ($id_mappings as $migration_id => $data) {
-      $migration = $migrations[$migration_id];
-
-      // Mark the dependent migrations as complete.
-      $migration->setMigrationResult(MigrationInterface::RESULT_COMPLETED);
-
-      $id_map = $migration->getIdMap();
-      $id_map->setMessage($this);
-      $source_ids = $migration->getSourcePlugin()->getIds();
-      foreach ($data as $id_mapping) {
-        $row = new Row(array_combine(array_keys($source_ids), $id_mapping[0]), $source_ids);
-        $id_map->saveIdMapping($row, $id_mapping[1]);
+      // Use loadMultiple() here in order to load all variants.
+      foreach (Migration::loadMultiple([$migration_id]) as $migration) {
+        // Mark the dependent migrations as complete.
+        $migration->setMigrationResult(MigrationInterface::RESULT_COMPLETED);
+
+        $id_map = $migration->getIdMap();
+        $id_map->setMessage($this);
+        $source_ids = $migration->getSourcePlugin()->getIds();
+        foreach ($data as $id_mapping) {
+          $row = new Row(array_combine(array_keys($source_ids), $id_mapping[0]), $source_ids);
+          $id_map->saveIdMapping($row, $id_mapping[1]);
+        }
       }
     }
   }

core/modules/migrate/tests/src/Unit/MigrationStorageTest.php

@@ -54,61 +54,43 @@ public function setUp() {
   }
 
   /**
-   * Tests expandDependencies() when variants exist.
+   * Tests getVariantIds() when variants exist.
    *
-   * @covers ::expandDependencies
+   * @covers ::getVariantIds
    */
-  public function testExpandDependenciesWithVariants() {
+  public function testGetVariantIdsWithVariants() {
     $this->query->method('execute')
       ->willReturn(['d6_node__page', 'd6_node__article']);
 
-    $dependencies = [
-      'required' => [
-        'd6_node:*',
-        'd6_user',
-      ],
-    ];
-    $dependencies = $this->storage->expandDependencies($dependencies);
-    $this->assertSame(['d6_node__page', 'd6_node__article', 'd6_user'],  $dependencies['required']);
+    $ids = $this->storage->getVariantIds(['d6_node:*', 'd6_user']);
+    $this->assertSame(['d6_node__page', 'd6_node__article', 'd6_user'],  $ids);
   }
 
   /**
-   * Tests expandDependencies() when no variants exist.
+   * Tests getVariantIds() when no variants exist.
    *
-   * @covers ::expandDependencies
+   * @covers ::getVariantIds
    */
-  public function testExpandDependenciesNoVariants() {
+  public function testGetVariantIdsNoVariants() {
     $this->query->method('execute')
       ->willReturn([]);
 
-    $dependencies = [
-      'required' => [
-        'd6_node:*',
-        'd6_user',
-      ],
-    ];
-    $dependencies = $this->storage->expandDependencies($dependencies);
-    $this->assertSame(['d6_user'],  $dependencies['required']);
+    $ids = $this->storage->getVariantIds(['d6_node:*', 'd6_user']);
+    $this->assertSame(['d6_user'],  $ids);
   }
 
   /**
-   * Tests expandDependencies() when no variants exist and there are no static
+   * Tests getVariantIds() when no variants exist and there are no static
    * (non-variant) dependencies.
    *
-   * @covers ::expandDependencies
+   * @covers ::getVariantIds
    */
-  public function testExpandDependenciesNoVariantsOrStaticDependencies() {
+  public function testGetVariantIdsNoVariantsOrStaticDependencies() {
     $this->query->method('execute')
       ->willReturn([]);
 
-    $dependencies = [
-      'required' => [
-        'd6_node:*',
-        'd6_node_revision:*',
-      ],
-    ];
-    $dependencies = $this->storage->expandDependencies($dependencies);
-    $this->assertSame([],  $dependencies['required']);
+    $ids = $this->storage->getVariantIds(['d6_node:*', 'd6_node_revision:*']);
+    $this->assertSame([],  $ids);
   }
 
 }
@@ -123,8 +105,8 @@ class TestMigrationStorage extends MigrationStorage {
   /**
    * {@inheritdoc}
    */
-  public function expandDependencies(array $dependencies) {
-    return parent::expandDependencies($dependencies);
+  public function getVariantIds(array $ids) {
+    return parent::getVariantIds($ids);
   }
 
 }

core/modules/migrate_drupal/migrate_drupal.module

@@ -7,16 +7,6 @@
 
 use Drupal\Core\Routing\RouteMatchInterface;
 
-/**
- * Implements hook_entity_type_alter().
- */
-function migrate_drupal_entity_type_alter(array &$entity_types) {
-  /** @var \Drupal\Core\Config\Entity\ConfigEntityType[] $entity_types */
-  $entity_types['migration']
-    ->setClass('Drupal\migrate_drupal\Entity\Migration')
-    ->setHandlerClass('storage', 'Drupal\migrate_drupal\MigrationStorage');
-}
-
 /**
  * Implements hook_help().
  */

core/modules/migrate_drupal/migrate_drupal.services.yml

 services:
-  plugin.manager.migrate.load:
-    class: Drupal\migrate\Plugin\MigratePluginManager
-    arguments: [load, '@container.namespaces', '@cache.discovery', '@module_handler']
   plugin.manager.migrate.cckfield:
     class: Drupal\migrate\Plugin\MigratePluginManager
     arguments: [cckfield, '@container.namespaces', '@cache.discovery', '@module_handler']

core/modules/migrate_drupal/src/Entity/Migration.php

-<?php
-
-/**
- * @file
- * Contains \Drupal\migrate_drupal\Entity\Migration.
- */
-
-namespace Drupal\migrate_drupal\Entity;
-
-use Drupal\migrate\Entity\Migration as BaseMigration;
-
-class Migration extends BaseMigration implements MigrationInterface {
-
-  /**
-   * The load plugin configuration, if any.
-   *
-   * @var array
-   */
-  protected $load = array();
-
-  /**
-   * The load plugin.
-   *
-   * @var \Drupal\migrate_drupal\Plugin\MigrateLoadInterface|false
-   */
-  protected $loadPlugin = FALSE;
-
-  /**
-   * {@inheritdoc}
-   */
-  public function getLoadPlugin() {
-    if ($this->load && !$this->loadPlugin) {
-      $this->loadPlugin = \Drupal::service('plugin.manager.migrate.load')->createInstance($this->load['plugin'], $this->load, $this);
-    }
-    return $this->loadPlugin;
-  }
-
-}

core/modules/migrate_drupal/src/Entity/MigrationInterface.php

-<?php
-/**
- * @file
- * Contains \Drupal\migrate_drupal\Entity\MigrationInterface.
- */
-
-namespace Drupal\migrate_drupal\Entity;
-
-use Drupal\migrate\Entity\MigrationInterface as BaseMigrationInterface;
-
-interface MigrationInterface extends BaseMigrationInterface {
-
-  /**
-   * Returns the initialized load plugin if there's one.
-   *
-   * @return \Drupal\migrate_drupal\Plugin\MigrateLoadInterface|false
-   */
-  public function getLoadPlugin();