Issue #2154209 by chx: Process refactor: multiple handling.

core/modules/migrate/lib/Drupal/migrate/Annotation/MigrateProcessPlugin.php

+<?php
+
+/**
+ * @file
+ * Contains \Drupal\migrate\Annotation\MigrateProcessPlugin.
+ */
+
+namespace Drupal\migrate\Annotation;
+
+use Drupal\Component\Annotation\Plugin;
+
+/**
+ * Defines a migration process plugin annotation object.
+ *
+ * @Annotation
+ */
+class MigrateProcessPlugin extends Plugin {
+
+  /**
+   * A unique identifier for the process plugin.
+   *
+   * @var string
+   */
+  public $id;
+
+  /**
+   * Whether the plugin handles multiples itself.
+   *
+   * Typically these plugins will expect an array as input and iterate over it
+   * themselves, changing the whole array. For example the 'iterator' and the
+   * 'flatten' plugins. If the plugin only need to change a single value it
+   * can skip setting this attribute and let
+   * \Drupal\migrate\MigrateExecutable::processRow() handle the iteration.
+   *
+   * @var bool (optional)
+   */
+  public $handle_multiples = FALSE;
+}

core/modules/migrate/lib/Drupal/migrate/MigrateExecutable.php

@@ -330,8 +330,27 @@ public function import() {
    */
   public function processRow(Row $row, array $process = NULL, $value = NULL) {
     foreach ($this->migration->getProcessPlugins($process) as $destination => $plugins) {
+      $multiple = FALSE;
       foreach ($plugins as $plugin) {
-        $value = $plugin->transform($value, $this, $row, $destination);
+        $definition = $plugin->getPluginDefinition();
+        // Many plugins expect a scalar value but the current value of the
+        // pipeline might be multiple scalars (this is set by the previous
+        // plugin) and in this case the current value needs to be iterated
+        // and each scalar separately transformed.
+        if ($multiple && !$definition['handle_multiples']) {
+          $new_value = array();
+          if (!is_array($value)) {
+            throw new MigrateException(sprintf('Pipeline failed for destination %s: %s got instead of an array,', $destination, $value));
+          }
+          foreach ($value as $scalar_value) {
+            $new_value[] = $plugin->transform($scalar_value, $this, $row, $destination);
+          }
+          $value = $new_value;
+        }
+        else {
+          $value = $plugin->transform($value, $this, $row, $destination);
+          $multiple = $multiple || $plugin->multiple();
+        }
       }
       $row->setDestinationProperty($destination, $value);
       // Reset the value.

core/modules/migrate/lib/Drupal/migrate/Plugin/MigratePluginManager.php

@@ -15,14 +15,14 @@
 use Drupal\migrate\Entity\MigrationInterface;
 
 /**
- * Manages migrate sources and steps.
+ * Manages migrate plugins.
  *
  * @see hook_migrate_info_alter()
  */
 class MigratePluginManager extends DefaultPluginManager {
 
   /**
-   * Constructs a MigraterPluginManager object.
+   * Constructs a MigratePluginManager object.
    *
    * @param string $type
    *   The type of the plugin: row, source, process, destination, entity_field, id_map.
@@ -35,9 +35,11 @@ class MigratePluginManager extends DefaultPluginManager {
    *   The language manager.
    * @param \Drupal\Core\Extension\ModuleHandlerInterface $module_handler
    *   The module handler to invoke the alter hook with.
+   * @param $annotation
+   *   The annotation class name.
    */
-  public function __construct($type, \Traversable $namespaces, CacheBackendInterface $cache_backend, LanguageManager $language_manager, ModuleHandlerInterface $module_handler) {
-    parent::__construct("Plugin/migrate/$type", $namespaces, 'Drupal\Component\Annotation\PluginID');
+  public function __construct($type, \Traversable $namespaces, CacheBackendInterface $cache_backend, LanguageManager $language_manager, ModuleHandlerInterface $module_handler, $annotation = 'Drupal\Component\Annotation\PluginID') {
+    parent::__construct("Plugin/migrate/$type", $namespaces, $annotation);
     $this->alterInfo($module_handler, 'migrate_' . $type . '_info');
     $this->setCacheBackend($cache_backend, $language_manager, 'migrate_plugins_' . $type);
   }

core/modules/migrate/lib/Drupal/migrate/Plugin/MigrateProcessInterface.php

@@ -33,4 +33,13 @@ interface MigrateProcessInterface extends PluginInspectionInterface {
    */
   public function transform($value, MigrateExecutable $migrate_executable, Row $row, $destination_property);
 
+  /**
+   * Indicates whether the returned value requires multiple handling.
+   *
+   * @return bool
+   *   TRUE when the returned value contains a list of values to be processed.
+   *   For example, when the 'source' property is a string and the value found
+   *   is an array.
+   */
+  public function multiple();
 }

core/modules/migrate/lib/Drupal/migrate/Plugin/migrate/process/DedupeBase.php

@@ -7,9 +7,8 @@
 
 namespace Drupal\migrate\Plugin\migrate\process;
 
-use Drupal\Core\Plugin\PluginBase;
+use Drupal\migrate\ProcessPluginBase;
 use Drupal\migrate\MigrateExecutable;
-use Drupal\migrate\Plugin\MigrateProcessInterface;
 use Drupal\migrate\Row;
 
 /**
@@ -20,7 +19,7 @@
  * existing filter format names and if it exists, a numeric postfix is added
  * and incremented until a unique value is created.
  */
-abstract class DedupeBase extends PluginBase implements MigrateProcessInterface {
+abstract class DedupeBase extends ProcessPluginBase {
 
   /**
    * {@inheritdoc}

core/modules/migrate/lib/Drupal/migrate/Plugin/migrate/process/DedupeEntity.php

@@ -10,7 +10,9 @@
 /**
  * Ensures value is not duplicated against an entity field.
  *
- * @PluginId("dedupe_entity")
+ * @MigrateProcessPlugin(
+ *   id = "dedupe_entity"
+ * )
  */
 class DedupeEntity extends DedupeBase {
 

core/modules/migrate/lib/Drupal/migrate/Plugin/migrate/process/DefaultValue.php

@@ -7,18 +7,19 @@
 
 namespace Drupal\migrate\Plugin\migrate\process;
 
-use Drupal\Core\Plugin\PluginBase;
+use Drupal\migrate\ProcessPluginBase;
 use Drupal\migrate\MigrateExecutable;
-use Drupal\migrate\Plugin\MigrateProcessInterface;
 use Drupal\migrate\Row;
 
 
 /**
  * This plugin sets missing values on the destination.
  *
- * @PluginId("default_value")
+ * @MigrateProcessPlugin(
+ *   id = "default_value"
+ * )
  */
-class DefaultValue extends PluginBase implements MigrateProcessInterface {
+class DefaultValue extends ProcessPluginBase {
 
  /**
    * {@inheritdoc}

core/modules/migrate/lib/Drupal/migrate/Plugin/migrate/process/Get.php

@@ -7,17 +7,23 @@
 
 namespace Drupal\migrate\Plugin\migrate\process;
 
-use Drupal\Core\Plugin\PluginBase;
+use Drupal\migrate\ProcessPluginBase;
 use Drupal\migrate\MigrateExecutable;
-use Drupal\migrate\Plugin\MigrateProcessInterface;
 use Drupal\migrate\Row;
 
 /**
  * This plugin copies from the source to the destination.
  *
- * @PluginId("get")
+ * @MigrateProcessPlugin(
+ *   id = "get"
+ * )
  */
-class Get extends PluginBase implements MigrateProcessInterface {
+class Get extends ProcessPluginBase {
+
+  /**
+   * @var bool
+   */
+  protected $multiple;
 
   /**
    * {@inheritdoc}
@@ -50,6 +56,17 @@ public function transform($value, MigrateExecutable $migrate_executable, Row $ro
         }
       }
     }
-    return is_string($source) ? $return[0] : $return;
+    if (is_string($source)) {
+      $this->multiple = is_array($return[0]);
+      return $return[0];
+    }
+    return $return;
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public function multiple() {
+    return $this->multiple;
   }
 }

core/modules/migrate/lib/Drupal/migrate/Plugin/migrate/process/Iterator.php

@@ -7,9 +7,8 @@
 
 namespace Drupal\migrate\Plugin\migrate\process;
 
-use Drupal\Core\Plugin\PluginBase;
+use Drupal\migrate\ProcessPluginBase;
 use Drupal\migrate\MigrateExecutable;
-use Drupal\migrate\Plugin\MigrateProcessInterface;
 use Drupal\migrate\Row;
 
 /**
@@ -17,9 +16,12 @@
  *
  * @see https://drupal.org/node/2135345
  *
- * @PluginId("iterator")
+ * @MigrateProcessPlugin(
+ *   id = "iterator",
+ *   handle_multiples = TRUE
+ * )
  */
-class Iterator extends PluginBase implements MigrateProcessInterface {
+class Iterator extends ProcessPluginBase {
 
   /**
    * Runs a process pipeline on each destination property per list item.
@@ -56,4 +58,10 @@ protected function transformKey($key, MigrateExecutable $migrate_executable, Row
     return $row->getDestinationProperty('key');
   }
 
+  /**
+   * {@inheritdoc}
+   */
+  public function multiple() {
+    return TRUE;
+  }
 }

core/modules/migrate/lib/Drupal/migrate/Plugin/migrate/process/MachineName.php

@@ -8,9 +8,8 @@
 namespace Drupal\migrate\Plugin\migrate\process;
 
 use Drupal\Core\Language\Language;
-use Drupal\Core\Plugin\PluginBase;
+use Drupal\migrate\ProcessPluginBase;
 use Drupal\migrate\MigrateExecutable;
-use Drupal\migrate\Plugin\MigrateProcessInterface;
 use Drupal\migrate\Row;
 
 /**
@@ -20,9 +19,11 @@
  * and replaced by an underscore and multiple underscores are collapsed into
  * one.
  *
- * @PluginId("machine_name")
+ * @MigrateProcessPlugin(
+ *   id = "machine_name"
+ * )
  */
-class MachineName extends PluginBase implements MigrateProcessInterface {
+class MachineName extends ProcessPluginBase {
 
   /**
    * @var \Drupal\Core\Transliteration\PHPTransliteration

core/modules/migrate/lib/Drupal/migrate/Plugin/migrate/process/StaticMap.php

@@ -8,10 +8,9 @@
 namespace Drupal\migrate\Plugin\migrate\process;
 
 use Drupal\Component\Utility\NestedArray;
-use Drupal\Core\Plugin\PluginBase;
+use Drupal\migrate\ProcessPluginBase;
 use Drupal\migrate\MigrateException;
 use Drupal\migrate\MigrateExecutable;
-use Drupal\migrate\Plugin\MigrateProcessInterface;
 use Drupal\migrate\Row;
 
 /**
@@ -19,23 +18,33 @@
  *
  * @see https://drupal.org/node/2143521
  *
- * @PluginId("static_map")
+ * @MigrateProcessPlugin(
+ *   id = "static_map"
+ * )
  */
-class StaticMap extends PluginBase implements MigrateProcessInterface {
+class StaticMap extends ProcessPluginBase {
 
   /**
    * {@inheritdoc}
    */
   public function transform($value, MigrateExecutable $migrate_executable, Row $row, $destination_property) {
-    if (!is_array($value)) {
-      $value = array($value);
+    $new_value = $value;
+    if (is_array($value)) {
+      if (!$value) {
+        throw new MigrateException('Can not lookup without a value.');
+      }
     }
-    if (!$value) {
-      throw new MigrateException('Can not lookup without a value.');
+    else {
+      $new_value = array($value);
     }
-    $new_value = NestedArray::getValue($this->configuration['map'], $value, $key_exists);
+    $new_value = NestedArray::getValue($this->configuration['map'], $new_value, $key_exists);
     if (!$key_exists) {
-      throw new MigrateException('Lookup failed.');
+      if (empty($this->configuration['bypass'])) {
+        throw new MigrateException('Lookup failed.');
+      }
+      else {
+        return $value;
+      }
     }
     return $new_value;
   }

core/modules/migrate/lib/Drupal/migrate/ProcessPluginBase.php

+<?php
+
+/**
+ * @file
+ * Contains \Drupal\migrate\Plugin\migrate\process\ProcessPluginBase.
+ */
+
+namespace Drupal\migrate;
+
+use Drupal\Core\Plugin\PluginBase;
+use Drupal\migrate\Plugin\MigrateProcessInterface;
+
+/**
+ * The base class for all migrate process plugins.
+ *
+ * Migrate process plugins are taking a value and transform them. For example,
+ * transform a human provided name into a machine name, look up an identifier
+ * in a previous migration and so on.
+ *
+ * @see https://drupal.org/node/2129651
+ */
+abstract class ProcessPluginBase extends PluginBase implements MigrateProcessInterface {
+
+  /**
+   * {@inheritdoc}
+   */
+  public function multiple() {
+    return FALSE;
+  }
+
+}

core/modules/migrate/migrate.services.yml

@@ -11,7 +11,7 @@ services:
     arguments: [source, '@container.namespaces', '@cache.cache', '@language_manager', '@module_handler']
   plugin.manager.migrate.process:
     class: Drupal\migrate\Plugin\MigratePluginManager
-    arguments: [process, '@container.namespaces', '@cache.cache', '@language_manager', '@module_handler']
+    arguments: [process, '@container.namespaces', '@cache.cache', '@language_manager', '@module_handler', 'Drupal\migrate\Annotation\MigrateProcessPlugin']
   plugin.manager.migrate.destination:
     class: Drupal\migrate\Plugin\MigratePluginManager
     arguments: [destination, '@container.namespaces', '@cache.cache', '@language_manager', '@module_handler']