Issue #2958672 by benjifisher, alisonjo2786, marvil07, heddn: Use migration lookup on text fields

src/Plugin/migrate/process/DomMigrationLookup.php

+<?php
+
+namespace Drupal\migrate_plus\Plugin\migrate\process;
+
+use Drupal\Component\Plugin\Exception\InvalidPluginDefinitionException;
+use Drupal\Core\Plugin\ContainerFactoryPluginInterface;
+use Drupal\migrate\MigrateException;
+use Drupal\migrate\MigrateExecutableInterface;
+use Drupal\migrate\Plugin\MigratePluginManagerInterface;
+use Drupal\migrate\Plugin\MigrationInterface;
+use Drupal\migrate\Row;
+use Symfony\Component\DependencyInjection\ContainerInterface;
+
+/**
+ * String replacements on a source dom based on migration lookup.
+ *
+ * Meant to be used after dom process plugin.
+ *
+ * Available configuration keys:
+ * - mode: What to modify. Possible values:
+ *   - attribute: One element attribute.
+ * - xpath: XPath query expression that will produce the \DOMNodeList to walk.
+ * - attribute_options: A map of options related to the attribute mode. Required
+ *   when mode is attribute. The keys can be:
+ *   - name: Name of the attribute to match and modify.
+ * - search: Regular expression to use. It should contain at least one
+ *   parenthesized subpattern which will be used as the ID passed to
+ *   migration_lookup process plugin.
+ * - replace: Default value to use for replacements on migrations, if not
+ *   specified on the migration. It should contain the '[mapped-id]' string
+ *   where the looked-up migration value will be placed.
+ * - migrations: A map of options indexed by migration machine name. Possible
+ *   option values are:
+ *   - replace: See replace option lines above.
+ * - no_stub: If TRUE, then do not create stub entities during migration lookup.
+ *   Optional, defaults to TRUE.
+ *
+ * Example:
+ *
+ * @code
+ * process:
+ *   'body/value':
+ *     -
+ *       plugin: dom
+ *       method: import
+ *       source: 'body/0/value'
+ *     -
+ *       plugin: dom_migration_lookup
+ *       mode: attribute
+ *       xpath: '//a'
+ *       attribute_options:
+ *         name: href
+ *       search: '@/user/(\d+)@'
+ *       replace: '/user/[mapped-id]'
+ *       migrations:
+ *         users:
+ *           replace: '/user/[mapped-id]'
+ *         people:
+ *           replace: '/people/[mapped-id]'
+ *     -
+ *       plugin: dom
+ *       method: export
+ * @endcode
+ *
+ * @MigrateProcessPlugin(
+ *   id = "dom_migration_lookup"
+ * )
+ */
+class DomMigrationLookup extends DomStrReplace implements ContainerFactoryPluginInterface {
+
+  /**
+   * The migration to be executed.
+   *
+   * @var \Drupal\migrate\Plugin\MigrationInterface
+   */
+  protected $migration;
+
+  /**
+   * The process plugin manager.
+   *
+   * @var \Drupal\migrate\Plugin\MigratePluginManagerInterface
+   */
+  protected $processPluginManager;
+
+  /**
+   * Parameters passed to transform method, except the first, value.
+   *
+   * This helps to pass values to another process plugin.
+   *
+   * @var array
+   */
+  protected $transformParameters;
+
+  /**
+   * {@inheritdoc}
+   */
+  public function __construct(array $configuration, $plugin_id, $plugin_definition, MigrationInterface $migration, MigratePluginManagerInterface $process_plugin_manager) {
+    $configuration += ['no_stub' => TRUE];
+    $default_replace_missing = empty($configuration['replace']);
+    if ($default_replace_missing) {
+      $configuration['replace'] = 'prevent-requirement-fail';
+    }
+    parent::__construct($configuration, $plugin_id, $plugin_definition);
+    if ($default_replace_missing) {
+      unset($this->configuration['replace']);
+    }
+    $this->migration = $migration;
+    $this->processPluginManager = $process_plugin_manager;
+    if (empty($this->configuration['migrations'])) {
+      throw new InvalidPluginDefinitionException(
+        $this->getPluginId(),
+        "Configuration option 'migration' is required."
+      );
+    }
+    if (!is_array($this->configuration['migrations'])) {
+      throw new InvalidPluginDefinitionException(
+        $this->getPluginId(),
+        "Configuration option 'migration' should be a keyed array."
+      );
+    }
+    // Add missing values if possible
+    $default_replace = isset($this->configuration['replace']) ? $this->configuration['replace'] : NULL;
+    foreach ($this->configuration['migrations'] as $migration_name => $configuration_item) {
+      if (!empty($configuration_item['replace'])) {
+        continue;
+      }
+      if (is_null($default_replace)) {
+        throw new InvalidPluginDefinitionException(
+          $this->getPluginId(),
+          "Please define either a global replace for all migrations, or a specific one for 'migrations.$migration_name'."
+        );
+      }
+      $this->configuration['migrations'][$migration_name]['replace'] = $default_replace;
+    }
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public static function create(ContainerInterface $container, array $configuration, $plugin_id, $plugin_definition, MigrationInterface $migration = NULL) {
+    return new static(
+      $configuration,
+      $plugin_id,
+      $plugin_definition,
+      $migration,
+      $container->get('plugin.manager.migrate.process')
+    );
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public function transform($value, MigrateExecutableInterface $migrate_executable, Row $row, $destination_property) {
+    $this->init($value, $destination_property);
+    $this->transformParameters = [
+      'migrate_executable' => $migrate_executable,
+      'row' => $row,
+      'destination_property' => $destination_property,
+    ];
+
+    foreach ($this->xpath->query($this->configuration['xpath']) as $html_node) {
+      $subject = $this->getSubject($html_node);
+      if (empty($subject)) {
+        // Could not find subject, skip processing.
+        continue;
+      }
+      $search = $this->getSearch();
+      if (!preg_match($search, $subject, $matches)) {
+        // No match found, skip processing.
+        continue;
+      }
+      $id = $matches[1];
+      // Walk through defined migrations looking for a map.
+      foreach ($this->configuration['migrations'] as $migration_name => $configuration) {
+        $mapped_id = $this->migrationLookup($id, $migration_name);
+        if (!is_null($mapped_id)) {
+          // Not using getReplace(), since this implementation depends on the
+          // migration.
+          $replace = str_replace('[mapped-id]', $mapped_id, $configuration['replace']);
+          $this->doReplace($html_node, $search, $replace, $subject);
+          break;
+        }
+      }
+    }
+
+    return $this->document;
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  protected function doReplace(\DOMElement $html_node, $search, $replace, $subject) {
+    $new_subject = preg_replace($search, $replace, $subject);
+    $this->postReplace($html_node, $new_subject);
+  }
+
+  /**
+   * Lookup the migration mapped ID on one migration.
+   *
+   * @param mixed $id
+   *   The ID to search with migration_lookup process plugin.
+   * @param string $migration_name
+   *   The migration to look into machine name.
+   *
+   * @return string|null
+   *   The found mapped ID, or NULL if not found on the provided migration.
+   */
+  protected function migrationLookup($id, $migration_name) {
+    $mapped_id = NULL;
+    $parameters = [
+      $id,
+      $this->transformParameters['migrate_executable'],
+      $this->transformParameters['row'],
+      $this->transformParameters['destination_property'],
+    ];
+    $plugin_configuration = [
+      'migration' => $migration_name,
+      'no_stub' => $this->configuration['no_stub'],
+    ];
+    $migration_lookup_plugin = $this->processPluginManager
+      ->createInstance('migration_lookup', $plugin_configuration, $this->migration);
+    $mapped_id = call_user_func_array([$migration_lookup_plugin, 'transform'], $parameters);
+    return $mapped_id;
+  }
+
+}

src/Plugin/migrate/process/DomStrReplace.php

@@ -16,8 +16,7 @@ use Drupal\migrate\Row;
  * Available configuration keys:
  * - mode: What to modify. Possible values:
  *   - attribute: One element attribute.
- * - expression: XPath query expression that will produce the \DOMNodeList to
- *   walk.
+ * - xpath: XPath query expression that will produce the \DOMNodeList to walk.
  * - attribute_options: A map of options related to the attribute mode. Required
  *   when mode is attribute. The keys can be:
  *   - name: Name of the attribute to match and modify.
@@ -38,7 +37,7 @@ use Drupal\migrate\Row;
  *     -
  *       plugin: dom_str_replace
  *       mode: attribute
- *       expression: '//a'
+ *       xpath: '//a'
  *       attribute_options:
  *         name: href
  *       search: 'foo'
@@ -46,7 +45,7 @@ use Drupal\migrate\Row;
  *     -
  *       plugin: dom_str_replace
  *       mode: attribute
- *       expression: '//a'
+ *       xpath: '//a'
  *       attribute_options:
  *         name: href
  *       regex: true
@@ -73,7 +72,7 @@ class DomStrReplace extends DomProcessBase {
       'regex' => FALSE,
     ];
     $options_validation = [
-      'expression' => NULL,
+      'xpath' => NULL,
       'mode' => ['attribute'],
       // @todo Move out once another mode is supported.
       // @see https://www.drupal.org/project/migrate_plus/issues/3042833
@@ -107,7 +106,7 @@ class DomStrReplace extends DomProcessBase {
   public function transform($value, MigrateExecutableInterface $migrate_executable, Row $row, $destination_property) {
     $this->init($value, $destination_property);
 
-    foreach ($this->xpath->query($this->configuration['expression']) as $html_node) {
+    foreach ($this->xpath->query($this->configuration['xpath']) as $html_node) {
       $subject = $this->getSubject($html_node);
       if (empty($subject)) {
         // Could not find subject, skip processing.

tests/src/Unit/process/DomMigrationLookupTest.php

+<?php
+
+namespace Drupal\Tests\migrate_plus\Unit\process;
+
+use Drupal\Component\Plugin\Exception\InvalidPluginDefinitionException;
+use Drupal\Component\Utility\Html;
+use Drupal\Tests\migrate\Unit\process\MigrateProcessTestCase;
+use Drupal\migrate\MigrateSkipRowException;
+use Drupal\migrate\Plugin\MigratePluginManager;
+use Drupal\migrate\Plugin\MigrateProcessInterface;
+use Drupal\migrate\Plugin\MigrationInterface;
+use Drupal\migrate_plus\Plugin\migrate\process\DomMigrationLookup;
+
+/**
+ * Tests the dom_migration_lookup process plugin.
+ *
+ * @group migrate
+ * @coversDefaultClass \Drupal\migrate_plus\Plugin\migrate\process\DomMigrationLookup
+ */
+class DomMigrationLookupTest extends MigrateProcessTestCase {
+
+  /**
+   * Example configuration for the dom_migration_lookup process plugin.
+   *
+   * @var array
+   */
+  protected $exampleConfiguration = [
+    'plugin' => 'dom_migration_lookup',
+    'mode' => 'attribute',
+    'xpath' => '//a',
+    'attribute_options' => [
+      'name' => 'href',
+    ],
+    'search' => '@/user/(\d+)@',
+    'replace' => '/user/[mapped-id]',
+    'migrations' => [
+      'users' => [],
+      'people' => [
+        'replace' => '/people/[mapped-id]',
+      ],
+    ],
+  ];
+
+  /**
+   * Mock a migration.
+   *
+   * @var \Drupal\migrate\Plugin\MigrationInterface
+   */
+  protected $migration;
+
+  /**
+   * Mock a process plugin manager.
+   *
+   * @var \Drupal\migrate\Plugin\MigratePluginManagerInterface
+   */
+  protected $processPluginManager;
+
+  /**
+   * {@inheritdoc}
+   */
+  protected function setUp() {
+    parent::setUp();
+
+    // Mock a  migration.
+    $prophecy = $this->prophesize(MigrationInterface::class);
+    $this->migration = $prophecy->reveal();
+
+    // Mock two migration lookup plugins.
+    $prophecy = $this->prophesize(MigrateProcessInterface::class);
+    $prophecy
+      ->transform('123', $this->migrateExecutable, $this->row, 'destinationproperty')
+      ->willReturn('321');
+    $prophecy
+      ->transform('456', $this->migrateExecutable, $this->row, 'destinationproperty')
+      ->willReturn(NULL);
+    $users_lookup_plugin = $prophecy->reveal();
+    $prophecy = $this->prophesize(MigrateProcessInterface::class);
+    $prophecy
+      ->transform('123', $this->migrateExecutable, $this->row, 'destinationproperty')
+      ->willReturn('ignored');
+    $prophecy
+      ->transform('456', $this->migrateExecutable, $this->row, 'destinationproperty')
+      ->willReturn('654');
+    $people_lookup_plugin = $prophecy->reveal();
+
+    // Mock a process plugin manager.
+    $prophecy = $this->prophesize(MigratePluginManager::class);
+    $users_configuration = [
+      'migration' => 'users',
+      'no_stub' => TRUE,
+    ];
+    $people_configuration = [
+      'migration' => 'people',
+      'no_stub' => TRUE,
+    ];
+    $prophecy
+      ->createInstance('migration_lookup', $users_configuration, $this->migration)
+      ->willReturn($users_lookup_plugin);
+    $prophecy
+      ->createInstance('migration_lookup', $people_configuration, $this->migration)
+      ->willReturn($people_lookup_plugin);
+    $this->processPluginManager = $prophecy->reveal();
+  }
+
+  /**
+   * @covers ::__construct
+   *
+   * @dataProvider providerTestConfigValidation
+   */
+  public function testConfigValidation(array $config_overrides, $message) {
+    $configuration = $config_overrides + $this->exampleConfiguration;
+    $value = '<p>A simple paragraph.</p>';
+    $this->setExpectedException(InvalidPluginDefinitionException::class, $message);
+    (new DomMigrationLookup($configuration, 'dom_migration_lookup', [], $this->migration, $this->processPluginManager))
+      ->transform($value, $this->migrateExecutable, $this->row, 'destinationproperty');
+  }
+
+  /**
+   * Dataprovider for testConfigValidation().
+   */
+  public function providerTestConfigValidation() {
+    $cases = [
+      'migrations-empty' => [
+        ['migrations' => []],
+        "Configuration option 'migration' is required."
+      ],
+      'migrations-invalid' => [
+        ['migrations' => 42],
+        "Configuration option 'migration' should be a keyed array."
+      ],
+      'replace-null' => [
+        ['replace' => NULL],
+        "Please define either a global replace for all migrations, or a specific one for 'migrations.users'."
+      ],
+    ];
+
+    return $cases;
+  }
+
+  /**
+   * @covers ::transform
+   */
+  public function testTransformInvalidInput() {
+    $value = 'string';
+    $this->setExpectedException(MigrateSkipRowException::class, 'The dom_migration_lookup plugin in the destinationproperty process pipeline requires a \DOMDocument object. You can use the dom plugin to convert a string to \DOMDocument.');
+    (new DomMigrationLookup($this->exampleConfiguration, 'dom_migration_lookup', [], $this->migration, $this->processPluginManager))
+      ->transform($value, $this->migrateExecutable, $this->row, 'destinationproperty');
+  }
+
+  /**
+   * @covers ::transform
+   *
+   * @dataProvider providerTestTransform
+   */
+  public function testTransform($config_overrides, $input_string, $output_string) {
+    $configuration = $config_overrides + $this->exampleConfiguration;
+    $value = Html::load($input_string);
+    $document = (new DomMigrationLookup($configuration, 'dom_migration_lookup', [], $this->migration, $this->processPluginManager))
+      ->transform($value, $this->migrateExecutable, $this->row, 'destinationproperty');
+    $this->assertTrue($document instanceof \DOMDocument);
+    $this->assertEquals($output_string, Html::serialize($document));
+  }
+
+  /**
+   * Dataprovider for testTransform().
+   */
+  public function providerTestTransform() {
+    $cases = [
+      'users-migration' => [
+        [],
+        '<a href="/user/123">text</a>',
+        '<a href="/user/321">text</a>',
+      ],
+      'people-migration' => [
+        [],
+        '<a href="https://www.example.com/user/456">text</a>',
+        '<a href="https://www.example.com/people/654">text</a>',
+      ],
+      'no-match' => [
+        ['search' => '@www\.mysite\.com/user/(\d+)@'],
+        '<a href="https://www.example.com/user/456">text</a>',
+        '<a href="https://www.example.com/user/456">text</a>',
+      ],
+    ];
+
+    return $cases;
+  }
+
+}

tests/src/Unit/process/DomStrReplaceTest.php

@@ -23,7 +23,7 @@ class DomStrReplaceTest extends MigrateProcessTestCase {
    */
   protected $exampleConfiguration = [
     'mode' => 'attribute',
-    'expression' => '//a',
+    'xpath' => '//a',
     'attribute_options' => [
       'name' => 'href',
     ],
@@ -49,9 +49,9 @@ class DomStrReplaceTest extends MigrateProcessTestCase {
    */
   public function providerTestConfigEmpty() {
     $cases = [
-      'expression-null' => [
-        ['expression' => NULL],
-        "Configuration option 'expression' is required.",
+      'xpath-null' => [
+        ['xpath' => NULL],
+        "Configuration option 'xpath' is required.",
       ],
       'mode-null' => [
         ['mode' => NULL],
@@ -83,7 +83,7 @@ class DomStrReplaceTest extends MigrateProcessTestCase {
    */
   public function testTransformInvalidInput() {
     $configuration = [
-      'expression' => '//a',
+      'xpath' => '//a',
       'mode' => 'attribute',
       'attribute_options' => [
         'name' => 'href',