Unverified Commit 63ecb2dd authored by larowlan's avatar larowlan

Issue #3004927 by mikelutz, quietone, heddn, larowlan, andypost: Create...

Issue #3004927 by mikelutz, quietone, heddn, larowlan, andypost: Create Migration Lookup and Stub services
parent c5ac07aa
......@@ -30,3 +30,9 @@ services:
plugin.manager.migration:
class: Drupal\migrate\Plugin\MigrationPluginManager
arguments: ['@module_handler', '@cache.discovery_migration', '@language_manager']
migrate.lookup:
class: Drupal\migrate\MigrateLookup
arguments: ['@plugin.manager.migration']
migrate.stub:
class: Drupal\migrate\MigrateStub
arguments: ['@plugin.manager.migration']
<?php
namespace Drupal\migrate;
use Drupal\Component\Plugin\Exception\PluginNotFoundException;
use Drupal\migrate\Plugin\MigrationInterface;
use Drupal\migrate\Plugin\MigrationPluginManagerInterface;
/**
* Provides a migration lookup service.
*/
class MigrateLookup implements MigrateLookupInterface {
/**
* The migration plugin manager.
*
* @var \Drupal\migrate\Plugin\MigrationPluginManagerInterface
*/
protected $migrationPluginManager;
/**
* Constructs a MigrateLookup object.
*
* @param \Drupal\migrate\Plugin\MigrationPluginManagerInterface $migration_plugin_manager
* The migration plugin manager.
*/
public function __construct(MigrationPluginManagerInterface $migration_plugin_manager) {
$this->migrationPluginManager = $migration_plugin_manager;
}
/**
* {@inheritdoc}
*/
public function lookup($migration_id, array $source_id_values) {
$results = [];
$migrations = $this->migrationPluginManager->createInstances($migration_id);
if (!$migrations) {
throw new PluginNotFoundException($migration_id);
}
foreach ($migrations as $migration) {
if ($result = $this->doLookup($migration, $source_id_values)) {
$results = array_merge($results, $result);
}
}
return $results;
}
/**
* Performs a lookup.
*
* @param \Drupal\migrate\Plugin\MigrationInterface $migration
* The migration upon which to perform the lookup.
* @param array $source_id_values
* The source ID values to look up.
*
* @return array
* An array of arrays of destination identifier values.
*
* @throws \Drupal\migrate\MigrateException
* Thrown when $source_id_values contains unknown keys, or the wrong number
* of keys.
*/
protected function doLookup(MigrationInterface $migration, array $source_id_values) {
$destination_keys = array_keys($migration->getDestinationPlugin()->getIds());
$indexed_ids = $migration->getIdMap()
->lookupDestinationIds($source_id_values);
$keyed_ids = [];
foreach ($indexed_ids as $id) {
$keyed_ids[] = array_combine($destination_keys, $id);
}
return $keyed_ids;
}
}
<?php
namespace Drupal\migrate;
/**
* Provides an interface for the migration lookup service.
*
* @package Drupal\migrate
*/
interface MigrateLookupInterface {
/**
* Retrieves destination ids from a migration lookup.
*
* @param string|string[] $migration_ids
* An array of migration plugin IDs to look up, or a single ID as a string.
* @param array $source_id_values
* An array of source id values.
*
* @return array
* An array of arrays of destination ids, or an empty array if none were
* found.
*
* @throws \Drupal\Component\Plugin\Exception\PluginException
* Thrown by the migration plugin manager on error, or if the migration(s)
* cannot be found.
* @throws \Drupal\migrate\MigrateException
* Thrown when $source_id_values contains unknown keys, or is the wrong
* length.
*/
public function lookup($migration_ids, array $source_id_values);
}
<?php
namespace Drupal\migrate;
use Drupal\Component\Plugin\Exception\PluginNotFoundException;
use Drupal\migrate\Plugin\MigrateIdMapInterface;
use Drupal\migrate\Plugin\MigrationInterface;
use Drupal\migrate\Plugin\MigrationPluginManagerInterface;
/**
* Provides the migrate stubbing service.
*/
class MigrateStub implements MigrateStubInterface {
/**
* The migration plugin manager.
*
* @var \Drupal\migrate\Plugin\MigrationPluginManagerInterface
*/
protected $migrationPluginManager;
/**
* Constructs a MigrationStub object.
*
* @param \Drupal\migrate\Plugin\MigrationPluginManagerInterface $migration_plugin_manager
* The migration plugin manager.
*/
public function __construct(MigrationPluginManagerInterface $migration_plugin_manager) {
$this->migrationPluginManager = $migration_plugin_manager;
}
/**
* Creates a stub.
*
* @param string $migration_id
* The migration to stub.
* @param array $source_ids
* An array of source ids.
* @param array $default_values
* (optional) An array of default values to add to the stub.
* @param bool $key_by_destination_ids
* (optional) NULL or TRUE to force indexing of the return array by
* destination id keys (default), or FALSE to return the raw return value of
* the destination plugin's ::import() method. The return value from
* MigrateDestinationInterface::import() is very poorly defined as "The
* entity ID or an indication of success". In practice, the mapping systems
* expect and all destination plugins return an array of destination
* identifiers. Unfortunately these arrays are inconsistently keyed. The
* core destination plugins return a numerically indexed array of
* destination identifiers, but several contrib destinations return an array
* of identifiers indexed by the destination keys. This method will
* generally index all return arrays for consistency and to provide as much
* information as possible, but this parameter is added for backwards
* compatibility to allow accessing the original array.
*
* @return array|false
* An array of destination ids for the new stub, keyed by destination id
* key, or false if the stub failed.
*
* @throws \Drupal\Component\Plugin\Exception\PluginException
* @throws \Drupal\Component\Plugin\Exception\PluginNotFoundException
* @throws \Drupal\migrate\MigrateException
*/
public function createStub($migration_id, array $source_ids, array $default_values = [], $key_by_destination_ids = NULL) {
$migrations = $this->migrationPluginManager->createInstances([$migration_id]);
if (!$migrations) {
throw new PluginNotFoundException($migration_id);
}
if (count($migrations) !== 1) {
throw new \LogicException(sprintf('Cannot stub derivable migration "%s". You must specify the id of a specific derivative to stub.', $migration_id));
}
$migration = reset($migrations);
$source_id_keys = array_keys($migration->getSourcePlugin()->getIds());
if (count($source_id_keys) !== count($source_ids)) {
throw new \InvalidArgumentException('Expected and provided source id counts do not match.');
}
if (array_keys($source_ids) === range(0, count($source_ids) - 1)) {
$source_ids = array_combine($source_id_keys, $source_ids);
}
$stub = $this->doCreateStub($migration, $source_ids, $default_values);
// If the return from ::import is numerically indexed, and we aren't
// requesting the raw return value, index it associatively using the
// destination id keys.
if (($key_by_destination_ids !== FALSE) && array_keys($stub) === range(0, count($stub) - 1)) {
$stub = array_combine(array_keys($migration->getDestinationPlugin()->getIds()), $stub);
}
return $stub;
}
/**
* Creates a stub.
*
* @param \Drupal\migrate\Plugin\MigrationInterface $migration
* The migration to use to create the stub.
* @param array $source_ids
* The source ids to map to the stub.
* @param array $default_values
* (optional) An array of values to include in the stub.
*
* @return array|bool
* An array of destination ids for the stub.
*
* @throws \Drupal\migrate\MigrateException
*/
protected function doCreateStub(MigrationInterface $migration, array $source_ids, array $default_values = []) {
$destination = $migration->getDestinationPlugin(TRUE);
$process = $migration->getProcess();
$id_map = $migration->getIdMap();
$migrate_executable = new MigrateExecutable($migration);
$row = new Row($source_ids + $migration->getSourceConfiguration(), $migration->getSourcePlugin()->getIds(), TRUE);
$migrate_executable->processRow($row, $process);
foreach ($default_values as $key => $value) {
$row->setDestinationProperty($key, $value);
}
$destination_ids = [];
try {
$destination_ids = $destination->import($row);
}
catch (\Exception $e) {
$id_map->saveMessage($row->getSourceIdValues(), $e->getMessage());
}
if ($destination_ids) {
$id_map->saveIdMapping($row, $destination_ids, MigrateIdMapInterface::STATUS_NEEDS_UPDATE);
return $destination_ids;
}
return FALSE;
}
}
<?php
namespace Drupal\migrate;
/**
* Provides an interface for the migrate stub service.
*/
interface MigrateStubInterface {
/**
* Creates a stub.
*
* @param string $migration_id
* The migration to stub.
* @param array $source_ids
* An array of source ids.
* @param array $default_values
* (optional) An array of default values to add to the stub.
*
* @return array|false
* An array of destination ids for the new stub, keyed by destination id
* key, or false if the stub failed.
*/
public function createStub($migration_id, array $source_ids, array $default_values = []);
}
......@@ -2,10 +2,13 @@
namespace Drupal\migrate\Plugin\migrate\process;
use Drupal\Component\Plugin\Exception\PluginNotFoundException;
use Drupal\Core\Plugin\ContainerFactoryPluginInterface;
use Drupal\migrate\MigrateException;
use Drupal\migrate\MigrateLookupInterface;
use Drupal\migrate\MigrateSkipProcessException;
use Drupal\migrate\Plugin\MigrationPluginManagerInterface;
use Drupal\migrate\Plugin\MigrateIdMapInterface;
use Drupal\migrate\MigrateSkipRowException;
use Drupal\migrate\MigrateStubInterface;
use Drupal\migrate\ProcessPluginBase;
use Drupal\migrate\Plugin\MigrationInterface;
use Drupal\migrate\MigrateExecutableInterface;
......@@ -106,18 +109,25 @@
class MigrationLookup extends ProcessPluginBase implements ContainerFactoryPluginInterface {
/**
* The migration plugin manager.
* The migration to be executed.
*
* @var \Drupal\migrate\Plugin\MigrationPluginManagerInterface
* @var \Drupal\migrate\Plugin\MigrationInterface
*/
protected $migrationPluginManager;
protected $migration;
/**
* The migration to be executed.
* The migrate lookup service.
*
* @var \Drupal\migrate\Plugin\MigrationInterface
* @var \Drupal\migrate\MigrateLookupInterface
*/
protected $migration;
protected $migrateLookup;
/**
* The migrate stub service.
*
* @var \Drupal\migrate\MigrateStubInterface
*/
protected $migrateStub;
/**
* Constructs a MigrationLookup object.
......@@ -130,13 +140,26 @@ class MigrationLookup extends ProcessPluginBase implements ContainerFactoryPlugi
* The plugin implementation definition.
* @param \Drupal\migrate\Plugin\MigrationInterface $migration
* The Migration the plugin is being used in.
* @param \Drupal\migrate\Plugin\MigrationPluginManagerInterface $migration_plugin_manager
* The Migration Plugin Manager Interface.
* @param \Drupal\migrate\MigrateLookupInterface $migrate_lookup
* The migrate lookup service.
* @param \Drupal\migrate\MigrateStubInterface $migrate_stub
* The migrate stub service.
*/
public function __construct(array $configuration, $plugin_id, $plugin_definition, MigrationInterface $migration, MigrationPluginManagerInterface $migration_plugin_manager) {
// @codingStandardsIgnoreLine
public function __construct(array $configuration, $plugin_id, $plugin_definition, MigrationInterface $migration, $migrate_lookup, $migrate_stub = NULL) {
parent::__construct($configuration, $plugin_id, $plugin_definition);
$this->migrationPluginManager = $migration_plugin_manager;
if (!$migrate_lookup instanceof MigrateLookupInterface) {
@trigger_error('Not passing the migrate lookup service as the fifth parameter to ' . __METHOD__ . ' is deprecated in drupal:8.8.0 and will throw a type error in drupal:9.0.0. Pass an instance of \\Drupal\\migrate\\MigrateLookupInterface. See https://www.drupal.org/node/3047268', E_USER_DEPRECATED);
$migrate_lookup = \Drupal::service('migrate.lookup');
}
if (!$migrate_stub instanceof MigrateStubInterface) {
@trigger_error('Not passing the migrate stub service as the sixth parameter to ' . __METHOD__ . ' is deprecated in drupal:8.8.0 and will throw a type error in drupal:9.0.0. Pass an instance of \\Drupal\\migrate\\MigrateStubInterface. See https://www.drupal.org/node/3047268', E_USER_DEPRECATED);
$migrate_stub = \Drupal::service('migrate.stub');
}
$this->migration = $migration;
$this->migrateLookup = $migrate_lookup;
$this->migrateStub = $migrate_stub;
}
/**
......@@ -148,7 +171,8 @@ public static function create(ContainerInterface $container, array $configuratio
$plugin_id,
$plugin_definition,
$migration,
$container->get('plugin.manager.migration')
$container->get('migrate.lookup'),
$container->get('migrate.stub')
);
}
......@@ -156,20 +180,14 @@ public static function create(ContainerInterface $container, array $configuratio
* {@inheritdoc}
*
* @throws \Drupal\migrate\MigrateSkipProcessException
* @throws \Drupal\migrate\MigrateException
*/
public function transform($value, MigrateExecutableInterface $migrate_executable, Row $row, $destination_property) {
$lookup_migrations_ids = $this->configuration['migration'];
if (!is_array($lookup_migrations_ids)) {
$lookup_migrations_ids = [$lookup_migrations_ids];
}
$lookup_migration_ids = (array) $this->configuration['migration'];
$self = FALSE;
/** @var \Drupal\migrate\Plugin\MigrationInterface[] $lookup_migrations */
$destination_ids = NULL;
$source_id_values = [];
$lookup_migrations = $this->migrationPluginManager->createInstances($lookup_migrations_ids);
foreach ($lookup_migrations as $lookup_migration_id => $lookup_migration) {
foreach ($lookup_migration_ids as $lookup_migration_id) {
if ($lookup_migration_id == $this->migration->id()) {
$self = TRUE;
}
......@@ -181,10 +199,24 @@ public function transform($value, MigrateExecutableInterface $migrate_executable
}
$this->skipInvalid($value);
$source_id_values[$lookup_migration_id] = $value;
// Break out of the loop as soon as a destination ID is found.
$lookup = $lookup_migration->getIdMap()->lookupDestinationIds($source_id_values[$lookup_migration_id]);
if (!empty($lookup)) {
$destination_ids = $lookup[0];
// Re-throw any PluginException as a MigrateException so the executable
// can shut down the migration.
try {
$destination_id_array = $this->migrateLookup->lookup($lookup_migration_id, $value);
}
catch (PluginNotFoundException $e) {
$destination_id_array = [];
}
catch (MigrateException $e) {
throw $e;
}
catch (\Exception $e) {
throw new MigrateException(sprintf('A %s was thrown while processing this migration lookup', gettype($e)), $e->getCode(), $e);
}
if ($destination_id_array) {
$destination_ids = array_values(reset($destination_id_array));
break;
}
}
......@@ -193,45 +225,35 @@ public function transform($value, MigrateExecutableInterface $migrate_executable
return NULL;
}
if (!$destination_ids && ($self || isset($this->configuration['stub_id']) || count($lookup_migrations) == 1)) {
if (!$destination_ids && ($self || isset($this->configuration['stub_id']) || count($lookup_migration_ids) == 1)) {
// If the lookup didn't succeed, figure out which migration will do the
// stubbing.
if ($self) {
$stub_migration = $this->migration;
$stub_migration = $this->migration->id();
}
elseif (isset($this->configuration['stub_id'])) {
$stub_migration = $lookup_migrations[$this->configuration['stub_id']];
$stub_migration = $this->configuration['stub_id'];
}
else {
$stub_migration = reset($lookup_migrations);
}
$destination_plugin = $stub_migration->getDestinationPlugin(TRUE);
// Only keep the process necessary to produce the destination ID.
$process = $stub_migration->getProcess();
// We already have the source ID values but need to key them for the Row
// constructor.
$source_ids = $stub_migration->getSourcePlugin()->getIds();
$values = [];
foreach (array_keys($source_ids) as $index => $source_id) {
$values[$source_id] = $source_id_values[$stub_migration->id()][$index];
$stub_migration = reset($lookup_migration_ids);
}
$stub_row = $this->createStubRow($values + $stub_migration->getSourceConfiguration(), $source_ids);
// Do a normal migration with the stub row.
$migrate_executable->processRow($stub_row, $process);
$destination_ids = [];
$id_map = $stub_migration->getIdMap();
// Rethrow any exception as a MigrateException so the executable can shut
// down the migration.
try {
$destination_ids = $destination_plugin->import($stub_row);
$destination_ids = $this->migrateStub->createStub($stub_migration, $source_id_values[$stub_migration], [], FALSE);
}
catch (\Exception $e) {
$id_map->saveMessage($stub_row->getSourceIdValues(), $e->getMessage());
catch (\LogicException $e) {
// For BC reasons, we must allow attempting to stub a derived migration.
$destination_ids = [];
}
if ($destination_ids) {
$id_map->saveIdMapping($stub_row, $destination_ids, MigrateIdMapInterface::STATUS_NEEDS_UPDATE);
catch (MigrateException $e) {
throw $e;
}
catch (MigrateSkipRowException $e) {
throw $e;
}
catch (\Exception $e) {
throw new MigrateException(sprintf('A(n) %s was thrown while attempting to stub.', gettype($e)), $e->getCode(), $e);
}
}
if ($destination_ids) {
......@@ -287,8 +309,14 @@ protected function isValid($value) {
*
* @return \Drupal\migrate\Row
* The stub row.
*
* @deprecated in drupal:8.8.0 and is removed from drupal:9.0.0. Use the
* migrate.stub service to create stubs.
*
* @see https://www.drupal.org/node/3047268
*/
protected function createStubRow(array $values, array $source_ids) {
@trigger_error(__METHOD__ . ' is deprecated in drupal:8.8.0 and is removed from drupal:9.0.0. Use the migrate.stub service to create stubs. See https://www.drupal.org/node/3047268', E_USER_DEPRECATED);
return new Row($values, $source_ids, TRUE);
}
......
name: 'Migration Lookup Test'
type: module
description: 'Provides test migrations to test migration lookup service.'
package: Testing
version: VERSION
core: 8.x
dependencies:
- drupal:migrate
id: sample_lookup_migration
label: Sample Lookup Migration
source:
plugin: embedded_data
data_rows:
- id: 17
nid: 1
title: Node 1
- id: 25
nid: 2
title: Node 2
ids:
id:
type: integer
process:
nid: nid
title: title
destination:
default_bundle: node_lookup
plugin: entity:node
id: sample_lookup_migration_2
label: Sample Lookup Migration
source:
plugin: embedded_data
data_rows:
- id: 27
nid: 3
title: Node 3
- id: 35
nid: 4
title: Node 4
ids:
id:
type: integer
process:
nid: nid
title: title
destination:
default_bundle: node_lookup
plugin: entity:node
id: sample_lookup_migration_multiple_source_ids
label: "Sample Lookup Migration With Multiple Source Ids."
source:
plugin: embedded_data
data_rows:
- id: 17
version_id: 17
nid: 1
title: "Node 1"
- id: 25
version_id: 25
nid: 2
title: "Node 2"
- id: 25
version_id: 26
nid: 3
title: "Node 3"
ids:
id:
type: integer
version_id:
type: integer
process:
nid: nid
title: title
destination:
default_bundle: node_lookup
plugin: entity:node
id: sample_lookup_migration_string_ids
label: Sample Lookup Migration with string ids
source:
plugin: embedded_data
data_rows:
- id: node1
nid: 10
title: Node 10
- id: node2
nid: 11
title: Node 11
ids:
id:
type: string
process:
nid: nid
title: title
destination:
default_bundle: node_lookup
plugin: entity:node
name: 'Migration Stub Test'
type: module
description: 'Provides test migrations to test migration stub service.'
package: Testing
version: VERSION
core: 8.x
dependencies:
- drupal:migrate
id: sample_stubbing_migration
label: "Sample Stubbing Migration"
source: