Commit 56f37d54 authored by Mike Ryan's avatar Mike Ryan

Issue #2623012 by mikeryan, KarenS, mbaynton: Implement interfaces and base...

Issue #2623012 by mikeryan, KarenS, mbaynton: Implement interfaces and base classes for URL-based sources
parent 2f33339f
......@@ -4,16 +4,26 @@ The migrate_example_advanced module demonstrates some techniques for Drupal 8
migrations beyond the basics in migrate_example. It includes a group of
migrations with a wine theme.
SETUP
-----
To demonstrate XML migrations as realistically as possible, the setup module
provides the source data as REST services. So the migrations' references to these
services can be set up accurately, if you install migrate_example_advanced via
drush be sure to use the --uri parameter, e.g.
drush en -y migrate_example_advanced --uri=http://d8.local:8083/
THE WINE SITE
-------------
In this scenario, we have a wine aficionado site which stores data in SQL tables
as well is pulling in additional data from XML files.
as well is pulling in additional data from XML services.
To make the example as simple as to run as possible, the SQL data is placed in
tables directly in your Drupal database - in most real-world scenarios, your
source data will be in an external database. The migrate_example_advanced_setup
submodule creates and populates these tables, as well as configuring your Drupal
8 site (creating node types, vocabularies, fields, etc.) to receive the data.
8 site (creating node types, vocabularies, fields, etc.) to receive the data,
and providing service endpoints for XML data.
STRUCTURE
---------
......@@ -43,4 +53,6 @@ Multiple vocabularies populated in one migration
------------------------------------------------
See migrate.migration.wine_terms.yml.
Importing from XML services
---------------------------
See migrate.migration.wine_role_xml.yml.
# This migration demonstrates importing from a monolithic JSON file.
id: wine_role_json
label: JSON feed of roles (positions)
third_party_settings:
migrate_plus:
migration_group: wine
source:
# We use the JSON source plugin.
plugin: url
data_fetcher_plugin: http
data_parser_plugin: json
# Normally, this is one or more fully-qualified URLs or file paths. Because
# we can't hardcode your local URL, we provide a relative path here which
# hook_install() will rewrite to a full URL for the current site.
urls: /migrate_example_advanced_position?_format=json
item_selector: 1
# Under 'fields', we list the data items to be imported. The first level keys
# are the source field names we want to populate (the names to be used as
# sources in the process configuration below). For each field we're importing,
# we provide a label (optional - this is for display in migration tools) and
# an xpath for retrieving that value. It's important to note that this xpath
# is relative to the elements retrieved by item_xpath.
fields:
-
name: machine_name
label: 'Unique position identifier'
selector: sourceid
-
name: friendly_name
label: 'Position name'
selector: name
# Under 'ids', we identify source fields populated above which will uniquely
# identify each imported item. The 'type' makes sure the migration map table
# uses the proper schema type for stored the IDs.
ids:
machine_name:
type: string
process:
# Note that the source field names here (machine_name and friendly_name) were
# defined by the 'fields' configuration for the source plugin above.
id: machine_name
label: friendly_name
destination:
plugin: entity:user_role
# This migration demonstrates importing from a monolithic XML file.
id: wine_role_xml
label: XML feed of roles (positions)
third_party_settings:
migrate_plus:
migration_group: wine
source:
# We use the XML data parser plugin.
plugin: url
data_fetcher_plugin: http
data_parser_plugin: xml
# Normally, this is one or more fully-qualified URLs or file paths. Because
# we can't hardcode your local URL, we provide a relative path here which
# hook_install() will rewrite to a full URL for the current site.
urls: /migrate_example_advanced_position?_format=xml
# Visit the URL above (relative to your site root) and look at it. You can see
# that <response> is the outer element, and each item we want to import is a
# <position> element. The item_xpath value is the xpath to use to query the
# desired elements.
item_selector: /response/position
# Under 'fields', we list the data items to be imported. The first level keys
# are the source field names we want to populate (the names to be used as
# sources in the process configuration below). For each field we're importing,
# we provide a label (optional - this is for display in migration tools) and
# an xpath for retrieving that value. It's important to note that this xpath
# is relative to the elements retrieved by item_xpath.
fields:
-
name: machine_name
label: 'Unique position identifier'
selector: sourceid
-
name: friendly_name
label: 'Position name'
selector: name
# Under 'ids', we identify source fields populated above which will uniquely
# identify each imported item. The 'type' makes sure the migration map table
# uses the proper schema type for stored the IDs.
ids:
machine_name:
type: string
process:
# Note that the source field names here (machine_name and friendly_name) were
# defined by the 'fields' configuration for the source plugin above.
id: machine_name
label: friendly_name
destination:
plugin: entity:user_role
# This migration demonstrates importing from an endpoint listing other endpoints
# containing individual item data.
id: wine_variety_list
label: XML feed of varieties
third_party_settings:
migrate_plus:
migration_group: wine
source:
# We use the XML source plugin.
plugin: xml
# Normally, this is one or more fully-qualified URLs or file paths. Because
# we can't hardcode your local URL, we provide a relative path here which
# hook_install() will rewrite to a full URL for the current site.
urls: /migrate_example_advanced_variety_list?_format=xml
item_url: /migrate_example_advanced_variety_list/:id?_format=xml
id_selector: /response/items
# Visit the URL above (relative to your site root) and look at it. You can see
# that <response> is the outer element, and each item we want to import is a
# <position> element. The item_xpath value is the xpath to use to query the
# desired elements.
item_selector: /response/variety
# Under 'fields', we list the data items to be imported. The first level keys
# are the source field names we want to populate (the names to be used as
# sources in the process configuration below). For each field we're importing,
# we provide a label (optional - this is for display in migration tools) and
# an xpath for retrieving that value. It's important to note that this xpath
# is relative to the elements retrieved by item_xpath.
fields:
category_name:
label:
selector: name
category_details:
label:
selector: details
category_parent:
label: 'Unique position identifier'
selector: parent
# Under 'ids', we identify source fields populated above which will uniquely
# identify each imported item. The 'type' makes sure the migration map table
# uses the proper schema type for stored the IDs.
ids:
category_name:
type: string
process:
vid:
plugin: default_value
default_value: migrate_example_wine_varieties
name: category_name
description: category_details
parent:
plugin: migration
migration: wine_terms
source: category_parent
destination:
plugin: entity:taxonomy_term
migration_dependencies:
require:
- wine_terms
# This migration demonstrates importing from multiple XML files.
id: wine_variety_multi_xml
label: XML feed of varieties
third_party_settings:
migrate_plus:
migration_group: wine
source:
# We use the XML source plugin.
plugin: url
data_fetcher_plugin: http
data_parser_plugin: xml
# Normally, this is one or more fully-qualified URLs or file paths. Because
# we can't hardcode your local URL, we provide a relative path here which
# hook_install() will rewrite to a full URL for the current site.
urls:
- /migrate_example_advanced_variety_multiple/red?_format=xml
- /migrate_example_advanced_variety_multiple/white?_format=xml
# Visit the URL above (relative to your site root) and look at it. You can see
# that <response> is the outer element, and each item we want to import is a
# <position> element. The item_xpath value is the xpath to use to query the
# desired elements.
item_selector: /response/variety
# Under 'fields', we list the data items to be imported. The first level keys
# are the source field names we want to populate (the names to be used as
# sources in the process configuration below). For each field we're importing,
# we provide a label (optional - this is for display in migration tools) and
# an xpath for retrieving that value. It's important to note that this xpath
# is relative to the elements retrieved by item_xpath.
fields:
-
name: category_name
label: Name
selector: name
-
name: category_details
label: Details
selector: details
-
name: category_parent
label: 'Unique position identifier'
selector: parent
# Under 'ids', we identify source fields populated above which will uniquely
# identify each imported item. The 'type' makes sure the migration map table
# uses the proper schema type for stored the IDs.
ids:
category_name:
type: string
process:
vid:
plugin: default_value
default_value: migrate_example_wine_varieties
name: category_name
description: category_details
parent:
plugin: migration
migration: wine_terms
source: category_parent
destination:
plugin: entity:taxonomy_term
migration_dependencies:
required:
- wine_terms
<?xml version="1.0" encoding="UTF-8"?>
<producer>
<name>Lolonis Winery</name>
<description>Makers of Ladybug Red</description>
<authorid>3</authorid>
<region>Redwood Valley</region>
</producer>
<?xml version="1.0" encoding="UTF-8"?>
<pr:producer xmlns:pr="http://www.wine.org/wine-producers">
<pr:name>Château Latour</pr:name>
<pr:description>Makers of grand vin Chateau Latour, Les Forts de Latour and Pauillac</pr:description>
<pr:authorid>3</pr:authorid>
<pr:region>Bordeaux</pr:region>
</pr:producer>
<?xml version="1.0" encoding="UTF-8"?>
<content>
<sourceid>0001</sourceid>
</content>
<?xml version="1.0" encoding="UTF-8"?>
<wn:content xmlns:wn="http://www.wine.org/wine">
<wn:sourceid>0002</wn:sourceid>
</wn:content>
......@@ -7,3 +7,5 @@ dependencies:
- migrate
- migrate_example_advanced_setup
- migrate_plus
- migrate_source_json
- migrate_source_xml
<?php
/**
* @file
* Install, update and uninstall functions for the migrate_example_advanced module.
*/
use Drupal\migrate\Entity\Migration;
/**
* Implements hook_install().
*/
function migrate_example_advanced_install() {
// We need the urls to be absolute for the XML source plugin to read them, but
// the static configuration files on disk can't know the server and port to
// use. So, in the .yml files we provide the REST resources relative to the
// site root and here rewrite them to fully-qualified paths.
/** @var \Drupal\migrate\Entity\MigrationInterface $wine_role_xml_migration */
$wine_role_xml_migration = Migration::load('wine_role_xml');
if ($wine_role_xml_migration) {
$source = $wine_role_xml_migration->get('source');
$request = \Drupal::request();
$source['urls'] = 'http://' . $request->getHttpHost() . $source['urls'];
$wine_role_xml_migration->set('source', $source);
$wine_role_xml_migration->save();
}
/** @var \Drupal\migrate\Entity\MigrationInterface $wine_role_json_migration */
$wine_role_json_migration = Migration::load('wine_role_json');
if ($wine_role_json_migration) {
$source = $wine_role_json_migration->get('source');
$request = \Drupal::request();
$source['urls'] = 'http://' . $request->getHttpHost() . $source['urls'];
$wine_role_json_migration->set('source', $source);
$wine_role_json_migration->save();
}
/** @var \Drupal\migrate\Entity\MigrationInterface $wine_variety_multi_xml_migration */
$wine_variety_multi_xml_migration = Migration::load('wine_variety_multi_xml');
if ($wine_variety_multi_xml_migration) {
$source = $wine_variety_multi_xml_migration->get('source');
$request = \Drupal::request();
$urls = [];
foreach ($source['urls'] as $url) {
$urls[] = 'http://' . $request->getHttpHost() . $url;
}
$source['urls'] = $urls;
$wine_variety_multi_xml_migration->set('source', $source);
$wine_variety_multi_xml_migration->save();
}
}
<?php
use Drupal\user\RoleInterface;
/**
* @file
......@@ -7,7 +8,10 @@
* a pure migration module.
*/
function migrate_example_advanced_schema() {
/**
* Implements hook_schema().
*/
function migrate_example_advanced_setup_schema() {
$schema['migrate_example_advanced_account'] = migrate_example_advanced_schema_account();
$schema['migrate_example_advanced_account_updates'] = migrate_example_advanced_schema_account_updates();
$schema['migrate_example_advanced_categories'] = migrate_example_advanced_schema_categories();
......@@ -28,7 +32,43 @@ function migrate_example_advanced_schema() {
return $schema;
}
function migrate_example_advanced_install() {
/**
* Implements hook_install().
*/
function migrate_example_advanced_setup_install() {
// Enable and configure REST resources providing source data.
$config = \Drupal::configFactory()->getEditable('rest.settings');
$resources = $config->get('resources');
$resources['migrate_example_advanced_position']['GET'] = [
'supported_formats' => ['json', 'xml'],
'supported_auth' => ['cookie'],
];
$resources['migrate_example_advanced_variety_multiple']['GET'] = [
'supported_formats' => ['json', 'xml'],
'supported_auth' => ['cookie'],
];
$resources['migrate_example_advanced_variety_list']['GET'] = [
'supported_formats' => ['json', 'xml'],
'supported_auth' => ['cookie'],
];
$resources['migrate_example_advanced_variety_items']['GET'] = [
'supported_formats' => ['json', 'xml'],
'supported_auth' => ['cookie'],
];
$config->set('resources', $resources);
$config->save();
// Don't require authentication for the services, so the migrations can easily
// be run from drush.
user_role_grant_permissions(RoleInterface::ANONYMOUS_ID,
['restful get migrate_example_advanced_position']);
user_role_grant_permissions(RoleInterface::ANONYMOUS_ID,
['restful get migrate_example_advanced_variety_multiple']);
user_role_grant_permissions(RoleInterface::ANONYMOUS_ID,
['restful get migrate_example_advanced_variety_list']);
user_role_grant_permissions(RoleInterface::ANONYMOUS_ID,
['restful get migrate_example_advanced_variety_items']);
// Populate our tables.
migrate_example_advanced_data_account();
migrate_example_advanced_data_account_updates();
......
<?php
/**
* @file
* Contains \Drupal\migrate_example_advanced_setup\Plugin\rest\resource\PositionResource.
*/
namespace Drupal\migrate_example_advanced_setup\Plugin\rest\resource;
use Drupal\rest\Plugin\ResourceBase;
use Drupal\rest\ResourceResponse;
/**
* Represents positions as resources.
*
* @RestResource(
* id = "migrate_example_advanced_position",
* label = @Translation("Advanced migration example - Position data"),
* uri_paths = {
* "canonical" = "/migrate_example_advanced_position"
* }
* )
*/
class PositionResource extends ResourceBase {
/**
* Responds to GET requests.
*
* @return \Drupal\rest\ResourceResponse
* The response containing the position data.
*/
public function get() {
$position1 = ['sourceid' => 'wine_taster', 'name' => 'Wine Taster'];
$position2 = ['sourceid' => 'vintner', 'name' => 'Vintner'];
$data = ['position' => [$position1, $position2]];
$response = new ResourceResponse($data, 200);
return $response;
}
}
<?php
/**
* @file
* Contains \Drupal\migrate_example_advanced_setup\Plugin\rest\resource\VarietyItems.
*/
namespace Drupal\migrate_example_advanced_setup\Plugin\rest\resource;
use Drupal\rest\Plugin\ResourceBase;
use Drupal\rest\ResourceResponse;
/**
* Provides varieties as two endpoints, one for reds and one for whites.
*
* @RestResource(
* id = "migrate_example_advanced_variety_items",
* label = @Translation("Advanced migration example - Variety data"),
* uri_paths = {
* "canonical" = "/migrate_example_advanced_variety_list/{variety}"
* }
* )
*/
class VarietyItems extends ResourceBase {
/**
* Responds to GET requests.
*
* @param string $variety
* Machine name of the variety to retrieve.
*
* @return \Drupal\rest\ResourceResponse
* The response containing the requested variety data.
*/
public function get($variety = NULL) {
$varieties = [
'retsina' => [
'name' => 'Retsina',
'parent' => 1, // categoryid for 'white'.
'details' => 'Greek',
],
'trebbiano' => [
'name' => 'Trebbiano',
'parent' => 1, // categoryid for 'white'.
'details' => 'Italian',
],
'valpolicella' => [
'name' => 'Valpolicella',
'parent' => 3, // categoryid for 'red'.
'details' => 'Italian Venoto region',
],
'bardolino' => [
'name' => 'Bardolino',
'parent' => 3, // categoryid for 'red'.
'details' => 'Italian Venoto region',
],
];
if (isset($varieties[$variety])) {
$data = ['variety' => $varieties[$variety]];
}
else {
$data = [];
}
$response = new ResourceResponse($data, 200);
return $response;
}
}
<?php
/**
* @file
* Contains \Drupal\migrate_example_advanced_setup\Plugin\rest\resource\VarietyList.
*/
namespace Drupal\migrate_example_advanced_setup\Plugin\rest\resource;
use Drupal\rest\Plugin\ResourceBase;
use Drupal\rest\ResourceResponse;
/**
* Provides varieties as two endpoints, one for reds and one for whites.
*
* @RestResource(
* id = "migrate_example_advanced_variety_list",
* label = @Translation("Advanced migration example - Variety list of data"),
* uri_paths = {
* "canonical" = "/migrate_example_advanced_variety_list"
* }
* )
*/
class VarietyList extends ResourceBase {
/**
* Responds to GET requests.
*
* @return \Drupal\rest\ResourceResponse
* The response containing the requested variety data.
*/
public function get() {
$data['items'] = ['retsina', 'trebbiano', 'valpolicella', 'bardolino'];
$response = new ResourceResponse($data, 200);
return $response;
}
}
<?php
/**
* @file
* Contains \Drupal\migrate_example_advanced_setup\Plugin\rest\resource\VarietyMultiFiles.
*/
namespace Drupal\migrate_example_advanced_setup\Plugin\rest\resource;
use Drupal\rest\Plugin\ResourceBase;
use Drupal\rest\ResourceResponse;
/**
* Provides varieties as two endpoints, one for reds and one for whites.
*
* @RestResource(
* id = "migrate_example_advanced_variety_multiple",
* label = @Translation("Advanced migration example - Variety data"),
* uri_paths = {
* "canonical" = "/migrate_example_advanced_variety_multiple/{type}"
* }
* )
*/
class VarietyMultiFiles extends ResourceBase {
/**
* Responds to GET requests.
*
* @param string $type
* 'red', 'white', or NULL to return all varieties.
*
* @return \Drupal\rest\ResourceResponse
* The response containing the requested variety data.
*/
public function get($type = NULL) {
$data = [];
if (strtolower($type) != 'white') {
$data['variety'][] = [
'name' => 'Amarone',
'parent' => 3, // categoryid for 'red'.
'details' => 'Italian Venoto region',