Issue #2746671 by Jo Fitzgerald, stella, andrewmacpherson, hussainweb, tom...

Issue #2746671 by Jo Fitzgerald, stella, andrewmacpherson, hussainweb, tom friedhof, chriscalip: CCK field data not available for D7 taxonomy term migrations

core/modules/field/migration_templates/d7_field_instance.yml

@@ -36,3 +36,4 @@ migration_dependencies:
   optional:
     - d7_node_type
     - d7_comment_type
+    - d7_taxonomy_vocabulary

core/modules/migrate/src/Plugin/migrate/process/Iterator.php

@@ -22,15 +22,17 @@ class Iterator extends ProcessPluginBase {
    * Runs a process pipeline on each destination property per list item.
    */
   public function transform($value, MigrateExecutableInterface $migrate_executable, Row $row, $destination_property) {
-    $return = array();
-    foreach ($value as $key => $new_value) {
-      $new_row = new Row($new_value, array());
-      $migrate_executable->processRow($new_row, $this->configuration['process']);
-      $destination = $new_row->getDestination();
-      if (array_key_exists('key', $this->configuration)) {
-        $key = $this->transformKey($key, $migrate_executable, $new_row);
+    $return = [];
+    if (!is_null($value)) {
+      foreach ($value as $key => $new_value) {
+        $new_row = new Row($new_value, []);
+        $migrate_executable->processRow($new_row, $this->configuration['process']);
+        $destination = $new_row->getDestination();
+        if (array_key_exists('key', $this->configuration)) {
+          $key = $this->transformKey($key, $migrate_executable, $new_row);
+        }
+        $return[$key] = $destination;
       }
-      $return[$key] = $destination;
     }
     return $return;
   }

core/modules/migrate_drupal/tests/fixtures/drupal7.php

@@ -3828,6 +3828,15 @@
   'data' => 'a:7:{s:5:"label";s:21:"Term Entity Reference";s:6:"widget";a:5:{s:6:"weight";s:2:"18";s:4:"type";s:33:"entityreference_autocomplete_tags";s:6:"module";s:15:"entityreference";s:6:"active";i:1;s:8:"settings";a:3:{s:14:"match_operator";s:8:"CONTAINS";s:4:"size";s:2:"60";s:4:"path";s:0:"";}}s:8:"settings";a:2:{s:9:"behaviors";a:1:{s:14:"taxonomy-index";a:1:{s:6:"status";b:1;}}s:18:"user_register_form";b:0;}s:7:"display";a:1:{s:7:"default";a:5:{s:5:"label";s:5:"above";s:4:"type";s:21:"entityreference_label";s:8:"settings";a:1:{s:4:"link";b:0;}s:6:"module";s:15:"entityreference";s:6:"weight";i:17;}}s:8:"required";i:0;s:11:"description";s:0:"";s:13:"default_value";N;}',
   'deleted' => '0',
 ))
+->values(array(
+  'id' => '41',
+  'field_id' => '20',
+  'field_name' => 'field_term_reference',
+  'entity_type' => 'taxonomy_term',
+  'bundle' => 'test_vocabulary',
+  'data' => 'a:7:{s:5:"label";s:14:"Term Reference";s:6:"widget";a:5:{s:6:"weight";s:2:"14";s:4:"type";s:21:"taxonomy_autocomplete";s:6:"module";s:8:"taxonomy";s:6:"active";i:0;s:8:"settings";a:2:{s:4:"size";i:60;s:17:"autocomplete_path";s:21:"taxonomy/autocomplete";}}s:8:"settings";a:1:{s:18:"user_register_form";b:0;}s:7:"display";a:1:{s:7:"default";a:4:{s:5:"label";s:5:"above";s:4:"type";s:6:"hidden";s:6:"weight";s:2:"13";s:8:"settings";a:0:{}}}s:8:"required";i:0;s:11:"description";s:0:"";s:13:"default_value";N;}',
+  'deleted' => '0',
+))
 ->execute();
 $connection->schema()->createTable('field_data_body', array(
@@ -4850,6 +4859,16 @@
   'delta' => '0',
   'field_integer_value' => '99',
 ))
+->values(array(
+  'entity_type' => 'taxonomy_term',
+  'bundle' => 'test_vocabulary',
+  'deleted' => '0',
+  'entity_id' => '4',
+  'revision_id' => '4',
+  'language' => 'und',
+  'delta' => '0',
+  'field_integer_value' => '6',
+))
 ->execute();
 $connection->schema()->createTable('field_data_field_integer_list', array(
@@ -5649,6 +5668,16 @@
   'delta' => '0',
   'field_term_reference_tid' => '4',
 ))
+->values(array(
+  'entity_type' => 'taxonomy_term',
+  'bundle' => 'test_vocabulary',
+  'deleted' => '0',
+  'entity_id' => '2',
+  'revision_id' => '2',
+  'language' => 'und',
+  'delta' => '0',
+  'field_term_reference_tid' => '3',
+))
 ->execute();
 $connection->schema()->createTable('field_data_field_text', array(
@@ -7023,6 +7052,16 @@
   'delta' => '0',
   'field_integer_value' => '99',
 ))
+->values(array(
+  'entity_type' => 'taxonomy_term',
+  'bundle' => 'test_vocabulary',
+  'deleted' => '0',
+  'entity_id' => '4',
+  'revision_id' => '4',
+  'language' => 'und',
+  'delta' => '0',
+  'field_integer_value' => '6',
+))
 ->execute();
 $connection->schema()->createTable('field_revision_field_integer_list', array(
@@ -7830,6 +7869,16 @@
   'delta' => '0',
   'field_term_reference_tid' => '4',
 ))
+->values(array(
+  'entity_type' => 'taxonomy_term',
+  'bundle' => 'test_vocabulary',
+  'deleted' => '0',
+  'entity_id' => '2',
+  'revision_id' => '2',
+  'language' => 'und',
+  'delta' => '0',
+  'field_term_reference_tid' => '3',
+))
 ->execute();
 $connection->schema()->createTable('field_revision_field_text', array(

core/modules/migrate_drupal_ui/src/Tests/d7/MigrateUpgrade7Test.php

@@ -43,8 +43,8 @@ protected function getEntityCounts() {
       'configurable_language' => 4,
       'contact_form' => 3,
       'editor' => 2,
-      'field_config' => 48,
-      'field_storage_config' => 36,
+      'field_config' => 49,
+      'field_storage_config' => 37,
       'file' => 1,
       'filter_format' => 7,
       'image_style' => 6,

core/modules/taxonomy/migration_templates/d6_taxonomy_term.yml

@@ -3,7 +3,7 @@ label: Taxonomy terms
 migration_tags:
   - Drupal 6
 source:
-  plugin: taxonomy_term
+  plugin: d6_taxonomy_term
 process:
   # If you are using this file to build a custom migration consider removing
   # the tid field to allow incremental migrations.

core/modules/taxonomy/migration_templates/d7_taxonomy_term.yml

@@ -2,8 +2,9 @@ id: d7_taxonomy_term
 label: Taxonomy terms
 migration_tags:
   - Drupal 7
+deriver: Drupal\taxonomy\Plugin\migrate\D7TaxonomyTermDeriver
 source:
-  plugin: taxonomy_term
+  plugin: d7_taxonomy_term
 process:
   # If you are using this file to build a custom migration consider removing
   # the tid field to allow incremental migrations.
@@ -35,3 +36,5 @@ destination:
 migration_dependencies:
   required:
     - d7_taxonomy_vocabulary
+  optional:
+    - d7_field_instance

core/modules/taxonomy/src/Plugin/migrate/D7TaxonomyTermDeriver.php

+<?php
+
+namespace Drupal\taxonomy\Plugin\migrate;
+
+use Drupal\Component\Plugin\Derivative\DeriverBase;
+use Drupal\Component\Plugin\Exception\PluginNotFoundException;
+use Drupal\Core\Database\DatabaseExceptionWrapper;
+use Drupal\Core\Plugin\Discovery\ContainerDeriverInterface;
+use Drupal\migrate\Exception\RequirementsException;
+use Drupal\migrate\Plugin\MigrationDeriverTrait;
+use Drupal\migrate_drupal\Plugin\MigrateCckFieldPluginManagerInterface;
+use Symfony\Component\DependencyInjection\ContainerInterface;
+
+/**
+ * Deriver for Drupal 7 taxonomy term migrations based on vocabularies.
+ */
+class D7TaxonomyTermDeriver extends DeriverBase implements ContainerDeriverInterface {
+  use MigrationDeriverTrait;
+
+  /**
+   * The base plugin ID this derivative is for.
+   *
+   * @var string
+   */
+  protected $basePluginId;
+
+  /**
+   * Already-instantiated cckfield plugins, keyed by ID.
+   *
+   * @var \Drupal\migrate_drupal\Plugin\MigrateCckFieldInterface[]
+   */
+  protected $cckPluginCache;
+
+  /**
+   * The CCK plugin manager.
+   *
+   * @var \Drupal\migrate_drupal\Plugin\MigrateCckFieldPluginManagerInterface
+   */
+  protected $cckPluginManager;
+
+  /**
+   * D7TaxonomyTermDeriver constructor.
+   *
+   * @param string $base_plugin_id
+   *   The base plugin ID for the plugin ID.
+   * @param \Drupal\migrate_drupal\Plugin\MigrateCckFieldPluginManagerInterface $cck_manager
+   *   The CCK plugin manager.
+   */
+  public function __construct($base_plugin_id, MigrateCckFieldPluginManagerInterface $cck_manager) {
+    $this->basePluginId = $base_plugin_id;
+    $this->cckPluginManager = $cck_manager;
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public static function create(ContainerInterface $container, $base_plugin_id) {
+    return new static(
+      $base_plugin_id,
+      $container->get('plugin.manager.migrate.cckfield')
+    );
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public function getDerivativeDefinitions($base_plugin_definition) {
+    $fields = [];
+    try {
+      $source_plugin = static::getSourcePlugin('d7_field_instance');
+      $source_plugin->checkRequirements();
+
+      // Read all field instance definitions in the source database.
+      foreach ($source_plugin as $row) {
+        if ($row->getSourceProperty('entity_type') == 'taxonomy_term') {
+          $fields[$row->getSourceProperty('bundle')][$row->getSourceProperty('field_name')] = $row->getSource();
+        }
+      }
+    }
+    catch (RequirementsException $e) {
+      // If checkRequirements() failed then the field module did not exist and
+      // we do not have any fields. Therefore, $fields will be empty and below
+      // we'll create a migration just for the node properties.
+    }
+
+    try {
+      foreach (static::getSourcePlugin('d7_taxonomy_vocabulary') as $row) {
+        $bundle = $row->getSourceProperty('machine_name');
+        $values = $base_plugin_definition;
+
+        $values['label'] = t('@label (@type)', [
+          '@label' => $values['label'],
+          '@type' => $row->getSourceProperty('name'),
+        ]);
+        $values['source']['bundle'] = $bundle;
+        $values['destination']['default_bundle'] = $bundle;
+
+        /** @var Migration $migration */
+        $migration = \Drupal::service('plugin.manager.migration')->createStubMigration($values);
+        if (isset($fields[$bundle])) {
+          foreach ($fields[$bundle] as $field_name => $info) {
+            $field_type = $info['type'];
+            try {
+              $plugin_id = $this->cckPluginManager->getPluginIdFromFieldType($field_type, ['core' => 7], $migration);
+              if (!isset($this->cckPluginCache[$field_type])) {
+                $this->cckPluginCache[$field_type] = $this->cckPluginManager->createInstance($plugin_id, ['core' => 7], $migration);
+              }
+              $this->cckPluginCache[$field_type]
+                ->processCckFieldValues($migration, $field_name, $info);
+            }
+            catch (PluginNotFoundException $ex) {
+              $migration->setProcessOfProperty($field_name, $field_name);
+            }
+          }
+        }
+        $this->derivatives[$bundle] = $migration->getPluginDefinition();
+      }
+    }
+    catch (DatabaseExceptionWrapper $e) {
+      // Once we begin iterating the source plugin it is possible that the
+      // source tables will not exist. This can happen when the
+      // MigrationPluginManager gathers up the migration definitions but we do
+      // not actually have a Drupal 7 source database.
+    }
+
+    return $this->derivatives;
+  }
+
+}

core/modules/taxonomy/src/Plugin/migrate/source/Term.php

@@ -14,6 +14,10 @@
  *   id = "taxonomy_term",
  *   source_provider = "taxonomy"
  * )
+ *
+ * @deprecated in Drupal 8.3.0, intended to be removed in Drupal 9.0.0.
+ *   Use \Drupal\taxonomy\Plugin\migrate\source\d6\Term or
+ *   \Drupal\taxonomy\Plugin\migrate\source\d7\Term.
  */
 class Term extends DrupalSqlBase {
 

core/modules/taxonomy/src/Plugin/migrate/source/d6/Term.php

+<?php
+
+namespace Drupal\taxonomy\Plugin\migrate\source\d6;
+
+use Drupal\migrate\Row;
+use Drupal\migrate_drupal\Plugin\migrate\source\DrupalSqlBase;
+
+/**
+ * Taxonomy term source from database.
+ *
+ * @todo Support term_relation, term_synonym table if possible.
+ *
+ * @MigrateSource(
+ *   id = "d6_taxonomy_term",
+ *   source_provider = "taxonomy"
+ * )
+ */
+class Term extends DrupalSqlBase {
+
+  /**
+   * {@inheritdoc}
+   */
+  public function query() {
+    $query = $this->select('term_data', 'td')
+      ->fields('td')
+      ->distinct()
+      ->orderBy('td.tid');
+
+    if (isset($this->configuration['bundle'])) {
+      $query->condition('td.vid', $this->configuration['bundle'], 'IN');
+    }
+
+    return $query;
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public function fields() {
+    $fields = [
+      'tid' => $this->t('The term ID.'),
+      'vid' => $this->t('Existing term VID'),
+      'name' => $this->t('The name of the term.'),
+      'description' => $this->t('The term description.'),
+      'weight' => $this->t('Weight'),
+      'parent' => $this->t("The Drupal term IDs of the term's parents."),
+    ];
+    return $fields;
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public function prepareRow(Row $row) {
+    // Find parents for this row.
+    $parents = $this->select('term_hierarchy', 'th')
+      ->fields('th', ['parent', 'tid'])
+      ->condition('tid', $row->getSourceProperty('tid'))
+      ->execute()
+      ->fetchCol();
+    $row->setSourceProperty('parent', $parents);
+
+    return parent::prepareRow($row);
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public function getIds() {
+    $ids['tid']['type'] = 'integer';
+    return $ids;
+  }
+
+}

core/modules/taxonomy/src/Plugin/migrate/source/d7/Term.php

+<?php
+
+namespace Drupal\taxonomy\Plugin\migrate\source\d7;
+
+use Drupal\migrate\Row;
+use Drupal\migrate_drupal\Plugin\migrate\source\d7\FieldableEntity;
+
+/**
+ * Taxonomy term source from database.
+ *
+ * @todo Support term_relation, term_synonym table if possible.
+ *
+ * @MigrateSource(
+ *   id = "d7_taxonomy_term",
+ *   source_provider = "taxonomy"
+ * )
+ */
+class Term extends FieldableEntity {
+
+  /**
+   * {@inheritdoc}
+   */
+  public function query() {
+    $query = $this->select('taxonomy_term_data', 'td')
+      ->fields('td')
+      ->distinct()
+      ->orderBy('tid');
+    $query->leftJoin('taxonomy_vocabulary', 'tv', 'td.vid = tv.vid');
+    $query->addField('tv', 'machine_name');
+
+    if (isset($this->configuration['bundle'])) {
+      $query->condition('tv.machine_name', $this->configuration['bundle'], 'IN');
+    }
+
+    return $query;
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public function fields() {
+    $fields = [
+      'tid' => $this->t('The term ID.'),
+      'vid' => $this->t('Existing term VID'),
+      'machine_name' => $this->t('Vocabulary machine name'),
+      'name' => $this->t('The name of the term.'),
+      'description' => $this->t('The term description.'),
+      'weight' => $this->t('Weight'),
+      'parent' => $this->t("The Drupal term IDs of the term's parents."),
+      'format' => $this->t("Format of the term description."),
+    ];
+    return $fields;
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public function prepareRow(Row $row) {
+    // Get Field API field values.
+    foreach (array_keys($this->getFields('taxonomy_term', $row->getSourceProperty('machine_name'))) as $field) {
+      $tid = $row->getSourceProperty('tid');
+      $row->setSourceProperty($field, $this->getFieldValues('taxonomy_term', $field, $tid));
+    }
+
+    // Find parents for this row.
+    $parents = $this->select('taxonomy_term_hierarchy', 'th')
+      ->fields('th', ['parent', 'tid'])
+      ->condition('tid', $row->getSourceProperty('tid'))
+      ->execute()
+      ->fetchCol();
+    $row->setSourceProperty('parent', $parents);
+
+    return parent::prepareRow($row);
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public function getIds() {
+    $ids['tid']['type'] = 'integer';
+    return $ids;
+  }
+
+}

core/modules/taxonomy/tests/src/Kernel/Migrate/d7/MigrateTaxonomyTermTest.php

@@ -13,7 +13,16 @@
  */
 class MigrateTaxonomyTermTest extends MigrateDrupal7TestBase {
 
-  public static $modules = array('taxonomy', 'text');
+  public static $modules = [
+    'comment',
+    'datetime',
+    'image',
+    'link',
+    'node',
+    'taxonomy',
+    'telephone',
+    'text',
+  ];
 
   /**
    * The cached taxonomy tree items, keyed by vid and tid.
@@ -28,7 +37,16 @@ class MigrateTaxonomyTermTest extends MigrateDrupal7TestBase {
   protected function setUp() {
     parent::setUp();
     $this->installEntitySchema('taxonomy_term');
-    $this->executeMigrations(['d7_taxonomy_vocabulary', 'd7_taxonomy_term']);
+    $this->installConfig(static::$modules);
+
+    $this->executeMigrations([
+      'd7_node_type',
+      'd7_comment_type',
+      'd7_field',
+      'd7_taxonomy_vocabulary',
+      'd7_field_instance',
+      'd7_taxonomy_term'
+    ]);
   }
 
   /**
@@ -48,8 +66,12 @@ protected function setUp() {
    *   The weight the migrated entity should have.
    * @param array $expected_parents
    *   The parent terms the migrated entity should have.
+   * @param int $expected_field_integer_value
+   *   The value the migrated entity field should have.
+   * @param int $expected_term_reference_tid
+   *   The term reference id the migrated entity field should have.
    */
-  protected function assertEntity($id, $expected_label, $expected_vid, $expected_description = '', $expected_format = NULL, $expected_weight = 0, $expected_parents = []) {
+  protected function assertEntity($id, $expected_label, $expected_vid, $expected_description = '', $expected_format = NULL, $expected_weight = 0, $expected_parents = [], $expected_field_integer_value = NULL, $expected_term_reference_tid = NULL) {
     /** @var \Drupal\taxonomy\TermInterface $entity */
     $entity = Term::load($id);
     $this->assertTrue($entity instanceof TermInterface);
@@ -60,6 +82,14 @@ protected function assertEntity($id, $expected_label, $expected_vid, $expected_d
     $this->assertEqual($expected_weight, $entity->getWeight());
     $this->assertIdentical($expected_parents, $this->getParentIDs($id));
     $this->assertHierarchy($expected_vid, $id, $expected_parents);
+    if (!is_null($expected_field_integer_value)) {
+      $this->assertTrue($entity->hasField('field_integer'));
+      $this->assertEquals($expected_field_integer_value, $entity->field_integer->value);
+    }
+    if (!is_null($expected_term_reference_tid)) {
+      $this->assertTrue($entity->hasField('field_integer'));
+      $this->assertEquals($expected_term_reference_tid, $entity->field_term_reference->target_id);
+    }
   }
 
   /**
@@ -67,9 +97,9 @@ protected function assertEntity($id, $expected_label, $expected_vid, $expected_d
    */
   public function testTaxonomyTerms() {
     $this->assertEntity(1, 'General discussion', 'forums', '', NULL, 2);
-    $this->assertEntity(2, 'Term1', 'test_vocabulary', 'The first term.', 'filtered_html');
+    $this->assertEntity(2, 'Term1', 'test_vocabulary', 'The first term.', 'filtered_html', 0, [], NULL, 3);
     $this->assertEntity(3, 'Term2', 'test_vocabulary', 'The second term.', 'filtered_html');
-    $this->assertEntity(4, 'Term3', 'test_vocabulary', 'The third term.', 'full_html', 0, [3]);
+    $this->assertEntity(4, 'Term3', 'test_vocabulary', 'The third term.', 'full_html', 0, [3], 6);
     $this->assertEntity(5, 'Custom Forum', 'forums', 'Where the cool kids are.', NULL, 3);
     $this->assertEntity(6, 'Games', 'forums', '', NULL, 4);
     $this->assertEntity(7, 'Minecraft', 'forums', '', NULL, 1, [6]);

core/modules/taxonomy/tests/src/Kernel/Plugin/migrate/source/TermSourceWithVocabularyFilterTest.php → core/modules/taxonomy/tests/src/Kernel/Plugin/migrate/source/d6/TermSourceWithVocabularyFilterTest.php

 <?php
 
-namespace Drupal\Tests\taxonomy\Kernel\Plugin\migrate\source;
+namespace Drupal\Tests\taxonomy\Kernel\Plugin\migrate\source\d6;
 
 /**
  * Tests the taxonomy term source with vocabulary filter.
  *
- * @covers \Drupal\taxonomy\Plugin\migrate\source\Term
+ * @covers \Drupal\taxonomy\Plugin\migrate\source\d6\Term
  * @group taxonomy
  */
 class TermSourceWithVocabularyFilterTest extends TermTest {
@@ -47,7 +47,7 @@ public function providerSource() {
 
     // Set up source plugin configuration.
     $tests[0]['configuration'] = [
-      'vocabulary' => [5],
+      'bundle' => [5],
     ];
 
     return $tests;

core/modules/taxonomy/tests/src/Kernel/Plugin/migrate/source/TermTest.php → core/modules/taxonomy/tests/src/Kernel/Plugin/migrate/source/d6/TermTest.php

 <?php
 
-namespace Drupal\Tests\taxonomy\Kernel\Plugin\migrate\source;
+namespace Drupal\Tests\taxonomy\Kernel\Plugin\migrate\source\d6;
 
 use Drupal\Tests\migrate\Kernel\MigrateSqlSourceTestBase;
 
 /**
  * Tests taxonomy term source plugin.
  *
- * @covers \Drupal\taxonomy\Plugin\migrate\source\Term
+ * @covers \Drupal\taxonomy\Plugin\migrate\source\d6\Term
  * @group taxonomy
  */
 class TermTest extends MigrateSqlSourceTestBase {

core/modules/taxonomy/tests/src/Kernel/Plugin/migrate/source/d7/TermSourceWithVocabularyFilterTest.php

+<?php
+
+namespace Drupal\Tests\taxonomy\Kernel\Plugin\migrate\source\d7;
+
+/**
+ * Tests the taxonomy term source with vocabulary filter.
+ *
+ * @covers \Drupal\taxonomy\Plugin\migrate\source\d7\Term
+ * @group taxonomy
+ */
+class TermSourceWithVocabularyFilterTest extends TermTest {
+
+  /**
+   * {@inheritdoc}
+   */
+  public static $modules = ['taxonomy', 'migrate_drupal'];
+
+  /**
+   * {@inheritdoc}
+   */
+  public function providerSource() {