Merge branch '3104268-sync-id-too-strict2' into '6.0.x'

Resolve #3104268 "Sync id too strict2"

See merge request !74

migrate_tools.module

@@ -96,6 +96,6 @@ function migrate_tools_migrate_prepare_row(Row $row, MigrateSourceInterface $sou
     // Keep track of all source rows here, as SourcePluginBase::next() might
     // skip some rows, and we need them all to detect missing items in source to
     // delete in destination.
-    $migrateTools->addToSyncSourceIds($migration->getPluginId(), $row->getSourceIdValues());
+    $migrateTools->addToSyncSourceIds($migration->getPluginId(), $row->getSourceIdValues(), $source);
   }
 }

src/MigrateTools.php

@@ -6,6 +6,7 @@ namespace Drupal\migrate_tools;
 
 use Drupal\Core\Database\Connection;
 use Drupal\migrate\Plugin\MigrateIdMapInterface;
+use Drupal\migrate\Plugin\MigrateSourceInterface;
 use Drupal\migrate\Plugin\MigrationInterface;
 
 /**
@@ -94,11 +95,13 @@ class MigrateTools {
    *   Migration ID.
    * @param array $sourceIds
    *   A set of SyncSourceIds. Gets serialized to retain its structure.
+   * @param \Drupal\migrate\Plugin\MigrateSourceInterface $source
+   *   The migrate source.
    *
    * @throws \Exception
    */
-  public function addToSyncSourceIds(string $migrationId, array $sourceIds): void
-  {
+  public function addToSyncSourceIds(string $migrationId, array $sourceIds, MigrateSourceInterface $source): void {
+    $sourceIds = $this->prepareSourceIdValues($sourceIds, $source);
     $this->bufferedSyncIdsEntries[] = [
       'migration_id' => $migrationId,
       // Serialize source IDs before saving them to retain their structure.
@@ -109,6 +112,37 @@ class MigrateTools {
     }
   }
 
+  /**
+   * Ensure all source IDs match the expected type.
+   *
+   * @param array $rowSourceIds
+   *   The source IDs from the current row.
+   * @param \Drupal\migrate\Plugin\MigrateSourceInterface $source
+   *   The migrate source.
+   *
+   * @return array
+   *   The updated source values.
+   *
+   * @see migrate_tools_migrate_prepare_row()
+   * @see https://www.drupal.org/node/3104268
+   */
+  protected function prepareSourceIdValues(array $rowSourceIds, MigrateSourceInterface $source): array {
+    $sourceIds = $source->getIds();
+    foreach ($rowSourceIds as $key => $value) {
+      if (!array_key_exists($key, $sourceIds)) {
+        continue;
+      }
+
+      // Cast the source ID as the configured type to avoid rollback
+      // and recreations when an exact match is not found.
+      if (in_array($sourceIds[$key]['type'], ['string', 'integer'], TRUE)) {
+        $rowSourceIds[$key] = $sourceIds[$key]['type'] === 'string' ? (string) $rowSourceIds[$key] : (int) $rowSourceIds[$key];
+      }
+    }
+
+    return $rowSourceIds;
+  }
+
   /**
    * Flushes any pending SyncSourceIds to the database.
    *

tests/modules/migrate_tools_test/config/install/migrate_plus.migration.mixed_terms.yml

+langcode: en
+status: true
+id: mixed_terms
+label: Mixed Terms
+class: null
+field_plugin_method: null
+cck_plugin_method: null
+migration_tags: []
+migration_group: default
+source:
+  plugin: embedded_data
+  data_rows:
+    -
+      name: 1
+    -
+      name: '2'
+    -
+      name: Orange
+    -
+      name: 4.1
+  ids:
+    name:
+      type: string
+  constants:
+    vocabulary: fruit
+process:
+  name: name
+  vid: constants/vocabulary
+destination:
+  plugin: entity:taxonomy_term
+migration_dependencies:
+  required: []
+  optional: []
+dependencies:
+  enforced:
+    module:
+      - migrate_tools_test

tests/src/Functional/DrushCommandsTest.php

@@ -104,6 +104,16 @@ final class DrushCommandsTest extends BrowserTestBase {
         'message_count' => 0,
         'last_imported' => '',
       ],
+      [
+        'group' => 'Default (default)',
+        'id' => 'mixed_terms',
+        'imported' => 0,
+        'status' => 'Idle',
+        'total' => 4,
+        'unprocessed' => 4,
+        'message_count' => 0,
+        'last_imported' => '',
+      ],
       [
         'group' => 'Default (default)',
         'id' => 'source_exception',
@@ -222,4 +232,32 @@ EOT;
     $this->assertCount(3, $id_map);
   }
 
+  /**
+   * Tests synced import with and without update enforced.
+   */
+  public function testSyncMixedKeysImport(): void {
+    $this->drush('mim', ['mixed_terms']);
+    $this->assertStringContainsString('1/4', $this->getErrorOutput());
+    $this->assertStringContainsString('4/4', $this->getErrorOutput());
+    $this->assertMatchesRegularExpression('/4\/4[^\n]+\[notice\][^\n]+Processed 4 items \(4 created, 0 updated, 0 failed, 0 ignored\) - done with \'mixed_terms\'/', $this->getErrorOutput());
+    $term = \Drupal::entityTypeManager()->getStorage('taxonomy_term')->load(1);
+    $this->assertEquals('1', $term->label());
+    $term = \Drupal::entityTypeManager()->getStorage('taxonomy_term')->load(2);
+    $this->assertEquals('2', $term->label());
+    $source = $this->container->get('config.factory')->getEditable('migrate_plus.migration.mixed_terms')->get('source');
+    $source['data_rows'][] = ['name' => 'Grape'];
+    $this->container->get('config.factory')->getEditable('migrate_plus.migration.mixed_terms')->set('source', $source)->save();
+    // Flush cache so the recently changed migration can be refreshed.
+    drupal_flush_all_caches();
+    $this->drush('mim', ['mixed_terms'], ['sync' => NULL, 'update' => NULL]);
+    $this->assertStringContainsString('1/5', $this->getErrorOutput());
+    $this->assertMatchesRegularExpression('/5\/5[^\n]+\[notice\][^\n]+Processed 5 items \(1 created, 4 updated, 0 failed, 0 ignored\) - done with \'mixed_terms\'/', $this->getErrorOutput());
+    $terms = \Drupal::entityTypeManager()->getStorage('taxonomy_term')->loadMultiple();
+    $this->assertEquals(5, count($terms));
+
+    // If keys are converted correctly terms should not get deleted and
+    // recreated, so we should still have IDs 1 to 5.
+    $this->assertEquals(range(1, 5), array_keys($terms));
+  }
+
 }