Issue #3104105 by egruel, phma, PCate, GaëlG, oxyc, J_Drupal, heddn, vasike:...

Issue #3104105 by egruel, phma, PCate, GaëlG, oxyc, J_Drupal, heddn, vasike: sync option doesnt work with track_changes

migrate_tools.module

@@ -15,6 +15,10 @@ use Drupal\migrate_tools\Form\MigrationEditForm;
 use Drupal\migrate_tools\Form\MigrationGroupAddForm;
 use Drupal\migrate_tools\Form\MigrationGroupDeleteForm;
 use Drupal\migrate_tools\Form\MigrationGroupEditForm;
+use Drupal\migrate\Plugin\MigrateSourceInterface;
+use Drupal\migrate\Plugin\MigrationInterface;
+use Drupal\migrate\Row;
+use Drupal\migrate_tools\EventSubscriber\MigrationImportSync;
 
 /**
  * Implements hook_entity_type_build().
@@ -70,3 +74,24 @@ function migrate_tools_migration_plugins_alter(array &$migrations): void {
     \Drupal::service('migrate_tools.shared_config_include_handler')->include($migrations[$id]);
   }
 }
+
+/**
+ * Implements hook_migrate_prepare_row().
+ *
+ * @see \Drupal\migrate_tools\EventSubscriber\MigrationImportSync
+ * @see \Drupal\migrate\Plugin\migrate\source\SourcePluginBase::next()
+ */
+function migrate_tools_migrate_prepare_row(Row $row, MigrateSourceInterface $source, MigrationInterface $migration) {
+
+  if (!empty($migration->syncSource)) {
+    // Keep track of all source rows here, as SourcePluginBase::next() might
+    // skip some rows, and we need them all to detect missing items in source to
+    // delete in destination.
+
+    $source_id_values = \Drupal::state()->get('migrate_tools_sync', []);
+
+    $source_id_values[] = $row->getSourceIdValues();
+
+    \Drupal::state()->set('migrate_tools_sync', $source_id_values);
+  }
+}

migrate_tools.services.yml

@@ -19,6 +19,7 @@ services:
       - { name: event_subscriber }
     arguments:
       - '@event_dispatcher'
+      - '@state'
 
   plugin.manager.migrate_shared_config:
     class: Drupal\migrate_tools\MigrateSharedConfigPluginManager

src/EventSubscriber/MigrationImportSync.php

@@ -4,6 +4,7 @@ declare(strict_types = 1);
 
 namespace Drupal\migrate_tools\EventSubscriber;
 
+use Drupal\Core\State\StateInterface;
 use Drupal\migrate\Event\MigrateEvents;
 use Drupal\migrate\Event\MigrateImportEvent;
 use Drupal\migrate\Event\MigrateRollbackEvent;
@@ -20,14 +21,25 @@ class MigrationImportSync implements EventSubscriberInterface {
 
   protected EventDispatcherInterface $dispatcher;
 
+  /**
+   * The state key/value store.
+   *
+   * @var \Drupal\Core\State\StateInterface
+   */
+  protected StateInterface $state;
+
   /**
    * MigrationImportSync constructor.
    *
    * @param \Symfony\Component\EventDispatcher\EventDispatcherInterface $dispatcher
    *   The event dispatcher.
+   * @param Drupal\Core\State\StateInterface $state
+   *   The Key/Value Store to use for tracking synced source rows.
    */
-  public function __construct(EventDispatcherInterface $dispatcher) {
+  public function __construct(EventDispatcherInterface $dispatcher, StateInterface $state) {
     $this->dispatcher = $dispatcher;
+    $this->state = $state;
+    $this->state->set('migrate_tools_sync', []);
   }
 
   /**
@@ -48,21 +60,37 @@ class MigrationImportSync implements EventSubscriberInterface {
   public function sync(MigrateImportEvent $event): void {
     $migration = $event->getMigration();
     if (!empty($migration->syncSource)) {
-      $id_map = $migration->getIdMap();
-      $id_map->prepareUpdate();
+
+      // Loop through the source to register existing source ids.
+      // @see migrate_tools_migrate_prepare_row().
       // Clone so that any generators aren't initialized prematurely.
       $source = clone $migration->getSourcePlugin();
       $source->rewind();
-      $source_id_values = [];
+
       while ($source->valid()) {
-        $source_id_values[] = $source->current()->getSourceIdValues();
         $source->next();
       }
+
+      $source_id_values = $this->state->get('migrate_tools_sync', []);
+
+      $id_map = $migration->getIdMap();
       $id_map->rewind();
       $destination = $migration->getDestinationPlugin();
+
       while ($id_map->valid()) {
         $map_source_id = $id_map->currentSource();
+
+        foreach ($source->getIds() as $id_key => $id_config) {
+          if ($id_config['type'] === 'string') {
+            $map_source_id[$id_key] = (string) $map_source_id[$id_key];
+          }
+          elseif ($id_config['type'] === 'integer') {
+            $map_source_id[$id_key] = (int) $map_source_id[$id_key];
+          }
+        }
+
         $destination_ids = $id_map->currentDestination();
+
         if ($destination_ids !== NULL && $destination_ids !== [] && !in_array($map_source_id, $source_id_values, TRUE)) {
           $this->dispatchRowDeleteEvent(MigrateEvents::PRE_ROW_DELETE, $migration, $destination_ids);
           if (class_exists(MigratePlusEvents::class)) {

tests/src/Functional/DrushCommandsGeneratorTest.php

@@ -83,7 +83,7 @@ EOD;
     file_put_contents('public://test.csv', $this->sourceData);
 
     // Execute sync migration.
-    $this->drush('mim', ['csv_source_test'], ['sync' => NULL]);
+    $this->drush('mim', ['csv_source_test'], ['sync' => NULL, 'update' => NULL]);
     $this->assertStringContainsString('1/4', $this->getErrorOutput());
     $this->assertStringContainsString('25% [notice] Rolled back 1 item - done with \'csv_source_test\'', $this->getErrorOutput());
     $this->assertStringContainsString('4/4', $this->getErrorOutput());
@@ -94,6 +94,21 @@ EOD;
     drupal_flush_all_caches();
     $this->assertEmpty(\Drupal::entityTypeManager()->getStorage('taxonomy_vocabulary')->load('genre'));
 
+    // Remove one vocab and replace with another (reverse previous change).
+    $this->sourceData = str_replace('fruit,Fruit,Fruit description,1,0', 'genre,Genre,Genre description,1,0', $this->sourceData);
+    file_put_contents('public://test.csv', $this->sourceData);
+
+    // Execute sync migration without update enforced.
+    $this->drush('mim', ['csv_source_test'], ['sync' => NULL]);
+    $this->assertStringContainsString('1/4', $this->getErrorOutput());
+    $this->assertStringContainsString('25% [notice] Rolled back 1 item - done with \'csv_source_test\'', $this->getErrorOutput());
+    $this->assertStringNotContainsString('3 updated', $this->getErrorOutput());
+    $this->assertStringContainsString('[notice] Processed 1 item (1 created, 0 updated, 0 failed, 0 ignored) - done with \'csv_source_test\'', $this->getErrorOutput());
+    $this->assertEquals(4, \Drupal::entityTypeManager()->getStorage('taxonomy_vocabulary')->getQuery()->accessCheck(TRUE)->count()->execute());
+    // Flush cache so recently deleted vocabulary actually goes away.
+    drupal_flush_all_caches();
+    $this->assertEmpty(\Drupal::entityTypeManager()->getStorage('taxonomy_vocabulary')->load('fruit'));
+
     /** @var \Drupal\migrate\Plugin\MigrateIdMapInterface $id_map */
     $id_map = $this->container->get('plugin.manager.migration')->createInstance('csv_source_test')->getIdMap();
     $this->assertCount(4, $id_map);

tests/src/Functional/DrushCommandsTest.php

@@ -183,7 +183,7 @@ EOT;
   }
 
   /**
-   * Tests synced import.
+   * Tests synced import with and without update enforced.
    */
   public function testSyncImport(): void {
     $this->drush('mim', ['fruit_terms']);
@@ -200,7 +200,7 @@ EOT;
     $this->container->get('config.factory')->getEditable('migrate_plus.migration.fruit_terms')->set('source', $source)->save();
     // Flush cache so the recently changed migration can be refreshed.
     drupal_flush_all_caches();
-    $this->drush('mim', ['fruit_terms'], ['sync' => NULL]);
+    $this->drush('mim', ['fruit_terms'], ['sync' => NULL, 'update' => NULL]);
     $this->assertStringContainsString('1/3', $this->getErrorOutput());
     $this->assertStringContainsString('4/4', $this->getErrorOutput());
     $this->assertStringContainsString('[notice] Processed 3 items (1 created, 2 updated, 0 failed, 0 ignored) - done with \'fruit_terms\'', $this->getErrorOutput());
@@ -208,6 +208,19 @@ EOT;
     $this->assertEquals(3, \Drupal::entityTypeManager()->getStorage('taxonomy_term')->getQuery()->accessCheck(TRUE)->count()->execute());
     $this->assertEmpty(\Drupal::entityTypeManager()->getStorage('taxonomy_term')->load(2));
 
+    unset($source['data_rows'][2]);
+    $source['data_rows'][] = ['name' => 'Pear'];
+    $this->container->get('config.factory')->getEditable('migrate_plus.migration.fruit_terms')->set('source', $source)->save();
+    // Flush cache so the recently changed migration can be refreshed.
+    drupal_flush_all_caches();
+    $this->drush('mim', ['fruit_terms'], ['sync' => NULL]);
+    $this->assertStringContainsString('1/3', $this->getErrorOutput());
+    $this->assertStringContainsString('[notice] Processed 1 item (1 created, 0 updated, 0 failed, 0 ignored) - done with \'fruit_terms\'', $this->getErrorOutput());
+    $this->assertStringNotContainsString('2 updated', $this->getErrorOutput());
+    $this->assertStringNotContainsString('5', $this->getErrorOutput());
+    $this->assertEquals(3, \Drupal::entityTypeManager()->getStorage('taxonomy_term')->getQuery()->accessCheck(TRUE)->count()->execute());
+    $this->assertEmpty(\Drupal::entityTypeManager()->getStorage('taxonomy_term')->load(3));
+
     /** @var \Drupal\migrate\Plugin\MigrateIdMapInterface $id_map */
     $id_map = $this->container->get('plugin.manager.migration')->createInstance('fruit_terms')->getIdMap();
     $this->assertCount(3, $id_map);