Migrate loop for large migrations.

scripts/migrations.sh

 #!/bin/bash
 
+# https://chromatichq.com/insights/migration-memory-management-batching-and-limits/
+# https://gist.github.com/adamzimmermann/e0f730425d0876991f1891a02bf372a4
+migrate_loop()
+{
+  # Better readability with separation.
+  echo "========================";
+  # Get the output of the drush status.
+  drush_output=$(drush ms "$1" --format string);
+
+  # Split output string into an array.
+  output=( $drush_output )
+  # Output the status items.
+  for index in "${!output[@]}"
+  do
+    if [ "$index" == "3" ]
+    then
+        echo "Migration: ${output[index]}";
+    fi
+
+    if [ "$index" == "4" ]
+    then
+        echo "Status: ${output[index]}";
+    fi
+
+    if [ "$index" == "5" ]
+    then
+        echo "Total: ${output[index]}";
+    fi
+
+    if [ "$index" == "6" ]
+    then
+        echo "Imported: ${output[index]}";
+    fi
+
+    if [ "$index" == "7" ]
+    then
+        echo "Remaining: ${output[index]}";
+    fi
+  done
+
+  # Check if all items were imported.
+  if [ "${output[7]}" == "0" ] || [ "${output[7]}" -lt "0" ]
+  then
+    echo "No items left to import.";
+  else
+    echo "There are ${output[7]} remaining ${output[3]} items to be imported.";
+    echo "Running command: drush migrate:import $1";
+    echo "...";
+    # Run the migration until it stops.
+    drush migrate:reset "$1";
+    drush migrate:import "$1" --feedback 100 --limit 1000;
+    # Run the check on this migration again.
+    migrate_loop "$1";
+  fi
+}
+
 TYPE="${1:-all}"
 
 if [[ "$TYPE" == "all" || "$TYPE" == "taxonomies" ]]; then
@@ -34,20 +90,19 @@ fi
 
 if [[ "$TYPE" == "all" || "$TYPE" == "users" ]]; then
   # Users and roles.
-  drush migrate:import drupalorg_migrate_user_roles
+  drush migrate:import drupalorg_migrate_user_roles --update
   # IMPORTANT: the following will take HOURS unless you truncate the number of D7-users somehow.
   # You can run these sql commands:
   #   DELETE FROM migrate.users WHERE uid > 500;
   #   UPDATE migrate.node SET uid = 1 where uid > 500;
   #   UPDATE migrate.files SET uid = 1 where uid > 500;
   #   UPDATE migrate.node_revision SET uid = 1 where uid > 500;
-  drush migrate:import drupalorg_migrate_users
+  migrate_loop drupalorg_migrate_users
 fi
 
 if [[ "$TYPE" == "all" || "$TYPE" == "files" ]]; then
-  # @TODO how to place files in server.
   # Files
-  drush migrate:import drupalorg_migrate_project_files
+  migrate_loop drupalorg_migrate_project_files
 fi
 
 if [[ "$TYPE" == "all" || "$TYPE" == "project-browser" ]]; then
@@ -65,10 +120,9 @@ fi
 
 if [[ "$TYPE" == "all" || "$TYPE" == "usage" ]]; then
   # Project usage data
-  drush migrate:import drupalorg_migrate_field_collection_release_files
-  drush migrate:import drupalorg_migrate_project_release
+  migrate_loop drupalorg_migrate_field_collection_release_files
+  migrate_loop drupalorg_migrate_project_release
   ## Needed to truncate the table locally due to memory limits. Just the delete query took minutes to run.
   ## DELETE FROM project_usage_week_release WHERE timestamp < 1633219200; -- October 2021
-  # @todo Maybe use high water property here https://www.drupal.org/docs/drupal-apis/migrate-api/migrate-api-overview#s-highwater-marks
-  drush migrate:import drupalorg_migrate_project_usage_week_release
+  migrate_loop drupalorg_migrate_project_usage_week_release
 fi

src/EventSubscriber/MemoryControlEventHandler.php

@@ -43,7 +43,7 @@ class MemoryControlEventHandler implements EventSubscriberInterface {
   public function reduceMemoryUsage(MigratePostRowSaveEvent $event): void {
     $this->iterationCountUntilCacheFlush--;
     if ($this->iterationCountUntilCacheFlush <= 0) {
-      dump(round(memory_get_usage()/1048576,2));
+      dump(round(memory_get_usage()/1048576,2). 'MB');
       // @todo if below not needed then remove all class + service.
 //      $this->memoryCache->deleteAll();
 //      $this->configFactory->clearStaticCache();