Issue #3404021: Update deploy:pre-hook command to use batch system

Closes #3404021

src/Commands/DrushPreDeployCommands.php

@@ -2,53 +2,57 @@
 
 namespace Drupal\drush_pre_deploy\Commands;
 
+use Consolidation\OutputFormatters\StructuredData\RowsOfFields;
+use Consolidation\OutputFormatters\StructuredData\UnstructuredListData;
+use Consolidation\SiteAlias\SiteAliasManagerAwareInterface;
+use Consolidation\SiteAlias\SiteAliasManagerAwareTrait;
 use Drupal\Core\Update\UpdateRegistry;
 use Drupal\Core\Utility\Error;
-use Drush\Drupal\Commands\core\DeployHookCommands;
+use Drush\Attributes as CLI;
+use Drush\Boot\DrupalBootLevels;
+use Drush\Commands\core\DocsCommands;
+use Drush\Commands\DrushCommands;
+use Drush\Drush;
 use Drush\Exceptions\UserAbortException;
-use Psr\Log\LogLevel;
-use Consolidation\SiteAlias\SiteAliasManagerAwareInterface;
-use Consolidation\SiteAlias\SiteAliasManagerAwareTrait;
-use Drupal\Core\Extension\ModuleHandlerInterface;
-use Drupal\Core\KeyValueStore\KeyValueFactoryInterface;
-use Consolidation\OutputFormatters\StructuredData\RowsOfFields;
 
 /**
  * Pre-deploy drush command class.
+ *
+ * @see \Drush\Commands\core\DeployHookCommands
+ * @see \Drush\Commands\core\DeployCommands
  */
-class DrushPreDeployCommands extends DeployHookCommands implements SiteAliasManagerAwareInterface {
+class DrushPreDeployCommands extends DrushCommands implements SiteAliasManagerAwareInterface {
 
   use SiteAliasManagerAwareTrait;
 
-  /**
-   * Command successfully completed some operation.
-   */
-  const SUCCESS = 'success';
+  public const UPDATE_TYPE = 'predeploy';
+
+  public const HOOK_METHOD_NAME = 'pre_deploy';
+
+  public const HOOK_STATUS = 'deploy:pre-hook-status';
+  public const HOOK = 'deploy:pre-hook';
+  public const BATCH_PROCESS = 'pre-deploy:batch-process';
+  public const MARK_COMPLETE = 'pre-deploy:mark-complete';
 
   /**
-   * {@inheritDoc}
+   * Get the pre-deploy hook update registry.
    */
-  public function __construct(string $root, string $site_path, ModuleHandlerInterface $moduleHandler, KeyValueFactoryInterface $keyValueFactory) {
-    $this->keyValue = $keyValueFactory->get('pre_deploy_hook');
-    $this->registry = new class(
-      $root,
-      $site_path,
-      array_keys($moduleHandler->getModuleList()),
-      $this->keyValue
+  public static function getRegistry(): UpdateRegistry {
+    $registry = new class (
+      \Drupal::getContainer()->getParameter('app.root'),
+      \Drupal::getContainer()->getParameter('site.path'),
+      array_keys(\Drupal::service('module_handler')->getModuleList()),
+      \Drupal::service('keyvalue')->get(self::HOOK_METHOD_NAME . '_hook')
     ) extends UpdateRegistry {
 
-      /**
-       * Sets the update registry type.
-       *
-       * @param string $type
-       *   The registry type.
-       */
-      public function setUpdateType($type) {
+      public function setUpdateType(string $type): void {
         $this->updateType = $type;
       }
 
     };
-    $this->registry->setUpdateType('predeploy');
+    $registry->setUpdateType(self::UPDATE_TYPE);
+
+    return $registry;
   }
 
   /**
@@ -70,11 +74,16 @@ class DrushPreDeployCommands extends DeployHookCommands implements SiteAliasMana
    *
    * @return \Consolidation\OutputFormatters\StructuredData\RowsOfFields
    *   A list of pending hooks.
-   *
-   * @phpcs:disable Generic.CodeAnalysis.UselessOverridingMethod.Found
    */
+  #[CLI\Command(name: self::HOOK_STATUS)]
+  #[CLI\Usage(name: 'drush ' . self::HOOK_STATUS, description: 'Prints information about pending pre-deploy hooks.')]
+  #[CLI\FieldLabels(labels: ['module' => 'Module', 'hook' => 'Hook', 'description' => 'Description'])]
+  #[CLI\DefaultTableFields(fields: ['module', 'hook', 'description'])]
+  #[CLI\FilterDefaultField(field: 'hook')]
+  #[CLI\Topics(topics: [DocsCommands::DEPLOY])]
+  #[CLI\Bootstrap(level: DrupalBootLevels::FULL)]
   public function status(): RowsOfFields {
-    $updates = $this->registry->getPendingUpdateInformation();
+    $updates = self::getRegistry()->getPendingUpdateInformation();
     $rows = [];
     foreach ($updates as $module => $update) {
       if (!empty($update['pending'])) {
@@ -92,30 +101,34 @@ class DrushPreDeployCommands extends DeployHookCommands implements SiteAliasMana
   }
 
   /**
-   * Runs pre-deploy hooks.
+   * Runs pending pre-deploy hooks.
    *
    * @usage deploy:pre-hook
    *   Runs pending pre-deploy hooks.
    *
    * @command deploy:pre-hook
+   * @topics docs:deploy
+   * @version 10.3
    * @bootstrap full
-   *
-   * @return int
-   *   0 for success, 1 for failure.
    */
+  #[CLI\Command(name: self::HOOK)]
+  #[CLI\Usage(name: 'drush ' . self::HOOK, description: 'Run pending pre-deploy hooks.')]
+  #[CLI\Topics(topics: [DocsCommands::DEPLOY])]
+  #[CLI\Version(version: '10.3')]
+  #[CLI\Bootstrap(level: DrupalBootLevels::FULL)]
   public function run(): int {
-    $pending = $this->registry->getPendingUpdateFunctions();
+    $pending = self::getRegistry()->getPendingUpdateFunctions();
 
     if (empty($pending)) {
       $this->logger()->success(dt('No pending pre-deploy hooks.'));
       return self::EXIT_SUCCESS;
     }
 
-    $process = $this->processManager()->drush($this->siteAliasManager()->getSelf(), 'deploy:pre-hook-status');
+    $process = $this->processManager()->drush($this->siteAliasManager()->getSelf(), self::HOOK_STATUS);
     $process->mustRun();
     $this->output()->writeln($process->getOutput());
 
-    if (!$this->io()->confirm(dt('Do you wish to run the specified pending pre deploy hooks?'))) {
+    if (!$this->io()->confirm(dt('Do you wish to run the specified pending pre-deploy hooks?'))) {
       throw new UserAbortException();
     }
 
@@ -124,11 +137,59 @@ class DrushPreDeployCommands extends DeployHookCommands implements SiteAliasMana
       $success = $this->doRunPendingHooks($pending);
     }
 
-    $level = $success ? self::SUCCESS : LogLevel::ERROR;
-    $this->logger()->log($level, dt('Finished performing pre deploy hooks.'));
+    $message = dt('Finished performing pre-deploy hooks.');
+    if ($success) {
+      $this->logger()->success($message);
+    }
+    else {
+      $this->logger()->error($message);
+    }
     return $success ? self::EXIT_SUCCESS : self::EXIT_FAILURE;
   }
 
+  /**
+   * Process operations in the specified batch set.
+   *
+   * @param string $batch_id
+   *   The batch id that will be processed.
+   *
+   * @command pre-deploy:batch-process
+   * @bootstrap full
+   * @hidden
+   */
+  #[CLI\Command(name: self::BATCH_PROCESS)]
+  #[CLI\Argument(name: 'batch_id', description: 'The batch id that will be processed.')]
+  #[CLI\Bootstrap(level: DrupalBootLevels::FULL)]
+  #[CLI\Help(hidden: true)]
+  public function process(string $batch_id, $options = ['format' => 'json']): UnstructuredListData {
+    $result = drush_batch_command($batch_id);
+    return new UnstructuredListData($result);
+  }
+
+  /**
+   * Mark all pre-deploy hooks as having run.
+   *
+   * @usage pre-deploy:mark-complete
+   *   Skip all pending pre-deploy hooks and mark them as complete.
+   *
+   * @command pre-deploy:mark-complete
+   * @topics docs:deploy
+   * @version 10.6.1
+   * @bootstrap full
+   */
+  #[CLI\Command(name: self::MARK_COMPLETE)]
+  #[CLI\Usage(name: 'drush ' . self::MARK_COMPLETE, description: 'Skip all pending pre-deploy hooks and mark them as complete.')]
+  #[CLI\Topics(topics: [DocsCommands::DEPLOY])]
+  #[CLI\Version(version: '10.6.1')]
+  #[CLI\Bootstrap(level: DrupalBootLevels::FULL)]
+  public function markComplete(): int {
+    $pending = self::getRegistry()->getPendingUpdateFunctions();
+    self::getRegistry()->registerInvokedUpdates($pending);
+
+    $this->logger()->success(dt('Marked %count pending pre-deploy hooks as complete.', ['%count' => count($pending)]));
+    return self::EXIT_SUCCESS;
+  }
+
   /**
    * Runs pending hooks.
    *
@@ -140,25 +201,40 @@ class DrushPreDeployCommands extends DeployHookCommands implements SiteAliasMana
    */
   protected function doRunPendingHooks(array $pending) {
     try {
+      $operations = [];
+      // Use a batch operation in order to reduce memory consumption.
+      // @see https://github.com/drush-ops/drush/pull/4800
       foreach ($pending as $function) {
-        $func = new \ReflectionFunction($function);
-        $this->logger()->notice('Predeploy hook started: ' . $func->getName());
-
-        // Pretend it is a batch operation to keep the same signature
-        // as the post update hooks.
-        $sandbox = [];
-        do {
-          $return = $function($sandbox);
-          if (!empty($return)) {
-            $this->logger()->notice($return);
-          }
-        } while (isset($sandbox['#finished']) && $sandbox['#finished'] < 1);
-
-        $this->registry->registerInvokedUpdates([$function]);
-        $this->logger()->debug('Performed: ' . $func->getName());
+        $operations[] = [[static::class, 'updateDoOnePreDeployHook'], [$function]];
+      }
+
+      $batch = [
+        'operations' => $operations,
+        'title' => 'Updating',
+        'init_message' => 'Starting pre-deploy hooks',
+        'error_message' => 'An unrecoverable error has occurred. You can find the error message below. It is advised to copy it to the clipboard for reference.',
+        'finished' => [$this, 'updateFinished'],
+      ];
+      batch_set($batch);
+      $result = drush_backend_batch_process(self::BATCH_PROCESS);
+
+      $success = FALSE;
+      if (!is_array($result)) {
+        $this->logger()->error(dt('Batch process did not return a result array. Returned: !type', ['!type' => gettype($result)]));
+      }
+      elseif (!empty($result[0]['#abort'])) {
+        // Whenever an error occurs the batch process does not continue, so
+        // this array should only contain a single item, but we still output
+        // all available data for completeness.
+        $this->logger()->error(dt('Update aborted by: !process', [
+          '!process' => implode(', ', $result[0]['#abort']),
+        ]));
+      }
+      else {
+        $success = TRUE;
       }
 
-      return TRUE;
+      return $success;
     }
     catch (\Throwable $e) {
       $variables = Error::decodeException($e);
@@ -168,4 +244,123 @@ class DrushPreDeployCommands extends DeployHookCommands implements SiteAliasMana
     }
   }
 
+  /**
+   * Batch command that executes a single pre-deploy hook.
+   *
+   * @param string $function
+   *   The pre-deploy-hook function to execute.
+   * @param array|\DrushBatchContext $context
+   *   The batch context object.
+   */
+  public static function updateDoOnePreDeployHook($function, $context) {
+    $ret = [];
+
+    // If this update was aborted in a previous step, or has a dependency that
+    // was aborted in a previous step, go no further.
+    if (!empty($context['results']['#abort'])) {
+      return;
+    }
+
+    $module = $name = $filename = NULL;
+    // Module names can include '_pre_deploy', so pre-deploy functions like
+    // module_pre_deploy_pre_deploy_name() or
+    // module_pre_deploy_update_pre_deploy_stuff() are ambiguous.
+    // Check every occurrence.
+    $hook_method_needle = '_' . self::HOOK_METHOD_NAME . '_';
+    $hook_method_needle_length = strlen($hook_method_needle);
+    $offset = 0;
+    while (($position = strpos($function, $hook_method_needle, $offset)) !== FALSE) {
+      // Trim off the trailing '_' so that it can be used for the next search.
+      $offset = $position + $hook_method_needle_length - 1;
+      $module = substr($function, 0, $position);
+      $name = substr($function, $position + $hook_method_needle_length);
+      $filename = $module . '.' . self::UPDATE_TYPE;
+      \Drupal::moduleHandler()->loadInclude($module, 'php', $filename);
+      if (function_exists($function)) {
+        break;
+      }
+    }
+
+    if ($module && $name && function_exists($function)) {
+      if (empty($context['results'][$module][$name]['type'])) {
+        Drush::logger()->notice("Pre-deploy hook started: $function");
+      }
+      try {
+        $ret['results']['query'] = $function($context['sandbox']);
+        $ret['results']['success'] = TRUE;
+        $ret['type'] = self::UPDATE_TYPE;
+
+        if (!isset($context['sandbox']['#finished']) || ($context['sandbox']['#finished'] >= 1)) {
+          self::getRegistry()->registerInvokedUpdates([$function]);
+        }
+      }
+      catch (\Exception $e) {
+        // @todo We may want to do different error handling for different exception
+        // types, but for now we'll just log the exception and return the
+        // message for printing.
+        // @see https://www.drupal.org/node/2564311
+        Drush::logger()->error($e->getMessage());
+
+        $variables = Error::decodeException($e);
+        $variables = array_filter($variables, static function ($key) {
+          return $key && ($key[0] === '@' || $key[0] === '%');
+        }, ARRAY_FILTER_USE_KEY);
+        // On windows there is a problem with json encoding a string with
+        // backslashes.
+        $variables['%file'] = strtr($variables['%file'], [DIRECTORY_SEPARATOR => '/']);
+        $ret['#abort'] = [
+          'success' => FALSE,
+          'query' => strip_tags((string) t('%type: @message in %function (line %line of %file).', $variables)),
+        ];
+      }
+    }
+    else {
+      $ret['#abort'] = ['success' => FALSE];
+      Drush::logger()->warning(dt('Pre-deploy hook function @function not found in file @filename', [
+        '@function' => $function,
+        '@filename' => "$filename.php",
+      ]));
+    }
+
+    if (isset($context['sandbox']['#finished'])) {
+      $context['finished'] = $context['sandbox']['#finished'];
+      unset($context['sandbox']['#finished']);
+    }
+    if (!isset($context['results'][$module][$name])) {
+      $context['results'][$module][$name] = [];
+    }
+    $context['results'][$module][$name] = array_merge($context['results'][$module][$name], $ret);
+
+    // Log the message that was returned.
+    if (!empty($ret['results']['query'])) {
+      Drush::logger()->notice(strip_tags((string) $ret['results']['query']));
+    }
+
+    if (!empty($ret['#abort'])) {
+      // Record this function in the list of updates that were aborted.
+      $context['results']['#abort'][] = $function;
+      $context['error_message'] = "Pre-deploy hook failed: $function";
+      if (!($context instanceof \DrushBatchContext)) {
+        // Log the error directly if it's not a batch context object.
+        // @see https://github.com/drush-ops/drush/pull/5354
+        Drush::logger()->error($context['error_message']);
+      }
+    }
+    elseif ($context['finished'] === 1) {
+      // Setting this value will output a success message.
+      // @see \DrushBatchContext::offsetSet()
+      $context['message'] = "Performed: $function";
+    }
+  }
+
+  /**
+   * Batch finished callback.
+   *
+   * @param bool $success
+   *   Whether the batch ended without a fatal error.
+   */
+  public function updateFinished(bool $success, array $results, array $operations): void {
+    // In theory there is nothing to do here.
+  }
+
 }