Issue #2912779 by SylvainM, JeroenT, betz, JulienD, fgm, matio89, geek-merlin,...

  "name": "drupal/stage_file_proxy",

  "description": "Provides stage_file_proxy module.",

  "type": "drupal-module",

  "license": "GPL-2.0+",

  "license": "GPL-2.0-or-later",

  "minimum-stability": "dev",

  "require": {

    "drupal/core": ">=8.7.7"

  },

  "extra": {

    "drush": {

      "services": {

          "stage_file_proxy.drush.services.yml": "^9 || ^10"

      }

    }

  }

}

<?php

namespace Drupal\stage_file_proxy\Commands;

use Drupal\Component\Utility\UrlHelper;

use Drupal\Core\Config\ConfigFactoryInterface;

use Drupal\Core\Database\Connection;

use Drupal\stage_file_proxy\FetchManagerInterface;

use Drush\Commands\DrushCommands;

use GuzzleHttp\Exception\ClientException;

use Psr\Log\LoggerInterface;

use Symfony\Component\Console\Helper\ProgressBar;

/**

 * Drush commands for Stage File Proxy.

 */

class StageFileProxyCommands extends DrushCommands {

  /**

   * The database service.

   *

   * @var \Drupal\Core\Database\Connection

   */

  protected $database;

  /**

   * The stage_file_proxy.fetch_manager service.

   *

   * @var \Drupal\stage_file_proxy\FetchManagerInterface

   */

  protected $fetchManager;

  /**

   * The logger.channel.stage_file_proxy service.

   *

   * @var \Psr\Log\LoggerInterface

   */

  protected $logger;

  /**

   * The module config.

   *

   * Not called "config": name is used by Drush to store a DrushConfig instance.

   *

   * @var \Drupal\Core\Config\ImmutableConfig

   */

  protected $moduleConfig;

  /**

   * The app root.

   *

   * @var string

   */

  protected $root;

  /**

   * StageFileProxyCommands constructor.

   *

   * @param \Drupal\Core\Config\ConfigFactoryInterface $configFactory

   *   The config.factory service.

   * @param \Drupal\Core\Database\Connection $database

   *   The database service.

   * @param \Drupal\stage_file_proxy\FetchManagerInterface $fetchManager

   *   The stage_file_proxy.fetch_manager service.

   * @param \Psr\Log\LoggerInterface $logger

   *   The logger.channel.stage_file_proxy service.

   * @param string $root

   *   The app root.

   */

  public function __construct(

    ConfigFactoryInterface $configFactory,

    Connection $database,

    FetchManagerInterface $fetchManager,

    LoggerInterface $logger,

    string $root

  ) {

    parent::__construct();

    $this->moduleConfig = $configFactory->get('stage_file_proxy.settings');

    $this->database = $database;

    $this->fetchManager = $fetchManager;

    $this->logger = $logger;

    $this->root = $root;

  }

  /**

   * Download all managed files from the origin.

   *

   * @command stage_file_proxy:dl

   * @aliases stage-file-proxy-dl,sfdl

   * @option skip-progress-bar Skip displaying a progress bar.

   */

  public function dl(array $command_options = ['skip-progress-bar' => FALSE]) {

    $logger = $this->logger();

    $server = $this->moduleConfig->get('origin');

    if (empty($server)) {

      throw new \Exception('Configure stage_file_proxy.settings.origin in your settings.php (see INSTALL.txt).');

    }

    $query = $this->database->select('file_managed', 'fm');

    $results = $query->fields('fm', ['uri'])

      ->orderBy('fm.fid', 'DESC')

      ->execute()

      ->fetchCol();

    $fileDir = $this->fetchManager->filePublicPath();

    $remoteFileDir = trim($this->moduleConfig->get('origin_dir'));

    if (!$remoteFileDir) {

      $remoteFileDir = $fileDir;

    }

    $gotFilesNumber = 0;

    $errorFilesNumber = 0;

    $notPublicFilesNumber = 0;

    $results_number = count($results);

    $publicPrefix = 'public://';

    $logger->notice('Downloading {count} files.', [

      'count' => $results_number,

    ]);

    $options = [

      'verify' => $this->moduleConfig->get('verify'),

    ];

    $progress_bar = NULL;

    if (!$command_options['skip-progress-bar']) {

      $progress_bar = new ProgressBar($this->output(), $results_number);

    }

    foreach ($results as $uri) {

      if (strpos($uri, $publicPrefix) !== 0) {

        $notPublicFilesNumber++;

        if ($progress_bar) {

          $progress_bar->advance();

        }

        continue;

      }

      $relativePath = mb_substr($uri, mb_strlen($publicPrefix));

      if (file_exists("{$this->root}/{$fileDir}/{$relativePath}")) {

        if ($progress_bar) {

          $progress_bar->advance();

        }

        continue;

      }

      try {

        if ($this->fetchManager->fetch($server, $remoteFileDir, $relativePath, $options)) {

          $gotFilesNumber++;

        }

        else {

          $errorFilesNumber++;

          $logger->error('Stage File Proxy encountered an unknown error by retrieving file {file}', [

            'file' => $server . '/' . UrlHelper::encodePath("{$remoteFileDir}/{$relativePath}"),

          ]);

        }

      }

      catch (ClientException $e) {

        $errorFilesNumber++;

        $logger->error($e->getMessage());

      }

      if ($progress_bar) {

        $progress_bar->advance();

      }

    }

    if ($progress_bar) {

      $progress_bar->finish();

    }

    $logger->notice('{gotFilesNumber} downloaded files.', [

      'gotFilesNumber' => $gotFilesNumber,

    ]);

    if ($errorFilesNumber) {

      $logger->error('{count} file(s) having an error, see log.', [

        'count' => $errorFilesNumber,

      ]);

    }

    if ($notPublicFilesNumber) {

      $logger->error('{count} file(s) not in public directory.', [

        'count' => $notPublicFilesNumber,

      ]);

    }

  }

}

<?php

/**

 * @file

 * Contains drush commands.

 */

use Drupal\Component\Utility\UrlHelper;

use Drupal\Core\Database\Database;

use GuzzleHttp\Exception\ClientException;

/**

 * Implements hook_drush_command().

 */

function stage_file_proxy_drush_command() {

  $items['stage-file-proxy-dl'] = [

    'description' => 'Download all managed files from the origin.',

  ];

  return $items;

}

/**

 * Download all managed files from the origin.

 */

function drush_stage_file_proxy_dl() {

  $server = Drupal::config('stage_file_proxy.settings')->get('origin');

  $options = [

    'verify' => \Drupal::config('stage_file_proxy.settings')->get('verify'),

  ];

  if (empty($server)) {

    \Drupal::messenger()->addError(

      dt('Configure stage_file_proxy.settings.origin in your settings.php (see INSTALL.txt).')

    );

    return;

  }

  $query = Database::getConnection('default')->select('file_managed', 'fm');

  $query->fields('fm', ['uri']);

  $query->orderBy('fm.fid', 'DESC');

  $results = $query->execute()->fetchCol();

  $fetch_manager = Drupal::service('stage_file_proxy.fetch_manager');

  $logger = Drupal::service('logger.channel.stage_file_proxy');

  $translation = Drupal::translation();

  $file_dir = $fetch_manager->filePublicPath();

  $remote_file_dir = trim(\Drupal::config('stage_file_proxy.settings')->get('origin_dir'));

  if (!$remote_file_dir) {

    $remote_file_dir = $file_dir;

  }

  $got_files_number = 0;

  $error_files_number = 0;

  $not_public_files_number = 0;

  foreach ($results as $uri) {

    if (strpos($uri, 'public://') !== 0) {

      $not_public_files_number++;

      continue;

    }

    $relative_path = mb_substr($uri, mb_strlen('public://'));

    if (file_exists(DRUPAL_ROOT . '/' . $file_dir . '/' . $relative_path)) {

      continue;

    }

    try {

      if ($fetch_manager->fetch($server, $remote_file_dir, $relative_path, $options)) {

        $got_files_number++;

      }

      else {

        $error_files_number++;

        $logger->error(

          'Stage File Proxy encountered an unknown error by retrieving file @file',

          ['@file' => $server . '/' . UrlHelper::encodePath($remote_file_dir . '/' . $relative_path)]

        );

      }

    }

    catch (ClientException $e) {

      $error_files_number++;

      $logger->error($e->getMessage());

    }

  }

  \Drupal::messenger()->addMessage(

    $translation->translate(

      '@got_files_number downladed files.',

      ['@got_files_number' => $got_files_number]

    )

  );

  if ($error_files_number) {

    \Drupal::messenger()->addError(

      $translation->formatPlural(

        $error_files_number,

        '@count file having an error, see log.',

        '@count files having an error, see log.'

      )

    );

  }

  if ($not_public_files_number) {

    \Drupal::messenger()->addError(

      $translation->formatPlural(

        $not_public_files_number,

        '@count file not in public directory.',

        '@count files not in public directory.'

      )

    );

  }

}

services:

  stage_file_proxy.command:

    class: Drupal\stage_file_proxy\Commands\StageFileProxyCommands

    arguments:

      - '@config.factory'

      - '@database'

      - '@stage_file_proxy.fetch_manager'

      - '@logger.channel.stage_file_proxy'

      - '@app.root'

    tags:

      -  { name: drush.command }

<?php

/**

 * @file

 * Contains drush commands.

 */

/**

 * Implements hook_drush_command().

 */

function stage_file_proxy_drush_command() {

  $items['stage-file-proxy-dl'] = [

    'description' => 'Download all managed files from the origin.',

  ];

  return $items;

}

/**

 * Download all managed files from the origin.

 */

function drush_stage_file_proxy_dl() {

  \Drupal::service('stage_file_proxy.command')->dl();

}