Skip to content
Snippets Groups Projects
ConverterCommand.php 25.6 KiB
Newer Older
namespace Drupal\automatic_updates\Development;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Filesystem\Filesystem;

/**
 * Converts the contrib module to core merge request.
 *
 * This command can be used in 2 ways.
 *
 * ##### Local conversion when working on core merge requests. #####
 *
 * This requires a local core clone and a local contrib clone. The contrib clone
 * should be on 3.0.x unless you are working on an issue that changes this file
 * itself. The core clone should be on the core merge request branch. The
 * contrib clone should NOT be inside the core clone. Both repositories should
 * have a clean git status.
 *
 * If just testing the conversion or if you want to run code checks or phpunit
 * tests locally on the core converted version you can just make a local branch
 * in the core of 11.x or use 11.x directly if you are going to commit.
 *
 * composer core-covert /path/to/core core-branch
 * For example to update the Package Manager core merge requests.
 *
 * 1. Checkout 3.0.x on the core repo
 * 2. Checkout the 3346707-package-manager branch from
 *    https://drupal.org/9/3346707
 * 3. Run:
 *     `composer core-convert /path/to/core 3346707-package-manager --package_manager_only`
 * 4. If the core checks pass the script will make a commit.
 * 5. Sanity check files changes in the commit to ensure they are to
 *    package manager only or related core composer files if needed.
 * 6. To be extra careful you can run some or all phpunit tests locally on the
 *    core converted version. Running the kernel tests or least the unit tests
 *    might be a good idea.
 * 7. Push the commit to the core merge request.
 *
 *
 * ##### .gitlab-ci.yml usage #####
 * Using the --gitlabci option tt is also used inside `.gitlab-ci.yml` to
 * convert the contrib module to the core merge request version and run tests.
 * This is to ensure, as much as possible, we can also convert to the core
 * version and expect code quality checks and tests to pass.
 *
 * The core clone should already have the core merge request locally.
 */

  private bool $package_manager_only;

  private string $core_target_branch;

  private string $contrib_dir;

  private bool $no_commit;

  private bool $skipCoreChecks;

  /**
   * {@inheritdoc}
   */
  protected function configure() {
    $this->addArgument('core_dir', InputArgument::REQUIRED, 'The path to the root of Drupal Core');
    $this->addArgument('core_branch', InputArgument::OPTIONAL, 'The core merge request branch');
    $this->addArgument('contrib_branch', InputArgument::OPTIONAL, 'The contrib branch to switch to', '3.0.x');
    $this->addOption('package_manager_only', NULL, InputOption::VALUE_NONE, 'Only convert package manager');
    $this->addOption('core_target_branch', NULL, InputOption::VALUE_REQUIRED, 'The core target branch', '11.x');
    $this->addOption('skip_core_checks', NULL, InputOption::VALUE_NONE, 'Skip core checks');
    $this->addOption('no_commit', NULL, InputOption::VALUE_NONE, 'Do not make commit');
    $this->addOption('gitlabci', NULL, InputOption::VALUE_NONE, 'Run in Gitlab CI');
  }

  /**
   * {@inheritdoc}
   */
  protected function execute(InputInterface $input, OutputInterface $output) {
    $this->core_dir = realpath($input->getArgument('core_dir'));
    $this->core_branch = $input->getArgument('core_branch');
    $this->contrib_branch = $input->getArgument('contrib_branch');
    $this->package_manager_only = $input->getOption('package_manager_only');
    $this->core_target_branch = $input->getOption('core_target_branch');
    $this->contrib_dir = realpath(__DIR__ . '/../..');
    $this->skipCoreChecks = $input->getOption('skip_core_checks');
    $this->no_commit = $input->getOption('no_commit');
    $this->gitlabci = $input->getOption('gitlabci');
    if (!$this->gitlabci) {
      if (empty($this->core_branch)) {
        throw new \Exception("core_branch is required if not on gitlabci");
      }
    }
    else {
      if (!empty($this->core_branch)) {
        throw new \Exception("branches are not allowed on gitlabci");
      }
    }
    // Esnure core_dir is a directory.
    if (!is_dir($this->core_dir)) {
      throw new \Exception("$this->core_dir is not a directory.");
    }
    if ($this->gitlabci) {
      $this->no_commit = TRUE;
      $this->skipCoreChecks = TRUE;
    }
    else {
      // Ensure the core directory is clean.
      static::switchToBranch($this->core_target_branch);
      // Git pull to ensure we are up to date.
      static::executeCommand('git pull');
      // change back the previous directory.
      chdir($this->contrib_dir);
      // Ensure we are on the correct branches.
      static::switchToBranches($this->core_dir, $this->core_branch, $this->contrib_branch);
      // Switch to the core directory and checkout the files and folders that this
      // conversion script will automatically update based on our composer.json
      // file and our dictionary.txt file.
      chdir($this->core_dir);
      static::executeCommand("git checkout {$this->core_target_branch} -- composer.json");
      static::executeCommand("git checkout {$this->core_target_branch} -- composer.lock");
      static::executeCommand("git checkout {$this->core_target_branch} -- composer");
      static::executeCommand("git checkout {$this->core_target_branch} -- core/composer.json");
      static::executeCommand("git checkout {$this->core_target_branch} -- core/misc/cspell/dictionary.txt");

      // Check out files that will be patched.
      static::executeCommand("git checkout {$this->core_target_branch} -- core/lib/Drupal/Core/EventSubscriber/FinishResponseSubscriber.php");
    }
    // Apply any core patches.
    $patches = glob(__DIR__ . '/../core-patches/*.patch');
    static::info("Applying core patches");
    print_r($patches);
    foreach ($patches as $file) {
      $patch_result = NULL;
      $output = [];
      // Patch but ignore previously applied patches.
      exec("patch -d{$this->core_dir} -p1 -N -r - < $file", $output, $patch_result);
      if ($patch_result !== 0) {
        if (!str_contains(implode("\n", $output), 'Ignoring previously applied (or reversed) patch.')) {
          throw new \Exception("Patch failed: $file");
        }
      }

    // Switch to the core directory and require all of the packages there are in
    // this module's composer.json require section.
    $require_section = $this->getContribModuleComposerJsonSection('require');
    // Hardcode symfony version to 6.4 for core.
    $core_symfony_version_update = function ($package_name, $version): string {
      if (str_starts_with($package_name, 'symfony/')) {
        return '^6.4';
      }
      return $version;
    };
    foreach ($require_section as $package_name => $version) {
      $version = $core_symfony_version_update($package_name, $version);
      static::executeCommand("composer require $package_name:$version --no-update --working-dir={$this->core_dir}/core");
    }
    // Run composer update for just this packages.
    static::executeCommand("composer update  " . implode(' ', array_keys($require_section)));

    $require_dev_section = $this->getContribModuleComposerJsonSection('require-dev');
    foreach ($require_dev_section as $package_name => $version) {
      $version = $core_symfony_version_update($package_name, $version);
      static::executeCommand("composer require --dev $package_name:$version");
    }
    $this->doConvert();
    return Command::SUCCESS;
  }

  /**
   * Executes a command and throws an exception if it fails.
   *
   * @param string $cmd
   *   The command to execute.
   */
  private static function executeCommand(string $cmd): void {
    $result = NULL;
    system($cmd, $result);
    if ($result !== 0) {
      throw new \Exception("Command failed: $cmd");
    }
  }

  /**
   * Prints message.
   *
   * @param string $msg
   *   The message to print.
   */
  private static function info(string $msg): void {
    print "\n$msg";
  }

  /**
   * Converts the contrib module to core merge request.
   */
    $old_machine_name = 'automatic_updates';
    $new_machine_name = 'auto_updates';
    self::info('Switched branches');
    $fs = new Filesystem();

    $core_module_path = static::getCoreModulePath($this->core_dir);
    $package_manager_core_path = $this->core_dir . "/core/modules/package_manager";
    // Remove old module.
    $fs->remove($core_module_path);
    self::info('Removed old core module');
    $fs->remove($package_manager_core_path);
    self::info("Removed package manager");

    $fs->mirror(self::getContribDir(), $core_module_path);
    self::info('Mirrored into core module');

      // This allows line that only need to be in the core MR to be commented
      // out.
      '// CORE_MR_ONLY:' => '',
      // Note this following line could leave 'CORE_MR_ONLY-10.1.x' comments in
      // the core MR, but we currently do not have any. Leaving
      // 'CORE_MR_ONLY-11.x' in is fine because we don't actually convert to
      // 10.1.x for a core MR. We might want to just remove the whole line if
      // it does not match the core target branch.
      "// CORE_MR_ONLY-{$this->core_target_branch}:" => '',
      $old_machine_name => $new_machine_name,
      'AutomaticUpdates' => 'AutoUpdates',
      "__DIR__ . '/../../../package_manager/tests" => "__DIR__ . '/../../../../Xpackage_manager/tests",
      "__DIR__ . '/../../../../package_manager/tests" => "__DIR__ . '/../../../../../Xpackage_manager/tests",
      "__DIR__ . '/../../../../../package_manager/tests" => "__DIR__ . '/../../../../../../Xpackage_manager/tests",
      '/Xpackage_manager' => '/package_manager',
    foreach ($replacements as $search => $replace) {
      static::renameFiles(static::getDirContents($core_module_path), $search, $replace);
      static::replaceContents(static::getDirContents($core_module_path, TRUE), $search, $replace);
    self::moveScripts($this->core_dir, $core_module_path, $this->package_manager_only);
      '.git',
      'composer.json',
      '.gitattributes',
      '.gitignore',
      'DEVELOPING.md',
      'phpstan.neon.dist',
      // @todo Move ComposerFixtureCreator to its location in core
      //   https://drupal.org/i/3347937.
      'scripts',
      'dictionary.txt',
    ];
    $removals = array_map(function ($path) use ($core_module_path) {
      return "$core_module_path/$path";
    }, $removals);
    $fs->remove($removals);
    self::info('Remove not needed');

    // Replace in file names and contents.
    static::replaceContents(
      [
        new \SplFileInfo("$core_module_path/auto_updates.info.yml"),
        new \SplFileInfo("$core_module_path/package_manager/package_manager.info.yml"),
      ],
      "core_version_requirement: ^10.1",
      "package: Core\nversion: VERSION\nlifecycle: experimental",
    );
    $fs->rename("$core_module_path/package_manager", $package_manager_core_path);
    static::copyGenericTest($package_manager_core_path, $this->core_dir);
    // Run phpcbf because removing code from merge request may result in unused
    // use statements or multiple empty lines.
    $contrib_dir = self::getContribDir();
    system("composer run phpcbf --working-dir=$contrib_dir -- $package_manager_core_path");
      static::copyGenericTest($core_module_path, $this->core_dir);
      system("composer run phpcbf --working-dir=$contrib_dir -- $$core_module_path");
    static::addWordsToDictionary($this->core_dir, $contrib_dir . "/dictionary.txt");
    if ($this->skipCoreChecks) {
      self::info('⚠️Skipped core checks');
      static::runCoreChecks($this->core_dir);
      self::info('Ran core checks');
    if (!$this->no_commit) {
      static::doMakeCommit($this->core_dir);
      self::info('Make commit');
      self::info("Done. Probably good but you should check before you push. These are the files present in the contrib module absent in core:");
      print shell_exec(sprintf("tree %s/package_manager > /tmp/contrib.txt  && tree %s/core/modules/package_manager > /tmp/core.txt && diff /tmp/contrib.txt /tmp/core.txt", self::getContribDir(), $this->core_dir));
      self::info('(Run diff /tmp/contrib.txt /tmp/core.txt to see that with color.');
    }

  }

  /**
   * Returns the path to the root of the contrib module.
   *
   * @return string
   *   The full path to the root of the contrib module.
   */
  private static function getContribDir(): string {
    return realpath(__DIR__ . '/../..');
  }

  /**
   * Returns the path where the contrib module will be placed in Drupal Core.
   *
   * @param string $core_dir
   *   The path to the root of Drupal Core.
   *
   * @return string
   *   The path where the contrib module will be placed in Drupal Core
   */
  private static function getCoreModulePath(string $core_dir): string {
    return $core_dir . '/core/modules/auto_updates';
  }

  /**
   * Replaces a string in the contents of the module files.
   *
   * @param array $files
   *   Files to replace.
   * @param string $search
   *   The string to be replaced.
   * @param string $replace
   *   The string to replace.
   */
  private static function replaceContents(array $files, string $search, string $replace): void {
    foreach ($files as $file) {
      $filePath = $file->getRealPath();
      file_put_contents($filePath, str_replace($search, $replace, file_get_contents($filePath)));
    }
  }

  /**
   * Renames the module files.
   *
   * @param array $files
   *   Files to replace.
   * @param string $old_pattern
   *   The old file name.
   * @param string $new_pattern
   *   The new file name.
   */
  private static function renameFiles(array $files, string $old_pattern, string $new_pattern): void {
    // Keep a record of the files and directories to change.
    // We will change all the files first, so we don't change the location of
    // any files in the middle. This probably won't work if we had nested
    // folders with the pattern on 2 folder levels, but we don't.
    $filesToChange = [];
    $dirsToChange = [];
    foreach ($files as $file) {
      $fileName = $file->getFilename();
      if ($fileName === '.') {
        $fullPath = $file->getPath();
        $parts = explode('/', $fullPath);
        $name = array_pop($parts);
        $path = "/" . implode('/', $parts);
      }
      else {
        $name = $fileName;
        $path = $file->getPath();
      }
      if (strpos($name, $old_pattern) !== FALSE) {
        $new_filename = str_replace($old_pattern, $new_pattern, $name);
        if ($file->isFile()) {
          $filesToChange[$file->getRealPath()] = $file->getPath() . "/$new_filename";
        }
        else {
          // Store directories by path depth.
          $depth = count(explode('/', $path));
          $dirsToChange[$depth][$file->getRealPath()] = "$path/$new_filename";
        }
      }
    }
    foreach ($filesToChange as $old => $new) {
      (new Filesystem())->rename($old, $new);
    }
    // Rename directories starting with the most nested to avoid renaming
    // parents directories first.
    krsort($dirsToChange);
    foreach ($dirsToChange as $dirs) {
      foreach ($dirs as $old => $new) {
        (new Filesystem())->rename($old, $new);
      }
    }
  }

  /**
   * Gets the contents of a directory.
   *
   * @param string $path
   *   The path of the directory.
   * @param bool $excludeDirs
   *   (optional) If TRUE, all directories will be excluded. Defaults to FALSE.
   *
   * @return \SplFileInfo[]
   *   Array of objects containing file information.
   */
  private static function getDirContents(string $path, bool $excludeDirs = FALSE): array {
    $rii = new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator($path));

    $files = [];
    /** @var \SplFileInfo $file */
    foreach ($rii as $file) {
      // Exclude the .git directories always.
      if ($file->getFilename() === '.git') {
        continue;
      }
      if (str_contains($file->getRealPath(), '/.git/') || str_ends_with($file->getRealPath(), '/.git')) {
        continue;
      }
      if ($excludeDirs && $file->isDir()) {
        continue;
      }
      $files[] = $file;
    }

    return $files;
  }

  /**
   * Ensures the git status is clean.
   *
   * @return bool
   *   TRUE if git status is clean , otherwise returns a exception.
   */
  private static function ensureGitClean(): bool {
    $status_output = shell_exec('git status');
    if (strpos($status_output, 'nothing to commit, working tree clean') === FALSE) {
      throw new \Exception("git not clean: " . $status_output);
    }
    return TRUE;
  }

  /**
   * Gets the current git branch.
   *
   * @return string
   *   The current git branch.
   */
  private static function getCurrentBranch(): string {
    return trim(shell_exec('git rev-parse --abbrev-ref HEAD'));
  }

  /**
   * Switches to the branches we need.
   *
   * @param string $core_dir
   *   The path to the root of Drupal Core.
   * @param string $core_branch
   *   The core merge request branch.
   * @param string $contrib_branch
   *   The contrib branch to switch to.
  private static function switchToBranches(string $core_dir, string $core_branch, string $contrib_branch): void {
    static::switchToBranch($contrib_branch);
    chdir($core_dir);
    static::switchToBranch($core_branch);
  }

  /**
   * Switches to a branches and makes sure it is clean.
   *
   * @param string $branch
   *   The branch to switch to.
   */
  private static function switchToBranch(string $branch): void {
    static::ensureGitClean();
    shell_exec("git checkout $branch");
    if ($branch !== static::getCurrentBranch()) {
      throw new \Exception("could not checkout $branch");
  /**
   * Makes the commit to the merge request.
   *
   * Should only be used if core code checks fail for a known reason that can
   * be ignored.
   *
   * @param \Composer\Script\Event $event
   *   The Composer event.
   */
  public static function makeCommit(Event $event): void {
    $args = $event->getArguments();
    $count_arg = count($args);
    if ($count_arg !== 1) {
      throw new \Exception("This scripts 1 required arguments: a directory that is a core clone");
    }
    $core_dir = $args[0];
    static::doMakeCommit($core_dir);
  }

  /**
   * Makes commit in the root of Drupal Core.
   *
   * @param string $core_dir
   *   The path to the root of Drupal Core.
   */
  private static function doMakeCommit(string $core_dir): void {
    chdir(self::getContribDir());
    self::ensureGitClean();
    $hash = trim(shell_exec('git rev-parse HEAD'));
    $message = trim(shell_exec("git show -s --format='%s'"));
    chdir($core_dir);
    // Make sure ALL files are committed, including the core/modules/package_manager/tests/fixtures/fake_site/core directory!
    shell_exec('git add -f core/modules/package_manager');
    shell_exec("git commit -m 'Contrib: $message - https://git.drupalcode.org/project/automatic_updates/-/commit/$hash'");
  }

  /**
   * Adds new words to cspell dictionary.
   *
   * @param string $core_dir
   *   The path to the root of Drupal Core.
   * @param string $dict_file_to_merge
   *   The path to the dictionary file with additional words.
   */
  private static function addWordsToDictionary(string $core_dir, string $dict_file_to_merge): void {
    if (!file_exists($dict_file_to_merge)) {
      throw new \LogicException(sprintf('%s does not exist', $dict_file_to_merge));
    }
    $dict_file = $core_dir . '/core/misc/cspell/dictionary.txt';
    $contents = file_get_contents($dict_file);
    $words = explode("\n", $contents);
    $words = array_filter($words);
    $new_words = explode("\n", file_get_contents($dict_file_to_merge));
    $words = array_merge($words, $new_words);
    $words = array_filter(array_unique($words));
    natcasesort($words);
    file_put_contents($dict_file, implode("\n", $words) . "\n");
  }

  /**
   * Runs code quality checks.
   *
   * @param string $core_dir
   *   The path to the root of Drupal Core.
   */
  private static function runCoreChecks(string $core_dir): void {
    chdir($core_dir);
    $result = NULL;
    system(' sh ./core/scripts/dev/commit-code-check.sh --branch 11.x', $result);
    if ($result !== 0) {
      print "😭commit-code-check.sh failed";
      print "Reset using this command in the core checkout:";
      print "  rm -rf core/modules/package_manager && git checkout -- core && cd core && yarn install && cd ..";
      exit(1);
    }
    print "🎉 commit-code-check.sh passed!";
  }

  /**
   * Removes lines from the module based on a starting and ending token.
   *
   * These are lines that are not needed in core at all.
   *
   * @param string $core_dir
   *   The path to the root of Drupal Core.
   */
  private static function removeLines(string $core_dir): void {
    $files = static::getDirContents(static::getCoreModulePath($core_dir), TRUE);
    foreach ($files as $file) {
      $filePath = $file->getRealPath();
      $contents = file_get_contents($filePath);
      $lines = explode("\n", $contents);
      $skip = FALSE;
      $newLines = [];
      foreach ($lines as $line) {
        if (str_contains($line, '// BEGIN: DELETE FROM CORE MERGE REQUEST')) {
          if ($skip) {
            throw new \Exception("Already found begin");
          }
          $skip = TRUE;
        }
        if (!$skip) {
          $newLines[] = $line;
        }
        if (str_contains($line, '// END: DELETE FROM CORE MERGE REQUEST')) {
          if (!$skip) {
            throw new \Exception("Didn't find matching begin");
          }
          $skip = FALSE;
        }
      }
      if ($skip) {
        throw new \Exception("Didn't find ending token");
      }
      // Remove extra blank.
      $newLineCnt = count($newLines);
      if ($newLineCnt > 1) {
        if ($newLines[count($newLines) - 1] === '' && $newLines[count($newLines) - 2] === '') {
          array_pop($newLines);
        }
      }
      else {
        print "\n**Small new line cnt: in $file**\n";
      file_put_contents($filePath, implode("\n", $newLines));
    }
  }

  /**
   * Move scripts.
   *
   * @param string $core_dir
   *   The core directory.
   * @param string $core_module_path
   *   The core module path.
   * @param bool $package_manager_only
   *   Whether we are only converting package manager.
   */
  protected static function moveScripts(string $core_dir, string $core_module_path, bool $package_manager_only): void {
    $fs = new Filesystem();
    $new_fixture_creator_path = "$core_dir/core/scripts/PackageManagerFixtureCreator.php";

    $move_files = [
      $core_module_path . '/scripts/PackageManagerFixtureCreator.php' => $new_fixture_creator_path,
    ];
    $new_auto_update_path = "$core_dir/core/scripts/auto-update";
    if ($package_manager_only) {
      if (file_exists($new_auto_update_path)) {
        $fs->remove($new_auto_update_path);
      }
    }
    else {
      $new_auto_update_path = "$core_dir/core/scripts/auto-update";
      $move_files[$core_module_path . '/auto-update'] = $new_auto_update_path;
    }
    foreach ($move_files as $old_file => $new_file) {
      $fs->remove($new_file);
      $fs->rename($old_file, $new_file);
      $fs->chmod($new_file, 0644);
    }
    $script_replacements = [
      "__DIR__ . '/../../../autoload.php'" => "__DIR__ . '/../../autoload.php'",
      "__DIR__ . '/../package_manager/tests/fixtures/fake_site'" => "__DIR__ . '/../modules/package_manager/tests/fixtures/fake_site'",
      "CORE_ROOT_PATH = __DIR__ . '/../../../'" => "CORE_ROOT_PATH = __DIR__ . '/../..'",
      "new Process(['composer', 'phpcbf'], self::FIXTURE_PATH);" => "new Process(['composer', 'phpcbf', self::FIXTURE_PATH], self::CORE_ROOT_PATH);",
    ];
    foreach ($script_replacements as $search => $replace) {
      static::replaceContents([new \SplFileInfo($new_fixture_creator_path)], $search, $replace);
    }
    if (!$package_manager_only) {
      static::replaceContents(
        [new \SplFileInfo($new_auto_update_path)],
        "__DIR__ . '/src/Commands'",
        "__DIR__ . '/../modules/auto_updates/src/Commands'"
      );
    }

  }

  /**
   * Copies a generic test into the new module.
   *
   * @param string $new_module_path
   *   The module path.
   * @param string $core_dir
   *   The core dir.
   */
  private static function copyGenericTest(string $new_module_path, string $core_dir): void {
    $parts = explode('/', $new_module_path);
    $module_name = array_pop($parts);
    $original_test = "$core_dir/core/modules/action/tests/src/Functional/GenericTest.php";
    $new_test = "$new_module_path/tests/src/Functional/GenericTest.php";
    $fs = new Filesystem();
    $fs->copy($original_test, $new_test);
    static::replaceContents([new \SplFileInfo($new_test)], 'action', $module_name);
  }

  private function getContribModuleComposerJsonSection(string $section): array {
    $composer_json = json_decode(file_get_contents($this->contrib_dir . '/composer.json'), TRUE);
    return $composer_json[$section];
  }