<?php declare(strict_types = 1); namespace Drupal\automatic_updates\Development; use Composer\Script\Event; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Input\InputOption; use Symfony\Component\Console\Output\OutputInterface; use Symfony\Component\Filesystem\Filesystem; /** * Converts the contrib module to core merge request. * * Cspell:disable. * * File usage: * * This command can be used in 2 ways. * * ##### Local conversion when working on core merge requests. ##### * * This requires a local core clone and a local contrib clone. The contrib clone * should be on 3.0.x unless you are working on an issue that changes this file * itself. The core clone should be on the core merge request branch. The * contrib clone should NOT be inside the core clone. Both repositories should * have a clean git status. * * If just testing the conversion or if you want to run code checks or phpunit * tests locally on the core converted version you can just make a local branch * in the core of 11.x or use 11.x directly if you are going to commit. * * @code * composer core-covert /path/to/core core-branch * @endcode * * For example to update the Package Manager core merge requests. * * 1. Checkout 3.0.x on the core repo * 2. Checkout the 3346707-package-manager branch from * https://drupal.org/9/3346707 * 3. Run: * `composer core-convert /path/to/core 3346707-package-manager --package_manager_only` * 4. If the core checks pass the script will make a commit. * 5. Sanity check files changes in the commit to ensure they are to * package manager only or related core composer files if needed. * 6. To be extra careful you can run some or all phpunit tests locally on the * core converted version. Running the kernel tests or least the unit tests * might be a good idea. * 7. Push the commit to the core merge request. * * * ##### .gitlab-ci.yml usage ##### * Using the --gitlabci option tt is also used inside `.gitlab-ci.yml` to * convert the contrib module to the core merge request version and run tests. * This is to ensure, as much as possible, we can also convert to the core * version and expect code quality checks and tests to pass. * * The core clone should already have the core merge request locally. */ class ConverterCommand extends Command { private string $core_dir; private string|null $core_branch; private string|null $contrib_branch; private bool $package_manager_only; private string $core_target_branch; private string $contrib_dir; private bool $no_commit; private bool $skipCoreChecks; private bool $gitlabci; /** * {@inheritdoc} */ protected function configure() { $this->addArgument('core_dir', InputArgument::REQUIRED, 'The path to the root of Drupal Core'); $this->addArgument('core_branch', InputArgument::OPTIONAL, 'The core merge request branch'); $this->addArgument('contrib_branch', InputArgument::OPTIONAL, 'The contrib branch to switch to', '3.0.x'); $this->addOption('package_manager_only', NULL, InputOption::VALUE_NONE, 'Only convert package manager'); $this->addOption('core_target_branch', NULL, InputOption::VALUE_REQUIRED, 'The core target branch', '11.x'); $this->addOption('skip_core_checks', NULL, InputOption::VALUE_NONE, 'Skip core checks'); $this->addOption('no_commit', NULL, InputOption::VALUE_NONE, 'Do not make commit'); $this->addOption('gitlabci', NULL, InputOption::VALUE_NONE, 'Run in Gitlab CI'); } /** * {@inheritdoc} */ protected function execute(InputInterface $input, OutputInterface $output) { $this->core_dir = realpath($input->getArgument('core_dir')); $this->core_branch = $input->getArgument('core_branch'); $this->contrib_branch = $input->getArgument('contrib_branch'); $this->package_manager_only = $input->getOption('package_manager_only'); $this->core_target_branch = $input->getOption('core_target_branch'); $this->contrib_dir = realpath(__DIR__ . '/../..'); $this->skipCoreChecks = $input->getOption('skip_core_checks'); $this->no_commit = $input->getOption('no_commit'); $this->gitlabci = $input->getOption('gitlabci'); if (!$this->gitlabci) { if (empty($this->core_branch)) { throw new \Exception("core_branch is required if not on gitlabci"); } } else { if (!empty($this->core_branch)) { throw new \Exception("branches are not allowed on gitlabci"); } } // Esnure core_dir is a directory. if (!is_dir($this->core_dir)) { throw new \Exception("$this->core_dir is not a directory."); } chdir($this->core_dir); if ($this->gitlabci) { $this->no_commit = TRUE; $this->skipCoreChecks = TRUE; } else { // Ensure the core directory is clean. static::switchToBranch($this->core_target_branch); // Git pull to ensure we are up to date. static::executeCommand('git pull'); // change back the previous directory. chdir($this->contrib_dir); // Ensure we are on the correct branches. static::switchToBranches($this->core_dir, $this->core_branch, $this->contrib_branch); // Switch to the core directory and checkout the files and folders that this // conversion script will automatically update based on our composer.json // file and our dictionary.txt file. chdir($this->core_dir); static::executeCommand("git checkout {$this->core_target_branch} -- composer.json"); static::executeCommand("git checkout {$this->core_target_branch} -- composer.lock"); static::executeCommand("git checkout {$this->core_target_branch} -- composer"); static::executeCommand("git checkout {$this->core_target_branch} -- core/composer.json"); static::executeCommand("git checkout {$this->core_target_branch} -- core/misc/cspell/dictionary.txt"); // Check out files that will be patched. static::executeCommand("git checkout {$this->core_target_branch} -- core/lib/Drupal/Core/EventSubscriber/FinishResponseSubscriber.php"); } // Apply any core patches. $patches = glob(__DIR__ . '/../core-patches/*.patch'); static::info("Applying core patches"); print_r($patches); foreach ($patches as $file) { $patch_result = NULL; $output = []; // Patch but ignore previously applied patches. exec("patch -d{$this->core_dir} -p1 -N -r - < $file", $output, $patch_result); if ($patch_result !== 0) { if (!str_contains(implode("\n", $output), 'Ignoring previously applied (or reversed) patch.')) { throw new \Exception("Patch failed: $file"); } } } // Switch to the core directory and require all of the packages there are in // this module's composer.json require section. $require_section = $this->getContribModuleComposerJsonSection('require'); // Hardcode symfony version to 6.4 for core. $core_symfony_version_update = function ($package_name, $version): string { if (str_starts_with($package_name, 'symfony/')) { return '^6.4'; } return $version; }; foreach ($require_section as $package_name => $version) { $version = $core_symfony_version_update($package_name, $version); static::executeCommand("composer require $package_name:$version --no-update --working-dir={$this->core_dir}/core"); } // Run composer update for just this packages. static::executeCommand("composer update " . implode(' ', array_keys($require_section))); $require_dev_section = $this->getContribModuleComposerJsonSection('require-dev'); foreach ($require_dev_section as $package_name => $version) { $version = $core_symfony_version_update($package_name, $version); static::executeCommand("composer require --dev $package_name:$version"); } $this->doConvert(); return Command::SUCCESS; } /** * Executes a command and throws an exception if it fails. * * @param string $cmd * The command to execute. */ private static function executeCommand(string $cmd): void { $result = NULL; system($cmd, $result); if ($result !== 0) { throw new \Exception("Command failed: $cmd"); } } /** * Prints message. * * @param string $msg * The message to print. */ private static function info(string $msg): void { print "\n$msg"; } /** * Converts the contrib module to core merge request. */ private function doConvert(): void { $old_machine_name = 'automatic_updates'; $new_machine_name = 'auto_updates'; self::info('Switched branches'); $fs = new Filesystem(); $core_module_path = static::getCoreModulePath($this->core_dir); $package_manager_core_path = $this->core_dir . "/core/modules/package_manager"; // Remove old module. $fs->remove($core_module_path); self::info('Removed old core module'); $fs->remove($package_manager_core_path); self::info("Removed package manager"); $fs->mirror(self::getContribDir(), $core_module_path); self::info('Mirrored into core module'); $replacements = [ // This allows line that only need to be in the core MR to be commented // out. '// CORE_MR_ONLY:' => '', // Note this following line could leave 'CORE_MR_ONLY-10.1.x' comments in // the core MR, but we currently do not have any. Leaving // 'CORE_MR_ONLY-11.x' in is fine because we don't actually convert to // 10.1.x for a core MR. We might want to just remove the whole line if // it does not match the core target branch. "// CORE_MR_ONLY-{$this->core_target_branch}:" => '', $old_machine_name => $new_machine_name, 'AutomaticUpdates' => 'AutoUpdates', "__DIR__ . '/../../../package_manager/tests" => "__DIR__ . '/../../../../Xpackage_manager/tests", "__DIR__ . '/../../../../package_manager/tests" => "__DIR__ . '/../../../../../Xpackage_manager/tests", "__DIR__ . '/../../../../../package_manager/tests" => "__DIR__ . '/../../../../../../Xpackage_manager/tests", '/Xpackage_manager' => '/package_manager', ]; foreach ($replacements as $search => $replace) { static::renameFiles(static::getDirContents($core_module_path), $search, $replace); static::replaceContents(static::getDirContents($core_module_path, TRUE), $search, $replace); } self::info('Replacements done.'); static::removeLines($this->core_dir); self::info('Remove unneeded lines'); self::moveScripts($this->core_dir, $core_module_path, $this->package_manager_only); self::info('Moved scripts'); // Remove unneeded. $removals = [ 'auto_updates_extensions', 'drupalci.yml', 'README.md', '.cspell.json', '.git', 'composer.json', '.gitattributes', '.gitignore', 'DEVELOPING.md', 'phpstan.neon.dist', // @todo Move ComposerFixtureCreator to its location in core // https://drupal.org/i/3347937. 'scripts', 'dictionary.txt', ]; $removals = array_map(function ($path) use ($core_module_path) { return "$core_module_path/$path"; }, $removals); $fs->remove($removals); self::info('Remove not needed'); // Replace in file names and contents. static::replaceContents( [ new \SplFileInfo("$core_module_path/auto_updates.info.yml"), new \SplFileInfo("$core_module_path/package_manager/package_manager.info.yml"), ], "core_version_requirement: ^10.1", "package: Core\nversion: VERSION\nlifecycle: experimental", ); $fs->rename("$core_module_path/package_manager", $package_manager_core_path); self::info('Move package manager'); static::copyGenericTest($package_manager_core_path, $this->core_dir); // Run phpcbf because removing code from merge request may result in unused // use statements or multiple empty lines. $contrib_dir = self::getContribDir(); system("composer run phpcbf --working-dir=$contrib_dir -- $package_manager_core_path"); if ($this->package_manager_only) { $fs->remove($core_module_path); } else { static::copyGenericTest($core_module_path, $this->core_dir); system("composer run phpcbf --working-dir=$contrib_dir -- $$core_module_path"); } static::addWordsToDictionary($this->core_dir, $contrib_dir . "/dictionary.txt"); self::info("Added to dictionary"); chdir($this->core_dir); if ($this->skipCoreChecks) { self::info('⚠️Skipped core checks'); } else { static::runCoreChecks($this->core_dir); self::info('Ran core checks'); } if (!$this->no_commit) { static::doMakeCommit($this->core_dir); self::info('Make commit'); self::info("Done. Probably good but you should check before you push. These are the files present in the contrib module absent in core:"); print shell_exec(sprintf("tree %s/package_manager > /tmp/contrib.txt && tree %s/core/modules/package_manager > /tmp/core.txt && diff /tmp/contrib.txt /tmp/core.txt", self::getContribDir(), $this->core_dir)); self::info('(Run diff /tmp/contrib.txt /tmp/core.txt to see that with color.'); } } /** * Returns the path to the root of the contrib module. * * @return string * The full path to the root of the contrib module. */ private static function getContribDir(): string { return realpath(__DIR__ . '/../..'); } /** * Returns the path where the contrib module will be placed in Drupal Core. * * @param string $core_dir * The path to the root of Drupal Core. * * @return string * The path where the contrib module will be placed in Drupal Core */ private static function getCoreModulePath(string $core_dir): string { return $core_dir . '/core/modules/auto_updates'; } /** * Replaces a string in the contents of the module files. * * @param array $files * Files to replace. * @param string $search * The string to be replaced. * @param string $replace * The string to replace. */ private static function replaceContents(array $files, string $search, string $replace): void { foreach ($files as $file) { $filePath = $file->getRealPath(); file_put_contents($filePath, str_replace($search, $replace, file_get_contents($filePath))); } } /** * Renames the module files. * * @param array $files * Files to replace. * @param string $old_pattern * The old file name. * @param string $new_pattern * The new file name. */ private static function renameFiles(array $files, string $old_pattern, string $new_pattern): void { // Keep a record of the files and directories to change. // We will change all the files first, so we don't change the location of // any files in the middle. This probably won't work if we had nested // folders with the pattern on 2 folder levels, but we don't. $filesToChange = []; $dirsToChange = []; foreach ($files as $file) { $fileName = $file->getFilename(); if ($fileName === '.') { $fullPath = $file->getPath(); $parts = explode('/', $fullPath); $name = array_pop($parts); $path = "/" . implode('/', $parts); } else { $name = $fileName; $path = $file->getPath(); } if (strpos($name, $old_pattern) !== FALSE) { $new_filename = str_replace($old_pattern, $new_pattern, $name); if ($file->isFile()) { $filesToChange[$file->getRealPath()] = $file->getPath() . "/$new_filename"; } else { // Store directories by path depth. $depth = count(explode('/', $path)); $dirsToChange[$depth][$file->getRealPath()] = "$path/$new_filename"; } } } foreach ($filesToChange as $old => $new) { (new Filesystem())->rename($old, $new); } // Rename directories starting with the most nested to avoid renaming // parents directories first. krsort($dirsToChange); foreach ($dirsToChange as $dirs) { foreach ($dirs as $old => $new) { (new Filesystem())->rename($old, $new); } } } /** * Gets the contents of a directory. * * @param string $path * The path of the directory. * @param bool $excludeDirs * (optional) If TRUE, all directories will be excluded. Defaults to FALSE. * * @return \SplFileInfo[] * Array of objects containing file information. */ private static function getDirContents(string $path, bool $excludeDirs = FALSE): array { $rii = new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator($path)); $files = []; /** @var \SplFileInfo $file */ foreach ($rii as $file) { // Exclude the .git directories always. if ($file->getFilename() === '.git') { continue; } if (str_contains($file->getRealPath(), '/.git/') || str_ends_with($file->getRealPath(), '/.git')) { continue; } if ($excludeDirs && $file->isDir()) { continue; } $files[] = $file; } return $files; } /** * Ensures the git status is clean. * * @return bool * TRUE if git status is clean , otherwise returns a exception. */ private static function ensureGitClean(): bool { $status_output = shell_exec('git status'); if (strpos($status_output, 'nothing to commit, working tree clean') === FALSE) { throw new \Exception("git not clean: " . $status_output); } return TRUE; } /** * Gets the current git branch. * * @return string * The current git branch. */ private static function getCurrentBranch(): string { return trim(shell_exec('git rev-parse --abbrev-ref HEAD')); } /** * Switches to the branches we need. * * @param string $core_dir * The path to the root of Drupal Core. * @param string $core_branch * The core merge request branch. * @param string $contrib_branch * The contrib branch to switch to. */ private static function switchToBranches(string $core_dir, string $core_branch, string $contrib_branch): void { static::switchToBranch($contrib_branch); chdir($core_dir); static::switchToBranch($core_branch); } /** * Switches to a branches and makes sure it is clean. * * @param string $branch * The branch to switch to. */ private static function switchToBranch(string $branch): void { static::ensureGitClean(); shell_exec("git checkout $branch"); if ($branch !== static::getCurrentBranch()) { throw new \Exception("could not checkout $branch"); } } /** * Makes the commit to the merge request. * * Should only be used if core code checks fail for a known reason that can * be ignored. * * @param \Composer\Script\Event $event * The Composer event. */ public static function makeCommit(Event $event): void { $args = $event->getArguments(); $count_arg = count($args); if ($count_arg !== 1) { throw new \Exception("This scripts 1 required arguments: a directory that is a core clone"); } $core_dir = $args[0]; static::doMakeCommit($core_dir); } /** * Makes commit in the root of Drupal Core. * * @param string $core_dir * The path to the root of Drupal Core. */ private static function doMakeCommit(string $core_dir): void { chdir(self::getContribDir()); self::ensureGitClean(); $hash = trim(shell_exec('git rev-parse HEAD')); $message = trim(shell_exec("git show -s --format='%s'")); chdir($core_dir); // Make sure ALL files are committed, including the core/modules/package_manager/tests/fixtures/fake_site/core directory! shell_exec('git add -f core/modules/package_manager'); shell_exec('git add .'); shell_exec("git commit -m 'Contrib: $message - https://git.drupalcode.org/project/automatic_updates/-/commit/$hash'"); } /** * Adds new words to cspell dictionary. * * @param string $core_dir * The path to the root of Drupal Core. * @param string $dict_file_to_merge * The path to the dictionary file with additional words. */ private static function addWordsToDictionary(string $core_dir, string $dict_file_to_merge): void { if (!file_exists($dict_file_to_merge)) { throw new \LogicException(sprintf('%s does not exist', $dict_file_to_merge)); } $dict_file = $core_dir . '/core/misc/cspell/dictionary.txt'; $contents = file_get_contents($dict_file); $words = explode("\n", $contents); $words = array_filter($words); $new_words = explode("\n", file_get_contents($dict_file_to_merge)); $words = array_merge($words, $new_words); $words = array_filter(array_unique($words)); natcasesort($words); file_put_contents($dict_file, implode("\n", $words) . "\n"); } /** * Runs code quality checks. * * @param string $core_dir * The path to the root of Drupal Core. */ private static function runCoreChecks(string $core_dir): void { chdir($core_dir); $result = NULL; system(' sh ./core/scripts/dev/commit-code-check.sh --branch 11.x', $result); if ($result !== 0) { print "😭commit-code-check.sh failed"; print "Reset using this command in the core checkout:"; print " rm -rf core/modules/package_manager && git checkout -- core && cd core && yarn install && cd .."; exit(1); } print "🎉 commit-code-check.sh passed!"; } /** * Removes lines from the module based on a starting and ending token. * * These are lines that are not needed in core at all. * * @param string $core_dir * The path to the root of Drupal Core. */ private static function removeLines(string $core_dir): void { $files = static::getDirContents(static::getCoreModulePath($core_dir), TRUE); foreach ($files as $file) { $filePath = $file->getRealPath(); $contents = file_get_contents($filePath); $lines = explode("\n", $contents); $skip = FALSE; $newLines = []; foreach ($lines as $line) { if (str_contains($line, '// BEGIN: DELETE FROM CORE MERGE REQUEST')) { if ($skip) { throw new \Exception("Already found begin"); } $skip = TRUE; } if (!$skip) { $newLines[] = $line; } if (str_contains($line, '// END: DELETE FROM CORE MERGE REQUEST')) { if (!$skip) { throw new \Exception("Didn't find matching begin"); } $skip = FALSE; } } if ($skip) { throw new \Exception("Didn't find ending token"); } // Remove extra blank. $newLineCnt = count($newLines); if ($newLineCnt > 1) { if ($newLines[count($newLines) - 1] === '' && $newLines[count($newLines) - 2] === '') { array_pop($newLines); } } else { print "\n**Small new line cnt: in $file**\n"; } file_put_contents($filePath, implode("\n", $newLines)); } } /** * Move scripts. * * @param string $core_dir * The core directory. * @param string $core_module_path * The core module path. * @param bool $package_manager_only * Whether we are only converting package manager. */ protected static function moveScripts(string $core_dir, string $core_module_path, bool $package_manager_only): void { $fs = new Filesystem(); $new_fixture_creator_path = "$core_dir/core/scripts/PackageManagerFixtureCreator.php"; $move_files = [ $core_module_path . '/scripts/PackageManagerFixtureCreator.php' => $new_fixture_creator_path, ]; $new_auto_update_path = "$core_dir/core/scripts/auto-update"; if ($package_manager_only) { if (file_exists($new_auto_update_path)) { $fs->remove($new_auto_update_path); } } else { $new_auto_update_path = "$core_dir/core/scripts/auto-update"; $move_files[$core_module_path . '/auto-update'] = $new_auto_update_path; } foreach ($move_files as $old_file => $new_file) { $fs->remove($new_file); $fs->rename($old_file, $new_file); $fs->chmod($new_file, 0644); } $script_replacements = [ "__DIR__ . '/../../../autoload.php'" => "__DIR__ . '/../../autoload.php'", "__DIR__ . '/../package_manager/tests/fixtures/fake_site'" => "__DIR__ . '/../modules/package_manager/tests/fixtures/fake_site'", "CORE_ROOT_PATH = __DIR__ . '/../../../'" => "CORE_ROOT_PATH = __DIR__ . '/../..'", "new Process(['composer', 'phpcbf'], self::FIXTURE_PATH);" => "new Process(['composer', 'phpcbf', self::FIXTURE_PATH], self::CORE_ROOT_PATH);", ]; foreach ($script_replacements as $search => $replace) { static::replaceContents([new \SplFileInfo($new_fixture_creator_path)], $search, $replace); } if (!$package_manager_only) { static::replaceContents( [new \SplFileInfo($new_auto_update_path)], "__DIR__ . '/src/Commands'", "__DIR__ . '/../modules/auto_updates/src/Commands'" ); } } /** * Copies a generic test into the new module. * * @param string $new_module_path * The module path. * @param string $core_dir * The core dir. */ private static function copyGenericTest(string $new_module_path, string $core_dir): void { $parts = explode('/', $new_module_path); $module_name = array_pop($parts); $original_test = "$core_dir/core/modules/action/tests/src/Functional/GenericTest.php"; $new_test = "$new_module_path/tests/src/Functional/GenericTest.php"; $fs = new Filesystem(); $fs->copy($original_test, $new_test); static::replaceContents([new \SplFileInfo($new_test)], 'action', $module_name); } private function getContribModuleComposerJsonSection(string $section): array { $composer_json = json_decode(file_get_contents($this->contrib_dir . '/composer.json'), TRUE); return $composer_json[$section]; } }