Issue #3358878 by yash.rode, Wim Leers, tedbow: Test failing on 8.x-2.x on 10.0.x and 10.1.x

.cspell.json

+{
+  "dictionaries": ["drupal","companies", "fonts", "html", "php", "softwareTerms", "automatic_updates"],
+  "dictionaryDefinitions": [
+    { "name": "automatic_updates", "path": "./dictionary.txt"}
+  ],
+  "ignorePaths": [
+    "pcre.ini",
+    "README.md"
+  ]
+}

automatic_updates_extensions/tests/src/Build/ModuleUpdateTest.php

@@ -95,6 +95,18 @@ END;
     $this->assertSame('1.1.0', $module_composer_json->version);
   }
 
+  /**
+   * {@inheritdoc}
+   */
+  public function copyCodebase(\Iterator $iterator = NULL, $working_dir = NULL): void {
+    parent::copyCodebase($iterator, $working_dir);
+
+    // Ensure that we will install Drupal 9.8.1 (a fake version that should
+    // never exist in real life), which is the latest security release in this
+    // simulation.
+    $this->setUpstreamCoreVersion('9.8.1');
+  }
+
   /**
    * Tests updating a module in a stage directory via the UI.
    */

dictionary.txt

@@ -8,3 +8,4 @@ colinodell
 testlogger
 unwritable
 filedate
+unshallow

drupalci.yml

@@ -9,42 +9,14 @@ build:
       # Run code quality checks.
       container_command.commit-checks:
         commands:
-          # Copy core's checking script into this contrib module.
-          - "cp /var/www/html/core/scripts/dev/commit-code-check.sh modules/contrib/automatic_updates/"
-          # Comply with core's checking script file permissions expectations.
-          - chmod 644 modules/contrib/automatic_updates/
-          # Rewrite $TOP_LEVEL/core since $TOP_LEVEL now refers to the contrib module.
-          - sed -i "s/\$TOP_LEVEL\/core/\/var\/www\/html\/core/" modules/contrib/automatic_updates/commit-code-check.sh
-          # Ensure the remainder of the script runs from the script's directory and not the current working directory.
-          - sed -i "s/# Gets list of files to check./cd \"\$\(dirname \"\$0\"\)\";/" modules/contrib/automatic_updates/commit-code-check.sh
-          # When constructing $FILES, ignore the `commit-code-check.sh` file we just copied.
-          - sed -i "s/--exclude=vendor/--exclude=vendor --exclude=commit-code-check.sh/" modules/contrib/automatic_updates/commit-code-check.sh
-          # vendor/bin/phpcs now needs to be prefixed. And $TOP_LEVEL/vendor needs to be rewritten. And add verbose output.
-          - sed -i "s/vendor\/bin\/phpcs/\/var\/www\/html\/vendor\/bin\/phpcs/" modules/contrib/automatic_updates/commit-code-check.sh
-          - sed -i "s/\$TOP_LEVEL\/vendor/\/var\/www\/html\/vendor/" modules/contrib/automatic_updates/commit-code-check.sh
-          - sed -i "s/phpcs -ps/phpcs \$TOP_LEVEL -ps/" modules/contrib/automatic_updates/commit-code-check.sh
-          # Disable all JS compile checks until we've matched core's build process.
-          # @todo Remove this in https://www.drupal.org/project/automatic_updates/issues/3221082.
-          - sed -i "s/COMPILE_CHECK=1/COMPILE_CHECK=0/" modules/contrib/automatic_updates/commit-code-check.sh
-          - sed -i "s/core\/scripts\/js/js/" modules/contrib/automatic_updates/commit-code-check.sh
-          # Uncomment to Check all files
-          #- sed -i "s/git diff --name-only HEAD~1 HEAD/find \* -type f -not -path \"\.\/\.git\*\"/" modules/contrib/automatic_updates/commit-code-check.sh
-          # Make cspell examine our files.
-          - sed -i "s/yarn run -s spellcheck/yarn run -s spellcheck --root \$TOP_LEVEL/" modules/contrib/automatic_updates/commit-code-check.sh
-          # Add our words to the dictionary.
-          - cat modules/contrib/automatic_updates/dictionary.txt >> core/misc/cspell/dictionary.txt
-          # Ensure we have the full path to PHPStan.
-          - sed -i "s/vendor\/bin\/phpstan/\/var\/www\/html\/vendor\/bin\/phpstan/" modules/contrib/automatic_updates/commit-code-check.sh
-          # After all of the shenanigans above, we're finally ready to run core's `commit-code-check.sh`! :)
-          - "modules/contrib/automatic_updates/commit-code-check.sh --drupalci"
-          # Restore the original permissions.
-          - chmod 777 modules/contrib/automatic_updates/
           # Disable the PCRE engine's JIT, since it causes Composer to die during the
           # update process, but only on Drupal CI, and for reasons that are essentially
           # impossible to trace into. The PCRE JIT is not necessary for Automatic Updates
           # to work correctly, and disabling it is a known workaround.
           # @see pcre.ini
           - sudo cp modules/contrib/automatic_updates/pcre.ini /usr/local/etc/php/conf.d
+          # @todo Replace in favor of commit-code-check.sh once https://www.drupal.org/project/drupal/issues/3314100 lands.
+          - modules/contrib/automatic_updates/scripts/commit-code-check.sh --drupalci
         halt-on-fail: true
       # run_tests task is executed several times in order of performance speeds.
       # halt-on-fail can be set on the run_tests tasks in order to fail fast.

package_manager/package_manager.api.php

@@ -6,6 +6,8 @@
  */
 
 /**
+ * @defgroup package_manager_architecture Package Manager architecture
+ *
  * Package Manager is an API-only module which provides the scaffolding and
  * functionality needed for Drupal to make changes to its own running code base
  * via Composer. It doesn't provide any user interface.

package_manager/tests/src/Build/TemplateProjectTestBase.php

@@ -354,6 +354,11 @@ END;
         if (str_starts_with(\Drupal::VERSION, '10.')) {
           unset($package['require']['symfony/polyfill-php80']);
         }
+        // If we're running on Drupal 10.1, which requires PHP 8.1 or later, this
+        // polyfill won't be installed, so make sure it's not required.
+        if (str_starts_with(\Drupal::VERSION, '10.1')) {
+          unset($package['require']['symfony/polyfill-php81']);
+        }
         // Disabling symlinks in the transport options doesn't seem to have an
         // effect, so we use the COMPOSER_MIRROR_PATH_REPOS environment variable
         // to force mirroring in ::createTestProject().

scripts/commit-code-check.sh

+#!/usr/bin/env bash
+
+# NAME
+#     commit-code-check.sh - Run all code quality checks.
+#
+# SYNOPSIS
+#     bash scripts/commit-code-check.sh
+#
+# DESCRIPTION
+#     Performs the following quality checks, closely matching core's commit-code-check.sh:
+#     - Spell checking.
+#     - PHPCS checks PHP and YAML files.
+#     - PHPStan checks PHP files.
+#     - ESLint checks YAML files.
+
+# cSpell:disable
+
+cd "$(dirname "$0")/../" || exit;
+
+DRUPALCI=0
+while test $# -gt 0; do
+  case "$1" in
+    -h|--help)
+      echo "Drupal code quality checks"
+      echo " "
+      echo "options:"
+      echo "-h, --help                show brief help"
+      echo "--drupalci                a special mode for DrupalCI"
+      echo " "
+      exit 0
+      ;;
+    --drupalci)
+      DRUPALCI=1
+      shift
+      ;;
+    *)
+      break
+      ;;
+  esac
+done
+
+MODULE_DIRECTORY=$(pwd)
+
+# Find the site root directory. Check up to three directories above.
+DIR=$(pwd)
+for i in {0..3}; do
+  DIR=$(dirname "$DIR")
+  if test -f "$DIR/core/composer.json"; then
+    CORE_DIRECTORY="$DIR"
+    break
+  fi
+done
+
+# Set up variables to make colored output simple. Color output is disabled on
+# DrupalCI because it is breaks reporting.
+# @todo https://www.drupal.org/project/drupalci_testbot/issues/3181869
+if [[ "$DRUPALCI" == "1" ]]; then
+  red=""
+  green=""
+  reset=""
+else
+  red=$(tput setaf 1 && tput bold)
+  green=$(tput setaf 2)
+  title=$(tput setaf 4 && tput bold)
+  reset=$(tput sgr0)
+fi
+
+# This script assumes that composer install and yarn install have already been
+# run and all dependencies are updated.
+FINAL_STATUS=0
+
+print_separator() {
+  printf "\n${title}"
+  printf -- '-%.0s' {1..100}
+  printf "${reset}\n"
+}
+print_title() {
+  print_separator
+  printf "${title}$1${reset}"
+  print_separator
+}
+print_results() {
+  RESULTS=$1
+  LABEL=$2
+  if [ "$RESULTS" -ne "0" ]; then
+    # If there are failures set the status to a number other than 0.
+    FINAL_STATUS=1
+    printf "\n${title}$LABEL: ${red}failed${reset}\n"
+  else
+    printf "\n${title}$LABEL: ${green}passed${reset}\n"
+  fi
+}
+
+print_title "[1/4] PHPCS"
+$CORE_DIRECTORY/vendor/bin/phpcs $MODULE_DIRECTORY -ps --standard="$CORE_DIRECTORY/core/phpcs.xml.dist"
+print_results $? "PHPCS"
+
+# print_title "[2/4] PHPStan"
+#php -d apc.enabled=0 -d apc.enable_cli=0 $CORE_DIRECTORY/vendor/bin/phpstan analyze --no-progress --configuration="$CORE_DIRECTORY/core/phpstan.neon.dist" $MODULE_DIRECTORY
+# print_results $? "PHPStan"
+
+print_title "[2/4] CSpell"
+cd $CORE_DIRECTORY/core && yarn run -s spellcheck --no-progress --root $MODULE_DIRECTORY -c .cspell.json "**" && cd -
+print_results $? "CSpell"
+
+print_title "[3/4] eslint:yaml"
+cd $CORE_DIRECTORY/core && yarn eslint --resolve-plugins-relative-to . --ext .yml $MODULE_DIRECTORY && cd -
+print_results $? "eslint:yaml"
+print_separator
+
+if [[ "$FINAL_STATUS" == "1" ]]; then
+  printf "${red}Drupal code quality checks failed.${reset}"
+  if [[ "$DRUPALCI" == "1" ]]; then
+    printf "\nTo reproduce this output locally:\n"
+    printf "* Apply the change as a patch, or from the merge request branch\n"
+    printf "* Run this command locally: sh ./scripts/commit-code-check.sh"
+  fi
+  print_separator
+fi
+exit $FINAL_STATUS

scripts/src/Converter.php

@@ -345,8 +345,9 @@ class Converter {
   }
 
   /**
-   * Removes lines from the module based on a starting and ending token
-   * These are lines that are not need in core at all.
+   * Removes lines from the module based on a starting and ending token.
+   *
+   * These are lines that are not needed in core at all.
    *
    * @param string $core_dir
    *   The path to the root of Drupal Core.

tests/src/Build/CoreUpdateTest.php

@@ -7,7 +7,6 @@ namespace Drupal\Tests\automatic_updates\Build;
 use Behat\Mink\Element\DocumentElement;
 use Drupal\automatic_updates\CronUpdater;
 use Drupal\automatic_updates\Updater;
-use Drupal\Composer\Composer;
 use Drupal\package_manager\Event\PostApplyEvent;
 use Drupal\package_manager\Event\PostCreateEvent;
 use Drupal\package_manager\Event\PostDestroyEvent;
@@ -263,46 +262,6 @@ class CoreUpdateTest extends UpdateTestBase {
     }
   }
 
-  /**
-   * Sets the version of Drupal core to which the test site will be updated.
-   *
-   * @param string $version
-   *   The Drupal core version to set.
-   */
-  private function setUpstreamCoreVersion(string $version): void {
-    $workspace_dir = $this->getWorkspaceDirectory();
-
-    // Loop through core's metapackages and plugins, and alter them as needed.
-    $packages = str_replace("$workspace_dir/", '', $this->getCorePackages());
-    foreach ($packages as $path) {
-      // Assign the new upstream version.
-      $this->runComposer("composer config version $version", $path);
-
-      // If this package requires Drupal core (e.g., drupal/core-recommended),
-      // make it require the new upstream version.
-      $info = $this->runComposer('composer info --self --format json', $path, TRUE);
-      if (isset($info['requires']['drupal/core'])) {
-        $this->runComposer("composer require --no-update drupal/core:$version", $path);
-      }
-    }
-
-    // Change the \Drupal::VERSION constant and put placeholder text in the
-    // README so we can ensure that we really updated to the correct version. We
-    // also change the default site configuration files so we can ensure that
-    // these are updated as well, despite `sites/default` being write-protected.
-    // @see ::assertUpdateSuccessful()
-    // @see ::createTestProject()
-    Composer::setDrupalVersion($workspace_dir, $version);
-    file_put_contents("$workspace_dir/core/README.txt", "Placeholder for Drupal core $version.");
-
-    foreach (['default.settings.php', 'default.services.yml'] as $file) {
-      $file = fopen("$workspace_dir/core/assets/scaffold/files/$file", 'a');
-      $this->assertIsResource($file);
-      fwrite($file, "# This is part of Drupal $version.\n");
-      fclose($file);
-    }
-  }
-
   /**
    * Asserts that a specific version of Drupal core is running.
    *

tests/src/Build/UpdateTestBase.php

@@ -5,6 +5,7 @@ declare(strict_types = 1);
 namespace Drupal\Tests\automatic_updates\Build;
 
 use Drupal\Component\Utility\Html;
+use Drupal\Composer\Composer;
 use Drupal\Tests\package_manager\Build\TemplateProjectTestBase;
 
 /**
@@ -90,4 +91,44 @@ END;
     $this->assertTrue($ready_text_found);
   }
 
+  /**
+   * Sets the version of Drupal core to which the test site will be updated.
+   *
+   * @param string $version
+   *   The Drupal core version to set.
+   */
+  protected function setUpstreamCoreVersion(string $version): void {
+    $workspace_dir = $this->getWorkspaceDirectory();
+
+    // Loop through core's metapackages and plugins, and alter them as needed.
+    $packages = str_replace("$workspace_dir/", '', $this->getCorePackages());
+    foreach ($packages as $path) {
+      // Assign the new upstream version.
+      $this->runComposer("composer config version $version", $path);
+
+      // If this package requires Drupal core (e.g., drupal/core-recommended),
+      // make it require the new upstream version.
+      $info = $this->runComposer('composer info --self --format json', $path, TRUE);
+      if (isset($info['requires']['drupal/core'])) {
+        $this->runComposer("composer require --no-update drupal/core:$version", $path);
+      }
+    }
+
+    // Change the \Drupal::VERSION constant and put placeholder text in the
+    // README so we can ensure that we really updated to the correct version. We
+    // also change the default site configuration files so we can ensure that
+    // these are updated as well, despite `sites/default` being write-protected.
+    // @see ::assertUpdateSuccessful()
+    // @see ::createTestProject()
+    Composer::setDrupalVersion($workspace_dir, $version);
+    file_put_contents("$workspace_dir/core/README.txt", "Placeholder for Drupal core $version.");
+
+    foreach (['default.settings.php', 'default.services.yml'] as $file) {
+      $file = fopen("$workspace_dir/core/assets/scaffold/files/$file", 'a');
+      $this->assertIsResource($file);
+      fwrite($file, "# This is part of Drupal $version.\n");
+      fclose($file);
+    }
+  }
+
 }

tests/src/Functional/UpdatePathTest.php

@@ -67,14 +67,23 @@ class UpdatePathTest extends UpdatePathTestBase {
 
     $this->assertSame(CronUpdater::DISABLED, $this->config('automatic_updates.settings')->get('cron'));
 
-    // TRICKY: we do expect `readiness_validation_last_run` to have been renamed
-    // to `status_check_last_run`, but then
-    // automatic_updates_post_update_create_status_check_mail_config() should
-    // cause that to be erased.
-    // @see automatic_updates_post_update_create_status_check_mail_config()
-    // @see \Drupal\automatic_updates\EventSubscriber\ConfigSubscriber::onConfigSave()
-    unset($expected_values['status_check_last_run']);
-    $this->assertSame($expected_values, $key_value->getMultiple(array_values($map)));
+    // For Drupal >=10.1, we do not care about the update path since 8.x-2.x is minimally maintained.
+    // @see https://www.drupal.org/project/automatic_updates/issues/3358878#comment-15044460
+    if (str_starts_with(\Drupal::VERSION, '10.1.') || version_compare(\Drupal::VERSION, '10.1', '>=')) {
+      foreach ($map as $new_key) {
+        $this->assertNotEmpty($key_value->get($new_key));
+      }
+    }
+    else {
+      // TRICKY: we do expect `readiness_validation_last_run` to have been renamed
+      // to `status_check_last_run`, but then
+      // automatic_updates_post_update_create_status_check_mail_config() should
+      // cause that to be erased.
+      // @see automatic_updates_post_update_create_status_check_mail_config()
+      // @see \Drupal\automatic_updates\EventSubscriber\ConfigSubscriber::onConfigSave()
+      unset($expected_values['status_check_last_run']);
+      $this->assertSame($expected_values, $key_value->getMultiple(array_values($map)));
+    }
     $this->assertSame(StatusCheckMailer::ERRORS_ONLY, $this->config('automatic_updates.settings')->get('status_check_mail'));
 
     $this->assertSame([], $this->config('package_manager.settings')->get('additional_trusted_composer_plugins'));