Issue #3477634 by mondrake, daffie: Ensure run-tests.sh and PHPUnit CLI run...

Issue #3477634 by mondrake, daffie: Ensure run-tests.sh and PHPUnit CLI run with the same list of tests to be executed

core/.phpstan-baseline.php

@@ -22388,22 +22388,16 @@
 	'path' => __DIR__ . '/modules/jsonapi/tests/src/Functional/ResourceTestBase.php',
 ];
 $ignoreErrors[] = [
-	'message' => '#^Method Drupal\\\\Tests\\\\jsonapi\\\\Functional\\\\RestJsonApiUnsupported\\:\\:assertAuthenticationEdgeCases\\(\\) has no return type specified\\.$#',
+	'message' => '#^Method Drupal\\\\Tests\\\\jsonapi\\\\Functional\\\\RestJsonApiUnsupportedTest\\:\\:assertResponseWhenMissingAuthentication\\(\\) has no return type specified\\.$#',
 	'identifier' => 'missingType.return',
 	'count' => 1,
-	'path' => __DIR__ . '/modules/jsonapi/tests/src/Functional/RestJsonApiUnsupported.php',
+	'path' => __DIR__ . '/modules/jsonapi/tests/src/Functional/RestJsonApiUnsupportedTest.php',
 ];
 $ignoreErrors[] = [
-	'message' => '#^Method Drupal\\\\Tests\\\\jsonapi\\\\Functional\\\\RestJsonApiUnsupported\\:\\:assertResponseWhenMissingAuthentication\\(\\) has no return type specified\\.$#',
+	'message' => '#^Method Drupal\\\\Tests\\\\jsonapi\\\\Functional\\\\RestJsonApiUnsupportedTest\\:\\:assertAuthenticationEdgeCases\\(\\) has no return type specified\\.$#',
 	'identifier' => 'missingType.return',
 	'count' => 1,
-	'path' => __DIR__ . '/modules/jsonapi/tests/src/Functional/RestJsonApiUnsupported.php',
-];
-$ignoreErrors[] = [
-	'message' => '#^Method Drupal\\\\Tests\\\\jsonapi\\\\Functional\\\\RestJsonApiUnsupported\\:\\:setUpAuthorization\\(\\) has no return type specified\\.$#',
-	'identifier' => 'missingType.return',
-	'count' => 1,
-	'path' => __DIR__ . '/modules/jsonapi/tests/src/Functional/RestJsonApiUnsupported.php',
+	'path' => __DIR__ . '/modules/jsonapi/tests/src/Functional/RestJsonApiUnsupportedTest.php',
 ];
 $ignoreErrors[] = [
 	'message' => '#^Method Drupal\\\\Tests\\\\jsonapi\\\\Functional\\\\ShortcutTest\\:\\:createEntityReferenceField\\(\\) has no return type specified\\.$#',
@@ -30476,10 +30470,10 @@
 	'path' => __DIR__ . '/modules/node/tests/src/Functional/NodeAccessCacheabilityTest.php',
 ];
 $ignoreErrors[] = [
-	'message' => '#^Method Drupal\\\\Tests\\\\node\\\\Functional\\\\NodeAccessCacheabilityWithNodeGrants\\:\\:createEntityReferenceField\\(\\) has no return type specified\\.$#',
+	'message' => '#^Method Drupal\\\\Tests\\\\node\\\\Functional\\\\NodeAccessCacheabilityWithNodeGrantsTest\\:\\:createEntityReferenceField\\(\\) has no return type specified\\.$#',
 	'identifier' => 'missingType.return',
 	'count' => 1,
-	'path' => __DIR__ . '/modules/node/tests/src/Functional/NodeAccessCacheabilityWithNodeGrants.php',
+	'path' => __DIR__ . '/modules/node/tests/src/Functional/NodeAccessCacheabilityWithNodeGrantsTest.php',
 ];
 $ignoreErrors[] = [
 	'message' => '#^Method Drupal\\\\Tests\\\\node\\\\Functional\\\\NodeAccessPagerTest\\:\\:addDefaultCommentField\\(\\) has no return type specified\\.$#',
@@ -55173,66 +55167,12 @@
 	'count' => 1,
 	'path' => __DIR__ . '/tests/Drupal/FunctionalTests/Installer/InstallerConfigDirectoryTestBase.php',
 ];
-$ignoreErrors[] = [
-	'message' => '#^Method Drupal\\\\FunctionalTests\\\\Installer\\\\InstallerExistingConfigProfileHookInstall\\:\\:setUpRequirementsProblem\\(\\) has no return type specified\\.$#',
-	'identifier' => 'missingType.return',
-	'count' => 1,
-	'path' => __DIR__ . '/tests/Drupal/FunctionalTests/Installer/InstallerExistingConfigProfileHookInstall.php',
-];
-$ignoreErrors[] = [
-	'message' => '#^Method Drupal\\\\FunctionalTests\\\\Installer\\\\InstallerExistingConfigProfileHookInstall\\:\\:setUpSettings\\(\\) has no return type specified\\.$#',
-	'identifier' => 'missingType.return',
-	'count' => 1,
-	'path' => __DIR__ . '/tests/Drupal/FunctionalTests/Installer/InstallerExistingConfigProfileHookInstall.php',
-];
-$ignoreErrors[] = [
-	'message' => '#^Method Drupal\\\\FunctionalTests\\\\Installer\\\\InstallerExistingConfigProfileHookInstall\\:\\:setUpSite\\(\\) has no return type specified\\.$#',
-	'identifier' => 'missingType.return',
-	'count' => 1,
-	'path' => __DIR__ . '/tests/Drupal/FunctionalTests/Installer/InstallerExistingConfigProfileHookInstall.php',
-];
-$ignoreErrors[] = [
-	'message' => '#^Method Drupal\\\\FunctionalTests\\\\Installer\\\\InstallerExistingConfigProfileHookInstall\\:\\:visitInstaller\\(\\) has no return type specified\\.$#',
-	'identifier' => 'missingType.return',
-	'count' => 1,
-	'path' => __DIR__ . '/tests/Drupal/FunctionalTests/Installer/InstallerExistingConfigProfileHookInstall.php',
-];
 $ignoreErrors[] = [
 	'message' => '#^Method Drupal\\\\FunctionalTests\\\\Installer\\\\InstallerExistingConfigSyncDirectoryMultilingualTest\\:\\:setUpProfile\\(\\) has no return type specified\\.$#',
 	'identifier' => 'missingType.return',
 	'count' => 1,
 	'path' => __DIR__ . '/tests/Drupal/FunctionalTests/Installer/InstallerExistingConfigSyncDirectoryMultilingualTest.php',
 ];
-$ignoreErrors[] = [
-	'message' => '#^Method Drupal\\\\FunctionalTests\\\\Installer\\\\InstallerExistingConfigSyncDirectoryProfileHookInstall\\:\\:setUpProfile\\(\\) has no return type specified\\.$#',
-	'identifier' => 'missingType.return',
-	'count' => 1,
-	'path' => __DIR__ . '/tests/Drupal/FunctionalTests/Installer/InstallerExistingConfigSyncDirectoryProfileHookInstall.php',
-];
-$ignoreErrors[] = [
-	'message' => '#^Method Drupal\\\\FunctionalTests\\\\Installer\\\\InstallerExistingConfigSyncDirectoryProfileHookInstall\\:\\:setUpRequirementsProblem\\(\\) has no return type specified\\.$#',
-	'identifier' => 'missingType.return',
-	'count' => 1,
-	'path' => __DIR__ . '/tests/Drupal/FunctionalTests/Installer/InstallerExistingConfigSyncDirectoryProfileHookInstall.php',
-];
-$ignoreErrors[] = [
-	'message' => '#^Method Drupal\\\\FunctionalTests\\\\Installer\\\\InstallerExistingConfigSyncDirectoryProfileHookInstall\\:\\:setUpSettings\\(\\) has no return type specified\\.$#',
-	'identifier' => 'missingType.return',
-	'count' => 1,
-	'path' => __DIR__ . '/tests/Drupal/FunctionalTests/Installer/InstallerExistingConfigSyncDirectoryProfileHookInstall.php',
-];
-$ignoreErrors[] = [
-	'message' => '#^Method Drupal\\\\FunctionalTests\\\\Installer\\\\InstallerExistingConfigSyncDirectoryProfileHookInstall\\:\\:setUpSite\\(\\) has no return type specified\\.$#',
-	'identifier' => 'missingType.return',
-	'count' => 1,
-	'path' => __DIR__ . '/tests/Drupal/FunctionalTests/Installer/InstallerExistingConfigSyncDirectoryProfileHookInstall.php',
-];
-$ignoreErrors[] = [
-	'message' => '#^Method Drupal\\\\FunctionalTests\\\\Installer\\\\InstallerExistingConfigSyncDirectoryProfileHookInstall\\:\\:visitInstaller\\(\\) has no return type specified\\.$#',
-	'identifier' => 'missingType.return',
-	'count' => 1,
-	'path' => __DIR__ . '/tests/Drupal/FunctionalTests/Installer/InstallerExistingConfigSyncDirectoryProfileHookInstall.php',
-];
 $ignoreErrors[] = [
 	'message' => '#^Method Drupal\\\\FunctionalTests\\\\Installer\\\\InstallerExistingConfigTestBase\\:\\:prepareEnvironment\\(\\) has no return type specified\\.$#',
 	'identifier' => 'missingType.return',

core/modules/jsonapi/tests/src/Functional/RestExportJsonApiUnsupported.php → core/modules/jsonapi/tests/src/Functional/RestExportJsonApiUnsupportedTest.php

@@ -13,7 +13,7 @@
  *
  * @internal
  */
-class RestExportJsonApiUnsupported extends ViewTestBase {
+class RestExportJsonApiUnsupportedTest extends ViewTestBase {
 
   /**
    * {@inheritdoc}

core/modules/jsonapi/tests/src/Functional/RestJsonApiUnsupported.php → core/modules/jsonapi/tests/src/Functional/RestJsonApiUnsupportedTest.php

@@ -18,7 +18,7 @@
  *
  * @internal
  */
-class RestJsonApiUnsupported extends ResourceTestBase {
+class RestJsonApiUnsupportedTest extends ResourceTestBase {
 
   use AnonResourceTestTrait;
 
@@ -50,7 +50,7 @@ class RestJsonApiUnsupported extends ResourceTestBase {
   /**
    * {@inheritdoc}
    */
-  protected function setUpAuthorization($method) {
+  protected function setUpAuthorization($method): void {
     switch ($method) {
       case 'GET':
         $this->grantPermissionsToTestedRole(['access content']);

core/modules/migrate_drupal_ui/tests/src/Functional/d6/Upgrade6TestWithContentModeration.php → core/modules/migrate_drupal_ui/tests/src/Functional/d6/Upgrade6WithContentModerationTest.php

@@ -13,7 +13,7 @@
  * @group migrate_drupal_ui
  * @group #slow
  */
-class Upgrade6TestWithContentModeration extends Upgrade6Test {
+class Upgrade6WithContentModerationTest extends Upgrade6Test {
 
   /**
    * {@inheritdoc}

core/modules/migrate_drupal_ui/tests/src/Functional/d7/Upgrade7TestWithContentModeration.php → core/modules/migrate_drupal_ui/tests/src/Functional/d7/Upgrade7WithContentModerationTest.php

@@ -13,7 +13,7 @@
  * @group migrate_drupal_ui
  * @group #slow
  */
-class Upgrade7TestWithContentModeration extends Upgrade7Test {
+class Upgrade7WithContentModerationTest extends Upgrade7Test {
 
   /**
    * {@inheritdoc}

core/modules/node/tests/src/Functional/NodeAccessCacheabilityWithNodeGrants.php → core/modules/node/tests/src/Functional/NodeAccessCacheabilityWithNodeGrantsTest.php

@@ -14,7 +14,7 @@
  *
  * @group node
  */
-class NodeAccessCacheabilityWithNodeGrants extends BrowserTestBase {
+class NodeAccessCacheabilityWithNodeGrantsTest extends BrowserTestBase {
 
   use EntityReferenceFieldCreationTrait;
 

core/modules/taxonomy/tests/src/Kernel/Migrate/d6/MigrateTermNodeComplete.php → core/modules/taxonomy/tests/src/Kernel/Migrate/d6/MigrateTermNodeCompleteTest.php

@@ -13,7 +13,7 @@
  *
  * @group migrate_drupal_6
  */
-class MigrateTermNodeComplete extends MigrateDrupal6TestBase {
+class MigrateTermNodeCompleteTest extends MigrateDrupal6TestBase {
 
   /**
    * {@inheritdoc}

core/phpunit.xml.dist

@@ -91,6 +91,7 @@
       <directory>modules/**/tests/src/Kernel</directory>
       <directory>recipes/*/tests/src/Kernel</directory>
       <directory>profiles/**/tests/src/Kernel</directory>
+      <directory>profiles/tests/testing/modules/*/tests/src/Kernel</directory>
       <directory>themes/**/tests/src/Kernel</directory>
       <directory>../modules/**/tests/src/Kernel</directory>
       <directory>../profiles/**/tests/src/Kernel</directory>
@@ -99,7 +100,12 @@
     <testsuite name="functional">
       <directory>tests/Drupal/FunctionalTests</directory>
       <directory>modules/**/tests/src/Functional</directory>
+      <directory>modules/config/tests/config_test/tests/src/Functional</directory>
+      <directory>modules/system/tests/modules/entity_test/tests/src/Functional</directory>
+      <directory>modules/layout_builder/modules/layout_builder_expose_all_field_blocks/tests/src/Functional</directory>
+      <directory>modules/navigation/modules/navigation_top_bar/tests/src/Functional</directory>
       <directory>profiles/**/tests/src/Functional</directory>
+      <directory>profiles/demo_umami/modules/demo_umami_content/tests/src/Functional</directory>
       <directory>recipes/*/tests/src/Functional</directory>
       <directory>themes/**/tests/src/Functional</directory>
       <directory>../modules/**/tests/src/Functional</directory>
@@ -118,6 +124,7 @@
     </testsuite>
     <testsuite name="build">
       <directory>tests/Drupal/BuildTests</directory>
+      <directory>modules/**/tests/src/Build</directory>
     </testsuite>
   </testsuites>
   <!-- Settings for coverage reports. -->

core/tests/Drupal/FunctionalTests/Asset/AssetOptimizationTestUmami.php → core/tests/Drupal/FunctionalTests/Asset/AssetOptimizationUmamiTest.php

@@ -14,7 +14,7 @@
  * @group asset
  * @group #slow
  */
-class AssetOptimizationTestUmami extends AssetOptimizationTest {
+class AssetOptimizationUmamiTest extends AssetOptimizationTest {
 
   /**
    * {@inheritdoc}

core/tests/Drupal/FunctionalTests/Installer/InstallerExistingConfigProfileHookInstall.php → core/tests/Drupal/FunctionalTests/Installer/InstallerExistingConfigProfileHookInstallTest.php

@@ -9,14 +9,14 @@
  *
  * @group Installer
  */
-class InstallerExistingConfigProfileHookInstall extends InstallerConfigDirectoryTestBase {
+class InstallerExistingConfigProfileHookInstallTest extends InstallerConfigDirectoryTestBase {
 
   protected $profile = 'config_profile_with_hook_install';
 
   /**
    * {@inheritdoc}
    */
-  protected function visitInstaller() {
+  protected function visitInstaller(): void {
     // Create an .install file with a hook_install() implementation.
     $path = $this->siteDirectory . '/profiles/' . $this->profile;
     $contents = <<<EOF
@@ -32,14 +32,14 @@ function config_profile_with_hook_install_install() {
   /**
    * Installer step: Configure settings.
    */
-  protected function setUpSettings() {
+  protected function setUpSettings(): void {
     // There are errors therefore there is nothing to do here.
   }
 
   /**
    * {@inheritdoc}
    */
-  protected function setUpRequirementsProblem() {
+  protected function setUpRequirementsProblem(): void {
     // The parent method asserts that there are no requirements errors, but
     // this test expects a requirements error in the test method below.
     // Therefore, we override this method to suppress the parent's assertions.
@@ -48,7 +48,7 @@ protected function setUpRequirementsProblem() {
   /**
    * Final installer step: Configure site.
    */
-  protected function setUpSite() {
+  protected function setUpSite(): void {
     // There are errors therefore there is nothing to do here.
   }
 

core/tests/Drupal/FunctionalTests/Installer/InstallerExistingConfigSyncDirectoryProfileHookInstall.php → core/tests/Drupal/FunctionalTests/Installer/InstallerExistingConfigSyncDirectoryProfileHookInstallTest.php

@@ -9,7 +9,7 @@
  *
  * @group Installer
  */
-class InstallerExistingConfigSyncDirectoryProfileHookInstall extends InstallerConfigDirectoryTestBase {
+class InstallerExistingConfigSyncDirectoryProfileHookInstallTest extends InstallerConfigDirectoryTestBase {
 
   /**
    * {@inheritdoc}
@@ -29,7 +29,7 @@ class InstallerExistingConfigSyncDirectoryProfileHookInstall extends InstallerCo
   /**
    * {@inheritdoc}
    */
-  protected function visitInstaller() {
+  protected function visitInstaller(): void {
     // Create an .install file with a hook_install() implementation.
     $path = $this->siteDirectory . '/profiles/' . $this->profile;
     $contents = <<<EOF
@@ -45,7 +45,7 @@ function testing_config_install_multilingual_install() {
   /**
    * Installer step: Select installation profile.
    */
-  protected function setUpProfile() {
+  protected function setUpProfile(): void {
     // This is the form we are testing so wait until the test method to do
     // assertions.
   }
@@ -53,21 +53,21 @@ protected function setUpProfile() {
   /**
    * Installer step: Requirements problem.
    */
-  protected function setUpRequirementsProblem() {
+  protected function setUpRequirementsProblem(): void {
     // This form will never be reached.
   }
 
   /**
    * Installer step: Configure settings.
    */
-  protected function setUpSettings() {
+  protected function setUpSettings(): void {
     // This form will never be reached.
   }
 
   /**
    * Final installer step: Configure site.
    */
-  protected function setUpSite() {
+  protected function setUpSite(): void {
     // This form will never be reached.
   }
 

core/tests/Drupal/FunctionalTests/Installer/TestingProfileHooks.php → core/tests/Drupal/FunctionalTests/Installer/TestingProfileHooksTest.php

@@ -11,7 +11,7 @@
  *
  * @group Installer
  */
-class TestingProfileHooks extends BrowserTestBase {
+class TestingProfileHooksTest extends BrowserTestBase {
 
   /**
    * {@inheritdoc}

core/tests/Drupal/KernelTests/Core/Test/PhpUnitTestDiscoveryTest.php

+<?php
+
+declare(strict_types=1);
+
+namespace Drupal\KernelTests\Core\Test;
+
+use Drupal\Core\Test\TestDiscovery;
+use Drupal\KernelTests\KernelTestBase;
+use Symfony\Component\Process\Process;
+
+/**
+ * Tests equality of test discovery between run-tests.sh and PHPUnit CLI.
+ *
+ * Generate the list of tests for all the tests that PHPUnit can discover.
+ * The goal here is to successfully generate the list, without any
+ * duplicate namespace errors, deprecation errors or so forth. This keeps
+ * us from committing tests which don't break under run-tests.sh, but do
+ * break under the PHPUnit CLI test runner tool. We then cross check the
+ * list thus generated, with the list generated by
+ * \Drupal\Core\Test\TestDiscovery, which is used by run-tests.sh, to ensure
+ * both methods will run the same tests,
+ *
+ * @group TestSuites
+ * @group Test
+ * @group #slow
+ */
+class PhpUnitTestDiscoveryTest extends KernelTestBase {
+
+  /**
+   * The filepath to the XML file to be used for dumping the test list.
+   */
+  private string $xmlOutputFile;
+
+  /**
+   * {@inheritdoc}
+   */
+  public function setUp(): void {
+    parent::setUp();
+    $xmlOutputFile = $this->container->getParameter('app.root') . DIRECTORY_SEPARATOR . 'test-list.xml';
+    touch($xmlOutputFile);
+    $this->xmlOutputFile = realpath($xmlOutputFile);
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public function tearDown(): void {
+    @unlink($this->xmlOutputFile);
+    parent::tearDown();
+  }
+
+  /**
+   * Tests equality of test discovery between run-tests.sh and PHPUnit CLI.
+   */
+  public function testPhpUnitTestDiscoveryEqualsInternal(): void {
+    // PHPUnit's own test discovery.
+    $process = new Process([
+      'vendor/bin/phpunit',
+      '--configuration',
+      'core',
+      '--list-tests-xml',
+      $this->xmlOutputFile,
+    ], $this->root);
+    $process
+      ->setTimeout(300)
+      ->setIdleTimeout(300)
+      ->run();
+    $this->assertEquals(0, $process->getExitCode(),
+      'COMMAND: ' . $process->getCommandLine() . "\n" .
+      'OUTPUT: ' . $process->getOutput() . "\n" .
+      'ERROR: ' . $process->getErrorOutput() . "\n"
+    );
+
+    $phpUnitXmlList = new \DOMDocument();
+    $phpUnitXmlList->loadXML(file_get_contents($this->xmlOutputFile));
+    $phpUnitList = [];
+    foreach ($phpUnitXmlList->getElementsByTagName('testCaseClass') as $node) {
+      $phpUnitList[] = $node->getAttribute('name');
+    }
+    asort($phpUnitList);
+
+    // Drupal's own test discovery, used by run-tests.sh.
+    $testDiscovery = new TestDiscovery(
+      $this->container->getParameter('app.root'),
+      $this->container->get('class_loader')
+    );
+    $internalList = [];
+    foreach ($testDiscovery->getTestClasses() as $group) {
+      foreach (array_keys($group) as $class) {
+        $internalList[] = $class;
+      }
+    }
+    $internalList = array_unique($internalList);
+    asort($internalList);
+
+    $this->assertEquals(array_values($phpUnitList), array_values($internalList));
+  }
+
+}

core/tests/Drupal/Tests/Core/Test/PhpUnitCliTest.php

-<?php
-
-declare(strict_types=1);
-
-namespace Drupal\Tests\Core\Test;
-
-use Drupal\Tests\UnitTestCase;
-use Symfony\Component\Process\Process;
-
-/**
- * @group TestSuites
- * @group Test
- * @group #slow
- */
-class PhpUnitCliTest extends UnitTestCase {
-
-  /**
-   * Ensure that the test suites are able to discover tests without incident.
-   *
-   * Generate the list of tests for all the tests that PHPUnit can discover.
-   * The goal here is to successfully generate the list, without any
-   * duplicate namespace errors, deprecation errors or so forth. This keeps
-   * us from committing tests which don't break under run-tests.sh, but do
-   * break under the PHPUnit CLI test runner tool.
-   */
-  public function testPhpUnitListTests(): void {
-    $command = [
-      'vendor/bin/phpunit',
-      '--configuration',
-      'core',
-      '--list-tests',
-    ];
-
-    $process = new Process($command, $this->root);
-    $process
-      ->setTimeout(300)
-      ->setIdleTimeout(300)
-      ->run();
-    $this->assertEquals(0, $process->getExitCode(),
-      'COMMAND: ' . $process->getCommandLine() . "\n" .
-      'OUTPUT: ' . $process->getOutput() . "\n" .
-      'ERROR: ' . $process->getErrorOutput() . "\n"
-    );
-  }
-
-}

core/tests/Drupal/Tests/Core/Theme/CoreThemesAutoloadedForTests.php → core/tests/Drupal/Tests/Core/Theme/CoreThemesAutoloadedForTestsTest.php

@@ -12,7 +12,7 @@
  *
  * @group Theme
  */
-class CoreThemesAutoloadedForTests extends UnitTestCase {
+class CoreThemesAutoloadedForTestsTest extends UnitTestCase {
 
   /**
    * Confirms that core/themes is autoloaded for tests.