Issue #3071193 by heddn, catch: Use live SHA512SUMS

config/install/automatic_updates.settings.yml

@@ -3,5 +3,5 @@ enable_psa: true
 notify: true
 check_frequency: 43200
 enable_readiness_checks: true
-download_uri: 'https://ftp.drupal.org/files/projects'
+hashes_uri: 'https://updates.drupal.org/release-hashes'
 ignored_paths: "modules/custom/*\nthemes/custom/*\nprofiles/custom/*"

config/schema/automatic_updates.schema.yml

@@ -17,9 +17,9 @@ automatic_updates.settings:
     enable_readiness_checks:
       type: boolean
       label: 'Enable readiness checks'
-    download_uri:
+    hashes_uri:
       type: string
-      label: 'Endpoint URI for file hashes and quasi patch files'
+      label: 'Endpoint URI for file hashes'
     ignored_paths:
       type: string
       label: 'List of files paths to ignore when running readiness checks'

src/ProjectInfoTrait.php

@@ -9,21 +9,65 @@ use PackageVersions\Versions;
  */
 trait ProjectInfoTrait {
 
+  /**
+   * Get extension list.
+   *
+   * @param string $extension_type
+   *   The extension type.
+   *
+   * @return \Drupal\Core\Extension\ExtensionList
+   *   The extension list service.
+   */
+  protected function getExtensionList($extension_type) {
+    if (isset($this->{$extension_type})) {
+      $list = $this->{$extension_type};
+    }
+    else {
+      $list = \Drupal::service("extension.list.$extension_type");
+    }
+    return $list;
+  }
+
+  /**
+   * Returns an array of info files information of available extensions.
+   *
+   * @param string $extension_type
+   *   The extension type.
+   *
+   * @return array
+   *   An associative array of extension information arrays, keyed by extension
+   *   name.
+   */
+  protected function getInfos($extension_type) {
+    $file_paths = $this->getExtensionList($extension_type)->getPathnames();
+    $infos = $this->getExtensionList($extension_type)->getAllAvailableInfo();
+    return array_map(function ($key, array $info) use ($file_paths) {
+      $info['packaged'] = $info['project'] ?? FALSE;
+      $info['project'] = $this->getProjectName($key, $info);
+      $info['install path'] = $file_paths[$key] ? dirname($file_paths[$key]) : '';
+      $info['version'] = $this->getExtensionVersion($info);
+      return $info;
+    }, array_keys($infos), $infos);
+  }
+
   /**
    * Get the extension version.
    *
-   * @param string $extension_name
-   *   The extension name.
    * @param array $info
    *   The extension's info.
    *
    * @return string|null
    *   The version or NULL if undefined.
    */
-  protected function getExtensionVersion($extension_name, array $info) {
+  protected function getExtensionVersion(array $info) {
+    $extension_name = $info['project'];
     if (isset($info['version']) && strpos($info['version'], '-dev') === FALSE) {
       return $info['version'];
     }
+    // Handle experimental modules from core.
+    if (substr($info['install path'], 0, 4) === "core") {
+      return $this->getExtensionList('module')->get('system')->info['version'];
+    }
     $composer_json = $this->getComposerJson($extension_name, $info);
     $extension_name = isset($composer_json['name']) ? $composer_json['name'] : $extension_name;
     try {
@@ -59,6 +103,9 @@ trait ProjectInfoTrait {
         $project_name = $this->getSuffix($composer_json['name'], '/', $extension_name);
       }
     }
+    if ($project_name === 'system') {
+      $project_name = 'drupal';
+    }
     return $project_name;
   }
 

src/ReadinessChecker/MissingProjectInfo.php

@@ -72,12 +72,12 @@ class MissingProjectInfo extends Filesystem {
   protected function missingProjectInfoCheck() {
     $messages = [];
     foreach ($this->getExtensionsTypes() as $extension_type) {
-      foreach ($this->getInfos($extension_type) as $extension_name => $info) {
-        if ($this->isIgnoredPath(drupal_get_path($info['type'], $extension_name))) {
+      foreach ($this->getInfos($extension_type) as $info) {
+        if ($this->isIgnoredPath($info['install path'])) {
           continue;
         }
-        if (!$this->getExtensionVersion($extension_name, $info)) {
-          $messages[] = $this->t('The project "@extension" can not be updated because its version is either undefined or a dev release.', ['@extension' => $extension_name]);
+        if (!$info['version']) {
+          $messages[] = $this->t('The project "@extension" can not be updated because its version is either undefined or a dev release.', ['@extension' => $info['name']]);
         }
       }
     }
@@ -94,18 +94,4 @@ class MissingProjectInfo extends Filesystem {
     return ['modules', 'profiles', 'themes'];
   }
 
-  /**
-   * Returns an array of info files information of available extensions.
-   *
-   * @param string $extension_type
-   *   The extension type.
-   *
-   * @return array
-   *   An associative array of extension information arrays, keyed by extension
-   *   name.
-   */
-  protected function getInfos($extension_type) {
-    return $this->{$extension_type}->getAllAvailableInfo();
-  }
-
 }

src/ReadinessChecker/ModifiedFiles.php

@@ -2,6 +2,7 @@
 
 namespace Drupal\automatic_updates\ReadinessChecker;
 
+use Drupal\automatic_updates\ProjectInfoTrait;
 use Drupal\automatic_updates\Services\ModifiedFilesInterface;
 use Drupal\Core\Extension\ExtensionList;
 use Drupal\Core\StringTranslation\StringTranslationTrait;
@@ -11,6 +12,7 @@ use Drupal\Core\StringTranslation\StringTranslationTrait;
  */
 class ModifiedFiles implements ReadinessCheckerInterface {
   use StringTranslationTrait;
+  use ProjectInfoTrait;
 
   /**
    * The modified files service.
@@ -24,21 +26,21 @@ class ModifiedFiles implements ReadinessCheckerInterface {
    *
    * @var \Drupal\Core\Extension\ExtensionList
    */
-  protected $modules;
+  protected $module;
 
   /**
    * The profile extension list.
    *
    * @var \Drupal\Core\Extension\ExtensionList
    */
-  protected $profiles;
+  protected $profile;
 
   /**
    * The theme extension list.
    *
    * @var \Drupal\Core\Extension\ExtensionList
    */
-  protected $themes;
+  protected $theme;
 
   /**
    * An array of array of strings of extension paths.
@@ -83,11 +85,10 @@ class ModifiedFiles implements ReadinessCheckerInterface {
   protected function modifiedFilesCheck() {
     $messages = [];
     $extensions = [];
-    $extensions['system'] = $this->modules->get('system')->info;
     foreach ($this->getExtensionsTypes() as $extension_type) {
-      foreach ($this->getInfos($extension_type) as $extension_name => $info) {
-        if (substr($this->getPath($extension_type, $extension_name), 0, 4) !== 'core') {
-          $extensions[$extension_name] = $info;
+      foreach ($this->getInfos($extension_type) as $info) {
+        if (substr($info['install path'], 0, 4) !== 'core' || $info['project'] === 'drupal') {
+          $extensions[$info['project']] = $info;
         }
       }
     }
@@ -104,39 +105,7 @@ class ModifiedFiles implements ReadinessCheckerInterface {
    *   The extension types.
    */
   protected function getExtensionsTypes() {
-    return ['modules', 'profiles', 'themes'];
-  }
-
-  /**
-   * Returns an array of info files information of available extensions.
-   *
-   * @param string $extension_type
-   *   The extension type.
-   *
-   * @return array
-   *   An associative array of extension information arrays, keyed by extension
-   *   name.
-   */
-  protected function getInfos($extension_type) {
-    return $this->{$extension_type}->getAllAvailableInfo();
-  }
-
-  /**
-   * Returns an extension file path.
-   *
-   * @param string $extension_type
-   *   The extension type.
-   * @param string $extension_name
-   *   The extension name.
-   *
-   * @return string
-   *   An extension file path or NULL if it does not exist.
-   */
-  protected function getPath($extension_type, $extension_name) {
-    if (!isset($this->paths[$extension_type])) {
-      $this->paths[$extension_type] = $this->{$extension_type}->getPathnames();
-    }
-    return isset($this->paths[$extension_type][$extension_name]) ? $this->paths[$extension_type][$extension_name] : NULL;
+    return ['module', 'profile', 'theme'];
   }
 
 }

src/Services/ModifiedFiles.php

@@ -72,6 +72,7 @@ class ModifiedFiles implements ModifiedFilesInterface {
    */
   public function getModifiedFiles(array $extensions = []) {
     $modified_files = [];
+    /** @var \GuzzleHttp\Promise\PromiseInterface[] $promises */
     $promises = $this->getHashRequests($extensions);
     // Wait until all the requests are finished.
     (new EachPromise($promises, [
@@ -86,13 +87,16 @@ class ModifiedFiles implements ModifiedFilesInterface {
   /**
    * Process checking hashes of files from external URL.
    *
-   * @param resource $resource
-   *   A resource handle.
+   * @param array $resource
+   *   An array of response resource and project info.
    * @param array $modified_files
    *   The list of modified files.
    */
-  protected function processHashes($resource, array &$modified_files) {
-    while (($line = fgets($resource)) !== FALSE) {
+  protected function processHashes(array $resource, array &$modified_files) {
+    $response = $resource['response'];
+    $info = $resource['info'];
+    $file_root = $info['install path'];
+    while (($line = fgets($response)) !== FALSE) {
       list($hash, $file) = preg_split('/\s+/', $line, 2);
       $file = trim($file);
       // If the line is empty, proceed to the next line.
@@ -107,15 +111,18 @@ class ModifiedFiles implements ModifiedFilesInterface {
       if ($this->isIgnoredPath($file)) {
         continue;
       }
-      $file_path = $this->drupalFinder->getDrupalRoot() . DIRECTORY_SEPARATOR . $file;
+      if ($info['project'] === 'drupal') {
+        $file_root = $this->drupalFinder->getDrupalRoot();
+      }
+      $file_path = $file_root . DIRECTORY_SEPARATOR . $file;
       if (!file_exists($file_path) || hash_file('sha512', $file_path) !== $hash) {
         $modified_files[] = $file_path;
       }
     }
-    if (!feof($resource)) {
+    if (!feof($response)) {
       $this->logger->error('Stream for resource closed prematurely.');
     }
-    fclose($resource);
+    fclose($response);
   }
 
   /**
@@ -131,9 +138,9 @@ class ModifiedFiles implements ModifiedFilesInterface {
    * @@codingStandardsIgnoreEnd
    */
   protected function getHashRequests(array $extensions) {
-    foreach ($extensions as $extension_name => $info) {
-      $url = $this->buildUrl($extension_name, $info);
-      yield $this->getPromise($url);
+    foreach ($extensions as $info) {
+      $url = $this->buildUrl($info);
+      yield $this->getPromise($url, $info);
     }
   }
 
@@ -142,36 +149,39 @@ class ModifiedFiles implements ModifiedFilesInterface {
    *
    * @param string $url
    *   The URL.
+   * @param array $info
+   *   The extension's info.
    *
    * @return \GuzzleHttp\Promise\PromiseInterface
    *   The promise.
    */
-  protected function getPromise($url) {
+  protected function getPromise($url, array $info) {
     return $this->httpClient->requestAsync('GET', $url, [
       'stream' => TRUE,
       'read_timeout' => 30,
     ])
-      ->then(function (ResponseInterface $response) {
-        return $response->getBody()->detach();
+      ->then(function (ResponseInterface $response) use ($info) {
+        return [
+          'response' => $response->getBody()->detach(),
+          'info' => $info,
+        ];
       });
   }
 
   /**
    * Build an extension's hash file URL.
    *
-   * @param string $extension_name
-   *   The extension name.
    * @param array $info
    *   The extension's info.
    *
    * @return string
    *   The URL endpoint with for an extension.
    */
-  protected function buildUrl($extension_name, array $info) {
-    $version = $this->getExtensionVersion($extension_name, $info);
-    $project_name = $this->getProjectName($extension_name, $info);
+  protected function buildUrl(array $info) {
+    $version = $info['version'];
+    $project_name = $info['project'];
     $hash_name = $this->getHashName($info);
-    $uri = ltrim($this->configFactory->get('automatic_updates.settings')->get('download_uri'), '/');
+    $uri = ltrim($this->configFactory->get('automatic_updates.settings')->get('hashes_uri'), '/');
     return Url::fromUri($uri . "/$project_name/$version/$hash_name")->toString();
   }
 
@@ -185,11 +195,11 @@ class ModifiedFiles implements ModifiedFilesInterface {
    *   The hash name.
    */
   protected function getHashName(array $info) {
-    $hash_name = 'SHA512SUMS';
-    if (isset($info['project'])) {
-      $hash_name .= '-package';
+    $hash_name = 'contents-sha512sums';
+    if ($info['packaged']) {
+      $hash_name .= '-packaged';
     }
-    return $hash_name;
+    return $hash_name . '.txt';
   }
 
 }

tests/modules/test_automatic_updates/src/Controller/HashesController.php

@@ -31,6 +31,14 @@ class HashesController extends ControllerBase {
       // Fake out a change in the LICENSE.txt.
       $response->setContent("2d4ce6b272311ca4159056fb75138eba1814b65323c35ae5e0978233918e45e62bb32fdd2e0e8f657954fd5823c045762b3b59645daf83246d88d8797726e02c  core/LICENSE.txt\n");
     }
+    elseif ($extension === 'ctools' && $version === '3.2') {
+      // Fake out a change in the LICENSE.txt.
+      $response->setContent("aee80b1f9f7f4a8a00dcf6e6ce6c41988dcaedc4de19d9d04460cbfb05d99829ffe8f9d038468eabbfba4d65b38e8dbef5ecf5eb8a1b891d9839cda6c48ee957  LICENSE.txt\n");
+    }
+    elseif ($extension === 'ctools' && $version === '3.1') {
+      // Fake out a change in the LICENSE.txt.
+      $response->setContent("c82147962109321f8fb9c802735d31aab659a1cc3cd13d36dc5371c8b682ff60f23d41c794f2d9dc970ef9634b7fc8bcf35e3b95132644fe2ec97a341658a3f6  LICENSE.txt\n");
+    }
     return $response;
   }
 

tests/src/Functional/ModifiedFilesTest.php

@@ -33,14 +33,33 @@ class ModifiedFilesTest extends BrowserTestBase {
       $this->container->get('config.factory')
     );
     $this->initHashesEndpoint($modified_files, 'core', '8.7.0');
-    $files = $modified_files->getModifiedFiles(['system' => []]);
+    $extensions = $modified_files->getInfos('module');
+    $extensions = array_filter($extensions, function (array $extension) {
+      return $extension['project'] === 'drupal';
+    });
+    $files = $modified_files->getModifiedFiles($extensions);
     $this->assertEmpty($files);
 
     // Hash doesn't match i.e. modified code, including contrib logic.
     $this->initHashesEndpoint($modified_files, 'core', '8.0.0');
-    $files = $modified_files->getModifiedFiles(['system' => []]);
+    $files = $modified_files->getModifiedFiles($extensions);
     $this->assertCount(1, $files);
     $this->assertStringEndsWith('core/LICENSE.txt', $files[0]);
+
+    // Test contrib hash matches.
+    $extensions = $modified_files->getInfos('module');
+    $extensions = array_filter($extensions, function (array $extension) {
+      return $extension['name'] === 'Chaos Tools';
+    });
+    $this->initHashesEndpoint($modified_files, 'ctools', '3.2');
+    $files = $modified_files->getModifiedFiles($extensions);
+    $this->assertEmpty($files);
+
+    // Test contrib doesn't match.
+    $this->initHashesEndpoint($modified_files, 'ctools', '3.1');
+    $files = $modified_files->getModifiedFiles($extensions);
+    $this->assertCount(1, $files);
+    $this->assertStringEndsWith('contrib/ctools/LICENSE.txt', $files[0]);
   }
 
   /**
@@ -77,8 +96,17 @@ class TestModifiedFiles extends ModifiedFiles {
   /**
    * {@inheritdoc}
    */
-  protected function buildUrl($extension_name, array $info) {
+  protected function buildUrl(array $info) {
     return $this->endpoint;
   }
 
+  // @codingStandardsIgnoreStart
+  /**
+   * {@inheritdoc}
+   */
+  public function getInfos($extension_type) {
+    return parent::getInfos($extension_type);
+  }
+  // codingStandardsIgnoreEnd
+
 }

tests/src/Kernel/ProjectInfoTraitTest.php

@@ -25,8 +25,9 @@ class ProjectInfoTraitTest extends KernelTestBase {
    */
   public function testTrait($expected, $info, $extension_name) {
     $class = new ProjectInfoTestClass();
-    $this->assertSame($expected['version'], $class->getExtensionVersion($extension_name, $info));
-    $this->assertSame($expected['project'], $class->getProjectName($extension_name, $info));
+    $project_name = $class->getProjectName($extension_name, $info);
+    $this->assertSame($expected['project'], $project_name);
+    $this->assertSame($expected['version'], $class->getExtensionVersion($info + ['project' => $project_name]));
   }
 
   /**
@@ -47,6 +48,7 @@ class ProjectInfoTraitTest extends KernelTestBase {
       'core' => '8.x',
       'configure' => 'entity.node_type.collection',
       'dependencies' => ['drupal:text'],
+      'install path' => '',
     ];
     $infos['node']['extension_name'] = 'node';
 
@@ -62,6 +64,7 @@ class ProjectInfoTraitTest extends KernelTestBase {
       'core' => '8.x',
       'configure' => 'update.settings',
       'dependencies' => ['file'],
+      'install path' => '',
     ];
     $infos['update']['extension_name'] = 'drupal/update';
 
@@ -80,6 +83,7 @@ class ProjectInfoTraitTest extends KernelTestBase {
       'required' => 'true',
       'configure' => 'system.admin_config_system',
       'dependencies' => [],
+      'install path' => '',
     ];
     $infos['system']['extension_name'] = 'system';
 
@@ -95,6 +99,7 @@ class ProjectInfoTraitTest extends KernelTestBase {
       'core' => '8.x',
       'configure' => 'automatic_updates.settings',
       'dependencies' => ['system', 'update'],
+      'install path' => '',
     ];
     $infos['automatic_updates']['extension_name'] = 'automatic_updates';
 
@@ -111,6 +116,7 @@ class ProjectInfoTraitTest extends KernelTestBase {
       'package' => 'Core',
       'core' => '8.x',
       'dependencies' => ['system'],
+      'install path' => '',
     ];
     $infos['ctools']['extension_name'] = 'ctools';
 
@@ -132,8 +138,8 @@ class ProjectInfoTestClass {
   /**
    * {@inheritdoc}
    */
-  public function getExtensionVersion($extension_name, array $info) {
-    return $this->getVersion($extension_name, $info);
+  public function getExtensionVersion(array $info) {
+    return $this->getVersion($info);
   }
 
   /**

tests/src/Kernel/ReadinessChecker/MissingProjectInfoTest.php

@@ -64,7 +64,9 @@ class TestMissingProjectInfo extends MissingProjectInfo {
         'description' => 'Handles general site configuration for administrators.',
         'package' => 'Core',
         'version' => 'VERSION',
-        'project' => 'drupal',
+        'packaged' => FALSE,
+        'project' => $this->getProjectName('system', []),
+        'install path' => drupal_get_path('module', 'system'),
         'core' => '8.x',
         'required' => 'true',
         'configure' => 'system.admin_config_system',