Issue #2188661 by sun, Berdir, andypost: Extension System, Part II: ExtensionDiscovery.

core/includes/bootstrap.inc

@@ -10,6 +10,7 @@
 use Drupal\Core\DrupalKernel;
 use Drupal\Core\Database\Database;
 use Drupal\Core\DependencyInjection\ContainerBuilder;
+use Drupal\Core\Extension\ExtensionDiscovery;
 use Drupal\Core\Utility\Title;
 use Drupal\Core\Utility\Error;
 use Symfony\Component\ClassLoader\ApcClassLoader;
@@ -648,7 +649,7 @@ function _drupal_request_initialize() {
 function drupal_get_filename($type, $name, $filename = NULL) {
   // The location of files will not change during the request, so do not use
   // drupal_static().
-  static $files = array(), $dirs = array();
+  static $files = array();
 
   // Profiles are converted into modules in system_rebuild_module_data().
   // @todo Remove false-exposure of profiles as modules.
@@ -660,72 +661,31 @@ function drupal_get_filename($type, $name, $filename = NULL) {
     $files[$type] = array();
   }
 
-  if (!empty($filename)) {
+  if (isset($filename)) {
     $files[$type][$name] = $filename;
   }
-  elseif (isset($files[$type][$name])) {
-    // nothing
-  }
-  else {
-    // Verify that we have an keyvalue service before using it. This is required
-    // because this function is called during installation.
-    // @todo Inject database connection into KeyValueStore\DatabaseStorage.
-    if (\Drupal::hasService('keyvalue') && function_exists('db_query')) {
-      if ($type == 'module') {
-        if (empty($files[$type])) {
-          $files[$type] = \Drupal::moduleHandler()->getModuleList();
-        }
-        if (isset($files[$type][$name])) {
-          return $files[$type][$name];
-        }
-      }
-      try {
-        $file_list = \Drupal::state()->get('system.' . $type . '.files');
-        if ($file_list && isset($file_list[$name]) && file_exists(DRUPAL_ROOT . '/' . $file_list[$name])) {
-          $files[$type][$name] = $file_list[$name];
-        }
-      }
-      catch (Exception $e) {
-        // The keyvalue service raised an exception because the backend might
-        // be down. We have a fallback for this case so we hide the error
-        // completely.
-      }
+  elseif (!isset($files[$type][$name])) {
+    // If the pathname of the requested extension is not known, try to retrieve
+    // the list of extension pathnames from various providers, checking faster
+    // providers first.
+    // Retrieve the current module list (derived from the service container).
+    if ($type == 'module' && \Drupal::hasService('module_handler')) {
+      $files[$type] += \Drupal::moduleHandler()->getModuleList();
+    }
+    // If still unknown, retrieve the file list prepared in state by
+    // system_rebuild_module_data() and system_rebuild_theme_data().
+    if (!isset($files[$type][$name]) && \Drupal::hasService('state')) {
+      $files[$type] += \Drupal::state()->get('system.' . $type . '.files', array());
     }
-    // Fallback to searching the filesystem if the database could not find the
-    // file or the file returned by the database is not found.
+    // If still unknown, perform a filesystem scan.
     if (!isset($files[$type][$name])) {
-      // We have consistent directory naming: modules, themes...
-      $dir = $type . 's';
-      if ($type == 'theme_engine') {
-        $dir = 'themes/engines';
-        $extension = 'engine';
+      $listing = new ExtensionDiscovery();
+      // Prevent an infinite recursion by this legacy function.
+      if ($original_type == 'profile') {
+        $listing->setProfileDirectories(array());
       }
-      elseif ($type == 'theme') {
-        $extension = 'info.yml';
-      }
-      // Profiles are converted into modules in system_rebuild_module_data().
-      // @todo Remove false-exposure of profiles as modules.
-      elseif ($original_type == 'profile') {
-        $dir = 'profiles';
-        $extension = 'profile';
-      }
-      else {
-        $extension = $type;
-      }
-
-      if (!isset($dirs[$dir][$extension])) {
-        $dirs[$dir][$extension] = TRUE;
-        if (!function_exists('drupal_system_listing')) {
-          require_once __DIR__ . '/common.inc';
-        }
-        // Scan the appropriate directories for all files with the requested
-        // extension, not just the file we are currently looking for. This
-        // prevents unnecessary scans from being repeated when this function is
-        // called more than once in the same page request.
-        $matches = drupal_system_listing("/^" . DRUPAL_PHP_FUNCTION_PATTERN . "\.$extension$/", $dir);
-        foreach ($matches as $matched_name => $file) {
-          $files[$type][$matched_name] = $file->uri;
-        }
+      foreach ($listing->scan($original_type) as $extension_name => $file) {
+        $files[$type][$extension_name] = $file->uri;
       }
     }
   }

core/includes/common.inc

@@ -19,7 +19,6 @@
 use Drupal\Component\Utility\NestedArray;
 use Drupal\Core\Datetime\DrupalDateTime;
 use Drupal\Core\Routing\GeneratorNotInitializedException;
-use Drupal\Core\SystemListingInfo;
 use Drupal\Core\Template\Attribute;
 use Drupal\Core\Render\Element;
 
@@ -3286,19 +3285,6 @@ function drupal_page_set_cache(Response $response, Request $request) {
   }
 }
 
-/**
- * This function is kept only for backward compatibility.
- *
- * @see \Drupal\Core\SystemListing::scan().
- */
-function drupal_system_listing($mask, $directory, $key = 'name', $min_depth = 1) {
-  // As SystemListing is required to build a dependency injection container
-  // from scratch and SystemListingInfo only extends SystemLising, this
-  // class needs to be hardwired.
-  $listing = new SystemListingInfo();
-  return $listing->scan($mask, $directory, $key, $min_depth);
-}
-
 /**
  * Sets the main page content value for later use.
  *

core/includes/install.core.inc

@@ -13,7 +13,7 @@
 use Drupal\Core\Language\Language;
 use Drupal\Core\Language\LanguageManager;
 use Drupal\Core\StringTranslation\Translator\FileTranslation;
-
+use Drupal\Core\Extension\ExtensionDiscovery;
 use Drupal\Core\DependencyInjection\ContainerBuilder;
 use Symfony\Component\DependencyInjection\ContainerInterface;
 use Symfony\Component\DependencyInjection\Reference;
@@ -494,7 +494,21 @@ function install_begin_request(&$install_state) {
     // Override the module list with a minimal set of modules.
     $module_handler->setModuleList(array('system' => 'core/modules/system/system.module'));
   }
-  $module_handler->load('system');
+  // After setting up a custom and finite module list in a custom low-level
+  // bootstrap like here, ensure to use ModuleHandler::loadAll() so that
+  // ModuleHandler::isLoaded() returns TRUE, since that is a condition being
+  // checked by other subsystems (e.g., the theme system).
+  $module_handler->loadAll();
+
+  // Add list of all available profiles to the installation state.
+  $listing = new ExtensionDiscovery();
+  $listing->setProfileDirectories(array());
+  $install_state['profiles'] += $listing->scan('profile');
+
+  // Prime drupal_get_filename()'s static cache.
+  foreach ($install_state['profiles'] as $name => $profile) {
+    drupal_get_filename('profile', $name, $profile->uri);
+  }
 
   // Prepare for themed output. We need to run this at the beginning of the
   // page request to avoid a different theme accidentally getting set. (We also
@@ -528,9 +542,6 @@ function install_begin_request(&$install_state) {
 
   // Modify the installation state as appropriate.
   $install_state['completed_task'] = $task;
-
-  // Add the list of available profiles to the installation state.
-  $install_state['profiles'] += drupal_system_listing('/^' . DRUPAL_PHP_FUNCTION_PATTERN . '\.profile$/', 'profiles');
 }
 
 /**

core/includes/install.inc

@@ -10,6 +10,7 @@
 use Drupal\Component\Utility\Settings;
 use Drupal\Core\Database\Database;
 use Drupal\Core\DrupalKernel;
+use Drupal\Core\Extension\ExtensionDiscovery;
 
 /**
  * Requirement severity -- Informational message only.
@@ -123,22 +124,17 @@ function drupal_detect_database_types() {
 }
 
 /**
- * Returns all supported database installer objects that are compiled into PHP.
+ * Returns all supported database driver installer objects.
  *
- * @return
- *  An array of database installer objects compiled into PHP.
+ * @return \Drupal\Core\Database\Install\Tasks[]
+ *   An array of available database driver installer objects.
  */
 function drupal_get_database_types() {
   $databases = array();
   $drivers = array();
 
-  // We define a driver as a directory in /core/includes/database that in turn
-  // contains a database.inc file. That allows us to drop in additional drivers
-  // without modifying the installer.
-  require_once __DIR__ . '/database.inc';
-  // Allow any valid PHP identifier.
-  // @see http://www.php.net/manual/en/language.variables.basics.php.
-  $mask = '/^[a-zA-Z_\x7f-\xff][a-zA-Z0-9_\x7f-\xff]*$/';
+  // The internal database driver name is any valid PHP identifier.
+  $mask = '/^' . DRUPAL_PHP_FUNCTION_PATTERN . '$/';
   $files = file_scan_directory(DRUPAL_ROOT . '/core/lib/Drupal/Core/Database/Driver', $mask, array('recurse' => FALSE));
   if (is_dir(DRUPAL_ROOT . '/drivers/lib/Drupal/Driver/Database')) {
     $files += file_scan_directory(DRUPAL_ROOT . '/drivers/lib/Drupal/Driver/Database/', $mask, array('recurse' => FALSE));
@@ -584,15 +580,16 @@ function drupal_verify_profile($install_state) {
   }
   $info = $install_state['profile_info'];
 
-  // Get a list of modules that exist in Drupal's assorted subdirectories.
+  // Get the list of available modules for the selected installation profile.
+  $listing = new ExtensionDiscovery();
   $present_modules = array();
-  foreach (drupal_system_listing('/^' . DRUPAL_PHP_FUNCTION_PATTERN . '\.module$/', 'modules') as $present_module) {
+  foreach ($listing->scan('module') as $present_module) {
     $present_modules[] = $present_module->name;
   }
 
   // The installation profile is also a module, which needs to be installed
   // after all the other dependencies have been installed.
-  $present_modules[] = drupal_get_profile();
+  $present_modules[] = $profile;
 
   // Verify that all of the profile's required modules are present.
   $missing_modules = array_diff($info['dependencies'], $present_modules);
@@ -974,9 +971,7 @@ function drupal_requirements_url($severity) {
 function drupal_check_profile($profile, array $install_state) {
   include_once __DIR__ . '/file.inc';
 
-  $profile_file = $install_state['profiles'][$profile]->uri;
-
-  if (!isset($profile) || !file_exists($profile_file)) {
+  if (!isset($profile) || !isset($install_state['profiles'][$profile])) {
     throw new Exception(install_no_profile_error());
   }
 

core/includes/module.inc

@@ -6,6 +6,7 @@
  */
 
 use Drupal\Core\Cache\Cache;
+use Drupal\Core\Extension\ExtensionDiscovery;
 
 /**
  * Builds a list of bootstrap modules and enabled modules and themes.
@@ -298,14 +299,19 @@ function module_uninstall($module_list = array(), $uninstall_dependents = TRUE)
  * Returns an array of modules required by core.
  */
 function drupal_required_modules() {
-  $files = drupal_system_listing('/^' . DRUPAL_PHP_FUNCTION_PATTERN . '\.info.yml$/', 'modules');
+  $listing = new ExtensionDiscovery();
+  $files = $listing->scan('module');
   $required = array();
 
-  // An installation profile is required and one must always be loaded.
-  $required[] = drupal_get_profile();
+  // Unless called by the installer, an installation profile is required and
+  // must always be loaded. drupal_get_profile() also returns the installation
+  // profile in the installer, but only after it has been selected.
+  if ($profile = drupal_get_profile()) {
+    $required[] = $profile;
+  }
 
   foreach ($files as $name => $file) {
-    $info = \Drupal::service('info_parser')->parse($file->uri);
+    $info = \Drupal::service('info_parser')->parse($file->getPathname());
     if (!empty($info) && !empty($info['required']) && $info['required']) {
       $required[] = $name;
     }

core/includes/theme.inc

@@ -12,6 +12,7 @@
 use Drupal\Component\Utility\Url;
 use Drupal\Core\Config\Config;
 use Drupal\Core\Language\Language;
+use Drupal\Core\Extension\Extension;
 use Drupal\Core\Extension\ExtensionNameLengthException;
 use Drupal\Core\Template\Attribute;
 use Drupal\Core\Template\RenderWrapper;
@@ -64,10 +65,10 @@
 /**
  * Determines if a theme is available to use.
  *
- * @param $theme
+ * @param string|\Drupal\Core\Extension\Extension $theme
  *   Either the name of a theme or a full theme object.
  *
- * @return
+ * @return bool
  *   Boolean TRUE if the theme is enabled or is the site administration theme;
  *   FALSE otherwise.
  *
@@ -77,7 +78,7 @@
  * @see \Drupal\Core\Theme\ThemeAccessCheck::checkAccess().
  */
 function drupal_theme_access($theme) {
-  if (is_object($theme)) {
+  if ($theme instanceof Extension) {
     $theme = $theme->name;
   }
   return \Drupal::service('access_check.theme')->checkAccess($theme);
@@ -120,19 +121,9 @@ function drupal_theme_initialize() {
  *
  * This function is useful to initialize a theme when no database is present.
  *
- * @param $theme
- *   An object with the following information:
- *     filename
- *       The .info.yml file for this theme. The 'path' to
- *       the theme will be in this file's directory. (Required)
- *     owner
- *       The path to the .theme file or the .engine file to load for
- *       the theme. (Required)
- *     stylesheet
- *       The primary stylesheet for the theme. (Optional)
- *     engine
- *       The name of theme engine to use. (Optional)
- * @param $base_theme
+ * @param \Drupal\Core\Extension\Extension $theme
+ *   The theme extension object.
+ * @param \Drupal\Core\Extension\Extension[] $base_theme
  *    An optional array of objects that represent the 'base theme' if the
  *    theme is meant to be derivative of another theme. It requires
  *    the same information as the $theme object. It should be in

core/lib/Drupal/Core/Config/InstallStorage.php

@@ -7,6 +7,8 @@
 
 namespace Drupal\Core\Config;
 
+use Drupal\Core\Extension\ExtensionDiscovery;
+
 /**
  * Storage controller used by the Drupal installer.
  *
@@ -110,9 +112,14 @@ public function listAll($prefix = '') {
    */
   protected function getAllFolders() {
     if (!isset($this->folders)) {
-      $this->folders = $this->getComponentNames('profile', array(drupal_get_profile()));
-      $this->folders += $this->getComponentNames('module', array_keys(drupal_system_listing('/^' . DRUPAL_PHP_FUNCTION_PATTERN . '\.module$/', 'modules', 'name', 0)));
-      $this->folders += $this->getComponentNames('theme', array_keys(drupal_system_listing('/^' . DRUPAL_PHP_FUNCTION_PATTERN . '\.info.yml$/', 'themes')));
+      $this->folders = array();
+      // @todo Refactor getComponentNames() to use the extension list directly.
+      if ($profile = drupal_get_profile()) {
+        $this->folders += $this->getComponentNames('profile', array($profile));
+      }
+      $listing = new ExtensionDiscovery();
+      $this->folders += $this->getComponentNames('module', array_keys($listing->scan('module')));
+      $this->folders += $this->getComponentNames('theme', array_keys($listing->scan('theme')));
     }
     return $this->folders;
   }

core/lib/Drupal/Core/DrupalKernel.php

@@ -13,6 +13,7 @@
 use Drupal\Core\DependencyInjection\ContainerBuilder;
 use Drupal\Core\DependencyInjection\ServiceProviderInterface;
 use Drupal\Core\DependencyInjection\YamlFileLoader;
+use Drupal\Core\Extension\ExtensionDiscovery;
 use Drupal\Core\Language\Language;
 use Symfony\Component\Config\Loader\LoaderInterface;
 use Symfony\Component\DependencyInjection\ParameterBag\ParameterBag;
@@ -83,7 +84,7 @@ class DrupalKernel implements DrupalKernelInterface, TerminableInterface {
    * An array of module data objects.
    *
    * The data objects have the same data structure as returned by
-   * file_scan_directory() but only the uri property is used.
+   * ExtensionDiscovery but only the uri property is used.
    *
    * @var array
    */
@@ -297,19 +298,28 @@ public function handle(Request $request, $type = HttpKernelInterface::MASTER_REQ
   protected function moduleData($module) {
     if (!$this->moduleData) {
       // First, find profiles.
-      $profiles_scanner = new SystemListing();
-      $all_profiles = $profiles_scanner->scan('/^' . DRUPAL_PHP_FUNCTION_PATTERN . '\.profile$/', 'profiles');
-      $profiles = array_keys(array_intersect_key($this->moduleList, $all_profiles));
+      $listing = new ExtensionDiscovery();
+      $listing->setProfileDirectories(array());
+      $all_profiles = $listing->scan('profile');
+      $profiles = array_intersect_key($all_profiles, $this->moduleList);
+
       // If a module is within a profile directory but specifies another
       // profile for testing, it needs to be found in the parent profile.
-      if (($parent_profile_config = $this->configStorage->read('simpletest.settings')) && isset($parent_profile_config['parent_profile']) && $parent_profile_config['parent_profile'] != $profiles[0]) {
+      $settings = $this->configStorage->read('simpletest.settings');
+      $parent_profile = !empty($settings['parent_profile']) ? $settings['parent_profile'] : NULL;
+      if ($parent_profile && !isset($profiles[$parent_profile])) {
         // In case both profile directories contain the same extension, the
         // actual profile always has precedence.
-        array_unshift($profiles, $parent_profile_config['parent_profile']);
+        $profiles = array($parent_profile => $all_profiles[$parent_profile]) + $profiles;
       }
+
+      $profile_directories = array_map(function ($profile) {
+        return $profile->getPath();
+      }, $profiles);
+      $listing->setProfileDirectories($profile_directories);
+
       // Now find modules.
-      $modules_scanner = new SystemListing($profiles);
-      $this->moduleData = $all_profiles + $modules_scanner->scan('/^' . DRUPAL_PHP_FUNCTION_PATTERN . '\.module$/', 'modules');
+      $this->moduleData = $profiles + $listing->scan('module');
     }
     return isset($this->moduleData[$module]) ? $this->moduleData[$module] : FALSE;
   }

core/lib/Drupal/Core/Extension/Discovery/RecursiveExtensionFilterIterator.php

+<?php
+
+/**
+ * @file
+ * Contains \Drupal\Core\Extension\Discovery\RecursiveExtensionFilterIterator.
+ */
+
+namespace Drupal\Core\Extension\Discovery;
+
+/**
+ * Filters a RecursiveDirectoryIterator to discover extensions.
+ *
+ * To ensure the best possible performance for extension discovery, this
+ * filter implementation hard-codes a range of assumptions about directories
+ * in which Drupal extensions may appear and in which not. Every unnecessary
+ * subdirectory tree recursion is avoided.
+ *
+ * The list of globally ignored directory names is defined in the
+ * RecursiveExtensionFilterIterator::$blacklist property.
+ *
+ * In addition, all 'config' directories are skipped, unless the directory path
+ * ends with 'modules/config', so as to still find the config module provided by
+ * Drupal core and still allow that module to be overridden with a custom config
+ * module.
+ *
+ * Lastly, ExtensionDiscovery instructs this filter to additionally skip all
+ * 'tests' directories at regular runtime, since just with Drupal core only, the
+ * discovery process yields 4x more extensions when tests are not ignored.
+ *
+ * @see ExtensionDiscovery::scan()
+ * @see ExtensionDiscovery::scanDirectory()
+ *
+ * @todo Use RecursiveCallbackFilterIterator instead of the $acceptTests
+ *   parameter forwarding once PHP 5.4 is available.
+ */
+class RecursiveExtensionFilterIterator extends \RecursiveFilterIterator {
+
+  /**
+   * List of base extension type directory names to scan.
+   *
+   * Only these directory names are considered when starting a filesystem
+   * recursion in a search path.
+   *
+   * @var array
+   */
+  protected $whitelist = array(
+    'profiles',
+    'modules',
+    'themes',
+  );
+
+  /**
+   * List of directory names to skip when recursing.
+   *
+   * These directories are globally ignored in the recursive filesystem scan;
+   * i.e., extensions (of all types) are not able to use any of these names,
+   * because their directory names will be skipped.
+   *
+   * @var array
+   */
+  protected $blacklist = array(
+    // Object-oriented code subdirectories.
+    'src',
+    'lib',
+    'vendor',
+    // Front-end.
+    'assets',
+    'css',
+    'files',
+    'images',
+    'js',
+    'misc',
+    'templates',
+    // Legacy subdirectories.
+    'includes',
+    // Test subdirectories.
+    'fixtures',
+    // @todo ./tests/Drupal should be ./tests/src/Drupal
+    'Drupal',
+  );
+
+  /**
+   * Whether to include test directories when recursing.
+   *
+   * @var bool
+   */
+  protected $acceptTests = FALSE;
+
+  /**
+   * Controls whether test directories will be scanned.
+   *
+   * @param bool $flag
+   *   Pass FALSE to skip all test directories in the discovery. If TRUE,
+   *   extensions in test directories will be discovered and only the global
+   *   directory blacklist in RecursiveExtensionFilterIterator::$blacklist is
+   *   applied.
+   */
+  public function acceptTests($flag = FALSE) {
+    $this->acceptTests = $flag;
+    if (!$this->acceptTests) {
+      $this->blacklist[] = 'tests';
+    }
+  }
+
+  /**
+   * Overrides \RecursiveFilterIterator::getChildren().
+   */
+  public function getChildren() {
+    $filter = parent::getChildren();
+    // Pass the $acceptTests flag forward to child iterators.
+    $filter->acceptTests($this->acceptTests);
+    return $filter;
+  }
+
+  /**
+   * Implements \FilterIterator::accept().
+   */
+  public function accept() {
+    $name = $this->current()->getFilename();
+    // FilesystemIterator::SKIP_DOTS only skips '.' and '..', but not hidden
+    // directories (like '.git').
+    if ($name[0] == '.') {
+      return FALSE;
+    }
+    if ($this->isDir()) {
+      // If this is a subdirectory of a base search path, only recurse into the
+      // fixed list of expected extension type directory names. Required for
+      // scanning the top-level/root directory; without this condition, we would
+      // recurse into the whole filesystem tree that possibly contains other
+      // files aside from Drupal.
+      if ($this->current()->getSubPath() == '') {
+        return in_array($name, $this->whitelist, TRUE);
+      }
+      // 'config' directories are special-cased here, because every extension
+      // contains one. However, those default configuration directories cannot
+      // contain extensions. The directory name cannot be globally skipped,
+      // because core happens to have a directory of an actual module that is
+      // named 'config'. By explicitly testing for that case, we can skip all
+      // other config directories, and at the same time, still allow the core
+      // config module to be overridden/replaced in a profile/site directory
+      // (whereas it must be located directly in a modules directory).
+      if ($name == 'config') {
+        return substr($this->current()->getPathname(), -14) == 'modules/config';
+      }
+      // Accept the directory unless the name is blacklisted.
+      return !in_array($name, $this->blacklist, TRUE);
+    }
+    else {
+      // Only accept extension info files.
+      return substr($name, -9) == '.info.yml';
+    }
+  }
+
+}

core/lib/Drupal/Core/Extension/Extension.php

+<?php
+
+/**
+ * @file
+ * Contains \Drupal\Core\Extension\Extension.
+ */
+
+namespace Drupal\Core\Extension;
+
+/**
+ * Defines an extension (file) object.
+ */
+class Extension implements \Serializable {
+
+  /**
+   * The type of the extension (e.g., 'module').
+   *
+   * @todo Replace all uses of $type with getType() method.
+   *
+   * @var string
+   */
+  public $type;
+
+  /**
+   * The relative pathname of the extension (e.g., 'core/modules/node/node.info.yml').
+   *
+   * @var string