Commit 68a543a3 authored by catch's avatar catch

Issue #2329453 by alexpott, joelpittet, hussainweb, kaidjohnson, andriyun,...

Issue #2329453 by alexpott, joelpittet, hussainweb, kaidjohnson, andriyun, JohnAlbin, mirie, steveoliver, quicksketch, geerlingguy, RobLoach, jenlampton, SebCorbin, pablo.guerino, afoster, markcarver, danbohea, andypost, catch, donquixote, marcvangend, David_Rothstein, LewisNyman, neclimdul: Ignore front end vendor folders to improve directory search performance
parent 066437b8
...@@ -10,6 +10,7 @@ ...@@ -10,6 +10,7 @@
use Drupal\Component\PhpStorage\FileStorage; use Drupal\Component\PhpStorage\FileStorage;
use Drupal\Component\Utility\Bytes; use Drupal\Component\Utility\Bytes;
use Drupal\Core\File\FileSystem; use Drupal\Core\File\FileSystem;
use Drupal\Core\Site\Settings;
use Drupal\Core\StreamWrapper\PublicStream; use Drupal\Core\StreamWrapper\PublicStream;
use Drupal\Core\StreamWrapper\PrivateStream; use Drupal\Core\StreamWrapper\PrivateStream;
...@@ -978,7 +979,7 @@ function file_unmanaged_save_data($data, $destination = NULL, $replace = FILE_EX ...@@ -978,7 +979,7 @@ function file_unmanaged_save_data($data, $destination = NULL, $replace = FILE_EX
* @param $options * @param $options
* An associative array of additional options, with the following elements: * An associative array of additional options, with the following elements:
* - 'nomask': The preg_match() regular expression for files to be excluded. * - 'nomask': The preg_match() regular expression for files to be excluded.
* There is no default. * Defaults to the 'file_scan_ignore_directories' setting.
* - 'callback': The callback function to call for each match. There is no * - 'callback': The callback function to call for each match. There is no
* default callback. * default callback.
* - 'recurse': When TRUE, the directory scan will recurse the entire tree * - 'recurse': When TRUE, the directory scan will recurse the entire tree
...@@ -1011,6 +1012,18 @@ function file_scan_directory($dir, $mask, $options = array(), $depth = 0) { ...@@ -1011,6 +1012,18 @@ function file_scan_directory($dir, $mask, $options = array(), $depth = 0) {
$dir_has_slash = (substr($dir, -1) === '/'); $dir_has_slash = (substr($dir, -1) === '/');
} }
// Allow directories specified in settings.php to be ignored. You can use this
// to not check for files in common special-purpose directories. For example,
// node_modules and bower_components. Ignoring irrelevant directories is a
// performance boost.
if (!isset($options['nomask'])) {
$ignore_directories = Settings::get('file_scan_ignore_directories', []);
array_walk($ignore_directories, function(&$value) {
$value = preg_quote($value, '/');
});
$default_nomask = '/^' . implode('|', $ignore_directories) . '$/';
}
$options['key'] = in_array($options['key'], array('uri', 'filename', 'name')) ? $options['key'] : 'uri'; $options['key'] = in_array($options['key'], array('uri', 'filename', 'name')) ? $options['key'] : 'uri';
$files = array(); $files = array();
// Avoid warnings when opendir does not have the permissions to open a // Avoid warnings when opendir does not have the permissions to open a
...@@ -1019,7 +1032,10 @@ function file_scan_directory($dir, $mask, $options = array(), $depth = 0) { ...@@ -1019,7 +1032,10 @@ function file_scan_directory($dir, $mask, $options = array(), $depth = 0) {
if ($handle = @opendir($dir)) { if ($handle = @opendir($dir)) {
while (FALSE !== ($filename = readdir($handle))) { while (FALSE !== ($filename = readdir($handle))) {
// Skip this file if it matches the nomask or starts with a dot. // Skip this file if it matches the nomask or starts with a dot.
if ($filename[0] != '.' && !(isset($options['nomask']) && preg_match($options['nomask'], $filename))) { if ($filename[0] != '.'
&& !(isset($options['nomask']) && preg_match($options['nomask'], $filename))
&& !(!empty($default_nomask) && preg_match($default_nomask, $filename))
) {
if ($depth == 0 && $dir_has_slash) { if ($depth == 0 && $dir_has_slash) {
$uri = "$dir$filename"; $uri = "$dir$filename";
} }
......
...@@ -81,6 +81,20 @@ class RecursiveExtensionFilterIterator extends \RecursiveFilterIterator { ...@@ -81,6 +81,20 @@ class RecursiveExtensionFilterIterator extends \RecursiveFilterIterator {
*/ */
protected $acceptTests = FALSE; protected $acceptTests = FALSE;
/**
* Construct a RecursiveExtensionFilterIterator.
*
* @param \RecursiveIterator $iterator
* The iterator to filter.
* @param array $blacklist
* (optional) Add to the blacklist of directories that should be filtered
* out during the iteration.
*/
public function __construct(\RecursiveIterator $iterator, array $blacklist = []) {
parent::__construct($iterator);
$this->blacklist = array_merge($this->blacklist, $blacklist);
}
/** /**
* Controls whether test directories will be scanned. * Controls whether test directories will be scanned.
* *
...@@ -102,6 +116,8 @@ public function acceptTests($flag = FALSE) { ...@@ -102,6 +116,8 @@ public function acceptTests($flag = FALSE) {
*/ */
public function getChildren() { public function getChildren() {
$filter = parent::getChildren(); $filter = parent::getChildren();
// Pass on the blacklist.
$filter->blacklist = $this->blacklist;
// Pass the $acceptTests flag forward to child iterators. // Pass the $acceptTests flag forward to child iterators.
$filter->acceptTests($this->acceptTests); $filter->acceptTests($this->acceptTests);
return $filter; return $filter;
......
...@@ -427,11 +427,17 @@ protected function scanDirectory($dir, $include_tests) { ...@@ -427,11 +427,17 @@ protected function scanDirectory($dir, $include_tests) {
$flags |= \FilesystemIterator::CURRENT_AS_SELF; $flags |= \FilesystemIterator::CURRENT_AS_SELF;
$directory_iterator = new \RecursiveDirectoryIterator($absolute_dir, $flags); $directory_iterator = new \RecursiveDirectoryIterator($absolute_dir, $flags);
// Allow directories specified in settings.php to be ignored. You can use
// this to not check for files in common special-purpose directories. For
// example, node_modules and bower_components. Ignoring irrelevant
// directories is a performance boost.
$ignore_directories = Settings::get('file_scan_ignore_directories', []);
// Filter the recursive scan to discover extensions only. // Filter the recursive scan to discover extensions only.
// Important: Without a RecursiveFilterIterator, RecursiveDirectoryIterator // Important: Without a RecursiveFilterIterator, RecursiveDirectoryIterator
// would recurse into the entire filesystem directory tree without any kind // would recurse into the entire filesystem directory tree without any kind
// of limitations. // of limitations.
$filter = new RecursiveExtensionFilterIterator($directory_iterator); $filter = new RecursiveExtensionFilterIterator($directory_iterator, $ignore_directories);
$filter->acceptTests($include_tests); $filter->acceptTests($include_tests);
// The actual recursive filesystem scan is only invoked by instantiating the // The actual recursive filesystem scan is only invoked by instantiating the
......
...@@ -51,4 +51,26 @@ function testDirectoryPrecedence() { ...@@ -51,4 +51,26 @@ function testDirectoryPrecedence() {
} }
} }
/**
* Tests that directories matching file_scan_ignore_directories are ignored
*/
public function testFileScanIgnoreDirectory() {
$listing = new ExtensionDiscovery(\Drupal::root(), FALSE);
$listing->setProfileDirectories(array('core/profiles/testing'));
$files = $listing->scan('module');
$this->assertArrayHasKey('drupal_system_listing_compatible_test', $files);
// Reset the static to force a rescan of the directories.
$reflected_class = new \ReflectionClass(ExtensionDiscovery::class);
$reflected_property = $reflected_class->getProperty('files');
$reflected_property->setAccessible(TRUE);
$reflected_property->setValue($reflected_class, []);
$this->setSetting('file_scan_ignore_directories', ['drupal_system_listing_compatible_test']);
$listing = new ExtensionDiscovery(\Drupal::root(), FALSE);
$listing->setProfileDirectories(array('core/profiles/testing'));
$files = $listing->scan('module');
$this->assertArrayNotHasKey('drupal_system_listing_compatible_test', $files);
}
} }
...@@ -142,4 +142,25 @@ function testOptionMinDepth() { ...@@ -142,4 +142,25 @@ function testOptionMinDepth() {
$this->assertTrue(empty($files), 'Minimum-depth of 1 successfully excludes files from current directory.'); $this->assertTrue(empty($files), 'Minimum-depth of 1 successfully excludes files from current directory.');
} }
/**
* Tests file_scan_directory() obeys 'file_scan_ignore_directories' setting.
*/
public function testIgnoreDirectories() {
$files = file_scan_directory('core/modules/system/tests/fixtures/IgnoreDirectories', '/\.txt$/');
$this->assertCount(2, $files, '2 text files found when not ignoring directories.');
$this->setSetting('file_scan_ignore_directories', ['frontend_framework']);
$files = file_scan_directory('core/modules/system/tests/fixtures/IgnoreDirectories', '/\.txt$/');
$this->assertCount(1, $files, '1 text files found when ignoring directories called "frontend_framework".');
// Ensure that the directories in file_scan_ignore_directories are escaped
// using preg_quote.
$this->setSetting('file_scan_ignore_directories', ['frontend.*']);
$files = file_scan_directory('core/modules/system/tests/fixtures/IgnoreDirectories', '/\.txt$/');
$this->assertCount(2, $files, '2 text files found when ignoring a directory that is not there.');
$files = file_scan_directory('core/modules/system/tests/fixtures/IgnoreDirectories', '/\.txt$/', ['nomask' => '/^something_thing_else$/']);
$this->assertCount(2, $files, '2 text files found when an "nomask" option is passed in.');
}
} }
...@@ -715,6 +715,21 @@ ...@@ -715,6 +715,21 @@
* example.org, with all subdomains included. * example.org, with all subdomains included.
*/ */
/**
* The default list of directories that will be ignored by Drupal's file API.
*
* By default ignore node_modules and bower_components folders to avoid issues
* with common frontend tools and recursive scanning of directories looking for
* extensions.
*
* @see file_scan_directory()
* @see \Drupal\Core\Extension\ExtensionDiscovery::scanDirectory()
*/
$settings['file_scan_ignore_directories'] = [
'node_modules',
'bower_components',
];
/** /**
* Load local development override configuration, if available. * Load local development override configuration, if available.
* *
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment