diff --git a/core/modules/migrate/src/Plugin/migrate/process/Download.php b/core/modules/migrate/src/Plugin/migrate/process/Download.php
new file mode 100644
index 0000000000000000000000000000000000000000..08c9b57b19af5aa29153fb55c4f972b5dd2b895b
--- /dev/null
+++ b/core/modules/migrate/src/Plugin/migrate/process/Download.php
@@ -0,0 +1,117 @@
+<?php
+
+namespace Drupal\migrate\Plugin\migrate\process;
+
+use Drupal\Core\File\FileSystemInterface;
+use Drupal\Core\Plugin\ContainerFactoryPluginInterface;
+use Drupal\migrate\MigrateException;
+use Drupal\migrate\MigrateExecutableInterface;
+use Drupal\migrate\ProcessPluginBase;
+use Drupal\migrate\Row;
+use GuzzleHttp\Client;
+use Symfony\Component\DependencyInjection\ContainerInterface;
+
+/**
+ * Downloads a file from a remote location into the local file system.
+ *
+ * @MigrateProcessPlugin(
+ * id = "download"
+ * )
+ */
+class Download extends ProcessPluginBase implements ContainerFactoryPluginInterface {
+
+ /**
+ * The file system service.
+ *
+ * @var \Drupal\Core\File\FileSystemInterface
+ */
+ protected $fileSystem;
+
+ /**
+ * The Guzzle HTTP Client service.
+ *
+ * @var \GuzzleHttp\Client
+ */
+ protected $httpClient;
+
+ /**
+ * Constructs a download process plugin.
+ *
+ * @param array $configuration
+ * The plugin configuration.
+ * @param string $plugin_id
+ * The plugin ID.
+ * @param mixed $plugin_definition
+ * The plugin definition.
+ * @param \Drupal\Core\File\FileSystemInterface $file_system
+ * The file system service.
+ * @param \GuzzleHttp\Client $http_client
+ * The HTTP client.
+ */
+ public function __construct(array $configuration, $plugin_id, array $plugin_definition, FileSystemInterface $file_system, Client $http_client) {
+ $configuration += [
+ 'rename' => FALSE,
+ 'guzzle_options' => [],
+ ];
+ parent::__construct($configuration, $plugin_id, $plugin_definition);
+ $this->fileSystem = $file_system;
+ $this->httpClient = $http_client;
+ }
+
+ /**
+ * {@inheritdoc}
+ */
+ public static function create(ContainerInterface $container, array $configuration, $plugin_id, $plugin_definition) {
+ return new static(
+ $configuration,
+ $plugin_id,
+ $plugin_definition,
+ $container->get('file_system'),
+ $container->get('http_client')
+ );
+ }
+
+ /**
+ * {@inheritdoc}
+ */
+ public function transform($value, MigrateExecutableInterface $migrate_executable, Row $row, $destination_property) {
+ // If we're stubbing a file entity, return a uri of NULL so it will get
+ // stubbed by the general process.
+ if ($row->isStub()) {
+ return NULL;
+ }
+ list($source, $destination) = $value;
+
+ // Modify the destination filename if necessary.
+ $replace = !empty($this->configuration['rename']) ?
+ FILE_EXISTS_RENAME :
+ FILE_EXISTS_REPLACE;
+ $final_destination = file_destination($destination, $replace);
+
+ // Try opening the file first, to avoid calling file_prepare_directory()
+ // unnecessarily. We're suppressing fopen() errors because we want to try
+ // to prepare the directory before we give up and fail.
+ $destination_stream = @fopen($final_destination, 'w');
+ if (!$destination_stream) {
+ // If fopen didn't work, make sure there's a writable directory in place.
+ $dir = $this->fileSystem->dirname($final_destination);
+ if (!file_prepare_directory($dir, FILE_CREATE_DIRECTORY | FILE_MODIFY_PERMISSIONS)) {
+ throw new MigrateException("Could not create or write to directory '$dir'");
+ }
+ // Let's try that fopen again.
+ $destination_stream = @fopen($final_destination, 'w');
+ if (!$destination_stream) {
+ throw new MigrateException("Could not write to file '$final_destination'");
+ }
+ }
+
+ // Stream the request body directly to the final destination stream.
+ $this->configuration['guzzle_options']['sink'] = $destination_stream;
+
+ // Make the request. Guzzle throws an exception for anything other than 200.
+ $this->httpClient->get($source, $this->configuration['guzzle_options']);
+
+ return $final_destination;
+ }
+
+}
diff --git a/core/modules/migrate/src/Plugin/migrate/process/FileCopy.php b/core/modules/migrate/src/Plugin/migrate/process/FileCopy.php
index 7c49715d6cf058f5d58e2dca6d2a69462c47ab12..c68fe344eac64b04219607b570acbda00e57bd84 100644
--- a/core/modules/migrate/src/Plugin/migrate/process/FileCopy.php
+++ b/core/modules/migrate/src/Plugin/migrate/process/FileCopy.php
@@ -8,6 +8,7 @@
use Drupal\Core\StreamWrapper\StreamWrapperManagerInterface;
use Drupal\migrate\MigrateException;
use Drupal\migrate\MigrateExecutableInterface;
+use Drupal\migrate\Plugin\MigrateProcessInterface;
use Drupal\migrate\ProcessPluginBase;
use Drupal\migrate\Row;
use Symfony\Component\DependencyInjection\ContainerInterface;
@@ -35,6 +36,13 @@ class FileCopy extends ProcessPluginBase implements ContainerFactoryPluginInterf
*/
protected $fileSystem;
+ /**
+ * An instance of the download process plugin.
+ *
+ * @var \Drupal\migrate\Plugin\MigrateProcessInterface
+ */
+ protected $downloadPlugin;
+
/**
* Constructs a file_copy process plugin.
*
@@ -48,8 +56,10 @@ class FileCopy extends ProcessPluginBase implements ContainerFactoryPluginInterf
* The stream wrapper manager service.
* @param \Drupal\Core\File\FileSystemInterface $file_system
* The file system service.
+ * @param \Drupal\migrate\Plugin\MigrateProcessInterface $download_plugin
+ * An instance of the download plugin for handling remote URIs.
*/
- public function __construct(array $configuration, $plugin_id, array $plugin_definition, StreamWrapperManagerInterface $stream_wrappers, FileSystemInterface $file_system) {
+ public function __construct(array $configuration, $plugin_id, array $plugin_definition, StreamWrapperManagerInterface $stream_wrappers, FileSystemInterface $file_system, MigrateProcessInterface $download_plugin) {
$configuration += array(
'move' => FALSE,
'rename' => FALSE,
@@ -58,6 +68,7 @@ public function __construct(array $configuration, $plugin_id, array $plugin_defi
parent::__construct($configuration, $plugin_id, $plugin_definition);
$this->streamWrapperManager = $stream_wrappers;
$this->fileSystem = $file_system;
+ $this->downloadPlugin = $download_plugin;
}
/**
@@ -69,7 +80,8 @@ public static function create(ContainerInterface $container, array $configuratio
$plugin_id,
$plugin_definition,
$container->get('stream_wrapper_manager'),
- $container->get('file_system')
+ $container->get('file_system'),
+ $container->get('plugin.manager.migrate.process')->createInstance('download')
);
}
@@ -84,8 +96,14 @@ public function transform($value, MigrateExecutableInterface $migrate_executable
}
list($source, $destination) = $value;
+ // If the source path or URI represents a remote resource, delegate to the
+ // download plugin.
+ if (!$this->isLocalUri($source)) {
+ return $this->downloadPlugin->transform($value, $migrate_executable, $row, $destination_property);
+ }
+
// Ensure the source file exists, if it's a local URI or path.
- if ($this->isLocalUri($source) && !file_exists($source)) {
+ if (!file_exists($source)) {
throw new MigrateException("File '$source' does not exist");
}
@@ -128,20 +146,14 @@ public function transform($value, MigrateExecutableInterface $migrate_executable
* File destination on success, FALSE on failure.
*/
protected function writeFile($source, $destination, $replace = FILE_EXISTS_REPLACE) {
- if ($this->configuration['move']) {
- return file_unmanaged_move($source, $destination, $replace);
- }
// Check if there is a destination available for copying. If there isn't,
// it already exists at the destination and the replace flag tells us to not
// replace it. In that case, return the original destination.
if (!($final_destination = file_destination($destination, $replace))) {
return $destination;
}
- // We can't use file_unmanaged_copy because it will break with remote Urls.
- if (@copy($source, $final_destination)) {
- return $final_destination;
- }
- return FALSE;
+ $function = 'file_unmanaged_' . ($this->configuration['move'] ? 'move' : 'copy');
+ return $function($source, $destination, $replace);
}
/**
@@ -187,8 +199,6 @@ protected function getDirectory($uri) {
/**
* Determines if the source and destination URIs represent identical paths.
*
- * If either URI is a remote stream, will return FALSE.
- *
* @param string $source
* The source URI.
* @param string $destination
@@ -199,10 +209,7 @@ protected function getDirectory($uri) {
* otherwise FALSE.
*/
protected function isLocationUnchanged($source, $destination) {
- if ($this->isLocalUri($source) && $this->isLocalUri($destination)) {
- return $this->fileSystem->realpath($source) === $this->fileSystem->realpath($destination);
- }
- return FALSE;
+ return $this->fileSystem->realpath($source) === $this->fileSystem->realpath($destination);
}
/**
@@ -219,6 +226,13 @@ protected function isLocationUnchanged($source, $destination) {
*/
protected function isLocalUri($uri) {
$scheme = $this->fileSystem->uriScheme($uri);
+
+ // The vfs scheme is vfsStream, which is used in testing. vfsStream is a
+ // simulated file system that exists only in memory, but should be treated
+ // as a local resource.
+ if ($scheme == 'vfs') {
+ $scheme = FALSE;
+ }
return $scheme === FALSE || $this->streamWrapperManager->getViaScheme($scheme) instanceof LocalStream;
}
diff --git a/core/modules/migrate/tests/src/Kernel/process/CopyFileTest.php b/core/modules/migrate/tests/src/Kernel/process/CopyFileTest.php
index 42c3f04fce470e8affa2f1bc552289be82edda17..427d413bc62898b1c5c9188972bb68b9b8f200a4 100644
--- a/core/modules/migrate/tests/src/Kernel/process/CopyFileTest.php
+++ b/core/modules/migrate/tests/src/Kernel/process/CopyFileTest.php
@@ -18,7 +18,7 @@ class CopyFileTest extends FileTestBase {
/**
* {@inheritdoc}
*/
- public static $modules = ['system'];
+ public static $modules = ['migrate', 'system'];
/**
* The file system service.
diff --git a/core/modules/migrate/tests/src/Kernel/process/FileCopyTest.php b/core/modules/migrate/tests/src/Kernel/process/FileCopyTest.php
new file mode 100644
index 0000000000000000000000000000000000000000..ee75e454548fcfb280ea83102beaa8417311f372
--- /dev/null
+++ b/core/modules/migrate/tests/src/Kernel/process/FileCopyTest.php
@@ -0,0 +1,180 @@
+<?php
+
+namespace Drupal\Tests\migrate\Kernel\process;
+
+use Drupal\Core\StreamWrapper\StreamWrapperInterface;
+use Drupal\KernelTests\Core\File\FileTestBase;
+use Drupal\migrate\Plugin\migrate\process\FileCopy;
+use Drupal\migrate\MigrateExecutableInterface;
+use Drupal\migrate\Plugin\MigrateProcessInterface;
+use Drupal\migrate\Row;
+
+/**
+ * Tests the file_copy process plugin.
+ *
+ * @group migrate
+ */
+class FileCopyTest extends FileTestBase {
+
+ /**
+ * {@inheritdoc}
+ */
+ public static $modules = ['migrate', 'system'];
+
+ /**
+ * The file system service.
+ *
+ * @var \Drupal\Core\File\FileSystemInterface
+ */
+ protected $fileSystem;
+
+ /**
+ * {@inheritdoc}
+ */
+ protected function setUp() {
+ parent::setUp();
+ $this->fileSystem = $this->container->get('file_system');
+ $this->container->get('stream_wrapper_manager')->registerWrapper('temporary', 'Drupal\Core\StreamWrapper\TemporaryStream', StreamWrapperInterface::LOCAL_NORMAL);
+ }
+
+ /**
+ * Test successful imports/copies.
+ */
+ public function testSuccessfulCopies() {
+ $file = $this->createUri(NULL, NULL, 'temporary');
+ $file_absolute = $this->fileSystem->realpath($file);
+ $data_sets = [
+ // Test a local to local copy.
+ [
+ $this->root . '/core/modules/simpletest/files/image-test.jpg',
+ 'public://file1.jpg'
+ ],
+ // Test a temporary file using an absolute path.
+ [
+ $file_absolute,
+ 'temporary://test.jpg'
+ ],
+ // Test a temporary file using a relative path.
+ [
+ $file_absolute,
+ 'temporary://core/modules/simpletest/files/test.jpg'
+ ],
+ ];
+ foreach ($data_sets as $data) {
+ list($source_path, $destination_path) = $data;
+ $actual_destination = $this->doTransform($source_path, $destination_path);
+ $message = sprintf('File %s exists', $destination_path);
+ $this->assertFileExists($destination_path, $message);
+ // Make sure we didn't accidentally do a move.
+ $this->assertFileExists($source_path, $message);
+ $this->assertSame($actual_destination, $destination_path, 'The import returned the copied filename.');
+ }
+ }
+
+ /**
+ * Test successful moves.
+ */
+ public function testSuccessfulMoves() {
+ $file_1 = $this->createUri(NULL, NULL, 'temporary');
+ $file_1_absolute = $this->fileSystem->realpath($file_1);
+ $file_2 = $this->createUri(NULL, NULL, 'temporary');
+ $file_2_absolute = $this->fileSystem->realpath($file_2);
+ $local_file = $this->createUri(NULL, NULL, 'public');
+ $data_sets = [
+ // Test a local to local copy.
+ [
+ $local_file,
+ 'public://file1.jpg'
+ ],
+ // Test a temporary file using an absolute path.
+ [
+ $file_1_absolute,
+ 'temporary://test.jpg'
+ ],
+ // Test a temporary file using a relative path.
+ [
+ $file_2_absolute,
+ 'temporary://core/modules/simpletest/files/test.jpg'
+ ],
+ ];
+ foreach ($data_sets as $data) {
+ list($source_path, $destination_path) = $data;
+ $actual_destination = $this->doTransform($source_path, $destination_path, ['move' => TRUE]);
+ $message = sprintf('File %s exists', $destination_path);
+ $this->assertFileExists($destination_path, $message);
+ $message = sprintf('File %s does not exist', $source_path);
+ $this->assertFileNotExists($source_path, $message);
+ $this->assertSame($actual_destination, $destination_path, 'The importer returned the moved filename.');
+ }
+ }
+
+ /**
+ * Test that non-existent files throw an exception.
+ *
+ * @expectedException \Drupal\migrate\MigrateException
+ *
+ * @expectedExceptionMessage File '/non/existent/file' does not exist
+ */
+ public function testNonExistentSourceFile() {
+ $source = '/non/existent/file';
+ $this->doTransform($source, 'public://wontmatter.jpg');
+ }
+
+ /**
+ * Test the 'rename' overwrite mode.
+ */
+ public function testRenameFile() {
+ $source = $this->createUri(NULL, NULL, 'temporary');
+ $destination = $this->createUri('foo.txt', NULL, 'public');
+ $expected_destination = 'public://foo_0.txt';
+ $actual_destination = $this->doTransform($source, $destination, ['rename' => TRUE]);
+ $this->assertFileExists($expected_destination, 'File was renamed on import');
+ $this->assertSame($actual_destination, $expected_destination, 'The importer returned the renamed filename.');
+ }
+
+ /**
+ * Tests that remote URIs are delegated to the download plugin.
+ */
+ public function testDownloadRemoteUri() {
+ $download_plugin = $this->getMock(MigrateProcessInterface::class);
+ $download_plugin->expects($this->once())->method('transform');
+
+ $plugin = new FileCopy(
+ [],
+ $this->randomMachineName(),
+ [],
+ $this->container->get('stream_wrapper_manager'),
+ $this->container->get('file_system'),
+ $download_plugin
+ );
+
+ $plugin->transform(
+ ['http://drupal.org/favicon.ico', '/destination/path'],
+ $this->getMock(MigrateExecutableInterface::class),
+ new Row([], []),
+ $this->randomMachineName()
+ );
+ }
+
+ /**
+ * Do an import using the destination.
+ *
+ * @param string $source_path
+ * Source path to copy from.
+ * @param string $destination_path
+ * The destination path to copy to.
+ * @param array $configuration
+ * Process plugin configuration settings.
+ *
+ * @return string
+ * The URI of the copied file.
+ */
+ protected function doTransform($source_path, $destination_path, $configuration = []) {
+ $plugin = FileCopy::create($this->container, $configuration, 'file_copy', []);
+ $executable = $this->prophesize(MigrateExecutableInterface::class)->reveal();
+ $row = new Row([], []);
+
+ return $plugin->transform([$source_path, $destination_path], $executable, $row, 'foobaz');
+ }
+
+}
diff --git a/core/modules/migrate/tests/src/Unit/process/DownloadTest.php b/core/modules/migrate/tests/src/Unit/process/DownloadTest.php
new file mode 100644
index 0000000000000000000000000000000000000000..e4de42e5b68394042f0c5c3cee2912e6ef814c07
--- /dev/null
+++ b/core/modules/migrate/tests/src/Unit/process/DownloadTest.php
@@ -0,0 +1,128 @@
+<?php
+
+namespace Drupal\Tests\migrate\Unit\process;
+
+use Drupal\Core\StreamWrapper\StreamWrapperInterface;
+use Drupal\KernelTests\Core\File\FileTestBase;
+use Drupal\migrate\MigrateException;
+use Drupal\migrate\Plugin\migrate\process\Download;
+use Drupal\migrate\MigrateExecutableInterface;
+use Drupal\migrate\Row;
+use GuzzleHttp\Client;
+use GuzzleHttp\Psr7\Response;
+
+/**
+ * Tests the download process plugin.
+ *
+ * @group migrate
+ */
+class DownloadTest extends FileTestBase {
+
+ /**
+ * {@inheritdoc}
+ */
+ public static $modules = ['system'];
+
+ /**
+ * {@inheritdoc}
+ */
+ protected function setUp() {
+ parent::setUp();
+ $this->container->get('stream_wrapper_manager')->registerWrapper('temporary', 'Drupal\Core\StreamWrapper\TemporaryStream', StreamWrapperInterface::LOCAL_NORMAL);
+ }
+
+ /**
+ * Tests a download that overwrites an existing local file.
+ */
+ public function testOverwritingDownload() {
+ // Create a pre-existing file at the destination, to test overwrite behavior.
+ $destination_uri = $this->createUri('existing_file.txt');
+
+ // Test destructive download.
+ $actual_destination = $this->doTransform($destination_uri);
+ $this->assertSame($destination_uri, $actual_destination, 'Import returned a destination that was not renamed');
+ $this->assertFileNotExists('public://existing_file_0.txt', 'Import did not rename the file');
+ }
+
+ /**
+ * Tests a download that renames the downloaded file if there's a collision.
+ */
+ public function testNonDestructiveDownload() {
+ // Create a pre-existing file at the destination, to test overwrite behavior.
+ $destination_uri = $this->createUri('another_existing_file.txt');
+
+ // Test non-destructive download.
+ $actual_destination = $this->doTransform($destination_uri, ['rename' => TRUE]);
+ $this->assertSame('public://another_existing_file_0.txt', $actual_destination, 'Import returned a renamed destination');
+ $this->assertFileExists($actual_destination, 'Downloaded file was created');
+ }
+
+ /**
+ * Tests that an exception is thrown if the destination URI is not writable.
+ */
+ public function testWriteProectedDestination() {
+ // Create a pre-existing file at the destination, to test overwrite behavior.
+ $destination_uri = $this->createUri('not-writable.txt');
+
+ // Make the destination non-writable.
+ $this->container
+ ->get('file_system')
+ ->chmod($destination_uri, 0444);
+
+ // Pass or fail, we'll need to make the file writable again so the test
+ // can clean up after itself.
+ $fix_permissions = function () use ($destination_uri) {
+ $this->container
+ ->get('file_system')
+ ->chmod($destination_uri, 0755);
+ };
+
+ try {
+ $this->doTransform($destination_uri);
+ $fix_permissions();
+ $this->fail('MigrateException was not thrown for non-writable destination URI.');
+ }
+ catch (MigrateException $e) {
+ $this->assertTrue(TRUE, 'MigrateException was thrown for non-writable destination URI.');
+ $fix_permissions();
+ }
+ }
+
+ /**
+ * Runs an input value through the download plugin.
+ *
+ * @param string $destination_uri
+ * The destination URI to download to.
+ * @param array $configuration
+ * (optional) Configuration for the download plugin.
+ *
+ * @return string
+ * The local URI of the downloaded file.
+ */
+ protected function doTransform($destination_uri, $configuration = []) {
+ // The HTTP client will return a file with contents 'It worked!'
+ $body = fopen('data://text/plain;base64,SXQgd29ya2VkIQ==', 'r');
+
+ // Prepare a mock HTTP client.
+ $this->container->set('http_client', $this->getMock(Client::class));
+ $this->container->get('http_client')
+ ->method('get')
+ ->willReturn(new Response(200, [], $body));
+
+ // Instantiate the plugin statically so it can pull dependencies out of
+ // the container.
+ $plugin = Download::create($this->container, $configuration, 'download', []);
+
+ // Execute the transformation.
+ $executable = $this->getMock(MigrateExecutableInterface::class);
+ $row = new Row([], []);
+
+ // Return the downloaded file's local URI.
+ $value = [
+ 'http://drupal.org/favicon.ico',
+ $destination_uri,
+ ];
+ return $plugin->transform($value, $executable, $row, 'foobaz');
+ }
+
+}