Commit 78714b90 authored by alexpott's avatar alexpott

Issue #2695297 by mikeryan, ohthehugemanatee, quietone, vasi, Sharique, chx,...

Issue #2695297 by mikeryan, ohthehugemanatee, quietone, vasi, Sharique, chx, benjy, phenaproxima, heddn, benjifisher: Refactor EntityFile and use process plugins instead
parent f2515062
# Every migration that saves into {file_managed} must have the d6_file
# migration as an optional dependency to ensure d6_file runs first.
# Every migration that references a file by Drupal 6 fid should specify this
# migration as an optional dependency.
id: d6_file
label: Files
migration_tags:
- Drupal 6
source:
plugin: d6_file
constants:
# source_base_path must be set by the tool configuring this migration. It
# represents the fully qualified path relative to which URIs in the files
# table are specified, and must end with a /. See source_full_path
# configuration in this migration's process pipeline as an example.
source_base_path: ''
process:
fid: fid
filename: filename
uri:
source_full_path:
-
plugin: concat
delimiter: /
source:
- constants/source_base_path
- filepath
-
plugin: urlencode
destination_full_path:
plugin: file_uri
source:
- filepath
- file_directory_path
- temp_directory_path
- is_public
uri:
plugin: file_copy
source:
- '@source_full_path'
- '@destination_full_path'
filemime: filemime
filesize: filesize
status: status
......
# Every migration that references a file by fid should specify this migration
# as an optional dependency.
# Every migration that references a file by Drupal 7 fid should specify this
# migration as an optional dependency.
id: d7_file
label: Files
migration_tags:
- Drupal 7
source:
plugin: d7_file
constants:
# source_base_path must be set by the tool configuring this migration. It
# represents the fully qualified path relative to which uris in the files
# table are specified, and must end with a /. See source_full_path
# configuration in this migration's process pipeline as an example.
source_base_path: ''
process:
fid: fid
filename: filename
uri: uri
source_full_path:
-
plugin: concat
delimiter: /
source:
- constants/source_base_path
- filepath
-
plugin: urlencode
uri:
plugin: file_copy
source:
- '@source_full_path'
- uri
filemime: filemime
# filesize is dynamically computed when file entities are saved, so there is
# no point in migrating it.
......@@ -22,5 +41,3 @@ process:
uid: uid
destination:
plugin: entity:file
source_path_property: filepath
urlencode: true
......@@ -3,75 +3,18 @@
namespace Drupal\file\Plugin\migrate\destination;
use Drupal\Component\Utility\Unicode;
use Drupal\Core\Entity\EntityManagerInterface;
use Drupal\Core\Entity\EntityStorageInterface;
use Drupal\Core\Field\FieldTypePluginManagerInterface;
use Drupal\Core\Field\Plugin\Field\FieldType\UriItem;
use Drupal\Core\File\FileSystemInterface;
use Drupal\Core\StreamWrapper\LocalStream;
use Drupal\Core\StreamWrapper\StreamWrapperManagerInterface;
use Drupal\migrate\Plugin\MigrationInterface;
use Drupal\migrate\Row;
use Drupal\migrate\MigrateException;
use Drupal\migrate\Plugin\migrate\destination\EntityContentBase;
use Symfony\Component\DependencyInjection\ContainerInterface;
/**
* Every migration that uses this destination must have an optional
* dependency on the d6_file migration to ensure it runs first.
*
* @MigrateDestination(
* id = "entity:file"
* )
*/
class EntityFile extends EntityContentBase {
/**
* @var \Drupal\Core\StreamWrapper\StreamWrapperManagerInterface
*/
protected $streamWrapperManager;
/**
* @var \Drupal\Core\File\FileSystemInterface
*/
protected $fileSystem;
/**
* {@inheritdoc}
*/
public function __construct(array $configuration, $plugin_id, $plugin_definition, MigrationInterface $migration, EntityStorageInterface $storage, array $bundles, EntityManagerInterface $entity_manager, FieldTypePluginManagerInterface $field_type_manager, StreamWrapperManagerInterface $stream_wrappers, FileSystemInterface $file_system) {
$configuration += array(
'source_base_path' => '',
'source_path_property' => 'filepath',
'destination_path_property' => 'uri',
'move' => FALSE,
'urlencode' => FALSE,
);
parent::__construct($configuration, $plugin_id, $plugin_definition, $migration, $storage, $bundles, $entity_manager, $field_type_manager);
$this->streamWrapperManager = $stream_wrappers;
$this->fileSystem = $file_system;
}
/**
* {@inheritdoc}
*/
public static function create(ContainerInterface $container, array $configuration, $plugin_id, $plugin_definition, MigrationInterface $migration = NULL) {
$entity_type = static::getEntityTypeId($plugin_id);
return new static(
$configuration,
$plugin_id,
$plugin_definition,
$migration,
$container->get('entity.manager')->getStorage($entity_type),
array_keys($container->get('entity.manager')->getBundleInfo($entity_type)),
$container->get('entity.manager'),
$container->get('plugin.manager.field.field_type'),
$container->get('stream_wrapper_manager'),
$container->get('file_system')
);
}
/**
* {@inheritdoc}
*/
......@@ -82,7 +25,12 @@ protected function getEntity(Row $row, array $old_destination_id_values) {
return parent::getEntity($row, $old_destination_id_values);
}
$destination = $row->getDestinationProperty($this->configuration['destination_path_property']);
// By default the entity key (fid) would be used, but we want to make sure
// we're loading the matching URI.
$destination = $row->getDestinationProperty('uri');
if (empty($destination)) {
throw new MigrateException('Destination property uri not provided');
}
$entity = $this->storage->loadByProperties(['uri' => $destination]);
if ($entity) {
return reset($entity);
......@@ -92,181 +40,6 @@ protected function getEntity(Row $row, array $old_destination_id_values) {
}
}
/**
* {@inheritdoc}
*/
public function import(Row $row, array $old_destination_id_values = array()) {
// For stub rows, there is no real file to deal with, let the stubbing
// process create the stub entity.
if ($row->isStub()) {
return parent::import($row, $old_destination_id_values);
}
$file = $row->getSourceProperty($this->configuration['source_path_property']);
$destination = $row->getDestinationProperty($this->configuration['destination_path_property']);
$source = $this->configuration['source_base_path'] . $file;
// Ensure the source file exists, if it's a local URI or path.
if ($this->isLocalUri($source) && !file_exists($source)) {
throw new MigrateException("File '$source' does not exist.");
}
// If the start and end file is exactly the same, there is nothing to do.
if ($this->isLocationUnchanged($source, $destination)) {
return parent::import($row, $old_destination_id_values);
}
$replace = $this->getOverwriteMode($row);
$success = $this->writeFile($source, $destination, $replace);
if (!$success) {
$dir = $this->getDirectory($destination);
if (file_prepare_directory($dir, FILE_CREATE_DIRECTORY)) {
$success = $this->writeFile($source, $destination, $replace);
}
else {
throw new MigrateException("Could not create directory '$dir'");
}
}
if ($success) {
return parent::import($row, $old_destination_id_values);
}
else {
throw new MigrateException("File $source could not be copied to $destination.");
}
}
/**
* Tries to move or copy a file.
*
* @param string $source
* The source path or URI.
* @param string $destination
* The destination path or URI.
* @param int $replace
* (optional) FILE_EXISTS_REPLACE (default) or FILE_EXISTS_RENAME.
*
* @return bool
* TRUE on success, FALSE on failure.
*/
protected function writeFile($source, $destination, $replace = FILE_EXISTS_REPLACE) {
if ($this->configuration['move']) {
return (boolean) file_unmanaged_move($source, $destination, $replace);
}
else {
$destination = file_destination($destination, $replace);
$source = $this->urlencode($source);
return @copy($source, $destination);
}
}
/**
* Determines how to handle file conflicts.
*
* @param \Drupal\migrate\Row $row
*
* @return int
* Either FILE_EXISTS_REPLACE (default) or FILE_EXISTS_RENAME, depending
* on the current configuration.
*/
protected function getOverwriteMode(Row $row) {
if (!empty($this->configuration['rename'])) {
$entity_id = $row->getDestinationProperty($this->getKey('id'));
if ($entity_id && ($entity = $this->storage->load($entity_id))) {
return FILE_EXISTS_RENAME;
}
}
return FILE_EXISTS_REPLACE;
}
/**
* Returns the directory component of a URI or path.
*
* For URIs like public://foo.txt, the full physical path of public://
* will be returned, since a scheme by itself will trip up certain file
* API functions (such as file_prepare_directory()).
*
* @param string $uri
* The URI or path.
*
* @return string|false
* The directory component of the path or URI, or FALSE if it could not
* be determined.
*/
protected function getDirectory($uri) {
$dir = $this->fileSystem->dirname($uri);
if (substr($dir, -3) == '://') {
return $this->fileSystem->realpath($dir);
}
else {
return $dir;
}
}
/**
* Returns if the source and destination URIs represent identical paths.
* If either URI is a remote stream, will return FALSE.
*
* @param string $source
* The source URI.
* @param string $destination
* The destination URI.
*
* @return bool
* TRUE if the source and destination URIs refer to the same physical path,
* otherwise FALSE.
*/
protected function isLocationUnchanged($source, $destination) {
if ($this->isLocalUri($source) && $this->isLocalUri($destination)) {
return $this->fileSystem->realpath($source) === $this->fileSystem->realpath($destination);
}
else {
return FALSE;
}
}
/**
* Returns if the given URI or path is considered local.
*
* A URI or path is considered local if it either has no scheme component,
* or the scheme is implemented by a stream wrapper which extends
* \Drupal\Core\StreamWrapper\LocalStream.
*
* @param string $uri
* The URI or path to test.
*
* @return bool
*/
protected function isLocalUri($uri) {
$scheme = $this->fileSystem->uriScheme($uri);
return $scheme === FALSE || $this->streamWrapperManager->getViaScheme($scheme) instanceof LocalStream;
}
/**
* Urlencode all the components of a remote filename.
*
* @param string $filename
* The filename of the file to be urlencoded.
*
* @return string
* The urlencoded filename.
*/
protected function urlencode($filename) {
// Only apply to a full URL
if ($this->configuration['urlencode'] && strpos($filename, '://')) {
$components = explode('/', $filename);
foreach ($components as $key => $component) {
$components[$key] = rawurlencode($component);
}
$filename = implode('/', $components);
// Actually, we don't want certain characters encoded
$filename = str_replace('%3A', ':', $filename);
$filename = str_replace('%3F', '?', $filename);
$filename = str_replace('%26', '&', $filename);
}
return $filename;
}
/**
* {@inheritdoc}
*/
......
......@@ -84,9 +84,7 @@ public function prepareRow(Row $row) {
// At this point, $path could be an absolute path or a relative path,
// depending on how the scheme's variable was set. So we need to shear out
// the source_base_path in order to make them all relative.
// @todo https://www.drupal.org/node/2577871 Don't depend on destination
// configuration and figure out if this is even needed at all?
$path = str_replace($this->migration->getDestinationConfiguration()['source_base_path'], NULL, $path);
$path = str_replace($this->configuration['constants']['source_base_path'], NULL, $path);
$row->setSourceProperty('filepath', $path);
return parent::prepareRow($row);
}
......
<?php
/**
* @file
* Contains \Drupal\Tests\file\Kernel\Migrate\EntityFileTest.
*/
namespace Drupal\Tests\file\Kernel\Migrate;
use Drupal\Core\StreamWrapper\StreamWrapperInterface;
use Drupal\migrate\Row;
use Drupal\file\Plugin\migrate\destination\EntityFile;
use Drupal\Core\Entity\ContentEntityInterface;
use Drupal\entity_test\Entity\EntityTest;
use Drupal\migrate\MigrateException;
use Drupal\KernelTests\KernelTestBase;
/**
* Tests the entity file destination plugin.
*
* @group file
*/
class EntityFileTest extends KernelTestBase {
/**
* Modules to install.
*
* @var array
*/
public static $modules = array('system', 'entity_test', 'user', 'file');
/**
* @var \Drupal\Tests\file\Kernel\Migrate\TestEntityFile $destination
*/
protected $destination;
/**
* {@inheritdoc}
*/
protected function setUp() {
parent::setUp();
\Drupal::getContainer()->get('stream_wrapper_manager')->registerWrapper('public', 'Drupal\Core\StreamWrapper\PublicStream', StreamWrapperInterface::NORMAL);
$this->destination = new TestEntityFile([]);
$this->installEntitySchema('file');
file_put_contents('/tmp/test-file.jpg', '');
}
/**
* Test successful imports/copies.
*/
public function testSuccessfulCopies() {
foreach ($this->localFileDataProvider() as $data) {
list($row_values, $destination_path, $expected, $source_base_path) = $data;
$this->doImport($row_values, $destination_path, $source_base_path);
$message = $expected ? sprintf('File %s exists', $destination_path) : sprintf('File %s does not exist', $destination_path);
$this->assertIdentical($expected, is_file($destination_path), $message);
}
}
/**
* The data provider for testing the file destination.
*
* @return array
* An array of file permutations to test.
*/
protected function localFileDataProvider() {
return [
// Test a local to local copy.
[['filepath' => 'core/modules/simpletest/files/image-test.jpg'], 'public://file1.jpg', TRUE, $this->root . '/'],
// Test a temporary file using an absolute path.
[['filepath' => '/tmp/test-file.jpg'], 'temporary://test.jpg', TRUE, ''],
// Test a temporary file using a relative path.
[['filepath' => 'test-file.jpg'], 'temporary://core/modules/simpletest/files/test.jpg', TRUE, '/tmp/'],
// Test a remote path to local.
[['filepath' => 'core/modules/simpletest/files/image-test.jpg'], 'public://remote-file.jpg', TRUE, $this->root . '/'],
// Test a remote path to local inside a folder that doesn't exist.
[['filepath' => 'core/modules/simpletest/files/image-test.jpg'], 'public://folder/remote-file.jpg', TRUE, $this->root . '/'],
];
}
/**
* Test that non-existent files throw an exception.
*/
public function testNonExistentSourceFile() {
$destination = '/non/existent/file';
try {
// If this test passes, doImport() will raise a MigrateException and
// we'll never reach fail().
$this->doImport(['filepath' => $destination], 'public://wontmatter.jpg');
$this->fail('Expected Drupal\migrate\MigrateException when importing ' . $destination);
}
catch (MigrateException $e) {
$this->assertIdentical($e->getMessage(), "File '$destination' does not exist.");
}
}
/**
* Tests various invocations of the writeFile() method.
*/
public function testWriteFile() {
$plugin = $this->destination;
$method = new \ReflectionMethod($plugin, 'writeFile');
$method->setAccessible(TRUE);
touch('temporary://baz.txt');
// Moving an actual file should return TRUE.
$plugin->configuration['move'] = TRUE;
$this->assertTrue($method->invoke($plugin, 'temporary://baz.txt', 'public://foo.txt'));
// Trying to move a non-existent file should return FALSE.
$this->assertFalse($method->invoke($plugin, 'temporary://invalid.txt', 'public://invalid.txt'));
// Copying over a file that already exists should replace the existing file.
$plugin->configuration['move'] = FALSE;
touch('temporary://baz.txt');
$this->assertTrue($method->invoke($plugin, 'temporary://baz.txt', 'public://foo.txt'));
// Copying over a file that already exists should rename the resulting file
// if FILE_EXISTS_RENAME is specified.
$method->invoke($plugin, 'temporary://baz.txt', 'public://foo.txt', FILE_EXISTS_RENAME);
$this->assertTrue(file_exists('public://foo_0.txt'));
// Trying to copy a non-existent file should return FALSE.
$this->assertFalse($method->invoke($plugin, 'temporary://invalid.txt', 'public://invalid.txt'));
}
/**
* Tests various invocations of the getOverwriteMode() method.
*/
public function testGetOverwriteMode() {
$plugin = $this->destination;
$method = new \ReflectionMethod($plugin, 'getOverwriteMode');
$method->setAccessible(TRUE);
$row = new Row([], []);
// If the plugin is not configured to rename the destination file, we should
// always get FILE_EXISTS_REPLACE.
$this->assertIdentical(FILE_EXISTS_REPLACE, $method->invoke($plugin, $row));
// When the plugin IS configured to rename the destination file, it should
// return FILE_EXISTS_RENAME if the destination entity already exists,
// and FILE_EXISTS_REPLACE otherwise.
$plugin->configuration['rename'] = TRUE;
$plugin->storage = \Drupal::entityManager()->getStorage('file');
/** @var \Drupal\file\FileInterface $file */
$file = $plugin->storage->create();
touch('public://foo.txt');
$file->setFileUri('public://foo.txt');
$file->save();
$row->setDestinationProperty($plugin->storage->getEntityType()->getKey('id'), $file->id());
$this->assertIdentical(FILE_EXISTS_RENAME, $method->invoke($plugin, $row));
unlink('public://foo.txt');
}
/**
* Tests various invocations of the getDirectory() method.
*/
public function testGetDirectory() {
$plugin = $this->destination;
$method = new \ReflectionMethod($plugin, 'getDirectory');
$method->setAccessible(TRUE);
$this->assertSame('public://foo', $method->invoke($plugin, 'public://foo/baz.txt'));
$this->assertSame('/path/to', $method->invoke($plugin, '/path/to/foo.txt'));
// A directory like public:// (no path) needs to resolve to a physical path.
$fs = \Drupal::getContainer()->get('file_system');
$this->assertSame($fs->realpath('public://'), $method->invoke($plugin, 'public://foo.txt'));
}
/**
* Tests various invocations of the isLocationUnchanged() method.
*/
public function testIsLocationUnchanged() {
$plugin = $this->destination;
$method = new \ReflectionMethod($plugin, 'isLocationUnchanged');
$method->setAccessible(TRUE);
$temporary_file = '/tmp/foo.txt';
touch($temporary_file);
$this->assertTrue($method->invoke($plugin, $temporary_file, 'temporary://foo.txt'));
unlink($temporary_file);
}
/**
* Tests various invocations of the isLocalUri() method.
*/
public function testIsLocalUri() {
$plugin = $this->destination;
$method = new \ReflectionMethod($plugin, 'isLocalUri');
$method->setAccessible(TRUE);
$this->assertTrue($method->invoke($plugin, 'public://foo.txt'));
$this->assertTrue($method->invoke($plugin, 'public://path/to/foo.txt'));
$this->assertTrue($method->invoke($plugin, 'temporary://foo.txt'));
$this->assertTrue($method->invoke($plugin, 'temporary://path/to/foo.txt'));
$this->assertTrue($method->invoke($plugin, 'foo.txt'));
$this->assertTrue($method->invoke($plugin, '/path/to/files/foo.txt'));
$this->assertTrue($method->invoke($plugin, 'relative/path/to/foo.txt'));
$this->assertFalse($method->invoke($plugin, 'http://www.example.com/foo.txt'));
}
/**
* Do an import using the destination.
*
* @param array $row_values
* An array of row values.
* @param string $destination_path
* The destination path to copy to.
* @param string $source_base_path
* The source base path.
* @return array
* An array of saved entities ids.
*
* @throws \Drupal\migrate\MigrateException
*/
protected function doImport($row_values, $destination_path, $source_base_path = '') {
$row = new Row($row_values, []);
$row->setDestinationProperty('uri', $destination_path);
$this->destination->configuration['source_base_path'] = $source_base_path;
// Importing asserts there are no errors, then we just check the file has
// been copied into place.
return $this->destination->import($row, array());
}
}
class TestEntityFile extends EntityFile {
/**
* This is needed to be passed to $this->save().
*
* @var \Drupal\Core\Entity\ContentEntityInterface
*/
public $mockEntity;
/**
* Make this public for easy writing during tests.
*
* @var array
*/
public $configuration;
/**
* @var \Drupal\Core\Entity\EntityStorageInterface
*/
public $storage;
public function __construct($configuration = []) {
$configuration += array(
'source_base_path' => '',
'source_path_property' => 'filepath',
'destination_path_property' => 'uri',
'move' => FALSE,
'urlencode' => FALSE,
);
$this->configuration = $configuration;
// We need a mock entity to be passed to save to prevent strict exceptions.
$this->mockEntity = EntityTest::create();
$this->streamWrapperManager = \Drupal::getContainer()->get('stream_wrapper_manager');
$this->fileSystem = \Drupal::getContainer()->get('file_system');
}
/**
* {@inheritdoc}
*/
protected function getEntity(Row $row, array $old_destination_id_values) {
return $this->mockEntity;
}
/**
* {@inheritdoc}
*/
protected function save(ContentEntityInterface $entity, array $old_destination_id_values = array()) {}
}
<?php
namespace Drupal\Tests\file\Kernel\Migrate\d6;
use Drupal\migrate\Plugin\MigrationInterface;
/**
* Helper for setting up a file migration test.
......@@ -14,15 +15,23 @@ protected function setUpMigratedFiles() {
$this->installEntitySchema('file');
$this->installConfig(['file']);
/** @var \Drupal\migrate\Plugin\MigrationInterface $migration */
$migration_plugin_manager = $this->container->get('plugin.manager.migration');
$this->executeMigration('d6_file');
}
/** @var \Drupal\migrate\Plugin\migration $migration */
$migration = $migration_plugin_manager->createInstance('d6_file');
$source = $migration->getSourceConfiguration();
$source['site_path'] = 'core/modules/simpletest';
$migration->set('source', $source);
$this->executeMigration($migration);
/**
* {@inheritdoc}
*/
protected function prepareMigration(MigrationInterface $migration) {
// File migrations need a source_base_path.
// @see MigrateUpgradeRunBatch::run
$destination = $migration->getDestinationConfiguration();
if ($destination['plugin'] === 'entity:file') {
// Make sure we have a single trailing slash.
$source = $migration->getSourceConfiguration();
$source['site_path'] = 'core/modules/simpletest';
$source['constants']['source_base_path'] = \Drupal::root() . '/';
$migration->set('source', $source);
}
}
}
......@@ -72,9 +72,8 @@ public function testFiles() {
$this->assertEntity(5, 'html-1.txt', '24', 'public://html-1.txt', 'text/plain', '1');
// Test that we can re-import and also test with file_directory_path set.
$migration_plugin_manager = $this->container->get('plugin.manager.migration');
\Drupal::database()
->truncate($migration_plugin_manager->createInstance('d6_file')->getIdMap()->mapTableName())
->truncate($this->getMigration('d6_file')->getIdMap()->mapTableName())