Skip to content
Snippets Groups Projects
Commit 8c17a63c authored by Mike Ryan's avatar Mike Ryan
Browse files

Issue #2609556 by mikeryan: Remove migrate_source_csv from migrate_plus

parent 7ed9742d
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
migrate_source_csv/migrate_source_csv.info.yml deleted 100644 → 0
+ 0
− 7
View file @ 7ed9742d
type: module
name: Migrate Source CSV
description: 'CSV source migration.'
package: Migration
core: 8.x
dependencies:
- migrate
migrate_source_csv/src/CSVFileObject.php deleted 100644 → 0
+ 0
− 120
View file @ 7ed9742d
<?php
/**
* @file
* Contains \Drupal\migrate_source_csv\CSVFileObject.php.
*/
namespace Drupal\migrate_source_csv;
/**
* Defines a CSV file object.
*
* @package Drupal\migrate_source_csv.
*
* Extends SPLFileObject to:
* - assume CSV format
* - skip header rows on rewind()
* - address columns by header row name instead of index.
*/
class CSVFileObject extends \SplFileObject {
/**
* The number of rows in the CSV file before the data starts.
*
* @var integer
*/
protected $headerRowCount = 0;
/**
* The human-readable column headers, keyed by column index in the CSV.
*
* @var array
*/
protected $columnNames = array();
/**
* {@inheritdoc}
*/
public function __construct($file_name) {
// Necessary to use this approach because SplFileObject doesn't like NULL
// arguments passed to it.
call_user_func_array(array('parent', '__construct'), func_get_args());
$this->setFlags(CSVFileObject::READ_CSV | CSVFileObject::READ_AHEAD | CSVFileObject::DROP_NEW_LINE | CSVFileObject::SKIP_EMPTY);
}
/**
* {@inheritdoc}
*/
public function rewind() {
$this->seek($this->getHeaderRowCount());
}
/**
* {@inheritdoc}
*/
public function current() {
$row = parent::current();
if ($row && !empty($this->getColumnNames())) {
// Only use columns specified in the defined CSV columns.
$row = array_intersect_key($row, $this->getColumnNames());
// Set meaningful keys for the columns mentioned in $this->csvColumns.
foreach ($this->getColumnNames() as $int => $values) {
// Copy value to more descriptive string based key and unset original.
$row[key($values)] = isset($row[$int]) ? $row[$int] : NULL;
unset($row[$int]);
}
}
return $row;
}
/**
* Return a count of all available source records.
*/
public function count() {
return iterator_count($this);
}
/**
* Number of header rows.
*
* @return int
* Get the number of header rows, zero if no header row.
*/
public function getHeaderRowCount() {
return $this->headerRowCount;
}
/**
* Number of header rows.
*
* @param int $header_row_count
* Set the number of header rows, zero if no header row.
*/
public function setHeaderRowCount($header_row_count) {
$this->headerRowCount = $header_row_count;
}
/**
* CSV column names.
*
* @return array
* Get CSV column names.
*/
public function getColumnNames() {
return $this->columnNames;
}
/**
* CSV column names.
*
* @param array $column_names
* Set CSV column names.
*/
public function setColumnNames(array $column_names) {
$this->columnNames = $column_names;
}
}
migrate_source_csv/src/Plugin/migrate/source/CSV.php deleted 100644 → 0
+ 0
− 140
View file @ 7ed9742d
<?php
/**
* @file
* Contains \Drupal\migrate_source_csv\Plugin\migrate\source\CSV.
*/
namespace Drupal\migrate_source_csv\Plugin\migrate\source;
use Drupal\migrate\Entity\MigrationInterface;
use Drupal\migrate\MigrateException;
use Drupal\migrate\Plugin\migrate\source\SourcePluginBase;
use Drupal\migrate_source_csv\CSVFileObject;
/**
* Source for CSV.
*
* If the CSV file contains non-ASCII characters, make sure it includes a
* UTF BOM (Byte Order Marker) so they are interpreted correctly.
*
* @MigrateSource(
* id = "csv"
* )
*/
class CSV extends SourcePluginBase {
/**
* List of available source fields.
*
* Keys are the field machine names as used in field mappings, values are
* descriptions.
*
* @var array
*/
protected $fields = array();
/**
* The source ids, as indexes.
*
* @var array
*/
protected $identifiers = array();
/**
* {@inheritdoc}
*/
public function __construct(array $configuration, $plugin_id, $plugin_definition, MigrationInterface $migration) {
parent::__construct($configuration, $plugin_id, $plugin_definition, $migration);
// Path is required.
if (empty($this->configuration['path'])) {
throw new MigrateException('You must declare the "path" to the source CSV file in your source settings.');
}
// Identifier field(s) are required.
if (empty($this->configuration['identifiers'])) {
throw new MigrateException('You must declare "identifiers" as a unique array of fields in your source settings.');
}
}
/**
* Return a string representing the source query.
*
* @return string
* The file path.
*/
public function __toString() {
return $this->configuration['path'];
}
/**
* {@inheritdoc}
*/
public function initializeIterator() {
// File handler using header-rows-respecting extension of SPLFileObject.
$file = new CSVFileObject($this->configuration['path']);
// Set basics of CSV behavior based on configuration.
$delimiter = !empty($this->configuration['delimiter']) ? $this->configuration['delimiter'] : ',';
$enclosure = !empty($this->configuration['enclosure']) ? $this->configuration['enclosure'] : '"';
$escape = !empty($this->configuration['escape']) ? $this->configuration['escape'] : '\\';
$file->setCsvControl($delimiter, $enclosure, $escape);
// Figure out what CSV column(s) to use.
if (!empty($this->configuration['header_row_count'])) {
$file->setHeaderRowCount($this->configuration['header_row_count']);
// Find the last header line.
$file->rewind();
$file->seek($file->getHeaderRowCount() - 1);
// Use the header row(s).
if (empty($this->configuration['column_names'])) {
$row = $file->current();
foreach ($row as $header) {
$header = trim($header);
$column_names[] = array($header => $header);
}
$file->setColumnNames($column_names);
}
}
// An explicit list of column name(s) is provided.
if (!empty($this->configuration['column_names'])) {
$file->setColumnNames($this->configuration['column_names']);
}
return $file;
}
/**
* {@inheritdoc}
*/
public function getIds() {
$ids = array();
foreach ($this->configuration['identifiers'] as $key) {
$ids[$key]['type'] = 'string';
}
return $ids;
}
/**
* {@inheritdoc}
*/
public function fields() {
$fields = array();
foreach ($this->getIterator()->getColumnNames() as $column) {
$key = key($column);
$fields[$key] = $key;
}
// Any caller-specified fields with the same names as extracted fields will
// override them; any others will be added.
if (!empty($this->configuration['fields'])) {
$fields = $this->configuration['fields'] + $fields;
}
return $fields;
}
}
migrate_source_csv/tests/src/Unit/CSVUnitTestCase.php deleted 100644 → 0
+ 0
− 86
View file @ 7ed9742d
<?php
/**
* @file
* Code for CSVFileObjectTest.php.
*/
namespace Drupal\Tests\migrate_source_csv\Unit;
use Drupal\Tests\UnitTestCase;
use org\bovigo\vfs\vfsStream;
/**
* Base unit test to build csv file contents.
*
* @group migrate_source_csv
*/
abstract class CSVUnitTestCase extends UnitTestCase {
/**
* The happy path file url.
*
* @var string
*/
protected $happyPath;
/**
* The un-happy path file url.
*
* @var string
*/
protected $sad;
/**
* {@inheritdoc}
*/
protected function setUp() {
$root_dir = vfsStream::setup('root');
$happy = <<<'EOD'
id,first_name,last_name,email,country,ip_address
1,Justin,Dean,jdean0@example.com,Indonesia,60.242.130.40
2,Joan,Jordan,jjordan1@example.com,Thailand,137.230.209.171
3,William,Ray,wray2@example.com,Germany,4.75.251.71
4,Jack,Collins,jcollins3@example.com,Indonesia,118.241.243.64
5,Jean,Moreno,jmoreno4@example.com,Portugal,12.24.215.20
6,Dennis,Mitchell,dmitchell5@example.com,Mexico,185.24.131.116
7,Harry,West,hwest6@example.com,Uzbekistan,101.74.110.171
8,Rebecca,Hunt,rhunt7@example.com,France,253.107.6.23
9,Rose,Rogers,rrogers8@example.com,China,21.2.126.228
10,Juan,Walker,jwalker9@example.com,Angola,192.118.77.225
11,Lois,Price,lpricea@example.com,Greece,231.185.100.19
12,Patricia,Bell,pbellb@example.com,Sweden,226.2.254.94
13,Gerald,Kelly,gkellyc@example.com,China,31.204.2.163
14,Kimberly,Jackson,kjacksond@example.com,Thailand,19.187.65.116
15,Jason,Mason,jmasone@example.com,Greece,225.129.68.203
EOD;
$sad = <<<'EOD'
1|%Justin%|Dean|jdean0@example.com|Indonesia|60.242.130.40
2|Joan|Jordan|jjordan1@example.com|Thailand|137.230.209.171
3|William|Ray|wray2@example.com|Germany|4.75.251.71
4|Jack|Collins|jcollins3@example.com|Indonesia|118.241.243.64
5|Jean|Moreno|jmoreno4@example.com|Portugal|12.24.215.20
6|Dennis|Mitchell|dmitchell5@example.com|Mexico|185.24.131.116
7|Harry|West|hwest6@example.com|Uzbekistan|101.74.110.171
8|Rebecca|Hunt|rhunt7@example.com|France|253.107.6.23
9|Rose|Rogers|rrogers8@example.com|China|21.2.126.228
10|Juan|Walker|jwalker9@example.com|Angola|192.118.77.225
11|Lois|Price|lpricea@example.com|Greece|231.185.100.19
12|Patricia|Bell|pbellb@example.com|Sweden|226.2.254.94
13|Gerald|Kelly|gkellyc@example.com|China|31.204.2.163
14|Kimberly|Jackson|kjacksond@example.com|Thailand|19.187.65.116
15|Jason|Mason|jmasone@example.com|Greece|225.129.68.203
EOD;
$this->happyPath = vfsStream::newFile('data.csv')
->at($root_dir)
->withContent($happy)
->url();
$this->sad = vfsStream::newFile('data_edge_case.csv')
->at($root_dir)
->withContent($sad)
->url();
}
}
migrate_source_csv/tests/src/Unit/Plugin/migrate/source/CSVTest.php deleted 100644 → 0
+ 0
− 273
View file @ 7ed9742d
<?php
/**
* @file
* Code for CSVTest.php.
*/
namespace Drupal\Tests\migrate_source_csv\Unit\Plugin\migrate\source;
use Drupal\migrate_source_csv\Plugin\migrate\source\CSV;
use Drupal\Tests\migrate_source_csv\Unit\CSVUnitTestCase;
/**
* @coversDefaultClass \Drupal\migrate_source_csv\Plugin\migrate\source\CSV
*
* @group migrate_source_csv
*/
class CSVTest extends CSVUnitTestCase {
/**
* The plugin id.
*
* @var string
*/
protected $pluginId;
/**
* The plugin definition.
*
* @var array
*/
protected $pluginDefinition;
/**
* The mock migration plugin.
*
* @var \Drupal\migrate\Entity\MigrationInterface
*/
protected $plugin;
/**
* {@inheritdoc}
*/
public function setUp() {
parent::setUp();
$this->pluginId = 'test csv migration';
$this->pluginDefinition = array();
$this->plugin = $this->getMock('\Drupal\migrate\Entity\MigrationInterface');
}
/**
* Tests the construction of CSV.
*
* @test
*
* @covers ::__construct
*/
public function create() {
$configuration = array(
'path' => $this->happyPath,
'identifiers' => array('id'),
'header_row_count' => 1,
);
$csv = new CSV($configuration, $this->pluginId, $this->pluginDefinition, $this->plugin);
$this->assertInstanceOf('\Drupal\migrate_source_csv\Plugin\migrate\source\CSV', $csv);
}
/**
* Tests that a missing path will throw an exception.
*
* @test
*
* @expectedException \Drupal\migrate\MigrateException
*
* @expectedExceptionMessage You must declare the "path" to the source CSV file in your source settings.
*/
public function migrateExceptionPathMissing() {
new CSV(array(), $this->pluginId, $this->pluginDefinition, $this->plugin);
}
/**
* Tests that missing identifiers will throw an exception.
*
* @test
*
* @expectedException \Drupal\migrate\MigrateException
*
* @expectedExceptionMessage You must declare "identifiers" as a unique array of fields in your source settings.
*/
public function migrateExceptionIdentifiersMissing() {
$configuration = array(
'path' => $this->happyPath,
);
new CSV($configuration, $this->pluginId, $this->pluginDefinition, $this->plugin);
}
/**
* Tests that toString functions as expected.
*
* @test
*
* @covers ::__toString
*/
public function toString() {
$configuration = array(
'path' => $this->happyPath,
'identifiers' => array('id'),
'header_row_count' => 1,
);
$csv = new CSV($configuration, $this->pluginId, $this->pluginDefinition, $this->plugin);
$this->assertEquals($configuration['path'], (string) $csv);
}
/**
* Tests initialization of the iterator.
*
* @test
*
* @covers ::initializeIterator
*/
public function initializeIterator() {
$configuration = array(
'path' => $this->happyPath,
'identifiers' => array('id'),
'header_row_count' => 1,
);
$config_common = array(
'path' => $this->sad,
'identifiers' => array('id'),
);
$config_delimiter = array('delimiter' => '|');
$config_enclosure = array('enclosure' => '%');
$config_escape = array('escape' => '`');
$csv = new CSV($config_common + $config_delimiter, $this->pluginId, $this->pluginDefinition, $this->plugin);
$this->assertEquals(current($config_delimiter), $csv->initializeIterator()
->getCsvControl()[0]);
$this->assertEquals('"', $csv->initializeIterator()->getCsvControl()[1]);
$csv = new CSV($config_common + $config_enclosure, $this->pluginId, $this->pluginDefinition, $this->plugin);
$this->assertEquals(',', $csv->initializeIterator()->getCsvControl()[0]);
$this->assertEquals(current($config_enclosure), $csv->initializeIterator()
->getCsvControl()[1]);
$csv = new CSV($config_common + $config_delimiter + $config_enclosure + $config_escape, $this->pluginId, $this->pluginDefinition, $this->plugin);
$csv_file_object = $csv->getIterator();
$row = array(
'1',
'Justin',
'Dean',
'jdean0@example.com',
'Indonesia',
'60.242.130.40',
);
$csv_file_object->rewind();
$current = $csv_file_object->current();
$this->assertArrayEquals($row, $current);
$csv = new CSV($configuration, $this->pluginId, $this->pluginDefinition, $this->plugin);
$csv_file_object = $csv->getIterator();
$row = array(
'id' => '1',
'first_name' => 'Justin',
'last_name' => 'Dean',
'email' => 'jdean0@example.com',
'country' => 'Indonesia',
'ip_address' => '60.242.130.40',
);
$second_row = array(
'id' => '2',
'first_name' => 'Joan',
'last_name' => 'Jordan',
'email' => 'jjordan1@example.com',
'country' => 'Thailand',
'ip_address' => '137.230.209.171',
);
$csv_file_object->rewind();
$current = $csv_file_object->current();
$this->assertArrayEquals($row, $current);
$csv_file_object->next();
$next = $csv_file_object->current();
$this->assertArrayEquals($second_row, $next);
$column_names = array(
'column_names' => array(
0 => array('id' => 'identifier'),
2 => array('last_name' => 'User last name'),
),
);
$csv = new CSV($configuration + $column_names, $this->pluginId, $this->pluginDefinition, $this->plugin);
$csv_file_object = $csv->getIterator();
$row = array(
'id' => '1',
'last_name' => 'Dean',
);
$second_row = array(
'id' => '2',
'last_name' => 'Jordan',
);
$csv_file_object->rewind();
$current = $csv_file_object->current();
$this->assertArrayEquals($row, $current);
$csv_file_object->next();
$next = $csv_file_object->current();
$this->assertArrayEquals($second_row, $next);
}
/**
* Tests that the identifier or key is properly identified.
*
* @test
*
* @covers ::getIds
*/
public function getIds() {
$configuration = array(
'path' => $this->happyPath,
'identifiers' => array('id'),
'header_row_count' => 1,
);
$csv = new CSV($configuration, $this->pluginId, $this->pluginDefinition, $this->plugin);
$expected = array('id' => array('type' => 'string'));
$this->assertArrayEquals($expected, $csv->getIds());
}
/**
* Tests that fields have a machine name and description.
*
* @test
*
* @covers ::fields
*/
public function fields() {
$configuration = array(
'path' => $this->happyPath,
'identifiers' => array('id'),
'header_row_count' => 1,
);
$fields = array(
'id' => 'identifier',
'first_name' => 'User first name',
);
$expected = $fields + array(
'last_name' => 'last_name',
'email' => 'email',
'country' => 'country',
'ip_address' => 'ip_address',
);
$csv = new CSV($configuration, $this->pluginId, $this->pluginDefinition, $this->plugin);
$csv = new CSV($configuration + array('fields' => $fields), $this->pluginId, $this->pluginDefinition, $this->plugin);
$this->assertArrayEquals($expected, $csv->fields());
$column_names = array(
0 => array('id' => 'identifier'),
2 => array('first_name' => 'User first name'),
);
$csv = new CSV($configuration + array('fields' => $fields, 'column_names' => $column_names), $this->pluginId, $this->pluginDefinition, $this->plugin);
$this->assertArrayEquals($fields, $csv->fields());
}
}
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment