Commit 3f369f17 authored by catch's avatar catch

Issue #2160345 by chx, YesCT, andypost: Clean up and test migrate executable and sql idmap.

parent 4f66a53b
......@@ -174,6 +174,16 @@ class Migration extends ConfigEntityBase implements MigrationInterface {
*/
public $sourceRowStatus = MigrateIdMapInterface::STATUS_IMPORTED;
/**
* The ratio of the memory limit at which an operation will be interrupted.
*
* Can be overridden by a Migration subclass if one would like to push the
* envelope. Defaults to 0.85.
*
* @var float
*/
protected $memoryThreshold = 0.85;
/**
* @var \Drupal\Core\KeyValueStore\KeyValueStoreInterface
*/
......@@ -184,6 +194,23 @@ class Migration extends ConfigEntityBase implements MigrationInterface {
*/
public $trackLastImported = FALSE;
/**
* The ratio of the time limit at which an operation will be interrupted.
*
* Can be overridden by a Migration subclass if one would like to push the
* envelope. Defaults to 0.9.
*
* @var float
*/
public $timeThreshold = 0.90;
/**
* The time limit when executing the migration.
*
* @var array
*/
public $limit = array();
/**
* {@inheritdoc}
*/
......
......@@ -63,8 +63,9 @@ public function getSourcePlugin();
* @param array $process
* A process configuration array.
*
* @return array
* A list of process plugins.
* @return \Drupal\migrate\Plugin\MigrateProcessInterface[][]
* An associative array. The keys are the destination property names. Values
* are process pipelines. Each pipeline contains an array of plugins.
*/
public function getProcessPlugins(array $process = NULL);
......
<?php
/**
* @file
* Contains \Drupal\migrate\MigrateSkipRowException.
*/
namespace Drupal\migrate;
/**
* This exception is thrown when a row should be skipped.
*/
class MigrateSkipRowException extends \Exception {
}
......@@ -207,7 +207,7 @@ public function lookupSourceID(array $destination_id_values);
* @return array
* The destination identifier values of the record, or NULL on failure.
*/
public function lookupDestinationID(array $source_id_values);
public function lookupDestinationId(array $source_id_values);
/**
* Removes any persistent storage used by this map.
......
......@@ -381,13 +381,13 @@ public function lookupSourceID(array $destination_id) {
}
$result = $query->execute();
$source_id = $result->fetchAssoc();
return array_values($source_id);
return array_values($source_id ?: array());
}
/**
* {@inheritdoc}
*/
public function lookupDestinationID(array $source_id) {
public function lookupDestinationId(array $source_id) {
$query = $this->getDatabase()->select($this->mapTableName, 'map')
->fields('map', $this->destinationIdFields);
foreach ($this->sourceIdFields as $key_name) {
......@@ -395,7 +395,7 @@ public function lookupDestinationID(array $source_id) {
}
$result = $query->execute();
$destination_id = $result->fetchAssoc();
return array_values($destination_id);
return array_values($destination_id ?: array());
}
/**
......@@ -444,9 +444,8 @@ public function saveMessage(array $source_id_values, $message, $level = Migratio
$count = 1;
foreach ($source_id_values as $id_value) {
$fields['sourceid' . $count++] = $id_value;
// If any key value is empty, we can't save - print out and abort.
if (empty($id_value)) {
print($message);
// If any key value is not set, we can't save.
if (!isset($id_value)) {
return;
}
}
......@@ -579,7 +578,7 @@ public function deleteDestination(array $destination_id) {
* {@inheritdoc}
*/
public function setUpdate(array $source_id) {
if (empty($source_ids)) {
if (empty($source_id)) {
throw new MigrateException('No source identifiers provided to update.');
}
$query = $this->getDatabase()
......@@ -695,16 +694,13 @@ public function key() {
* from rewind().
*/
public function next() {
$this->currentRow = $this->result->fetchObject();
$this->currentRow = $this->result->fetchAssoc();
$this->currentKey = array();
if (!is_object($this->currentRow)) {
$this->currentRow = NULL;
}
else {
if ($this->currentRow) {
foreach ($this->sourceIdFields as $map_field) {
$this->currentKey[$map_field] = $this->currentRow->$map_field;
$this->currentKey[$map_field] = $this->currentRow[$map_field];
// Leave only destination fields.
unset($this->currentRow->$map_field);
unset($this->currentRow[$map_field]);
}
}
}
......@@ -717,7 +713,7 @@ public function next() {
*/
public function valid() {
// @todo Check numProcessed against itemlimit.
return !is_null($this->currentRow);
return $this->currentRow !== FALSE;
}
}
<?php
/**
* @file
* Contains \Drupal\migrate\Tests\FakeConnection.
*/
namespace Drupal\migrate\Tests;
use Drupal\Core\Database\Connection;
/**
* Defines a fake connection for use during testing.
*/
class FakeConnection extends Connection {
/**
* Table prefix on the database.
*
* @var string
*/
protected $tablePrefix;
/**
* Connection options for the database.
*
* @var array
*/
protected $connectionOptions;
/**
* Constructs a FakeConnection.
*
* @param array $database_contents
* The database contents faked as an array. Each key is a table name, each
* value is a list of table rows.
* @param array $connection_options
* (optional) The array of connection options for the database.
* @param string $prefix
* (optional) The table prefix on the database.
*/
public function __construct(array $database_contents, $connection_options = array(), $prefix = '') {
$this->databaseContents = $database_contents;
$this->connectionOptions = $connection_options;
$this->tablePrefix = $prefix;
}
/**
* {@inheritdoc}
*/
public function getConnectionOptions() {
return $this->connectionOptions;
}
/**
* {@inheritdoc}
*/
public function tablePrefix($table = 'default') {
return $this->tablePrefix;
}
/**
* {@inheritdoc}
*/
public function select($table, $alias = NULL, array $options = array()) {
return new FakeSelect($this->databaseContents, $table, $alias);
}
/**
* {@inheritdoc}
*/
public function insert($table, array $options = array()) {
return new FakeInsert($this->databaseContents, $table);
}
/**
* {@inheritdoc}
*/
public function merge($table, array $options = array()) {
return new FakeMerge($this->databaseContents, $table);
}
/**
* {@inheritdoc}
*/
public function update($table, array $options = array()) {
return new FakeUpdate($this->databaseContents, $table);
}
/**
* {@inheritdoc}
*/
public function truncate($table, array $options = array()) {
return new FakeTruncate($this->databaseContents, $table);
}
/**
* {@inheritdoc}
*/
public function schema() {
return new FakeDatabaseSchema($this->databaseContents);
}
/**
* {@inheritdoc}
*/
public function query($query, array $args = array(), $options = array()) {
throw new \Exception('Method not supported');
}
/**
* {@inheritdoc}
*/
public function queryRange($query, $from, $count, array $args = array(), array $options = array()) {
throw new \Exception('Method not supported');
}
/**
* {@inheritdoc}
*/
public function queryTemporary($query, array $args = array(), array $options = array()) {
throw new \Exception('Method not supported');
}
/**
* {@inheritdoc}
*/
public function driver() {
throw new \Exception('Method not supported');
}
/**
* {@inheritdoc}
*/
public function databaseType() {
throw new \Exception('Method not supported');
}
/**
* {@inheritdoc}
*/
public function createDatabase($database) {
// There is nothing to do.
}
/**
* {@inheritdoc}
*/
public function mapConditionOperator($operator) {
throw new \Exception('Method not supported');
}
/**
* {@inheritdoc}
*/
public function nextId($existing_id = 0) {
throw new \Exception('Method not supported');
}
}
......@@ -2,10 +2,9 @@
/**
* @file
* Contains \Drupal\migrate\Tests\FakeSelect.
* Contains \Drupal\migrate\Tests\FakeDatabaseSchema.
*/
namespace Drupal\migrate\Tests;
use Drupal\Core\Database\Schema;
......@@ -19,11 +18,18 @@ class FakeDatabaseSchema extends Schema {
*/
public $databaseContents;
public function __construct($database_contents) {
/**
* Constructs a fake database schema.
*
* @param array $database_contents
* The database contents faked as an array. Each key is a table name, each
* value is a list of table rows.
*/
public function __construct(array &$database_contents) {
$this->uniqueIdentifier = uniqid('', TRUE);
// @todo Maybe we can generate an internal representation.
$this->databaseContents = $database_contents;
$this->databaseContents = &$database_contents;
}
public function tableExists($table) {
......
<?php
/**
* @file
* Contains \Drupal\migrate\Tests\FakeInsert.
*/
namespace Drupal\migrate\Tests;
use Drupal\Core\Database\Query\Insert;
use Drupal\Core\Database\Query\SelectInterface;
/**
* Defines FakeInsert for use in database tests.
*/
class FakeInsert extends Insert {
/**
* The database contents.
*
* @var array
*/
protected $databaseContents;
/**
* The database table to insert into.
*
* @var string
*/
protected $table;
/**
* Constructs a fake insert object.
*
* @param array $database_contents
* The database contents faked as an array. Each key is a table name, each
* value is a list of table rows.
* @param string $table
* The table to insert into.
* @param array $options
* (optional) The database options. Not used.
*/
public function __construct(array &$database_contents, $table, array $options = array()) {
$this->databaseContents = &$database_contents;
$this->table = $table;
}
/**
* {@inheritdoc}
*/
public function useDefaults(array $fields) {
throw new \Exception('This method is not supported');
}
/**
* {@inheritdoc}
*/
public function from(SelectInterface $query) {
throw new \Exception('This method is not supported');
}
/**
* {@inheritdoc}
*/
public function execute() {
foreach ($this->insertValues as $values) {
$this->databaseContents[$this->table][] = array_combine($this->insertFields, $values);
}
}
}
<?php
/**
* @file
* Contains \Drupal\migrate\Tests\FakeMerge.
*/
namespace Drupal\migrate\Tests;
use Drupal\Core\Database\Query\Condition;
use Drupal\Core\Database\Query\InvalidMergeQueryException;
use Drupal\Core\Database\Query\Merge;
/**
* Defines FakeMerge for use in database tests.
*/
class FakeMerge extends Merge {
/**
* Constructs a fake merge object and initializes the conditions.
*
* @param array $database_contents
* The database contents faked as an array. Each key is a table name, each
* value is a list of table rows.
* @param string $table
* The database table to merge into.
*/
function __construct(array &$database_contents, $table) {
$this->databaseContents = &$database_contents;
$this->table = $table;
$this->conditionTable = $table;
$this->condition = new Condition('AND');
}
/**
* {@inheritdoc}
*/
public function execute() {
if (!count($this->condition)) {
throw new InvalidMergeQueryException(t('Invalid merge query: no conditions'));
}
$select = new FakeSelect($this->databaseContents, $this->conditionTable, 'c');
$count = $select
->condition($this->condition)
->countQuery()
->execute()
->fetchField();
if ($count) {
$update = new FakeUpdate($this->databaseContents, $this->table);
$update
->fields($this->updateFields)
->condition($this->condition)
->execute();
return self::STATUS_UPDATE;
}
$insert = new FakeInsert($this->databaseContents, $this->table);
$insert->fields($this->insertFields)->execute();
return self::STATUS_INSERT;
}
}
......@@ -19,7 +19,7 @@ class FakeSelect extends Select {
* Contents of the pseudo-database.
*
* Keys are table names and values are arrays of rows in the table.
* Every row there contains all table fields keyed by field name.
* Every row there contains all table field values keyed by field name.
*
* @code
* array(
......@@ -51,21 +51,20 @@ class FakeSelect extends Select {
/**
* Constructs a new FakeSelect.
*
* @param array $database_contents
* An array of mocked database content.
* @param string $table
* The base table name used within fake select.
* @param string $alias
* The base table alias used within fake select.
* @param array $database_contents
* An array of mocked database content.
*
* @param string $conjunction
* The operator to use to combine conditions: 'AND' or 'OR'.
*/
public function __construct($table, $alias, array $database_contents, $conjunction = 'AND') {
public function __construct(array $database_contents, $table, $alias, $conjunction = 'AND') {
$this->databaseContents = $database_contents;
$this->addJoin(NULL, $table, $alias);
$this->where = new Condition($conjunction);
$this->having = new Condition($conjunction);
$this->databaseContents = $database_contents;
}
/**
......
......@@ -50,8 +50,11 @@ public function fetchField($index = 0) {
* {@inheritdoc}
*/
public function fetchAssoc() {
$return = $this->current();
$this->next();
$return = FALSE;
if ($this->valid()) {
$return = $this->current();
$this->next();
}
return $return;
}
......@@ -90,6 +93,14 @@ public function fetchAllAssoc($key, $fetch = NULL) {
return $return;
}
/**
* {@inheritdoc}
*/
public function fetchObject() {
$return = $this->fetchAssoc();
return $return === FALSE ? FALSE : (object) $return;
}
/**
* {@inheritdoc}
*/
......
<?php
/**
* @file
* Contains \Drupal\migrate\Tests\FakeTruncate.
*/
namespace Drupal\migrate\Tests;
/**
* Defines FakeTruncate for use in database tests.
*/
class FakeTruncate {
/**
* Constructs a FakeTruncate object.
*
* @param array $database_contents
* The database contents faked as an array. Each key is a table name, each
* value is a list of table rows.
* @param $table
* The table to truncate.
*/
public function __construct(array &$database_contents, $table) {
$this->databaseContents = &$database_contents;
$this->table = $table;
}
/**
* Executes the TRUNCATE query.
*/
public function execute() {
$this->databaseContents[$this->table] = array();
}
}
<?php
/**
* @file
* Contains \Drupal\migrate\Tests\FakeUpdate.
*/
namespace Drupal\migrate\Tests;
use Drupal\Core\Database\Query\Condition;
use Drupal\Core\Database\Query\SelectInterface;
use Drupal\Core\Database\Query\Update;
/**
* Defines FakeUpdate for use in database tests.
*/
class FakeUpdate extends Update {
/**
* The database table to update.
*
* @var string
*/
protected $table;
/**
* The database contents.
*
* @var array
*/
protected $databaseContents;
/**
* Constructs a FakeUpdate object and initializes the condition.
*
* @param array $database_contents