Commit 0d61dd81 authored by alexpott's avatar alexpott

Issue #2143415 by YesCT, mikeryan, marvil07, bdone, chx: Migrate fixes

parent 958056da
......@@ -167,12 +167,12 @@ class Migration extends ConfigEntityBase implements MigrationInterface {
public $systemOfRecord = self::SOURCE;
/**
* Specify value of needs_update for current map row. Usually set by
* Specify value of source_row_status for current map row. Usually set by
* MigrateFieldHandler implementations.
*
* @var int
*/
public $needsUpdate = MigrateIdMapInterface::STATUS_IMPORTED;
public $sourceRowStatus = MigrateIdMapInterface::STATUS_IMPORTED;
/**
* @var \Drupal\Core\KeyValueStore\KeyValueStoreInterface
......
......@@ -16,7 +16,7 @@
class MigrateExecutable {
/**
* The migration to do.
* The configuration of the migration to do.
*
* @var \Drupal\migrate\Entity\MigrationInterface
*/
......@@ -44,7 +44,7 @@ class MigrateExecutable {
*
* @var int
*/
protected $needsUpdate;
protected $sourceRowStatus;
/**
* The number of rows processed.
......@@ -78,6 +78,7 @@ class MigrateExecutable {
/**
* The fraction of the memory limit at which an operation will be interrupted.
*
* Can be overridden by a Migration subclass if one would like to push the
* envelope. Defaults to 85%.
*
......@@ -94,6 +95,7 @@ class MigrateExecutable {
/**
* The fraction of the time limit at which an operation will be interrupted.
*
* Can be overridden by a Migration subclass if one would like to push the
* envelope. Defaults to 90%.
*
......@@ -109,11 +111,15 @@ class MigrateExecutable {
protected $timeLimit;
/**
* The configuration values of the source.
*
* @var array
*/
protected $sourceIdValues;
/**
* The number of rows processed since feedback was given.
*
* @var int
*/
protected $processed_since_feedback = 0;
......@@ -126,8 +132,12 @@ class MigrateExecutable {
protected $translationManager;
/**
* @param MigrationInterface $migration
* @param MigrateMessageInterface $message
* Constructs a MigrateExecutable and verifies and sets the memory limit.
*
* @param \Drupal\migrate\Entity\MigrationInterface $migration
* The migration to run.
* @param \Drupal\migrate\MigrateMessageInterface $message
* The message to record.
*
* @throws \Drupal\migrate\MigrateException
*/
......@@ -161,7 +171,12 @@ public function __construct(MigrationInterface $migration, MigrateMessageInterfa
}
/**
* Returns the source.
*
* Makes sure source is initialized based on migration settings.
*
* @return \Drupal\migrate\Source
* The source.
*/
public function getSource() {
if (!isset($this->source)) {
......@@ -236,7 +251,7 @@ public function import() {
$destination_id_values = $destination->import($row);
// @TODO handle the successful but no ID case like config.
if ($destination_id_values) {
$id_map->saveIdMapping($row, $destination_id_values, $this->needsUpdate, $this->rollbackAction);
$id_map->saveIdMapping($row, $destination_id_values, $this->sourceRowStatus, $this->rollbackAction);
$this->successes_since_feedback++;
$this->total_successes++;
}
......@@ -266,7 +281,7 @@ public function import() {
// Reset row properties.
unset($sourceValues, $destinationValues);
$this->needsUpdate = MigrateIdMapInterface::STATUS_IMPORTED;
$this->sourceRowStatus = MigrateIdMapInterface::STATUS_IMPORTED;
// TODO: Temporary. Remove when http://drupal.org/node/375494 is committed.
// TODO: Should be done in MigrateDestinationEntity
......@@ -353,9 +368,9 @@ protected function timeOptionExceeded() {
}
public function getTimeLimit() {
if (isset($this->options['limit']) &&
($this->options['limit']['unit'] == 'seconds' || $this->options['limit']['unit'] == 'second')) {
return $this->options['limit']['value'];
$limit = $this->migration->get('limit');
if (isset($limit['unit']) && isset($limit['value']) && ($limit['unit'] == 'seconds' || $limit['unit'] == 'second')) {
return $limit['value'];
}
else {
return NULL;
......
......@@ -221,7 +221,7 @@ public function destroy();
*
* @todo Remove this as this is SQL only and so doesn't belong to the interface.
*/
public function getQualifiedMapTable();
public function getQualifiedMapTableName();
/**
* Sets the migrate message.
......
......@@ -28,6 +28,9 @@ public function fields();
* Returns the iterator that will yield the row arrays to be processed.
*
* @return \Iterator
*
* @throws \Exception
* Cannot obtain a valid iterator.
*/
public function getIterator();
......
......@@ -127,9 +127,9 @@ protected function runQuery() {
$delimiter = ' AND ';
}
$alias = $this->query->leftJoin($this->migration->getIdMap()->getQualifiedMapTable(), 'map', $map_join);
$alias = $this->query->leftJoin($this->migration->getIdMap()->getQualifiedMapTableName(), 'map', $map_join);
$conditions->isNull($alias . '.sourceid1');
$conditions->condition($alias . '.needs_update', MigrateIdMapInterface::STATUS_NEEDS_UPDATE);
$conditions->condition($alias . '.source_row_status', MigrateIdMapInterface::STATUS_NEEDS_UPDATE);
$condition_added = TRUE;
// And as long as we have the map table, add its data to the row.
......@@ -143,7 +143,7 @@ protected function runQuery() {
$map_key = 'destid' . $count++;
$this->query->addField($alias, $map_key, "migrate_map_$map_key");
}
$this->query->addField($alias, 'needs_update', 'migrate_map_needs_update');
$this->query->addField($alias, 'source_row_status', 'migrate_map_source_row_status');
}
// 3. If we are using highwater marks, also include rows above the mark.
// But, include all rows if the highwater mark is not set.
......
......@@ -44,7 +44,7 @@ class Row {
protected $idMap = array(
'original_hash' => '',
'hash' => '',
'needs_update' => MigrateIdMapInterface::STATUS_NEEDS_UPDATE,
'source_row_status' => MigrateIdMapInterface::STATUS_NEEDS_UPDATE,
);
/**
......@@ -248,7 +248,7 @@ public function changed() {
* TRUE if the row needs updating, FALSE otherwise.
*/
public function needsUpdate() {
return $this->idMap['needs_update'] == MigrateIdMapInterface::STATUS_NEEDS_UPDATE;
return $this->idMap['source_row_status'] == MigrateIdMapInterface::STATUS_NEEDS_UPDATE;
}
/**
......
......@@ -19,7 +19,7 @@ class MigrateTestBase extends WebTestBase {
*
* @var array
*/
var $databaseDumpFiles = array();
public $databaseDumpFiles = array();
public static $modules = array('migrate');
......
......@@ -14,11 +14,14 @@ class FakeDatabaseSchema extends Schema {
/**
* As set on MigrateSqlSourceTestCase::databaseContents.
*
* @var array
*/
protected $databaseContents;
public $databaseContents;
public function __construct($database_contents) {
$this->uniqueIdentifier = uniqid('', TRUE);
// @todo Maybe we can generate an internal representation.
$this->databaseContents = $database_contents;
}
......@@ -84,7 +87,7 @@ public function dropPrimaryKey($table) {
}
public function dropTable($table) {
throw new \Exception(sprintf('Unsupported method "%s"', __METHOD__));
unset($this->databaseContents[$table]);
}
public function dropUniqueKey($table, $name) {
......@@ -92,7 +95,13 @@ public function dropUniqueKey($table, $name) {
}
public function fieldExists($table, $column) {
throw new \Exception(sprintf('Unsupported method "%s"', __METHOD__));
if (!empty($this->databaseContents[$table])) {
$row = reset($this->databaseContents[$table]);
return isset($row[$column]);
}
else {
throw new \Exception("Can't determine whether field exists with an empty / nonexistent table.");
}
}
public function fieldNames($fields) {
......@@ -132,7 +141,7 @@ public function renameTable($table, $new_name) {
}
public function uniqueIdentifier() {
throw new \Exception(sprintf('Unsupported method "%s"', __METHOD__));
return $this->uniqueIdentifier;
}
/**
......
......@@ -53,10 +53,8 @@ class FakeSelect extends Select {
*
* @param string $table
* The base table name used within fake select.
*
* @param string $alias
* The base table alias used within fake select.
*
* @param array $database_contents
* An array of mocked database content.
*
......@@ -169,7 +167,6 @@ public function execute() {
* of JOINs.
*/
protected function executeJoins() {
// @TODO add support for all_fields.
$fields = array();
foreach ($this->fields as $field_info) {
$this->fieldsWithTable[$field_info['table'] . '.' . $field_info['field']] = $field_info;
......@@ -183,6 +180,10 @@ protected function executeJoins() {
'table' => $alias,
'field' => $field,
);
$this->fieldsWithTable["$alias.$field"] = array(
'table' => $alias,
'field' => $field,
);
}
}
}
......@@ -508,4 +509,19 @@ public static function getInfo() {
);
}
/**
* {@inheritdoc}
*/
public function fields($table_alias, array $fields = array()) {
if (!$fields) {
$table = $this->tables[$table_alias]['table'];
if (!empty($this->databaseContents[$table])) {
$fields = array_keys(reset($this->databaseContents[$table]));
}
else {
throw new \Exception('All fields on empty table is not supported.');
}
}
return parent::fields($table_alias, $fields);
}
}
......@@ -42,6 +42,13 @@ class MigrateExecutableTest extends MigrateTestCase {
*/
protected $executable;
protected $mapJoinable = FALSE;
protected $migrationConfiguration = array(
'id' => 'test',
'limit' => array('units' => 'seconds', 'value' => 5),
);
/**
* {@inheritdoc}
*/
......@@ -54,7 +61,7 @@ public static function getInfo() {
}
protected function setUp() {
$this->migration = $this->getMock('Drupal\migrate\Entity\MigrationInterface');
$this->migration = $this->getMigration();
$this->message = $this->getMock('Drupal\migrate\MigrateMessageInterface');
$id_map = $this->getMock('Drupal\migrate\Plugin\MigrateIdMapInterface');
......@@ -71,10 +78,11 @@ protected function setUp() {
*/
public function testImportWithFailingRewind() {
$iterator = $this->getMock('\Iterator');
$exception_message = $this->getRandomGenerator()->string();
$iterator->expects($this->once())
->method('valid')
->will($this->returnCallback(function() {
throw new \Exception('invalid source iteration');
->will($this->returnCallback(function() use ($exception_message) {
throw new \Exception($exception_message);
}));
$source = $this->getMock('Drupal\migrate\Plugin\MigrateSourceInterface');
$source->expects($this->any())
......@@ -88,7 +96,7 @@ public function testImportWithFailingRewind() {
// Ensure that a message with the proper message was added.
$this->message->expects($this->once())
->method('display')
->with('Migration failed with source plugin exception: invalid source iteration');
->with("Migration failed with source plugin exception: $exception_message");
$result = $this->executable->import();
$this->assertEquals(MigrationInterface::RESULT_FAILED, $result);
......
......@@ -33,7 +33,7 @@ protected function getMigration() {
$idmap = $this->getMock('Drupal\migrate\Plugin\MigrateIdMapInterface');
if ($this->mapJoinable) {
$idmap->expects($this->once())
->method('getQualifiedMapTable')
->method('getQualifiedMapTableName')
->will($this->returnValue('test_map'));
}
......@@ -59,8 +59,12 @@ protected function getDatabase($database_contents) {
->disableOriginalConstructor()
->getMock();
$database->databaseContents = &$database_contents;
// Although select doesn't modify the contents of the database, it still
// needs to be a reference so that we can select previously inserted or
// updated rows.
$database->expects($this->any())
->method('select')->will($this->returnCallback(function ($base_table, $base_alias) use ($database_contents) {
->method('select')->will($this->returnCallback(function ($base_table, $base_alias) use (&$database_contents) {
return new FakeSelect($base_table, $base_alias, $database_contents);
}));
$database->expects($this->any())
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment