Commit 1787b905 authored by webchick's avatar webchick

Issue #2541580 by mikeryan: Remove obsoleted functionality from core Migrate

parent 7bc3890c
......@@ -31,20 +31,6 @@ class MigrateExecutable implements MigrateExecutableInterface {
*/
protected $migration;
/**
* The number of successfully imported rows since feedback was given.
*
* @var int
*/
protected $successesSinceFeedback;
/**
* The number of rows that were successfully processed.
*
* @var int
*/
protected $totalSuccesses;
/**
* Status of one row.
*
......@@ -55,15 +41,6 @@ class MigrateExecutable implements MigrateExecutableInterface {
*/
protected $sourceRowStatus;
/**
* The number of rows processed.
*
* The total attempted, whether or not they were successful.
*
* @var int
*/
protected $totalProcessed;
/**
* The queued messages not yet saved.
*
......@@ -75,44 +52,6 @@ class MigrateExecutable implements MigrateExecutableInterface {
*/
protected $queuedMessages = array();
/**
* The options that can be set when executing the migration.
*
* Values can be set for:
* - 'limit': Sets a time limit.
*
* @var array
*/
protected $options;
/**
* The PHP max_execution_time.
*
* @var int
*/
protected $maxExecTime;
/**
* The ratio of the memory limit at which an operation will be interrupted.
*
* @var float
*/
protected $memoryThreshold = 0.85;
/**
* The ratio of the time limit at which an operation will be interrupted.
*
* @var float
*/
public $timeThreshold = 0.90;
/**
* The time limit when executing the migration.
*
* @var array
*/
public $limit = array();
/**
* The configuration values of the source.
*
......@@ -120,20 +59,6 @@ class MigrateExecutable implements MigrateExecutableInterface {
*/
protected $sourceIdValues;
/**
* The number of rows processed since feedback was given.
*
* @var int
*/
protected $processedSinceFeedback = 0;
/**
* The PHP memory_limit expressed in bytes.
*
* @var int
*/
protected $memoryLimit;
/**
* The rollback action to be saved for the current row.
*
......@@ -202,31 +127,6 @@ public function __construct(MigrationInterface $migration, MigrateMessageInterfa
$this->message = $message;
$this->migration->getIdMap()->setMessage($message);
$this->eventDispatcher = $event_dispatcher;
// Record the memory limit in bytes
$limit = trim(ini_get('memory_limit'));
if ($limit == '-1') {
$this->memoryLimit = PHP_INT_MAX;
}
else {
if (!is_numeric($limit)) {
$last = strtolower(substr($limit, -1));
switch ($last) {
case 'g':
$limit *= 1024;
case 'm':
$limit *= 1024;
case 'k':
$limit *= 1024;
break;
default:
throw new MigrateException($this->t('Invalid PHP memory_limit !limit',
array('!limit' => $limit)));
}
}
$this->memoryLimit = $limit;
}
// Record the maximum execution time limit.
$this->maxExecTime = ini_get('max_execution_time');
}
/**
......@@ -321,8 +221,6 @@ public function import() {
if ($destination_id_values !== TRUE) {
$id_map->saveIdMapping($row, $destination_id_values, $this->sourceRowStatus, $this->rollbackAction);
}
$this->successesSinceFeedback++;
$this->totalSuccesses++;
}
else {
$id_map->saveIdMapping($row, array(), MigrateIdMapInterface::STATUS_FAILED, $this->rollbackAction);
......@@ -343,8 +241,6 @@ public function import() {
$this->handleException($e);
}
}
$this->totalProcessed++;
$this->processedSinceFeedback++;
if ($high_water_property = $this->migration->get('highWaterProperty')) {
$this->migration->saveHighWater($row->getSourceProperty($high_water_property['name']));
}
......@@ -353,12 +249,6 @@ public function import() {
unset($sourceValues, $destinationValues);
$this->sourceRowStatus = MigrateIdMapInterface::STATUS_IMPORTED;
if (($return = $this->checkStatus()) != MigrationInterface::RESULT_COMPLETED) {
break;
}
if ($this->timeOptionExceeded()) {
break;
}
try {
$source->next();
}
......@@ -370,11 +260,6 @@ public function import() {
}
}
/**
* @TODO uncomment this
*/
#$this->progressMessage($return);
$this->migration->setMigrationResult($return);
$this->getEventDispatcher()->dispatch(MigrateEvents::POST_IMPORT, new MigrateImportEvent($this->migration));
return $return;
......@@ -441,40 +326,6 @@ protected function currentSourceIds() {
return $this->getSource()->getCurrentIds();
}
/**
* Tests whether we've exceeded the designated time limit.
*
* @return bool
* TRUE if the threshold is exceeded, FALSE if not.
*/
protected function timeOptionExceeded() {
// If there is no time limit, then it is not exceeded.
if (!$time_limit = $this->getTimeLimit()) {
return FALSE;
}
// Calculate if the time limit is exceeded.
$time_elapsed = $this->getTimeElapsed();
if ($time_elapsed >= $time_limit) {
return TRUE;
}
else {
return FALSE;
}
}
/**
* {@inheritdoc}
*/
public function getTimeLimit() {
$limit = $this->limit;
if (isset($limit['unit']) && isset($limit['value']) && ($limit['unit'] == 'seconds' || $limit['unit'] == 'second')) {
return $limit['value'];
}
else {
return NULL;
}
}
/**
* {@inheritdoc}
*/
......@@ -499,143 +350,6 @@ public function saveQueuedMessages() {
$this->queuedMessages = array();
}
/**
* Checks for exceptional conditions, and display feedback.
*
* Standard top-of-loop stuff, common between rollback and import.
*/
protected function checkStatus() {
if ($this->memoryExceeded()) {
return MigrationInterface::RESULT_INCOMPLETE;
}
if ($this->maxExecTimeExceeded()) {
return MigrationInterface::RESULT_INCOMPLETE;
}
/*
* @TODO uncomment this
if ($this->getStatus() == MigrationInterface::STATUS_STOPPING) {
return MigrationBase::RESULT_STOPPED;
}
*/
// If feedback is requested, produce a progress message at the proper time
/*
* @TODO uncomment this
if (isset($this->feedback)) {
if (($this->feedback_unit == 'seconds' && time() - $this->lastfeedback >= $this->feedback) ||
($this->feedback_unit == 'items' && $this->processed_since_feedback >= $this->feedback)) {
$this->progressMessage(MigrationInterface::RESULT_INCOMPLETE);
}
}
*/
return MigrationInterface::RESULT_COMPLETED;
}
/**
* Tests whether we've exceeded the desired memory threshold.
*
* If so, output a message.
*
* @return bool
* TRUE if the threshold is exceeded, otherwise FALSE.
*/
protected function memoryExceeded() {
$usage = $this->getMemoryUsage();
$pct_memory = $usage / $this->memoryLimit;
if (!$threshold = $this->memoryThreshold) {
return FALSE;
}
if ($pct_memory > $threshold) {
$this->message->display(
$this->t('Memory usage is !usage (!pct% of limit !limit), reclaiming memory.',
array('!pct' => round($pct_memory*100),
'!usage' => $this->formatSize($usage),
'!limit' => $this->formatSize($this->memoryLimit))),
'warning');
$usage = $this->attemptMemoryReclaim();
$pct_memory = $usage / $this->memoryLimit;
// Use a lower threshold - we don't want to be in a situation where we keep
// coming back here and trimming a tiny amount
if ($pct_memory > (0.90 * $threshold)) {
$this->message->display(
$this->t('Memory usage is now !usage (!pct% of limit !limit), not enough reclaimed, starting new batch',
array('!pct' => round($pct_memory*100),
'!usage' => $this->formatSize($usage),
'!limit' => $this->formatSize($this->memoryLimit))),
'warning');
return TRUE;
}
else {
$this->message->display(
$this->t('Memory usage is now !usage (!pct% of limit !limit), reclaimed enough, continuing',
array('!pct' => round($pct_memory*100),
'!usage' => $this->formatSize($usage),
'!limit' => $this->formatSize($this->memoryLimit))),
'warning');
return FALSE;
}
}
else {
return FALSE;
}
}
/**
* Returns the memory usage so far.
*
* @return int
* The memory usage.
*/
protected function getMemoryUsage() {
return memory_get_usage();
}
/**
* Tries to reclaim memory.
*
* @return int
* The memory usage after reclaim.
*/
protected function attemptMemoryReclaim() {
// First, try resetting Drupal's static storage - this frequently releases
// plenty of memory to continue.
drupal_static_reset();
// @TODO: explore resetting the container.
return memory_get_usage();
}
/**
* Generates a string representation for the given byte count.
*
* @param int $size
* A size in bytes.
*
* @return string
* A translated string representation of the size.
*/
protected function formatSize($size) {
return format_size($size);
}
/**
* Tests whether we're approaching the PHP maximum execution time limit.
*
* @return bool
* TRUE if the threshold is exceeded, FALSE if not.
*/
protected function maxExecTimeExceeded() {
return $this->maxExecTime && (($this->getTimeElapsed() / $this->maxExecTime) > $this->timeThreshold);
}
/**
* Returns the time elapsed.
*
* This allows a test to set a fake elapsed time.
*/
protected function getTimeElapsed() {
return time() - REQUEST_TIME;
}
/**
* Takes an Exception object and both saves and displays it.
*
......
......@@ -36,14 +36,6 @@ public function import();
*/
public function processRow(Row $row, array $process = NULL, $value = NULL);
/**
* Returns the time limit.
*
* @return null|int
* The time limit, NULL if no limit or if the units were not in seconds.
*/
public function getTimeLimit();
/**
* Passes messages through to the map class.
*
......
......@@ -55,33 +55,6 @@ public function getIds();
*/
public function fields(MigrationInterface $migration = NULL);
/**
* Allows pre-processing of an import.
*
* Derived classes may implement preImport() to do any processing they need
* done before over all source rows.
*/
public function preImport();
/**
* Allows pre-processing of a rollback.
*/
public function preRollback();
/**
* Allows post-processing of an import.
*
* Derived classes may implement postImport(), to do any processing they need
* done after looping over all source rows.
*/
public function postImport();
/**
* Allows post-processing of a rollback.
*/
public function postRollback();
/**
* Import the row.
*
......
......@@ -59,34 +59,6 @@ public function checkRequirements() {
}
}
/**
* Modify the Row before it is imported.
*/
public function preImport() {
// By default we do nothing.
}
/**
* Modify the Row before it is rolled back.
*/
public function preRollback() {
// By default we do nothing.
}
/**
* {@inheritdoc}
*/
public function postImport() {
// By default we do nothing.
}
/**
* {@inheritdoc}
*/
public function postRollback() {
// By default we do nothing.
}
/**
* {@inheritdoc}
*/
......@@ -94,25 +66,4 @@ public function rollbackMultiple(array $destination_identifiers) {
// By default we do nothing.
}
/**
* {@inheritdoc}
*/
public function getCreated() {
// TODO: Implement getCreated() method.
}
/**
* {@inheritdoc}
*/
public function getUpdated() {
// TODO: Implement getUpdated() method.
}
/**
* {@inheritdoc}
*/
public function resetStats() {
// TODO: Implement resetStats() method.
}
}
......@@ -59,20 +59,6 @@ abstract class SourcePluginBase extends PluginBase implements MigrateSourceInter
*/
protected $currentSourceIds;
/**
* Number of rows intentionally ignored (prepareRow() returned FALSE)
*
* @var int
*/
protected $numIgnored = 0;
/**
* Number of rows we've at least looked at.
*
* @var int
*/
protected $numProcessed = 0;
/**
* The high water mark at the beginning of the import operation.
*
......@@ -214,7 +200,6 @@ public function prepareRow(Row $row) {
$id_map->delete($this->currentSourceIds, TRUE);
$this->migrateExecutable->saveQueuedMessages();
$id_map->saveIdMapping($row, array(), MigrateIdMapInterface::STATUS_IGNORED, $this->migrateExecutable->rollbackAction);
$this->numIgnored++;
$this->currentRow = NULL;
$this->currentSourceIds = NULL;
$result = FALSE;
......@@ -226,7 +211,6 @@ public function prepareRow(Row $row) {
// after hashes).
$row->rehash();
}
$this->numProcessed++;
return $result;
}
......@@ -280,8 +264,6 @@ public function valid() {
*/
public function rewind() {
$this->idMap = $this->migration->getIdMap();
$this->numProcessed = 0;
$this->numIgnored = 0;
$this->getIterator()->rewind();
$this->next();
}
......@@ -377,27 +359,6 @@ public function getCurrentIds() {
return $this->currentSourceIds;
}
/**
* Getter for numIgnored data member.
*/
public function getIgnored() {
return $this->numIgnored;
}
/**
* Getter for numProcessed data member.
*/
public function getProcessed() {
return $this->numProcessed;
}
/**
* Reset numIgnored back to 0.
*/
public function resetStats() {
$this->numIgnored = 0;
}
/**
* Get the source count.
*
......
......@@ -28,11 +28,6 @@ abstract class SqlBase extends SourcePluginBase {
*/
protected $query;
/**
* @var \Drupal\migrate\Entity\MigrationInterface
*/
protected $migration;
/**
* @var \Drupal\Core\Database\Connection
*/
......
<?php
/**
* @file
* Contains \Drupal\Tests\migrate\Unit\MigrateExecutableMemoryExceededTest.
*/
namespace Drupal\Tests\migrate\Unit;
/**
* Tests the \Drupal\migrate\MigrateExecutable::memoryExceeded() method.
*
* @group migrate
*/
class MigrateExecutableMemoryExceededTest extends MigrateTestCase {
/**
* The mocked migration entity.
*
* @var \Drupal\migrate\Entity\MigrationInterface|\PHPUnit_Framework_MockObject_MockObject
*/
protected $migration;
/**
* The mocked migrate message.
*
* @var \Drupal\migrate\MigrateMessageInterface|\PHPUnit_Framework_MockObject_MockObject
*/
protected $message;
/**
* The tested migrate executable.
*
* @var \Drupal\Tests\migrate\Unit\TestMigrateExecutable
*/
protected $executable;
/**
* The migration configuration, initialized to set the ID to test.
*
* @var array
*/
protected $migrationConfiguration = array(
'id' => 'test',
);
/**
* php.init memory_limit value.
*/
protected $memoryLimit = 10000000;
/**
* {@inheritdoc}
*/
protected function setUp() {
parent::setUp();
$this->migration = $this->getMigration();
$this->message = $this->getMock('Drupal\migrate\MigrateMessageInterface');
$this->executable = new TestMigrateExecutable($this->migration, $this->message);
$this->executable->setStringTranslation($this->getStringTranslationStub());
}
/**
* Runs the actual test.
*
* @param string $message
* The second message to assert.
* @param bool $memory_exceeded
* Whether to test the memory exceeded case.
* @param int $memory_usage_first
* (optional) The first memory usage value.
* @param int $memory_usage_second
* (optional) The fake amount of memory usage reported after memory reclaim.
* @param int $memory_limit
* (optional) The memory limit.
*/
protected function runMemoryExceededTest($message, $memory_exceeded, $memory_usage_first = NULL, $memory_usage_second = NULL, $memory_limit = NULL) {
$this->executable->setMemoryLimit($memory_limit ?: $this->memoryLimit);
$this->executable->setMemoryUsage($memory_usage_first ?: $this->memoryLimit, $memory_usage_second ?: $this->memoryLimit);
$this->executable->setMemoryThreshold(0.85);
if ($message) {
$this->executable->message->expects($this->at(0))
->method('display')
->with($this->stringContains('reclaiming memory'));
$this->executable->message->expects($this->at(1))
->method('display')
->with($this->stringContains($message));
}
else {
$this->executable->message->expects($this->never())
->method($this->anything());
}
$result = $this->executable->memoryExceeded();
$this->assertEquals($memory_exceeded, $result);
}
/**
* Tests memoryExceeded method when a new batch is needed.
*/
public function testMemoryExceededNewBatch() {
// First case try reset and then start new batch.
$this->runMemoryExceededTest('starting new batch', TRUE);
}
/**
* Tests memoryExceeded method when enough is cleared.
*/
public function testMemoryExceededClearedEnough() {
$this->runMemoryExceededTest('reclaimed enough', FALSE, $this->memoryLimit, $this->memoryLimit * 0.75);
}
/**
* Tests memoryExceeded when memory usage is not exceeded.
*/
public function testMemoryNotExceeded() {
$this->runMemoryExceededTest('', FALSE, floor($this->memoryLimit * 0.85) - 1);
}
}
......@@ -53,8 +53,6 @@ protected function setUp() {
$event_dispatcher = $this->getMock('Symfony\Component\EventDispatcher\EventDispatcherInterface');
$this->executable = new TestMigrateExecutable($this->migration, $this->message, $event_dispatcher);
$this->executable->setStringTranslation($this->getStringTranslationStub());
$this->executable->setTimeThreshold(0.1);
$this->executable->limit = array('unit' => 'second', 'value' => 1);
}
/**
......@@ -120,11 +118,6 @@ public function testImportWithValidRow() {
->will($this->returnValue($destination));
$this->assertSame(MigrationInterface::RESULT_COMPLETED, $this->executable->import());
$this->assertSame(1, $this->executable->getSuccessesSinceFeedback());
$this->assertSame(1, $this->executable->getTotalSuccesses());
$this->assertSame(1, $this->executable->getTotalProcessed());
$this->assertSame(1, $this->executable->getProcessedSinceFeedback());
}
/**
......@@ -170,11 +163,6 @@ public function testImportWithValidRowWithoutDestinationId() {
->method('saveIdMapping');
$this->assertSame(MigrationInterface::RESULT_COMPLETED, $this->executable->import());
$this->assertSame(1, $this->executable->getSuccessesSinceFeedback());
$this->assertSame(1, $this->executable->getTotalSuccesses());
$this->assertSame(1, $this->executable->getTotalProcessed());
$this->assertSame(1, $this->executable->getProcessedSinceFeedback());
}
/**
......@@ -346,44 +334,6 @@ public function testImportWithValidRowWithException() {
$this->assertSame(MigrationInterface::RESULT_COMPLETED, $this->executable->import());
}
/**
* Tests time limit option method.
*/
public function testTimeOptionExceeded() {
// Assert time limit of one second (test configuration default) is exceeded.
$this->executable->setTimeElapsed(1);
$this->assertTrue($this->executable->timeOptionExceeded());
// Assert time limit not exceeded.
$this->executable->limit = array('unit' => 'seconds', 'value' => (int) $_SERVER['REQUEST_TIME'] - 3600);
$this->assertFalse($this->executable->timeOptionExceeded());
// Assert no time limit.
$this->executable->limit = array();
$this->assertFalse($this->executable->timeOptionExceeded());