Commit 2066518e authored by webchick's avatar webchick

Issue #2545632 by neclimdul, hussainweb, mikeryan, phenaproxima, benjy: Revert...

Issue #2545632 by neclimdul, hussainweb, mikeryan, phenaproxima, benjy: Revert removal of memory reclamation in migrate system
parent 125844e9
......@@ -52,6 +52,20 @@ class MigrateExecutable implements MigrateExecutableInterface {
*/
protected $queuedMessages = array();
/**
* The ratio of the memory limit at which an operation will be interrupted.
*
* @var float
*/
protected $memoryThreshold = 0.85;
/**
* The PHP memory_limit expressed in bytes.
*
* @var int
*/
protected $memoryLimit;
/**
* The configuration values of the source.
*
......@@ -127,6 +141,30 @@ public function __construct(MigrationInterface $migration, MigrateMessageInterfa
$this->message = $message;
$this->migration->getIdMap()->setMessage($message);
$this->eventDispatcher = $event_dispatcher;
// Record the memory limit in bytes
$limit = trim(ini_get('memory_limit'));
if ($limit == '-1') {
$this->memoryLimit = PHP_INT_MAX;
}
else {
if (!is_numeric($limit)) {
$last = strtolower(substr($limit, -1));
switch ($last) {
case 'g':
$limit *= 1024;
case 'm':
$limit *= 1024;
case 'k':
$limit *= 1024;
break;
default:
$limit = PHP_INT_MAX;
$this->message->display($this->t('Invalid PHP memory_limit !limit, setting to unlimited.',
array('!limit' => $limit)));
}
}
$this->memoryLimit = $limit;
}
}
/**
......@@ -249,6 +287,10 @@ public function import() {
unset($sourceValues, $destinationValues);
$this->sourceRowStatus = MigrateIdMapInterface::STATUS_IMPORTED;
if (($return = $this->checkStatus()) != MigrationInterface::RESULT_COMPLETED) {
break;
}
try {
$source->next();
}
......@@ -370,4 +412,100 @@ protected function handleException(\Exception $exception, $save = TRUE) {
$this->message->display($message, 'error');
}
/**
* Checks for exceptional conditions, and display feedback.
*/
protected function checkStatus() {
if ($this->memoryExceeded()) {
return MigrationInterface::RESULT_INCOMPLETE;
}
return MigrationInterface::RESULT_COMPLETED;
}
/**
* Tests whether we've exceeded the desired memory threshold.
*
* If so, output a message.
*
* @return bool
* TRUE if the threshold is exceeded, otherwise FALSE.
*/
protected function memoryExceeded() {
$usage = $this->getMemoryUsage();
$pct_memory = $usage / $this->memoryLimit;
if (!$threshold = $this->memoryThreshold) {
return FALSE;
}
if ($pct_memory > $threshold) {
$this->message->display(
$this->t('Memory usage is !usage (!pct% of limit !limit), reclaiming memory.',
array('!pct' => round($pct_memory*100),
'!usage' => $this->formatSize($usage),
'!limit' => $this->formatSize($this->memoryLimit))),
'warning');
$usage = $this->attemptMemoryReclaim();
$pct_memory = $usage / $this->memoryLimit;
// Use a lower threshold - we don't want to be in a situation where we keep
// coming back here and trimming a tiny amount
if ($pct_memory > (0.90 * $threshold)) {
$this->message->display(
$this->t('Memory usage is now !usage (!pct% of limit !limit), not enough reclaimed, starting new batch',
array('!pct' => round($pct_memory*100),
'!usage' => $this->formatSize($usage),
'!limit' => $this->formatSize($this->memoryLimit))),
'warning');
return TRUE;
}
else {
$this->message->display(
$this->t('Memory usage is now !usage (!pct% of limit !limit), reclaimed enough, continuing',
array('!pct' => round($pct_memory*100),
'!usage' => $this->formatSize($usage),
'!limit' => $this->formatSize($this->memoryLimit))),
'warning');
return FALSE;
}
}
else {
return FALSE;
}
}
/**
* Returns the memory usage so far.
*
* @return int
* The memory usage.
*/
protected function getMemoryUsage() {
return memory_get_usage();
}
/**
* Tries to reclaim memory.
*
* @return int
* The memory usage after reclaim.
*/
protected function attemptMemoryReclaim() {
// First, try resetting Drupal's static storage - this frequently releases
// plenty of memory to continue.
drupal_static_reset();
// @TODO: explore resetting the container.
return memory_get_usage();
}
/**
* Generates a string representation for the given byte count.
*
* @param int $size
* A size in bytes.
*
* @return string
* A translated string representation of the size.
*/
protected function formatSize($size) {
return format_size($size);
}
}
<?php
/**
* @file
* Contains \Drupal\Tests\migrate\Unit\MigrateExecutableMemoryExceededTest.
*/
namespace Drupal\Tests\migrate\Unit;
/**
* Tests the \Drupal\migrate\MigrateExecutable::memoryExceeded() method.
*
* @group migrate
*/
class MigrateExecutableMemoryExceededTest extends MigrateTestCase {
/**
* The mocked migration entity.
*
* @var \Drupal\migrate\Entity\MigrationInterface|\PHPUnit_Framework_MockObject_MockObject
*/
protected $migration;
/**
* The mocked migrate message.
*
* @var \Drupal\migrate\MigrateMessageInterface|\PHPUnit_Framework_MockObject_MockObject
*/
protected $message;
/**
* The tested migrate executable.
*
* @var \Drupal\Tests\migrate\Unit\TestMigrateExecutable
*/
protected $executable;
/**
* The migration configuration, initialized to set the ID to test.
*
* @var array
*/
protected $migrationConfiguration = array(
'id' => 'test',
);
/**
* php.init memory_limit value.
*/
protected $memoryLimit = 10000000;
/**
* {@inheritdoc}
*/
protected function setUp() {
parent::setUp();
$this->migration = $this->getMigration();
$this->message = $this->getMock('Drupal\migrate\MigrateMessageInterface');
$this->executable = new TestMigrateExecutable($this->migration, $this->message);
$this->executable->setStringTranslation($this->getStringTranslationStub());
}
/**
* Runs the actual test.
*
* @param string $message
* The second message to assert.
* @param bool $memory_exceeded
* Whether to test the memory exceeded case.
* @param int $memory_usage_first
* (optional) The first memory usage value.
* @param int $memory_usage_second
* (optional) The fake amount of memory usage reported after memory reclaim.
* @param int $memory_limit
* (optional) The memory limit.
*/
protected function runMemoryExceededTest($message, $memory_exceeded, $memory_usage_first = NULL, $memory_usage_second = NULL, $memory_limit = NULL) {
$this->executable->setMemoryLimit($memory_limit ?: $this->memoryLimit);
$this->executable->setMemoryUsage($memory_usage_first ?: $this->memoryLimit, $memory_usage_second ?: $this->memoryLimit);
$this->executable->setMemoryThreshold(0.85);
if ($message) {
$this->executable->message->expects($this->at(0))
->method('display')
->with($this->stringContains('reclaiming memory'));
$this->executable->message->expects($this->at(1))
->method('display')
->with($this->stringContains($message));
}
else {
$this->executable->message->expects($this->never())
->method($this->anything());
}
$result = $this->executable->memoryExceeded();
$this->assertEquals($memory_exceeded, $result);
}
/**
* Tests memoryExceeded method when a new batch is needed.
*/
public function testMemoryExceededNewBatch() {
// First case try reset and then start new batch.
$this->runMemoryExceededTest('starting new batch', TRUE);
}
/**
* Tests memoryExceeded method when enough is cleared.
*/
public function testMemoryExceededClearedEnough() {
$this->runMemoryExceededTest('reclaimed enough', FALSE, $this->memoryLimit, $this->memoryLimit * 0.75);
}
/**
* Tests memoryExceeded when memory usage is not exceeded.
*/
public function testMemoryNotExceeded() {
$this->runMemoryExceededTest('', FALSE, floor($this->memoryLimit * 0.85) - 1);
}
}
......@@ -8,16 +8,27 @@
namespace Drupal\Tests\migrate\Unit;
use Drupal\Core\StringTranslation\TranslationInterface;
use Drupal\migrate\Entity\MigrationInterface;
use Drupal\migrate\MigrateExecutable;
use Drupal\migrate\MigrateMessageInterface;
use Symfony\Component\EventDispatcher\EventDispatcherInterface;
/**
* Tests MigrateExecutable.
*/
class TestMigrateExecutable extends MigrateExecutable {
/**
* The fake memory usage in bytes.
*
* @var int
*/
protected $memoryUsage;
/**
* The cleared memory usage.
*
* @var int
*/
protected $clearedMemoryUsage;
/**
* Sets the string translation service.
*
......@@ -59,4 +70,68 @@ public function handleException(\Exception $exception, $save = TRUE) {
$this->message->display($message);
}
/**
* Allows access to the protected memoryExceeded method.
*
* @return bool
* The memoryExceeded value.
*/
public function memoryExceeded() {
return parent::memoryExceeded();
}
/**
* {@inheritdoc}
*/
protected function attemptMemoryReclaim() {
return $this->clearedMemoryUsage;
}
/**
* {@inheritdoc}
*/
protected function getMemoryUsage() {
return $this->memoryUsage;
}
/**
* Sets the fake memory usage.
*
* @param int $memory_usage
* The fake memory usage value.
* @param int $cleared_memory_usage
* (optional) The fake cleared memory value.
*/
public function setMemoryUsage($memory_usage, $cleared_memory_usage = NULL) {
$this->memoryUsage = $memory_usage;
$this->clearedMemoryUsage = $cleared_memory_usage;
}
/**
* Sets the memory limit.
*
* @param int $memory_limit
* The memory limit.
*/
public function setMemoryLimit($memory_limit) {
$this->memoryLimit = $memory_limit;
}
/**
* Sets the memory threshold.
*
* @param float $threshold
* The new threshold.
*/
public function setMemoryThreshold($threshold) {
$this->memoryThreshold = $threshold;
}
/**
* {@inheritdoc}
*/
protected function formatSize($size) {
return $size;
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment