Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
D
drupal
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Locked Files
Custom Issue Tracker
Custom Issue Tracker
Labels
Merge Requests
315
Merge Requests
315
Requirements
Requirements
List
Security & Compliance
Security & Compliance
Dependency List
License Compliance
Analytics
Analytics
Code Review
Insights
Issue
Repository
Value Stream
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Commits
Open sidebar
project
drupal
Commits
260f9367
Commit
260f9367
authored
Mar 15, 2015
by
alexpott
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Issue
#2427335
by benjy, chx, dawehner: Combine legacy Source class into SourcePluginBase
parent
08ce5a70
Changes
26
Hide whitespace changes
Inline
Side-by-side
Showing
26 changed files
with
684 additions
and
520 deletions
+684
-520
core/modules/migrate/src/MigrateExecutable.php
core/modules/migrate/src/MigrateExecutable.php
+8
-6
core/modules/migrate/src/Plugin/MigrateSourceInterface.php
core/modules/migrate/src/Plugin/MigrateSourceInterface.php
+1
-1
core/modules/migrate/src/Plugin/migrate/source/EmptySource.php
...modules/migrate/src/Plugin/migrate/source/EmptySource.php
+1
-1
core/modules/migrate/src/Plugin/migrate/source/SourcePluginBase.php
...es/migrate/src/Plugin/migrate/source/SourcePluginBase.php
+392
-3
core/modules/migrate/src/Plugin/migrate/source/SqlBase.php
core/modules/migrate/src/Plugin/migrate/source/SqlBase.php
+1
-13
core/modules/migrate/src/Row.php
core/modules/migrate/src/Row.php
+3
-0
core/modules/migrate/src/Source.php
core/modules/migrate/src/Source.php
+0
-449
core/modules/migrate/tests/src/Unit/MigrateExecutableTest.php
.../modules/migrate/tests/src/Unit/MigrateExecutableTest.php
+10
-9
core/modules/migrate/tests/src/Unit/MigrateSourceTest.php
core/modules/migrate/tests/src/Unit/MigrateSourceTest.php
+245
-0
core/modules/migrate/tests/src/Unit/MigrateSqlSourceTestCase.php
...dules/migrate/tests/src/Unit/MigrateSqlSourceTestCase.php
+1
-16
core/modules/migrate/tests/src/Unit/TestMigrateExecutable.php
.../modules/migrate/tests/src/Unit/TestMigrateExecutable.php
+1
-1
core/modules/migrate_drupal/src/Plugin/migrate/source/Variable.php
...les/migrate_drupal/src/Plugin/migrate/source/Variable.php
+1
-1
core/modules/migrate_drupal/src/Plugin/migrate/source/d6/Block.php
...les/migrate_drupal/src/Plugin/migrate/source/d6/Block.php
+2
-2
core/modules/migrate_drupal/src/Plugin/migrate/source/d6/CommentVariable.php
...e_drupal/src/Plugin/migrate/source/d6/CommentVariable.php
+1
-1
core/modules/migrate_drupal/src/Plugin/migrate/source/d6/ContactSettings.php
...e_drupal/src/Plugin/migrate/source/d6/ContactSettings.php
+1
-1
core/modules/migrate_drupal/src/Plugin/migrate/source/d6/FieldInstancePerFormDisplay.php
.../Plugin/migrate/source/d6/FieldInstancePerFormDisplay.php
+1
-1
core/modules/migrate_drupal/src/Plugin/migrate/source/d6/FieldInstancePerViewMode.php
...src/Plugin/migrate/source/d6/FieldInstancePerViewMode.php
+1
-1
core/modules/migrate_drupal/src/Plugin/migrate/source/d6/File.php
...ules/migrate_drupal/src/Plugin/migrate/source/d6/File.php
+2
-2
core/modules/migrate_drupal/src/Plugin/migrate/source/d6/Node.php
...ules/migrate_drupal/src/Plugin/migrate/source/d6/Node.php
+2
-2
core/modules/migrate_drupal/src/Plugin/migrate/source/d6/NodeType.php
.../migrate_drupal/src/Plugin/migrate/source/d6/NodeType.php
+2
-2
core/modules/migrate_drupal/src/Plugin/migrate/source/d6/Role.php
...ules/migrate_drupal/src/Plugin/migrate/source/d6/Role.php
+2
-2
core/modules/migrate_drupal/src/Plugin/migrate/source/d6/UploadInstance.php
...te_drupal/src/Plugin/migrate/source/d6/UploadInstance.php
+1
-1
core/modules/migrate_drupal/src/Plugin/migrate/source/d6/UserPictureFile.php
...e_drupal/src/Plugin/migrate/source/d6/UserPictureFile.php
+2
-2
core/modules/migrate_drupal/src/Plugin/migrate/source/d6/UserPictureInstance.php
...upal/src/Plugin/migrate/source/d6/UserPictureInstance.php
+1
-1
core/modules/migrate_drupal/src/Plugin/migrate/source/d6/ViewMode.php
.../migrate_drupal/src/Plugin/migrate/source/d6/ViewMode.php
+1
-1
core/modules/migrate_drupal/src/Plugin/migrate/source/d6/ViewModeBase.php
...rate_drupal/src/Plugin/migrate/source/d6/ViewModeBase.php
+1
-1
No files found.
core/modules/migrate/src/MigrateExecutable.php
View file @
260f9367
...
...
@@ -162,7 +162,7 @@ class MigrateExecutable implements MigrateExecutableInterface {
/**
* The source.
*
* @var \Drupal\migrate\
Sour
ce
* @var \Drupal\migrate\
Plugin\MigrateSourceInterfa
ce
*/
protected
$source
;
...
...
@@ -219,12 +219,16 @@ public function __construct(MigrationInterface $migration, MigrateMessageInterfa
*
* Makes sure source is initialized based on migration settings.
*
* @return \Drupal\migrate\
Sour
ce
* @return \Drupal\migrate\
Plugin\MigrateSourceInterfa
ce
* The source.
*/
protected
function
getSource
()
{
if
(
!
isset
(
$this
->
source
))
{
$this
->
source
=
new
Source
(
$this
->
migration
,
$this
);
$this
->
source
=
$this
->
migration
->
getSourcePlugin
();
// @TODO, find out how to remove this.
// @see https://drupal.org/node/2443617
$this
->
source
->
migrateExecutable
=
$this
;
}
return
$this
->
source
;
}
...
...
@@ -256,13 +260,11 @@ public function import() {
}
catch
(
\
Exception
$e
)
{
$this
->
message
->
display
(
$this
->
t
(
'Migration failed with source plugin exception: !e'
,
array
(
'!e'
=>
$e
->
getMessage
())),
'error'
);
$this
->
t
(
'Migration failed with source plugin exception: !e'
,
array
(
'!e'
=>
$e
->
getMessage
())),
'error'
);
return
MigrationInterface
::
RESULT_FAILED
;
}
$destination
=
$this
->
migration
->
getDestinationPlugin
();
while
(
$source
->
valid
())
{
$row
=
$source
->
current
();
if
(
$this
->
sourceIdValues
=
$row
->
getSourceIdValues
())
{
...
...
core/modules/migrate/src/Plugin/MigrateSourceInterface.php
View file @
260f9367
...
...
@@ -19,7 +19,7 @@
*
* @ingroup migration
*/
interface
MigrateSourceInterface
extends
\
Countable
,
PluginInspectionInterface
{
interface
MigrateSourceInterface
extends
\
Countable
,
\
Iterator
,
PluginInspectionInterface
{
/**
* Returns available fields on the source.
...
...
core/modules/migrate/src/Plugin/migrate/source/EmptySource.php
View file @
260f9367
...
...
@@ -30,7 +30,7 @@ public function fields() {
/**
* {@inheritdoc}
*/
public
function
get
Iterator
()
{
public
function
initialize
Iterator
()
{
return
new
\
ArrayIterator
(
array
(
array
(
'id'
=>
''
)));
}
...
...
core/modules/migrate/src/Plugin/migrate/source/SourcePluginBase.php
View file @
260f9367
...
...
@@ -9,6 +9,8 @@
use
Drupal\Core\Plugin\PluginBase
;
use
Drupal\migrate\Entity\MigrationInterface
;
use
Drupal\migrate\MigrateException
;
use
Drupal\migrate\Plugin\MigrateIdMapInterface
;
use
Drupal\migrate\Plugin\MigrateSourceInterface
;
use
Drupal\migrate\Row
;
...
...
@@ -22,7 +24,7 @@
*
* @ingroup migration
*/
abstract
class
SourcePluginBase
extends
PluginBase
implements
MigrateSourceInterface
{
abstract
class
SourcePluginBase
extends
PluginBase
implements
MigrateSourceInterface
{
/**
* @var \Drupal\Core\Extension\ModuleHandlerInterface
...
...
@@ -34,14 +36,154 @@ abstract class SourcePluginBase extends PluginBase implements MigrateSourceInter
*/
protected
$migration
;
/**
* The name and type of the highwater property in the source.
*
* @var array
*
* @see $originalHighwater
*/
protected
$highWaterProperty
;
/**
* The current row from the query
*
* @var \Drupal\Migrate\Row
*/
protected
$currentRow
;
/**
* The primary key of the current row
*
* @var array
*/
protected
$currentSourceIds
;
/**
* Number of rows intentionally ignored (prepareRow() returned FALSE)
*
* @var int
*/
protected
$numIgnored
=
0
;
/**
* Number of rows we've at least looked at.
*
* @var int
*/
protected
$numProcessed
=
0
;
/**
* The high water mark at the beginning of the import operation.
*
* If the source has a property for tracking changes (like Drupal ha
* node.changed) then this is the highest value of those imported so far.
*
* @var int
*/
protected
$originalHighWater
;
/**
* List of source IDs to process.
*
* @var array
*/
protected
$idList
=
array
();
/**
* Whether this instance should cache the source count.
*
* @var boolean
*/
protected
$cacheCounts
=
FALSE
;
/**
* Key to use for caching counts.
*
* @var string
*/
protected
$cacheKey
;
/**
* Whether this instance should not attempt to count the source.
*
* @var boolean
*/
protected
$skipCount
=
FALSE
;
/**
* If TRUE, we will maintain hashed source rows to determine whether incoming
* data has changed.
*
* @var bool
*/
protected
$trackChanges
=
FALSE
;
/**
* By default, next() will directly read the map row and add it to the data
* row. A source plugin implementation may do this itself (in particular, the
* SQL source can incorporate the map table into the query) - if so, it should
* set this TRUE so we don't duplicate the effort.
*
* @var bool
*/
protected
$mapRowAdded
=
FALSE
;
/**
* @var \Drupal\Core\Cache\CacheBackendInterface
*/
protected
$cache
;
/**
* @var \Drupal\migrate\Plugin\MigrateIdMapInterface
*/
protected
$idMap
;
/**
* @var \Iterator
*/
protected
$iterator
;
// @TODO, find out how to remove this.
// @see https://drupal.org/node/2443617
public
$migrateExecutable
;
/**
* {@inheritdoc}
*/
public
function
__construct
(
array
$configuration
,
$plugin_id
,
$plugin_definition
,
MigrationInterface
$migration
)
{
parent
::
__construct
(
$configuration
,
$plugin_id
,
$plugin_definition
);
$this
->
migration
=
$migration
;
// Set up some defaults based on the source configuration.
$this
->
cacheCounts
=
!
empty
(
$configuration
[
'cache_counts'
]);
$this
->
skipCount
=
!
empty
(
$configuration
[
'skip_count'
]);
$this
->
cacheKey
=
!
empty
(
$configuration
[
'cache_key'
])
?
!
empty
(
$configuration
[
'cache_key'
])
:
NULL
;
$this
->
trackChanges
=
!
empty
(
$configuration
[
'track_changes'
])
?
$configuration
[
'track_changes'
]
:
FALSE
;
// Pull out the current highwater mark if we have a highwater property.
if
(
$this
->
highWaterProperty
=
$this
->
migration
->
get
(
'highWaterProperty'
))
{
$this
->
originalHighWater
=
$this
->
migration
->
getHighWater
();
}
if
(
$id_list
=
$this
->
migration
->
get
(
'idlist'
))
{
$this
->
idList
=
$id_list
;
}
// Don't allow the use of both highwater and track changes together.
if
(
$this
->
highWaterProperty
&&
$this
->
trackChanges
)
{
throw
new
MigrateException
(
'You should either use a highwater mark or track changes not both. They are both designed to solve the same problem'
);
}
}
/**
* Initialize the iterator with the source data.
*
* @return array
* An array of the data for this source.
*/
protected
abstract
function
initializeIterator
();
/**
* Get the module handler.
*
...
...
@@ -59,12 +201,259 @@ protected function getModuleHandler() {
* {@inheritdoc}
*/
public
function
prepareRow
(
Row
$row
)
{
$result
=
TRUE
;
$result_hook
=
$this
->
getModuleHandler
()
->
invokeAll
(
'migrate_prepare_row'
,
array
(
$row
,
$this
,
$this
->
migration
));
$result_named_hook
=
$this
->
getModuleHandler
()
->
invokeAll
(
'migrate_'
.
$this
->
migration
->
id
()
.
'_prepare_row'
,
array
(
$row
,
$this
,
$this
->
migration
));
// If any of the hooks returned false, we want to skip the row.
// We're explicitly skipping this row - keep track in the map table.
if
((
$result_hook
&&
in_array
(
FALSE
,
$result_hook
))
||
(
$result_named_hook
&&
in_array
(
FALSE
,
$result_named_hook
)))
{
return
FALSE
;
// Make sure we replace any previous messages for this item with any
// new ones.
$id_map
=
$this
->
migration
->
getIdMap
();
$id_map
->
delete
(
$this
->
currentSourceIds
,
TRUE
);
$this
->
migrateExecutable
->
saveQueuedMessages
();
$id_map
->
saveIdMapping
(
$row
,
array
(),
MigrateIdMapInterface
::
STATUS_IGNORED
,
$this
->
migrateExecutable
->
rollbackAction
);
$this
->
numIgnored
++
;
$this
->
currentRow
=
NULL
;
$this
->
currentSourceIds
=
NULL
;
$result
=
FALSE
;
}
elseif
(
$this
->
trackChanges
)
{
// When tracking changed data, We want to quietly skip (rather than
// "ignore") rows with changes. The caller needs to make that decision,
// so we need to provide them with the necessary information (before and
// after hashes).
$row
->
rehash
();
}
$this
->
numProcessed
++
;
return
$result
;
}
/**
* Returns the iterator that will yield the row arrays to be processed.
*
* @return \Iterator
*/
public
function
getIterator
()
{
if
(
!
isset
(
$this
->
iterator
))
{
$this
->
iterator
=
$this
->
initializeIterator
();
}
return
$this
->
iterator
;
}
/**
* {@inheritdoc}
*/
public
function
current
()
{
return
$this
->
currentRow
;
}
/**
* Get the iterator key.
*
* Implementation of Iterator::key - called when entering a loop iteration,
* returning the key of the current row. It must be a scalar - we will
* serialize to fulfill the requirement, but using getCurrentIds() is
* preferable.
*/
public
function
key
()
{
return
serialize
(
$this
->
currentSourceIds
);
}
/**
* Whether the iterator is currently valid.
*
* Implementation of Iterator::valid() - called at the top of the loop,
* returning TRUE to process the loop and FALSE to terminate it
*/
public
function
valid
()
{
return
isset
(
$this
->
currentRow
);
}
/**
* Rewind the iterator.
*
* Implementation of Iterator::rewind() - subclasses of MigrateSource should
* implement performRewind() to do any class-specific setup for iterating
* source records.
*/
public
function
rewind
()
{
$this
->
idMap
=
$this
->
migration
->
getIdMap
();
$this
->
numProcessed
=
0
;
$this
->
numIgnored
=
0
;
$this
->
getIterator
()
->
rewind
();
$this
->
next
();
}
/**
* {@inheritdoc}
*
* The migration iterates over rows returned by the source plugin. This
* method determines the next row which will be processed and imported into
* the system.
*
* The method tracks the source and destination IDs using the ID map plugin.
*
* This also takes care about highwater support. Highwater allows to reimport
* rows from a previous migration run, which got changed in the meantime.
* This is done by specifying a highwater field, which is compared with the
* last time, the migration got executed (originalHighWater).
*/
public
function
next
()
{
$this
->
currentSourceIds
=
NULL
;
$this
->
currentRow
=
NULL
;
$source_configuration
=
$this
->
migration
->
get
(
'source'
);
// In order to find the next row we want to process, we ask the source
// plugin for the next possible row.
while
(
!
isset
(
$this
->
currentRow
)
&&
$this
->
getIterator
()
->
valid
())
{
$row_data
=
$this
->
getIterator
()
->
current
()
+
$source_configuration
;
$this
->
getIterator
()
->
next
();
$row
=
new
Row
(
$row_data
,
$this
->
migration
->
getSourcePlugin
()
->
getIds
(),
$this
->
migration
->
get
(
'destinationIds'
));
// Populate the source key for this row.
$this
->
currentSourceIds
=
$row
->
getSourceIdValues
();
// Pick up the existing map row, if any, unless getNextRow() did it.
if
(
!
$this
->
mapRowAdded
&&
(
$id_map
=
$this
->
idMap
->
getRowBySource
(
$this
->
currentSourceIds
)))
{
$row
->
setIdMap
(
$id_map
);
}
// In case we have specified an ID list, but the ID given by the source is
// not in there, we skip the row.
$id_in_the_list
=
$this
->
idList
&&
in_array
(
reset
(
$this
->
currentSourceIds
),
$this
->
idList
);
if
(
$this
->
idList
&&
!
$id_in_the_list
)
{
continue
;
}
// Preparing the row gives source plugins the chance to skip.
if
(
$this
->
prepareRow
(
$row
)
===
FALSE
)
{
continue
;
}
// Check whether the row needs processing.
// 1. Explicitly specified IDs.
// 2. This row has not been imported yet.
// 3. Explicitly set to update.
// 4. The row is newer than the current highwater mark.
// 5. If no such property exists then try by checking the hash of the row.
if
(
$id_in_the_list
||
!
$row
->
getIdMap
()
||
$row
->
needsUpdate
()
||
$this
->
aboveHighwater
(
$row
)
||
$this
->
rowChanged
(
$row
)
)
{
$this
->
currentRow
=
$row
->
freezeSource
();
}
}
}
/**
* Check if the incoming data is newer than what we've previously imported.
*
* @param \Drupal\migrate\Row $row
* The row we're importing.
*
* @return bool
* TRUE if the highwater value in the row is greater than our current value.
*/
protected
function
aboveHighwater
(
Row
$row
)
{
return
$this
->
highWaterProperty
&&
$row
->
getSourceProperty
(
$this
->
highWaterProperty
[
'name'
])
>
$this
->
originalHighWater
;
}
/**
* Check if the incoming row has changed since our last import.
*
* @param \Drupal\migrate\Row $row
* The row we're importing.
*
* @return bool
* TRUE if the row has changed otherwise FALSE.
*/
protected
function
rowChanged
(
Row
$row
)
{
return
$this
->
trackChanges
&&
$row
->
changed
();
}
/**
* Getter for currentSourceIds data member.
*/
public
function
getCurrentIds
()
{
return
$this
->
currentSourceIds
;
}
/**
* Getter for numIgnored data member.
*/
public
function
getIgnored
()
{
return
$this
->
numIgnored
;
}
/**
* Getter for numProcessed data member.
*/
public
function
getProcessed
()
{
return
$this
->
numProcessed
;
}
/**
* Reset numIgnored back to 0.
*/
public
function
resetStats
()
{
$this
->
numIgnored
=
0
;
}
/**
* Get the source count.
*
* Return a count of available source records, from the cache if appropriate.
* Returns -1 if the source is not countable.
*
* @param bool $refresh
* Whether or not to refresh the count.
*
* @return int
* The count.
*/
public
function
count
(
$refresh
=
FALSE
)
{
if
(
$this
->
skipCount
)
{
return
-
1
;
}
if
(
!
isset
(
$this
->
cacheKey
))
{
$this
->
cacheKey
=
hash
(
'sha256'
,
$this
->
getPluginId
());
}
// If a refresh is requested, or we're not caching counts, ask the derived
// class to get the count from the source.
if
(
$refresh
||
!
$this
->
cacheCounts
)
{
$count
=
$this
->
getIterator
()
->
count
();
$this
->
getCache
()
->
set
(
$this
->
cacheKey
,
$count
,
'cache'
);
}
else
{
// Caching is in play, first try to retrieve a cached count.
$cache_object
=
$this
->
getCache
()
->
get
(
$this
->
cacheKey
,
'cache'
);
if
(
is_object
(
$cache_object
))
{
// Success.
$count
=
$cache_object
->
data
;
}
else
{
// No cached count, ask the derived class to count 'em up, and cache
// the result.
$count
=
$this
->
getIterator
()
->
count
();
$this
->
getCache
()
->
set
(
$this
->
cacheKey
,
$count
,
'cache'
);
}
}
return
$count
;
}
/**
* Get the cache object.
*
* @return \Drupal\Core\Cache\CacheBackendInterface
* The cache object.
*/
protected
function
getCache
()
{
if
(
!
isset
(
$this
->
cache
))
{
$this
->
cache
=
\
Drupal
::
cache
(
'migrate'
);
}
return
$this
->
cache
;
}
}
core/modules/migrate/src/Plugin/migrate/source/SqlBase.php
View file @
260f9367
...
...
@@ -112,7 +112,7 @@ protected function prepareQuery() {
* We could simply execute the query and be functionally correct, but
* we will take advantage of the PDO-based API to optimize the query up-front.
*/
protected
function
runQuery
()
{
protected
function
initializeIterator
()
{
$this
->
prepareQuery
();
$high_water_property
=
$this
->
migration
->
get
(
'highWaterProperty'
);
...
...
@@ -203,18 +203,6 @@ public function count() {
return
$this
->
query
()
->
countQuery
()
->
execute
()
->
fetchField
();
}
/**
* Returns the iterator that will yield the row arrays to be processed.
*
* @return \Iterator
*/
public
function
getIterator
()
{
if
(
!
isset
(
$this
->
iterator
))
{
$this
->
iterator
=
$this
->
runQuery
();
}
return
$this
->
iterator
;
}
/**
* Check if we can join against the map table.
*
...
...
core/modules/migrate/src/Row.php
View file @
260f9367
...
...
@@ -176,9 +176,12 @@ public function setSourceProperty($property, $data) {
/**
* Freezes the source.
*
* @return $this
*/
public
function
freezeSource
()
{
$this
->
frozen
=
TRUE
;
return
$this
;
}
/**
...
...
core/modules/migrate/src/Source.php
deleted
100644 → 0
View file @
08ce5a70
<?php
/**
* @file
* Contains \Drupal\migrate\Plugin\migrate\source\SourceBase.
*/
namespace
Drupal\migrate
;
use
Drupal\migrate\Entity\MigrationInterface
;
use
Drupal\migrate\Plugin\MigrateIdMapInterface
;
/**
* Source is a caching / decision making wrapper around the source plugin.
*
* Derived classes are expected to define __toString(), returning a string
* describing the source and significant options, i.e. the query.
*
* @see \Drupal\migrate\MigrateSourceInterface
*/
class
Source
implements
\
Iterator
,
\
Countable
{
/**
* The current row from the quey
*
* @var \Drupal\Migrate\Row
*/
protected
$currentRow
;
/**
* The primary key of the current row
*
* @var array
*/
protected
$currentIds
;
/**
* Number of rows intentionally ignored (prepareRow() returned FALSE)
*
* @var int
*/
protected
$numIgnored
=
0
;
/**
* Number of rows we've at least looked at.
*
* @var int
*/
protected
$numProcessed
=
0
;
/**
* The high water mark at the beginning of the import operation.
*
* @var
*/
protected
$originalHighWater
=
''
;
/**
* List of source IDs to process.