Commit c487b0dc authored by Chris Burgess's avatar Chris Burgess
Browse files

Issue #3262781: Refactor pipeline graphing

parent dfc56055

migrate_visualize.theme.inc

+16 −13
Original line number Diff line number Diff line
@@ -19,10 +19,10 @@ function _migrate_visualize_graph_type_attributes() { 
    'destination:field' => ['color' => 'darkgoldenrod3', 'shape' => 'box'],

    'destination:meta' => ['color' => 'white', 'shape' => 'cds'],

    'process' => ['color' => 'blue', 'shape' => 'box'],

    'process:pipeline' => ['color' => 'blue', 'shape' => 'box'],

    'process:meta' => ['color' => 'white', 'shape' => 'box'],

    'process:pipeline:step' => ['color' => 'blue', 'shape' => 'box'],

    'process:pipeline' => ['color' => 'blue', 'shape' => 'box'],

    'process:pipeline' => ['color' => 1, 'shape' => 'box'],

    'process:meta' => ['color' => 2, 'shape' => 'box'],

    'process:pipeline:step' => ['color' => 3, 'shape' => 'box'],

    'process:pipeline' => ['color' => 4, 'shape' => 'box'],

    'plugin' => ['color' => 'cornflowerblue2', 'shape' => 'box'],

    'destination' => ['color' => 'darkgoldenrod3', 'shape' => 'box'],

  // 'source' => ['colorscheme' => 'bupu9',],
@@ -77,19 +77,22 @@ function template_preprocess_migration_visualize_graphviz(array &$variables) { 
    }

  }



  $graphvizAttributes = [

    'colorscheme' => 'dark28',

    'penwidth' => 2,

  ];



  /** @var \Fhaculty\Graph\Edge\Directed $edge */

  foreach ($graph->getEdges() as $k => $edge) {

    if (!$edge->getAttribute('graphviz.color')) {

      $edge->setAttribute('graphviz.colorscheme', 'oranges9');

    $edge->setAttribute('graphviz.color', $k % 6 + 2);

    foreach ($graphvizAttributes as $name => $value) {

      if (!$edge->getAttribute($name)) {

        $edge->setAttribute("graphviz.{$name}", $value);

      }

    if ($edge->getAttribute('migrate_visualize.debug')) {

      $edge->setAttribute('migrate_visualize.label', t('(@debug)', [

        '@label' => $edge->getAttribute('migrate_visualize.label'),

        '@debug' => $edge->getAttribute('migrate_visualize.debug'),

      ]));

    }

    $edge->setAttribute('graphviz.penwidth', '2');

    if ($comment = $edge->getAttribute('migrate_visualize.debug')) {

      $edge->setAttribute('graphviz.comment', $comment);

    }

    if ($label = $edge->getAttribute('migrate_visualize.label')) {

      $edge->setAttribute('graphviz.label', $label);

    }

src/Controller/VisualizeController.php

+1 −1
Original line number Diff line number Diff line
@@ -59,7 +59,7 @@ class VisualizeController extends ControllerBase { 
      ->getForm('Drupal\migrate_visualize\Form\VisualizeMigrationSwitcherForm');



    $migrationInstance = $this->migrationPluginManager->createInstance($migration);

    if ($migration !== '' && $migrationInstance === false) {

    if ($migration !== '' && $migrationInstance === FALSE) {

      $this->messenger()

        ->addError($this->t('Migration not found: @migration', [

          '@migration' => $migration,

src/MigrateGraph.php

+93 −92
Original line number Diff line number Diff line
@@ -97,7 +97,7 @@ class MigrateGraph { 


    $this->graphSourcePlugin($this->migration->getSourcePlugin(), $this->migration->getSourceConfiguration());

    $this->graphDestinationPlugin($this->migration->getDestinationPlugin(), $this->migration->getDestinationConfiguration());

    $this->graphProcessPlugins($this->migration->getProcessPlugins());

    $this->graphPipeline($this->migration->getProcessPlugins());

  }



  /**
@@ -188,7 +188,6 @@ class MigrateGraph { 
        $entityType = explode(':', $configuration['plugin'])[1];

        $entityFieldManager = \Drupal::service('entity_field.manager');

        $fields = $entityFieldManager->getFieldDefinitions($entityType, $configuration['default_bundle']);



        foreach ($fields as $fieldId => $field) {

          $fieldVertex = $graph->createVertex("destination:field:{$fieldId}");

          $fieldVertex->setAttribute('migrate_visualize.discovery', 'destination');
@@ -205,40 +204,67 @@ class MigrateGraph { 
  /**

   * Graph the process plugins.

   *

   * @var array $processPlugins

   *   The process pipelines for this migration.

   * @var array $pipeline

   *   The process pipeline for this migration.

   */

  protected function graphProcessPlugins($processPlugins) : void {

  protected function graphPipeline($pipeline) : void {

    $graph = $this->getGraph();

    /** @var \Drupal\migrate\Plugin\MigrateProcessInterface[] $processPlugin */

    foreach ($processPlugins as $pipelineId => $pipeline) {

      $pipelineVertex = $graph->createVertex("process:field:{$pipelineId}");

      $pipelineVertex->setAttribute('migrate_visualize.type', 'process:pipeline');

      $pipelineVertex->setAttribute('migrate_visualize.label', t("@fieldId", [

        '@fieldId' => $pipelineId,

    /** @var \Drupal\migrate\Plugin\MigrateProcessInterface[] $plugins */

    foreach ($pipeline as $destinationPropertyName => $plugins) {

      $processVertexId = "process:field:{$destinationPropertyName}";

      $processVertex = $graph->createVertex($processVertexId);

      $processVertex->setAttribute('migrate_visualize.type', 'process:pipeline');

      $processVertex->setAttribute('migrate_visualize.label', t("@fieldId", [

      // . ' (' . __LINE__ . ')',

        '@fieldId' => $destinationPropertyName,

      ]));



      unset($previousStep);

      foreach ($plugins as $pipelineStepId => $plugin) {

        $this->graphProcessPlugin($destinationPropertyName, $plugin, $processVertex, $pipelineStepId, $plugins);

      }



      $destinationVertexId = "destination:field:{$destinationPropertyName}";



      if ($graph->hasVertex($destinationVertexId)) {

        $destinationVertex = $graph->getVertex($destinationVertexId);

      }

      else {

        $destinationVertex = $graph->createVertex($destinationVertexId);

        $destinationVertex->setAttribute('migrate_visualize.type', 'destination:field');

        $destinationVertex->setAttribute('migrate_visualize.key', "{$destinationVertexId}");

        $destinationVertex->setAttribute('migrate_visualize.label', "{$destinationPropertyName}");

        $destinationVertex->setAttribute('migrate_visualize.discovery', 'process');

        $destinationVertex->setAttribute('migrate_visualize.debug', __LINE__);

      }

      $pipelineDestinationEdge = $processVertex->createEdgeTo($destinationVertex);

    }

  }



  /**

   * IDK why but seems like if there's >1 steps in the pipeline, the first will be a get plugin

   * with the config of the second. This prevents duplication on steps 0,1 of the graph.

   */

      // If (count($pipeline) > 1) {

      //  array_shift($pipeline);

      // }

      foreach ($pipeline as $pipelineStepId => $processPlugin) {

  protected function graphProcessPlugin($destinationPropertyName, $plugin, $destinationVertex, $pipelineStepId, $plugins) {

    $graph = $this->getGraph();



    /** @var ProcessPluginBase $processPlugin */

        $configuration = self::extractProtectedProperty($processPlugin, 'configuration');

        $stepId = "process:field:{$pipelineId}:{$pipelineStepId}";

        $stepVertex = $graph->createVertex($stepId);

    $configuration = self::extractProtectedProperty($plugin, 'configuration');

    $definition = $plugin->getPluginDefinition();

    $pluginId = $definition['id'];



    // @see https://www.drupal.org/project/migrate_visualize/issues/3262781

    if ($pluginId === 'get' && $configuration['plugin'] !== 'get') {

      return;

    }



        $stepVertex->setAttribute('migrate_visualize.type', 'process:pipeline:step');

        $stepVertex->setAttribute('migrate_visualize.label', $this->getProcessPluginStepLabel($processPlugin, $configuration));

    $pluginVertexId = "process:field:{$destinationPropertyName}:{$pipelineStepId}";

    $pluginVertex = $graph->createVertex($pluginVertexId);



    $pluginVertex->setAttribute('migrate_visualize.type', 'process:pipeline:step');

    $pluginVertex->setAttribute('migrate_visualize.label', $this->getProcessPluginStepLabel($plugin, $configuration));



    // Connect to sources.

    foreach ($this->sourceKeys as $key) {

          unset($sourceVertexId);

      if (isset($configuration[$key])) {

        if (is_string($configuration[$key])) {

          $sources = [$configuration[$key]];
@@ -259,55 +285,30 @@ class MigrateGraph { 
          else {

            $sourceVertex = $graph->createVertex($sourceVertexId);

            $sourceVertex->setAttribute('migrate_visualize.type', 'source:field');

                $sourceVertex->setAttribute('migrate_visualize.key', "{$sourceVertexId}");

            $sourceVertex->setAttribute('migrate_visualize.key', $sourceVertexId);

            $sourceVertex->setAttribute('migrate_visualize.label', "{$sourceField}");

          }

              if (!$sourceVertex->hasEdgeTo($stepVertex)) {

                $edge = $sourceVertex->createEdgeTo($stepVertex);

                // $edge->setAttribute('migrate_visualize.debug', __LINE__);

                // $edge->setAttribute('graphviz.color', 'blue');

          if (!$sourceVertex->hasEdgeTo($pluginVertex)) {

            $edge = $sourceVertex->createEdgeTo($pluginVertex);

          }

        }

      }

    }



        // Last step in pipeline.

        if ($pipelineStepId == array_key_last($pipeline)) {

          $edge = $stepVertex->createEdgeTo($pipelineVertex);

          // $edge->setAttribute('migrate_visualize.debug', __LINE__);

          // $edge->setAttribute('graphviz.color', 'red');



          $destinationVertexId = "destination:field:{$pipelineId}";

          if ($graph->hasVertex($destinationVertexId)) {

            $destinationVertex = $graph->getVertex($destinationVertexId);

          }

          else {

            $destinationVertex = $graph->createVertex($destinationVertexId);

            $destinationVertex->setAttribute('migrate_visualize.type', 'destination:field');

            $destinationVertex->setAttribute('migrate_visualize.key', "{$destinationVertexId}");

            $destinationVertex->setAttribute('migrate_visualize.label', "{$pipelineId}");

            $destinationVertex->setAttribute('migrate_visualize.discovery', 'process');

          }



          if (isset($sourceVertexId) && $graph->hasVertex($sourceVertexId)) {

            if (!$sourceVertex->hasEdgeTo($stepVertex)) {

              $edge = $sourceVertex->createEdgeTo($stepVertex);

              // $edge->setAttribute('migrate_visualize.debug', __LINE__);

              // $edge->setAttribute('graphviz.color', 'green');

            }

    // Previous step exists.

    if (isset($previousPluginVertex)) {

      $previousPluginEdge = $previousPluginVertex->createEdgeTo($pluginVertex);

    }

    $previousPluginVertex = $pluginVertex;



          $edge = $pipelineVertex->createEdgeTo($destinationVertex);

          // $edge->setAttribute('migrate_visualize.debug', __LINE__);

          // $edge->setAttribute('graphviz.color', 'red');

        }

    // Last step in pipeline.

    if (count($plugins) == ($pipelineStepId + 1)) {

      $stepPipelineEdge = $pluginVertex->createEdgeTo($destinationVertex);



        // Previous step exists.

        if (isset($previousStep)) {

          $edge = $previousStep->createEdgeTo($stepVertex);

          // $edge->setAttribute('migrate_visualize.debug', __LINE__);

      if ($graph->hasVertex($sourceVertexId)) {

        if (!$sourceVertex->hasEdgeTo($pluginVertex)) {

          $sourceStepEdge = $sourceVertex->createEdgeTo($pluginVertex);

        }

        $previousStep = $stepVertex;

      }

    }

  }

tests/src/Functional/MigrateGraphTest.php

0 → 100644
+46 −0
Original line number Diff line number Diff line 
<?php



namespace Drupal\Tests\migrate_visualize\Functional;



use Drupal\Tests\BrowserTestBase;



/**

 * Tests Migrate Visualize settings form.

 *

 * @group migrate_visualize

 */

class MigrateGraphTest extends BrowserTestBase {



  /**

   * {@inheritdoc}

   */

  protected static $modules = [

    'field',

    'node',

    'migrate',

    'migrate_visualize',

    'migrate_visualize_test',

    'path',

  ];



  /**

   * {@inheritdoc}

   */

  protected $defaultTheme = 'stark';



  /**

   * Test access to migrate visualize settings form.

   */

  public function testMigrateGraphService() {

    $migrationPluginManager = \Drupal::service('plugin.manager.migration');

    $migrateGraphService = \Drupal::service('migrate_visualize.migrate_graph');



    $migration = $migrationPluginManager->createInstance('dummy');

    $migrateGraphService->graph($migration);

    $graph = $migrateGraphService->getGraph();



    $this->assertEquals(6, count($graph->getVertices()), 'dummy: number of vertices');

    $this->assertEquals(3, count($graph->getEdges()), 'dummy: number of edges');

  }



}