Commit d0d41569 authored by Ben Mullins's avatar Ben Mullins Committed by Tim Plunkett
Browse files

Issue #3292562 by bnjmnm, tim.plunkett, narendraR, chrisfromredfin: Make... 

Issue #3292562 by bnjmnm, tim.plunkett, narendraR, chrisfromredfin: Make fixture generation a drush command
parent 9e4b10cc

fixtures/categories.json

+1 −1

File changed.

Preview size limit exceeded, changes collapsed.

fixtures/project_data.json

+1 −1

File changed.

Preview size limit exceeded, changes collapsed.

project_browser.install

+5 −192
Original line number Diff line number Diff line
@@ -5,10 +5,7 @@ 
 * Contains install and update functions for Project Browser.

 */



use Composer\Semver\Semver;

use Drupal\Component\Serialization\Json;

use Drupal\Component\Utility\Unicode;

use Drupal\Component\Utility\Xss;

use Drupal\Core\Database\Database;



/**
@@ -222,193 +219,9 @@ function project_browser_update_9004() { 
  _project_browser_populate_from_fixture();

}



/**

 * Change this to an update hook and run update.php to create a new fixture.

 *

 * It's only here because an update hook is the easiest way to set up a batch.

 */

function project_browser_hacky_fixture_maker(&$sandbox) {

  // Initialize some sandbox values on first iteration.

  if (!isset($sandbox['progress'])) {

    // The count of nodes visited so far.

    $sandbox['progress'] = 0;

    // Total nodes that must be visited.

    $sandbox['max'] = 6400;

    // A place to store messages during the run.

    $sandbox['messages'] = [];

    // Last node read via the query.

    $sandbox['current_page'] = 0;



    $sandbox['projects'] = [];

  }



  $current_source = \Drupal::service('project_browser.enabled_source')->getCurrentSource();

  if ($current_source && $current_source->getPluginId() === 'drupalorg_mockapi') {

    $query = [

      'page' => $sandbox['current_page'],

      'field_project_type' => 'full',

      'limit' => 50,

      'field_project_has_releases' => 1,

      'field_project_has_issue_queue' => 1,

      'type' => 'project_module',

      'status' => 1,

      'sort' => 'changed',

      'direction' => 'DESC',

    ];

    $eariest_possible_timestamp_reached = NULL;

    $drupal_org_response = $current_source->getProjectsFromSource($query);

    $returned_projects = $drupal_org_response['list'];



    if ($returned_projects) {

      foreach ($returned_projects as &$project) {

        $project['project_usage_total'] = 0;

        if (array_key_exists('project_usage', $project)) {

          foreach ($project['project_usage'] as $usage) {

            $project['project_usage_total'] += $usage;

          }

        }



        if (empty($project['body']['value'])) {

          $project['body']['value'] = '';

        }

        if (empty($project['body']['summary'])) {

          $project['body']['summary'] = $project['body']['value'];

        }

        $project['body']['summary'] = Xss::filter(strip_tags($project['body']['summary']));

        $project['body']['summary'] = Unicode::truncate($project['body']['summary'], 200, TRUE, TRUE);



        // Once we hit projects that haven't been updated since March 13, 2020, we

        // know they aren't compatible because it is before

        // https://www.drupal.org/node/3119415.

        if ($project['changed'] < 1583985600) {

          $eariest_possible_timestamp_reached = TRUE;

        }

      }

      $sandbox['current_page'] += 1;



      $projects_to_store = (array) $returned_projects;



      // Rewrite the projects array so each project has added release data and

      // unnecessary values are removed to conserve space.

      $projects_to_store = array_map(function ($a_project) use ($current_source) {

        $the_project = (array) $a_project;

        $releases = $current_source->getProjectReleasesFromSource($the_project['field_project_machine_name']);

        if (!empty($releases['releases'])) {

          $compatible_releases = array_filter($releases['releases'], function ($release) {

            if (!empty($release['core_compatibility'])) {

              try {

                // Apparently there are multiple projects that have invalid

                // version strings.

                return Semver::satisfies(\Drupal::VERSION, $release['core_compatibility']);

              }

              catch (\Exception $exception) {

                // Don't include releases with invalid compatibility strings.

                return FALSE;

              }

            }

            return FALSE;

          });

          if (empty($compatible_releases)) {

            // Don't include projects without any compatible releases.

            return NULL;

          }

        }

        else {

          // Don't include projects without releases.

          return NULL;

        }



        // To keep filesize down, remove unnecessary items from release data.

        $the_project['releases'] = array_map(function ($release) {

          return [

            'version' => $release['version'],

            'status' => $release['status'],

            'date' => $release['date'] ?? NULL,

            'core_compatibility' => $release['core_compatibility'],

          ];

        }, $compatible_releases);

        $current_source->truncateProjectData($the_project);

        return $the_project;

      }, $projects_to_store);



      $projects_to_store = array_filter($projects_to_store);

      $sandbox['projects'] = array_merge($sandbox['projects'], $projects_to_store);

      $sandbox['progress'] += count($sandbox['projects']);

      $sandbox['#finished'] = count($sandbox['projects']) >= $sandbox['max'] || $eariest_possible_timestamp_reached ? TRUE : (count($sandbox['projects']) / $sandbox['max']);

    }

    else {

      $sandbox['#finished'] = TRUE;

    }

  }

  else {

    $sandbox['#finished'] = TRUE;

  }



  if ($sandbox['#finished'] === TRUE) {

    $module_path = \Drupal::service('module_handler')->getModule('project_browser')->getPath();

    file_put_contents($module_path . '/fixtures/project_data.json', '[]');

    file_put_contents($module_path . '/fixtures/categories.json', '[]');



    $projects = $sandbox['projects'];



    $category_values = [];



    // Map fixture values to DB columns.

    $values = array_map(function ($project) use (&$category_values) {

      if (!empty($project['taxonomy_vocabulary_3'])) {

        foreach ($project['taxonomy_vocabulary_3'] as $category) {

          $category_values[] = [

            'tid' => $category['id'],

            'pid' => $project['nid'],

          ];

        }

      }



      return [

        'nid' => $project['nid'],

        'title' => $project['title'],

        'author' => (string) @$project['author']['name'],

        'created' => $project['created'],

        'changed' => $project['changed'],

        'project_usage_total' => $project['project_usage_total'] ?? 0,

        'taxonomy_vocabulary_44' => $project['taxonomy_vocabulary_44']['id'],

        'taxonomy_vocabulary_46' => $project['taxonomy_vocabulary_46']['id'],

        'status' => $project['status'],

        'field_security_advisory_coverage' => $project['field_security_advisory_coverage'],

        'flag_project_star_user_count' => $project['flag_project_star_user_count'] ?? 0,

        'field_project_type' => $project['field_project_type'] ?? '',

        'project_data' => serialize($project),

      ];

    }, $projects);



    $used_nids = [];

    foreach ($values as $record) {

      if (in_array($record['nid'], $used_nids)) {

        continue;

      }

      $used_nids[] = $record['nid'];

      $temp_array = Json::decode(file_get_contents($module_path . '/fixtures/project_data.json'));

      array_push($temp_array, (object) $record);

      $all_records = Json::encode($temp_array);

      file_put_contents($module_path . '/fixtures/project_data.json', $all_records);

    }



    $used_primary = [];

    foreach ($category_values as $record) {

      if (in_array($record['tid'] . $record['pid'], $used_primary)) {

        continue;

      }

      $used_primary[] = $record['tid'] . $record['pid'];

      $temp_array = Json::decode(file_get_contents($module_path . '/fixtures/categories.json'));

      array_push($temp_array, (object) $record);

      $all_categories = Json::encode($temp_array);

      file_put_contents($module_path . '/fixtures/categories.json', $all_categories);

    }

    $final_message = 'Added ' . count($sandbox['projects']) . ' projects';

    return t('The batch_example demonstration update did what it was supposed to do: @message', ['@message' => $final_message]);

  }

  else {

    $last = $sandbox['projects'][array_key_last($sandbox['projects'])];

    return 'Page: ' . $sandbox['current_page'] . ' | Projects added:' . count($sandbox['projects']) . ' | ' . $last['changed'] . ' | ' . $sandbox['#finished'] * 100 . '%';

  }

function project_browser_update_9005() {

  $connection = Database::getConnection();

  $connection->truncate('project_browser_projects')->execute();

  $connection->truncate('project_browser_categories')->execute();

  _project_browser_populate_from_fixture();

}

src/Commands/UpdateFixtureCommands.php

+304 −0
Original line number Diff line number Diff line
@@ -2,6 +2,11 @@ 


namespace Drupal\project_browser\Commands;



use Composer\Semver\Semver;

use Drupal\Component\Serialization\Json;

use Drupal\Component\Utility\Unicode;

use Drupal\Component\Utility\Xss;

use Drupal\Core\Database\Database;

use Drupal\Core\Logger\LoggerChannelFactoryInterface;

use Drupal\Core\StringTranslation\StringTranslationTrait;

use Drupal\project_browser\EnabledSourceHandler;
@@ -83,4 +88,303 @@ class UpdateFixtureCommands extends DrushCommands { 
    $this->logger()->notice($this->t('Completed.'));

  }



  /**

   * Generate new fixtures.

   *

   * @command update:generate-fixture

   * @aliases pb-fixture

   *

   * @usage update:generate-fixture

   */

  public function generateFixture() {

    $this->logger()->notice($this->t('Begin Fixture Generation'));

    $sandbox = [];



    while (empty($sandbox) || $sandbox['#finished'] !== TRUE) {

      $progress_message = $this->hackyFixtureMaker($sandbox);



      $this->logger()->notice($progress_message);



      if ($sandbox['#finished'] === TRUE || $progress_message === 'Fixture generation complete') {

        $this->populateFromFixture();

        $this->logger()->notice($this->t('Writing fixture to database'));

        break;

      }

    }

    $this->logger()->notice($this->t('Updating project_browser.install'));

    $module_path = \Drupal::service('module_handler')->getModule('project_browser')->getPath();

    $install_file_contents = file_get_contents($module_path . '/project_browser.install');

    preg_match_all('/project_browser_update_(\d+)\(\)/', $install_file_contents, $matches);

    $update_hooks = $matches[1];

    sort($update_hooks);

    $new_hook_id = end($update_hooks) + 1;

    $update_hook = <<<'UPDATE'



function project_browser_update_{$new_hook_id}() {

  $connection = Database::getConnection();

  $connection->truncate('project_browser_projects')->execute();

  $connection->truncate('project_browser_categories')->execute();

  _project_browser_populate_from_fixture();

}



UPDATE;

    // Add update hook to the install file.

    file_put_contents($module_path . '/project_browser.install', $install_file_contents . str_replace('{$new_hook_id}', $new_hook_id, $update_hook));

  }



  /**

   * Batch operation for creating fixtures.

   *

   * @param array $sandbox

   *   The batch sandbox.

   *

   * @return string

   *   Progress messages.

   */

  private function hackyFixtureMaker(&$sandbox) {

    // Initialize some sandbox values on first iteration.

    if (!isset($sandbox['progress'])) {

      // The count of nodes visited so far.

      $sandbox['progress'] = 0;

      // Total nodes that must be visited.

      $sandbox['max'] = 7500;

      // A place to store messages during the run.

      $sandbox['messages'] = [];

      // Last node read via the query.

      $sandbox['current_page'] = 0;



      $sandbox['projects'] = [];

    }



    $current_source = \Drupal::service('project_browser.enabled_source')->getCurrentSource();

    if ($current_source && $current_source->getPluginId() === 'drupalorg_mockapi') {

      $query = [

        'page' => $sandbox['current_page'],

        'field_project_type' => 'full',

        'limit' => 50,

        'field_project_has_releases' => 1,

        'field_project_has_issue_queue' => 1,

        'type' => 'project_module',

        'status' => 1,

        'sort' => 'changed',

        'direction' => 'DESC',

      ];

      $eariest_possible_timestamp_reached = NULL;

      $drupal_org_response = $current_source->getProjectsFromSource($query);

      $returned_projects = $drupal_org_response['list'];



      if ($returned_projects) {

        foreach ($returned_projects as &$project) {

          $project['project_usage_total'] = 0;

          if (array_key_exists('project_usage', $project)) {

            foreach ($project['project_usage'] as $usage) {

              $project['project_usage_total'] += $usage;

            }

          }



          if (empty($project['body']['value'])) {

            $project['body']['value'] = '';

          }

          if (empty($project['body']['summary'])) {

            $project['body']['summary'] = $project['body']['value'];

          }

          $project['body']['summary'] = Xss::filter(strip_tags($project['body']['summary']));

          $project['body']['summary'] = Unicode::truncate($project['body']['summary'], 200, TRUE, TRUE);



          // Once we hit projects that haven't been updated since March 13, 2020, we

          // know they aren't compatible because it is before

          // https://www.drupal.org/node/3119415.

          if ($project['changed'] < 1583985600) {

            $eariest_possible_timestamp_reached = TRUE;

          }

        }

        $sandbox['current_page'] += 1;



        $projects_to_store = (array) $returned_projects;



        // Rewrite the projects array so each project has added release data and

        // unnecessary values are removed to conserve space.

        $projects_to_store = array_map(function ($a_project) use ($current_source) {

          $the_project = (array) $a_project;

          $releases = $current_source->getProjectReleasesFromSource($the_project['field_project_machine_name']);

          if (!empty($releases['releases'])) {

            $compatible_releases = array_filter($releases['releases'], function ($release) {

              if (!empty($release['core_compatibility'])) {

                try {

                  // Apparently there are multiple projects that have invalid

                  // version strings.

                  return Semver::satisfies(\Drupal::VERSION, $release['core_compatibility']);

                }

                catch (\Exception $exception) {

                  // Don't include releases with invalid compatibility strings.

                  return FALSE;

                }

              }

              return FALSE;

            });

            if (empty($compatible_releases)) {

              // Don't include projects without any compatible releases.

              return NULL;

            }

          }

          else {

            // Don't include projects without releases.

            return NULL;

          }



          // To keep filesize down, remove unnecessary items from release data.

          $the_project['releases'] = array_map(function ($release) {

            return [

              'version' => $release['version'],

              'status' => $release['status'],

              'date' => $release['date'] ?? NULL,

              'core_compatibility' => $release['core_compatibility'],

            ];

          }, $compatible_releases);

          $current_source->truncateProjectData($the_project);

          return $the_project;

        }, $projects_to_store);



        $projects_to_store = array_filter($projects_to_store);

        $sandbox['projects'] = array_merge($sandbox['projects'], $projects_to_store);

        $sandbox['progress'] += count($sandbox['projects']);

        $sandbox['#finished'] = count($sandbox['projects']) >= $sandbox['max'] || $eariest_possible_timestamp_reached ? TRUE : (count($sandbox['projects']) / $sandbox['max']);

      }

      else {

        $sandbox['#finished'] = TRUE;

      }

    }

    else {

      $sandbox['#finished'] = TRUE;

    }



    if ($sandbox['#finished'] === TRUE) {

      $module_path = \Drupal::service('module_handler')->getModule('project_browser')->getPath();

      file_put_contents($module_path . '/fixtures/project_data.json', '[]');

      file_put_contents($module_path . '/fixtures/categories.json', '[]');



      $projects = $sandbox['projects'];



      $category_values = [];



      // Map fixture values to DB columns.

      $values = array_map(function ($project) use (&$category_values) {

        if (!empty($project['taxonomy_vocabulary_3'])) {

          foreach ($project['taxonomy_vocabulary_3'] as $category) {

            $category_values[] = [

              'tid' => $category['id'],

              'pid' => $project['nid'],

            ];

          }

        }



        return [

          'nid' => $project['nid'],

          'title' => $project['title'],

          'author' => (string) @$project['author']['name'],

          'created' => $project['created'],

          'changed' => $project['changed'],

          'project_usage_total' => $project['project_usage_total'] ?? 0,

          'taxonomy_vocabulary_44' => $project['taxonomy_vocabulary_44']['id'],

          'taxonomy_vocabulary_46' => $project['taxonomy_vocabulary_46']['id'],

          'status' => $project['status'],

          'field_security_advisory_coverage' => $project['field_security_advisory_coverage'],

          'flag_project_star_user_count' => $project['flag_project_star_user_count'] ?? 0,

          'field_project_type' => $project['field_project_type'] ?? '',

          'project_data' => serialize($project),

        ];

      }, $projects);



      $used_nids = [];

      $this->logger()->notice('creating project_data.json file');

      $temp_array = Json::decode(file_get_contents($module_path . '/fixtures/project_data.json'));

      foreach ($values as $record) {

        if (in_array($record['nid'], $used_nids)) {

          continue;

        }

        $used_nids[] = $record['nid'];

        array_push($temp_array, (object) $record);

      }



      $all_records = Json::encode($temp_array);

      file_put_contents($module_path . '/fixtures/project_data.json', $all_records);



      $used_primary = [];

      $this->logger()->notice('creating categories.json file');

      $temp_array = Json::decode(file_get_contents($module_path . '/fixtures/categories.json'));



      foreach ($category_values as $record) {

        if (in_array($record['tid'] . $record['pid'], $used_primary)) {

          continue;

        }

        $used_primary[] = $record['tid'] . $record['pid'];

        array_push($temp_array, (object) $record);

        $all_categories = Json::encode($temp_array);

      }

      file_put_contents($module_path . '/fixtures/categories.json', $all_categories);



      return 'Fixture generation complete';

    }

    else {

      $last = $sandbox['projects'][array_key_last($sandbox['projects'])];

      return 'Page: ' . $sandbox['current_page'] . ' | Projects added:' . count($sandbox['projects']) . ' | ' . $last['changed'] . ' | ' . $sandbox['#finished'] * 100 . '%';

    }

  }



  /**

   * Updates database from fixtures.

   */

  private function populateFromFixture() {

    $connection = Database::getConnection();

    $connection->truncate('project_browser_projects')->execute();

    $connection->truncate('project_browser_categories')->execute();

    $module_path = \Drupal::service('module_handler')->getModule('project_browser')->getPath();

    $most_recent_change = 0;



    $projects = Json::decode(file_get_contents($module_path . '/fixtures/project_data.json'));

    $projects_chunk = array_chunk($projects, 1000);

    foreach ($projects_chunk as $chunk_projects) {

      // Insert fixture data to the database.

      $query = $connection->insert('project_browser_projects')->fields([

        'nid',

        'title',

        'author',

        'created',

        'changed',

        'project_usage_total',

        'maintenance_status',

        'development_status',

        'status',

        'field_security_advisory_coverage',

        'flag_project_star_user_count',

        'field_project_type',

        'project_data',

      ]);

      foreach ($chunk_projects as $project) {

        if ($project['changed'] > $most_recent_change) {

          $most_recent_change = $project['changed'];

        }

        // Map from fixture format to the expected by the database.

        $project_data = unserialize($project['project_data']);

        $project['maintenance_status'] = $project_data['taxonomy_vocabulary_44']['id'];

        $project['development_status'] = $project_data['taxonomy_vocabulary_46']['id'];



        $query->values($project);

      }

      $query->execute();

    }



    $categories = Json::decode(file_get_contents($module_path . '/fixtures/categories.json'));

    $category_query = $connection->insert('project_browser_categories')->fields([

      'tid',

      'pid',

    ]);

    foreach ($categories as $category) {

      $category_query->values((array) $category);

    }

    $category_query->execute();



    \Drupal::state()->set('project_browser.last_imported', $most_recent_change);

  }



}