Add a processed-keycloak-users-json option to integrity check command

A local file that will be used instead of KC REST API to retrieve users data.

ℹ It may be easier to review with git diff --ignore-all-space.

Closes #3448936

drupalorg/drupalorg.drush.inc

@@ -164,8 +164,12 @@ function drupalorg_drush_command() {
       'arguments' => [
         'differences-log-file' => 'Optional. File where the found differences information will be written to.',
       ],
+      'options' => [
+        'processed-keycloak-users-json' => 'Optional. Use a local file that contains Keycloak exported data for users, processed by migration script.',
+      ],
       'examples' => [
         'drush drupalorg-keycloak-integrity-check differences.log' => 'Find integrity check differences and write it to the passed file.',
+        'drush drupalorg-keycloak-integrity-check --processed-keycloak-users-json="/path/to/kc-user-data.json" differences.log' => 'Find integrity check differences and write it to the passed file, using kc-user-data.json as Keycloak user data source instead of its REST API.',
       ],
     ],
   ];
@@ -1961,91 +1965,134 @@ function drush_drupalorg_keycloak_integrity_check($log_filename = null) {
   ini_set('memory_limit', '-1');
 
   drush_log(dt('1. Start Keycloak users retrieval'), LogLevel::OK);
-  $integration = new KeycloakIntegration();
-  // More than 1k will take a lot more time on the KC side reaching time outs
-  // of 20s wich are already big; use a conservative value below that point.
-  $chunk_size = 900;
-  $retrieved_users = 0;
-  $keycloak_users_count = $integration->countUsers();
   $keycloak_users_data = [];
-  try {
-    do {
-      $keycloak_users_raw = $integration->getUsers([
-        'first' => $retrieved_users,
-        'max' => $chunk_size,
-      ]);
-      $keycloak_users = json_decode($keycloak_users_raw, TRUE);
-      unset($keycloak_users_raw);
+  $kc_user_data_filename = drush_get_option('processed-keycloak-users-json');
+  if (!empty($kc_user_data_filename)) {
+      $keycloak_users = json_decode(file_get_contents($kc_user_data_filename), TRUE);
+      $t1 = time();
+      $m1 = memory_get_usage();
+      drush_log(dt('(keycloak local 1) time: @time s || memory: @memory b', ['@time' => $t1 - $t0, '@memory' => $m1]), LogLevel::INFO);
+      if (is_null($keycloak_users)) {
+          drush_set_error('DRUPALORG_KEYCLOAK_INTEGRITY_KC_GET_USERS_FAIL', dt('Cannot find valid contents on passed user data file.'));
+          return FALSE;
+      }
+      $usernames_to_ignore = [
+        // Related to setup KC clients.
+        'service-account-drupalorg-drush',
+        'service-account-realm-management',
+        'service-account-migrator',
+      ];
       foreach ($keycloak_users as $keycloak_user) {
-        $keycloak_users_data[$keycloak_user['id']] = [
-          !empty($keycloak_user['attributes']['sub'][0]) ? $keycloak_user['attributes']['sub'][0] : '',
-          $keycloak_user['id'],
+        if (in_array($keycloak_user['username'], $usernames_to_ignore)) {
+          continue;
+        }
+        $keycloak_users_data[$keycloak_user['uuid']] = [
+          (string) $keycloak_user['sub'],
+          $keycloak_user['uuid'],
           $keycloak_user['username'],
           $keycloak_user['email'],
           $keycloak_user['enabled'],
-          $keycloak_user['createdTimestamp'],
-          !empty($keycloak_user['attributes']['zoneinfo'][0]) ? $keycloak_user['attributes']['zoneinfo'][0] : '',
-          !empty($keycloak_user['firstName']) ? $keycloak_user['firstName'] : '',
-          !empty($keycloak_user['lastName']) ? $keycloak_user['lastName'] : '',
+          $keycloak_user['created'],
+          (string) $keycloak_user['zoneinfo'],
+          (string) $keycloak_user['firstName'],
+          (string) $keycloak_user['lastName'],
           $keycloak_user['emailVerified'],
-          // Set as not confirmed by default.
-          false,
+          $keycloak_user['confirmed'],
           $keycloak_user['totp'],
-          !empty($keycloak_user['attributes']['picture'][0]) ? $keycloak_user['attributes']['picture'][0] : '',
+          (string) $keycloak_user['picture'],
         ];
-        $retrieved_users++;
-      }
-      unset($keycloak_users);
-    } while ($retrieved_users < $keycloak_users_count);
-  }
-  catch (Exception $exception) {
-    drush_log(dt('Cannot retrieve Keycloak users: @message.', [
-      '@message' => $exception->getMessage(),
-    ]), LogLevel::ERROR);
-    drush_set_error('DRUPALORG_KEYCLOAK_INTEGRITY_KC_GET_USERS_FAIL', dt('Cannot continue without data from Keycloak.'));
-    return FALSE;
-  }
-  $t1 = time();
-  $m1 = memory_get_usage();
-  drush_log(dt('(keycloak 1) time: @time s || memory: @memory b', ['@time' => $t1 - $t0, '@memory' => $m1]), LogLevel::INFO);
-
-  // There does not seem to be a way to get a given KC group total members
-  // count without walking the whole set, and it is not part of the data in the
-  // user representation obtained in the code above.
-  // For now, walk the set, making the maximum be the already known total set
-  // of users, and break the loop as soon as no results are returned.
-  $confirmed_group_uuid = variable_get('drupalorg_keycloak_trusted_group', '2141a3e3-dd22-48bc-9023-11233f577d61');
-  $retrieved_users_in_group = 0;
-  try {
-    do {
-      $keycloak_users_raw = $integration->getGroupMembers($confirmed_group_uuid, [
-        'first' => $retrieved_users_in_group,
-        'max' => $chunk_size,
-      ]);
-      $keycloak_users = json_decode($keycloak_users_raw, TRUE);
-      unset($keycloak_users_raw);
-      if (empty($keycloak_users)) {
-        // No more data to walk.
-        break;
-      }
-      foreach ($keycloak_users as $keycloak_user) {
-        // Override flag indicating the user is confirmed.
-        $keycloak_users_data[$keycloak_user['id']][10] = true;
-        $retrieved_users_in_group++;
       }
-      unset($keycloak_users);
-    } while ($retrieved_users_in_group < $keycloak_users_count);
+      $keycloak_users_count = count($keycloak_users_data);
+      $t2 = time();
+      $m2 = memory_get_usage();
+      drush_log(dt('(keycloak local 2) time: @time s || memory: @memory b', ['@time' => $t2 - $t1, '@memory' => $m2]), LogLevel::INFO);
   }
-  catch (Exception $exception) {
-    drush_log(dt('Cannot retrieve confirmed group data from Keycloak: @message.', [
-      '@message' => $exception->getMessage(),
-    ]), LogLevel::ERROR);
-    drush_set_error('DRUPALORG_KEYCLOAK_INTEGRITY_KC_GET_GROUP_USERS_FAIL', dt('Cannot continue without group data from Keycloak.'));
-    return FALSE;
+  else {
+    $integration = new KeycloakIntegration();
+    // More than 1k will take a lot more time on the KC side reaching time outs
+    // of 20s wich are already big; use a conservative value below that point.
+    $chunk_size = 900;
+    $retrieved_users = 0;
+    $keycloak_users_count = $integration->countUsers();
+    try {
+      do {
+        $keycloak_users_raw = $integration->getUsers([
+          'first' => $retrieved_users,
+          'max' => $chunk_size,
+        ]);
+        $keycloak_users = json_decode($keycloak_users_raw, TRUE);
+        unset($keycloak_users_raw);
+        foreach ($keycloak_users as $keycloak_user) {
+          $keycloak_users_data[$keycloak_user['id']] = [
+            !empty($keycloak_user['attributes']['sub'][0]) ? $keycloak_user['attributes']['sub'][0] : '',
+            $keycloak_user['id'],
+            $keycloak_user['username'],
+            $keycloak_user['email'],
+            $keycloak_user['enabled'],
+            $keycloak_user['createdTimestamp'],
+            !empty($keycloak_user['attributes']['zoneinfo'][0]) ? $keycloak_user['attributes']['zoneinfo'][0] : '',
+            !empty($keycloak_user['firstName']) ? $keycloak_user['firstName'] : '',
+            !empty($keycloak_user['lastName']) ? $keycloak_user['lastName'] : '',
+            $keycloak_user['emailVerified'],
+            // Set as not confirmed by default.
+            false,
+            $keycloak_user['totp'],
+            !empty($keycloak_user['attributes']['picture'][0]) ? $keycloak_user['attributes']['picture'][0] : '',
+          ];
+          $retrieved_users++;
+        }
+        unset($keycloak_users);
+      } while ($retrieved_users < $keycloak_users_count);
+    }
+    catch (Exception $exception) {
+      drush_log(dt('Cannot retrieve Keycloak users: @message.', [
+        '@message' => $exception->getMessage(),
+      ]), LogLevel::ERROR);
+      drush_set_error('DRUPALORG_KEYCLOAK_INTEGRITY_KC_GET_USERS_FAIL', dt('Cannot continue without data from Keycloak.'));
+      return FALSE;
+    }
+    $t1 = time();
+    $m1 = memory_get_usage();
+    drush_log(dt('(keycloak rest 1) time: @time s || memory: @memory b', ['@time' => $t1 - $t0, '@memory' => $m1]), LogLevel::INFO);
+
+    // There does not seem to be a way to get a given KC group total members
+    // count without walking the whole set, and it is not part of the data in the
+    // user representation obtained in the code above.
+    // For now, walk the set, making the maximum be the already known total set
+    // of users, and break the loop as soon as no results are returned.
+    $confirmed_group_uuid = variable_get('drupalorg_keycloak_trusted_group', '2141a3e3-dd22-48bc-9023-11233f577d61');
+    $retrieved_users_in_group = 0;
+    try {
+      do {
+        $keycloak_users_raw = $integration->getGroupMembers($confirmed_group_uuid, [
+          'first' => $retrieved_users_in_group,
+          'max' => $chunk_size,
+        ]);
+        $keycloak_users = json_decode($keycloak_users_raw, TRUE);
+        unset($keycloak_users_raw);
+        if (empty($keycloak_users)) {
+          // No more data to walk.
+          break;
+        }
+        foreach ($keycloak_users as $keycloak_user) {
+          // Override flag indicating the user is confirmed.
+          $keycloak_users_data[$keycloak_user['id']][10] = true;
+          $retrieved_users_in_group++;
+        }
+        unset($keycloak_users);
+      } while ($retrieved_users_in_group < $keycloak_users_count);
+    }
+    catch (Exception $exception) {
+      drush_log(dt('Cannot retrieve confirmed group data from Keycloak: @message.', [
+        '@message' => $exception->getMessage(),
+      ]), LogLevel::ERROR);
+      drush_set_error('DRUPALORG_KEYCLOAK_INTEGRITY_KC_GET_GROUP_USERS_FAIL', dt('Cannot continue without group data from Keycloak.'));
+      return FALSE;
+    }
+    $t2 = time();
+    $m2 = memory_get_usage();
+    drush_log(dt('(keycloak rest 2) time: @time s || memory: @memory b', ['@time' => $t2 - $t1, '@memory' => $m2]), LogLevel::INFO);
   }
-  $t2 = time();
-  $m2 = memory_get_usage();
-  drush_log(dt('(keycloak 2) time: @time s || memory: @memory b', ['@time' => $t2 - $t1, '@memory' => $m2]), LogLevel::INFO);
 
   drush_log(dt('2. Start Drupal users retrieval'), LogLevel::OK);
   $pre_auth_role = variable_get('drupalorg_crosssite_email_unverified_rid', 39);