Issue #2640516 by jcandan, m.lebedev, berenddeboer, scott_euser, segi, Ruslan...

Issue #2640516 by jcandan, m.lebedev, berenddeboer, scott_euser, segi, Ruslan Piskarov, weekbeforenext, ronaldmulero, DuaelFr, Rob230, quadbyte, badrange, drclaw, sdstyles, szloredan, pixlkat, heddn, Grayside, robin.ingelbrecht, auth, monymirza, KevinVanRansbeeck, sleopold, mortona2k, prudloff, hmdnawaz, Peter Törnstrand, longwave, googletorp: Support paging through multiple requests

migrate_example/tests/src/Kernel/MigrateExampleTest.php

@@ -46,6 +46,8 @@ final class MigrateExampleTest extends MigrateDrupalTestBase {
     \Drupal::service('module_installer')->install(['migrate_example_setup']);
     $this->installConfig(['migrate_example_setup']);
 
+    $this->startCollectingMessages();
+
     // Execute "beer" migrations from 'migrate_example' module.
     $this->executeMigration('beer_user');
     $this->executeMigrations([

src/DataFetcherPluginBase.php

@@ -24,4 +24,11 @@ abstract class DataFetcherPluginBase extends PluginBase implements DataFetcherPl
     return new static($configuration, $plugin_id, $plugin_definition);
   }
 
+  /**
+   * {@inheritdoc}
+   */
+  public function getNextUrls(string $url): array {
+    return [];
+  }
+
 }

src/DataFetcherPluginInterface.php

@@ -51,4 +51,17 @@ interface DataFetcherPluginInterface {
    */
   public function getResponse(string $url): ResponseInterface;
 
+  /**
+   * Collect next urls from the metadata of a paged response.
+   *
+   * Examples of this include HTTP headers and file naming conventions.
+   *
+   * @param string $url
+   *   URL of the resource to check for pager metadata.
+   *
+   * @return array
+   *   Array of URIs.
+   */
+  public function getNextUrls(string $url): array;
+
 }

src/DataParserPluginBase.php

@@ -151,6 +151,9 @@ abstract class DataParserPluginBase extends PluginBase implements DataParserPlug
       }
 
       if ($this->openSourceUrl($this->urls[$this->activeUrl])) {
+        if (!empty($this->configuration['pager'])) {
+          $this->addNextUrls($this->activeUrl);
+        }
         // We have a valid source.
         return TRUE;
       }
@@ -159,6 +162,36 @@ abstract class DataParserPluginBase extends PluginBase implements DataParserPlug
     return FALSE;
   }
 
+  /**
+   * Add next page of source data following the active URL.
+   *
+   * @param int $activeUrl
+   *   The index within the source URL array to insert the next URL resource.
+   *   This is parameterized to enable custom plugins to control the ordering of
+   *   next URLs injected into the source URL backlog.
+   */
+  protected function addNextUrls(int $activeUrl = 0): void {
+    $next_urls = $this->getNextUrls($this->urls[$this->activeUrl]);
+
+    if (!empty($next_urls)) {
+      array_splice($this->urls, $activeUrl + 1, 0, $next_urls);
+      $this->urls = array_values(array_unique($this->urls));
+    }
+  }
+
+  /**
+   * Collected the next urls from a paged response.
+   *
+   * @param string $url
+   *   URL of the currently active source.
+   *
+   * @return array
+   *   Array of URLs representing next paged resources.
+   */
+  protected function getNextUrls(string $url): array {
+    return $this->getDataFetcherPlugin()->getNextUrls($url);
+  }
+
   /**
    * {@inheritdoc}
    */

src/Plugin/migrate_plus/data_fetcher/Http.php

@@ -119,4 +119,25 @@ class Http extends DataFetcherPluginBase implements ContainerFactoryPluginInterf
     return (string) $this->getResponse($url)->getBody();
   }
 
+  /**
+   * {@inheritdoc}
+   */
+  public function getNextUrls(string $url): array {
+    $next_urls = [];
+
+    $headers = $this->getResponse($url)->getHeader('Link');
+    if (!empty($headers)) {
+      $headers = explode(',', $headers[0]);
+      foreach ($headers as $header) {
+        $matches = [];
+        preg_match('/^<(.*)>; rel="next"$/', trim($header), $matches);
+        if (!empty($matches) && !empty($matches[1])) {
+          $next_urls[] = $matches[1];
+        }
+      }
+    }
+
+    return array_merge(parent::getNextUrls($url), $next_urls);
+  }
+
 }

src/Plugin/migrate_plus/data_parser/Json.php

@@ -4,7 +4,10 @@ declare(strict_types = 1);
 
 namespace Drupal\migrate_plus\Plugin\migrate_plus\data_parser;
 
+use Drupal\Component\Utility\UrlHelper;
+use Drupal\Core\Url;
 use Drupal\Core\Plugin\ContainerFactoryPluginInterface;
+use Drupal\migrate\MigrateException;
 use Drupal\migrate_plus\DataParserPluginBase;
 
 /**
@@ -22,40 +25,69 @@ class Json extends DataParserPluginBase implements ContainerFactoryPluginInterfa
    */
   protected ?\ArrayIterator $iterator = NULL;
 
+  /**
+   * The currently saved source url (as a string).
+   *
+   * @var string
+   */
+  protected $currentUrl;
+
+  /**
+   * The active url's source data.
+   *
+   * @var array
+   */
+  protected $sourceData;
+
   /**
    * Retrieves the JSON data and returns it as an array.
    *
    * @param string $url
    *   URL of a JSON feed.
+   * @param string|int $item_selector
+   *   Selector within the data content at which useful data is found.
    *
    * @throws \GuzzleHttp\Exception\RequestException
    */
-  protected function getSourceData(string $url): array {
-    $response = $this->getDataFetcherPlugin()->getResponseContent($url);
+  protected function getSourceData(string $url, string|int $item_selector) {
+    // Use cached source data if this is the first request or URL is same as the
+    // last time we made the request.
+    if ($this->currentUrl != $url || !$this->sourceData) {
+      $response = $this->getDataFetcherPlugin()->getResponseContent($url);
 
-    // Convert objects to associative arrays.
-    $source_data = json_decode($response, TRUE, 512, JSON_THROW_ON_ERROR);
+      // Convert objects to associative arrays.
+      $this->sourceData = json_decode($response, TRUE);
 
-    // If json_decode() has returned NULL, it might be that the data isn't
-    // valid utf8 - see http://php.net/manual/en/function.json-decode.php#86997.
-    if (is_null($source_data)) {
+      // If json_decode() has returned NULL, it might be that the data isn't
+      // valid utf8 - see http://php.net/manual/en/function.json-decode.php#86997.
+      if (!$this->sourceData) {
       $utf8response = mb_convert_encoding($response, 'UTF-8');
-      $source_data = json_decode($utf8response, TRUE, 512, JSON_THROW_ON_ERROR);
+        $this->sourceData = json_decode($utf8response, TRUE);
+      }
+      $this->currentUrl = $url;
     }
 
     // Backwards-compatibility for depth selection.
-    if (is_int($this->itemSelector)) {
-      return $this->selectByDepth($source_data);
+    if (is_numeric($this->itemSelector)) {
+      return $this->selectByDepth($this->sourceData, (string) $item_selector);
+    }
+
+    // If the item_selector is an empty string, return all.
+    if ($item_selector == '') {
+      return $this->sourceData;
     }
 
     // Otherwise, we're using xpath-like selectors.
-    $selectors = explode('/', trim((string) $this->itemSelector, '/'));
+    $selectors = explode('/', trim($item_selector, '/'));
+    $return = $this->sourceData;
     foreach ($selectors as $selector) {
-      if (is_array($source_data) && array_key_exists($selector, $source_data)) {
-        $source_data = $source_data[$selector];
+      // If the item_selector is missing, return an empty array.
+      if (!isset($return[$selector])) {
+        return [];
       }
+      $return = $return[$selector];
     }
-    return $source_data;
+    return $return;
   }
 
   /**
@@ -66,7 +98,7 @@ class Json extends DataParserPluginBase implements ContainerFactoryPluginInterfa
    *
    *   Selected items at the requested depth of the JSON feed.
    */
-  protected function selectByDepth(array $raw_data): array {
+  protected function selectByDepth(array $raw_data, string $item_selector): array {
     // Return the results in a recursive iterator that can traverse
     // multidimensional arrays.
     $iterator = new \RecursiveIteratorIterator(
@@ -76,7 +108,7 @@ class Json extends DataParserPluginBase implements ContainerFactoryPluginInterfa
     // Backwards-compatibility - an integer item_selector is interpreted as a
     // depth. When there is an array of items at the expected depth, pull that
     // array out as a distinct item.
-    $identifierDepth = $this->itemSelector;
+    $identifierDepth = $item_selector;
     $iterator->rewind();
     while ($iterator->valid()) {
       $item = $iterator->current();
@@ -93,7 +125,11 @@ class Json extends DataParserPluginBase implements ContainerFactoryPluginInterfa
    */
   protected function openSourceUrl(string $url): bool {
     // (Re)open the provided URL.
-    $source_data = $this->getSourceData($url);
+    $source_data = $this->getSourceData($url, $this->itemSelector);
+    // Ensure there is source data at the current url.
+    if (is_null($source_data)) {
+      return FALSE;
+    }
     $this->iterator = new \ArrayIterator($source_data);
     return TRUE;
   }
@@ -103,7 +139,7 @@ class Json extends DataParserPluginBase implements ContainerFactoryPluginInterfa
    */
   protected function fetchNextRow(): void {
     $current = $this->iterator->current();
-    if ($current) {
+    if (is_array($current)) {
       foreach ($this->fieldSelectors() as $field_name => $selector) {
         $field_data = $current;
         $field_selectors = explode('/', trim((string) $selector, '/'));
@@ -124,4 +160,167 @@ class Json extends DataParserPluginBase implements ContainerFactoryPluginInterfa
     }
   }
 
+  /**
+   * {@inheritdoc}
+   */
+  protected function getNextUrls(string $url): array {
+    $next_urls = [];
+
+    // If a pager selector is provided, get the data from the source.
+    $selector_data = NULL;
+    if (!empty($this->configuration['pager']['selector'])) {
+      $selector_data = $this->getSourceData($url, $this->configuration['pager']['selector']);
+    }
+
+    // Logic for each type of pager.
+    switch ($this->configuration['pager']['type']) {
+      case 'urls':
+        if (NULL !== $selector_data) {
+          if (is_array($selector_data)) {
+            $next_urls = $selector_data;
+          }
+          elseif (filter_var($selector_data, FILTER_VALIDATE_URL)) {
+            $next_urls[] = $selector_data;
+          }
+        }
+        break;
+
+      case 'cursor':
+        if (NULL !== $selector_data && is_scalar($selector_data)) {
+          // Just use 'cursor' as a default parameter key if not provided.
+          $key = !empty($this->configuration['pager']['key']) ? $this->configuration['pager']['key'] : 'cursor';
+          // Parse the url and replace the cursor param value and rebuild the url.
+          $path = UrlHelper::parse($url);
+          $path['query'][$key] = $selector_data;
+          $next_urls[] = Url::fromUri($path['path'], [
+            'query' => $path['query'],
+            'fragment' => $path['fragment'],
+          ])->toString();
+        }
+        break;
+
+      case 'page':
+        if (NULL !== $selector_data && is_scalar($selector_data)) {
+          // Just use 'page' as a default parameter key if not provided.
+          $key = !empty($this->configuration['pager']['key']) ? $this->configuration['pager']['key'] : 'page';
+          // Define the max page to generate.
+          $max = $selector_data + 1;
+          if (!empty($this->configuration['pager']['selector_max'])) {
+            $max = $this->getSourceData($url, $this->configuration['pager']['selector_max']);
+          }
+
+          // Parse the url and replace the page param value and rebuild the url.
+          $path = UrlHelper::parse($url);
+          for ($page = $selector_data + 1; $page < $max; ++$page) {
+            $path['query'][$key] = $page;
+            $next_urls[] = Url::fromUri($path['path'], [
+              'query' => $path['query'],
+              'fragment' => $path['fragment'],
+            ])->toString();
+          }
+        }
+        break;
+
+      case 'paginator':
+        // The first pass uses the endpoint's default size.
+        // @todo Handle first URL set page size on first pass.
+        if (!isset($this->configuration['pager']['default_num_items'])) {
+          throw new MigrateException('Pager "default_num_items" must be configured.');
+        }
+        $num_items = $this->configuration['pager']['default_num_items'];
+
+        // Use 'page' as a default page parameter key if not provided.
+        $page_key = !empty($this->configuration['pager']['page_key']) ? $this->configuration['pager']['page_key'] : 'page';
+
+        // Set default paginator type.
+        $paginator_type_options = ['page_number', 'starting_item'];
+        $paginator_type = $paginator_type_options[0];
+        // Check configured paginator type.
+        if (!empty($this->configuration['pager']['paginator_type'])) {
+          if (!in_array($this->configuration['pager']['paginator_type'], $paginator_type_options)) {
+            // Not set to one of the two available options.
+            throw new MigrateException(
+              'Pager "paginator_type" must be configured as either "page_number" or "starting_item" ("page_number" is default).'
+            );
+          }
+          $paginator_type = $this->configuration['pager']['paginator_type'];
+        }
+
+        // Use 'pagesize' as a default page parameter key if not provided.
+        $size_key = !empty($this->configuration['pager']['size_key']) ? $this->configuration['pager']['size_key'] : 'pagesize';
+
+        // Parse the url.
+        $path = UrlHelper::parse($url);
+
+        $curr_page = !empty($path['query'][$page_key]) ? $path['query'][$page_key] : 0;
+
+        // @todo Use core's QueryBase and pager.
+        // @see contrib module external_entities \Entity\Query\External\Query.php for example.
+        $next_start = $curr_page + $num_items;
+        $next_end = $num_items;
+        // Use "page_number" when the pager uses page numbers to determine
+        // the item to start at, use "starting_item" when the pager uses the
+        // item number to start at.
+        if ($paginator_type === 'page_number') {
+          $next_start = $curr_page + 1;
+        }
+
+        // Replace the paginator param value.
+        $path['query'][$page_key] = $next_start;
+        // Replace the size param value.
+        $path['query'][$size_key] = $next_end;
+
+        // If we have a selector that tells us the number of rows returned in
+        // the current request, use that to decide if we should add the next
+        // url to the array.
+        if (NULL !== $selector_data) {
+          if (is_scalar($selector_data)) {
+            // If we have a numeric number of rows and the current page is still
+            // a full page (i.e. the number of items, $selector_data, in this
+            // page equals the number of items configured, $num_items), advance
+            // to the next page.
+            if ($selector_data == $num_items) {
+              $next_urls[] = Url::fromUri($path['path'], [
+                'query' => $path['query'],
+                'fragment' => $path['fragment'],
+              ])->toString();
+            }
+          }
+          else {
+            // If we have an array of rows
+            if (count($selector_data) > 0) {
+              $next_urls[] = Url::fromUri($path['path'], [
+                'query' => $path['query'],
+                'fragment' => $path['fragment'],
+              ])->toString();
+            }
+          }
+        }
+        else {
+          // Rebuild the url.
+          $next_urls[] = Url::fromUri($path['path'], [
+            'query' => $path['query'],
+            'fragment' => $path['fragment'],
+          ])->toString();
+
+          // Service may return 404 for last page, ensure next_urls are valid.
+          foreach ($next_urls as $key => $next_url) {
+            try {
+              $response = $this->getDataFetcherPlugin()->getResponse($next_url);
+              if ($response->getStatusCode() !== 200) {
+                unset($next_urls[$key]);
+              }
+            }
+            catch (\Exception $e) {
+              unset($next_urls[$key]);
+            }
+          }
+        }
+        break;
+    }
+
+    return array_merge(parent::getNextUrls($url), $next_urls);
+  }
+
+
 }

tests/modules/migrate_plus_http_test/migrate_plus_http_test.info.yml

+type: module
+name: Migrate Plus Http Test
+description: 'Test module to test Migrate Plus Http data_source.'
+package: Testing
+core_version_requirement: '>=9.1'
+dependencies:
+  - drupal:migrate
+  - migrate_plus:migrate_plus

tests/modules/migrate_plus_http_test/migrate_plus_http_test.routing.yml

+
+migrate_plus_http_test.json_first:
+  path: '/migrate-plus-http-test/json-first'
+  defaults:
+    _controller: '\Drupal\migrate_plus_http_test\Controller\HttpResponses::content'
+    _title: 'JSON First'
+  requirements:
+    _permission: 'access content'
+
+migrate_plus_http_test.json_second:
+  path: '/migrate-plus-http-test/json-second'
+  defaults:
+    _controller: '\Drupal\migrate_plus_http_test\Controller\HttpResponses::second'
+    _title: 'JSON Second'
+  requirements:
+    _permission: 'access content'
+
+migrate_plus_http_test.json_third:
+  path: '/migrate-plus-http-test/json-third'
+  defaults:
+    _controller: '\Drupal\migrate_plus_http_test\Controller\HttpResponses::third'
+    _title: 'JSON Third'
+  requirements:
+    _permission: 'access content'
+
+migrate_plus_http_test.json_fourth:
+  path: '/migrate-plus-http-test/json-fourth'
+  defaults:
+    _controller: '\Drupal\migrate_plus_http_test\Controller\HttpResponses::fourth'
+    _title: 'JSON Fourth'
+  requirements:
+    _permission: 'access content'
+
+migrate_plus_http_test.json_fifth:
+  path: '/migrate-plus-http-test/json-fifth'
+  defaults:
+    _controller: '\Drupal\migrate_plus_http_test\Controller\HttpResponses::fifth'
+    _title: 'JSON Fifth'
+  requirements:
+    _permission: 'access content'

tests/modules/migrate_plus_http_test/src/Controller/HttpResponses.php

+<?php
+
+namespace Drupal\migrate_plus_http_test\Controller;
+
+use Drupal\Core\Controller\ControllerBase;
+use Drupal\Core\Url;
+use Symfony\Component\HttpFoundation\JsonResponse;
+use Symfony\Component\HttpFoundation\Request;
+use Symfony\Component\HttpKernel\Exception\NotFoundHttpException;
+
+/**
+ * An example controller.
+ */
+class HttpResponses extends ControllerBase {
+
+  /**
+   * Returns a render-able array for a test page.
+   */
+  public function content() {
+
+    $url = Url::fromRoute('migrate_plus_http_test.json_second', [], ['absolute' => TRUE]);
+
+    $data = [
+      "status" => "ok",
+      'nextUrl' => $url->toString(),
+      "data" => $this->generateRowsData(3),
+    ];
+
+    return new JsonResponse($data);
+  }
+
+  public function second(){
+    $data = [
+      "status" => "ok",
+      'nextUrl' => NULL,
+      "data" => $this->generateRowsData(3,4),
+    ];
+
+    return new JsonResponse($data);
+  }
+
+  public function third(Request $request){
+    $page = $request->query->get('page') ?? 0;
+    $first = $page*3 +1;
+
+    $data = [
+      "status" => "ok",
+      'currentPage' => $page,
+      'numPages' => 2,
+      'nextPage' => $page+1,
+      "data" => $this->generateRowsData(3,$first),
+    ];
+
+    if($page == 2){
+      unset($data['nextPage']);
+    }
+
+    return new JsonResponse($data);
+  }
+
+  public function fourth( Request $request ) {
+    $page = $request->query->get('cursor') ?? 0;
+    $first = $page*3 +1;
+
+    $data = [
+      "status" => "ok",
+      'nextPage' => $page+1,
+      "data" => $this->generateRowsData(3,$first),
+    ];
+
+    if($page == 2){
+      unset($data['nextPage']);
+    }
+
+    return new JsonResponse($data);
+  }
+
+  public function fifth( Request $request ) {
+    $page = $request->query->get('page') ?? 0;
+    $first = $page*3 +1;
+
+    $data = [
+      "status" => "ok",
+      'numItems' => 3,
+      "data" => $this->generateRowsData(3,$first),
+    ];
+
+    if($page == 2){
+      $data['numItems'] = 0;
+      $data["data"] = [];
+    }
+
+    if($page == 3){
+      throw new NotFoundHttpException();
+    }
+
+    return new JsonResponse($data);
+  }
+
+  protected function generateRowsData(int $length, int $first = 1){
+    $data = [];
+    $last = $first + $length;
+    for($i = $first; $i < $last; $i++){
+      $data[] = [
+        "id" => $i,
+      ];
+    }
+
+    return $data;
+  }
+
+}

tests/src/Functional/HttpTest.php

+<?php
+
+declare(strict_types = 1);
+
+namespace Drupal\Tests\migrate_plus\Functional;
+
+use Drupal\Core\Url;
+use Drupal\Tests\BrowserTestBase;
+use Drupal\migrate_plus\DataParserPluginManager;
+
+/**
+ * Tests the http data_fetcher from the url plugin.
+ *
+ * @group migrate_plus
+ */
+final class HttpTest extends BrowserTestBase {
+
+  /**
+   * {@inheritdoc}
+   */
+  protected static $modules = [
+    'node',
+    'migrate_plus',
+    'migrate_plus_http_test',
+  ];
+
+  /**
+   * A user with permission to administer site configuration.
+   *
+   * @var \Drupal\user\UserInterface
+   */
+  protected $user;
+
+  /**
+   * {@inheritdoc}
+   */
+  protected $defaultTheme = 'stark';
+
+  /**
+   * The plugin manager.
+   */
+  protected ?DataParserPluginManager $pluginManager = NULL;
+
+  /**
+   * {@inheritdoc}
+   */
+  protected function setUp(): void {
+    parent::setUp();
+
+    $this->pluginManager = $this->container->get('plugin.manager.migrate_plus.data_parser');
+
+    $this->user = $this->drupalCreateUser(['administer site configuration', 'access content']);
+    $this->drupalLogin($this->user);
+  }
+
+  public function testUrlPagerMissingSelector(): void {
+
+    $url = Url::fromRoute('migrate_plus_http_test.json_first', [], ['absolute' => TRUE]);
+
+    $conf = $this->getBaseConfiguration();
+    $conf['urls'][] = $url->toString();
+    $conf['pager'] = [
+      'type' => 'urls'
+    ];
+
+    $result = $this->getPluginResults($conf);
+
+    $expected = [
+      [ "Id" => 1 ],
+      [ "Id" => 2 ],
+      [ "Id" => 3 ],
+    ];
+
+    $this->assertEquals($expected, $result);
+  }
+
+  public function testUrlsPagerMissingSelectorAttribute(){
+
+    $url = Url::fromRoute('migrate_plus_http_test.json_first', [], ['absolute' => TRUE]);
+
+    $conf = $this->getBaseConfiguration();
+    $conf['urls'][] = $url->toString();
+    $conf['pager'] = [
+      'type' => 'urls',
+      'selector' => 'thisIsMissing'
+    ];
+
+    $result = $this->getPluginResults($conf);
+
+    $expected = [
+      [ "Id" => 1 ],
+      [ "Id" => 2 ],
+      [ "Id" => 3 ],
+    ];
+
+    $this->assertEquals($expected, $result);
+
+  }
+
+  public function testUrlsPager(): void {
+
+    $url = Url::fromRoute('migrate_plus_http_test.json_first', [], ['absolute' => TRUE]);
+
+    $conf = $this->getBaseConfiguration();
+    $conf['urls'][] = $url->toString();
+    $conf['pager'] = [
+      'type' => 'urls',
+      'selector' => 'nextUrl'
+    ];
+
+    $result = $this->getPluginResults($conf);
+
+    $expected = [
+      [ "Id" => 1 ],
+      [ "Id" => 2 ],
+      [ "Id" => 3 ],
+      [ "Id" => 4 ],
+      [ "Id" => 5 ],
+      [ "Id" => 6 ],
+    ];
+
+    $this->assertEquals($expected, $result);
+  }
+
+  public function testCursorPager(){
+
+    $url = Url::fromRoute('migrate_plus_http_test.json_third', [], ['absolute' => TRUE]);
+
+    $conf = $this->getBaseConfiguration();
+    $conf['urls'][] = $url->toString();
+    $conf['pager'] = [
+      'type' => 'cursor',
+      'selector' => 'nextPage',
+      'key' => 'page'
+    ];
+
+    $expected = [
+      [ "Id" => 1 ],
+      [ "Id" => 2 ],
+      [ "Id" => 3 ],
+      [ "Id" => 4 ],
+      [ "Id" => 5 ],
+      [ "Id" => 6 ],
+      [ "Id" => 7 ],
+      [ "Id" => 8 ],
+      [ "Id" => 9 ],
+    ];
+
+    $result = $this->getPluginResults($conf);
+    $this->assertEquals( $expected, $result);
+  }
+
+  public function testCursorPagerMissingKey(){
+    $url = Url::fromRoute('migrate_plus_http_test.json_fourth', [], ['absolute' => TRUE]);
+
+    $conf = $this->getBaseConfiguration();
+    $conf['urls'][] = $url->toString();
+    $conf['pager'] = [
+      'type' => 'cursor',
+      'selector' => 'nextPage'
+    ];
+
+    $expected = [
+      [ "Id" => 1 ],
+      [ "Id" => 2 ],
+      [ "Id" => 3 ],
+      [ "Id" => 4 ],
+      [ "Id" => 5 ],
+      [ "Id" => 6 ],
+      [ "Id" => 7 ],
+      [ "Id" => 8 ],
+      [ "Id" => 9 ],
+    ];
+
+    $conf = $this->getBaseConfiguration();
+    $conf['urls'][] = $url->toString();
+    $conf['pager'] = [
+      'type' => 'cursor',
+      'selector' => 'nextPage'
+    ];
+
+    $result = $this->getPluginResults($conf);
+    $this->assertEquals( $expected, $result);
+  }
+
+  public function testCursorPagerMissingSelector(){
+
+    $url = Url::fromRoute('migrate_plus_http_test.json_third', [], ['absolute' => TRUE]);
+
+    $conf = $this->getBaseConfiguration();
+    $conf['urls'][] = $url->toString();
+    $conf['pager'] = [
+      'type' => 'cursor',
+    ];
+
+    $expected = [
+      [ "Id" => 1 ],
+      [ "Id" => 2 ],
+      [ "Id" => 3 ],
+    ];
+
+    $result = $this->getPluginResults($conf);
+    $this->assertEquals( $expected, $result);
+  }
+
+  public function testCursorPagerMissingSelectorAttribute(){
+
+    $url = Url::fromRoute('migrate_plus_http_test.json_third', [], ['absolute' => TRUE]);
+
+    $conf = $this->getBaseConfiguration();
+    $conf['urls'][] = $url->toString();
+    $conf['pager'] = [
+      'type' => 'cursor',
+      'selector' => 'missingAttribute'
+    ];
+
+    $expected = [
+      [ "Id" => 1 ],
+      [ "Id" => 2 ],
+      [ "Id" => 3 ],
+    ];
+
+    $result = $this->getPluginResults($conf);
+    $this->assertEquals( $expected, $result);
+  }
+
+  public function testPagePager(){
+
+    $url = Url::fromRoute('migrate_plus_http_test.json_third', [], ['absolute' => TRUE]);
+
+    $conf = $this->getBaseConfiguration();
+    $conf['urls'][] = $url->toString();
+    $conf['pager'] = [
+      'type' => 'page',
+      'selector' => 'currentPage',
+      'key' => 'page'
+    ];
+
+    $expected = [
+      [ "Id" => 1 ],
+      [ "Id" => 2 ],
+      [ "Id" => 3 ],
+    ];
+
+    $result = $this->getPluginResults($conf);
+    $this->assertEquals( $expected, $result);
+
+  }
+
+  public function testPagePagerSelectorMax(){
+
+    $url = Url::fromRoute('migrate_plus_http_test.json_third', [], ['absolute' => TRUE]);
+
+    $conf = $this->getBaseConfiguration();
+    $conf['urls'][] = $url->toString();
+    $conf['pager'] = [
+      'type' => 'page',
+      'selector' => 'currentPage',
+      'selector_max' => 'numPages',
+      'key' => 'page'
+    ];
+
+    $expected = [
+      [ "Id" => 1 ],
+      [ "Id" => 2 ],
+      [ "Id" => 3 ],
+      [ "Id" => 4 ],
+      [ "Id" => 5 ],
+      [ "Id" => 6 ],
+    ];
+
+    $result = $this->getPluginResults($conf);
+    $this->assertEquals( $expected, $result);
+
+  }
+
+  public function testPaginationPagerNumItemsSelector(){
+
+    $url = Url::fromRoute('migrate_plus_http_test.json_fifth', [], ['absolute' => TRUE]);
+
+    $conf = $this->getBaseConfiguration();
+    $conf['urls'][] = $url->toString();
+    $conf['pager'] = [
+      'type' => 'paginator',
+      'selector' => 'numItems',
+      'default_num_items' => 3,
+      'page_key' => 'page'
+    ];
+
+    $expected = [
+      [ "Id" => 1 ],
+      [ "Id" => 2 ],
+      [ "Id" => 3 ],
+      [ "Id" => 4 ],
+      [ "Id" => 5 ],
+      [ "Id" => 6 ],
+    ];
+
+    $result = $this->getPluginResults($conf);
+    $this->assertEquals( $expected, $result);
+
+  }
+
+  public function testPaginationPagerRowArraySelector(){
+
+    $url = Url::fromRoute('migrate_plus_http_test.json_fifth', [], ['absolute' => TRUE]);
+
+    $conf = $this->getBaseConfiguration();
+    $conf['urls'][] = $url->toString();
+    $conf['pager'] = [
+      'type' => 'paginator',
+      'selector' => 'data',
+      'default_num_items' => 3,
+      'page_key' => 'page'
+    ];
+
+    $expected = [
+      [ "Id" => 1 ],
+      [ "Id" => 2 ],
+      [ "Id" => 3 ],
+      [ "Id" => 4 ],
+      [ "Id" => 5 ],
+      [ "Id" => 6 ],
+    ];
+
+    $result = $this->getPluginResults($conf);
+    $this->assertEquals( $expected, $result);
+
+  }
+
+  public function testPaginationPagerNoSelector(){
+
+    $url = Url::fromRoute('migrate_plus_http_test.json_fifth', [], ['absolute' => TRUE]);
+
+    $conf = $this->getBaseConfiguration();
+    $conf['urls'][] = $url->toString();
+    $conf['pager'] = [
+      'type' => 'paginator',
+      'default_num_items' => 3,
+      'page_key' => 'page'
+    ];
+
+    $expected = [
+      [ "Id" => 1 ],
+      [ "Id" => 2 ],
+      [ "Id" => 3 ],
+      [ "Id" => 4 ],
+      [ "Id" => 5 ],
+      [ "Id" => 6 ],
+    ];
+
+    $result = $this->getPluginResults($conf);
+    $this->assertEquals( $expected, $result);
+  }
+
+  protected function getBaseConfiguration( ): array{
+    return [
+      'plugin' => 'url',
+      'data_fetcher_plugin' => 'http',
+      'data_parser_plugin' => 'json',
+      'pager' => [],
+      'urls' => [],
+      'ids' => [
+        'Id' => [
+          'type' => 'integer'
+        ]
+      ],
+      'fields' => [
+        [
+          'name' => 'Id',
+          'label' => 'ID',
+          'selector' => '/id'
+        ],
+      ],
+      'item_selector' => 'data',
+    ];
+  }
+
+  protected function getPluginResults( array $configuration ): array{
+    $json_parser = $this->pluginManager->createInstance('json', $configuration);
+    $data = [];
+    foreach ($json_parser as $item) {
+      $data[] = $item;
+    }
+    return $data;
+  }
+
+}

tests/src/Kernel/Plugin/migrate_plus/data_parser/JsonTest.php

@@ -202,16 +202,7 @@ final class JsonTest extends KernelTestBase {
       'item_selector not available' => [
         'item_selector' => '/data_unavailable',
         'fields' => $fields,
-        'expected' => [
-          [
-            'id' => '',
-            'title' => '',
-          ],
-          [
-            'id' => '',
-            'title' => '',
-          ],
-        ],
+        'expected' => [],
       ],
       'item_selector 2nd level' => [
         'item_selector' => '/data/0/items',