Add exhaustive response option to paginated API's

package.json

@@ -82,6 +82,7 @@
     "p-map": "^3.0.0",
     "p-wait-for": "^3.1.0",
     "parallel-transform": "^1.1.0",
+    "parse-link-header": "^1.0.1",
     "pump": "^3.0.0",
     "qs": "^6.9.3",
     "rimraf": "^3.0.2",

src/index.test.js

@@ -298,6 +298,78 @@ test('Can parse text responses', async (t) => {
   t.true(scope.isDone())
 })
 
+test('Can retrieve multiple pages', async t => {
+  // Expected responses are arrays, since pagination makes sense in the context
+  // calls that return multiple items:
+  // https://docs.netlify.com/api/get-started/#pagination
+  const expectedResponsePages = [
+    [
+      { id: '1', content: 'page 1' },
+      { id: '2', content: 'page 1' }
+    ],
+    [
+      { id: '3', content: 'page 2' },
+      { id: '4', content: 'page 2' }
+    ],
+    [{ id: '5', content: 'page 3' }]
+  ]
+  const expectedResponse = expectedResponsePages.reduce((response, page) => response.concat(page), [])
+
+  const baseUrl = `${pathPrefix}/sites`
+  const baseUrlFull = `${origin}${baseUrl}`
+  const scope = nock(origin)
+    .persist()
+    .get(baseUrl)
+    .reply(200, expectedResponsePages[0], {
+      Link: `<${baseUrlFull}?page=3>; rel="last", <${baseUrlFull}?page=2>; rel="next"`
+    })
+    .get(`${baseUrl}?page=2`)
+    .reply(200, expectedResponsePages[1], {
+      Link: `<${baseUrlFull}?page=3>; rel="last", <${baseUrlFull}?page=3>; rel="next"`
+    })
+    .get(`${baseUrl}?page=3`)
+    .reply(200, expectedResponsePages[2], {
+      Link: `<${baseUrlFull}?page=3>; rel="last"`
+    })
+    .persist(false)
+
+  const client = getClient()
+  const response = await client.listSites({}, { exhaustive: true })
+
+  t.deepEqual(response, expectedResponse)
+  t.true(scope.isDone())
+})
+
+test('Retrieves the first page if exhaustive is not specified', async t => {
+  // The entire dataset, split into multiple pages.
+  const datasetPages = [
+    [
+      { id: '1', content: 'page 1' },
+      { id: '2', content: 'page 1' }
+    ],
+    [
+      { id: '3', content: 'page 2' },
+      { id: '4', content: 'page 2' }
+    ],
+    [{ id: '5', content: 'page 3' }]
+  ]
+  const expectedResponse = datasetPages[0]
+
+  const baseUrl = `${pathPrefix}/sites`
+  const baseUrlFull = `${origin}${baseUrl}`
+  const scope = nock(origin)
+    .get(baseUrl)
+    .reply(200, expectedResponse, {
+      Link: `<${baseUrlFull}?page=3>; rel="last", <${baseUrlFull}?page=2>; rel="next"`
+    })
+
+  const client = getClient()
+  const response = await client.listSites()
+
+  t.deepEqual(response, expectedResponse)
+  t.true(scope.isDone())
+})
+
 test('Handle error empty responses', async (t) => {
   const accountId = uuidv4()
   const status = 404

src/methods/index.js

@@ -1,5 +1,6 @@
 // Webpack will sometimes export default exports in different places
 const fetch = require('node-fetch').default || require('node-fetch')
+const parseLinkHeader = require('parse-link-header')
 
 const { getOperations } = require('../operations')
 
@@ -30,12 +31,58 @@ const getMethod = function (method, NetlifyApi) {
 }
 
 const callMethod = async function (method, NetlifyApi, params, opts) {
-  const requestParams = { ...NetlifyApi.globalParams, ...params }
+  const { exhaustive = false } = opts || {}
   const url = getUrl(method, NetlifyApi, requestParams)
-  const response = await makeRequestOrRetry({ url, method, NetlifyApi, requestParams, opts })
+  const { parsedResponse, headers } = await retrieveResponse({ url, method, NetlifyApi, requestParams, opts })
+  if (!exhaustive || !Array.isArray(parsedResponse)) {
+    // If the user did not enable the retrieval of all items, or
+    // if the response is a single object/item, then
+    // we can skip the pagination logic.
+    return parsedResponse
+  }
+
+  return await retrieveResponseForNextPages({ method, NetlifyApi, requestParams, opts, parsedResponse, headers })
+}
 
+const retrieveResponse = async function({ url, method, NetlifyApi, requestParams, opts }) {
+  const response = await makeRequestOrRetry({ url, method, NetlifyApi, requestParams, opts })
   const parsedResponse = await parseResponse(response)
-  return parsedResponse
+  return {
+    parsedResponse,
+    headers: response.headers
+  }
+}
+
+const retrieveResponseForNextPages = async function({
+  method,
+  NetlifyApi,
+  requestParams,
+  opts,
+  parsedResponse,
+  headers
+}) {
+  // Responses for each page are accumulated in a flattened manner.
+  let results = parsedResponse
+
+  // The API directly provides the link to the next page (if any)
+  // in the `Link` header.
+  let url = getNextPageUrl(headers)
+
+  // For paginated results, we retrieve all the pages for this
+  // method until we exhaust the entire dataset.
+  while (url) {
+    const { parsedResponse, headers } = await retrieveResponse({ url, method, NetlifyApi, requestParams, opts })
+    results = results.concat(parsedResponse)
+    url = getNextPageUrl(headers)
+  }
+
+  return results
+}
+
+const getNextPageUrl = function(headers) {
+  const linkHeader = headers.get('link') || ''
+  const { next = {} } = parseLinkHeader(linkHeader) || {}
+  return next.url
 }
 
 const getOpts = function ({ verb, parameters }, NetlifyApi, { body }, opts) {