From 49e7651dcfd947870c6b239616be3c2d82ec8662 Mon Sep 17 00:00:00 2001
From: Jay Vercellone <juanjov@gmail.com>
Date: Sat, 3 Oct 2020 20:02:21 -0700
Subject: [PATCH 1/2] Add exhaustive response option to paginated API's
---
package-lock.json | 8 +++++
package.json | 1 +
src/index.test.js | 72 ++++++++++++++++++++++++++++++++++++++++++++
src/methods/index.js | 52 ++++++++++++++++++++++++++++++--
4 files changed, 131 insertions(+), 2 deletions(-)
diff --git a/package-lock.json b/package-lock.json
index d653c32..b32941f 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -8251,6 +8251,14 @@
"json-parse-better-errors": "^1.0.1"
}
},
+ "parse-link-header": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/parse-link-header/-/parse-link-header-1.0.1.tgz",
+ "integrity": "sha1-vt/g0hGK64S+deewJUGeyKYRQKc=",
+ "requires": {
+ "xtend": "~4.0.1"
+ }
+ },
"parse-ms": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/parse-ms/-/parse-ms-2.1.0.tgz",
diff --git a/package.json b/package.json
index e96ccc1..d569064 100644
--- a/package.json
+++ b/package.json
@@ -72,6 +72,7 @@
"p-map": "^3.0.0",
"p-wait-for": "^3.1.0",
"parallel-transform": "^1.1.0",
+ "parse-link-header": "^1.0.1",
"pump": "^3.0.0",
"qs": "^6.9.3",
"rimraf": "^3.0.2",
diff --git a/src/index.test.js b/src/index.test.js
index 062b8a2..9ac5f3c 100644
--- a/src/index.test.js
+++ b/src/index.test.js
@@ -320,6 +320,78 @@ test('Can parse text responses', async t => {
t.true(scope.isDone())
})
+test('Can retrieve multiple pages', async t => {
+ // Expected responses are arrays, since pagination makes sense in the context
+ // calls that return multiple items:
+ // https://docs.netlify.com/api/get-started/#pagination
+ const expectedResponsePages = [
+ [
+ { id: '1', content: 'page 1' },
+ { id: '2', content: 'page 1' }
+ ],
+ [
+ { id: '3', content: 'page 2' },
+ { id: '4', content: 'page 2' }
+ ],
+ [{ id: '5', content: 'page 3' }]
+ ]
+ const expectedResponse = expectedResponsePages.flatMap(x => x)
+
+ const baseUrl = `${pathPrefix}/sites`
+ const baseUrlFull = `${origin}${baseUrl}`
+ const scope = nock(origin)
+ .persist()
+ .get(baseUrl)
+ .reply(200, expectedResponsePages[0], {
+ Link: `<${baseUrlFull}?page=3>; rel="last", <${baseUrlFull}?page=2>; rel="next"`
+ })
+ .get(`${baseUrl}?page=2`)
+ .reply(200, expectedResponsePages[1], {
+ Link: `<${baseUrlFull}?page=3>; rel="last", <${baseUrlFull}?page=3>; rel="next"`
+ })
+ .get(`${baseUrl}?page=3`)
+ .reply(200, expectedResponsePages[2], {
+ Link: `<${baseUrlFull}?page=3>; rel="last"`
+ })
+ .persist(false)
+
+ const client = getClient()
+ const response = await client.listSites({}, { exhaustive: true })
+
+ t.deepEqual(response, expectedResponse)
+ t.true(scope.isDone())
+})
+
+test('Retrieves the first page if exhaustive is not specified', async t => {
+ // The entire dataset, split into multiple pages.
+ const datasetPages = [
+ [
+ { id: '1', content: 'page 1' },
+ { id: '2', content: 'page 1' }
+ ],
+ [
+ { id: '3', content: 'page 2' },
+ { id: '4', content: 'page 2' }
+ ],
+ [{ id: '5', content: 'page 3' }]
+ ]
+ const expectedResponse = datasetPages[0]
+
+ const baseUrl = `${pathPrefix}/sites`
+ const baseUrlFull = `${origin}${baseUrl}`
+ const scope = nock(origin)
+ .get(baseUrl)
+ .reply(200, expectedResponse, {
+ Link: `<${baseUrlFull}?page=3>; rel="last", <${baseUrlFull}?page=2>; rel="next"`
+ })
+
+ const client = getClient()
+ const response = await client.listSites()
+
+ t.deepEqual(response, expectedResponse)
+ t.true(scope.isDone())
+})
+
test('Handle error empty responses', async t => {
const account_id = '8'
const status = 404
diff --git a/src/methods/index.js b/src/methods/index.js
index f2a5381..c551ac5 100644
--- a/src/methods/index.js
+++ b/src/methods/index.js
@@ -1,4 +1,5 @@
const fetch = require('node-fetch').default || require('node-fetch') // Webpack will sometimes export default exports in different places
+const parseLinkHeader = require('parse-link-header')
const { getOperations } = require('../operations')
@@ -29,12 +30,59 @@ const getMethod = function(method, NetlifyApi) {
}
const callMethod = async function(method, NetlifyApi, params, opts) {
+ const { exhaustive = false } = opts || {}
const requestParams = Object.assign({}, NetlifyApi.globalParams, params)
const url = getUrl(method, NetlifyApi, requestParams)
- const response = await makeRequestOrRetry({ url, method, NetlifyApi, requestParams, opts })
+ const { parsedResponse, headers } = await retrieveResponse({ url, method, NetlifyApi, requestParams, opts })
+ if (!exhaustive || !Array.isArray(parsedResponse)) {
+ // If the user did not enable the retrieval of all items, or
+ // if the response is a single object/item, then
+ // we can skip the pagination logic.
+ return parsedResponse
+ }
+
+ return await retrieveResponseForNextPages({ method, NetlifyApi, requestParams, opts, parsedResponse, headers })
+}
+const retrieveResponse = async function({ url, method, NetlifyApi, requestParams, opts }) {
+ const response = await makeRequestOrRetry({ url, method, NetlifyApi, requestParams, opts })
const parsedResponse = await parseResponse(response)
- return parsedResponse
+ return {
+ parsedResponse,
+ headers: response.headers
+ }
+}
+
+const retrieveResponseForNextPages = async function({
+ method,
+ NetlifyApi,
+ requestParams,
+ opts,
+ parsedResponse,
+ headers
+}) {
+ // Responses for each page are accumulated, until finally flattened.
+ const results = [parsedResponse]
+
+ // The API directly provides the link to the next page (if any)
+ // in the `Link` header.
+ let url = getNextPageUrl(headers)
+
+ // For paginated results, we retrieve all the pages for this
+ // method until we exhaust the entire dataset.
+ while (url) {
+ const { parsedResponse, headers } = await retrieveResponse({ url, method, NetlifyApi, requestParams, opts })
+ results.push(parsedResponse)
+ url = getNextPageUrl(headers)
+ }
+
+ return results.flatMap(i => i)
+}
+
+const getNextPageUrl = function(headers) {
+ const linkHeader = headers.get('link') || ''
+ const { next = {} } = parseLinkHeader(linkHeader) || {}
+ return next.url
}
const getOpts = function({ verb, parameters }, NetlifyApi, { body }, opts) {
From caaa6d5a78e688fd5f380c5ea60fa796e481a137 Mon Sep 17 00:00:00 2001
From: Jay Vercellone <juanjov@gmail.com>
Date: Sat, 3 Oct 2020 21:53:53 -0700
Subject: [PATCH 2/2] Replace usage of flatMap for good 'ol concat
---
src/index.test.js | 2 +-
src/methods/index.js | 8 ++++----
2 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/src/index.test.js b/src/index.test.js
index 9ac5f3c..84cbb57 100644
--- a/src/index.test.js
+++ b/src/index.test.js
@@ -335,7 +335,7 @@ test('Can retrieve multiple pages', async t => {
],
[{ id: '5', content: 'page 3' }]
]
- const expectedResponse = expectedResponsePages.flatMap(x => x)
+ const expectedResponse = expectedResponsePages.reduce((response, page) => response.concat(page), [])
const baseUrl = `${pathPrefix}/sites`
const baseUrlFull = `${origin}${baseUrl}`
diff --git a/src/methods/index.js b/src/methods/index.js
index c551ac5..a5522f7 100644
--- a/src/methods/index.js
+++ b/src/methods/index.js
@@ -61,8 +61,8 @@ const retrieveResponseForNextPages = async function({
parsedResponse,
headers
}) {
- // Responses for each page are accumulated, until finally flattened.
- const results = [parsedResponse]
+ // Responses for each page are accumulated in a flattened manner.
+ let results = parsedResponse
// The API directly provides the link to the next page (if any)
// in the `Link` header.
@@ -72,11 +72,11 @@ const retrieveResponseForNextPages = async function({
// method until we exhaust the entire dataset.
while (url) {
const { parsedResponse, headers } = await retrieveResponse({ url, method, NetlifyApi, requestParams, opts })
- results.push(parsedResponse)
+ results = results.concat(parsedResponse)
url = getNextPageUrl(headers)
}
- return results.flatMap(i => i)
+ return results
}
const getNextPageUrl = function(headers) {