diff --git a/package-lock.json b/package-lock.json index a0c9bac..317704f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9252,6 +9252,14 @@ "json-parse-better-errors": "^1.0.1" } }, + "parse-link-header": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parse-link-header/-/parse-link-header-1.0.1.tgz", + "integrity": "sha1-vt/g0hGK64S+deewJUGeyKYRQKc=", + "requires": { + "xtend": "~4.0.1" + } + }, "parse-ms": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/parse-ms/-/parse-ms-2.1.0.tgz", diff --git a/package.json b/package.json index 9d318d5..5a39284 100644 --- a/package.json +++ b/package.json @@ -82,6 +82,7 @@ "p-map": "^3.0.0", "p-wait-for": "^3.1.0", "parallel-transform": "^1.1.0", + "parse-link-header": "^1.0.1", "pump": "^3.0.0", "qs": "^6.9.3", "rimraf": "^3.0.2", diff --git a/src/index.test.js b/src/index.test.js index 86967f1..da2a47f 100644 --- a/src/index.test.js +++ b/src/index.test.js @@ -298,6 +298,78 @@ test('Can parse text responses', async (t) => { t.true(scope.isDone()) }) +test('Can retrieve multiple pages', async t => { + // Expected responses are arrays, since pagination makes sense in the context + // calls that return multiple items: + // https://docs.netlify.com/api/get-started/#pagination + const expectedResponsePages = [ + [ + { id: '1', content: 'page 1' }, + { id: '2', content: 'page 1' } + ], + [ + { id: '3', content: 'page 2' }, + { id: '4', content: 'page 2' } + ], + [{ id: '5', content: 'page 3' }] + ] + const expectedResponse = expectedResponsePages.reduce((response, page) => response.concat(page), []) + + const baseUrl = `${pathPrefix}/sites` + const baseUrlFull = `${origin}${baseUrl}` + const scope = nock(origin) + .persist() + .get(baseUrl) + .reply(200, expectedResponsePages[0], { + Link: `<${baseUrlFull}?page=3>; rel="last", <${baseUrlFull}?page=2>; rel="next"` + }) + .get(`${baseUrl}?page=2`) + .reply(200, expectedResponsePages[1], { + Link: `<${baseUrlFull}?page=3>; rel="last", <${baseUrlFull}?page=3>; rel="next"` + }) + .get(`${baseUrl}?page=3`) + .reply(200, expectedResponsePages[2], { + Link: `<${baseUrlFull}?page=3>; rel="last"` + }) + .persist(false) + + const client = getClient() + const response = await client.listSites({}, { exhaustive: true }) + + t.deepEqual(response, expectedResponse) + t.true(scope.isDone()) +}) + +test('Retrieves the first page if exhaustive is not specified', async t => { + // The entire dataset, split into multiple pages. + const datasetPages = [ + [ + { id: '1', content: 'page 1' }, + { id: '2', content: 'page 1' } + ], + [ + { id: '3', content: 'page 2' }, + { id: '4', content: 'page 2' } + ], + [{ id: '5', content: 'page 3' }] + ] + const expectedResponse = datasetPages[0] + + const baseUrl = `${pathPrefix}/sites` + const baseUrlFull = `${origin}${baseUrl}` + const scope = nock(origin) + .get(baseUrl) + .reply(200, expectedResponse, { + Link: `<${baseUrlFull}?page=3>; rel="last", <${baseUrlFull}?page=2>; rel="next"` + }) + + const client = getClient() + const response = await client.listSites() + + t.deepEqual(response, expectedResponse) + t.true(scope.isDone()) +}) + test('Handle error empty responses', async (t) => { const accountId = uuidv4() const status = 404 diff --git a/src/methods/index.js b/src/methods/index.js index de87915..5954b41 100644 --- a/src/methods/index.js +++ b/src/methods/index.js @@ -1,5 +1,6 @@ // Webpack will sometimes export default exports in different places const fetch = require('node-fetch').default || require('node-fetch') +const parseLinkHeader = require('parse-link-header') const { getOperations } = require('../operations') @@ -30,12 +31,58 @@ const getMethod = function (method, NetlifyApi) { } const callMethod = async function (method, NetlifyApi, params, opts) { - const requestParams = { ...NetlifyApi.globalParams, ...params } + const { exhaustive = false } = opts || {} const url = getUrl(method, NetlifyApi, requestParams) - const response = await makeRequestOrRetry({ url, method, NetlifyApi, requestParams, opts }) + const { parsedResponse, headers } = await retrieveResponse({ url, method, NetlifyApi, requestParams, opts }) + if (!exhaustive || !Array.isArray(parsedResponse)) { + // If the user did not enable the retrieval of all items, or + // if the response is a single object/item, then + // we can skip the pagination logic. + return parsedResponse + } + + return await retrieveResponseForNextPages({ method, NetlifyApi, requestParams, opts, parsedResponse, headers }) +} +const retrieveResponse = async function({ url, method, NetlifyApi, requestParams, opts }) { + const response = await makeRequestOrRetry({ url, method, NetlifyApi, requestParams, opts }) const parsedResponse = await parseResponse(response) - return parsedResponse + return { + parsedResponse, + headers: response.headers + } +} + +const retrieveResponseForNextPages = async function({ + method, + NetlifyApi, + requestParams, + opts, + parsedResponse, + headers +}) { + // Responses for each page are accumulated in a flattened manner. + let results = parsedResponse + + // The API directly provides the link to the next page (if any) + // in the `Link` header. + let url = getNextPageUrl(headers) + + // For paginated results, we retrieve all the pages for this + // method until we exhaust the entire dataset. + while (url) { + const { parsedResponse, headers } = await retrieveResponse({ url, method, NetlifyApi, requestParams, opts }) + results = results.concat(parsedResponse) + url = getNextPageUrl(headers) + } + + return results +} + +const getNextPageUrl = function(headers) { + const linkHeader = headers.get('link') || '' + const { next = {} } = parseLinkHeader(linkHeader) || {} + return next.url } const getOpts = function ({ verb, parameters }, NetlifyApi, { body }, opts) {