Skip to content

Commit a94079a

Browse files
feat: support deploy config API with Blobs (#5565)
* feat: support deploy config API with Blobs * feat: set `experimentalRegion`
1 parent 626fa44 commit a94079a

File tree

7 files changed

+158
-40
lines changed

7 files changed

+158
-40
lines changed

packages/build/src/plugins_core/blobs_upload/index.ts

Lines changed: 26 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import pMap from 'p-map'
55
import semver from 'semver'
66

77
import { log, logError } from '../../log/logger.js'
8-
import { anyBlobsToUpload, getBlobsDir } from '../../utils/blobs.js'
8+
import { scanForBlobs } from '../../utils/blobs.js'
99
import { CoreStep, CoreStepCondition, CoreStepFunction } from '../types.js'
1010

1111
import { getKeysToUpload, getFileWithMetadata } from './utils.js'
@@ -26,22 +26,41 @@ const coreStep: CoreStepFunction = async function ({
2626
// for cli deploys with `netlify deploy --build` the `NETLIFY_API_HOST` is undefined
2727
const apiHost = NETLIFY_API_HOST || 'api.netlify.com'
2828

29-
const storeOpts: { siteID: string; deployID: string; token: string; apiURL: string; fetch?: any } = {
29+
const storeOpts: Parameters<typeof getDeployStore>[0] = {
3030
siteID: SITE_ID,
3131
deployID: deployId,
3232
token: NETLIFY_API_TOKEN,
3333
apiURL: `https://${apiHost}`,
3434
}
35+
36+
// If we don't have native `fetch` in the global scope, add a polyfill.
3537
if (semver.lt(nodeVersion, '18.0.0')) {
3638
const nodeFetch = await import('node-fetch')
39+
40+
// @ts-expect-error The types between `node-fetch` and the native `fetch`
41+
// are not a 100% match, even though the APIs are mostly compatible.
3742
storeOpts.fetch = nodeFetch.default
3843
}
3944

40-
const blobStore = getDeployStore(storeOpts)
41-
const blobsDir = getBlobsDir(buildDir, packagePath)
42-
const keys = await getKeysToUpload(blobsDir)
45+
const blobs = await scanForBlobs(buildDir, packagePath)
4346

4447
// We checked earlier, but let's be extra safe
48+
if (blobs === null) {
49+
if (!quiet) {
50+
log(logs, 'No blobs to upload to deploy store.')
51+
}
52+
return {}
53+
}
54+
55+
// If using the deploy config API, configure the store to use the region that
56+
// was configured for the deploy.
57+
if (!blobs.isLegacyDirectory) {
58+
storeOpts.experimentalRegion = 'auto'
59+
}
60+
61+
const blobStore = getDeployStore(storeOpts)
62+
const keys = await getKeysToUpload(blobs.directory)
63+
4564
if (keys.length === 0) {
4665
if (!quiet) {
4766
log(logs, 'No blobs to upload to deploy store.')
@@ -57,7 +76,7 @@ const coreStep: CoreStepFunction = async function ({
5776
if (debug && !quiet) {
5877
log(logs, `- Uploading blob ${key}`, { indent: true })
5978
}
60-
const { data, metadata } = await getFileWithMetadata(blobsDir, key)
79+
const { data, metadata } = await getFileWithMetadata(blobs.directory, key)
6180
await blobStore.set(key, data, { metadata })
6281
}
6382

@@ -81,7 +100,7 @@ const deployAndBlobsPresent: CoreStepCondition = async ({
81100
buildDir,
82101
packagePath,
83102
constants: { NETLIFY_API_TOKEN },
84-
}) => Boolean(NETLIFY_API_TOKEN && deployId && (await anyBlobsToUpload(buildDir, packagePath)))
103+
}) => Boolean(NETLIFY_API_TOKEN && deployId && (await scanForBlobs(buildDir, packagePath)))
85104

86105
export const uploadBlobs: CoreStep = {
87106
event: 'onPostBuild',

packages/build/src/plugins_core/pre_cleanup/index.ts

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,21 @@
11
import { rm } from 'node:fs/promises'
22

3-
import { anyBlobsToUpload, getBlobsDir } from '../../utils/blobs.js'
3+
import { scanForBlobs, getBlobsDirs } from '../../utils/blobs.js'
44
import { CoreStep, CoreStepCondition, CoreStepFunction } from '../types.js'
55

66
const coreStep: CoreStepFunction = async ({ buildDir, packagePath }) => {
7-
const blobsDir = getBlobsDir(buildDir, packagePath)
7+
const blobsDirs = getBlobsDirs(buildDir, packagePath)
88
try {
9-
await rm(blobsDir, { recursive: true, force: true })
9+
await Promise.all(blobsDirs.map((dir) => rm(dir, { recursive: true, force: true })))
1010
} catch {
1111
// Ignore errors if it fails, we can continue anyway.
1212
}
1313

1414
return {}
1515
}
1616

17-
const blobsPresent: CoreStepCondition = ({ buildDir, packagePath }) => anyBlobsToUpload(buildDir, packagePath)
17+
const blobsPresent: CoreStepCondition = async ({ buildDir, packagePath }) =>
18+
Boolean(await scanForBlobs(buildDir, packagePath))
1819

1920
export const preCleanup: CoreStep = {
2021
event: 'onPreBuild',

packages/build/src/utils/blobs.ts

Lines changed: 33 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,19 +2,44 @@ import { resolve } from 'node:path'
22

33
import { fdir } from 'fdir'
44

5-
const BLOBS_PATH = '.netlify/blobs/deploy'
5+
const LEGACY_BLOBS_PATH = '.netlify/blobs/deploy'
6+
const DEPLOY_CONFIG_BLOBS_PATH = '.netlify/deploy/v1/blobs/deploy'
67

7-
/** Retrieve the absolute path of the deploy scoped internal blob directory */
8-
export const getBlobsDir = (buildDir: string, packagePath?: string) => resolve(buildDir, packagePath || '', BLOBS_PATH)
8+
/** Retrieve the absolute path of the deploy scoped internal blob directories */
9+
export const getBlobsDirs = (buildDir: string, packagePath?: string) => [
10+
resolve(buildDir, packagePath || '', DEPLOY_CONFIG_BLOBS_PATH),
11+
resolve(buildDir, packagePath || '', LEGACY_BLOBS_PATH),
12+
]
913

1014
/**
11-
* Detect if there are any blobs to upload
15+
* Detect if there are any blobs to upload, and if so, what directory they're
16+
* in and whether that directory is the legacy `.netlify/blobs` path or the
17+
* newer deploy config API endpoint.
18+
*
1219
* @param buildDir The build directory. (current working directory where the build is executed)
1320
* @param packagePath An optional package path for mono repositories
1421
* @returns
1522
*/
16-
export const anyBlobsToUpload = async function (buildDir: string, packagePath?: string) {
17-
const blobsDir = getBlobsDir(buildDir, packagePath)
18-
const { files } = await new fdir().onlyCounts().crawl(blobsDir).withPromise()
19-
return files > 0
23+
export const scanForBlobs = async function (buildDir: string, packagePath?: string) {
24+
const blobsDir = resolve(buildDir, packagePath || '', DEPLOY_CONFIG_BLOBS_PATH)
25+
const blobsDirScan = await new fdir().onlyCounts().crawl(blobsDir).withPromise()
26+
27+
if (blobsDirScan.files > 0) {
28+
return {
29+
directory: blobsDir,
30+
isLegacyDirectory: false,
31+
}
32+
}
33+
34+
const legacyBlobsDir = resolve(buildDir, packagePath || '', LEGACY_BLOBS_PATH)
35+
const legacyBlobsDirScan = await new fdir().onlyCounts().crawl(legacyBlobsDir).withPromise()
36+
37+
if (legacyBlobsDirScan.files > 0) {
38+
return {
39+
directory: legacyBlobsDir,
40+
isLegacyDirectory: true,
41+
}
42+
}
43+
44+
return null
2045
}
Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
11
import { mkdir, writeFile } from 'node:fs/promises'
22

3-
await mkdir('.netlify/blobs/deploy/nested', { recursive: true })
3+
await mkdir('.netlify/deploy/v1/blobs/deploy/nested', { recursive: true })
44

55
await Promise.all([
6-
writeFile('.netlify/blobs/deploy/something.txt', 'some value'),
7-
writeFile('.netlify/blobs/deploy/with-metadata.txt', 'another value'),
8-
writeFile('.netlify/blobs/deploy/$with-metadata.txt.json', JSON.stringify({ "meta": "data", "number": 1234 })),
9-
writeFile('.netlify/blobs/deploy/nested/file.txt', 'file value'),
10-
writeFile('.netlify/blobs/deploy/nested/$file.txt.json', JSON.stringify({ "some": "metadata" })),
6+
writeFile('.netlify/deploy/v1/blobs/deploy/something.txt', 'some value'),
7+
writeFile('.netlify/deploy/v1/blobs/deploy/with-metadata.txt', 'another value'),
8+
writeFile('.netlify/deploy/v1/blobs/deploy/$with-metadata.txt.json', JSON.stringify({ "meta": "data", "number": 1234 })),
9+
writeFile('.netlify/deploy/v1/blobs/deploy/nested/file.txt', 'file value'),
10+
writeFile('.netlify/deploy/v1/blobs/deploy/nested/$file.txt.json', JSON.stringify({ "some": "metadata" })),
1111
])
Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
import { mkdir, writeFile } from 'node:fs/promises'
2+
3+
await mkdir('.netlify/blobs/deploy/nested', { recursive: true })
4+
5+
await Promise.all([
6+
writeFile('.netlify/blobs/deploy/something.txt', 'some value'),
7+
writeFile('.netlify/blobs/deploy/with-metadata.txt', 'another value'),
8+
writeFile('.netlify/blobs/deploy/$with-metadata.txt.json', JSON.stringify({ "meta": "data", "number": 1234 })),
9+
writeFile('.netlify/blobs/deploy/nested/file.txt', 'file value'),
10+
writeFile('.netlify/blobs/deploy/nested/$file.txt.json', JSON.stringify({ "some": "metadata" })),
11+
])
Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
[build]
2+
command = "node build.mjs"
3+
base = "/"
4+
publish = "/dist"

packages/build/tests/blobs_upload/tests.js

Lines changed: 73 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -14,23 +14,24 @@ const TOKEN = 'test'
1414

1515
test.beforeEach(async (t) => {
1616
const port = await getPort()
17-
t.context.blobRequestCount = { set: 0, get: 0 }
17+
t.context.blobRequests = {}
1818

1919
const tmpDir = await tmp.dir()
2020
t.context.blobServer = new BlobsServer({
2121
port,
2222
token: TOKEN,
2323
directory: tmpDir.path,
24-
onRequest: ({ type }) => {
25-
t.context.blobRequestCount[type] = (t.context.blobRequestCount[type] || 0) + 1
24+
onRequest: ({ type, url }) => {
25+
t.context.blobRequests[type] = t.context.blobRequests[type] || []
26+
t.context.blobRequests[type].push(url)
2627
},
2728
})
2829

2930
await t.context.blobServer.start()
3031

3132
process.env.NETLIFY_BLOBS_CONTEXT = Buffer.from(
3233
JSON.stringify({
33-
edgeURL: `http://localhost:${port}`,
34+
apiURL: `http://localhost:${port}`,
3435
}),
3536
).toString('base64')
3637
})
@@ -50,27 +51,74 @@ test.serial("blobs upload, don't run when deploy id is provided and no files in
5051
.runBuildProgrammatic()
5152

5253
t.true(success)
53-
t.is(t.context.blobRequestCount.set, 0)
54+
t.is(t.context.blobRequests.set, undefined)
5455

5556
t.false(stdout.join('\n').includes('Uploading blobs to deploy store'))
5657
})
5758

58-
test.serial("blobs upload, don't run when there are files but deploy id is not provided", async (t) => {
59-
const fixture = await new Fixture('./fixtures/src_with_blobs').withCopyRoot({ git: false })
59+
test.serial(
60+
"blobs upload, don't run when there are files but deploy id is not provided using legacy API",
61+
async (t) => {
62+
const fixture = await new Fixture('./fixtures/src_with_blobs_legacy').withCopyRoot({ git: false })
63+
64+
const {
65+
success,
66+
logs: { stdout },
67+
} = await fixture.withFlags({ token: TOKEN, offline: true, cwd: fixture.repositoryRoot }).runBuildProgrammatic()
68+
69+
t.true(success)
70+
71+
const blobsDir = join(fixture.repositoryRoot, '.netlify', 'blobs', 'deploy')
72+
await t.notThrowsAsync(access(blobsDir))
73+
74+
t.is(t.context.blobRequests.set, undefined)
75+
76+
t.false(stdout.join('\n').includes('Uploading blobs to deploy store'))
77+
},
78+
)
79+
80+
test.serial('blobs upload, uploads files to deploy store using legacy API', async (t) => {
81+
const fixture = await new Fixture('./fixtures/src_with_blobs_legacy').withCopyRoot({ git: false })
6082

6183
const {
6284
success,
6385
logs: { stdout },
64-
} = await fixture.withFlags({ token: TOKEN, offline: true, cwd: fixture.repositoryRoot }).runBuildProgrammatic()
86+
} = await fixture
87+
.withFlags({ deployId: 'abc123', siteId: 'test', token: TOKEN, offline: true, cwd: fixture.repositoryRoot })
88+
.runBuildProgrammatic()
6589

6690
t.true(success)
91+
t.is(t.context.blobRequests.set.length, 6)
6792

68-
const blobsDir = join(fixture.repositoryRoot, '.netlify', 'blobs', 'deploy')
69-
await t.notThrowsAsync(access(blobsDir))
93+
const regionRequests = t.context.blobRequests.set.filter((urlPath) => {
94+
const url = new URL(urlPath, 'http://localhost')
7095

71-
t.is(t.context.blobRequestCount.set, 0)
96+
return url.searchParams.has('region')
97+
})
7298

73-
t.false(stdout.join('\n').includes('Uploading blobs to deploy store'))
99+
t.is(regionRequests.length, 0)
100+
101+
const storeOpts = { deployID: 'abc123', siteID: 'test', token: TOKEN }
102+
if (semver.lt(nodeVersion, '18.0.0')) {
103+
const nodeFetch = await import('node-fetch')
104+
storeOpts.fetch = nodeFetch.default
105+
}
106+
107+
const store = getDeployStore(storeOpts)
108+
109+
const blob1 = await store.getWithMetadata('something.txt')
110+
t.is(blob1.data, 'some value')
111+
t.deepEqual(blob1.metadata, {})
112+
113+
const blob2 = await store.getWithMetadata('with-metadata.txt')
114+
t.is(blob2.data, 'another value')
115+
t.deepEqual(blob2.metadata, { meta: 'data', number: 1234 })
116+
117+
const blob3 = await store.getWithMetadata('nested/file.txt')
118+
t.is(blob3.data, 'file value')
119+
t.deepEqual(blob3.metadata, { some: 'metadata' })
120+
121+
t.true(stdout.join('\n').includes('Uploading blobs to deploy store'))
74122
})
75123

76124
test.serial('blobs upload, uploads files to deploy store', async (t) => {
@@ -84,7 +132,17 @@ test.serial('blobs upload, uploads files to deploy store', async (t) => {
84132
.runBuildProgrammatic()
85133

86134
t.true(success)
87-
t.is(t.context.blobRequestCount.set, 3)
135+
136+
// 3 requests for getting pre-signed URLs + 3 requests for hitting them.
137+
t.is(t.context.blobRequests.set.length, 6)
138+
139+
const regionAutoRequests = t.context.blobRequests.set.filter((urlPath) => {
140+
const url = new URL(urlPath, 'http://localhost')
141+
142+
return url.searchParams.get('region') === 'auto'
143+
})
144+
145+
t.is(regionAutoRequests.length, 3)
88146

89147
const storeOpts = { deployID: 'abc123', siteID: 'test', token: TOKEN }
90148
if (semver.lt(nodeVersion, '18.0.0')) {
@@ -118,7 +176,7 @@ test.serial('blobs upload, cancels deploy if blob metadata is malformed', async
118176
const blobsDir = join(fixture.repositoryRoot, '.netlify', 'blobs', 'deploy')
119177
await t.notThrowsAsync(access(blobsDir))
120178

121-
t.is(t.context.blobRequestCount.set, 0)
179+
t.is(t.context.blobRequests.set, undefined)
122180

123181
t.false(success)
124182
t.is(severityCode, 4)
@@ -136,7 +194,7 @@ if (semver.gte(nodeVersion, '16.9.0')) {
136194
.runBuildProgrammatic()
137195

138196
t.true(success)
139-
t.is(t.context.blobRequestCount.set, 3)
197+
t.is(t.context.blobRequests.set.length, 6)
140198

141199
const storeOpts = { deployID: 'abc123', siteID: 'test', token: TOKEN }
142200
if (semver.lt(nodeVersion, '18.0.0')) {

0 commit comments

Comments
 (0)