Skip to content
This repository was archived by the owner on Oct 10, 2022. It is now read-only.
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion .eslintrc.js
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ module.exports = {
'max-nested-callbacks': 0,
'max-statements': 0,
'no-await-in-loop': 0,
'no-magic-numbers': 0,
'no-param-reassign': 0,
'fp/no-class': 0,
'fp/no-let': 0,
Expand Down
31 changes: 31 additions & 0 deletions src/deploy/constants.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
// Local deploy timeout: 20 mins
const DEFAULT_DEPLOY_TIMEOUT = 1.2e6
// Concurrent file hash calls
const DEFAULT_CONCURRENT_HASH = 1e2
// Number of concurrent uploads
const DEFAULT_CONCURRENT_UPLOAD = 5
// Number of files
const DEFAULT_SYNC_LIMIT = 1e2
// Number of times to retry an upload
const DEFAULT_MAX_RETRY = 5

const UPLOAD_RANDOM_FACTOR = 0.5
// 5 seconds
const UPLOAD_INITIAL_DELAY = 5e3
// 1.5 minute
const UPLOAD_MAX_DELAY = 9e4

// 1 second
const DEPLOY_POLL = 1e3

module.exports = {
DEFAULT_DEPLOY_TIMEOUT,
DEFAULT_CONCURRENT_HASH,
DEFAULT_CONCURRENT_UPLOAD,
DEFAULT_SYNC_LIMIT,
DEFAULT_MAX_RETRY,
UPLOAD_RANDOM_FACTOR,
UPLOAD_INITIAL_DELAY,
UPLOAD_MAX_DELAY,
DEPLOY_POLL,
}
3 changes: 2 additions & 1 deletion src/deploy/hash_files.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,12 @@ const { promisify } = require('util')
const walker = require('folder-walker')
const pump = promisify(require('pump'))

const { DEFAULT_CONCURRENT_HASH } = require('./constants')
const { hasherCtor, manifestCollectorCtor, fileFilterCtor, fileNormalizerCtor } = require('./hasher_segments')

const hashFiles = async (dir, configPath, opts) => {
opts = {
concurrentHash: 100,
concurrentHash: DEFAULT_CONCURRENT_HASH,
assetType: 'file',
statusCb: () => {},
...opts,
Expand Down
3 changes: 2 additions & 1 deletion src/deploy/hash_fns.js
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,12 @@ const zipIt = require('@netlify/zip-it-and-ship-it')
const fromArray = require('from2-array')
const pump = promisify(require('pump'))

const { DEFAULT_CONCURRENT_HASH } = require('./constants')
const { hasherCtor, manifestCollectorCtor } = require('./hasher_segments')

const hashFns = async (dir, opts) => {
opts = {
concurrentHash: 100,
concurrentHash: DEFAULT_CONCURRENT_HASH,
assetType: 'function',
hashAlgorithm: 'sha256',
// tmpDir,
Expand Down
25 changes: 13 additions & 12 deletions src/deploy/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,17 @@ const cleanDeep = require('clean-deep')
const rimraf = promisify(require('rimraf'))
const tempy = require('tempy')

const {
DEFAULT_DEPLOY_TIMEOUT,
DEFAULT_CONCURRENT_HASH,
DEFAULT_CONCURRENT_UPLOAD,
DEFAULT_SYNC_LIMIT,
DEFAULT_MAX_RETRY,
} = require('./constants')
const hashFiles = require('./hash_files')
const hashFns = require('./hash_fns')
const uploadFiles = require('./upload_files')
const { waitForDiff } = require('./util')
const { waitForDeploy, getUploadList, defaultFilter } = require('./util')
const { waitForDiff, waitForDeploy, getUploadList, defaultFilter } = require('./util')

const deploySite = async (api, siteId, dir, opts) => {
opts = {
Expand All @@ -18,17 +24,12 @@ const deploySite = async (api, siteId, dir, opts) => {
// API calls this the 'title'
message: undefined,
tmpDir: tempy.directory(),
// local deploy timeout: 20 mins
deployTimeout: 1.2e6,
// concurrent file hash calls
concurrentHash: 100,
// Number of concurrent uploads
concurrentUpload: 5,
deployTimeout: DEFAULT_DEPLOY_TIMEOUT,
concurrentHash: DEFAULT_CONCURRENT_HASH,
concurrentUpload: DEFAULT_CONCURRENT_UPLOAD,
filter: defaultFilter,
// number of files
syncFileLimit: 100,
// number of times to retry an upload
maxRetry: 5,
syncFileLimit: DEFAULT_SYNC_LIMIT,
maxRetry: DEFAULT_MAX_RETRY,
statusCb: () => {
/* default to noop */
// statusObj: {
Expand Down
8 changes: 5 additions & 3 deletions src/deploy/upload_files.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ const fs = require('fs')
const backoff = require('backoff')
const pMap = require('p-map')

const { UPLOAD_RANDOM_FACTOR, UPLOAD_INITIAL_DELAY, UPLOAD_MAX_DELAY } = require('./constants')

const uploadFiles = async (api, deployId, uploadList, { concurrentUpload, statusCb, maxRetry }) => {
if (!concurrentUpload || !statusCb || !maxRetry) throw new Error('Missing required option concurrentUpload')
statusCb({
Expand Down Expand Up @@ -70,9 +72,9 @@ const retryUpload = (uploadFn, maxRetry) =>
new Promise((resolve, reject) => {
let lastError
const fibonacciBackoff = backoff.fibonacci({
randomisationFactor: 0.5,
initialDelay: 5000,
maxDelay: 90000,
randomisationFactor: UPLOAD_RANDOM_FACTOR,
initialDelay: UPLOAD_INITIAL_DELAY,
maxDelay: UPLOAD_MAX_DELAY,
})

const tryUpload = async () => {
Expand Down
6 changes: 4 additions & 2 deletions src/deploy/util.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ const { basename, sep } = require('path')

const pWaitFor = require('p-wait-for')

const { DEPLOY_POLL } = require('./constants')

// Default filter when scanning for files
const defaultFilter = (filePath) => {
if (filePath == null) {
Expand Down Expand Up @@ -60,7 +62,7 @@ const waitForDiff = async (api, deployId, siteId, timeout) => {
}

await pWaitFor(loadDeploy, {
interval: 1000,
interval: DEPLOY_POLL,
timeout,
message: 'Timeout while waiting for deploy',
})
Expand Down Expand Up @@ -97,7 +99,7 @@ const waitForDeploy = async (api, deployId, siteId, timeout) => {
}

await pWaitFor(loadDeploy, {
interval: 1000,
interval: DEPLOY_POLL,
timeout,
message: 'Timeout while waiting for deploy',
})
Expand Down
7 changes: 6 additions & 1 deletion src/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ class NetlifyAPI {
}

async getAccessToken(ticket, opts) {
opts = { poll: 1000, timeout: 3.6e6, ...opts }
opts = { poll: DEFAULT_TICKET_POLL, timeout: DEFAULT_TICKET_TIMEOUT, ...opts }

const { id } = ticket

Expand Down Expand Up @@ -98,6 +98,11 @@ class NetlifyAPI {
}
}

// 1 second
const DEFAULT_TICKET_POLL = 1e3
// 1 hour
const DEFAULT_TICKET_TIMEOUT = 3.6e6

module.exports = NetlifyAPI

module.exports.methods = getOperations()