Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
32 commits
Select commit Hold shift + click to select a range
1a0f86a
Skip compile & packaging if --no-build is set (#560)
jamesmbourne May 7, 2020
b44716b
Serialized compile to address #299
asprouse Jun 14, 2019
4d43773
Add serializedCompile documentation
asprouse Jun 14, 2019
49b2ef8
Update README.md
miguel-a-calles-mba Jul 12, 2020
468cf39
Update README.md
miguel-a-calles-mba Jul 12, 2020
9b2fe1a
Option to exclude files using regular expression
dseeker Jan 25, 2020
01d8d13
Add some tests & fix linter
j0k3r Jul 24, 2020
789f8a3
Improve cleanup process
neilime Nov 11, 2018
678eaa6
fix for dependencies with no version detected
brysgo Jun 25, 2020
19a0088
fix tests
brysgo Jun 25, 2020
ecf6c2b
Update yarn.js
miguel-a-calles-mba Aug 2, 2020
32d5071
Update yarn.js
miguel-a-calles-mba Aug 2, 2020
fb877e4
Update compile.js
vicary Jul 3, 2020
4af95b6
Update README.md
vicary Jul 3, 2020
d1ce814
Rename to concurrency
vicary Jul 3, 2020
883ea07
Update Configuration.test.js
vicary Jul 3, 2020
f8db180
Update validate.js
vicary Jul 3, 2020
df63d72
Update compile.test.js
vicary Jul 3, 2020
874a241
bluebird doesn't like undefined concurrency
vicary Jul 3, 2020
23bb9a3
Support icloud drive
Oct 4, 2020
2fdc3eb
Serialized compile to address #299
asprouse Jun 14, 2019
6ab5e92
compatible with serialized compile
vicary Oct 4, 2020
ca76a25
audit fix
vicary Oct 4, 2020
a234211
docs for compatibility with serilized-compile
vicary Oct 4, 2020
4ac320b
fix tests
vicary Oct 4, 2020
ba280ad
Option to override concurrency setting via serverless CLI
coyoteecd Jul 6, 2020
17c6209
Remove compile-concurrency option, since the same can be achieved via…
coyoteecd Jul 6, 2020
92d1cd1
Move the parsing and default value for concurrency option in Configur…
coyoteecd Oct 14, 2020
79d3f65
revert reformatting
vicary Jan 6, 2021
a31173e
revert dropped commits in intermediate fork parent
vicary Jan 6, 2021
b8ffa34
Remove dropped configurations
vicary Jan 6, 2021
b84a6f8
Revert lock file to upstream master
vicary Jan 6, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
node_modules
node_modules.nosync
dist
.webpack
.serverless
Expand Down
12 changes: 12 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -527,6 +527,18 @@ if you are trying to override the entry in webpack.config.js with other unsuppor
The individual packaging needs more time at the packaging phase, but you'll
get that paid back twice at runtime.

#### Individual packaging concurrency/serializdCompile

```yaml
# serverless.yml
custom:
webpack:
concurrency: 5
serializedCompile: true # for backward compatibility, same as concurrency: 1
```

This runs webpack builds in parallel, throttled to `concurrency`. It reduces memory usage and in some cases impoves overall build performance.

## Usage

### Automatic bundling
Expand Down
2 changes: 1 addition & 1 deletion index.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ describe('ServerlessWebpack', () => {

_.set(serverless, 'service.custom.webpack.webpackConfig', 'webpack.config.ts');

const badDeps = function() {
const badDeps = function () {
new ServerlessWebpack(serverless, {});
};

Expand Down
22 changes: 21 additions & 1 deletion lib/Configuration.js
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@ const DefaultConfig = {
packager: 'npm',
packagerOptions: {},
keepOutputDirectory: false,
config: null
config: null,
concurrency: Infinity
};

class Configuration {
Expand All @@ -37,6 +38,21 @@ class Configuration {
}
}

// Concurrency may be passed via CLI, e.g.
// custom:
// webpack:
// concurrency: ${opt:compile-concurrency, 7}
// In this case it is typed as a string and we have to validate it
if (this._config.concurrency !== undefined) {
this._config.concurrency = Number(this._config.concurrency);
if (isNaN(this._config.concurrency) || this._config.concurrency < 1) {
throw new Error('concurrency option must be a positive number');
}
} else if (this._config.serializedCompile === true) {
// Backwards compatibility with serializedCompile setting
this._config.concurrency = 1;
}

// Set defaults for all missing properties
_.defaults(this._config, DefaultConfig);
}
Expand Down Expand Up @@ -73,6 +89,10 @@ class Configuration {
return this._config.keepOutputDirectory;
}

get concurrency() {
return this._config.concurrency;
}

toJSON() {
return _.omitBy(this._config, _.isNil);
}
Expand Down
46 changes: 42 additions & 4 deletions lib/Configuration.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ describe('Configuration', () => {
packager: 'npm',
packagerOptions: {},
keepOutputDirectory: false,
config: null
config: null,
concurrency: Infinity
};
});

Expand Down Expand Up @@ -68,7 +69,8 @@ describe('Configuration', () => {
packager: 'npm',
packagerOptions: {},
keepOutputDirectory: false,
config: null
config: null,
concurrency: Infinity
});
});
});
Expand All @@ -88,7 +90,8 @@ describe('Configuration', () => {
packager: 'npm',
packagerOptions: {},
keepOutputDirectory: false,
config: null
config: null,
concurrency: Infinity
});
});

Expand All @@ -107,8 +110,43 @@ describe('Configuration', () => {
packager: 'npm',
packagerOptions: {},
keepOutputDirectory: false,
config: null
config: null,
concurrency: Infinity
});
});

it('should accept a numeric string as concurrency value', () => {
const testCustom = {
webpack: {
includeModules: { forceInclude: ['mod1'] },
webpackConfig: 'myWebpackFile.js',
concurrency: '3'
}
};
const config = new Configuration(testCustom);
expect(config._config.concurrency).to.equal(3);
});

it('should not accept an invalid string as concurrency value', () => {
const testCustom = {
webpack: {
includeModules: { forceInclude: ['mod1'] },
webpackConfig: 'myWebpackFile.js',
concurrency: '3abc'
}
};
expect(() => new Configuration(testCustom)).throws();
});

it('should not accept a non-positive number as concurrency value', () => {
const testCustom = {
webpack: {
includeModules: { forceInclude: ['mod1'] },
webpackConfig: 'myWebpackFile.js',
concurrency: 0
}
};
expect(() => new Configuration(testCustom)).throws();
});
});
});
82 changes: 50 additions & 32 deletions lib/compile.js
Original file line number Diff line number Diff line change
Expand Up @@ -5,44 +5,62 @@ const BbPromise = require('bluebird');
const webpack = require('webpack');
const tty = require('tty');

const defaultStatsConfig = {
colors: tty.isatty(process.stdout.fd),
hash: false,
version: false,
chunks: false,
children: false
};

function ensureArray(obj) {
return _.isArray(obj) ? obj : [obj];
}

function getStatsLogger(statsConfig, consoleLog) {
return stats => consoleLog(stats.toString(statsConfig || defaultStatsConfig));
}

function webpackCompile(config, logStats) {
return BbPromise
.fromCallback(cb => webpack(config).run(cb))
.then(stats => {
// ensure stats in any array in the case of multiCompile
stats = stats.stats ? stats.stats : [stats];

_.forEach(stats, compileStats => {
logStats(compileStats);
if (compileStats.hasErrors()) {
throw new Error('Webpack compilation error, see stats above');
}
});

return stats;
});
}

function webpackConcurrentCompile(configs, logStats, concurrency) {
return BbPromise
.map(configs, config => webpackCompile(config, logStats), { concurrency })
.then(stats => _.flatten(stats));
}

module.exports = {
compile() {
this.serverless.cli.log('Bundling with Webpack...');

const compiler = webpack(this.webpackConfig);

return BbPromise.fromCallback(cb => compiler.run(cb)).then(stats => {
if (!this.multiCompile) {
stats = { stats: [stats] };
}

const compileOutputPaths = [];
const consoleStats = this.webpackConfig.stats ||
_.get(this, 'webpackConfig[0].stats') || {
colors: tty.isatty(process.stdout.fd),
hash: false,
version: false,
chunks: false,
children: false
};

_.forEach(stats.stats, compileStats => {
const statsOutput = compileStats.toString(consoleStats);
if (statsOutput) {
this.serverless.cli.consoleLog(statsOutput);
}
const configs = ensureArray(this.webpackConfig);
const logStats = getStatsLogger(configs[0].stats, this.serverless.cli.consoleLog);

if (compileStats.compilation.errors.length) {
throw new Error('Webpack compilation error, see above');
}
if (!this.configuration) {
return BbPromise.reject('Missing plugin configuration');
}
const concurrency = this.configuration.concurrency;

compileOutputPaths.push(compileStats.compilation.compiler.outputPath);
return webpackConcurrentCompile(configs, logStats, concurrency)
.then(stats => {
this.compileStats = { stats };
return BbPromise.resolve();
});

this.compileOutputPaths = compileOutputPaths;
this.compileStats = stats;

return BbPromise.resolve();
});
}
};
12 changes: 6 additions & 6 deletions lib/packagers/yarn.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -134,11 +134,11 @@ describe('yarn', () => {
acorn@^2.1.0, acorn@^2.4.0:
version "2.7.0"
resolved "https://registry.yarnpkg.com/acorn/-/acorn-2.7.0.tgz#ab6e7d9d886aaca8b085bc3312b79a198433f0e7"

acorn@^3.0.4:
version "3.3.0"
resolved "https://registry.yarnpkg.com/acorn/-/acorn-3.3.0.tgz#45e37fb39e8da3f25baee3ff5369e2bb5f22017a"

otherModule@file:../../otherModule/the-new-version:
version "1.2.0"

Expand All @@ -162,7 +162,7 @@ describe('yarn', () => {
request "^2.83.0"
ulid "^0.1.0"
uuid "^3.1.0"

acorn@^5.0.0, acorn@^5.5.0:
version "5.5.3"
resolved "https://registry.yarnpkg.com/acorn/-/acorn-5.5.3.tgz#f473dd47e0277a08e28e9bec5aeeb04751f0b8c9"
Expand All @@ -172,11 +172,11 @@ describe('yarn', () => {
acorn@^2.1.0, acorn@^2.4.0:
version "2.7.0"
resolved "https://registry.yarnpkg.com/acorn/-/acorn-2.7.0.tgz#ab6e7d9d886aaca8b085bc3312b79a198433f0e7"

acorn@^3.0.4:
version "3.3.0"
resolved "https://registry.yarnpkg.com/acorn/-/acorn-3.3.0.tgz#45e37fb39e8da3f25baee3ff5369e2bb5f22017a"

otherModule@file:../../project/../../otherModule/the-new-version:
version "1.2.0"

Expand All @@ -200,7 +200,7 @@ describe('yarn', () => {
request "^2.83.0"
ulid "^0.1.0"
uuid "^3.1.0"

acorn@^5.0.0, acorn@^5.5.0:
version "5.5.3"
resolved "https://registry.yarnpkg.com/acorn/-/acorn-5.5.3.tgz#f473dd47e0277a08e28e9bec5aeeb04751f0b8c9"
Expand Down
8 changes: 7 additions & 1 deletion lib/validate.js
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,7 @@ module.exports = {
this.webpackConfig.output.path = path.join(this.serverless.config.servicePath, this.options.out);
}

// Skip compilation with --no-build
if (this.skipCompile) {
this.serverless.cli.log('Skipping build and using existing compiled output');
if (!fse.pathExistsSync(this.webpackConfig.output.path)) {
Expand All @@ -187,8 +188,13 @@ module.exports = {

// In case of individual packaging we have to create a separate config for each function
if (_.has(this.serverless, 'service.package') && this.serverless.service.package.individually) {
this.options.verbose && this.serverless.cli.log('Using multi-compile (individual packaging)');
this.multiCompile = true;
this.options.verbose &&
this.serverless.cli.log(
`Using ${
this.configuration.concurrency !== Infinity ? 'concurrent' : 'multi'
}-compile (individual packaging)`
);

if (this.webpackConfig.entry && !_.isEqual(this.webpackConfig.entry, entries)) {
return BbPromise.reject(
Expand Down
Loading