diff --git a/.gitignore b/.gitignore index e701029f9..88ece127c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ node_modules +node_modules.nosync dist .webpack .serverless diff --git a/README.md b/README.md index 3cb9c3750..dec7e6206 100644 --- a/README.md +++ b/README.md @@ -527,6 +527,18 @@ if you are trying to override the entry in webpack.config.js with other unsuppor The individual packaging needs more time at the packaging phase, but you'll get that paid back twice at runtime. +#### Individual packaging concurrency/serializdCompile + +```yaml +# serverless.yml +custom: + webpack: + concurrency: 5 + serializedCompile: true # for backward compatibility, same as concurrency: 1 +``` + +This runs webpack builds in parallel, throttled to `concurrency`. It reduces memory usage and in some cases impoves overall build performance. + ## Usage ### Automatic bundling diff --git a/index.test.js b/index.test.js index ab0a03bcc..56760dbd1 100644 --- a/index.test.js +++ b/index.test.js @@ -85,7 +85,7 @@ describe('ServerlessWebpack', () => { _.set(serverless, 'service.custom.webpack.webpackConfig', 'webpack.config.ts'); - const badDeps = function() { + const badDeps = function () { new ServerlessWebpack(serverless, {}); }; diff --git a/lib/Configuration.js b/lib/Configuration.js index a3fde813e..48941e4e2 100644 --- a/lib/Configuration.js +++ b/lib/Configuration.js @@ -14,7 +14,8 @@ const DefaultConfig = { packager: 'npm', packagerOptions: {}, keepOutputDirectory: false, - config: null + config: null, + concurrency: Infinity }; class Configuration { @@ -37,6 +38,21 @@ class Configuration { } } + // Concurrency may be passed via CLI, e.g. + // custom: + // webpack: + // concurrency: ${opt:compile-concurrency, 7} + // In this case it is typed as a string and we have to validate it + if (this._config.concurrency !== undefined) { + this._config.concurrency = Number(this._config.concurrency); + if (isNaN(this._config.concurrency) || this._config.concurrency < 1) { + throw new Error('concurrency option must be a positive number'); + } + } else if (this._config.serializedCompile === true) { + // Backwards compatibility with serializedCompile setting + this._config.concurrency = 1; + } + // Set defaults for all missing properties _.defaults(this._config, DefaultConfig); } @@ -73,6 +89,10 @@ class Configuration { return this._config.keepOutputDirectory; } + get concurrency() { + return this._config.concurrency; + } + toJSON() { return _.omitBy(this._config, _.isNil); } diff --git a/lib/Configuration.test.js b/lib/Configuration.test.js index b978eff8a..a4b4f023d 100644 --- a/lib/Configuration.test.js +++ b/lib/Configuration.test.js @@ -19,7 +19,8 @@ describe('Configuration', () => { packager: 'npm', packagerOptions: {}, keepOutputDirectory: false, - config: null + config: null, + concurrency: Infinity }; }); @@ -68,7 +69,8 @@ describe('Configuration', () => { packager: 'npm', packagerOptions: {}, keepOutputDirectory: false, - config: null + config: null, + concurrency: Infinity }); }); }); @@ -88,7 +90,8 @@ describe('Configuration', () => { packager: 'npm', packagerOptions: {}, keepOutputDirectory: false, - config: null + config: null, + concurrency: Infinity }); }); @@ -107,8 +110,43 @@ describe('Configuration', () => { packager: 'npm', packagerOptions: {}, keepOutputDirectory: false, - config: null + config: null, + concurrency: Infinity }); }); + + it('should accept a numeric string as concurrency value', () => { + const testCustom = { + webpack: { + includeModules: { forceInclude: ['mod1'] }, + webpackConfig: 'myWebpackFile.js', + concurrency: '3' + } + }; + const config = new Configuration(testCustom); + expect(config._config.concurrency).to.equal(3); + }); + + it('should not accept an invalid string as concurrency value', () => { + const testCustom = { + webpack: { + includeModules: { forceInclude: ['mod1'] }, + webpackConfig: 'myWebpackFile.js', + concurrency: '3abc' + } + }; + expect(() => new Configuration(testCustom)).throws(); + }); + + it('should not accept a non-positive number as concurrency value', () => { + const testCustom = { + webpack: { + includeModules: { forceInclude: ['mod1'] }, + webpackConfig: 'myWebpackFile.js', + concurrency: 0 + } + }; + expect(() => new Configuration(testCustom)).throws(); + }); }); }); diff --git a/lib/compile.js b/lib/compile.js index e5c2363cf..73902ab56 100644 --- a/lib/compile.js +++ b/lib/compile.js @@ -5,44 +5,62 @@ const BbPromise = require('bluebird'); const webpack = require('webpack'); const tty = require('tty'); +const defaultStatsConfig = { + colors: tty.isatty(process.stdout.fd), + hash: false, + version: false, + chunks: false, + children: false +}; + +function ensureArray(obj) { + return _.isArray(obj) ? obj : [obj]; +} + +function getStatsLogger(statsConfig, consoleLog) { + return stats => consoleLog(stats.toString(statsConfig || defaultStatsConfig)); +} + +function webpackCompile(config, logStats) { + return BbPromise + .fromCallback(cb => webpack(config).run(cb)) + .then(stats => { + // ensure stats in any array in the case of multiCompile + stats = stats.stats ? stats.stats : [stats]; + + _.forEach(stats, compileStats => { + logStats(compileStats); + if (compileStats.hasErrors()) { + throw new Error('Webpack compilation error, see stats above'); + } + }); + + return stats; + }); +} + +function webpackConcurrentCompile(configs, logStats, concurrency) { + return BbPromise + .map(configs, config => webpackCompile(config, logStats), { concurrency }) + .then(stats => _.flatten(stats)); +} + module.exports = { compile() { this.serverless.cli.log('Bundling with Webpack...'); - const compiler = webpack(this.webpackConfig); - - return BbPromise.fromCallback(cb => compiler.run(cb)).then(stats => { - if (!this.multiCompile) { - stats = { stats: [stats] }; - } - - const compileOutputPaths = []; - const consoleStats = this.webpackConfig.stats || - _.get(this, 'webpackConfig[0].stats') || { - colors: tty.isatty(process.stdout.fd), - hash: false, - version: false, - chunks: false, - children: false - }; - - _.forEach(stats.stats, compileStats => { - const statsOutput = compileStats.toString(consoleStats); - if (statsOutput) { - this.serverless.cli.consoleLog(statsOutput); - } + const configs = ensureArray(this.webpackConfig); + const logStats = getStatsLogger(configs[0].stats, this.serverless.cli.consoleLog); - if (compileStats.compilation.errors.length) { - throw new Error('Webpack compilation error, see above'); - } + if (!this.configuration) { + return BbPromise.reject('Missing plugin configuration'); + } + const concurrency = this.configuration.concurrency; - compileOutputPaths.push(compileStats.compilation.compiler.outputPath); + return webpackConcurrentCompile(configs, logStats, concurrency) + .then(stats => { + this.compileStats = { stats }; + return BbPromise.resolve(); }); - - this.compileOutputPaths = compileOutputPaths; - this.compileStats = stats; - - return BbPromise.resolve(); - }); } }; diff --git a/lib/packagers/yarn.test.js b/lib/packagers/yarn.test.js index 52445ad4a..63fb47435 100644 --- a/lib/packagers/yarn.test.js +++ b/lib/packagers/yarn.test.js @@ -134,11 +134,11 @@ describe('yarn', () => { acorn@^2.1.0, acorn@^2.4.0: version "2.7.0" resolved "https://registry.yarnpkg.com/acorn/-/acorn-2.7.0.tgz#ab6e7d9d886aaca8b085bc3312b79a198433f0e7" - + acorn@^3.0.4: version "3.3.0" resolved "https://registry.yarnpkg.com/acorn/-/acorn-3.3.0.tgz#45e37fb39e8da3f25baee3ff5369e2bb5f22017a" - + otherModule@file:../../otherModule/the-new-version: version "1.2.0" @@ -162,7 +162,7 @@ describe('yarn', () => { request "^2.83.0" ulid "^0.1.0" uuid "^3.1.0" - + acorn@^5.0.0, acorn@^5.5.0: version "5.5.3" resolved "https://registry.yarnpkg.com/acorn/-/acorn-5.5.3.tgz#f473dd47e0277a08e28e9bec5aeeb04751f0b8c9" @@ -172,11 +172,11 @@ describe('yarn', () => { acorn@^2.1.0, acorn@^2.4.0: version "2.7.0" resolved "https://registry.yarnpkg.com/acorn/-/acorn-2.7.0.tgz#ab6e7d9d886aaca8b085bc3312b79a198433f0e7" - + acorn@^3.0.4: version "3.3.0" resolved "https://registry.yarnpkg.com/acorn/-/acorn-3.3.0.tgz#45e37fb39e8da3f25baee3ff5369e2bb5f22017a" - + otherModule@file:../../project/../../otherModule/the-new-version: version "1.2.0" @@ -200,7 +200,7 @@ describe('yarn', () => { request "^2.83.0" ulid "^0.1.0" uuid "^3.1.0" - + acorn@^5.0.0, acorn@^5.5.0: version "5.5.3" resolved "https://registry.yarnpkg.com/acorn/-/acorn-5.5.3.tgz#f473dd47e0277a08e28e9bec5aeeb04751f0b8c9" diff --git a/lib/validate.js b/lib/validate.js index 80c0eb784..0659c3c05 100644 --- a/lib/validate.js +++ b/lib/validate.js @@ -171,6 +171,7 @@ module.exports = { this.webpackConfig.output.path = path.join(this.serverless.config.servicePath, this.options.out); } + // Skip compilation with --no-build if (this.skipCompile) { this.serverless.cli.log('Skipping build and using existing compiled output'); if (!fse.pathExistsSync(this.webpackConfig.output.path)) { @@ -187,8 +188,13 @@ module.exports = { // In case of individual packaging we have to create a separate config for each function if (_.has(this.serverless, 'service.package') && this.serverless.service.package.individually) { - this.options.verbose && this.serverless.cli.log('Using multi-compile (individual packaging)'); this.multiCompile = true; + this.options.verbose && + this.serverless.cli.log( + `Using ${ + this.configuration.concurrency !== Infinity ? 'concurrent' : 'multi' + }-compile (individual packaging)` + ); if (this.webpackConfig.entry && !_.isEqual(this.webpackConfig.entry, entries)) { return BbPromise.reject( diff --git a/tests/compile.test.js b/tests/compile.test.js index c0bcb672a..b9104b5b2 100644 --- a/tests/compile.test.js +++ b/tests/compile.test.js @@ -65,6 +65,7 @@ describe('compile', () => { it('should compile with webpack from a context configuration', () => { const testWebpackConfig = 'testconfig'; module.webpackConfig = testWebpackConfig; + module.configuration = { concurrency: Infinity }; return expect(module.compile()).to.be.fulfilled.then(() => { expect(webpackMock).to.have.been.calledWith(testWebpackConfig); expect(webpackMock.compilerMock.run).to.have.been.calledOnce; @@ -72,8 +73,16 @@ describe('compile', () => { }); }); + it('should fail if configuration is missing', () => { + const testWebpackConfig = 'testconfig'; + module.webpackConfig = testWebpackConfig; + module.configuration = undefined; + return expect(module.compile()).to.be.rejectedWith('Missing plugin configuration'); + }); + it('should fail if there are compilation errors', () => { module.webpackConfig = 'testconfig'; + module.configuration = { concurrency: Infinity }; // We stub errors here. It will be reset again in afterEach() sandbox.stub(webpackMock.statsMock.compilation, 'errors').value(['error']); return expect(module.compile()).to.be.rejectedWith(/compilation error/); @@ -81,19 +90,51 @@ describe('compile', () => { it('should work with multi compile', () => { const testWebpackConfig = 'testconfig'; - const multiStats = [ - { - compilation: { - errors: [], - compiler: { - outputPath: 'statsMock-outputPath' - } - }, - toString: sandbox.stub().returns('testStats') - } - ]; + const multiStats = { + stats: [ + { + compilation: { + errors: [], + compiler: { + outputPath: 'statsMock-outputPath' + } + }, + toString: sandbox.stub().returns('testStats'), + hasErrors: _.constant(false) + } + ] + }; + module.webpackConfig = testWebpackConfig; + module.multiCompile = true; + module.configuration = { concurrency: Infinity }; + webpackMock.compilerMock.run.reset(); + webpackMock.compilerMock.run.yields(null, multiStats); + return expect(module.compile()).to.be.fulfilled.then(() => { + expect(webpackMock).to.have.been.calledWith(testWebpackConfig); + expect(webpackMock.compilerMock.run).to.have.been.calledOnce; + return null; + }); + }); + + it('should work with concurrent compile', () => { + const testWebpackConfig = 'testconfig'; + const multiStats = { + stats: [ + { + compilation: { + errors: [], + compiler: { + outputPath: 'statsMock-outputPath' + } + }, + toString: sandbox.stub().returns('testStats'), + hasErrors: _.constant(false) + } + ] + }; module.webpackConfig = testWebpackConfig; module.multiCompile = true; + module.configuration = { concurrency: 1 }; webpackMock.compilerMock.run.reset(); webpackMock.compilerMock.run.yields(null, multiStats); return expect(module.compile()).to.be.fulfilled.then(() => { @@ -114,10 +155,12 @@ describe('compile', () => { outputPath: 'statsMock-outputPath' } }, - toString: sandbox.stub().returns('testStats') + toString: sandbox.stub().returns('testStats'), + hasErrors: _.constant(false) }; module.webpackConfig = testWebpackConfig; + module.configuration = { concurrency: Infinity }; webpackMock.compilerMock.run.reset(); webpackMock.compilerMock.run.yields(null, mockStats); return expect(module.compile()) @@ -128,7 +171,7 @@ describe('compile', () => { return expect(module.compile()).to.be.fulfilled; }) .then(() => { - expect(webpackMock).to.have.been.calledWith([testWebpackConfig]); + expect(webpackMock).to.have.been.calledWith(testWebpackConfig); expect(mockStats.toString.args).to.eql([ [testWebpackConfig.stats], [testWebpackConfig.stats] ]); return null; }); diff --git a/tests/webpack.mock.js b/tests/webpack.mock.js index abf911e6b..23e26518f 100644 --- a/tests/webpack.mock.js +++ b/tests/webpack.mock.js @@ -9,7 +9,10 @@ const StatsMock = () => ({ outputPath: 'statsMock-outputPath' } }, - toString: sinon.stub().returns('testStats') + toString: sinon.stub().returns('testStats'), + hasErrors() { + return Boolean(this.compilation.errors.length); + } }); const CompilerMock = (sandbox, statsMock) => ({