Skip to content

Commit 1601c02

Browse files
authored
feat: Chunked execution of linters on Windows only (#439)
* feat: Chunked execution of linters on Windows only * test: Add tests for resolveTaskFn Move some tests from makeCmdTasks to resolveTaskFn. * chore: Add babel-plugin-transform-object-rest-spread
1 parent 0924e78 commit 1601c02

18 files changed

+444
-326
lines changed

.babelrc

+2-1
Original file line numberDiff line numberDiff line change
@@ -5,5 +5,6 @@
55
"node": 6
66
}
77
}]
8-
]
8+
],
9+
"plugins": ["transform-object-rest-spread"]
910
}

README.md

+2-2
Original file line numberDiff line numberDiff line change
@@ -130,12 +130,12 @@ To set options and keep lint-staged extensible, advanced format can be used. Thi
130130
## Options
131131

132132
* `concurrent`_true_ — runs linters for each glob pattern simultaneously. If you don’t want this, you can set `concurrent: false`
133-
* `chunkSize` — Max allowed chunk size based on number of files for glob pattern. This is important on windows based systems to avoid command length limitations. See [#147](https://github.com/okonet/lint-staged/issues/147)
133+
* `chunkSize` — Max allowed chunk size based on number of files for glob pattern. This option is only applicable on Windows based systems to avoid command length limitations. See [#147](https://github.com/okonet/lint-staged/issues/147)
134134
* `globOptions``{ matchBase: true, dot: true }`[micromatch options](https://github.com/micromatch/micromatch#options) to
135135
customize how glob patterns match files.
136136
* `ignore` - `['**/docs/**/*.js']` - array of glob patterns to entirely ignore from any task.
137137
* `linters``Object` — keys (`String`) are glob patterns, values (`Array<String> | String`) are commands to execute.
138-
* `subTaskConcurrency``1` — Controls concurrency for processing chunks generated for each linter. Execution is **not** concurrent by default(see [#225](https://github.com/okonet/lint-staged/issues/225))
138+
* `subTaskConcurrency``1` — Controls concurrency for processing chunks generated for each linter. This option is only applicable on Windows. Execution is **not** concurrent by default(see [#225](https://github.com/okonet/lint-staged/issues/225))
139139

140140
## Filtering files
141141

package.json

+2
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
"execa": "^0.9.0",
3535
"find-parent-dir": "^0.3.0",
3636
"is-glob": "^4.0.0",
37+
"is-windows": "^1.0.2",
3738
"jest-validate": "^22.4.0",
3839
"listr": "^0.13.0",
3940
"lodash": "^4.17.5",
@@ -49,6 +50,7 @@
4950
"stringify-object": "^3.2.2"
5051
},
5152
"devDependencies": {
53+
"babel-plugin-transform-object-rest-spread": "^6.26.0",
5254
"babel-preset-env": "^1.6.0",
5355
"commitizen": "^2.9.6",
5456
"consolemock": "^1.0.2",

src/index.js

+1
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ const debug = require('debug')('lint-staged')
1212
// Force colors for packages that depend on https://www.npmjs.com/package/supports-color
1313
// but do this only in TTY mode
1414
if (process.stdout.isTTY) {
15+
// istanbul ignore next
1516
process.env.FORCE_COLOR = true
1617
}
1718

src/makeCmdTasks.js

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
'use strict'
2+
3+
const resolveTaskFn = require('./resolveTaskFn')
4+
const resolveGitDir = require('./resolveGitDir')
5+
6+
const debug = require('debug')('lint-staged:make-cmd-tasks')
7+
8+
/**
9+
* Creates and returns an array of listr tasks which map to the given commands.
10+
*
11+
* @param {Array<string>|string} commands
12+
* @param {Array<string>} pathsToLint
13+
* @param {Object} [options]
14+
* @param {number} options.chunkSize
15+
* @param {number} options.subTaskConcurrency
16+
*/
17+
module.exports = function makeCmdTasks(
18+
commands,
19+
pathsToLint,
20+
{ chunkSize = Number.MAX_SAFE_INTEGER, subTaskConcurrency = 1 } = {}
21+
) {
22+
debug('Creating listr tasks for commands %o', commands)
23+
24+
const gitDir = resolveGitDir()
25+
const lintersArray = Array.isArray(commands) ? commands : [commands]
26+
27+
return lintersArray.map(linter => ({
28+
title: linter,
29+
task: resolveTaskFn({
30+
linter,
31+
gitDir,
32+
pathsToLint,
33+
chunkSize,
34+
subTaskConcurrency
35+
})
36+
}))
37+
}

src/printErrors.js

+1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
'use strict'
22

3+
// istanbul ignore next
34
// Work-around for duplicated error logs, see #142
45
const errMsg = err => (err.privateMsg != null ? err.privateMsg : err.message)
56

src/resolveTaskFn.js

+118
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
'use strict'
2+
3+
const chunk = require('lodash/chunk')
4+
const dedent = require('dedent')
5+
const isWindows = require('is-windows')
6+
const execa = require('execa')
7+
const symbols = require('log-symbols')
8+
const pMap = require('p-map')
9+
const calcChunkSize = require('./calcChunkSize')
10+
const findBin = require('./findBin')
11+
12+
const debug = require('debug')('lint-staged:task')
13+
14+
/**
15+
* Execute the given linter binary with arguments and file paths using execa and
16+
* return the promise.
17+
*
18+
* @param {string} bin
19+
* @param {Array<string>} args
20+
* @param {Object} execaOptions
21+
* @param {Array<string>} pathsToLint
22+
* @return {Promise}
23+
*/
24+
function execLinter(bin, args, execaOptions, pathsToLint) {
25+
const binArgs = args.concat(pathsToLint)
26+
27+
debug('bin:', bin)
28+
debug('args: %O', binArgs)
29+
debug('opts: %o', execaOptions)
30+
31+
return execa(bin, binArgs, Object.assign({}, execaOptions))
32+
}
33+
34+
const successMsg = linter => `${symbols.success} ${linter} passed!`
35+
36+
/**
37+
* Create and returns an error instance with given stdout and stderr. If we set
38+
* the message on the error instance, it gets logged multiple times(see #142).
39+
* So we set the actual error message in a private field and extract it later,
40+
* log only once.
41+
*
42+
* @param {string} linter
43+
* @param {string} errStdout
44+
* @param {string} errStderr
45+
* @returns {Error}
46+
*/
47+
function makeErr(linter, errStdout, errStderr) {
48+
const err = new Error()
49+
err.privateMsg = dedent`
50+
${symbols.error} "${linter}" found some errors. Please fix them and try committing again.
51+
${errStdout}
52+
${errStderr}
53+
`
54+
return err
55+
}
56+
57+
/**
58+
* Returns the task function for the linter. It handles chunking for file paths
59+
* if the OS is Windows.
60+
*
61+
* @param {Object} options
62+
* @param {string} options.linter
63+
* @param {string} options.gitDir
64+
* @param {Array<string>} options.pathsToLint
65+
* @param {number} options.chunkSize
66+
* @param {number} options.subTaskConcurrency
67+
* @returns {function(): Promise<string>}
68+
*/
69+
module.exports = function resolveTaskFn(options) {
70+
const { linter, gitDir, pathsToLint } = options
71+
const { bin, args } = findBin(linter)
72+
73+
const execaOptions = { reject: false }
74+
// Only use gitDir as CWD if we are using the git binary
75+
// e.g `npm` should run tasks in the actual CWD
76+
if (/git(\.exe)?$/i.test(bin) && gitDir !== process.cwd()) {
77+
execaOptions.cwd = gitDir
78+
}
79+
80+
if (!isWindows()) {
81+
debug('%s OS: %s; File path chunking unnecessary', symbols.success, process.platform)
82+
return () =>
83+
execLinter(bin, args, execaOptions, pathsToLint).then(result => {
84+
if (!result.failed) return successMsg(linter)
85+
86+
throw makeErr(linter, result.stdout, result.stderr)
87+
})
88+
}
89+
90+
const { chunkSize, subTaskConcurrency: concurrency } = options
91+
92+
const filePathChunks = chunk(pathsToLint, calcChunkSize(pathsToLint, chunkSize))
93+
const mapper = execLinter.bind(null, bin, args, execaOptions)
94+
95+
debug(
96+
'OS: %s; Creating linter task with %d chunked file paths',
97+
process.platform,
98+
filePathChunks.length
99+
)
100+
return () =>
101+
pMap(filePathChunks, mapper, { concurrency })
102+
.catch(err => {
103+
/* This will probably never be called. But just in case.. */
104+
throw new Error(dedent`
105+
${symbols.error} ${linter} got an unexpected error.
106+
${err.message}
107+
`)
108+
})
109+
.then(results => {
110+
const errors = results.filter(res => res.failed)
111+
if (errors.length === 0) return successMsg(linter)
112+
113+
const errStdout = errors.map(err => err.stdout).join('')
114+
const errStderr = errors.map(err => err.stderr).join('')
115+
116+
throw makeErr(linter, errStdout, errStderr)
117+
})
118+
}

src/runAll.js

+15-9
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ const sgf = require('staged-git-files')
44
const Listr = require('listr')
55
const has = require('lodash/has')
66
const pify = require('pify')
7-
const runScript = require('./runScript')
7+
const makeCmdTasks = require('./makeCmdTasks')
88
const generateTasks = require('./generateTasks')
99
const resolveGitDir = require('./resolveGitDir')
1010

@@ -22,7 +22,7 @@ module.exports = function runAll(config) {
2222
throw new Error('Invalid config provided to runAll! Use getConfig instead.')
2323
}
2424

25-
const { concurrent, renderer } = config
25+
const { concurrent, renderer, chunkSize, subTaskConcurrency } = config
2626
const gitDir = resolveGitDir()
2727
debug('Resolved git directory to be `%s`', gitDir)
2828

@@ -35,13 +35,19 @@ module.exports = function runAll(config) {
3535
const tasks = generateTasks(config, filenames).map(task => ({
3636
title: `Running tasks for ${task.pattern}`,
3737
task: () =>
38-
new Listr(runScript(task.commands, task.fileList, config), {
39-
// In sub-tasks we don't want to run concurrently
40-
// and we want to abort on errors
41-
dateFormat: false,
42-
concurrent: false,
43-
exitOnError: true
44-
}),
38+
new Listr(
39+
makeCmdTasks(task.commands, task.fileList, {
40+
chunkSize,
41+
subTaskConcurrency
42+
}),
43+
{
44+
// In sub-tasks we don't want to run concurrently
45+
// and we want to abort on errors
46+
dateFormat: false,
47+
concurrent: false,
48+
exitOnError: true
49+
}
50+
),
4551
skip: () => {
4652
if (task.fileList.length === 0) {
4753
return `No staged files match ${task.pattern}`

src/runScript.js

-79
This file was deleted.

test/findBin.spec.js

-2
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,6 @@ import makeConsoleMock from 'consolemock'
22
import npmWhichMock from 'npm-which'
33
import findBin from '../src/findBin'
44

5-
jest.mock('npm-which')
6-
75
describe('findBin', () => {
86
it('should return path to bin', () => {
97
const { bin, args } = findBin('my-linter')

test/index.spec.js

-2
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,6 @@ const replaceSerializer = (from, to) => ({
88
print: val => val.replace(from, to)
99
})
1010

11-
jest.mock('cosmiconfig')
12-
1311
const mockCosmiconfigWith = result => {
1412
cosmiconfig.mockImplementationOnce(() => ({
1513
load: () => Promise.resolve(result)

test/makeCmdTasks.spec.js

+44
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
import execa from 'execa'
2+
import makeCmdTasks from '../src/makeCmdTasks'
3+
4+
describe('makeCmdTasks', () => {
5+
beforeEach(() => {
6+
execa.mockClear()
7+
})
8+
9+
it('should return an array', () => {
10+
expect(makeCmdTasks('test', ['test.js'])).toBeInstanceOf(Array)
11+
})
12+
13+
it('should work with a single command', async () => {
14+
expect.assertions(4)
15+
const res = makeCmdTasks('test', ['test.js'])
16+
expect(res.length).toBe(1)
17+
const [linter] = res
18+
expect(linter.title).toBe('test')
19+
expect(linter.task).toBeInstanceOf(Function)
20+
const taskPromise = linter.task()
21+
expect(taskPromise).toBeInstanceOf(Promise)
22+
await taskPromise
23+
})
24+
25+
it('should work with multiple commands', async () => {
26+
expect.assertions(9)
27+
const res = makeCmdTasks(['test', 'test2'], ['test.js'])
28+
expect(res.length).toBe(2)
29+
const [linter1, linter2] = res
30+
expect(linter1.title).toBe('test')
31+
expect(linter2.title).toBe('test2')
32+
33+
let taskPromise = linter1.task()
34+
expect(taskPromise).toBeInstanceOf(Promise)
35+
await taskPromise
36+
expect(execa).toHaveBeenCalledTimes(1)
37+
expect(execa).lastCalledWith('test', ['test.js'], { reject: false })
38+
taskPromise = linter2.task()
39+
expect(taskPromise).toBeInstanceOf(Promise)
40+
await taskPromise
41+
expect(execa).toHaveBeenCalledTimes(2)
42+
expect(execa).lastCalledWith('test2', ['test.js'], { reject: false })
43+
})
44+
})

0 commit comments

Comments
 (0)