Skip to content

Commit 38d8332

Browse files
emmericpemmericp
committed
Add multi-process support to --check.
Set the parameter --num_threads to the desired number of worker tasks to potentially speed up --check. This works by spawning multiple sub-proccesses that each run the desired diagnostics on a subset of the workspace. Each process will still load and compile the entire workspace, so there are diminishing returns and memory usage increases linearly with the number of threads. Overall this can reduce the runtime by about ~50% for my projects, example results: Workspace 1, dominated by a few large/complex files 1 thread: 49.7 seconds 2 threads: 31.8 seconds 4 threads: 23.6 seconds 8 threads: 24.4 seconds Workspace 2, large number of small-ish files 1 thread: 96.0 seconds 2 threads: 76.5 seconds 4 threads: 49.5 seconds 8 threads: 38.1 seconds
1 parent 6b5e195 commit 38d8332

File tree

7 files changed

+256
-111
lines changed

7 files changed

+256
-111
lines changed

‎locale/en-us/script.lua

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -650,6 +650,8 @@ CLI_CHECK_PROGRESS =
650650
'Found {} problems in {} files'
651651
CLI_CHECK_RESULTS =
652652
'Diagnosis complete, {} problems found, see {}'
653+
CLI_CHECK_MULTIPLE_WORKERS =
654+
'Starting {} worker tasks, progress output will be disabled. This may take a few minutes.'
653655
CLI_DOC_INITING =
654656
'Loading documents ...'
655657
CLI_DOC_DONE =

‎locale/pt-br/script.lua

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -650,6 +650,8 @@ CLI_CHECK_PROGRESS = -- TODO: need translate!
650650
'Found {} problems in {} files'
651651
CLI_CHECK_RESULTS =
652652
'Diagnóstico completo, {} problemas encontrados, veja {}'
653+
CLI_CHECK_MULTIPLE_WORKERS = -- TODO: need translate!
654+
'Starting {} worker tasks, progress output will be disabled. This may take a few minutes.'
653655
CLI_DOC_INITING = -- TODO: need translate!
654656
'Loading documents ...'
655657
CLI_DOC_DONE = -- TODO: need translate!

‎locale/zh-cn/script.lua

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -650,6 +650,8 @@ CLI_CHECK_PROGRESS = -- TODO: need translate!
650650
'Found {} problems in {} files'
651651
CLI_CHECK_RESULTS =
652652
'诊断完成，共有 {} 个问题，请查看 {}'
653+
CLI_CHECK_MULTIPLE_WORKERS = -- TODO: need translate!
654+
'Starting {} worker tasks, progress output will be disabled. This may take a few minutes.'
653655
CLI_DOC_INITING =
654656
'加载文档 ...'
655657
CLI_DOC_DONE =

‎locale/zh-tw/script.lua

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -650,6 +650,8 @@ CLI_CHECK_PROGRESS = -- TODO: need translate!
650650
'Found {} problems in {} files'
651651
CLI_CHECK_RESULTS =
652652
'診斷完成，共有 {} 個問題，請查看 {}'
653+
CLI_CHECK_MULTIPLE_WORKERS = -- TODO: need translate!
654+
'Starting {} worker tasks, progress output will be disabled. This may take a few minutes.'
653655
CLI_DOC_INITING = -- TODO: need translate!
654656
'Loading documents ...'
655657
CLI_DOC_DONE = -- TODO: need translate!

‎script/cli/check.lua

Lines changed: 77 additions & 111 deletions
Original file line numberDiff line numberDiff line change
@@ -1,130 +1,96 @@
1-
local lclient = require 'lclient'()
2-
local furi = require 'file-uri'
3-
local ws = require 'workspace'
4-
local files = require 'files'
5-
local diag = require 'provider.diagnostic'
6-
local util = require 'utility'
7-
local jsonb = require 'json-beautify'
8-
local lang = require 'language'
9-
local define = require 'proto.define'
10-
local config = require 'config.config'
11-
local fs = require 'bee.filesystem'
12-
local provider = require 'provider'
1+
local lang = require 'language'
2+
local platform = require 'bee.platform'
3+
local subprocess = require 'bee.subprocess'
4+
local json = require 'json'
5+
local jsonb = require 'json-beautify'
6+
local util = require 'utility'
137

14-
require 'plugin'
15-
require 'vm'
168

17-
lang(LOCALE)
9+
local numThreads = tonumber(NUM_THREADS or 1)
1810

19-
if type(CHECK) ~= 'string' then
20-
print(lang.script('CLI_CHECK_ERROR_TYPE', type(CHECK)))
21-
return
11+
local exe = arg[-1]
12+
-- TODO: is this necessary? got it from the shell.lua helper in bee.lua tests
13+
if platform.os == 'windows' and not exe:match('%.[eE][xX][eE]$') then
14+
exe = exe..'.exe'
2215
end
2316

24-
local rootPath = fs.absolute(fs.path(CHECK)):string()
25-
local rootUri = furi.encode(rootPath)
26-
if not rootUri then
27-
print(lang.script('CLI_CHECK_ERROR_URI', rootPath))
28-
return
17+
local function logFileForThread(threadId)
18+
return LOGPATH .. '/check-partial-' .. threadId .. '.json'
2919
end
30-
rootUri = rootUri:gsub("/$", "")
3120

32-
if CHECKLEVEL then
33-
if not define.DiagnosticSeverity[CHECKLEVEL] then
34-
print(lang.script('CLI_CHECK_ERROR_LEVEL', 'Error, Warning, Information, Hint'))
35-
return
21+
local function buildArgs(threadId)
22+
local args = {exe}
23+
local skipNext = false
24+
for i = 1, #arg do
25+
local arg = arg[i]
26+
-- --check needs to be transformed into --check_worker
27+
if arg:lower():match('^%-%-check$') or arg:lower():match('^%-%-check=') then
28+
args[#args + 1] = arg:gsub('%-%-%w*', '--check_worker')
29+
-- --check_out_path needs to be removed if we have more than one thread
30+
elseif arg:lower():match('%-%-check_out_path') and numThreads > 1 then
31+
if not arg:match('%-%-%w*=') then
32+
skipNext = true
33+
end
34+
else
35+
if skipNext then
36+
skipNext = false
37+
else
38+
args[#args + 1] = arg
39+
end
40+
end
41+
end
42+
args[#args + 1] = '--thread_id'
43+
args[#args + 1] = tostring(threadId)
44+
if numThreads > 1 then
45+
args[#args + 1] = '--quiet'
46+
args[#args + 1] = '--check_out_path'
47+
args[#args + 1] = logFileForThread(threadId)
3648
end
49+
return args
3750
end
38-
local checkLevel = define.DiagnosticSeverity[CHECKLEVEL] or define.DiagnosticSeverity.Warning
39-
40-
util.enableCloseFunction()
4151

42-
local lastClock = os.clock()
43-
local results = {}
44-
45-
local function errorhandler(err)
46-
print(err)
47-
print(debug.traceback())
52+
if numThreads > 1 then
53+
print(lang.script('CLI_CHECK_MULTIPLE_WORKERS', numThreads))
4854
end
4955

50-
---@async
51-
xpcall(lclient.start, errorhandler, lclient, function (client)
52-
client:registerFakers()
53-
54-
client:initialize {
55-
rootUri = rootUri,
56-
}
57-
58-
client:register('textDocument/publishDiagnostics', function (params)
59-
results[params.uri] = params.diagnostics
60-
end)
61-
62-
io.write(lang.script('CLI_CHECK_INITING'))
63-
64-
provider.updateConfig(rootUri)
65-
66-
ws.awaitReady(rootUri)
67-
68-
local disables = util.arrayToHash(config.get(rootUri, 'Lua.diagnostics.disable'))
69-
for name, serverity in pairs(define.DiagnosticDefaultSeverity) do
70-
serverity = config.get(rootUri, 'Lua.diagnostics.severity')[name] or 'Warning'
71-
if serverity:sub(-1) == '!' then
72-
serverity = serverity:sub(1, -2)
73-
end
74-
if define.DiagnosticSeverity[serverity] > checkLevel then
75-
disables[name] = true
76-
end
56+
local procs = {}
57+
for i = 1, numThreads do
58+
local process, err = subprocess.spawn({buildArgs(i)})
59+
if err then
60+
print(err)
7761
end
78-
config.set(rootUri, 'Lua.diagnostics.disable', util.getTableKeys(disables, true))
79-
80-
local uris = files.getChildFiles(rootUri)
81-
local max = #uris
82-
for i, uri in ipairs(uris) do
83-
files.open(uri)
84-
diag.doDiagnostic(uri, true)
85-
-- Print regularly but always print the last entry to ensure that logs written to files don't look incomplete.
86-
if os.clock() - lastClock > 0.2 or i == #uris then
87-
lastClock = os.clock()
88-
client:update()
89-
local output = '\x0D'
90-
.. ('>'):rep(math.ceil(i / max * 20))
91-
.. ('='):rep(20 - math.ceil(i / max * 20))
92-
.. ' '
93-
.. ('0'):rep(#tostring(max) - #tostring(i))
94-
.. tostring(i) .. '/' .. tostring(max)
95-
io.write(output)
96-
local filesWithErrors = 0
97-
local errors = 0
98-
for _, diags in pairs(results) do
99-
filesWithErrors = filesWithErrors + 1
100-
errors = errors + #diags
101-
end
102-
if errors > 0 then
103-
local errorDetails = ' [' .. lang.script('CLI_CHECK_PROGRESS', errors, filesWithErrors) .. ']'
104-
io.write(errorDetails)
105-
end
106-
io.flush()
107-
end
62+
if process then
63+
procs[#procs + 1] = process
10864
end
109-
io.write('\x0D')
110-
end)
65+
end
11166

112-
local count = 0
113-
for uri, result in pairs(results) do
114-
count = count + #result
115-
if #result == 0 then
116-
results[uri] = nil
117-
end
67+
for _, process in ipairs(procs) do
68+
process:wait()
11869
end
11970

120-
if count == 0 then
121-
print(lang.script('CLI_CHECK_SUCCESS'))
122-
else
123-
local outpath = CHECK_OUT_PATH
124-
if outpath == nil then
125-
outpath = LOGPATH .. '/check.json'
126-
end
127-
util.saveFile(outpath, jsonb.beautify(results))
71+
local outpath = CHECK_OUT_PATH
72+
if outpath == nil then
73+
outpath = LOGPATH .. '/check.json'
74+
end
12875

129-
print(lang.script('CLI_CHECK_RESULTS', count, outpath))
76+
if numThreads > 1 then
77+
local mergedResults = {}
78+
local count = 0
79+
for i = 1, numThreads do
80+
local result = json.decode(util.loadFile(logFileForThread(i)) or '[]')
81+
for k, v in pairs(result) do
82+
local entries = mergedResults[k] or {}
83+
mergedResults[k] = entries
84+
for _, entry in ipairs(v) do
85+
entries[#entries + 1] = entry
86+
count = count + 1
87+
end
88+
end
89+
end
90+
util.saveFile(outpath, jsonb.beautify(mergedResults))
91+
if count == 0 then
92+
print(lang.script('CLI_CHECK_SUCCESS'))
93+
else
94+
print(lang.script('CLI_CHECK_RESULTS', count, outpath))
95+
end
13096
end

0 commit comments

Comments
 (0)