-
-
Notifications
You must be signed in to change notification settings - Fork 306
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add multi-process support to --check.
Set the parameter --num_threads to the desired number of worker tasks to potentially speed up --check. This works by spawning multiple sub-proccesses that each run the desired diagnostics on a subset of the workspace. Each process will still load and compile the entire workspace, so there are diminishing returns and memory usage increases linearly with the number of threads. Overall this can reduce the runtime by about ~50% for my projects, example results: Workspace 1, dominated by a few large/complex files 1 thread: 49.7 seconds 2 threads: 31.8 seconds 4 threads: 23.6 seconds 8 threads: 24.4 seconds Workspace 2, large number of small-ish files 1 thread: 96.0 seconds 2 threads: 76.5 seconds 4 threads: 49.5 seconds 8 threads: 38.1 seconds
- Loading branch information
Showing
7 changed files
with
256 additions
and
111 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,130 +1,96 @@ | ||
| local lclient = require 'lclient'() | ||
| local furi = require 'file-uri' | ||
| local ws = require 'workspace' | ||
| local files = require 'files' | ||
| local diag = require 'provider.diagnostic' | ||
| local util = require 'utility' | ||
| local jsonb = require 'json-beautify' | ||
| local lang = require 'language' | ||
| local define = require 'proto.define' | ||
| local config = require 'config.config' | ||
| local fs = require 'bee.filesystem' | ||
| local provider = require 'provider' | ||
| local lang = require 'language' | ||
| local platform = require 'bee.platform' | ||
| local subprocess = require 'bee.subprocess' | ||
| local json = require 'json' | ||
| local jsonb = require 'json-beautify' | ||
| local util = require 'utility' | ||
|
|
||
| require 'plugin' | ||
| require 'vm' | ||
|
|
||
| lang(LOCALE) | ||
| local numThreads = tonumber(NUM_THREADS or 1) | ||
|
|
||
| if type(CHECK) ~= 'string' then | ||
| print(lang.script('CLI_CHECK_ERROR_TYPE', type(CHECK))) | ||
| return | ||
| local exe = arg[-1] | ||
| -- TODO: is this necessary? got it from the shell.lua helper in bee.lua tests | ||
| if platform.os == 'windows' and not exe:match('%.[eE][xX][eE]$') then | ||
| exe = exe..'.exe' | ||
| end | ||
|
|
||
| local rootPath = fs.absolute(fs.path(CHECK)):string() | ||
| local rootUri = furi.encode(rootPath) | ||
| if not rootUri then | ||
| print(lang.script('CLI_CHECK_ERROR_URI', rootPath)) | ||
| return | ||
| local function logFileForThread(threadId) | ||
| return LOGPATH .. '/check-partial-' .. threadId .. '.json' | ||
| end | ||
| rootUri = rootUri:gsub("/$", "") | ||
|
|
||
| if CHECKLEVEL then | ||
| if not define.DiagnosticSeverity[CHECKLEVEL] then | ||
| print(lang.script('CLI_CHECK_ERROR_LEVEL', 'Error, Warning, Information, Hint')) | ||
| return | ||
| local function buildArgs(threadId) | ||
| local args = {exe} | ||
| local skipNext = false | ||
| for i = 1, #arg do | ||
| local arg = arg[i] | ||
| -- --check needs to be transformed into --check_worker | ||
| if arg:lower():match('^%-%-check$') or arg:lower():match('^%-%-check=') then | ||
| args[#args + 1] = arg:gsub('%-%-%w*', '--check_worker') | ||
| -- --check_out_path needs to be removed if we have more than one thread | ||
| elseif arg:lower():match('%-%-check_out_path') and numThreads > 1 then | ||
| if not arg:match('%-%-%w*=') then | ||
| skipNext = true | ||
| end | ||
| else | ||
| if skipNext then | ||
| skipNext = false | ||
| else | ||
| args[#args + 1] = arg | ||
| end | ||
| end | ||
| end | ||
| args[#args + 1] = '--thread_id' | ||
| args[#args + 1] = tostring(threadId) | ||
| if numThreads > 1 then | ||
| args[#args + 1] = '--quiet' | ||
| args[#args + 1] = '--check_out_path' | ||
| args[#args + 1] = logFileForThread(threadId) | ||
| end | ||
| return args | ||
| end | ||
| local checkLevel = define.DiagnosticSeverity[CHECKLEVEL] or define.DiagnosticSeverity.Warning | ||
|
|
||
| util.enableCloseFunction() | ||
|
|
||
| local lastClock = os.clock() | ||
| local results = {} | ||
|
|
||
| local function errorhandler(err) | ||
| print(err) | ||
| print(debug.traceback()) | ||
| if numThreads > 1 then | ||
| print(lang.script('CLI_CHECK_MULTIPLE_WORKERS', numThreads)) | ||
| end | ||
|
|
||
| ---@async | ||
| xpcall(lclient.start, errorhandler, lclient, function (client) | ||
| client:registerFakers() | ||
|
|
||
| client:initialize { | ||
| rootUri = rootUri, | ||
| } | ||
|
|
||
| client:register('textDocument/publishDiagnostics', function (params) | ||
| results[params.uri] = params.diagnostics | ||
| end) | ||
|
|
||
| io.write(lang.script('CLI_CHECK_INITING')) | ||
|
|
||
| provider.updateConfig(rootUri) | ||
|
|
||
| ws.awaitReady(rootUri) | ||
|
|
||
| local disables = util.arrayToHash(config.get(rootUri, 'Lua.diagnostics.disable')) | ||
| for name, serverity in pairs(define.DiagnosticDefaultSeverity) do | ||
| serverity = config.get(rootUri, 'Lua.diagnostics.severity')[name] or 'Warning' | ||
| if serverity:sub(-1) == '!' then | ||
| serverity = serverity:sub(1, -2) | ||
| end | ||
| if define.DiagnosticSeverity[serverity] > checkLevel then | ||
| disables[name] = true | ||
| end | ||
| local procs = {} | ||
| for i = 1, numThreads do | ||
| local process, err = subprocess.spawn({buildArgs(i)}) | ||
| if err then | ||
| print(err) | ||
| end | ||
| config.set(rootUri, 'Lua.diagnostics.disable', util.getTableKeys(disables, true)) | ||
|
|
||
| local uris = files.getChildFiles(rootUri) | ||
| local max = #uris | ||
| for i, uri in ipairs(uris) do | ||
| files.open(uri) | ||
| diag.doDiagnostic(uri, true) | ||
| -- Print regularly but always print the last entry to ensure that logs written to files don't look incomplete. | ||
| if os.clock() - lastClock > 0.2 or i == #uris then | ||
| lastClock = os.clock() | ||
| client:update() | ||
| local output = '\x0D' | ||
| .. ('>'):rep(math.ceil(i / max * 20)) | ||
| .. ('='):rep(20 - math.ceil(i / max * 20)) | ||
| .. ' ' | ||
| .. ('0'):rep(#tostring(max) - #tostring(i)) | ||
| .. tostring(i) .. '/' .. tostring(max) | ||
| io.write(output) | ||
| local filesWithErrors = 0 | ||
| local errors = 0 | ||
| for _, diags in pairs(results) do | ||
| filesWithErrors = filesWithErrors + 1 | ||
| errors = errors + #diags | ||
| end | ||
| if errors > 0 then | ||
| local errorDetails = ' [' .. lang.script('CLI_CHECK_PROGRESS', errors, filesWithErrors) .. ']' | ||
| io.write(errorDetails) | ||
| end | ||
| io.flush() | ||
| end | ||
| if process then | ||
| procs[#procs + 1] = process | ||
| end | ||
| io.write('\x0D') | ||
| end) | ||
| end | ||
|
|
||
| local count = 0 | ||
| for uri, result in pairs(results) do | ||
| count = count + #result | ||
| if #result == 0 then | ||
| results[uri] = nil | ||
| end | ||
| for _, process in ipairs(procs) do | ||
| process:wait() | ||
| end | ||
|
|
||
| if count == 0 then | ||
| print(lang.script('CLI_CHECK_SUCCESS')) | ||
| else | ||
| local outpath = CHECK_OUT_PATH | ||
| if outpath == nil then | ||
| outpath = LOGPATH .. '/check.json' | ||
| end | ||
| util.saveFile(outpath, jsonb.beautify(results)) | ||
| local outpath = CHECK_OUT_PATH | ||
| if outpath == nil then | ||
| outpath = LOGPATH .. '/check.json' | ||
| end | ||
|
|
||
| print(lang.script('CLI_CHECK_RESULTS', count, outpath)) | ||
| if numThreads > 1 then | ||
| local mergedResults = {} | ||
| local count = 0 | ||
| for i = 1, numThreads do | ||
| local result = json.decode(util.loadFile(logFileForThread(i)) or '[]') | ||
| for k, v in pairs(result) do | ||
| local entries = mergedResults[k] or {} | ||
| mergedResults[k] = entries | ||
| for _, entry in ipairs(v) do | ||
| entries[#entries + 1] = entry | ||
| count = count + 1 | ||
| end | ||
| end | ||
| end | ||
| util.saveFile(outpath, jsonb.beautify(mergedResults)) | ||
| if count == 0 then | ||
| print(lang.script('CLI_CHECK_SUCCESS')) | ||
| else | ||
| print(lang.script('CLI_CHECK_RESULTS', count, outpath)) | ||
| end | ||
| end |
Oops, something went wrong.