From 3ba58f657238a9163bce5016b5d7a136b592eb8f Mon Sep 17 00:00:00 2001 From: Kartik Raj Date: Wed, 15 Nov 2023 16:54:15 -0800 Subject: [PATCH 01/30] Add utilities to exec and spawn inside worker threads --- src/client/common/process/worker/main.ts | 27 +++ .../common/process/worker/plainExecWorker.ts | 16 ++ .../process/worker/rawProcessApiWrapper.ts | 43 ++++ .../common/process/worker/shellExecWorker.ts | 16 ++ src/client/common/process/worker/types.ts | 38 ++++ .../process/worker/workerRawProcessApis.ts | 213 ++++++++++++++++++ 6 files changed, 353 insertions(+) create mode 100644 src/client/common/process/worker/main.ts create mode 100644 src/client/common/process/worker/plainExecWorker.ts create mode 100644 src/client/common/process/worker/rawProcessApiWrapper.ts create mode 100644 src/client/common/process/worker/shellExecWorker.ts create mode 100644 src/client/common/process/worker/types.ts create mode 100644 src/client/common/process/worker/workerRawProcessApis.ts diff --git a/src/client/common/process/worker/main.ts b/src/client/common/process/worker/main.ts new file mode 100644 index 000000000000..d8f158a73c70 --- /dev/null +++ b/src/client/common/process/worker/main.ts @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { Worker } from 'worker_threads'; +import { traceError } from '../../../logging'; + +// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/explicit-module-boundary-types +export async function executeWorkerFile(workerFileName: string, workerData: any): Promise { + return new Promise((resolve, reject) => { + const worker = new Worker(workerFileName, { workerData }); + worker.on('message', (res: { err: Error; res: unknown }) => { + if (res.err) { + reject(res.err); + } + resolve(res.res); + }); + worker.on('error', (ex: Error) => { + traceError(`Error in worker ${workerFileName}`, ex); + reject(ex); + }); + worker.on('exit', (code) => { + if (code !== 0) { + reject(new Error(`Worker ${workerFileName} stopped with exit code ${code}`)); + } + }); + }); +} diff --git a/src/client/common/process/worker/plainExecWorker.ts b/src/client/common/process/worker/plainExecWorker.ts new file mode 100644 index 000000000000..69374eba53d7 --- /dev/null +++ b/src/client/common/process/worker/plainExecWorker.ts @@ -0,0 +1,16 @@ +import { parentPort, workerData } from 'worker_threads'; +import { workerPlainExec } from './workerRawProcessApis'; + +workerPlainExec(workerData.file, workerData.args, workerData.options, workerData.defaultEnv, workerData.disposables) + .then((res) => { + if (!parentPort) { + throw new Error('Not in a worker thread'); + } + parentPort.postMessage({ res }); + }) + .catch((err) => { + if (!parentPort) { + throw new Error('Not in a worker thread'); + } + parentPort.postMessage({ err }); + }); diff --git a/src/client/common/process/worker/rawProcessApiWrapper.ts b/src/client/common/process/worker/rawProcessApiWrapper.ts new file mode 100644 index 000000000000..dc984a1439d8 --- /dev/null +++ b/src/client/common/process/worker/rawProcessApiWrapper.ts @@ -0,0 +1,43 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { SpawnOptions } from 'child_process'; +import * as path from 'path'; +import { WorkspaceService } from '../../application/workspace'; +import { ProcessLogger } from '../logger'; +import { executeWorkerFile } from './main'; +import { EnvironmentVariables, ExecutionResult, IDisposable, ShellOptions } from './types'; + +export function shellExec( + command: string, + options: ShellOptions, + defaultEnv?: EnvironmentVariables, + disposables?: Set, +): Promise> { + const processLogger = new ProcessLogger(new WorkspaceService()); + processLogger.logProcess(command, undefined, options); + return executeWorkerFile(path.join(__dirname, 'shellExecWorker.js'), { + command, + options, + defaultEnv, + disposables, + }); +} + +export function plainExec( + file: string, + args: string[], + options: SpawnOptions & { doNotLog?: boolean } = {}, + defaultEnv?: EnvironmentVariables, + disposables?: Set, +): Promise> { + const processLogger = new ProcessLogger(new WorkspaceService()); + processLogger.logProcess(file, args, options); + return executeWorkerFile(path.join(__dirname, 'plainExecWorker.js'), { + file, + args, + options, + defaultEnv, + disposables, + }); +} diff --git a/src/client/common/process/worker/shellExecWorker.ts b/src/client/common/process/worker/shellExecWorker.ts new file mode 100644 index 000000000000..739d0bc1b6d5 --- /dev/null +++ b/src/client/common/process/worker/shellExecWorker.ts @@ -0,0 +1,16 @@ +import { parentPort, workerData } from 'worker_threads'; +import { workerShellExec } from './workerRawProcessApis'; + +workerShellExec(workerData.command, workerData.options, workerData.defaultEnv, workerData.disposables) + .then((res) => { + if (!parentPort) { + throw new Error('Not in a worker thread'); + } + parentPort.postMessage(res); + }) + .catch((ex) => { + if (!parentPort) { + throw new Error('Not in a worker thread'); + } + parentPort.postMessage(ex); + }); diff --git a/src/client/common/process/worker/types.ts b/src/client/common/process/worker/types.ts new file mode 100644 index 000000000000..5c58aec10214 --- /dev/null +++ b/src/client/common/process/worker/types.ts @@ -0,0 +1,38 @@ +/* eslint-disable @typescript-eslint/no-empty-function */ +/* eslint-disable @typescript-eslint/explicit-module-boundary-types */ +import { ExecOptions, SpawnOptions as ChildProcessSpawnOptions } from 'child_process'; + +export function noop() {} +export interface IDisposable { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + dispose(): void | undefined | Promise; +} +export type EnvironmentVariables = Record; +export class StdErrError extends Error { + // eslint-disable-next-line @typescript-eslint/no-useless-constructor + constructor(message: string) { + super(message); + } +} + +export type SpawnOptions = ChildProcessSpawnOptions & { + encoding?: string; + // /** + // * Can't use `CancellationToken` here as it comes from vscode which is not available in worker threads. + // */ + // token?: CancellationToken; + mergeStdOutErr?: boolean; + throwOnStdErr?: boolean; + extraVariables?: NodeJS.ProcessEnv; + // /** + // * Can't use `OutputChannel` here as it comes from vscode which is not available in worker threads. + // */ + // outputChannel?: OutputChannel; + stdinStr?: string; +}; +export type ShellOptions = ExecOptions & { throwOnStdErr?: boolean }; + +export type ExecutionResult = { + stdout: T; + stderr?: T; +}; diff --git a/src/client/common/process/worker/workerRawProcessApis.ts b/src/client/common/process/worker/workerRawProcessApis.ts new file mode 100644 index 000000000000..1c1c6ca5b46b --- /dev/null +++ b/src/client/common/process/worker/workerRawProcessApis.ts @@ -0,0 +1,213 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// !!!! IMPORTANT: DO NOT IMPORT FROM VSCODE MODULE AS IT IS NOT AVAILABLE INSIDE WORKER THREADS !!!! + +import { exec, execSync, spawn } from 'child_process'; +import { Readable } from 'stream'; +import { createDeferred } from '../../utils/async'; +import { DEFAULT_ENCODING } from '../constants'; +import { decodeBuffer } from '../decoder'; +import { + ShellOptions, + SpawnOptions, + EnvironmentVariables, + IDisposable, + noop, + StdErrError, + ExecutionResult, +} from './types'; + +const PS_ERROR_SCREEN_BOGUS = /your [0-9]+x[0-9]+ screen size is bogus\. expect trouble/; + +function getDefaultOptions(options: T, defaultEnv?: EnvironmentVariables): T { + const defaultOptions = { ...options }; + const execOptions = defaultOptions as SpawnOptions; + if (execOptions) { + execOptions.encoding = + typeof execOptions.encoding === 'string' && execOptions.encoding.length > 0 + ? execOptions.encoding + : DEFAULT_ENCODING; + const { encoding } = execOptions; + delete execOptions.encoding; + execOptions.encoding = encoding; + } + if (!defaultOptions.env || Object.keys(defaultOptions.env).length === 0) { + const env = defaultEnv || process.env; + defaultOptions.env = { ...env }; + } else { + defaultOptions.env = { ...defaultOptions.env }; + } + + if (execOptions && execOptions.extraVariables) { + defaultOptions.env = { ...defaultOptions.env, ...execOptions.extraVariables }; + } + + // Always ensure we have unbuffered output. + defaultOptions.env.PYTHONUNBUFFERED = '1'; + if (!defaultOptions.env.PYTHONIOENCODING) { + defaultOptions.env.PYTHONIOENCODING = 'utf-8'; + } + + return defaultOptions; +} + +export function workerShellExec( + command: string, + options: ShellOptions, + defaultEnv?: EnvironmentVariables, + disposables?: Set, +): Promise> { + const shellOptions = getDefaultOptions(options, defaultEnv); + return new Promise((resolve, reject) => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const callback = (e: any, stdout: any, stderr: any) => { + if (e && e !== null) { + reject(e); + } else if (shellOptions.throwOnStdErr && stderr && stderr.length) { + reject(new Error(stderr)); + } else { + stdout = filterOutputUsingCondaRunMarkers(stdout); + // Make sure stderr is undefined if we actually had none. This is checked + // elsewhere because that's how exec behaves. + resolve({ stderr: stderr && stderr.length > 0 ? stderr : undefined, stdout }); + } + }; + let procExited = false; + const proc = exec(command, shellOptions, callback); // NOSONAR + proc.once('close', () => { + procExited = true; + }); + proc.once('exit', () => { + procExited = true; + }); + proc.once('error', () => { + procExited = true; + }); + const disposable: IDisposable = { + dispose: () => { + // If process has not exited nor killed, force kill it. + if (!procExited && !proc.killed) { + if (proc.pid) { + killPid(proc.pid); + } else { + proc.kill(); + } + } + }, + }; + if (disposables) { + disposables.add(disposable); + } + }); +} + +export function workerPlainExec( + file: string, + args: string[], + options: SpawnOptions & { doNotLog?: boolean } = {}, + defaultEnv?: EnvironmentVariables, + disposables?: Set, +): Promise> { + const spawnOptions = getDefaultOptions(options, defaultEnv); + const encoding = spawnOptions.encoding ? spawnOptions.encoding : 'utf8'; + const proc = spawn(file, args, spawnOptions); + // Listen to these errors (unhandled errors in streams tears down the process). + // Errors will be bubbled up to the `error` event in `proc`, hence no need to log. + proc.stdout?.on('error', noop); + proc.stderr?.on('error', noop); + const deferred = createDeferred>(); + const disposable: IDisposable = { + dispose: () => { + // If process has not exited nor killed, force kill it. + if (!proc.killed && !deferred.completed) { + if (proc.pid) { + killPid(proc.pid); + } else { + proc.kill(); + } + } + }, + }; + disposables?.add(disposable); + const internalDisposables: IDisposable[] = []; + + // eslint-disable-next-line @typescript-eslint/ban-types + const on = (ee: Readable | null, name: string, fn: Function) => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + ee?.on(name, fn as any); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + internalDisposables.push({ dispose: () => ee?.removeListener(name, fn as any) as any }); + }; + + // Tokens not supported yet as they come from vscode module which is not available. + // if (options.token) { + // internalDisposables.push(options.token.onCancellationRequested(disposable.dispose)); + // } + + const stdoutBuffers: Buffer[] = []; + on(proc.stdout, 'data', (data: Buffer) => { + stdoutBuffers.push(data); + }); + const stderrBuffers: Buffer[] = []; + on(proc.stderr, 'data', (data: Buffer) => { + if (options.mergeStdOutErr) { + stdoutBuffers.push(data); + stderrBuffers.push(data); + } else { + stderrBuffers.push(data); + } + }); + + proc.once('close', () => { + if (deferred.completed) { + return; + } + const stderr: string | undefined = + stderrBuffers.length === 0 ? undefined : decodeBuffer(stderrBuffers, encoding); + if ( + stderr && + stderr.length > 0 && + options.throwOnStdErr && + // ignore this specific error silently; see this issue for context: https://github.com/microsoft/vscode/issues/75932 + !(PS_ERROR_SCREEN_BOGUS.test(stderr) && stderr.replace(PS_ERROR_SCREEN_BOGUS, '').trim().length === 0) + ) { + deferred.reject(new StdErrError(stderr)); + } else { + let stdout = decodeBuffer(stdoutBuffers, encoding); + stdout = filterOutputUsingCondaRunMarkers(stdout); + deferred.resolve({ stdout, stderr }); + } + internalDisposables.forEach((d) => d.dispose()); + disposable.dispose(); + }); + proc.once('error', (ex) => { + deferred.reject(ex); + internalDisposables.forEach((d) => d.dispose()); + disposable.dispose(); + }); + + return deferred.promise; +} + +function filterOutputUsingCondaRunMarkers(stdout: string) { + // These markers are added if conda run is used or `interpreterInfo.py` is + // run, see `get_output_via_markers.py`. + const regex = />>>PYTHON-EXEC-OUTPUT([\s\S]*)<<= 2 ? match[1].trim() : undefined; + return filteredOut !== undefined ? filteredOut : stdout; +} + +function killPid(pid: number): void { + try { + if (process.platform === 'win32') { + // Windows doesn't support SIGTERM, so execute taskkill to kill the process + execSync(`taskkill /pid ${pid} /T /F`); // NOSONAR + } else { + process.kill(pid); + } + } catch { + console.warn('Unable to kill process with pid', pid); + } +} From 384e56af37269d5f1446d18293138c368af38103 Mon Sep 17 00:00:00 2001 From: Kartik Raj Date: Wed, 15 Nov 2023 17:31:54 -0800 Subject: [PATCH 02/30] Shell exec and exec using worker threads in discovery --- .../common/externalDependencies.ts | 46 ++++++++----------- .../common/windowsRegistry.ts | 2 +- 2 files changed, 20 insertions(+), 28 deletions(-) diff --git a/src/client/pythonEnvironments/common/externalDependencies.ts b/src/client/pythonEnvironments/common/externalDependencies.ts index 357967236c9e..70ca21d4a8a5 100644 --- a/src/client/pythonEnvironments/common/externalDependencies.ts +++ b/src/client/pythonEnvironments/common/externalDependencies.ts @@ -3,7 +3,6 @@ import * as fsapi from 'fs-extra'; import * as path from 'path'; -import { Worker } from 'worker_threads'; import * as vscode from 'vscode'; import { IWorkspaceService } from '../../common/application/types'; import { ExecutionResult, IProcessServiceFactory, ShellOptions, SpawnOptions } from '../../common/process/types'; @@ -12,6 +11,9 @@ import { chain, iterable } from '../../common/utils/async'; import { getOSType, OSType } from '../../common/utils/platform'; import { IServiceContainer } from '../../ioc/types'; import { traceError, traceVerbose } from '../../logging'; +import { DiscoveryUsingWorkers } from '../../common/experiments/groups'; +import { plainExec, shellExec } from '../../common/process/worker/rawProcessApiWrapper'; +import { IEnvironmentVariablesProvider } from '../../common/variables/types'; let internalServiceContainer: IServiceContainer; export function initializeExternalDependencies(serviceContainer: IServiceContainer): void { @@ -21,13 +23,25 @@ export function initializeExternalDependencies(serviceContainer: IServiceContain // processes export async function shellExecute(command: string, options: ShellOptions = {}): Promise> { - const service = await internalServiceContainer.get(IProcessServiceFactory).create(); - return service.shellExec(command, options); + if (inExperiment(DiscoveryUsingWorkers.experiment)) { + const service = await internalServiceContainer.get(IProcessServiceFactory).create(); + return service.shellExec(command, options); + } + const envVarsService = internalServiceContainer.get(IEnvironmentVariablesProvider); + const envs = await envVarsService.getEnvironmentVariables(); + options.env = { ...options.env, ...envs }; + return shellExec(command, options); } export async function exec(file: string, args: string[], options: SpawnOptions = {}): Promise> { - const service = await internalServiceContainer.get(IProcessServiceFactory).create(); - return service.exec(file, args, options); + if (inExperiment(DiscoveryUsingWorkers.experiment)) { + const service = await internalServiceContainer.get(IProcessServiceFactory).create(); + return service.exec(file, args, options); + } + const envVarsService = internalServiceContainer.get(IEnvironmentVariablesProvider); + const envs = await envVarsService.getEnvironmentVariables(); + options.env = { ...options.env, ...envs }; + return plainExec(file, args, options); } export function inExperiment(experimentName: string): boolean { @@ -201,25 +215,3 @@ export function onDidChangePythonSetting(name: string, callback: () => void, roo } }); } - -// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/explicit-module-boundary-types -export async function executeWorkerFile(workerFileName: string, workerData: any): Promise { - return new Promise((resolve, reject) => { - const worker = new Worker(workerFileName, { workerData }); - worker.on('message', (res: { err: Error; res: unknown }) => { - if (res.err) { - reject(res.err); - } - resolve(res.res); - }); - worker.on('error', (ex: Error) => { - traceError(`Error in worker ${workerFileName}`, ex); - reject(ex); - }); - worker.on('exit', (code) => { - if (code !== 0) { - reject(new Error(`Worker ${workerFileName} stopped with exit code ${code}`)); - } - }); - }); -} diff --git a/src/client/pythonEnvironments/common/windowsRegistry.ts b/src/client/pythonEnvironments/common/windowsRegistry.ts index efac5bb3209f..ac58963a6c14 100644 --- a/src/client/pythonEnvironments/common/windowsRegistry.ts +++ b/src/client/pythonEnvironments/common/windowsRegistry.ts @@ -5,7 +5,7 @@ import { HKCU, HKLM, Options, REG_SZ, Registry, RegistryItem } from 'winreg'; import * as path from 'path'; import { createDeferred } from '../../common/utils/async'; -import { executeWorkerFile } from './externalDependencies'; +import { executeWorkerFile } from '../../common/process/worker/main'; export { HKCU, HKLM, REG_SZ, Options }; From 487afe8ef12a848f33e2338d9f3043c7421da3fa Mon Sep 17 00:00:00 2001 From: Kartik Raj Date: Wed, 15 Nov 2023 17:51:20 -0800 Subject: [PATCH 03/30] Use worker threads for fetching conda environments --- src/client/common/process/worker/plainExecWorker.ts | 10 ++++++++-- .../common/process/worker/rawProcessApiWrapper.ts | 10 +++------- src/client/common/process/worker/shellExecWorker.ts | 8 ++++---- .../common/process/worker/workerRawProcessApis.ts | 4 ++-- .../pythonEnvironments/common/externalDependencies.ts | 10 +++++----- 5 files changed, 22 insertions(+), 20 deletions(-) diff --git a/src/client/common/process/worker/plainExecWorker.ts b/src/client/common/process/worker/plainExecWorker.ts index 69374eba53d7..9893142ed51c 100644 --- a/src/client/common/process/worker/plainExecWorker.ts +++ b/src/client/common/process/worker/plainExecWorker.ts @@ -1,7 +1,13 @@ import { parentPort, workerData } from 'worker_threads'; -import { workerPlainExec } from './workerRawProcessApis'; +import { _workerPlainExecImpl } from './workerRawProcessApis'; -workerPlainExec(workerData.file, workerData.args, workerData.options, workerData.defaultEnv, workerData.disposables) +_workerPlainExecImpl( + workerData.file, + workerData.args, + workerData.options, + workerData.defaultEnv, + workerData.disposables, +) .then((res) => { if (!parentPort) { throw new Error('Not in a worker thread'); diff --git a/src/client/common/process/worker/rawProcessApiWrapper.ts b/src/client/common/process/worker/rawProcessApiWrapper.ts index dc984a1439d8..ab478fc1aa77 100644 --- a/src/client/common/process/worker/rawProcessApiWrapper.ts +++ b/src/client/common/process/worker/rawProcessApiWrapper.ts @@ -6,13 +6,12 @@ import * as path from 'path'; import { WorkspaceService } from '../../application/workspace'; import { ProcessLogger } from '../logger'; import { executeWorkerFile } from './main'; -import { EnvironmentVariables, ExecutionResult, IDisposable, ShellOptions } from './types'; +import { EnvironmentVariables, ExecutionResult, ShellOptions } from './types'; -export function shellExec( +export function workerShellExec( command: string, options: ShellOptions, defaultEnv?: EnvironmentVariables, - disposables?: Set, ): Promise> { const processLogger = new ProcessLogger(new WorkspaceService()); processLogger.logProcess(command, undefined, options); @@ -20,16 +19,14 @@ export function shellExec( command, options, defaultEnv, - disposables, }); } -export function plainExec( +export function workerPlainExec( file: string, args: string[], options: SpawnOptions & { doNotLog?: boolean } = {}, defaultEnv?: EnvironmentVariables, - disposables?: Set, ): Promise> { const processLogger = new ProcessLogger(new WorkspaceService()); processLogger.logProcess(file, args, options); @@ -38,6 +35,5 @@ export function plainExec( args, options, defaultEnv, - disposables, }); } diff --git a/src/client/common/process/worker/shellExecWorker.ts b/src/client/common/process/worker/shellExecWorker.ts index 739d0bc1b6d5..ffaa5f435484 100644 --- a/src/client/common/process/worker/shellExecWorker.ts +++ b/src/client/common/process/worker/shellExecWorker.ts @@ -1,16 +1,16 @@ import { parentPort, workerData } from 'worker_threads'; -import { workerShellExec } from './workerRawProcessApis'; +import { _workerShellExecImpl } from './workerRawProcessApis'; -workerShellExec(workerData.command, workerData.options, workerData.defaultEnv, workerData.disposables) +_workerShellExecImpl(workerData.command, workerData.options, workerData.defaultEnv, workerData.disposables) .then((res) => { if (!parentPort) { throw new Error('Not in a worker thread'); } - parentPort.postMessage(res); + parentPort.postMessage({ res }); }) .catch((ex) => { if (!parentPort) { throw new Error('Not in a worker thread'); } - parentPort.postMessage(ex); + parentPort.postMessage({ ex }); }); diff --git a/src/client/common/process/worker/workerRawProcessApis.ts b/src/client/common/process/worker/workerRawProcessApis.ts index 1c1c6ca5b46b..5b04aaa40b0a 100644 --- a/src/client/common/process/worker/workerRawProcessApis.ts +++ b/src/client/common/process/worker/workerRawProcessApis.ts @@ -52,7 +52,7 @@ function getDefaultOptions(options: T, de return defaultOptions; } -export function workerShellExec( +export function _workerShellExecImpl( command: string, options: ShellOptions, defaultEnv?: EnvironmentVariables, @@ -102,7 +102,7 @@ export function workerShellExec( }); } -export function workerPlainExec( +export function _workerPlainExecImpl( file: string, args: string[], options: SpawnOptions & { doNotLog?: boolean } = {}, diff --git a/src/client/pythonEnvironments/common/externalDependencies.ts b/src/client/pythonEnvironments/common/externalDependencies.ts index 70ca21d4a8a5..8392ac5c0c73 100644 --- a/src/client/pythonEnvironments/common/externalDependencies.ts +++ b/src/client/pythonEnvironments/common/externalDependencies.ts @@ -12,7 +12,7 @@ import { getOSType, OSType } from '../../common/utils/platform'; import { IServiceContainer } from '../../ioc/types'; import { traceError, traceVerbose } from '../../logging'; import { DiscoveryUsingWorkers } from '../../common/experiments/groups'; -import { plainExec, shellExec } from '../../common/process/worker/rawProcessApiWrapper'; +import { workerPlainExec, workerShellExec } from '../../common/process/worker/rawProcessApiWrapper'; import { IEnvironmentVariablesProvider } from '../../common/variables/types'; let internalServiceContainer: IServiceContainer; @@ -23,25 +23,25 @@ export function initializeExternalDependencies(serviceContainer: IServiceContain // processes export async function shellExecute(command: string, options: ShellOptions = {}): Promise> { - if (inExperiment(DiscoveryUsingWorkers.experiment)) { + if (!inExperiment(DiscoveryUsingWorkers.experiment)) { const service = await internalServiceContainer.get(IProcessServiceFactory).create(); return service.shellExec(command, options); } const envVarsService = internalServiceContainer.get(IEnvironmentVariablesProvider); const envs = await envVarsService.getEnvironmentVariables(); options.env = { ...options.env, ...envs }; - return shellExec(command, options); + return workerShellExec(command, options); } export async function exec(file: string, args: string[], options: SpawnOptions = {}): Promise> { - if (inExperiment(DiscoveryUsingWorkers.experiment)) { + if (!inExperiment(DiscoveryUsingWorkers.experiment)) { const service = await internalServiceContainer.get(IProcessServiceFactory).create(); return service.exec(file, args, options); } const envVarsService = internalServiceContainer.get(IEnvironmentVariablesProvider); const envs = await envVarsService.getEnvironmentVariables(); options.env = { ...options.env, ...envs }; - return plainExec(file, args, options); + return workerPlainExec(file, args, options); } export function inExperiment(experimentName: string): boolean { From f8ea50e76bba9539abe34c6fd1b93e76c3f0144d Mon Sep 17 00:00:00 2001 From: Kartik Raj Date: Wed, 15 Nov 2023 18:07:36 -0800 Subject: [PATCH 04/30] Simplify --- src/client/pythonEnvironments/base/locator.ts | 2 +- .../base/locators/lowLevel/condaLocator.ts | 9 +++++++-- .../lowLevel/windowsRegistryLocator.ts | 7 ++++++- .../base/locators/wrappers.ts | 11 +--------- .../common/environmentManagers/conda.ts | 20 +++++++++++-------- .../common/externalDependencies.ts | 9 +++++++-- .../lowLevel/condaLocator.testvirtualenvs.ts | 11 ++++++++++ .../windowsRegistryLocator.testvirtualenvs.ts | 2 +- 8 files changed, 46 insertions(+), 25 deletions(-) diff --git a/src/client/pythonEnvironments/base/locator.ts b/src/client/pythonEnvironments/base/locator.ts index 8fefe8e2d0dd..ab3b17629bc5 100644 --- a/src/client/pythonEnvironments/base/locator.ts +++ b/src/client/pythonEnvironments/base/locator.ts @@ -178,7 +178,7 @@ export interface ILocator extends * @param query - if provided, the locator will limit results to match * @returns - the fast async iterator of Python envs, which may have incomplete info */ - iterEnvs(query?: QueryForEvent, useWorkerThreads?: boolean): IPythonEnvsIterator; + iterEnvs(query?: QueryForEvent): IPythonEnvsIterator; } export type ICompositeLocator = Omit, 'providerId'>; diff --git a/src/client/pythonEnvironments/base/locators/lowLevel/condaLocator.ts b/src/client/pythonEnvironments/base/locators/lowLevel/condaLocator.ts index 7cac0cb7df90..651a43ff8868 100644 --- a/src/client/pythonEnvironments/base/locators/lowLevel/condaLocator.ts +++ b/src/client/pythonEnvironments/base/locators/lowLevel/condaLocator.ts @@ -6,6 +6,8 @@ import { BasicEnvInfo, IPythonEnvsIterator } from '../../locator'; import { Conda, getCondaEnvironmentsTxt } from '../../../common/environmentManagers/conda'; import { traceError, traceVerbose } from '../../../../logging'; import { FSWatchingLocator } from './fsWatchingLocator'; +import { DiscoveryUsingWorkers } from '../../../../common/experiments/groups'; +import { inExperiment } from '../../../common/externalDependencies'; export class CondaEnvironmentLocator extends FSWatchingLocator { public readonly providerId: string = 'conda-envs'; @@ -19,8 +21,11 @@ export class CondaEnvironmentLocator extends FSWatchingLocator { } // eslint-disable-next-line class-methods-use-this - public async *doIterEnvs(): IPythonEnvsIterator { - const conda = await Conda.getConda(); + public async *doIterEnvs( + _: unknown, + useWorkerThreads = inExperiment(DiscoveryUsingWorkers.experiment), + ): IPythonEnvsIterator { + const conda = await Conda.getConda(undefined, useWorkerThreads); if (conda === undefined) { traceVerbose(`Couldn't locate the conda binary.`); return; diff --git a/src/client/pythonEnvironments/base/locators/lowLevel/windowsRegistryLocator.ts b/src/client/pythonEnvironments/base/locators/lowLevel/windowsRegistryLocator.ts index 947d90ee9cb5..16b2167021db 100644 --- a/src/client/pythonEnvironments/base/locators/lowLevel/windowsRegistryLocator.ts +++ b/src/client/pythonEnvironments/base/locators/lowLevel/windowsRegistryLocator.ts @@ -7,12 +7,17 @@ import { BasicEnvInfo, IPythonEnvsIterator, Locator } from '../../locator'; import { getRegistryInterpreters } from '../../../common/windowsUtils'; import { traceError, traceVerbose } from '../../../../logging'; import { isMicrosoftStoreDir } from '../../../common/environmentManagers/microsoftStoreEnv'; +import { inExperiment } from '../../../common/externalDependencies'; +import { DiscoveryUsingWorkers } from '../../../../common/experiments/groups'; export class WindowsRegistryLocator extends Locator { public readonly providerId: string = 'windows-registry'; // eslint-disable-next-line class-methods-use-this - public iterEnvs(_?: unknown, useWorkerThreads = true): IPythonEnvsIterator { + public iterEnvs( + _?: unknown, + useWorkerThreads = inExperiment(DiscoveryUsingWorkers.experiment), + ): IPythonEnvsIterator { const iterator = async function* () { traceVerbose('Searching for windows registry interpreters'); const interpreters = await getRegistryInterpreters(useWorkerThreads); diff --git a/src/client/pythonEnvironments/base/locators/wrappers.ts b/src/client/pythonEnvironments/base/locators/wrappers.ts index 1334c60fa2e0..bfaede584f6f 100644 --- a/src/client/pythonEnvironments/base/locators/wrappers.ts +++ b/src/client/pythonEnvironments/base/locators/wrappers.ts @@ -3,13 +3,10 @@ // eslint-disable-next-line max-classes-per-file import { Uri } from 'vscode'; -import { DiscoveryUsingWorkers } from '../../../common/experiments/groups'; import { IDisposable } from '../../../common/types'; import { iterEmpty } from '../../../common/utils/async'; import { getURIFilter } from '../../../common/utils/misc'; import { Disposables } from '../../../common/utils/resourceLifecycle'; -import { traceLog } from '../../../logging'; -import { inExperiment } from '../../common/externalDependencies'; import { PythonEnvInfo } from '../info'; import { BasicEnvInfo, ILocator, IPythonEnvsIterator, PythonLocatorQuery } from '../locator'; import { combineIterators, Locators } from '../locators'; @@ -20,8 +17,6 @@ import { LazyResourceBasedLocator } from './common/resourceBasedLocator'; */ export class ExtensionLocators extends Locators { - private readonly useWorkerThreads: boolean; - constructor( // These are expected to be low-level locators (e.g. system). private readonly nonWorkspace: ILocator[], @@ -30,10 +25,6 @@ export class ExtensionLocators extends Locators { private readonly workspace: ILocator, ) { super([...nonWorkspace, workspace]); - this.useWorkerThreads = inExperiment(DiscoveryUsingWorkers.experiment); - if (this.useWorkerThreads) { - traceLog('Using worker threads for discovery...'); - } } public iterEnvs(query?: PythonLocatorQuery): IPythonEnvsIterator { @@ -42,7 +33,7 @@ export class ExtensionLocators extends Locators { const nonWorkspace = query?.providerId ? this.nonWorkspace.filter((locator) => query.providerId === locator.providerId) : this.nonWorkspace; - iterators.push(...nonWorkspace.map((loc) => loc.iterEnvs(query, this.useWorkerThreads))); + iterators.push(...nonWorkspace.map((loc) => loc.iterEnvs(query))); } return combineIterators(iterators); } diff --git a/src/client/pythonEnvironments/common/environmentManagers/conda.ts b/src/client/pythonEnvironments/common/environmentManagers/conda.ts index 842f81b99a4e..0adf19e71bd0 100644 --- a/src/client/pythonEnvironments/common/environmentManagers/conda.ts +++ b/src/client/pythonEnvironments/common/environmentManagers/conda.ts @@ -272,9 +272,9 @@ export class Conda { }); } - public static async getConda(shellPath?: string): Promise { + public static async getConda(shellPath?: string, useWorkerThreads?: boolean): Promise { if (Conda.condaPromise.get(shellPath) === undefined || isTestExecution()) { - Conda.condaPromise.set(shellPath, Conda.locate(shellPath)); + Conda.condaPromise.set(shellPath, Conda.locate(shellPath, useWorkerThreads)); } return Conda.condaPromise.get(shellPath); } @@ -285,7 +285,7 @@ export class Conda { * * @return A Conda instance corresponding to the binary, if successful; otherwise, undefined. */ - private static async locate(shellPath?: string): Promise { + private static async locate(shellPath?: string, useWorkerThreads?: boolean): Promise { traceVerbose(`Searching for conda.`); const home = getUserHomeDir(); const customCondaPath = getPythonSetting(CONDAPATH_SETTING_KEY); @@ -395,7 +395,7 @@ export class Conda { try { if (condaBatFile) { const condaBat = new Conda(condaBatFile, undefined, shellPath); - await condaBat.getInfo(); + await condaBat.getInfo(useWorkerThreads); conda = new Conda(condaPath, condaBatFile, shellPath); } } catch (ex) { @@ -421,10 +421,10 @@ export class Conda { * Retrieves global information about this conda. * Corresponds to "conda info --json". */ - public async getInfo(useCache?: boolean): Promise { + public async getInfo(useCache?: boolean, useWorkerThreads?: boolean): Promise { let condaInfoCached = this.condaInfoCached.get(this.shellPath); if (!useCache || !condaInfoCached) { - condaInfoCached = this.getInfoImpl(this.command, this.shellPath); + condaInfoCached = this.getInfoImpl(this.command, this.shellPath, useWorkerThreads); this.condaInfoCached.set(this.shellPath, condaInfoCached); } return condaInfoCached; @@ -435,12 +435,16 @@ export class Conda { */ @cache(30_000, true, 10_000) // eslint-disable-next-line class-methods-use-this - private async getInfoImpl(command: string, shellPath: string | undefined): Promise { + private async getInfoImpl( + command: string, + shellPath: string | undefined, + useWorkerThreads?: boolean, + ): Promise { const options: SpawnOptions = { timeout: CONDA_GENERAL_TIMEOUT }; if (shellPath) { options.shell = shellPath; } - const resultPromise = exec(command, ['info', '--json'], options); + const resultPromise = exec(command, ['info', '--json'], options, useWorkerThreads); // It has been observed that specifying a timeout is still not reliable to terminate the Conda process, see #27915. // Hence explicitly continue execution after timeout has been reached. const success = await Promise.race([ diff --git a/src/client/pythonEnvironments/common/externalDependencies.ts b/src/client/pythonEnvironments/common/externalDependencies.ts index 8392ac5c0c73..04e2d59b3f6c 100644 --- a/src/client/pythonEnvironments/common/externalDependencies.ts +++ b/src/client/pythonEnvironments/common/externalDependencies.ts @@ -33,8 +33,13 @@ export async function shellExecute(command: string, options: ShellOptions = {}): return workerShellExec(command, options); } -export async function exec(file: string, args: string[], options: SpawnOptions = {}): Promise> { - if (!inExperiment(DiscoveryUsingWorkers.experiment)) { +export async function exec( + file: string, + args: string[], + options: SpawnOptions = {}, + useWorkerThreads = inExperiment(DiscoveryUsingWorkers.experiment), +): Promise> { + if (!useWorkerThreads) { const service = await internalServiceContainer.get(IProcessServiceFactory).create(); return service.exec(file, args, options); } diff --git a/src/test/pythonEnvironments/base/locators/lowLevel/condaLocator.testvirtualenvs.ts b/src/test/pythonEnvironments/base/locators/lowLevel/condaLocator.testvirtualenvs.ts index 1b2ea8715d35..330f1627ca81 100644 --- a/src/test/pythonEnvironments/base/locators/lowLevel/condaLocator.testvirtualenvs.ts +++ b/src/test/pythonEnvironments/base/locators/lowLevel/condaLocator.testvirtualenvs.ts @@ -13,6 +13,8 @@ import { PythonEnvsChangedEvent } from '../../../../../client/pythonEnvironments import { TEST_TIMEOUT } from '../../../../constants'; import { traceWarn } from '../../../../../client/logging'; import { TEST_LAYOUT_ROOT } from '../../../common/commonTestConstants'; +import { getEnvs } from '../../common'; +import { assertBasicEnvsEqual } from '../envTestUtils'; class CondaEnvs { private readonly condaEnvironmentsTxt; @@ -111,4 +113,13 @@ suite('Conda Env Watcher', async () => { assert.deepEqual(actualEvent!, expectedEvent, 'Unexpected event emitted'); }); + + test('Worker thread to fetch conda environments is working', async () => { + locator = new CondaEnvironmentLocator(); + const items = await getEnvs(locator.doIterEnvs(undefined, false)); + const workerItems = await getEnvs(locator.doIterEnvs(undefined, true)); + console.log('Number of items Conda locator returned:', items.length); + // Make sure items returned when using worker threads v/s not are the same. + assertBasicEnvsEqual(items, workerItems); + }).timeout(TEST_TIMEOUT * 2); }); diff --git a/src/test/pythonEnvironments/base/locators/lowLevel/windowsRegistryLocator.testvirtualenvs.ts b/src/test/pythonEnvironments/base/locators/lowLevel/windowsRegistryLocator.testvirtualenvs.ts index 6f0ff0e1bb5e..693d7c1b7fee 100644 --- a/src/test/pythonEnvironments/base/locators/lowLevel/windowsRegistryLocator.testvirtualenvs.ts +++ b/src/test/pythonEnvironments/base/locators/lowLevel/windowsRegistryLocator.testvirtualenvs.ts @@ -18,7 +18,7 @@ suite('Windows Registry Locator', async () => { return undefined; }); - test('Make sure worker thread to fetch environments is working', async () => { + test('Worker thread to fetch registry interpreters is working', async () => { const items = await getEnvs(locator.iterEnvs(undefined, false)); const workerItems = await getEnvs(locator.iterEnvs(undefined, true)); console.log('Number of items Windows registry locator returned:', items.length); From 6869e48518a3d3163e829fab02186fa076a50806 Mon Sep 17 00:00:00 2001 From: Kartik Raj Date: Wed, 15 Nov 2023 18:10:06 -0800 Subject: [PATCH 05/30] z --- .../base/locators/lowLevel/condaLocator.testvirtualenvs.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/pythonEnvironments/base/locators/lowLevel/condaLocator.testvirtualenvs.ts b/src/test/pythonEnvironments/base/locators/lowLevel/condaLocator.testvirtualenvs.ts index 330f1627ca81..8003e1fc7e94 100644 --- a/src/test/pythonEnvironments/base/locators/lowLevel/condaLocator.testvirtualenvs.ts +++ b/src/test/pythonEnvironments/base/locators/lowLevel/condaLocator.testvirtualenvs.ts @@ -52,7 +52,7 @@ class CondaEnvs { } } -suite('Conda Env Watcher', async () => { +suite('Conda Env Locator', async () => { let locator: CondaEnvironmentLocator; let condaEnvsTxt: CondaEnvs; From c9f0a599f2f68affd5bac9713163f9eb14cb6510 Mon Sep 17 00:00:00 2001 From: Kartik Raj Date: Wed, 15 Nov 2023 18:11:38 -0800 Subject: [PATCH 06/30] z --- .../common/environmentManagers/conda.unit.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/test/pythonEnvironments/common/environmentManagers/conda.unit.test.ts b/src/test/pythonEnvironments/common/environmentManagers/conda.unit.test.ts index 1e9de68ad77a..ca0e24d5f3d3 100644 --- a/src/test/pythonEnvironments/common/environmentManagers/conda.unit.test.ts +++ b/src/test/pythonEnvironments/common/environmentManagers/conda.unit.test.ts @@ -156,7 +156,6 @@ suite('Conda and its environments are located correctly', () => { const isFile = typeof dir[name] === 'string'; return { name, - path: dir.name?.toString() ?? '', isFile: () => isFile, isDirectory: () => !isFile, isBlockDevice: () => false, From 828f75cd536d59fa35831a217bcce39aed76153c Mon Sep 17 00:00:00 2001 From: Kartik Raj Date: Wed, 15 Nov 2023 18:15:00 -0800 Subject: [PATCH 07/30] z --- .../common/environmentManagers/conda.unit.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/pythonEnvironments/common/environmentManagers/conda.unit.test.ts b/src/test/pythonEnvironments/common/environmentManagers/conda.unit.test.ts index ca0e24d5f3d3..1e9de68ad77a 100644 --- a/src/test/pythonEnvironments/common/environmentManagers/conda.unit.test.ts +++ b/src/test/pythonEnvironments/common/environmentManagers/conda.unit.test.ts @@ -156,6 +156,7 @@ suite('Conda and its environments are located correctly', () => { const isFile = typeof dir[name] === 'string'; return { name, + path: dir.name?.toString() ?? '', isFile: () => isFile, isDirectory: () => !isFile, isBlockDevice: () => false, From 2ab4f31817f7c0564ca11ecad57bbcdf4946e36f Mon Sep 17 00:00:00 2001 From: Kartik Raj Date: Wed, 15 Nov 2023 18:47:13 -0800 Subject: [PATCH 08/30] Wrap setting api --- .../common/environmentManagers/conda.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/client/pythonEnvironments/common/environmentManagers/conda.ts b/src/client/pythonEnvironments/common/environmentManagers/conda.ts index 0adf19e71bd0..163e809afd1a 100644 --- a/src/client/pythonEnvironments/common/environmentManagers/conda.ts +++ b/src/client/pythonEnvironments/common/environmentManagers/conda.ts @@ -285,10 +285,15 @@ export class Conda { * * @return A Conda instance corresponding to the binary, if successful; otherwise, undefined. */ - private static async locate(shellPath?: string, useWorkerThreads?: boolean): Promise { + private static async locate(shellPath?: string, useWorkerThreads = true): Promise { traceVerbose(`Searching for conda.`); const home = getUserHomeDir(); - const customCondaPath = getPythonSetting(CONDAPATH_SETTING_KEY); + let customCondaPath: string | undefined = 'conda'; + try { + customCondaPath = getPythonSetting(CONDAPATH_SETTING_KEY); + } catch (ex) { + traceError(`Failed to get conda path setting, ${ex}`); + } const suffix = getOSType() === OSType.Windows ? 'Scripts\\conda.exe' : 'bin/conda'; // Produce a list of candidate binaries to be probed by exec'ing them. @@ -307,7 +312,7 @@ export class Conda { } async function* getCandidatesFromRegistry() { - const interps = await getRegistryInterpreters(true); + const interps = await getRegistryInterpreters(useWorkerThreads); const candidates = interps .filter((interp) => interp.interpreterPath && interp.distroOrgName === 'ContinuumAnalytics') .map((interp) => path.join(path.win32.dirname(interp.interpreterPath), suffix)); From 9a310c1e4247f40579520d1e10fc8c96115e14bb Mon Sep 17 00:00:00 2001 From: Kartik Raj Date: Wed, 15 Nov 2023 18:59:00 -0800 Subject: [PATCH 09/30] Fixes for tests --- .../common/environmentManagers/conda.ts | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/client/pythonEnvironments/common/environmentManagers/conda.ts b/src/client/pythonEnvironments/common/environmentManagers/conda.ts index 163e809afd1a..bc98e57829e9 100644 --- a/src/client/pythonEnvironments/common/environmentManagers/conda.ts +++ b/src/client/pythonEnvironments/common/environmentManagers/conda.ts @@ -265,7 +265,12 @@ export class Conda { * @param command - Command used to spawn conda. This has the same meaning as the * first argument of spawn() - i.e. it can be a full path, or just a binary name. */ - constructor(readonly command: string, shellCommand?: string, private readonly shellPath?: string) { + constructor( + readonly command: string, + shellCommand?: string, + private readonly shellPath?: string, + private readonly useWorkerThreads = true, + ) { this.shellCommand = shellCommand ?? command; onDidChangePythonSetting(CONDAPATH_SETTING_KEY, () => { Conda.condaPromise = new Map>(); @@ -390,7 +395,7 @@ export class Conda { // Probe the candidates, and pick the first one that exists and does what we need. for await (const condaPath of getCandidates()) { traceVerbose(`Probing conda binary: ${condaPath}`); - let conda = new Conda(condaPath, undefined, shellPath); + let conda = new Conda(condaPath, undefined, shellPath, useWorkerThreads); try { await conda.getInfo(); if (getOSType() === OSType.Windows && (isTestExecution() || condaPath !== customCondaPath)) { @@ -400,7 +405,7 @@ export class Conda { try { if (condaBatFile) { const condaBat = new Conda(condaBatFile, undefined, shellPath); - await condaBat.getInfo(useWorkerThreads); + await condaBat.getInfo(); conda = new Conda(condaPath, condaBatFile, shellPath); } } catch (ex) { @@ -426,10 +431,10 @@ export class Conda { * Retrieves global information about this conda. * Corresponds to "conda info --json". */ - public async getInfo(useCache?: boolean, useWorkerThreads?: boolean): Promise { + public async getInfo(useCache?: boolean): Promise { let condaInfoCached = this.condaInfoCached.get(this.shellPath); if (!useCache || !condaInfoCached) { - condaInfoCached = this.getInfoImpl(this.command, this.shellPath, useWorkerThreads); + condaInfoCached = this.getInfoImpl(this.command, this.shellPath); this.condaInfoCached.set(this.shellPath, condaInfoCached); } return condaInfoCached; @@ -440,16 +445,12 @@ export class Conda { */ @cache(30_000, true, 10_000) // eslint-disable-next-line class-methods-use-this - private async getInfoImpl( - command: string, - shellPath: string | undefined, - useWorkerThreads?: boolean, - ): Promise { + private async getInfoImpl(command: string, shellPath: string | undefined): Promise { const options: SpawnOptions = { timeout: CONDA_GENERAL_TIMEOUT }; if (shellPath) { options.shell = shellPath; } - const resultPromise = exec(command, ['info', '--json'], options, useWorkerThreads); + const resultPromise = exec(command, ['info', '--json'], options, this.useWorkerThreads); // It has been observed that specifying a timeout is still not reliable to terminate the Conda process, see #27915. // Hence explicitly continue execution after timeout has been reached. const success = await Promise.race([ From 0ca12ac31b702e010b5ccfc95a769a72c79e4269 Mon Sep 17 00:00:00 2001 From: Kartik Raj Date: Wed, 15 Nov 2023 19:00:04 -0800 Subject: [PATCH 10/30] improve test --- .../base/locators/lowLevel/condaLocator.testvirtualenvs.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/pythonEnvironments/base/locators/lowLevel/condaLocator.testvirtualenvs.ts b/src/test/pythonEnvironments/base/locators/lowLevel/condaLocator.testvirtualenvs.ts index 8003e1fc7e94..81060a0635c8 100644 --- a/src/test/pythonEnvironments/base/locators/lowLevel/condaLocator.testvirtualenvs.ts +++ b/src/test/pythonEnvironments/base/locators/lowLevel/condaLocator.testvirtualenvs.ts @@ -121,5 +121,6 @@ suite('Conda Env Locator', async () => { console.log('Number of items Conda locator returned:', items.length); // Make sure items returned when using worker threads v/s not are the same. assertBasicEnvsEqual(items, workerItems); + assert(workerItems.length > 0, 'No environments found'); }).timeout(TEST_TIMEOUT * 2); }); From 2fa06d8803ddb6a84fcbf4582525c1910a95418a Mon Sep 17 00:00:00 2001 From: Kartik Raj Date: Thu, 16 Nov 2023 20:33:53 +0000 Subject: [PATCH 11/30] z --- .github/workflows/pr-check.yml | 494 --------------------------------- 1 file changed, 494 deletions(-) diff --git a/.github/workflows/pr-check.yml b/.github/workflows/pr-check.yml index 3283d3281e5b..cd0d8ba44c6a 100644 --- a/.github/workflows/pr-check.yml +++ b/.github/workflows/pr-check.yml @@ -33,497 +33,3 @@ jobs: node_version: ${{ env.NODE_VERSION}} vsix_name: ${{ env.VSIX_NAME }} artifact_name: ${{ env.ARTIFACT_NAME_VSIX }} - - lint: - name: Lint - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Lint - uses: ./.github/actions/lint - with: - node_version: ${{ env.NODE_VERSION }} - - check-types: - name: Check Python types - runs-on: ubuntu-latest - steps: - - name: Use Python ${{ env.PYTHON_VERSION }} - uses: actions/setup-python@v4 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: Checkout - uses: actions/checkout@v4 - - - name: Install base Python requirements - uses: brettcannon/pip-secure-install@v1 - with: - options: '-t ./pythonFiles/lib/python --no-cache-dir --implementation py' - - - name: Install Jedi requirements - uses: brettcannon/pip-secure-install@v1 - with: - requirements-file: './pythonFiles/jedilsp_requirements/requirements.txt' - options: '-t ./pythonFiles/lib/jedilsp --no-cache-dir --implementation py' - - - name: Install other Python requirements - run: | - python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --no-cache-dir --implementation py --no-deps --upgrade --pre debugpy - python -m pip install --upgrade -r build/test-requirements.txt - - - name: Run Pyright - uses: jakebailey/pyright-action@v1 - with: - version: 1.1.308 - working-directory: 'pythonFiles' - - python-tests: - name: Python Tests - # The value of runs-on is the OS of the current job (specified in the strategy matrix below) instead of being hardcoded. - runs-on: ${{ matrix.os }} - defaults: - run: - working-directory: ${{ env.special-working-directory }} - strategy: - fail-fast: false - matrix: - # We're not running CI on macOS for now because it's one less matrix entry to lower the number of runners used, - # macOS runners are expensive, and we assume that Ubuntu is enough to cover the Unix case. - os: [ubuntu-latest, windows-latest] - # Run the tests on the oldest and most recent versions of Python. - python: ['3.8', '3.x', '3.12-dev'] - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - path: ${{ env.special-working-directory-relative }} - - - name: Use Python ${{ matrix.python }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python }} - - - name: Install base Python requirements - uses: brettcannon/pip-secure-install@v1 - with: - requirements-file: '"${{ env.special-working-directory-relative }}/requirements.txt"' - options: '-t "${{ env.special-working-directory-relative }}/pythonFiles/lib/python" --no-cache-dir --implementation py' - - - name: Install test requirements - run: python -m pip install --upgrade -r build/test-requirements.txt - - - name: Run Python unit tests - run: python pythonFiles/tests/run_all.py - - tests: - name: Tests - # The value of runs-on is the OS of the current job (specified in the strategy matrix below) instead of being hardcoded. - runs-on: ${{ matrix.os }} - defaults: - run: - working-directory: ${{ env.special-working-directory }} - strategy: - fail-fast: false - matrix: - # We're not running CI on macOS for now because it's one less matrix entry to lower the number of runners used, - # macOS runners are expensive, and we assume that Ubuntu is enough to cover the Unix case. - os: [ubuntu-latest, windows-latest] - # Run the tests on the oldest and most recent versions of Python. - python: ['3.x'] - test-suite: [ts-unit, venv, single-workspace, debugger, functional] - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - path: ${{ env.special-working-directory-relative }} - - - name: Install Node - uses: actions/setup-node@v4 - with: - node-version: ${{ env.NODE_VERSION }} - cache: 'npm' - cache-dependency-path: ${{ env.special-working-directory-relative }}/package-lock.json - - - name: Install dependencies (npm ci) - run: npm ci - - - name: Compile - run: npx gulp prePublishNonBundle - - - name: Use Python ${{ matrix.python }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python }} - - - name: Install debugpy - run: | - # We need to have debugpy so that tests relying on it keep passing, but we don't need install_debugpy's logic in the test phase. - python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --no-cache-dir --implementation py --no-deps --upgrade --pre debugpy - - - name: Download get-pip.py - run: | - python -m pip install wheel - python -m pip install -r build/build-install-requirements.txt - python ./pythonFiles/download_get_pip.py - shell: bash - - - name: Install base Python requirements - uses: brettcannon/pip-secure-install@v1 - with: - requirements-file: '"${{ env.special-working-directory-relative }}/requirements.txt"' - options: '-t "${{ env.special-working-directory-relative }}/pythonFiles/lib/python" --no-cache-dir --implementation py' - - - name: Install Jedi requirements - uses: brettcannon/pip-secure-install@v1 - with: - requirements-file: '"${{ env.special-working-directory-relative }}/pythonFiles/jedilsp_requirements/requirements.txt"' - options: '-t "${{ env.special-working-directory-relative }}/pythonFiles/lib/jedilsp" --no-cache-dir --implementation py' - - - name: Install test requirements - run: python -m pip install --upgrade -r build/test-requirements.txt - - - name: Install debugpy wheels (Python ${{ matrix.python }}) - run: | - python -m pip install wheel - python -m pip --disable-pip-version-check install -r build/build-install-requirements.txt - python ./pythonFiles/install_debugpy.py - shell: bash - if: matrix.test-suite == 'debugger' - - - name: Install functional test requirements - run: python -m pip install --upgrade -r ./build/functional-test-requirements.txt - if: matrix.test-suite == 'functional' - - - name: Prepare pipenv for venv tests - env: - TEST_FILES_SUFFIX: testvirtualenvs - PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' - shell: pwsh - if: matrix.test-suite == 'venv' - run: | - python -m pip install pipenv - python -m pipenv run python ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} pipenvPath - - - name: Prepare poetry for venv tests - env: - TEST_FILES_SUFFIX: testvirtualenvs - shell: pwsh - if: matrix.test-suite == 'venv' - run: | - python -m pip install poetry - Move-Item -Path ".\build\ci\pyproject.toml" -Destination . - poetry env use python - - - name: Prepare virtualenv for venv tests - env: - TEST_FILES_SUFFIX: testvirtualenvs - PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' - shell: pwsh - if: matrix.test-suite == 'venv' - run: | - python -m pip install virtualenv - python -m virtualenv .virtualenv/ - if ('${{ matrix.os }}' -match 'windows-latest') { - & ".virtualenv/Scripts/python.exe" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} virtualEnvPath - } else { - & ".virtualenv/bin/python" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} virtualEnvPath - } - - - name: Prepare venv for venv tests - env: - TEST_FILES_SUFFIX: testvirtualenvs - PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' - shell: pwsh - if: matrix.test-suite == 'venv' && startsWith(matrix.python, 3.) - run: | - python -m venv .venv - if ('${{ matrix.os }}' -match 'windows-latest') { - & ".venv/Scripts/python.exe" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} venvPath - } else { - & ".venv/bin/python" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} venvPath - } - - - name: Prepare conda for venv tests - env: - TEST_FILES_SUFFIX: testvirtualenvs - PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' - shell: pwsh - if: matrix.test-suite == 'venv' - run: | - # 1. For `terminalActivation.testvirtualenvs.test.ts` - if ('${{ matrix.os }}' -match 'windows-latest') { - $condaPythonPath = Join-Path -Path $Env:CONDA -ChildPath python.exe - $condaExecPath = Join-Path -Path $Env:CONDA -ChildPath Scripts | Join-Path -ChildPath conda - } else{ - $condaPythonPath = Join-Path -Path $Env:CONDA -ChildPath bin | Join-Path -ChildPath python - $condaExecPath = Join-Path -Path $Env:CONDA -ChildPath bin | Join-Path -ChildPath conda - } - & $condaPythonPath ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} condaExecPath $condaExecPath - & $condaPythonPath ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} condaPath - & $condaExecPath init --all - - - name: Set CI_PYTHON_PATH and CI_DISABLE_AUTO_SELECTION - run: | - echo "CI_PYTHON_PATH=python" >> $GITHUB_ENV - echo "CI_DISABLE_AUTO_SELECTION=1" >> $GITHUB_ENV - shell: bash - if: matrix.test-suite != 'ts-unit' - - # Run TypeScript unit tests only for Python 3.X. - - name: Run TypeScript unit tests - run: npm run test:unittests - if: matrix.test-suite == 'ts-unit' && startsWith(matrix.python, 3.) - - # The virtual environment based tests use the `testSingleWorkspace` set of tests - # with the environment variable `TEST_FILES_SUFFIX` set to `testvirtualenvs`, - # which is set in the "Prepare environment for venv tests" step. - # We also use a third-party GitHub Action to install xvfb on Linux, - # run tests and then clean up the process once the tests ran. - # See https://github.com/GabrielBB/xvfb-action - - name: Run venv tests - env: - TEST_FILES_SUFFIX: testvirtualenvs - CI_PYTHON_VERSION: ${{ matrix.python }} - uses: GabrielBB/xvfb-action@v1.6 - with: - run: npm run testSingleWorkspace - working-directory: ${{ env.special-working-directory }} - if: matrix.test-suite == 'venv' - - - name: Run single-workspace tests - env: - CI_PYTHON_VERSION: ${{ matrix.python }} - uses: GabrielBB/xvfb-action@v1.6 - with: - run: npm run testSingleWorkspace - working-directory: ${{ env.special-working-directory }} - if: matrix.test-suite == 'single-workspace' - - - name: Run debugger tests - env: - CI_PYTHON_VERSION: ${{ matrix.python }} - uses: GabrielBB/xvfb-action@v1.6 - with: - run: npm run testDebugger - working-directory: ${{ env.special-working-directory }} - if: matrix.test-suite == 'debugger' - - # Run TypeScript functional tests - - name: Run TypeScript functional tests - run: npm run test:functional - if: matrix.test-suite == 'functional' - - smoke-tests: - name: Smoke tests - # The value of runs-on is the OS of the current job (specified in the strategy matrix below) instead of being hardcoded. - runs-on: ${{ matrix.os }} - needs: [build-vsix] - strategy: - fail-fast: false - matrix: - # We're not running CI on macOS for now because it's one less matrix entry to lower the number of runners used, - # macOS runners are expensive, and we assume that Ubuntu is enough to cover the UNIX case. - os: [ubuntu-latest, windows-latest] - steps: - # Need the source to have the tests available. - - name: Checkout - uses: actions/checkout@v4 - - - name: Smoke tests - uses: ./.github/actions/smoke-tests - with: - node_version: ${{ env.NODE_VERSION }} - artifact_name: ${{ env.ARTIFACT_NAME_VSIX }} - - ### Coverage run - coverage: - name: Coverage - # The value of runs-on is the OS of the current job (specified in the strategy matrix below) instead of being hardcoded. - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - # Only run coverage on linux for PRs - os: [ubuntu-latest] - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Install Node - uses: actions/setup-node@v4 - with: - node-version: ${{ env.NODE_VERSION }} - cache: 'npm' - - - name: Install dependencies (npm ci) - run: npm ci - - - name: Compile - run: npx gulp prePublishNonBundle - - - name: Use Python ${{ env.PYTHON_VERSION }} - uses: actions/setup-python@v4 - with: - python-version: ${{ env.PYTHON_VERSION }} - cache: 'pip' - cache-dependency-path: | - requirements.txt - pythonFiles/jedilsp_requirements/requirements.txt - build/test-requirements.txt - build/functional-test-requirements.txt - - - name: Install base Python requirements - uses: brettcannon/pip-secure-install@v1 - with: - options: '-t ./pythonFiles/lib/python --implementation py' - - - name: Install Jedi requirements - uses: brettcannon/pip-secure-install@v1 - with: - requirements-file: './pythonFiles/jedilsp_requirements/requirements.txt' - options: '-t ./pythonFiles/lib/jedilsp --implementation py' - - - name: Install debugpy - run: | - # We need to have debugpy so that tests relying on it keep passing, but we don't need install_debugpy's logic in the test phase. - python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --implementation py --no-deps --upgrade --pre debugpy - - - name: Install test requirements - run: python -m pip install --upgrade -r build/test-requirements.txt - - - name: Install functional test requirements - run: python -m pip install --upgrade -r ./build/functional-test-requirements.txt - - - name: Prepare pipenv for venv tests - env: - TEST_FILES_SUFFIX: testvirtualenvs - PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' - shell: pwsh - run: | - python -m pip install pipenv - python -m pipenv run python ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} pipenvPath - - - name: Prepare poetry for venv tests - env: - TEST_FILES_SUFFIX: testvirtualenvs - shell: pwsh - run: | - python -m pip install poetry - Move-Item -Path ".\build\ci\pyproject.toml" -Destination . - poetry env use python - - - name: Prepare virtualenv for venv tests - env: - TEST_FILES_SUFFIX: testvirtualenvs - PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' - shell: pwsh - run: | - python -m pip install virtualenv - python -m virtualenv .virtualenv/ - if ('${{ matrix.os }}' -match 'windows-latest') { - & ".virtualenv/Scripts/python.exe" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} virtualEnvPath - } else { - & ".virtualenv/bin/python" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} virtualEnvPath - } - - - name: Prepare venv for venv tests - env: - TEST_FILES_SUFFIX: testvirtualenvs - PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' - shell: pwsh - run: | - python -m venv .venv - if ('${{ matrix.os }}' -match 'windows-latest') { - & ".venv/Scripts/python.exe" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} venvPath - } else { - & ".venv/bin/python" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} venvPath - } - - - name: Prepare conda for venv tests - env: - TEST_FILES_SUFFIX: testvirtualenvs - PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' - shell: pwsh - run: | - # 1. For `terminalActivation.testvirtualenvs.test.ts` - if ('${{ matrix.os }}' -match 'windows-latest') { - $condaPythonPath = Join-Path -Path $Env:CONDA -ChildPath python.exe - $condaExecPath = Join-Path -Path $Env:CONDA -ChildPath Scripts | Join-Path -ChildPath conda - } else{ - $condaPythonPath = Join-Path -Path $Env:CONDA -ChildPath bin | Join-Path -ChildPath python - $condaExecPath = Join-Path -Path $Env:CONDA -ChildPath bin | Join-Path -ChildPath conda - } - & $condaPythonPath ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} condaExecPath $condaExecPath - & $condaPythonPath ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} condaPath - & $condaExecPath init --all - - - name: Run TypeScript unit tests - run: npm run test:unittests:cover - - - name: Run Python unit tests - run: | - python pythonFiles/tests/run_all.py - - # The virtual environment based tests use the `testSingleWorkspace` set of tests - # with the environment variable `TEST_FILES_SUFFIX` set to `testvirtualenvs`, - # which is set in the "Prepare environment for venv tests" step. - # We also use a third-party GitHub Action to install xvfb on Linux, - # run tests and then clean up the process once the tests ran. - # See https://github.com/GabrielBB/xvfb-action - - name: Run venv tests - env: - TEST_FILES_SUFFIX: testvirtualenvs - CI_PYTHON_VERSION: ${{ env.PYTHON_VERSION }} - CI_DISABLE_AUTO_SELECTION: 1 - uses: GabrielBB/xvfb-action@v1.6 - with: - run: npm run testSingleWorkspace:cover - - - name: Run single-workspace tests - env: - CI_PYTHON_VERSION: ${{ env.PYTHON_VERSION }} - CI_DISABLE_AUTO_SELECTION: 1 - uses: GabrielBB/xvfb-action@v1.6 - with: - run: npm run testSingleWorkspace:cover - - # Enable these tests when coverage is setup for multiroot workspace tests - # - name: Run multi-workspace tests - # env: - # CI_PYTHON_VERSION: ${{ env.PYTHON_VERSION }} - # CI_DISABLE_AUTO_SELECTION: 1 - # uses: GabrielBB/xvfb-action@v1.6 - # with: - # run: npm run testMultiWorkspace:cover - - # Enable these tests when coverage is setup for debugger tests - # - name: Run debugger tests - # env: - # CI_PYTHON_VERSION: ${{ env.PYTHON_VERSION }} - # CI_DISABLE_AUTO_SELECTION: 1 - # uses: GabrielBB/xvfb-action@v1.6 - # with: - # run: npm run testDebugger:cover - - # Run TypeScript functional tests - - name: Run TypeScript functional tests - env: - CI_PYTHON_VERSION: ${{ env.PYTHON_VERSION }} - CI_DISABLE_AUTO_SELECTION: 1 - run: npm run test:functional:cover - - - name: Generate coverage reports - run: npm run test:cover:report - - - name: Upload HTML report - uses: actions/upload-artifact@v3 - with: - name: ${{ runner.os }}-coverage-report-html - path: ./coverage - retention-days: 1 From ba00d7e3eef895b807da8d6202baf0002f471312 Mon Sep 17 00:00:00 2001 From: Kartik Raj Date: Thu, 16 Nov 2023 20:34:18 +0000 Subject: [PATCH 12/30] z --- src/client/common/process/worker/rawProcessApiWrapper.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/client/common/process/worker/rawProcessApiWrapper.ts b/src/client/common/process/worker/rawProcessApiWrapper.ts index ab478fc1aa77..2069cc73cabb 100644 --- a/src/client/common/process/worker/rawProcessApiWrapper.ts +++ b/src/client/common/process/worker/rawProcessApiWrapper.ts @@ -13,8 +13,6 @@ export function workerShellExec( options: ShellOptions, defaultEnv?: EnvironmentVariables, ): Promise> { - const processLogger = new ProcessLogger(new WorkspaceService()); - processLogger.logProcess(command, undefined, options); return executeWorkerFile(path.join(__dirname, 'shellExecWorker.js'), { command, options, @@ -28,8 +26,6 @@ export function workerPlainExec( options: SpawnOptions & { doNotLog?: boolean } = {}, defaultEnv?: EnvironmentVariables, ): Promise> { - const processLogger = new ProcessLogger(new WorkspaceService()); - processLogger.logProcess(file, args, options); return executeWorkerFile(path.join(__dirname, 'plainExecWorker.js'), { file, args, From 446162ed2972b056ac6b26fc7078bdca5b26df98 Mon Sep 17 00:00:00 2001 From: Kartik Raj Date: Thu, 16 Nov 2023 20:43:44 +0000 Subject: [PATCH 13/30] z --- src/client/common/process/worker/rawProcessApiWrapper.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/client/common/process/worker/rawProcessApiWrapper.ts b/src/client/common/process/worker/rawProcessApiWrapper.ts index 2069cc73cabb..9b54a7dbfb9a 100644 --- a/src/client/common/process/worker/rawProcessApiWrapper.ts +++ b/src/client/common/process/worker/rawProcessApiWrapper.ts @@ -3,8 +3,6 @@ import { SpawnOptions } from 'child_process'; import * as path from 'path'; -import { WorkspaceService } from '../../application/workspace'; -import { ProcessLogger } from '../logger'; import { executeWorkerFile } from './main'; import { EnvironmentVariables, ExecutionResult, ShellOptions } from './types'; From 4f8fbd97ffe3327d15e258264669867ee0c723fc Mon Sep 17 00:00:00 2001 From: Kartik Raj Date: Thu, 16 Nov 2023 20:50:52 +0000 Subject: [PATCH 14/30] check --- .../common/process/worker/rawProcessApiWrapper.ts | 9 ++++++--- .../common/externalDependencies.ts | 14 +++++++++++--- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/client/common/process/worker/rawProcessApiWrapper.ts b/src/client/common/process/worker/rawProcessApiWrapper.ts index 9b54a7dbfb9a..5c4580c4e05f 100644 --- a/src/client/common/process/worker/rawProcessApiWrapper.ts +++ b/src/client/common/process/worker/rawProcessApiWrapper.ts @@ -3,14 +3,17 @@ import { SpawnOptions } from 'child_process'; import * as path from 'path'; +import { IProcessLogger } from '../types'; import { executeWorkerFile } from './main'; import { EnvironmentVariables, ExecutionResult, ShellOptions } from './types'; export function workerShellExec( command: string, options: ShellOptions, + processLogger: IProcessLogger, defaultEnv?: EnvironmentVariables, ): Promise> { + processLogger.logProcess(command, undefined, options); return executeWorkerFile(path.join(__dirname, 'shellExecWorker.js'), { command, options, @@ -21,13 +24,13 @@ export function workerShellExec( export function workerPlainExec( file: string, args: string[], - options: SpawnOptions & { doNotLog?: boolean } = {}, - defaultEnv?: EnvironmentVariables, + options: SpawnOptions = {}, + processLogger: IProcessLogger, ): Promise> { + processLogger.logProcess(file, args, options); return executeWorkerFile(path.join(__dirname, 'plainExecWorker.js'), { file, args, options, - defaultEnv, }); } diff --git a/src/client/pythonEnvironments/common/externalDependencies.ts b/src/client/pythonEnvironments/common/externalDependencies.ts index 04e2d59b3f6c..f102f1df5059 100644 --- a/src/client/pythonEnvironments/common/externalDependencies.ts +++ b/src/client/pythonEnvironments/common/externalDependencies.ts @@ -5,7 +5,13 @@ import * as fsapi from 'fs-extra'; import * as path from 'path'; import * as vscode from 'vscode'; import { IWorkspaceService } from '../../common/application/types'; -import { ExecutionResult, IProcessServiceFactory, ShellOptions, SpawnOptions } from '../../common/process/types'; +import { + ExecutionResult, + IProcessLogger, + IProcessServiceFactory, + ShellOptions, + SpawnOptions, +} from '../../common/process/types'; import { IDisposable, IConfigurationService, IExperimentService } from '../../common/types'; import { chain, iterable } from '../../common/utils/async'; import { getOSType, OSType } from '../../common/utils/platform'; @@ -27,10 +33,11 @@ export async function shellExecute(command: string, options: ShellOptions = {}): const service = await internalServiceContainer.get(IProcessServiceFactory).create(); return service.shellExec(command, options); } + const logger = internalServiceContainer.get(IProcessLogger); const envVarsService = internalServiceContainer.get(IEnvironmentVariablesProvider); const envs = await envVarsService.getEnvironmentVariables(); options.env = { ...options.env, ...envs }; - return workerShellExec(command, options); + return workerShellExec(command, options, logger); } export async function exec( @@ -43,10 +50,11 @@ export async function exec( const service = await internalServiceContainer.get(IProcessServiceFactory).create(); return service.exec(file, args, options); } + const logger = internalServiceContainer.get(IProcessLogger); const envVarsService = internalServiceContainer.get(IEnvironmentVariablesProvider); const envs = await envVarsService.getEnvironmentVariables(); options.env = { ...options.env, ...envs }; - return workerPlainExec(file, args, options); + return workerPlainExec(file, args, options, logger); } export function inExperiment(experimentName: string): boolean { From 67cd46ce7f33357347a1edd0c3d26ace6742e68a Mon Sep 17 00:00:00 2001 From: Kartik Raj Date: Thu, 16 Nov 2023 20:54:29 +0000 Subject: [PATCH 15/30] run all --- .github/workflows/pr-check.yml | 494 +++++++++++++++++++++++++++++++++ 1 file changed, 494 insertions(+) diff --git a/.github/workflows/pr-check.yml b/.github/workflows/pr-check.yml index cd0d8ba44c6a..3283d3281e5b 100644 --- a/.github/workflows/pr-check.yml +++ b/.github/workflows/pr-check.yml @@ -33,3 +33,497 @@ jobs: node_version: ${{ env.NODE_VERSION}} vsix_name: ${{ env.VSIX_NAME }} artifact_name: ${{ env.ARTIFACT_NAME_VSIX }} + + lint: + name: Lint + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Lint + uses: ./.github/actions/lint + with: + node_version: ${{ env.NODE_VERSION }} + + check-types: + name: Check Python types + runs-on: ubuntu-latest + steps: + - name: Use Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Checkout + uses: actions/checkout@v4 + + - name: Install base Python requirements + uses: brettcannon/pip-secure-install@v1 + with: + options: '-t ./pythonFiles/lib/python --no-cache-dir --implementation py' + + - name: Install Jedi requirements + uses: brettcannon/pip-secure-install@v1 + with: + requirements-file: './pythonFiles/jedilsp_requirements/requirements.txt' + options: '-t ./pythonFiles/lib/jedilsp --no-cache-dir --implementation py' + + - name: Install other Python requirements + run: | + python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --no-cache-dir --implementation py --no-deps --upgrade --pre debugpy + python -m pip install --upgrade -r build/test-requirements.txt + + - name: Run Pyright + uses: jakebailey/pyright-action@v1 + with: + version: 1.1.308 + working-directory: 'pythonFiles' + + python-tests: + name: Python Tests + # The value of runs-on is the OS of the current job (specified in the strategy matrix below) instead of being hardcoded. + runs-on: ${{ matrix.os }} + defaults: + run: + working-directory: ${{ env.special-working-directory }} + strategy: + fail-fast: false + matrix: + # We're not running CI on macOS for now because it's one less matrix entry to lower the number of runners used, + # macOS runners are expensive, and we assume that Ubuntu is enough to cover the Unix case. + os: [ubuntu-latest, windows-latest] + # Run the tests on the oldest and most recent versions of Python. + python: ['3.8', '3.x', '3.12-dev'] + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + path: ${{ env.special-working-directory-relative }} + + - name: Use Python ${{ matrix.python }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} + + - name: Install base Python requirements + uses: brettcannon/pip-secure-install@v1 + with: + requirements-file: '"${{ env.special-working-directory-relative }}/requirements.txt"' + options: '-t "${{ env.special-working-directory-relative }}/pythonFiles/lib/python" --no-cache-dir --implementation py' + + - name: Install test requirements + run: python -m pip install --upgrade -r build/test-requirements.txt + + - name: Run Python unit tests + run: python pythonFiles/tests/run_all.py + + tests: + name: Tests + # The value of runs-on is the OS of the current job (specified in the strategy matrix below) instead of being hardcoded. + runs-on: ${{ matrix.os }} + defaults: + run: + working-directory: ${{ env.special-working-directory }} + strategy: + fail-fast: false + matrix: + # We're not running CI on macOS for now because it's one less matrix entry to lower the number of runners used, + # macOS runners are expensive, and we assume that Ubuntu is enough to cover the Unix case. + os: [ubuntu-latest, windows-latest] + # Run the tests on the oldest and most recent versions of Python. + python: ['3.x'] + test-suite: [ts-unit, venv, single-workspace, debugger, functional] + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + path: ${{ env.special-working-directory-relative }} + + - name: Install Node + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + cache: 'npm' + cache-dependency-path: ${{ env.special-working-directory-relative }}/package-lock.json + + - name: Install dependencies (npm ci) + run: npm ci + + - name: Compile + run: npx gulp prePublishNonBundle + + - name: Use Python ${{ matrix.python }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} + + - name: Install debugpy + run: | + # We need to have debugpy so that tests relying on it keep passing, but we don't need install_debugpy's logic in the test phase. + python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --no-cache-dir --implementation py --no-deps --upgrade --pre debugpy + + - name: Download get-pip.py + run: | + python -m pip install wheel + python -m pip install -r build/build-install-requirements.txt + python ./pythonFiles/download_get_pip.py + shell: bash + + - name: Install base Python requirements + uses: brettcannon/pip-secure-install@v1 + with: + requirements-file: '"${{ env.special-working-directory-relative }}/requirements.txt"' + options: '-t "${{ env.special-working-directory-relative }}/pythonFiles/lib/python" --no-cache-dir --implementation py' + + - name: Install Jedi requirements + uses: brettcannon/pip-secure-install@v1 + with: + requirements-file: '"${{ env.special-working-directory-relative }}/pythonFiles/jedilsp_requirements/requirements.txt"' + options: '-t "${{ env.special-working-directory-relative }}/pythonFiles/lib/jedilsp" --no-cache-dir --implementation py' + + - name: Install test requirements + run: python -m pip install --upgrade -r build/test-requirements.txt + + - name: Install debugpy wheels (Python ${{ matrix.python }}) + run: | + python -m pip install wheel + python -m pip --disable-pip-version-check install -r build/build-install-requirements.txt + python ./pythonFiles/install_debugpy.py + shell: bash + if: matrix.test-suite == 'debugger' + + - name: Install functional test requirements + run: python -m pip install --upgrade -r ./build/functional-test-requirements.txt + if: matrix.test-suite == 'functional' + + - name: Prepare pipenv for venv tests + env: + TEST_FILES_SUFFIX: testvirtualenvs + PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' + shell: pwsh + if: matrix.test-suite == 'venv' + run: | + python -m pip install pipenv + python -m pipenv run python ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} pipenvPath + + - name: Prepare poetry for venv tests + env: + TEST_FILES_SUFFIX: testvirtualenvs + shell: pwsh + if: matrix.test-suite == 'venv' + run: | + python -m pip install poetry + Move-Item -Path ".\build\ci\pyproject.toml" -Destination . + poetry env use python + + - name: Prepare virtualenv for venv tests + env: + TEST_FILES_SUFFIX: testvirtualenvs + PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' + shell: pwsh + if: matrix.test-suite == 'venv' + run: | + python -m pip install virtualenv + python -m virtualenv .virtualenv/ + if ('${{ matrix.os }}' -match 'windows-latest') { + & ".virtualenv/Scripts/python.exe" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} virtualEnvPath + } else { + & ".virtualenv/bin/python" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} virtualEnvPath + } + + - name: Prepare venv for venv tests + env: + TEST_FILES_SUFFIX: testvirtualenvs + PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' + shell: pwsh + if: matrix.test-suite == 'venv' && startsWith(matrix.python, 3.) + run: | + python -m venv .venv + if ('${{ matrix.os }}' -match 'windows-latest') { + & ".venv/Scripts/python.exe" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} venvPath + } else { + & ".venv/bin/python" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} venvPath + } + + - name: Prepare conda for venv tests + env: + TEST_FILES_SUFFIX: testvirtualenvs + PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' + shell: pwsh + if: matrix.test-suite == 'venv' + run: | + # 1. For `terminalActivation.testvirtualenvs.test.ts` + if ('${{ matrix.os }}' -match 'windows-latest') { + $condaPythonPath = Join-Path -Path $Env:CONDA -ChildPath python.exe + $condaExecPath = Join-Path -Path $Env:CONDA -ChildPath Scripts | Join-Path -ChildPath conda + } else{ + $condaPythonPath = Join-Path -Path $Env:CONDA -ChildPath bin | Join-Path -ChildPath python + $condaExecPath = Join-Path -Path $Env:CONDA -ChildPath bin | Join-Path -ChildPath conda + } + & $condaPythonPath ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} condaExecPath $condaExecPath + & $condaPythonPath ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} condaPath + & $condaExecPath init --all + + - name: Set CI_PYTHON_PATH and CI_DISABLE_AUTO_SELECTION + run: | + echo "CI_PYTHON_PATH=python" >> $GITHUB_ENV + echo "CI_DISABLE_AUTO_SELECTION=1" >> $GITHUB_ENV + shell: bash + if: matrix.test-suite != 'ts-unit' + + # Run TypeScript unit tests only for Python 3.X. + - name: Run TypeScript unit tests + run: npm run test:unittests + if: matrix.test-suite == 'ts-unit' && startsWith(matrix.python, 3.) + + # The virtual environment based tests use the `testSingleWorkspace` set of tests + # with the environment variable `TEST_FILES_SUFFIX` set to `testvirtualenvs`, + # which is set in the "Prepare environment for venv tests" step. + # We also use a third-party GitHub Action to install xvfb on Linux, + # run tests and then clean up the process once the tests ran. + # See https://github.com/GabrielBB/xvfb-action + - name: Run venv tests + env: + TEST_FILES_SUFFIX: testvirtualenvs + CI_PYTHON_VERSION: ${{ matrix.python }} + uses: GabrielBB/xvfb-action@v1.6 + with: + run: npm run testSingleWorkspace + working-directory: ${{ env.special-working-directory }} + if: matrix.test-suite == 'venv' + + - name: Run single-workspace tests + env: + CI_PYTHON_VERSION: ${{ matrix.python }} + uses: GabrielBB/xvfb-action@v1.6 + with: + run: npm run testSingleWorkspace + working-directory: ${{ env.special-working-directory }} + if: matrix.test-suite == 'single-workspace' + + - name: Run debugger tests + env: + CI_PYTHON_VERSION: ${{ matrix.python }} + uses: GabrielBB/xvfb-action@v1.6 + with: + run: npm run testDebugger + working-directory: ${{ env.special-working-directory }} + if: matrix.test-suite == 'debugger' + + # Run TypeScript functional tests + - name: Run TypeScript functional tests + run: npm run test:functional + if: matrix.test-suite == 'functional' + + smoke-tests: + name: Smoke tests + # The value of runs-on is the OS of the current job (specified in the strategy matrix below) instead of being hardcoded. + runs-on: ${{ matrix.os }} + needs: [build-vsix] + strategy: + fail-fast: false + matrix: + # We're not running CI on macOS for now because it's one less matrix entry to lower the number of runners used, + # macOS runners are expensive, and we assume that Ubuntu is enough to cover the UNIX case. + os: [ubuntu-latest, windows-latest] + steps: + # Need the source to have the tests available. + - name: Checkout + uses: actions/checkout@v4 + + - name: Smoke tests + uses: ./.github/actions/smoke-tests + with: + node_version: ${{ env.NODE_VERSION }} + artifact_name: ${{ env.ARTIFACT_NAME_VSIX }} + + ### Coverage run + coverage: + name: Coverage + # The value of runs-on is the OS of the current job (specified in the strategy matrix below) instead of being hardcoded. + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + # Only run coverage on linux for PRs + os: [ubuntu-latest] + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Node + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + cache: 'npm' + + - name: Install dependencies (npm ci) + run: npm ci + + - name: Compile + run: npx gulp prePublishNonBundle + + - name: Use Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: 'pip' + cache-dependency-path: | + requirements.txt + pythonFiles/jedilsp_requirements/requirements.txt + build/test-requirements.txt + build/functional-test-requirements.txt + + - name: Install base Python requirements + uses: brettcannon/pip-secure-install@v1 + with: + options: '-t ./pythonFiles/lib/python --implementation py' + + - name: Install Jedi requirements + uses: brettcannon/pip-secure-install@v1 + with: + requirements-file: './pythonFiles/jedilsp_requirements/requirements.txt' + options: '-t ./pythonFiles/lib/jedilsp --implementation py' + + - name: Install debugpy + run: | + # We need to have debugpy so that tests relying on it keep passing, but we don't need install_debugpy's logic in the test phase. + python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --implementation py --no-deps --upgrade --pre debugpy + + - name: Install test requirements + run: python -m pip install --upgrade -r build/test-requirements.txt + + - name: Install functional test requirements + run: python -m pip install --upgrade -r ./build/functional-test-requirements.txt + + - name: Prepare pipenv for venv tests + env: + TEST_FILES_SUFFIX: testvirtualenvs + PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' + shell: pwsh + run: | + python -m pip install pipenv + python -m pipenv run python ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} pipenvPath + + - name: Prepare poetry for venv tests + env: + TEST_FILES_SUFFIX: testvirtualenvs + shell: pwsh + run: | + python -m pip install poetry + Move-Item -Path ".\build\ci\pyproject.toml" -Destination . + poetry env use python + + - name: Prepare virtualenv for venv tests + env: + TEST_FILES_SUFFIX: testvirtualenvs + PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' + shell: pwsh + run: | + python -m pip install virtualenv + python -m virtualenv .virtualenv/ + if ('${{ matrix.os }}' -match 'windows-latest') { + & ".virtualenv/Scripts/python.exe" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} virtualEnvPath + } else { + & ".virtualenv/bin/python" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} virtualEnvPath + } + + - name: Prepare venv for venv tests + env: + TEST_FILES_SUFFIX: testvirtualenvs + PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' + shell: pwsh + run: | + python -m venv .venv + if ('${{ matrix.os }}' -match 'windows-latest') { + & ".venv/Scripts/python.exe" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} venvPath + } else { + & ".venv/bin/python" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} venvPath + } + + - name: Prepare conda for venv tests + env: + TEST_FILES_SUFFIX: testvirtualenvs + PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' + shell: pwsh + run: | + # 1. For `terminalActivation.testvirtualenvs.test.ts` + if ('${{ matrix.os }}' -match 'windows-latest') { + $condaPythonPath = Join-Path -Path $Env:CONDA -ChildPath python.exe + $condaExecPath = Join-Path -Path $Env:CONDA -ChildPath Scripts | Join-Path -ChildPath conda + } else{ + $condaPythonPath = Join-Path -Path $Env:CONDA -ChildPath bin | Join-Path -ChildPath python + $condaExecPath = Join-Path -Path $Env:CONDA -ChildPath bin | Join-Path -ChildPath conda + } + & $condaPythonPath ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} condaExecPath $condaExecPath + & $condaPythonPath ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} condaPath + & $condaExecPath init --all + + - name: Run TypeScript unit tests + run: npm run test:unittests:cover + + - name: Run Python unit tests + run: | + python pythonFiles/tests/run_all.py + + # The virtual environment based tests use the `testSingleWorkspace` set of tests + # with the environment variable `TEST_FILES_SUFFIX` set to `testvirtualenvs`, + # which is set in the "Prepare environment for venv tests" step. + # We also use a third-party GitHub Action to install xvfb on Linux, + # run tests and then clean up the process once the tests ran. + # See https://github.com/GabrielBB/xvfb-action + - name: Run venv tests + env: + TEST_FILES_SUFFIX: testvirtualenvs + CI_PYTHON_VERSION: ${{ env.PYTHON_VERSION }} + CI_DISABLE_AUTO_SELECTION: 1 + uses: GabrielBB/xvfb-action@v1.6 + with: + run: npm run testSingleWorkspace:cover + + - name: Run single-workspace tests + env: + CI_PYTHON_VERSION: ${{ env.PYTHON_VERSION }} + CI_DISABLE_AUTO_SELECTION: 1 + uses: GabrielBB/xvfb-action@v1.6 + with: + run: npm run testSingleWorkspace:cover + + # Enable these tests when coverage is setup for multiroot workspace tests + # - name: Run multi-workspace tests + # env: + # CI_PYTHON_VERSION: ${{ env.PYTHON_VERSION }} + # CI_DISABLE_AUTO_SELECTION: 1 + # uses: GabrielBB/xvfb-action@v1.6 + # with: + # run: npm run testMultiWorkspace:cover + + # Enable these tests when coverage is setup for debugger tests + # - name: Run debugger tests + # env: + # CI_PYTHON_VERSION: ${{ env.PYTHON_VERSION }} + # CI_DISABLE_AUTO_SELECTION: 1 + # uses: GabrielBB/xvfb-action@v1.6 + # with: + # run: npm run testDebugger:cover + + # Run TypeScript functional tests + - name: Run TypeScript functional tests + env: + CI_PYTHON_VERSION: ${{ env.PYTHON_VERSION }} + CI_DISABLE_AUTO_SELECTION: 1 + run: npm run test:functional:cover + + - name: Generate coverage reports + run: npm run test:cover:report + + - name: Upload HTML report + uses: actions/upload-artifact@v3 + with: + name: ${{ runner.os }}-coverage-report-html + path: ./coverage + retention-days: 1 From 567938cd3869abfe7e9e7adf14814e23c366e832 Mon Sep 17 00:00:00 2001 From: Kartik Raj Date: Thu, 16 Nov 2023 22:35:03 +0000 Subject: [PATCH 16/30] Fix unit --- .../base/locators/lowLevel/condaLocator.unit.test.ts | 1 + .../base/locators/lowLevel/windowsRegistryLocator.unit.test.ts | 2 ++ 2 files changed, 3 insertions(+) diff --git a/src/test/pythonEnvironments/base/locators/lowLevel/condaLocator.unit.test.ts b/src/test/pythonEnvironments/base/locators/lowLevel/condaLocator.unit.test.ts index 276f28cd665b..a3215e47ebfa 100644 --- a/src/test/pythonEnvironments/base/locators/lowLevel/condaLocator.unit.test.ts +++ b/src/test/pythonEnvironments/base/locators/lowLevel/condaLocator.unit.test.ts @@ -17,6 +17,7 @@ suite('Conda Python Version Parser Tests', () => { setup(() => { readFileStub = sinon.stub(externalDeps, 'readFile'); + sinon.stub(externalDeps, 'inExperiment').returns(false); pathExistsStub = sinon.stub(externalDeps, 'pathExists'); pathExistsStub.resolves(true); diff --git a/src/test/pythonEnvironments/base/locators/lowLevel/windowsRegistryLocator.unit.test.ts b/src/test/pythonEnvironments/base/locators/lowLevel/windowsRegistryLocator.unit.test.ts index c4621b267ad6..a69a643f52d4 100644 --- a/src/test/pythonEnvironments/base/locators/lowLevel/windowsRegistryLocator.unit.test.ts +++ b/src/test/pythonEnvironments/base/locators/lowLevel/windowsRegistryLocator.unit.test.ts @@ -11,6 +11,7 @@ import { WindowsRegistryLocator } from '../../../../../client/pythonEnvironments import { createBasicEnv } from '../../common'; import { TEST_LAYOUT_ROOT } from '../../../common/commonTestConstants'; import { assertBasicEnvsEqual } from '../envTestUtils'; +import * as externalDependencies from '../../../../../client/pythonEnvironments/common/externalDependencies'; suite('Windows Registry', () => { let stubReadRegistryValues: sinon.SinonStub; @@ -200,6 +201,7 @@ suite('Windows Registry', () => { } setup(async () => { + sinon.stub(externalDependencies, 'inExperiment').returns(false); stubReadRegistryValues = sinon.stub(winreg, 'readRegistryValues'); stubReadRegistryKeys = sinon.stub(winreg, 'readRegistryKeys'); stubReadRegistryValues.callsFake(fakeRegistryValues); From ece6a1c5f558a4f0afea04ad485164b176056761 Mon Sep 17 00:00:00 2001 From: Kartik Raj Date: Fri, 17 Nov 2023 00:04:39 +0000 Subject: [PATCH 17/30] Fix --- .../base/locators/lowLevel/condaLocator.unit.test.ts | 3 +-- .../common/environmentManagers/conda.unit.test.ts | 5 +++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/test/pythonEnvironments/base/locators/lowLevel/condaLocator.unit.test.ts b/src/test/pythonEnvironments/base/locators/lowLevel/condaLocator.unit.test.ts index a3215e47ebfa..1bf3ef19398d 100644 --- a/src/test/pythonEnvironments/base/locators/lowLevel/condaLocator.unit.test.ts +++ b/src/test/pythonEnvironments/base/locators/lowLevel/condaLocator.unit.test.ts @@ -24,8 +24,7 @@ suite('Conda Python Version Parser Tests', () => { }); teardown(() => { - readFileStub.restore(); - pathExistsStub.restore(); + sinon.restore(); }); interface ICondaPythonVersionTestData { diff --git a/src/test/pythonEnvironments/common/environmentManagers/conda.unit.test.ts b/src/test/pythonEnvironments/common/environmentManagers/conda.unit.test.ts index 1e9de68ad77a..268c5333a42f 100644 --- a/src/test/pythonEnvironments/common/environmentManagers/conda.unit.test.ts +++ b/src/test/pythonEnvironments/common/environmentManagers/conda.unit.test.ts @@ -593,6 +593,11 @@ suite('Conda and its environments are located correctly', () => { }, }, }; + sinon.stub(externalDependencies, 'inExperiment').returns(false); + }); + + teardown(() => { + sinon.restore(); }); test('Must compute conda environment name from prefix', async () => { From 384d4220dea21212ee95c33fa85487a8b4176141 Mon Sep 17 00:00:00 2001 From: Kartik Raj Date: Fri, 17 Nov 2023 03:54:12 +0000 Subject: [PATCH 18/30] Fix venv tests --- .github/workflows/build.yml | 2 +- .github/workflows/pr-check.yml | 4 ++-- src/client/common/constants.ts | 3 ++- .../lowLevel/condaLocator.testvirtualenvs.ts | 22 ++++++++++++++++++- 4 files changed, 26 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d17d5fff4b92..2a155b5ebf98 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -282,7 +282,7 @@ jobs: shell: pwsh if: matrix.test-suite == 'venv' run: | - # 1. For `terminalActivation.testvirtualenvs.test.ts` + # 1. For `*.testvirtualenvs.test.ts` if ('${{ matrix.os }}' -match 'windows-latest') { $condaPythonPath = Join-Path -Path $Env:CONDA -ChildPath python.exe $condaExecPath = Join-Path -Path $Env:CONDA -ChildPath Scripts | Join-Path -ChildPath conda diff --git a/.github/workflows/pr-check.yml b/.github/workflows/pr-check.yml index 3283d3281e5b..eb8cce0bce39 100644 --- a/.github/workflows/pr-check.yml +++ b/.github/workflows/pr-check.yml @@ -255,7 +255,7 @@ jobs: shell: pwsh if: matrix.test-suite == 'venv' run: | - # 1. For `terminalActivation.testvirtualenvs.test.ts` + # 1. For `*.testvirtualenvs.test.ts` if ('${{ matrix.os }}' -match 'windows-latest') { $condaPythonPath = Join-Path -Path $Env:CONDA -ChildPath python.exe $condaExecPath = Join-Path -Path $Env:CONDA -ChildPath Scripts | Join-Path -ChildPath conda @@ -451,7 +451,7 @@ jobs: PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' shell: pwsh run: | - # 1. For `terminalActivation.testvirtualenvs.test.ts` + # 1. For `*.testvirtualenvs.test.ts` if ('${{ matrix.os }}' -match 'windows-latest') { $condaPythonPath = Join-Path -Path $Env:CONDA -ChildPath python.exe $condaExecPath = Join-Path -Path $Env:CONDA -ChildPath Scripts | Join-Path -ChildPath conda diff --git a/src/client/common/constants.ts b/src/client/common/constants.ts index 6fc743fb8a0a..ac9387784c2c 100644 --- a/src/client/common/constants.ts +++ b/src/client/common/constants.ts @@ -94,7 +94,8 @@ export namespace ThemeIcons { export const DEFAULT_INTERPRETER_SETTING = 'python'; -export const isCI = process.env.TRAVIS === 'true' || process.env.TF_BUILD !== undefined; +export const isCI = + process.env.TRAVIS === 'true' || process.env.TF_BUILD !== undefined || process.env.GITHUB_ACTIONS === 'true'; export function isTestExecution(): boolean { return process.env.VSC_PYTHON_CI_TEST === '1' || isUnitTestExecution(); diff --git a/src/test/pythonEnvironments/base/locators/lowLevel/condaLocator.testvirtualenvs.ts b/src/test/pythonEnvironments/base/locators/lowLevel/condaLocator.testvirtualenvs.ts index 81060a0635c8..a89a7ca9718a 100644 --- a/src/test/pythonEnvironments/base/locators/lowLevel/condaLocator.testvirtualenvs.ts +++ b/src/test/pythonEnvironments/base/locators/lowLevel/condaLocator.testvirtualenvs.ts @@ -1,3 +1,4 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. @@ -10,11 +11,14 @@ import { CondaEnvironmentLocator } from '../../../../../client/pythonEnvironment import { sleep } from '../../../../core'; import { createDeferred, Deferred } from '../../../../../client/common/utils/async'; import { PythonEnvsChangedEvent } from '../../../../../client/pythonEnvironments/base/watcher'; -import { TEST_TIMEOUT } from '../../../../constants'; +import { EXTENSION_ROOT_DIR_FOR_TESTS, TEST_TIMEOUT } from '../../../../constants'; import { traceWarn } from '../../../../../client/logging'; import { TEST_LAYOUT_ROOT } from '../../../common/commonTestConstants'; import { getEnvs } from '../../common'; import { assertBasicEnvsEqual } from '../envTestUtils'; +import { PYTHON_VIRTUAL_ENVS_LOCATION } from '../../../../ciConstants'; +import { isCI } from '../../../../../client/common/constants'; +import * as externalDependencies from '../../../../../client/pythonEnvironments/common/externalDependencies'; class CondaEnvs { private readonly condaEnvironmentsTxt; @@ -55,6 +59,10 @@ class CondaEnvs { suite('Conda Env Locator', async () => { let locator: CondaEnvironmentLocator; let condaEnvsTxt: CondaEnvs; + const envsLocation = + PYTHON_VIRTUAL_ENVS_LOCATION !== undefined + ? path.join(EXTENSION_ROOT_DIR_FOR_TESTS, PYTHON_VIRTUAL_ENVS_LOCATION) + : path.join(EXTENSION_ROOT_DIR_FOR_TESTS, 'src', 'tmp', 'envPaths.json'); async function waitForChangeToBeDetected(deferred: Deferred) { const timeout = setTimeout(() => { @@ -63,11 +71,23 @@ suite('Conda Env Locator', async () => { }, TEST_TIMEOUT); await deferred.promise; } + let envPaths: any; + + suiteSetup(async () => { + if (isCI) { + envPaths = await fs.readJson(envsLocation); + } + }); setup(async () => { sinon.stub(platformUtils, 'getUserHomeDir').returns(TEST_LAYOUT_ROOT); condaEnvsTxt = new CondaEnvs(); await condaEnvsTxt.cleanUp(); + if (isCI) { + console.log('I am in CI'); + console.log(JSON.stringify(envPaths)); + sinon.stub(externalDependencies, 'getPythonSetting').returns(envPaths.condaExecPath); + } }); async function setupLocator(onChanged: (e: PythonEnvsChangedEvent) => Promise) { From d08a51dae0c529c00de79d7741a477a03c32574c Mon Sep 17 00:00:00 2001 From: Kartik Raj Date: Fri, 17 Nov 2023 04:04:06 +0000 Subject: [PATCH 19/30] Remove console statements --- .../base/locators/lowLevel/condaLocator.testvirtualenvs.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/test/pythonEnvironments/base/locators/lowLevel/condaLocator.testvirtualenvs.ts b/src/test/pythonEnvironments/base/locators/lowLevel/condaLocator.testvirtualenvs.ts index a89a7ca9718a..b3e1084a56be 100644 --- a/src/test/pythonEnvironments/base/locators/lowLevel/condaLocator.testvirtualenvs.ts +++ b/src/test/pythonEnvironments/base/locators/lowLevel/condaLocator.testvirtualenvs.ts @@ -84,8 +84,6 @@ suite('Conda Env Locator', async () => { condaEnvsTxt = new CondaEnvs(); await condaEnvsTxt.cleanUp(); if (isCI) { - console.log('I am in CI'); - console.log(JSON.stringify(envPaths)); sinon.stub(externalDependencies, 'getPythonSetting').returns(envPaths.condaExecPath); } }); From ee6cc127d62f6f20c02dfde18d919f2e4ac8b592 Mon Sep 17 00:00:00 2001 From: Kartik Raj Date: Fri, 17 Nov 2023 04:09:36 +0000 Subject: [PATCH 20/30] z --- .vscode/settings.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 3251f7efcad5..50864f0b5cd8 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -25,7 +25,8 @@ "[python]": { "editor.formatOnSave": true, "editor.codeActionsOnSave": { - "source.organizeImports.isort": true + "source.fixAll.eslint": "explicit", + "source.organizeImports.isort": "explicit" }, "editor.defaultFormatter": "ms-python.black-formatter", }, @@ -51,7 +52,7 @@ "prettier.printWidth": 120, "prettier.singleQuote": true, "editor.codeActionsOnSave": { - "source.fixAll.eslint": true + "source.fixAll.eslint": "explicit" }, "python.languageServer": "Default", "typescript.preferences.importModuleSpecifier": "relative", From 4c742ce87c7acb470669c54170fba3fb24435a45 Mon Sep 17 00:00:00 2001 From: Kartik Raj Date: Fri, 17 Nov 2023 04:12:47 +0000 Subject: [PATCH 21/30] cleanup --- src/client/common/process/worker/main.ts | 8 ++++---- src/client/common/process/worker/plainExecWorker.ts | 8 +------- src/client/common/process/worker/rawProcessApiWrapper.ts | 4 +--- src/client/common/process/worker/shellExecWorker.ts | 2 +- 4 files changed, 7 insertions(+), 15 deletions(-) diff --git a/src/client/common/process/worker/main.ts b/src/client/common/process/worker/main.ts index d8f158a73c70..f6a61b766e38 100644 --- a/src/client/common/process/worker/main.ts +++ b/src/client/common/process/worker/main.ts @@ -8,11 +8,11 @@ import { traceError } from '../../../logging'; export async function executeWorkerFile(workerFileName: string, workerData: any): Promise { return new Promise((resolve, reject) => { const worker = new Worker(workerFileName, { workerData }); - worker.on('message', (res: { err: Error; res: unknown }) => { - if (res.err) { - reject(res.err); + worker.on('message', (msg: { err: Error; res: unknown }) => { + if (msg.err) { + reject(msg.err); } - resolve(res.res); + resolve(msg.res); }); worker.on('error', (ex: Error) => { traceError(`Error in worker ${workerFileName}`, ex); diff --git a/src/client/common/process/worker/plainExecWorker.ts b/src/client/common/process/worker/plainExecWorker.ts index 9893142ed51c..f44ea15f9653 100644 --- a/src/client/common/process/worker/plainExecWorker.ts +++ b/src/client/common/process/worker/plainExecWorker.ts @@ -1,13 +1,7 @@ import { parentPort, workerData } from 'worker_threads'; import { _workerPlainExecImpl } from './workerRawProcessApis'; -_workerPlainExecImpl( - workerData.file, - workerData.args, - workerData.options, - workerData.defaultEnv, - workerData.disposables, -) +_workerPlainExecImpl(workerData.file, workerData.args, workerData.options) .then((res) => { if (!parentPort) { throw new Error('Not in a worker thread'); diff --git a/src/client/common/process/worker/rawProcessApiWrapper.ts b/src/client/common/process/worker/rawProcessApiWrapper.ts index 5c4580c4e05f..bc3040a2bfee 100644 --- a/src/client/common/process/worker/rawProcessApiWrapper.ts +++ b/src/client/common/process/worker/rawProcessApiWrapper.ts @@ -5,19 +5,17 @@ import { SpawnOptions } from 'child_process'; import * as path from 'path'; import { IProcessLogger } from '../types'; import { executeWorkerFile } from './main'; -import { EnvironmentVariables, ExecutionResult, ShellOptions } from './types'; +import { ExecutionResult, ShellOptions } from './types'; export function workerShellExec( command: string, options: ShellOptions, processLogger: IProcessLogger, - defaultEnv?: EnvironmentVariables, ): Promise> { processLogger.logProcess(command, undefined, options); return executeWorkerFile(path.join(__dirname, 'shellExecWorker.js'), { command, options, - defaultEnv, }); } diff --git a/src/client/common/process/worker/shellExecWorker.ts b/src/client/common/process/worker/shellExecWorker.ts index ffaa5f435484..f4e9809a29a5 100644 --- a/src/client/common/process/worker/shellExecWorker.ts +++ b/src/client/common/process/worker/shellExecWorker.ts @@ -1,7 +1,7 @@ import { parentPort, workerData } from 'worker_threads'; import { _workerShellExecImpl } from './workerRawProcessApis'; -_workerShellExecImpl(workerData.command, workerData.options, workerData.defaultEnv, workerData.disposables) +_workerShellExecImpl(workerData.command, workerData.options, workerData.defaultEnv) .then((res) => { if (!parentPort) { throw new Error('Not in a worker thread'); From 7580f323b167a58534fea9a860fd98472e3a6a6e Mon Sep 17 00:00:00 2001 From: Kartik Raj Date: Fri, 17 Nov 2023 05:02:20 +0000 Subject: [PATCH 22/30] Add more tests --- src/client/common/process/proc.ts | 9 ++++- src/client/common/process/types.ts | 3 +- .../process/worker/rawProcessApiWrapper.ts | 10 +---- .../common/externalDependencies.ts | 37 +++++-------------- src/test/common/process/proc.exec.test.ts | 22 +++++++++++ 5 files changed, 42 insertions(+), 39 deletions(-) diff --git a/src/client/common/process/proc.ts b/src/client/common/process/proc.ts index 18add7daf6fa..4a5aa984fa44 100644 --- a/src/client/common/process/proc.ts +++ b/src/client/common/process/proc.ts @@ -7,6 +7,7 @@ import { IDisposable } from '../types'; import { EnvironmentVariables } from '../variables/types'; import { execObservable, killPid, plainExec, shellExec } from './rawProcessApis'; import { ExecutionResult, IProcessService, ObservableExecutionResult, ShellOptions, SpawnOptions } from './types'; +import { workerPlainExec, workerShellExec } from './worker/rawProcessApiWrapper'; export class ProcessService extends EventEmitter implements IProcessService { private processesToKill = new Set(); @@ -47,14 +48,20 @@ export class ProcessService extends EventEmitter implements IProcessService { } public exec(file: string, args: string[], options: SpawnOptions = {}): Promise> { + this.emit('exec', file, args, options); + if (options.useWorker) { + return workerPlainExec(file, args, options); + } const execOptions = { ...options, doNotLog: true }; const promise = plainExec(file, args, execOptions, this.env, this.processesToKill); - this.emit('exec', file, args, options); return promise; } public shellExec(command: string, options: ShellOptions = {}): Promise> { this.emit('exec', command, undefined, options); + if (options.useWorker) { + return workerShellExec(command, options); + } const disposables = new Set(); const shellOptions = { ...options, doNotLog: true }; return shellExec(command, shellOptions, this.env, disposables).finally(() => { diff --git a/src/client/common/process/types.ts b/src/client/common/process/types.ts index d4b742718e36..9263e69cbe21 100644 --- a/src/client/common/process/types.ts +++ b/src/client/common/process/types.ts @@ -26,9 +26,10 @@ export type SpawnOptions = ChildProcessSpawnOptions & { extraVariables?: NodeJS.ProcessEnv; outputChannel?: OutputChannel; stdinStr?: string; + useWorker?: boolean; }; -export type ShellOptions = ExecOptions & { throwOnStdErr?: boolean }; +export type ShellOptions = ExecOptions & { throwOnStdErr?: boolean; useWorker?: boolean }; export type ExecutionResult = { stdout: T; diff --git a/src/client/common/process/worker/rawProcessApiWrapper.ts b/src/client/common/process/worker/rawProcessApiWrapper.ts index bc3040a2bfee..b4fa2ab97db0 100644 --- a/src/client/common/process/worker/rawProcessApiWrapper.ts +++ b/src/client/common/process/worker/rawProcessApiWrapper.ts @@ -3,16 +3,10 @@ import { SpawnOptions } from 'child_process'; import * as path from 'path'; -import { IProcessLogger } from '../types'; import { executeWorkerFile } from './main'; import { ExecutionResult, ShellOptions } from './types'; -export function workerShellExec( - command: string, - options: ShellOptions, - processLogger: IProcessLogger, -): Promise> { - processLogger.logProcess(command, undefined, options); +export function workerShellExec(command: string, options: ShellOptions): Promise> { return executeWorkerFile(path.join(__dirname, 'shellExecWorker.js'), { command, options, @@ -23,9 +17,7 @@ export function workerPlainExec( file: string, args: string[], options: SpawnOptions = {}, - processLogger: IProcessLogger, ): Promise> { - processLogger.logProcess(file, args, options); return executeWorkerFile(path.join(__dirname, 'plainExecWorker.js'), { file, args, diff --git a/src/client/pythonEnvironments/common/externalDependencies.ts b/src/client/pythonEnvironments/common/externalDependencies.ts index f102f1df5059..2ecbbdf0cb11 100644 --- a/src/client/pythonEnvironments/common/externalDependencies.ts +++ b/src/client/pythonEnvironments/common/externalDependencies.ts @@ -5,21 +5,13 @@ import * as fsapi from 'fs-extra'; import * as path from 'path'; import * as vscode from 'vscode'; import { IWorkspaceService } from '../../common/application/types'; -import { - ExecutionResult, - IProcessLogger, - IProcessServiceFactory, - ShellOptions, - SpawnOptions, -} from '../../common/process/types'; +import { ExecutionResult, IProcessServiceFactory, ShellOptions, SpawnOptions } from '../../common/process/types'; import { IDisposable, IConfigurationService, IExperimentService } from '../../common/types'; import { chain, iterable } from '../../common/utils/async'; import { getOSType, OSType } from '../../common/utils/platform'; import { IServiceContainer } from '../../ioc/types'; import { traceError, traceVerbose } from '../../logging'; import { DiscoveryUsingWorkers } from '../../common/experiments/groups'; -import { workerPlainExec, workerShellExec } from '../../common/process/worker/rawProcessApiWrapper'; -import { IEnvironmentVariablesProvider } from '../../common/variables/types'; let internalServiceContainer: IServiceContainer; export function initializeExternalDependencies(serviceContainer: IServiceContainer): void { @@ -29,32 +21,21 @@ export function initializeExternalDependencies(serviceContainer: IServiceContain // processes export async function shellExecute(command: string, options: ShellOptions = {}): Promise> { - if (!inExperiment(DiscoveryUsingWorkers.experiment)) { - const service = await internalServiceContainer.get(IProcessServiceFactory).create(); - return service.shellExec(command, options); - } - const logger = internalServiceContainer.get(IProcessLogger); - const envVarsService = internalServiceContainer.get(IEnvironmentVariablesProvider); - const envs = await envVarsService.getEnvironmentVariables(); - options.env = { ...options.env, ...envs }; - return workerShellExec(command, options, logger); + const useWorker = inExperiment(DiscoveryUsingWorkers.experiment); + const service = await internalServiceContainer.get(IProcessServiceFactory).create(); + options = { ...options, useWorker }; + return service.shellExec(command, options); } export async function exec( file: string, args: string[], options: SpawnOptions = {}, - useWorkerThreads = inExperiment(DiscoveryUsingWorkers.experiment), + useWorker = inExperiment(DiscoveryUsingWorkers.experiment), ): Promise> { - if (!useWorkerThreads) { - const service = await internalServiceContainer.get(IProcessServiceFactory).create(); - return service.exec(file, args, options); - } - const logger = internalServiceContainer.get(IProcessLogger); - const envVarsService = internalServiceContainer.get(IEnvironmentVariablesProvider); - const envs = await envVarsService.getEnvironmentVariables(); - options.env = { ...options.env, ...envs }; - return workerPlainExec(file, args, options, logger); + const service = await internalServiceContainer.get(IProcessServiceFactory).create(); + options = { ...options, useWorker }; + return service.exec(file, args, options); } export function inExperiment(experimentName: string): boolean { diff --git a/src/test/common/process/proc.exec.test.ts b/src/test/common/process/proc.exec.test.ts index c193df95d080..7e771e884b82 100644 --- a/src/test/common/process/proc.exec.test.ts +++ b/src/test/common/process/proc.exec.test.ts @@ -34,6 +34,16 @@ suite('ProcessService Observable', () => { expect(result.stderr).to.equal(undefined, 'stderr not undefined'); }); + test('When using worker threads, exec should output print statements', async () => { + const procService = new ProcessService(); + const printOutput = '1234'; + const result = await procService.exec(pythonPath, ['-c', `print("${printOutput}")`], { useWorker: true }); + + expect(result).not.to.be.an('undefined', 'result is undefined'); + expect(result.stdout.trim()).to.be.equal(printOutput, 'Invalid output'); + expect(result.stderr).to.equal(undefined, 'stderr not undefined'); + }); + test('exec should output print unicode characters', async function () { // This test has not been working for many months in Python 2.7 under // Windows. Tracked by #2546. (unicode under Py2.7 is tough!) @@ -241,6 +251,18 @@ suite('ProcessService Observable', () => { expect(result.stderr).to.equal(undefined, 'stderr not empty'); expect(result.stdout.trim()).to.be.equal(printOutput, 'Invalid output'); }); + test('When using worker threads, shellExec should be able to run python and filter output using conda related markers', async () => { + const procService = new ProcessService(); + const printOutput = '1234'; + const result = await procService.shellExec( + `"${pythonPath}" -c "print('>>>PYTHON-EXEC-OUTPUT');print('${printOutput}');print('<< { const procService = new ProcessService(); const result = procService.shellExec('invalid command'); From e16b48e29a2559e778ee04bbfb2f4c887b12af4f Mon Sep 17 00:00:00 2001 From: Kartik Raj Date: Sat, 18 Nov 2023 00:44:50 +0000 Subject: [PATCH 23/30] Add fixes --- .gitignore | 1 + build/webpack/webpack.extension.config.js | 8 +++ package-lock.json | 51 +++++++++++-------- package.json | 1 + src/client/common/process/worker/main.ts | 15 +++++- ...plainExecWorker.ts => plainExec.worker.ts} | 0 .../process/worker/rawProcessApiWrapper.ts | 4 +- ...shellExecWorker.ts => shellExec.worker.ts} | 0 ...ryKeysWorker.ts => registryKeys.worker.ts} | 0 ...luesWorker.ts => registryValues.worker.ts} | 0 .../common/windowsRegistry.ts | 4 +- 11 files changed, 59 insertions(+), 25 deletions(-) rename src/client/common/process/worker/{plainExecWorker.ts => plainExec.worker.ts} (100%) rename src/client/common/process/worker/{shellExecWorker.ts => shellExec.worker.ts} (100%) rename src/client/pythonEnvironments/common/{registryKeysWorker.ts => registryKeys.worker.ts} (100%) rename src/client/pythonEnvironments/common/{registryValuesWorker.ts => registryValues.worker.ts} (100%) diff --git a/.gitignore b/.gitignore index 046d01588573..69166ccac1a4 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ log.log **/node_modules *.pyc *.vsix +envVars.txt **/.vscode/.ropeproject/** **/testFiles/**/.cache/** *.noseids diff --git a/build/webpack/webpack.extension.config.js b/build/webpack/webpack.extension.config.js index a33508e5d96a..082ce52a4d32 100644 --- a/build/webpack/webpack.extension.config.js +++ b/build/webpack/webpack.extension.config.js @@ -19,6 +19,10 @@ const config = { target: 'node', entry: { extension: './src/client/extension.ts', + 'shellExec.worker': './src/client/common/process/worker/shellExec.worker.ts', + 'plainExec.worker': './src/client/common/process/worker/plainExec.worker.ts', + 'registryKeys.worker': 'src/client/pythonEnvironments/common/registryKeys.worker.ts', + 'registryValues.worker': 'src/client/pythonEnvironments/common/registryValues.worker.ts', }, devtool: 'source-map', node: { @@ -51,6 +55,10 @@ const config = { }, ], }, + { + test: /\.worker\.js$/, + use: { loader: 'worker-loader' }, + }, ], }, externals: [ diff --git a/package-lock.json b/package-lock.json index 250ba6866f28..dacf9be4e84f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -106,7 +106,6 @@ "typemoq": "^2.1.0", "typescript": "4.5.5", "uuid": "^8.3.2", - "vscode-debugadapter-testsupport": "^1.27.0", "webpack": "^5.76.0", "webpack-bundle-analyzer": "^4.5.0", "webpack-cli": "^4.9.2", @@ -114,6 +113,7 @@ "webpack-merge": "^5.8.0", "webpack-node-externals": "^3.0.0", "webpack-require-from": "^1.8.6", + "worker-loader": "^3.0.8", "yargs": "^15.3.1" }, "engines": { @@ -14513,16 +14513,6 @@ "integrity": "sha512-2ham8XPWTONajOR0ohOKOHXkm3+gaBmGut3SRuu75xLd/RRaY6vqgh8NBYYk7+RW3u5AtzPQZG8F10LHkl0lAQ==", "dev": true }, - "node_modules/vscode-debugadapter-testsupport": { - "version": "1.35.0", - "resolved": "https://registry.npmjs.org/vscode-debugadapter-testsupport/-/vscode-debugadapter-testsupport-1.35.0.tgz", - "integrity": "sha512-4emLt6JOk4iKqC2aWNJupOtrK6JwYAZ6KppqvKASN6B1s063VoqI18QhUB1CeoKwNaN1LIG3VPv2xM8HKOjyDA==", - "deprecated": "This package has been renamed to @vscode/debugadapter-testsupport, please update to the new name", - "dev": true, - "dependencies": { - "vscode-debugprotocol": "1.35.0" - } - }, "node_modules/vscode-debugprotocol": { "version": "1.35.0", "resolved": "https://registry.npmjs.org/vscode-debugprotocol/-/vscode-debugprotocol-1.35.0.tgz", @@ -14970,6 +14960,26 @@ "wipe-node-cache": "^2.1.0" } }, + "node_modules/worker-loader": { + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/worker-loader/-/worker-loader-3.0.8.tgz", + "integrity": "sha512-XQyQkIFeRVC7f7uRhFdNMe/iJOdO6zxAaR3EWbDp45v3mDhrTi+++oswKNxShUNjPC/1xUp5DB29YKLhFo129g==", + "dev": true, + "dependencies": { + "loader-utils": "^2.0.0", + "schema-utils": "^3.0.0" + }, + "engines": { + "node": ">= 10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "webpack": "^4.0.0 || ^5.0.0" + } + }, "node_modules/workerpool": { "version": "6.2.0", "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.2.0.tgz", @@ -26646,15 +26656,6 @@ "integrity": "sha512-2ham8XPWTONajOR0ohOKOHXkm3+gaBmGut3SRuu75xLd/RRaY6vqgh8NBYYk7+RW3u5AtzPQZG8F10LHkl0lAQ==", "dev": true }, - "vscode-debugadapter-testsupport": { - "version": "1.35.0", - "resolved": "https://registry.npmjs.org/vscode-debugadapter-testsupport/-/vscode-debugadapter-testsupport-1.35.0.tgz", - "integrity": "sha512-4emLt6JOk4iKqC2aWNJupOtrK6JwYAZ6KppqvKASN6B1s063VoqI18QhUB1CeoKwNaN1LIG3VPv2xM8HKOjyDA==", - "dev": true, - "requires": { - "vscode-debugprotocol": "1.35.0" - } - }, "vscode-debugprotocol": { "version": "1.35.0", "resolved": "https://registry.npmjs.org/vscode-debugprotocol/-/vscode-debugprotocol-1.35.0.tgz", @@ -26980,6 +26981,16 @@ "wipe-node-cache": "^2.1.0" } }, + "worker-loader": { + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/worker-loader/-/worker-loader-3.0.8.tgz", + "integrity": "sha512-XQyQkIFeRVC7f7uRhFdNMe/iJOdO6zxAaR3EWbDp45v3mDhrTi+++oswKNxShUNjPC/1xUp5DB29YKLhFo129g==", + "dev": true, + "requires": { + "loader-utils": "^2.0.0", + "schema-utils": "^3.0.0" + } + }, "workerpool": { "version": "6.2.0", "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.2.0.tgz", diff --git a/package.json b/package.json index 068f04114cae..f009f198faf0 100644 --- a/package.json +++ b/package.json @@ -1659,6 +1659,7 @@ "typescript": "4.5.5", "uuid": "^8.3.2", "webpack": "^5.76.0", + "worker-loader": "^3.0.8", "webpack-bundle-analyzer": "^4.5.0", "webpack-cli": "^4.9.2", "webpack-fix-default-import-plugin": "^1.0.3", diff --git a/src/client/common/process/worker/main.ts b/src/client/common/process/worker/main.ts index f6a61b766e38..1694607b6899 100644 --- a/src/client/common/process/worker/main.ts +++ b/src/client/common/process/worker/main.ts @@ -2,13 +2,25 @@ // Licensed under the MIT License. import { Worker } from 'worker_threads'; -import { traceError } from '../../../logging'; +import { traceVerbose, traceError } from '../../../logging/index'; +/** + * Executes a worker file. Make sure to declare the worker file as a entry in the webpack config. + * @param workerFileName Filename of the worker file to execute, it has to end with ".worker.js" for webpack to bundle it. + * @param workerData Arguments to the worker file. + * @returns + */ // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/explicit-module-boundary-types export async function executeWorkerFile(workerFileName: string, workerData: any): Promise { + if (!workerFileName.endsWith('.worker.js')) { + throw new Error('Worker file must end with ".worker.js" for webpack to bundle webworkers'); + } return new Promise((resolve, reject) => { + traceVerbose(`Starting worker ${workerFileName} with data ${JSON.stringify(workerData)}`); const worker = new Worker(workerFileName, { workerData }); + traceVerbose(`Started worker ${workerFileName}`); worker.on('message', (msg: { err: Error; res: unknown }) => { + traceVerbose(`Worker ${workerFileName} sent message ${JSON.stringify(msg)}`); if (msg.err) { reject(msg.err); } @@ -19,6 +31,7 @@ export async function executeWorkerFile(workerFileName: string, workerData: any) reject(ex); }); worker.on('exit', (code) => { + traceVerbose(`Worker ${workerFileName} exited with code ${code}`); if (code !== 0) { reject(new Error(`Worker ${workerFileName} stopped with exit code ${code}`)); } diff --git a/src/client/common/process/worker/plainExecWorker.ts b/src/client/common/process/worker/plainExec.worker.ts similarity index 100% rename from src/client/common/process/worker/plainExecWorker.ts rename to src/client/common/process/worker/plainExec.worker.ts diff --git a/src/client/common/process/worker/rawProcessApiWrapper.ts b/src/client/common/process/worker/rawProcessApiWrapper.ts index b4fa2ab97db0..e6476df5d8fa 100644 --- a/src/client/common/process/worker/rawProcessApiWrapper.ts +++ b/src/client/common/process/worker/rawProcessApiWrapper.ts @@ -7,7 +7,7 @@ import { executeWorkerFile } from './main'; import { ExecutionResult, ShellOptions } from './types'; export function workerShellExec(command: string, options: ShellOptions): Promise> { - return executeWorkerFile(path.join(__dirname, 'shellExecWorker.js'), { + return executeWorkerFile(path.join(__dirname, 'shellExec.worker.js'), { command, options, }); @@ -18,7 +18,7 @@ export function workerPlainExec( args: string[], options: SpawnOptions = {}, ): Promise> { - return executeWorkerFile(path.join(__dirname, 'plainExecWorker.js'), { + return executeWorkerFile(path.join(__dirname, 'plainExec.worker.js'), { file, args, options, diff --git a/src/client/common/process/worker/shellExecWorker.ts b/src/client/common/process/worker/shellExec.worker.ts similarity index 100% rename from src/client/common/process/worker/shellExecWorker.ts rename to src/client/common/process/worker/shellExec.worker.ts diff --git a/src/client/pythonEnvironments/common/registryKeysWorker.ts b/src/client/pythonEnvironments/common/registryKeys.worker.ts similarity index 100% rename from src/client/pythonEnvironments/common/registryKeysWorker.ts rename to src/client/pythonEnvironments/common/registryKeys.worker.ts diff --git a/src/client/pythonEnvironments/common/registryValuesWorker.ts b/src/client/pythonEnvironments/common/registryValues.worker.ts similarity index 100% rename from src/client/pythonEnvironments/common/registryValuesWorker.ts rename to src/client/pythonEnvironments/common/registryValues.worker.ts diff --git a/src/client/pythonEnvironments/common/windowsRegistry.ts b/src/client/pythonEnvironments/common/windowsRegistry.ts index ac58963a6c14..801ef0c907b1 100644 --- a/src/client/pythonEnvironments/common/windowsRegistry.ts +++ b/src/client/pythonEnvironments/common/windowsRegistry.ts @@ -39,7 +39,7 @@ export async function readRegistryValues(options: Options, useWorkerThreads: boo }); return deferred.promise; } - return executeWorkerFile(path.join(__dirname, 'registryValuesWorker.js'), options); + return executeWorkerFile(path.join(__dirname, 'registryValues.worker.js'), options); } export async function readRegistryKeys(options: Options, useWorkerThreads: boolean): Promise { @@ -56,5 +56,5 @@ export async function readRegistryKeys(options: Options, useWorkerThreads: boole }); return deferred.promise; } - return executeWorkerFile(path.join(__dirname, 'registryKeysWorker.js'), options); + return executeWorkerFile(path.join(__dirname, 'registryKeys.worker.js'), options); } From fa3415364dd43a234de7c2389f54b6a7d8cebfc5 Mon Sep 17 00:00:00 2001 From: Kartik Raj Date: Sat, 18 Nov 2023 00:50:41 +0000 Subject: [PATCH 24/30] s --- .github/workflows/codeql-analysis.yml | 68 ---- .github/workflows/pr-check.yml | 475 +------------------------- .github/workflows/pr-labels.yml | 21 -- 3 files changed, 1 insertion(+), 563 deletions(-) delete mode 100644 .github/workflows/codeql-analysis.yml delete mode 100644 .github/workflows/pr-labels.yml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml deleted file mode 100644 index 5b037d5a1d0b..000000000000 --- a/.github/workflows/codeql-analysis.yml +++ /dev/null @@ -1,68 +0,0 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -name: 'CodeQL' - -on: - push: - branches: - - main - - release-* - - release/* - pull_request: - # The branches below must be a subset of the branches above - branches: [main] - schedule: - - cron: '0 3 * * 0' - -permissions: - security-events: write - -jobs: - analyze: - name: Analyze - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - # Override automatic language detection by changing the below list - # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python'] - language: ['javascript', 'python'] - # Learn more... - # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v2 - with: - languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - # queries: ./path/to/local/query, your-org/your-repo/queries@main - - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - #- name: Autobuild - # uses: github/codeql-action/autobuild@v1 - - # ℹī¸ Command-line programs to run using the OS shell. - # 📚 https://git.io/JvXDl - - # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines - # and modify them (or add more) to build your code if your project - # uses a compiled language - - #- run: | - # make bootstrap - # make release - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/pr-check.yml b/.github/workflows/pr-check.yml index eb8cce0bce39..86501cf11ed5 100644 --- a/.github/workflows/pr-check.yml +++ b/.github/workflows/pr-check.yml @@ -1,7 +1,6 @@ name: PR/CI Check on: - pull_request: push: branches-ignore: - main @@ -34,290 +33,6 @@ jobs: vsix_name: ${{ env.VSIX_NAME }} artifact_name: ${{ env.ARTIFACT_NAME_VSIX }} - lint: - name: Lint - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Lint - uses: ./.github/actions/lint - with: - node_version: ${{ env.NODE_VERSION }} - - check-types: - name: Check Python types - runs-on: ubuntu-latest - steps: - - name: Use Python ${{ env.PYTHON_VERSION }} - uses: actions/setup-python@v4 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: Checkout - uses: actions/checkout@v4 - - - name: Install base Python requirements - uses: brettcannon/pip-secure-install@v1 - with: - options: '-t ./pythonFiles/lib/python --no-cache-dir --implementation py' - - - name: Install Jedi requirements - uses: brettcannon/pip-secure-install@v1 - with: - requirements-file: './pythonFiles/jedilsp_requirements/requirements.txt' - options: '-t ./pythonFiles/lib/jedilsp --no-cache-dir --implementation py' - - - name: Install other Python requirements - run: | - python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --no-cache-dir --implementation py --no-deps --upgrade --pre debugpy - python -m pip install --upgrade -r build/test-requirements.txt - - - name: Run Pyright - uses: jakebailey/pyright-action@v1 - with: - version: 1.1.308 - working-directory: 'pythonFiles' - - python-tests: - name: Python Tests - # The value of runs-on is the OS of the current job (specified in the strategy matrix below) instead of being hardcoded. - runs-on: ${{ matrix.os }} - defaults: - run: - working-directory: ${{ env.special-working-directory }} - strategy: - fail-fast: false - matrix: - # We're not running CI on macOS for now because it's one less matrix entry to lower the number of runners used, - # macOS runners are expensive, and we assume that Ubuntu is enough to cover the Unix case. - os: [ubuntu-latest, windows-latest] - # Run the tests on the oldest and most recent versions of Python. - python: ['3.8', '3.x', '3.12-dev'] - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - path: ${{ env.special-working-directory-relative }} - - - name: Use Python ${{ matrix.python }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python }} - - - name: Install base Python requirements - uses: brettcannon/pip-secure-install@v1 - with: - requirements-file: '"${{ env.special-working-directory-relative }}/requirements.txt"' - options: '-t "${{ env.special-working-directory-relative }}/pythonFiles/lib/python" --no-cache-dir --implementation py' - - - name: Install test requirements - run: python -m pip install --upgrade -r build/test-requirements.txt - - - name: Run Python unit tests - run: python pythonFiles/tests/run_all.py - - tests: - name: Tests - # The value of runs-on is the OS of the current job (specified in the strategy matrix below) instead of being hardcoded. - runs-on: ${{ matrix.os }} - defaults: - run: - working-directory: ${{ env.special-working-directory }} - strategy: - fail-fast: false - matrix: - # We're not running CI on macOS for now because it's one less matrix entry to lower the number of runners used, - # macOS runners are expensive, and we assume that Ubuntu is enough to cover the Unix case. - os: [ubuntu-latest, windows-latest] - # Run the tests on the oldest and most recent versions of Python. - python: ['3.x'] - test-suite: [ts-unit, venv, single-workspace, debugger, functional] - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - path: ${{ env.special-working-directory-relative }} - - - name: Install Node - uses: actions/setup-node@v4 - with: - node-version: ${{ env.NODE_VERSION }} - cache: 'npm' - cache-dependency-path: ${{ env.special-working-directory-relative }}/package-lock.json - - - name: Install dependencies (npm ci) - run: npm ci - - - name: Compile - run: npx gulp prePublishNonBundle - - - name: Use Python ${{ matrix.python }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python }} - - - name: Install debugpy - run: | - # We need to have debugpy so that tests relying on it keep passing, but we don't need install_debugpy's logic in the test phase. - python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --no-cache-dir --implementation py --no-deps --upgrade --pre debugpy - - - name: Download get-pip.py - run: | - python -m pip install wheel - python -m pip install -r build/build-install-requirements.txt - python ./pythonFiles/download_get_pip.py - shell: bash - - - name: Install base Python requirements - uses: brettcannon/pip-secure-install@v1 - with: - requirements-file: '"${{ env.special-working-directory-relative }}/requirements.txt"' - options: '-t "${{ env.special-working-directory-relative }}/pythonFiles/lib/python" --no-cache-dir --implementation py' - - - name: Install Jedi requirements - uses: brettcannon/pip-secure-install@v1 - with: - requirements-file: '"${{ env.special-working-directory-relative }}/pythonFiles/jedilsp_requirements/requirements.txt"' - options: '-t "${{ env.special-working-directory-relative }}/pythonFiles/lib/jedilsp" --no-cache-dir --implementation py' - - - name: Install test requirements - run: python -m pip install --upgrade -r build/test-requirements.txt - - - name: Install debugpy wheels (Python ${{ matrix.python }}) - run: | - python -m pip install wheel - python -m pip --disable-pip-version-check install -r build/build-install-requirements.txt - python ./pythonFiles/install_debugpy.py - shell: bash - if: matrix.test-suite == 'debugger' - - - name: Install functional test requirements - run: python -m pip install --upgrade -r ./build/functional-test-requirements.txt - if: matrix.test-suite == 'functional' - - - name: Prepare pipenv for venv tests - env: - TEST_FILES_SUFFIX: testvirtualenvs - PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' - shell: pwsh - if: matrix.test-suite == 'venv' - run: | - python -m pip install pipenv - python -m pipenv run python ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} pipenvPath - - - name: Prepare poetry for venv tests - env: - TEST_FILES_SUFFIX: testvirtualenvs - shell: pwsh - if: matrix.test-suite == 'venv' - run: | - python -m pip install poetry - Move-Item -Path ".\build\ci\pyproject.toml" -Destination . - poetry env use python - - - name: Prepare virtualenv for venv tests - env: - TEST_FILES_SUFFIX: testvirtualenvs - PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' - shell: pwsh - if: matrix.test-suite == 'venv' - run: | - python -m pip install virtualenv - python -m virtualenv .virtualenv/ - if ('${{ matrix.os }}' -match 'windows-latest') { - & ".virtualenv/Scripts/python.exe" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} virtualEnvPath - } else { - & ".virtualenv/bin/python" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} virtualEnvPath - } - - - name: Prepare venv for venv tests - env: - TEST_FILES_SUFFIX: testvirtualenvs - PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' - shell: pwsh - if: matrix.test-suite == 'venv' && startsWith(matrix.python, 3.) - run: | - python -m venv .venv - if ('${{ matrix.os }}' -match 'windows-latest') { - & ".venv/Scripts/python.exe" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} venvPath - } else { - & ".venv/bin/python" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} venvPath - } - - - name: Prepare conda for venv tests - env: - TEST_FILES_SUFFIX: testvirtualenvs - PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' - shell: pwsh - if: matrix.test-suite == 'venv' - run: | - # 1. For `*.testvirtualenvs.test.ts` - if ('${{ matrix.os }}' -match 'windows-latest') { - $condaPythonPath = Join-Path -Path $Env:CONDA -ChildPath python.exe - $condaExecPath = Join-Path -Path $Env:CONDA -ChildPath Scripts | Join-Path -ChildPath conda - } else{ - $condaPythonPath = Join-Path -Path $Env:CONDA -ChildPath bin | Join-Path -ChildPath python - $condaExecPath = Join-Path -Path $Env:CONDA -ChildPath bin | Join-Path -ChildPath conda - } - & $condaPythonPath ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} condaExecPath $condaExecPath - & $condaPythonPath ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} condaPath - & $condaExecPath init --all - - - name: Set CI_PYTHON_PATH and CI_DISABLE_AUTO_SELECTION - run: | - echo "CI_PYTHON_PATH=python" >> $GITHUB_ENV - echo "CI_DISABLE_AUTO_SELECTION=1" >> $GITHUB_ENV - shell: bash - if: matrix.test-suite != 'ts-unit' - - # Run TypeScript unit tests only for Python 3.X. - - name: Run TypeScript unit tests - run: npm run test:unittests - if: matrix.test-suite == 'ts-unit' && startsWith(matrix.python, 3.) - - # The virtual environment based tests use the `testSingleWorkspace` set of tests - # with the environment variable `TEST_FILES_SUFFIX` set to `testvirtualenvs`, - # which is set in the "Prepare environment for venv tests" step. - # We also use a third-party GitHub Action to install xvfb on Linux, - # run tests and then clean up the process once the tests ran. - # See https://github.com/GabrielBB/xvfb-action - - name: Run venv tests - env: - TEST_FILES_SUFFIX: testvirtualenvs - CI_PYTHON_VERSION: ${{ matrix.python }} - uses: GabrielBB/xvfb-action@v1.6 - with: - run: npm run testSingleWorkspace - working-directory: ${{ env.special-working-directory }} - if: matrix.test-suite == 'venv' - - - name: Run single-workspace tests - env: - CI_PYTHON_VERSION: ${{ matrix.python }} - uses: GabrielBB/xvfb-action@v1.6 - with: - run: npm run testSingleWorkspace - working-directory: ${{ env.special-working-directory }} - if: matrix.test-suite == 'single-workspace' - - - name: Run debugger tests - env: - CI_PYTHON_VERSION: ${{ matrix.python }} - uses: GabrielBB/xvfb-action@v1.6 - with: - run: npm run testDebugger - working-directory: ${{ env.special-working-directory }} - if: matrix.test-suite == 'debugger' - - # Run TypeScript functional tests - - name: Run TypeScript functional tests - run: npm run test:functional - if: matrix.test-suite == 'functional' - smoke-tests: name: Smoke tests # The value of runs-on is the OS of the current job (specified in the strategy matrix below) instead of being hardcoded. @@ -328,7 +43,7 @@ jobs: matrix: # We're not running CI on macOS for now because it's one less matrix entry to lower the number of runners used, # macOS runners are expensive, and we assume that Ubuntu is enough to cover the UNIX case. - os: [ubuntu-latest, windows-latest] + os: [ubuntu-latest] steps: # Need the source to have the tests available. - name: Checkout @@ -339,191 +54,3 @@ jobs: with: node_version: ${{ env.NODE_VERSION }} artifact_name: ${{ env.ARTIFACT_NAME_VSIX }} - - ### Coverage run - coverage: - name: Coverage - # The value of runs-on is the OS of the current job (specified in the strategy matrix below) instead of being hardcoded. - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - # Only run coverage on linux for PRs - os: [ubuntu-latest] - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Install Node - uses: actions/setup-node@v4 - with: - node-version: ${{ env.NODE_VERSION }} - cache: 'npm' - - - name: Install dependencies (npm ci) - run: npm ci - - - name: Compile - run: npx gulp prePublishNonBundle - - - name: Use Python ${{ env.PYTHON_VERSION }} - uses: actions/setup-python@v4 - with: - python-version: ${{ env.PYTHON_VERSION }} - cache: 'pip' - cache-dependency-path: | - requirements.txt - pythonFiles/jedilsp_requirements/requirements.txt - build/test-requirements.txt - build/functional-test-requirements.txt - - - name: Install base Python requirements - uses: brettcannon/pip-secure-install@v1 - with: - options: '-t ./pythonFiles/lib/python --implementation py' - - - name: Install Jedi requirements - uses: brettcannon/pip-secure-install@v1 - with: - requirements-file: './pythonFiles/jedilsp_requirements/requirements.txt' - options: '-t ./pythonFiles/lib/jedilsp --implementation py' - - - name: Install debugpy - run: | - # We need to have debugpy so that tests relying on it keep passing, but we don't need install_debugpy's logic in the test phase. - python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --implementation py --no-deps --upgrade --pre debugpy - - - name: Install test requirements - run: python -m pip install --upgrade -r build/test-requirements.txt - - - name: Install functional test requirements - run: python -m pip install --upgrade -r ./build/functional-test-requirements.txt - - - name: Prepare pipenv for venv tests - env: - TEST_FILES_SUFFIX: testvirtualenvs - PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' - shell: pwsh - run: | - python -m pip install pipenv - python -m pipenv run python ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} pipenvPath - - - name: Prepare poetry for venv tests - env: - TEST_FILES_SUFFIX: testvirtualenvs - shell: pwsh - run: | - python -m pip install poetry - Move-Item -Path ".\build\ci\pyproject.toml" -Destination . - poetry env use python - - - name: Prepare virtualenv for venv tests - env: - TEST_FILES_SUFFIX: testvirtualenvs - PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' - shell: pwsh - run: | - python -m pip install virtualenv - python -m virtualenv .virtualenv/ - if ('${{ matrix.os }}' -match 'windows-latest') { - & ".virtualenv/Scripts/python.exe" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} virtualEnvPath - } else { - & ".virtualenv/bin/python" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} virtualEnvPath - } - - - name: Prepare venv for venv tests - env: - TEST_FILES_SUFFIX: testvirtualenvs - PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' - shell: pwsh - run: | - python -m venv .venv - if ('${{ matrix.os }}' -match 'windows-latest') { - & ".venv/Scripts/python.exe" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} venvPath - } else { - & ".venv/bin/python" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} venvPath - } - - - name: Prepare conda for venv tests - env: - TEST_FILES_SUFFIX: testvirtualenvs - PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' - shell: pwsh - run: | - # 1. For `*.testvirtualenvs.test.ts` - if ('${{ matrix.os }}' -match 'windows-latest') { - $condaPythonPath = Join-Path -Path $Env:CONDA -ChildPath python.exe - $condaExecPath = Join-Path -Path $Env:CONDA -ChildPath Scripts | Join-Path -ChildPath conda - } else{ - $condaPythonPath = Join-Path -Path $Env:CONDA -ChildPath bin | Join-Path -ChildPath python - $condaExecPath = Join-Path -Path $Env:CONDA -ChildPath bin | Join-Path -ChildPath conda - } - & $condaPythonPath ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} condaExecPath $condaExecPath - & $condaPythonPath ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} condaPath - & $condaExecPath init --all - - - name: Run TypeScript unit tests - run: npm run test:unittests:cover - - - name: Run Python unit tests - run: | - python pythonFiles/tests/run_all.py - - # The virtual environment based tests use the `testSingleWorkspace` set of tests - # with the environment variable `TEST_FILES_SUFFIX` set to `testvirtualenvs`, - # which is set in the "Prepare environment for venv tests" step. - # We also use a third-party GitHub Action to install xvfb on Linux, - # run tests and then clean up the process once the tests ran. - # See https://github.com/GabrielBB/xvfb-action - - name: Run venv tests - env: - TEST_FILES_SUFFIX: testvirtualenvs - CI_PYTHON_VERSION: ${{ env.PYTHON_VERSION }} - CI_DISABLE_AUTO_SELECTION: 1 - uses: GabrielBB/xvfb-action@v1.6 - with: - run: npm run testSingleWorkspace:cover - - - name: Run single-workspace tests - env: - CI_PYTHON_VERSION: ${{ env.PYTHON_VERSION }} - CI_DISABLE_AUTO_SELECTION: 1 - uses: GabrielBB/xvfb-action@v1.6 - with: - run: npm run testSingleWorkspace:cover - - # Enable these tests when coverage is setup for multiroot workspace tests - # - name: Run multi-workspace tests - # env: - # CI_PYTHON_VERSION: ${{ env.PYTHON_VERSION }} - # CI_DISABLE_AUTO_SELECTION: 1 - # uses: GabrielBB/xvfb-action@v1.6 - # with: - # run: npm run testMultiWorkspace:cover - - # Enable these tests when coverage is setup for debugger tests - # - name: Run debugger tests - # env: - # CI_PYTHON_VERSION: ${{ env.PYTHON_VERSION }} - # CI_DISABLE_AUTO_SELECTION: 1 - # uses: GabrielBB/xvfb-action@v1.6 - # with: - # run: npm run testDebugger:cover - - # Run TypeScript functional tests - - name: Run TypeScript functional tests - env: - CI_PYTHON_VERSION: ${{ env.PYTHON_VERSION }} - CI_DISABLE_AUTO_SELECTION: 1 - run: npm run test:functional:cover - - - name: Generate coverage reports - run: npm run test:cover:report - - - name: Upload HTML report - uses: actions/upload-artifact@v3 - with: - name: ${{ runner.os }}-coverage-report-html - path: ./coverage - retention-days: 1 diff --git a/.github/workflows/pr-labels.yml b/.github/workflows/pr-labels.yml deleted file mode 100644 index 730b8e5c5832..000000000000 --- a/.github/workflows/pr-labels.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: 'PR labels' -on: - pull_request: - types: - - 'opened' - - 'reopened' - - 'labeled' - - 'unlabeled' - - 'synchronize' - -jobs: - classify: - name: 'Classify PR' - runs-on: ubuntu-latest - steps: - - name: 'PR impact specified' - uses: mheap/github-action-required-labels@v5 - with: - mode: exactly - count: 1 - labels: 'bug, debt, feature-request, no-changelog' From c10dc6a1b14122a4644de6e5c4ce18a555bf4a32 Mon Sep 17 00:00:00 2001 From: Kartik Raj Date: Sat, 18 Nov 2023 00:57:20 +0000 Subject: [PATCH 25/30] Revert "s" This reverts commit fa3415364dd43a234de7c2389f54b6a7d8cebfc5. --- .github/workflows/codeql-analysis.yml | 68 ++++ .github/workflows/pr-check.yml | 475 +++++++++++++++++++++++++- .github/workflows/pr-labels.yml | 21 ++ 3 files changed, 563 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/codeql-analysis.yml create mode 100644 .github/workflows/pr-labels.yml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 000000000000..5b037d5a1d0b --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,68 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +name: 'CodeQL' + +on: + push: + branches: + - main + - release-* + - release/* + pull_request: + # The branches below must be a subset of the branches above + branches: [main] + schedule: + - cron: '0 3 * * 0' + +permissions: + security-events: write + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + # Override automatic language detection by changing the below list + # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python'] + language: ['javascript', 'python'] + # Learn more... + # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + #- name: Autobuild + # uses: github/codeql-action/autobuild@v1 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/pr-check.yml b/.github/workflows/pr-check.yml index 86501cf11ed5..eb8cce0bce39 100644 --- a/.github/workflows/pr-check.yml +++ b/.github/workflows/pr-check.yml @@ -1,6 +1,7 @@ name: PR/CI Check on: + pull_request: push: branches-ignore: - main @@ -33,6 +34,290 @@ jobs: vsix_name: ${{ env.VSIX_NAME }} artifact_name: ${{ env.ARTIFACT_NAME_VSIX }} + lint: + name: Lint + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Lint + uses: ./.github/actions/lint + with: + node_version: ${{ env.NODE_VERSION }} + + check-types: + name: Check Python types + runs-on: ubuntu-latest + steps: + - name: Use Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Checkout + uses: actions/checkout@v4 + + - name: Install base Python requirements + uses: brettcannon/pip-secure-install@v1 + with: + options: '-t ./pythonFiles/lib/python --no-cache-dir --implementation py' + + - name: Install Jedi requirements + uses: brettcannon/pip-secure-install@v1 + with: + requirements-file: './pythonFiles/jedilsp_requirements/requirements.txt' + options: '-t ./pythonFiles/lib/jedilsp --no-cache-dir --implementation py' + + - name: Install other Python requirements + run: | + python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --no-cache-dir --implementation py --no-deps --upgrade --pre debugpy + python -m pip install --upgrade -r build/test-requirements.txt + + - name: Run Pyright + uses: jakebailey/pyright-action@v1 + with: + version: 1.1.308 + working-directory: 'pythonFiles' + + python-tests: + name: Python Tests + # The value of runs-on is the OS of the current job (specified in the strategy matrix below) instead of being hardcoded. + runs-on: ${{ matrix.os }} + defaults: + run: + working-directory: ${{ env.special-working-directory }} + strategy: + fail-fast: false + matrix: + # We're not running CI on macOS for now because it's one less matrix entry to lower the number of runners used, + # macOS runners are expensive, and we assume that Ubuntu is enough to cover the Unix case. + os: [ubuntu-latest, windows-latest] + # Run the tests on the oldest and most recent versions of Python. + python: ['3.8', '3.x', '3.12-dev'] + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + path: ${{ env.special-working-directory-relative }} + + - name: Use Python ${{ matrix.python }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} + + - name: Install base Python requirements + uses: brettcannon/pip-secure-install@v1 + with: + requirements-file: '"${{ env.special-working-directory-relative }}/requirements.txt"' + options: '-t "${{ env.special-working-directory-relative }}/pythonFiles/lib/python" --no-cache-dir --implementation py' + + - name: Install test requirements + run: python -m pip install --upgrade -r build/test-requirements.txt + + - name: Run Python unit tests + run: python pythonFiles/tests/run_all.py + + tests: + name: Tests + # The value of runs-on is the OS of the current job (specified in the strategy matrix below) instead of being hardcoded. + runs-on: ${{ matrix.os }} + defaults: + run: + working-directory: ${{ env.special-working-directory }} + strategy: + fail-fast: false + matrix: + # We're not running CI on macOS for now because it's one less matrix entry to lower the number of runners used, + # macOS runners are expensive, and we assume that Ubuntu is enough to cover the Unix case. + os: [ubuntu-latest, windows-latest] + # Run the tests on the oldest and most recent versions of Python. + python: ['3.x'] + test-suite: [ts-unit, venv, single-workspace, debugger, functional] + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + path: ${{ env.special-working-directory-relative }} + + - name: Install Node + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + cache: 'npm' + cache-dependency-path: ${{ env.special-working-directory-relative }}/package-lock.json + + - name: Install dependencies (npm ci) + run: npm ci + + - name: Compile + run: npx gulp prePublishNonBundle + + - name: Use Python ${{ matrix.python }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} + + - name: Install debugpy + run: | + # We need to have debugpy so that tests relying on it keep passing, but we don't need install_debugpy's logic in the test phase. + python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --no-cache-dir --implementation py --no-deps --upgrade --pre debugpy + + - name: Download get-pip.py + run: | + python -m pip install wheel + python -m pip install -r build/build-install-requirements.txt + python ./pythonFiles/download_get_pip.py + shell: bash + + - name: Install base Python requirements + uses: brettcannon/pip-secure-install@v1 + with: + requirements-file: '"${{ env.special-working-directory-relative }}/requirements.txt"' + options: '-t "${{ env.special-working-directory-relative }}/pythonFiles/lib/python" --no-cache-dir --implementation py' + + - name: Install Jedi requirements + uses: brettcannon/pip-secure-install@v1 + with: + requirements-file: '"${{ env.special-working-directory-relative }}/pythonFiles/jedilsp_requirements/requirements.txt"' + options: '-t "${{ env.special-working-directory-relative }}/pythonFiles/lib/jedilsp" --no-cache-dir --implementation py' + + - name: Install test requirements + run: python -m pip install --upgrade -r build/test-requirements.txt + + - name: Install debugpy wheels (Python ${{ matrix.python }}) + run: | + python -m pip install wheel + python -m pip --disable-pip-version-check install -r build/build-install-requirements.txt + python ./pythonFiles/install_debugpy.py + shell: bash + if: matrix.test-suite == 'debugger' + + - name: Install functional test requirements + run: python -m pip install --upgrade -r ./build/functional-test-requirements.txt + if: matrix.test-suite == 'functional' + + - name: Prepare pipenv for venv tests + env: + TEST_FILES_SUFFIX: testvirtualenvs + PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' + shell: pwsh + if: matrix.test-suite == 'venv' + run: | + python -m pip install pipenv + python -m pipenv run python ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} pipenvPath + + - name: Prepare poetry for venv tests + env: + TEST_FILES_SUFFIX: testvirtualenvs + shell: pwsh + if: matrix.test-suite == 'venv' + run: | + python -m pip install poetry + Move-Item -Path ".\build\ci\pyproject.toml" -Destination . + poetry env use python + + - name: Prepare virtualenv for venv tests + env: + TEST_FILES_SUFFIX: testvirtualenvs + PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' + shell: pwsh + if: matrix.test-suite == 'venv' + run: | + python -m pip install virtualenv + python -m virtualenv .virtualenv/ + if ('${{ matrix.os }}' -match 'windows-latest') { + & ".virtualenv/Scripts/python.exe" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} virtualEnvPath + } else { + & ".virtualenv/bin/python" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} virtualEnvPath + } + + - name: Prepare venv for venv tests + env: + TEST_FILES_SUFFIX: testvirtualenvs + PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' + shell: pwsh + if: matrix.test-suite == 'venv' && startsWith(matrix.python, 3.) + run: | + python -m venv .venv + if ('${{ matrix.os }}' -match 'windows-latest') { + & ".venv/Scripts/python.exe" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} venvPath + } else { + & ".venv/bin/python" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} venvPath + } + + - name: Prepare conda for venv tests + env: + TEST_FILES_SUFFIX: testvirtualenvs + PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' + shell: pwsh + if: matrix.test-suite == 'venv' + run: | + # 1. For `*.testvirtualenvs.test.ts` + if ('${{ matrix.os }}' -match 'windows-latest') { + $condaPythonPath = Join-Path -Path $Env:CONDA -ChildPath python.exe + $condaExecPath = Join-Path -Path $Env:CONDA -ChildPath Scripts | Join-Path -ChildPath conda + } else{ + $condaPythonPath = Join-Path -Path $Env:CONDA -ChildPath bin | Join-Path -ChildPath python + $condaExecPath = Join-Path -Path $Env:CONDA -ChildPath bin | Join-Path -ChildPath conda + } + & $condaPythonPath ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} condaExecPath $condaExecPath + & $condaPythonPath ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} condaPath + & $condaExecPath init --all + + - name: Set CI_PYTHON_PATH and CI_DISABLE_AUTO_SELECTION + run: | + echo "CI_PYTHON_PATH=python" >> $GITHUB_ENV + echo "CI_DISABLE_AUTO_SELECTION=1" >> $GITHUB_ENV + shell: bash + if: matrix.test-suite != 'ts-unit' + + # Run TypeScript unit tests only for Python 3.X. + - name: Run TypeScript unit tests + run: npm run test:unittests + if: matrix.test-suite == 'ts-unit' && startsWith(matrix.python, 3.) + + # The virtual environment based tests use the `testSingleWorkspace` set of tests + # with the environment variable `TEST_FILES_SUFFIX` set to `testvirtualenvs`, + # which is set in the "Prepare environment for venv tests" step. + # We also use a third-party GitHub Action to install xvfb on Linux, + # run tests and then clean up the process once the tests ran. + # See https://github.com/GabrielBB/xvfb-action + - name: Run venv tests + env: + TEST_FILES_SUFFIX: testvirtualenvs + CI_PYTHON_VERSION: ${{ matrix.python }} + uses: GabrielBB/xvfb-action@v1.6 + with: + run: npm run testSingleWorkspace + working-directory: ${{ env.special-working-directory }} + if: matrix.test-suite == 'venv' + + - name: Run single-workspace tests + env: + CI_PYTHON_VERSION: ${{ matrix.python }} + uses: GabrielBB/xvfb-action@v1.6 + with: + run: npm run testSingleWorkspace + working-directory: ${{ env.special-working-directory }} + if: matrix.test-suite == 'single-workspace' + + - name: Run debugger tests + env: + CI_PYTHON_VERSION: ${{ matrix.python }} + uses: GabrielBB/xvfb-action@v1.6 + with: + run: npm run testDebugger + working-directory: ${{ env.special-working-directory }} + if: matrix.test-suite == 'debugger' + + # Run TypeScript functional tests + - name: Run TypeScript functional tests + run: npm run test:functional + if: matrix.test-suite == 'functional' + smoke-tests: name: Smoke tests # The value of runs-on is the OS of the current job (specified in the strategy matrix below) instead of being hardcoded. @@ -43,7 +328,7 @@ jobs: matrix: # We're not running CI on macOS for now because it's one less matrix entry to lower the number of runners used, # macOS runners are expensive, and we assume that Ubuntu is enough to cover the UNIX case. - os: [ubuntu-latest] + os: [ubuntu-latest, windows-latest] steps: # Need the source to have the tests available. - name: Checkout @@ -54,3 +339,191 @@ jobs: with: node_version: ${{ env.NODE_VERSION }} artifact_name: ${{ env.ARTIFACT_NAME_VSIX }} + + ### Coverage run + coverage: + name: Coverage + # The value of runs-on is the OS of the current job (specified in the strategy matrix below) instead of being hardcoded. + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + # Only run coverage on linux for PRs + os: [ubuntu-latest] + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Node + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + cache: 'npm' + + - name: Install dependencies (npm ci) + run: npm ci + + - name: Compile + run: npx gulp prePublishNonBundle + + - name: Use Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: 'pip' + cache-dependency-path: | + requirements.txt + pythonFiles/jedilsp_requirements/requirements.txt + build/test-requirements.txt + build/functional-test-requirements.txt + + - name: Install base Python requirements + uses: brettcannon/pip-secure-install@v1 + with: + options: '-t ./pythonFiles/lib/python --implementation py' + + - name: Install Jedi requirements + uses: brettcannon/pip-secure-install@v1 + with: + requirements-file: './pythonFiles/jedilsp_requirements/requirements.txt' + options: '-t ./pythonFiles/lib/jedilsp --implementation py' + + - name: Install debugpy + run: | + # We need to have debugpy so that tests relying on it keep passing, but we don't need install_debugpy's logic in the test phase. + python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --implementation py --no-deps --upgrade --pre debugpy + + - name: Install test requirements + run: python -m pip install --upgrade -r build/test-requirements.txt + + - name: Install functional test requirements + run: python -m pip install --upgrade -r ./build/functional-test-requirements.txt + + - name: Prepare pipenv for venv tests + env: + TEST_FILES_SUFFIX: testvirtualenvs + PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' + shell: pwsh + run: | + python -m pip install pipenv + python -m pipenv run python ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} pipenvPath + + - name: Prepare poetry for venv tests + env: + TEST_FILES_SUFFIX: testvirtualenvs + shell: pwsh + run: | + python -m pip install poetry + Move-Item -Path ".\build\ci\pyproject.toml" -Destination . + poetry env use python + + - name: Prepare virtualenv for venv tests + env: + TEST_FILES_SUFFIX: testvirtualenvs + PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' + shell: pwsh + run: | + python -m pip install virtualenv + python -m virtualenv .virtualenv/ + if ('${{ matrix.os }}' -match 'windows-latest') { + & ".virtualenv/Scripts/python.exe" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} virtualEnvPath + } else { + & ".virtualenv/bin/python" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} virtualEnvPath + } + + - name: Prepare venv for venv tests + env: + TEST_FILES_SUFFIX: testvirtualenvs + PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' + shell: pwsh + run: | + python -m venv .venv + if ('${{ matrix.os }}' -match 'windows-latest') { + & ".venv/Scripts/python.exe" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} venvPath + } else { + & ".venv/bin/python" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} venvPath + } + + - name: Prepare conda for venv tests + env: + TEST_FILES_SUFFIX: testvirtualenvs + PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' + shell: pwsh + run: | + # 1. For `*.testvirtualenvs.test.ts` + if ('${{ matrix.os }}' -match 'windows-latest') { + $condaPythonPath = Join-Path -Path $Env:CONDA -ChildPath python.exe + $condaExecPath = Join-Path -Path $Env:CONDA -ChildPath Scripts | Join-Path -ChildPath conda + } else{ + $condaPythonPath = Join-Path -Path $Env:CONDA -ChildPath bin | Join-Path -ChildPath python + $condaExecPath = Join-Path -Path $Env:CONDA -ChildPath bin | Join-Path -ChildPath conda + } + & $condaPythonPath ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} condaExecPath $condaExecPath + & $condaPythonPath ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} condaPath + & $condaExecPath init --all + + - name: Run TypeScript unit tests + run: npm run test:unittests:cover + + - name: Run Python unit tests + run: | + python pythonFiles/tests/run_all.py + + # The virtual environment based tests use the `testSingleWorkspace` set of tests + # with the environment variable `TEST_FILES_SUFFIX` set to `testvirtualenvs`, + # which is set in the "Prepare environment for venv tests" step. + # We also use a third-party GitHub Action to install xvfb on Linux, + # run tests and then clean up the process once the tests ran. + # See https://github.com/GabrielBB/xvfb-action + - name: Run venv tests + env: + TEST_FILES_SUFFIX: testvirtualenvs + CI_PYTHON_VERSION: ${{ env.PYTHON_VERSION }} + CI_DISABLE_AUTO_SELECTION: 1 + uses: GabrielBB/xvfb-action@v1.6 + with: + run: npm run testSingleWorkspace:cover + + - name: Run single-workspace tests + env: + CI_PYTHON_VERSION: ${{ env.PYTHON_VERSION }} + CI_DISABLE_AUTO_SELECTION: 1 + uses: GabrielBB/xvfb-action@v1.6 + with: + run: npm run testSingleWorkspace:cover + + # Enable these tests when coverage is setup for multiroot workspace tests + # - name: Run multi-workspace tests + # env: + # CI_PYTHON_VERSION: ${{ env.PYTHON_VERSION }} + # CI_DISABLE_AUTO_SELECTION: 1 + # uses: GabrielBB/xvfb-action@v1.6 + # with: + # run: npm run testMultiWorkspace:cover + + # Enable these tests when coverage is setup for debugger tests + # - name: Run debugger tests + # env: + # CI_PYTHON_VERSION: ${{ env.PYTHON_VERSION }} + # CI_DISABLE_AUTO_SELECTION: 1 + # uses: GabrielBB/xvfb-action@v1.6 + # with: + # run: npm run testDebugger:cover + + # Run TypeScript functional tests + - name: Run TypeScript functional tests + env: + CI_PYTHON_VERSION: ${{ env.PYTHON_VERSION }} + CI_DISABLE_AUTO_SELECTION: 1 + run: npm run test:functional:cover + + - name: Generate coverage reports + run: npm run test:cover:report + + - name: Upload HTML report + uses: actions/upload-artifact@v3 + with: + name: ${{ runner.os }}-coverage-report-html + path: ./coverage + retention-days: 1 diff --git a/.github/workflows/pr-labels.yml b/.github/workflows/pr-labels.yml new file mode 100644 index 000000000000..730b8e5c5832 --- /dev/null +++ b/.github/workflows/pr-labels.yml @@ -0,0 +1,21 @@ +name: 'PR labels' +on: + pull_request: + types: + - 'opened' + - 'reopened' + - 'labeled' + - 'unlabeled' + - 'synchronize' + +jobs: + classify: + name: 'Classify PR' + runs-on: ubuntu-latest + steps: + - name: 'PR impact specified' + uses: mheap/github-action-required-labels@v5 + with: + mode: exactly + count: 1 + labels: 'bug, debt, feature-request, no-changelog' From cd275eb0ee657a9c87d76dfb5a19cedb4b0febcc Mon Sep 17 00:00:00 2001 From: Kartik Raj Date: Fri, 17 Nov 2023 17:46:59 -0800 Subject: [PATCH 26/30] Minmize logging --- src/client/common/process/rawProcessApis.ts | 1 - src/client/common/process/worker/main.ts | 8 ++++---- src/client/common/terminal/shellDetector.ts | 7 +++---- .../common/terminal/shellDetectors/baseShellDetector.ts | 3 --- .../terminal/shellDetectors/settingsShellDetector.ts | 2 -- .../shellDetectors/userEnvironmentShellDetector.ts | 2 -- src/client/common/utils/decorators.ts | 3 +-- src/client/interpreter/activation/service.ts | 1 - src/client/interpreter/virtualEnvs/activatedEnvLaunch.ts | 5 +---- .../base/locators/composite/envsCollectionService.ts | 3 +-- .../base/locators/lowLevel/windowsKnownPathsLocator.ts | 3 --- .../common/environmentManagers/pipenv.ts | 4 ++-- src/client/terminals/envCollectionActivation/service.ts | 2 ++ 13 files changed, 14 insertions(+), 30 deletions(-) diff --git a/src/client/common/process/rawProcessApis.ts b/src/client/common/process/rawProcessApis.ts index 27cd88d42851..5e3641328b69 100644 --- a/src/client/common/process/rawProcessApis.ts +++ b/src/client/common/process/rawProcessApis.ts @@ -56,7 +56,6 @@ export function shellExec( disposables?: Set, ): Promise> { const shellOptions = getDefaultOptions(options, defaultEnv); - traceVerbose(`Shell Exec: ${command} with options: ${JSON.stringify(shellOptions, null, 4)}`); if (!options.doNotLog) { const processLogger = new ProcessLogger(new WorkspaceService()); processLogger.logProcess(command, undefined, shellOptions); diff --git a/src/client/common/process/worker/main.ts b/src/client/common/process/worker/main.ts index 1694607b6899..52eb629d44ab 100644 --- a/src/client/common/process/worker/main.ts +++ b/src/client/common/process/worker/main.ts @@ -2,6 +2,7 @@ // Licensed under the MIT License. import { Worker } from 'worker_threads'; +import * as path from 'path'; import { traceVerbose, traceError } from '../../../logging/index'; /** @@ -16,11 +17,10 @@ export async function executeWorkerFile(workerFileName: string, workerData: any) throw new Error('Worker file must end with ".worker.js" for webpack to bundle webworkers'); } return new Promise((resolve, reject) => { - traceVerbose(`Starting worker ${workerFileName} with data ${JSON.stringify(workerData)}`); const worker = new Worker(workerFileName, { workerData }); - traceVerbose(`Started worker ${workerFileName}`); + const id = worker.threadId; + traceVerbose(`Worker id ${id} for file ${path.basename(workerFileName)} with data ${JSON.stringify(workerData)}`); worker.on('message', (msg: { err: Error; res: unknown }) => { - traceVerbose(`Worker ${workerFileName} sent message ${JSON.stringify(msg)}`); if (msg.err) { reject(msg.err); } @@ -31,7 +31,7 @@ export async function executeWorkerFile(workerFileName: string, workerData: any) reject(ex); }); worker.on('exit', (code) => { - traceVerbose(`Worker ${workerFileName} exited with code ${code}`); + traceVerbose(`Worker id ${id} exited with code ${code}`); if (code !== 0) { reject(new Error(`Worker ${workerFileName} stopped with exit code ${code}`)); } diff --git a/src/client/common/terminal/shellDetector.ts b/src/client/common/terminal/shellDetector.ts index 98cda5953fe8..71d978eb6959 100644 --- a/src/client/common/terminal/shellDetector.ts +++ b/src/client/common/terminal/shellDetector.ts @@ -53,9 +53,6 @@ export class ShellDetector { for (const detector of shellDetectors) { shell = detector.identify(telemetryProperties, terminal); - traceVerbose( - `${detector}. Shell identified as ${shell} ${terminal ? `(Terminal name is ${terminal.name})` : ''}`, - ); if (shell && shell !== TerminalShellType.other) { telemetryProperties.failed = false; break; @@ -66,7 +63,9 @@ export class ShellDetector { // This impacts executing code in terminals and activation of environments in terminal. // So, the better this works, the better it is for the user. sendTelemetryEvent(EventName.TERMINAL_SHELL_IDENTIFICATION, undefined, telemetryProperties); - traceVerbose(`Shell identified as '${shell}'`); + traceVerbose( + `Shell identified as ${shell} ${terminal ? `(Terminal name is ${terminal.name})` : ''}`, + ); // If we could not identify the shell, use the defaults. if (shell === undefined || shell === TerminalShellType.other) { diff --git a/src/client/common/terminal/shellDetectors/baseShellDetector.ts b/src/client/common/terminal/shellDetectors/baseShellDetector.ts index 29a5e548891e..4262bdf80364 100644 --- a/src/client/common/terminal/shellDetectors/baseShellDetector.ts +++ b/src/client/common/terminal/shellDetectors/baseShellDetector.ts @@ -5,7 +5,6 @@ import { injectable, unmanaged } from 'inversify'; import { Terminal } from 'vscode'; -import { traceVerbose } from '../../../logging'; import { IShellDetector, ShellIdentificationTelemetry, TerminalShellType } from '../types'; /* @@ -75,7 +74,5 @@ export function identifyShellFromShellPath(shellPath: string): TerminalShellType return matchedShell; }, TerminalShellType.other); - traceVerbose(`Shell path '${shellPath}', base path '${basePath}'`); - traceVerbose(`Shell path identified as shell '${shell}'`); return shell; } diff --git a/src/client/common/terminal/shellDetectors/settingsShellDetector.ts b/src/client/common/terminal/shellDetectors/settingsShellDetector.ts index 7ffc168db28b..3eeb9d2e85da 100644 --- a/src/client/common/terminal/shellDetectors/settingsShellDetector.ts +++ b/src/client/common/terminal/shellDetectors/settingsShellDetector.ts @@ -5,7 +5,6 @@ import { inject, injectable } from 'inversify'; import { Terminal } from 'vscode'; -import { traceVerbose } from '../../../logging'; import { IWorkspaceService } from '../../application/types'; import { IPlatformService } from '../../platform/types'; import { OSType } from '../../utils/platform'; @@ -62,7 +61,6 @@ export class SettingsShellDetector extends BaseShellDetector { } else { telemetryProperties.shellIdentificationSource = 'settings'; } - traceVerbose(`Shell path from user settings '${shellPath}'`); return shell; } } diff --git a/src/client/common/terminal/shellDetectors/userEnvironmentShellDetector.ts b/src/client/common/terminal/shellDetectors/userEnvironmentShellDetector.ts index 7d8ed34ebf62..bed2848ece92 100644 --- a/src/client/common/terminal/shellDetectors/userEnvironmentShellDetector.ts +++ b/src/client/common/terminal/shellDetectors/userEnvironmentShellDetector.ts @@ -5,7 +5,6 @@ import { inject, injectable } from 'inversify'; import { Terminal } from 'vscode'; -import { traceVerbose } from '../../../logging'; import { IPlatformService } from '../../platform/types'; import { ICurrentProcess } from '../../types'; import { OSType } from '../../utils/platform'; @@ -41,7 +40,6 @@ export class UserEnvironmentShellDetector extends BaseShellDetector { if (shell !== TerminalShellType.other) { telemetryProperties.shellIdentificationSource = 'environment'; } - traceVerbose(`Shell path from user env '${shellPath}'`); return shell; } } diff --git a/src/client/common/utils/decorators.ts b/src/client/common/utils/decorators.ts index 689eb9acad44..44a82ee13760 100644 --- a/src/client/common/utils/decorators.ts +++ b/src/client/common/utils/decorators.ts @@ -1,5 +1,5 @@ import '../../common/extensions'; -import { traceError, traceVerbose } from '../../logging'; +import { traceError } from '../../logging'; import { isTestExecution } from '../constants'; import { createDeferred, Deferred } from './async'; import { getCacheKeyFromFunctionArgs, getGlobalCacheStore } from './cacheUtils'; @@ -161,7 +161,6 @@ export function cache(expiryDurationMs: number, cachePromise = false, expiryDura } const cachedItem = cacheStoreForMethods.get(key); if (cachedItem && (cachedItem.expiry > Date.now() || expiryDurationMs === -1)) { - traceVerbose(`Cached data exists ${key}`); return Promise.resolve(cachedItem.data); } const expiryMs = diff --git a/src/client/interpreter/activation/service.ts b/src/client/interpreter/activation/service.ts index ff03e95836fd..586bad0d765c 100644 --- a/src/client/interpreter/activation/service.ts +++ b/src/client/interpreter/activation/service.ts @@ -294,7 +294,6 @@ export class EnvironmentActivationService implements IEnvironmentActivationServi const oldWarnings = env[PYTHON_WARNINGS]; env[PYTHON_WARNINGS] = 'ignore'; - traceVerbose(`${hasCustomEnvVars ? 'Has' : 'No'} Custom Env Vars`); traceVerbose(`Activating Environment to capture Environment variables, ${command}`); // Do some wrapping of the call. For two reasons: diff --git a/src/client/interpreter/virtualEnvs/activatedEnvLaunch.ts b/src/client/interpreter/virtualEnvs/activatedEnvLaunch.ts index b4dcfe36e095..d5470e528ab9 100644 --- a/src/client/interpreter/virtualEnvs/activatedEnvLaunch.ts +++ b/src/client/interpreter/virtualEnvs/activatedEnvLaunch.ts @@ -91,12 +91,9 @@ export class ActivatedEnvironmentLaunch implements IActivatedEnvironmentLaunch { } if (process.env.VSCODE_CLI !== '1') { // We only want to select the interpreter if VS Code was launched from the command line. - traceVerbose( - 'VS Code was not launched from the command line, not selecting activated interpreter', - JSON.stringify(process.env, undefined, 4), - ); return undefined; } + traceVerbose('VS Code was not launched from the command line'); const prefix = await this.getPrefixOfSelectedActivatedEnv(); if (!prefix) { this._promptIfApplicable().ignoreErrors(); diff --git a/src/client/pythonEnvironments/base/locators/composite/envsCollectionService.ts b/src/client/pythonEnvironments/base/locators/composite/envsCollectionService.ts index fb1a791d07ed..cc37b1f82cfd 100644 --- a/src/client/pythonEnvironments/base/locators/composite/envsCollectionService.ts +++ b/src/client/pythonEnvironments/base/locators/composite/envsCollectionService.ts @@ -88,14 +88,13 @@ export class EnvsCollectionService extends PythonEnvsWatcher { traceError(`Failed to resolve ${path}`, ex); return undefined; }); - traceVerbose(`Resolved ${path} to ${JSON.stringify(resolved)}`); + traceVerbose(`Resolved ${path} using downstream locator`); if (resolved) { this.cache.addEnv(resolved, true); } diff --git a/src/client/pythonEnvironments/base/locators/lowLevel/windowsKnownPathsLocator.ts b/src/client/pythonEnvironments/base/locators/lowLevel/windowsKnownPathsLocator.ts index 5bfc62d99d48..3e6393ccf479 100644 --- a/src/client/pythonEnvironments/base/locators/lowLevel/windowsKnownPathsLocator.ts +++ b/src/client/pythonEnvironments/base/locators/lowLevel/windowsKnownPathsLocator.ts @@ -16,7 +16,6 @@ import { Locators } from '../../locators'; import { getEnvs } from '../../locatorUtils'; import { PythonEnvsChangedEvent } from '../../watcher'; import { DirFilesLocator } from './filesLocator'; -import { traceVerbose } from '../../../../logging'; /** * A locator for Windows locators found under the $PATH env var. @@ -93,9 +92,7 @@ function getDirFilesLocator( // rather than in each low-level locator. In the meantime we // take a naive approach. async function* iterEnvs(query: PythonLocatorQuery): IPythonEnvsIterator { - traceVerbose('Searching for windows path interpreters'); yield* await getEnvs(locator.iterEnvs(query)).then((res) => { - traceVerbose('Finished searching for windows path interpreters'); return res; }); } diff --git a/src/client/pythonEnvironments/common/environmentManagers/pipenv.ts b/src/client/pythonEnvironments/common/environmentManagers/pipenv.ts index 80a185dc2991..d8b1b2ff649e 100644 --- a/src/client/pythonEnvironments/common/environmentManagers/pipenv.ts +++ b/src/client/pythonEnvironments/common/environmentManagers/pipenv.ts @@ -3,7 +3,7 @@ import * as path from 'path'; import { getEnvironmentVariable } from '../../../common/utils/platform'; -import { traceError } from '../../../logging'; +import { traceError, traceVerbose } from '../../../logging'; import { arePathsSame, normCasePath, pathExists, readFile } from '../externalDependencies'; function getSearchHeight() { @@ -85,7 +85,7 @@ async function getProjectDir(envFolder: string): Promise { } const projectDir = (await readFile(dotProjectFile)).trim(); if (!(await pathExists(projectDir))) { - traceError( + traceVerbose( `The .project file inside environment folder: ${envFolder} doesn't contain a valid path to the project`, ); return undefined; diff --git a/src/client/terminals/envCollectionActivation/service.ts b/src/client/terminals/envCollectionActivation/service.ts index c9fa125324a0..446bc994c384 100644 --- a/src/client/terminals/envCollectionActivation/service.ts +++ b/src/client/terminals/envCollectionActivation/service.ts @@ -402,6 +402,8 @@ function shouldSkip(env: string) { 'SHLVL', // Even though this maybe returned, setting it can result in output encoding errors in terminal. 'PYTHONUTF8', + // We have deactivate service which takes care of setting it. + '_OLD_VIRTUAL_PATH' ].includes(env); } From 9276e3af09c9598686443823b6a07d8011b4d591 Mon Sep 17 00:00:00 2001 From: Kartik Raj Date: Sat, 18 Nov 2023 09:14:12 +0000 Subject: [PATCH 27/30] lint --- .../base/locators/lowLevel/windowsKnownPathsLocator.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/client/pythonEnvironments/base/locators/lowLevel/windowsKnownPathsLocator.ts b/src/client/pythonEnvironments/base/locators/lowLevel/windowsKnownPathsLocator.ts index 3e6393ccf479..63789920caea 100644 --- a/src/client/pythonEnvironments/base/locators/lowLevel/windowsKnownPathsLocator.ts +++ b/src/client/pythonEnvironments/base/locators/lowLevel/windowsKnownPathsLocator.ts @@ -92,9 +92,7 @@ function getDirFilesLocator( // rather than in each low-level locator. In the meantime we // take a naive approach. async function* iterEnvs(query: PythonLocatorQuery): IPythonEnvsIterator { - yield* await getEnvs(locator.iterEnvs(query)).then((res) => { - return res; - }); + yield* await getEnvs(locator.iterEnvs(query)).then((res) => res); } return { providerId: locator.providerId, From 98b629691f0bc71fd467ed6cecdbd5e3b8e50fa4 Mon Sep 17 00:00:00 2001 From: Kartik Raj Date: Sat, 18 Nov 2023 09:23:55 +0000 Subject: [PATCH 28/30] prettier --- src/client/common/process/worker/main.ts | 4 +++- src/client/common/terminal/shellDetector.ts | 4 +--- src/client/terminals/envCollectionActivation/service.ts | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/client/common/process/worker/main.ts b/src/client/common/process/worker/main.ts index 52eb629d44ab..324673618942 100644 --- a/src/client/common/process/worker/main.ts +++ b/src/client/common/process/worker/main.ts @@ -19,7 +19,9 @@ export async function executeWorkerFile(workerFileName: string, workerData: any) return new Promise((resolve, reject) => { const worker = new Worker(workerFileName, { workerData }); const id = worker.threadId; - traceVerbose(`Worker id ${id} for file ${path.basename(workerFileName)} with data ${JSON.stringify(workerData)}`); + traceVerbose( + `Worker id ${id} for file ${path.basename(workerFileName)} with data ${JSON.stringify(workerData)}`, + ); worker.on('message', (msg: { err: Error; res: unknown }) => { if (msg.err) { reject(msg.err); diff --git a/src/client/common/terminal/shellDetector.ts b/src/client/common/terminal/shellDetector.ts index 71d978eb6959..ad515d42c734 100644 --- a/src/client/common/terminal/shellDetector.ts +++ b/src/client/common/terminal/shellDetector.ts @@ -63,9 +63,7 @@ export class ShellDetector { // This impacts executing code in terminals and activation of environments in terminal. // So, the better this works, the better it is for the user. sendTelemetryEvent(EventName.TERMINAL_SHELL_IDENTIFICATION, undefined, telemetryProperties); - traceVerbose( - `Shell identified as ${shell} ${terminal ? `(Terminal name is ${terminal.name})` : ''}`, - ); + traceVerbose(`Shell identified as ${shell} ${terminal ? `(Terminal name is ${terminal.name})` : ''}`); // If we could not identify the shell, use the defaults. if (shell === undefined || shell === TerminalShellType.other) { diff --git a/src/client/terminals/envCollectionActivation/service.ts b/src/client/terminals/envCollectionActivation/service.ts index 446bc994c384..68c098c4c928 100644 --- a/src/client/terminals/envCollectionActivation/service.ts +++ b/src/client/terminals/envCollectionActivation/service.ts @@ -403,7 +403,7 @@ function shouldSkip(env: string) { // Even though this maybe returned, setting it can result in output encoding errors in terminal. 'PYTHONUTF8', // We have deactivate service which takes care of setting it. - '_OLD_VIRTUAL_PATH' + '_OLD_VIRTUAL_PATH', ].includes(env); } From 2374105f6cafbbee8959793f4b16983258480aa9 Mon Sep 17 00:00:00 2001 From: Kartik Raj Date: Sat, 18 Nov 2023 19:50:58 +0000 Subject: [PATCH 29/30] Add logging for known path locators --- .../locators/lowLevel/windowsKnownPathsLocator.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/client/pythonEnvironments/base/locators/lowLevel/windowsKnownPathsLocator.ts b/src/client/pythonEnvironments/base/locators/lowLevel/windowsKnownPathsLocator.ts index 63789920caea..e196d827d897 100644 --- a/src/client/pythonEnvironments/base/locators/lowLevel/windowsKnownPathsLocator.ts +++ b/src/client/pythonEnvironments/base/locators/lowLevel/windowsKnownPathsLocator.ts @@ -16,6 +16,7 @@ import { Locators } from '../../locators'; import { getEnvs } from '../../locatorUtils'; import { PythonEnvsChangedEvent } from '../../watcher'; import { DirFilesLocator } from './filesLocator'; +import { traceVerbose } from '../../../../logging'; /** * A locator for Windows locators found under the $PATH env var. @@ -62,7 +63,15 @@ export class WindowsPathEnvVarLocator implements ILocator, IDispos // Note that we do no filtering here, including to check if files // are valid executables. That is left to callers (e.g. composite // locators). - return this.locators.iterEnvs(query); + const it = this.locators.iterEnvs(query); + async function* iterator(it: IPythonEnvsIterator) { + traceVerbose(`Searching windows known paths locator`); + for await (const env of it) { + yield env; + } + traceVerbose(`Finished searching windows known paths locator`); + } + return iterator(it); } } From e9193db9506cf8c5133604717133a53d4c77eaa9 Mon Sep 17 00:00:00 2001 From: Kartik Raj Date: Sat, 18 Nov 2023 19:53:13 +0000 Subject: [PATCH 30/30] sa --- .../base/locators/lowLevel/windowsKnownPathsLocator.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/client/pythonEnvironments/base/locators/lowLevel/windowsKnownPathsLocator.ts b/src/client/pythonEnvironments/base/locators/lowLevel/windowsKnownPathsLocator.ts index e196d827d897..b2f5123069b0 100644 --- a/src/client/pythonEnvironments/base/locators/lowLevel/windowsKnownPathsLocator.ts +++ b/src/client/pythonEnvironments/base/locators/lowLevel/windowsKnownPathsLocator.ts @@ -63,7 +63,6 @@ export class WindowsPathEnvVarLocator implements ILocator, IDispos // Note that we do no filtering here, including to check if files // are valid executables. That is left to callers (e.g. composite // locators). - const it = this.locators.iterEnvs(query); async function* iterator(it: IPythonEnvsIterator) { traceVerbose(`Searching windows known paths locator`); for await (const env of it) { @@ -71,7 +70,7 @@ export class WindowsPathEnvVarLocator implements ILocator, IDispos } traceVerbose(`Finished searching windows known paths locator`); } - return iterator(it); + return iterator(this.locators.iterEnvs(query)); } }