Skip to content

Intercept output of interpreterInfo.py script to filter out stdout #18281

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jan 13, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions news/2 Fixes/18234.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Ensures interpreters are discovered even when running `interpreterInfo.py` script prints more than just the script output.
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,10 @@
import runpy
import sys

# Activating conda can print out stuff before the actual output is
# printed. Hence, printing out markers to make it more resilient to
# pull the output.
print(">>>CONDA-RUN-OUTPUT", end="")
# Sometimes executing scripts can print out stuff before the actual output is
# printed. For eg. when activating conda. Hence, printing out markers to make
# it more resilient to pull the output.
print(">>>PYTHON-EXEC-OUTPUT", end="")

module = sys.argv[1]
if module == "-c":
Expand All @@ -21,4 +21,4 @@
else:
runpy.run_module(module, run_name="__main__", alter_sys=True)

print("<<<CONDA-RUN-OUTPUT", end="")
print("<<<PYTHON-EXEC-OUTPUT", end="")
2 changes: 2 additions & 0 deletions src/client/common/process/internal/scripts/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ export type InterpreterInfoJson = {
is64Bit: boolean;
};

export const OUTPUT_MARKER_SCRIPT = path.join(_SCRIPTS_DIR, 'get_output_via_markers.py');

export function interpreterInfo(): [string[], (out: string) => InterpreterInfoJson | undefined] {
const script = path.join(SCRIPTS_DIR, 'interpreterInfo.py');
const args = [script];
Expand Down
9 changes: 5 additions & 4 deletions src/client/common/process/rawProcessApis.ts
Original file line number Diff line number Diff line change
Expand Up @@ -160,16 +160,17 @@ export function plainExec(
}

function filterOutputUsingCondaRunMarkers(stdout: string) {
// These markers are added if conda run is used, see `conda_run_script.py`.
const regex = />>>CONDA-RUN-OUTPUT([\s\S]*)<<<CONDA-RUN-OUTPUT/;
// These markers are added if conda run is used or `interpreterInfo.py` is
// run, see `get_output_via_markers.py`.
const regex = />>>PYTHON-EXEC-OUTPUT([\s\S]*)<<<PYTHON-EXEC-OUTPUT/;
const match = stdout.match(regex);
const filteredOut = match !== null && match.length >= 2 ? match[1] : '';
return filteredOut.length ? filteredOut : stdout;
}

function removeCondaRunMarkers(out: string) {
out = out.replace('>>>CONDA-RUN-OUTPUT', '');
return out.replace('<<<CONDA-RUN-OUTPUT', '');
out = out.replace('>>>PYTHON-EXEC-OUTPUT', '');
return out.replace('<<<PYTHON-EXEC-OUTPUT', '');
}

export function execObservable(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import { traceError, traceInfo } from '../../../logging';
import { Conda, CONDA_ACTIVATION_TIMEOUT, isCondaEnvironment } from '../../common/environmentManagers/conda';
import { PythonEnvInfo, PythonEnvKind } from '.';
import { normCasePath } from '../../common/externalDependencies';
import { OUTPUT_MARKER_SCRIPT } from '../../../common/process/internal/scripts';

export enum EnvironmentInfoServiceQueuePriority {
Default,
Expand All @@ -24,7 +25,7 @@ export interface IEnvironmentInfoService {
}

async function buildEnvironmentInfo(env: PythonEnvInfo): Promise<InterpreterInformation | undefined> {
const python = [env.executable.filename];
const python = [env.executable.filename, OUTPUT_MARKER_SCRIPT];
const interpreterInfo = await getInterpreterInfo(buildPythonExecInfo(python, undefined, env.executable.filename));
return interpreterInfo;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import { EnvironmentType, PythonEnvironment } from '../../info';
import { cache } from '../../../common/utils/decorators';
import { isTestExecution } from '../../../common/constants';
import { traceError, traceVerbose } from '../../../logging';
import { _SCRIPTS_DIR } from '../../../common/process/internal/scripts/constants';
import { OUTPUT_MARKER_SCRIPT } from '../../../common/process/internal/scripts';

export const AnacondaCompanyName = 'Anaconda, Inc.';

Expand Down Expand Up @@ -205,8 +205,6 @@ export const CONDA_RUN_VERSION = '4.9.0';
export const CONDA_ACTIVATION_TIMEOUT = 45000;
const CONDA_GENERAL_TIMEOUT = 50000;

export const CONDA_RUN_SCRIPT = path.join(_SCRIPTS_DIR, 'conda_run_script.py');

/** Wraps the "conda" utility, and exposes its functionality.
*/
export class Conda {
Expand Down Expand Up @@ -418,7 +416,7 @@ export class Conda {
} else {
args.push('-p', env.prefix);
}
return [this.command, 'run', ...args, '--no-capture-output', 'python', CONDA_RUN_SCRIPT];
return [this.command, 'run', ...args, '--no-capture-output', 'python', OUTPUT_MARKER_SCRIPT];
}

/**
Expand Down
6 changes: 3 additions & 3 deletions src/test/common/process/proc.exec.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ suite('ProcessService Observable', () => {
this.timeout(7000);
const procService = new ProcessService(new BufferDecoder());
const pythonCode = [
'print(">>>CONDA-RUN-OUTPUT")',
'print(">>>PYTHON-EXEC-OUTPUT")',
'import sys',
'import time',
'print("1")',
Expand All @@ -155,7 +155,7 @@ suite('ProcessService Observable', () => {
'time.sleep(1)',
'sys.stderr.write("c")',
'sys.stderr.flush()',
'print("<<<CONDA-RUN-OUTPUT")',
'print("<<<PYTHON-EXEC-OUTPUT")',
];
const result = await procService.exec(pythonPath, ['-c', pythonCode.join(';')]);
const expectedStdout = ['1', '2', '3'];
Expand Down Expand Up @@ -235,7 +235,7 @@ suite('ProcessService Observable', () => {
const procService = new ProcessService(new BufferDecoder());
const printOutput = '1234';
const result = await procService.shellExec(
`"${pythonPath}" -c "print('>>>CONDA-RUN-OUTPUT');print('${printOutput}');print('<<<CONDA-RUN-OUTPUT')"`,
`"${pythonPath}" -c "print('>>>PYTHON-EXEC-OUTPUT');print('${printOutput}');print('<<<PYTHON-EXEC-OUTPUT')"`,
);

expect(result).not.to.be.an('undefined', 'result is undefined');
Expand Down
4 changes: 2 additions & 2 deletions src/test/common/process/proc.observable.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -199,7 +199,7 @@ suite('ProcessService', () => {
this.timeout(20000);
const procService = new ProcessService(new BufferDecoder());
const pythonCode = [
'print(">>>CONDA-RUN-OUTPUT")',
'print(">>>PYTHON-EXEC-OUTPUT")',
'import sys',
'import time',
'print("1")',
Expand All @@ -220,7 +220,7 @@ suite('ProcessService', () => {
'sys.stderr.write("c")',
'sys.stderr.flush()',
'time.sleep(2)',
'print("<<<CONDA-RUN-OUTPUT")',
'print("<<<PYTHON-EXEC-OUTPUT")',
];
const result = procService.execObservable(pythonPath, ['-c', pythonCode.join(';')]);
const outputs = [
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,11 @@ import { PythonEnvKind } from '../../../../client/pythonEnvironments/base/info';
import { getEnvs } from '../../../../client/pythonEnvironments/base/locatorUtils';
import * as externalDependencies from '../../../../client/pythonEnvironments/common/externalDependencies';
import * as windowsUtils from '../../../../client/pythonEnvironments/common/windowsUtils';
import {
Conda,
CondaInfo,
CONDA_RUN_SCRIPT,
} from '../../../../client/pythonEnvironments/common/environmentManagers/conda';
import { Conda, CondaInfo } from '../../../../client/pythonEnvironments/common/environmentManagers/conda';
import { CondaEnvironmentLocator } from '../../../../client/pythonEnvironments/base/locators/lowLevel/condaLocator';
import { createBasicEnv } from '../../base/common';
import { assertBasicEnvsEqual } from '../../base/locators/envTestUtils';
import { OUTPUT_MARKER_SCRIPT } from '../../../../client/common/process/internal/scripts';

suite('Conda and its environments are located correctly', () => {
// getOSType() is stubbed to return this.
Expand Down Expand Up @@ -484,14 +481,14 @@ suite('Conda and its environments are located correctly', () => {
expect(args).to.not.equal(undefined);
assert.deepStrictEqual(
args,
['conda', 'run', '-n', 'envName', '--no-capture-output', 'python', CONDA_RUN_SCRIPT],
['conda', 'run', '-n', 'envName', '--no-capture-output', 'python', OUTPUT_MARKER_SCRIPT],
'Incorrect args for case 1',
);

args = await conda?.getRunPythonArgs({ name: '', prefix: 'envPrefix' });
assert.deepStrictEqual(
args,
['conda', 'run', '-p', 'envPrefix', '--no-capture-output', 'python', CONDA_RUN_SCRIPT],
['conda', 'run', '-p', 'envPrefix', '--no-capture-output', 'python', OUTPUT_MARKER_SCRIPT],
'Incorrect args for case 2',
);
});
Expand Down