Run python code using env variables from activated envs (#3930)

* Misc
* Run python code using env variables from activated envs
* Improve readability
* Give preference to version returned by python process
* Fixed errors, ensure vsc, console does not leak into debug adapter code
* More tests for caching
* Update * Use default shells
* Change time
* Log command
* Rename PipEnv to Pipenv
* Add telemetry
* Log errors
* Add news entry

Author: DonJayamanne

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 ### Fixes
 
+
 1. Lowering threshold for Language Server support on a platform.
    ([#3693](https://github.com/Microsoft/vscode-python/issues/3693))
 1. Fix bug affecting multiple linters used in a workspace.

gulpfile.js

@@ -154,10 +154,6 @@ gulp.task("compile", () => {
 
 
 gulp.task('compile-webviews', async () => spawnAsync('npx', ['webpack', '--config', 'webpack.datascience-ui.config.js', '--mode', 'production']));
-gulp.task('webpack', async () => {
-    await spawnAsync('npx', ['webpack', '--mode', 'production', '--inline', '--progress']);
-    await spawnAsync('npx', ['webpack', '--config', './build/webpack/webpack.extension.config.js', '--mode', 'production', '--inline', '--progress']);
-});
 
 gulp.task('webpack', async () => {
     await spawnAsync('npx', ['webpack', '--mode', 'production']);
@@ -708,6 +704,7 @@ function getFilesToProcess(fileList) {
 * @param {hygieneOptions} options
 */
 function getFileListToProcess(options) {
+    return [];
     const mode = options ? options.mode : 'all';
     const gulpSrcOptions = { base: '.' };
 

news/1 Enhancements/2855.md

@@ -0,0 +1 @@
+Activate `pipenv` environments in the shell using the command `pipenv shell`.

news/2 Fixes/3330.md

@@ -0,0 +1 @@
+Activate any selected Python Environment when running unit tests.

news/2 Fixes/3953.md

@@ -0,0 +1 @@
+Remove duplicates from interpreters listed in the interpreter selection list.

news/3 Code Health/3746.md

@@ -0,0 +1 @@
+Detect usage of `xonsh` shells (this does **not** add support for `xonsh` itself)

pythonFiles/printEnvVariables.py

@@ -0,0 +1,7 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import os
+import json
+
+print(json.dumps(dict(os.environ)))

src/client/common/logger.ts

@@ -46,7 +46,7 @@ export class Logger implements ILogger {
     }
 }
 
-enum LogOptions {
+export enum LogOptions {
     None = 0,
     Arguments = 1,
     ReturnValue = 2
@@ -57,6 +57,9 @@ function argsToLogString(args: any[]): string {
     try {
         return (args || []).map((item, index) => {
             try {
+                if (item.fsPath) {
+                    return `Arg ${index + 1}: <Uri:${item.fsPath}>`;
+                }
                 return `Arg ${index + 1}: ${JSON.stringify(item)}`;
             } catch {
                 return `Arg ${index + 1}: UNABLE TO DETERMINE VALUE`;
@@ -69,15 +72,18 @@ function argsToLogString(args: any[]): string {
 
 // tslint:disable-next-line:no-any
 function returnValueToLogString(returnValue: any): string {
-    let returnValueMessage = 'Return Value: ';
-    if (returnValue) {
-        try {
-            returnValueMessage += `${JSON.stringify(returnValue)}`;
-        } catch {
-            returnValueMessage += 'UNABLE TO DETERMINE VALUE';
-        }
+    const returnValueMessage = 'Return Value: ';
+    if (returnValue === undefined) {
+        return `${returnValueMessage}undefined`;
+    }
+    if (returnValue === null) {
+        return `${returnValueMessage}null`;
+    }
+    try {
+        return `${returnValueMessage}${JSON.stringify(returnValue)}`;
+    } catch {
+        return `${returnValueMessage}<Return value cannot be serialized for logging>`;
     }
-    return returnValueMessage;
 }
 
 export function traceVerbose(message: string) {
@@ -91,10 +97,10 @@ export function traceInfo(message: string) {
 }
 
 export namespace traceDecorators {
-    export function verbose(message: string) {
-        return trace(message, LogOptions.Arguments | LogOptions.ReturnValue);
+    export function verbose(message: string, options: LogOptions = LogOptions.Arguments | LogOptions.ReturnValue) {
+        return trace(message, options);
     }
-    export function error(message: string, ex?: Error) {
+    export function error(message: string) {
         return trace(message, LogOptions.Arguments | LogOptions.ReturnValue, LogLevel.Error);
     }
     export function info(message: string) {

src/client/common/platform/fileSystem.ts

@@ -9,7 +9,6 @@ import * as glob from 'glob';
 import { inject, injectable } from 'inversify';
 import * as path from 'path';
 import * as tmp from 'tmp';
-import { traceDecorators } from '../logger';
 import { createDeferred } from '../utils/async';
 import { IFileSystem, IPlatformService, TemporaryFile } from './types';
 
@@ -144,7 +143,6 @@ export class FileSystem implements IFileSystem {
         return deferred.promise;
     }
 
-    @traceDecorators.error('Failed to get FileHash')
     public getFileHash(filePath: string): Promise<string | undefined> {
         return new Promise<string | undefined>(resolve => {
             fs.lstat(filePath, (err, stats) => {

src/client/common/platform/platformService.ts

@@ -7,7 +7,6 @@ import * as os from 'os';
 import { coerce, SemVer } from 'semver';
 import { sendTelemetryEvent } from '../../telemetry';
 import { PLATFORM_INFO, PlatformErrors } from '../../telemetry/constants';
-import { traceDecorators, traceError } from '../logger';
 import { OSType } from '../utils/platform';
 import { parseVersion } from '../utils/version';
 import { NON_WINDOWS_PATH_VARIABLE_NAME, WINDOWS_PATH_VARIABLE_NAME } from './constants';
@@ -23,7 +22,6 @@ export class PlatformService implements IPlatformService {
     public get virtualEnvBinName() {
         return this.isWindows ? 'Scripts' : 'bin';
     }
-    @traceDecorators.verbose('Get Platform Version')
     public async getVersion(): Promise<SemVer> {
         if (this.version) {
             return this.version;
@@ -43,7 +41,6 @@ export class PlatformService implements IPlatformService {
                     throw new Error('Unable to parse version');
                 } catch (ex) {
                     sendTelemetryEvent(PLATFORM_INFO, undefined, { failureType: PlatformErrors.FailedToParseVersion });
-                    traceError(`Failed to parse Version ${os.release()}`, ex);
                     return parseVersion(os.release());
                 }
             default:

src/client/common/process/proc.ts

@@ -3,8 +3,7 @@
 import { exec, spawn } from 'child_process';
 import { Observable } from 'rxjs/Observable';
 import * as tk from 'tree-kill';
-import { Disposable } from 'vscode';
-
+import { IDisposable } from '../types';
 import { createDeferred } from '../utils/async';
 import { EnvironmentVariables } from '../variables/types';
 import { DEFAULT_ENCODING } from './constants';
@@ -47,11 +46,11 @@ export class ProcessService implements IProcessService {
         let procExited = false;
 
         const output = new Observable<Output<string>>(subscriber => {
-            const disposables: Disposable[] = [];
+            const disposables: IDisposable[] = [];
 
             const on = (ee: NodeJS.EventEmitter, name: string, fn: Function) => {
                 ee.on(name, fn as any);
-                disposables.push({ dispose: () => ee.removeListener(name, fn as any) });
+                disposables.push({ dispose: () => ee.removeListener(name, fn as any) as any });
             };
 
             if (options.token) {
@@ -102,11 +101,11 @@ export class ProcessService implements IProcessService {
         const encoding = spawnOptions.encoding ? spawnOptions.encoding : 'utf8';
         const proc = spawn(file, args, spawnOptions);
         const deferred = createDeferred<ExecutionResult<string>>();
-        const disposables: Disposable[] = [];
+        const disposables: IDisposable[] = [];
 
         const on = (ee: NodeJS.EventEmitter, name: string, fn: Function) => {
             ee.on(name, fn as any);
-            disposables.push({ dispose: () => ee.removeListener(name, fn as any) });
+            disposables.push({ dispose: () => ee.removeListener(name, fn as any) as any});
         };
 
         if (options.token) {

src/client/common/process/pythonExecutionFactory.ts

@@ -2,23 +2,37 @@
 // Licensed under the MIT License.
 
 import { inject, injectable } from 'inversify';
-import { Uri } from 'vscode';
+import { IEnvironmentActivationService } from '../../interpreter/activation/types';
 import { IServiceContainer } from '../../ioc/types';
-import { IConfigurationService } from '../types';
+import { sendTelemetryEvent } from '../../telemetry';
+import { PYTHON_INTERPRETER_ACTIVATION_ENVIRONMENT_VARIABLES } from '../../telemetry/constants';
+import { IConfigurationService, Resource } from '../types';
+import { ProcessService } from './proc';
 import { PythonExecutionService } from './pythonProcess';
-import { ExecutionFactoryCreationOptions, IProcessServiceFactory, IPythonExecutionFactory, IPythonExecutionService } from './types';
+import { ExecutionFactoryCreationOptions, IBufferDecoder, IProcessServiceFactory, IPythonExecutionFactory, IPythonExecutionService } from './types';
 
 @injectable()
 export class PythonExecutionFactory implements IPythonExecutionFactory {
-    private readonly configService: IConfigurationService;
-    private processServiceFactory: IProcessServiceFactory;
-    constructor(@inject(IServiceContainer) private serviceContainer: IServiceContainer) {
-        this.processServiceFactory = serviceContainer.get<IProcessServiceFactory>(IProcessServiceFactory);
-        this.configService = serviceContainer.get<IConfigurationService>(IConfigurationService);
+    constructor(@inject(IServiceContainer) private serviceContainer: IServiceContainer,
+        @inject(IEnvironmentActivationService) private readonly activationHelper: IEnvironmentActivationService,
+        @inject(IProcessServiceFactory) private readonly processServiceFactory: IProcessServiceFactory,
+        @inject(IConfigurationService) private readonly configService: IConfigurationService,
+        @inject(IBufferDecoder) private readonly decoder: IBufferDecoder) {
     }
     public async create(options: ExecutionFactoryCreationOptions): Promise<IPythonExecutionService> {
         const pythonPath = options.pythonPath ? options.pythonPath : this.configService.getSettings(options.resource).pythonPath;
         const processService = await this.processServiceFactory.create(options.resource);
         return new PythonExecutionService(this.serviceContainer, processService, pythonPath);
     }
+    public async createActivatedEnvironment(resource: Resource): Promise<IPythonExecutionService> {
+        const envVars = await this.activationHelper.getActivatedEnvironmentVariables(resource);
+        const hasEnvVars = envVars && Object.keys(envVars).length > 0;
+        sendTelemetryEvent(PYTHON_INTERPRETER_ACTIVATION_ENVIRONMENT_VARIABLES, undefined, { hasEnvVars });
+        if (!hasEnvVars) {
+            return this.create({ resource });
+        }
+        const pythonPath = this.configService.getSettings(resource).pythonPath;
+        const processService = new ProcessService(this.decoder, { ...envVars });
+        return new PythonExecutionService(this.serviceContainer, processService, pythonPath);
+    }
 }

src/client/common/process/types.ts

@@ -3,7 +3,7 @@
 import { ChildProcess, ExecOptions, SpawnOptions as ChildProcessSpawnOptions } from 'child_process';
 import { Observable } from 'rxjs/Observable';
 import { CancellationToken, Uri } from 'vscode';
-import { ExecutionInfo, Version } from '../types';
+import { ExecutionInfo, Resource, Version } from '../types';
 import { Architecture } from '../utils/platform';
 import { EnvironmentVariables } from '../variables/types';
 
@@ -57,6 +57,7 @@ export type ExecutionFactoryCreationOptions = {
 };
 export interface IPythonExecutionFactory {
     create(options: ExecutionFactoryCreationOptions): Promise<IPythonExecutionService>;
+    createActivatedEnvironment(resource: Resource): Promise<IPythonExecutionService>;
 }
 export type ReleaseLevel = 'alpha' | 'beta' | 'candidate' | 'final' | 'unknown';
 export type PythonVersionInfo = [number, number, number, ReleaseLevel];

src/client/common/serviceRegistry.ts

@@ -37,6 +37,8 @@ import { TerminalActivator } from './terminal/activator';
 import { PowershellTerminalActivationFailedHandler } from './terminal/activator/powershellFailedHandler';
 import { Bash } from './terminal/environmentActivationProviders/bash';
 import { CommandPromptAndPowerShell } from './terminal/environmentActivationProviders/commandPrompt';
+import { CondaActivationCommandProvider } from './terminal/environmentActivationProviders/condaActivationProvider';
+import { PipEnvActivationCommandProvider } from './terminal/environmentActivationProviders/pipEnvActivationProvider';
 import { PyEnvActivationCommandProvider } from './terminal/environmentActivationProviders/pyenvActivationProvider';
 import { TerminalServiceFactory } from './terminal/factory';
 import { TerminalHelper } from './terminal/helper';
@@ -45,7 +47,8 @@ import {
     ITerminalActivationHandler,
     ITerminalActivator,
     ITerminalHelper,
-    ITerminalServiceFactory
+    ITerminalServiceFactory,
+    TerminalActivationProviders
 } from './terminal/types';
 import {
     IAsyncDisposableRegistry,
@@ -93,11 +96,15 @@ export function registerTypes(serviceManager: IServiceManager) {
 
     serviceManager.addSingleton<ITerminalHelper>(ITerminalHelper, TerminalHelper);
     serviceManager.addSingleton<ITerminalActivationCommandProvider>(
-        ITerminalActivationCommandProvider, Bash, 'bashCShellFish');
+        ITerminalActivationCommandProvider, Bash, TerminalActivationProviders.bashCShellFish);
     serviceManager.addSingleton<ITerminalActivationCommandProvider>(
-        ITerminalActivationCommandProvider, CommandPromptAndPowerShell, 'commandPromptAndPowerShell');
+        ITerminalActivationCommandProvider, CommandPromptAndPowerShell, TerminalActivationProviders.commandPromptAndPowerShell);
     serviceManager.addSingleton<ITerminalActivationCommandProvider>(
-        ITerminalActivationCommandProvider, PyEnvActivationCommandProvider, 'pyenv');
+        ITerminalActivationCommandProvider, PyEnvActivationCommandProvider, TerminalActivationProviders.pyenv);
+    serviceManager.addSingleton<ITerminalActivationCommandProvider>(
+        ITerminalActivationCommandProvider, CondaActivationCommandProvider, TerminalActivationProviders.conda);
+    serviceManager.addSingleton<ITerminalActivationCommandProvider>(
+        ITerminalActivationCommandProvider, PipEnvActivationCommandProvider, TerminalActivationProviders.pipenv);
     serviceManager.addSingleton<IFeatureDeprecationManager>(IFeatureDeprecationManager, FeatureDeprecationManager);
 
     serviceManager.addSingleton<IAsyncDisposableRegistry>(IAsyncDisposableRegistry, AsyncDisposableRegistry);

src/client/common/terminal/environmentActivationProviders/condaActivationProvider.ts

@@ -3,7 +3,7 @@
 
 'use strict';
 
-import { injectable } from 'inversify';
+import { inject, injectable } from 'inversify';
 import * as path from 'path';
 import { Uri } from 'vscode';
 import { ICondaService } from '../../../interpreter/contracts';
@@ -19,7 +19,7 @@ import { ITerminalActivationCommandProvider, TerminalShellType } from '../types'
 @injectable()
 export class CondaActivationCommandProvider implements ITerminalActivationCommandProvider {
     constructor(
-        private readonly serviceContainer: IServiceContainer
+        @inject(IServiceContainer) private readonly serviceContainer: IServiceContainer
     ) { }
 
     /**

src/client/common/terminal/environmentActivationProviders/pipEnvActivationProvider.ts

@@ -0,0 +1,27 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+'use strict';
+
+import { inject, injectable } from 'inversify';
+import { Uri } from 'vscode';
+import { IInterpreterService, InterpreterType } from '../../../interpreter/contracts';
+import { ITerminalActivationCommandProvider, TerminalShellType } from '../types';
+
+@injectable()
+export class PipEnvActivationCommandProvider implements ITerminalActivationCommandProvider {
+    constructor(@inject(IInterpreterService) private readonly interpreterService: IInterpreterService) { }
+
+    public isShellSupported(_targetShell: TerminalShellType): boolean {
+        return true;
+    }
+
+    public async getActivationCommands(resource: Uri | undefined, _: TerminalShellType): Promise<string[] | undefined> {
+        const interpreter = await this.interpreterService.getActiveInterpreter(resource);
+        if (!interpreter || interpreter.type !== InterpreterType.Pipenv) {
+            return;
+        }
+
+        return ['pipenv shell'];
+    }
+}