Add support for detection and selection of conda environments lacking a python interpreter (#18427)

* Support conda environments without a python executable

* Fix interpreter display

* Fix areSameEnv

* Add support to select env folders as interpreterPaths

* Allow selecting such environments

* Handle resolving environment path

* Document resolver works for both env path and executable

* Fxi bug

* Simplify conda.ts

* Define identifiers for an environment

* Fix getCondaEnvironment

* Introduce an id property to environments

* Update proposed discovery API to handle environments

* Fix bug with interpreter display

* Normalize path passed in resolveEnv

* Update environment details API

* Dont use pythonPath for getting active interpreter for activation commands

* Support conda activation

* Support ${command:python.interpreterPath} with this envs

* Fix getActiveItem

* Add comment justifying using `.pythonPath` for middleware

* Fix shebang codelens

* Fix startup telemetry

* Do not support pip installer for such environments

* Trigger discovery once installation is finished for such environments

* Automatically install python into environment once it is selected

* Add telemetry for new interpreters discovered

* Add telemtry if such an env is selected

* Fix bugs and cache installer

* Fix compile errors, tests, and add tests

* Fix some tests

* Ignore id when comparing envs

* Phew, fixed terrible tests in module installer

* Fix more tests

* MOre

* Skip resolver tests on linux

* Fix tests

* If environment doesn't contain python do not support pip installer

* Skip module install tests for python as it is not a module

* News entry

* Remove unnecssary commnet

* Fix bug introduced by merges

* Code reviews

Author: Kartik Raj

news/1 Enhancements/18357.md

@@ -0,0 +1 @@
+Add support for detection and selection of conda environments lacking a python interpreter.

package.nls.json

@@ -45,6 +45,7 @@
     "Pylance.pylanceRevertToJedi": "Revert to Jedi",
     "Experiments.inGroup": "Experiment '{0}' is active",
     "Experiments.optedOutOf": "Experiment '{0}' is inactive",
+    "Interpreters.installingPython": "Installing Python into Environment...",
     "Interpreters.clearAtWorkspace": "Clear at workspace level",
     "Interpreters.RefreshingInterpreters": "Refreshing Python Interpreters",
     "Interpreters.entireWorkspace": "Select at workspace level",

src/client/activation/languageClientMiddlewareBase.ts

@@ -80,6 +80,8 @@ export class LanguageClientMiddlewareBase implements Middleware {
                     const uri = item.scopeUri ? Uri.parse(item.scopeUri) : undefined;
                     // For backwards compatibility, set python.pythonPath to the configured
                     // value as though it were in the user's settings.json file.
+                    // As this is for backwards compatibility, `ConfigService.pythonPath`
+                    // can be considered as active interpreter path.
                     settings[i].pythonPath = configService.getSettings(uri).pythonPath;
 
                     const env = await envService.getEnvironmentVariables(uri);

src/client/apiTypes.ts

@@ -4,6 +4,7 @@
 import { Event, Uri } from 'vscode';
 import { Resource } from './common/types';
 import { IDataViewerDataProvider, IJupyterUriProvider } from './jupyter/types';
+import { EnvPathType, PythonEnvKind } from './pythonEnvironments/base/info';
 
 /*
  * Do not introduce any breaking changes to this API.
@@ -87,28 +88,39 @@ export interface IExtensionApi {
     };
 }
 
-export interface InterpreterDetailsOptions {
+export interface EnvironmentDetailsOptions {
     useCache: boolean;
 }
 
-export interface InterpreterDetails {
-    path: string;
+export interface EnvironmentDetails {
+    interpreterPath: string;
+    envFolderPath?: string;
     version: string[];
-    environmentType: string[];
+    environmentType: PythonEnvKind[];
     metadata: Record<string, unknown>;
 }
 
-export interface InterpretersChangedParams {
+export interface EnvironmentsChangedParams {
+    /**
+     * Path to environment folder or path to interpreter that uniquely identifies an environment.
+     * Virtual environments lacking an interpreter are identified by environment folder paths,
+     * whereas other envs can be identified using interpreter path.
+     */
     path?: string;
     type: 'add' | 'remove' | 'update' | 'clear-all';
 }
 
-export interface ActiveInterpreterChangedParams {
-    interpreterPath?: string;
+export interface ActiveEnvironmentChangedParams {
+    /**
+     * Path to environment folder or path to interpreter that uniquely identifies an environment.
+     * Virtual environments lacking an interpreter are identified by environment folder paths,
+     * whereas other envs can be identified using interpreter path.
+     */
+    path: string;
     resource?: Uri;
 }
 
-export interface RefreshInterpretersOptions {
+export interface RefreshEnvironmentsOptions {
     clearCache?: boolean;
 }
 
@@ -122,57 +134,60 @@ export interface IProposedExtensionAPI {
          * returns what ever is set for the workspace.
          * @param resource : Uri of a file or workspace
          */
-        getActiveInterpreterPath(resource?: Resource): Promise<string | undefined>;
+        getActiveEnvironmentPath(resource?: Resource): Promise<EnvPathType | undefined>;
         /**
          * Returns details for the given interpreter. Details such as absolute interpreter path,
          * version, type (conda, pyenv, etc). Metadata such as `sysPrefix` can be found under
          * metadata field.
-         * @param interpreterPath : Path of the interpreter whose details you need.
+         * @param path : Path to environment folder or path to interpreter whose details you need.
          * @param options : [optional]
          *     * useCache : When true, cache is checked first for any data, returns even if there
          *                  is partial data.
          */
-        getInterpreterDetails(
-            interpreterPath: string,
-            options?: InterpreterDetailsOptions,
-        ): Promise<InterpreterDetails | undefined>;
+        getEnvironmentDetails(
+            path: string,
+            options?: EnvironmentDetailsOptions,
+        ): Promise<EnvironmentDetails | undefined>;
         /**
-         * Returns paths to interpreters found by the extension at the time of calling. This API
-         * will *not* trigger a refresh. If a refresh is going on it will *not* wait for the refresh
-         * to finish. This will return what is known so far. To get complete list `await` on promise
-         * returned by `getRefreshPromise()`.
+         * Returns paths to environments that uniquely identifies an environment found by the extension
+         * at the time of calling. This API will *not* trigger a refresh. If a refresh is going on it
+         * will *not* wait for the refresh to finish. This will return what is known so far. To get
+         * complete list `await` on promise returned by `getRefreshPromise()`.
+         *
+         * Virtual environments lacking an interpreter are identified by environment folder paths,
+         * whereas other envs can be identified using interpreter path.
          */
-        getInterpreterPaths(): Promise<string[] | undefined>;
+        getEnvironmentPaths(): Promise<EnvPathType[] | undefined>;
         /**
-         * Sets the active interpreter path for the python extension. Configuration target will
-         * always be the workspace.
-         * @param interpreterPath : Interpreter path to set for a given workspace.
+         * Sets the active environment path for the python extension. Configuration target will
+         * always be the workspace folder.
+         * @param path : Interpreter path to set for a given workspace.
          * @param resource : [optional] Uri of a file ro workspace to scope to a particular workspace
          *                   folder.
          */
-        setActiveInterpreter(interpreterPath: string, resource?: Resource): Promise<void>;
+        setActiveEnvironment(path: string, resource?: Resource): Promise<void>;
         /**
          * This API will re-trigger environment discovery. Extensions can wait on the returned
-         * promise to get the updated interpreters list. If there is a refresh already going on
+         * promise to get the updated environment list. If there is a refresh already going on
          * then it returns the promise for that refresh.
          * @param options : [optional]
          *     * clearCache : When true, this will clear the cache before interpreter refresh
          *                    is triggered.
          */
-        refreshInterpreters(options?: RefreshInterpretersOptions): Promise<string[] | undefined>;
+        refreshEnvironment(options?: RefreshEnvironmentsOptions): Promise<EnvPathType[] | undefined>;
         /**
          * Returns a promise for the ongoing refresh. Returns `undefined` if there are no active
          * refreshes going on.
          */
         getRefreshPromise(): Promise<void> | undefined;
         /**
-         * This event is triggered when the known interpreters list changes, like when a interpreter
-         * is found, existing interpreter is removed, or some details changed on an interpreter.
+         * This event is triggered when the known environment list changes, like when a environment
+         * is found, existing environment is removed, or some details changed on an environment.
          */
-        onDidInterpretersChanged: Event<InterpretersChangedParams[]>;
+        onDidEnvironmentsChanged: Event<EnvironmentsChangedParams[]>;
         /**
-         * This event is triggered when the active interpreter changes.
+         * This event is triggered when the active environment changes.
          */
-        onDidActiveInterpreterChanged: Event<ActiveInterpreterChangedParams>;
+        onDidActiveEnvironmentChanged: Event<ActiveEnvironmentChangedParams>;
     };
 }

src/client/common/installer/condaInstaller.ts

@@ -75,7 +75,7 @@ export class CondaInstaller extends ModuleInstaller {
 
         const pythonPath = isResource(resource)
             ? this.serviceContainer.get<IConfigurationService>(IConfigurationService).getSettings(resource).pythonPath
-            : resource.path;
+            : resource.id ?? '';
         const condaLocatorService = this.serviceContainer.get<IComponentAdapter>(IComponentAdapter);
         const info = await condaLocatorService.getCondaEnvironment(pythonPath);
         const args = [flags & ModuleInstallFlags.upgrade ? 'update' : 'install'];
@@ -127,7 +127,7 @@ export class CondaInstaller extends ModuleInstaller {
         const condaService = this.serviceContainer.get<IComponentAdapter>(IComponentAdapter);
         const pythonPath = isResource(resource)
             ? this.serviceContainer.get<IConfigurationService>(IConfigurationService).getSettings(resource).pythonPath
-            : resource.path;
+            : resource.id ?? '';
         return condaService.isCondaEnvironment(pythonPath);
     }
 }

src/client/common/installer/moduleInstaller.ts

@@ -15,18 +15,22 @@ import { wrapCancellationTokens } from '../cancellation';
 import { STANDARD_OUTPUT_CHANNEL } from '../constants';
 import { IFileSystem } from '../platform/types';
 import * as internalPython from '../process/internal/python';
+import { IProcessServiceFactory } from '../process/types';
 import { ITerminalServiceFactory, TerminalCreationOptions } from '../terminal/types';
 import { ExecutionInfo, IConfigurationService, IOutputChannel, Product } from '../types';
 import { Products } from '../utils/localize';
 import { isResource } from '../utils/misc';
 import { ProductNames } from './productNames';
-import { IModuleInstaller, InterpreterUri, ModuleInstallFlags } from './types';
+import { IModuleInstaller, InstallOptions, InterpreterUri, ModuleInstallFlags } from './types';
 
 @injectable()
 export abstract class ModuleInstaller implements IModuleInstaller {
     public abstract get priority(): number;
+
     public abstract get name(): string;
+
     public abstract get displayName(): string;
+
     public abstract get type(): ModuleInstallerType;
 
     constructor(protected serviceContainer: IServiceContainer) {}
@@ -36,24 +40,18 @@ export abstract class ModuleInstaller implements IModuleInstaller {
         resource?: InterpreterUri,
         cancel?: CancellationToken,
         flags?: ModuleInstallFlags,
+        options?: InstallOptions,
     ): Promise<void> {
+        const shouldExecuteInTerminal = !options?.installAsProcess;
         const name =
-            typeof productOrModuleName == 'string'
+            typeof productOrModuleName === 'string'
                 ? productOrModuleName
                 : translateProductToModule(productOrModuleName);
         const productName = typeof productOrModuleName === 'string' ? name : ProductNames.get(productOrModuleName);
         sendTelemetryEvent(EventName.PYTHON_INSTALL_PACKAGE, undefined, { installer: this.displayName, productName });
         const uri = isResource(resource) ? resource : undefined;
-        const options: TerminalCreationOptions = {};
-        if (isResource(resource)) {
-            options.resource = uri;
-        } else {
-            options.interpreter = resource;
-        }
         const executionInfo = await this.getExecutionInfo(name, resource, flags);
-        const terminalService = this.serviceContainer
-            .get<ITerminalServiceFactory>(ITerminalServiceFactory)
-            .getTerminalService(options);
+
         const install = async (token?: CancellationToken) => {
             const executionInfoArgs = await this.processInstallArgs(executionInfo.args, resource);
             if (executionInfo.moduleName) {
@@ -64,25 +62,38 @@ export abstract class ModuleInstaller implements IModuleInstaller {
                 const interpreter = isResource(resource)
                     ? await interpreterService.getActiveInterpreter(resource)
                     : resource;
-                const pythonPath = isResource(resource) ? settings.pythonPath : resource.path;
+                const interpreterPath = interpreter?.path ?? settings.pythonPath;
+                const pythonPath = isResource(resource) ? interpreterPath : resource.path;
                 const args = internalPython.execModule(executionInfo.moduleName, executionInfoArgs);
                 if (!interpreter || interpreter.envType !== EnvironmentType.Unknown) {
-                    await terminalService.sendCommand(pythonPath, args, token);
+                    await this.executeCommand(shouldExecuteInTerminal, resource, pythonPath, args, token);
                 } else if (settings.globalModuleInstallation) {
                     const fs = this.serviceContainer.get<IFileSystem>(IFileSystem);
                     if (await fs.isDirReadonly(path.dirname(pythonPath)).catch((_err) => true)) {
                         this.elevatedInstall(pythonPath, args);
                     } else {
-                        await terminalService.sendCommand(pythonPath, args, token);
+                        await this.executeCommand(shouldExecuteInTerminal, resource, pythonPath, args, token);
                     }
                 } else if (name === translateProductToModule(Product.pip)) {
                     // Pip should always be installed into the specified environment.
-                    await terminalService.sendCommand(pythonPath, args, token);
+                    await this.executeCommand(shouldExecuteInTerminal, resource, pythonPath, args, token);
                 } else {
-                    await terminalService.sendCommand(pythonPath, args.concat(['--user']), token);
+                    await this.executeCommand(
+                        shouldExecuteInTerminal,
+                        resource,
+                        pythonPath,
+                        args.concat(['--user']),
+                        token,
+                    );
                 }
             } else {
-                await terminalService.sendCommand(executionInfo.execPath!, executionInfoArgs, token);
+                await this.executeCommand(
+                    shouldExecuteInTerminal,
+                    resource,
+                    executionInfo.execPath!,
+                    executionInfoArgs,
+                    token,
+                );
             }
         };
 
@@ -103,6 +114,7 @@ export abstract class ModuleInstaller implements IModuleInstaller {
             await install(cancel);
         }
     }
+
     public abstract isSupported(resource?: InterpreterUri): Promise<boolean>;
 
     protected elevatedInstall(execPath: string, args: string[]) {
@@ -131,11 +143,13 @@ export abstract class ModuleInstaller implements IModuleInstaller {
             }
         });
     }
+
     protected abstract getExecutionInfo(
         moduleName: string,
         resource?: InterpreterUri,
         flags?: ModuleInstallFlags,
     ): Promise<ExecutionInfo>;
+
     private async processInstallArgs(args: string[], resource?: InterpreterUri): Promise<string[]> {
         const indexOfPylint = args.findIndex((arg) => arg.toUpperCase() === 'PYLINT');
         if (indexOfPylint === -1) {
@@ -152,6 +166,32 @@ export abstract class ModuleInstaller implements IModuleInstaller {
         }
         return args;
     }
+
+    private async executeCommand(
+        executeInTerminal: boolean,
+        resource: InterpreterUri | undefined,
+        command: string,
+        args: string[],
+        token?: CancellationToken,
+    ) {
+        const options: TerminalCreationOptions = {};
+        if (isResource(resource)) {
+            options.resource = resource;
+        } else {
+            options.interpreter = resource;
+        }
+        if (executeInTerminal) {
+            const terminalService = this.serviceContainer
+                .get<ITerminalServiceFactory>(ITerminalServiceFactory)
+                .getTerminalService(options);
+
+            terminalService.sendCommand(command, args, token);
+        } else {
+            const processServiceFactory = this.serviceContainer.get<IProcessServiceFactory>(IProcessServiceFactory);
+            const processService = await processServiceFactory.create(options.resource);
+            await processService.exec(command, args);
+        }
+    }
 }
 
 export function translateProductToModule(product: Product): string {
@@ -204,6 +244,8 @@ export function translateProductToModule(product: Product): string {
             return 'pip';
         case Product.ensurepip:
             return 'ensurepip';
+        case Product.python:
+            return 'python';
         default: {
             throw new Error(`Product ${product} cannot be installed as a Python Module.`);
         }

src/client/common/installer/pipInstaller.ts

@@ -3,7 +3,7 @@
 
 import { inject, injectable } from 'inversify';
 import { IServiceContainer } from '../../ioc/types';
-import { ModuleInstallerType } from '../../pythonEnvironments/info';
+import { EnvironmentType, ModuleInstallerType } from '../../pythonEnvironments/info';
 import { IWorkspaceService } from '../application/types';
 import { IPythonExecutionFactory } from '../process/types';
 import { ExecutionInfo, IInstaller, Product } from '../types';
@@ -16,6 +16,26 @@ import { ProductNames } from './productNames';
 import { sendTelemetryEvent } from '../../telemetry';
 import { EventName } from '../../telemetry/constants';
 import { IInterpreterService } from '../../interpreter/contracts';
+import { isParentPath } from '../platform/fs-paths';
+
+async function doesEnvironmentContainPython(serviceContainer: IServiceContainer, resource: InterpreterUri) {
+    const interpreterService = serviceContainer.get<IInterpreterService>(IInterpreterService);
+    const environment = isResource(resource) ? await interpreterService.getActiveInterpreter(resource) : resource;
+    if (!environment) {
+        return undefined;
+    }
+    if (
+        environment.envPath?.length &&
+        environment.envType === EnvironmentType.Conda &&
+        !isParentPath(environment?.path, environment.envPath)
+    ) {
+        // For conda environments not containing a python interpreter, do not use pip installer due to bugs in `conda run`:
+        // https://github.com/microsoft/vscode-python/issues/18479#issuecomment-1044427511
+        // https://github.com/conda/conda/issues/11211
+        return false;
+    }
+    return true;
+}
 
 @injectable()
 export class PipInstaller extends ModuleInstaller {
@@ -36,7 +56,10 @@ export class PipInstaller extends ModuleInstaller {
     constructor(@inject(IServiceContainer) serviceContainer: IServiceContainer) {
         super(serviceContainer);
     }
-    public isSupported(resource?: InterpreterUri): Promise<boolean> {
+    public async isSupported(resource?: InterpreterUri): Promise<boolean> {
+        if ((await doesEnvironmentContainPython(this.serviceContainer, resource)) === false) {
+            return false;
+        }
         return this.isPipAvailable(resource);
     }
     protected async getExecutionInfo(