Added support for A/B testing and staged rollouts of new functionality (#5530)

* Added functionality

* Added functionality

* News entry

* Code reviews

* Addressed more comments

* Added LS-enabled exp

* Added tests for experiments

* Corrected check and tests

* Updated tests

* Added inExperiment tests

* Fixed bugs and updated tests

* Send telemetry and log experiment group

* Code reviews

* More reviews

* Accept suggestions

* Send telemetry for the control group appropriately

* Updated tests:

* Ensure experiment groups are logged in the beginning

* Modified decorator

* Code reviews

* Added docstrings

* Minor fixing

* code reviews... more coming

* Reverted suggestion and added tests:

* Add more doc strings

* Refactoring and read config file

* Added docstrings...

* Simplified stuff

* Local experiments file docstring

* Updated tests

* Added more tests

* Added check for if experiments are valid

* Added tests for registering IOCs

* Move HTTP Client

* Added tests

* More tests

* Refactored tests

* Added crypto tests

* Added httpClient.getJSON() tests

* Apply quotes

* Fix CI

* Commit experiments file

* Indent tests

* Fix launch.json

Author: Kartik Raj

.vscode/launch.json

@@ -189,4 +189,4 @@
             ]
         }
     ]
-}
+}

experiments.json

@@ -0,0 +1,3 @@
+[
+    {}
+]

news/1 Enhancements/5042.md

@@ -0,0 +1 @@
+Add support for A/B testing and staged rollouts of new functionality

package.nls.json

@@ -102,6 +102,7 @@
     "DataScience.importingFormat": "Importing {0}",
     "DataScience.startingJupyter": "Starting Jupyter server",
     "DataScience.connectingToJupyter": "Connecting to Jupyter server",
+    "Experiments.inGroup": "User belongs to experiment group '{0}'",
     "Interpreters.RefreshingInterpreters": "Refreshing Python Interpreters",
     "Interpreters.LoadingInterpreters": "Loading Python Interpreters",
     "Common.doNotShowAgain": "Do not show again",

src/client/activation/activationManager.ts

@@ -26,7 +26,7 @@ export class ExtensionActivationManager implements IExtensionActivationManager {
         @inject(IInterpreterAutoSelectionService) private readonly autoSelection: IInterpreterAutoSelectionService,
         @inject(IApplicationDiagnostics) private readonly appDiagnostics: IApplicationDiagnostics,
         @inject(IWorkspaceService) private readonly workspaceService: IWorkspaceService
-    ) {}
+    ) { }
 
     public dispose() {
         while (this.disposables.length > 0) {

src/client/activation/activationService.ts

@@ -9,8 +9,10 @@ import { LSNotSupportedDiagnosticServiceId } from '../application/diagnostics/ch
 import { IDiagnosticsService } from '../application/diagnostics/types';
 import { IApplicationShell, ICommandManager, IWorkspaceService } from '../common/application/types';
 import { STANDARD_OUTPUT_CHANNEL } from '../common/constants';
+import { LSControl, LSEnabled } from '../common/experimentGroups';
 import '../common/extensions';
-import { IConfigurationService, IDisposableRegistry, IOutputChannel, IPersistentStateFactory, IPythonSettings, Resource } from '../common/types';
+import { traceError } from '../common/logger';
+import { IConfigurationService, IDisposableRegistry, IExperimentsManager, IOutputChannel, IPersistentStateFactory, IPythonSettings, Resource } from '../common/types';
 import { swallowExceptions } from '../common/utils/decorators';
 import { IServiceContainer } from '../ioc/types';
 import { sendTelemetryEvent } from '../telemetry';
@@ -33,7 +35,8 @@ export class LanguageServerExtensionActivationService implements IExtensionActiv
     private resource!: Resource;
 
     constructor(@inject(IServiceContainer) private serviceContainer: IServiceContainer,
-        @inject(IPersistentStateFactory) private stateFactory: IPersistentStateFactory) {
+        @inject(IPersistentStateFactory) private stateFactory: IPersistentStateFactory,
+        @inject(IExperimentsManager) private readonly abExperiments: IExperimentsManager) {
         this.workspaceService = this.serviceContainer.get<IWorkspaceService>(IWorkspaceService);
         this.output = this.serviceContainer.get<OutputChannel>(IOutputChannel, STANDARD_OUTPUT_CHANNEL);
         this.appShell = this.serviceContainer.get<IApplicationShell>(IApplicationShell);
@@ -113,6 +116,39 @@ export class LanguageServerExtensionActivationService implements IExtensionActiv
             sendTelemetryEvent(EventName.PYTHON_LANGUAGE_SERVER_SWITCHED, undefined, { change: message });
         }
     }
+
+    /**
+     * Checks if user has not manually set `jediEnabled` setting
+     * @param resource
+     * @returns `true` if user has NOT manually added the setting and is using default configuration, `false` if user has `jediEnabled` setting added
+     */
+    public isJediUsingDefaultConfiguration(resource?: Uri): boolean {
+        const settings = this.workspaceService.getConfiguration('python', resource).inspect<boolean>('jediEnabled');
+        if (!settings) {
+            traceError('WorkspaceConfiguration.inspect returns `undefined` for setting `python.jediEnabled`');
+            return false;
+        }
+        return (settings.globalValue === undefined && settings.workspaceValue === undefined && settings.workspaceFolderValue === undefined);
+    }
+
+    /**
+     * Checks if user is using Jedi as intellisense
+     * @returns `true` if user is using jedi, `false` if user is using language server
+     */
+    public useJedi(): boolean {
+        if (this.isJediUsingDefaultConfiguration()) {
+            if (this.abExperiments.inExperiment(LSEnabled)) {
+                return false;
+            }
+            // Send telemetry if user is in control group
+            this.abExperiments.sendTelemetryIfInExperiment(LSControl);
+        }
+        const configurationService = this.serviceContainer.get<IConfigurationService>(IConfigurationService);
+        const enabled = configurationService.getSettings(this.resource).jediEnabled;
+        this.trackLangaugeServerSwitch(enabled).ignoreErrors();
+        return enabled;
+    }
+
     protected onWorkspaceFoldersChanged() {
         //If an activated workspace folder was removed, dispose its activator
         const workspaceKeys = this.workspaceService.workspaceFolders!.map(workspaceFolder => this.getWorkspacePathKey(workspaceFolder.uri));
@@ -153,12 +189,6 @@ export class LanguageServerExtensionActivationService implements IExtensionActiv
             this.serviceContainer.get<ICommandManager>(ICommandManager).executeCommand('workbench.action.reloadWindow');
         }
     }
-    private useJedi(): boolean {
-        const configurationService = this.serviceContainer.get<IConfigurationService>(IConfigurationService);
-        const enabled = configurationService.getSettings(this.resource).jediEnabled;
-        this.trackLangaugeServerSwitch(enabled).ignoreErrors();
-        return enabled;
-    }
     private getWorkspacePathKey(resource: Resource): string {
         return this.workspaceService.getWorkspaceFolderIdentifier(resource, workspacePathNameForGlobalWorkspaces);
     }