Result resolver feature branch

pythonFiles/unittestadapter/execution.py

@@ -242,7 +242,6 @@ def run_tests(
     try:
         client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         client_socket.connect(("localhost", run_test_ids_port_int))
-        print(f"CLIENT: Server listening on port {run_test_ids_port_int}...")
         buffer = b""
 
         while True:
@@ -263,7 +262,6 @@ def run_tests(
                 buffer = b""
 
                 # Process the JSON data
-                print(f"Received JSON data: {test_ids_from_buffer}")
                 break
             except json.JSONDecodeError:
                 # JSON decoding error, the complete JSON object is not yet received

src/client/testing/common/socketServer.ts

@@ -123,7 +123,7 @@ export class UnitTestSocketServer extends EventEmitter implements IUnitTestSocke
             if ((socket as any).id) {
                 destroyedSocketId = (socket as any).id;
             }
-            this.log('socket disconnected', destroyedSocketId.toString());
+            this.log('socket disconnected', destroyedSocketId?.toString());
             if (socket && socket.destroy) {
                 socket.destroy();
             }

src/client/testing/testController/common/resultResolver.ts

@@ -0,0 +1,233 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+import { CancellationToken, TestController, TestItem, Uri, TestMessage, Location, TestRun } from 'vscode';
+import * as util from 'util';
+import { DiscoveredTestPayload, ExecutionTestPayload, ITestResultResolver } from './types';
+import { TestProvider } from '../../types';
+import { traceError, traceLog } from '../../../logging';
+import { Testing } from '../../../common/utils/localize';
+import { clearAllChildren, createErrorTestItem, getTestCaseNodes } from './testItemUtilities';
+import { sendTelemetryEvent } from '../../../telemetry';
+import { EventName } from '../../../telemetry/constants';
+import { splitLines } from '../../../common/stringUtils';
+import { buildErrorNodeOptions, fixLogLines, populateTestTree } from './utils';
+
+export class PythonResultResolver implements ITestResultResolver {
+    testController: TestController;
+
+    testProvider: TestProvider;
+
+    public runIdToTestItem: Map<string, TestItem>;
+
+    public runIdToVSid: Map<string, string>;
+
+    public vsIdToRunId: Map<string, string>;
+
+    constructor(testController: TestController, testProvider: TestProvider, private workspaceUri: Uri) {
+        this.testController = testController;
+        this.testProvider = testProvider;
+
+        this.runIdToTestItem = new Map<string, TestItem>();
+        this.runIdToVSid = new Map<string, string>();
+        this.vsIdToRunId = new Map<string, string>();
+    }
+
+    public resolveDiscovery(payload: DiscoveredTestPayload, token?: CancellationToken): Promise<void> {
+        const workspacePath = this.workspaceUri.fsPath;
+        traceLog('Using result resolver for discovery');
+
+        const rawTestData = payload;
+        if (!rawTestData) {
+            // No test data is available
+            return Promise.resolve();
+        }
+
+        // Check if there were any errors in the discovery process.
+        if (rawTestData.status === 'error') {
+            const testingErrorConst =
+                this.testProvider === 'pytest' ? Testing.errorPytestDiscovery : Testing.errorUnittestDiscovery;
+            const { errors } = rawTestData;
+            traceError(testingErrorConst, '\r\n', errors!.join('\r\n\r\n'));
+
+            let errorNode = this.testController.items.get(`DiscoveryError:${workspacePath}`);
+            const message = util.format(
+                `${testingErrorConst} ${Testing.seePythonOutput}\r\n`,
+                errors!.join('\r\n\r\n'),
+            );
+
+            if (errorNode === undefined) {
+                const options = buildErrorNodeOptions(this.workspaceUri, message, this.testProvider);
+                errorNode = createErrorTestItem(this.testController, options);
+                this.testController.items.add(errorNode);
+            }
+            errorNode.error = message;
+        } else {
+            // Remove the error node if necessary,
+            // then parse and insert test data.
+            this.testController.items.delete(`DiscoveryError:${workspacePath}`);
+
+            if (rawTestData.tests) {
+                // If the test root for this folder exists: Workspace refresh, update its children.
+                // Otherwise, it is a freshly discovered workspace, and we need to create a new test root and populate the test tree.
+                populateTestTree(this.testController, rawTestData.tests, undefined, this, token);
+            } else {
+                // Delete everything from the test controller.
+                this.testController.items.replace([]);
+            }
+        }
+
+        sendTelemetryEvent(EventName.UNITTEST_DISCOVERY_DONE, undefined, {
+            tool: this.testProvider,
+            failed: false,
+        });
+        return Promise.resolve();
+    }
+
+    public resolveExecution(payload: ExecutionTestPayload, runInstance: TestRun): Promise<void> {
+        const rawTestExecData = payload;
+        if (rawTestExecData !== undefined && rawTestExecData.result !== undefined) {
+            // Map which holds the subtest information for each test item.
+            const subTestStats: Map<string, { passed: number; failed: number }> = new Map();
+
+            // iterate through payload and update the UI accordingly.
+            for (const keyTemp of Object.keys(rawTestExecData.result)) {
+                const testCases: TestItem[] = [];
+
+                // grab leaf level test items
+                this.testController.items.forEach((i) => {
+                    const tempArr: TestItem[] = getTestCaseNodes(i);
+                    testCases.push(...tempArr);
+                });
+
+                if (
+                    rawTestExecData.result[keyTemp].outcome === 'failure' ||
+                    rawTestExecData.result[keyTemp].outcome === 'passed-unexpected'
+                ) {
+                    const rawTraceback = rawTestExecData.result[keyTemp].traceback ?? '';
+                    const traceback = splitLines(rawTraceback, {
+                        trim: false,
+                        removeEmptyEntries: true,
+                    }).join('\r\n');
+
+                    const text = `${rawTestExecData.result[keyTemp].test} failed: ${
+                        rawTestExecData.result[keyTemp].message ?? rawTestExecData.result[keyTemp].outcome
+                    }\r\n${traceback}\r\n`;
+                    const message = new TestMessage(text);
+
+                    // note that keyTemp is a runId for unittest library...
+                    const grabVSid = this.runIdToVSid.get(keyTemp);
+                    // search through freshly built array of testItem to find the failed test and update UI.
+                    testCases.forEach((indiItem) => {
+                        if (indiItem.id === grabVSid) {
+                            if (indiItem.uri && indiItem.range) {
+                                message.location = new Location(indiItem.uri, indiItem.range);
+                                runInstance.failed(indiItem, message);
+                                runInstance.appendOutput(fixLogLines(text));
+                            }
+                        }
+                    });
+                } else if (
+                    rawTestExecData.result[keyTemp].outcome === 'success' ||
+                    rawTestExecData.result[keyTemp].outcome === 'expected-failure'
+                ) {
+                    const grabTestItem = this.runIdToTestItem.get(keyTemp);
+                    const grabVSid = this.runIdToVSid.get(keyTemp);
+                    if (grabTestItem !== undefined) {
+                        testCases.forEach((indiItem) => {
+                            if (indiItem.id === grabVSid) {
+                                if (indiItem.uri && indiItem.range) {
+                                    runInstance.passed(grabTestItem);
+                                    runInstance.appendOutput('Passed here');
+                                }
+                            }
+                        });
+                    }
+                } else if (rawTestExecData.result[keyTemp].outcome === 'skipped') {
+                    const grabTestItem = this.runIdToTestItem.get(keyTemp);
+                    const grabVSid = this.runIdToVSid.get(keyTemp);
+                    if (grabTestItem !== undefined) {
+                        testCases.forEach((indiItem) => {
+                            if (indiItem.id === grabVSid) {
+                                if (indiItem.uri && indiItem.range) {
+                                    runInstance.skipped(grabTestItem);
+                                    runInstance.appendOutput('Skipped here');
+                                }
+                            }
+                        });
+                    }
+                } else if (rawTestExecData.result[keyTemp].outcome === 'subtest-failure') {
+                    // split on " " since the subtest ID has the parent test ID in the first part of the ID.
+                    const parentTestCaseId = keyTemp.split(' ')[0];
+                    const parentTestItem = this.runIdToTestItem.get(parentTestCaseId);
+                    const data = rawTestExecData.result[keyTemp];
+                    // find the subtest's parent test item
+                    if (parentTestItem) {
+                        const subtestStats = subTestStats.get(parentTestCaseId);
+                        if (subtestStats) {
+                            subtestStats.failed += 1;
+                        } else {
+                            subTestStats.set(parentTestCaseId, { failed: 1, passed: 0 });
+                            runInstance.appendOutput(fixLogLines(`${parentTestCaseId} [subtests]:\r\n`));
+                            // clear since subtest items don't persist between runs
+                            clearAllChildren(parentTestItem);
+                        }
+                        const subtestId = keyTemp;
+                        const subTestItem = this.testController?.createTestItem(subtestId, subtestId);
+                        runInstance.appendOutput(fixLogLines(`${subtestId} Failed\r\n`));
+                        // create a new test item for the subtest
+                        if (subTestItem) {
+                            const traceback = data.traceback ?? '';
+                            const text = `${data.subtest} Failed: ${data.message ?? data.outcome}\r\n${traceback}\r\n`;
+                            runInstance.appendOutput(fixLogLines(text));
+                            parentTestItem.children.add(subTestItem);
+                            runInstance.started(subTestItem);
+                            const message = new TestMessage(rawTestExecData?.result[keyTemp].message ?? '');
+                            if (parentTestItem.uri && parentTestItem.range) {
+                                message.location = new Location(parentTestItem.uri, parentTestItem.range);
+                            }
+                            runInstance.failed(subTestItem, message);
+                        } else {
+                            throw new Error('Unable to create new child node for subtest');
+                        }
+                    } else {
+                        throw new Error('Parent test item not found');
+                    }
+                } else if (rawTestExecData.result[keyTemp].outcome === 'subtest-success') {
+                    // split on " " since the subtest ID has the parent test ID in the first part of the ID.
+                    const parentTestCaseId = keyTemp.split(' ')[0];
+                    const parentTestItem = this.runIdToTestItem.get(parentTestCaseId);
+
+                    // find the subtest's parent test item
+                    if (parentTestItem) {
+                        const subtestStats = subTestStats.get(parentTestCaseId);
+                        if (subtestStats) {
+                            subtestStats.passed += 1;
+                        } else {
+                            subTestStats.set(parentTestCaseId, { failed: 0, passed: 1 });
+                            runInstance.appendOutput(fixLogLines(`${parentTestCaseId} [subtests]:\r\n`));
+                            // clear since subtest items don't persist between runs
+                            clearAllChildren(parentTestItem);
+                        }
+                        const subtestId = keyTemp;
+                        const subTestItem = this.testController?.createTestItem(subtestId, subtestId);
+                        // create a new test item for the subtest
+                        if (subTestItem) {
+                            parentTestItem.children.add(subTestItem);
+                            runInstance.started(subTestItem);
+                            runInstance.passed(subTestItem);
+                            runInstance.appendOutput(fixLogLines(`${subtestId} Passed\r\n`));
+                        } else {
+                            throw new Error('Unable to create new child node for subtest');
+                        }
+                    } else {
+                        throw new Error('Parent test item not found');
+                    }
+                }
+            }
+        }
+        return Promise.resolve();
+    }
+}
+
+// had to switch the order of the original parameter since required param cannot follow optional.

src/client/testing/testController/common/server.ts

@@ -9,7 +9,7 @@ import {
     IPythonExecutionFactory,
     SpawnOptions,
 } from '../../../common/process/types';
-import { traceError, traceInfo, traceLog, traceVerbose } from '../../../logging';
+import { traceError, traceInfo, traceLog } from '../../../logging';
 import { DataReceivedEvent, ITestServer, TestCommandOptions } from './types';
 import { ITestDebugLauncher, LaunchOptions } from '../../common/types';
 import { UNITTEST_PROVIDER } from '../../common/constants';
@@ -24,6 +24,10 @@ export class PythonTestServer implements ITestServer, Disposable {
 
     private ready: Promise<void>;
 
+    private _onRunDataReceived: EventEmitter<DataReceivedEvent> = new EventEmitter<DataReceivedEvent>();
+
+    private _onDiscoveryDataReceived: EventEmitter<DataReceivedEvent> = new EventEmitter<DataReceivedEvent>();
+
     constructor(private executionFactory: IPythonExecutionFactory, private debugLauncher: ITestDebugLauncher) {
         this.server = net.createServer((socket: net.Socket) => {
             let buffer: Buffer = Buffer.alloc(0); // Buffer to accumulate received data
@@ -48,11 +52,28 @@ export class PythonTestServer implements ITestServer, Disposable {
                         rawData = rpcHeaders.remainingRawData;
                         const rpcContent = jsonRPCContent(rpcHeaders.headers, rawData);
                         const extractedData = rpcContent.extractedJSON;
+                        // do not send until we have the full content
                         if (extractedData.length === Number(totalContentLength)) {
-                            // do not send until we have the full content
-                            traceVerbose(`Received data from test server: ${extractedData}`);
-                            this._onDataReceived.fire({ uuid, data: extractedData });
-                            this.uuids = this.uuids.filter((u) => u !== uuid);
+                            // if the rawData includes tests then this is a discovery request
+                            if (rawData.includes(`"tests":`)) {
+                                this._onDiscoveryDataReceived.fire({
+                                    uuid,
+                                    data: rpcContent.extractedJSON,
+                                });
+                                // if the rawData includes result then this is a run request
+                            } else if (rawData.includes(`"result":`)) {
+                                this._onRunDataReceived.fire({
+                                    uuid,
+                                    data: rpcContent.extractedJSON,
+                                });
+                            } else {
+                                traceLog(
+                                    `Error processing test server request: request is not recognized as discovery or run.`,
+                                );
+                                this._onDataReceived.fire({ uuid: '', data: '' });
+                                return;
+                            }
+                            // this.uuids = this.uuids.filter((u) => u !== uuid); WHERE DOES THIS GO??
                             buffer = Buffer.alloc(0);
                         } else {
                             break;
@@ -97,6 +118,18 @@ export class PythonTestServer implements ITestServer, Disposable {
         return uuid;
     }
 
+    public deleteUUID(uuid: string): void {
+        this.uuids = this.uuids.filter((u) => u !== uuid);
+    }
+
+    public get onRunDataReceived(): Event<DataReceivedEvent> {
+        return this._onRunDataReceived.event;
+    }
+
+    public get onDiscoveryDataReceived(): Event<DataReceivedEvent> {
+        return this._onDiscoveryDataReceived.event;
+    }
+
     public dispose(): void {
         this.server.close();
         this._onDataReceived.dispose();

src/client/testing/testController/common/types.ts

@@ -172,12 +172,21 @@ export type TestCommandOptionsPytest = {
  */
 export interface ITestServer {
     readonly onDataReceived: Event<DataReceivedEvent>;
+    readonly onRunDataReceived: Event<DataReceivedEvent>;
+    readonly onDiscoveryDataReceived: Event<DataReceivedEvent>;
     sendCommand(options: TestCommandOptions, runTestIdsPort?: string, callback?: () => void): Promise<void>;
     serverReady(): Promise<void>;
     getPort(): number;
     createUUID(cwd: string): string;
+    deleteUUID(uuid: string): void;
+}
+export interface ITestResultResolver {
+    runIdToVSid: Map<string, string>;
+    runIdToTestItem: Map<string, TestItem>;
+    vsIdToRunId: Map<string, string>;
+    resolveDiscovery(payload: DiscoveredTestPayload, token?: CancellationToken): Promise<void>;
+    resolveExecution(payload: ExecutionTestPayload, runInstance: TestRun): Promise<void>;
 }
-
 export interface ITestDiscoveryAdapter {
     // ** first line old method signature, second line new method signature
     discoverTests(uri: Uri): Promise<DiscoveredTestPayload>;
@@ -192,6 +201,7 @@ export interface ITestExecutionAdapter {
         uri: Uri,
         testIds: string[],
         debugBool?: boolean,
+        runInstance?: TestRun,
         executionFactory?: IPythonExecutionFactory,
         debugLauncher?: ITestDebugLauncher,
     ): Promise<ExecutionTestPayload>;