Unittest discovery using the new test adapter (#18253)

* Basic test discovery (no server yet)

* Clean up test discovery script

* Add default test results array value (py side)

* Add localization

* Build tree on the Python side + move utils out

* Basic test discovery

* Add test tree update support

* Multiroot workspace support

* Make sure we don't shadow the "type" keyword

* Move discovery in a function

* Use pathlib everywhere

* News entry

* Add docstrings to utils tests + fix nested case

* Rename utils tests data + delete unused files

* Discovery tests

* Update comments + requirements

* Do not run test discovery for now

* Re-generate requirements using python 3.7

* Sort Python imports on save

* Get new server architecture up

* Add comment for tornado

* Apply Python suggestions from code review

Co-authored-by: Brett Cannon <[email protected]>

* Pass test discovery adapter to workspace adapter

* Wrap nodes under test provider

* Let the OS pick a port

* traceLog when response processing errors out

Co-authored-by: Brett Cannon <[email protected]>

Author: kimadeline

.vscode/settings.json

@@ -23,7 +23,10 @@
         ".vscode test": true
     },
     "[python]": {
-        "editor.formatOnSave": true
+        "editor.formatOnSave": true,
+        "editor.codeActionsOnSave": {
+            "source.organizeImports": true
+        }
     },
     "[typescript]": {
         "editor.defaultFormatter": "esbenp.prettier-vscode",

news/1 Enhancements/17242.md

@@ -0,0 +1 @@
+Rewrite support for unittest test discovery.

package.nls.json

@@ -71,6 +71,7 @@
     "Common.moreInfo": "More Info",
     "Common.and": "and",
     "Common.ok": "Ok",
+    "Common.error": "Error",
     "Common.install": "Install",
     "Common.learnMore": "Learn more",
     "Common.reportThisIssue": "Report this issue",
@@ -157,6 +158,9 @@
     "debug.pyramidEnterDevelopmentIniPathInvalidFilePathError": "Enter a valid file path",
     "Testing.configureTests": "Configure Test Framework",
     "Testing.testNotConfigured": "No test framework configured.",
+    "Testing.cancelUnittestDiscovery": "Canceled unittest test discovery",
+    "Testing.errorUnittestDiscovery": "Unittest test discovery error",
+    "Testing.seePythonOutput": "(see Output > Python)",
     "Common.openOutputPanel": "Show output",
     "LanguageService.statusItem.name": "Python IntelliSense Status",
     "LanguageService.statusItem.text": "Partial Mode",

pythonFiles/testing_tools/socket_manager.py

@@ -0,0 +1,44 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import socket
+import sys
+
+
+class SocketManager(object):
+    """Create a socket and connect to the given address.
+
+    The address is a (host: str, port: int) tuple.
+    Example usage:
+
+    ```
+    with SocketManager(("localhost", 6767)) as sock:
+        request = json.dumps(payload)
+        result = s.socket.sendall(request.encode("utf-8"))
+    ```
+    """
+
+    def __init__(self, addr):
+        self.addr = addr
+        self.socket = None
+
+    def __enter__(self):
+        self.socket = socket.socket(
+            socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP
+        )
+        if sys.platform == "win32":
+            addr_use = socket.SO_EXCLUSIVEADDRUSE
+        else:
+            addr_use = socket.SO_REUSEADDR
+        self.socket.setsockopt(socket.SOL_SOCKET, addr_use, 1)
+        self.socket.connect(self.addr)
+
+        return self
+
+    def __exit__(self, *_):
+        if self.socket:
+            try:
+                self.socket.shutdown(socket.SHUT_RDWR)
+            except Exception:
+                pass
+            self.socket.close()

pythonFiles/tests/unittestadapter/.data/discovery_empty.py

@@ -0,0 +1,15 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import unittest
+
+
+class DiscoveryEmpty(unittest.TestCase):
+    """Test class for the test_empty_discovery test.
+
+    The discover_tests function should return a dictionary with a "success" status, no errors, and no test tree
+    if unittest discovery was performed successfully but no tests were found.
+    """
+
+    def something(self) -> bool:
+        return True

pythonFiles/tests/unittestadapter/.data/discovery_error/file_one.py

@@ -0,0 +1,20 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import unittest
+
+import something_else  # type: ignore
+
+
+class DiscoveryErrorOne(unittest.TestCase):
+    """Test class for the test_error_discovery test.
+
+    The discover_tests function should return a dictionary with an "error" status, the discovered tests, and a list of errors
+    if unittest discovery failed at some point.
+    """
+
+    def test_one(self) -> None:
+        self.assertGreater(2, 1)
+
+    def test_two(self) -> None:
+        self.assertNotEqual(2, 1)

pythonFiles/tests/unittestadapter/.data/discovery_error/file_two.py

@@ -0,0 +1,18 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import unittest
+
+
+class DiscoveryErrorTwo(unittest.TestCase):
+    """Test class for the test_error_discovery test.
+
+    The discover_tests function should return a dictionary with an "error" status, the discovered tests, and a list of errors
+    if unittest discovery failed at some point.
+    """
+
+    def test_one(self) -> None:
+        self.assertGreater(2, 1)
+
+    def test_two(self) -> None:
+        self.assertNotEqual(2, 1)

pythonFiles/tests/unittestadapter/.data/discovery_simple.py

@@ -0,0 +1,18 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import unittest
+
+
+class DiscoverySimple(unittest.TestCase):
+    """Test class for the test_simple_discovery test.
+
+    The discover_tests function should return a dictionary with a "success" status, no errors, and a test tree
+    if unittest discovery was performed successfully.
+    """
+
+    def test_one(self) -> None:
+        self.assertGreater(2, 1)
+
+    def test_two(self) -> None:
+        self.assertNotEqual(2, 1)

pythonFiles/tests/unittestadapter/.data/utils_decorated_tree.py

@@ -0,0 +1,29 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import unittest
+from functools import wraps
+
+
+def my_decorator(f):
+    @wraps(f)
+    def wrapper(*args, **kwds):
+        print("Calling decorated function")
+        return f(*args, **kwds)
+
+    return wrapper
+
+
+class TreeOne(unittest.TestCase):
+    """Test class for the test_build_decorated_tree test.
+
+    build_test_tree should build a test tree with these test cases.
+    """
+
+    @my_decorator
+    def test_one(self) -> None:
+        self.assertGreater(2, 1)
+
+    @my_decorator
+    def test_two(self) -> None:
+        self.assertNotEqual(2, 1)

pythonFiles/tests/unittestadapter/.data/utils_nested_cases/file_one.py

@@ -0,0 +1,17 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import unittest
+
+
+class CaseTwoFileOne(unittest.TestCase):
+    """Test class for the test_nested_test_cases test.
+
+    get_test_case should return tests from the test suites in this folder.
+    """
+
+    def test_one(self) -> None:
+        self.assertGreater(2, 1)
+
+    def test_two(self) -> None:
+        self.assertNotEqual(2, 1)

pythonFiles/tests/unittestadapter/.data/utils_nested_cases/folder/__init__.py

@@ -0,0 +1,2 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.

pythonFiles/tests/unittestadapter/.data/utils_nested_cases/folder/file_two.py

@@ -0,0 +1,17 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import unittest
+
+
+class CaseTwoFileTwo(unittest.TestCase):
+    """Test class for the test_nested_test_cases test.
+
+    get_test_case should return tests from the test suites in this folder.
+    """
+
+    def test_one(self) -> None:
+        self.assertGreater(2, 1)
+
+    def test_two(self) -> None:
+        self.assertNotEqual(2, 1)

pythonFiles/tests/unittestadapter/.data/utils_simple_cases.py

@@ -0,0 +1,17 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import unittest
+
+
+class CaseOne(unittest.TestCase):
+    """Test class for the test_simple_test_cases test.
+
+    get_test_case should return tests from the test suite.
+    """
+
+    def test_one(self) -> None:
+        self.assertGreater(2, 1)
+
+    def test_two(self) -> None:
+        self.assertNotEqual(2, 1)

pythonFiles/tests/unittestadapter/.data/utils_simple_tree.py

@@ -0,0 +1,17 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import unittest
+
+
+class TreeOne(unittest.TestCase):
+    """Test class for the test_build_simple_tree test.
+
+    build_test_tree should build a test tree with these test cases.
+    """
+
+    def test_one(self) -> None:
+        self.assertGreater(2, 1)
+
+    def test_two(self) -> None:
+        self.assertNotEqual(2, 1)

pythonFiles/tests/unittestadapter/__init__.py

@@ -0,0 +1,2 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.

pythonFiles/tests/unittestadapter/conftest.py

@@ -0,0 +1,8 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import sys
+
+# Ignore the contents of this folder for Python 2 tests.
+if sys.version_info[0] < 3:
+    collect_ignore_glob = ["*.py"]

pythonFiles/tests/unittestadapter/helpers.py

@@ -0,0 +1,32 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import pathlib
+
+TEST_DATA_PATH = pathlib.Path(__file__).parent / ".data"
+
+
+def is_same_tree(tree1, tree2) -> bool:
+    """Helper function to test if two test trees are the same.
+
+    `is_same_tree` starts by comparing the root attributes, and then checks if all children are the same.
+    """
+    # Compare the root.
+    if any(tree1[key] != tree2[key] for key in ["path", "name", "type_"]):
+        return False
+
+    # Compare child test nodes if they exist, otherwise compare test items.
+    if "children" in tree1 and "children" in tree2:
+        children1 = tree1["children"]
+        children2 = tree2["children"]
+
+        # Compare test nodes.
+        if len(children1) != len(children2):
+            return False
+        else:
+            return all(is_same_tree(*pair) for pair in zip(children1, children2))
+    elif "id_" in tree1 and "id_" in tree2:
+        # Compare test items.
+        return all(tree1[key] == tree2[key] for key in ["id_", "lineno"])
+
+    return False