Pass PTVSD folder name to the new debug adapter executable (#7152)

* Get PR validation going, fix unit tests later
* Don't flatten the folder structure too much
* Longer comment
* Fix service registry unit tests
* Add fallback to base PTVSD path
* Update ptvsd folder retrieval (cache path)
* Fix existing tests
* Mock things in setup() function when possible
* Don't mix then and await
* More tests
* Lol undo suite name change
* don't call __iter__ directly
* Update src/client/debugger/extension/adapter/factory.ts
Co-Authored-By: Eric Snow <[email protected]>
* Renaming in python scripts (code review)
* Add fallback to SpecifierSet
* TS code review
* Python unit tests + error handling
* Move to a folder and fix things
* Use latest version of ptvsd
* Undo formatting changes
* Update packaging version
* Add path to pythonfiles
* forgot python subfolder
* try updating pytest in test-requirements
* Add packaging to test requirements
* Forgot copyright
* Undo pytest version change
* Install ptvsd wheels during tests step not compile
* yml indentation
* Apply suggestions from code review
Co-Authored-By: Eric Snow <[email protected]>
* More code review changes
* more review changes
* More review changes
* more changes
* remove need for ptvsd_version function
* Forgot to import re
* Fix tests
* Apply suggestions from code review
Co-Authored-By: Eric Snow <[email protected]>
* move path constants to __init__.py
* Add safeguard for ptvsd version during dev
* Add wheel support for remote debugging
* Fix tests
* Revert remote debugging changes
* Add todo comment
* Disable linting for the todo

Author: kimadeline

build/ci/templates/steps/build_compile.yml

@@ -36,15 +36,6 @@ steps:
       displayName: "pip install requirements"
       condition: and(succeeded(), eq(variables['build'], 'true'))
 
-    - task: PythonScript@0
-      displayName: "Install PTVSD wheels"
-      inputs:
-          scriptSource: "filePath"
-          scriptPath: "./pythonFiles/install_ptvsd.py"
-          arguments: "--ci"
-          failOnStderr: true
-      condition: and(succeeded(), eq(variables['build'], 'true'))
-
     - bash: npm run clean
       displayName: "Clean"
       condition: and(succeeded(), eq(variables['build'], 'true'))

build/ci/templates/test_phases.yml

@@ -138,6 +138,15 @@ steps:
     displayName: 'pip install ipython requirements'
     condition: and(succeeded(), eq(variables['NeedsIPythonReqs'], 'true'))
 
+  - task: PythonScript@0
+    displayName: "Install PTVSD wheels"
+    inputs:
+        scriptSource: "filePath"
+        scriptPath: "./pythonFiles/install_ptvsd.py"
+        arguments: "--ci"
+        failOnStderr: true
+    condition: contains(variables['TestsToRun'], 'testUnitTests')
+
   # Run the Python unit tests in our codebase. Produces a JUnit-style log file that
   # will be uploaded after all tests are complete.
   #

build/test-requirements.txt

@@ -15,3 +15,4 @@ rope
 flask
 django
 isort
+packaging==19.2

pythonFiles/install_ptvsd.py

@@ -5,50 +5,56 @@
 import urllib.request
 import sys
 
-ROOT_DIRNAME = path.dirname(path.dirname(path.abspath(__file__)))
-REQUIREMENTS_PATH = path.join(ROOT_DIRNAME, "requirements.txt")
-PYTHONFILES_PATH = path.join(ROOT_DIRNAME, "pythonFiles", "lib", "python")
+ROOT = path.dirname(path.dirname(path.abspath(__file__)))
+REQUIREMENTS = path.join(ROOT, "requirements.txt")
+PYTHONFILES = path.join(ROOT, "pythonFiles", "lib", "python")
 PYPI_PTVSD_URL = "https://pypi.org/pypi/ptvsd/json"
 
 
 def install_ptvsd():
-    # If we are in CI use the packaging module installed in PYTHONFILES_PATH.
+    # If we are in CI use the packaging module installed in PYTHONFILES.
     if len(sys.argv) == 2 and sys.argv[1] == "--ci":
-        sys.path.insert(0, PYTHONFILES_PATH)
+        sys.path.insert(0, PYTHONFILES)
     from packaging.requirements import Requirement
 
-    with open(REQUIREMENTS_PATH, "r", encoding="utf-8") as requirements:
-        for line in requirements:
-            package_requirement = Requirement(line)
-            if package_requirement.name != "ptvsd":
-                continue
-            requirement_specifier = package_requirement.specifier
-            ptvsd_version = next(requirement_specifier.__iter__()).version
+    with open(REQUIREMENTS, "r", encoding="utf-8") as reqsfile:
+        for line in reqsfile:
+            pkgreq = Requirement(line)
+            if pkgreq.name == "ptvsd":
+                specs = pkgreq.specifier
+                version = next(iter(specs)).version
+                break
+
+    try:
+        version
+    except NameError:
+        raise Exception("ptvsd requirement not found.")
 
     # Response format: https://warehouse.readthedocs.io/api-reference/json/#project
     with urllib.request.urlopen(PYPI_PTVSD_URL) as response:
         json_response = json.loads(response.read())
     releases = json_response["releases"]
 
     # Release metadata format: https://github.com/pypa/interoperability-peps/blob/master/pep-0426-core-metadata.rst
-    for wheel_info in releases[ptvsd_version]:
+    for wheel_info in releases[version]:
         # Download only if it's a 3.7 wheel.
         if not wheel_info["python_version"].endswith(("37", "3.7")):
             continue
         filename = wheel_info["filename"].rpartition(".")[0]  # Trim the file extension.
-        ptvsd_path = path.join(PYTHONFILES_PATH, filename)
+        ptvsd_path = path.join(PYTHONFILES, filename)
 
         with urllib.request.urlopen(wheel_info["url"]) as wheel_response:
             wheel_file = BytesIO(wheel_response.read())
-        # Extract only the contents of the ptvsd subfolder.
-        prefix = path.join(f"ptvsd-{ptvsd_version}.data", "purelib", "ptvsd")
+        # Extract only the contents of the purelib subfolder (parent folder of ptvsd),
+        # since ptvsd files rely on the presence of a 'ptvsd' folder.
+        prefix = path.join(f"ptvsd-{version}.data", "purelib")
 
         with ZipFile(wheel_file, "r") as wheel:
             for zip_info in wheel.infolist():
-                if not zip_info.filename.startswith(prefix):
-                    continue
+                # Normalize path for Windows, the wheel folder structure uses forward slashes.
+                normalized = path.normpath(zip_info.filename)
                 # Flatten the folder structure.
-                zip_info.filename = zip_info.filename.split(prefix)[-1]
+                zip_info.filename = normalized.split(prefix)[-1]
                 wheel.extract(zip_info, ptvsd_path)
 
 

pythonFiles/ptvsd_folder_name.py

@@ -0,0 +1,53 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import sys
+import os.path
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+PYTHONFILES = os.path.join(ROOT, "pythonFiles", "lib", "python")
+REQUIREMENTS = os.path.join(ROOT, "requirements.txt")
+
+sys.path.insert(0, PYTHONFILES)
+
+from packaging.requirements import Requirement
+from packaging.tags import sys_tags
+
+sys.path.remove(PYTHONFILES)
+
+
+def ptvsd_folder_name():
+    """Return the folder name for the bundled PTVSD wheel compatible with the new debug adapter."""
+
+    with open(REQUIREMENTS, "r", encoding="utf-8") as reqsfile:
+        for line in reqsfile:
+            pkgreq = Requirement(line)
+            if pkgreq.name == "ptvsd":
+                specs = pkgreq.specifier
+                try:
+                    spec, = specs
+                    version = spec.version
+                except:
+                    # Fallpack to use base PTVSD path.
+                    print(PYTHONFILES, end="")
+                    return
+                break
+
+    try:
+        for tag in sys_tags():
+            folder_name = f"ptvsd-{version}-{tag.interpreter}-{tag.abi}-{tag.platform}"
+            folder_path = os.path.join(PYTHONFILES, folder_name)
+            if os.path.exists(folder_path):
+                print(folder_path, end="")
+                return
+    except:
+        # Fallback to use base PTVSD path no matter the exception.
+        print(PYTHONFILES, end="")
+        return
+
+    # Default fallback to use base PTVSD path.
+    print(PYTHONFILES, end="")
+
+
+if __name__ == "__main__":
+    ptvsd_folder_name()

pythonFiles/tests/__init__.py

@@ -1,2 +1,13 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
+import os.path
+
+TEST_ROOT = os.path.dirname(__file__)
+SRC_ROOT = os.path.dirname(TEST_ROOT)
+PROJECT_ROOT = os.path.dirname(SRC_ROOT)
+IPYTHON_ROOT = os.path.join(SRC_ROOT, "ipython")
+TESTING_TOOLS_ROOT = os.path.join(SRC_ROOT, "testing_tools")
+DEBUG_ADAPTER_ROOT = os.path.join(SRC_ROOT, "debug_adapter")
+
+PYTHONFILES = os.path.join(SRC_ROOT, "lib", "python")
+REQUIREMENTS = os.path.join(PROJECT_ROOT, "requirements.txt")

pythonFiles/tests/__main__.py

@@ -2,26 +2,22 @@
 # Licensed under the MIT License.
 
 import argparse
-import os.path
 import sys
 
 import pytest
 
-
-TEST_ROOT = os.path.dirname(__file__)
-SRC_ROOT = os.path.dirname(TEST_ROOT)
-PROJECT_ROOT = os.path.dirname(SRC_ROOT)
-IPYTHON_ROOT = os.path.join(SRC_ROOT, 'ipython')
-TESTING_TOOLS_ROOT = os.path.join(SRC_ROOT, 'testing_tools')
+from . import DEBUG_ADAPTER_ROOT, IPYTHON_ROOT, SRC_ROOT, TEST_ROOT, TESTING_TOOLS_ROOT
 
 
 def parse_args():
     parser = argparse.ArgumentParser()
     # To mark a test as functional:  (decorator) @pytest.mark.functional
-    parser.add_argument('--functional', dest='markers',
-                        action='append_const', const='functional')
-    parser.add_argument('--no-functional', dest='markers',
-                        action='append_const', const='not functional')
+    parser.add_argument(
+        "--functional", dest="markers", action="append_const", const="functional"
+    )
+    parser.add_argument(
+        "--no-functional", dest="markers", action="append_const", const="not functional"
+    )
     args, remainder = parser.parse_known_args()
 
     ns = vars(args)
@@ -32,20 +28,18 @@ def parse_args():
 def main(pytestargs, markers=None):
     sys.path.insert(1, IPYTHON_ROOT)
     sys.path.insert(1, TESTING_TOOLS_ROOT)
+    sys.path.insert(1, DEBUG_ADAPTER_ROOT)
 
-    pytestargs = [
-        '--rootdir', SRC_ROOT,
-        TEST_ROOT,
-        ] + pytestargs
+    pytestargs = ["--rootdir", SRC_ROOT, TEST_ROOT] + pytestargs
     for marker in reversed(markers or ()):
         pytestargs.insert(0, marker)
-        pytestargs.insert(0, '-m')
+        pytestargs.insert(0, "-m")
 
     ec = pytest.main(pytestargs)
     return ec
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     mainkwargs, pytestargs = parse_args()
     ec = main(pytestargs, **mainkwargs)
     sys.exit(ec)

pythonFiles/tests/debug_adapter/__init__.py

@@ -0,0 +1,2 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.

pythonFiles/tests/debug_adapter/test_ptvsd_folder_name.py

@@ -0,0 +1,81 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+import sys
+
+if sys.version_info[:2] != (3, 7):
+    import unittest
+
+    raise unittest.SkipTest("PTVSD wheels shipped for Python 3.7 only")
+
+import os.path
+import pytest
+import re
+
+from unittest.mock import patch, mock_open
+from packaging.tags import sys_tags
+from ptvsd_folder_name import ptvsd_folder_name
+
+from .. import PYTHONFILES, REQUIREMENTS
+
+
+def open_requirements_with_ptvsd():
+    return patch(
+        "ptvsd_folder_name.open", mock_open(read_data="jedi==0.15.1\nptvsd==5.0.0")
+    )
+
+
+def open_requirements_without_ptvsd():
+    return patch("ptvsd_folder_name.open", mock_open(read_data="jedi==0.15.1\n"))
+
+
+class TestPtvsdFolderName:
+    """Unit tests for the script retrieving the PTVSD folder name for the PTVSD wheels experiment."""
+
+    def test_requirement_exists_folder_exists(self, capsys):
+        # Return the first constructed folder path as existing.
+
+        patcher = patch("os.path.exists")
+        mock_exists = patcher.start()
+        mock_exists.side_effect = lambda p: True
+        tag = next(sys_tags())
+        folder = "ptvsd-5.0.0-{}-{}-{}".format(tag.interpreter, tag.abi, tag.platform)
+
+        with open_requirements_with_ptvsd():
+            ptvsd_folder_name()
+
+        patcher.stop()
+        expected = os.path.join(PYTHONFILES, folder)
+        captured = capsys.readouterr()
+        assert captured.out == expected
+
+    def test_ptvsd_requirement_once(self):
+        reqs = [
+            line
+            for line in open(REQUIREMENTS, "r", encoding="utf-8")
+            if re.match("ptvsd==", line)
+        ]
+        assert len(reqs) == 1
+
+    def test_no_ptvsd_requirement(self, capsys):
+        with open_requirements_without_ptvsd() as p:
+            ptvsd_folder_name()
+
+        expected = PYTHONFILES
+        captured = capsys.readouterr()
+        assert captured.out == expected
+
+    def test_no_wheel_folder(self, capsys):
+        # Return none of of the constructed paths as existing,
+        # ptvsd_folder_name() should return the path to default ptvsd.
+        patcher = patch("os.path.exists")
+        mock_no_exist = patcher.start()
+        mock_no_exist.side_effect = lambda p: False
+
+        with open_requirements_with_ptvsd() as p:
+            ptvsd_folder_name()
+
+        patcher.stop()
+        expected = PYTHONFILES
+        captured = capsys.readouterr()
+        assert captured.out == expected
+

pythonFiles/tests/debug_adapter/test_ptvsd_folder_name_functional.py

@@ -0,0 +1,59 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import sys
+
+if sys.version_info[:2] != (3, 7):
+    import unittest
+
+    raise unittest.SkipTest("PTVSD wheels shipped for Python 3.7 only")
+
+import os.path
+import pytest
+import subprocess
+
+from packaging.requirements import Requirement
+from .. import PYTHONFILES, REQUIREMENTS, SRC_ROOT
+
+ARGV = ["python", os.path.join(SRC_ROOT, "ptvsd_folder_name.py")]
+PREFIX = "ptvsd=="
+
+with open(REQUIREMENTS, "r", encoding="utf-8") as reqsfile:
+    for line in reqsfile:
+        if line.startswith(PREFIX):
+            VERSION = line[len(PREFIX) :].strip()
+            break
+
+
+def ptvsd_paths(*platforms):
+    paths = set()
+    for platform in platforms:
+        folder = "ptvsd-{}-cp37-cp37m-{}".format(VERSION, platform)
+        paths.add(os.path.join(PYTHONFILES, folder))
+    return paths
+
+
+@pytest.mark.functional
+class TestPtvsdFolderNameFunctional:
+    """Functional tests for the script retrieving the PTVSD folder name for the PTVSD wheels experiment."""
+
+    def test_ptvsd_folder_name_nofail(self):
+        output = subprocess.check_output(ARGV, universal_newlines=True)
+        assert output != PYTHONFILES
+
+    @pytest.mark.skipif(sys.platform != "darwin", reason="macOS functional test")
+    def test_ptvsd_folder_name_macos(self):
+        output = subprocess.check_output(ARGV, universal_newlines=True)
+        assert output in ptvsd_paths("macosx_10_13_x86_64")
+
+    @pytest.mark.skipif(sys.platform != "win32", reason="Windows functional test")
+    def test_ptvsd_folder_name_windows(self):
+        output = subprocess.check_output(ARGV, universal_newlines=True)
+        assert output in ptvsd_paths("win32", "win_amd64")
+
+    @pytest.mark.skipif(sys.platform != "linux", reason="Linux functional test")
+    def test_ptvsd_folder_name_linux(self):
+        output = subprocess.check_output(ARGV, universal_newlines=True)
+        assert output in ptvsd_paths(
+            "manylinux1_i686", "manylinux1_x86_64", "manylinux2010_x86_64"
+        )

requirements.txt

@@ -4,4 +4,4 @@ isort==4.3.21
 ptvsd==5.0.0a4
 pyparsing==2.4.0
 six==1.12.0
-packaging==19.1
+packaging==19.2

src/client/api.ts

@@ -41,6 +41,8 @@ export function buildApi(ready: Promise<any>) {
             return Promise.reject(ex);
         }),
         debug: {
+            // tslint:disable-next-line:no-suspicious-comment
+            // TODO: Add support for ptvsd wheels experiment, see https://github.com/microsoft/vscode-python/issues/7549
             async getRemoteLauncherCommand(host: string, port: number, waitUntilDebuggerAttaches: boolean = true): Promise<string[]> {
                 return new RemoteDebuggerExternalLauncherScriptProvider().getLauncherArgs({ host, port, waitUntilDebuggerAttaches });
             }