From 6d09c219de68696b9d7c74ada8b79837127441ae Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Wed, 16 May 2018 05:16:32 -0400 Subject: [PATCH 1/5] Make pep561 tests parrallel and sandboxed Each test is run in a seperate virtualenv which is deleted after the tests. This means they can be run in parrallel, and caused about a 30% speedup on my machine (4x parrallel). This also will allow for the tests to scale better with egg support, and bypass issues with uninstalling some types of packages. --- mypy/sitepkgs.py | 3 +- mypy/test/testpep561.py | 144 +++++++++++++++++++++------------------- 2 files changed, 79 insertions(+), 68 deletions(-) diff --git a/mypy/sitepkgs.py b/mypy/sitepkgs.py index 8a36bffb98c9..382fab8d8bc9 100644 --- a/mypy/sitepkgs.py +++ b/mypy/sitepkgs.py @@ -7,7 +7,8 @@ possible. """ - +import sys +sys.path = sys.path[1:] # we don't want to pick up mypy.types from distutils.sysconfig import get_python_lib import site MYPY = False diff --git a/mypy/test/testpep561.py b/mypy/test/testpep561.py index 5432a53722fc..7fdedaebe7e8 100644 --- a/mypy/test/testpep561.py +++ b/mypy/test/testpep561.py @@ -1,8 +1,10 @@ from contextlib import contextmanager import os +import random import shutil +import string import sys -from typing import Iterator, List +from typing import Iterator, List, Generator from unittest import TestCase, main import mypy.api @@ -19,112 +21,120 @@ def check_mypy_run(cmd_line: List[str], - expected_out: str, + python_executable: str = sys.executable, + expected_out: str = '', expected_err: str = '', expected_returncode: int = 1) -> None: """Helper to run mypy and check the output.""" + if python_executable != sys.executable: + cmd_line.append('--python-executable={}'.format(python_executable)) out, err, returncode = mypy.api.run(cmd_line) assert out == expected_out, err assert err == expected_err, out assert returncode == expected_returncode, returncode -def is_in_venv() -> bool: - """Returns whether we are running inside a venv. +def random_dir() -> str: + dir = ''.join(random.sample(string.ascii_lowercase + string.digits + '_-.', 10)) + return dir - Based on https://stackoverflow.com/a/42580137. - """ - if hasattr(sys, 'real_prefix'): - return True - else: - return hasattr(sys, 'base_prefix') and sys.base_prefix != sys.prefix +class TestPEP561(TestCase): + test_file = 'simple.py' + base_dir = 'test-packages-data' + @contextmanager + def virtualenv(self, python_executable: str = sys.executable) -> Generator[str, None, None]: + """Context manager that creates a virtualenv in a randomly named directory + + returns the path to the created Python executable""" + venv_dir = random_dir() # a unique name for the virtualenv directory + run_command([sys.executable, '-m', 'virtualenv', '-p{}'.format(python_executable), + venv_dir], cwd=os.getcwd()) + if sys.platform == 'win32': + yield os.path.abspath(os.path.join(venv_dir, 'Scripts', 'python')) + else: + yield os.path.abspath(os.path.join(venv_dir, 'bin', 'python')) + shutil.rmtree(venv_dir) -class TestPEP561(TestCase): @contextmanager def install_package(self, pkg: str, python_executable: str = sys.executable) -> Iterator[None]: """Context manager to temporarily install a package from test-data/packages/pkg/""" working_dir = os.path.join(package_path, pkg) install_cmd = [python_executable, '-m', 'pip', 'install', '.'] - # if we aren't in a virtualenv, install in the - # user package directory so we don't need sudo - if not is_in_venv() or python_executable != sys.executable: - install_cmd.append('--user') returncode, lines = run_command(install_cmd, cwd=working_dir) if returncode != 0: self.fail('\n'.join(lines)) - try: - yield - finally: - returncode, lines = run_command([python_executable, '-m', 'pip', 'uninstall', - '-y', pkg], cwd=package_path) - if returncode != 0: - self.fail('\n'.join(lines)) + yield + + def setUp(self) -> None: + self.test_dir = os.path.abspath(random_dir()) + if not os.path.exists(self.test_dir): + os.mkdir(self.test_dir) + self.old_cwd = os.getcwd() + os.chdir(self.test_dir) + with open(self.test_file, 'w') as f: + f.write(SIMPLE_PROGRAM) + + def tearDown(self) -> None: + os.chdir(self.old_cwd) + shutil.rmtree(self.test_dir) def test_get_pkg_dirs(self) -> None: """Check that get_package_dirs works.""" dirs = _get_site_packages_dirs(sys.executable) assert dirs - def test_typed_pkg(self) -> None: - """Tests type checking based on installed packages. - - This test CANNOT be split up, concurrency means that simultaneously - installing/uninstalling will break tests. - """ - test_file = 'simple.py' - if not os.path.isdir('test-packages-data'): - os.mkdir('test-packages-data') - old_cwd = os.getcwd() - os.chdir('test-packages-data') - with open(test_file, 'w') as f: - f.write(SIMPLE_PROGRAM) - try: - with self.install_package('typedpkg-stubs'): + def test_typedpkg_stub_package(self) -> None: + with self.virtualenv() as python_executable: + with self.install_package('typedpkg-stubs', python_executable): check_mypy_run( - [test_file], + [self.test_file], + python_executable, "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n" ) - # The Python 2 tests are intentionally placed after a Python 3 test to check - # the package_dir_cache is behaving correctly. - python2 = try_find_python2_interpreter() - if python2: - with self.install_package('typedpkg-stubs', python2): + def test_typedpkg(self) -> None: + with self.virtualenv() as python_executable: + with self.install_package('typedpkg', python_executable): + check_mypy_run( + [self.test_file], + python_executable, + "simple.py:4: error: Revealed type is 'builtins.tuple[builtins.str]'\n" + ) + + def test_stub_and_typed_pkg(self) -> None: + with self.virtualenv() as python_executable: + with self.install_package('typedpkg', python_executable): + with self.install_package('typedpkg-stubs', python_executable): check_mypy_run( - ['--python-executable={}'.format(python2), test_file], + [self.test_file], + python_executable, "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n" ) - with self.install_package('typedpkg', python2): + + def test_typedpkg_stubs_python2(self) -> None: + python2 = try_find_python2_interpreter() + if python2: + with self.virtualenv(python2) as py2: + with self.install_package('typedpkg-stubs', py2): check_mypy_run( - ['--python-executable={}'.format(python2), 'simple.py'], - "simple.py:4: error: Revealed type is 'builtins.tuple[builtins.str]'\n" + [self.test_file], + py2, + "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n" ) - with self.install_package('typedpkg', python2): - with self.install_package('typedpkg-stubs', python2): - check_mypy_run( - ['--python-executable={}'.format(python2), test_file], - "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n" - ) - - with self.install_package('typedpkg'): - check_mypy_run( - [test_file], - "simple.py:4: error: Revealed type is 'builtins.tuple[builtins.str]'\n" - ) - - with self.install_package('typedpkg'): - with self.install_package('typedpkg-stubs'): + def test_typedpkg_python2(self) -> None: + python2 = try_find_python2_interpreter() + if python2: + with self.virtualenv(python2) as py2: + with self.install_package('typedpkg', py2): check_mypy_run( - [test_file], - "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n" + [self.test_file], + py2, + "simple.py:4: error: Revealed type is 'builtins.tuple[builtins.str]'\n" ) - finally: - os.chdir(old_cwd) - shutil.rmtree('test-packages-data') if __name__ == '__main__': From 0bca04cebdf630fe348d5a347f496fbf729d73da Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Wed, 16 May 2018 07:07:40 -0400 Subject: [PATCH 2/5] Use temp files and directories --- mypy/test/testpep561.py | 61 ++++++++++++++++++----------------------- test-requirements.txt | 1 + 2 files changed, 27 insertions(+), 35 deletions(-) diff --git a/mypy/test/testpep561.py b/mypy/test/testpep561.py index 7fdedaebe7e8..445dcb0897fc 100644 --- a/mypy/test/testpep561.py +++ b/mypy/test/testpep561.py @@ -4,12 +4,13 @@ import shutil import string import sys +import tempfile from typing import Iterator, List, Generator from unittest import TestCase, main import mypy.api from mypy.build import FindModuleCache, _get_site_packages_dirs -from mypy.test.config import package_path +from mypy.test.config import package_path, test_temp_dir from mypy.test.helpers import run_command from mypy.util import try_find_python2_interpreter @@ -34,28 +35,21 @@ def check_mypy_run(cmd_line: List[str], assert returncode == expected_returncode, returncode -def random_dir() -> str: - dir = ''.join(random.sample(string.ascii_lowercase + string.digits + '_-.', 10)) - return dir - - class TestPEP561(TestCase): - test_file = 'simple.py' - base_dir = 'test-packages-data' @contextmanager def virtualenv(self, python_executable: str = sys.executable) -> Generator[str, None, None]: - """Context manager that creates a virtualenv in a randomly named directory + """Context manager that creates a virtualenv in a temporary directory returns the path to the created Python executable""" - venv_dir = random_dir() # a unique name for the virtualenv directory - run_command([sys.executable, '-m', 'virtualenv', '-p{}'.format(python_executable), - venv_dir], cwd=os.getcwd()) - if sys.platform == 'win32': - yield os.path.abspath(os.path.join(venv_dir, 'Scripts', 'python')) - else: - yield os.path.abspath(os.path.join(venv_dir, 'bin', 'python')) - shutil.rmtree(venv_dir) + with tempfile.TemporaryDirectory() as venv_dir: + run_command([sys.executable, '-m', 'virtualenv', '-p{}'.format(python_executable), + venv_dir], cwd=os.getcwd()) + if sys.platform == 'win32': + python = os.path.abspath(os.path.join(venv_dir, 'Scripts', 'python')) + else: + python = os.path.abspath(os.path.join(venv_dir, 'bin', 'python')) + yield python @contextmanager def install_package(self, pkg: str, @@ -69,17 +63,14 @@ def install_package(self, pkg: str, yield def setUp(self) -> None: - self.test_dir = os.path.abspath(random_dir()) - if not os.path.exists(self.test_dir): - os.mkdir(self.test_dir) - self.old_cwd = os.getcwd() - os.chdir(self.test_dir) - with open(self.test_file, 'w') as f: + self.tempfile = tempfile.NamedTemporaryFile() + with open(self.tempfile.name, 'w') as f: f.write(SIMPLE_PROGRAM) + self.msg_list = "{}:4: error: Revealed type is 'builtins.list[builtins.str]'\n".format(self.tempfile.name) + self.msg_tuple = "{}:4: error: Revealed type is 'builtins.tuple[builtins.str]'\n".format(self.tempfile.name) def tearDown(self) -> None: - os.chdir(self.old_cwd) - shutil.rmtree(self.test_dir) + self.tempfile.close() def test_get_pkg_dirs(self) -> None: """Check that get_package_dirs works.""" @@ -90,18 +81,18 @@ def test_typedpkg_stub_package(self) -> None: with self.virtualenv() as python_executable: with self.install_package('typedpkg-stubs', python_executable): check_mypy_run( - [self.test_file], + [self.tempfile.name], python_executable, - "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n" + self.msg_list, ) def test_typedpkg(self) -> None: with self.virtualenv() as python_executable: with self.install_package('typedpkg', python_executable): check_mypy_run( - [self.test_file], + [self.tempfile.name], python_executable, - "simple.py:4: error: Revealed type is 'builtins.tuple[builtins.str]'\n" + self.msg_tuple, ) def test_stub_and_typed_pkg(self) -> None: @@ -109,9 +100,9 @@ def test_stub_and_typed_pkg(self) -> None: with self.install_package('typedpkg', python_executable): with self.install_package('typedpkg-stubs', python_executable): check_mypy_run( - [self.test_file], + [self.tempfile.name], python_executable, - "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n" + self.msg_list, ) def test_typedpkg_stubs_python2(self) -> None: @@ -120,9 +111,9 @@ def test_typedpkg_stubs_python2(self) -> None: with self.virtualenv(python2) as py2: with self.install_package('typedpkg-stubs', py2): check_mypy_run( - [self.test_file], + [self.tempfile.name], py2, - "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n" + self.msg_list, ) def test_typedpkg_python2(self) -> None: @@ -131,9 +122,9 @@ def test_typedpkg_python2(self) -> None: with self.virtualenv(python2) as py2: with self.install_package('typedpkg', py2): check_mypy_run( - [self.test_file], + [self.tempfile.name], py2, - "simple.py:4: error: Revealed type is 'builtins.tuple[builtins.str]'\n" + self.msg_tuple, ) diff --git a/test-requirements.txt b/test-requirements.txt index 881027780e36..dfa56ea196f1 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -9,3 +9,4 @@ pytest-cov>=2.4.0 typed-ast>=1.1.0,<1.2.0 typing>=3.5.2; python_version < '3.5' py>=1.5.2 +virtualenv From 0b75f548a73b8e081c3ff8d2a3dd0592e13937d7 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Wed, 16 May 2018 07:33:05 -0400 Subject: [PATCH 3/5] Change tempfile management --- mypy/test/testpep561.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/mypy/test/testpep561.py b/mypy/test/testpep561.py index 445dcb0897fc..5bf4d161c3f1 100644 --- a/mypy/test/testpep561.py +++ b/mypy/test/testpep561.py @@ -63,14 +63,17 @@ def install_package(self, pkg: str, yield def setUp(self) -> None: - self.tempfile = tempfile.NamedTemporaryFile() - with open(self.tempfile.name, 'w') as f: - f.write(SIMPLE_PROGRAM) - self.msg_list = "{}:4: error: Revealed type is 'builtins.list[builtins.str]'\n".format(self.tempfile.name) - self.msg_tuple = "{}:4: error: Revealed type is 'builtins.tuple[builtins.str]'\n".format(self.tempfile.name) + self.temp_file_dir = tempfile.TemporaryDirectory() + self.tempfile = os.path.join(self.temp_file_dir.name, 'simple.py') + with open(self.tempfile, 'w+') as file: + file.write(SIMPLE_PROGRAM) + self.msg_list = \ + "{}:4: error: Revealed type is 'builtins.list[builtins.str]'\n".format(self.tempfile) + self.msg_tuple = \ + "{}:4: error: Revealed type is 'builtins.tuple[builtins.str]'\n".format(self.tempfile) def tearDown(self) -> None: - self.tempfile.close() + self.temp_file_dir.cleanup() def test_get_pkg_dirs(self) -> None: """Check that get_package_dirs works.""" @@ -81,7 +84,7 @@ def test_typedpkg_stub_package(self) -> None: with self.virtualenv() as python_executable: with self.install_package('typedpkg-stubs', python_executable): check_mypy_run( - [self.tempfile.name], + [self.tempfile], python_executable, self.msg_list, ) @@ -90,7 +93,7 @@ def test_typedpkg(self) -> None: with self.virtualenv() as python_executable: with self.install_package('typedpkg', python_executable): check_mypy_run( - [self.tempfile.name], + [self.tempfile], python_executable, self.msg_tuple, ) @@ -100,7 +103,7 @@ def test_stub_and_typed_pkg(self) -> None: with self.install_package('typedpkg', python_executable): with self.install_package('typedpkg-stubs', python_executable): check_mypy_run( - [self.tempfile.name], + [self.tempfile], python_executable, self.msg_list, ) @@ -111,7 +114,7 @@ def test_typedpkg_stubs_python2(self) -> None: with self.virtualenv(python2) as py2: with self.install_package('typedpkg-stubs', py2): check_mypy_run( - [self.tempfile.name], + [self.tempfile], py2, self.msg_list, ) @@ -122,7 +125,7 @@ def test_typedpkg_python2(self) -> None: with self.virtualenv(python2) as py2: with self.install_package('typedpkg', py2): check_mypy_run( - [self.tempfile.name], + [self.tempfile], py2, self.msg_tuple, ) From 873646e9adb7c6f471381210b87e2292f4a57b87 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Thu, 17 May 2018 13:36:11 -0400 Subject: [PATCH 4/5] Remove unused import and fast path yields --- mypy/test/testpep561.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/mypy/test/testpep561.py b/mypy/test/testpep561.py index 5bf4d161c3f1..c813a0671dcb 100644 --- a/mypy/test/testpep561.py +++ b/mypy/test/testpep561.py @@ -10,7 +10,7 @@ import mypy.api from mypy.build import FindModuleCache, _get_site_packages_dirs -from mypy.test.config import package_path, test_temp_dir +from mypy.test.config import package_path from mypy.test.helpers import run_command from mypy.util import try_find_python2_interpreter @@ -46,10 +46,9 @@ def virtualenv(self, python_executable: str = sys.executable) -> Generator[str, run_command([sys.executable, '-m', 'virtualenv', '-p{}'.format(python_executable), venv_dir], cwd=os.getcwd()) if sys.platform == 'win32': - python = os.path.abspath(os.path.join(venv_dir, 'Scripts', 'python')) + yield os.path.abspath(os.path.join(venv_dir, 'Scripts', 'python')) else: - python = os.path.abspath(os.path.join(venv_dir, 'bin', 'python')) - yield python + yield os.path.abspath(os.path.join(venv_dir, 'bin', 'python')) @contextmanager def install_package(self, pkg: str, From 1bf934408327c171f6c281cad9c5515a5b1a6095 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Fri, 15 Jun 2018 14:05:37 -0700 Subject: [PATCH 5/5] Make install_package a normal function --- mypy/test/testpep561.py | 66 ++++++++++++++++++++--------------------- 1 file changed, 32 insertions(+), 34 deletions(-) diff --git a/mypy/test/testpep561.py b/mypy/test/testpep561.py index c813a0671dcb..d2dacf0ade15 100644 --- a/mypy/test/testpep561.py +++ b/mypy/test/testpep561.py @@ -50,16 +50,14 @@ def virtualenv(self, python_executable: str = sys.executable) -> Generator[str, else: yield os.path.abspath(os.path.join(venv_dir, 'bin', 'python')) - @contextmanager def install_package(self, pkg: str, - python_executable: str = sys.executable) -> Iterator[None]: + python_executable: str = sys.executable) -> None: """Context manager to temporarily install a package from test-data/packages/pkg/""" working_dir = os.path.join(package_path, pkg) install_cmd = [python_executable, '-m', 'pip', 'install', '.'] returncode, lines = run_command(install_cmd, cwd=working_dir) if returncode != 0: self.fail('\n'.join(lines)) - yield def setUp(self) -> None: self.temp_file_dir = tempfile.TemporaryDirectory() @@ -81,53 +79,53 @@ def test_get_pkg_dirs(self) -> None: def test_typedpkg_stub_package(self) -> None: with self.virtualenv() as python_executable: - with self.install_package('typedpkg-stubs', python_executable): - check_mypy_run( - [self.tempfile], - python_executable, - self.msg_list, - ) + self.install_package('typedpkg-stubs', python_executable) + check_mypy_run( + [self.tempfile], + python_executable, + self.msg_list, + ) def test_typedpkg(self) -> None: with self.virtualenv() as python_executable: - with self.install_package('typedpkg', python_executable): - check_mypy_run( - [self.tempfile], - python_executable, - self.msg_tuple, - ) + self.install_package('typedpkg', python_executable) + check_mypy_run( + [self.tempfile], + python_executable, + self.msg_tuple, + ) def test_stub_and_typed_pkg(self) -> None: with self.virtualenv() as python_executable: - with self.install_package('typedpkg', python_executable): - with self.install_package('typedpkg-stubs', python_executable): - check_mypy_run( - [self.tempfile], - python_executable, - self.msg_list, - ) + self.install_package('typedpkg', python_executable) + self.install_package('typedpkg-stubs', python_executable) + check_mypy_run( + [self.tempfile], + python_executable, + self.msg_list, + ) def test_typedpkg_stubs_python2(self) -> None: python2 = try_find_python2_interpreter() if python2: with self.virtualenv(python2) as py2: - with self.install_package('typedpkg-stubs', py2): - check_mypy_run( - [self.tempfile], - py2, - self.msg_list, - ) + self.install_package('typedpkg-stubs', py2) + check_mypy_run( + [self.tempfile], + py2, + self.msg_list, + ) def test_typedpkg_python2(self) -> None: python2 = try_find_python2_interpreter() if python2: with self.virtualenv(python2) as py2: - with self.install_package('typedpkg', py2): - check_mypy_run( - [self.tempfile], - py2, - self.msg_tuple, - ) + self.install_package('typedpkg', py2) + check_mypy_run( + [self.tempfile], + py2, + self.msg_tuple, + ) if __name__ == '__main__':