Skip to content

Commit 595545f

Browse files
emmatypingemmatyping
authored
Make pep561 tests parrallel and sandboxed (#5060)
Each test is run in a seperate virtualenv which is deleted after the tests. This allows for scaling and faster test runs.
1 parent 1b21e8c commit 595545f

File tree

3 files changed

+82
-79
lines changed

3 files changed

+82
-79
lines changed

‎mypy/sitepkgs.py

+2-1
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,8 @@
77
possible.
88
"""
99

10-
10+
import sys
11+
sys.path = sys.path[1:] # we don't want to pick up mypy.types
1112
from distutils.sysconfig import get_python_lib
1213
import site
1314
MYPY = False

‎mypy/test/testpep561.py

+79-78
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,11 @@
11
from contextlib import contextmanager
22
import os
3+
import random
34
import shutil
5+
import string
46
import sys
5-
from typing import Iterator, List
7+
import tempfile
8+
from typing import Iterator, List, Generator
69
from unittest import TestCase, main
710

811
import mypy.api
@@ -19,113 +22,111 @@
1922

2023

2124
def check_mypy_run(cmd_line: List[str],
22-
expected_out: str,
25+
python_executable: str = sys.executable,
26+
expected_out: str = '',
2327
expected_err: str = '',
2428
expected_returncode: int = 1) -> None:
2529
"""Helper to run mypy and check the output."""
30+
if python_executable != sys.executable:
31+
cmd_line.append('--python-executable={}'.format(python_executable))
2632
out, err, returncode = mypy.api.run(cmd_line)
2733
assert out == expected_out, err
2834
assert err == expected_err, out
2935
assert returncode == expected_returncode, returncode
3036

3137

32-
def is_in_venv() -> bool:
33-
"""Returns whether we are running inside a venv.
34-
35-
Based on https://stackoverflow.com/a/42580137.
36-
37-
"""
38-
if hasattr(sys, 'real_prefix'):
39-
return True
40-
else:
41-
return hasattr(sys, 'base_prefix') and sys.base_prefix != sys.prefix
42-
43-
4438
class TestPEP561(TestCase):
39+
4540
@contextmanager
41+
def virtualenv(self, python_executable: str = sys.executable) -> Generator[str, None, None]:
42+
"""Context manager that creates a virtualenv in a temporary directory
43+
44+
returns the path to the created Python executable"""
45+
with tempfile.TemporaryDirectory() as venv_dir:
46+
run_command([sys.executable, '-m', 'virtualenv', '-p{}'.format(python_executable),
47+
venv_dir], cwd=os.getcwd())
48+
if sys.platform == 'win32':
49+
yield os.path.abspath(os.path.join(venv_dir, 'Scripts', 'python'))
50+
else:
51+
yield os.path.abspath(os.path.join(venv_dir, 'bin', 'python'))
52+
4653
def install_package(self, pkg: str,
47-
python_executable: str = sys.executable) -> Iterator[None]:
54+
python_executable: str = sys.executable) -> None:
4855
"""Context manager to temporarily install a package from test-data/packages/pkg/"""
4956
working_dir = os.path.join(package_path, pkg)
5057
install_cmd = [python_executable, '-m', 'pip', 'install', '.']
51-
# if we aren't in a virtualenv, install in the
52-
# user package directory so we don't need sudo
53-
if not is_in_venv() or python_executable != sys.executable:
54-
install_cmd.append('--user')
5558
returncode, lines = run_command(install_cmd, cwd=working_dir)
5659
if returncode != 0:
5760
self.fail('\n'.join(lines))
58-
try:
59-
yield
60-
finally:
61-
returncode, lines = run_command([python_executable, '-m', 'pip', 'uninstall',
62-
'-y', pkg], cwd=package_path)
63-
if returncode != 0:
64-
self.fail('\n'.join(lines))
61+
62+
def setUp(self) -> None:
63+
self.temp_file_dir = tempfile.TemporaryDirectory()
64+
self.tempfile = os.path.join(self.temp_file_dir.name, 'simple.py')
65+
with open(self.tempfile, 'w+') as file:
66+
file.write(SIMPLE_PROGRAM)
67+
self.msg_list = \
68+
"{}:4: error: Revealed type is 'builtins.list[builtins.str]'\n".format(self.tempfile)
69+
self.msg_tuple = \
70+
"{}:4: error: Revealed type is 'builtins.tuple[builtins.str]'\n".format(self.tempfile)
71+
72+
def tearDown(self) -> None:
73+
self.temp_file_dir.cleanup()
6574

6675
def test_get_pkg_dirs(self) -> None:
6776
"""Check that get_package_dirs works."""
6877
dirs = _get_site_packages_dirs(sys.executable)
6978
assert dirs
7079

71-
def test_typed_pkg(self) -> None:
72-
"""Tests type checking based on installed packages.
73-
74-
This test CANNOT be split up, concurrency means that simultaneously
75-
installing/uninstalling will break tests.
76-
"""
77-
test_file = 'simple.py'
78-
if not os.path.isdir('test-packages-data'):
79-
os.mkdir('test-packages-data')
80-
old_cwd = os.getcwd()
81-
os.chdir('test-packages-data')
82-
with open(test_file, 'w') as f:
83-
f.write(SIMPLE_PROGRAM)
84-
try:
85-
with self.install_package('typedpkg-stubs'):
80+
def test_typedpkg_stub_package(self) -> None:
81+
with self.virtualenv() as python_executable:
82+
self.install_package('typedpkg-stubs', python_executable)
83+
check_mypy_run(
84+
[self.tempfile],
85+
python_executable,
86+
self.msg_list,
87+
)
88+
89+
def test_typedpkg(self) -> None:
90+
with self.virtualenv() as python_executable:
91+
self.install_package('typedpkg', python_executable)
92+
check_mypy_run(
93+
[self.tempfile],
94+
python_executable,
95+
self.msg_tuple,
96+
)
97+
98+
def test_stub_and_typed_pkg(self) -> None:
99+
with self.virtualenv() as python_executable:
100+
self.install_package('typedpkg', python_executable)
101+
self.install_package('typedpkg-stubs', python_executable)
102+
check_mypy_run(
103+
[self.tempfile],
104+
python_executable,
105+
self.msg_list,
106+
)
107+
108+
def test_typedpkg_stubs_python2(self) -> None:
109+
python2 = try_find_python2_interpreter()
110+
if python2:
111+
with self.virtualenv(python2) as py2:
112+
self.install_package('typedpkg-stubs', py2)
86113
check_mypy_run(
87-
[test_file],
88-
"simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n"
114+
[self.tempfile],
115+
py2,
116+
self.msg_list,
89117
)
90118

91-
# The Python 2 tests are intentionally placed after a Python 3 test to check
92-
# the package_dir_cache is behaving correctly.
93-
python2 = try_find_python2_interpreter()
94-
if python2:
95-
with self.install_package('typedpkg-stubs', python2):
96-
check_mypy_run(
97-
['--python-executable={}'.format(python2), test_file],
98-
"simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n"
99-
)
100-
with self.install_package('typedpkg', python2):
101-
check_mypy_run(
102-
['--python-executable={}'.format(python2), 'simple.py'],
103-
"simple.py:4: error: Revealed type is 'builtins.tuple[builtins.str]'\n"
104-
)
105-
106-
with self.install_package('typedpkg', python2):
107-
with self.install_package('typedpkg-stubs', python2):
108-
check_mypy_run(
109-
['--python-executable={}'.format(python2), test_file],
110-
"simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n"
111-
)
112-
113-
with self.install_package('typedpkg'):
119+
def test_typedpkg_python2(self) -> None:
120+
python2 = try_find_python2_interpreter()
121+
if python2:
122+
with self.virtualenv(python2) as py2:
123+
self.install_package('typedpkg', py2)
114124
check_mypy_run(
115-
[test_file],
116-
"simple.py:4: error: Revealed type is 'builtins.tuple[builtins.str]'\n"
125+
[self.tempfile],
126+
py2,
127+
self.msg_tuple,
117128
)
118129

119-
with self.install_package('typedpkg'):
120-
with self.install_package('typedpkg-stubs'):
121-
check_mypy_run(
122-
[test_file],
123-
"simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n"
124-
)
125-
finally:
126-
os.chdir(old_cwd)
127-
shutil.rmtree('test-packages-data')
128-
129130

130131
if __name__ == '__main__':
131132
main()

‎test-requirements.txt

+1
Original file line numberDiff line numberDiff line change
@@ -9,3 +9,4 @@ pytest-cov>=2.4.0
99
typed-ast>=1.1.0,<1.2.0
1010
typing>=3.5.2; python_version < '3.5'
1111
py>=1.5.2
12+
virtualenv

0 commit comments

Comments
 (0)