diff --git a/.travis.yml b/.travis.yml index 0b60e361e889..c8ec76a98ace 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,9 +11,13 @@ python: install: - pip install -r test-requirements.txt - - python setup.py install script: + - python setup.py install + - mkdir elsewhere + - cd elsewhere + - mkdir tmp-test-dirs + - ln -s ../runtests.py - python runtests.py -v notifications: diff --git a/mypy/build.py b/mypy/build.py index e611da81235e..514aa4b3665f 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -95,7 +95,6 @@ def build(program_path: str, argument: str = None, program_text: Union[str, bytes] = None, alt_lib_path: str = None, - bin_dir: str = None, pyversion: Tuple[int, int] = defaults.PYTHON3_VERSION, custom_typing_module: str = None, report_dirs: Dict[str, str] = {}, @@ -116,8 +115,6 @@ def build(program_path: str, program_text: the main source file contents; if omitted, read from file alt_lib_dir: an additional directory for looking up library modules (takes precedence over other directories) - bin_dir: directory containing the mypy script, used for finding data - directories; if omitted, use '.' as the data directory pyversion: Python version (major, minor) custom_typing_module: if not None, use this module id as an alias for typing flags: list of build options (e.g. COMPILE_ONLY) @@ -125,7 +122,7 @@ def build(program_path: str, flags = flags or [] module = module or '__main__' - data_dir = default_data_dir(bin_dir) + data_dir = default_data_dir() # Determine the default module search path. lib_path = default_lib_path(data_dir, target, pyversion, python_path) @@ -176,26 +173,19 @@ def build(program_path: str, return result -def default_data_dir(bin_dir: str) -> str: - # TODO fix this logic - if not bin_dir: - # Default to directory containing this file's parent. - return os.path.dirname(os.path.dirname(__file__)) - base = os.path.basename(bin_dir) - dir = os.path.dirname(bin_dir) - if (sys.platform == 'win32' and base.lower() == 'mypy' - and not os.path.isdir(os.path.join(dir, 'stubs'))): - # Installed, on Windows. - return os.path.join(dir, 'Lib', 'mypy') - elif base == 'mypy': - # Assume that we have a repo check out or unpacked source tarball. - return os.path.dirname(bin_dir) - elif base == 'bin': - # Installed to somewhere (can be under /usr/local or anywhere). - return os.path.join(dir, 'lib', 'mypy') - elif base == 'python3': - # Assume we installed python3 with brew on os x - return os.path.join(os.path.dirname(dir), 'lib', 'mypy') +def is_installed() -> bool: + return 'site-packages' in __file__ or 'dist-packages' in __file__ + + +def default_data_dir() -> str: + if is_installed(): + # we are installed + rv = os.path.join(sys.prefix, 'lib', 'mypy') + else: + # we are from from a source checkout + rv = os.path.dirname(os.path.dirname(__file__)) + if os.path.isdir(os.path.join(rv, 'stubs')): + return rv else: # Don't know where to find the data files! raise RuntimeError("Broken installation: can't determine base dir") diff --git a/mypy/main.py b/mypy/main.py index feaa2fcc8f4a..0ca1e48a819e 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -29,11 +29,10 @@ def __init__(self) -> None: def main() -> None: - bin_dir = find_bin_directory() path, module, program_text, options = process_options(sys.argv[1:]) try: if options.target == build.TYPE_CHECK: - type_check_only(path, module, program_text, bin_dir, options) + type_check_only(path, module, program_text, options) else: raise RuntimeError('unsupported target %d' % options.target) except CompileError as e: @@ -42,21 +41,6 @@ def main() -> None: sys.exit(1) -def find_bin_directory() -> str: - """Find the directory that contains this script. - - This is used by build to find stubs and other data files. - """ - script = __file__ - # Follow up to 5 symbolic links (cap to avoid cycles). - for i in range(5): - if os.path.islink(script): - script = readlinkabs(script) - else: - break - return os.path.dirname(script) - - def readlinkabs(link: str) -> str: """Return an absolute path to symbolic link destination.""" # Adapted from code by Greg Smith. @@ -68,12 +52,11 @@ def readlinkabs(link: str) -> str: def type_check_only(path: str, module: str, program_text: str, - bin_dir: str, options: Options) -> None: + options: Options) -> None: # Type check the program and dependencies and translate to Python. build.build(path, module=module, program_text=program_text, - bin_dir=bin_dir, target=build.TYPE_CHECK, pyversion=options.pyversion, custom_typing_module=options.custom_typing_module, diff --git a/mypy/test/config.py b/mypy/test/config.py index 83962d6972f8..27e9b3e962b8 100644 --- a/mypy/test/config.py +++ b/mypy/test/config.py @@ -4,10 +4,8 @@ import typing -PREFIX = '' - # Location of test data files such as test case descriptions. -test_data_prefix = os.path.join(PREFIX, 'mypy', 'test', 'data') +test_data_prefix = os.path.join(os.path.dirname(__file__), 'data') assert os.path.isdir(test_data_prefix), \ 'Test data prefix ({}) not set correctly'.format(test_data_prefix) diff --git a/mypy/test/helpers.py b/mypy/test/helpers.py index b6b1e5a850cf..ddef334801a3 100644 --- a/mypy/test/helpers.py +++ b/mypy/test/helpers.py @@ -21,8 +21,6 @@ def assert_string_arrays_equal(expected: List[str], actual: List[str], Display any differences in a human-readable form. """ - actual = clean_up(actual) - if actual != expected: num_skip_start = num_skipped_prefix_lines(expected, actual) num_skip_end = num_skipped_suffix_lines(expected, actual) @@ -171,7 +169,6 @@ def assert_string_arrays_equal_wildcards(expected: List[str], msg: str) -> None: # Like above, but let a line with only '...' in expected match any number # of lines in actual. - actual = clean_up(actual) while actual != [] and actual[-1] == '': actual = actual[:-1] @@ -181,25 +178,6 @@ def assert_string_arrays_equal_wildcards(expected: List[str], assert_string_arrays_equal(expected, actual, msg) -def clean_up(a): - """Remove common directory prefix from all strings in a. - - This uses a naive string replace; it seems to work well enough. Also - remove trailing carriage returns. - """ - res = [] - for s in a: - prefix = config.PREFIX + os.sep - ss = s - for p in prefix, prefix.replace(os.sep, '/'): - if p != '/' and p != '//' and p != '\\' and p != '\\\\': - ss = ss.replace(p, '') - # Ignore spaces at end of line. - ss = re.sub(' +$', '', ss) - res.append(re.sub('\\r$', '', ss)) - return res - - def match_array(pattern: List[str], target: List[str]) -> List[str]: """Expand '...' wildcards in pattern by matching against target.""" diff --git a/mypy/test/testpythoneval.py b/mypy/test/testpythoneval.py index 07a4a0dc4f47..48f45b51c51b 100644 --- a/mypy/test/testpythoneval.py +++ b/mypy/test/testpythoneval.py @@ -17,6 +17,7 @@ import typing +from mypy.build import is_installed from mypy.myunit import Suite, SkipTestCaseException from mypy.test.config import test_data_prefix, test_temp_dir from mypy.test.data import parse_test_cases @@ -72,7 +73,7 @@ def test_python_evaluation(testcase): # Type check the program. # This uses the same PYTHONPATH as the current process. process = subprocess.Popen([python3_path, - os.path.join(testcase.old_cwd, 'scripts', 'mypy')] + '-m', 'mypy'] + args + [program], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -83,11 +84,14 @@ def test_python_evaluation(testcase): if not process.wait(): # Set up module path for the execution. # This needs the typing module but *not* the mypy module. - vers_dir = '2.7' if py2 else '3.2' - typing_path = os.path.join(testcase.old_cwd, 'lib-typing', vers_dir) - assert os.path.isdir(typing_path) - env = os.environ.copy() - env['PYTHONPATH'] = typing_path + if is_installed(): + env = None + else: + vers_dir = '2.7' if py2 else '3.2' + typing_path = os.path.join(testcase.old_cwd, 'lib-typing', vers_dir) + assert os.path.isdir(typing_path) + env = os.environ.copy() + env['PYTHONPATH'] = typing_path process = subprocess.Popen([interpreter, program], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -110,6 +114,10 @@ def try_find_python2_interpreter(): stderr=subprocess.STDOUT) stdout, stderr = process.communicate() if b'Python 2.7' in stdout: + if is_installed(): + print('WARNING: python2 interpreter found, but' + ' typing is not installed for python2') + return None return interpreter except OSError: pass diff --git a/runtests.py b/runtests.py index 462fa1491369..89a08e4c7f03 100755 --- a/runtests.py +++ b/runtests.py @@ -6,7 +6,16 @@ if True: # When this is run as a script, `typing` is not available yet. import sys - from os.path import join, isdir + from os.path import ( + basename, + dirname, + isabs, + isdir, + join, + realpath, + relpath, + splitext, + ) def get_versions(): # type: () -> typing.List[str] major = sys.version_info[0] @@ -31,6 +40,10 @@ def get_versions(): # type: () -> typing.List[str] import os +# Allow this to be symlinked to support running an installed version. +SOURCE_DIR = dirname(realpath(__file__)) + + # Ideally, all tests would be `discover`able so that they can be driven # (and parallelized) by an external test driver. @@ -45,7 +58,6 @@ def __init__(self, whitelist: List[str], blacklist: List[str], self.waiter = Waiter(verbosity=verbosity, xfail=xfail) self.versions = get_versions() self.cwd = os.getcwd() - self.mypy = os.path.join(self.cwd, 'scripts', 'mypy') self.env = dict(os.environ) def prepend_path(self, name: str, paths: List[str]) -> None: @@ -74,7 +86,7 @@ def add_mypy(self, name: str, *args: str, cwd: Optional[str] = None) -> None: if not self.allow(name): return largs = list(args) - largs[0:0] = [sys.executable, self.mypy] + largs[0:0] = ['mypy', '--use-python-path'] env = self.env self.waiter.add(LazySubprocess(name, largs, cwd=cwd, env=env)) @@ -96,7 +108,7 @@ def add_mypy_mod(self, name: str, *args: str, cwd: Optional[str] = None) -> None if not self.allow(name): return largs = list(args) - largs[0:0] = [sys.executable, self.mypy, '-m'] + largs[0:0] = ['mypy', '--use-python-path', '-m'] env = self.env self.waiter.add(LazySubprocess(name, largs, cwd=cwd, env=env)) @@ -118,7 +130,7 @@ def add_mypy_string(self, name: str, *args: str, cwd: Optional[str] = None) -> N if not self.allow(name): return largs = list(args) - largs[0:0] = [sys.executable, self.mypy, '-c'] + largs[0:0] = ['mypy', '--use-python-path', '-c'] env = self.env self.waiter.add(LazySubprocess(name, largs, cwd=cwd, env=env)) @@ -144,6 +156,19 @@ def add_python2(self, name: str, *args: str, cwd: Optional[str] = None) -> None: env = self.env self.waiter.add(LazySubprocess(name, largs, cwd=cwd, env=env)) + def add_myunit(self, name: str, *args: str, cwd: Optional[str] = None, + script: bool = True) -> None: + name = 'run %s' % name + if not self.allow(name): + return + largs = list(args) + if script: + largs[0:0] = ['myunit'] + else: + largs[0:0] = [sys.executable, '-m' 'mypy.myunit'] + env = self.env + self.waiter.add(LazySubprocess(name, largs, cwd=cwd, env=env)) + def add_flake8(self, name: str, file: str, cwd: Optional[str] = None) -> None: name = 'lint %s' % name if not self.allow(name): @@ -159,28 +184,30 @@ def list_tasks(self) -> None: def add_basic(driver: Driver) -> None: if False: - driver.add_mypy('file setup.py', 'setup.py') - driver.add_flake8('file setup.py', 'setup.py') - driver.add_mypy('file runtests.py', 'runtests.py') - driver.add_flake8('file runtests.py', 'runtests.py') - driver.add_mypy('legacy entry script', 'scripts/mypy') - driver.add_flake8('legacy entry script', 'scripts/mypy') - driver.add_mypy('legacy myunit script', 'scripts/myunit') - driver.add_flake8('legacy myunit script', 'scripts/myunit') + driver.add_mypy('file setup.py', join(SOURCE_DIR, 'setup.py')) + driver.add_flake8('file setup.py', join(SOURCE_DIR, 'setup.py')) + driver.add_mypy('file runtests.py', join(SOURCE_DIR, 'runtests.py')) + driver.add_flake8('file runtests.py', join(SOURCE_DIR, 'runtests.py')) + driver.add_mypy('legacy entry script', join(SOURCE_DIR, 'scripts/mypy')) + driver.add_flake8('legacy entry script', join(SOURCE_DIR, 'scripts/mypy')) + driver.add_mypy('legacy myunit script', join(SOURCE_DIR, 'scripts/myunit')) + driver.add_flake8('legacy myunit script', join(SOURCE_DIR, 'scripts/myunit')) driver.add_mypy_mod('entry mod mypy', 'mypy') driver.add_mypy_mod('entry mod mypy.stubgen', 'mypy.stubgen') driver.add_mypy_mod('entry mod mypy.myunit', 'mypy.myunit') def find_files(base: str, prefix: str = '', suffix: str = '') -> List[str]: + base = join(SOURCE_DIR, base) return [join(root, f) for root, dirs, files in os.walk(base) for f in files if f.startswith(prefix) and f.endswith(suffix)] -def file_to_module(file: str) -> str: - rv = os.path.splitext(file)[0].replace(os.sep, '.') +def file_to_module(file: str, ignore: str = '') -> str: + file = relpath(file, join(SOURCE_DIR, ignore)) + rv = splitext(file)[0].replace(os.sep, '.') if rv.endswith('.__init__'): rv = rv[:-len('.__init__')] return rv @@ -210,9 +237,12 @@ def add_myunit(driver: Driver) -> None: elif mod == 'mypy.test.testpythoneval': # Run Python evaluation integration tests separetely since they are much slower # than proper unit tests. - driver.add_python_mod('eval-test %s' % mod, 'mypy.myunit', '-m', mod, *driver.arglist) + + # testpythoneval requires lib-typing/2.7 to be available. Ick! + driver.add_myunit('eval-test %s' % mod, '-m', mod, *driver.arglist, + cwd=SOURCE_DIR, script=False) else: - driver.add_python_mod('unit-test %s' % mod, 'mypy.myunit', '-m', mod, *driver.arglist) + driver.add_myunit('unit-test %s' % mod, '-m', mod, *driver.arglist) def add_stubs(driver: Driver) -> None: @@ -223,7 +253,7 @@ def add_stubs(driver: Driver) -> None: for pfx in ['', 'third-party-']: stubdir = join('stubs', pfx + version) for f in find_files(stubdir, suffix='.pyi'): - module = file_to_module(f[len(stubdir) + 1:]) + module = file_to_module(f, stubdir) if module not in seen: seen.add(module) driver.add_mypy_string( @@ -234,21 +264,21 @@ def add_stubs(driver: Driver) -> None: def add_libpython(driver: Driver) -> None: seen = set() # type: Set[str] for version in driver.versions: - libpython_dir = join(driver.cwd, 'lib-python', version) + libpython_dir = join('lib-python', version) for f in find_files(libpython_dir, prefix='test_', suffix='.py'): - module = file_to_module(f[len(libpython_dir) + 1:]) + module = file_to_module(f, libpython_dir) if module not in seen: seen.add(module) driver.add_mypy_mod( 'libpython (%s) module %s' % (version, module), module, - cwd=libpython_dir) + cwd=join(SOURCE_DIR, libpython_dir)) def add_samples(driver: Driver) -> None: for f in find_files('samples', suffix='.py'): if 'codec' in f: - cwd, bf = os.path.dirname(f), os.path.basename(f) + cwd, bf = dirname(f), basename(f) bf = bf[:-len('.py')] driver.add_mypy_string('codec file %s' % f, 'import mypy.codec.register, %s' % bf, @@ -276,7 +306,7 @@ def sanity() -> None: return failed = False for p in paths.split(os.pathsep): - if not os.path.isabs(p): + if not isabs(p): print('Relative PYTHONPATH entry %r' % p) failed = True if failed: @@ -328,7 +358,9 @@ def main() -> None: whitelist.append('') driver = Driver(whitelist=whitelist, blacklist=blacklist, arglist=arglist, - verbosity=verbosity, xfail=[]) + verbosity=verbosity, xfail=[ + 'run2 unittest mypy.codec.test.test_function_translation', + ]) driver.prepend_path('PATH', [join(driver.cwd, 'scripts')]) driver.prepend_path('MYPYPATH', [driver.cwd]) driver.prepend_path('PYTHONPATH', [driver.cwd]) diff --git a/setup.py b/setup.py index 0a0caa15075a..0fecc9c8a593 100644 --- a/setup.py +++ b/setup.py @@ -66,6 +66,22 @@ def find_data_files(base, globs): 'Topic :: Software Development', ] +# TODO recurse automatically? +# If we switch to setuptools, use find_packages? +packages = [ + 'mypy', 'mypy.test', + 'mypy.myunit', + 'mypy.codec', 'mypy.codec.test', +] + +package_data = { + 'mypy.test': [ + 'data/*.test', + 'data/fixtures/*.py', + 'data/lib-stub/*.py', + ], +} + setup(name='mypy-lang', version=version, description=description, @@ -77,8 +93,9 @@ def find_data_files(base, globs): platforms=['POSIX'], package_dir={'': 'lib-typing/3.2', 'mypy': 'mypy'}, py_modules=['typing'], - packages=['mypy'], - scripts=['scripts/mypy'], + packages=packages, + scripts=['scripts/mypy', 'scripts/myunit'], data_files=data_files, + package_data=package_data, classifiers=classifiers, )