Externalize flake8 linting, introduce additional checks (#2638)

This change is a step towards removing `runtests.py` (see #1673).

The exclusion list in the flake8 configuration in `setup.cfg` has been updated
to enable running the linter from the root of the project by simply invoking
`flake8`.  This enables it to leverage its own file discovery and its own
multiprocessing queue without excessive subprocessing for linting every file.
This gives a minor speed up in local test runs. Before:

  total time in lint: 130.914682

After:

  total time in lint: 20.379915

There's an additional speedup on Travis because linting is now only performed
on Python 3.6.

More importantly, this means flake8 is now running over all files unless
explicitly excluded in `setup.cfg`. This will help avoiding unintentional
omissions in the future (see comments on #2637).

Note: running `flake8` as a single lazy subprocess in `runtests.py` doesn't
sacrifice any parallelism because the linter has its own process pool.

Minimal whitespace changes were required to `mypy_extensions.py` but in return
flake8 will check it now exactly like it checks the rest of the `mypy/*`
codebase.  Those are also done on #2637 but that hasn't landed yet.

Finally, flake8-bugbear and flake8-pyi were added to test requirements to make
the linter configuration consistent with typeshed.  I hope the additional
checks will speed up future pull requests by automating bigger parts of the
code review.  The pyi plugin enables forward reference support when linting
.pyi files.  That means it's now possible to run `flake8` inside the typeshed
submodule or on arbitrary .pyi files during development (which your editor
could do for you), for example on fixtures.  See discussion on #2629 on checks
that are disabled and why.

Author: ambv

.travis.yml

@@ -5,7 +5,8 @@ python:
   - "3.4"
   # Specifically request 3.5.1 because we need to be compatible with that.
   - "3.5.1"
-  - "3.6-dev"
+  - "3.6"
+  - "3.7-dev"
   # Pypy build is disabled because it doubles the travis build time, and it rarely fails
   # unless one one of the other builds fails.
   # - "pypy3"
@@ -15,4 +16,5 @@ install:
   - python setup.py install
 
 script:
-  - python runtests.py
+  - python runtests.py -x lint
+  - if [[ $TRAVIS_PYTHON_VERSION == '3.6' ]]; then flake8; fi

extensions/mypy_extensions.py

@@ -22,9 +22,11 @@ def _check_fails(cls, other):
         pass
     return False
 
+
 def _dict_new(cls, *args, **kwargs):
     return dict(*args, **kwargs)
 
+
 def _typeddict_new(cls, _typename, _fields=None, **kwargs):
     if _fields is None:
         _fields = kwargs
@@ -33,6 +35,7 @@ def _typeddict_new(cls, _typename, _fields=None, **kwargs):
                         " but not both")
     return _TypedDictMeta(_typename, (), {'__annotations__': dict(_fields)})
 
+
 class _TypedDictMeta(type):
     def __new__(cls, name, bases, ns):
         # Create new typed dict class object.

runtests.py

@@ -151,11 +151,11 @@ def add_python2(self, name: str, *args: str, cwd: Optional[str] = None) -> None:
         env = self.env
         self.waiter.add(LazySubprocess(name, largs, cwd=cwd, env=env))
 
-    def add_flake8(self, name: str, file: str, cwd: Optional[str] = None) -> None:
-        name = 'lint %s' % name
+    def add_flake8(self, cwd: Optional[str] = None) -> None:
+        name = 'lint'
         if not self.allow(name):
             return
-        largs = ['flake8', file]
+        largs = ['flake8', '-j{}'.format(self.waiter.limit)]
         env = self.env
         self.waiter.add(LazySubprocess(name, largs, cwd=cwd, env=env))
 
@@ -167,13 +167,9 @@ def list_tasks(self) -> None:
 def add_basic(driver: Driver) -> None:
     if False:
         driver.add_mypy('file setup.py', 'setup.py')
-    driver.add_flake8('file setup.py', 'setup.py')
     driver.add_mypy('file runtests.py', 'runtests.py')
-    driver.add_flake8('file runtests.py', 'runtests.py')
     driver.add_mypy('legacy entry script', 'scripts/mypy')
-    driver.add_flake8('legacy entry script', 'scripts/mypy')
     driver.add_mypy('legacy myunit script', 'scripts/myunit')
-    driver.add_flake8('legacy myunit script', 'scripts/myunit')
     # needs typed_ast installed:
     driver.add_mypy('fast-parse', '--fast-parse', 'test-data/samples/hello.py')
 
@@ -206,7 +202,6 @@ def add_imports(driver: Driver) -> None:
         mod = file_to_module(f)
         if not mod.endswith('.__main__'):
             driver.add_python_string('import %s' % mod, 'import %s' % mod)
-        driver.add_flake8('module %s' % mod, f)
 
 
 PYTEST_FILES = ['mypy/test/{}.py'.format(name) for name in [
@@ -411,6 +406,7 @@ def main() -> None:
     add_stubs(driver)
     add_stdlibsamples(driver)
     add_samples(driver)
+    driver.add_flake8()
 
     if list_only:
         driver.list_tasks()

setup.cfg

@@ -1,7 +1,24 @@
 [flake8]
 max-line-length = 99
-exclude = mypy/codec/*
-# Thing to ignore:
+# typeshed and unit test fixtures have .pyi-specific flake8 configuration
+exclude =
+  # Sphinx configuration is irrelevant
+  docs/source/conf.py,
+  # external library with incompatible style
+  lib-typing/*,
+  # conflicting styles
+  misc/*,
+  # external library with incompatible style
+  pinfer/*,
+  # conflicting styles
+  scripts/*,
+  # tests have more relaxed styling requirements
+  # fixtures have their own .pyi-specific configuration
+  test-data/*,
+  # typeshed has its own .pyi-specific configuration
+  typeshed/*
+
+# Things to ignore:
 #   E251: spaces around default arg value (against our style)
 #   E128: continuation line under-indented (too noisy)
 #   F401: unused identifiers (useless, as it doesn't see inside # type: comments)
@@ -10,8 +27,10 @@ exclude = mypy/codec/*
 #   W503: line break before binary operator
 #   E704: multiple statements on one line (def)
 #   E402: module level import not at top of file
-#   B???: flake8-bugbear errors
-ignore = E251,E128,F401,W601,E701,W503,E704,E402,B
+#   B3??: Python 3 compatibility warnings
+#   B006: use of mutable defaults in function signatures
+#   B007: Loop control variable not used within the loop body.
+ignore = E251,E128,F401,W601,E701,W503,E704,E402,B3,B006,B007
 
 [coverage:run]
 branch = true

test-data/.flake8

@@ -0,0 +1 @@
+../typeshed/.flake8

test-requirements.txt

@@ -1,4 +1,6 @@
 flake8
+flake8-bugbear; python_version >= '3.5'
+flake8-pyi; python_version >= '3.5'
 lxml
 typed-ast>=0.6.1; sys_platform != 'win32'
 pytest>=2.8