Merge branch 'master' into chore/1954/Add_performance_regressions

Author: Pierre-Sassoulas

CONTRIBUTORS.txt

@@ -379,3 +379,5 @@ contributors:
 * Matthew Beckers (mattlbeck): contributor
 
 * Yang Yang: contributor
+
+* Andrew J. Simmons (anjsimmo): contributor

ChangeLog

@@ -7,6 +7,15 @@ What's New in Pylint 2.5.0?
 
 Release date: TBA
 
+* Fix a false negative for ``undefined-variable`` when using class attribute in comprehension.
+
+  Close #3494
+
+* Fix a false positive for ``undefined-variable`` when using class attribute in decorator or as type hint.
+
+  Close #511
+  Close #1976
+
 * Remove HTML quoting of messages in JSON output.
 
   Close #2769

Makefile

@@ -50,7 +50,7 @@ $(PKG_SDIST):
 
 lint: $(PIP)
 	$(PIP) install .
-	$(PYVE)/bin/pylint lint.py || true  # for now ignore errors
+	$(PYVE)/bin/pylint lint/__init__.py || true  # for now ignore errors
 
 clean:
 	rm -rf $(PYVE)

pylint/__init__.py

@@ -15,7 +15,6 @@
 
 
 def run_pylint():
-    """run pylint"""
     from pylint.lint import Run as PylintRun
 
     try:
@@ -25,7 +24,6 @@ def run_pylint():
 
 
 def run_epylint():
-    """run pylint"""
     from pylint.epylint import Run as EpylintRun
 
     EpylintRun()

pylint/__main__.py

@@ -15,5 +15,4 @@
 if sys.path[0] == "" or sys.path[0] == os.getcwd():
     sys.path.pop(0)
 
-
 pylint.run_pylint()

pylint/checkers/format.py

@@ -1291,7 +1291,7 @@ def is_line_length_check_activated(pylint_pattern_match_object) -> bool:
                 if pragma.action == "disable" and "line-too-long" in pragma.messages:
                     return False
         except PragmaParserError:
-            # Printing usefull informations dealing with this error is done in lint.py
+            # Printing useful information dealing with this error is done in the lint package
             pass
         return True
 

pylint/checkers/misc.py

@@ -162,7 +162,7 @@ def process_tokens(self, tokens):
                         ):
                             values.extend(pragma_repr.messages)
                     except PragmaParserError:
-                        # Printing usefull informations dealing with this error is done in lint.py
+                        # Printing useful information dealing with this error is done in the lint package
                         pass
                     values = [_val.upper() for _val in values]
                     if set(values) & set(self.config.notes):

pylint/checkers/variables.py

@@ -973,13 +973,11 @@ def visit_name(self, node):
             # if the current scope is a class scope but it's not the inner
             # scope, ignore it. This prevents to access this scope instead of
             # the globals one in function members when there are some common
-            # names. The only exception is when the starting scope is a
-            # comprehension and its direct outer scope is a class
-            if (
-                current_consumer.scope_type == "class"
-                and i != start_index
-                and not (base_scope_type == "comprehension" and i == start_index - 1)
-            ):
+            # names.
+            if current_consumer.scope_type == "class" and i != start_index:
+                # The only exceptions are: when the variable forms an iter within a
+                # comprehension scope; and/or when used as a default, decorator,
+                # or annotation within a function.
                 if self._ignore_class_scope(node):
                     continue
 
@@ -1249,16 +1247,53 @@ def _allow_global_unused_variables(self):
 
     @staticmethod
     def _defined_in_function_definition(node, frame):
-        in_annotation_or_default = False
+        in_annotation_or_default_or_decorator = False
         if isinstance(frame, astroid.FunctionDef) and node.statement() is frame:
-            in_annotation_or_default = (
-                node in frame.args.annotations
-                or node in frame.args.posonlyargs_annotations
-                or node in frame.args.kwonlyargs_annotations
-                or node is frame.args.varargannotation
-                or node is frame.args.kwargannotation
-            ) or frame.args.parent_of(node)
-        return in_annotation_or_default
+            in_annotation_or_default_or_decorator = (
+                (
+                    node in frame.args.annotations
+                    or node in frame.args.posonlyargs_annotations
+                    or node in frame.args.kwonlyargs_annotations
+                    or node is frame.args.varargannotation
+                    or node is frame.args.kwargannotation
+                )
+                or frame.args.parent_of(node)
+                or (frame.decorators and frame.decorators.parent_of(node))
+                or (
+                    frame.returns
+                    and (node is frame.returns or frame.returns.parent_of(node))
+                )
+            )
+        return in_annotation_or_default_or_decorator
+
+    @staticmethod
+    def _in_lambda_or_comprehension_body(
+        node: astroid.node_classes.NodeNG, frame: astroid.node_classes.NodeNG
+    ) -> bool:
+        """return True if node within a lambda/comprehension body (or similar) and thus should not have access to class attributes in frame"""
+        child = node
+        parent = node.parent
+        while parent is not None:
+            if parent is frame:
+                return False
+            if isinstance(parent, astroid.Lambda) and not child is parent.args:
+                # Body of lambda should not have access to class attributes.
+                return True
+            if (
+                isinstance(parent, astroid.node_classes.Comprehension)
+                and not child is parent.iter
+            ):
+                # Only iter of list/set/dict/generator comprehension should have access.
+                return True
+            if isinstance(parent, astroid.scoped_nodes.ComprehensionScope) and not (
+                parent.generators and child is parent.generators[0]
+            ):
+                # Body of list/set/dict/generator comprehension should not have access to class attributes.
+                # Furthermore, only the first generator (if multiple) in comprehension should have access.
+                return True
+            child = parent
+            parent = parent.parent
+        return False
 
     @staticmethod
     def _is_variable_violation(
@@ -1419,13 +1454,19 @@ def _ignore_class_scope(self, node):
 
         name = node.name
         frame = node.statement().scope()
-        in_annotation_or_default = self._defined_in_function_definition(node, frame)
-        if in_annotation_or_default:
+        in_annotation_or_default_or_decorator = self._defined_in_function_definition(
+            node, frame
+        )
+        if in_annotation_or_default_or_decorator:
             frame_locals = frame.parent.scope().locals
         else:
             frame_locals = frame.locals
         return not (
-            (isinstance(frame, astroid.ClassDef) or in_annotation_or_default)
+            (
+                isinstance(frame, astroid.ClassDef)
+                or in_annotation_or_default_or_decorator
+            )
+            and not self._in_lambda_or_comprehension_body(node, frame)
             and name in frame_locals
         )
 

pylint/lint/__init__.py

@@ -0,0 +1,76 @@
+# Copyright (c) 2006-2015 LOGILAB S.A. (Paris, FRANCE) <[email protected]>
+# Copyright (c) 2008 Fabrice Douchant <[email protected]>
+# Copyright (c) 2009 Vincent
+# Copyright (c) 2009 Mads Kiilerich <[email protected]>
+# Copyright (c) 2011-2014 Google, Inc.
+# Copyright (c) 2012 David Pursehouse <[email protected]>
+# Copyright (c) 2012 Kevin Jing Qiu <[email protected]>
+# Copyright (c) 2012 FELD Boris <[email protected]>
+# Copyright (c) 2012 JT Olds <[email protected]>
+# Copyright (c) 2014-2018 Claudiu Popa <[email protected]>
+# Copyright (c) 2014-2015 Michal Nowikowski <[email protected]>
+# Copyright (c) 2014 Brett Cannon <[email protected]>
+# Copyright (c) 2014 Alexandru Coman <[email protected]>
+# Copyright (c) 2014 Daniel Harding <[email protected]>
+# Copyright (c) 2014 Arun Persaud <[email protected]>
+# Copyright (c) 2014 Dan Goldsmith <[email protected]>
+# Copyright (c) 2015-2016 Florian Bruhin <[email protected]>
+# Copyright (c) 2015 Aru Sahni <[email protected]>
+# Copyright (c) 2015 Steven Myint <[email protected]>
+# Copyright (c) 2015 Simu Toni <[email protected]>
+# Copyright (c) 2015 Mihai Balint <[email protected]>
+# Copyright (c) 2015 Ionel Cristian Maries <[email protected]>
+# Copyright (c) 2016-2017 Łukasz Rogalski <[email protected]>
+# Copyright (c) 2016 Glenn Matthews <[email protected]>
+# Copyright (c) 2016 Alan Evangelista <[email protected]>
+# Copyright (c) 2017-2018 Ville Skyttä <[email protected]>
+# Copyright (c) 2017-2018 hippo91 <[email protected]>
+# Copyright (c) 2017 Daniel Miller <[email protected]>
+# Copyright (c) 2017 Roman Ivanov <[email protected]>
+# Copyright (c) 2017 Ned Batchelder <[email protected]>
+# Copyright (c) 2018 Randall Leeds <[email protected]>
+# Copyright (c) 2018 Mike Frysinger <[email protected]>
+# Copyright (c) 2018 ssolanki <[email protected]>
+# Copyright (c) 2018 Ville Skyttä <[email protected]>
+# Copyright (c) 2018 Sushobhit <[email protected]>
+# Copyright (c) 2018 Anthony Sottile <[email protected]>
+# Copyright (c) 2018 Jason Owen <[email protected]>
+# Copyright (c) 2018 Gary Tyler McLeod <[email protected]>
+# Copyright (c) 2018 Yuval Langer <[email protected]>
+# Copyright (c) 2018 Nick Drozd <[email protected]>
+# Copyright (c) 2018 kapsh <[email protected]>
+
+# Licensed under the GPL: https://www.gnu.org/licenses/old-licenses/gpl-2.0.html
+# For details: https://github.com/PyCQA/pylint/blob/master/COPYING
+
+""" pylint [options] modules_or_packages
+
+  Check that module(s) satisfy a coding standard (and more !).
+
+    pylint --help
+
+  Display this help message and exit.
+
+    pylint --help-msg <msg-id>[,<msg-id>]
+
+  Display help messages about given message identifiers and exit.
+"""
+import sys
+
+from pylint.lint.check_parallel import check_parallel
+from pylint.lint.pylinter import PyLinter
+from pylint.lint.report_functions import (
+    report_messages_by_module_stats,
+    report_messages_stats,
+    report_total_messages_stats,
+)
+from pylint.lint.run import Run
+from pylint.lint.utils import (
+    ArgumentPreprocessingError,
+    _patch_sys_path,
+    fix_import_path,
+    preprocess_options,
+)
+
+if __name__ == "__main__":
+    Run(sys.argv[1:])

pylint/lint/check_parallel.py

@@ -0,0 +1,113 @@
+import collections
+import functools
+
+from pylint import reporters
+from pylint.lint.utils import _patch_sys_path
+from pylint.message import Message
+
+try:
+    import multiprocessing
+except ImportError:
+    multiprocessing = None  # type: ignore
+
+# PyLinter object used by worker processes when checking files using multiprocessing
+# should only be used by the worker processes
+_worker_linter = None
+
+
+def _get_new_args(message):
+    location = (
+        message.abspath,
+        message.path,
+        message.module,
+        message.obj,
+        message.line,
+        message.column,
+    )
+    return (message.msg_id, message.symbol, location, message.msg, message.confidence)
+
+
+def _merge_stats(stats):
+    merged = {}
+    by_msg = collections.Counter()
+    for stat in stats:
+        message_stats = stat.pop("by_msg", {})
+        by_msg.update(message_stats)
+
+        for key, item in stat.items():
+            if key not in merged:
+                merged[key] = item
+            elif isinstance(item, dict):
+                merged[key].update(item)
+            else:
+                merged[key] = merged[key] + item
+
+    merged["by_msg"] = by_msg
+    return merged
+
+
+def _worker_initialize(linter, arguments=None):
+    global _worker_linter  # pylint: disable=global-statement
+    _worker_linter = linter
+
+    # On the worker process side the messages are just collected and passed back to
+    # parent process as _worker_check_file function's return value
+    _worker_linter.set_reporter(reporters.CollectingReporter())
+    _worker_linter.open()
+
+    # Patch sys.path so that each argument is importable just like in single job mode
+    _patch_sys_path(arguments or ())
+
+
+def _worker_check_single_file(file_item):
+    name, filepath, modname = file_item
+
+    _worker_linter.open()
+    _worker_linter.check_single_file(name, filepath, modname)
+
+    msgs = [_get_new_args(m) for m in _worker_linter.reporter.messages]
+    return (
+        _worker_linter.current_name,
+        msgs,
+        _worker_linter.stats,
+        _worker_linter.msg_status,
+    )
+
+
+def check_parallel(linter, jobs, files, arguments=None):
+    """Use the given linter to lint the files with given amount of workers (jobs)"""
+    # The reporter does not need to be passed to worker processess, i.e. the reporter does
+    # not need to be pickleable
+    original_reporter = linter.reporter
+    linter.reporter = None
+
+    # The linter is inherited by all the pool's workers, i.e. the linter
+    # is identical to the linter object here. This is requred so that
+    # a custom PyLinter object can be used.
+    initializer = functools.partial(_worker_initialize, arguments=arguments)
+    with multiprocessing.Pool(jobs, initializer=initializer, initargs=[linter]) as pool:
+        # ..and now when the workers have inherited the linter, the actual reporter
+        # can be set back here on the parent process so that results get stored into
+        # correct reporter
+        linter.set_reporter(original_reporter)
+        linter.open()
+
+        all_stats = []
+
+        for module, messages, stats, msg_status in pool.imap_unordered(
+            _worker_check_single_file, files
+        ):
+            linter.set_current_module(module)
+            for msg in messages:
+                msg = Message(*msg)
+                linter.reporter.handle_message(msg)
+
+            all_stats.append(stats)
+            linter.msg_status |= msg_status
+
+    linter.stats = _merge_stats(all_stats)
+
+    # Insert stats data to local checkers.
+    for checker in linter.get_checkers():
+        if checker is not linter:
+            checker.stats = linter.stats