From b94e281401f7b04222659dbf7726936a67345ec6 Mon Sep 17 00:00:00 2001 From: Cheryl Sabella Date: Fri, 30 Jun 2017 06:41:29 -0400 Subject: [PATCH 1/9] bpo-6691: Support for nested classes and functions in pyclbr --- Doc/library/pyclbr.rst | 81 +++++++++++------------ Lib/pyclbr.py | 143 +++++++++++++++++++++++++++------------- Lib/test/test_pyclbr.py | 119 +++++++++++++++++++++++++++++++++ 3 files changed, 255 insertions(+), 88 deletions(-) diff --git a/Doc/library/pyclbr.rst b/Doc/library/pyclbr.rst index 32842717bc6fad..331788e6b1cea9 100644 --- a/Doc/library/pyclbr.rst +++ b/Doc/library/pyclbr.rst @@ -39,78 +39,77 @@ modules. path. -.. _pyclbr-class-objects: +.. _pyclbr-object-objects: -Class Objects -------------- - -The :class:`Class` objects used as values in the dictionary returned by -:func:`readmodule` and :func:`readmodule_ex` provide the following data -attributes: - - -.. attribute:: Class.module +Object Objects +-------------- +The class :class:`Object` is the base class for the classes +:class:`Class` and :class:`Function`. It provides the following +data members: - The name of the module defining the class described by the class descriptor. +.. attribute:: Object.module -.. attribute:: Class.name + The name of the module defining the object described. - The name of the class. +.. attribute:: Object.name -.. attribute:: Class.super + The name of the object. - A list of :class:`Class` objects which describe the immediate base - classes of the class being described. Classes which are named as - superclasses but which are not discoverable by :func:`readmodule` are - listed as a string with the class name instead of as :class:`Class` - objects. +.. attribute:: Object.file -.. attribute:: Class.methods + Name of the file in which the object was defined. - A dictionary mapping method names to line numbers. +.. attribute:: Object.lineno -.. attribute:: Class.file + The line number in the file named by :attr:`~Object.file` where + the definition of the object started. - Name of the file containing the ``class`` statement defining the class. +.. attribute:: Object.parent -.. attribute:: Class.lineno + The parent of this object, if any. - The line number of the ``class`` statement within the file named by - :attr:`~Class.file`. +.. attribute:: Object.objects -.. _pyclbr-function-objects: + A dictionary mapping object names to the objects that are defined inside the + namespace created by the current object. -Function Objects ----------------- -The :class:`Function` objects used as values in the dictionary returned by -:func:`readmodule_ex` provide the following attributes: +.. _pyclbr-class-objects: +Class Objects +------------- -.. attribute:: Function.module +The :class:`Class` objects used as values in the dictionary returned by +:func:`readmodule` and :func:`readmodule_ex` provide the following extra +data members: - The name of the module defining the function described by the function - descriptor. +.. attribute:: Class.super -.. attribute:: Function.name + A list of :class:`Class` objects which describe the immediate base + classes of the class being described. Classes which are named as + superclasses but which are not discoverable by :func:`readmodule` are + listed as a string with the class name instead of as :class:`Class` + objects. - The name of the function. +.. attribute:: Class.methods -.. attribute:: Function.file + A dictionary mapping method names to line numbers. - Name of the file containing the ``def`` statement defining the function. +.. _pyclbr-function-objects: -.. attribute:: Function.lineno +Function Objects +---------------- - The line number of the ``def`` statement within the file named by - :attr:`~Function.file`. +The :class:`Function` objects used as values in the dictionary returned by +:func:`readmodule_ex` provide only the members already defined by +:class:`Class` objects. diff --git a/Lib/pyclbr.py b/Lib/pyclbr.py index d7dba9761851ea..69636e2622ff83 100644 --- a/Lib/pyclbr.py +++ b/Lib/pyclbr.py @@ -15,14 +15,23 @@ is present for packages: the key '__path__' has a list as its value which contains the package search path. -A class is described by the class Class in this module. Instances -of this class have the following instance variables: - module -- the module name - name -- the name of the class - super -- a list of super classes (Class instances) +Classes and functions have a common superclass in this module, the Object +class. Every instance of this class have the following instance variables: + module -- the module name + name -- the name of the object + file -- the file in which the object was defined + lineno -- the line in the file on which the definition of the object + started + parent -- the parent of this object, if any + objects -- the other classes and function this object may contain +The 'objects' attribute is a dictionary where each key/value pair corresponds +to the name of the object and the object itself. + +A class is described by the class Class in this module. Instances +of this class have the following instance variables (plus the ones from +Object): + super -- a list of super classes (Class instances) methods -- a dictionary of methods - file -- the file in which the class was defined - lineno -- the line in the file on which the class statement occurred The dictionary of methods uses the method names as keys and the line numbers on which the method was defined as values. If the name of a super class is not recognized, the corresponding @@ -32,11 +41,6 @@ shouldn't happen often. A function is described by the class Function in this module. -Instances of this class have the following instance variables: - module -- the module name - name -- the name of the class - file -- the file in which the class was defined - lineno -- the line in the file on which the class statement occurred """ import io @@ -45,33 +49,54 @@ import tokenize from token import NAME, DEDENT, OP -__all__ = ["readmodule", "readmodule_ex", "Class", "Function"] +__all__ = ["readmodule", "readmodule_ex", "Object", "Class", "Function"] _modules = {} # cache of modules we've seen -# each Python class is represented by an instance of this class -class Class: - '''Class to represent a Python class.''' - def __init__(self, module, name, super, file, lineno): + +class Object: + """Class to represent a Python object.""" + def __init__(self, module, name, file, lineno, parent): self.module = module self.name = name + self.file = file + self.lineno = lineno + self.parent = parent + self.objects = {} + + def _addobject(self, name, obj): + self.objects[name] = obj + + +# each Python class is represented by an instance of this class +class Class(Object): + '''Class to represent a Python class.''' + def __init__(self, module, name, super, file, lineno, parent=None): + Object.__init__(self, module, name, file, lineno, parent) if super is None: super = [] self.super = super self.methods = {} - self.file = file - self.lineno = lineno def _addmethod(self, name, lineno): self.methods[name] = lineno -class Function: + +class Function(Object): '''Class to represent a top-level Python function''' - def __init__(self, module, name, file, lineno): - self.module = module - self.name = name - self.file = file - self.lineno = lineno + def __init__(self, module, name, file, lineno, parent=None): + Object.__init__(self, module, name, file, lineno, parent) + + +def _newfunction(ob, name, lineno): + '''Helper function for creating a nested function or a method.''' + return Function(ob.module, name, ob.file, lineno, ob) + + +def _newclass(ob, name, super, lineno): + '''Helper function for creating a nested class.''' + return Class(ob.module, name, super, ob.file, lineno, ob) + def readmodule(module, path=None): '''Backwards compatible interface. @@ -138,7 +163,6 @@ def _readmodule(module, path, inpackage=None): search_path = path else: search_path = path + sys.path - # XXX This will change once issue19944 lands. spec = importlib.util._find_spec_from_path(fullmodule, search_path) _modules[fullmodule] = dict # is module a package? @@ -174,17 +198,22 @@ def _readmodule(module, path, inpackage=None): tokentype, meth_name, start = next(g)[0:3] if tokentype != NAME: continue # Syntax error + cur_func = None if stack: - cur_class = stack[-1][0] - if isinstance(cur_class, Class): - # it's a method - cur_class._addmethod(meth_name, lineno) - # else it's a nested def + cur_obj = stack[-1][0] + if isinstance(cur_obj, Object): + # It's a nested function or a method. + cur_func = _newfunction(cur_obj, meth_name, lineno) + cur_obj._addobject(meth_name, cur_func) + + if isinstance(cur_obj, Class): + # it's a method + cur_obj._addmethod(meth_name, lineno) else: # it's a function - dict[meth_name] = Function(fullmodule, meth_name, - fname, lineno) - stack.append((None, thisindent)) # Marker for nested fns + cur_func = Function(fullmodule, meth_name, fname, lineno) + dict[meth_name] = cur_func + stack.append((cur_func, thisindent)) # Marker for nested fns. elif token == 'class': lineno, thisindent = start # close previous nested classes and defs @@ -235,9 +264,16 @@ def _readmodule(module, path, inpackage=None): super.append(token) # expressions in the base list are not supported inherit = names - cur_class = Class(fullmodule, class_name, inherit, - fname, lineno) - if not stack: + if stack: + cur_obj = stack[-1][0] + if isinstance(cur_obj, Object): + # Either a nested class or a class inside a function. + cur_class = _newclass(cur_obj, class_name, inherit, + lineno) + cur_obj._addobject(class_name, cur_class) + else: + cur_class = Class(fullmodule, class_name, inherit, + fname, lineno) dict[class_name] = cur_class stack.append((cur_class, thisindent)) elif token == 'import' and start[1] == 0: @@ -284,6 +320,7 @@ def _readmodule(module, path, inpackage=None): f.close() return dict + def _getnamelist(g): # Helper to get a comma-separated list of dotted names plus 'as' # clauses. Return a list of pairs (name, name2) where name2 is @@ -304,6 +341,7 @@ def _getnamelist(g): break return names + def _getname(g): # Helper to get a dotted name, return a pair (name, token) where # name is the dotted name, or None if there was no dotted name, @@ -323,10 +361,10 @@ def _getname(g): parts.append(token) return (".".join(parts), token) + def _main(): # Main program for testing. import os - from operator import itemgetter mod = sys.argv[1] if os.path.exists(mod): path = [os.path.dirname(mod)] @@ -336,17 +374,28 @@ def _main(): else: path = [] dict = readmodule_ex(mod, path) - objs = list(dict.values()) - objs.sort(key=lambda a: getattr(a, 'lineno', 0)) - for obj in objs: + lineno_key = lambda a: getattr(a, 'lineno', 0) + objs = sorted(dict.values(), key=lineno_key, reverse=True) + indent_level = 2 + while objs: + obj = objs.pop() + if isinstance(obj, list): + # Value of a __path__ key + continue + if not hasattr(obj, 'indent'): + obj.indent = 0 + + if isinstance(obj, Object): + new_objs = sorted(obj.objects.values(), + key=lineno_key, reverse=True) + for ob in new_objs: + ob.indent = obj.indent + indent_level + objs.extend(new_objs) if isinstance(obj, Class): - print("class", obj.name, obj.super, obj.lineno) - methods = sorted(obj.methods.items(), key=itemgetter(1)) - for name, lineno in methods: - if name != "__path__": - print(" def", name, lineno) + print("{}class {} {} {}" + .format(' ' * obj.indent, obj.name, obj.super, obj.lineno)) elif isinstance(obj, Function): - print("def", obj.name, obj.lineno) + print("{}def {} {}".format(' ' * obj.indent, obj.name, obj.lineno)) if __name__ == "__main__": _main() diff --git a/Lib/test/test_pyclbr.py b/Lib/test/test_pyclbr.py index 9c216d3eb7b81a..de863aee256ea5 100644 --- a/Lib/test/test_pyclbr.py +++ b/Lib/test/test_pyclbr.py @@ -2,10 +2,14 @@ Test cases for pyclbr.py Nick Mathewson ''' + +import os import sys from types import FunctionType, MethodType, BuiltinFunctionType import pyclbr from unittest import TestCase, main as unittest_main +from test import support +from functools import partial StaticMethodType = type(staticmethod(lambda: None)) ClassMethodType = type(classmethod(lambda c: None)) @@ -150,6 +154,121 @@ def test_decorators(self): # self.checkModule('test.pyclbr_input', ignore=['om']) + def test_nested(self): + + def clbr_from_tuple(t, store, parent=None, lineno=1): + '''Create pyclbr objects from the given tuple t.''' + name = t[0] + obj = pickp(name) + if parent is not None: + store = store[parent].objects + ob_name = name.split()[1] + store[ob_name] = obj(name=ob_name, lineno=lineno, parent=parent) + parent = ob_name + + for item in t[1:]: + lineno += 1 + if isinstance(item, str): + obj = pickp(item) + ob_name = item.split()[1] + store[parent].objects[ob_name] = obj( + name=ob_name, lineno=lineno, parent=parent) + else: + lineno = clbr_from_tuple(item, store, parent, lineno) + + return lineno + + def tuple_to_py(t, output, indent=0): + '''Write python code to output according to the given tuple.''' + name = t[0] + output.write('{}{}():'.format(' ' * indent, name)) + indent += 2 + + if not t[1:]: + output.write(' pass') + output.write('\n') + + for item in t[1:]: + if isinstance(item, str): + output.write('{}{}(): pass\n'.format(' ' * indent, item)) + else: + tuple_to_py(item, output, indent) + + # Nested "thing" to test. + sample = ( + ("class A", + ("class B", + ("def a", + "def b")), + "def c"), + ("def d", + ("def e", + ("class C", + "def f") + ), + "def g") + ) + + pclass = partial(pyclbr.Class, module=None, file=None, super=None) + pfunc = partial(pyclbr.Function, module=None, file=None) + + def pickp(name): + return pclass if name.startswith('class') else pfunc + + # Create a module for storing the Python code. + dirname = os.path.abspath(support.TESTFN) + modname = 'notsupposedtoexist' + fname = os.path.join(dirname, modname) + os.extsep + 'py' + os.mkdir(dirname) + + # Create pyclbr objects from the sample above, and also convert + # the same sample above to Python code and write it to fname. + d = {} + lineno = 1 + with open(fname, 'w') as output: + for t in sample: + newlineno = clbr_from_tuple(t, d, lineno=lineno) + lineno = newlineno + 1 + tuple_to_py(t, output) + + # Get the data returned by readmodule_ex to compare against + # our generated data. + try: + with support.DirsOnSysPath(dirname): + d_cmp = pyclbr.readmodule_ex(modname) + finally: + support.unlink(fname) + support.rmtree(dirname) + + # Finally perform the tests. + def check_objects(ob1, ob2): + self.assertEqual(ob1.lineno, ob2.lineno) + if ob1.parent is None: + self.assertIsNone(ob2.parent) + else: + # ob1 must come from our generated data since the parent + # attribute is always a string, while the ob2 must come + # from pyclbr which is always an Object instance. + self.assertEqual(ob1.parent, ob2.parent.name) + self.assertEqual( + ob1.__class__.__name__, + ob2.__class__.__name__) + self.assertEqual(ob1.objects.keys(), ob2.objects.keys()) + for name, obj in list(ob1.objects.items()): + obj_cmp = ob2.objects.pop(name) + del ob1.objects[name] + check_objects(obj, obj_cmp) + + for name, obj in list(d.items()): + self.assertIn(name, d_cmp) + obj_cmp = d_cmp.pop(name) + del d[name] + check_objects(obj, obj_cmp) + self.assertFalse(obj.objects) + self.assertFalse(obj_cmp.objects) + self.assertFalse(d) + self.assertFalse(d_cmp) + def test_others(self): cm = self.checkModule From a726dc1ae5be9131badd78d02a308fe64121c7ed Mon Sep 17 00:00:00 2001 From: Cheryl Sabella Date: Sat, 1 Jul 2017 13:29:04 -0400 Subject: [PATCH 2/9] bpo-6691: Support for nested classes and functions in pyclbr --- Doc/library/pyclbr.rst | 49 ++++++++++------ Lib/pyclbr.py | 120 ++++++++++++++++++++-------------------- Lib/test/test_pyclbr.py | 38 ++++++------- 3 files changed, 112 insertions(+), 95 deletions(-) diff --git a/Doc/library/pyclbr.rst b/Doc/library/pyclbr.rst index 331788e6b1cea9..f0df880d72e8cf 100644 --- a/Doc/library/pyclbr.rst +++ b/Doc/library/pyclbr.rst @@ -11,13 +11,12 @@ -------------- The :mod:`pyclbr` module can be used to determine some limited information -about the classes, methods and top-level functions defined in a module. The -information provided is sufficient to implement a traditional three-pane -class browser. The information is extracted from the source code rather -than by importing the module, so this module is safe to use with untrusted -code. This restriction makes it impossible to use this module with modules -not implemented in Python, including all standard and optional extension -modules. +about the classes, methods, and functions defined in a module. The +information provided is sufficient to implement a module browser. The +information is extracted from the source code rather than by importing the +module, so this module is safe to use with untrusted code. This restriction +makes it impossible to use this module with modules not implemented in Python, +including all standard and optional extension modules. .. function:: readmodule(module, path=None) @@ -32,8 +31,8 @@ modules. .. function:: readmodule_ex(module, path=None) Like :func:`readmodule`, but the returned dictionary, in addition to - mapping class names to class descriptor objects, also maps top-level - function names to function descriptor objects. Moreover, if the module + mapping class names to class descriptor objects, also maps function + names to function descriptor objects. Moreover, if the module being read is a package, the key ``'__path__'`` in the returned dictionary has as its value a list which contains the package search path. @@ -73,21 +72,33 @@ data members: The parent of this object, if any. + .. versionadded:: 3.7 -.. attribute:: Object.objects + +.. attribute:: Object.children A dictionary mapping object names to the objects that are defined inside the namespace created by the current object. + .. versionadded:: 3.7 + + +.. versionchanged:: 3.7 + :class:`Object` was added as a base class for :class:`Class` and + :class:`Function` and, except as otherwise noted, the attributes + were previously common to those two classes. + + .. _pyclbr-class-objects: Class Objects ------------- -The :class:`Class` objects used as values in the dictionary returned by -:func:`readmodule` and :func:`readmodule_ex` provide the following extra -data members: +:class:`Class` is a subclass of :class:`Object` whose objects are used as values +in the dictionary returned by :func:`readmodule` and :func:`readmodule_ex`. +In addition to the attributes from :class:`Object`, :class:`Class` objects +also provide the following attributes: .. attribute:: Class.super @@ -104,12 +115,18 @@ data members: A dictionary mapping method names to line numbers. +.. versionchanged:: 3.7 + :class:`Class` became a subclass of :class:`Object`. + + .. _pyclbr-function-objects: Function Objects ---------------- -The :class:`Function` objects used as values in the dictionary returned by -:func:`readmodule_ex` provide only the members already defined by -:class:`Class` objects. +:class:`Function` is a subclass of :class:`Object` whose objects are used as +values in the dictionary returned by :func:`readmodule_ex`. The only instance +attributes are those from :class:`Object`. +.. versionchanged:: 3.7 + :class:`Function` became a subclass of :class:`Object`. diff --git a/Lib/pyclbr.py b/Lib/pyclbr.py index 69636e2622ff83..e0f5a1ce667c59 100644 --- a/Lib/pyclbr.py +++ b/Lib/pyclbr.py @@ -1,7 +1,7 @@ -"""Parse a Python module and describe its classes and methods. +"""Parse a Python module and describe its classes and functions. Parse enough of a Python file to recognize imports and class and -method definitions, and to find out the superclasses of a class. +function definitions, and to find out the superclasses of a class. The interface consists of a single function: readmodule_ex(module [, path]) @@ -9,38 +9,38 @@ list of directories where the module is to be searched. If present, path is prepended to the system search path sys.path. The return value is a dictionary. The keys of the dictionary are the names of -the classes defined in the module (including classes that are defined -via the from XXX import YYY construct). The values are class -instances of the class Class defined here. One special key/value pair -is present for packages: the key '__path__' has a list as its value -which contains the package search path. +the classes and functions defined in the module (including classes that +are defined via the from XXX import YYY construct). The values are class +instances of the class Class and function instances of the class Function, +respectively. One special key/value pair is present for packages: the +key '__path__' has a list as its value which contains the package search +path. Classes and functions have a common superclass in this module, the Object -class. Every instance of this class have the following instance variables: +class. Every instance of this class has the following instance variables: module -- the module name name -- the name of the object file -- the file in which the object was defined lineno -- the line in the file on which the definition of the object started parent -- the parent of this object, if any - objects -- the other classes and function this object may contain -The 'objects' attribute is a dictionary where each key/value pair corresponds -to the name of the object and the object itself. + children -- the nested objects (classes and functions) contained + in this object +The 'children' attribute is a dictionary mapping object names to objects. A class is described by the class Class in this module. Instances -of this class have the following instance variables (plus the ones from -Object): +of this class have the attributes from Object, plus the following: super -- a list of super classes (Class instances) methods -- a dictionary of methods -The dictionary of methods uses the method names as keys and the line -numbers on which the method was defined as values. +'methods' maps method names to the line number where the definition begins. If the name of a super class is not recognized, the corresponding entry in the list of super classes is not a class instance but a string giving the name of the super class. Since import statements are recognized and imported modules are scanned as well, this shouldn't happen often. -A function is described by the class Function in this module. +A function is described by the class Function in this module. The +only instance attributes are those of Object. """ import io @@ -55,27 +55,25 @@ class Object: - """Class to represent a Python object.""" + """Class to represent a Python class or function.""" def __init__(self, module, name, file, lineno, parent): self.module = module self.name = name self.file = file self.lineno = lineno self.parent = parent - self.objects = {} + self.children = {} - def _addobject(self, name, obj): - self.objects[name] = obj + def _addchild(self, name, obj): + self.children[name] = obj -# each Python class is represented by an instance of this class +# Each Python class is represented by an instance of this class. class Class(Object): '''Class to represent a Python class.''' def __init__(self, module, name, super, file, lineno, parent=None): Object.__init__(self, module, name, file, lineno, parent) - if super is None: - super = [] - self.super = super + self.super = [] if super is None else super self.methods = {} def _addmethod(self, name, lineno): @@ -127,7 +125,7 @@ def _readmodule(module, path, inpackage=None): package search path; otherwise, we are searching for a top-level module, and PATH is combined with sys.path. ''' - # Compute the full module name (prepending inpackage if set) + # Compute the full module name (prepending inpackage if set). if inpackage is not None: fullmodule = "%s.%s" % (inpackage, module) else: @@ -137,15 +135,15 @@ def _readmodule(module, path, inpackage=None): if fullmodule in _modules: return _modules[fullmodule] - # Initialize the dict for this module's contents - dict = {} + # Initialize the dict for this module's contents. + tree = {} - # Check if it is a built-in module; we don't do much for these + # Check if it is a built-in module; we don't do much for these. if module in sys.builtin_module_names and inpackage is None: - _modules[module] = dict - return dict + _modules[module] = tree + return tree - # Check for a dotted module name + # Check for a dotted module name. i = module.rfind('.') if i >= 0: package = module[:i] @@ -157,27 +155,26 @@ def _readmodule(module, path, inpackage=None): raise ImportError('No package named {}'.format(package)) return _readmodule(submodule, parent['__path__'], package) - # Search the path for the module + # Search the path for the module. f = None if inpackage is not None: search_path = path else: search_path = path + sys.path spec = importlib.util._find_spec_from_path(fullmodule, search_path) - _modules[fullmodule] = dict + _modules[fullmodule] = tree # is module a package? if spec.submodule_search_locations is not None: - dict['__path__'] = spec.submodule_search_locations + tree['__path__'] = spec.submodule_search_locations try: source = spec.loader.get_source(fullmodule) if source is None: - return dict + return tree except (AttributeError, ImportError): - # not Python source, can't do anything with this module - return dict + # not Python source, can't do anything with this module. + return tree fname = spec.loader.get_filename(fullmodule) - f = io.StringIO(source) stack = [] # stack of (class, indent) pairs @@ -195,7 +192,7 @@ def _readmodule(module, path, inpackage=None): # close previous nested classes and defs while stack and stack[-1][1] >= thisindent: del stack[-1] - tokentype, meth_name, start = next(g)[0:3] + tokentype, func_name, start = next(g)[0:3] if tokentype != NAME: continue # Syntax error cur_func = None @@ -203,26 +200,26 @@ def _readmodule(module, path, inpackage=None): cur_obj = stack[-1][0] if isinstance(cur_obj, Object): # It's a nested function or a method. - cur_func = _newfunction(cur_obj, meth_name, lineno) - cur_obj._addobject(meth_name, cur_func) + cur_func = _newfunction(cur_obj, func_name, lineno) + cur_obj._addchild(func_name, cur_func) if isinstance(cur_obj, Class): # it's a method - cur_obj._addmethod(meth_name, lineno) + cur_obj._addmethod(func_name, lineno) else: # it's a function - cur_func = Function(fullmodule, meth_name, fname, lineno) - dict[meth_name] = cur_func + cur_func = Function(fullmodule, func_name, fname, lineno) + tree[func_name] = cur_func stack.append((cur_func, thisindent)) # Marker for nested fns. elif token == 'class': lineno, thisindent = start - # close previous nested classes and defs + # Close previous nested classes and defs. while stack and stack[-1][1] >= thisindent: del stack[-1] tokentype, class_name, start = next(g)[0:3] if tokentype != NAME: continue # Syntax error - # parse what follows the class name + # Parse what follows the class name. tokentype, token, start = next(g)[0:3] inherit = None if token == '(': @@ -234,9 +231,9 @@ def _readmodule(module, path, inpackage=None): tokentype, token, start = next(g)[0:3] if token in (')', ',') and level == 1: n = "".join(super) - if n in dict: + if n in tree: # we know this super class - n = dict[n] + n = tree[n] else: c = n.split('.') if len(c) > 1: @@ -270,11 +267,11 @@ def _readmodule(module, path, inpackage=None): # Either a nested class or a class inside a function. cur_class = _newclass(cur_obj, class_name, inherit, lineno) - cur_obj._addobject(class_name, cur_class) + cur_obj._addchild(class_name, cur_class) else: cur_class = Class(fullmodule, class_name, inherit, fname, lineno) - dict[class_name] = cur_class + tree[class_name] = cur_class stack.append((cur_class, thisindent)) elif token == 'import' and start[1] == 0: modules = _getnamelist(g) @@ -298,27 +295,27 @@ def _readmodule(module, path, inpackage=None): continue names = _getnamelist(g) try: - # Recursively read the imported module + # Recursively read the imported module. d = _readmodule(mod, path, inpackage) except: # If we can't find or parse the imported module, # too bad -- don't die here. continue - # add any classes that were defined in the imported module - # to our name space if they were mentioned in the list + # Add any classes that were defined in the imported module + # to our name space if they were mentioned in the list. for n, n2 in names: if n in d: - dict[n2 or n] = d[n] + tree[n2 or n] = d[n] elif n == '*': # don't add names that start with _ for n in d: if n[0] != '_': - dict[n] = d[n] + tree[n] = d[n] except StopIteration: pass f.close() - return dict + return tree def _getnamelist(g): @@ -365,7 +362,10 @@ def _getname(g): def _main(): # Main program for testing. import os - mod = sys.argv[1] + try: + mod = sys.argv[1] + except: + mod = __file__ if os.path.exists(mod): path = [os.path.dirname(mod)] mod = os.path.basename(mod) @@ -373,9 +373,9 @@ def _main(): mod = mod[:-3] else: path = [] - dict = readmodule_ex(mod, path) + tree = readmodule_ex(mod, path) lineno_key = lambda a: getattr(a, 'lineno', 0) - objs = sorted(dict.values(), key=lineno_key, reverse=True) + objs = sorted(tree.values(), key=lineno_key, reverse=True) indent_level = 2 while objs: obj = objs.pop() @@ -386,7 +386,7 @@ def _main(): obj.indent = 0 if isinstance(obj, Object): - new_objs = sorted(obj.objects.values(), + new_objs = sorted(obj.children.values(), key=lineno_key, reverse=True) for ob in new_objs: ob.indent = obj.indent + indent_level diff --git a/Lib/test/test_pyclbr.py b/Lib/test/test_pyclbr.py index de863aee256ea5..a11a4d879a4f8d 100644 --- a/Lib/test/test_pyclbr.py +++ b/Lib/test/test_pyclbr.py @@ -156,39 +156,39 @@ def test_decorators(self): def test_nested(self): - def clbr_from_tuple(t, store, parent=None, lineno=1): + def clbr_from_tuple(tup, store, parent=None, lineno=1): '''Create pyclbr objects from the given tuple t.''' - name = t[0] + name = tup[0] obj = pickp(name) if parent is not None: - store = store[parent].objects + store = store[parent].children ob_name = name.split()[1] store[ob_name] = obj(name=ob_name, lineno=lineno, parent=parent) parent = ob_name - for item in t[1:]: + for item in tup[1:]: lineno += 1 if isinstance(item, str): obj = pickp(item) ob_name = item.split()[1] - store[parent].objects[ob_name] = obj( + store[parent].children[ob_name] = obj( name=ob_name, lineno=lineno, parent=parent) else: lineno = clbr_from_tuple(item, store, parent, lineno) return lineno - def tuple_to_py(t, output, indent=0): + def tuple_to_py(tup, output, indent=0): '''Write python code to output according to the given tuple.''' - name = t[0] + name = tup[0] output.write('{}{}():'.format(' ' * indent, name)) indent += 2 - if not t[1:]: + if not tup[1:]: output.write(' pass') output.write('\n') - for item in t[1:]: + for item in tup[1:]: if isinstance(item, str): output.write('{}{}(): pass\n'.format(' ' * indent, item)) else: @@ -200,7 +200,7 @@ def tuple_to_py(t, output, indent=0): ("class B", ("def a", "def b")), - "def c"), + "def c"), ("def d", ("def e", ("class C", @@ -226,10 +226,10 @@ def pickp(name): d = {} lineno = 1 with open(fname, 'w') as output: - for t in sample: - newlineno = clbr_from_tuple(t, d, lineno=lineno) + for tup in sample: + newlineno = clbr_from_tuple(tup, d, lineno=lineno) lineno = newlineno + 1 - tuple_to_py(t, output) + tuple_to_py(tup, output) # Get the data returned by readmodule_ex to compare against # our generated data. @@ -253,10 +253,10 @@ def check_objects(ob1, ob2): self.assertEqual( ob1.__class__.__name__, ob2.__class__.__name__) - self.assertEqual(ob1.objects.keys(), ob2.objects.keys()) - for name, obj in list(ob1.objects.items()): - obj_cmp = ob2.objects.pop(name) - del ob1.objects[name] + self.assertEqual(ob1.children.keys(), ob2.children.keys()) + for name, obj in list(ob1.children.items()): + obj_cmp = ob2.children.pop(name) + del ob1.children[name] check_objects(obj, obj_cmp) for name, obj in list(d.items()): @@ -264,8 +264,8 @@ def check_objects(ob1, ob2): obj_cmp = d_cmp.pop(name) del d[name] check_objects(obj, obj_cmp) - self.assertFalse(obj.objects) - self.assertFalse(obj_cmp.objects) + self.assertFalse(obj.children) + self.assertFalse(obj_cmp.children) self.assertFalse(d) self.assertFalse(d_cmp) From 7105eedda0cdb0cee3cd3108fcda673817d4d800 Mon Sep 17 00:00:00 2001 From: Cheryl Sabella Date: Sat, 1 Jul 2017 18:15:33 -0400 Subject: [PATCH 3/9] Split _readmodule into two functions --- Lib/pyclbr.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Lib/pyclbr.py b/Lib/pyclbr.py index e0f5a1ce667c59..b3e441530d73e8 100644 --- a/Lib/pyclbr.py +++ b/Lib/pyclbr.py @@ -175,6 +175,11 @@ def _readmodule(module, path, inpackage=None): return tree fname = spec.loader.get_filename(fullmodule) + return _create_tree(fullmodule, path, fname, source, tree, inpackage) + + +def _create_tree(fullmodule, path, fname, source, tree, inpackage): + "Create the tree for each module." f = io.StringIO(source) stack = [] # stack of (class, indent) pairs From bcc1760bcbbe581fe30527ee4d757b330eb2ff20 Mon Sep 17 00:00:00 2001 From: Cheryl Sabella Date: Sun, 2 Jul 2017 08:15:18 -0400 Subject: [PATCH 4/9] Remove conditional for isinstance --- Lib/pyclbr.py | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/Lib/pyclbr.py b/Lib/pyclbr.py index b3e441530d73e8..0695fb7716814e 100644 --- a/Lib/pyclbr.py +++ b/Lib/pyclbr.py @@ -203,14 +203,11 @@ def _create_tree(fullmodule, path, fname, source, tree, inpackage): cur_func = None if stack: cur_obj = stack[-1][0] - if isinstance(cur_obj, Object): - # It's a nested function or a method. - cur_func = _newfunction(cur_obj, func_name, lineno) - cur_obj._addchild(func_name, cur_func) - - if isinstance(cur_obj, Class): - # it's a method - cur_obj._addmethod(func_name, lineno) + cur_func = _newfunction(cur_obj, func_name, lineno) + cur_obj._addchild(func_name, cur_func) + if isinstance(cur_obj, Class): + # it's a method + cur_obj._addmethod(func_name, lineno) else: # it's a function cur_func = Function(fullmodule, func_name, fname, lineno) @@ -268,11 +265,9 @@ def _create_tree(fullmodule, path, fname, source, tree, inpackage): inherit = names if stack: cur_obj = stack[-1][0] - if isinstance(cur_obj, Object): - # Either a nested class or a class inside a function. - cur_class = _newclass(cur_obj, class_name, inherit, - lineno) - cur_obj._addchild(class_name, cur_class) + cur_class = _newclass(cur_obj, class_name, inherit, + lineno) + cur_obj._addchild(class_name, cur_class) else: cur_class = Class(fullmodule, class_name, inherit, fname, lineno) From 83b10c7a43fb85eb1b077a3ce6a70b49d2ae2905 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sun, 2 Jul 2017 20:41:14 -0400 Subject: [PATCH 5/9] Edit docstrings and comments. --- Lib/pyclbr.py | 151 ++++++++++++++++++++++++-------------------------- 1 file changed, 71 insertions(+), 80 deletions(-) diff --git a/Lib/pyclbr.py b/Lib/pyclbr.py index 0695fb7716814e..e68c45df8e4947 100644 --- a/Lib/pyclbr.py +++ b/Lib/pyclbr.py @@ -4,43 +4,38 @@ function definitions, and to find out the superclasses of a class. The interface consists of a single function: - readmodule_ex(module [, path]) + readmodule_ex(module, path=None) where module is the name of a Python module, and path is an optional list of directories where the module is to be searched. If present, -path is prepended to the system search path sys.path. The return -value is a dictionary. The keys of the dictionary are the names of -the classes and functions defined in the module (including classes that -are defined via the from XXX import YYY construct). The values are class -instances of the class Class and function instances of the class Function, -respectively. One special key/value pair is present for packages: the -key '__path__' has a list as its value which contains the package search -path. - -Classes and functions have a common superclass in this module, the Object -class. Every instance of this class has the following instance variables: - module -- the module name - name -- the name of the object - file -- the file in which the object was defined - lineno -- the line in the file on which the definition of the object - started - parent -- the parent of this object, if any - children -- the nested objects (classes and functions) contained - in this object -The 'children' attribute is a dictionary mapping object names to objects. - -A class is described by the class Class in this module. Instances -of this class have the attributes from Object, plus the following: - super -- a list of super classes (Class instances) - methods -- a dictionary of methods -'methods' maps method names to the line number where the definition begins. +path is prepended to the system search path sys.path. The return value +is a dictionary. The keys of the dictionary are the names of the +classes and functions defined in the module (including classes that are +defined via the from XXX import YYY construct). The values are +instances of classes Class and Function. One special key/value pair is +present for packages: the key '__path__' has a list as its value which +contains the package search path. + +Classes and Functions have a common superclass: Object. Every instance +has the following attributes: + module -- name of the module; + name -- name of the object; + file -- file in which the object is defined; + lineno -- line in the file where the object's definition starts; + parent -- parent of this object, if any; + children -- nested objects contained in this object. +The 'children' attribute is a dictionary mapping names to objects. + +Instances of Function describe function with the attributes from Object. + +Instances of Class describe classes with the attributes from Object, +plus the following: + super -- list of super classes (Class instances if possible); + methods -- mapping of method names to beginning line numbers. If the name of a super class is not recognized, the corresponding entry in the list of super classes is not a class instance but a string giving the name of the super class. Since import statements are recognized and imported modules are scanned as well, this shouldn't happen often. - -A function is described by the class Function in this module. The -only instance attributes are those of Object. """ import io @@ -51,11 +46,11 @@ __all__ = ["readmodule", "readmodule_ex", "Object", "Class", "Function"] -_modules = {} # cache of modules we've seen +_modules = {} # Initialize cache of modules we've seen. class Object: - """Class to represent a Python class or function.""" + "Informaton about Python class or function." def __init__(self, module, name, file, lineno, parent): self.module = module self.name = name @@ -68,9 +63,8 @@ def _addchild(self, name, obj): self.children[name] = obj -# Each Python class is represented by an instance of this class. class Class(Object): - '''Class to represent a Python class.''' + "Information about a Python class." def __init__(self, module, name, super, file, lineno, parent=None): Object.__init__(self, module, name, file, lineno, parent) self.super = [] if super is None else super @@ -81,26 +75,26 @@ def _addmethod(self, name, lineno): class Function(Object): - '''Class to represent a top-level Python function''' + "Information about a Python function, including methods." def __init__(self, module, name, file, lineno, parent=None): Object.__init__(self, module, name, file, lineno, parent) def _newfunction(ob, name, lineno): - '''Helper function for creating a nested function or a method.''' + "Return a Function after expanding ob to 3 arguments." return Function(ob.module, name, ob.file, lineno, ob) def _newclass(ob, name, super, lineno): - '''Helper function for creating a nested class.''' + "Return a Class after expanding ob to 3 arguments." return Class(ob.module, name, super, ob.file, lineno, ob) def readmodule(module, path=None): - '''Backwards compatible interface. + """Return Class objects for the top-level classes in module. - Call readmodule_ex() and then only keep Class objects from the - resulting dictionary.''' + This is the original interface, before Functions were added. + """ res = {} for key, value in _readmodule(module, path or []).items(): @@ -109,29 +103,29 @@ def readmodule(module, path=None): return res def readmodule_ex(module, path=None): - '''Read a module file and return a dictionary of classes. + """Return a dictionary with all functions and classes in module. - Search for MODULE in PATH and sys.path, read and parse the - module and return a dictionary with one entry for each class - found in the module. - ''' + Search for module in PATH + sys.path. + If possible, include imported superclasses. + Do this by reading source, without importing (and executing) it. + """ return _readmodule(module, path or []) def _readmodule(module, path, inpackage=None): - '''Do the hard work for readmodule[_ex]. + """Do the hard work for readmodule[_ex]. - If INPACKAGE is given, it must be the dotted name of the package in + If inpackage is given, it must be the dotted name of the package in which we are searching for a submodule, and then PATH must be the package search path; otherwise, we are searching for a top-level - module, and PATH is combined with sys.path. - ''' + module, and path is combined with sys.path. + """ # Compute the full module name (prepending inpackage if set). if inpackage is not None: fullmodule = "%s.%s" % (inpackage, module) else: fullmodule = module - # Check in the cache + # Check in the cache. if fullmodule in _modules: return _modules[fullmodule] @@ -163,7 +157,7 @@ def _readmodule(module, path, inpackage=None): search_path = path + sys.path spec = importlib.util._find_spec_from_path(fullmodule, search_path) _modules[fullmodule] = tree - # is module a package? + # Is module a package? if spec.submodule_search_locations is not None: tree['__path__'] = spec.submodule_search_locations try: @@ -171,7 +165,7 @@ def _readmodule(module, path, inpackage=None): if source is None: return tree except (AttributeError, ImportError): - # not Python source, can't do anything with this module. + # If module is not Python source, we cannot do anything. return tree fname = spec.loader.get_filename(fullmodule) @@ -179,40 +173,40 @@ def _readmodule(module, path, inpackage=None): def _create_tree(fullmodule, path, fname, source, tree, inpackage): - "Create the tree for each module." + "Return the tree for a particular module." f = io.StringIO(source) - stack = [] # stack of (class, indent) pairs + stack = [] # Initialize stack of (class, indent) pairs. g = tokenize.generate_tokens(f.readline) try: for tokentype, token, start, _end, _line in g: if tokentype == DEDENT: lineno, thisindent = start - # close nested classes and defs + # Close previous nested classes and defs. while stack and stack[-1][1] >= thisindent: del stack[-1] elif token == 'def': lineno, thisindent = start - # close previous nested classes and defs + # Close previous nested classes and defs. while stack and stack[-1][1] >= thisindent: del stack[-1] tokentype, func_name, start = next(g)[0:3] if tokentype != NAME: - continue # Syntax error + continue # Skip def with syntax error. cur_func = None if stack: cur_obj = stack[-1][0] cur_func = _newfunction(cur_obj, func_name, lineno) cur_obj._addchild(func_name, cur_func) if isinstance(cur_obj, Class): - # it's a method + # Function is a method. cur_obj._addmethod(func_name, lineno) else: - # it's a function + # It is just a function. cur_func = Function(fullmodule, func_name, fname, lineno) tree[func_name] = cur_func - stack.append((cur_func, thisindent)) # Marker for nested fns. + stack.append((cur_func, thisindent)) elif token == 'class': lineno, thisindent = start # Close previous nested classes and defs. @@ -220,28 +214,26 @@ def _create_tree(fullmodule, path, fname, source, tree, inpackage): del stack[-1] tokentype, class_name, start = next(g)[0:3] if tokentype != NAME: - continue # Syntax error + continue # Skip class with syntax error. # Parse what follows the class name. tokentype, token, start = next(g)[0:3] inherit = None if token == '(': - names = [] # List of superclasses - # there's a list of superclasses + names = [] # Initialize list of superclasses. level = 1 - super = [] # Tokens making up current superclass + super = [] # Tokens making up current superclass. while True: tokentype, token, start = next(g)[0:3] if token in (')', ',') and level == 1: n = "".join(super) if n in tree: - # we know this super class + # We know this super class. n = tree[n] else: c = n.split('.') if len(c) > 1: - # super class is of the form - # module.class: look in module for - # class + # Super class form is module.class: + # look in module for class. m = c[-2] c = c[-1] if m in _modules: @@ -258,10 +250,10 @@ def _create_tree(fullmodule, path, fname, source, tree, inpackage): break elif token == ',' and level == 1: pass - # only use NAME and OP (== dot) tokens for type name + # Only use NAME and OP (== dot) tokens for type name. elif tokentype in (NAME, OP) and level == 1: super.append(token) - # expressions in the base list are not supported + # Expressions in the base list are not supported. inherit = names if stack: cur_obj = stack[-1][0] @@ -277,7 +269,7 @@ def _create_tree(fullmodule, path, fname, source, tree, inpackage): modules = _getnamelist(g) for mod, _mod2 in modules: try: - # Recursively read the imported module + # Recursively read the imported module. if inpackage is None: _readmodule(mod, path) else: @@ -307,7 +299,7 @@ def _create_tree(fullmodule, path, fname, source, tree, inpackage): if n in d: tree[n2 or n] = d[n] elif n == '*': - # don't add names that start with _ + # Don't add names that start with _. for n in d: if n[0] != '_': tree[n] = d[n] @@ -319,9 +311,10 @@ def _create_tree(fullmodule, path, fname, source, tree, inpackage): def _getnamelist(g): - # Helper to get a comma-separated list of dotted names plus 'as' - # clauses. Return a list of pairs (name, name2) where name2 is - # the 'as' name, or None if there is no 'as' clause. + """Return list of (dotted-name, as-name or None) tuples for token source g. + + An as-name is the name that follows 'as' in an as clause. + """ names = [] while True: name, token = _getname(g) @@ -340,9 +333,7 @@ def _getnamelist(g): def _getname(g): - # Helper to get a dotted name, return a pair (name, token) where - # name is the dotted name, or None if there was no dotted name, - # and token is the next input token. + "Return (dotted-name or None, next-token) tuple for token source g." parts = [] tokentype, token = next(g)[0:2] if tokentype != NAME and token != '*': @@ -360,7 +351,7 @@ def _getname(g): def _main(): - # Main program for testing. + "Print module output (default this file) for quick visual check." import os try: mod = sys.argv[1] @@ -380,7 +371,7 @@ def _main(): while objs: obj = objs.pop() if isinstance(obj, list): - # Value of a __path__ key + # Value is a __path__ key. continue if not hasattr(obj, 'indent'): obj.indent = 0 From 2c285203bff523153fcb103875b2acc291627478 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Mon, 3 Jul 2017 14:03:30 -0400 Subject: [PATCH 6/9] Revise doc --- Doc/library/pyclbr.rst | 151 +++++++++++++++++++++++------------------ 1 file changed, 84 insertions(+), 67 deletions(-) diff --git a/Doc/library/pyclbr.rst b/Doc/library/pyclbr.rst index f0df880d72e8cf..c53e4620516c29 100644 --- a/Doc/library/pyclbr.rst +++ b/Doc/library/pyclbr.rst @@ -10,123 +10,140 @@ -------------- -The :mod:`pyclbr` module can be used to determine some limited information -about the classes, methods, and functions defined in a module. The -information provided is sufficient to implement a module browser. The -information is extracted from the source code rather than by importing the -module, so this module is safe to use with untrusted code. This restriction -makes it impossible to use this module with modules not implemented in Python, -including all standard and optional extension modules. +The :mod:`pyclbr` module provides limited information about the +functions, classes, and methods defined in a python-coded module. The +information is sufficient to implement a module browser. The +information is extracted from the python source code rather than by +importing the module, so this module is safe to use with untrusted code. +This restriction makes it impossible to use this module with modules not +implemented in Python, including all standard and optional extension +modules. .. function:: readmodule(module, path=None) - Read a module and return a dictionary mapping class names to class - descriptor objects. The parameter *module* should be the name of a - module as a string; it may be the name of a module within a package. The - *path* parameter should be a sequence, and is used to augment the value - of ``sys.path``, which is used to locate module source code. + Return a dictionary mapping module-level class names to class + descriptors. If possible, descriptors for imported base classes are + included. Parameter *module* is a string with the name of the module + to read; it may be the name of a module within a package. If given, + *path* is a sequence of directory paths prepended to ``sys.path``, + which is used to locate the module source code. .. function:: readmodule_ex(module, path=None) - Like :func:`readmodule`, but the returned dictionary, in addition to - mapping class names to class descriptor objects, also maps function - names to function descriptor objects. Moreover, if the module - being read is a package, the key ``'__path__'`` in the returned - dictionary has as its value a list which contains the package search - path. + Return a dictionary-based tree containing a function or class + descriptors for each function and class defined in the module with a + ``def`` or ``class`` statement. The returned dictionary maps + module-level function and class names to their descriptors. Nested + objects are entered into the children dictionary of their parent. As + with readmodule, *module* names the module to be read and *path* is + prepended to sys.path. If the module being read is a package, the + returned dictionary has a key ``'__path__'`` whose value is a list + containing the package search path. +The descriptors returned by these functions are instances of +Function and Class classes. Users are not expected to create instances +of these classes. -.. _pyclbr-object-objects: -Object Objects --------------- -The class :class:`Object` is the base class for the classes -:class:`Class` and :class:`Function`. It provides the following -data members: +.. _pyclbr-function-objects: + +Function Objects +---------------- +Class :class:`Function` instances describe functions defined by def +statements. They have the following attributes: -.. attribute:: Object.module +.. attribute:: Function.file - The name of the module defining the object described. + Name of the file in which the function is defined. -.. attribute:: Object.name +.. attribute:: Function.module - The name of the object. + The name of the module defining the function described. -.. attribute:: Object.file +.. attribute:: Function.name - Name of the file in which the object was defined. + The name of the function. -.. attribute:: Object.lineno +.. attribute:: Function.lineno - The line number in the file named by :attr:`~Object.file` where - the definition of the object started. + The line number in the file where the definition starts. -.. attribute:: Object.parent +.. attribute:: Function.parent - The parent of this object, if any. + For top-level functions, None. For nested functions, the parent. .. versionadded:: 3.7 -.. attribute:: Object.children +.. attribute:: Function.children - A dictionary mapping object names to the objects that are defined inside the - namespace created by the current object. + A dictionary mapping names to descriptors for nested functions and + classes. .. versionadded:: 3.7 -.. versionchanged:: 3.7 - :class:`Object` was added as a base class for :class:`Class` and - :class:`Function` and, except as otherwise noted, the attributes - were previously common to those two classes. - - - .. _pyclbr-class-objects: Class Objects ------------- +Class :class:`Class` instances describe classes defined by class +statements. They have the same attributes as Functions and two more. -:class:`Class` is a subclass of :class:`Object` whose objects are used as values -in the dictionary returned by :func:`readmodule` and :func:`readmodule_ex`. -In addition to the attributes from :class:`Object`, :class:`Class` objects -also provide the following attributes: +.. attribute:: Class.file -.. attribute:: Class.super + Name of the file in which the class is defined. - A list of :class:`Class` objects which describe the immediate base - classes of the class being described. Classes which are named as - superclasses but which are not discoverable by :func:`readmodule` are - listed as a string with the class name instead of as :class:`Class` - objects. +.. attribute:: Class.module -.. attribute:: Class.methods + The name of the module defining the class described. - A dictionary mapping method names to line numbers. +.. attribute:: Class.name -.. versionchanged:: 3.7 - :class:`Class` became a subclass of :class:`Object`. + The name of the class. -.. _pyclbr-function-objects: +.. attribute:: Class.lineno -Function Objects ----------------- + The line number in the file where the definition starts. + + +.. attribute:: Class.parent -:class:`Function` is a subclass of :class:`Object` whose objects are used as -values in the dictionary returned by :func:`readmodule_ex`. The only instance -attributes are those from :class:`Object`. + For top-level classes, None. For nested classes, the parent. + + .. versionadded:: 3.7 + + +.. attribute:: Class.children + + A dictionary mapping names to descriptors for nested functions and + classes. + + .. versionadded:: 3.7 + + +.. attribute:: Class.super + + A list of :class:`Class` objects which describe the immediate base + classes of the class being described. Classes which are named as + superclasses but which are not discoverable by :func:`readmodule_ex` + are listed as a string with the class name instead of as + :class:`Class` objects. + + +.. attribute:: Class.methods -.. versionchanged:: 3.7 - :class:`Function` became a subclass of :class:`Object`. + A dictionary mapping method names to line numbers. This can be + derived from the newer children dictionary, but remains for + back-compatibility. From 082e4698dece030653636f510fef743a2a1344ed Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Mon, 3 Jul 2017 14:06:39 -0400 Subject: [PATCH 7/9] Replace Object with _Object --- Lib/pyclbr.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Lib/pyclbr.py b/Lib/pyclbr.py index e68c45df8e4947..72fb53affedf24 100644 --- a/Lib/pyclbr.py +++ b/Lib/pyclbr.py @@ -15,7 +15,7 @@ present for packages: the key '__path__' has a list as its value which contains the package search path. -Classes and Functions have a common superclass: Object. Every instance +Classes and Functions have a common superclass: _Object. Every instance has the following attributes: module -- name of the module; name -- name of the object; @@ -25,9 +25,9 @@ children -- nested objects contained in this object. The 'children' attribute is a dictionary mapping names to objects. -Instances of Function describe function with the attributes from Object. +Instances of Function describe functions with the attributes from _Object. -Instances of Class describe classes with the attributes from Object, +Instances of Class describe classes with the attributes from _Object, plus the following: super -- list of super classes (Class instances if possible); methods -- mapping of method names to beginning line numbers. @@ -44,12 +44,12 @@ import tokenize from token import NAME, DEDENT, OP -__all__ = ["readmodule", "readmodule_ex", "Object", "Class", "Function"] +__all__ = ["readmodule", "readmodule_ex", "Class", "Function"] _modules = {} # Initialize cache of modules we've seen. -class Object: +class _Object: "Informaton about Python class or function." def __init__(self, module, name, file, lineno, parent): self.module = module @@ -63,10 +63,10 @@ def _addchild(self, name, obj): self.children[name] = obj -class Class(Object): +class Class(_Object): "Information about a Python class." def __init__(self, module, name, super, file, lineno, parent=None): - Object.__init__(self, module, name, file, lineno, parent) + _Object.__init__(self, module, name, file, lineno, parent) self.super = [] if super is None else super self.methods = {} @@ -74,10 +74,10 @@ def _addmethod(self, name, lineno): self.methods[name] = lineno -class Function(Object): +class Function(_Object): "Information about a Python function, including methods." def __init__(self, module, name, file, lineno, parent=None): - Object.__init__(self, module, name, file, lineno, parent) + _Object.__init__(self, module, name, file, lineno, parent) def _newfunction(ob, name, lineno): @@ -376,7 +376,7 @@ def _main(): if not hasattr(obj, 'indent'): obj.indent = 0 - if isinstance(obj, Object): + if isinstance(obj, _Object): new_objs = sorted(obj.children.values(), key=lineno_key, reverse=True) for ob in new_objs: From ed1b89208985f9d08426f03d2aa43b6008e8ea83 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Mon, 3 Jul 2017 15:23:29 -0400 Subject: [PATCH 8/9] Expand docstring for _create_tree --- Lib/pyclbr.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/Lib/pyclbr.py b/Lib/pyclbr.py index 72fb53affedf24..d4c22fd1fb0214 100644 --- a/Lib/pyclbr.py +++ b/Lib/pyclbr.py @@ -173,7 +173,17 @@ def _readmodule(module, path, inpackage=None): def _create_tree(fullmodule, path, fname, source, tree, inpackage): - "Return the tree for a particular module." + """Return the tree for a particular module. + + fullmodule (full module name), inpackage+module, becomes o.module. + path is passed to recursive calls of _readmodule. + fname becomes o.file. + source is tokenized. Imports cause recursive calls to _readmodule. + tree is {} or {'__path__': }. + inpackage, None or string, is passed to recursive calls of _readmodule. + + The effect of recursive calls is mutation of global _modules. + """ f = io.StringIO(source) stack = [] # Initialize stack of (class, indent) pairs. From 259616c97f4e7df12927b2ff62595f7f1453d1a6 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Mon, 3 Jul 2017 20:11:01 -0400 Subject: [PATCH 9/9] Expand helper functions; replace new tests. --- Doc/library/pyclbr.rst | 4 + Lib/pyclbr.py | 43 +++++----- Lib/test/test_pyclbr.py | 173 ++++++++++++++-------------------------- 3 files changed, 85 insertions(+), 135 deletions(-) diff --git a/Doc/library/pyclbr.rst b/Doc/library/pyclbr.rst index c53e4620516c29..ea34dd0638caf7 100644 --- a/Doc/library/pyclbr.rst +++ b/Doc/library/pyclbr.rst @@ -42,6 +42,10 @@ modules. returned dictionary has a key ``'__path__'`` whose value is a list containing the package search path. +.. versionadded:: 3.7 + Descriptors for nested definitions. They are accessed through the + new children attibute. Each has a new parent attribute. + The descriptors returned by these functions are instances of Function and Class classes. Users are not expected to create instances of these classes. diff --git a/Lib/pyclbr.py b/Lib/pyclbr.py index d4c22fd1fb0214..2c798df233b8eb 100644 --- a/Lib/pyclbr.py +++ b/Lib/pyclbr.py @@ -63,6 +63,12 @@ def _addchild(self, name, obj): self.children[name] = obj +class Function(_Object): + "Information about a Python function, including methods." + def __init__(self, module, name, file, lineno, parent=None): + _Object.__init__(self, module, name, file, lineno, parent) + + class Class(_Object): "Information about a Python class." def __init__(self, module, name, super, file, lineno, parent=None): @@ -74,21 +80,19 @@ def _addmethod(self, name, lineno): self.methods[name] = lineno -class Function(_Object): - "Information about a Python function, including methods." - def __init__(self, module, name, file, lineno, parent=None): - _Object.__init__(self, module, name, file, lineno, parent) - - -def _newfunction(ob, name, lineno): - "Return a Function after expanding ob to 3 arguments." - return Function(ob.module, name, ob.file, lineno, ob) - - -def _newclass(ob, name, super, lineno): - "Return a Class after expanding ob to 3 arguments." - return Class(ob.module, name, super, ob.file, lineno, ob) +def _nest_function(ob, func_name, lineno): + "Return a Function after nesting within ob." + newfunc = Function(ob.module, func_name, ob.file, lineno, ob) + ob._addchild(func_name, newfunc) + if isinstance(ob, Class): + ob._addmethod(func_name, lineno) + return newfunc +def _nest_class(ob, class_name, lineno, super=None): + "Return a Class after nesting within ob." + newclass = Class(ob.module, class_name, super, ob.file, lineno, ob) + ob._addchild(class_name, newclass) + return newclass def readmodule(module, path=None): """Return Class objects for the top-level classes in module. @@ -207,11 +211,7 @@ def _create_tree(fullmodule, path, fname, source, tree, inpackage): cur_func = None if stack: cur_obj = stack[-1][0] - cur_func = _newfunction(cur_obj, func_name, lineno) - cur_obj._addchild(func_name, cur_func) - if isinstance(cur_obj, Class): - # Function is a method. - cur_obj._addmethod(func_name, lineno) + cur_func = _nest_function(cur_obj, func_name, lineno) else: # It is just a function. cur_func = Function(fullmodule, func_name, fname, lineno) @@ -267,9 +267,8 @@ def _create_tree(fullmodule, path, fname, source, tree, inpackage): inherit = names if stack: cur_obj = stack[-1][0] - cur_class = _newclass(cur_obj, class_name, inherit, - lineno) - cur_obj._addchild(class_name, cur_class) + cur_class = _nest_class( + cur_obj, class_name, lineno, inherit) else: cur_class = Class(fullmodule, class_name, inherit, fname, lineno) diff --git a/Lib/test/test_pyclbr.py b/Lib/test/test_pyclbr.py index a11a4d879a4f8d..238eb71cd87a92 100644 --- a/Lib/test/test_pyclbr.py +++ b/Lib/test/test_pyclbr.py @@ -5,6 +5,7 @@ import os import sys +from textwrap import dedent from types import FunctionType, MethodType, BuiltinFunctionType import pyclbr from unittest import TestCase, main as unittest_main @@ -155,119 +156,65 @@ def test_decorators(self): self.checkModule('test.pyclbr_input', ignore=['om']) def test_nested(self): - - def clbr_from_tuple(tup, store, parent=None, lineno=1): - '''Create pyclbr objects from the given tuple t.''' - name = tup[0] - obj = pickp(name) - if parent is not None: - store = store[parent].children - ob_name = name.split()[1] - store[ob_name] = obj(name=ob_name, lineno=lineno, parent=parent) - parent = ob_name - - for item in tup[1:]: - lineno += 1 - if isinstance(item, str): - obj = pickp(item) - ob_name = item.split()[1] - store[parent].children[ob_name] = obj( - name=ob_name, lineno=lineno, parent=parent) - else: - lineno = clbr_from_tuple(item, store, parent, lineno) - - return lineno - - def tuple_to_py(tup, output, indent=0): - '''Write python code to output according to the given tuple.''' - name = tup[0] - output.write('{}{}():'.format(' ' * indent, name)) - indent += 2 - - if not tup[1:]: - output.write(' pass') - output.write('\n') - - for item in tup[1:]: - if isinstance(item, str): - output.write('{}{}(): pass\n'.format(' ' * indent, item)) - else: - tuple_to_py(item, output, indent) - - # Nested "thing" to test. - sample = ( - ("class A", - ("class B", - ("def a", - "def b")), - "def c"), - ("def d", - ("def e", - ("class C", - "def f") - ), - "def g") - ) - - pclass = partial(pyclbr.Class, module=None, file=None, super=None) - pfunc = partial(pyclbr.Function, module=None, file=None) - - def pickp(name): - return pclass if name.startswith('class') else pfunc - - # Create a module for storing the Python code. - dirname = os.path.abspath(support.TESTFN) - modname = 'notsupposedtoexist' - fname = os.path.join(dirname, modname) + os.extsep + 'py' - os.mkdir(dirname) - - # Create pyclbr objects from the sample above, and also convert - # the same sample above to Python code and write it to fname. - d = {} - lineno = 1 - with open(fname, 'w') as output: - for tup in sample: - newlineno = clbr_from_tuple(tup, d, lineno=lineno) - lineno = newlineno + 1 - tuple_to_py(tup, output) - - # Get the data returned by readmodule_ex to compare against - # our generated data. - try: - with support.DirsOnSysPath(dirname): - d_cmp = pyclbr.readmodule_ex(modname) - finally: - support.unlink(fname) - support.rmtree(dirname) - - # Finally perform the tests. - def check_objects(ob1, ob2): - self.assertEqual(ob1.lineno, ob2.lineno) - if ob1.parent is None: - self.assertIsNone(ob2.parent) - else: - # ob1 must come from our generated data since the parent - # attribute is always a string, while the ob2 must come - # from pyclbr which is always an Object instance. - self.assertEqual(ob1.parent, ob2.parent.name) - self.assertEqual( - ob1.__class__.__name__, - ob2.__class__.__name__) - self.assertEqual(ob1.children.keys(), ob2.children.keys()) - for name, obj in list(ob1.children.items()): - obj_cmp = ob2.children.pop(name) - del ob1.children[name] - check_objects(obj, obj_cmp) - - for name, obj in list(d.items()): - self.assertIn(name, d_cmp) - obj_cmp = d_cmp.pop(name) - del d[name] - check_objects(obj, obj_cmp) - self.assertFalse(obj.children) - self.assertFalse(obj_cmp.children) - self.assertFalse(d) - self.assertFalse(d_cmp) + mb = pyclbr + # Set arguments for descriptor creation and _creat_tree call. + m, p, f, t, i = 'test', '', 'test.py', {}, None + source = dedent("""\ + def f0: + def f1(a,b,c): + def f2(a=1, b=2, c=3): pass + return f1(a,b,d) + class c1: pass + class C0: + "Test class." + def F1(): + "Method." + return 'return' + class C1(): + class C2: + "Class nested within nested class." + def F3(): return 1+1 + + """) + actual = mb._create_tree(m, p, f, source, t, i) + + # Create descriptors, linked together, and expected dict. + f0 = mb.Function(m, 'f0', f, 1) + f1 = mb._nest_function(f0, 'f1', 2) + f2 = mb._nest_function(f1, 'f2', 3) + c1 = mb._nest_class(f0, 'c1', 5) + C0 = mb.Class(m, 'C0', None, f, 6) + F1 = mb._nest_function(C0, 'F1', 8) + C1 = mb._nest_class(C0, 'C1', 11) + C2 = mb._nest_class(C1, 'C2', 12) + F3 = mb._nest_function(C2, 'F3', 14) + expected = {'f0':f0, 'C0':C0} + + def compare(parent1, children1, parent2, children2): + """Return equality of tree pairs. + + Each parent,children pair define a tree. The parents are + assumed equal. Comparing the children dictionaries as such + does not work due to comparison by identity and double + linkage. We separate comparing string and number attributes + from comparing the children of input children. + """ + self.assertEqual(children1.keys(), children2.keys()) + for ob in children1.values(): + self.assertIs(ob.parent, parent1) + for ob in children2.values(): + self.assertIs(ob.parent, parent2) + for key in children1.keys(): + o1, o2 = children1[key], children2[key] + t1 = type(o1), o1.name, o1.file, o1.module, o1.lineno + t2 = type(o2), o2.name, o2.file, o2.module, o2.lineno + self.assertEqual(t1, t2) + if type(o1) is mb.Class: + self.assertEqual(o1.methods, o2.methods) + # Skip superclasses for now as not part of example + compare(o1, o1.children, o2, o2.children) + + compare(None, actual, None, expected) def test_others(self): cm = self.checkModule