Merge pull request #632 from nsoranzo/python312-imp-module_past

Python 3.12 support

Author: edschofield

setup.py

@@ -103,6 +103,11 @@
     "Programming Language :: Python :: 3.5",
     "Programming Language :: Python :: 3.6",
     "Programming Language :: Python :: 3.7",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
     "License :: OSI Approved",
     "License :: OSI Approved :: MIT License",
     "Development Status :: 4 - Beta",

src/future/backports/test/support.py

@@ -28,10 +28,6 @@
 # import collections.abc    # not present on Py2.7
 import re
 import subprocess
-try:
-    from imp import cache_from_source
-except ImportError:
-    from importlib.util import cache_from_source
 import time
 try:
     import sysconfig
@@ -344,37 +340,6 @@ def rmtree(path):
         if error.errno != errno.ENOENT:
             raise
 
-def make_legacy_pyc(source):
-    """Move a PEP 3147 pyc/pyo file to its legacy pyc/pyo location.
-
-    The choice of .pyc or .pyo extension is done based on the __debug__ flag
-    value.
-
-    :param source: The file system path to the source file.  The source file
-        does not need to exist, however the PEP 3147 pyc file must exist.
-    :return: The file system path to the legacy pyc file.
-    """
-    pyc_file = cache_from_source(source)
-    up_one = os.path.dirname(os.path.abspath(source))
-    legacy_pyc = os.path.join(up_one, source + ('c' if __debug__ else 'o'))
-    os.rename(pyc_file, legacy_pyc)
-    return legacy_pyc
-
-def forget(modname):
-    """'Forget' a module was ever imported.
-
-    This removes the module from sys.modules and deletes any PEP 3147 or
-    legacy .pyc and .pyo files.
-    """
-    unload(modname)
-    for dirname in sys.path:
-        source = os.path.join(dirname, modname + '.py')
-        # It doesn't matter if they exist or not, unlink all possible
-        # combinations of PEP 3147 and legacy pyc and pyo files.
-        unlink(source + 'c')
-        unlink(source + 'o')
-        unlink(cache_from_source(source, debug_override=True))
-        unlink(cache_from_source(source, debug_override=False))
 
 # On some platforms, should not run gui test even if it is allowed
 # in `use_resources'.

src/future/moves/test/support.py

@@ -1,9 +1,18 @@
 from __future__ import absolute_import
+
+import sys
+
 from future.standard_library import suspend_hooks
 from future.utils import PY3
 
 if PY3:
     from test.support import *
+    if sys.version_info[:2] >= (3, 10):
+        from test.support.os_helper import (
+            EnvironmentVarGuard,
+            TESTFN,
+        )
+        from test.support.warnings_helper import check_warnings
 else:
     __future_module__ = True
     with suspend_hooks():

src/future/standard_library/__init__.py

@@ -62,12 +62,7 @@
 
 import sys
 import logging
-try:
-    import importlib
-except ImportError:
-    import imp
 import contextlib
-import types
 import copy
 import os
 
@@ -82,6 +77,9 @@
 
 from future.utils import PY2, PY3
 
+if PY2:
+    import imp
+
 # The modules that are defined under the same names on Py3 but with
 # different contents in a significant way (e.g. submodules) are:
 #   pickle (fast one)
@@ -300,11 +298,8 @@ def _find_and_load_module(self, name, path=None):
                 flog.debug('What to do here?')
 
         name = bits[0]
-        try:
-            module_info = imp.find_module(name, path)
-            return imp.load_module(name, *module_info)
-        except AttributeError:
-            return importlib.import_module(name, path)
+        module_info = imp.find_module(name, path)
+        return imp.load_module(name, *module_info)
 
 
 class hooks(object):

src/past/translation/__init__.py

@@ -32,12 +32,7 @@
 Inspired by and based on ``uprefix`` by Vinay M. Sajip.
 """
 
-try:
-    import imp
-except ImportError:
-    import importlib
 import logging
-import marshal
 import os
 import sys
 import copy
@@ -46,6 +41,17 @@
 
 from libfuturize import fixes
 
+try:
+    from importlib.machinery import (
+        PathFinder,
+        SourceFileLoader,
+    )
+except ImportError:
+    PathFinder = None
+    SourceFileLoader = object
+
+if sys.version_info[:2] < (3, 4):
+    import imp
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.DEBUG)
@@ -228,6 +234,81 @@ def detect_python2(source, pathname):
         return False
 
 
+def transform(source, pathname):
+    # This implementation uses lib2to3,
+    # you can override and use something else
+    # if that's better for you
+
+    # lib2to3 likes a newline at the end
+    RTs.setup()
+    source += '\n'
+    try:
+        tree = RTs._rt.refactor_string(source, pathname)
+    except ParseError as e:
+        if e.msg != 'bad input' or e.value != '=':
+            raise
+        tree = RTs._rtp.refactor_string(source, pathname)
+    # could optimise a bit for only doing str(tree) if
+    # getattr(tree, 'was_changed', False) returns True
+    return str(tree)[:-1]  # remove added newline
+
+
+class PastSourceFileLoader(SourceFileLoader):
+    exclude_paths = []
+    include_paths = []
+
+    def _convert_needed(self):
+        fullname = self.name
+        if any(fullname.startswith(path) for path in self.exclude_paths):
+            convert = False
+        elif any(fullname.startswith(path) for path in self.include_paths):
+            convert = True
+        else:
+            convert = False
+        return convert
+
+    def _exec_transformed_module(self, module):
+        source = self.get_source(self.name)
+        pathname = self.path
+        if detect_python2(source, pathname):
+            source = transform(source, pathname)
+        code = compile(source, pathname, "exec")
+        exec(code, module.__dict__)
+
+    # For Python 3.3
+    def load_module(self, fullname):
+        logger.debug("Running load_module for %s", fullname)
+        if fullname in sys.modules:
+            mod = sys.modules[fullname]
+        else:
+            if self._convert_needed():
+                logger.debug("Autoconverting %s", fullname)
+                mod = imp.new_module(fullname)
+                sys.modules[fullname] = mod
+
+                # required by PEP 302
+                mod.__file__ = self.path
+                mod.__loader__ = self
+                if self.is_package(fullname):
+                    mod.__path__ = []
+                    mod.__package__ = fullname
+                else:
+                    mod.__package__ = fullname.rpartition('.')[0]
+                self._exec_transformed_module(mod)
+            else:
+                mod = super().load_module(fullname)
+        return mod
+
+    # For Python >=3.4
+    def exec_module(self, module):
+        logger.debug("Running exec_module for %s", module)
+        if self._convert_needed():
+            logger.debug("Autoconverting %s", self.name)
+            self._exec_transformed_module(module)
+        else:
+            super().exec_module(module)
+
+
 class Py2Fixer(object):
     """
     An import hook class that uses lib2to3 for source-to-source translation of
@@ -261,151 +342,30 @@ def exclude(self, paths):
         """
         self.exclude_paths += paths
 
+    # For Python 3.3
     def find_module(self, fullname, path=None):
-        logger.debug('Running find_module: {0}...'.format(fullname))
-        if '.' in fullname:
-            parent, child = fullname.rsplit('.', 1)
-            if path is None:
-                loader = self.find_module(parent, path)
-                mod = loader.load_module(parent)
-                path = mod.__path__
-            fullname = child
-
-        # Perhaps we should try using the new importlib functionality in Python
-        # 3.3: something like this?
-        # thing = importlib.machinery.PathFinder.find_module(fullname, path)
-        try:
-            self.found = imp.find_module(fullname, path)
-        except Exception as e:
-            logger.debug('Py2Fixer could not find {0}')
-            logger.debug('Exception was: {0})'.format(fullname, e))
+        logger.debug("Running find_module: (%s, %s)", fullname, path)
+        loader = PathFinder.find_module(fullname, path)
+        if not loader:
+            logger.debug("Py2Fixer could not find %s", fullname)
             return None
-        self.kind = self.found[-1][-1]
-        if self.kind == imp.PKG_DIRECTORY:
-            self.pathname = os.path.join(self.found[1], '__init__.py')
-        elif self.kind == imp.PY_SOURCE:
-            self.pathname = self.found[1]
-        return self
-
-    def transform(self, source):
-        # This implementation uses lib2to3,
-        # you can override and use something else
-        # if that's better for you
-
-        # lib2to3 likes a newline at the end
-        RTs.setup()
-        source += '\n'
-        try:
-            tree = RTs._rt.refactor_string(source, self.pathname)
-        except ParseError as e:
-            if e.msg != 'bad input' or e.value != '=':
-                raise
-            tree = RTs._rtp.refactor_string(source, self.pathname)
-        # could optimise a bit for only doing str(tree) if
-        # getattr(tree, 'was_changed', False) returns True
-        return str(tree)[:-1] # remove added newline
-
-    def load_module(self, fullname):
-        logger.debug('Running load_module for {0}...'.format(fullname))
-        if fullname in sys.modules:
-            mod = sys.modules[fullname]
-        else:
-            if self.kind in (imp.PY_COMPILED, imp.C_EXTENSION, imp.C_BUILTIN,
-                             imp.PY_FROZEN):
-                convert = False
-            # elif (self.pathname.startswith(_stdlibprefix)
-            #       and 'site-packages' not in self.pathname):
-            #     # We assume it's a stdlib package in this case. Is this too brittle?
-            #     # Please file a bug report at https://github.com/PythonCharmers/python-future
-            #     # if so.
-            #     convert = False
-            # in theory, other paths could be configured to be excluded here too
-            elif any([fullname.startswith(path) for path in self.exclude_paths]):
-                convert = False
-            elif any([fullname.startswith(path) for path in self.include_paths]):
-                convert = True
-            else:
-                convert = False
-            if not convert:
-                logger.debug('Excluded {0} from translation'.format(fullname))
-                mod = imp.load_module(fullname, *self.found)
-            else:
-                logger.debug('Autoconverting {0} ...'.format(fullname))
-                mod = imp.new_module(fullname)
-                sys.modules[fullname] = mod
-
-                # required by PEP 302
-                mod.__file__ = self.pathname
-                mod.__name__ = fullname
-                mod.__loader__ = self
-
-                # This:
-                #     mod.__package__ = '.'.join(fullname.split('.')[:-1])
-                # seems to result in "SystemError: Parent module '' not loaded,
-                # cannot perform relative import" for a package's __init__.py
-                # file. We use the approach below. Another option to try is the
-                # minimal load_module pattern from the PEP 302 text instead.
-
-                # Is the test in the next line more or less robust than the
-                # following one? Presumably less ...
-                # ispkg = self.pathname.endswith('__init__.py')
-
-                if self.kind == imp.PKG_DIRECTORY:
-                    mod.__path__ = [ os.path.dirname(self.pathname) ]
-                    mod.__package__ = fullname
-                else:
-                    #else, regular module
-                    mod.__path__ = []
-                    mod.__package__ = fullname.rpartition('.')[0]
+        loader.__class__ = PastSourceFileLoader
+        loader.exclude_paths = self.exclude_paths
+        loader.include_paths = self.include_paths
+        return loader
+
+    # For Python >=3.4
+    def find_spec(self, fullname, path=None, target=None):
+        logger.debug("Running find_spec: (%s, %s, %s)", fullname, path, target)
+        spec = PathFinder.find_spec(fullname, path, target)
+        if not spec:
+            logger.debug("Py2Fixer could not find %s", fullname)
+            return None
+        spec.loader.__class__ = PastSourceFileLoader
+        spec.loader.exclude_paths = self.exclude_paths
+        spec.loader.include_paths = self.include_paths
+        return spec
 
-                try:
-                    cachename = imp.cache_from_source(self.pathname)
-                    if not os.path.exists(cachename):
-                        update_cache = True
-                    else:
-                        sourcetime = os.stat(self.pathname).st_mtime
-                        cachetime = os.stat(cachename).st_mtime
-                        update_cache = cachetime < sourcetime
-                    # # Force update_cache to work around a problem with it being treated as Py3 code???
-                    # update_cache = True
-                    if not update_cache:
-                        with open(cachename, 'rb') as f:
-                            data = f.read()
-                            try:
-                                code = marshal.loads(data)
-                            except Exception:
-                                # pyc could be corrupt. Regenerate it
-                                update_cache = True
-                    if update_cache:
-                        if self.found[0]:
-                            source = self.found[0].read()
-                        elif self.kind == imp.PKG_DIRECTORY:
-                            with open(self.pathname) as f:
-                                source = f.read()
-
-                        if detect_python2(source, self.pathname):
-                            source = self.transform(source)
-
-                        code = compile(source, self.pathname, 'exec')
-
-                        dirname = os.path.dirname(cachename)
-                        try:
-                            if not os.path.exists(dirname):
-                                os.makedirs(dirname)
-                            with open(cachename, 'wb') as f:
-                                data = marshal.dumps(code)
-                                f.write(data)
-                        except Exception:   # could be write-protected
-                            pass
-                    exec(code, mod.__dict__)
-                except Exception as e:
-                    # must remove module from sys.modules
-                    del sys.modules[fullname]
-                    raise # keep it simple
-
-        if self.found[0]:
-            self.found[0].close()
-        return mod
 
 _hook = Py2Fixer()
 

tests/test_future/test_builtins.py

@@ -1303,8 +1303,11 @@ def test_pow(self):
         self.assertAlmostEqual(pow(-1, 0.5), 1j)
         self.assertAlmostEqual(pow(-1, 1/3), 0.5 + 0.8660254037844386j)
 
-        # Raises TypeError in Python < v3.5, ValueError in v3.5:
-        self.assertRaises((TypeError, ValueError), pow, -1, -2, 3)
+        # Raises TypeError in Python < v3.5, ValueError in v3.5-v3.7:
+        if sys.version_info[:2] < (3, 8):
+            self.assertRaises((TypeError, ValueError), pow, -1, -2, 3)
+        else:
+            self.assertEqual(pow(-1, -2, 3), 1)
         self.assertRaises(ValueError, pow, 1, 2, 0)
 
         self.assertRaises(TypeError, pow)

tests/test_future/test_standard_library.py

@@ -9,7 +9,6 @@
 
 import sys
 import tempfile
-import os
 import copy
 import textwrap
 from subprocess import CalledProcessError
@@ -448,13 +447,11 @@ def test_reload(self):
         reload has been moved to the imp module
         """
         try:
-            import imp
-            imp.reload(imp)
-            self.assertTrue(True)
+            from importlib import reload
         except ImportError:
-            import importlib
-            importlib.reload(importlib)
-            self.assertTrue(True)
+            from imp import reload
+        reload(sys)
+        self.assertTrue(True)
 
     def test_install_aliases(self):
         """

tests/test_future/test_urllib2.py

@@ -691,10 +691,6 @@ def connect_ftp(self, user, passwd, host, port, dirs,
         h = NullFTPHandler(data)
         h.parent = MockOpener()
 
-        # MIME guessing works in Python 3.8!
-        guessed_mime = None
-        if sys.hexversion >= 0x03080000:
-            guessed_mime = "image/gif"
         for url, host, port, user, passwd, type_, dirs, filename, mimetype in [
             ("ftp://localhost/foo/bar/baz.html",
              "localhost", ftplib.FTP_PORT, "", "", "I",
@@ -713,7 +709,7 @@ def connect_ftp(self, user, passwd, host, port, dirs,
              ["foo", "bar"], "", None),
             ("ftp://localhost/baz.gif;type=a",
              "localhost", ftplib.FTP_PORT, "", "", "A",
-             [], "baz.gif", guessed_mime),
+             [], "baz.gif", None),
             ]:
             req = Request(url)
             req.timeout = None

tests/test_future/test_urllib_toplevel.py

@@ -781,8 +781,11 @@ def test_unquoting(self):
                          "%s" % result)
         self.assertRaises((TypeError, AttributeError), urllib_parse.unquote, None)
         self.assertRaises((TypeError, AttributeError), urllib_parse.unquote, ())
-        with support.check_warnings(('', BytesWarning), quiet=True):
-            self.assertRaises((TypeError, AttributeError), urllib_parse.unquote, bytes(b''))
+        if sys.version_info[:2] < (3, 9):
+            with support.check_warnings(('', BytesWarning), quiet=True):
+                self.assertRaises((TypeError, AttributeError), urllib_parse.unquote, bytes(b''))
+        else:
+            self.assertEqual(urllib_parse.unquote(bytes(b"")), "")
 
     def test_unquoting_badpercent(self):
         # Test unquoting on bad percent-escapes

tests/test_future/test_utils.py

@@ -150,7 +150,7 @@ class Timeout(BaseException):
         self.assertRaises(Timeout, raise_, Timeout())
 
         if PY3:
-            self.assertRaisesRegexp(
+            self.assertRaisesRegex(
                 TypeError, "class must derive from BaseException",
                 raise_, int)
 

tests/test_past/test_builtins.py

@@ -6,7 +6,6 @@
 from past.builtins import apply, cmp, execfile, intern, raw_input
 from past.builtins import reduce, reload, unichr, unicode, xrange
 
-from future import standard_library
 from future.backports.test.support import TESTFN     #, run_unittest
 import tempfile
 import os

tests/test_past/test_translation.py

@@ -7,18 +7,18 @@
 import os
 import textwrap
 import sys
-import pprint
 import tempfile
-import os
 import io
-from subprocess import Popen, PIPE
-
-from past import utils
-from past.builtins import basestring, str as oldstr, unicode
 
+from future.tests.base import (
+    expectedFailurePY3,
+    unittest,
+)
+from past.builtins import (
+    str as oldstr,
+    unicode,
+)
 from past.translation import install_hooks, remove_hooks, common_substring
-from future.tests.base import (unittest, CodeHandler, skip26,
-                               expectedFailurePY3, expectedFailurePY26)
 
 
 class TestTranslate(unittest.TestCase):
@@ -58,8 +58,8 @@ def write_and_import(self, code, modulename='mymodule'):
         sys.path.insert(0, self.tempdir)
         try:
             module = __import__(modulename)
-        except SyntaxError:
-            print('Bombed!')
+        except SyntaxError as e:
+            print('Import failed: %s' % e)
         else:
             print('Succeeded!')
         finally: