gh-123987: Fix issues in importlib.resources.

Author: jaraco

Lib/importlib/resources/__init__.py

@@ -1,4 +1,11 @@
-"""Read resources contained within a package."""
+"""
+Read resources contained within a package.
+
+This codebase is shared between importlib.resources in the stdlib
+and importlib_resources in PyPI. See
+https://github.com/python/importlib_metadata/wiki/Development-Methodology
+for more detail.
+"""
 
 from ._common import (
     as_file,

Lib/importlib/resources/_common.py

@@ -66,10 +66,10 @@ def get_resource_reader(package: types.ModuleType) -> Optional[ResourceReader]:
     # zipimport.zipimporter does not support weak references, resulting in a
     # TypeError.  That seems terrible.
     spec = package.__spec__
-    reader = getattr(spec.loader, 'get_resource_reader', None)  # type: ignore
+    reader = getattr(spec.loader, 'get_resource_reader', None)  # type: ignore[union-attr]
     if reader is None:
         return None
-    return reader(spec.name)  # type: ignore
+    return reader(spec.name)  # type: ignore[union-attr]
 
 
 @functools.singledispatch
@@ -93,12 +93,13 @@ def _infer_caller():
     """
 
     def is_this_file(frame_info):
-        return frame_info.filename == __file__
+        return frame_info.filename == stack[0].filename
 
     def is_wrapper(frame_info):
         return frame_info.function == 'wrapper'
 
-    not_this_file = itertools.filterfalse(is_this_file, inspect.stack())
+    stack = inspect.stack()
+    not_this_file = itertools.filterfalse(is_this_file, stack)
     # also exclude 'wrapper' due to singledispatch in the call stack
     callers = itertools.filterfalse(is_wrapper, not_this_file)
     return next(callers).frame

Lib/importlib/resources/readers.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import collections
 import contextlib
 import itertools
@@ -6,6 +8,7 @@
 import re
 import warnings
 import zipfile
+from collections.abc import Iterator
 
 from . import abc
 
@@ -135,27 +138,31 @@ class NamespaceReader(abc.TraversableResources):
     def __init__(self, namespace_path):
         if 'NamespacePath' not in str(namespace_path):
             raise ValueError('Invalid path')
-        self.path = MultiplexedPath(*map(self._resolve, namespace_path))
+        self.path = MultiplexedPath(*filter(bool, map(self._resolve, namespace_path)))
 
     @classmethod
-    def _resolve(cls, path_str) -> abc.Traversable:
+    def _resolve(cls, path_str) -> abc.Traversable | None:
         r"""
         Given an item from a namespace path, resolve it to a Traversable.
 
         path_str might be a directory on the filesystem or a path to a
         zipfile plus the path within the zipfile, e.g. ``/foo/bar`` or
         ``/foo/baz.zip/inner_dir`` or ``foo\baz.zip\inner_dir\sub``.
+
+        path_str might also be a sentinel used by editable packages to
+        trigger other behaviors (see python/importlib_resources#311).
+        In that case, return None.
         """
-        (dir,) = (cand for cand in cls._candidate_paths(path_str) if cand.is_dir())
-        return dir
+        dirs = (cand for cand in cls._candidate_paths(path_str) if cand.is_dir())
+        return next(dirs, None)
 
     @classmethod
-    def _candidate_paths(cls, path_str):
+    def _candidate_paths(cls, path_str: str) -> Iterator[abc.Traversable]:
         yield pathlib.Path(path_str)
         yield from cls._resolve_zip_path(path_str)
 
     @staticmethod
-    def _resolve_zip_path(path_str):
+    def _resolve_zip_path(path_str: str):
         for match in reversed(list(re.finditer(r'[\\/]', path_str))):
             with contextlib.suppress(
                 FileNotFoundError,

Lib/importlib/resources/simple.py

@@ -77,7 +77,7 @@ class ResourceHandle(Traversable):
 
     def __init__(self, parent: ResourceContainer, name: str):
         self.parent = parent
-        self.name = name  # type: ignore
+        self.name = name  # type: ignore[misc]
 
     def is_file(self):
         return True

Lib/test/test_importlib/resources/_path.py

@@ -2,15 +2,44 @@
 import functools
 
 from typing import Dict, Union
+from typing import runtime_checkable
+from typing import Protocol
 
 
 ####
-# from jaraco.path 3.4.1
+# from jaraco.path 3.7.1
 
-FilesSpec = Dict[str, Union[str, bytes, 'FilesSpec']]  # type: ignore
 
+class Symlink(str):
+    """
+    A string indicating the target of a symlink.
+    """
+
+
+FilesSpec = Dict[str, Union[str, bytes, Symlink, 'FilesSpec']]
+
+
+@runtime_checkable
+class TreeMaker(Protocol):
+    def __truediv__(self, *args, **kwargs): ...  # pragma: no cover
+
+    def mkdir(self, **kwargs): ...  # pragma: no cover
+
+    def write_text(self, content, **kwargs): ...  # pragma: no cover
+
+    def write_bytes(self, content): ...  # pragma: no cover
 
-def build(spec: FilesSpec, prefix=pathlib.Path()):
+    def symlink_to(self, target): ...  # pragma: no cover
+
+
+def _ensure_tree_maker(obj: Union[str, TreeMaker]) -> TreeMaker:
+    return obj if isinstance(obj, TreeMaker) else pathlib.Path(obj)  # type: ignore[return-value]
+
+
+def build(
+    spec: FilesSpec,
+    prefix: Union[str, TreeMaker] = pathlib.Path(),  # type: ignore[assignment]
+):
     """
     Build a set of files/directories, as described by the spec.
 
@@ -25,21 +54,25 @@ def build(spec: FilesSpec, prefix=pathlib.Path()):
     ...             "__init__.py": "",
     ...         },
     ...         "baz.py": "# Some code",
-    ...     }
+    ...         "bar.py": Symlink("baz.py"),
+    ...     },
+    ...     "bing": Symlink("foo"),
     ... }
     >>> target = getfixture('tmp_path')
     >>> build(spec, target)
     >>> target.joinpath('foo/baz.py').read_text(encoding='utf-8')
     '# Some code'
+    >>> target.joinpath('bing/bar.py').read_text(encoding='utf-8')
+    '# Some code'
     """
     for name, contents in spec.items():
-        create(contents, pathlib.Path(prefix) / name)
+        create(contents, _ensure_tree_maker(prefix) / name)
 
 
 @functools.singledispatch
 def create(content: Union[str, bytes, FilesSpec], path):
     path.mkdir(exist_ok=True)
-    build(content, prefix=path)  # type: ignore
+    build(content, prefix=path)  # type: ignore[arg-type]
 
 
 @create.register
@@ -52,5 +85,10 @@ def _(content: str, path):
     path.write_text(content, encoding='utf-8')
 
 
+@create.register
+def _(content: Symlink, path):
+    path.symlink_to(content)
+
+
 # end from jaraco.path
 ####

Lib/test/test_importlib/resources/test_files.py

@@ -1,3 +1,7 @@
+import os
+import pathlib
+import py_compile
+import shutil
 import textwrap
 import unittest
 import warnings
@@ -7,6 +11,7 @@
 from importlib import resources
 from importlib.resources.abc import Traversable
 from . import util
+from test.support import os_helper, import_helper
 
 
 @contextlib.contextmanager
@@ -55,6 +60,26 @@ class OpenZipTests(FilesTests, util.ZipSetup, unittest.TestCase):
 class OpenNamespaceTests(FilesTests, util.DiskSetup, unittest.TestCase):
     MODULE = 'namespacedata01'
 
+    def test_non_paths_in_dunder_path(self):
+        """
+        Non-path items in a namespace package's ``__path__`` are ignored.
+
+        As reported in python/importlib_resources#311, some tools
+        like Setuptools, when creating editable packages, will inject
+        non-paths into a namespace package's ``__path__``, a
+        sentinel like
+        ``__editable__.sample_namespace-1.0.finder.__path_hook__``
+        to cause the ``PathEntryFinder`` to be called when searching
+        for packages. In that case, resources should still be loadable.
+        """
+        import namespacedata01
+
+        namespacedata01.__path__.append(
+            '__editable__.sample_namespace-1.0.finder.__path_hook__'
+        )
+
+        resources.files(namespacedata01)
+
 
 class OpenNamespaceZipTests(FilesTests, util.ZipSetup, unittest.TestCase):
     ZIP_MODULE = 'namespacedata01'
@@ -81,7 +106,7 @@ def test_module_resources(self):
         """
         A module can have resources found adjacent to the module.
         """
-        import mod
+        import mod  # type: ignore[import-not-found]
 
         actual = resources.files(mod).joinpath('res.txt').read_text(encoding='utf-8')
         assert actual == self.spec['res.txt']
@@ -97,8 +122,8 @@ class ModuleFilesZipTests(DirectSpec, util.ZipSetup, ModulesFiles, unittest.Test
 
 class ImplicitContextFiles:
     set_val = textwrap.dedent(
-        """
-        import importlib.resources as res
+        f"""
+        import {resources.__name__} as res
         val = res.files().joinpath('res.txt').read_text(encoding='utf-8')
         """
     )
@@ -108,6 +133,10 @@ class ImplicitContextFiles:
             'submod.py': set_val,
             'res.txt': 'resources are the best',
         },
+        'frozenpkg': {
+            '__init__.py': set_val.replace(resources.__name__, 'c_resources'),
+            'res.txt': 'resources are the best',
+        },
     }
 
     def test_implicit_files_package(self):
@@ -122,6 +151,32 @@ def test_implicit_files_submodule(self):
         """
         assert importlib.import_module('somepkg.submod').val == 'resources are the best'
 
+    def _compile_importlib(self):
+        """
+        Make a compiled-only copy of the importlib resources package.
+        """
+        bin_site = self.fixtures.enter_context(os_helper.temp_dir())
+        c_resources = pathlib.Path(bin_site, 'c_resources')
+        sources = pathlib.Path(resources.__file__).parent
+        shutil.copytree(sources, c_resources, ignore=lambda *_: ['__pycache__'])
+
+        for dirpath, _, filenames in os.walk(c_resources):
+            for filename in filenames:
+                source_path = pathlib.Path(dirpath) / filename
+                cfile = source_path.with_suffix('.pyc')
+                py_compile.compile(source_path, cfile)
+                pathlib.Path.unlink(source_path)
+        self.fixtures.enter_context(import_helper.DirsOnSysPath(bin_site))
+
+    def test_implicit_files_with_compiled_importlib(self):
+        """
+        Caller detection works for compiled-only resources module.
+
+        python/cpython#123085
+        """
+        self._compile_importlib()
+        assert importlib.import_module('frozenpkg').val == 'resources are the best'
+
 
 class ImplicitContextFilesDiskTests(
     DirectSpec, util.DiskSetup, ImplicitContextFiles, unittest.TestCase

Misc/NEWS.d/next/Library/2024-09-12-14-24-25.gh-issue-123987.7_OD1p.rst

@@ -0,0 +1,3 @@
+Fixed issue in NamespaceReader where a non-path item in a namespace path,
+such as a sentinel added by an editable installer, would break resource
+loading.