diff --git a/README.md b/README.md index 8bab031c..7d2587bf 100644 --- a/README.md +++ b/README.md @@ -220,6 +220,24 @@ wheel.exclude = [] # The build tag to use for the wheel. If empty, no build tag is used. wheel.build-tag = "" +# Do automatic repairs of the compiled binaries and libraries. +wheel.repair.enable = false + +# Patch the dynamic links to libraries installed in the current wheel. +wheel.repair.in-wheel = true + +# Patch the dynamic links to libraries in other wheels. +wheel.repair.cross-wheel = false + +# A list of external library files that will be bundled in the wheel. +wheel.repair.bundle-external = [] + +# Automatically patch every top-level packages/modules to import the dlls on Windows wheels. +wheel.repair.patch-imports = true + +# The generated file containing any necessary top-level imports. +wheel.repair.imports-file = "" + # If CMake is less than this value, backport a copy of FindPython. backport.find-python = "3.26.1" diff --git a/docs/api/scikit_build_core.repair_wheel.rst b/docs/api/scikit_build_core.repair_wheel.rst new file mode 100644 index 00000000..a6068ff0 --- /dev/null +++ b/docs/api/scikit_build_core.repair_wheel.rst @@ -0,0 +1,42 @@ +scikit\_build\_core.repair\_wheel package +========================================= + +.. automodule:: scikit_build_core.repair_wheel + :members: + :show-inheritance: + :undoc-members: + +Submodules +---------- + +scikit\_build\_core.repair\_wheel.darwin module +----------------------------------------------- + +.. automodule:: scikit_build_core.repair_wheel.darwin + :members: + :show-inheritance: + :undoc-members: + +scikit\_build\_core.repair\_wheel.linux module +---------------------------------------------- + +.. automodule:: scikit_build_core.repair_wheel.linux + :members: + :show-inheritance: + :undoc-members: + +scikit\_build\_core.repair\_wheel.rpath module +---------------------------------------------- + +.. automodule:: scikit_build_core.repair_wheel.rpath + :members: + :show-inheritance: + :undoc-members: + +scikit\_build\_core.repair\_wheel.windows module +------------------------------------------------ + +.. automodule:: scikit_build_core.repair_wheel.windows + :members: + :show-inheritance: + :undoc-members: diff --git a/docs/api/scikit_build_core.rst b/docs/api/scikit_build_core.rst index e1d25de2..b5afb3cb 100644 --- a/docs/api/scikit_build_core.rst +++ b/docs/api/scikit_build_core.rst @@ -18,6 +18,7 @@ Subpackages scikit_build_core.file_api scikit_build_core.hatch scikit_build_core.metadata + scikit_build_core.repair_wheel scikit_build_core.resources scikit_build_core.settings scikit_build_core.setuptools diff --git a/docs/guide/dynamic_link.md b/docs/guide/dynamic_link.md index 2cff7297..bc9fb962 100644 --- a/docs/guide/dynamic_link.md +++ b/docs/guide/dynamic_link.md @@ -33,6 +33,89 @@ name collision if the same library is being bundled by a different package, and check if the packages confirm to standards like [PEP600] (`manylinux_X_Y`). These tools do not allow to have cross wheel library dependency. +## scikit-build-core wheel repair + +:::{warning} + +This feature is experimental and API and effects may change. + +::: + +scikit-build-core also provides a built-in wheel repair which is enabled from +`wheel.repair.enable`. Unlike the [wheel repair tools], this feature uses the +linking information used during the CMake steps. + +:::{note} + +Executables, libraries, dependencies installed in `${SKBUILD_SCRIPTS_DIR}` or +`${SKBUILD_DATA_DIR}` are not considered. Only files in `wheel.install-dir` or +`${SKBUILD_PLATLIB_DIR}` are considered. + +::: + +So far there are 3 repair features implemented, which can be activated +independently. + +### `wheel.repair.in-wheel` + +If this feature is enabled, it patches the executable/libraries so that, if the +dependency is packaged in the _same_ wheel, the executable/libraries point to +the dependency files inside the wheel. + +### `wheel.repair.cross-wheel` + +If this feature is enabled, it patches the executable/libraries so that, if the +dependency is packaged in a _different_ wheel available from +`build-system.requires`, the executable/libraries point to the dependency files +in that other wheel. + +The same/compatible library that was used in the `build-system.requires` should +be used in the project's dependencies. The link to the other wheel will have +priority, but if that wheel is not installed or is incompatible, it will +fall-through to the system dependencies. + +### `wheel.repair.bundle-external` + +This feature is enabled by providing a list of regex patterns of the dynamic +libraries that should be bundled. Only the filename is considered for the regex +matching. The dependency files are then copied to a folder `{project.name}.libs` +and the dependents are patched to point to there. + +External libraries linked from a different wheel available from +`build-system.requires` are not considered. + +:::{warning} + +Unlike the [wheel repair tools], this feature does not mangle the library names, +which may cause issues if multiple dependencies link to the same library with +the same `SONAME`/`SOVERSION` (usually just the library file name). + +::: + +### Windows repairs + +The windows wheel repairs are done by adding `os.add_dll_directory` commands to +the top-level python package/modules in the current wheel. Thus, the library +linkage is only available when executing a python script/module that import the +current wheel's top-level python package/modules. + +In contrast, in Unix systems the libraries and executable are patched directly +and are available outside of the python environment as well. + +### Beware of library load order + +Beware if there are multiple dynamic libraries in other wheels or even on the +system with the same `SONAME`/`SOVERSION` (usually just the library file name). +Depending on the order of python or other script execution, the other libraries +(not the ones that were patched to be linked to) may be loaded first, and when +your libraries are loaded, the dependencies that have already been loaded will +be used instead of the ones that were patched to be linked to. + +If you want to avoid this, consider using the [wheel repair tools] which always +bundle and mangle the libraries appropriately to preserve the consistency. +However, this also makes it impossible to link/fallback to system libraries or +link to a shared library in a different wheel. + ## Manual patching You can manually make a relative RPath. This has the benefit of working when not @@ -71,5 +154,6 @@ os.add_dll_directory(str(dependency_dll_path)) [cibuildwheel]: https://cibuildwheel.pypa.io/en/stable/ [repair wheel]: https://cibuildwheel.pypa.io/en/stable/options/#repair-wheel-command [PEP600]: https://peps.python.org/pep-0600 +[wheel repair tools]: #wheel-repair-tools diff --git a/docs/reference/configs.md b/docs/reference/configs.md index 8c6e8b85..0e4e1f61 100644 --- a/docs/reference/configs.md +++ b/docs/reference/configs.md @@ -574,4 +574,71 @@ print(mk_skbuild_docs()) This value is used to construct ``SKBUILD_SABI_COMPONENT`` CMake variable. ``` +## wheel.repair + +```{eval-rst} +.. confval:: wheel.repair.bundle-external + :type: ``list[str]`` + + A list of external library files that will be bundled in the wheel. + + Each entry is treated as a regex pattern, and only the filenames are considered + for the match. The libraries are taken from the CMake dependency during the CMake + build. The bundled libraries are installed under ``site-packages/${name}.libs`` +``` + +```{eval-rst} +.. confval:: wheel.repair.cross-wheel + :type: ``bool`` + :default: false + + Patch the dynamic links to libraries in other wheels. + + .. note:: + This may result in incompatible wheels. Use this only if the + wheels are strongly linked to each other and strict manylinux compliance is + not required. +``` + +```{eval-rst} +.. confval:: wheel.repair.enable + :type: ``bool`` + :default: false + + Do automatic repairs of the compiled binaries and libraries. + + .. warning:: + This is an experimental feature gated by :confval:`experimental` +``` + +```{eval-rst} +.. confval:: wheel.repair.imports-file + :type: ``Path`` + + The generated file containing any necessary top-level imports. + + This files should be imported as early as possible in all top-level modules and packages. + + On Windows wheels, this file contains all ``os.add_dll_directory`` needed in the current wheel. + On other OS, this is an empty file. +``` + +```{eval-rst} +.. confval:: wheel.repair.in-wheel + :type: ``bool`` + :default: true + + Patch the dynamic links to libraries installed in the current wheel. +``` + +```{eval-rst} +.. confval:: wheel.repair.patch-imports + :type: ``bool`` + :default: true + + Automatically patch every top-level packages/modules to import the dlls on Windows wheels. + + Alternatively, set this to ``false`` and use :confval:`wheel.repair.imports-file` instead. +``` + diff --git a/pyproject.toml b/pyproject.toml index acf0f935..87f3a139 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,6 +69,8 @@ test-hatchling = [ test-meta = [ "hatch-fancy-pypi-readme>=22.3", "setuptools-scm", + "auditwheel; platform_system=='Linux'", + "delocate; platform_system=='Darwin'", ] test-numpy = [ "numpy; python_version<'3.14' and platform_python_implementation!='PyPy' and (platform_system != 'Windows' or platform_machine != 'ARM64')", @@ -185,7 +187,15 @@ disallow_untyped_defs = true disallow_incomplete_defs = true [[tool.mypy.overrides]] -module = ["numpy", "pathspec", "setuptools_scm", "hatch_fancy_pypi_readme", "virtualenv"] +module = [ + "numpy", + "pathspec", + "setuptools_scm", + "hatch_fancy_pypi_readme", + "virtualenv", + "auditwheel.*", + "delocate.*", +] ignore_missing_imports = true diff --git a/src/scikit_build_core/build/__init__.py b/src/scikit_build_core/build/__init__.py index c3e82c98..18c6eb7a 100644 --- a/src/scikit_build_core/build/__init__.py +++ b/src/scikit_build_core/build/__init__.py @@ -148,6 +148,7 @@ def get_requires_for_build_sdist( return [ *cmake_requires, *requires.dynamic_metadata(), + *requires.other_dynamic_requires(), ] @@ -166,6 +167,7 @@ def get_requires_for_build_wheel( return [ *cmake_requires, *requires.dynamic_metadata(), + *requires.other_dynamic_requires(), ] @@ -184,4 +186,5 @@ def get_requires_for_build_editable( return [ *cmake_requires, *requires.dynamic_metadata(), + *requires.other_dynamic_requires(), ] diff --git a/src/scikit_build_core/build/wheel.py b/src/scikit_build_core/build/wheel.py index 47e223bb..cb8e1e38 100644 --- a/src/scikit_build_core/build/wheel.py +++ b/src/scikit_build_core/build/wheel.py @@ -20,6 +20,7 @@ from ..cmake import CMake, CMaker from ..errors import FailedLiveProcessError from ..format import pyproject_format +from ..repair_wheel import WheelRepairer from ..settings.skbuild_read_settings import SettingsReader from ._editable import editable_redirect, libdir_to_installed, mapping_to_modules from ._init import setup_logging @@ -487,6 +488,21 @@ def _build_wheel_impl_impl( ), wheel_dirs["metadata"], ) as wheel: + if ( + cmake is not None + and settings.wheel.repair.enable + and settings.experimental + ): + repairer = WheelRepairer.get_wheel_repairer( + name=normalized_name, + settings=settings, + wheel=wheel, + builder=builder, + install_dir=install_dir, + wheel_dirs=wheel_dirs, + ) + repairer.repair_wheel() + wheel.build(wheel_dirs, exclude=settings.wheel.exclude) str_pkgs = ( diff --git a/src/scikit_build_core/builder/__main__.py b/src/scikit_build_core/builder/__main__.py index 967d8d7a..2f9f00ac 100644 --- a/src/scikit_build_core/builder/__main__.py +++ b/src/scikit_build_core/builder/__main__.py @@ -31,7 +31,12 @@ def main() -> None: if Path("pyproject.toml").is_file(): req = GetRequires() - all_req = [*req.cmake(), *req.ninja(), *req.dynamic_metadata()] + all_req = [ + *req.cmake(), + *req.ninja(), + *req.dynamic_metadata(), + *req.other_dynamic_requires(), + ] rich_print(f"{{bold.red}}Get Requires:{{normal}} {all_req!r}") ip_program_search(color="magenta") diff --git a/src/scikit_build_core/builder/get_requires.py b/src/scikit_build_core/builder/get_requires.py index aff5c2c6..2a394dd3 100644 --- a/src/scikit_build_core/builder/get_requires.py +++ b/src/scikit_build_core/builder/get_requires.py @@ -4,6 +4,8 @@ import functools import importlib.util import os +import platform +import shutil import sysconfig from typing import TYPE_CHECKING, Literal @@ -137,10 +139,7 @@ def ninja(self) -> Generator[str, None, None]: return yield f"ninja{ninja_verset}" - def dynamic_metadata(self) -> Generator[str, None, None]: - if self.settings.fail: - return - + def other_dynamic_requires(self) -> Generator[str, None, None]: for build_require in self.settings.build.requires: yield build_require.format( **pyproject_format( @@ -148,6 +147,20 @@ def dynamic_metadata(self) -> Generator[str, None, None]: ) ) + if self.settings.wheel.repair.enable: + platform_system = platform.system() + if platform_system == "Linux": + yield "auditwheel" + patchelf_path = shutil.which("patchelf") + if patchelf_path is None: + yield "patchelf" + elif platform_system == "Darwin": + yield "delocate" + + def dynamic_metadata(self) -> Generator[str, None, None]: + if self.settings.fail: + return + for dynamic_metadata in self.settings.metadata.values(): if "provider" in dynamic_metadata: config = dynamic_metadata.copy() diff --git a/src/scikit_build_core/cmake.py b/src/scikit_build_core/cmake.py index c613075f..0aa8ce53 100644 --- a/src/scikit_build_core/cmake.py +++ b/src/scikit_build_core/cmake.py @@ -266,7 +266,7 @@ def configure( self.file_api = load_reply_dir(self._file_api_query) except ExceptionGroup as exc: logger.warning("Could not parse CMake file-api") - logger.debug(str(exc)) + logger.debug(str(exc.exceptions)) def _compute_build_args( self, diff --git a/src/scikit_build_core/hatch/plugin.py b/src/scikit_build_core/hatch/plugin.py index affefb46..8e6ffa40 100644 --- a/src/scikit_build_core/hatch/plugin.py +++ b/src/scikit_build_core/hatch/plugin.py @@ -113,7 +113,11 @@ def dependencies(self) -> list[str]: # These are only injected if cmake is required cmake_requires = [*requires.cmake(), *requires.ninja()] if required else [] - return [*cmake_requires, *requires.dynamic_metadata()] + return [ + *cmake_requires, + *requires.dynamic_metadata(), + *requires.other_dynamic_requires(), + ] def initialize(self, version: str, build_data: dict[str, Any]) -> None: if version == "editable": diff --git a/src/scikit_build_core/repair_wheel/__init__.py b/src/scikit_build_core/repair_wheel/__init__.py new file mode 100644 index 00000000..a3f0e1f7 --- /dev/null +++ b/src/scikit_build_core/repair_wheel/__init__.py @@ -0,0 +1,396 @@ +""" +Repair wheel +""" + +from __future__ import annotations + +import dataclasses +import functools +import os +import platform +import re +import shutil +import sysconfig +from abc import ABC, abstractmethod +from pathlib import Path +from typing import TYPE_CHECKING, ClassVar, Final + +from .._logging import logger +from .darwin import MacOSWheelRepairer +from .linux import LinuxWheelRepairer +from .windows import WindowsWheelRepairer + +if TYPE_CHECKING: + from ..build._wheelfile import WheelWriter + from ..builder.builder import Builder + from ..file_api.model.codemodel import Configuration, Target + from ..settings.skbuild_model import ScikitBuildSettings + + +__all__ = [ + "LinuxWheelRepairer", + "MacOSWheelRepairer", + "WheelRepairer", + "WindowsWheelRepairer", +] + +DIR = Path(__file__).parent.resolve() + + +@functools.lru_cache(1) +def _get_buildenv_platlib() -> str: + # Normally we could `sysconfig.get_path("platlib")` directly, but pip fake-venv breaks it + platlib_path = sysconfig.get_path("platlib") + purelib_path = sysconfig.get_path("purelib") + real_purelib_path = DIR.parent.parent + if real_purelib_path.samefile(purelib_path): + # Here is the normal state if we are in a real venv + return str(Path(platlib_path).resolve()) + # Otherwise we need to trick it to giving us the real path + data_path = sysconfig.get_path("data") + platlib_relative_path = Path(platlib_path).relative_to(data_path) + purelib_relative_path = Path(purelib_path).relative_to(data_path) + + # removesuffix(purelib_relative_path) + if str(real_purelib_path).rfind(str(purelib_relative_path)) == -1: + logger.warning( + "Could not figure out the true build-env path:\n" + "sysconfig_purelib = {sysconfig_purelib}\n" + "scikit-build-core_purelib = {real_purelib}\n", + sysconfig_purelib=purelib_path, + real_purelib=real_purelib_path, + ) + return platlib_path + real_root = str(real_purelib_path)[: -len(str(purelib_relative_path))] + real_platlib_path = str(Path(real_root) / platlib_relative_path) + # Yet another dirty trick necessary + real_platlib_path = real_platlib_path.replace( + os.path.normpath("/overlay/"), + os.path.normpath("/normal/"), + ) + logger.debug("Calculated real_platlib_path = {}", real_platlib_path) + return str(real_platlib_path) + + +@dataclasses.dataclass() +class WheelRepairer(ABC): + """Abstract wheel repairer.""" + + name: str + """Normalized project name.""" + settings: ScikitBuildSettings + """Pyproject settings.""" + wheel: WheelWriter + """The current wheel creator.""" + builder: Builder + """CMake builder used.""" + install_dir: Path + """Wheel install directory of the CMake project.""" + wheel_dirs: dict[str, Path] + """Wheel packaging directories.""" + _platform_repairers: ClassVar[dict[str, type[WheelRepairer]]] = {} + """Dictionary of platform specific repairers""" + _platform: ClassVar[str | None] = None + """The ``platform.system()`` corresponding to the current repairer.""" + _initialized: Final[bool] = False + """Whether all ``WheelRepairer`` have been initialized.""" + _filter_targets: ClassVar[bool] = True + """Whether to filter the targets before calling ``patch_target``.""" + + def __init_subclass__(cls) -> None: + if cls._platform: + WheelRepairer._platform_repairers[cls._platform] = cls + + @functools.cached_property + def bundled_libs_path(self) -> Path: + """Staging path for the bundled library directory.""" + return Path(self.wheel_dirs["platlib"]) / f"{self.name}.libs" + + @functools.cached_property + def bundle_external(self) -> list[re.Pattern[str]]: + """List of compiled regex patterns of the library files to bundle.""" + patterns = [] + for pattern_str in self.settings.wheel.repair.bundle_external: + try: + pattern = re.compile(pattern_str) + except re.error as exc: + logger.warning( + 'Skipping "{pattern}" as an invalid pattern', + pattern=pattern_str, + ) + logger.debug(str(exc)) + continue + patterns.append(pattern) + return patterns + + @functools.cached_property + def configuration(self) -> Configuration: + """Current file-api configuration.""" + assert self.builder.config.file_api + reply = self.builder.config.file_api.reply + assert reply.codemodel_v2 + return next( + conf + for conf in reply.codemodel_v2.configurations + if conf.name == self.builder.config.build_type + ) + + @property + def targets(self) -> list[Target]: + """All targets found from file-api.""" + return self.configuration.targets + + def path_relative_site_packages( + self, + path: Path, + relative_to: Path | None = None, + ) -> Path: + """ + Transform an absolute path to a relative one in the final site-packages. + + It accounts for the temporary wheel install directory and the current build environment + (isolated or not). + + If ``relative_to`` is not passed, the root path is the ``platlib`` wheel path. If it is + a relative path, it is considered as relative to ``install-dir``. + + :raises ValueError: if ``path`` does not belong to the current site-packages + """ + assert path.is_absolute(), "Path must be absolute" + if relative_to is None: + relative_to = self.wheel_dirs["platlib"] + if not relative_to.is_absolute(): + relative_to = self.install_dir / relative_to + # Make sure relative_to is relative to platlib path, otherwise throw the ValueError + relative_to.relative_to(self.wheel_dirs["platlib"]) + + try: + # Try to get the relative path in the wheel install platlib + path.relative_to(self.wheel_dirs["platlib"]) + except ValueError: + # Otherwise check if the path is relative to build environment + path = path.relative_to(_get_buildenv_platlib()) + # Mock the path to be in the wheel install platlib + path = self.wheel_dirs["platlib"] / path + return Path(os.path.relpath(path, relative_to)) + + def path_is_in_site_packages(self, path: Path) -> bool: + """Check if a path belongs to the current site-packages.""" + try: + self.path_relative_site_packages(path) + except ValueError: + return False + else: + return True + + def get_wheel_install_paths(self, target: Target) -> set[Path]: + """Get a target's install paths that belong to the wheel.""" + if not target.install: + return set() + install_paths = [] + for dest in target.install.destinations: + path = dest.path + if path.is_absolute(): + try: + path = path.relative_to(self.install_dir) + except ValueError: + continue + install_paths.append(path) + return set(install_paths) + + def get_library_dependencies(self, target: Target) -> list[Target]: + """Get a target's library dependencies that need to be patched.""" + dependencies = [] + for dep in target.dependencies: + dep_target = next(targ for targ in self.targets if targ.id == dep.id) + if dep_target.type == "EXECUTABLE": + logger.warning("Handling executable dependencies not supported yet.") + continue + if dep_target.type != "SHARED_LIBRARY": + continue + dep_install_paths = self.get_wheel_install_paths(dep_target) + if not dep_install_paths: + logger.warning( + "Cannot patch dependency {dep} of target {target} because " + "the dependency is not installed in the wheel", + dep=dep_target.name, + target=target.name, + ) + continue + if len(dep_install_paths) > 1: + logger.warning( + "Cannot patch dependency {dep} of target {target} because " + "the dependency is installed in multiple locations on the wheel", + dep=dep_target.name, + target=target.name, + ) + continue + dependencies.append(dep_target) + return dependencies + + def try_bundle(self, external_lib: Path) -> Path | None: + """ + Try to bundle an external library file. + + :param external_lib: path to actual external library to bundle + :returns: ``None`` if the library is not bundled, otherwise the path + to the bundled file + """ + assert external_lib.is_absolute() + if not external_lib.exists(): + logger.warning( + "External library file does not exist: {external_lib}", + external_lib=external_lib, + ) + return None + if external_lib.is_dir(): + logger.debug( + "Skip bundling directory: {external_lib}", + external_lib=external_lib, + ) + return None + libname = external_lib.name + bundled_lib = self.bundled_libs_path / libname + if bundled_lib.exists(): + # If we have already bundled the library no need to do it again + return bundled_lib + for pattern in self.bundle_external: + if pattern.match(libname): + logger.debug( + 'Bundling library matching "{pattern}": {external_lib}', + external_lib=external_lib, + pattern=pattern.pattern, + ) + shutil.copy(external_lib, bundled_lib) + return bundled_lib + logger.debug( + "Skip bundling: {external_lib}", + external_lib=external_lib, + ) + return None + + def get_package_lib_path( + self, original_lib: Path, relative_to: Path | None = None + ) -> Path | None: + """ + Get the file path of a library to be used. + + This checks for the settings in ``settings.wheel.repair`` returning either: + - If the dependency should be skipped: ``None`` + - If ``original_lib`` is a library in another wheel: a relative path to the original library file + - If ``original_lib`` is a library to be bundled: a relative path to the bundled library file + + The relative paths are relative to ``relative_to`` or the ``platlib`` wheel path if not passed. + """ + if not original_lib.is_absolute() or not original_lib.exists(): + logger.debug( + "Could not handle {original_lib} because it is either relative or does not exist.", + original_lib=original_lib, + ) + return None + if self.path_is_in_site_packages(original_lib): + # The other library is in another wheel + if not self.settings.wheel.repair.cross_wheel: + logger.debug( + "Skipping {original_lib} because it is in another wheel.", + original_lib=original_lib, + ) + return None + final_lib = original_lib + # Otherwise, check if we need to bundle the external library + elif not self.bundle_external or not ( + final_lib := self.try_bundle(original_lib) # type: ignore[assignment] + ): + logger.debug( + "Skipping {original_lib} because it is not being bundled.", + original_lib=original_lib, + ) + return None + return self.path_relative_site_packages(final_lib, relative_to=relative_to) + + def repair_wheel(self) -> None: + """Repair the current wheel.""" + if self.bundle_external: + self.bundled_libs_path.mkdir(exist_ok=True) + + for target in self.targets: + if self._filter_targets: + if target.type == "STATIC_LIBRARY": + logger.debug( + "Handling static library {target} not supported yet.", + target=target.name, + ) + continue + if target.type not in ( + "SHARED_LIBRARY", + "MODULE_LIBRARY", + "EXECUTABLE", + ): + continue + if not target.install: + logger.debug( + "Skip patching {target} because it is not being installed.", + target=target.name, + ) + continue + self.patch_target(target) + + @abstractmethod + def patch_target(self, target: Target) -> None: + """Patch a specific target""" + + @classmethod + def get_wheel_repairer( + cls, + name: str, + settings: ScikitBuildSettings, + wheel: WheelWriter, + builder: Builder, + install_dir: Path, + wheel_dirs: dict[str, Path], + ) -> WheelRepairer: + """Construct the platform specific wheel repairer""" + if "platlib" not in wheel_dirs: + # This should only happen if the user explicitly disabled platlib + logger.warning( + "Wheel repairer is implemented only if `wheel.platlib` is True." + ) + return NoopWheelRepairer( + name=name, + settings=settings, + wheel=wheel, + builder=builder, + install_dir=install_dir, + wheel_dirs=wheel_dirs, + ) + + if not ( + repairer_cls := WheelRepairer._platform_repairers.get(platform.system()) + ): + return NoopWheelRepairer( + name=name, + settings=settings, + wheel=wheel, + builder=builder, + install_dir=install_dir, + wheel_dirs=wheel_dirs, + ) + return repairer_cls( + name=name, + settings=settings, + wheel=wheel, + builder=builder, + install_dir=install_dir, + wheel_dirs=wheel_dirs, + ) + + +class NoopWheelRepairer(WheelRepairer): + """Dummy wheel repairer that just shows a warning.""" + + def repair_wheel(self) -> None: + # Do nothing + logger.warning("Unknown platform {}. Not doing any repair.", platform.system()) + + def patch_target(self, target: Target) -> None: + pass diff --git a/src/scikit_build_core/repair_wheel/darwin.py b/src/scikit_build_core/repair_wheel/darwin.py new file mode 100644 index 00000000..7512985b --- /dev/null +++ b/src/scikit_build_core/repair_wheel/darwin.py @@ -0,0 +1,44 @@ +""" +Repair MacOS RPATH +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from .rpath import RpathWheelRepairer + +if TYPE_CHECKING: + from pathlib import Path + +__all__ = ["MacOSWheelRepairer"] + + +def __dir__() -> list[str]: + return __all__ + + +class MacOSWheelRepairer(RpathWheelRepairer): + """ + Adjust the RPATH with @loader_path. + """ + + # TODO: Tighten multi-architecture assumption. + + _platform = "Darwin" + _origin_symbol = "@loader_path" + + def get_library_rpath(self, artifact: Path) -> list[str]: + from delocate.tools import get_rpaths + + # Using the deprecated method here in order to support python 3.8 + return list(get_rpaths(str(artifact))) + + def patch_library_rpath(self, artifact: Path, rpaths: list[str]) -> None: + from delocate.tools import _delete_rpaths, add_rpath + + original_rpaths = self.get_library_rpath(artifact) + _delete_rpaths(str(artifact), set(original_rpaths)) + final_rpaths = set(rpaths) + for rpath in final_rpaths: + add_rpath(str(artifact), rpath) diff --git a/src/scikit_build_core/repair_wheel/linux.py b/src/scikit_build_core/repair_wheel/linux.py new file mode 100644 index 00000000..7c649956 --- /dev/null +++ b/src/scikit_build_core/repair_wheel/linux.py @@ -0,0 +1,44 @@ +""" +Repair Linux RPATH +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from .._shutil import Run +from .rpath import RpathWheelRepairer + +if TYPE_CHECKING: + from pathlib import Path + +__all__ = ["LinuxWheelRepairer"] + + +def __dir__() -> list[str]: + return __all__ + + +class LinuxWheelRepairer(RpathWheelRepairer): + """ + Adjust the RPATH with $ORIGIN. + """ + + _platform = "Linux" + _origin_symbol = "$ORIGIN" + + def get_library_rpath(self, artifact: Path) -> list[str]: + from auditwheel.elfutils import elf_read_rpaths + + return [ + path + for dt_rpaths in elf_read_rpaths(artifact).values() + for path in dt_rpaths + ] + + def patch_library_rpath(self, artifact: Path, rpaths: list[str]) -> None: + final_rpaths = set(rpaths) + if final_rpaths: + run = Run() + run.live("patchelf", "--remove-rpath", artifact) + run.live("patchelf", "--set-rpath", ":".join(final_rpaths), artifact) diff --git a/src/scikit_build_core/repair_wheel/rpath.py b/src/scikit_build_core/repair_wheel/rpath.py new file mode 100644 index 00000000..06b4d1d6 --- /dev/null +++ b/src/scikit_build_core/repair_wheel/rpath.py @@ -0,0 +1,195 @@ +""" +Repair RPATH systems +""" + +from __future__ import annotations + +import shlex +from abc import ABC, abstractmethod +from pathlib import Path +from typing import TYPE_CHECKING, ClassVar + +from .._logging import logger +from . import WheelRepairer + +if TYPE_CHECKING: + from ..file_api.model.codemodel import Target + +__all__ = ["RpathWheelRepairer"] + + +def __dir__() -> list[str]: + return __all__ + + +class RpathWheelRepairer(WheelRepairer, ABC): + """ + Adjust the RPATH with an equivalent $ORIGIN/@loader_path. + """ + + _origin_symbol: ClassVar[str] + """The equivalent symbol for the $ORIGIN/@loader_path in the RPATH.""" + + @abstractmethod + def get_library_rpath(self, artifact: Path) -> list[str]: + """Get the current rpaths.""" + + @abstractmethod + def patch_library_rpath(self, artifact: Path, rpaths: list[str]) -> None: + """Patch the rpaths of a specific library.""" + + def get_dependency_rpaths(self, target: Target, install_path: Path) -> list[str]: + """Get the rpaths due to target link dependencies.""" + target_path = self.install_dir / install_path + rpaths = [] + for dep_target in self.get_library_dependencies(target): + dep_install_paths = self.get_wheel_install_paths(dep_target) + assert len(dep_install_paths) == 1 + dep_install_path = self.install_dir / next(iter(dep_install_paths)) + rpath = self.path_relative_site_packages(dep_install_path, target_path) + new_rpath_str = f"{self._origin_symbol}/{rpath}" + rpaths.append(new_rpath_str) + return rpaths + + def get_package_rpaths(self, target: Target, install_path: Path) -> list[str]: + """ + Get the rpaths due to external package linkage. + + Have to use the linker flags until the package targets are exposed. + https://gitlab.kitware.com/cmake/cmake/-/issues/26755 + """ + if not target.link: + return [] + rpaths = [] + for link_command in target.link.commandFragments: + if link_command.role == "flags": + if not link_command.fragment: + logger.debug( + "Skipping {target} link-flags: {flags}", + target=target.name, + flags=link_command.fragment, + ) + continue + if link_command.role != "libraries": + logger.warning( + "File-api link role {role} is not supported. " + "Target={target}, command={command}", + target=target.name, + role=link_command.role, + command=link_command.fragment, + ) + continue + # TODO: These should be abstracted somehow? + # CMake 3.15 didn't seem to space-separate the flags + for link_part in shlex.split(link_command.fragment): + # Try to parse `-Wl,-rpath` flags + if link_part.startswith("-Wl,-rpath,"): + # removeprefix(`-Wl,-rpath,`) but compatible with Python 3.9 + check_rpaths = link_part[len("-Wl,-rpath,") :] + for rpath_str in check_rpaths.split(":"): + if not rpath_str: + # Skip empty rpaths. Most likely will have on at the end + continue + rpath = Path(rpath_str) + # Relative paths should be covered by `get_dependency_rpaths` so we skip them. + if not rpath.is_absolute(): + continue + # Get the relative rpath to either the cross-wheel or bundled file + if not ( + rpath := self.get_package_lib_path( # type: ignore[assignment] + rpath, relative_to=install_path + ) + ): + continue + new_rpath_str = f"{self._origin_symbol}/{rpath}" + rpaths.append(new_rpath_str) + continue + # The remaining case should be a path + try: + # TODO: how to best catch if a string is a valid path? + rpath = Path(link_part) + except Exception as exc: + logger.warning( + "Could not parse link-library as a path: {fragment}\nexc = {exc}", + fragment=link_command.fragment, + exc=exc, + ) + continue + if not rpath.is_absolute(): + # Relative paths should be covered by `get_dependency_rpaths` so we skip them. + continue + # Get the relative rpath to either the cross-wheel or bundled file + if not ( + rpath := self.get_package_lib_path(rpath, relative_to=install_path) # type: ignore[assignment] + ): + continue + new_rpath_str = f"{self._origin_symbol}/{rpath.parent}" + rpaths.append(new_rpath_str) + return rpaths + + def get_existing_rpaths(self, artifact: Path) -> list[str]: + """ + Get the rpaths that are already present in the artifact. + + Keep any rpaths that contain ``_origin_symbol``, or are outside the site_packages. + Convert the paths that point to site_packages to contain ``_origin_symbol`` + """ + patched_rpaths = [] + for rpath_str in self.get_library_rpath(artifact): + # If the rpath is already relative keep it + # TODO: maybe abstract this to include other symbols to skip? + if rpath_str.startswith(self._origin_symbol): + patched_rpaths.append(rpath_str) + continue + # Otherwise check if we need to patch it + rpath_path = Path(rpath_str) + if not self.path_is_in_site_packages(rpath_path): + # If it does not point to wheel install path, just keep it + patched_rpaths.append(rpath_str) + continue + # Otherwise change the RPATH to point use $ORIGIN + new_rpath = self.path_relative_site_packages(rpath_path, artifact.parent) + new_rpath_str = f"{self._origin_symbol}/{new_rpath}" + patched_rpaths.append(new_rpath_str) + return patched_rpaths + + def patch_target(self, target: Target) -> None: + # Get the target install paths where the $ORIGIN is calculated from + target_install_paths = self.get_wheel_install_paths(target) + if not target_install_paths: + logger.debug( + "Skip patching {target} because all install paths are outside the wheel.", + target=target.name, + ) + return + if len(set(target.artifacts)) != 1: + logger.warning( + "Unexpected multiple artifacts for target {target}: {artifacts}", + target=target.name, + artifacts=[item.path for item in target.artifacts], + ) + return + artifact = target.artifacts[0] + for install_path in target_install_paths: + target_path = self.install_dir / install_path + artifact_path = target_path / artifact.path.name + if self.settings.wheel.repair.in_wheel: + dependency_rpaths = self.get_dependency_rpaths(target, install_path) + else: + dependency_rpaths = [] + package_rpaths = self.get_package_rpaths(target, install_path) + existing_rpaths = self.get_existing_rpaths(artifact_path) + logger.debug( + "Patching rpaths for artifact {artifact}\n" + "existing={existing_rpaths}\n" + "dependency={dependency_rpaths}\n" + "package={package_rpaths}\n", + artifact=artifact_path, + existing_rpaths=existing_rpaths, + dependency_rpaths=dependency_rpaths, + package_rpaths=package_rpaths, + ) + self.patch_library_rpath( + artifact=artifact_path, + rpaths=[*existing_rpaths, *dependency_rpaths, *package_rpaths], + ) diff --git a/src/scikit_build_core/repair_wheel/windows.py b/src/scikit_build_core/repair_wheel/windows.py new file mode 100644 index 00000000..812aadf9 --- /dev/null +++ b/src/scikit_build_core/repair_wheel/windows.py @@ -0,0 +1,299 @@ +""" +Repair Windows dll path +""" + +from __future__ import annotations + +import dataclasses +import functools +import os.path +import textwrap +from pathlib import Path +from typing import TYPE_CHECKING, ClassVar + +from .._logging import logger +from . import WheelRepairer, _get_buildenv_platlib + +if TYPE_CHECKING: + import re + + from ..file_api.model.codemodel import Target + +__all__ = ["WindowsWheelRepairer"] + + +def __dir__() -> list[str]: + return __all__ + + +@dataclasses.dataclass +class WindowsWheelRepairer(WheelRepairer): + """ + Patch the package and top-level python module files with ``os.add_dll_directory``. + """ + + _platform = "Windows" + + PATCH_PY_FILE: ClassVar[str] = textwrap.dedent("""\ + # start scikit-build-core Windows patch + def _skbuild_patch_dll_dir(): + import os + import os.path + + mod_dir = os.path.abspath(os.path.dirname(__file__)) + path_to_platlib = os.path.normpath({path_to_platlib!r}) + dll_paths = {dll_paths!r} + for path in dll_paths: + path = os.path.normpath(path) + path = os.path.join(mod_dir, path_to_platlib, path) + os.add_dll_directory(path) + + _skbuild_patch_dll_dir() + del _skbuild_patch_dll_dir + # end scikit-build-core Windows patch + """) + dll_dirs: set[Path] = dataclasses.field(default_factory=set, init=False) + """All dll paths used relative to ``platlib``.""" + + @functools.cached_property + def bundle_external(self) -> list[re.Pattern[str]]: + if self.settings.wheel.repair.bundle_external: + logger.warning("Bundling Windows dll files is not supported yet.") + return [] + + def try_bundle(self, external_lib: Path) -> Path | None: + # Everything should be gated by `bundle_external` so this should not be called + # TODO: figure out a better way to find the corresponding dll file of the linked lib file + raise NotImplementedError + + def get_dll_path_from_lib(self, lib_path: Path) -> Path | None: + """Guess the dll path from lib path.""" + # TODO: rework the logic of this to work with `try_bundle` + dll_path = None + platlib = Path(_get_buildenv_platlib()) + lib_path = lib_path.relative_to(platlib) + # Change the `.lib` to `.dll` + if ".dll" in (suffixes := lib_path.suffixes): + # In some cases like msys, they use `.dll.a`, in which case we can't use `with_suffix` + if suffixes[-2] != ".dll": + logger.warning( + "Expected .dll suffix to be the penultimate extension, instead got: {lib_path}", + lib_path=lib_path, + ) + return None + # Drop the last suffix it should then be just .dll file + dll_name = lib_path.stem + else: + dll_name = lib_path.with_suffix(".dll").name + # Try to find the dll in the same package directory + if len(lib_path.parts) > 1: + pkg_dir = lib_path.parts[0] + for root, _, files in os.walk(platlib / pkg_dir): + if dll_name in files: + dll_path = Path(root) / dll_name + break + else: + logger.debug( + "Did not find the dll file under {pkg_dir}", + pkg_dir=pkg_dir, + ) + if not dll_path: + logger.debug( + "Looking for {dll_name} in all platlib path.", + dll_name=dll_name, + ) + for root, _, files in os.walk(platlib): + if dll_name in files: + dll_path = Path(root) / dll_name + break + else: + logger.warning( + "Could not find dll file {dll_name} corresponding to {lib_path}", + dll_name=dll_name, + lib_path=lib_path, + ) + return None + logger.debug( + "Found dll file {dll_path}", + dll_path=dll_path, + ) + return self.path_relative_site_packages(dll_path) + + def get_library_dependencies(self, target: Target) -> list[Target]: + msg = "get_library_dependencies is not generalized for Windows." + raise NotImplementedError(msg) + + def get_dependency_dll(self, target: Target) -> list[Path]: + """Get the dll due to target link dependencies.""" + dll_paths = [] + for dep in target.dependencies: + dep_target = next(targ for targ in self.targets if targ.id == dep.id) + if dep_target.type != "SHARED_LIBRARY": + logger.debug( + "Skipping dependency {dep_target} of type {type}", + dep_target=dep_target.name, + type=dep_target.type, + ) + continue + if not dep_target.install: + logger.warning( + "Dependency {dep_target} is not installed", + dep_target=dep_target.name, + ) + continue + dll_artifact = next( + artifact.path.name + for artifact in dep_target.artifacts + if artifact.path.suffix == ".dll" + ) + for install_path in self.get_wheel_install_paths(dep_target): + dep_install_path = self.install_dir / install_path + if (dep_install_path / dll_artifact).exists(): + break + else: + logger.warning( + "Could not find installed {dll_artifact} location in install paths: {install_path}", + dll_artifact=dll_artifact, + install_path=[ + dest.path for dest in dep_target.install.destinations + ], + ) + continue + dll_path = self.path_relative_site_packages(dep_install_path) + dll_paths.append(dll_path) + return dll_paths + + def get_package_dll(self, target: Target) -> list[Path]: + """ + Get the dll due to external package linkage. + + Have to use the guess the dll paths until the package targets are exposed. + https://gitlab.kitware.com/cmake/cmake/-/issues/26755 + """ + if not target.link: + return [] + dll_paths = [] + for link_command in target.link.commandFragments: + if link_command.role == "flags": + if not link_command.fragment: + logger.debug( + "Skipping {target} link-flags: {flags}", + target=target.name, + flags=link_command.fragment, + ) + continue + if link_command.role != "libraries": + logger.warning( + "File-api link role {role} is not supported. " + "Target={target}, command={command}", + target=target.name, + role=link_command.role, + command=link_command.fragment, + ) + continue + # The remaining case should be a path + try: + # TODO: how to best catch if a string is a valid path? + lib_path = Path(link_command.fragment) + except Exception as exc: + logger.warning( + "Could not parse link-library as a path: {fragment}\nexc = {exc}", + fragment=link_command.fragment, + exc=exc, + ) + continue + if not lib_path.is_absolute(): + # If the link_command is a space-separated list of libraries, this should be skipped + logger.debug( + "Skipping non-absolute-path library: {fragment}", + fragment=link_command.fragment, + ) + continue + # TODO: Handle this better when revisiting `try_bundle` + if not self.get_package_lib_path(lib_path): + continue + dll_path = self.get_dll_path_from_lib(lib_path) + if not dll_path: + continue + dll_paths.append(dll_path.parent) + return dll_paths + + def patch_target(self, target: Target) -> None: + # Here we just gather all dll paths needed for each target + if self.settings.wheel.repair.in_wheel: + dependency_dlls = self.get_dependency_dll(target) + else: + dependency_dlls = [] + package_dlls = self.get_package_dll(target) + + if not package_dlls and not dependency_dlls: + logger.warning( + "No dll files found for target {target}", + target=target.name, + ) + return + logger.debug( + "Found dlls for target {target}:\n" + "package_dlls={package_dlls}\n" + "dependency_dlls={dependency_dlls}\n", + target=target.name, + package_dlls=package_dlls, + dependency_dlls=dependency_dlls, + ) + self.dll_dirs.update(package_dlls) + self.dll_dirs.update(dependency_dlls) + + def patch_python_file(self, file: Path) -> None: + """ + Patch python package or top-level module. + + Make sure the python files have an appropriate ``os.add_dll_directory`` + for the scripts directory. + """ + assert self.dll_dirs + assert all(not path.is_absolute() for path in self.dll_dirs) + logger.debug( + "Patching python file: {file}", + file=file, + ) + platlib = Path(self.wheel_dirs["platlib"]) + content = file.read_text() + mod_dir = file.parent + path_to_platlib = os.path.relpath(platlib, mod_dir) + patch_script = self.PATCH_PY_FILE.format( + path_to_platlib=path_to_platlib, + dll_paths=[str(path) for path in self.dll_dirs], + ) + # TODO: Account for the header comments, __future__.annotations, etc. + with file.open("w") as f: + f.write(f"{patch_script}\n" + content) + + def repair_wheel(self) -> None: + super().repair_wheel() + platlib = Path(self.wheel_dirs["platlib"]) + if not self.dll_dirs: + logger.debug( + "Skipping wheel repair because no site-package dlls were found." + ) + return + logger.debug( + "Patching dll directories: {dll_dirs}", + dll_dirs=self.dll_dirs, + ) + if self.settings.wheel.repair.imports_file: + imports_file = platlib / self.settings.wheel.repair.imports_file + self.patch_python_file(imports_file) + else: + for path in platlib.iterdir(): + assert isinstance(path, Path) + if path.is_dir(): + pkg_file = path / "__init__.py" + if not pkg_file.exists(): + logger.debug( + "Ignoring non-python package: {pkg_file}", + pkg_file=pkg_file, + ) + continue + self.patch_python_file(pkg_file) + elif path.suffix == ".py": + self.patch_python_file(path) diff --git a/src/scikit_build_core/resources/scikit-build.schema.json b/src/scikit_build_core/resources/scikit-build.schema.json index 5065b415..142f2a84 100644 --- a/src/scikit_build_core/resources/scikit-build.schema.json +++ b/src/scikit_build_core/resources/scikit-build.schema.json @@ -239,6 +239,44 @@ "type": "string", "default": "", "description": "The build tag to use for the wheel. If empty, no build tag is used." + }, + "repair": { + "type": "object", + "additionalProperties": false, + "properties": { + "enable": { + "type": "boolean", + "default": false, + "description": "Do automatic repairs of the compiled binaries and libraries." + }, + "in-wheel": { + "type": "boolean", + "default": true, + "description": "Patch the dynamic links to libraries installed in the current wheel." + }, + "cross-wheel": { + "type": "boolean", + "default": false, + "description": "Patch the dynamic links to libraries in other wheels." + }, + "bundle-external": { + "type": "array", + "items": { + "type": "string" + }, + "description": "A list of external library files that will be bundled in the wheel." + }, + "patch-imports": { + "type": "boolean", + "default": true, + "description": "Automatically patch every top-level packages/modules to import the dlls on Windows wheels." + }, + "imports-file": { + "type": "string", + "description": "The generated file containing any necessary top-level imports." + } + }, + "description": "Wheel repair options" } } }, @@ -566,6 +604,9 @@ }, "exclude": { "$ref": "#/$defs/inherit" + }, + "repair": { + "$ref": "#/$defs/inherit" } } }, diff --git a/src/scikit_build_core/settings/skbuild_model.py b/src/scikit_build_core/settings/skbuild_model.py index a83abd2f..3375dc64 100644 --- a/src/scikit_build_core/settings/skbuild_model.py +++ b/src/scikit_build_core/settings/skbuild_model.py @@ -222,6 +222,58 @@ class SDistSettings: """ +@dataclasses.dataclass +class WheelRepair: + enable: bool = False + """ + Do automatic repairs of the compiled binaries and libraries. + + .. warning:: + This is an experimental feature gated by :confval:`experimental` + """ + + in_wheel: bool = True + """ + Patch the dynamic links to libraries installed in the current wheel. + """ + + cross_wheel: bool = False + """ + Patch the dynamic links to libraries in other wheels. + + .. note:: + This may result in incompatible wheels. Use this only if the + wheels are strongly linked to each other and strict manylinux compliance is + not required. + """ + + bundle_external: List[str] = dataclasses.field(default_factory=list) + """ + A list of external library files that will be bundled in the wheel. + + Each entry is treated as a regex pattern, and only the filenames are considered + for the match. The libraries are taken from the CMake dependency during the CMake + build. The bundled libraries are installed under ``site-packages/${name}.libs`` + """ + + patch_imports: bool = True + """ + Automatically patch every top-level packages/modules to import the dlls on Windows wheels. + + Alternatively, set this to ``false`` and use :confval:`wheel.repair.imports-file` instead. + """ + + imports_file: Optional[Path] = None + """ + The generated file containing any necessary top-level imports. + + This files should be imported as early as possible in all top-level modules and packages. + + On Windows wheels, this file contains all ``os.add_dll_directory`` needed in the current wheel. + On other OS, this is an empty file. + """ + + @dataclasses.dataclass class WheelSettings: packages: Optional[Union[List[str], Dict[str, str]]] = dataclasses.field( @@ -314,6 +366,11 @@ class WheelSettings: The build tag to use for the wheel. If empty, no build tag is used. """ + repair: WheelRepair = dataclasses.field(default_factory=WheelRepair) + """ + Wheel repair options + """ + @dataclasses.dataclass class BackportSettings: diff --git a/tests/conftest.py b/tests/conftest.py index db9eda95..22c95f84 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,6 +4,7 @@ import dataclasses import importlib.util import os +import platform import shutil import subprocess import sys @@ -56,6 +57,11 @@ def pep518_wheelhouse(tmp_path_factory: pytest.TempPathFactory) -> Path: "virtualenv", "wheel", ] + if platform.system() == "Linux": + packages.append("auditwheel") + packages.append("patchelf") + if platform.system() == "Darwin": + packages.append("delocate") if importlib.util.find_spec("cmake") is not None: packages.append("cmake") @@ -346,6 +352,15 @@ def package_pep639_pure(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Pack return package +@pytest.fixture +def repair_wheel(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> PackageInfo: + package = PackageInfo( + "repair_wheel", + ) + process_package(package, tmp_path, monkeypatch) + return package + + def which_mock(name: str) -> str | None: if name in {"ninja", "ninja-build", "cmake3", "samu", "gmake", "make"}: return None diff --git a/tests/packages/repair_wheel/CMakeLists.txt b/tests/packages/repair_wheel/CMakeLists.txt new file mode 100644 index 00000000..6b805057 --- /dev/null +++ b/tests/packages/repair_wheel/CMakeLists.txt @@ -0,0 +1,57 @@ +cmake_minimum_required(VERSION 3.15...3.26) + +project(${SKBUILD_PROJECT_NAME} LANGUAGES CXX) + +set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON) + +include(GNUInstallDirs) +include(CMakePackageConfigHelpers) + +find_package(Base CONFIG REQUIRED) +find_package(Python COMPONENTS Interpreter Development.Module) + +add_executable(main src/main.cpp) +add_library(other SHARED src/other.cpp) +python_add_library(_module MODULE src/module.cpp WITH_SOABI) + +target_link_libraries(other PRIVATE base::base) +target_link_libraries(main PRIVATE other) +target_link_libraries(_module PRIVATE base::base) + +install(TARGETS main other) +install(TARGETS _module DESTINATION ".") + +if(DO_MANUAL_REPAIR) + if(APPLE) + set(origin_token "@loader_path") + else() + set(origin_token "$ORIGIN") + endif() + set_property( + TARGET main PROPERTY INSTALL_RPATH + "${origin_token}/../${CMAKE_INSTALL_LIBDIR}") + set_property( + TARGET other + PROPERTY INSTALL_RPATH + "${origin_token}/../../base_project/${CMAKE_INSTALL_LIBDIR}") + set_property( + TARGET _module + PROPERTY INSTALL_RPATH + "${origin_token}/../base_project/${CMAKE_INSTALL_LIBDIR}") + if(WIN32) + install(TARGETS other RUNTIME DESTINATION ${SKBUILD_SCRIPTS_DIR}) + file( + WRITE ${CMAKE_CURRENT_BINARY_DIR}/__init__.py + " +import os +import sysconfig +from pathlib import Path + +base_project_bindir = Path(__file__).parent / \"../base_project/${CMAKE_INSTALL_BINDIR}\" +project_bindir = Path(__file__).parent / \"${CMAKE_INSTALL_BINDIR}\" +os.add_dll_directory(str(base_project_bindir)) +os.add_dll_directory(str(project_bindir)) +") + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/__init__.py DESTINATION ".") + endif() +endif() diff --git a/tests/packages/repair_wheel/extern/BaseConfig.cmake.in b/tests/packages/repair_wheel/extern/BaseConfig.cmake.in new file mode 100644 index 00000000..e95d6a15 --- /dev/null +++ b/tests/packages/repair_wheel/extern/BaseConfig.cmake.in @@ -0,0 +1,3 @@ +@PACKAGE_INIT@ + +include(${CMAKE_CURRENT_LIST_DIR}/BaseTargets.cmake) diff --git a/tests/packages/repair_wheel/extern/CMakeLists.txt b/tests/packages/repair_wheel/extern/CMakeLists.txt new file mode 100644 index 00000000..cf8a0b69 --- /dev/null +++ b/tests/packages/repair_wheel/extern/CMakeLists.txt @@ -0,0 +1,30 @@ +cmake_minimum_required(VERSION 3.15...3.26) + +project(Base LANGUAGES CXX) + +include(GNUInstallDirs) +include(CMakePackageConfigHelpers) + +set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON) + +add_library(base SHARED) + +target_sources(base PRIVATE base.cpp) +set_property( + TARGET base + APPEND + PROPERTY PUBLIC_HEADER base.h) +target_include_directories( + base PUBLIC $ + $) + +install(TARGETS base EXPORT BaseTargets) +install( + EXPORT BaseTargets + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/base + NAMESPACE base::) +configure_package_config_file( + BaseConfig.cmake.in BaseConfig.cmake + INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/Base) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/BaseConfig.cmake + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/base) diff --git a/tests/packages/repair_wheel/extern/base.cpp b/tests/packages/repair_wheel/extern/base.cpp new file mode 100644 index 00000000..f5a28a7a --- /dev/null +++ b/tests/packages/repair_wheel/extern/base.cpp @@ -0,0 +1,7 @@ +#include + +#include "base.h" + +void base::hello() { + std::cout << "Hello, World!" << std::endl; +} diff --git a/tests/packages/repair_wheel/extern/base.h b/tests/packages/repair_wheel/extern/base.h new file mode 100644 index 00000000..8932831f --- /dev/null +++ b/tests/packages/repair_wheel/extern/base.h @@ -0,0 +1,5 @@ +#pragma once + +namespace base { + void hello(); +} diff --git a/tests/packages/repair_wheel/extern/base_project/__init__.py b/tests/packages/repair_wheel/extern/base_project/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/packages/repair_wheel/extern/pyproject.toml b/tests/packages/repair_wheel/extern/pyproject.toml new file mode 100644 index 00000000..693d4c84 --- /dev/null +++ b/tests/packages/repair_wheel/extern/pyproject.toml @@ -0,0 +1,13 @@ +[build-system] +requires = ["scikit-build-core"] +build-backend = "scikit_build_core.build" + +[project] +name = "base_project" +version = "0.1.0" + +[project.entry-points."cmake.root"] +Base = "base_project" + +[tool.scikit-build] +wheel.install-dir = "base_project" diff --git a/tests/packages/repair_wheel/pyproject.toml b/tests/packages/repair_wheel/pyproject.toml new file mode 100644 index 00000000..1b8ced18 --- /dev/null +++ b/tests/packages/repair_wheel/pyproject.toml @@ -0,0 +1,23 @@ +[build-system] +requires = ["scikit-build-core"] +build-backend = "scikit_build_core.build" + +[project] +name = "repair_wheel" +version = "0.1.0" +dependencies = ["base_project"] + +[tool.scikit-build] +build.requires = ["base_project @ {root:uri}/extern"] +wheel.install-dir = "repair_wheel" +wheel.repair.enable = true +wheel.repair.cross-wheel = true +experimental = true + +[project.scripts] +main = "repair_wheel.__main__:run" + +[[tool.scikit-build.overrides]] +if.env.MANUAL = true +wheel.repair.enable = false +cmake.define.DO_MANUAL_REPAIR = true diff --git a/tests/packages/repair_wheel/python/repair_wheel/__init__.py b/tests/packages/repair_wheel/python/repair_wheel/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/packages/repair_wheel/python/repair_wheel/__main__.py b/tests/packages/repair_wheel/python/repair_wheel/__main__.py new file mode 100644 index 00000000..815b0555 --- /dev/null +++ b/tests/packages/repair_wheel/python/repair_wheel/__main__.py @@ -0,0 +1,15 @@ +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + + +def run() -> None: + exe_path = Path(__file__).parent + exe_path = next((exe_path / "bin").glob("main*")) + sys.exit(subprocess.call([str(exe_path), *sys.argv[2:]])) + + +if __name__ == "__main__": + run() diff --git a/tests/packages/repair_wheel/python/repair_wheel/_module.pyi b/tests/packages/repair_wheel/python/repair_wheel/_module.pyi new file mode 100644 index 00000000..6e27daa8 --- /dev/null +++ b/tests/packages/repair_wheel/python/repair_wheel/_module.pyi @@ -0,0 +1 @@ +def hello() -> None: ... diff --git a/tests/packages/repair_wheel/src/main.cpp b/tests/packages/repair_wheel/src/main.cpp new file mode 100644 index 00000000..a16e6435 --- /dev/null +++ b/tests/packages/repair_wheel/src/main.cpp @@ -0,0 +1,6 @@ +#include "other.h" + +int main() { + other::hello(); + return 0; +} diff --git a/tests/packages/repair_wheel/src/module.cpp b/tests/packages/repair_wheel/src/module.cpp new file mode 100644 index 00000000..2615051e --- /dev/null +++ b/tests/packages/repair_wheel/src/module.cpp @@ -0,0 +1,20 @@ +#define PY_SSIZE_T_CLEAN +#include + +#include + +static PyObject *hello(PyObject *self, PyObject *args){ + base::hello(); + Py_RETURN_NONE; +} + +static PyMethodDef repair_wheel_methods[] = { + {"hello", hello, METH_NOARGS, "Say hello"}, + {NULL, NULL, 0, NULL}}; + +static struct PyModuleDef repair_wheel_module = {PyModuleDef_HEAD_INIT, "_module", + NULL, -1, repair_wheel_methods}; + +PyMODINIT_FUNC PyInit__module(void) { + return PyModule_Create(&repair_wheel_module); +} diff --git a/tests/packages/repair_wheel/src/other.cpp b/tests/packages/repair_wheel/src/other.cpp new file mode 100644 index 00000000..3467f8b8 --- /dev/null +++ b/tests/packages/repair_wheel/src/other.cpp @@ -0,0 +1,7 @@ +#include + +#include "other.h" + +void other::hello() { + base::hello(); +} diff --git a/tests/packages/repair_wheel/src/other.h b/tests/packages/repair_wheel/src/other.h new file mode 100644 index 00000000..209f2555 --- /dev/null +++ b/tests/packages/repair_wheel/src/other.h @@ -0,0 +1,5 @@ +#pragma once + +namespace other { + void hello(); +} diff --git a/tests/test_dynamic_metadata.py b/tests/test_dynamic_metadata.py index 9038edc5..b40f64c8 100644 --- a/tests/test_dynamic_metadata.py +++ b/tests/test_dynamic_metadata.py @@ -375,18 +375,18 @@ def test_build_requires_field(override, monkeypatch) -> None: settings_reader.validate_may_exit() if override is None: - assert set(GetRequires().dynamic_metadata()) == { + assert set(GetRequires().other_dynamic_requires()) == { "foo", } elif override == "env": # evaluate ../foo as uri foo_path = pyproject_path.absolute().parent.parent / "foo" foo_path = foo_path.absolute() - assert set(GetRequires().dynamic_metadata()) == { + assert set(GetRequires().other_dynamic_requires()) == { f"foo @ {foo_path.as_uri()}", } elif override == "sdist": - assert set(GetRequires().dynamic_metadata()) == { + assert set(GetRequires().other_dynamic_requires()) == { # TODO: Check if special handling should be done for sdist "foo", } diff --git a/tests/test_repair_wheel.py b/tests/test_repair_wheel.py new file mode 100644 index 00000000..3f777cea --- /dev/null +++ b/tests/test_repair_wheel.py @@ -0,0 +1,81 @@ +import platform +import shutil +from pathlib import Path + +import pytest + +DIR = Path(__file__).parent.resolve() +REPAIR_WHEEL = DIR / "packages/repair_wheel" + + +@pytest.mark.compile +@pytest.mark.configure +@pytest.mark.parametrize( + ("backend", "with_isolation", "manual_repair"), + [ + ("pip", True, False), + ("pip", False, False), + ("pip", True, True), + ("build", True, False), + ("build", False, False), + ], + ids=[ + "pip-isolated", + "pip-not-isolated", + "manual", + "build-isolated", + "build-not-isolated", + ], +) +@pytest.mark.usefixtures("repair_wheel") +def test_full_build( + backend, + isolated, + with_isolation, + manual_repair, + monkeypatch, + tmp_path, +): + monkeypatch.setenv("MANUAL", f"{manual_repair}") + if backend == "pip": + isolated.install("pip>=23") + elif backend == "build": + isolated.install("build[virtualenv]") + else: + raise NotImplementedError + + if not with_isolation: + isolated.install("scikit-build-core") + if platform.system() == "Linux": + isolated.install("auditwheel") + if shutil.which("patchelf") is None: + isolated.install("patchelf") + if platform.system() == "Darwin": + isolated.install("delocate") + isolated.install("./extern", isolated=with_isolation) + + if backend == "pip": + isolated.install( + "-v", + "./extern", + ".", + isolated=with_isolation, + ) + elif backend == "build": + dist = tmp_path / "dist" + build_args = ["-v", "--wheel", f"--outdir={dist}"] + if with_isolation: + isolated.module("build", *build_args, "./extern") + else: + build_args.append("--no-isolation") + isolated.module("build", *build_args, ".") + wheels = list(dist.glob("*.whl")) + isolated.install(*wheels) + + if platform.system() != "Windows": + # Requires a more specialized patch + isolated.run("main") + isolated.module("repair_wheel") + isolated.execute( + "from repair_wheel._module import hello; hello()", + )