From 1f62c18e2ae5a0cc46ad5ac56c059b9bbf438dcb Mon Sep 17 00:00:00 2001 From: Mark Williams Date: Sat, 27 Jan 2018 16:30:17 -0800 Subject: [PATCH 01/11] Script to generate symbol_versions from lib_whitelist. This script is derived from the explanation in ec6171f's commit message. Thanks to @ehashman for describing the source of these symbols! --- scripts/calculate_symbol_versions.py | 91 ++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 scripts/calculate_symbol_versions.py diff --git a/scripts/calculate_symbol_versions.py b/scripts/calculate_symbol_versions.py new file mode 100644 index 00000000..6b33e241 --- /dev/null +++ b/scripts/calculate_symbol_versions.py @@ -0,0 +1,91 @@ +""" +Calculate symbol_versions for a policy in policy.json by collection +defined version (.gnu.version_d) from libraries in lib_whitelist. +This should be run inside a manylinux Docker container. +""" +import argparse +import os +import platform +import json +from elftools.elf.elffile import ELFFile + +if platform.architecture()[0] == '64bit': + LIBRARY_PATHS = ['/lib64', '/usr/lib64'] +else: + LIBRARY_PATHS = ['/lib', '/usr/lib'] + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument("policy", help="The policy name") +parser.add_argument("policyjson", help="The policy.json file.") + + +def load_policies(path): + with open(path) as f: + return json.load(f) + + +def choose_policy(name, policies): + try: + return next(policy for policy in policies if policy['name'] == name) + except StopIteration: + raise RuntimeError("Unknown policy {}".format(name)) + + +def find_library(library): + for p in LIBRARY_PATHS: + path = os.path.join(p, library) + if os.path.exists(path): + return path + else: + raise RuntimeError("Unknown library {}".format(library)) + + +def versionify(version_string): + return [int(n) for n in version_string.split('.')] + + +def calculate_symbol_versions(libraries, symbol_versions, arch): + calculated_symbol_versions = {k: set() for k in symbol_versions} + prefixes = ['/lib', '/usr/lib'] + if arch == '64bit': + prefixes = [p + '64' for p in prefixes] + + for library in libraries: + library_path = find_library(library) + with open(library_path, 'rb') as f: + e = ELFFile(f) + section = e.get_section_by_name('.gnu.version_d') + if section: + for _, verdef_iter in section.iter_versions(): + for vernaux in verdef_iter: + for symbol_name in symbol_versions: + try: + name, version = vernaux.name.split('_', 1) + except ValueError: + pass + if name in calculated_symbol_versions \ + and version != 'PRIVATE': + calculated_symbol_versions[name].add(version) + return { + k: sorted(v, key=versionify) + for k, v in calculated_symbol_versions.items() + } + + +def main(): + args = parser.parse_args() + policies = load_policies(args.policyjson) + policy = choose_policy(args.policy, policies) + arch, _ = platform.architecture() + print( + json.dumps( + calculate_symbol_versions( + policy['lib_whitelist'], + policy['symbol_versions'], + arch, + ) + ) + ) + + +main() From 3a98ce45d6152c85a2255aecc938d9080f887f3f Mon Sep 17 00:00:00 2001 From: Mark Williams Date: Sat, 27 Jan 2018 16:32:13 -0800 Subject: [PATCH 02/11] First draft of manylinux2 policy. - Remove libpanel and libncurses from the lib_whitelist (#94) - Generate symbol_versions - Bump priority to 200. --- auditwheel/policy/policy.json | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/auditwheel/policy/policy.json b/auditwheel/policy/policy.json index fe13834e..d1315272 100644 --- a/auditwheel/policy/policy.json +++ b/auditwheel/policy/policy.json @@ -25,5 +25,26 @@ "libX11.so.6", "libXext.so.6", "libXrender.so.1", "libICE.so.6", "libSM.so.6", "libGL.so.1", "libgobject-2.0.so.0", "libgthread-2.0.so.0", "libglib-2.0.so.0", "libresolv.so.2" + ]}, + {"name": "manylinux2", + "priority": 200, + "symbol_versions": { + "GLIBC": ["2.2.5", "2.2.6", "2.3", "2.3.2", "2.3.3", "2.3.4", "2.4", + "2.5", "2.6", "2.7", "2.8", "2.9", "2.10", "2.11", "2.12"], + "CXXABI": ["1.3", "1.3.1", "1.3.2", "1.3.3"], + "GLIBCXX": ["3.4", "3.4.1", "3.4.2", "3.4.3", "3.4.4", "3.4.5", + "3.4.6", "3.4.7", "3.4.8", "3.4.9", "3.4.10", "3.4.11", + "3.4.12", "3.4.13"], + "GCC": ["3.0", "3.3", "3.3.1", "3.4", "3.4.2", "3.4.4", "4.0.0", + "4.2.0", "4.3.0"] + }, + "lib_whitelist": [ + "libgcc_s.so.1", + "libstdc++.so.6", + "libm.so.6", "libdl.so.2", "librt.so.1", "libcrypt.so.1", + "libc.so.6", "libnsl.so.1", "libutil.so.1", "libpthread.so.0", + "libX11.so.6", "libXext.so.6", "libXrender.so.1", "libICE.so.6", + "libSM.so.6", "libGL.so.1", "libgobject-2.0.so.0", + "libgthread-2.0.so.0", "libglib-2.0.so.0", "libresolv.so.2" ]} ] From c3fc172228c05c6481677c55a7022aaa8b4bcade Mon Sep 17 00:00:00 2001 From: Mark Williams Date: Sun, 28 Jan 2018 17:24:06 -0800 Subject: [PATCH 03/11] Remove unused import. --- auditwheel/main_show.py | 1 - 1 file changed, 1 deletion(-) diff --git a/auditwheel/main_show.py b/auditwheel/main_show.py index c68811b3..80462368 100644 --- a/auditwheel/main_show.py +++ b/auditwheel/main_show.py @@ -13,7 +13,6 @@ def printp(text): def execute(args, p): import json - from functools import reduce from collections import OrderedDict from os.path import isfile, basename from .policy import (load_policies, get_priority_by_name, From 780f63057894ef091ab6b8db12233936072e16c4 Mon Sep 17 00:00:00 2001 From: Mark Williams Date: Sun, 28 Jan 2018 17:24:52 -0800 Subject: [PATCH 04/11] linux ABI tags should be removed from manylinux2 wheels. --- auditwheel/policy/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/auditwheel/policy/__init__.py b/auditwheel/policy/__init__.py index c25e5809..20ad6073 100644 --- a/auditwheel/policy/__init__.py +++ b/auditwheel/policy/__init__.py @@ -16,7 +16,9 @@ _PLATFORM_REPLACEMENT_MAP = { 'manylinux1_x86_64': ['linux_x86_64'], + 'manylinux2_x86_64': ['linux_x86_64'], 'manylinux1_i686': ['linux_i686'], + 'manylinux2_i686': ['linux_i686'], } # XXX: this could be weakened. The show command _could_ run on OS X or From 4f794daf8e2c374c330df1b4f36b3acabb888749 Mon Sep 17 00:00:00 2001 From: Mark Williams Date: Mon, 29 Jan 2018 00:33:48 -0800 Subject: [PATCH 05/11] Update tests for manylinux2. test_manylinux uses an interim manylinux2 image stored on Docker Hub, and a fork of pip that understands the manylinux2 ABI tag. --- tests/test_manylinux.py | 18 ++++++++++++------ tests/test_pyfpe.py | 2 +- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/tests/test_manylinux.py b/tests/test_manylinux.py index 62ef9340..23741469 100644 --- a/tests/test_manylinux.py +++ b/tests/test_manylinux.py @@ -9,7 +9,7 @@ VERBOSE = True ENCODING = 'utf-8' -MANYLINUX_IMAGE_ID = 'quay.io/pypa/manylinux1_x86_64' +MANYLINUX_IMAGE_ID = 'markrwilliams/manylinux2:x86_64' DOCKER_CONTAINER_NAME = 'auditwheel-test-manylinux' PYTHON_IMAGE_ID = 'python:3.5' PATH = ('/opt/python/cp35-cp35m/bin:/opt/rh/devtoolset-2/root/usr/bin:' @@ -132,19 +132,25 @@ def test_build_repair_numpy(docker_container): orig_wheel = filenames[0] assert 'manylinux' not in orig_wheel - # Repair the wheel using the manylinux1 container + # Repair the wheel using the manylinux2 container docker_exec(manylinux_id, 'auditwheel repair -w /io /io/' + orig_wheel) filenames = os.listdir(io_folder) assert len(filenames) == 2 - repaired_wheels = [fn for fn in filenames if 'manylinux1' in fn] - assert repaired_wheels == ['numpy-1.11.0-cp35-cp35m-manylinux1_x86_64.whl'] + repaired_wheels = [fn for fn in filenames if 'manylinux2' in fn] + assert repaired_wheels == ['numpy-1.11.0-cp35-cp35m-manylinux2_x86_64.whl'] repaired_wheel = repaired_wheels[0] output = docker_exec(manylinux_id, 'auditwheel show /io/' + repaired_wheel) assert ( - 'numpy-1.11.0-cp35-cp35m-manylinux1_x86_64.whl is consistent' - ' with the following platform tag: "manylinux1_x86_64"' + 'numpy-1.11.0-cp35-cp35m-manylinux2_x86_64.whl is consistent' + ' with the following platform tag: "manylinux2_x86_64"' ) in output.replace('\n', ' ') + # TODO: Remove once pip supports manylinux2 + docker_exec( + python_id, + "pip install git+https://github.com/markrwilliams/pip.git@manylinux2", + ) + # Check that the repaired numpy wheel can be installed and executed # on a modern linux image. docker_exec(python_id, 'pip install /io/' + repaired_wheel) diff --git a/tests/test_pyfpe.py b/tests/test_pyfpe.py index d8584207..472f98ab 100644 --- a/tests/test_pyfpe.py +++ b/tests/test_pyfpe.py @@ -4,5 +4,5 @@ def test_analyze_wheel_abi(): winfo = analyze_wheel_abi('tests/fpewheel-0.0.0-cp35-cp35m-linux_x86_64.whl') - assert winfo.sym_tag == 'manylinux1_x86_64' # for external symbols, it could get manylinux1 + assert winfo.sym_tag == 'manylinux2_x86_64' # for external symbols, it could get manylinux2 assert winfo.pyfpe_tag == 'linux_x86_64' # but for having the pyfpe reference, it gets just linux From 647431465a6b83e0a279d5d6201e69a91591178a Mon Sep 17 00:00:00 2001 From: William T Olson Date: Tue, 15 May 2018 14:31:34 -0400 Subject: [PATCH 06/11] Rename manylinux2 tag as manylinux2010 --- auditwheel/policy/__init__.py | 4 ++-- auditwheel/policy/policy.json | 2 +- tests/test_manylinux.py | 14 +++++++------- tests/test_pyfpe.py | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/auditwheel/policy/__init__.py b/auditwheel/policy/__init__.py index 20ad6073..016e5125 100644 --- a/auditwheel/policy/__init__.py +++ b/auditwheel/policy/__init__.py @@ -16,9 +16,9 @@ _PLATFORM_REPLACEMENT_MAP = { 'manylinux1_x86_64': ['linux_x86_64'], - 'manylinux2_x86_64': ['linux_x86_64'], + 'manylinux2010_x86_64': ['linux_x86_64'], 'manylinux1_i686': ['linux_i686'], - 'manylinux2_i686': ['linux_i686'], + 'manylinux2010_i686': ['linux_i686'], } # XXX: this could be weakened. The show command _could_ run on OS X or diff --git a/auditwheel/policy/policy.json b/auditwheel/policy/policy.json index d1315272..39095563 100644 --- a/auditwheel/policy/policy.json +++ b/auditwheel/policy/policy.json @@ -26,7 +26,7 @@ "libSM.so.6", "libGL.so.1", "libgobject-2.0.so.0", "libgthread-2.0.so.0", "libglib-2.0.so.0", "libresolv.so.2" ]}, - {"name": "manylinux2", + {"name": "manylinux2010", "priority": 200, "symbol_versions": { "GLIBC": ["2.2.5", "2.2.6", "2.3", "2.3.2", "2.3.3", "2.3.4", "2.4", diff --git a/tests/test_manylinux.py b/tests/test_manylinux.py index 23741469..01f6cbc5 100644 --- a/tests/test_manylinux.py +++ b/tests/test_manylinux.py @@ -9,7 +9,7 @@ VERBOSE = True ENCODING = 'utf-8' -MANYLINUX_IMAGE_ID = 'markrwilliams/manylinux2:x86_64' +MANYLINUX_IMAGE_ID = 'zombiefeynman/manylinux2010_x86_64' DOCKER_CONTAINER_NAME = 'auditwheel-test-manylinux' PYTHON_IMAGE_ID = 'python:3.5' PATH = ('/opt/python/cp35-cp35m/bin:/opt/rh/devtoolset-2/root/usr/bin:' @@ -132,23 +132,23 @@ def test_build_repair_numpy(docker_container): orig_wheel = filenames[0] assert 'manylinux' not in orig_wheel - # Repair the wheel using the manylinux2 container + # Repair the wheel using the manylinux2010 container docker_exec(manylinux_id, 'auditwheel repair -w /io /io/' + orig_wheel) filenames = os.listdir(io_folder) assert len(filenames) == 2 - repaired_wheels = [fn for fn in filenames if 'manylinux2' in fn] - assert repaired_wheels == ['numpy-1.11.0-cp35-cp35m-manylinux2_x86_64.whl'] + repaired_wheels = [fn for fn in filenames if 'manylinux2010' in fn] + assert repaired_wheels == ['numpy-1.11.0-cp35-cp35m-manylinux2010_x86_64.whl'] repaired_wheel = repaired_wheels[0] output = docker_exec(manylinux_id, 'auditwheel show /io/' + repaired_wheel) assert ( - 'numpy-1.11.0-cp35-cp35m-manylinux2_x86_64.whl is consistent' - ' with the following platform tag: "manylinux2_x86_64"' + 'numpy-1.11.0-cp35-cp35m-manylinux2010_x86_64.whl is consistent' + ' with the following platform tag: "manylinux2010_x86_64"' ) in output.replace('\n', ' ') # TODO: Remove once pip supports manylinux2 docker_exec( python_id, - "pip install git+https://github.com/markrwilliams/pip.git@manylinux2", + "pip install git+https://github.com/wtolson/pip.git@manylinux2010", ) # Check that the repaired numpy wheel can be installed and executed diff --git a/tests/test_pyfpe.py b/tests/test_pyfpe.py index 472f98ab..3db09ad9 100644 --- a/tests/test_pyfpe.py +++ b/tests/test_pyfpe.py @@ -4,5 +4,5 @@ def test_analyze_wheel_abi(): winfo = analyze_wheel_abi('tests/fpewheel-0.0.0-cp35-cp35m-linux_x86_64.whl') - assert winfo.sym_tag == 'manylinux2_x86_64' # for external symbols, it could get manylinux2 + assert winfo.sym_tag == 'manylinux2010_x86_64' # for external symbols, it could get manylinux2010 assert winfo.pyfpe_tag == 'linux_x86_64' # but for having the pyfpe reference, it gets just linux From 55970f201cb26854b8f6b1a694090ece00addb6d Mon Sep 17 00:00:00 2001 From: William T Olson Date: Tue, 5 Jun 2018 09:51:23 -0700 Subject: [PATCH 07/11] Test audit wheel with manylinux1 and manylinux2010 containers --- tests/test_manylinux.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/test_manylinux.py b/tests/test_manylinux.py index 01f6cbc5..06b4f1ae 100644 --- a/tests/test_manylinux.py +++ b/tests/test_manylinux.py @@ -9,7 +9,8 @@ VERBOSE = True ENCODING = 'utf-8' -MANYLINUX_IMAGE_ID = 'zombiefeynman/manylinux2010_x86_64' +MANYLINUX1_IMAGE_ID = 'quay.io/pypa/manylinux1_x86_64' +MANYLINUX2010_IMAGE_ID = 'zombiefeynman/manylinux2010_x86_64' DOCKER_CONTAINER_NAME = 'auditwheel-test-manylinux' PYTHON_IMAGE_ID = 'python:3.5' PATH = ('/opt/python/cp35-cp35m/bin:/opt/rh/devtoolset-2/root/usr/bin:' @@ -61,8 +62,8 @@ def docker_exec(container_id, cmd): return output -@pytest.yield_fixture -def docker_container(): +@pytest.fixture(params=[MANYLINUX1_IMAGE_ID, MANYLINUX2010_IMAGE_ID]) +def docker_container(request): if find_executable("docker") is None: pytest.skip('docker is required') if not op.exists(WHEEL_CACHE_FOLDER): @@ -80,7 +81,7 @@ def docker_container(): # This container will be used to build and repair manylinux compatible # wheels manylinux_id = docker_start( - MANYLINUX_IMAGE_ID, + request.param, volumes={'/io': io_folder, '/auditwheel_src': src_folder}, env_variables={'PATH': PATH}) # Install the development version of auditwheel from source: From bdee5a1dcdcc2fa55ae38714ae96d401bb9a8388 Mon Sep 17 00:00:00 2001 From: Elana Hashman Date: Sat, 17 Nov 2018 13:00:33 -0500 Subject: [PATCH 08/11] Make manylinux2010 policy lower precedence than manylinux1 --- auditwheel/policy/policy.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auditwheel/policy/policy.json b/auditwheel/policy/policy.json index 39095563..333b165c 100644 --- a/auditwheel/policy/policy.json +++ b/auditwheel/policy/policy.json @@ -27,7 +27,7 @@ "libgthread-2.0.so.0", "libglib-2.0.so.0", "libresolv.so.2" ]}, {"name": "manylinux2010", - "priority": 200, + "priority": 90, "symbol_versions": { "GLIBC": ["2.2.5", "2.2.6", "2.3", "2.3.2", "2.3.3", "2.3.4", "2.4", "2.5", "2.6", "2.7", "2.8", "2.9", "2.10", "2.11", "2.12"], From b96413d182e886d15c0efde44f406da1c85dd99c Mon Sep 17 00:00:00 2001 From: Elana Hashman Date: Sat, 17 Nov 2018 17:52:10 -0500 Subject: [PATCH 09/11] Fix some logging issues --- auditwheel/lddtree.py | 2 +- auditwheel/main.py | 6 ++---- auditwheel/repair.py | 4 ++-- auditwheel/wheel_abi.py | 3 +-- auditwheel/wheeltools.py | 8 ++++---- 5 files changed, 10 insertions(+), 13 deletions(-) diff --git a/auditwheel/lddtree.py b/auditwheel/lddtree.py index 0b5259f9..ca1456b6 100644 --- a/auditwheel/lddtree.py +++ b/auditwheel/lddtree.py @@ -201,7 +201,7 @@ def load_ld_paths(root: str='/', prefix: str='') -> Dict[str, List[str]]: root=root) # the trusted directories are not necessarily in ld.so.conf ldpaths['conf'].extend(['/lib', '/lib64/', '/usr/lib', '/usr/lib64']) - log.info('linker ldpaths: %s', ldpaths) + log.debug('linker ldpaths: %s', ldpaths) return ldpaths diff --git a/auditwheel/main.py b/auditwheel/main.py index c110ec82..498c9d97 100644 --- a/auditwheel/main.py +++ b/auditwheel/main.py @@ -34,12 +34,10 @@ def main(): args = p.parse_args() logging.disable(logging.NOTSET) - if args.verbose >= 2: + if args.verbose >= 1: logging.basicConfig(level=logging.DEBUG) - elif args.verbose == 1: - logging.basicConfig(level=logging.INFO) else: - logging.basicConfig(level=logging.WARN) + logging.basicConfig(level=logging.INFO) if not hasattr(args, 'func'): p.print_help() diff --git a/auditwheel/repair.py b/auditwheel/repair.py index 7e3a4bb1..46e3fd71 100644 --- a/auditwheel/repair.py +++ b/auditwheel/repair.py @@ -133,7 +133,7 @@ def copylib(src_path, dest_dir): if os.path.exists(dest_path): return new_soname, dest_path - logger.info('Grafting: %s -> %s', src_path, dest_path) + logger.debug('Grafting: %s -> %s', src_path, dest_path) rpaths = elf_read_rpaths(src_path) shutil.copy2(src_path, dest_path) @@ -151,5 +151,5 @@ def copylib(src_path, dest_dir): def patchelf_set_rpath(fn, libdir): rpath = pjoin('$ORIGIN', relpath(libdir, dirname(fn))) - logger.info('Setting RPATH: %s to "%s"', fn, rpath) + logger.debug('Setting RPATH: %s to "%s"', fn, rpath) check_call(['patchelf', '--force-rpath', '--set-rpath', rpath, fn]) diff --git a/auditwheel/wheel_abi.py b/auditwheel/wheel_abi.py index b0acb108..81ea8609 100644 --- a/auditwheel/wheel_abi.py +++ b/auditwheel/wheel_abi.py @@ -43,7 +43,7 @@ def get_wheel_elfdata(wheel_fn: str): 'The wheel has to be platlib compliant in order to be repaired by auditwheel.') % so_path_split[-1]) - log.info('processing: %s', fn) + log.debug('processing: %s', fn) elftree = lddtree(fn) for key, value in elf_find_versioned_symbols(elf): @@ -103,7 +103,6 @@ def analyze_wheel_abi(wheel_fn: str): for fn in elftree_by_fn.keys(): update(external_refs, external_refs_by_fn[fn]) - log.info(json.dumps(external_refs, indent=4)) log.debug('external reference info') log.debug(json.dumps(external_refs, indent=4)) diff --git a/auditwheel/wheeltools.py b/auditwheel/wheeltools.py index 24f4b96c..5b244f0c 100644 --- a/auditwheel/wheeltools.py +++ b/auditwheel/wheeltools.py @@ -204,7 +204,7 @@ def add_platforms(wheel_ctx, platforms, remove_platforms=()): parsed_fname = WHEEL_INFO_RE(wheel_fname) fparts = parsed_fname.groupdict() original_fname_tags = fparts['plat'].split('.') - logger.info('Previous filename tags:', ', '.join(original_fname_tags)) + logger.info('Previous filename tags: %s', ', '.join(original_fname_tags)) fname_tags = {tag for tag in original_fname_tags if tag not in remove_platforms} fname_tags |= set(platforms) @@ -216,7 +216,7 @@ def add_platforms(wheel_ctx, platforms, remove_platforms=()): definitely_not_purelib = True if fname_tags != original_fname_tags: - logger.info('New filename tags:', ', '.join(fname_tags)) + logger.info('New filename tags: %s', ', '.join(fname_tags)) else: logger.info('No filename tags change needed.') @@ -227,7 +227,7 @@ def add_platforms(wheel_ctx, platforms, remove_platforms=()): out_wheel = pjoin(out_dir, out_wheel_fname) in_info_tags = [tag for name, tag in info.items() if name == 'Tag'] - logger.info('Previous WHEEL info tags:', ', '.join(in_info_tags)) + logger.info('Previous WHEEL info tags: %s', ', '.join(in_info_tags)) # Python version, C-API version combinations pyc_apis = ['-'.join(tag.split('-')[:2]) for tag in in_info_tags] # unique Python version, C-API version combinations @@ -248,7 +248,7 @@ def add_platforms(wheel_ctx, platforms, remove_platforms=()): info['Root-Is-Purelib'] = 'False' logger.info('Changed wheel type to Platlib') - logger.info('New WHEEL info tags:', ', '.join(info.get_all('Tag'))) + logger.info('New WHEEL info tags: %s', ', '.join(info.get_all('Tag'))) write_pkg_info(info_fname, info) else: logger.info('No WHEEL info change needed.') From 7047b957ad353996f126f4836657d1b78ae302ca Mon Sep 17 00:00:00 2001 From: Elana Hashman Date: Sat, 17 Nov 2018 17:52:24 -0500 Subject: [PATCH 10/11] Update tests to account for correct manylinux2010 behaviour --- auditwheel/main_repair.py | 12 +++++++ tests/test_manylinux.py | 75 ++++++++++++++++++++++++--------------- tests/test_pyfpe.py | 2 +- 3 files changed, 60 insertions(+), 29 deletions(-) diff --git a/auditwheel/main_repair.py b/auditwheel/main_repair.py index 424a03a8..3b1a7712 100644 --- a/auditwheel/main_repair.py +++ b/auditwheel/main_repair.py @@ -79,4 +79,16 @@ def execute(args, p): update_tags=args.UPDATE_TAGS) if out_wheel is not None: + analyzed_tag = analyze_wheel_abi(out_wheel).overall_tag + if reqd_tag < get_priority_by_name(analyzed_tag): + logger.info(('Wheel is eligible for a higher priority tag. ' + 'You requested %s but I have found this wheel is ' + 'eligible for %s.'), + args.PLAT, analyzed_tag) + out_wheel = repair_wheel(args.WHEEL_FILE, + abi=analyzed_tag, + lib_sdir=args.LIB_SDIR, + out_dir=args.WHEEL_DIR, + update_tags=args.UPDATE_TAGS) + logger.info('\nFixed-up wheel written to %s', out_wheel) diff --git a/tests/test_manylinux.py b/tests/test_manylinux.py index eeefdc7f..36b2acf6 100644 --- a/tests/test_manylinux.py +++ b/tests/test_manylinux.py @@ -14,7 +14,12 @@ ENCODING = 'utf-8' MANYLINUX1_IMAGE_ID = 'quay.io/pypa/manylinux1_x86_64' +#TODO: use the pypa manylinux2010 image on release MANYLINUX2010_IMAGE_ID = 'zombiefeynman/manylinux2010_x86_64' +MANYLINUX_IMAGES = { + 'manylinux1': MANYLINUX1_IMAGE_ID, + 'manylinux2010': MANYLINUX2010_IMAGE_ID, +} DOCKER_CONTAINER_NAME = 'auditwheel-test-manylinux' PYTHON_IMAGE_ID = 'python:3.5' PATH = ('/opt/python/cp35-cp35m/bin:/opt/rh/devtoolset-2/root/usr/bin:' @@ -62,7 +67,7 @@ def docker_exec(container_id, cmd): return output -@pytest.fixture(params=[MANYLINUX1_IMAGE_ID, MANYLINUX2010_IMAGE_ID]) +@pytest.fixture(params=MANYLINUX_IMAGES.keys()) def docker_container(request): if find_executable("docker") is None: pytest.skip('docker is required') @@ -73,6 +78,7 @@ def docker_container(request): pytest.skip('Can only be run from the source folder') io_folder = tempfile.mkdtemp(prefix='tmp_auditwheel_test_manylinux_', dir=src_folder) + policy = request.param manylinux_id, python_id = None, None try: # Launch a docker container with volumes and pre-configured Python @@ -81,7 +87,7 @@ def docker_container(request): # This container will be used to build and repair manylinux compatible # wheels manylinux_id = docker_start( - request.param, + MANYLINUX_IMAGES[policy], volumes={'/io': io_folder, '/auditwheel_src': src_folder}, env_variables={'PATH': PATH}) # Install the development version of auditwheel from source: @@ -94,7 +100,7 @@ def docker_container(request): PYTHON_IMAGE_ID, volumes={'/io': io_folder, '/auditwheel_src': src_folder}) docker_exec(python_id, 'pip install -U pip') - yield manylinux_id, python_id, io_folder + yield policy, manylinux_id, python_id, io_folder finally: for container_id in [manylinux_id, python_id]: if container_id is None: @@ -112,7 +118,7 @@ def test_build_repair_numpy(docker_container): # First build numpy from source as a naive linux wheel that is tied # to system libraries (atlas, libgfortran...) - manylinux_id, python_id, io_folder = docker_container + policy, manylinux_id, python_id, io_folder = docker_container docker_exec(manylinux_id, 'yum install -y atlas atlas-devel') if op.exists(op.join(WHEEL_CACHE_FOLDER, ORIGINAL_NUMPY_WHEEL)): @@ -133,25 +139,29 @@ def test_build_repair_numpy(docker_container): orig_wheel = filenames[0] assert 'manylinux' not in orig_wheel - # Repair the wheel using the manylinux2010 container - docker_exec(manylinux_id, 'auditwheel repair -w /io /io/' + orig_wheel) + # Repair the wheel using the manylinux container + repair_command = ( + 'auditwheel repair --plat {policy}_x86_64 -w /io /io/{orig_wheel}' + ).format(policy=policy, orig_wheel=orig_wheel) + docker_exec(manylinux_id, repair_command) filenames = os.listdir(io_folder) - assert len(filenames) == 2 - repaired_wheels = [fn for fn in filenames if 'manylinux2010' in fn] - assert repaired_wheels == ['numpy-1.11.0-cp35-cp35m-manylinux2010_x86_64.whl'] + + if policy == 'manylinux1': + expected_files = 2 + elif policy == 'manylinux2010': + expected_files = 3 # We end up repairing the wheel twice to manylinux1 + + assert len(filenames) == expected_files + # Regardless of build environment, wheel only needs manylinux1 symbols + repaired_wheels = [fn for fn in filenames if 'manylinux1' in fn] + assert repaired_wheels == ['numpy-1.11.0-cp35-cp35m-manylinux1_x86_64.whl'] repaired_wheel = repaired_wheels[0] output = docker_exec(manylinux_id, 'auditwheel show /io/' + repaired_wheel) assert ( - 'numpy-1.11.0-cp35-cp35m-manylinux2010_x86_64.whl is consistent' - ' with the following platform tag: "manylinux2010_x86_64"' + 'numpy-1.11.0-cp35-cp35m-manylinux1_x86_64.whl is consistent' + ' with the following platform tag: "manylinux1_x86_64"' ) in output.replace('\n', ' ') - # TODO: Remove once pip supports manylinux2 - docker_exec( - python_id, - "pip install git+https://github.com/wtolson/pip.git@manylinux2010", - ) - # Check that the repaired numpy wheel can be installed and executed # on a modern linux image. docker_exec(python_id, 'pip install /io/' + repaired_wheel) @@ -173,7 +183,7 @@ def test_build_repair_numpy(docker_container): def test_build_wheel_with_binary_executable(docker_container): # Test building a wheel that contains a binary executable (e.g., a program) - manylinux_id, python_id, io_folder = docker_container + policy, manylinux_id, python_id, io_folder = docker_container docker_exec(manylinux_id, 'yum install -y gsl-devel') docker_exec(manylinux_id, ['bash', '-c', 'cd /auditwheel_src/tests/testpackage && python setup.py bdist_wheel -d /io']) @@ -183,21 +193,30 @@ def test_build_wheel_with_binary_executable(docker_container): orig_wheel = filenames[0] assert 'manylinux' not in orig_wheel - # Repair the wheel using the manylinux1 container - docker_exec(manylinux_id, 'auditwheel repair -w /io /io/' + orig_wheel) + # Repair the wheel using the appropriate manylinux container + repair_command = ( + 'auditwheel repair --plat {policy}_x86_64 -w /io /io/{orig_wheel}' + ).format(policy=policy, orig_wheel=orig_wheel) + docker_exec(manylinux_id, repair_command) filenames = os.listdir(io_folder) assert len(filenames) == 2 - repaired_wheels = [fn for fn in filenames if 'manylinux1' in fn] - assert repaired_wheels == ['testpackage-0.0.1-py3-none-manylinux1_x86_64.whl'] + repaired_wheels = [fn for fn in filenames if policy in fn] + # Wheel picks up newer symbols when built in manylinux2010 + expected_wheel_name = 'testpackage-0.0.1-py3-none-%s_x86_64.whl' % policy + assert repaired_wheels == [expected_wheel_name] repaired_wheel = repaired_wheels[0] output = docker_exec(manylinux_id, 'auditwheel show /io/' + repaired_wheel) assert ( - 'testpackage-0.0.1-py3-none-manylinux1_x86_64.whl is consistent' - ' with the following platform tag: "manylinux1_x86_64"' - ) in output.replace('\n', ' ') + 'testpackage-0.0.1-py3-none-{policy}_x86_64.whl is consistent' + ' with the following platform tag: "{policy}_x86_64"' + ).format(policy=policy) in output.replace('\n', ' ') + + # TODO: Remove once pip supports manylinux2010 + docker_exec( + python_id, + "pip install git+https://github.com/wtolson/pip.git@manylinux2010", + ) - # Check that the repaired numpy wheel can be installed and executed - # on a modern linux image. docker_exec(python_id, 'pip install /io/' + repaired_wheel) output = docker_exec( python_id, ['python', '-c', 'from testpackage import runit; print(runit(1.5))']).strip() @@ -205,7 +224,7 @@ def test_build_wheel_with_binary_executable(docker_container): def test_build_repair_pure_wheel(docker_container): - manylinux_id, python_id, io_folder = docker_container + policy, manylinux_id, python_id, io_folder = docker_container if op.exists(op.join(WHEEL_CACHE_FOLDER, ORIGINAL_SIX_WHEEL)): # If six has already been built and put in cache, let's reuse this. diff --git a/tests/test_pyfpe.py b/tests/test_pyfpe.py index 3db09ad9..d8584207 100644 --- a/tests/test_pyfpe.py +++ b/tests/test_pyfpe.py @@ -4,5 +4,5 @@ def test_analyze_wheel_abi(): winfo = analyze_wheel_abi('tests/fpewheel-0.0.0-cp35-cp35m-linux_x86_64.whl') - assert winfo.sym_tag == 'manylinux2010_x86_64' # for external symbols, it could get manylinux2010 + assert winfo.sym_tag == 'manylinux1_x86_64' # for external symbols, it could get manylinux1 assert winfo.pyfpe_tag == 'linux_x86_64' # but for having the pyfpe reference, it gets just linux From b4cba09cf28d66e18b8bdb424307d626802dd1a5 Mon Sep 17 00:00:00 2001 From: Elana Hashman Date: Sun, 18 Nov 2018 12:40:13 -0500 Subject: [PATCH 11/11] Also test per-policy for pure wheels --- tests/test_manylinux.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_manylinux.py b/tests/test_manylinux.py index 36b2acf6..410a7ddf 100644 --- a/tests/test_manylinux.py +++ b/tests/test_manylinux.py @@ -241,8 +241,11 @@ def test_build_repair_pure_wheel(docker_container): orig_wheel = filenames[0] assert 'manylinux' not in orig_wheel - # Repair the wheel using the manylinux1 container - docker_exec(manylinux_id, 'auditwheel repair -w /io /io/' + orig_wheel) + # Repair the wheel using the manylinux container + repair_command = ( + 'auditwheel repair --plat {policy}_x86_64 -w /io /io/{orig_wheel}' + ).format(policy=policy, orig_wheel=orig_wheel) + docker_exec(manylinux_id, repair_command) filenames = os.listdir(io_folder) assert len(filenames) == 1 # no new wheels assert filenames == [ORIGINAL_SIX_WHEEL]