diff --git a/.github/workflows/docs-ci.yml b/.github/workflows/docs-ci.yml index 18a44aa0..511b7c28 100644 --- a/.github/workflows/docs-ci.yml +++ b/.github/workflows/docs-ci.yml @@ -13,10 +13,10 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index 3a4fe279..95857301 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -1,27 +1,83 @@ -name: Release library as a PyPI wheel and sdist on GH release creation +name: Create library release archives, create a GH release and publish PyPI wheel and sdist on tag in main branch + + +# This is executed automatically on a tag in the main branch + +# Summary of the steps: +# - build wheels and sdist +# - upload wheels and sdist to PyPI +# - create gh-release and upload wheels and dists there +# TODO: smoke test wheels and sdist +# TODO: add changelog to release text body + +# WARNING: this is designed only for packages building as pure Python wheels on: - release: - types: [created] + workflow_dispatch: + push: + tags: + - "v*.*.*" jobs: - build-and-publish-to-pypi: + build-pypi-distribs: name: Build and publish library to PyPI runs-on: ubuntu-20.04 + + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.9 + + - name: Install pypa/build + run: python -m pip install build --user + + - name: Build a binary wheel and a source tarball + run: python -m build --sdist --wheel --outdir dist/ + + - name: Upload built archives + uses: actions/upload-artifact@v3 + with: + name: pypi_archives + path: dist/* + + + create-gh-release: + name: Create GH release + needs: + - build-pypi-distribs + runs-on: ubuntu-20.04 + + steps: + - name: Download built archives + uses: actions/download-artifact@v3 + with: + name: pypi_archives + path: dist + + - name: Create GH release + uses: softprops/action-gh-release@v1 + with: + draft: true + files: dist/* + + + create-pypi-release: + name: Create PyPI release + needs: + - create-gh-release + runs-on: ubuntu-20.04 + steps: - - uses: actions/checkout@master - - name: Set up Python - uses: actions/setup-python@v1 - with: - python-version: 3.9 - - name: Install pypa/build - run: python -m pip install build --user - - name: Build a binary wheel and a source tarball - run: python -m build --sdist --wheel --outdir dist/ - . - - name: Publish distribution to PyPI - if: startsWith(github.ref, 'refs/tags') - uses: pypa/gh-action-pypi-publish@master - with: - password: ${{ secrets.PYPI_API_TOKEN }} - + - name: Download built archives + uses: actions/download-artifact@v3 + with: + name: pypi_archives + path: dist + + - name: Publish to PyPI + if: startsWith(github.ref, 'refs/tags') + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.gitignore b/.gitignore index 339dca50..2d48196f 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ *.py[cod] # virtualenv and other misc bits +/src/*.egg-info *.egg-info /dist /build diff --git a/.readthedocs.yml b/.readthedocs.yml index 1b71cd9e..8ab23688 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -5,6 +5,17 @@ # Required version: 2 +# Build in latest ubuntu/python +build: + os: ubuntu-22.04 + tools: + python: "3.11" + +# Build PDF & ePub +formats: + - epub + - pdf + # Where the Sphinx conf.py file is located sphinx: configuration: docs/source/conf.py diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..cc36c355 --- /dev/null +++ b/Makefile @@ -0,0 +1,54 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/skeleton for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +# Python version can be specified with `$ PYTHON_EXE=python3.x make conf` +PYTHON_EXE?=python3 +VENV=venv +ACTIVATE?=. ${VENV}/bin/activate; + +dev: + @echo "-> Configure the development envt." + ./configure --dev + +isort: + @echo "-> Apply isort changes to ensure proper imports ordering" + ${VENV}/bin/isort --sl -l 100 src tests setup.py + +black: + @echo "-> Apply black code formatter" + ${VENV}/bin/black -l 100 src tests setup.py + +doc8: + @echo "-> Run doc8 validation" + @${ACTIVATE} doc8 --max-line-length 100 --ignore-path docs/_build/ --quiet docs/ + +valid: isort black + +check: + @echo "-> Run pycodestyle (PEP8) validation" + @${ACTIVATE} pycodestyle --max-line-length=100 --exclude=.eggs,venv,lib,thirdparty,docs,migrations,settings.py,.cache . + @echo "-> Run isort imports ordering validation" + @${ACTIVATE} isort --sl --check-only -l 100 setup.py src tests . + @echo "-> Run black validation" + @${ACTIVATE} black --check --check -l 100 src tests setup.py + +clean: + @echo "-> Clean the Python env" + ./configure --clean + +test: + @echo "-> Run the test suite" + ${VENV}/bin/pytest -vvs + +docs: + rm -rf docs/_build/ + @${ACTIVATE} sphinx-build docs/ docs/_build/ + +.PHONY: conf dev check valid black isort clean test docs diff --git a/README.rst b/README.rst index 2e4c5954..7bb19a77 100644 --- a/README.rst +++ b/README.rst @@ -1,8 +1,8 @@ -FetchCode: Utilities to fetch code from purls, URLs and version control repos. +FetchCode: Utilities to fetch code from purls, URLs and version control repos. ================================================================================ FetchCode is a library to reliably fetch code via HTTP, FTP and version control -systems. It can work using plain HTTP and FTP URLs, as well as +systems. It can work using plain HTTP and FTP URLs, as well as `Package URLs `_ and version control (VCS) URLs as used in Python pip and as specified in `SPDX Package Download Location `_ diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ea96efc4..764883de 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -9,33 +9,41 @@ jobs: - template: etc/ci/azure-posix.yml parameters: - job_name: ubuntu18_cpython - image_name: ubuntu-18.04 - python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] + job_name: ubuntu20_cpython + image_name: ubuntu-20.04 + python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] test_suites: all: venv/bin/pytest -n 2 -vvs - template: etc/ci/azure-posix.yml parameters: - job_name: ubuntu20_cpython - image_name: ubuntu-20.04 - python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] + job_name: ubuntu22_cpython + image_name: ubuntu-22.04 + python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] + test_suites: + all: venv/bin/pytest -n 2 -vvs + + - template: etc/ci/azure-posix.yml + parameters: + job_name: macos11_cpython + image_name: macOS-11 + python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] test_suites: all: venv/bin/pytest -n 2 -vvs - template: etc/ci/azure-posix.yml parameters: job_name: macos12_cpython - image_name: macos-12 - python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] + image_name: macOS-12 + python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] test_suites: all: venv/bin/pytest -n 2 -vvs - template: etc/ci/azure-posix.yml parameters: - job_name: macos11_cpython - image_name: macos-11 - python_versions: ['3.7', '3.8', '3.9', '3.10'] + job_name: macos13_cpython + image_name: macOS-13 + python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -43,7 +51,7 @@ jobs: parameters: job_name: win2019_cpython image_name: windows-2019 - python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] + python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] test_suites: all: venv\Scripts\pytest -n 2 -vvs @@ -51,6 +59,6 @@ jobs: parameters: job_name: win2022_cpython image_name: windows-2022 - python_versions: ['3.7', '3.8', '3.9', '3.10'] + python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] test_suites: all: venv\Scripts\pytest -n 2 -vvs diff --git a/configure b/configure index d1b4fda2..926a894e 100755 --- a/configure +++ b/configure @@ -36,7 +36,7 @@ DOCS_REQUIREMENTS="--editable .[docs] --constraint requirements.txt" VIRTUALENV_DIR=venv # Cleanable files and directories to delete with the --clean option -CLEANABLE="build venv" +CLEANABLE="build dist venv .cache .eggs" # extra arguments passed to pip PIP_EXTRA_ARGS=" " @@ -52,13 +52,20 @@ CFG_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" CFG_BIN_DIR=$CFG_ROOT_DIR/$VIRTUALENV_DIR/bin +################################ +# Install with or without and index. With "--no-index" this is using only local wheels +# This is an offline mode with no index and no network operations +# NO_INDEX="--no-index " +NO_INDEX="" + + ################################ # Thirdparty package locations and index handling -# Find packages from the local thirdparty directory or from thirdparty.aboutcode.org -if [ -d "$CFG_ROOT_DIR/thirdparty" ]; then - PIP_EXTRA_ARGS="--find-links $CFG_ROOT_DIR/thirdparty" +# Find packages from the local thirdparty directory if present +THIRDPARDIR=$CFG_ROOT_DIR/thirdparty +if [[ "$(echo $THIRDPARDIR/*.whl)x" != "$THIRDPARDIR/*.whlx" ]]; then + PIP_EXTRA_ARGS="$NO_INDEX --find-links $THIRDPARDIR" fi -PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS --find-links https://thirdparty.aboutcode.org/pypi/simple/links.html" ################################ @@ -183,6 +190,7 @@ while getopts :-: optchar; do esac done + PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS" find_python diff --git a/configure.bat b/configure.bat index 487e78a3..5e95b311 100644 --- a/configure.bat +++ b/configure.bat @@ -34,7 +34,7 @@ set "DOCS_REQUIREMENTS=--editable .[docs] --constraint requirements.txt" set "VIRTUALENV_DIR=venv" @rem # Cleanable files and directories to delete with the --clean option -set "CLEANABLE=build venv" +set "CLEANABLE=build dist venv .cache .eggs" @rem # extra arguments passed to pip set "PIP_EXTRA_ARGS= " @@ -52,11 +52,10 @@ set "CFG_BIN_DIR=%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\Scripts" @rem ################################ @rem # Thirdparty package locations and index handling -@rem # Find packages from the local thirdparty directory or from thirdparty.aboutcode.org +@rem # Find packages from the local thirdparty directory if exist "%CFG_ROOT_DIR%\thirdparty" ( set PIP_EXTRA_ARGS=--find-links "%CFG_ROOT_DIR%\thirdparty" ) -set "PIP_EXTRA_ARGS=%PIP_EXTRA_ARGS% --find-links https://thirdparty.aboutcode.org/pypi/simple/links.html" @rem ################################ @@ -69,7 +68,6 @@ if not defined CFG_QUIET ( @rem ################################ @rem # Main command line entry point set "CFG_REQUIREMENTS=%REQUIREMENTS%" -set "NO_INDEX=--no-index" :again if not "%1" == "" ( diff --git a/docs/source/conf.py b/docs/source/conf.py index 6f5d217a..efc87bdc 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -28,16 +28,22 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ -'sphinx.ext.intersphinx', + "sphinx.ext.intersphinx", + "sphinx_reredirects", ] + +# Redirects for olds pages +# See https://documatt.gitlab.io/sphinx-reredirects/usage.html +redirects = {} + # This points to aboutcode.readthedocs.io # In case of "undefined label" ERRORS check docs on intersphinx to troubleshoot # Link was created at commit - https://github.com/nexB/aboutcode/commit/faea9fcf3248f8f198844fe34d43833224ac4a83 intersphinx_mapping = { - 'aboutcode': ('https://aboutcode.readthedocs.io/en/latest/', None), - 'scancode-workbench': ('https://scancode-workbench.readthedocs.io/en/develop/', None), + "aboutcode": ("https://aboutcode.readthedocs.io/en/latest/", None), + "scancode-workbench": ("https://scancode-workbench.readthedocs.io/en/develop/", None), } @@ -62,7 +68,7 @@ # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ["_static"] -master_doc = 'index' +master_doc = "index" html_context = { "display_github": True, @@ -72,9 +78,7 @@ "conf_py_path": "/docs/source/", # path in the checkout to the docs root } -html_css_files = [ - '_static/theme_overrides.css' - ] +html_css_files = ["_static/theme_overrides.css"] # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. @@ -97,3 +101,9 @@ .. role:: img-title-para """ + +# -- Options for LaTeX output ------------------------------------------------- + +latex_elements = { + 'classoptions': ',openany,oneside' +} \ No newline at end of file diff --git a/etc/scripts/README.rst b/etc/scripts/README.rst index edf82e44..5e54a2cc 100755 --- a/etc/scripts/README.rst +++ b/etc/scripts/README.rst @@ -21,7 +21,7 @@ Pre-requisites virtualenv or in the the main configured development virtualenv. These requireements need to be installed:: - pip install --requirement etc/release/requirements.txt + pip install --requirement etc/scripts/requirements.txt TODO: we need to pin the versions of these tools @@ -34,7 +34,7 @@ Scripts ~~~~~~~ **gen_requirements.py**: create/update requirements files from currently - installed requirements. + installed requirements. **gen_requirements_dev.py** does the same but can subtract the main requirements to get extra requirements used in only development. @@ -50,7 +50,7 @@ The sequence of commands to run are: ./configure --clean ./configure - python etc/release/gen_requirements.py --site-packages-dir + python etc/scripts/gen_requirements.py --site-packages-dir * You can optionally install or update extra main requirements after the ./configure step such that these are included in the generated main requirements. @@ -59,7 +59,7 @@ The sequence of commands to run are: ./configure --clean ./configure --dev - python etc/release/gen_requirements_dev.py --site-packages-dir + python etc/scripts/gen_requirements_dev.py --site-packages-dir * You can optionally install or update extra dev requirements after the ./configure step such that these are included in the generated dev diff --git a/etc/scripts/fetch_thirdparty.py b/etc/scripts/fetch_thirdparty.py index 26d520f7..eedf05c6 100644 --- a/etc/scripts/fetch_thirdparty.py +++ b/etc/scripts/fetch_thirdparty.py @@ -12,6 +12,7 @@ import itertools import os import sys +from collections import defaultdict import click @@ -110,7 +111,39 @@ is_flag=True, help="Use on disk cached PyPI indexes list of packages and versions and do not refetch if present.", ) - +@click.option( + "--sdist-only", + "sdist_only", + type=str, + metavar="SDIST", + default=tuple(), + show_default=False, + multiple=True, + help="Package name(s) that come only in sdist format (no wheels). " + "The command will not fail and exit if no wheel exists for these names", +) +@click.option( + "--wheel-only", + "wheel_only", + type=str, + metavar="WHEEL", + default=tuple(), + show_default=False, + multiple=True, + help="Package name(s) that come only in wheel format (no sdist). " + "The command will not fail and exit if no sdist exists for these names", +) +@click.option( + "--no-dist", + "no_dist", + type=str, + metavar="DIST", + default=tuple(), + show_default=False, + multiple=True, + help="Package name(s) that do not come either in wheel or sdist format. " + "The command will not fail and exit if no distribution exists for these names", +) @click.help_option("-h", "--help") def fetch_thirdparty( requirements_files, @@ -123,6 +156,9 @@ def fetch_thirdparty( sdists, index_urls, use_cached_index, + sdist_only, + wheel_only, + no_dist, ): """ Download to --dest THIRDPARTY_DIR the PyPI wheels, source distributions, @@ -205,58 +241,62 @@ def fetch_thirdparty( ) repos.append(repo) - wheels_fetched = [] - wheels_not_found = [] - - sdists_fetched = [] - sdists_not_found = [] + wheels_or_sdist_not_found = defaultdict(list) for name, version in sorted(required_name_versions): nv = name, version print(f"Processing: {name} @ {version}") if wheels: for environment in environments: + if TRACE: print(f" ==> Fetching wheel for envt: {environment}") - fwfns = utils_thirdparty.download_wheel( + + fetched = utils_thirdparty.download_wheel( name=name, version=version, environment=environment, dest_dir=dest_dir, repos=repos, ) - if fwfns: - wheels_fetched.extend(fwfns) - else: - wheels_not_found.append(f"{name}=={version} for: {environment}") + if not fetched: + wheels_or_sdist_not_found[f"{name}=={version}"].append(environment) if TRACE: print(f" NOT FOUND") - if sdists: + if (sdists or + (f"{name}=={version}" in wheels_or_sdist_not_found and name in sdist_only) + ): if TRACE: print(f" ==> Fetching sdist: {name}=={version}") + fetched = utils_thirdparty.download_sdist( name=name, version=version, dest_dir=dest_dir, repos=repos, ) - if fetched: - sdists_fetched.append(fetched) - else: - sdists_not_found.append(f"{name}=={version}") + if not fetched: + wheels_or_sdist_not_found[f"{name}=={version}"].append("sdist") if TRACE: print(f" NOT FOUND") - if wheels and wheels_not_found: - print(f"==> MISSING WHEELS") - for wh in wheels_not_found: - print(f" {wh}") - - if sdists and sdists_not_found: - print(f"==> MISSING SDISTS") - for sd in sdists_not_found: - print(f" {sd}") + mia = [] + for nv, dists in wheels_or_sdist_not_found.items(): + name, _, version = nv.partition("==") + if name in no_dist: + continue + sdist_missing = sdists and "sdist" in dists and not name in wheel_only + if sdist_missing: + mia.append(f"SDist missing: {nv} {dists}") + wheels_missing = wheels and any(d for d in dists if d != "sdist") and not name in sdist_only + if wheels_missing: + mia.append(f"Wheels missing: {nv} {dists}") + + if mia: + for m in mia: + print(m) + raise Exception(mia) print(f"==> FETCHING OR CREATING ABOUT AND LICENSE FILES") utils_thirdparty.fetch_abouts_and_licenses(dest_dir=dest_dir, use_cached_index=use_cached_index) diff --git a/etc/scripts/gen_pypi_simple.py b/etc/scripts/gen_pypi_simple.py index 03312ab3..214d90dc 100644 --- a/etc/scripts/gen_pypi_simple.py +++ b/etc/scripts/gen_pypi_simple.py @@ -118,7 +118,7 @@ def build_per_package_index(pkg_name, packages, base_url): """ document.append(header) - for package in packages: + for package in sorted(packages, key=lambda p: p.archive_file): document.append(package.simple_index_entry(base_url)) footer = """ @@ -141,8 +141,8 @@ def build_links_package_index(packages_by_package_name, base_url): """ document.append(header) - for _name, packages in packages_by_package_name.items(): - for package in packages: + for _name, packages in sorted(packages_by_package_name.items(), key=lambda i: i[0]): + for package in sorted(packages, key=lambda p: p.archive_file): document.append(package.simple_index_entry(base_url)) footer = """ diff --git a/etc/scripts/requirements.txt b/etc/scripts/requirements.txt index ebb404b7..7c514da9 100644 --- a/etc/scripts/requirements.txt +++ b/etc/scripts/requirements.txt @@ -8,4 +8,5 @@ pip setuptools twine wheel -build \ No newline at end of file +build +packvers diff --git a/etc/scripts/utils_dejacode.py b/etc/scripts/utils_dejacode.py index f28e2479..c42e6c93 100644 --- a/etc/scripts/utils_dejacode.py +++ b/etc/scripts/utils_dejacode.py @@ -15,7 +15,7 @@ import requests import saneyaml -from packaging import version as packaging_version +from packvers import version as packaging_version """ Utility to create and retrieve package and ABOUT file data from DejaCode. diff --git a/etc/scripts/utils_pip_compatibility_tags.py b/etc/scripts/utils_pip_compatibility_tags.py index 5d5eb34c..af42a0cd 100644 --- a/etc/scripts/utils_pip_compatibility_tags.py +++ b/etc/scripts/utils_pip_compatibility_tags.py @@ -27,7 +27,7 @@ import re -from packaging.tags import ( +from packvers.tags import ( compatible_tags, cpython_tags, generic_tags, diff --git a/etc/scripts/utils_requirements.py b/etc/scripts/utils_requirements.py index 7c99a33b..0fc25a35 100644 --- a/etc/scripts/utils_requirements.py +++ b/etc/scripts/utils_requirements.py @@ -8,6 +8,8 @@ # See https://github.com/nexB/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # + +import os import re import subprocess diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index 9cbda374..addf8e5e 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -28,8 +28,8 @@ from commoncode import fileutils from commoncode.hash import multi_checksums from commoncode.text import python_safe_name -from packaging import tags as packaging_tags -from packaging import version as packaging_version +from packvers import tags as packaging_tags +from packvers import version as packaging_version import utils_pip_compatibility_tags @@ -115,10 +115,9 @@ TRACE_ULTRA_DEEP = False # Supported environments -PYTHON_VERSIONS = "36", "37", "38", "39", "310" +PYTHON_VERSIONS = "37", "38", "39", "310" PYTHON_DOT_VERSIONS_BY_VER = { - "36": "3.6", "37": "3.7", "38": "3.8", "39": "3.9", @@ -134,7 +133,6 @@ def get_python_dot_version(version): ABIS_BY_PYTHON_VERSION = { - "36": ["cp36", "cp36m", "abi3"], "37": ["cp37", "cp37m", "abi3"], "38": ["cp38", "cp38m", "abi3"], "39": ["cp39", "cp39m", "abi3"], @@ -311,6 +309,7 @@ def download_sdist(name, version, dest_dir=THIRDPARTY_DIR, repos=tuple()): return fetched_sdist_filename + ################################################################################ # # Core models @@ -911,7 +910,7 @@ def load_pkginfo_data(self, dest_dir=THIRDPARTY_DIR): declared_license = [raw_data["License"]] + [ c for c in classifiers if c.startswith("License") ] - license_expression = compute_normalized_license_expression(declared_license) + license_expression = get_license_expression(declared_license) other_classifiers = [c for c in classifiers if not c.startswith("License")] holder = raw_data["Author"] @@ -1064,16 +1063,16 @@ def get_sdist_name_ver_ext(filename): if version.isalpha(): return False - # non-pep 440 version + # non-pep 440 version if "-" in version: return False - # single version + # single version if version.isdigit() and len(version) == 1: return False - # r1 version - if len(version) == 2 and version[0]=="r" and version[1].isdigit(): + # r1 version + if len(version) == 2 and version[0] == "r" and version[1].isdigit(): return False # dotless version (but calver is OK) @@ -1336,10 +1335,10 @@ def package_from_dists(cls, dists): For example: >>> w1 = Wheel(name='bitarray', version='0.8.1', build='', - ... python_versions=['cp36'], abis=['cp36m'], + ... python_versions=['cp38'], abis=['cp38m'], ... platforms=['linux_x86_64']) >>> w2 = Wheel(name='bitarray', version='0.8.1', build='', - ... python_versions=['cp36'], abis=['cp36m'], + ... python_versions=['cp38'], abis=['cp38m'], ... platforms=['macosx_10_9_x86_64', 'macosx_10_10_x86_64']) >>> sd = Sdist(name='bitarray', version='0.8.1') >>> package = PypiPackage.package_from_dists(dists=[w1, w2, sd]) @@ -1588,6 +1587,7 @@ def tags(self): ) ) + ################################################################################ # # PyPI repo and link index for package wheels and sources @@ -1629,7 +1629,9 @@ class PypiSimpleRepository: use_cached_index = attr.ib( type=bool, default=False, - metadata=dict(help="If True, use any existing on-disk cached PyPI index files. Otherwise, fetch and cache."), + metadata=dict( + help="If True, use any existing on-disk cached PyPI index files. Otherwise, fetch and cache." + ), ) def _get_package_versions_map(self, name): @@ -1674,6 +1676,7 @@ def get_package_version(self, name, version=None): """ if not version: versions = list(self._get_package_versions_map(name).values()) + # return the latest version return versions and versions[-1] else: return self._get_package_versions_map(name).get(version) @@ -1724,7 +1727,9 @@ class LinksRepository: use_cached_index = attr.ib( type=bool, default=False, - metadata=dict(help="If True, use any existing on-disk cached index files. Otherwise, fetch and cache."), + metadata=dict( + help="If True, use any existing on-disk cached index files. Otherwise, fetch and cache." + ), ) def __attrs_post_init__(self): @@ -1790,6 +1795,7 @@ def from_url(cls, url=ABOUT_BASE_URL, _LINKS_REPO={}, use_cached_index=False): _LINKS_REPO[url] = cls(url=url, use_cached_index=use_cached_index) return _LINKS_REPO[url] + ################################################################################ # Globals for remote repos to be lazily created and cached on first use for the # life of the session together with some convenience functions. @@ -1803,6 +1809,7 @@ def get_local_packages(directory=THIRDPARTY_DIR): """ return list(PypiPackage.packages_from_dir(directory=directory)) + ################################################################################ # # Basic file and URL-based operations using a persistent file-based Cache @@ -1953,6 +1960,7 @@ def fetch_and_save( fo.write(content) return content + ################################################################################ # # Functions to update or fetch ABOUT and license files @@ -2051,7 +2059,9 @@ def get_other_dists(_package, _dist): # if has key data we may look to improve later, but we can move on if local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir=dest_dir) - local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index) + local_dist.fetch_license_files( + dest_dir=dest_dir, use_cached_index=use_cached_index + ) continue # lets try to fetch remotely @@ -2089,7 +2099,9 @@ def get_other_dists(_package, _dist): # if has key data we may look to improve later, but we can move on if local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir=dest_dir) - local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index) + local_dist.fetch_license_files( + dest_dir=dest_dir, use_cached_index=use_cached_index + ) continue # try to get data from pkginfo (no license though) @@ -2107,6 +2119,7 @@ def get_other_dists(_package, _dist): lic_errs = "\n".join(lic_errs) print(f"Failed to fetch some licenses:: {lic_errs}") + ################################################################################ # # Functions to build new Python wheels including native on multiple OSes @@ -2210,6 +2223,7 @@ def download_wheels_with_pip( downloaded = existing ^ set(os.listdir(dest_dir)) return sorted(downloaded), error + ################################################################################ # # Functions to check for problems @@ -2258,16 +2272,16 @@ def find_problems( check_about(dest_dir=dest_dir) -def compute_normalized_license_expression(declared_licenses): +def get_license_expression(declared_licenses): """ Return a normalized license expression or None. """ if not declared_licenses: return try: - from packagedcode import pypi + from packagedcode.licensing import get_only_expression_from_extracted_license - return pypi.compute_normalized_license(declared_licenses) + return get_only_expression_from_extracted_license(declared_licenses) except ImportError: # Scancode is not installed, clean and join all the licenses lics = [python_safe_name(l).lower() for l in declared_licenses] diff --git a/requirements-dev.txt b/requirements-dev.txt index fe92ed8e..5a2b6959 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,4 +1,4 @@ -aboutcode-toolkit==7.0.1 +aboutcode-toolkit==7.0.2 bleach==4.1.0 build==0.7.0 commonmark==0.9.1 diff --git a/setup.cfg b/setup.cfg index d2562b0f..e4b7ed61 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,6 +38,7 @@ license_files = NOTICE AUTHORS.rst CHANGELOG.rst + CODE_OF_CONDUCT.rst [options] package_dir = @@ -48,7 +49,7 @@ zip_safe = false setup_requires = setuptools_scm[toml] >= 4 -python_requires = >=3.6.* +python_requires = >=3.7 install_requires = attrs @@ -64,10 +65,15 @@ where = src testing = pytest >= 6, != 7.0.0 pytest-xdist >= 2 - aboutcode-toolkit >= 6.0.0 + aboutcode-toolkit >= 7.0.2 + pycodestyle >= 2.8.0 + twine black + isort docs = - Sphinx >= 3.3.1 - sphinx-rtd-theme >= 0.5.0 - doc8 >= 0.8.1 + Sphinx>=5.0.2 + sphinx-rtd-theme>=1.0.0 + sphinx-reredirects >= 0.1.2 + doc8>=0.11.2 +