Merge latest skeleton in python-inspector

.gitignore

@@ -2,6 +2,7 @@
 *.py[cod]
 
 # virtualenv and other misc bits
+/src/*.egg-info
 *.egg-info
 /dist
 /build

azure-pipelines.yml

@@ -31,13 +31,21 @@ jobs:
           test_suites:
               all: venv/bin/pytest -n 2 -vvs --reruns 2
 
+    - template: etc/ci/azure-posix.yml
+      parameters:
+          job_name: macos1015_cpython
+          image_name: macos-10.15
+          python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11']
+          test_suites:
+              all: venv/bin/pytest -n 2 -vvs --reruns 2
+
     - template: etc/ci/azure-posix.yml
       parameters:
           job_name: macos11_cpython
           image_name: macos-11
           python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11']
           test_suites:
-              all: venv/bin/pytest -n 2 -vvs --reruns 2
+              all: venv/bin/pytest -n 2 -vvs
 
     - template: etc/ci/azure-posix.yml
       parameters:

etc/scripts/README.rst

@@ -21,7 +21,7 @@ Pre-requisites
     virtualenv or in the the main configured development virtualenv.
     These requireements need to be installed::
 
-        pip install --requirement etc/release/requirements.txt
+        pip install --requirement etc/scripts/requirements.txt
 
 TODO: we need to pin the versions of these tools
 
@@ -34,7 +34,7 @@ Scripts
 ~~~~~~~
 
 **gen_requirements.py**: create/update requirements files from currently
-  installed requirements. 
+  installed requirements.
 
 **gen_requirements_dev.py** does the same but can subtract the main requirements
   to get extra requirements used in only development.
@@ -50,7 +50,7 @@ The sequence of commands to run are:
 
     ./configure --clean
     ./configure
-    python etc/release/gen_requirements.py --site-packages-dir <path to site-packages dir>
+    python etc/scripts/gen_requirements.py --site-packages-dir <path to site-packages dir>
 
 * You can optionally install or update extra main requirements after the
   ./configure step such that these are included in the generated main requirements.
@@ -59,7 +59,7 @@ The sequence of commands to run are:
 
     ./configure --clean
     ./configure --dev
-    python etc/release/gen_requirements_dev.py --site-packages-dir <path to site-packages dir>
+    python etc/scripts/gen_requirements_dev.py --site-packages-dir <path to site-packages dir>
 
 * You can optionally install or update extra dev requirements after the
   ./configure step such that these are included in the generated dev

etc/scripts/fetch_thirdparty.py

@@ -12,6 +12,7 @@
 import itertools
 import os
 import sys
+from collections import defaultdict
 
 import click
 
@@ -110,6 +111,39 @@
     is_flag=True,
     help="Use on disk cached PyPI indexes list of packages and versions and do not refetch if present.",
 )
+@click.option(
+    "--sdist-only",
+    "sdist_only",
+    type=str,
+    metavar="SDIST",
+    default=tuple(),
+    show_default=False,
+    multiple=True,
+    help="Package name(s) that come only in sdist format (no wheels). "
+         "The command will not fail and exit if no wheel exists for these names",
+)
+@click.option(
+    "--wheel-only",
+    "wheel_only",
+    type=str,
+    metavar="WHEEL",
+    default=tuple(),
+    show_default=False,
+    multiple=True,
+    help="Package name(s) that come only in wheel format (no sdist). "
+         "The command will not fail and exit if no sdist exists for these names",
+)
+@click.option(
+    "--no-dist",
+    "no_dist",
+    type=str,
+    metavar="DIST",
+    default=tuple(),
+    show_default=False,
+    multiple=True,
+    help="Package name(s) that do not come either in wheel or sdist format. "
+         "The command will not fail and exit if no distribution exists for these names",
+)
 @click.help_option("-h", "--help")
 def fetch_thirdparty(
     requirements_files,
@@ -122,6 +156,9 @@ def fetch_thirdparty(
     sdists,
     index_urls,
     use_cached_index,
+    sdist_only,
+    wheel_only,
+    no_dist,
 ):
     """
     Download to --dest THIRDPARTY_DIR the PyPI wheels, source distributions,
@@ -204,58 +241,62 @@ def fetch_thirdparty(
             )
             repos.append(repo)
 
-    wheels_fetched = []
-    wheels_not_found = []
-
-    sdists_fetched = []
-    sdists_not_found = []
+    wheels_or_sdist_not_found = defaultdict(list)
 
     for name, version in sorted(required_name_versions):
         nv = name, version
         print(f"Processing: {name} @ {version}")
         if wheels:
             for environment in environments:
+
                 if TRACE:
                     print(f"  ==> Fetching wheel for envt: {environment}")
-                fwfns = utils_thirdparty.download_wheel(
+
+                fetched = utils_thirdparty.download_wheel(
                     name=name,
                     version=version,
                     environment=environment,
                     dest_dir=dest_dir,
                     repos=repos,
                 )
-                if fwfns:
-                    wheels_fetched.extend(fwfns)
-                else:
-                    wheels_not_found.append(f"{name}=={version} for: {environment}")
+                if not fetched:
+                    wheels_or_sdist_not_found[f"{name}=={version}"].append(environment)
                     if TRACE:
                         print(f"      NOT FOUND")
 
-        if sdists:
+        if (sdists or
+            (f"{name}=={version}" in wheels_or_sdist_not_found and name in sdist_only)
+         ):
             if TRACE:
                 print(f"  ==> Fetching sdist: {name}=={version}")
+
             fetched = utils_thirdparty.download_sdist(
                 name=name,
                 version=version,
                 dest_dir=dest_dir,
                 repos=repos,
             )
-            if fetched:
-                sdists_fetched.append(fetched)
-            else:
-                sdists_not_found.append(f"{name}=={version}")
+            if not fetched:
+                wheels_or_sdist_not_found[f"{name}=={version}"].append("sdist")
                 if TRACE:
                     print(f"      NOT FOUND")
 
-    if wheels and wheels_not_found:
-        print(f"==> MISSING WHEELS")
-        for wh in wheels_not_found:
-            print(f"  {wh}")
+    mia = []
+    for nv, dists in wheels_or_sdist_not_found.items():
+        name, _, version = nv.partition("==")
+        if name in no_dist:
+            continue
+        sdist_missing = sdists and "sdist" in dists and not name in wheel_only
+        if sdist_missing:
+            mia.append(f"SDist missing: {nv} {dists}")
+        wheels_missing = wheels and any(d for d in dists if d != "sdist") and not name in sdist_only
+        if wheels_missing:
+            mia.append(f"Wheels missing: {nv} {dists}")
 
-    if sdists and sdists_not_found:
-        print(f"==> MISSING SDISTS")
-        for sd in sdists_not_found:
-            print(f"  {sd}")
+    if mia:
+        for m in mia:
+            print(m)
+        raise Exception(mia)
 
     print(f"==> FETCHING OR CREATING ABOUT AND LICENSE FILES")
     utils_thirdparty.fetch_abouts_and_licenses(dest_dir=dest_dir, use_cached_index=use_cached_index)

etc/scripts/gen_pypi_simple.py

@@ -118,7 +118,7 @@ def build_per_package_index(pkg_name, packages, base_url):
   <body>"""
     document.append(header)
 
-    for package in packages:
+    for package in sorted(packages, key=lambda p: p.archive_file):
         document.append(package.simple_index_entry(base_url))
 
     footer = """  </body>
@@ -141,8 +141,8 @@ def build_links_package_index(packages_by_package_name, base_url):
   <body>"""
     document.append(header)
 
-    for _name, packages in packages_by_package_name.items():
-        for package in packages:
+    for _name, packages in sorted(packages_by_package_name.items(), key=lambda i: i[0]):
+        for package in sorted(packages, key=lambda p: p.archive_file):
             document.append(package.simple_index_entry(base_url))
 
     footer = """  </body>

etc/scripts/requirements.txt

@@ -8,4 +8,5 @@ pip
 setuptools
 twine
 wheel
-build
+build
+packvers

etc/scripts/utils_dejacode.py

@@ -15,7 +15,7 @@
 import requests
 import saneyaml
 
-from packaging import version as packaging_version
+from packvers import version as packaging_version
 
 """
 Utility to create and retrieve package and ABOUT file data from DejaCode.

etc/scripts/utils_pip_compatibility_tags.py

@@ -27,7 +27,7 @@
 
 import re
 
-from packaging.tags import (
+from packvers.tags import (
     compatible_tags,
     cpython_tags,
     generic_tags,

etc/scripts/utils_requirements.py

@@ -8,6 +8,7 @@
 # See https://github.com/nexB/skeleton for support or download.
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
+
 import os
 import re
 import subprocess

etc/scripts/utils_thirdparty.py

@@ -28,8 +28,8 @@
 from commoncode import fileutils
 from commoncode.hash import multi_checksums
 from commoncode.text import python_safe_name
-from packaging import tags as packaging_tags
-from packaging import version as packaging_version
+from packvers import tags as packaging_tags
+from packvers import version as packaging_version
 
 import utils_pip_compatibility_tags
 
@@ -115,10 +115,9 @@
 TRACE_ULTRA_DEEP = False
 
 # Supported environments
-PYTHON_VERSIONS = "36", "37", "38", "39", "310"
+PYTHON_VERSIONS = "37", "38", "39", "310"
 
 PYTHON_DOT_VERSIONS_BY_VER = {
-    "36": "3.6",
     "37": "3.7",
     "38": "3.8",
     "39": "3.9",
@@ -134,7 +133,6 @@ def get_python_dot_version(version):
 
 
 ABIS_BY_PYTHON_VERSION = {
-    "36": ["cp36", "cp36m", "abi3"],
     "37": ["cp37", "cp37m", "abi3"],
     "38": ["cp38", "cp38m", "abi3"],
     "39": ["cp39", "cp39m", "abi3"],
@@ -912,7 +910,7 @@ def load_pkginfo_data(self, dest_dir=THIRDPARTY_DIR):
         declared_license = [raw_data["License"]] + [
             c for c in classifiers if c.startswith("License")
         ]
-        license_expression = compute_normalized_license_expression(declared_license)
+        license_expression = get_license_expression(declared_license)
         other_classifiers = [c for c in classifiers if not c.startswith("License")]
 
         holder = raw_data["Author"]
@@ -1337,10 +1335,10 @@ def package_from_dists(cls, dists):
 
         For example:
         >>> w1 = Wheel(name='bitarray', version='0.8.1', build='',
-        ...    python_versions=['cp36'], abis=['cp36m'],
+        ...    python_versions=['cp38'], abis=['cp38m'],
         ...    platforms=['linux_x86_64'])
         >>> w2 = Wheel(name='bitarray', version='0.8.1', build='',
-        ...    python_versions=['cp36'], abis=['cp36m'],
+        ...    python_versions=['cp38'], abis=['cp38m'],
         ...    platforms=['macosx_10_9_x86_64', 'macosx_10_10_x86_64'])
         >>> sd = Sdist(name='bitarray', version='0.8.1')
         >>> package = PypiPackage.package_from_dists(dists=[w1, w2, sd])
@@ -2274,16 +2272,16 @@ def find_problems(
     check_about(dest_dir=dest_dir)
 
 
-def compute_normalized_license_expression(declared_licenses):
+def get_license_expression(declared_licenses):
     """
     Return a normalized license expression or None.
     """
     if not declared_licenses:
         return
     try:
-        from packagedcode import pypi
+        from packagedcode.licensing import get_only_expression_from_extracted_license
 
-        return pypi.compute_normalized_license(declared_licenses)
+        return get_only_expression_from_extracted_license(declared_licenses)
     except ImportError:
         # Scancode is not installed, clean and join all the licenses
         lics = [python_safe_name(l).lower() for l in declared_licenses]

requirements.txt

@@ -1,27 +1,27 @@
-attrs==22.1.0
+attrs==22.2.0
 beautifulsoup4==4.11.1
-certifi==2022.6.15
-charset-normalizer==2.1.0
+certifi==2022.12.7
+charset-normalizer==3.0.1
 click==8.1.3
-colorama==0.4.5
-commoncode==30.2.0
+colorama==0.4.6
+commoncode==31.0.0
 dparse2==0.7.0
-idna==3.3
-importlib-metadata==4.12.0
+idna==3.4
+importlib-metadata==6.0.0
 intbitset==3.0.2
-packageurl-python==0.10.0
+packageurl-python==0.10.4
 packaging==21.3
 packvers==21.5
 pip-requirements-parser==32.0.1
 pkginfo2==30.0.0
 pyparsing==3.0.9
 PyYAML==6.0
-requests==2.28.1
-resolvelib==0.8.1
-saneyaml==0.5.2
+requests==2.28.2
+resolvelib==0.9.0
+saneyaml==0.6.0
 soupsieve==2.3.2.post1
 text-unidecode==1.3
 tinynetrc==1.3.1
 toml==0.10.2
-urllib3==1.26.11
-zipp==3.8.1
+urllib3==1.26.14
+zipp==3.12.0

setup.cfg

@@ -51,7 +51,7 @@ zip_safe = false
 
 setup_requires = setuptools_scm[toml] >= 4
 
-python_requires = >=3.6.*
+python_requires = >=3.7
 
 install_requires =
     attrs >= 18.1, !=20.1.0