diff --git a/README.md b/README.md index 5c42502..00c0fe1 100644 --- a/README.md +++ b/README.md @@ -4,33 +4,76 @@ ![security check](https://github.com/gardenlinux/parse_features_lib/actions/workflows/bandit.yml/badge.svg) # Parse features lib -This library helps you to work with the gardenlinux/features folder. It parses all info.yamls and builds a tree. -Features (planned): -* validate CNAMEs -* validate info.yamls -* Deduct dependencies from cname_base +This library includes tooling to build and distribute [Garden Linux](https://github.com/gardenlinux/gardenlinux). + +Features: + +- compare APT repositories +- parse features +- parse flavors +- push OCI artifacts to a registry ## Quickstart + +### Example: get a list of features for a given cname + **Inclusion via poetry**: -`parse_features_lib = { git = "https://github.com/gardenlinux/parse_features_lib", rev="main" }` +`gardenlinux = { git = "https://github.com/gardenlinux/python_gardenlinux_lib", rev="0.6.0" }` + ```python -import parse_features_lib +from gardenlinux.features import Parser + +cname = "aws-gardener_prod" +feature_list = Parser().filter_as_list(cname) +print(f"features of {cname}:") +for feature in feature_list: + print(feature) +``` + +## Developer Documentation + +The library is documented with docstrings, which are used to generate the developer documentation available [here](https://gardenlinux.github.io/python-gardenlinux-lib/). + +## Push OCI artifacts to a registry -if __name__ == "__main__": - # Step 1: parse the "features directory" and get the full graph containing all features - all_features = parse_features_lib.read_feature_files("features") +this tool helps you to push oci artifacts. - # Step 2: supply desired features and get all their dependencies - dependencies = parse_features_lib.filter_graph(all_features, {"gardener", "_prod", "server", "ociExample"}) +### Installation - # Step 3: play with the retrieved data. - for feature, info in dependencies.nodes(data="content"): - if "oci_artifacts" in info: - print(feature, info["oci_artifacts"]) +```bash +git clone https://github.com/gardenlinux/python-gardenlinux-lib.git +mkdir venv +python -m venv venv +source venv/bin/activate.sh +poetry install +gl-oci --help ``` -## Developer Documentation -The library is documented with docstrings, which are used to generate the developer documentation available [here](https://gardenlinux.github.io/python-gardenlinux-lib/). +### Usage + +The process to push a Gardenlinux build-output folder to an OCI registry is split into two steps: In the first step all files are pushed to the registry and a manifest that includes all those pushed files (layers) is created and pushed as well. An index entry that links to this manifest is created offline and written to a local file but not pushed to any index. This push to an index can be done in the second step where the local file containing the index entry is read and pushed to an index. The seperation into two steps was done because pushing of manifests takes long and writes to dedicated resources (possible to run in parallel). Updating the index on the other hand is quick but writes to a share resource (not possible to run in parallel). By splitting the process up into two steps it is possible to run the slow part in parallel and the quick part sequentially. + +#### 1. Push layers + manifest + +To push layers you have to supply the directory with the build outputs `--dir`. Also you have to supply cname (`--cname`), architecture `--arch` and version `--version` of the build. This information will be included in the manifest. You have to supply an endpoint where the artifacts shall be pushed to `--container`, for example `ghcr.io/gardenlinux/gardenlinux`. You can disable enforced HTTPS connections to your registry with `--insecure True`. You can supply `--cosign_file ` if you want to have the hash saved in ``. This can be handy to read the hash later to sign the manifest with cosign. With `--manifest_file ` you tell the program in which file to store the manifests index entry. This is the file that can be used in the next step to update the index. You can use the environment variable GL_CLI_REGISTRY_TOKEN to authenticate against the registry. Below is an example of a full program call of `push-manifest` + +```bash +GL_CLI_REGISTRY_TOKEN=asdf123 gl-oci push-manifest --dir build-metal-gardener_prod --container ghcr.io/gardenlinux/gl-oci --arch amd64 --version 1592.1 --cname metal-gardener_prod --cosign_file digest --manifest_file oci_manifest_entry_metal.json +``` +#### 2. Update index with manifest entry + +Parameters that are the same as for `push-manifest`: + +- env-var `GL_CLI_REGISTRY_TOKEN` +- `--version` +- `--container` +- `--manifest-file` this time this parameter adjusts the manifest entry file to be read from instead of being written to + +A full example looks like this: + +```bash +GL_CLI_REGISTRY_TOKEN=asdf123 gl-oci update-index --container ghcr.io/gardenlinux/gl-oci --version 1592.1 --manifest_file oci_manifest_entry_metal.json +``` diff --git a/poetry.lock b/poetry.lock index 72732c2..42b72cc 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.1.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. [[package]] name = "alabaster" @@ -6,7 +6,6 @@ version = "0.7.16" description = "A light, configurable Sphinx theme" optional = false python-versions = ">=3.9" -groups = ["docs"] files = [ {file = "alabaster-0.7.16-py3-none-any.whl", hash = "sha256:b46733c07dce03ae4e150330b975c75737fa60f0a7c591b6c8bf4928a28e2c92"}, {file = "alabaster-0.7.16.tar.gz", hash = "sha256:75a8b99c28a5dad50dd7f8ccdd447a121ddb3892da9e53d1ca5cca3106d58d65"}, @@ -18,7 +17,6 @@ version = "0.5" description = "Python library to query APT repositories" optional = false python-versions = ">=3.5" -groups = ["main"] files = [ {file = "apt-repo-0.5.tar.gz", hash = "sha256:b566195884b8ea59e6b831f814fd106c7e683dccc86ca95f6494a447572f30ea"}, ] @@ -29,19 +27,18 @@ version = "25.1.0" description = "Classes Without Boilerplate" optional = false python-versions = ">=3.8" -groups = ["main"] files = [ {file = "attrs-25.1.0-py3-none-any.whl", hash = "sha256:c75a69e28a550a7e93789579c22aa26b0f5b83b75dc4e08fe092980051e1090a"}, {file = "attrs-25.1.0.tar.gz", hash = "sha256:1c97078a80c814273a76b2a298a932eb681c87415c11dee0a6921de7f1b02c3e"}, ] [package.extras] -benchmark = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-xdist[psutil]"] -cov = ["cloudpickle ; platform_python_implementation == \"CPython\"", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-xdist[psutil]"] -dev = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pre-commit-uv", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-xdist[psutil]"] +benchmark = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +cov = ["cloudpickle", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +dev = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pre-commit-uv", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier (<24.7)"] -tests = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-xdist[psutil]"] -tests-mypy = ["mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\""] +tests = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +tests-mypy = ["mypy (>=1.11.1)", "pytest-mypy-plugins"] [[package]] name = "babel" @@ -49,14 +46,13 @@ version = "2.17.0" description = "Internationalization utilities" optional = false python-versions = ">=3.8" -groups = ["docs"] files = [ {file = "babel-2.17.0-py3-none-any.whl", hash = "sha256:4d0b53093fdfb4b21c92b5213dba5a1b23885afa8383709427046b21c366e5f2"}, {file = "babel-2.17.0.tar.gz", hash = "sha256:0c54cffb19f690cdcc52a3b50bcbf71e07a808d1c80d549f2459b9d2cf0afb9d"}, ] [package.extras] -dev = ["backports.zoneinfo ; python_version < \"3.9\"", "freezegun (>=1.0,<2.0)", "jinja2 (>=3.0)", "pytest (>=6.0)", "pytest-cov", "pytz", "setuptools", "tzdata ; sys_platform == \"win32\""] +dev = ["backports.zoneinfo", "freezegun (>=1.0,<2.0)", "jinja2 (>=3.0)", "pytest (>=6.0)", "pytest-cov", "pytz", "setuptools", "tzdata"] [[package]] name = "bandit" @@ -64,7 +60,6 @@ version = "1.8.3" description = "Security oriented static analyser for python code." optional = false python-versions = ">=3.9" -groups = ["dev"] files = [ {file = "bandit-1.8.3-py3-none-any.whl", hash = "sha256:28f04dc0d258e1dd0f99dee8eefa13d1cb5e3fde1a5ab0c523971f97b289bcd8"}, {file = "bandit-1.8.3.tar.gz", hash = "sha256:f5847beb654d309422985c36644649924e0ea4425c76dec2e89110b87506193a"}, @@ -80,7 +75,7 @@ stevedore = ">=1.20.0" baseline = ["GitPython (>=3.1.30)"] sarif = ["jschema-to-python (>=1.2.3)", "sarif-om (>=1.0.4)"] test = ["beautifulsoup4 (>=4.8.0)", "coverage (>=4.5.4)", "fixtures (>=3.0.0)", "flake8 (>=4.0.0)", "pylint (==1.9.4)", "stestr (>=2.5.0)", "testscenarios (>=0.5.0)", "testtools (>=2.3.0)"] -toml = ["tomli (>=1.1.0) ; python_version < \"3.11\""] +toml = ["tomli (>=1.1.0)"] yaml = ["PyYAML"] [[package]] @@ -89,7 +84,6 @@ version = "24.10.0" description = "The uncompromising code formatter." optional = false python-versions = ">=3.9" -groups = ["dev"] files = [ {file = "black-24.10.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e6668650ea4b685440857138e5fe40cde4d652633b1bdffc62933d0db4ed9812"}, {file = "black-24.10.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1c536fcf674217e87b8cc3657b81809d3c085d7bf3ef262ead700da345bfa6ea"}, @@ -136,7 +130,6 @@ version = "1.36.21" description = "The AWS SDK for Python" optional = false python-versions = ">=3.8" -groups = ["main"] files = [ {file = "boto3-1.36.21-py3-none-any.whl", hash = "sha256:f94faa7cf932d781f474d87f8b4c14a033af95ac1460136b40d75e7a30086ef0"}, {file = "boto3-1.36.21.tar.gz", hash = "sha256:41eb2b73eb612d300e629e3328b83f1ffea0fc6633e75c241a72a76746c1db26"}, @@ -156,7 +149,6 @@ version = "1.36.21" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">=3.8" -groups = ["main"] files = [ {file = "botocore-1.36.21-py3-none-any.whl", hash = "sha256:24a7052e792639dc2726001bd474cd0aaa959c1e18ddd92c17f3adc6efa1b132"}, {file = "botocore-1.36.21.tar.gz", hash = "sha256:da746240e2ad64fd4997f7f3664a0a8e303d18075fc1d473727cb6375080ea16"}, @@ -176,7 +168,6 @@ version = "2025.1.31" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" -groups = ["main", "docs"] files = [ {file = "certifi-2025.1.31-py3-none-any.whl", hash = "sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe"}, {file = "certifi-2025.1.31.tar.gz", hash = "sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651"}, @@ -188,8 +179,6 @@ version = "1.17.1" description = "Foreign Function Interface for Python calling C code." optional = false python-versions = ">=3.8" -groups = ["main"] -markers = "platform_python_implementation != \"PyPy\"" files = [ {file = "cffi-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:df8b1c11f177bc2313ec4b2d46baec87a5f3e71fc8b45dab2ee7cae86d9aba14"}, {file = "cffi-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f2cdc858323644ab277e9bb925ad72ae0e67f69e804f4898c070998d50b1a67"}, @@ -269,7 +258,6 @@ version = "3.4.1" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." optional = false python-versions = ">=3.7" -groups = ["main", "docs"] files = [ {file = "charset_normalizer-3.4.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:91b36a978b5ae0ee86c394f5a54d6ef44db1de0815eb43de826d41d21e4af3de"}, {file = "charset_normalizer-3.4.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7461baadb4dc00fd9e0acbe254e3d7d2112e7f92ced2adc96e54ef6501c5f176"}, @@ -371,7 +359,6 @@ version = "8.1.8" description = "Composable command line interface toolkit" optional = false python-versions = ">=3.7" -groups = ["dev"] files = [ {file = "click-8.1.8-py3-none-any.whl", hash = "sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2"}, {file = "click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a"}, @@ -386,12 +373,10 @@ version = "0.4.6" description = "Cross-platform colored terminal text." optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" -groups = ["main", "dev", "docs"] files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] -markers = {main = "sys_platform == \"win32\"", dev = "platform_system == \"Windows\"", docs = "sys_platform == \"win32\""} [[package]] name = "cryptography" @@ -399,7 +384,6 @@ version = "44.0.1" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false python-versions = "!=3.9.0,!=3.9.1,>=3.7" -groups = ["main"] files = [ {file = "cryptography-44.0.1-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:bf688f615c29bfe9dfc44312ca470989279f0e94bb9f631f85e3459af8efc009"}, {file = "cryptography-44.0.1-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dd7c7e2d71d908dc0f8d2027e1604102140d84b155e658c20e8ad1304317691f"}, @@ -438,10 +422,10 @@ files = [ cffi = {version = ">=1.12", markers = "platform_python_implementation != \"PyPy\""} [package.extras] -docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=3.0.0) ; python_version >= \"3.8\""] +docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=3.0.0)"] docstest = ["pyenchant (>=3)", "readme-renderer (>=30.0)", "sphinxcontrib-spelling (>=7.3.1)"] -nox = ["nox (>=2024.4.15)", "nox[uv] (>=2024.3.2) ; python_version >= \"3.8\""] -pep8test = ["check-sdist ; python_version >= \"3.8\"", "click (>=8.0.1)", "mypy (>=1.4)", "ruff (>=0.3.6)"] +nox = ["nox (>=2024.4.15)", "nox[uv] (>=2024.3.2)"] +pep8test = ["check-sdist", "click (>=8.0.1)", "mypy (>=1.4)", "ruff (>=0.3.6)"] sdist = ["build (>=1.0.0)"] ssh = ["bcrypt (>=3.1.5)"] test = ["certifi (>=2024)", "cryptography-vectors (==44.0.1)", "pretend (>=0.7)", "pytest (>=7.4.0)", "pytest-benchmark (>=4.0)", "pytest-cov (>=2.10.1)", "pytest-xdist (>=3.5.0)"] @@ -453,7 +437,6 @@ version = "0.20.1" description = "Docutils -- Python Documentation Utilities" optional = false python-versions = ">=3.7" -groups = ["docs"] files = [ {file = "docutils-0.20.1-py3-none-any.whl", hash = "sha256:96f387a2c5562db4476f09f13bbab2192e764cac08ebbf3a34a95d9b1e4a59d6"}, {file = "docutils-0.20.1.tar.gz", hash = "sha256:f08a4e276c3a1583a86dce3e34aba3fe04d02bba2dd51ed16106244e8a923e3b"}, @@ -465,8 +448,6 @@ version = "1.2.2" description = "Backport of PEP 654 (exception groups)" optional = false python-versions = ">=3.7" -groups = ["main"] -markers = "python_version == \"3.10\"" files = [ {file = "exceptiongroup-1.2.2-py3-none-any.whl", hash = "sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b"}, {file = "exceptiongroup-1.2.2.tar.gz", hash = "sha256:47c2edf7c6738fafb49fd34290706d1a1a2f4d1c6df275526b62cbb4aa5393cc"}, @@ -481,7 +462,6 @@ version = "4.0.12" description = "Git Object Database" optional = false python-versions = ">=3.7" -groups = ["main"] files = [ {file = "gitdb-4.0.12-py3-none-any.whl", hash = "sha256:67073e15955400952c6565cc3e707c554a4eea2e428946f7a4c162fab9bd9bcf"}, {file = "gitdb-4.0.12.tar.gz", hash = "sha256:5ef71f855d191a3326fcfbc0d5da835f26b13fbcba60c32c21091c349ffdb571"}, @@ -496,7 +476,6 @@ version = "3.1.44" description = "GitPython is a Python library used to interact with Git repositories" optional = false python-versions = ">=3.7" -groups = ["main"] files = [ {file = "GitPython-3.1.44-py3-none-any.whl", hash = "sha256:9e0e10cda9bed1ee64bc9a6de50e7e38a9c9943241cd7f585f6df3ed28011110"}, {file = "gitpython-3.1.44.tar.gz", hash = "sha256:c87e30b26253bf5418b01b0660f818967f3c503193838337fe5e573331249269"}, @@ -507,7 +486,7 @@ gitdb = ">=4.0.1,<5" [package.extras] doc = ["sphinx (>=7.1.2,<7.2)", "sphinx-autodoc-typehints", "sphinx_rtd_theme"] -test = ["coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock ; python_version < \"3.8\"", "mypy", "pre-commit", "pytest (>=7.3.1)", "pytest-cov", "pytest-instafail", "pytest-mock", "pytest-sugar", "typing-extensions ; python_version < \"3.11\""] +test = ["coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock", "mypy", "pre-commit", "pytest (>=7.3.1)", "pytest-cov", "pytest-instafail", "pytest-mock", "pytest-sugar", "typing-extensions"] [[package]] name = "idna" @@ -515,7 +494,6 @@ version = "3.10" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.6" -groups = ["main", "docs"] files = [ {file = "idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3"}, {file = "idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9"}, @@ -530,7 +508,6 @@ version = "1.4.1" description = "Getting image size from png/jpeg/jpeg2000/gif file" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -groups = ["docs"] files = [ {file = "imagesize-1.4.1-py2.py3-none-any.whl", hash = "sha256:0d8d18d08f840c19d0ee7ca1fd82490fdc3729b7ac93f49870406ddde8ef8d8b"}, {file = "imagesize-1.4.1.tar.gz", hash = "sha256:69150444affb9cb0d5cc5a92b3676f0b2fb7cd9ae39e947a5e11a36b4497cd4a"}, @@ -542,7 +519,6 @@ version = "2.0.0" description = "brain-dead simple config-ini parsing" optional = false python-versions = ">=3.7" -groups = ["main"] files = [ {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, @@ -554,7 +530,6 @@ version = "3.1.5" description = "A very fast and expressive template engine." optional = false python-versions = ">=3.7" -groups = ["docs"] files = [ {file = "jinja2-3.1.5-py3-none-any.whl", hash = "sha256:aba0f4dc9ed8013c424088f68a5c226f7d6097ed89b246d7749c2ec4175c6adb"}, {file = "jinja2-3.1.5.tar.gz", hash = "sha256:8fefff8dc3034e27bb80d67c671eb8a9bc424c0ef4c0826edbff304cceff43bb"}, @@ -572,7 +547,6 @@ version = "1.0.1" description = "JSON Matching Expressions" optional = false python-versions = ">=3.7" -groups = ["main"] files = [ {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, @@ -584,7 +558,6 @@ version = "4.23.0" description = "An implementation of JSON Schema validation for Python" optional = false python-versions = ">=3.8" -groups = ["main"] files = [ {file = "jsonschema-4.23.0-py3-none-any.whl", hash = "sha256:fbadb6f8b144a8f8cf9f0b89ba94501d143e50411a1278633f56a7acf7fd5566"}, {file = "jsonschema-4.23.0.tar.gz", hash = "sha256:d71497fef26351a33265337fa77ffeb82423f3ea21283cd9467bb03999266bc4"}, @@ -606,7 +579,6 @@ version = "2024.10.1" description = "The JSON Schema meta-schemas and vocabularies, exposed as a Registry" optional = false python-versions = ">=3.9" -groups = ["main"] files = [ {file = "jsonschema_specifications-2024.10.1-py3-none-any.whl", hash = "sha256:a09a0680616357d9a0ecf05c12ad234479f549239d0f5b55f3deea67475da9bf"}, {file = "jsonschema_specifications-2024.10.1.tar.gz", hash = "sha256:0f38b83639958ce1152d02a7f062902c41c8fd20d558b0c34344292d417ae272"}, @@ -621,7 +593,6 @@ version = "3.0.0" description = "Python port of markdown-it. Markdown parsing, done right!" optional = false python-versions = ">=3.8" -groups = ["dev"] files = [ {file = "markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb"}, {file = "markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1"}, @@ -646,7 +617,6 @@ version = "3.0.2" description = "Safely add untrusted strings to HTML/XML markup." optional = false python-versions = ">=3.9" -groups = ["docs"] files = [ {file = "MarkupSafe-3.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7e94c425039cde14257288fd61dcfb01963e658efbc0ff54f5306b06054700f8"}, {file = "MarkupSafe-3.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9e2d922824181480953426608b81967de705c3cef4d1af983af849d7bd619158"}, @@ -717,7 +687,6 @@ version = "0.1.2" description = "Markdown URL utilities" optional = false python-versions = ">=3.7" -groups = ["dev"] files = [ {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, @@ -729,7 +698,6 @@ version = "1.0.0" description = "Type system extensions for programs checked with the mypy type checker." optional = false python-versions = ">=3.5" -groups = ["dev"] files = [ {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, @@ -741,7 +709,6 @@ version = "3.4.2" description = "Python package for creating and manipulating graphs and networks" optional = false python-versions = ">=3.10" -groups = ["main"] files = [ {file = "networkx-3.4.2-py3-none-any.whl", hash = "sha256:df5d4365b724cf81b8c6a7312509d0c22386097011ad1abe274afd5e9d3bbc5f"}, {file = "networkx-3.4.2.tar.gz", hash = "sha256:307c3669428c5362aab27c8a1260aa8f47c4e91d3891f48be0141738d8d053e1"}, @@ -755,13 +722,22 @@ example = ["cairocffi (>=1.7)", "contextily (>=1.6)", "igraph (>=0.11)", "momepy extra = ["lxml (>=4.6)", "pydot (>=3.0.1)", "pygraphviz (>=1.14)", "sympy (>=1.10)"] test = ["pytest (>=7.2)", "pytest-cov (>=4.0)"] +[[package]] +name = "opencontainers" +version = "0.0.14" +description = "Python module for oci specifications" +optional = false +python-versions = "*" +files = [ + {file = "opencontainers-0.0.14.tar.gz", hash = "sha256:fde3b8099b56b5c956415df8933e2227e1914e805a277b844f2f9e52341738f2"}, +] + [[package]] name = "oras" version = "0.2.0" description = "OCI Registry as Storage Python SDK" optional = false python-versions = "*" -groups = ["main"] files = [] develop = false @@ -786,7 +762,6 @@ version = "24.2" description = "Core utilities for Python packages" optional = false python-versions = ">=3.8" -groups = ["main", "dev", "docs"] files = [ {file = "packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759"}, {file = "packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f"}, @@ -798,7 +773,6 @@ version = "0.12.1" description = "Utility library for gitignore style pattern matching of file paths." optional = false python-versions = ">=3.8" -groups = ["dev"] files = [ {file = "pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08"}, {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, @@ -810,7 +784,6 @@ version = "6.1.1" description = "Python Build Reasonableness" optional = false python-versions = ">=2.6" -groups = ["dev"] files = [ {file = "pbr-6.1.1-py2.py3-none-any.whl", hash = "sha256:38d4daea5d9fa63b3f626131b9d34947fd0c8be9b05a29276870580050a25a76"}, {file = "pbr-6.1.1.tar.gz", hash = "sha256:93ea72ce6989eb2eed99d0f75721474f69ad88128afdef5ac377eb797c4bf76b"}, @@ -825,7 +798,6 @@ version = "4.3.6" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." optional = false python-versions = ">=3.8" -groups = ["dev"] files = [ {file = "platformdirs-4.3.6-py3-none-any.whl", hash = "sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb"}, {file = "platformdirs-4.3.6.tar.gz", hash = "sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907"}, @@ -842,7 +814,6 @@ version = "1.5.0" description = "plugin and hook calling mechanisms for python" optional = false python-versions = ">=3.8" -groups = ["main"] files = [ {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, @@ -858,8 +829,6 @@ version = "2.22" description = "C parser in Python" optional = false python-versions = ">=3.8" -groups = ["main"] -markers = "platform_python_implementation != \"PyPy\"" files = [ {file = "pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc"}, {file = "pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6"}, @@ -871,7 +840,6 @@ version = "2.19.1" description = "Pygments is a syntax highlighting package written in Python." optional = false python-versions = ">=3.8" -groups = ["dev", "docs"] files = [ {file = "pygments-2.19.1-py3-none-any.whl", hash = "sha256:9ea1544ad55cecf4b8242fab6dd35a93bbce657034b0611ee383099054ab6d8c"}, {file = "pygments-2.19.1.tar.gz", hash = "sha256:61c16d2a8576dc0649d9f39e089b5f02bcd27fba10d8fb4dcc28173f7a45151f"}, @@ -886,7 +854,6 @@ version = "8.3.4" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.8" -groups = ["main"] files = [ {file = "pytest-8.3.4-py3-none-any.whl", hash = "sha256:50e16d954148559c9a74109af1eaf0c945ba2d8f30f0a3d3335edde19788b6f6"}, {file = "pytest-8.3.4.tar.gz", hash = "sha256:965370d062bce11e73868e0335abac31b4d3de0e82f4007408d242b4f8610761"}, @@ -909,7 +876,6 @@ version = "2.9.0.post0" description = "Extensions to the standard Python datetime module" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -groups = ["main"] files = [ {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, @@ -924,7 +890,6 @@ version = "1.0.1" description = "Read key-value pairs from a .env file and set them as environment variables" optional = false python-versions = ">=3.8" -groups = ["main"] files = [ {file = "python-dotenv-1.0.1.tar.gz", hash = "sha256:e324ee90a023d808f1959c46bcbc04446a10ced277783dc6ee09987c37ec10ca"}, {file = "python_dotenv-1.0.1-py3-none-any.whl", hash = "sha256:f7b63ef50f1b690dddf550d03497b66d609393b40b564ed0d674909a68ebf16a"}, @@ -939,7 +904,6 @@ version = "6.0.2" description = "YAML parser and emitter for Python" optional = false python-versions = ">=3.8" -groups = ["main", "dev"] files = [ {file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"}, {file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"}, @@ -1002,7 +966,6 @@ version = "0.36.2" description = "JSON Referencing + Python" optional = false python-versions = ">=3.9" -groups = ["main"] files = [ {file = "referencing-0.36.2-py3-none-any.whl", hash = "sha256:e8699adbbf8b5c7de96d8ffa0eb5c158b3beafce084968e2ea8bb08c6794dcd0"}, {file = "referencing-0.36.2.tar.gz", hash = "sha256:df2e89862cd09deabbdba16944cc3f10feb6b3e6f18e902f7cc25609a34775aa"}, @@ -1019,7 +982,6 @@ version = "2.32.3" description = "Python HTTP for Humans." optional = false python-versions = ">=3.8" -groups = ["main", "docs"] files = [ {file = "requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"}, {file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"}, @@ -1041,7 +1003,6 @@ version = "14.0.0" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" optional = false python-versions = ">=3.8.0" -groups = ["dev"] files = [ {file = "rich-14.0.0-py3-none-any.whl", hash = "sha256:1c9491e1951aac09caffd42f448ee3d04e58923ffe14993f6e83068dc395d7e0"}, {file = "rich-14.0.0.tar.gz", hash = "sha256:82f1bc23a6a21ebca4ae0c45af9bdbc492ed20231dcb63f297d6d1021a9d5725"}, @@ -1061,7 +1022,6 @@ version = "0.22.3" description = "Python bindings to Rust's persistent data structures (rpds)" optional = false python-versions = ">=3.9" -groups = ["main"] files = [ {file = "rpds_py-0.22.3-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:6c7b99ca52c2c1752b544e310101b98a659b720b21db00e65edca34483259967"}, {file = "rpds_py-0.22.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:be2eb3f2495ba669d2a985f9b426c1797b7d48d6963899276d22f23e33d47e37"}, @@ -1174,7 +1134,6 @@ version = "0.11.2" description = "An Amazon S3 Transfer Manager" optional = false python-versions = ">=3.8" -groups = ["main"] files = [ {file = "s3transfer-0.11.2-py3-none-any.whl", hash = "sha256:be6ecb39fadd986ef1701097771f87e4d2f821f27f6071c872143884d2950fbc"}, {file = "s3transfer-0.11.2.tar.gz", hash = "sha256:3b39185cb72f5acc77db1a58b6e25b977f28d20496b6e58d6813d75f464d632f"}, @@ -1192,20 +1151,19 @@ version = "80.7.1" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.9" -groups = ["dev"] files = [ {file = "setuptools-80.7.1-py3-none-any.whl", hash = "sha256:ca5cc1069b85dc23070a6628e6bcecb3292acac802399c7f8edc0100619f9009"}, {file = "setuptools-80.7.1.tar.gz", hash = "sha256:f6ffc5f0142b1bd8d0ca94ee91b30c0ca862ffd50826da1ea85258a06fd94552"}, ] [package.extras] -check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\"", "ruff (>=0.8.0) ; sys_platform != \"cygwin\""] -core = ["importlib_metadata (>=6) ; python_version < \"3.10\"", "jaraco.functools (>=4)", "jaraco.text (>=3.7)", "more_itertools", "more_itertools (>=8.8)", "packaging (>=24.2)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1) ; python_version < \"3.11\"", "wheel (>=0.43.0)"] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)", "ruff (>=0.8.0)"] +core = ["importlib_metadata (>=6)", "jaraco.functools (>=4)", "jaraco.text (>=3.7)", "more_itertools", "more_itertools (>=8.8)", "packaging (>=24.2)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"] cover = ["pytest-cov"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] enabler = ["pytest-enabler (>=2.2)"] -test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21) ; python_version >= \"3.9\" and sys_platform != \"cygwin\"", "jaraco.envs (>=2.2)", "jaraco.path (>=3.7.2)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf ; sys_platform != \"cygwin\"", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] -type = ["importlib_metadata (>=7.0.2) ; python_version < \"3.10\"", "jaraco.develop (>=7.21) ; sys_platform != \"cygwin\"", "mypy (==1.14.*)", "pytest-mypy"] +test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.7.2)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] +type = ["importlib_metadata (>=7.0.2)", "jaraco.develop (>=7.21)", "mypy (==1.14.*)", "pytest-mypy"] [[package]] name = "six" @@ -1213,7 +1171,6 @@ version = "1.17.0" description = "Python 2 and 3 compatibility utilities" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -groups = ["main"] files = [ {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, @@ -1225,7 +1182,6 @@ version = "5.0.2" description = "A pure Python implementation of a sliding window memory map manager" optional = false python-versions = ">=3.7" -groups = ["main"] files = [ {file = "smmap-5.0.2-py3-none-any.whl", hash = "sha256:b30115f0def7d7531d22a0fb6502488d879e75b260a9db4d0819cfb25403af5e"}, {file = "smmap-5.0.2.tar.gz", hash = "sha256:26ea65a03958fa0c8a1c7e8c7a58fdc77221b8910f6be2131affade476898ad5"}, @@ -1237,7 +1193,6 @@ version = "2.2.0" description = "This package provides 29 stemmers for 28 languages generated from Snowball algorithms." optional = false python-versions = "*" -groups = ["docs"] files = [ {file = "snowballstemmer-2.2.0-py2.py3-none-any.whl", hash = "sha256:c8e1716e83cc398ae16824e5572ae04e0d9fc2c6b985fb0f900f5f0c96ecba1a"}, {file = "snowballstemmer-2.2.0.tar.gz", hash = "sha256:09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1"}, @@ -1249,7 +1204,6 @@ version = "7.4.7" description = "Python documentation generator" optional = false python-versions = ">=3.9" -groups = ["docs"] files = [ {file = "sphinx-7.4.7-py3-none-any.whl", hash = "sha256:c2419e2135d11f1951cd994d6eb18a1835bd8fdd8429f9ca375dc1f3281bd239"}, {file = "sphinx-7.4.7.tar.gz", hash = "sha256:242f92a7ea7e6c5b406fdc2615413890ba9f699114a9c09192d7dfead2ee9cfe"}, @@ -1285,7 +1239,6 @@ version = "2.0.0" description = "Read the Docs theme for Sphinx" optional = false python-versions = ">=3.6" -groups = ["docs"] files = [ {file = "sphinx_rtd_theme-2.0.0-py2.py3-none-any.whl", hash = "sha256:ec93d0856dc280cf3aee9a4c9807c60e027c7f7b461b77aeffed682e68f0e586"}, {file = "sphinx_rtd_theme-2.0.0.tar.gz", hash = "sha256:bd5d7b80622406762073a04ef8fadc5f9151261563d47027de09910ce03afe6b"}, @@ -1305,7 +1258,6 @@ version = "2.0.0" description = "sphinxcontrib-applehelp is a Sphinx extension which outputs Apple help books" optional = false python-versions = ">=3.9" -groups = ["docs"] files = [ {file = "sphinxcontrib_applehelp-2.0.0-py3-none-any.whl", hash = "sha256:4cd3f0ec4ac5dd9c17ec65e9ab272c9b867ea77425228e68ecf08d6b28ddbdb5"}, {file = "sphinxcontrib_applehelp-2.0.0.tar.gz", hash = "sha256:2f29ef331735ce958efa4734873f084941970894c6090408b079c61b2e1c06d1"}, @@ -1322,7 +1274,6 @@ version = "2.0.0" description = "sphinxcontrib-devhelp is a sphinx extension which outputs Devhelp documents" optional = false python-versions = ">=3.9" -groups = ["docs"] files = [ {file = "sphinxcontrib_devhelp-2.0.0-py3-none-any.whl", hash = "sha256:aefb8b83854e4b0998877524d1029fd3e6879210422ee3780459e28a1f03a8a2"}, {file = "sphinxcontrib_devhelp-2.0.0.tar.gz", hash = "sha256:411f5d96d445d1d73bb5d52133377b4248ec79db5c793ce7dbe59e074b4dd1ad"}, @@ -1339,7 +1290,6 @@ version = "2.1.0" description = "sphinxcontrib-htmlhelp is a sphinx extension which renders HTML help files" optional = false python-versions = ">=3.9" -groups = ["docs"] files = [ {file = "sphinxcontrib_htmlhelp-2.1.0-py3-none-any.whl", hash = "sha256:166759820b47002d22914d64a075ce08f4c46818e17cfc9470a9786b759b19f8"}, {file = "sphinxcontrib_htmlhelp-2.1.0.tar.gz", hash = "sha256:c9e2916ace8aad64cc13a0d233ee22317f2b9025b9cf3295249fa985cc7082e9"}, @@ -1356,7 +1306,6 @@ version = "4.1" description = "Extension to include jQuery on newer Sphinx releases" optional = false python-versions = ">=2.7" -groups = ["docs"] files = [ {file = "sphinxcontrib-jquery-4.1.tar.gz", hash = "sha256:1620739f04e36a2c779f1a131a2dfd49b2fd07351bf1968ced074365933abc7a"}, {file = "sphinxcontrib_jquery-4.1-py2.py3-none-any.whl", hash = "sha256:f936030d7d0147dd026a4f2b5a57343d233f1fc7b363f68b3d4f1cb0993878ae"}, @@ -1371,7 +1320,6 @@ version = "1.0.1" description = "A sphinx extension which renders display math in HTML via JavaScript" optional = false python-versions = ">=3.5" -groups = ["docs"] files = [ {file = "sphinxcontrib-jsmath-1.0.1.tar.gz", hash = "sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8"}, {file = "sphinxcontrib_jsmath-1.0.1-py2.py3-none-any.whl", hash = "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178"}, @@ -1386,7 +1334,6 @@ version = "2.0.0" description = "sphinxcontrib-qthelp is a sphinx extension which outputs QtHelp documents" optional = false python-versions = ">=3.9" -groups = ["docs"] files = [ {file = "sphinxcontrib_qthelp-2.0.0-py3-none-any.whl", hash = "sha256:b18a828cdba941ccd6ee8445dbe72ffa3ef8cbe7505d8cd1fa0d42d3f2d5f3eb"}, {file = "sphinxcontrib_qthelp-2.0.0.tar.gz", hash = "sha256:4fe7d0ac8fc171045be623aba3e2a8f613f8682731f9153bb2e40ece16b9bbab"}, @@ -1403,7 +1350,6 @@ version = "2.0.0" description = "sphinxcontrib-serializinghtml is a sphinx extension which outputs \"serialized\" HTML files (json and pickle)" optional = false python-versions = ">=3.9" -groups = ["docs"] files = [ {file = "sphinxcontrib_serializinghtml-2.0.0-py3-none-any.whl", hash = "sha256:6e2cb0eef194e10c27ec0023bfeb25badbbb5868244cf5bc5bdc04e4464bf331"}, {file = "sphinxcontrib_serializinghtml-2.0.0.tar.gz", hash = "sha256:e9d912827f872c029017a53f0ef2180b327c3f7fd23c87229f7a8e8b70031d4d"}, @@ -1420,7 +1366,6 @@ version = "5.4.1" description = "Manage dynamic plugins for Python applications" optional = false python-versions = ">=3.9" -groups = ["dev"] files = [ {file = "stevedore-5.4.1-py3-none-any.whl", hash = "sha256:d10a31c7b86cba16c1f6e8d15416955fc797052351a56af15e608ad20811fcfe"}, {file = "stevedore-5.4.1.tar.gz", hash = "sha256:3135b5ae50fe12816ef291baff420acb727fcd356106e3e9cbfa9e5985cd6f4b"}, @@ -1435,8 +1380,6 @@ version = "2.2.1" description = "A lil' TOML parser" optional = false python-versions = ">=3.8" -groups = ["main", "dev", "docs"] -markers = "python_version == \"3.10\"" files = [ {file = "tomli-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249"}, {file = "tomli-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6"}, @@ -1478,12 +1421,10 @@ version = "4.12.2" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" -groups = ["main", "dev"] files = [ {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, ] -markers = {main = "python_version < \"3.13\"", dev = "python_version == \"3.10\""} [[package]] name = "urllib3" @@ -1491,19 +1432,18 @@ version = "2.3.0" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.9" -groups = ["main", "docs"] files = [ {file = "urllib3-2.3.0-py3-none-any.whl", hash = "sha256:1cee9ad369867bfdbbb48b7dd50374c0967a0bb7710050facf0dd6911440e3df"}, {file = "urllib3-2.3.0.tar.gz", hash = "sha256:f8c5449b3cf0861679ce7e0503c7b44b5ec981bec0d1d3795a07f1ba96f0204d"}, ] [package.extras] -brotli = ["brotli (>=1.0.9) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; platform_python_implementation != \"CPython\""] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] h2 = ["h2 (>=4,<5)"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] zstd = ["zstandard (>=0.18.0)"] [metadata] -lock-version = "2.1" +lock-version = "2.0" python-versions = "^3.10" -content-hash = "3912bc8cb9cf35d5cdabd52236d746dff0a0a99db04fde2e568a9fd8568faa0e" +content-hash = "ec3774eef21c857812edb50d1b3392f8cbaf17002add991254a5e13f3fe8dc2e" diff --git a/pyproject.toml b/pyproject.toml index e25d873..278af7f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,7 @@ boto3 = "*" [tool.poetry.group.dev.dependencies] bandit = "^1.8.3" black = "^24.8.0" +opencontainers = "^0.0.14" [tool.poetry.group.docs.dependencies] sphinx-rtd-theme = "^2.0.0" @@ -31,6 +32,7 @@ sphinx-rtd-theme = "^2.0.0" gl-cname = "gardenlinux.features.cname_main:main" gl-features-parse = "gardenlinux.features.__main__:main" gl-flavors-parse = "gardenlinux.flavors.__main__:main" +gl-oci = "gardenlinux.oci.__main__:main" flavors-parse = "gardenlinux.flavors.__main__:main" [tool.pytest.ini_options] diff --git a/src/gardenlinux/constants.py b/src/gardenlinux/constants.py index 49bfbed..50521bb 100644 --- a/src/gardenlinux/constants.py +++ b/src/gardenlinux/constants.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- +ARCHS = ["amd64", "arm64"] + # GardenLinux "bare" feature BARE_FLAVOR_FEATURE_CONTENT = {"description": "Bare flavor", "type": "platform"} @@ -55,8 +57,19 @@ # It is important that this list is sorted in descending length of the entries GL_MEDIA_TYPES = [ + "secureboot.aws-efivars", + "secureboot.kek.auth", "gcpimage.tar.gz.log", + "secureboot.pk.auth", + "secureboot.kek.crt", + "secureboot.kek.der", + "secureboot.db.auth", "firecracker.tar.gz", + "secureboot.pk.crt", + "secureboot.pk.der", + "secureboot.db.crt", + "secureboot.db.der", + "secureboot.db.arn", "platform.test.log", "platform.test.xml", "gcpimage.tar.gz", @@ -65,11 +78,15 @@ "pxe.tar.gz.log", "root.squashfs", "manifest.log", + "squashfs.log", "release.log", + "vmlinuz.log", + "initrd.log", "pxe.tar.gz", "qcow2.log", "test-log", "boot.efi", + "squashfs", "manifest", "vmdk.log", "tar.log", @@ -122,12 +139,30 @@ "vhd.log": "application/io.gardenlinux.log", "ova.log": "application/io.gardenlinux.log", "vmlinuz": "application/io.gardenlinux.kernel", + "vmlinuz.log": "application/io.gardenlinux.log", "initrd": "application/io.gardenlinux.initrd", + "initrd.log": "application/io.gardenlinux.log", "root.squashfs": "application/io.gardenlinux.squashfs", + "squashfs": "application/io.gardenlinux.squashfs", + "squashfs.log": "application/io.gardenlinux.log", "boot.efi": "application/io.gardenlinux.efi", "platform.test.log": "application/io.gardenlinux.io.platform.test.log", "platform.test.xml": "application/io.gardenlinux.io.platform.test.xml", "chroot.test.log": "application/io.gardenlinux.io.chroot.test.log", "chroot.test.xml": "application/io.gardenlinux.io.chroot.test.xml", "oci.log": "application/io.gardenlinux.log", + "secureboot.pk.crt": "application/io.gardenlinux.cert.secureboot.pk.crt", + "secureboot.pk.der": "application/io.gardenlinux.cert.secureboot.pk.der", + "secureboot.pk.auth": "application/io.gardenlinux.cert.secureboot.pk.auth", + "secureboot.kek.crt": "application/io.gardenlinux.cert.secureboot.kek.crt", + "secureboot.kek.der": "application/io.gardenlinux.cert.secureboot.kek.der", + "secureboot.kek.auth": "application/io.gardenlinux.cert.secureboot.kek.auth", + "secureboot.db.crt": "application/io.gardenlinux.cert.secureboot.db.crt", + "secureboot.db.der": "application/io.gardenlinux.cert.secureboot.db.der", + "secureboot.db.auth": "application/io.gardenlinux.cert.secureboot.db.auth", + "secureboot.db.arn": "application/io.gardenlinux.cert.secureboot.db.arn", + "secureboot.aws-efivars": "application/io.gardenlinux.cert.secureboot.aws-efivars", } + +OCI_ANNOTATION_SIGNATURE_KEY = "io.gardenlinux.oci.signature" +OCI_ANNOTATION_SIGNED_STRING_KEY = "io.gardenlinux.oci.signed-string" diff --git a/src/gardenlinux/features/parser.py b/src/gardenlinux/features/parser.py index a016e15..a49d5a5 100644 --- a/src/gardenlinux/features/parser.py +++ b/src/gardenlinux/features/parser.py @@ -9,7 +9,11 @@ import subprocess import yaml -from ..constants import BARE_FLAVOR_FEATURE_CONTENT, BARE_FLAVOR_LIBC_FEATURE_CONTENT +from ..constants import ( + ARCHS, + BARE_FLAVOR_FEATURE_CONTENT, + BARE_FLAVOR_LIBC_FEATURE_CONTENT, +) from ..logger import LoggerSetup @@ -217,6 +221,57 @@ def get_cname_as_feature_set(cname): cname = cname.replace("_", "-_") return set(cname.split("-")) + @staticmethod + def get_flavor_from_cname(cname: str, get_arch: bool = True) -> str: + """ + Extracts the flavor from a canonical name. + + This method parses a Garden Linux canonical name (cname) and extracts + the flavor component, with or without the architecture suffix. + + Example canonical names: + - "aws-gardener_prod-amd64" + - "azure-gardener_prod_tpm2_trustedboot-amd64-1312.2-80ffcc87" + + The flavor is the platform plus feature string (e.g., "aws-gardener_prod") + + Args: + cname (str): Canonical name of an image + get_arch (bool): Whether to include the architecture in the returned flavor + If True: returns "aws-gardener_prod-amd64" + If False: returns "aws-gardener_prod" + + Returns: + str: The extracted flavor string, with or without architecture + """ + # Use regex to extract components from the canonical name + # This handles complex cnames with version and commit hash + re_match = re.match( + "([a-zA-Z0-9]+([\\_\\-][a-zA-Z0-9]+)*?)(-([a-z0-9]+)(-([a-z0-9.]+)-([a-z0-9]+))*)?$", + cname, + ) + + assert re_match, f"Not a valid GardenLinux canonical name {cname}" + + if re_match.lastindex == 1: + data_splitted = re_match[1].split("-", 1) + + flavor = data_splitted[0] + + if len(data_splitted) > 1: + if get_arch is True: + arch = data_splitted[1] + else: + flavor += "-" + data_splitted[1] + else: + arch = re_match[4] + flavor = re_match[1] + # Add architecture if requested + if get_arch and arch: + return f"{flavor}-{arch}" + else: + return flavor + @staticmethod def _get_filter_set_callable(filter_set, additional_filter_func): def filter_func(node): diff --git a/src/gardenlinux/oci/__init__.py b/src/gardenlinux/oci/__init__.py new file mode 100644 index 0000000..40a96af --- /dev/null +++ b/src/gardenlinux/oci/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/src/gardenlinux/oci/__main__.py b/src/gardenlinux/oci/__main__.py new file mode 100755 index 0000000..7968d8c --- /dev/null +++ b/src/gardenlinux/oci/__main__.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 + +import os +import click + +from pygments.lexer import default + +from .registry import GlociRegistry + + +@click.group() +def cli(): + pass + + +@cli.command() +@click.option( + "--container", + required=True, + type=click.Path(), + help="Container Name", +) +@click.option( + "--version", + required=True, + type=click.Path(), + help="Version of image", +) +@click.option( + "--commit", + required=False, + type=click.Path(), + default=None, + help="Commit of image", +) +@click.option( + "--arch", + required=True, + type=click.Path(), + help="Target Image CPU Architecture", +) +@click.option( + "--cname", required=True, type=click.Path(), help="Canonical Name of Image" +) +@click.option("--dir", "directory", required=True, help="path to the build artifacts") +@click.option( + "--cosign_file", + required=False, + help="A file where the pushed manifests digests is written to. The content can be used by an external tool (e.g. cosign) to sign the manifests contents", +) +@click.option( + "--manifest_file", + default="manifests/manifest.json", + help="A file where the index entry for the pushed manifest is written to.", +) +@click.option( + "--insecure", + default=False, + help="Use HTTP to communicate with the registry", +) +def push_manifest( + container, + version, + commit, + arch, + cname, + directory, + cosign_file, + manifest_file, + insecure, +): + """push artifacts from a dir to a registry, get the index-entry for the manifest in return""" + container_name = f"{container}:{version}" + registry = GlociRegistry( + container_name=container_name, + token=os.getenv("GL_CLI_REGISTRY_TOKEN"), + insecure=insecure, + ) + digest = registry.push_from_dir( + arch, version, cname, directory, manifest_file, commit=commit + ) + if cosign_file: + print(digest, file=open(cosign_file, "w")) + + +@cli.command() +@click.option( + "--container", + "container", + required=True, + type=click.Path(), + help="Container Name", +) +@click.option( + "--version", + "version", + required=True, + type=click.Path(), + help="Version of image", +) +@click.option( + "--manifest_folder", + default="manifests", + help="A folder where the index entries are read from.", +) +@click.option( + "--insecure", + default=False, + help="Use HTTP to communicate with the registry", +) +def update_index(container, version, manifest_folder, insecure): + """push a index entry from a list of files to an index""" + container_name = f"{container}:{version}" + registry = GlociRegistry( + container_name=container_name, + token=os.getenv("GL_CLI_REGISTRY_TOKEN"), + insecure=insecure, + ) + registry.update_index(manifest_folder) + + +def main(): + """Entry point for the gl-oci command.""" + cli() + + +if __name__ == "__main__": + cli() diff --git a/src/gardenlinux/oci/checksum.py b/src/gardenlinux/oci/checksum.py new file mode 100644 index 0000000..abfa909 --- /dev/null +++ b/src/gardenlinux/oci/checksum.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- + +import hashlib + + +def verify_sha256(checksum: str, data: bytes): + data_checksum = f"sha256:{hashlib.sha256(data).hexdigest()}" + if checksum != data_checksum: + raise ValueError(f"Invalid checksum. {checksum} != {data_checksum}") + + +def calculate_sha256(file_path: str) -> str: + """Calculate the SHA256 checksum of a file.""" + sha256_hash = hashlib.sha256() + with open(file_path, "rb") as f: + for byte_block in iter(lambda: f.read(4096), b""): + sha256_hash.update(byte_block) + return sha256_hash.hexdigest() diff --git a/src/gardenlinux/oci/registry.py b/src/gardenlinux/oci/registry.py new file mode 100644 index 0000000..c6c3430 --- /dev/null +++ b/src/gardenlinux/oci/registry.py @@ -0,0 +1,703 @@ +# -*- coding: utf-8 -*- + +import base64 +import copy +import hashlib +import json +import logging +import os +import shutil +import sys +import tarfile +import tempfile +import uuid +from enum import Enum, auto +from platform import architecture +from typing import Optional, Tuple + +import jsonschema +import oras.auth +import oras.client +import oras.defaults +import oras.oci +import oras.provider +import oras.utils +import requests +from oras.container import Container as OrasContainer +from oras.decorator import ensure_container +from oras.provider import Registry +from oras.schemas import manifest as oras_manifest_schema + +from gardenlinux.features import Parser +from ..constants import OCI_ANNOTATION_SIGNATURE_KEY, OCI_ANNOTATION_SIGNED_STRING_KEY +from .checksum import ( + calculate_sha256, + verify_sha256, +) +from python_gardenlinux_lib.features.parse_features import get_oci_metadata_from_fileset +from .schemas import ( + EmptyIndex, + EmptyManifestMetadata, + EmptyPlatform, +) +from .schemas import index as indexSchema + + +class ManifestState(Enum): + Incomplete = auto() + Complete = auto() + Final = auto() + + +logger = logging.getLogger(__name__) +logging.basicConfig(stream=sys.stdout, level=logging.INFO) + + +def get_image_state(manifest: dict) -> str: + if "annotations" not in manifest: + logger.warning("No annotations set for manifest.") + return "UNDEFINED" + if "image_state" not in manifest["annotations"]: + logger.warning("No image_state set for manifest.") + return "UNDEFINED" + return manifest["annotations"]["image_state"] + + +def NewPlatform(architecture: str, version: str) -> dict: + platform = copy.deepcopy(EmptyPlatform) + platform["architecture"] = architecture + platform["os.version"] = version + return platform + + +def NewManifestMetadata( + digest: str, size: int, annotations: dict, platform_data: dict +) -> dict: + manifest_meta_data = copy.deepcopy(EmptyManifestMetadata) + manifest_meta_data["mediaType"] = "application/vnd.oci.image.manifest.v1+json" + manifest_meta_data["digest"] = digest + manifest_meta_data["size"] = size + manifest_meta_data["annotations"] = annotations + manifest_meta_data["platform"] = platform_data + return manifest_meta_data + + +def NewIndex() -> dict: + index = copy.deepcopy(EmptyIndex) + index["mediaType"] = "application/vnd.oci.image.index.v1+json" + return index + + +def create_config_from_dict(conf: dict, annotations: dict) -> Tuple[dict, str]: + """ + Write a new OCI configuration to file, and generate oci metadata for it + For reference see https://github.com/opencontainers/image-spec/blob/main/config.md + annotations, mediatype, size, digest are not part of digest and size calculation, + and therefore must be attached to the output dict and not written to the file. + + :param conf: dict with custom configuration (the payload of the configuration) + :param annotations: dict with custom annotations to be attached to metadata part of config + + """ + config_path = os.path.join(os.path.curdir, str(uuid.uuid4())) + with open(config_path, "w") as fp: + json.dump(conf, fp) + conf["annotations"] = annotations + conf["mediaType"] = oras.defaults.unknown_config_media_type + conf["size"] = oras.utils.get_size(config_path) + conf["digest"] = f"sha256:{oras.utils.get_file_hash(config_path)}" + return conf, config_path + + +def construct_manifest_entry_signed_data_string( + cname: str, version: str, new_manifest_metadata: dict, architecture: str +) -> str: + data_to_sign = ( + f"version:{version} cname:{cname} architecture:{architecture} manifest-size" + f":{new_manifest_metadata['size']} manifest-digest:{new_manifest_metadata['digest']}" + ) + return data_to_sign + + +def construct_layer_signed_data_string( + cname: str, version: str, architecture: str, media_type: str, checksum_sha256: str +) -> str: + data_to_sign = f"version:{version} cname:{cname} architecture:{architecture} media_type:{media_type} digest:{checksum_sha256}" + return data_to_sign + + +class GlociRegistry(Registry): + def __init__( + self, + container_name: str, + insecure: bool = False, + token: Optional[str] = None, + config_path: Optional[str] = None, + ): + super().__init__(auth_backend="token", insecure=insecure) + self.container = OrasContainer(container_name) + self.container_name = container_name + self.registry_url = self.container.registry + self.config_path = config_path + if not token: + logger.info("No Token provided.") + else: + self.token = base64.b64encode(token.encode("utf-8")).decode("utf-8") + self.auth.set_token_auth(self.token) + + @ensure_container + def get_manifest_json( + self, container: OrasContainer, allowed_media_type: Optional[list[str]] = None + ): + if not allowed_media_type: + default_image_index_media_type = ( + "application/vnd.oci.image.manifest.v1+json" + ) + allowed_media_type = [default_image_index_media_type] + # self.load_configs(container) + headers = {"Accept": ";".join(allowed_media_type)} + headers.update(self.headers) + get_manifest = f"{self.prefix}://{container.manifest_url()}" + response = self.do_request(get_manifest, "GET", headers=headers) + self._check_200_response(response) + return response + + @ensure_container + def get_manifest_size( + self, container: OrasContainer, allowed_media_type: Optional[list[str]] = None + ): + response = self.get_manifest_json(container, allowed_media_type) + if response is None: + return 0 + return len(response.content) + + @ensure_container + def get_digest( + self, container: OrasContainer, allowed_media_type: Optional[list[str]] = None + ): + response = self.get_manifest_json(container, allowed_media_type) + if response is None: + return "" + return f"sha256:{hashlib.sha256(response.content).hexdigest()}" + + def get_index(self, allowed_media_type: Optional[list[str]] = None): + """ + Returns the manifest for a cname+arch combination of a container + Will return None if no result was found + + TODO: refactor: use get_manifest_json and call it with index mediatype. + """ + if not allowed_media_type: + default_image_index_media_type = "application/vnd.oci.image.index.v1+json" + allowed_media_type = [default_image_index_media_type] + + headers = {"Accept": ";".join(allowed_media_type)} + manifest_url = f"{self.prefix}://{self.container.manifest_url()}" + response = self.do_request(manifest_url, "GET", headers=headers) + try: + self._check_200_response(response) + index = response.json() + return index + + except ValueError: + logger.info("Index not found, creating new Index!") + return NewIndex() + + @ensure_container + def get_manifest_meta_data_by_cname( + self, + container: OrasContainer, + cname: str, + version: str, + arch: str, + allowed_media_type: Optional[list[str]] = None, + ): + """ + Returns the manifest for a cname+arch combination of a container + Will return None if no result was found + """ + index = self.get_index(allowed_media_type=allowed_media_type) + + if "manifests" not in index: + logger.debug("Index is empty") + return None + + for manifest_meta in index["manifests"]: + # Annotations are optional: + # https://github.com/opencontainers/image-spec/blob/v1.0.1/descriptor.md#properties + + if "annotations" in manifest_meta: + if ( + "cname" in manifest_meta["annotations"] + and "architecture" in manifest_meta["annotations"] + and "os.version" in manifest_meta["platform"] + and manifest_meta["annotations"]["cname"] == cname + and manifest_meta["annotations"]["architecture"] == arch + and manifest_meta["platform"]["os.version"] == version + ): + return manifest_meta + + return None + + @ensure_container + def get_manifest_by_digest( + self, + container: OrasContainer, + digest: str, + allowed_media_type: Optional[list[str]] = None, + ): + if not allowed_media_type: + default_image_manifest_media_type = ( + "application/vnd.oci.image.manifest.v1+json" + ) + allowed_media_type = [default_image_manifest_media_type] + + manifest_url = f"{self.prefix}://{container.get_blob_url(digest)}".replace( + "/blobs/", "/manifests/" + ) + headers = {"Accept": ";".join(allowed_media_type)} + response = self.do_request(manifest_url, "GET", headers=headers, stream=False) + self._check_200_response(response) + manifest = response.json() + verify_sha256(digest, response.content) + jsonschema.validate(manifest, schema=oras_manifest_schema) + return manifest + + @ensure_container + def get_manifest_by_cname( + self, + container: OrasContainer, + cname: str, + version: str, + arch: str, + allowed_media_type: Optional[list[str]] = None, + ): + """ + Returns the manifest for a cname+arch combination of a container + Will return None if no result was found + """ + if not allowed_media_type: + default_image_manifest_media_type = ( + "application/vnd.oci.image.manifest.v1+json" + ) + allowed_media_type = [default_image_manifest_media_type] + manifest_meta = self.get_manifest_meta_data_by_cname( + container, cname, version, arch + ) + if manifest_meta is None: + logger.error(f"No manifest found for {cname}-{arch}") + return None + if "digest" not in manifest_meta: + logger.error("No digest found in metadata!") + manifest_digest = manifest_meta["digest"] + return self.get_manifest_by_digest( + container, manifest_digest, allowed_media_type=allowed_media_type + ) + + def change_state(self, cname: str, version: str, architecture: str, new_state: str): + manifest_container = OrasContainer( + f"{self.container_name}-{cname}-{architecture}" + ) + manifest = self.get_manifest_by_cname( + manifest_container, cname, version, architecture + ) + + if "annotations" not in manifest: + logger.warning("No annotations found in manifest, init annotations now.") + manifest["annotations"] = {} + + def attach_layer( + self, + cname: str, + version: str, + architecture: str, + file_path: str, + media_type: str, + ): + if not os.path.exists(file_path): + exit(f"{file_path} does not exist.") + + manifest_container = OrasContainer( + f"{self.container_name}-{cname}-{architecture}" + ) + + manifest = self.get_manifest_by_cname( + self.container, cname, version, architecture + ) + + layer = self.create_layer(file_path, cname, version, architecture, media_type) + self._check_200_response(self.upload_blob(file_path, self.container, layer)) + + manifest["layers"].append(layer) + + old_manifest_digest = self.get_digest(manifest_container) + self._check_200_response(self.upload_manifest(manifest, manifest_container)) + + new_manifest_metadata = self.get_manifest_meta_data_by_cname( + self.container, cname, version, architecture + ) + new_manifest_metadata["digest"] = self.get_digest(manifest_container) + new_manifest_metadata["size"] = self.get_manifest_size(manifest_container) + new_manifest_metadata["platform"] = NewPlatform(architecture, version) + + new_index = self.update_index(old_manifest_digest, new_manifest_metadata) + self._check_200_response(self.upload_index(new_index)) + + print(f"Successfully attached {file_path} to {manifest_container}") + + def sign_manifest_entry( + self, new_manifest_metadata: dict, version: str, architecture: str, cname: str + ): + data_to_sign = construct_manifest_entry_signed_data_string( + cname, version, new_manifest_metadata, architecture + ) + signature = self.signer.sign_data(data_to_sign) + new_manifest_metadata["annotations"].update( + { + OCI_ANNOTATION_SIGNATURE_KEY: signature, + OCI_ANNOTATION_SIGNED_STRING_KEY: data_to_sign, + } + ) + + def sign_layer( + self, + layer: dict, + cname: str, + version: str, + architecture: str, + checksum_sha256: str, + media_type: str, + ): + data_to_sign = construct_layer_signed_data_string( + cname, version, architecture, media_type, checksum_sha256 + ) + signature = self.signer.sign_data(data_to_sign) + layer["annotations"].update( + { + OCI_ANNOTATION_SIGNATURE_KEY: signature, + OCI_ANNOTATION_SIGNED_STRING_KEY: data_to_sign, + } + ) + + def verify_manifest_meta_signature(self, manifest_meta: dict): + if "annotations" not in manifest_meta: + raise ValueError("manifest does not contain annotations") + if OCI_ANNOTATION_SIGNATURE_KEY not in manifest_meta["annotations"]: + raise ValueError("manifest is not signed") + if OCI_ANNOTATION_SIGNED_STRING_KEY not in manifest_meta["annotations"]: + raise ValueError("manifest is not signed") + signature = manifest_meta["annotations"][OCI_ANNOTATION_SIGNATURE_KEY] + signed_data = manifest_meta["annotations"][OCI_ANNOTATION_SIGNED_STRING_KEY] + cname = manifest_meta["annotations"]["cname"] + version = manifest_meta["platform"]["os.version"] + architecture = manifest_meta["annotations"]["architecture"] + signed_data_expected = construct_manifest_entry_signed_data_string( + cname, version, manifest_meta, architecture + ) + if signed_data_expected != signed_data: + raise ValueError( + f"Signed data does not match expected signed data.\n{signed_data} != {signed_data_expected}" + ) + self.signer.verify_signature(signed_data, signature) + + def verify_manifest_signature(self, manifest: dict): + if "layers" not in manifest: + raise ValueError("manifest does not contain layers") + if "annotations" not in manifest: + raise ValueError("manifest does not contain annotations") + + cname = manifest["annotations"]["cname"] + version = manifest["annotations"]["version"] + architecture = manifest["annotations"]["architecture"] + for layer in manifest["layers"]: + if "annotations" not in layer: + raise ValueError(f"layer does not contain annotations. layer: {layer}") + if OCI_ANNOTATION_SIGNATURE_KEY not in layer["annotations"]: + raise ValueError(f"layer is not signed. layer: {layer}") + if OCI_ANNOTATION_SIGNED_STRING_KEY not in layer["annotations"]: + raise ValueError(f"layer is not signed. layer: {layer}") + media_type = layer["mediaType"] + checksum_sha256 = layer["digest"].removeprefix("sha256:") + signature = layer["annotations"][OCI_ANNOTATION_SIGNATURE_KEY] + signed_data = layer["annotations"][OCI_ANNOTATION_SIGNED_STRING_KEY] + signed_data_expected = construct_layer_signed_data_string( + cname, version, architecture, media_type, checksum_sha256 + ) + if signed_data_expected != signed_data: + raise ValueError( + f"Signed data does not match expected signed data. {signed_data} != {signed_data_expected}" + ) + self.signer.verify_signature(signed_data, signature) + + @ensure_container + def remove_container(self, container: OrasContainer): + self.delete_tag(container.manifest_url()) + + def status_all(self): + """ + Validate if container is valid + - all manifests require a info.yaml in the layers + - info.yaml needs to be signed (TODO) + - all layers listed in info.yaml must exist + - all mediatypes of layers listed in info.yaml must be set correctly + """ + index = self.get_index() + + if "manifests" not in index: + logger.info("No manifests in index") + return + for manifest_meta in index["manifests"]: + manifest_digest = manifest_meta["digest"] + manifest = self.get_manifest_by_digest(self.container, manifest_digest) + image_state = get_image_state(manifest) + print(f"{manifest_digest}:\t{image_state}") + + def upload_index(self, index: dict) -> requests.Response: + jsonschema.validate(index, schema=indexSchema) + headers = { + "Content-Type": "application/vnd.oci.image.index.v1+json", + "Content-Length": str(len(index)), + } + tag = self.container.digest or self.container.tag + + index_url = ( + f"{self.container.registry}/v2/{self.container.api_prefix}/manifests/{tag}" + ) + response = self.do_request( + f"{self.prefix}://{index_url}", # noqa + "PUT", + headers=headers, + json=index, + ) + return response + + def push_image_manifest( + self, + architecture: str, + cname: str, + version: str, + build_artifacts_dir: str, + oci_metadata: list, + feature_set: str, + manifest_file: str, + commit: Optional[str] = None, + ): + """ + creates and pushes an image manifest + + :param oci_metadata: a list of filenames and their OCI metadata, can be constructed with + parse_features.get_oci_metadata or parse_features.get_oci_metadata_from_fileset + :param str architecture: target architecture of the image + :param str cname: canonical name of the target image + :param str build_artifacts_dir: directory where the build artifacts are located + :param str feature_set: the expanded list of the included features of this manifest. It will be set in the + manifest itself and in the index entry for this manifest + :param str commit: the commit hash of the image + :param str manifest_file: the file where the manifest is written to + :returns the digest of the pushed manifest + """ + + # Handle null commit value + if commit is None: + commit = "" + + # TODO: construct oci_artifacts default data + + manifest_image = oras.oci.NewManifest() + total_size = 0 + + # For each file, create sign, attach and push a layer + for artifact in oci_metadata: + annotations_input = artifact["annotations"] + media_type = artifact["media_type"] + file_path = os.path.join(build_artifacts_dir, artifact["file_name"]) + + if not os.path.exists(file_path): + raise ValueError(f"{file_path} does not exist.") + + cleanup_blob = False + if os.path.isdir(file_path): + file_path = oras.utils.make_targz(file_path) + cleanup_blob = True + + # Create and sign layer information + layer = self.create_layer( + file_path, cname, version, architecture, media_type + ) + total_size += int(layer["size"]) + + if annotations_input: + layer["annotations"].update(annotations_input) + # Attach this layer to the manifest that is currently created (and pushed later) + manifest_image["layers"].append(layer) + logger.debug(f"Layer: {layer}") + # Push + response = self.upload_blob(file_path, self.container, layer) + self._check_200_response(response) + logger.info(f"Pushed {artifact["file_name"]} {layer["digest"]}") + if cleanup_blob and os.path.exists(file_path): + os.remove(file_path) + # This ends up in the manifest + flavor = Parser.get_flavor_from_cname(cname, get_arch=True) + manifest_image["annotations"] = {} + manifest_image["annotations"]["version"] = version + manifest_image["annotations"]["cname"] = cname + manifest_image["annotations"]["architecture"] = architecture + manifest_image["annotations"]["feature_set"] = feature_set + manifest_image["annotations"]["flavor"] = flavor + manifest_image["annotations"]["commit"] = commit + description = ( + f"Image: {cname} " + f"Flavor: {flavor} " + f"Architecture: {architecture} " + f"Features: {feature_set} " + f"Commit: {commit} " + ) + manifest_image["annotations"][ + "org.opencontainers.image.description" + ] = description + + config_annotations = {"cname": cname, "architecture": architecture} + conf, config_file = create_config_from_dict(dict(), config_annotations) + + response = self.upload_blob(config_file, self.container, conf) + + os.remove(config_file) + self._check_200_response(response) + + manifest_image["config"] = conf + + manifest_container = OrasContainer( + f"{self.container_name}-{cname}-{architecture}" + ) + + local_digest = f"sha256:{hashlib.sha256(json.dumps(manifest_image).encode('utf-8')).hexdigest()}" + + self._check_200_response( + self.upload_manifest(manifest_image, manifest_container) + ) + logger.info(f"Successfully pushed {self.container} {local_digest}") + + # This ends up in the index-entry for the manifest + metadata_annotations = {"cname": cname, "architecture": architecture} + metadata_annotations["feature_set"] = feature_set + manifest_digest = self.get_digest(manifest_container) + if manifest_digest != local_digest: + raise ValueError("local and remotely calculated digests do not match") + manifest_index_metadata = NewManifestMetadata( + manifest_digest, + self.get_manifest_size(manifest_container), + metadata_annotations, + NewPlatform(architecture, version), + ) + + print(json.dumps(manifest_index_metadata), file=open(manifest_file, "w")) + logger.info(f"Index entry written to {manifest_file}") + + return local_digest + + def update_index(self, manifest_folder): + """ + replaces an old manifest entry with a new manifest entry + """ + index = self.get_index() + # Ensure mediaType is set for existing indices + if "mediaType" not in index: + index["mediaType"] = "application/vnd.oci.image.index.v1+json" + + new_entries = 0 + + for file in os.listdir(manifest_folder): + manifest_metadata = json.loads( + open(manifest_folder + "/" + file, "r").read() + ) + # Skip if manifest with same digest already exists + found = False + for entry in index["manifests"]: + if entry["digest"] == manifest_metadata["digest"]: + found = True + break + if found: + logger.info( + f"Skipping manifest with digest {manifest_metadata["digest"]} - already exists" + ) + continue + index["manifests"].append(manifest_metadata) + logger.info( + f"Index appended locally {manifest_metadata["annotations"]["cname"]}" + ) + new_entries += 1 + + self._check_200_response(self.upload_index(index)) + logger.info(f"Index pushed with {new_entries} new entries") + + def create_layer( + self, + file_path: str, + cname: str, + version: str, + architecture: str, + media_type: str, + ): + checksum_sha256 = calculate_sha256(file_path) + layer = oras.oci.NewLayer(file_path, media_type, is_dir=False) + layer["annotations"] = { + oras.defaults.annotation_title: os.path.basename(file_path), + } + return layer + + def push_from_dir( + self, + architecture: str, + version: str, + cname: str, + directory: str, + manifest_file: str, + commit: Optional[str] = None, + ): + # Step 1 scan and extract nested artifacts: + for file in os.listdir(directory): + try: + if file.endswith(".pxe.tar.gz"): + logger.info(f"Found nested artifact {file}") + nested_tar_obj = tarfile.open(f"{directory}/{file}") + nested_tar_obj.extractall(filter="data", path=directory) + nested_tar_obj.close() + except (OSError, tarfile.FilterError, tarfile.TarError) as e: + print(f"Failed to extract nested artifact {file}", e) + exit(1) + + try: + oci_metadata = get_oci_metadata_from_fileset( + os.listdir(directory), architecture + ) + + features = "" + for artifact in oci_metadata: + if artifact["media_type"] == "application/io.gardenlinux.release": + file = open(f"{directory}/{artifact["file_name"]}", "r") + lines = file.readlines() + for line in lines: + if line.strip().startswith("GARDENLINUX_FEATURES="): + features = line.strip().removeprefix( + "GARDENLINUX_FEATURES=" + ) + break + file.close() + + flavor = Parser.get_flavor_from_cname(cname, get_arch=True) + + digest = self.push_image_manifest( + architecture, + cname, + version, + directory, + oci_metadata, + features, + manifest_file, + commit=commit, + ) + except Exception as e: + print("Error: ", e) + exit(1) + return digest diff --git a/src/gardenlinux/oci/schemas.py b/src/gardenlinux/oci/schemas.py new file mode 100644 index 0000000..c5801cf --- /dev/null +++ b/src/gardenlinux/oci/schemas.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- + +# for reference: +# https://json-schema.org/understanding-json-schema/reference/object + +# TODO: add more schemas, and validate dicts with schemas before accessing them. + +schema_url = "http://json-schema.org/draft-07/schema" + +platformProperties = { + "architecture": {"type": "string"}, + "os": {"type": "string"}, + "os.version": {"type": "string"}, + "variant": {"type": "string"}, +} + +manifestMetaProperties = { + "mediaType": {"type": "string"}, + "platform": {"type": "object", "properties": platformProperties}, +} + +indexProperties = { + "schemaVersion": {"type": "number"}, + "mediaType": {"type": "string"}, + "subject": {"type": ["null", "object"]}, + "manifests": {"type": "array", "items": manifestMetaProperties}, + "annotations": {"type": ["object", "null", "array"]}, +} + + +index = { + "$schema": schema_url, + "title": "Index Schema", + "type": "object", + "required": [ + "schemaVersion", + "manifests", + "mediaType", + ], + "properties": indexProperties, + "additionalProperties": True, +} + +EmptyPlatform = { + "architecture": "", + "os": "gardenlinux", + "os.version": "experimental", +} + +EmptyManifestMetadata = { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "", + "size": 0, + "annotations": {}, +} + +EmptyIndex = { + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.index.v1+json", + "manifests": [], +} diff --git a/tests/conftest.py b/tests/conftest.py index 1424d4c..3eb053f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,11 +13,8 @@ from cryptography.hazmat.primitives.asymmetric import rsa from dotenv import load_dotenv -from .helper import call_command, spawn_background_process - -TEST_DATA_DIR = "test-data" -GL_ROOT_DIR = f"{TEST_DATA_DIR}/gardenlinux" -CERT_DIR = f"{TEST_DATA_DIR}/cert" +from .constants import TEST_DATA_DIR, GL_ROOT_DIR, CERT_DIR +from .helper import call_command, spawn_background_process, generate_test_certificates def generate_test_certificates(): @@ -112,6 +109,7 @@ def zot_session(): def pytest_sessionstart(session): generate_test_certificates() call_command("./test-data/build-test-data.sh --dummy") + call_command("mkdir -p manifests") def pytest_sessionfinish(session): @@ -119,3 +117,5 @@ def pytest_sessionfinish(session): os.remove(CERT_DIR + "/oci-sign.crt") if os.path.isfile(CERT_DIR + "/oci-sign.key"): os.remove(CERT_DIR + "/oci-sign.key") + if os.path.isdir("./manifests"): + shutil.rmtree("./manifests") diff --git a/tests/constants.py b/tests/constants.py new file mode 100644 index 0000000..55ae23c --- /dev/null +++ b/tests/constants.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- + +TEST_DATA_DIR = "test-data" +GL_ROOT_DIR = f"{TEST_DATA_DIR}/gardenlinux" +CERT_DIR = f"{TEST_DATA_DIR}/cert" + +ZOT_CONFIG_FILE = f"{TEST_DATA_DIR}/zot/config.json" +REGISTRY = "127.0.0.1:18081" +REGISTRY_URL = f"http://{REGISTRY}" +REPO_NAME = "gardenlinux-example" +CONTAINER_NAME_ZOT_EXAMPLE = f"{REGISTRY}/{REPO_NAME}" +GARDENLINUX_ROOT_DIR_EXAMPLE = f"{TEST_DATA_DIR}/gardenlinux/.build" diff --git a/tests/helper.py b/tests/helper.py index a1e65ed..d6976fb 100644 --- a/tests/helper.py +++ b/tests/helper.py @@ -3,8 +3,7 @@ import os -ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) -ZOT_CONFIG_FILE = f"/zot/config.json" +from .constants import CERT_DIR, GL_ROOT_DIR, ZOT_CONFIG_FILE def spawn_background_process(cmd, stdout=None, stderr=None): @@ -25,3 +24,34 @@ def call_command(cmd): except subprocess.CalledProcessError as e: error_message = e.stderr.decode("utf-8") return f"An error occurred: {error_message}" + + +def generate_test_certificates(): + """Generate self-signed certificates for testing""" + os.makedirs(CERT_DIR, exist_ok=True) + key_path = os.path.join(CERT_DIR, "oci-sign.key") + cert_path = os.path.join(CERT_DIR, "oci-sign.crt") + cmd = [ + "openssl", + "req", + "-x509", + "-newkey", + "rsa:4096", + "-keyout", + key_path, + "-out", + cert_path, + "-days", + "365", + "-nodes", + "-subj", + "/CN=Garden Linux test signing key for oci", + ] + try: + subprocess.run(cmd, check=True) + # Set proper permissions + os.chmod(key_path, 0o600) + print(f"Generated test certificates in {CERT_DIR}") + except subprocess.CalledProcessError as e: + print(f"Error generating certificates: {e}") + raise diff --git a/tests/test_deduce_image_type.py b/tests/test_deduce_image_type.py index d05f9be..393440a 100644 --- a/tests/test_deduce_image_type.py +++ b/tests/test_deduce_image_type.py @@ -4,7 +4,7 @@ ) import pytest -from tests.conftest import GL_ROOT_DIR +from .constants import GL_ROOT_DIR @pytest.mark.parametrize( diff --git a/tests/test_get_features_dict.py b/tests/test_get_features_dict.py index 629752b..0649f8e 100644 --- a/tests/test_get_features_dict.py +++ b/tests/test_get_features_dict.py @@ -1,7 +1,7 @@ import pytest from gardenlinux.features import Parser -from tests.conftest import GL_ROOT_DIR +from .constants import GL_ROOT_DIR @pytest.mark.parametrize( diff --git a/tests/test_get_oci_metadata.py b/tests/test_get_oci_metadata.py index 10b544f..aba9142 100644 --- a/tests/test_get_oci_metadata.py +++ b/tests/test_get_oci_metadata.py @@ -1,7 +1,7 @@ import pytest from python_gardenlinux_lib.features.parse_features import get_oci_metadata -from tests.conftest import GL_ROOT_DIR +from .constants import GL_ROOT_DIR @pytest.mark.parametrize( diff --git a/tests/test_oci.py b/tests/test_oci.py new file mode 100644 index 0000000..7029724 --- /dev/null +++ b/tests/test_oci.py @@ -0,0 +1,93 @@ +import pytest +from click.testing import CliRunner +import sys +import logging + +sys.path.append("src") + +from gardenlinux.oci.__main__ import cli as gl_oci +from .constants import CONTAINER_NAME_ZOT_EXAMPLE, GARDENLINUX_ROOT_DIR_EXAMPLE + + +@pytest.mark.usefixtures("zot_session") +@pytest.mark.parametrize( + "version, cname, arch", + [ + ("today", "aws-gardener_prod", "arm64"), + ("today", "aws-gardener_prod", "amd64"), + # ("today", "gcp-gardener_prod", "arm64"), + # ("today", "gcp-gardener_prod", "amd64"), + # ("today", "azure-gardener_prod", "arm64"), + # ("today", "azure-gardener_prod", "amd64"), + # ("today", "openstack-gardener_prod", "arm64"), + # ("today", "openstack-gardener_prod", "amd64"), + # ("today", "openstackbaremetal-gardener_prod", "arm64"), + # ("today", "openstackbaremetal-gardener_prod", "amd64"), + # ("today", "metal-kvm_dev", "arm64"), + # ("today", "metal-kvm_dev", "amd64"), + ], +) +def test_push_manifest_and_index(version, arch, cname): + logger = logging.getLogger(__name__) + runner = CliRunner() + result = runner.invoke( + gl_oci, + [ + "push-manifest", + "--container", + CONTAINER_NAME_ZOT_EXAMPLE, + "--version", + version, + "--arch", + arch, + "--cname", + cname, + "--dir", + GARDENLINUX_ROOT_DIR_EXAMPLE, + "--insecure", + "True", + "--cosign_file", + "digest", + "--manifest_file", + "manifests/manifest.json", + ], + catch_exceptions=False, + ) + print(f"Output: {result.output}") + if result.exit_code != 0: + print(f"Exit Code: {result.exit_code}") + if result.exception: + print(result.exception) + try: + print(f"Output: {result.stderr}") + except ValueError: + print("No stderr captured.") + assert result.exit_code == 0 + + logger.info("Pushed manifests") + + result = runner.invoke( + gl_oci, + [ + "update-index", + "--container", + CONTAINER_NAME_ZOT_EXAMPLE, + "--version", + version, + "--insecure", + "True", + "--manifest_folder", + "manifests", + ], + catch_exceptions=False, + ) + print(f"Output: {result.output}") + if result.exit_code != 0: + print(f"Exit Code: {result.exit_code}") + if result.exception: + print(result.exception) + try: + print(f"Output: {result.stderr}") + except ValueError: + print("No stderr captured.") + assert result.exit_code == 0 diff --git a/tests/test_push_image.py b/tests/test_push_image.py.old similarity index 100% rename from tests/test_push_image.py rename to tests/test_push_image.py.old