port oci cli back from python_gardenlinux_cli

Author: yeoldegrove

README.md

@@ -4,26 +4,33 @@
 ![security check](https://github.com/gardenlinux/parse_features_lib/actions/workflows/bandit.yml/badge.svg)
 
 # Parse features lib
-This library helps you to work with the gardenlinux/features folder. It parses all info.yamls and builds a tree.
 
-Features (planned):
-* validate CNAMEs
-* validate info.yamls
-* Deduct dependencies from cname_base
+This library includes tooling to build and distribute [Garden Linux](https://github.com/gardenlinux/gardenlinux).
+
+Features:
+
+-   compare APT repositories
+-   parse features
+-   parse flavors
+-   push OCI artifacts to a registry
 
 ## Quickstart
+
+### Example: parse features
+
 **Inclusion via poetry**:
 
-`parse_features_lib = { git  = "https://github.com/gardenlinux/parse_features_lib", rev="main" }`
+`python_gardenlinux_lib = { git  = "https://github.com/gardenlinux/python_gardenlinux_lib", rev="0.6.0" }`
+
 ```python
-import parse_features_lib
+from python_gardenlinux_lib.parse_features import read_feature_files, filter_graph
 
 if __name__ == "__main__":
     # Step 1: parse the "features directory" and get the full graph containing all features
-    all_features = parse_features_lib.read_feature_files("features")
+    all_features = parse_features.read_feature_files("features")
 
     # Step 2: supply desired features and get all their dependencies
-    dependencies = parse_features_lib.filter_graph(all_features, {"gardener", "_prod", "server", "ociExample"})
+    dependencies = parse_features.filter_graph(all_features, {"gardener", "_prod", "server", "ociExample"})
 
     # Step 3: play with the retrieved data.
     for feature, info in dependencies.nodes(data="content"):
@@ -32,5 +39,47 @@ if __name__ == "__main__":
 ```
 
 ## Developer Documentation
-The library is documented with docstrings, which are used to generate the developer documentation available [here](https://gardenlinux.github.io/python-gardenlinux-lib/). 
 
+The library is documented with docstrings, which are used to generate the developer documentation available [here](https://gardenlinux.github.io/python-gardenlinux-lib/).
+
+## Push OCI artifacts to a registry
+
+this tool helps you to push oci artifacts.
+
+### Installation
+
+```bash
+git clone https://github.com/gardenlinux/python-gardenlinux-lib.git
+mkdir venv
+python -m venv venv
+source venv/bin/activate.sh
+poetry install
+gl-oci --help
+```
+
+### Usage
+
+The process to push a Gardenlinux build-output folder to an OCI registry is split into two steps: In the first step all files are pushed to the registry and a manifest that includes all those pushed files (layers) is created and pushed as well. An index entry that links to this manifest is created offline and written to a local file but not pushed to any index. This push to an index can be done in the second step where the local file containing the index entry is read and pushed to an index. The seperation into two steps was done because pushing of manifests takes long and writes to dedicated resources (possible to run in parallel). Updating the index on the other hand is quick but writes to a share resource (not possible to run in parallel). By splitting the process up into two steps it is possible to run the slow part in parallel and the quick part sequentially.
+
+#### 1. Push layers + manifest
+
+To push layers you have to supply the directory with the build outputs `--dir`. Also you have to supply cname (`--cname`), architecture `--arch` and version `--version` of the build. This information will be included in the manifest. You have to supply an endpoint where the artifacts shall be pushed to `--container`, for example `ghcr.io/gardenlinux/gardenlinux`. You can disable enforced HTTPS connections to your registry with `--insecure True`. You can supply `--cosign_file <filename>` if you want to have the hash saved in `<filename>`. This can be handy to read the hash later to sign the manifest with cosign. With `--manifest_file <filename>` you tell the program in which file to store the manifests index entry. This is the file that can be used in the next step to update the index. You can use the environment variable GLOCI_REGISTRY_TOKEN to authenticate against the registry. Below is an example of a full program call of `push-manifest`
+
+```bash
+GLOCI_REGISTRY_TOKEN=asdf123 gl-oci push-manifest --dir build-metal-gardener_prod --container ghcr.io/gardenlinux/gl-oci --arch amd64 --version 1592.1 --cname metal-gardener_prod --cosign_file digest --manifest_file oci_manifest_entry_metal.json
+```
+
+#### 2. Update index with manifest entry
+
+Parameters that are the same as for `push-manifest`:
+
+-   env-var `GLOCI_REGISTRY_TOKEN`
+-   `--version`
+-   `--container`
+-   `--manifest-file` this time this parameter adjusts the manifest entry file to be read from instead of being written to
+
+A full example looks like this:
+
+```bash
+GLOCI_REGISTRY_TOKEN=asdf123 gl-oci update-index --container ghcr.io/gardenlinux/gl-oci --version 1592.1 --manifest_file oci_manifest_entry_metal.json
+```

pyproject.toml

@@ -29,6 +29,7 @@ black = "^24.8.0"
 gl-cname = "python_gardenlinux_lib.cname:main"
 gl-flavors-parse = "python_gardenlinux_lib.flavors.parse_flavors:main"
 flavors-parse = "python_gardenlinux_lib.flavors.parse_flavors:main"
+gl-oci = "python_gardenlinux_lib.oci.cli:main"
 
 [tool.pytest.ini_options]
 pythonpath = [

src/python_gardenlinux_lib/cname.py

@@ -21,7 +21,7 @@ def main():
 
     re_match = re.match(
         "([a-zA-Z0-9]+(-[a-zA-Z0-9\\_\\-]*?)?)(-([a-z0-9]+)(-([a-z0-9.]+)-([a-z0-9]+))*)?$",
-        args.cname
+        args.cname,
     )
 
     assert re_match, f"not a valid cname {args.cname}"
@@ -72,11 +72,13 @@ def main():
 
     print(cname)
 
+
 def get_cname_base(sorted_features):
     return reduce(
-        lambda a, b : a + ("-" if not b.startswith("_") else "") + b, sorted_features
+        lambda a, b: a + ("-" if not b.startswith("_") else "") + b, sorted_features
     )
 
+
 def get_minimal_feature_set(graph):
     return set([node for (node, degree) in graph.in_degree() if degree == 0])
 

src/python_gardenlinux_lib/constants.py

@@ -2,8 +2,19 @@
 
 # It is important that this list is sorted in descending length of the entries
 GL_MEDIA_TYPES = [
+    "secureboot.aws-efivars",
+    "secureboot.kek.auth",
     "gcpimage.tar.gz.log",
+    "secureboot.pk.auth",
+    "secureboot.kek.crt",
+    "secureboot.kek.der",
+    "secureboot.db.auth",
     "firecracker.tar.gz",
+    "secureboot.pk.crt",
+    "secureboot.pk.der",
+    "secureboot.db.crt",
+    "secureboot.db.der",
+    "secureboot.db.arn",
     "platform.test.log",
     "platform.test.xml",
     "gcpimage.tar.gz",
@@ -12,11 +23,15 @@
     "pxe.tar.gz.log",
     "root.squashfs",
     "manifest.log",
+    "squashfs.log",
     "release.log",
+    "vmlinuz.log",
+    "initrd.log",
     "pxe.tar.gz",
     "qcow2.log",
     "test-log",
     "boot.efi",
+    "squashfs",
     "manifest",
     "vmdk.log",
     "tar.log",
@@ -69,12 +84,27 @@
     "vhd.log": "application/io.gardenlinux.log",
     "ova.log": "application/io.gardenlinux.log",
     "vmlinuz": "application/io.gardenlinux.kernel",
+    "vmlinuz.log": "application/io.gardenlinux.log",
     "initrd": "application/io.gardenlinux.initrd",
+    "initrd.log": "application/io.gardenlinux.log",
     "root.squashfs": "application/io.gardenlinux.squashfs",
+    "squashfs": "application/io.gardenlinux.squashfs",
+    "squashfs.log": "application/io.gardenlinux.log",
     "boot.efi": "application/io.gardenlinux.efi",
     "platform.test.log": "application/io.gardenlinux.io.platform.test.log",
     "platform.test.xml": "application/io.gardenlinux.io.platform.test.xml",
     "chroot.test.log": "application/io.gardenlinux.io.chroot.test.log",
     "chroot.test.xml": "application/io.gardenlinux.io.chroot.test.xml",
     "oci.log": "application/io.gardenlinux.log",
+    "secureboot.pk.crt": "application/io.gardenlinux.cert.secureboot.pk.crt",
+    "secureboot.pk.der": "application/io.gardenlinux.cert.secureboot.pk.der",
+    "secureboot.pk.auth": "application/io.gardenlinux.cert.secureboot.pk.auth",
+    "secureboot.kek.crt": "application/io.gardenlinux.cert.secureboot.kek.crt",
+    "secureboot.kek.der": "application/io.gardenlinux.cert.secureboot.kek.der",
+    "secureboot.kek.auth": "application/io.gardenlinux.cert.secureboot.kek.auth",
+    "secureboot.db.crt": "application/io.gardenlinux.cert.secureboot.db.crt",
+    "secureboot.db.der": "application/io.gardenlinux.cert.secureboot.db.der",
+    "secureboot.db.auth": "application/io.gardenlinux.cert.secureboot.db.auth",
+    "secureboot.db.arn": "application/io.gardenlinux.cert.secureboot.db.arn",
+    "secureboot.aws-efivars": "application/io.gardenlinux.cert.secureboot.aws-efivars",
 }

src/python_gardenlinux_lib/oras/crypto.py renamed to src/python_gardenlinux_lib/crypto.py

@@ -38,6 +38,7 @@ def get_gardenlinux_commit(gardenlinux_root: str, limit: Optional[int] = None) -
     else:
         return commit_str
 
+
 def get_features_dict(
     cname: str, gardenlinux_root: str, feature_dir_name: str = "features"
 ) -> dict:
@@ -61,6 +62,7 @@ def get_features_dict(
 
     return features_by_type
 
+
 def get_features_graph(
     cname: str, gardenlinux_root: str, feature_dir_name: str = "features"
 ) -> networkx.graph:
@@ -78,6 +80,7 @@ def get_features_graph(
 
     return graph
 
+
 def get_features_list(
     cname: str, gardenlinux_root: str, feature_dir_name: str = "features"
 ) -> list:
@@ -93,6 +96,7 @@ def get_features_list(
 
     return features
 
+
 def get_features(
     cname: str, gardenlinux_root: str, feature_dir_name: str = "features"
 ) -> str:
@@ -107,6 +111,7 @@ def get_features(
 
     return ",".join(features)
 
+
 def construct_layer_metadata(
     filetype: str, cname: str, version: str, arch: str, commit: str
 ) -> dict:
@@ -125,6 +130,7 @@ def construct_layer_metadata(
         "annotations": {"io.gardenlinux.image.layer.architecture": arch},
     }
 
+
 def construct_layer_metadata_from_filename(filename: str, arch: str) -> dict:
     """
     :param str filename: filename of the blob
@@ -138,6 +144,7 @@ def construct_layer_metadata_from_filename(filename: str, arch: str) -> dict:
         "annotations": {"io.gardenlinux.image.layer.architecture": arch},
     }
 
+
 def get_file_set_from_cname(cname: str, version: str, arch: str, gardenlinux_root: str):
     """
     :param str cname: the target cname of the image
@@ -160,6 +167,7 @@ def get_file_set_from_cname(cname: str, version: str, arch: str, gardenlinux_roo
             )
     return file_set
 
+
 def get_oci_metadata_from_fileset(fileset: list, arch: str):
     """
     :param str arch: arch of the target image
@@ -175,6 +183,7 @@ def get_oci_metadata_from_fileset(fileset: list, arch: str):
 
     return oci_layer_metadata_list
 
+
 def get_oci_metadata(cname: str, version: str, arch: str, gardenlinux_root: str):
     """
     :param str cname: the target cname of the image
@@ -197,6 +206,7 @@ def get_oci_metadata(cname: str, version: str, arch: str, gardenlinux_root: str)
 
     return oci_layer_metadata_list
 
+
 def lookup_media_type_for_filetype(filetype: str) -> str:
     """
     :param str filetype: filetype of the target layer
@@ -209,6 +219,7 @@ def lookup_media_type_for_filetype(filetype: str) -> str:
             f"media type for {filetype} is not defined. You may want to add the definition to parse_features_lib"
         )
 
+
 def lookup_media_type_for_file(filename: str) -> str:
     """
     :param str filename: filename of the target layer
@@ -222,6 +233,7 @@ def lookup_media_type_for_file(filename: str) -> str:
             f"media type for {filename} is not defined. You may want to add the definition to parse_features_lib"
         )
 
+
 def deduce_feature_name(feature_dir: str):
     """
     :param str feature_dir: Directory of single Feature
@@ -232,20 +244,23 @@ def deduce_feature_name(feature_dir: str):
         raise ValueError("Expected name from parse_feature_yaml function to be set")
     return parsed["name"]
 
+
 def deduce_archive_filetypes(feature_dir):
     """
     :param str feature_dir: Directory of single Feature
     :return: str list of filetype for archive
     """
     return deduce_filetypes_from_string(feature_dir, "image")
 
+
 def deduce_image_filetypes(feature_dir):
     """
     :param str feature_dir: Directory of single Feature
     :return: str list of filetype for image
     """
     return deduce_filetypes_from_string(feature_dir, "convert")
 
+
 def deduce_filetypes(feature_dir):
     """
     :param str feature_dir: Directory of single Feature
@@ -260,6 +275,7 @@ def deduce_filetypes(feature_dir):
     image_file_types.extend(archive_file_types)
     return image_file_types
 
+
 def deduce_filetypes_from_string(feature_dir: str, script_base_name: str):
     """
     Garden Linux features can optionally have an image.<filetype> or convert.<filetype> script,
@@ -283,6 +299,7 @@ def deduce_filetypes_from_string(feature_dir: str, script_base_name: str):
 
     return sorted(result)
 
+
 def read_feature_files(feature_dir):
     """
     Legacy function copied from gardenlinux/builder
@@ -312,6 +329,7 @@ def read_feature_files(feature_dir):
         raise ValueError("Graph is not directed acyclic graph")
     return feature_graph
 
+
 def parse_feature_yaml(feature_yaml_file: str):
     """
     Legacy function copied from gardenlinux/builder
@@ -328,9 +346,11 @@ def parse_feature_yaml(feature_yaml_file: str):
         content = yaml.safe_load(f)
     return {"name": name, "content": content}
 
+
 def __get_node_features(node):
     return node.get("content", {}).get("features", {})
 
+
 def filter_graph(feature_graph, feature_set, ignore_excludes=False):
     filter_set = set(feature_graph.nodes())
 
@@ -369,29 +389,35 @@ def filter_func(node):
         raise ValueError("Including explicitly excluded feature")
     return graph
 
+
 def sort_set(input_set, order_list):
     return [item for item in order_list if item in input_set]
 
+
 def __sort_key(graph, node):
     prefix_map = {"platform": "0", "element": "1", "flag": "2"}
     node_type = __get_node_type(graph.nodes.get(node, {}))
     prefix = prefix_map[node_type]
     return f"{prefix}-{node}"
 
+
 def sort_nodes(graph):
     def key_function(node):
         return __sort_key(graph, node)
 
     return list(networkx.lexicographical_topological_sort(graph, key=key_function))
 
+
 def __reverse_cname_base(cname):
     cname = cname.replace("_", "-_")
     return set(cname.split("-"))
 
+
 def __reverse_sort_nodes(graph):
     reverse_graph = graph.reverse()
     assert networkx.is_directed_acyclic_graph(reverse_graph)
     return sort_nodes(reverse_graph)
 
+
 def __get_node_type(node):
     return node.get("content", {}).get("type")