diff --git a/charts/gitops-runtime/Chart.yaml b/charts/gitops-runtime/Chart.yaml index ce12596a..425ba4c1 100644 --- a/charts/gitops-runtime/Chart.yaml +++ b/charts/gitops-runtime/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 appVersion: 0.1.29 description: A Helm chart for Codefresh gitops runtime name: gitops-runtime -version: 0.2.6-alpha +version: 0.2.7-alpha home: https://github.com/codefresh-io/gitops-runtime-helm icon: https://avatars1.githubusercontent.com/u/11412079?v=3 keywords: @@ -16,9 +16,9 @@ annotations: artifacthub.io/prerelease: "true" artifacthub.io/changes: | - kind: changed - description: updated `argo-cd` to `v2.6.0-cap-CR-18430-del-app` (fix application/git-source deletion) - - kind: fixed - description: Fix delete runtime hook when using custom CA + description: Add private registry utililies + - kind: changed + description: Installer image for hooks now runs rootless dependencies: - name: argo-cd repository: https://codefresh-io.github.io/argo-helm diff --git a/charts/gitops-runtime/README.md b/charts/gitops-runtime/README.md index 8c57e8e7..389b7c6a 100644 --- a/charts/gitops-runtime/README.md +++ b/charts/gitops-runtime/README.md @@ -1,27 +1,30 @@ -# gitops-runtime +## Codefresh gitops runtime +![Version: 0.2.7-alpha](https://img.shields.io/badge/Version-0.2.7--alpha-informational?style=flat-square) ![AppVersion: 0.1.29](https://img.shields.io/badge/AppVersion-0.1.29-informational?style=flat-square) -![Version: 0.2.6-alpha](https://img.shields.io/badge/Version-0.2.6--alpha-informational?style=flat-square) ![AppVersion: 0.1.29](https://img.shields.io/badge/AppVersion-0.1.29-informational?style=flat-square) +## Codefresh official documentation: +Prior to running the installation please see the official documentation at: https://codefresh.io/docs/docs/installation/gitops/hybrid-gitops-helm-installation/ -A Helm chart for Codefresh gitops runtime +## Using with private registries - Helper utility +The GitOps Runtime comprises multiple subcharts and container images. Subcharts also vary in values structure, making it difficult to override image specific values to use private registries. +We have created a helper utility to resolve this issue: +- The utility create values files in the correct structure, overriding the registry for each image. When installing the chart, you can then provide those values files to override all images. +- The utility also creates other files with data to help you identify and correctly mirror all the images. -**Homepage:** +#### Usage -## Maintainers +The utility is packaged in a container image. Below are instructions on executing the utility using Docker: -| Name | Email | Url | -| ---- | ------ | --- | -| codefresh | | | +``` +docker run -v :/output quay.io/codefresh/gitops-runtime-private-registry-utils:0.2.7-alpha +``` +`output_dir` - is a local directory where the utility will output files.
+`local_registry` - is your local registry where you want to mirror the images to -## Requirements - -| Repository | Name | Version | -|------------|------|---------| -| https://bitnami-labs.github.io/sealed-secrets/ | sealed-secrets | 2.7.3 | -| https://chartmuseum.codefresh.io/codefresh-tunnel-client | tunnel-client(codefresh-tunnel-client) | 0.1.12 | -| https://codefresh-io.github.io/argo-helm | argo-cd | 5.29.2-cap-CR-18430 | -| https://codefresh-io.github.io/argo-helm | argo-events | 2.0.5-1-cf-init | -| https://codefresh-io.github.io/argo-helm | argo-rollouts | 2.22.1-1-cap-sw | -| https://codefresh-io.github.io/argo-helm | argo-workflows | 0.22.9-1-CR-17426 | +The utility will output 4 files into the folder: +1. `image-list.txt` - is the list of all images used in this version of the chart. Those are the images that you need to mirror. +2. `image-mirror.csv` - is a csv file with 2 fields - source_image and target_image. source_image is the image with the original registry and target_image is the image with the private registry. Can be used as an input file for a mirroring script. +3. `values-images-no-tags.yaml` - a values file with all image values with the private registry **excluding tags**. If provided through --values to helm install/upgrade command - it will override all images to use the private registry. +4. `values-images-with-tags.yaml` - The same as 3 but with tags **included**. ## Values @@ -184,6 +187,3 @@ A Helm chart for Codefresh gitops runtime | tunnel-client | object | `{"enabled":true,"libraryMode":true,"tunnelServer":{"host":"register-tunnels.cf-cd.com","subdomainHost":"tunnels.cf-cd.com"}}` | Tunnel based runtime. Not supported for on-prem platform. In on-prem use ingress based runtimes. | | tunnel-client.enabled | bool | `true` | Will only be used if global.runtime.ingress.enabled = false | | tunnel-client.libraryMode | bool | `true` | Do not change this value! Breaks chart logic | - ----------------------------------------------- -Autogenerated from chart metadata using [helm-docs v1.9.1](https://github.com/norwoodj/helm-docs/releases/v1.9.1) diff --git a/charts/gitops-runtime/README.md.gotmpl b/charts/gitops-runtime/README.md.gotmpl new file mode 100644 index 00000000..71d07799 --- /dev/null +++ b/charts/gitops-runtime/README.md.gotmpl @@ -0,0 +1,29 @@ +## Codefresh gitops runtime +{{ template "chart.versionBadge" . }}{{ template "chart.typeBadge" . }}{{ template "chart.appVersionBadge" . }} + +## Codefresh official documentation: +Prior to running the installation please see the official documentation at: https://codefresh.io/docs/docs/installation/gitops/hybrid-gitops-helm-installation/ + +## Using with private registries - Helper utility +The GitOps Runtime comprises multiple subcharts and container images. Subcharts also vary in values structure, making it difficult to override image specific values to use private registries. +We have created a helper utility to resolve this issue: +- The utility create values files in the correct structure, overriding the registry for each image. When installing the chart, you can then provide those values files to override all images. +- The utility also creates other files with data to help you identify and correctly mirror all the images. + +#### Usage + +The utility is packaged in a container image. Below are instructions on executing the utility using Docker: + +``` +docker run -v :/output quay.io/codefresh/gitops-runtime-private-registry-utils:0.2.7-alpha +``` +`output_dir` - is a local directory where the utility will output files.
+`local_registry` - is your local registry where you want to mirror the images to + +The utility will output 4 files into the folder: +1. `image-list.txt` - is the list of all images used in this version of the chart. Those are the images that you need to mirror. +2. `image-mirror.csv` - is a csv file with 2 fields - source_image and target_image. source_image is the image with the original registry and target_image is the image with the private registry. Can be used as an input file for a mirroring script. +3. `values-images-no-tags.yaml` - a values file with all image values with the private registry **excluding tags**. If provided through --values to helm install/upgrade command - it will override all images to use the private registry. +4. `values-images-with-tags.yaml` - The same as 3 but with tags **included**. + +{{ template "chart.valuesSection" . }} diff --git a/charts/gitops-runtime/ci/values-all-images.yaml b/charts/gitops-runtime/ci/values-all-images.yaml index 345138bc..69d212b5 100644 --- a/charts/gitops-runtime/ci/values-all-images.yaml +++ b/charts/gitops-runtime/ci/values-all-images.yaml @@ -1,3 +1,4 @@ +# Values file used to render all image values global: codefresh: accountId: 628a80b693a15c0f9c13ab75 # Codefresh Account id for ilia-codefresh for now, needs to be some test account @@ -13,7 +14,6 @@ global: runtime: name: default - cluster: test-cluster ingress: enabled: false diff --git a/installer-image/Dockerfile b/installer-image/Dockerfile index f08a922c..3b7a71eb 100644 --- a/installer-image/Dockerfile +++ b/installer-image/Dockerfile @@ -2,9 +2,7 @@ FROM --platform=$BUILDPLATFORM debian:bullseye-slim RUN apt-get update -y && apt-get install curl -y ARG CF_CLI_VERSION=v0.1.25 ARG KUBECTL_VERSION=v1.26.0 -ARG ARGOCD_VERSION=v2.4.15 ARG TARGETARCH -RUN echo "${TARGETARCH}" RUN curl -L --output - https://github.com/codefresh-io/cli-v2/releases/download/${CF_CLI_VERSION}/cf-linux-${TARGETARCH}.tar.gz | tar zx && mv ./cf-linux-${TARGETARCH} /usr/local/bin/cf RUN curl -LO https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/${TARGETARCH}/kubectl && chmod +x kubectl && mv ./kubectl /usr/local/bin/kubectl -RUN curl -sSL -o /usr/local/bin/argocd https://github.com/argoproj/argo-cd/releases/download/${ARGOCD_VERSION}/argocd-linux-${TARGETARCH} && chmod +x /usr/local/bin/argocd \ No newline at end of file +USER 1000 \ No newline at end of file diff --git a/scripts/all-image-values.sh b/scripts/all-image-values.sh deleted file mode 100755 index b66e25b0..00000000 --- a/scripts/all-image-values.sh +++ /dev/null @@ -1,4 +0,0 @@ -MYDIR=$(dirname $0) - -$MYDIR/output-calculated-values.sh /tmp/gitops-runtime-values.yaml > /dev/null -python3 $MYDIR/helper-scripts/yaml-filter.py /tmp/gitops-runtime-values.yaml image.repository,image.registry,image.tag,argo-events.configs.nats.versions,argo-events.configs.jetstream.versions \ No newline at end of file diff --git a/scripts/list-images-for-mirrror.sh b/scripts/list-images-for-mirrror.sh deleted file mode 100755 index 7ef7f8bd..00000000 --- a/scripts/list-images-for-mirrror.sh +++ /dev/null @@ -1,6 +0,0 @@ -MYDIR=$(dirname $0) -CHARTDIR="${MYDIR}/../charts/gitops-runtime" -VALUESFILE="${CHARTDIR}/ci/values-all-images.yaml" -helm dependency update $CHARTDIR > /dev/null -helm template --values $VALUESFILE $CHARTDIR | grep -E -i ".*Image: " | awk -F ':' '{if ($2) printf "%s:%s\n", $2, $3}' | tr -d "\"|[:blank:]" | sort -u |uniq -i -# Current limitation - the image for argoexec is missing, since it's sent as a parameter in the command in the template, and it's hard to match it with a regex. \ No newline at end of file diff --git a/scripts/private-registry-utils/.dockerignore b/scripts/private-registry-utils/.dockerignore new file mode 100644 index 00000000..9b54c5ba --- /dev/null +++ b/scripts/private-registry-utils/.dockerignore @@ -0,0 +1,2 @@ +README.md +Dockerfile diff --git a/scripts/private-registry-utils/Dockerfile b/scripts/private-registry-utils/Dockerfile new file mode 100644 index 00000000..04ceadfb --- /dev/null +++ b/scripts/private-registry-utils/Dockerfile @@ -0,0 +1,13 @@ +FROM --platform=$BUILDPLATFORM python:3.11.3-slim-bullseye +ARG TARGETARCH +RUN cd /tmp && python3 -c "from urllib.request import urlretrieve; urlretrieve('https://get.helm.sh/helm-v3.12.0-linux-${TARGETARCH}.tar.gz', 'helm-v3.12.0-linux-${TARGETARCH}.tar.gz')" && tar -xvf helm-v3.12.0-linux-${TARGETARCH}.tar.gz && chmod +x linux-${TARGETARCH}/helm && mv linux-${TARGETARCH}/helm /usr/local/bin/helm && rm -rf /tmp/* +COPY charts/gitops-runtime /chart +RUN helm dependency update /chart +COPY scripts/private-registry-utils/python-requirements.txt /scripts/python-requirements.txt +RUN pip3 install -r /scripts/python-requirements.txt +COPY scripts/private-registry-utils /scripts +RUN chmod -R +x /scripts +WORKDIR /scripts +# Output calculated values and filter image values +RUN ./output-calculated-values.sh ./all-values.yaml && python3 ./helper-scripts/yaml-filter.py all-values.yaml image.repository,image.registry,image.tag,argo-events.configs.nats.versions,argo-events.configs.jetstream.versions > all-image-values.yaml +ENTRYPOINT ["python3", "private-registry-utils.py", "all-image-values.yaml"] diff --git a/scripts/private-registry-utils/README.md b/scripts/private-registry-utils/README.md new file mode 100644 index 00000000..52f09558 --- /dev/null +++ b/scripts/private-registry-utils/README.md @@ -0,0 +1,15 @@ +Utilities to assist with image mirroring to private registries + +## How it works? + +1. All calculated values are outputed using output-calculated-values.sh. The script contains a special template that handles image values. Specifically it derives tags for images that don't have explicit tags sepcified from the subchart appVersion. +2. Those values are then filtered using the helper script yaml-filter.py to filter only the values that contain images. +3. image-values-utils.py then uses the output of the previous steps and can generate: + 1. A list of all images + 2. Values files with and without tags including a custom registry for each image. + 3. A csv file with source and target image names (can help with image mirroring). + +## Usage +1. Build the image with buildcontext being the root of the repo `docker build -f scripts/private-registry-utils/Dockerfile -t .` For example: `docker build -f scripts/private-registry-utils/Dockerfile -t gitops-runtime-priv-reg .` +2. Execute with: `docker run -v :/output -it ` for example: `docker run -v /tmp/output:/output -it gitops-runtime-priv-reg myregisry.example.com` +3. See the generated files in local output folder diff --git a/scripts/helper-scripts/yaml-filter.py b/scripts/private-registry-utils/helper-scripts/yaml-filter.py similarity index 100% rename from scripts/helper-scripts/yaml-filter.py rename to scripts/private-registry-utils/helper-scripts/yaml-filter.py diff --git a/scripts/output-calculated-values.sh b/scripts/private-registry-utils/output-calculated-values.sh similarity index 96% rename from scripts/output-calculated-values.sh rename to scripts/private-registry-utils/output-calculated-values.sh index cfd12040..91a3c918 100755 --- a/scripts/output-calculated-values.sh +++ b/scripts/private-registry-utils/output-calculated-values.sh @@ -1,6 +1,6 @@ #!/bin/bash MYDIR=$(dirname $0) -CHARTDIR="${MYDIR}/../charts/gitops-runtime" +CHARTDIR="/chart" VALUESFILE="${CHARTDIR}/ci/values-all-images.yaml" OUTPUTFILE=$1 # This template prints all values and also sets tags for all images with non-empty repository value, where the tag is empty and should be derived from the appVersion of the subchart. @@ -47,6 +47,5 @@ END ) echo -e "$ALL_VALUES_TEMPLATE" > $CHARTDIR/templates/all-values.yaml -helm dependency update $CHARTDIR helm template --values $VALUESFILE --set getImages=true --show-only templates/all-values.yaml $CHARTDIR > $OUTPUTFILE rm $CHARTDIR/templates/all-values.yaml \ No newline at end of file diff --git a/scripts/private-registry-utils/private-registry-utils.py b/scripts/private-registry-utils/private-registry-utils.py new file mode 100644 index 00000000..edb9cd55 --- /dev/null +++ b/scripts/private-registry-utils/private-registry-utils.py @@ -0,0 +1,150 @@ +import argparse +import yaml +import sys +import re +import csv + +DEFAULT_REGISTRY_URL = "registry.example.com" + +def remove_duplicates(list_of_dicts, key): + seen = set() + deduplicated_list = [] + + for d in list_of_dicts: + if d[key] not in seen: + deduplicated_list.append(d) + seen.add(d[key]) + + return deduplicated_list + +def replace_registry_in_image(image_string, new_registry): + if '/' in image_string: + parts = image_string.split('/') + if len(parts) >= 2: + parts[0] = new_registry + return '/'.join(parts) + else: + return new_registry + '/' + image_string + +# Try to identify whether a string is a docker image +def is_docker_image(image_string): + pattern = r'^[a-zA-Z0-9/_-]+:[a-zA-Z0-9_.-]+$' + return bool(re.match(pattern, image_string)) + + +def recurse_replace_registry(currValue,new_registry): + if type(currValue) is dict: + for key in currValue.keys(): + if key == "registry": + currValue[key]=new_registry + elif key == "repository" and "registry" not in currValue: + currValue[key] = replace_registry_in_image(currValue[key],new_registry) + elif type(currValue[key]) is str: + if is_docker_image(currValue[key]): + currValue[key] = replace_registry_in_image(currValue[key],new_registry) + else: + recurse_replace_registry(currValue[key],new_registry) + elif type(currValue) is list: + for item in currValue: + recurse_replace_registry(item,new_registry) + +def recurse_remove_tags(currValue): + if type(currValue) is dict: + if "tag" in currValue: + currValue.pop("tag") + else: + for key in currValue.keys(): + recurse_remove_tags(currValue[key]) + elif type(currValue) is list: + for item in currValue: + recurse_remove_tags(item) + + +def recurse_get_source_target(currValue,new_registry,lstSourceTarget): + sourceImage = "" + if type(currValue) is dict: + for key in currValue.keys(): + if key == "registry": + sourceImage += currValue[key] + "/" + if key == "repository": + sourceImage += currValue[key] + if key == "tag": + sourceImage += ":" + currValue[key] + + elif type(currValue[key]) is str: + if is_docker_image(currValue[key]): + sourceImage = currValue[key] + + recurse_get_source_target(currValue[key],new_registry,lstSourceTarget) + + if len(sourceImage) > 0: + lstSourceTarget.append({"source_image": sourceImage, "target_image": replace_registry_in_image(sourceImage,new_registry)}) + + + elif type(currValue) is list: + for item in currValue: + recurse_get_source_target(item,new_registry,lstSourceTarget) + +def generate_file_from_field(list_of_dicts, field_name, output_file): + with open(output_file, 'w+') as file: + for d in list_of_dicts: + field_value = d.get(field_name) + if field_value: + file.write(str(field_value) + '\n') + +def generate_image_values(dictImageValues,outputDir): + + with open(f"{outputDir}/values-images-with-tags.yaml", 'w+') as file: + yaml.dump(dictImageValues, file) + + recurse_remove_tags(dictImageValues) + + with open(f"{outputDir}/values-images-no-tags.yaml", 'w+') as file: + yaml.dump(dictImageValues, file) + +def generate_image_list(): + # Code for generating image list + print("Generating image list") + +def generate_mirror_csv(lstSourceTarget, outputDir): + + fields = ['source_image', 'target_image'] + + with open(f"{outputDir}/image-mirror.csv", 'w+') as csvfile: + writer = csv.DictWriter(csvfile, fieldnames = fields) + writer.writeheader() + writer.writerows(lstSourceTarget) + +def main(): + + parser = argparse.ArgumentParser(description="Codefresh gitops runtime - private registry utils") + parser.add_argument("images_values_file", help="Input values.yaml file") + parser.add_argument("private_registry_url", nargs="?", default=DEFAULT_REGISTRY_URL, help="Private Registry URL") + parser.add_argument("--output-dir", help="Output directory",default="/output") + parser.add_argument("--action", choices=["generate-image-values", "generate-image-list", "generate-mirror-csv"], help="Action to execute") + args = parser.parse_args() + + # Open yaml + with open(args.images_values_file, 'r') as stream: + try: + dictImageValues=yaml.safe_load(stream) + except yaml.YAMLError as e: + print(e) + + lstSourceTarget = [] + recurse_get_source_target(dictImageValues,args.private_registry_url,lstSourceTarget) + lstSourceTarget = remove_duplicates(lstSourceTarget, "source_image") + recurse_replace_registry(dictImageValues,args.private_registry_url) + + + if args.action == "generate-mirror-csv" or args.action is None: + generate_mirror_csv(lstSourceTarget,args.output_dir) + + if args.action == "generate-image-list" or args.action is None: + generate_file_from_field(lstSourceTarget,"source_image", f"{args.output_dir}/image-list.txt") + + if args.action == "generate-image-values" or args.action is None: + generate_image_values(dictImageValues,args.output_dir) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/private-registry-utils/python-requirements.txt b/scripts/private-registry-utils/python-requirements.txt new file mode 100644 index 00000000..a3db8990 --- /dev/null +++ b/scripts/private-registry-utils/python-requirements.txt @@ -0,0 +1 @@ +PyYAML==6.0 \ No newline at end of file