From d1505fb8f284b09e0e5119bcd52d5e1d1282de16 Mon Sep 17 00:00:00 2001 From: Bohdan Pysarenko Date: Wed, 12 Feb 2025 19:45:44 +0200 Subject: [PATCH 01/11] add new option to app-proxy --- charts/gitops-runtime/README.md | 1 + .../templates/_components/cap-app-proxy/_config.yaml | 3 +++ .../environment-variables/_main-container.yaml | 6 ++++++ charts/gitops-runtime/values.yaml | 4 ++++ 4 files changed, 14 insertions(+) diff --git a/charts/gitops-runtime/README.md b/charts/gitops-runtime/README.md index 8d944b9c..07539a3f 100644 --- a/charts/gitops-runtime/README.md +++ b/charts/gitops-runtime/README.md @@ -90,6 +90,7 @@ sealed-secrets: | app-proxy.config.env | string | `"production"` | | | app-proxy.config.logLevel | string | `"info"` | Log Level | | app-proxy.config.skipGitPermissionValidation | string | `"false"` | Skit git permissions validation | +| app-proxy.cron.clusterChunkSize | int | `0` | App-proxy will split and refresh the cluster list with chunks with size `clusterChunkSize`. When the default value is used the cluster list will be refreshed with one chunk (no split; not recommended when cluster list is too large). | | app-proxy.env | object | `{}` | | | app-proxy.extraVolumeMounts | list | `[]` | Extra volume mounts for main container | | app-proxy.extraVolumes | list | `[]` | extra volumes | diff --git a/charts/gitops-runtime/templates/_components/cap-app-proxy/_config.yaml b/charts/gitops-runtime/templates/_components/cap-app-proxy/_config.yaml index 8485c411..8822c8e5 100644 --- a/charts/gitops-runtime/templates/_components/cap-app-proxy/_config.yaml +++ b/charts/gitops-runtime/templates/_components/cap-app-proxy/_config.yaml @@ -4,6 +4,9 @@ argoCdUsername: {{ .Values.config.argoCdUsername }} argoWorkflowsInsecure: {{ .Values.config.argoWorkflowsInsecure | quote }} argoWorkflowsUrl: {{ default "" .Values.config.argoWorkflowsUrl }} cors: {{ .Values.global.codefresh.url }} + {{- if ((.Values.cron).clusterChunkSize) }} +clusterChunkSize: {{ .Values.cron.clusterChunkSize | quote }} + {{- end }} env: {{ .Values.config.env | quote}} isConfigurationRuntime: {{ .Values.global.runtime.isConfigurationRuntime | quote }} runtimeName: {{ required "global.runtime.name is required" .Values.global.runtime.name | quote}} diff --git a/charts/gitops-runtime/templates/_components/cap-app-proxy/environment-variables/_main-container.yaml b/charts/gitops-runtime/templates/_components/cap-app-proxy/environment-variables/_main-container.yaml index 7cef698f..9b1bc3f3 100644 --- a/charts/gitops-runtime/templates/_components/cap-app-proxy/environment-variables/_main-container.yaml +++ b/charts/gitops-runtime/templates/_components/cap-app-proxy/environment-variables/_main-container.yaml @@ -36,6 +36,12 @@ CF_HOST: key: base-url optional: true CLUSTER: https://kubernetes.default.svc +CLUSTER_CHUNK_SIZE: + valueFrom: + configMapKeyRef: + name: cap-app-proxy-cm + key: clusterChunkSize + optional: true CORS: valueFrom: configMapKeyRef: diff --git a/charts/gitops-runtime/values.yaml b/charts/gitops-runtime/values.yaml index 9834d49d..a857657f 100644 --- a/charts/gitops-runtime/values.yaml +++ b/charts/gitops-runtime/values.yaml @@ -481,6 +481,10 @@ app-proxy: skipGitPermissionValidation: "false" # -- Log Level logLevel: "info" + + # cron: + # -- chunk size of the cluster lists to refresh the clusters in portion during cron job + # clusterChunkSize: 5 env: {} From b8078d0f531e10289dd2ad7aa0f8836792536ee6 Mon Sep 17 00:00:00 2001 From: Bohdan Pysarenko Date: Mon, 17 Feb 2025 17:51:45 +0200 Subject: [PATCH 02/11] enable chunks --- charts/gitops-runtime/values.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/charts/gitops-runtime/values.yaml b/charts/gitops-runtime/values.yaml index a857657f..591274de 100644 --- a/charts/gitops-runtime/values.yaml +++ b/charts/gitops-runtime/values.yaml @@ -482,9 +482,11 @@ app-proxy: # -- Log Level logLevel: "info" - # cron: - # -- chunk size of the cluster lists to refresh the clusters in portion during cron job - # clusterChunkSize: 5 + cron: + # -- chunk size of the cluster lists to refresh the clusters in portion during cron job: + # - reduce this value if you have a lot of clusters and the cron job is failing with payload too large error + # - use 0 to sync all clusters at once + clusterChunkSize: 50 env: {} From 75d7183da5b062c8cc8c14ba0f6ec160b7b72773 Mon Sep 17 00:00:00 2001 From: Bohdan Pysarenko Date: Tue, 18 Feb 2025 11:55:41 +0200 Subject: [PATCH 03/11] fix lint --- charts/gitops-runtime/values.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/gitops-runtime/values.yaml b/charts/gitops-runtime/values.yaml index 46e600cc..166deef7 100644 --- a/charts/gitops-runtime/values.yaml +++ b/charts/gitops-runtime/values.yaml @@ -481,11 +481,11 @@ app-proxy: skipGitPermissionValidation: "false" # -- Log Level logLevel: "info" - + cron: # -- chunk size of the cluster lists to refresh the clusters in portion during cron job: - # - reduce this value if you have a lot of clusters and the cron job is failing with payload too large error - # - use 0 to sync all clusters at once + # - reduce this value if you have a lot of clusters and the cron job is failing with payload too large error + # - use 0 to sync all clusters at once clusterChunkSize: 50 env: {} From 4ca52f2a1fb2730401989b5e03ed37f918aa3159 Mon Sep 17 00:00:00 2001 From: Bohdan Pysarenko Date: Tue, 18 Feb 2025 12:17:07 +0200 Subject: [PATCH 04/11] fix lint --- charts/gitops-runtime/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/gitops-runtime/values.yaml b/charts/gitops-runtime/values.yaml index 166deef7..0ae347d9 100644 --- a/charts/gitops-runtime/values.yaml +++ b/charts/gitops-runtime/values.yaml @@ -485,7 +485,7 @@ app-proxy: cron: # -- chunk size of the cluster lists to refresh the clusters in portion during cron job: # - reduce this value if you have a lot of clusters and the cron job is failing with payload too large error - # - use 0 to sync all clusters at once + # - use 0 to sync all clusters at once clusterChunkSize: 50 env: {} From e63579785e2cecdb0846acaa70f5ec2b2827a23d Mon Sep 17 00:00:00 2001 From: Bohdan Pysarenko Date: Tue, 18 Feb 2025 16:45:34 +0200 Subject: [PATCH 05/11] upd --- charts/gitops-runtime/README.md | 2 +- .../templates/_components/cap-app-proxy/_config.yaml | 4 ++-- charts/gitops-runtime/values.yaml | 5 ++--- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/charts/gitops-runtime/README.md b/charts/gitops-runtime/README.md index 07539a3f..24120fa2 100644 --- a/charts/gitops-runtime/README.md +++ b/charts/gitops-runtime/README.md @@ -90,7 +90,7 @@ sealed-secrets: | app-proxy.config.env | string | `"production"` | | | app-proxy.config.logLevel | string | `"info"` | Log Level | | app-proxy.config.skipGitPermissionValidation | string | `"false"` | Skit git permissions validation | -| app-proxy.cron.clusterChunkSize | int | `0` | App-proxy will split and refresh the cluster list with chunks with size `clusterChunkSize`. When the default value is used the cluster list will be refreshed with one chunk (no split; not recommended when cluster list is too large). | +| app-proxy.config.clusterChunkSize | int | `0` | Number of clusters per request. App-proxy will split and refresh the cluster list with chunks with size `clusterChunkSize`. When the default value is used the cluster list will be refreshed with one chunk (no split; not recommended when cluster list is too large). | | app-proxy.env | object | `{}` | | | app-proxy.extraVolumeMounts | list | `[]` | Extra volume mounts for main container | | app-proxy.extraVolumes | list | `[]` | extra volumes | diff --git a/charts/gitops-runtime/templates/_components/cap-app-proxy/_config.yaml b/charts/gitops-runtime/templates/_components/cap-app-proxy/_config.yaml index 8822c8e5..59a89cdc 100644 --- a/charts/gitops-runtime/templates/_components/cap-app-proxy/_config.yaml +++ b/charts/gitops-runtime/templates/_components/cap-app-proxy/_config.yaml @@ -4,8 +4,8 @@ argoCdUsername: {{ .Values.config.argoCdUsername }} argoWorkflowsInsecure: {{ .Values.config.argoWorkflowsInsecure | quote }} argoWorkflowsUrl: {{ default "" .Values.config.argoWorkflowsUrl }} cors: {{ .Values.global.codefresh.url }} - {{- if ((.Values.cron).clusterChunkSize) }} -clusterChunkSize: {{ .Values.cron.clusterChunkSize | quote }} + {{- if ((.Values.config).clusterChunkSize) }} +clusterChunkSize: {{ .Values.config.clusterChunkSize | quote }} {{- end }} env: {{ .Values.config.env | quote}} isConfigurationRuntime: {{ .Values.global.runtime.isConfigurationRuntime | quote }} diff --git a/charts/gitops-runtime/values.yaml b/charts/gitops-runtime/values.yaml index 0ae347d9..49457da0 100644 --- a/charts/gitops-runtime/values.yaml +++ b/charts/gitops-runtime/values.yaml @@ -481,9 +481,8 @@ app-proxy: skipGitPermissionValidation: "false" # -- Log Level logLevel: "info" - - cron: - # -- chunk size of the cluster lists to refresh the clusters in portion during cron job: + # -- define cluster list size per request to report the cluster state to platform, e.g. + # if you have 90 clusters and set clusterChunkSize: 40, it means cron job will report cluster state to platform in 3 iterations (40,40,10) # - reduce this value if you have a lot of clusters and the cron job is failing with payload too large error # - use 0 to sync all clusters at once clusterChunkSize: 50 From 2768bd7aa2df82b6cab8a6c9dee7d1cca8c03988 Mon Sep 17 00:00:00 2001 From: Bohdan Pysarenko Date: Wed, 19 Feb 2025 15:37:38 +0200 Subject: [PATCH 06/11] bump docker image --- installer-image/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/installer-image/Dockerfile b/installer-image/Dockerfile index 09d4b8cd..bd091a13 100644 --- a/installer-image/Dockerfile +++ b/installer-image/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:bullseye-slim +FROM debian:bookworm-slim ARG CF_CLI_VERSION=v0.1.70 ARG TARGETARCH @@ -9,4 +9,4 @@ COPY --from=bitnami/kubectl:1.32.1 /opt/bitnami/kubectl/bin/kubectl /usr/local/b RUN adduser --shell /bin/bash codefresh USER codefresh -WORKDIR /home/codefresh +WORKDIR /home/codefresh \ No newline at end of file From fcc694f3a62a8318c42fe8f60db36f61c36c1bc7 Mon Sep 17 00:00:00 2001 From: Bohdan Pysarenko Date: Wed, 19 Feb 2025 15:51:45 +0200 Subject: [PATCH 07/11] set strict debian version --- installer-image/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/installer-image/Dockerfile b/installer-image/Dockerfile index bd091a13..efb6dd31 100644 --- a/installer-image/Dockerfile +++ b/installer-image/Dockerfile @@ -1,4 +1,5 @@ -FROM debian:bookworm-slim +#bullseye-slim +FROM debian:11.11-slim ARG CF_CLI_VERSION=v0.1.70 ARG TARGETARCH From 21ce1212c160c690ccc6a91c5a198719bcb274f7 Mon Sep 17 00:00:00 2001 From: Bohdan Pysarenko Date: Wed, 19 Feb 2025 16:15:49 +0200 Subject: [PATCH 08/11] revert docker image --- installer-image/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installer-image/Dockerfile b/installer-image/Dockerfile index efb6dd31..6682eafe 100644 --- a/installer-image/Dockerfile +++ b/installer-image/Dockerfile @@ -1,5 +1,5 @@ #bullseye-slim -FROM debian:11.11-slim +FROM debian:11.10-slim ARG CF_CLI_VERSION=v0.1.70 ARG TARGETARCH From c4bd1b3524b3ad9ce6ae62de3565380bec42d2e9 Mon Sep 17 00:00:00 2001 From: Bohdan Pysarenko Date: Wed, 19 Feb 2025 18:42:38 +0200 Subject: [PATCH 09/11] change base image --- installer-image/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/installer-image/Dockerfile b/installer-image/Dockerfile index 6682eafe..00a801be 100644 --- a/installer-image/Dockerfile +++ b/installer-image/Dockerfile @@ -1,5 +1,5 @@ -#bullseye-slim -FROM debian:11.10-slim +#bookworm-slim +FROM debian:12.9-slim ARG CF_CLI_VERSION=v0.1.70 ARG TARGETARCH From 3b48a48ae4d048ce958ca9abe0acf27bc75e1888 Mon Sep 17 00:00:00 2001 From: Bohdan Pysarenko Date: Thu, 20 Feb 2025 10:43:49 +0200 Subject: [PATCH 10/11] fix comments --- .../templates/_components/cap-app-proxy/_config.yaml | 4 ++-- installer-image/Dockerfile | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/gitops-runtime/templates/_components/cap-app-proxy/_config.yaml b/charts/gitops-runtime/templates/_components/cap-app-proxy/_config.yaml index 59a89cdc..22a4b3b0 100644 --- a/charts/gitops-runtime/templates/_components/cap-app-proxy/_config.yaml +++ b/charts/gitops-runtime/templates/_components/cap-app-proxy/_config.yaml @@ -4,8 +4,8 @@ argoCdUsername: {{ .Values.config.argoCdUsername }} argoWorkflowsInsecure: {{ .Values.config.argoWorkflowsInsecure | quote }} argoWorkflowsUrl: {{ default "" .Values.config.argoWorkflowsUrl }} cors: {{ .Values.global.codefresh.url }} - {{- if ((.Values.config).clusterChunkSize) }} -clusterChunkSize: {{ .Values.config.clusterChunkSize | quote }} + {{- with .Values.config.clusterChunkSize }} +clusterChunkSize: {{ . | quote }} {{- end }} env: {{ .Values.config.env | quote}} isConfigurationRuntime: {{ .Values.global.runtime.isConfigurationRuntime | quote }} diff --git a/installer-image/Dockerfile b/installer-image/Dockerfile index 00a801be..5e69fc19 100644 --- a/installer-image/Dockerfile +++ b/installer-image/Dockerfile @@ -10,4 +10,4 @@ COPY --from=bitnami/kubectl:1.32.1 /opt/bitnami/kubectl/bin/kubectl /usr/local/b RUN adduser --shell /bin/bash codefresh USER codefresh -WORKDIR /home/codefresh \ No newline at end of file +WORKDIR /home/codefresh From 3c28b4508fd474c586b4c7721c4652ca622e60d4 Mon Sep 17 00:00:00 2001 From: Bohdan Pysarenko Date: Thu, 20 Feb 2025 15:28:09 +0200 Subject: [PATCH 11/11] fix --- .../templates/_components/cap-app-proxy/_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/gitops-runtime/templates/_components/cap-app-proxy/_config.yaml b/charts/gitops-runtime/templates/_components/cap-app-proxy/_config.yaml index 22a4b3b0..9beeca42 100644 --- a/charts/gitops-runtime/templates/_components/cap-app-proxy/_config.yaml +++ b/charts/gitops-runtime/templates/_components/cap-app-proxy/_config.yaml @@ -29,7 +29,7 @@ enrichmentJiraEnrichmentImage: {{ printf "%s/%s:%s" $enrichmentValues.config.ima {{- define "cap-app-proxy.resources.configmap" }} {{- $documentedConfigs := (include "cap-app-proxy.resources.configmap-documented-configs" . | fromYaml ) }} -{{- $overrides := .Values.config }} +{{- $overrides := omit .Values.config "clusterChunkSize" }} {{- $mergedConfig := mergeOverwrite $documentedConfigs $overrides }} apiVersion: v1 kind: ConfigMap