Sje/hotfix multiple registries

.werft/workspace-run-integration-tests.yaml

@@ -160,13 +160,14 @@ pod:
           cd "${TEST_PATH}"
           set +e
           go test -v ./... "${args[@]}" 2>&1 | tee "${TEST_NAME}".log | werft log slice "test-${TEST_NAME}"
+          RC=${PIPESTATUS[0]}
           set -e
 
           RUN_COUNT=$((RUN_COUNT+1))
-          if [ "${PIPESTATUS[0]}" -ne "0" ]; then
+          if [ "${RC}" -ne "0" ]; then
             FAILURE_COUNT=$((FAILURE_COUNT+1))
             FAILURE_TESTS["${TEST_NAME}"]=$(grep "\-\-\- FAIL: " "${TEST_PATH}"/"${TEST_NAME}".log)
-            werft log slice "test-${TEST_NAME}" --fail "${PIPESTATUS[0]}"
+            werft log slice "test-${TEST_NAME}" --fail "${RC}"
           else
             werft log slice "test-${TEST_NAME}" --done
           fi

install/installer/cmd/mirror_list.go

@@ -113,7 +113,9 @@ func renderAllKubernetesObject(cfgVersion string, cfg *configv1.Config) ([]strin
 					},
 				},
 				S3Storage: &configv1.S3Storage{
-					Bucket: "some-bucket",
+					Bucket:   "some-bucket",
+					Region:   "some-region",
+					Endpoint: "some-url",
 					Certificate: configv1.ObjectRef{
 						Kind: configv1.ObjectRefSecret,
 						Name: "value",

install/installer/leeway.Dockerfile

@@ -5,7 +5,7 @@
 FROM alpine:3.15
 COPY --from=alpine/helm:3.8.0 /usr/bin/helm /usr/bin/helm
 COPY install-installer--app/installer install-installer--app/provenance-bundle.jsonl /app/
-RUN apk add --no-cache curl jq yq  \
+RUN apk add --no-cache curl jq openssh-keygen yq  \
     && curl -L "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" -o /usr/local/bin/kubectl \
     && chmod +x /usr/local/bin/kubectl
 ENTRYPOINT [ "/app/installer" ]

install/installer/pkg/components/docker-registry/helm.go

@@ -43,18 +43,23 @@ var Helm = common.CompositeHelmFunc(
 
 		inCluster := pointer.BoolDeref(cfg.Config.ContainerRegistry.InCluster, false)
 		s3Storage := cfg.Config.ContainerRegistry.S3Storage
+		enablePersistence := "true"
 
 		if inCluster && s3Storage != nil {
+			enablePersistence = "false"
 			registryValues = append(registryValues,
-				helm.KeyValue("docker-registry.s3.region", cfg.Config.Metadata.Region),
+				helm.KeyValue("docker-registry.s3.region", s3Storage.Region),
 				helm.KeyValue("docker-registry.s3.bucket", s3Storage.Bucket),
+				helm.KeyValue("docker-registry.s3.regionEndpoint", s3Storage.Endpoint),
 				helm.KeyValue("docker-registry.s3.encrypt", "true"),
 				helm.KeyValue("docker-registry.s3.secure", "true"),
 				helm.KeyValue("docker-registry.storage", "s3"),
 				helm.KeyValue("docker-registry.secrets.s3.secretRef", s3Storage.Certificate.Name),
 			)
 		}
 
+		registryValues = append(registryValues, helm.KeyValue("docker-registry.persistence.enabled", enablePersistence))
+
 		return &common.HelmConfig{
 			Enabled: inCluster,
 			Values: &values.Options{

install/installer/pkg/config/v1/config.go

@@ -214,6 +214,8 @@ type ContainerRegistryExternal struct {
 
 type S3Storage struct {
 	Bucket      string    `json:"bucket" validate:"required"`
+	Region      string    `json:"region" validate:"required"`
+	Endpoint    string    `json:"endpoint" validate:"required"`
 	Certificate ObjectRef `json:"certificate" validate:"required"`
 }
 

install/installer/third_party/charts/docker-registry/values.yaml

@@ -2,6 +2,4 @@
 # Licensed under the GNU Affero General Public License (AGPL).
 # See License-AGPL.txt in the project root for license information.
 
-docker-registry:
-  persistence:
-    enabled: true
+docker-registry: {}

install/installer/third_party/charts/minio/Chart.yaml

@@ -8,5 +8,5 @@ name: minio
 version: 1.0.0
 dependencies:
   - name: minio
-    version: 9.0.6
-    repository: https://charts.bitnami.com/bitnami
+    version: 11.6.3
+    repository: https://charts.bitnami.com/bitnami

install/installer/third_party/charts/mysql/Chart.yaml

@@ -8,5 +8,5 @@ name: mysql
 version: 1.0.0
 dependencies:
   - name: mysql
-    version: 8.6.2
-    repository: https://charts.bitnami.com/bitnami
+    version: 9.1.2
+    repository: https://charts.bitnami.com/bitnami

install/installer/third_party/charts/rabbitmq/Chart.yaml

@@ -8,5 +8,5 @@ name: rabbitmq
 version: 1.0.0
 dependencies:
   - name: rabbitmq
-    version: 8.24.6
+    version: 10.1.1
     repository: https://charts.bitnami.com/bitnami

install/kots/manifests/gitpod-installation-status.yaml

@@ -30,7 +30,7 @@ spec:
       containers:
         - name: installation-status
           # This will normally be the release tag
-          image: "eu.gcr.io/gitpod-core-dev/build/installer:release-2022.04.1.2"
+          image: "eu.gcr.io/gitpod-core-dev/build/installer:sje-registry-s3-fork.3"
           command:
             - /bin/sh
             - -c

install/kots/manifests/gitpod-installer-job.yaml

@@ -28,7 +28,7 @@ spec:
       containers:
         - name: installer
           # This will normally be the release tag
-          image: "eu.gcr.io/gitpod-core-dev/build/installer:release-2022.04.1.2"
+          image: "eu.gcr.io/gitpod-core-dev/build/installer:sje-registry-s3-fork.3"
           volumeMounts:
             - mountPath: /config-patch
               name: config-patch
@@ -146,18 +146,36 @@ spec:
               then
                 echo "Gitpod: configuring mirrored container registry"
 
-                yq e -i ".containerRegistry.inCluster = false" "${CONFIG_FILE}"
-                yq e -i ".containerRegistry.external.url = \"{{repl LocalRegistryAddress }}\"" "${CONFIG_FILE}"
-                yq e -i ".containerRegistry.external.certificate.kind = \"secret\"" "${CONFIG_FILE}"
-                yq e -i ".containerRegistry.external.certificate.name = \"{{repl ImagePullSecretName }}\"" "${CONFIG_FILE}"
                 yq e -i ".repository = \"{{repl LocalRegistryAddress }}\"" "${CONFIG_FILE}"
                 yq e -i ".imagePullSecrets[0].kind = \"secret\"" "${CONFIG_FILE}"
                 yq e -i ".imagePullSecrets[0].name = \"{{repl ImagePullSecretName }}\"" "${CONFIG_FILE}"
                 yq e -i '.dropImageRepo = true' "${CONFIG_FILE}"
-              elif [ '{{repl ConfigOptionEquals "reg_incluster" "0" }}' = "true" ];
+              fi
+
+              # Output the local registry secret - this is proxy.replicated.com if user hasn't set their own
+              echo "{{repl LocalRegistryImagePullSecret }}" | base64 -d > /tmp/kotsregistry.json
+
+              # Add the registries to the server allowlist
+              yq e -i ".experimental.webApp.server.defaultBaseImageRegistryWhitelist += $(cat /tmp/kotsregistry.json | jq '.auths' | jq -rc 'keys')" "${CONFIG_FILE}"
+
+              if [ '{{repl ConfigOptionEquals "reg_incluster" "0" }}' = "true" ];
               then
                 echo "Gitpod: configuring external container registry"
 
+                # Create a container-registry secret merging the external registry and KOTS registry keys
+                echo '{{repl printf "{\"auths\": {\"%s\": {\"username\": \"%s\", \"password\": %s, \"auth\": \"%s\"}}}" (ConfigOption "reg_server" | default (ConfigOption "reg_url")) (ConfigOption "reg_username") (ConfigOption "reg_password" | toJson) (printf "%s:%s" (ConfigOption "reg_username") (ConfigOption "reg_password") | Base64Encode) }}' \
+                  | yq -o=json '.' - \
+                  > /tmp/gitpodregistry.json
+
+                cat /tmp/kotsregistry.json /tmp/gitpodregistry.json | jq -s '.[0] * .[1]' - - > /tmp/container-registry-secret
+
+                echo "Gitpod: create the container-registry secret"
+                kubectl create secret docker-registry container-registry \
+                  --namespace "{{repl Namespace }}" \
+                  --from-file=.dockerconfigjson=/tmp/container-registry-secret \
+                  -o yaml --dry-run=client | \
+                  kubectl replace --namespace "{{repl Namespace }}" --force -f -
+
                 yq e -i ".containerRegistry.inCluster = false" "${CONFIG_FILE}"
                 yq e -i ".containerRegistry.external.url = \"{{repl ConfigOption "reg_url" }}\"" "${CONFIG_FILE}"
                 yq e -i ".containerRegistry.external.certificate.kind = \"secret\"" "${CONFIG_FILE}"
@@ -167,6 +185,8 @@ spec:
                 then
                   echo "Gitpod: configuring container registry S3 backend"
 
+                  yq e -i ".containerRegistry.s3storage.region = \"{{repl ConfigOption "reg_incluster_storage_s3_region" }}\"" "${CONFIG_FILE}"
+                  yq e -i ".containerRegistry.s3storage.endpoint = \"{{repl ConfigOption "reg_incluster_storage_s3_endpoint" }}\"" "${CONFIG_FILE}"
                   yq e -i ".containerRegistry.s3storage.bucket = \"{{repl ConfigOption "reg_incluster_storage_s3_bucketname" }}\"" "${CONFIG_FILE}"
                   yq e -i ".containerRegistry.s3storage.certificate.kind = \"secret\"" "${CONFIG_FILE}"
                   yq e -i ".containerRegistry.s3storage.certificate.name = \"container-registry-s3-backend\"" "${CONFIG_FILE}"
@@ -211,7 +231,6 @@ spec:
               if [ '{{repl ConfigOptionEquals "ssh_gateway" "1" }}' = "true" ];
               then
                 echo "Gitpod: Generate SSH host key"
-                apk update && apk add --no-cache openssh-keygen # TODO: Move installation of openssh-keygen to installer image
                 ssh-keygen -t rsa -q -N "" -f host.key
                 kubectl create secret generic ssh-gateway-host-key --from-file=host.key -n {{repl Namespace }} || echo "SSH Gateway Host Key secret has not been created. Does it exist already?"
                 yq e -i '.sshGatewayHostKey.kind = "secret"' "${CONFIG_FILE}"
@@ -264,14 +283,31 @@ spec:
               EOF
 
               echo "Gitpod: render Kubernetes manifests"
-              /app/installer render -c "${CONFIG_FILE}" --namespace {{repl Namespace }} > "${GITPOD_OBJECTS}/templates/gitpod.yaml"
+              /app/installer render -c "${CONFIG_FILE}" --namespace {{repl Namespace }} --use-experimental-config > "${GITPOD_OBJECTS}/templates/gitpod.yaml"
 
               # Workaround for #8532 and #8529
               echo "Gitpod: Remove the StatefulSet status object for OpenVSX Proxy"
               yq eval-all --inplace \
                 'del(select(.kind == "StatefulSet" and .metadata.name == "openvsx-proxy").status)' \
                 "${GITPOD_OBJECTS}/templates/gitpod.yaml"
 
+              if [ '{{repl ConfigOptionEquals "reg_incluster" "1" }}' = "true" ];
+              then
+                echo "Gitpod: Add the local registry secret to the in-cluster registry secret"
+
+                # Get the in-cluster registry secret
+                yq eval-all '(select(.kind == "Secret" and .metadata.name == "builtin-registry-auth") | .data.".dockerconfigjson")' \
+                  "${GITPOD_OBJECTS}/templates/gitpod.yaml" \
+                  | base64 -d \
+                  > /tmp/gitpodregistry.json
+
+                export REGISTRY_SECRET=$(cat /tmp/kotsregistry.json /tmp/gitpodregistry.json | jq -s '.[0] * .[1]' - - | base64 -w 0)
+
+                echo "Gitpod: update the in-cluster registry secret"
+                yq eval-all --inplace '(select(.kind == "Secret" and .metadata.name == "builtin-registry-auth") | .data.".dockerconfigjson") |= env(REGISTRY_SECRET)' \
+                  "${GITPOD_OBJECTS}/templates/gitpod.yaml"
+              fi
+
               echo "Gitpod: Escape any Golang template values"
               sed -i -r 's/(.*\{\{.*)/{{`\1`}}/' "${GITPOD_OBJECTS}/templates/gitpod.yaml"
 

install/kots/manifests/kots-config.yaml

@@ -28,7 +28,6 @@ spec:
         - name: reg_incluster
           title: Use in-cluster container registry
           type: bool
-          when: '{{repl eq HasLocalRegistry false }}'
           default: "1"
           help_text: You may either use an in-cluster container registry or configure your own external container registry for better performance. This container registry must be accessible from your Kubernetes cluster.
           recommended: false
@@ -45,6 +44,22 @@ spec:
             - name: s3
               title: S3
 
+        # S3 help_text
+        - name: reg_incluster_storage_s3_region
+          title: Storage region
+          type: text
+          required: true
+          when: '{{repl (ConfigOptionEquals "reg_incluster_storage" "s3") }}'
+          help_text: ID of the region where your storage exists, such as `eu-west-2`.
+
+        - name: reg_incluster_storage_s3_endpoint
+          title: Endpoint
+          type: text
+          required: true
+          value: s3.amazonaws.com
+          when: '{{repl (ConfigOptionEquals "reg_incluster_storage" "s3") }}'
+          help_text: The endpoint used to connect to the S3 storage.
+
         - name: reg_incluster_storage_s3_bucketname
           title: S3 bucket name
           type: text
@@ -69,27 +84,27 @@ spec:
         - name: reg_url
           title: Container registry URL
           type: text
-          when: '{{repl and (eq HasLocalRegistry false) (ConfigOptionEquals "reg_incluster" "0") }}'
+          when: '{{repl ConfigOptionEquals "reg_incluster" "0" }}'
           required: true
           help_text: The container registry URL. This will usually be the fully qualified domain of your registry.
 
         - name: reg_server
           title: Container registry server
           type: text
-          when: '{{repl and (eq HasLocalRegistry false) (ConfigOptionEquals "reg_incluster" "0") }}'
+          when: '{{repl ConfigOptionEquals "reg_incluster" "0" }}'
           help_text: The container registry server. This is used when [generating your credentials](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/#create-a-secret-by-providing-credentials-on-the-command-line). Depending upon your provider, this may or may not be the same as the registry URL. If not specified, the URL will be used.
 
         - name: reg_username
           title: Container registry username
           type: text
-          when: '{{repl and (eq HasLocalRegistry false) (ConfigOptionEquals "reg_incluster" "0") }}'
+          when: '{{repl ConfigOptionEquals "reg_incluster" "0" }}'
           required: true
           help_text: The username for your container registry.
 
         - name: reg_password
           title: Container registry password
           type: password
-          when: '{{repl and (eq HasLocalRegistry false) (ConfigOptionEquals "reg_incluster" "0") }}'
+          when: '{{repl ConfigOptionEquals "reg_incluster" "0" }}'
           required: true
           help_text: The password for your container registry.
 

test/tests/workspace/cgroup_v2_test.go

@@ -13,13 +13,13 @@ import (
 	"testing"
 	"time"
 
+	"github.com/google/go-cmp/cmp"
 	"sigs.k8s.io/e2e-framework/pkg/envconf"
 	"sigs.k8s.io/e2e-framework/pkg/features"
 
 	agent "github.com/gitpod-io/gitpod/test/pkg/agent/workspace/api"
 	"github.com/gitpod-io/gitpod/test/pkg/integration"
-	"github.com/gitpod-io/gitpod/test/tests/workspace/common"
-	"github.com/google/go-cmp/cmp"
+	"github.com/gitpod-io/gitpod/test/pkg/integration/common"
 )
 
 func TestCgroupV2(t *testing.T) {

test/tests/workspace/contexts_test.go

@@ -46,7 +46,7 @@ func TestGitHubContexts(t *testing.T) {
 			ContextURL:    "github.com/gitpod-io/gitpod-test-repo/issues/88",
 			WorkspaceRoot: "/workspace/gitpod-test-repo",
 			ExpectedBranchFunc: func(username string) string {
-				return fmt.Sprintf("%s/integration-tests-test-context-88", username)
+				return fmt.Sprintf("%s/integration-88", username)
 			},
 		},
 		{
@@ -108,7 +108,7 @@ func runContextTests(t *testing.T, tests []ContextTest) {
 						t.SkipNow()
 					}
 
-					t.Parallel()
+					// t.Parallel()
 
 					ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
 					defer cancel()
@@ -129,11 +129,6 @@ func runContextTests(t *testing.T, tests []ContextTest) {
 					}
 					defer stopWS(false) // we do not wait for stopped here as it does not matter for this test case and speeds things up
 
-					_, err = integration.WaitForWorkspaceStart(ctx, nfo.LatestInstance.ID, api)
-					if err != nil {
-						t.Fatal(err)
-					}
-
 					rsa, closer, err := integration.Instrument(integration.ComponentWorkspace, "workspace", cfg.Namespace(), kubeconfig, cfg.Client(), integration.WithInstanceID(nfo.LatestInstance.ID))
 					if err != nil {
 						t.Fatal(err)
@@ -157,7 +152,7 @@ func runContextTests(t *testing.T, tests []ContextTest) {
 						expectedBranch = test.ExpectedBranchFunc(username)
 					}
 					if actBranch != expectedBranch {
-						t.Fatalf("expected branch '%s', got '%s'!", test.ExpectedBranch, actBranch)
+						t.Fatalf("expected branch '%s', got '%s'!", expectedBranch, actBranch)
 					}
 				})
 			}

test/tests/workspace/git_test.go

@@ -152,11 +152,6 @@ func TestGitActions(t *testing.T) {
 
 					defer stopWS(false)
 
-					_, err = integration.WaitForWorkspaceStart(ctx, nfo.LatestInstance.ID, api)
-					if err != nil {
-						t.Fatal(err)
-					}
-
 					rsa, closer, err := integration.Instrument(integration.ComponentWorkspace, "workspace", cfg.Namespace(), kubeconfig, cfg.Client(), integration.WithInstanceID(nfo.LatestInstance.ID))
 					if err != nil {
 						t.Fatal(err)

test/tests/workspace/k3s_test.go

@@ -14,7 +14,7 @@ import (
 
 	agent "github.com/gitpod-io/gitpod/test/pkg/agent/workspace/api"
 	"github.com/gitpod-io/gitpod/test/pkg/integration"
-	"github.com/gitpod-io/gitpod/test/tests/workspace/common"
+	"github.com/gitpod-io/gitpod/test/pkg/integration/common"
 )
 
 func TestK3s(t *testing.T) {