Add self-hosted upgrade tests

nandajavarma · nandajavarma · commit bd3127524eb2 · 2022-06-06T16:25:35.000Z
diff --git a/.werft/build.ts b/.werft/build.ts
@@ -9,6 +9,7 @@ import { validateChanges } from './jobs/build/validate-changes';
 import { prepare } from './jobs/build/prepare';
 import { deployToPreviewEnvironment } from './jobs/build/deploy-to-preview-environment';
 import { triggerIntegrationTests } from './jobs/build/trigger-integration-tests';
+import { triggerUpgradeTests } from './jobs/build/self-hosted-upgrade-tests';
 import { jobConfig } from './jobs/build/job-config';
 import { typecheckWerftJobs } from './jobs/build/typecheck-werft-jobs';
 
@@ -72,4 +73,7 @@ async function run(context: any) {
     }
 
     await triggerIntegrationTests(werft, config, context.Owner)
+
+    // this will trigger an upgrade test on a self-hosted gitpod instance on a new cluster.
+    await triggerUpgradeTests(werft, config, context.Owner)
 }
diff --git a/.werft/installer-tests.ts b/.werft/installer-tests.ts
@@ -5,6 +5,8 @@ import { Werft } from "./util/werft";
 const testConfig: string = process.argv.length > 2 ? process.argv[2] : "STANDARD_K3S_TEST";
 // we can provide the version of the gitpod to install (eg: 2022.4.2)
 const version: string = process.argv.length > 3 ? process.argv[3] : "";
+// we are assuming that if a version is provided, this is to test upgrade to the latest
+const channel: string = version != "" ? "beta" : "unstable";
 
 const makefilePath: string = join("install/tests");
 
@@ -43,12 +45,12 @@ const INFRA_PHASES: { [name: string]: InfraConfig } = {
     },
     INSTALL_GITPOD_IGNORE_PREFLIGHTS: {
         phase: "install-gitpod-without-preflights",
-        makeTarget: `kots-install channel=unstable version=${version} preflights=false`, // this is a bit of a hack, for now we pass params like this
+        makeTarget: `kots-install channel=${channel} version=${version} preflights=false`, // this is a bit of a hack, for now we pass params like this
         description: "Install gitpod using kots community edition without preflights",
     },
     INSTALL_GITPOD: {
         phase: "install-gitpod",
-        makeTarget: `kots-install channel=unstable version=${version} preflights=true`,
+        makeTarget: `kots-install channel=${channel} version=${version} preflights=true`,
         description: "Install gitpod using kots community edition",
     },
     CHECK_INSTALLATION: {
@@ -57,6 +59,11 @@ const INFRA_PHASES: { [name: string]: InfraConfig } = {
         makeTarget: "check-gitpod-installation",
         description: "Check gitpod installation",
     },
+    KOTS_UPGRADE: {
+        phase: "kots-upgrade",
+        makeTarget: "kots-uprgade",
+        description: "Upgrade Gitpod installation to latest version using KOTS CLI",
+    },
     RUN_INTEGRATION_TESTS: {
         phase: "run-integration-tests",
         makeTarget: "run-tests",
@@ -95,6 +102,19 @@ const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = {
             "DESTROY",
         ],
     },
+    STANDARD_GKE_UPGRADE_TEST: {
+        DESCRIPTION: `Deploy Gitpod on GKE, and test upgrade from ${version} to latest version`,
+        PHASES: [
+            "STANDARD_GKE_CLUSTER",
+            "CERT_MANAGER",
+            "GCP_MANAGED_DNS",
+            "INSTALL_GITPOD",
+            "CHECK_INSTALLATION",
+            "KOTS_UPGRADE",
+            "CHECK_INSTALLATION",
+            "DESTROY",
+        ],
+    },
     STANDARD_K3S_TEST: {
         DESCRIPTION:
             "Deploy Gitpod on a K3s cluster, created on a GCP instance," +
diff --git a/.werft/jobs/build/job-config.ts b/.werft/jobs/build/job-config.ts
@@ -3,32 +3,34 @@ import { Werft } from "../../util/werft";
 import { previewNameFromBranchName } from "../../util/preview";
 
 export interface JobConfig {
-    analytics: string
+    analytics: string;
     buildConfig: any;
-    cleanSlateDeployment: boolean
-    coverageOutput: string
+    cleanSlateDeployment: boolean;
+    coverageOutput: string;
     dontTest: boolean;
     dynamicCPULimits: boolean;
-    installEELicense: boolean
-    localAppVersion: string
+    installEELicense: boolean;
+    localAppVersion: string;
     mainBuild: boolean;
     noPreview: boolean;
     publishRelease: boolean;
-    publishToJBMarketplace: string
-    publishToNpm: string
+    publishToJBMarketplace: string;
+    publishToNpm: string;
     publishToKots: boolean;
-    retag: string
+    retag: string;
     storage: string;
     version: string;
-    withContrib: boolean
+    withContrib: boolean;
     withIntegrationTests: boolean;
-    withObservability: boolean
-    withPayment: boolean
-    withVM: boolean
+    withUpgradeTests: boolean;
+    fromVersion: string;
+    withObservability: boolean;
+    withPayment: boolean;
+    withVM: boolean;
     workspaceFeatureFlags: string[];
-    previewEnvironment: PreviewEnvironmentConfig,
-    repository: Repository
-    observability: Observability
+    previewEnvironment: PreviewEnvironmentConfig;
+    repository: Repository;
+    observability: Observability;
 }
 
 export interface PreviewEnvironmentConfig {
@@ -45,14 +47,14 @@ export interface Repository {
 
 export interface Observability {
     // The branch of gitpod-io/observability to use
-    branch: string
+    branch: string;
 }
 
 export function jobConfig(werft: Werft, context: any): JobConfig {
-    const sliceId = 'Parsing job configuration'
-    werft.phase('Job configuration')
-    werft.log(sliceId , "Parsing the job configuration")
-    const version = parseVersion(context)
+    const sliceId = "Parsing job configuration";
+    werft.phase("Job configuration");
+    werft.log(sliceId, "Parsing the job configuration");
+    const version = parseVersion(context);
     const repo = `${context.Repository.host}/${context.Repository.owner}/${context.Repository.repo}`;
     const mainBuild = repo === "github.com/gitpod-io/gitpod" && context.Repository.ref.includes("refs/heads/main");
 
@@ -64,7 +66,7 @@ export function jobConfig(werft: Werft, context: any): JobConfig {
         if (!raw) {
             return [];
         }
-        return raw.split(",").map(e => e.trim());
+        return raw.split(",").map((e) => e.trim());
     })();
 
     const coverageOutput = exec("mktemp -d", { silent: true }).stdout.trim();
@@ -76,13 +78,15 @@ export function jobConfig(werft: Werft, context: any): JobConfig {
     const noPreview = ("no-preview" in buildConfig && buildConfig["no-preview"] !== "false") || publishRelease;
     const storage = buildConfig["storage"] || "";
     const withIntegrationTests = "with-integration-tests" in buildConfig && !mainBuild;
+    const withUpgradeTests = "with-upgrade-tests" in buildConfig && !mainBuild;
+    const fromVersion = withUpgradeTests ? buildConfig["from-version"] : "";
     const publishToNpm = "publish-to-npm" in buildConfig || mainBuild;
     const publishToJBMarketplace = "publish-to-jb-marketplace" in buildConfig || mainBuild;
     const publishToKots = "publish-to-kots" in buildConfig;
     const analytics = buildConfig["analytics"];
-    const localAppVersion = mainBuild || ("with-localapp-version" in buildConfig) ? version : "unknown";
-    const retag = ("with-retag" in buildConfig) ? "" : "--dont-retag";
-    const cleanSlateDeployment = mainBuild || ("with-clean-slate-deployment" in buildConfig);
+    const localAppVersion = mainBuild || "with-localapp-version" in buildConfig ? version : "unknown";
+    const retag = "with-retag" in buildConfig ? "" : "--dont-retag";
+    const cleanSlateDeployment = mainBuild || "with-clean-slate-deployment" in buildConfig;
     const installEELicense = !("without-ee-license" in buildConfig) || mainBuild;
     const withPayment = "with-payment" in buildConfig && !mainBuild;
     const withObservability = "with-observability" in buildConfig && !mainBuild;
@@ -91,24 +95,24 @@ export function jobConfig(werft: Werft, context: any): JobConfig {
         repo: context.Repository.repo,
         ref: context.Repository.ref,
         branch: context.Repository.ref,
-    }
+    };
     const refsPrefix = "refs/heads/";
     if (repository.branch.startsWith(refsPrefix)) {
         repository.branch = repository.branch.substring(refsPrefix.length);
     }
     const withoutVM = "without-vm" in buildConfig;
     const withVM = !withoutVM || mainBuild;
 
-    const previewName = previewNameFromBranchName(repository.branch)
+    const previewName = previewNameFromBranchName(repository.branch);
     const previewEnvironmentNamespace = withVM ? `default` : `staging-${previewName}`;
     const previewEnvironment = {
         destname: previewName,
-        namespace: previewEnvironmentNamespace
-    }
+        namespace: previewEnvironmentNamespace,
+    };
 
     const observability: Observability = {
-        branch: context.Annotations.withObservabilityBranch || "main"
-    }
+        branch: context.Annotations.withObservabilityBranch || "main",
+    };
 
     const jobConfig = {
         analytics,
@@ -137,19 +141,21 @@ export function jobConfig(werft: Werft, context: any): JobConfig {
         withPayment,
         withVM,
         workspaceFeatureFlags,
-    }
+    };
 
     werft.log("job config", JSON.stringify(jobConfig));
-    const globalAttributes = Object.fromEntries(Object.entries(jobConfig).map((kv) => {
-        const [key, value] = kv
-        return [`werft.job.config.${key}`, value]
-    }))
-    globalAttributes['werft.job.config.branch'] = context.Repository.ref
-    werft.addAttributes(globalAttributes)
+    const globalAttributes = Object.fromEntries(
+        Object.entries(jobConfig).map((kv) => {
+            const [key, value] = kv;
+            return [`werft.job.config.${key}`, value];
+        }),
+    );
+    globalAttributes["werft.job.config.branch"] = context.Repository.ref;
+    werft.addAttributes(globalAttributes);
 
-    werft.done(sliceId)
+    werft.done(sliceId);
 
-    return jobConfig
+    return jobConfig;
 }
 
 function parseVersion(context: any) {
@@ -163,5 +169,5 @@ function parseVersion(context: any) {
     if (version.substr(0, PREFIX_TO_STRIP.length) === PREFIX_TO_STRIP) {
         version = version.substr(PREFIX_TO_STRIP.length);
     }
-    return version
+    return version;
 }
diff --git a/.werft/jobs/build/self-hosted-upgrade-tests.ts b/.werft/jobs/build/self-hosted-upgrade-tests.ts
@@ -0,0 +1,34 @@
+import { exec } from "../../util/shell";
+import { Werft } from "../../util/werft";
+import { JobConfig } from "./job-config";
+
+const phases = {
+    TRIGGER_UPGRADE_TESTS: "trigger self-hosted upgrade tests",
+};
+
+/**
+ * Trigger self hosted upgrade tests
+ */
+export async function triggerUpgradeTests(werft: Werft, config: JobConfig, username: string) {
+    werft.phase(phases.TRIGGER_UPGRADE_TESTS, "Trigger upgrade tests on self-hosted gitpod");
+
+    if (!config.withUpgradeTests || !config.fromVersion) {
+        werft.log(phases.TRIGGER_UPGRADE_TESTS, "Skipped upgrade tests");
+        werft.done(phases.TRIGGER_UPGRADE_TESTS);
+        return;
+    }
+
+    try {
+        const annotation = `-a fromVersion=${config.fromVersion}`;
+        exec(`werft run --remote-job-path .werft/run-sh-upgrade-tests-gke.yaml ${annotation} github`, {
+            slice: phases.TRIGGER_UPGRADE_TESTS,
+        }).trim();
+
+        werft.done(phases.TRIGGER_UPGRADE_TESTS);
+    } catch (err) {
+        if (!config.mainBuild) {
+            werft.fail(phases.TRIGGER_UPGRADE_TESTS, err);
+        }
+        exec("exit 0");
+    }
+}
diff --git a/.werft/run-sh-upgrade-tests-gke.yaml b/.werft/run-sh-upgrade-tests-gke.yaml
@@ -0,0 +1,68 @@
+# debug using `werft run github -f -s .werft/installer-tests.ts -j .werft/run-sh-upgrade-tests-gke.yaml -a debug=true`
+pod:
+  serviceAccount: werft
+  affinity:
+  nodeAffinity:
+    requiredDuringSchedulingIgnoredDuringExecution:
+      nodeSelectorTerms:
+      - matchExpressions:
+        - key: dev/workload
+          operator: In
+          values:
+          - "builds"
+  securityContext:
+    runAsUser: 0
+  volumes:
+  - name: sh-playground-sa-perm
+    secret:
+      secretName: sh-playground-sa-perm
+  - name: sh-playground-dns-perm
+    secret:
+      secretName: sh-playground-dns-perm
+  containers:
+  - name: nightly-test
+    image: eu.gcr.io/gitpod-core-dev/dev/dev-environment:cw-werft-cred.0
+    workingDir: /workspace
+    imagePullPolicy: Always
+    volumeMounts:
+    - name: sh-playground-sa-perm
+      mountPath: /mnt/secrets/sh-playground-sa-perm
+    - name: sh-playground-dns-perm # this sa is used for the DNS management
+      mountPath: /mnt/secrets/sh-playground-dns-perm
+    env:
+    - name: WERFT_HOST
+      value: "werft.werft.svc.cluster.local:7777"
+    - name: GOOGLE_APPLICATION_CREDENTIALS
+      value: "/mnt/secrets/sh-playground-sa-perm/sh-sa.json"
+    - name: TF_VAR_sa_creds
+      value: "/mnt/secrets/sh-playground-sa-perm/sh-sa.json"
+    - name: TF_VAR_dns_sa_creds
+      value: "/mnt/secrets/sh-playground-dns-perm/sh-dns-sa.json"
+    - name: WERFT_K8S_NAMESPACE
+      value: "werft"
+    - name: WERFT_K8S_LABEL
+      value: "component=werft"
+    - name: NODENAME
+      valueFrom:
+        fieldRef:
+          fieldPath: spec.nodeName
+    command:
+      - bash
+      - -c
+      - |
+        sleep 1
+        set -Eeuo pipefail
+
+        sudo chown -R gitpod:gitpod /workspace
+        sudo apt update && apt install gettext-base
+
+        export TF_VAR_TEST_ID=$(echo $RANDOM | md5sum | head -c 5; echo)
+
+        (cd .werft && yarn install && mv node_modules ..) | werft log slice prep
+        printf '{{ toJson . }}' > context.json
+
+        FROM_VERSION="{{ .Annotations.fromVersion }}"
+        npx ts-node .werft/installer-tests.ts "STANDARD_GKE_UPGRADE_TEST" ${FROM_VERSION}
+# The bit below makes this a cron job
+# plugins:
+#   cron: "15 4 * * *"
diff --git a/install/infra/terraform/gke/main.tf b/install/infra/terraform/gke/main.tf
@@ -48,7 +48,7 @@ resource "google_container_cluster" "gitpod-cluster" {
     resource_limits {
       resource_type = "cpu"
       minimum       = 2
-      maximum       = 8
+      maximum       = 16
     }
 
     resource_limits {
@@ -58,7 +58,7 @@ resource "google_container_cluster" "gitpod-cluster" {
     }
   }
 
-  node_version = var.kubernetes_version
+  min_master_version = var.kubernetes_version
   # the default nodepool is used as the services nodepool
   remove_default_node_pool = false
   node_config {
diff --git a/install/tests/Makefile b/install/tests/Makefile
@@ -92,6 +92,10 @@ check-gitpod-installation: delete-cm-setup check-kots-app check-env-sub-domain
 run-tests:
 	./tests.sh ${KUBECONFIG}
 
+kots-uprgade:
+	@echo "Upgrade gitpod KOTS app to latest"
+	kubectl kots upstream upgrade --kubeconfig=${KUBECONFIG} gitpod -n gitpod --deploy
+
 cleanup:
 	terraform workspace select $(TF_VAR_TEST_ID)
 	which ${KUBECONFIG} && terraform destroy -target=module.externaldns -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file"

Original file line number	Diff line number	Diff line change
`@@ -9,6 +9,7 @@ import { validateChanges } from './jobs/build/validate-changes';`
`9`	`9`	`import { prepare } from './jobs/build/prepare';`
`10`	`10`	`import { deployToPreviewEnvironment } from './jobs/build/deploy-to-preview-environment';`
`11`	`11`	`import { triggerIntegrationTests } from './jobs/build/trigger-integration-tests';`
	`12`	`+import { triggerUpgradeTests } from './jobs/build/self-hosted-upgrade-tests';`
`12`	`13`	`import { jobConfig } from './jobs/build/job-config';`
`13`	`14`	`import { typecheckWerftJobs } from './jobs/build/typecheck-werft-jobs';`
`14`	`15`
`@@ -72,4 +73,7 @@ async function run(context: any) {`
`72`	`73`	`}`
`73`	`74`
`74`	`75`	`await triggerIntegrationTests(werft, config, context.Owner)`
	`76`	`+`
	`77`	`+ // this will trigger an upgrade test on a self-hosted gitpod instance on a new cluster.`
	`78`	`+ await triggerUpgradeTests(werft, config, context.Owner)`
`75`	`79`	`}`
Original file line number	Diff line number	Diff line change
`@@ -48,7 +48,7 @@ resource "google_container_cluster" "gitpod-cluster" {`
`48`	`48`	`resource_limits {`
`49`	`49`	`resource_type = "cpu"`
`50`	`50`	`minimum = 2`
`51`		`- maximum = 8`
	`51`	`+ maximum = 16`
`52`	`52`	`}`
`53`	`53`
`54`	`54`	`resource_limits {`
`@@ -58,7 +58,7 @@ resource "google_container_cluster" "gitpod-cluster" {`
`58`	`58`	`}`
`59`	`59`	`}`
`60`	`60`
`61`		`- node_version = var.kubernetes_version`
	`61`	`+ min_master_version = var.kubernetes_version`
`62`	`62`	`# the default nodepool is used as the services nodepool`
`63`	`63`	`remove_default_node_pool = false`
`64`	`64`	`node_config {`