[installer-tests] add werft flag to trigger self-hosted preview

Author: nandajavarma

.github/workflows/delete-kots-channel.yml

@@ -16,6 +16,17 @@ jobs:
           tar xf replicated.tar.gz replicated && rm replicated.tar.gz
           mv replicated /usr/local/bin/replicated
 
+      - name: Delete replicated license
+        run: |
+          CUSTOMER_ID=$(replicated customer ls | grep ${{ github.event.ref }} | awk '{print $1}')
+          if [ "${CUSTOMER_ID}" = "" ]; then
+            echo "No license found"
+            exit 0
+          fi
+          curl --request POST \
+              --url https://api.replicated.com/vendor/v3/customer/${CUSTOMER_ID}/archive \
+              --header 'Authorization: ${{ env.REPLICATED_API_TOKEN }}'
+
       - name: Delete Replicated channel
         run: |
           CHANNEL_ID=$(replicated channel inspect ${{ github.event.ref }} \

.werft/build.ts

@@ -9,7 +9,7 @@ import { validateChanges } from "./jobs/build/validate-changes";
 import { prepare } from "./jobs/build/prepare";
 import { deployToPreviewEnvironment } from "./jobs/build/deploy-to-preview-environment";
 import { triggerIntegrationTests } from "./jobs/build/trigger-integration-tests";
-import { triggerUpgradeTests } from "./jobs/build/self-hosted-upgrade-tests";
+import { triggerSelfHostedPreview, triggerUpgradeTests } from "./jobs/build/self-hosted-upgrade-tests";
 import { jobConfig } from "./jobs/build/job-config";
 import { typecheckWerftJobs } from "./jobs/build/typecheck-werft-jobs";
 
@@ -60,6 +60,11 @@ async function run(context: any) {
     await typecheckWerftJobs(werft);
     await buildAndPublish(werft, config);
 
+    if (config.withSelfHostedPreview) {
+        await triggerSelfHostedPreview(werft, config, context.Owner);
+        return
+    }
+
     if (!config.withPreview || config.publishRelease) {
         werft.phase("deploy", "not deploying");
         console.log("running without preview environment or publish-release is set");

.werft/installer-tests.ts

@@ -3,6 +3,7 @@ import * as https from "https";
 import { join } from "path";
 import { exec } from "./util/shell";
 import { Werft } from "./util/werft";
+import { deleteReplicatedLicense } from "./jobs/build/self-hosted-upgrade-tests";
 
 const context: any = JSON.parse(fs.readFileSync("context.json").toString());
 
@@ -476,6 +477,8 @@ function cleanup() {
 
     werft.done(phase.phase);
 
+    deleteReplicatedLicense(werft, process.env["TF_VAR_TEST_ID"]);
+
     return ret;
 }
 

.werft/jobs/build/job-config.ts

@@ -6,6 +6,7 @@ export interface JobConfig {
     analytics: string;
     buildConfig: any;
     cleanSlateDeployment: boolean;
+    cluster: string;
     coverageOutput: string;
     dontTest: boolean;
     fromVersion: string;
@@ -24,6 +25,7 @@ export interface JobConfig {
     withContrib: boolean;
     withIntegrationTests: boolean;
     withUpgradeTests: boolean;
+    withSelfHostedPreview: boolean;
     withObservability: boolean;
     withPayment: boolean;
     workspaceFeatureFlags: string[];
@@ -83,10 +85,12 @@ export function jobConfig(werft: Werft, context: any): JobConfig {
     const withIntegrationTests = "with-integration-tests" in buildConfig && !mainBuild;
     const withUpgradeTests = "with-upgrade-tests" in buildConfig && !mainBuild;
     const fromVersion = withUpgradeTests ? buildConfig["from-version"] : "";
-    const replicatedChannel = withUpgradeTests ? buildConfig["channel"] : "";
+    const replicatedChannel = buildConfig["channel"];
+    const cluster = buildConfig["cluster"];
+    const withSelfHostedPreview = "with-sh-preview" in buildConfig;
     const publishToNpm = "publish-to-npm" in buildConfig || mainBuild;
     const publishToJBMarketplace = "publish-to-jb-marketplace" in buildConfig || mainBuild;
-    const publishToKots = "publish-to-kots" in buildConfig || mainBuild;
+    const publishToKots = "publish-to-kots" in buildConfig || withSelfHostedPreview || mainBuild;
     const analytics = buildConfig["analytics"];
     const localAppVersion = mainBuild || "with-localapp-version" in buildConfig ? version : "unknown";
     const retag = "with-retag" in buildConfig ? "" : "--dont-retag";
@@ -122,6 +126,7 @@ export function jobConfig(werft: Werft, context: any): JobConfig {
         analytics,
         buildConfig,
         cleanSlateDeployment,
+        cluster,
         coverageOutput,
         dontTest,
         fromVersion,
@@ -145,6 +150,7 @@ export function jobConfig(werft: Werft, context: any): JobConfig {
         withObservability,
         withPayment,
         withUpgradeTests,
+        withSelfHostedPreview,
         workspaceFeatureFlags,
         withLargeVM,
     };

.werft/jobs/build/self-hosted-upgrade-tests.ts

@@ -3,24 +3,29 @@ import { Werft } from "../../util/werft";
 import { JobConfig } from "./job-config";
 
 interface config {
+    cloud: string,
     phase: string;
     description: string;
 }
 
 const phases: { [name: string]: config } = {
     gke: {
+        cloud: "gcp",
         phase: "trigger upgrade test in GKE",
         description: "Triggers upgrade test on supplied version from Beta channel on GKE cluster",
     },
     aks: {
+        cloud: "azure",
         phase: "trigger upgrade test in AKS",
         description: "Triggers upgrade test on supplied version from Beta channel on AKS cluster",
     },
     k3s: {
+        cloud: "k3s",
         phase: "trigger upgrade test in K3S",
         description: "Triggers upgrade test on supplied version from Beta channel on K3S cluster",
     },
     eks: {
+        cloud: "aws",
         phase: "trigger upgrade test in EKS",
         description: "Triggers upgrade test on supplied version from Beta channel on EKS cluster",
     },
@@ -67,3 +72,110 @@ export async function triggerUpgradeTests(werft: Werft, config: JobConfig, usern
         }
     }
 }
+
+export async function triggerSelfHostedPreview(werft: Werft, config: JobConfig, username: string) {
+    const replicatedChannel =  config.replicatedChannel || config.repository.branch;
+    const cluster =  config.cluster || "k3s";
+    const formattedBranch = config.repository.branch.replace("/", "-").slice(0,10)
+    const phase = phases[cluster]
+    const subdomain =  `${formattedBranch}x-${phase.cloud}`
+
+    const replicatedApp = process.env.REPLICATED_APP
+
+    var licenseFlag: string = ""
+    var annotation: string = ""
+
+
+    if(!["stable", "unstable", "beta"].includes(replicatedChannel.toLowerCase())){
+        werft.phase("get-replicated-license", `Create and download replicated license for ${replicatedChannel}`);
+
+        const customerID = getCustomerID(subdomain)
+
+        if(customerID == "") {
+            exec(`replicated customer create --app ${replicatedApp} --channel ${replicatedChannel} --name ${subdomain}`,
+                { slice: "get-replicated-license"})
+        }
+
+        exec(`replicated customer download-license --app ${replicatedApp} --customer ${subdomain} > license.yaml`,
+            { slice: "get-replicated-license", dontCheckRc: true})
+
+        exec(`install -D license.yaml install/licenses/${replicatedChannel}.yaml`,
+            { slice: "get-replicated-license"},
+        )
+        werft.done("get-replicated-license");
+
+        licenseFlag = `-s install/licenses/${replicatedChannel}.yaml`
+    }
+
+    exec(`git config --global user.name "${username}"`);
+
+    annotation = `${annotation} -a channel=${replicatedChannel} -a preview=true -a skipTests=true -a deps=external`;
+
+    werft.phase("self-hosted-preview", `Create self-hosted preview in ${cluster}`);
+
+    annotation = `${annotation} -a cluster=${cluster} -a updateGitHubStatus=gitpod-io/gitpod -a subdomain=${subdomain}`
+
+    const testFile: string = `.werft/${cluster}-installer-tests.yaml`;
+
+    try {
+        exec(
+            `werft run --remote-job-path ${testFile} ${annotation} github ${licenseFlag}`,
+            {
+                slice: "self-hosted-preview"
+            },
+        ).trim();
+
+        werft.done("self-hosted-preview");
+    } catch (err) {
+        if (!config.mainBuild) {
+            werft.fail("self-hosted-preview", err);
+        }
+        console.log("Deleting the created license ", subdomain)
+        deleteReplicatedLicense(werft, subdomain)
+        exec("exit 0");
+    }
+}
+
+export async function deleteReplicatedLicense(werft: Werft, licenseName: string) {
+    var customerID: string
+
+    if(licenseName == "") {
+        console.log("No customerID or license name found, skipping replicated license cleanup")
+        return
+    }
+
+    customerID = getCustomerID(licenseName)
+
+    if(customerID == "") {
+        console.log("Could not find license, skipping replicated license cleanup")
+        return
+    }
+
+    console.log("trying to cleanup replicated license")
+    werft.phase("delete-replicated-license", "Deletes the replicated license created")
+    const ret = exec(`curl --request POST \
+            --url https://api.replicated.com/vendor/v3/customer/${customerID}/archive \
+            --header 'Authorization: ${ process.env.REPLICATED_API_TOKEN }'`,
+             {slice: "delete-replicated-license", dontCheckRc: true})
+    if(ret.code){
+        werft.fail("delete-replicated-license", "Could not delete the replciated license")
+        return
+    }
+
+    werft.done("delete-replicated-license")
+}
+
+function getCustomerID(licenseName: string): string {
+    var customerID: string = ""
+    const replicatedApp = process.env.REPLICATED_APP
+
+    const response = exec(`replicated customer ls --app ${replicatedApp} | grep ${licenseName} | awk '{print $1}'`,
+                        { slice: "get-replicated-license", dontCheckRc: true})
+
+    const customerIDS = response.stdout.split("\n").filter(item => item);
+    if(customerIDS.length > 0) {
+        customerID = customerIDS[0].trim()
+    }
+
+    return customerID
+}

install/tests/Makefile

@@ -282,6 +282,7 @@ kots-install: install-kots-cli
 	kubectl kots remove ${app} -n gitpod --force --kubeconfig=${KUBECONFIG} || echo "No kots app existing, Installing"
 	kubectl kots install ${app}/${channel} \
 	--skip-rbac-check ${version-flag} ${preflight-flag} \
+					--wait-duration "10m" \
 					--namespace gitpod --kubeconfig=${KUBECONFIG} \
                     --name gitpod --shared-password gitpod \
 					--license-file ${license-file} \
@@ -291,9 +292,13 @@ kots-install: install-kots-cli
 time_to_sleep_azure := 600 # azure seem to take more time to fullfil DNS propogation
 time_to_sleep := 300
 
+wait_time := 180
+wait_time_azure := 180
+
 delete-cm-setup: sleeptime=$(if $(time_to_sleep_$(cloud)),$(time_to_sleep_$(cloud)),${time_to_sleep})
+delete-cm-setup: waittime=$(if $(wait_time_$(cloud)),$(wait_time_$(cloud)),${wait_time})
 delete-cm-setup:
-	sleep 180 && kubectl --kubeconfig=${KUBECONFIG} delete pods --all -n cert-manager && sleep ${sleeptime};
+	sleep ${waittime} && kubectl --kubeconfig=${KUBECONFIG} delete pods --all -n cert-manager && sleep ${sleeptime};
 
 gitpod-debug-info:
 	@echo "Gitpod is not ready"

install/tests/cleanup.sh

@@ -35,4 +35,14 @@ for i in $(gsutil ls gs://nightly-tests/tf-state); do
     export TF_VAR_TEST_ID=$TF_VAR_TEST_ID
 
     make cleanup cloud=$cloud
+
+    CUSTOMERID=$(replicated customer ls --app "${REPLICATED_APP}" | grep "$TF_VAR_TEST_ID" | awk '{print $1}')
+
+    [ -z "$CUSTOMERID" ] && continue
+
+    echo "Trying to archive replicated license"
+
+    curl --request POST \
+    --url https://api.replicated.com/vendor/v3/customer/$CUSTOMERID/archive \
+    --header "Authorization: ${REPLICATED_API_TOKEN}" || echo "Couldn't delete replicated licese"
 done

install/tests/main.tf

@@ -13,9 +13,6 @@ variable "project" { default = "sh-automated-tests" }
 variable "sa_creds" { default = null }
 variable "dns_sa_creds" { default = null }
 
-variable "domain" { default = "tests.gitpod-self-hosted.com" }
-variable "gcp_zone" { default = "tests-gitpod-self-hosted-com" }
-
 variable "eks_node_image_id" {
   default = null
 }