Merge branch 'main' into jp/sync-profile

Author: jeanp413

.github/workflows/code-nightly.yaml

@@ -31,7 +31,7 @@ jobs:
       - run: |
           gcloud auth configure-docker --quiet
           export LEEWAY_WORKSPACE_ROOT=$(pwd)
-          headCommit=$(curl -H 'Accept: application/vnd.github.VERSION.sha' https://github.com/api/repos/gitpod-io/openvscode-server/commits/gp-code/release/1.71)
+          headCommit=$(curl -H 'Accept: application/vnd.github.VERSION.sha' https://github.com/api/repos/gitpod-io/openvscode-server/commits/gp-code/main)
           cd components/ide/code
           leeway build -Dversion=nightly -DimageRepoBase=eu.gcr.io/gitpod-core-dev/build -DcodeCommit=$headCommit -DcodeQuality=insider .:docker
       - name: Get previous job's status

.github/workflows/delete-kots-channel.yml

@@ -16,6 +16,17 @@ jobs:
           tar xf replicated.tar.gz replicated && rm replicated.tar.gz
           mv replicated /usr/local/bin/replicated
 
+      - name: Delete replicated license
+        run: |
+          CUSTOMER_ID=$(replicated customer ls | grep ${{ github.event.ref }} | awk '{print $1}')
+          if [ "${CUSTOMER_ID}" = "" ]; then
+            echo "No license found"
+            exit 0
+          fi
+          curl --request POST \
+              --url https://api.replicated.com/vendor/v3/customer/${CUSTOMER_ID}/archive \
+              --header 'Authorization: ${{ env.REPLICATED_API_TOKEN }}'
+
       - name: Delete Replicated channel
         run: |
           CHANNEL_ID=$(replicated channel inspect ${{ github.event.ref }} \

.gitpod.yml

@@ -58,7 +58,9 @@ tasks:
     init: yarn --network-timeout 100000 && yarn build
   - name: Go
     before: pre-commit install --install-hooks
-    init: leeway exec --filter-type go -v -- go mod verify
+    init: |
+      ./components/gitpod-protocol/go/scripts/generate-config.sh
+      leeway exec --filter-type go -v -- go mod verify
     openMode: split-right
 vscode:
   extensions:

.pre-commit-config.yaml

@@ -45,11 +45,6 @@ repos:
 
   - repo: local
     hooks:
-      - id: generate-gitpod-config-types
-        name: generate-gitpod-config-types
-        entry: ./components/gitpod-protocol/hack/generate-config.sh
-        language: script
-        pass_filenames: false
       - id: license-header
         name: license-header
         entry: leeway run components:update-license-header

.werft/build.ts

@@ -9,7 +9,7 @@ import { validateChanges } from "./jobs/build/validate-changes";
 import { prepare } from "./jobs/build/prepare";
 import { deployToPreviewEnvironment } from "./jobs/build/deploy-to-preview-environment";
 import { triggerIntegrationTests } from "./jobs/build/trigger-integration-tests";
-import { triggerUpgradeTests } from "./jobs/build/self-hosted-upgrade-tests";
+import { triggerSelfHostedPreview, triggerUpgradeTests } from "./jobs/build/self-hosted-upgrade-tests";
 import { jobConfig } from "./jobs/build/job-config";
 import { typecheckWerftJobs } from "./jobs/build/typecheck-werft-jobs";
 
@@ -60,6 +60,11 @@ async function run(context: any) {
     await typecheckWerftJobs(werft);
     await buildAndPublish(werft, config);
 
+    if (config.withSelfHostedPreview) {
+        await triggerSelfHostedPreview(werft, config, context.Owner);
+        return
+    }
+
     if (!config.withPreview || config.publishRelease) {
         werft.phase("deploy", "not deploying");
         console.log("running without preview environment or publish-release is set");

.werft/installer-tests.ts

@@ -3,6 +3,7 @@ import * as https from "https";
 import { join } from "path";
 import { exec } from "./util/shell";
 import { Werft } from "./util/werft";
+import { deleteReplicatedLicense } from "./jobs/build/self-hosted-upgrade-tests";
 
 const context: any = JSON.parse(fs.readFileSync("context.json").toString());
 
@@ -476,6 +477,8 @@ function cleanup() {
 
     werft.done(phase.phase);
 
+    deleteReplicatedLicense(werft, process.env["TF_VAR_TEST_ID"]);
+
     return ret;
 }
 

.werft/jobs/build/deploy-to-preview-environment.ts

@@ -212,7 +212,6 @@ export async function deployToPreviewEnvironment(werft: Werft, jobConfig: JobCon
         previewName: previewNameFromBranchName(jobConfig.repository.branch),
         stackdriverServiceAccount: STACKDRIVER_SERVICEACCOUNT,
         werft: werft,
-        installationMethod: jobConfig.observability.installationMethod,
     });
     const sliceID = "observability";
     monitoringSatelliteInstaller

.werft/jobs/build/installer/installer.ts

@@ -167,6 +167,7 @@ EOF`);
 
     private configureIDE(slice: string) {
         exec(`yq w -i ${this.options.installerConfigPath} experimental.ide.resolveLatest false`, { slice });
+        exec(`yq w -i ${this.options.installerConfigPath} experimental.ide.ideMetrics.enabledErrorReporting true`, { slice });
     }
 
     private configureObservability(slice: string) {

.werft/jobs/build/job-config.ts

@@ -6,6 +6,7 @@ export interface JobConfig {
     analytics: string;
     buildConfig: any;
     cleanSlateDeployment: boolean;
+    cluster: string;
     coverageOutput: string;
     dontTest: boolean;
     fromVersion: string;
@@ -24,6 +25,7 @@ export interface JobConfig {
     withContrib: boolean;
     withIntegrationTests: boolean;
     withUpgradeTests: boolean;
+    withSelfHostedPreview: boolean;
     withObservability: boolean;
     withPayment: boolean;
     workspaceFeatureFlags: string[];
@@ -45,13 +47,9 @@ export interface Repository {
     branch: string;
 }
 
-export type ObservabilityInstallationMethod = "jsonnet" | "observability-installer";
-
 export interface Observability {
     // The branch of gitpod-io/observability to use
     branch: string;
-    // What tool to use to install monitoring-satellite from gitpod-io/observability
-    installationMethod: ObservabilityInstallationMethod;
 }
 
 export function jobConfig(werft: Werft, context: any): JobConfig {
@@ -83,10 +81,12 @@ export function jobConfig(werft: Werft, context: any): JobConfig {
     const withIntegrationTests = "with-integration-tests" in buildConfig && !mainBuild;
     const withUpgradeTests = "with-upgrade-tests" in buildConfig && !mainBuild;
     const fromVersion = withUpgradeTests ? buildConfig["from-version"] : "";
-    const replicatedChannel = withUpgradeTests ? buildConfig["channel"] : "";
+    const replicatedChannel = buildConfig["channel"];
+    const cluster = buildConfig["cluster"];
+    const withSelfHostedPreview = "with-sh-preview" in buildConfig;
     const publishToNpm = "publish-to-npm" in buildConfig || mainBuild;
     const publishToJBMarketplace = "publish-to-jb-marketplace" in buildConfig || mainBuild;
-    const publishToKots = "publish-to-kots" in buildConfig || mainBuild;
+    const publishToKots = "publish-to-kots" in buildConfig || withSelfHostedPreview || mainBuild;
     const analytics = buildConfig["analytics"];
     const localAppVersion = mainBuild || "with-localapp-version" in buildConfig ? version : "unknown";
     const retag = "with-retag" in buildConfig ? "" : "--dont-retag";
@@ -115,13 +115,13 @@ export function jobConfig(werft: Werft, context: any): JobConfig {
 
     const observability: Observability = {
         branch: context.Annotations.withObservabilityBranch || "main",
-        installationMethod: context.Annotations.observabilityInstallationMethod || "jsonnet",
     };
 
     const jobConfig = {
         analytics,
         buildConfig,
         cleanSlateDeployment,
+        cluster,
         coverageOutput,
         dontTest,
         fromVersion,
@@ -145,6 +145,7 @@ export function jobConfig(werft: Werft, context: any): JobConfig {
         withObservability,
         withPayment,
         withUpgradeTests,
+        withSelfHostedPreview,
         workspaceFeatureFlags,
         withLargeVM,
     };

.werft/jobs/build/self-hosted-upgrade-tests.ts

@@ -3,24 +3,29 @@ import { Werft } from "../../util/werft";
 import { JobConfig } from "./job-config";
 
 interface config {
+    cloud: string,
     phase: string;
     description: string;
 }
 
 const phases: { [name: string]: config } = {
     gke: {
+        cloud: "gcp",
         phase: "trigger upgrade test in GKE",
         description: "Triggers upgrade test on supplied version from Beta channel on GKE cluster",
     },
     aks: {
+        cloud: "azure",
         phase: "trigger upgrade test in AKS",
         description: "Triggers upgrade test on supplied version from Beta channel on AKS cluster",
     },
     k3s: {
+        cloud: "k3s",
         phase: "trigger upgrade test in K3S",
         description: "Triggers upgrade test on supplied version from Beta channel on K3S cluster",
     },
     eks: {
+        cloud: "aws",
         phase: "trigger upgrade test in EKS",
         description: "Triggers upgrade test on supplied version from Beta channel on EKS cluster",
     },
@@ -67,3 +72,110 @@ export async function triggerUpgradeTests(werft: Werft, config: JobConfig, usern
         }
     }
 }
+
+export async function triggerSelfHostedPreview(werft: Werft, config: JobConfig, username: string) {
+    const replicatedChannel =  config.replicatedChannel || config.repository.branch;
+    const cluster =  config.cluster || "k3s";
+    const formattedBranch = config.repository.branch.replace("/", "-").slice(0,10)
+    const phase = phases[cluster]
+    const subdomain =  `${formattedBranch}x-${phase.cloud}`
+
+    const replicatedApp = process.env.REPLICATED_APP
+
+    var licenseFlag: string = ""
+    var annotation: string = ""
+
+
+    if(!["stable", "unstable", "beta"].includes(replicatedChannel.toLowerCase())){
+        werft.phase("get-replicated-license", `Create and download replicated license for ${replicatedChannel}`);
+
+        const customerID = getCustomerID(subdomain)
+
+        if(customerID == "") {
+            exec(`replicated customer create --app ${replicatedApp} --channel ${replicatedChannel} --name ${subdomain}`,
+                { slice: "get-replicated-license"})
+        }
+
+        exec(`replicated customer download-license --app ${replicatedApp} --customer ${subdomain} > license.yaml`,
+            { slice: "get-replicated-license", dontCheckRc: true})
+
+        exec(`install -D license.yaml install/licenses/${replicatedChannel}.yaml`,
+            { slice: "get-replicated-license"},
+        )
+        werft.done("get-replicated-license");
+
+        licenseFlag = `-s install/licenses/${replicatedChannel}.yaml`
+    }
+
+    exec(`git config --global user.name "${username}"`);
+
+    annotation = `${annotation} -a channel=${replicatedChannel} -a preview=true -a skipTests=true -a deps=external`;
+
+    werft.phase("self-hosted-preview", `Create self-hosted preview in ${cluster}`);
+
+    annotation = `${annotation} -a cluster=${cluster} -a updateGitHubStatus=gitpod-io/gitpod -a subdomain=${subdomain}`
+
+    const testFile: string = `.werft/${cluster}-installer-tests.yaml`;
+
+    try {
+        exec(
+            `werft run --remote-job-path ${testFile} ${annotation} github ${licenseFlag}`,
+            {
+                slice: "self-hosted-preview"
+            },
+        ).trim();
+
+        werft.done("self-hosted-preview");
+    } catch (err) {
+        if (!config.mainBuild) {
+            werft.fail("self-hosted-preview", err);
+        }
+        console.log("Deleting the created license ", subdomain)
+        deleteReplicatedLicense(werft, subdomain)
+        exec("exit 0");
+    }
+}
+
+export async function deleteReplicatedLicense(werft: Werft, licenseName: string) {
+    var customerID: string
+
+    if(licenseName == "") {
+        console.log("No customerID or license name found, skipping replicated license cleanup")
+        return
+    }
+
+    customerID = getCustomerID(licenseName)
+
+    if(customerID == "") {
+        console.log("Could not find license, skipping replicated license cleanup")
+        return
+    }
+
+    console.log("trying to cleanup replicated license")
+    werft.phase("delete-replicated-license", "Deletes the replicated license created")
+    const ret = exec(`curl --request POST \
+            --url https://api.replicated.com/vendor/v3/customer/${customerID}/archive \
+            --header 'Authorization: ${ process.env.REPLICATED_API_TOKEN }'`,
+             {slice: "delete-replicated-license", dontCheckRc: true})
+    if(ret.code){
+        werft.fail("delete-replicated-license", "Could not delete the replciated license")
+        return
+    }
+
+    werft.done("delete-replicated-license")
+}
+
+function getCustomerID(licenseName: string): string {
+    var customerID: string = ""
+    const replicatedApp = process.env.REPLICATED_APP
+
+    const response = exec(`replicated customer ls --app ${replicatedApp} | grep ${licenseName} | awk '{print $1}'`,
+                        { slice: "get-replicated-license", dontCheckRc: true})
+
+    const customerIDS = response.stdout.split("\n").filter(item => item);
+    if(customerIDS.length > 0) {
+        customerID = customerIDS[0].trim()
+    }
+
+    return customerID
+}