Fix ability to run integration tests in Werft

Author: mads-hartmann

.github/pull_request_template.md

@@ -36,3 +36,5 @@ Optional annotations to add to the werft job.
 * with-preview - whether to create a preview environment for this PR
 -->
 - [ ] /werft with-preview
+- [ ] /werft with-integration-tests=all
+      Valid options are `all`, `workspace`, `webapp`, `ide`

.werft/build.ts

@@ -8,7 +8,7 @@ import { buildAndPublish } from "./jobs/build/build-and-publish";
 import { validateChanges } from "./jobs/build/validate-changes";
 import { prepare } from "./jobs/build/prepare";
 import { deployToPreviewEnvironment } from "./jobs/build/deploy-to-preview-environment";
-import { triggerIntegrationTests } from "./jobs/build/trigger-integration-tests";
+import { runIntegrationTests } from "./jobs/build/trigger-integration-tests";
 import { triggerSelfHostedPreview, triggerUpgradeTests } from "./jobs/build/self-hosted-upgrade-tests";
 import { jobConfig } from "./jobs/build/job-config";
 import { typecheckWerftJobs } from "./jobs/build/typecheck-werft-jobs";
@@ -82,5 +82,5 @@ async function run(context: any) {
         throw e;
     }
 
-    await triggerIntegrationTests(werft, config, context.Owner);
+    await runIntegrationTests(werft, config, context.Owner);
 }

.werft/jobs/build/job-config.ts

@@ -2,6 +2,8 @@ import { exec } from "../../util/shell";
 import { Werft } from "../../util/werft";
 import { previewNameFromBranchName } from "../../util/preview";
 
+type WithIntegrationTests = "skip" | "all" | "workspace" | "ide" | "webapp";
+
 export interface JobConfig {
     analytics: string;
     buildConfig: any;
@@ -23,7 +25,7 @@ export interface JobConfig {
     storage: string;
     version: string;
     withContrib: boolean;
-    withIntegrationTests: boolean;
+    withIntegrationTests: WithIntegrationTests;
     withUpgradeTests: boolean;
     withSelfHostedPreview: boolean;
     withObservability: boolean;
@@ -78,7 +80,7 @@ export function jobConfig(werft: Werft, context: any): JobConfig {
     const withContrib = "with-contrib" in buildConfig || mainBuild;
     const withPreview = "with-preview" in buildConfig && !mainBuild;
     const storage = buildConfig["storage"] || "";
-    const withIntegrationTests = "with-integration-tests" in buildConfig && !mainBuild;
+    const withIntegrationTests = parseWithIntegrationTests(werft, sliceId, buildConfig["with-integration-tests"]);
     const withUpgradeTests = "with-upgrade-tests" in buildConfig && !mainBuild;
     const fromVersion = withUpgradeTests ? buildConfig["from-version"] : "";
     const replicatedChannel = buildConfig["channel"];
@@ -151,7 +153,7 @@ export function jobConfig(werft: Werft, context: any): JobConfig {
     };
 
     werft.logOutput(sliceId, JSON.stringify(jobConfig, null, 2));
-    werft.log(sliceId, "Expand to see the parsed configuration")
+    werft.log(sliceId, "Expand to see the parsed configuration");
     const globalAttributes = Object.fromEntries(
         Object.entries(jobConfig).map((kv) => {
             const [key, value] = kv;
@@ -179,3 +181,23 @@ function parseVersion(context: any) {
     }
     return version;
 }
+
+export function parseWithIntegrationTests(werft: Werft, sliceID: string, value?: string): WithIntegrationTests {
+    switch (value) {
+        case null:
+        case undefined:
+            return "skip";
+        case "skip":
+        case "all":
+        case "webapp":
+        case "ide":
+        case "webapp":
+            return value;
+        case "":
+            werft.log(sliceID, "with-integration-tests was set but no value was provided - falling back to 'all'");
+            return "all";
+        default:
+            werft.log(sliceID, `Unknown value for with-integration-tests: '${value}' - falling back to 'all'`);
+            return "all";
+    }
+}

.werft/jobs/build/trigger-integration-tests.ts

@@ -1,50 +1,36 @@
 import { exec } from "../../util/shell";
 import { Werft } from "../../util/werft";
 import { JobConfig } from "./job-config";
+import { PREVIEW_K3S_KUBECONFIG_PATH } from "./const";
 
 const phases = {
-    TRIGGER_INTEGRATION_TESTS: "trigger integration tests",
+    RUN_INTEGRATION_TESTS: "Run integration tests",
 };
 
 /**
  * Trigger integration tests
  */
-export async function triggerIntegrationTests(werft: Werft, config: JobConfig, username: string) {
-    werft.phase(phases.TRIGGER_INTEGRATION_TESTS, "Trigger integration tests");
+export async function runIntegrationTests(werft: Werft, config: JobConfig, username: string) {
+    werft.phase(phases.RUN_INTEGRATION_TESTS, "Run integration tests");
 
-    if (!config.withIntegrationTests) {
+    if (config.withIntegrationTests == "skip") {
         // If we're skipping integration tests we wont trigger the job, which in turn won't create the
         // ci/werft/run-integration-tests Github Check. As ci/werft/run-integration-tests is a required
         // check this means you can't merge your PR without override checks.
-        werft.log(phases.TRIGGER_INTEGRATION_TESTS, "Skipped integration tests");
-        werft.done(phases.TRIGGER_INTEGRATION_TESTS);
+        werft.log(phases.RUN_INTEGRATION_TESTS, "Skipped integration tests");
+        werft.done(phases.RUN_INTEGRATION_TESTS);
         return;
     }
 
     try {
-        const imageVersion = exec(
-            `docker run --rm eu.gcr.io/gitpod-core-dev/build/versions:${config.version} cat /versions.yaml | yq r - 'components.integrationTest.version'`,
-            { silent: true },
-        ).stdout.trim();
-
-        exec(`git config --global user.name "${username}"`);
-        const annotations = [
-            `version="${imageVersion}"`,
-            `namespace="${config.previewEnvironment.namespace}"`,
-            `username="${username}"`,
-            `updateGitHubStatus="gitpod-io/gitpod"`,
-        ]
-            .map((annotation) => `-a ${annotation}`)
-            .join(" ");
-        exec(`werft run --remote-job-path .werft/run-integration-tests.yaml ${annotations} github`, {
-            slice: phases.TRIGGER_INTEGRATION_TESTS,
-        }).trim();
-
-        werft.done(phases.TRIGGER_INTEGRATION_TESTS);
+        exec(
+            `KUBECONFIG="${PREVIEW_K3S_KUBECONFIG_PATH}" GOOGLE_APPLICATION_CREDENTIALS=/home/gitpod/.config/gcloud/legacy_credentials/[email protected]/adc.json /workspace/test/run.sh ${config.withIntegrationTests}`,
+        );
+        werft.done(phases.RUN_INTEGRATION_TESTS);
     } catch (err) {
         if (!config.mainBuild) {
-            werft.fail(phases.TRIGGER_INTEGRATION_TESTS, err);
+            werft.fail(phases.RUN_INTEGRATION_TESTS, err);
         }
-        exec("exit 0");
+        throw err;
     }
 }

.werft/run-integration-tests.yaml

@@ -21,33 +21,22 @@ Such tests are for example:
 
 ### Automatically at Gitpod
 
-There is a [werft job](../.werft/run-integration-tests.yaml) that runs the integration tests against `core-dev` preview environments.
+You can opt-in to run the integrations tests as part of the build job. that runs the integration tests against preview environments.
 
  > For tests that require an existing user the framework tries to automatically select one from the DB.
  > - On preview envs make sure to create one before running tests against it!
  > - If it's important to use a certain user (with fixed settings, for example) pass the additional `username` parameter.
 
 Example command:
-```
-werft run github -j .werft/run-integration-tests.yaml -a namespace=staging-gpl-2658-int-tests -a version=gpl-2658-int-tests.57 -f
+
+```sh
+werft run github -a with-preview=true -a with-integration-tests=true -f
 ```
 
 ### Manually
 
 You may want to run tests to assert whether a Gitpod installation is successfully integrated.
 
-#### Using a pod
-
-Best for when you want to validate an environment.
-
-1. Update image name in `integration.yaml` for job `integration-job` to latest built by werft.
-2. Optionally add your username in that job argument or any other additional params.
-2. Apply yaml file that will add all necessary permissions and create a job that will run tests.
-   * [`kubectl apply -f ./integration.yaml`](./integration.yaml)
-3. Check logs to inspect test results like so `kubectl logs -f jobs/integration-job`.
-4. Tear down the integration user and job when testing is done.
-   * [`kubectl delete -f ./integration.yaml`](./integration.yaml)
-
 #### Go test
 
 This is best for when you're actively developing Gitpod.

test/README.md

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+THIS_DIR="$(dirname "$0")"
+
+FAILURE_COUNT=0
+
+#TODO: parse args here and add to TEST_LIST based on teams passed in
+#TODO: Also update docs with examples
+
+if [[ "${TEST_LIST-}" == "" ]]; then
+  TEST_LIST="$THIS_DIR/tests/components/* $THIS_DIR/tests/ide/* $THIS_DIR/tests/workspace"
+fi
+
+args=()
+args+=( "-kubeconfig=${KUBECONFIG:-/home/gitpod/.kube/config}" )
+args+=( "-namespace=default" )
+[[ "${USERNAME:-}" != "" ]] && args+=( "-username=$USERNAME" )
+args+=( "-timeout=60m" )
+args+=( "-p=1" )
+
+for TEST_PATH in ${TEST_LIST}
+do
+  TEST_NAME=$(basename "${TEST_PATH}")
+  echo "running integration for ${TEST_NAME}" | werft log slice "test-${TEST_NAME}"
+
+  cd "${TEST_PATH}"
+  set +e
+  go test -v ./... "${args[@]}" 2>&1 | tee "${TEST_NAME}.log" | werft log slice "test-${TEST_NAME}"
+  RC=${PIPESTATUS[0]}
+  set -e
+  cd -
+
+  if [ "${RC}" -ne "0" ]; then
+    FAILURE_COUNT=$((FAILURE_COUNT+1))
+    werft log slice "test-${TEST_NAME}" --fail "${RC}"
+  else
+    werft log slice "test-${TEST_NAME}" --done
+  fi
+done
+
+exit $FAILURE_COUNT