Fix ability to run integration tests in Werft

Author: mads-hartmann

.github/pull_request_template.md

@@ -36,3 +36,5 @@ Optional annotations to add to the werft job.
 * with-preview - whether to create a preview environment for this PR
 -->
 - [ ] /werft with-preview
+- [ ] /werft with-integration-tests=all
+      Valid options are `all`, `workspace`, `webapp`, `ide`

.werft/build.ts

@@ -8,7 +8,7 @@ import { buildAndPublish } from "./jobs/build/build-and-publish";
 import { validateChanges } from "./jobs/build/validate-changes";
 import { prepare } from "./jobs/build/prepare";
 import { deployToPreviewEnvironment } from "./jobs/build/deploy-to-preview-environment";
-import { triggerIntegrationTests } from "./jobs/build/trigger-integration-tests";
+import { runIntegrationTests } from "./jobs/build/trigger-integration-tests";
 import { triggerSelfHostedPreview, triggerUpgradeTests } from "./jobs/build/self-hosted-upgrade-tests";
 import { jobConfig } from "./jobs/build/job-config";
 import { typecheckWerftJobs } from "./jobs/build/typecheck-werft-jobs";
@@ -82,5 +82,5 @@ async function run(context: any) {
         throw e;
     }
 
-    await triggerIntegrationTests(werft, config, context.Owner);
+    await runIntegrationTests(werft, config, context.Owner);
 }

.werft/build.yaml

@@ -221,6 +221,18 @@ pod:
       value: "/mnt/secrets/github-token-gitpod-bot/token"
     - name: WERFT_CREDENTIAL_HELPER
       value: "/workspace/dev/preview/werft-credential-helper.sh"
+    # When running the build with 'with-integration-tests' these are used
+    # to specify what Gitpod user to use in those tests.
+    - name: INTEGRATION_TEST_USERNAME
+      valueFrom:
+        secretKeyRef:
+          name: integration-test-user
+          key: username
+    - name: INTEGRATION_TEST_USER_TOKEN
+      valueFrom:
+        secretKeyRef:
+          name: integration-test-user
+          key: token
     command:
       - bash
       - -c

.werft/jobs/build/job-config.ts

@@ -2,6 +2,8 @@ import { exec } from "../../util/shell";
 import { Werft } from "../../util/werft";
 import { previewNameFromBranchName } from "../../util/preview";
 
+type WithIntegrationTests = "skip" | "all" | "workspace" | "ide" | "webapp";
+
 export interface JobConfig {
     analytics: string;
     buildConfig: any;
@@ -23,7 +25,7 @@ export interface JobConfig {
     storage: string;
     version: string;
     withContrib: boolean;
-    withIntegrationTests: boolean;
+    withIntegrationTests: WithIntegrationTests;
     withUpgradeTests: boolean;
     withSelfHostedPreview: boolean;
     withObservability: boolean;
@@ -78,7 +80,7 @@ export function jobConfig(werft: Werft, context: any): JobConfig {
     const withContrib = "with-contrib" in buildConfig || mainBuild;
     const withPreview = "with-preview" in buildConfig && !mainBuild;
     const storage = buildConfig["storage"] || "";
-    const withIntegrationTests = "with-integration-tests" in buildConfig && !mainBuild;
+    const withIntegrationTests = parseWithIntegrationTests(werft, sliceId, buildConfig["with-integration-tests"]);
     const withUpgradeTests = "with-upgrade-tests" in buildConfig && !mainBuild;
     const fromVersion = withUpgradeTests ? buildConfig["from-version"] : "";
     const replicatedChannel = buildConfig["channel"];
@@ -151,7 +153,7 @@ export function jobConfig(werft: Werft, context: any): JobConfig {
     };
 
     werft.logOutput(sliceId, JSON.stringify(jobConfig, null, 2));
-    werft.log(sliceId, "Expand to see the parsed configuration")
+    werft.log(sliceId, "Expand to see the parsed configuration");
     const globalAttributes = Object.fromEntries(
         Object.entries(jobConfig).map((kv) => {
             const [key, value] = kv;
@@ -179,3 +181,23 @@ function parseVersion(context: any) {
     }
     return version;
 }
+
+export function parseWithIntegrationTests(werft: Werft, sliceID: string, value?: string): WithIntegrationTests {
+    switch (value) {
+        case null:
+        case undefined:
+            return "skip";
+        case "skip":
+        case "all":
+        case "webapp":
+        case "ide":
+        case "webapp":
+            return value;
+        case "":
+            werft.log(sliceID, "with-integration-tests was set but no value was provided - falling back to 'all'");
+            return "all";
+        default:
+            werft.log(sliceID, `Unknown value for with-integration-tests: '${value}' - falling back to 'all'`);
+            return "all";
+    }
+}

.werft/jobs/build/trigger-integration-tests.ts

@@ -1,50 +1,36 @@
 import { exec } from "../../util/shell";
 import { Werft } from "../../util/werft";
 import { JobConfig } from "./job-config";
+import { PREVIEW_K3S_KUBECONFIG_PATH } from "./const";
 
 const phases = {
-    TRIGGER_INTEGRATION_TESTS: "trigger integration tests",
+    RUN_INTEGRATION_TESTS: "Run integration tests",
 };
 
 /**
  * Trigger integration tests
  */
-export async function triggerIntegrationTests(werft: Werft, config: JobConfig, username: string) {
-    werft.phase(phases.TRIGGER_INTEGRATION_TESTS, "Trigger integration tests");
+export async function runIntegrationTests(werft: Werft, config: JobConfig, username: string) {
+    werft.phase(phases.RUN_INTEGRATION_TESTS, "Run integration tests");
 
-    if (!config.withIntegrationTests) {
+    if (config.withIntegrationTests == "skip") {
         // If we're skipping integration tests we wont trigger the job, which in turn won't create the
         // ci/werft/run-integration-tests Github Check. As ci/werft/run-integration-tests is a required
         // check this means you can't merge your PR without override checks.
-        werft.log(phases.TRIGGER_INTEGRATION_TESTS, "Skipped integration tests");
-        werft.done(phases.TRIGGER_INTEGRATION_TESTS);
+        werft.log(phases.RUN_INTEGRATION_TESTS, "Skipped integration tests");
+        werft.done(phases.RUN_INTEGRATION_TESTS);
         return;
     }
 
     try {
-        const imageVersion = exec(
-            `docker run --rm eu.gcr.io/gitpod-core-dev/build/versions:${config.version} cat /versions.yaml | yq r - 'components.integrationTest.version'`,
-            { silent: true },
-        ).stdout.trim();
-
-        exec(`git config --global user.name "${username}"`);
-        const annotations = [
-            `version="${imageVersion}"`,
-            `namespace="${config.previewEnvironment.namespace}"`,
-            `username="${username}"`,
-            `updateGitHubStatus="gitpod-io/gitpod"`,
-        ]
-            .map((annotation) => `-a ${annotation}`)
-            .join(" ");
-        exec(`werft run --remote-job-path .werft/run-integration-tests.yaml ${annotations} github`, {
-            slice: phases.TRIGGER_INTEGRATION_TESTS,
-        }).trim();
-
-        werft.done(phases.TRIGGER_INTEGRATION_TESTS);
+        exec(
+            `KUBECONFIG="${PREVIEW_K3S_KUBECONFIG_PATH}" GOOGLE_APPLICATION_CREDENTIALS=/home/gitpod/.config/gcloud/legacy_credentials/[email protected]/adc.json /workspace/test/run.sh ${config.withIntegrationTests}`,
+        );
+        werft.done(phases.RUN_INTEGRATION_TESTS);
     } catch (err) {
         if (!config.mainBuild) {
-            werft.fail(phases.TRIGGER_INTEGRATION_TESTS, err);
+            werft.fail(phases.RUN_INTEGRATION_TESTS, err);
         }
-        exec("exit 0");
+        throw err;
     }
 }

.werft/run-integration-tests.yaml

@@ -21,52 +21,64 @@ Such tests are for example:
 
 ### Automatically at Gitpod
 
-There is a [werft job](../.werft/run-integration-tests.yaml) that runs the integration tests against `core-dev` preview environments.
+You can opt-in to run the integrations tests as part of the build job. that runs the integration tests against preview environments.
 
  > For tests that require an existing user the framework tries to automatically select one from the DB.
  > - On preview envs make sure to create one before running tests against it!
  > - If it's important to use a certain user (with fixed settings, for example) pass the additional `username` parameter.
 
 Example command:
-```
-werft run github -j .werft/run-integration-tests.yaml -a namespace=staging-gpl-2658-int-tests -a version=gpl-2658-int-tests.57 -f
+
+```sh
+werft job run github -a with-preview=true -a with-integration-tests=webapp -f
 ```
 
 ### Manually
 
 You may want to run tests to assert whether a Gitpod installation is successfully integrated.
 
-#### Using a pod
-
-Best for when you want to validate an environment.
-
-1. Update image name in `integration.yaml` for job `integration-job` to latest built by werft.
-2. Optionally add your username in that job argument or any other additional params.
-2. Apply yaml file that will add all necessary permissions and create a job that will run tests.
-   * [`kubectl apply -f ./integration.yaml`](./integration.yaml)
-3. Check logs to inspect test results like so `kubectl logs -f jobs/integration-job`.
-4. Tear down the integration user and job when testing is done.
-   * [`kubectl delete -f ./integration.yaml`](./integration.yaml)
-
 #### Go test
 
 This is best for when you're actively developing Gitpod.
+
 Test will work if images that they use are already cached by Gitpod instance. If not, they might fail if it takes too long to pull an image.
+
 There are 4 different types of tests:
+
 1. Enterprise specific, that require valid license to be installed. Run those with `-enterprise=true`
 2. Tests that require correct user (user should have github OAuth integration setup with gitpod). Run those with `-username=<gitpod_username>`. Make sure to load https://github.com/gitpod-io/gitpod-test-repo and https://github.com/gitpod-io/gitpod workspaces inside your gitpod that you are testing to preload those images onto your node. Wait for it to finish pulling those image, this will ensure that test will not fail due to timeout while waiting to pull an image for the first time.
 3. To test gitlab integration, add `-gitlab=true`
 4. All other tests.
 
-To run the tests:
-1. Clone this repo (`git clone [email protected]:gitpod-io/gitpod.git`), and `cd` to `./gitpod/test`
-2. Run the tests like so
-   ```console
-   go test -v ./... \
-     -kubeconfig=<path_to_kube_config_file> \
-     -namespace=<namespace_where_gitpod_is_installed> \
-     -username=<gitpod_user_with_oauth_setup> \
-     -enterprise=<true|false> \
-     -gitlab=<true|false>
-   ```
-3. If you want to run specific test, add `-run <test>` before `-kubeconfig` parameter.
+If you want to run an entire test suite, the easiest is to use `./test/run.sh`:
+
+```sh
+# This will run all test suites
+./test/run.sh
+
+# This will run only the webapp test suite
+./test/run.sh webapp
+```
+
+If you're iterating on a single test, the easiest is to use `go test` directly. If your integration tests depends on having having a user token available, then you'll have to set USER_TOKEN manually (see run.sh on how to fetch the credentials that are used during our build)
+
+```sh
+cd test
+go test -v ./... \
+    -run <test> \
+    -kubeconfig=/home/gitpod/.kube/config \
+    -namespace=default \
+    -username=<gitpod_user_with_oauth_setup> \
+    -enterprise=<true|false> \
+    -gitlab=<true|false>
+```
+
+A concrete example would be
+
+```sh
+cd test
+go test -v ./... \
+    -run TestAdminBlockUser \
+    -kubeconfig=/home/gitpod/.kube/config \
+    -namespace=default
+```