diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 8be4f46c27052d..42cfca3b3ed9ec 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -36,3 +36,5 @@ Optional annotations to add to the werft job. * with-preview - whether to create a preview environment for this PR --> - [ ] /werft with-preview +- [ ] /werft with-integration-tests=all + Valid options are `all`, `workspace`, `webapp`, `ide` diff --git a/.werft/build.ts b/.werft/build.ts index e3b546eae88017..e32cb600e73ee3 100644 --- a/.werft/build.ts +++ b/.werft/build.ts @@ -8,7 +8,7 @@ import { buildAndPublish } from "./jobs/build/build-and-publish"; import { validateChanges } from "./jobs/build/validate-changes"; import { prepare } from "./jobs/build/prepare"; import { deployToPreviewEnvironment } from "./jobs/build/deploy-to-preview-environment"; -import { triggerIntegrationTests } from "./jobs/build/trigger-integration-tests"; +import { runIntegrationTests } from "./jobs/build/trigger-integration-tests"; import { triggerSelfHostedPreview, triggerUpgradeTests } from "./jobs/build/self-hosted-upgrade-tests"; import { jobConfig } from "./jobs/build/job-config"; @@ -81,5 +81,5 @@ async function run(context: any) { throw e; } - await triggerIntegrationTests(werft, config, context.Owner); + await runIntegrationTests(werft, config, context.Owner); } diff --git a/.werft/build.yaml b/.werft/build.yaml index fb7afe22ae43dd..6d9984465fdab8 100644 --- a/.werft/build.yaml +++ b/.werft/build.yaml @@ -221,6 +221,18 @@ pod: value: "/mnt/secrets/github-token-gitpod-bot/token" - name: WERFT_CREDENTIAL_HELPER value: "/workspace/dev/preview/werft-credential-helper.sh" + # When running the build with 'with-integration-tests' these are used + # to specify what Gitpod user to use in those tests. + - name: INTEGRATION_TEST_USERNAME + valueFrom: + secretKeyRef: + name: integration-test-user + key: username + - name: INTEGRATION_TEST_USER_TOKEN + valueFrom: + secretKeyRef: + name: integration-test-user + key: token command: - bash - -c diff --git a/.werft/jobs/build/job-config.ts b/.werft/jobs/build/job-config.ts index c6a4d87ba051db..f1a9d1ad360051 100644 --- a/.werft/jobs/build/job-config.ts +++ b/.werft/jobs/build/job-config.ts @@ -2,6 +2,8 @@ import { exec } from "../../util/shell"; import { Werft } from "../../util/werft"; import { previewNameFromBranchName } from "../../util/preview"; +type WithIntegrationTests = "skip" | "all" | "workspace" | "ide" | "webapp"; + export interface JobConfig { analytics: string; buildConfig: any; @@ -23,7 +25,7 @@ export interface JobConfig { storage: string; version: string; withContrib: boolean; - withIntegrationTests: boolean; + withIntegrationTests: WithIntegrationTests; withUpgradeTests: boolean; withSelfHostedPreview: boolean; withObservability: boolean; @@ -78,7 +80,7 @@ export function jobConfig(werft: Werft, context: any): JobConfig { const withContrib = "with-contrib" in buildConfig || mainBuild; const withPreview = "with-preview" in buildConfig && !mainBuild; const storage = buildConfig["storage"] || ""; - const withIntegrationTests = "with-integration-tests" in buildConfig && !mainBuild; + const withIntegrationTests = parseWithIntegrationTests(werft, sliceId, buildConfig["with-integration-tests"]); const withUpgradeTests = "with-upgrade-tests" in buildConfig && !mainBuild; const fromVersion = withUpgradeTests ? buildConfig["from-version"] : ""; const replicatedChannel = buildConfig["channel"]; @@ -151,7 +153,7 @@ export function jobConfig(werft: Werft, context: any): JobConfig { }; werft.logOutput(sliceId, JSON.stringify(jobConfig, null, 2)); - werft.log(sliceId, "Expand to see the parsed configuration") + werft.log(sliceId, "Expand to see the parsed configuration"); const globalAttributes = Object.fromEntries( Object.entries(jobConfig).map((kv) => { const [key, value] = kv; @@ -179,3 +181,24 @@ function parseVersion(context: any) { } return version; } + +export function parseWithIntegrationTests(werft: Werft, sliceID: string, value?: string): WithIntegrationTests { + switch (value) { + case null: + case undefined: + werft.log(sliceID, "with-integration-tests was not set - will use 'skip'"); + return "skip"; + case "skip": + case "all": + case "webapp": + case "ide": + case "webapp": + return value; + case "": + werft.log(sliceID, "with-integration-tests was set but no value was provided - falling back to 'all'"); + return "all"; + default: + werft.log(sliceID, `Unknown value for with-integration-tests: '${value}' - falling back to 'all'`); + return "all"; + } +} diff --git a/.werft/jobs/build/trigger-integration-tests.ts b/.werft/jobs/build/trigger-integration-tests.ts index d6ccaa11ec67db..1981ddd0c7133d 100644 --- a/.werft/jobs/build/trigger-integration-tests.ts +++ b/.werft/jobs/build/trigger-integration-tests.ts @@ -1,50 +1,36 @@ import { exec } from "../../util/shell"; import { Werft } from "../../util/werft"; import { JobConfig } from "./job-config"; +import { PREVIEW_K3S_KUBECONFIG_PATH } from "./const"; const phases = { - TRIGGER_INTEGRATION_TESTS: "trigger integration tests", + RUN_INTEGRATION_TESTS: "Run integration tests", }; /** * Trigger integration tests */ -export async function triggerIntegrationTests(werft: Werft, config: JobConfig, username: string) { - werft.phase(phases.TRIGGER_INTEGRATION_TESTS, "Trigger integration tests"); +export async function runIntegrationTests(werft: Werft, config: JobConfig, username: string) { + werft.phase(phases.RUN_INTEGRATION_TESTS, "Run integration tests"); - if (!config.withIntegrationTests) { + if (config.withIntegrationTests == "skip") { // If we're skipping integration tests we wont trigger the job, which in turn won't create the // ci/werft/run-integration-tests Github Check. As ci/werft/run-integration-tests is a required // check this means you can't merge your PR without override checks. - werft.log(phases.TRIGGER_INTEGRATION_TESTS, "Skipped integration tests"); - werft.done(phases.TRIGGER_INTEGRATION_TESTS); + werft.log(phases.RUN_INTEGRATION_TESTS, "Skipped integration tests"); + werft.done(phases.RUN_INTEGRATION_TESTS); return; } try { - const imageVersion = exec( - `docker run --rm eu.gcr.io/gitpod-core-dev/build/versions:${config.version} cat /versions.yaml | yq r - 'components.integrationTest.version'`, - { silent: true }, - ).stdout.trim(); - - exec(`git config --global user.name "${username}"`); - const annotations = [ - `version="${imageVersion}"`, - `namespace="${config.previewEnvironment.namespace}"`, - `username="${username}"`, - `updateGitHubStatus="gitpod-io/gitpod"`, - ] - .map((annotation) => `-a ${annotation}`) - .join(" "); - exec(`werft run --remote-job-path .werft/run-integration-tests.yaml ${annotations} github`, { - slice: phases.TRIGGER_INTEGRATION_TESTS, - }).trim(); - - werft.done(phases.TRIGGER_INTEGRATION_TESTS); + exec( + `KUBECONFIG="${PREVIEW_K3S_KUBECONFIG_PATH}" GOOGLE_APPLICATION_CREDENTIALS=/home/gitpod/.config/gcloud/legacy_credentials/cd-gitpod-deployer@gitpod-core-dev.iam.gserviceaccount.com/adc.json /workspace/test/run.sh ${config.withIntegrationTests}`, + ); + werft.done(phases.RUN_INTEGRATION_TESTS); } catch (err) { if (!config.mainBuild) { - werft.fail(phases.TRIGGER_INTEGRATION_TESTS, err); + werft.fail(phases.RUN_INTEGRATION_TESTS, err); } - exec("exit 0"); + throw err; } } diff --git a/.werft/run-integration-tests.yaml b/.werft/run-integration-tests.yaml deleted file mode 100644 index 6b66360de70fcf..00000000000000 --- a/.werft/run-integration-tests.yaml +++ /dev/null @@ -1,87 +0,0 @@ -args: - - name: version - desc: "The version of the integration tests to use" - required: true - - name: namespace - desc: "The namespace to run the integration test against" - required: true - - name: username - desc: "The username to run the integration test with" - required: false -pod: - serviceAccount: werft - nodeSelector: - dev/workload: builds - imagePullSecrets: - - name: eu-gcr-io-pull-secret - volumes: - - name: gcp-sa - secret: - secretName: gcp-sa-gitpod-dev-deployer - - name: config - emptyDir: {} - initContainers: - - name: gcloud - image: eu.gcr.io/gitpod-core-dev/dev/dev-environment:aa-unset-goflags.0 - workingDir: /workspace - imagePullPolicy: IfNotPresent - volumeMounts: - - name: gcp-sa - mountPath: /mnt/secrets/gcp-sa - readOnly: true - - name: config - mountPath: /config - readOnly: false - command: - - bash - - -c - - | - - echo "[prep] preparing config." - - gcloud auth activate-service-account --key-file /mnt/secrets/gcp-sa/service-account.json - cp -R /home/gitpod/.config/gcloud /config/gcloud - cp /home/gitpod/.kube/config /config/kubeconfig - - echo "[prep] copied config..." - containers: - - name: tests - image: eu.gcr.io/gitpod-core-dev/build/integration-tests:{{ .Annotations.version }} - workingDir: /workspace - imagePullPolicy: IfNotPresent - volumeMounts: - - name: config - mountPath: /config - readOnly: true - command: - - /bin/bash - - -c - - | - sleep 1 - set -Eeuo pipefail - - echo "[prep] receiving config..." - export GOOGLE_APPLICATION_CREDENTIALS="/config/gcloud/legacy_credentials/cd-gitpod-deployer@gitpod-core-dev.iam.gserviceaccount.com/adc.json" - echo "[prep] received config." - - USERNAME="{{ .Annotations.username }}" - if [[ "$USERNAME" == "" ]]; then - USERNAME="" - fi - echo "[prep] using username: $USERNAME" - - args=() - args+=( '-kubeconfig=/config/kubeconfig' ) - args+=( "-namespace={{ .Annotations.namespace }}" ) - [[ "$USERNAME" != "" ]] && args+=( "-username=$USERNAME" ) - echo "[prep] args: ${args[@]}" - echo "[prep|DONE]" - - /entrypoint.sh "${args[@]}" 2>&1 | ts "[int-tests] " - - RC=${PIPESTATUS[0]} - if [ $RC -eq 1 ]; then - echo "[int-tests|FAIL]" - else - echo "[int-tests|DONE]" - fi diff --git a/test/README.md b/test/README.md index c70ec1a0a3ac73..4378eae807d6a0 100644 --- a/test/README.md +++ b/test/README.md @@ -21,52 +21,64 @@ Such tests are for example: ### Automatically at Gitpod -There is a [werft job](../.werft/run-integration-tests.yaml) that runs the integration tests against `core-dev` preview environments. +You can opt-in to run the integrations tests as part of the build job. that runs the integration tests against preview environments. > For tests that require an existing user the framework tries to automatically select one from the DB. > - On preview envs make sure to create one before running tests against it! > - If it's important to use a certain user (with fixed settings, for example) pass the additional `username` parameter. Example command: -``` -werft run github -j .werft/run-integration-tests.yaml -a namespace=staging-gpl-2658-int-tests -a version=gpl-2658-int-tests.57 -f + +```sh +werft job run github -a with-preview=true -a with-integration-tests=webapp -f ``` ### Manually You may want to run tests to assert whether a Gitpod installation is successfully integrated. -#### Using a pod - -Best for when you want to validate an environment. - -1. Update image name in `integration.yaml` for job `integration-job` to latest built by werft. -2. Optionally add your username in that job argument or any other additional params. -2. Apply yaml file that will add all necessary permissions and create a job that will run tests. - * [`kubectl apply -f ./integration.yaml`](./integration.yaml) -3. Check logs to inspect test results like so `kubectl logs -f jobs/integration-job`. -4. Tear down the integration user and job when testing is done. - * [`kubectl delete -f ./integration.yaml`](./integration.yaml) - #### Go test This is best for when you're actively developing Gitpod. + Test will work if images that they use are already cached by Gitpod instance. If not, they might fail if it takes too long to pull an image. + There are 4 different types of tests: + 1. Enterprise specific, that require valid license to be installed. Run those with `-enterprise=true` 2. Tests that require correct user (user should have github OAuth integration setup with gitpod). Run those with `-username=`. Make sure to load https://github.com/gitpod-io/gitpod-test-repo and https://github.com/gitpod-io/gitpod workspaces inside your gitpod that you are testing to preload those images onto your node. Wait for it to finish pulling those image, this will ensure that test will not fail due to timeout while waiting to pull an image for the first time. 3. To test gitlab integration, add `-gitlab=true` 4. All other tests. -To run the tests: -1. Clone this repo (`git clone git@github.com:gitpod-io/gitpod.git`), and `cd` to `./gitpod/test` -2. Run the tests like so - ```console - go test -v ./... \ - -kubeconfig= \ - -namespace= \ - -username= \ - -enterprise= \ - -gitlab= - ``` -3. If you want to run specific test, add `-run ` before `-kubeconfig` parameter. +If you want to run an entire test suite, the easiest is to use `./test/run.sh`: + +```sh +# This will run all test suites +./test/run.sh + +# This will run only the webapp test suite +./test/run.sh webapp +``` + +If you're iterating on a single test, the easiest is to use `go test` directly. If your integration tests depends on having having a user token available, then you'll have to set USER_TOKEN manually (see run.sh on how to fetch the credentials that are used during our build) + +```sh +cd test +go test -v ./... \ + -run \ + -kubeconfig=/home/gitpod/.kube/config \ + -namespace=default \ + -username= \ + -enterprise= \ + -gitlab= +``` + +A concrete example would be + +```sh +cd test +go test -v ./... \ + -run TestAdminBlockUser \ + -kubeconfig=/home/gitpod/.kube/config \ + -namespace=default +``` diff --git a/test/integration.yaml b/test/integration.yaml deleted file mode 100644 index 1388e8b5439126..00000000000000 --- a/test/integration.yaml +++ /dev/null @@ -1,62 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: integration-svc ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: integration-role -rules: - - apiGroups: - - '' - resources: - - 'pods' - - 'secrets' - - 'services' - - 'configmaps' - - 'endpoints' - verbs: - - 'list' - - 'get' - - apiGroups: - - '' - resources: - - 'pods/portforward' - - 'pods/exec' - verbs: - - 'create' ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: integration-rolebind -subjects: - - kind: ServiceAccount - name: integration-svc -roleRef: - kind: Role - name: integration-role - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: integration-job -spec: - template: - spec: - serviceAccountName: integration-svc - containers: - - name: tests - image: eu.gcr.io/gitpod-core-dev/build/integration-tests:to-test-cgv2.4 - imagePullPolicy: Always - #args: ["-username=sagor999"] - #args: ["-enterprise=true"] - #args: ["-gitlab=true"] - resources: - requests: - cpu: 2 - memory: 2Gi - restartPolicy: Never - backoffLimit: 0 diff --git a/test/run.sh b/test/run.sh new file mode 100755 index 00000000000000..2590418299a5c9 --- /dev/null +++ b/test/run.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# +# Convenience script to run our integration tests suites. +# +# Usage: +# +# run.sh [suite] +# +# Examples +# +# run.sh This will run all test suites +# run.sh ide This will run just the 'ide' test suite +# + +set -euo pipefail + +THIS_DIR="$(dirname "$0")" +TEST_SUITE=${1:-'all'} +FAILURE_COUNT=0 +LOGS_DIR=$(mktemp -d) + +WEBAPP_TEST_LIST="$THIS_DIR/tests/components/database $THIS_DIR/tests/components/server" +IDE_TEST_LIST="$THIS_DIR/tests/ide/vscode $THIS_DIR/tests/ide/jetbrains" +WORKSPACE_TEST_LIST="$THIS_DIR/tests/components/content-service $THIS_DIR/tests/components/image-builder $THIS_DIR/tests/components/ws-daemon $THIS_DIR/tests/components/ws-manager $THIS_DIR/tests/workspace" + +case $TEST_SUITE in + "webapp") + TEST_LIST="$WEBAPP_TEST_LIST" + ;; + "ide") + TEST_LIST="$IDE_TEST_LIST" + ;; + "workspace") + TEST_LIST="${WORKSPACE_TEST_LIST}" + ;; + "" | "all") + TEST_LIST="${WEBAPP_TEST_LIST} ${IDE_TEST_LIST} ${WORKSPACE_TEST_LIST}" + ;; + *) + echo "Unknown test suite ${TEST_SUITE}" + exit 1 +esac + +args=() +args+=( "-kubeconfig=${KUBECONFIG:-/home/gitpod/.kube/config}" ) +args+=( "-namespace=default" ) +args+=( "-timeout=60m" ) +args+=( "-p=1" ) + +if [[ "${GITPOD_REPO_ROOT:-}" != "" ]]; then + echo "Running in Gitpod workspace. Fetching USERNAME and USER_TOKEN" | werft log slice "test-setup" + USERNAME="$(kubectl --context=dev -n werft get secret integration-test-user -o jsonpath='{.data.username}' | base64 -d)" + USER_TOKEN="$(kubectl --context=dev -n werft get secret integration-test-user -o jsonpath='{.data.token}' | base64 -d)" + export USER_TOKEN +else + echo "Running in Werft. Using INTEGRATION_TEST_USERNAME and INTEGRATION_TEST_USER_TOKEN for USERNAME and USER_TOKEN" | werft log slice "test-setup" + USERNAME="${INTEGRATION_TEST_USERNAME}" + USER_TOKEN="${INTEGRATION_TEST_USER_TOKEN}" + export USERNAME + export USER_TOKEN +fi +werft log slice "test-setup" --done + +[[ "$USERNAME" != "" ]] && args+=( "-username=$USERNAME" ) + +for TEST_PATH in ${TEST_LIST} +do + TEST_NAME=$(basename "${TEST_PATH}") + LOG_FILE="${LOGS_DIR}/${TEST_NAME}.log" + echo "running integration for ${TEST_NAME} - log file at ${LOG_FILE}" | werft log slice "test-${TEST_NAME}" + + cd "${TEST_PATH}" + set +e + go test -v ./... "${args[@]}" 2>&1 | tee "${LOG_FILE}" | werft log slice "test-${TEST_NAME}" + RC=${PIPESTATUS[0]} + set -e + cd - + + if [ "${RC}" -ne "0" ]; then + FAILURE_COUNT=$((FAILURE_COUNT+1)) + werft log slice "test-${TEST_NAME}" --fail "${RC}" + else + werft log slice "test-${TEST_NAME}" --done + fi +done + +exit $FAILURE_COUNT