[#2658] Basic integration tests

.werft/build.js

@@ -107,6 +107,7 @@ async function build(context, version) {
     const noPreview = "no-preview" in buildConfig || publishRelease;
     const registryFacadeHandover = "registry-facade-handover" in buildConfig;
     const storage = buildConfig["storage"] || "";
+    const withIntegrationTests = buildConfig["with-integration-tests"] == "true";
     werft.log("job config", JSON.stringify({
         buildConfig,
         version,
@@ -119,6 +120,7 @@ async function build(context, version) {
         noPreview,
         registryFacadeHandover,
         storage: storage,
+        withIntegrationTests,
     }));
 
     /**
@@ -162,21 +164,35 @@ async function build(context, version) {
     if (noPreview) {
         werft.phase("deploy", "not deploying");
         console.log("no-preview or publish-release is set");
-    } else {
-        await deployToDev(version, workspaceFeatureFlags, dynamicCPULimits, registryFacadeHandover, storage);
+
+        return
     }
-}
+
+    const destname = version.split(".")[0];
+    const namespace = `staging-${destname}`;
+    const domain = `${destname}.staging.gitpod-dev.com`;
+    const url = `https://${domain}`;
+    const deploymentConfig = {
+      version,
+      destname,
+      namespace,
+      domain,
+      url,
+    };
+    await deployToDev(deploymentConfig, workspaceFeatureFlags, dynamicCPULimits, registryFacadeHandover, storage);
 
+    if (withIntegrationTests) {
+        exec(`git config --global user.name "${context.Owner}"`);
+        exec(`werft run --follow-with-prefix="int-tests: " --remote-job-path .werft/run-integration-tests.yaml -a version=${deploymentConfig.version} -a namespace=${deploymentConfig.namespace} github`);
+    }
+}
 
 /**
  * Deploy dev
  */
-async function deployToDev(version, workspaceFeatureFlags, dynamicCPULimits, registryFacadeHandover, storage) {
+async function deployToDev(deploymentConfig, workspaceFeatureFlags, dynamicCPULimits, registryFacadeHandover, storage) {
     werft.phase("deploy", "deploying to dev");
-    const destname = version.split(".")[0];
-    const namespace = `staging-${destname}`;
-    const domain = `${destname}.staging.gitpod-dev.com`;
-    const url = `https://${domain}`;
+    const { version, destname, namespace, domain, url } = deploymentConfig;
     const wsdaemonPort = `1${Math.floor(Math.random()*1000)}`;
     const registryProxyPort = `2${Math.floor(Math.random()*1000)}`;
     const registryNodePort = `${30000 + Math.floor(Math.random()*1000)}`;

.werft/run-integration-tests.yaml

@@ -0,0 +1,79 @@
+args:
+- name: version
+  desc: "The version of the integration tests to use"
+  required: true
+- name: namespace
+  desc: "The namespace to run the integration test against"
+  required: true
+- name: username
+  desc: "The username to run the integration test with"
+  required: false
+pod:
+  serviceAccount: werft
+  nodeSelector:
+    dev/workload: builds
+  imagePullSecrets:
+  - name: eu-gcr-io-pull-secret
+  volumes:
+  - name: gcp-sa
+    secret:
+      secretName: gcp-sa-gitpod-dev-deployer
+  - name: config
+    emptyDir: {}
+  initContainers:
+  - name: gcloud
+    image: eu.gcr.io/gitpod-core-dev/dev/dev-environment:gpl-bump-helm.12
+    workingDir: /workspace
+    imagePullPolicy: Always
+    volumeMounts:
+    - name: gcp-sa
+      mountPath: /mnt/secrets/gcp-sa
+      readOnly: true
+    - name: config
+      mountPath: /config
+      readOnly: false
+    command:
+    - bash
+    - -c
+    - |
+
+      echo "[prep] preparing config."
+
+      gcloud auth activate-service-account --key-file /mnt/secrets/gcp-sa/service-account.json
+      cp -R /home/gitpod/.config/gcloud /config/gcloud
+      cp /home/gitpod/.kube/config /config/kubeconfig
+
+      echo "[prep] copied config..."
+  containers:
+  - name: tests
+    image: eu.gcr.io/gitpod-core-dev/build/integration-tests:{{ .Annotations.version }}
+    workingDir: /workspace
+    imagePullPolicy: Always
+    volumeMounts:
+    - name: config
+      mountPath: /config
+      readOnly: true
+    command:
+    - /bin/bash
+    - -c
+    - |
+      sleep 1
+      set -Eeuo pipefail
+
+      echo "[prep] receiving config..."
+      mkdir /root/.config
+      cp -R /config/gcloud /root/.config/gcloud
+      export GOOGLE_APPLICATION_CREDENTIALS=/config/gcloud/legacy_credentials/[email protected]/adc.json
+      echo "[prep] received config."
+
+      USERNAME="{{ .Annotations.username }}"
+      if [[ "$USERNAME" == "<no value>" ]]; then
+        USERNAME=""
+      fi
+      echo "[prep] using username: $USERNAME"
+      echo "[prep|DONE]"
+
+      /entrypoint.sh -kubeconfig=/config/kubeconfig -namespace={{ .Annotations.namespace }} -username=$USERNAME 2>&1 | ts "[int-tests] "
+
+      RC=${PIPESTATUS[0]}
+      if [ $RC -eq 1 ]; then echo "[int-tests|FAIL]"; else echo "[int-tests|DONE]"; fi

components/gitpod-protocol/go/reconnecting-ws.go

@@ -124,6 +124,8 @@ func (rc *ReconnectingWebsocket) Dial() {
 		case err := <-rc.errCh:
 			log.WithError(err).WithField("url", rc.url).Warn("connection has been closed, reconnecting...")
 			conn.Close()
+
+			time.Sleep(1 * time.Second)
 			conn = rc.connect()
 		}
 	}

test/BUILD.yaml

@@ -28,6 +28,8 @@ packages:
       dontTest: true
   - name: docker
     type: docker
+    srcs:
+      - entrypoint.sh
     deps:
       - :app
     argdeps:

test/README.md

@@ -15,4 +15,17 @@ Such tests are for example:
 - instrumentation: agents that are compiled before/during the test, uploaded to a pod and executed there.
                    They communicate with the test using net/rpc.
 - API access: to all internal APIs, including ws-manager, ws-daemon, image-builder, registry-facade, server
-- DB access to the Gitpod DB
+- DB access to the Gitpod DB
+
+## Run
+
+There is a [werft job](../.werft/run-integration-tests.yaml) that runs the integration tests against `core-dev` preview environments.
+
+ > For tests that require an existing user the framework tries to automatically select one from the DB.
+ > - On preview envs make sure to create one before running tests against it!
+ > - If it's important to use a certain user (with fixed settings, for example) pass the additional `username` parameter.
+
+Example command:
+```
+werft job run github -j .werft/run-integration-tests.yaml -a namespace=staging-gpl-2658-int-tests -a version=gpl-2658-int-tests.57 -f
+```

test/entrypoint.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+set -ex
+
+for i in $(ls /tests/*.test); do
+    $i $*;
+done

test/go.sum

@@ -106,6 +106,7 @@ github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWo
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/gitpod-io/gitpod v0.6.0-beta3 h1:WGLut/rxCjgg+RDHzMzu1jgztbIlQU8U/aFm4bVudmc=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=

test/leeway.Dockerfile

@@ -7,8 +7,11 @@ FROM alpine:3.13
 # Ensure latest packages are present, like security updates.
 RUN  apk upgrade --no-cache \
   && apk add --no-cache ca-certificates
+
+# convenience scripting tools
+RUN apk add --no-cache bash moreutils
 
 COPY test--app/bin /tests
 ENV PATH=$PATH:/tests
-RUN sh -c "echo '#!/bin/sh' > /entrypoint.sh; echo 'set -ex' >> /entrypoint.sh; echo 'for i in \$(ls /tests/*.test); do \$i \$*; done' >> /entrypoint.sh; chmod +x /entrypoint.sh"
+COPY entrypoint.sh /entrypoint.sh
 ENTRYPOINT [ "/entrypoint.sh" ]