Splitting the test executions

Author: nandajavarma

.werft/installer-tests.ts

@@ -186,11 +186,50 @@ const INFRA_PHASES: { [name: string]: InfraConfig } = {
 
 
 const TESTS: { [name: string]: InfraConfig } = {
-    // TODO(nvn): make this more atomic
-    INTEGRATION_TESTS: {
-        phase: "run-tests",
-        makeTarget: "run-tests",
-        description: "Run the integration tests",
+    WORKSPACE_TEST: {
+        phase: "run-workspace-tests",
+        makeTarget: "run-workspace-tests",
+        description: "Run the test for workspaces",
+    },
+    VSCODE_IDE_TEST: {
+        phase: "run-vscode-ide-tests",
+        makeTarget: "run-vscode-ide-tests",
+        description: "Run the test for vscode IDE",
+    },
+    JB_IDE_TEST: {
+        phase: "run-jb-ide-tests",
+        makeTarget: "run-jb-ide-tests",
+        description: "Run the test for jetbrains IDE",
+    },
+    CONTENTSERVICE_TEST: {
+        phase: "run-cs-component-tests",
+        makeTarget: "run-cs-component-tests",
+        description: "Run the test for content-service component",
+    },
+    DB_TEST: {
+        phase: "run-db-component-tests",
+        makeTarget: "run-db-component-tests",
+        description: "Run the test for database component",
+    },
+    IMAGEBUILDER_TEST: {
+        phase: "run-ib-component-tests",
+        makeTarget: "run-ib-component-tests",
+        description: "Run the test for image-builder component",
+    },
+    SERVER_TEST: {
+        phase: "run-server-component-tests",
+        makeTarget: "run-server-component-tests",
+        description: "Run the test for server component",
+    },
+    WS_DAEMON_TEST: {
+        phase: "run-wsd-component-tests",
+        makeTarget: "run-wsd-component-tests",
+        description: "Run the test for ws-daemon component",
+    },
+    WS_MNGR_TEST: {
+        phase: "run-wsm-component-tests",
+        makeTarget: "run-wsm-component-tests",
+        description: "Run the test for ws-manager component",
     },
 }
 
@@ -250,6 +289,8 @@ if (upgrade === "true") {
             `werft log result -d  "self-hosted preview url" url "https://${process.env["TF_VAR_TEST_ID"]}.tests.gitpod-self-hosted.com"`,
         );
 
+        exec(`werft log result -d  "Terraform state" url "Terraform state file name is ${process.env["TF_VAR_TEST_ID"]}"`);
+
         werft.done("print-output");
     } else {
         // if we are not doing preview, we delete the infrastructure
@@ -261,7 +302,7 @@ function runIntegrationTests() {
     werft.phase("run-integration-tests", "Run all existing integration tests");
     for (let test in TESTS) {
         const testPhase = TESTS[test];
-        // we just let tests fail
+        // todo(nvn): handle the test failures by alerting teams
         const ret = callMakeTargets(testPhase.phase, testPhase.description, testPhase.makeTarget);
         if (ret) {
             exec(

install/tests/Makefile

@@ -221,8 +221,36 @@ check-gitpod-installation: delete-cm-setup check-kots-app check-env-sub-domain
 	@echo "Curling http://${TF_VAR_TEST_ID}.tests.gitpod-self-hosted.com/api/version"
 	curl -i -X GET http://${TF_VAR_TEST_ID}.tests.gitpod-self-hosted.com/api/version || { echo "Curling Gitpod endpoint failed"; exit 1; }
 
-run-tests:
-	./tests.sh ${KUBECONFIG}
+define runtests
+	./tests.sh ${KUBECONFIG} $(1)
+endef
+
+run-workspace-tests:
+	$(call runtests,"test/tests/workspace/")
+
+run-vscode-ide-tests:
+	$(call runtests,"test/tests/ide/vscode/")
+
+run-jb-ide-tests:
+	$(call runtests,"test/tests/ide/jetbrains/")
+
+run-cs-component-tests:
+	$(call runtests,"test/tests/components/content-service/")
+
+run-db-component-tests:
+	$(call runtests,"test/tests/components/database/")
+
+run-ib-component-tests:
+	$(call runtests,"test/tests/components/image-builder/")
+
+run-server-component-tests:
+	$(call runtests,"test/tests/components/content-service/")
+
+run-wsd-component-tests:
+	$(call runtests,"test/tests/components/ws-daemon/")
+
+run-wsm-component-tests:
+	$(call runtests,"test/tests/components/ws-manager/")
 
 kots-upgrade:
 	@echo "Upgrade gitpod KOTS app to latest"

install/tests/install/infra/terraform/eks/providers.tf

@@ -3,35 +3,25 @@
 # Licensed under the GNU Affero General Public License (AGPL).
 # See License-AGPL.txt in the project root for license information.
 
-# exclude 'e' (exit on any error)
-# there are many test binaries, each can have failures
-
-TEST_DIR="../../test/tests"
-
-# shellcheck disable=SC2045
-FAILURE_COUNT=0
 CURRENT=$(pwd)
-for i in $(find ${TEST_DIR} -type d -links 2 ! -empty | sort); do
-    # Will print */ if no directories are available
-    TEST_NAME=$(basename "${i}")
-    echo "running integration for ${TEST_NAME}"
+TESTPATH="../../$2"
+DATE_BIN=$(command -v date)
+DATE=$(${DATE_BIN} +%y-%m-%d--%H:%M:%S)
+
+echo "Starting test on ${DATE} for ${TESTPATH}"
 
-    cd "${i}" || echo "Path invalid for ${TEST_NAME}"
-    go test -v ./... "-kubeconfig=$1" -namespace=gitpod -username=gitpod-integration-test 2>&0
-    TEST_STATUS=$?
-    if [ "$TEST_STATUS" -ne "0" ]; then
-        FAILURE_COUNT=$((FAILURE_COUNT+1))
-        echo "Test failed at $(date)"
-    else
-        echo "Test succeeded at $(date)"
-    fi;
+cd "${TESTPATH}" ||  echo "Path invalid ${TESTPATH}"
 
-    cd "${CURRENT}" || echo "Couldn't move back to test dir"
-done
+go test -v ./... "-kubeconfig=$1" -namespace=gitpod -username=gitpod-integration-test 2>&0 -coverprofile=coverage.out
 
-if [ "$FAILURE_COUNT" -ne "0" ]; then
-    echo "Test suite ended with failure at $(date)"
-    exit $FAILURE_COUNT
+TEST_STATUS=$?
+echo ${TEST_STATUS}
+if [ "$TEST_STATUS" -ne "0" ]; then
+    echo "Test failed for ${TESTPATH} at ${DATE}"
+    exit 1
+else
+    echo "Test succeeded for ${TESTPATH} at ${DATE}"
+    exit 0
 fi;
 
-echo "Test suite ended with success at $(date)"
+cd "${CURRENT}" || echo "Couldn't move back to test dir"