Skip to content

Commit c9a41bb

Browse files
nandajavarmanandajavarma
committed
[installer-tests] add slack webhook
1 parent 62f576f commit c9a41bb

File tree

3 files changed

+190
-22
lines changed

3 files changed

+190
-22
lines changed

‎.werft/installer-tests.ts

Lines changed: 168 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import * as fs from "fs";
2+
import * as https from "https";
23
import { join } from "path";
34
import { exec } from "./util/shell";
45
import { Werft } from "./util/werft";
@@ -16,6 +17,11 @@ const upgrade: string = annotations.upgrade || "false"; // setting to true will
1617
const skipTests: string = annotations.skipTests || "false"; // setting to true skips the integration tests
1718
const deps: string = annotations.deps || ""; // options: ["external", "internal"] setting to `external` will ensure that all resource dependencies(storage, db, registry) will be external. if unset, a random selection will be used
1819

20+
const slackHook: { [name: string]: string} = {
21+
"self-hosted-jobs": process.env.SH_SLACK_NOTIFICATION_PATH.trim(),
22+
"workspace-jobs": process.env.WS_SLACK_NOTIFICATION_PATH.trim(),
23+
"ide-jobs": process.env.IDE_SLACK_NOTIFICATION_PATH.trim(),
24+
}
1925

2026
const makefilePath: string = join("install/tests");
2127

@@ -25,6 +31,7 @@ interface InfraConfig {
2531
phase: string;
2632
makeTarget: string;
2733
description: string;
34+
slackhook?: string;
2835
}
2936

3037
interface TestConfig {
@@ -33,12 +40,14 @@ interface TestConfig {
3340
CLOUD: string;
3441
}
3542

43+
const op: string = preview == "true" ? "Preview" : "Test"
44+
3645
// Each of the TEST_CONFIGURATIONS define an integration test end-to-end
3746
// It should be a combination of multiple INFRA_PHASES, order of PHASES slice is important
3847
const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = {
3948
STANDARD_GKE_TEST: {
4049
CLOUD: "gcp",
41-
DESCRIPTION: "Deploy Gitpod on GKE, with managed DNS, and run integration tests",
50+
DESCRIPTION: `${op} Gitpod on a GKE managed cluster`,
4251
PHASES: [
4352
"STANDARD_GKE_CLUSTER",
4453
"CERT_MANAGER",
@@ -51,9 +60,7 @@ const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = {
5160
},
5261
STANDARD_K3S_TEST: {
5362
CLOUD: "gcp", // the cloud provider is still GCP
54-
DESCRIPTION:
55-
"Deploy Gitpod on a K3s cluster, created on a GCP instance," +
56-
" with managed DNS and run integrations tests",
63+
DESCRIPTION: `${op} Gitpod on a K3s cluster in a GCP instance`,
5764
PHASES: [
5865
"STANDARD_K3S_CLUSTER_ON_GCP",
5966
"CERT_MANAGER",
@@ -65,7 +72,7 @@ const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = {
6572
},
6673
STANDARD_AKS_TEST: {
6774
CLOUD: "azure",
68-
DESCRIPTION: "Deploy Gitpod on AKS, with managed DNS, and run integration tests",
75+
DESCRIPTION: `${op} Gitpod on an AKS managed cluster`,
6976
PHASES: [
7077
"STANDARD_AKS_CLUSTER",
7178
"CERT_MANAGER",
@@ -79,7 +86,7 @@ const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = {
7986
},
8087
STANDARD_EKS_TEST: {
8188
CLOUD: "aws",
82-
DESCRIPTION: "Create an EKS cluster",
89+
DESCRIPTION: `${op} Gitpod on an EKS managed cluster`,
8390
PHASES: [
8491
"STANDARD_EKS_CLUSTER",
8592
"CERT_MANAGER",
@@ -189,47 +196,52 @@ const TESTS: { [name: string]: InfraConfig } = {
189196
WORKSPACE_TEST: {
190197
phase: "run-workspace-tests",
191198
makeTarget: "run-workspace-tests",
192-
description: "Run the test for workspaces",
199+
description: "Run the \`workspace\` tests \`test/tests/workspace\`",
200+
slackhook: slackHook["workspace-jobs"],
193201
},
194202
VSCODE_IDE_TEST: {
195203
phase: "run-vscode-ide-tests",
196204
makeTarget: "run-vscode-ide-tests",
197-
description: "Run the test for vscode IDE",
205+
description: "Run the \`vscode IDE\` tests \`test/tests/ide/vscode\`",
206+
slackhook: slackHook["ide-jobs"],
198207
},
199208
JB_IDE_TEST: {
200209
phase: "run-jb-ide-tests",
201210
makeTarget: "run-jb-ide-tests",
202-
description: "Run the test for jetbrains IDE",
211+
description: "Run the \`jetbrains IDE\` tests \`test/tests/ide/jetbrains\`",
212+
slackhook: slackHook["ide-jobs"],
203213
},
204214
CONTENTSERVICE_TEST: {
205215
phase: "run-cs-component-tests",
206216
makeTarget: "run-cs-component-tests",
207-
description: "Run the test for content-service component",
217+
description: "Run the \`content-service\` tests \`test/tests/components/content-service\`",
208218
},
209219
DB_TEST: {
210220
phase: "run-db-component-tests",
211221
makeTarget: "run-db-component-tests",
212-
description: "Run the test for database component",
222+
description: "Run the \`database\` tests \`test/tests/components/database\`",
213223
},
214224
IMAGEBUILDER_TEST: {
215225
phase: "run-ib-component-tests",
216226
makeTarget: "run-ib-component-tests",
217-
description: "Run the test for image-builder component",
227+
description: "Run the \`image-builder\` tests \`test/tests/components/image-builder\`",
218228
},
219229
SERVER_TEST: {
220230
phase: "run-server-component-tests",
221231
makeTarget: "run-server-component-tests",
222-
description: "Run the test for server component",
232+
description: "Run the \`server\` tests \`test/tests/components/server\`",
223233
},
224234
WS_DAEMON_TEST: {
225235
phase: "run-wsd-component-tests",
226236
makeTarget: "run-wsd-component-tests",
227-
description: "Run the test for ws-daemon component",
237+
description: "Run the \`ws-daemon\` tests \`test/tests/components/ws-daemon\`",
238+
slackhook: slackHook["workspace-jobs"],
228239
},
229240
WS_MNGR_TEST: {
230241
phase: "run-wsm-component-tests",
231242
makeTarget: "run-wsm-component-tests",
232-
description: "Run the test for ws-manager component",
243+
description: "Run the \`ws-manager\` tests \`test/tests/components/ws-manager\`",
244+
slackhook: slackHook["workspace-jobs"],
233245
},
234246
}
235247

@@ -246,27 +258,38 @@ installerTests(TEST_CONFIGURATIONS[testConfig]).catch((err) => {
246258

247259
export async function installerTests(config: TestConfig) {
248260
console.log(config.DESCRIPTION);
249-
// these phases set up the infrastructure
261+
250262
werft.phase(`create-${cloud}-infra`, `Create the infrastructure in ${cloud}`);
263+
251264
for (let phase of config.PHASES) {
252265
const phaseSteps = INFRA_PHASES[phase];
253266
const ret = callMakeTargets(phaseSteps.phase, phaseSteps.description, phaseSteps.makeTarget);
254267
if (ret) {
255268
// there is not point in continuing if one stage fails for infra setup
256-
werft.fail(`create-${cloud}-infra`, "Cluster creation failed");
257-
break;
269+
const err: Error = new Error("Cluster creation failed")
270+
werft.fail(`create-${cloud}-infra`, err.message);
271+
272+
sendFailureSlackAlert(phaseSteps.description, err, slackHook["self-hosted-jobs"]).catch((error: Error) => {
273+
console.error("Failed to send message to Slack", error);
274+
});
275+
276+
throw err
258277
}
259278
}
260279
werft.done(`create-${cloud}-infra`);
261280

262-
if (upgrade === "true") {
281+
if (upgrade === "true") {
263282
// we could run integration tests in the current setup
264283
// but since we run nightly tests on unstable setups, feels unnecessary
265284
// runIntegrationTests()
266285

267286
const upgradePhase = INFRA_PHASES["KOTS_UPGRADE"];
268287
const ret = callMakeTargets(upgradePhase.phase, upgradePhase.description, upgradePhase.makeTarget);
269288
if (ret) {
289+
sendFailureSlackAlert(upgradePhase.description, new Error("Upgrade test failed"), slackHook["self-hosted-jobs"]).catch((error: Error) => {
290+
console.error("Failed to send message to Slack", error);
291+
});
292+
270293
return;
271294
}
272295
}
@@ -289,6 +312,11 @@ if (upgrade === "true") {
289312
`werft log result -d "self-hosted preview url" url "https://${process.env["TF_VAR_TEST_ID"]}.tests.gitpod-self-hosted.com"`,
290313
);
291314

315+
sendPreviewSlackAlert().catch((error: Error) => {
316+
console.error("Failed to send message to Slack", error);
317+
});
318+
319+
292320
exec(`werft log result -d "Terraform state" url "Terraform state file name is ${process.env["TF_VAR_TEST_ID"]}"`);
293321

294322
werft.done("print-output");
@@ -302,12 +330,15 @@ function runIntegrationTests() {
302330
werft.phase("run-integration-tests", "Run all existing integration tests");
303331
for (let test in TESTS) {
304332
const testPhase = TESTS[test];
305-
// todo(nvn): handle the test failures by alerting teams
306333
const ret = callMakeTargets(testPhase.phase, testPhase.description, testPhase.makeTarget);
307334
if (ret) {
308335
exec(
309336
`werft log result -d "failed test" url "${testPhase.description}(Phase ${testPhase.phase}) failed. Please refer logs."`,
310337
);
338+
339+
sendFailureSlackAlert(testPhase.description, new Error("Integration test failed"), testPhase.slackhook).catch((error: Error) => {
340+
console.error("Failed to send message to Slack", error);
341+
});
311342
}
312343
}
313344

@@ -383,3 +414,120 @@ function cleanup() {
383414

384415
return ret;
385416
}
417+
418+
export async function sendFailureSlackAlert(phase: string, err: Error, hook: string): Promise<void> {
419+
if (typeof hook === 'undefined' || hook === null) {
420+
return
421+
}
422+
423+
const repo = context.Repository.host + "/" + context.Repository.owner + "/" + context.Repository.repo;
424+
const data = JSON.stringify({
425+
blocks: [
426+
{
427+
type: "section",
428+
text: {
429+
type: "mrkdwn",
430+
text: ":X: *self-hosted test failure*\n_Test configuration:_ `" + config.DESCRIPTION + "`\n_Build:_ `" + context.Name + "`",
431+
},
432+
accessory: {
433+
type: "button",
434+
text: {
435+
type: "plain_text",
436+
text: "Go to Werft",
437+
emoji: true
438+
},
439+
value: "click_me_123",
440+
url: "https://werft.gitpod-dev.com/job/" + context.Name,
441+
action_id: "button-action"
442+
}
443+
},
444+
{
445+
type: "section",
446+
fields: [
447+
{
448+
type: "mrkdwn",
449+
text: "*Failed step:*\n" + phase + "\n",
450+
},
451+
{
452+
type: "mrkdwn",
453+
text: "*Error:*\n" + err + "\n",
454+
},
455+
]
456+
},
457+
],
458+
});
459+
return new Promise((resolve, reject) => {
460+
const req = https.request(
461+
{
462+
hostname: "hooks.slack.com",
463+
port: 443,
464+
path: hook,
465+
method: "POST",
466+
headers: {
467+
"Content-Type": "application/json",
468+
"Content-Length": data.length,
469+
},
470+
},
471+
() => resolve(),
472+
);
473+
req.on("error", (error: Error) => reject(error));
474+
req.write(data);
475+
req.end();
476+
});
477+
}
478+
479+
export async function sendPreviewSlackAlert(): Promise<void> {
480+
const data = JSON.stringify({
481+
blocks: [
482+
{
483+
type: "section",
484+
text: {
485+
type: "mrkdwn",
486+
text: ":white_check_mark: *self-hosted preview environment\n_Test configuration:_ `" + config.DESCRIPTION + "`*\n_Build:_ `" + context.Name + "`",
487+
},
488+
accessory: {
489+
type: "button",
490+
text: {
491+
type: "plain_text",
492+
text: "Go to Werft",
493+
emoji: true
494+
},
495+
value: "click_me_123",
496+
url: "https://werft.gitpod-dev.com/job/" + context.Name,
497+
action_id: "button-action"
498+
}
499+
},
500+
{
501+
type: "section",
502+
fields: [
503+
{
504+
type: "mrkdwn",
505+
text: "*URL:*\n<https://" + process.env["TF_VAR_TEST_ID"] + ".tests.gitpod-self-hosted.com|Access preview setup>",
506+
},
507+
{
508+
type: "mrkdwn",
509+
text: "*Terraform workspace:*\n`" + process.env["TF_VAR_TEST_ID"] + "`\n",
510+
},
511+
]
512+
},
513+
],
514+
});
515+
return new Promise((resolve, reject) => {
516+
const req = https.request(
517+
{
518+
hostname: "hooks.slack.com",
519+
port: 443,
520+
path: process.env.SH_SLACK_NOTIFICATION_PATH.trim(),
521+
method: "POST",
522+
headers: {
523+
"Content-Type": "application/json",
524+
"Content-Length": data.length,
525+
},
526+
},
527+
() => resolve(),
528+
);
529+
req.on("error", (error: Error) => reject(error));
530+
req.write(data);
531+
req.end();
532+
});
533+
}

‎.werft/self-hosted-installer-tests.yaml

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,21 @@ pod:
111111
secretKeyRef:
112112
name: aws-credentials
113113
key: aws-region
114+
- name: IDE_SLACK_NOTIFICATION_PATH
115+
valueFrom:
116+
secretKeyRef:
117+
name: slack-webhook-urls
118+
key: ide_jobs
119+
- name: WS_SLACK_NOTIFICATION_PATH
120+
valueFrom:
121+
secretKeyRef:
122+
name: slack-webhook-urls
123+
key: workspace_jobs
124+
- name: SH_SLACK_NOTIFICATION_PATH
125+
valueFrom:
126+
secretKeyRef:
127+
name: slack-webhook-urls
128+
key: self_hosted_jobs
114129
command:
115130
- bash
116131
- -c

‎install/tests/Makefile

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -214,8 +214,13 @@ delete-cm-setup: sleeptime=$(if $(time_to_sleep_$(cloud)),$(time_to_sleep_$(clou
214214
delete-cm-setup:
215215
sleep 180 && kubectl --kubeconfig=${KUBECONFIG} delete pods --all -n cert-manager && sleep ${sleeptime};
216216

217+
gitpod-debug-info:
218+
@echo "Gitpod is not ready"
219+
@kubectl get pods -n gitpod
220+
@kubectl get certificate -n gitpod
221+
217222
check-kots-app:
218-
kubectl kots get --kubeconfig=${KUBECONFIG} app gitpod -n gitpod | grep gitpod | awk '{print $$2}' | grep "ready" || { echo "Gitpod is not ready"; exit 1; }
223+
kubectl kots get --kubeconfig=${KUBECONFIG} app gitpod -n gitpod | grep gitpod | awk '{print $$2}' | grep "ready" || { $(MAKE) gitpod-debug-info; exit 1; }
219224

220225
check-gitpod-installation: delete-cm-setup check-kots-app check-env-sub-domain
221226
@echo "Curling http://${TF_VAR_TEST_ID}.tests.gitpod-self-hosted.com/api/version"
@@ -244,7 +249,7 @@ run-ib-component-tests:
244249
$(call runtests,"test/tests/components/image-builder/")
245250

246251
run-server-component-tests:
247-
$(call runtests,"test/tests/components/content-service/")
252+
$(call runtests,"test/tests/components/server/")
248253

249254
run-wsd-component-tests:
250255
$(call runtests,"test/tests/components/ws-daemon/")

0 commit comments

Comments
 (0)