Skip to content

Add self-hosted upgrade tests #10485

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jun 21, 2022
Merged

Add self-hosted upgrade tests #10485

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 41 additions & 35 deletions .werft/build.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,58 +1,64 @@
import * as fs from 'fs';
import { SpanStatusCode } from '@opentelemetry/api';
import { Werft } from './util/werft';
import { reportBuildFailureInSlack } from './util/slack';
import * as Tracing from './observability/tracing'
import * as VM from './vm/vm'
import { buildAndPublish } from './jobs/build/build-and-publish';
import { validateChanges } from './jobs/build/validate-changes';
import { prepare } from './jobs/build/prepare';
import { deployToPreviewEnvironment } from './jobs/build/deploy-to-preview-environment';
import { triggerIntegrationTests } from './jobs/build/trigger-integration-tests';
import { jobConfig } from './jobs/build/job-config';
import { typecheckWerftJobs } from './jobs/build/typecheck-werft-jobs';
import * as fs from "fs";
import { SpanStatusCode } from "@opentelemetry/api";
import { Werft } from "./util/werft";
import { reportBuildFailureInSlack } from "./util/slack";
import * as Tracing from "./observability/tracing";
import * as VM from "./vm/vm";
import { buildAndPublish } from "./jobs/build/build-and-publish";
import { validateChanges } from "./jobs/build/validate-changes";
import { prepare } from "./jobs/build/prepare";
import { deployToPreviewEnvironment } from "./jobs/build/deploy-to-preview-environment";
import { triggerIntegrationTests } from "./jobs/build/trigger-integration-tests";
import { triggerUpgradeTests } from "./jobs/build/self-hosted-upgrade-tests";
import { jobConfig } from "./jobs/build/job-config";
import { typecheckWerftJobs } from "./jobs/build/typecheck-werft-jobs";

// Will be set once tracing has been initialized
let werft: Werft
const context: any = JSON.parse(fs.readFileSync('context.json').toString());
let werft: Werft;
const context: any = JSON.parse(fs.readFileSync("context.json").toString());

Tracing.initialize()
.then(() => {
werft = new Werft("build")
werft = new Werft("build");
})
.then(() => run(context))
.catch((err) => {
werft.rootSpan.setStatus({
code: SpanStatusCode.ERROR,
message: err
})
message: err,
});

console.log('Error', err)
console.log("Error", err);

if (context.Repository.ref === "refs/heads/main") {
reportBuildFailureInSlack(context, err).catch((error: Error) => {
console.error("Failed to send message to Slack", error)
console.error("Failed to send message to Slack", error);
});
}

// Explicitly not using process.exit as we need to flush tracing, see tracing.js
process.exitCode = 1
process.exitCode = 1;
})
.finally(() => {
werft.phase("Stop kubectl port forwards", "Stopping kubectl port forwards")
VM.stopKubectlPortForwards()
werft.phase("Stop kubectl port forwards", "Stopping kubectl port forwards");
VM.stopKubectlPortForwards();

werft.phase("Flushing telemetry", "Flushing telemetry before stopping job")
werft.endAllSpans()
})
werft.phase("Flushing telemetry", "Flushing telemetry before stopping job");
werft.endAllSpans();
});

async function run(context: any) {
const config = jobConfig(werft, context)
const config = jobConfig(werft, context);

await validateChanges(werft, config)
await prepare(werft, config)
await typecheckWerftJobs(werft)
await buildAndPublish(werft, config)
await validateChanges(werft, config);
await prepare(werft, config);
if (config.withUpgradeTests) {
// this will trigger an upgrade test on a self-hosted gitpod instance on a new cluster.
await triggerUpgradeTests(werft, config, context.Owner);
return;
}
await typecheckWerftJobs(werft);
await buildAndPublish(werft, config);

if (!config.withPreview || config.publishRelease) {
werft.phase("deploy", "not deploying");
Expand All @@ -61,15 +67,15 @@ async function run(context: any) {
}

try {
await deployToPreviewEnvironment(werft, config)
await deployToPreviewEnvironment(werft, config);
} catch (e) {
// We currently don't support concurrent deployments to the same preview environment.
// Until we do we don't want errors to mark the main build as failed.
if (config.mainBuild) {
return
return;
}
throw e
throw e;
}

await triggerIntegrationTests(werft, config, context.Owner)
await triggerIntegrationTests(werft, config, context.Owner);
}
165 changes: 104 additions & 61 deletions .werft/installer-tests.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,10 @@ import { Werft } from "./util/werft";

const testConfig: string = process.argv.length > 2 ? process.argv[2] : "STANDARD_K3S_TEST";
// we can provide the version of the gitpod to install (eg: 2022.4.2)
const version: string = process.argv.length > 3 ? process.argv[3] : "";
// "-" is the default value which will install the latest version
const version: string = process.argv.length > 3 ? process.argv[3] : "-";

const channel: string = process.argv.length > 4 ? process.argv[4] : "unstable";

const makefilePath: string = join("install/tests");

Expand All @@ -16,6 +19,78 @@ interface InfraConfig {
description: string;
}

interface TestConfig {
DESCRIPTION: string;
PHASES: string[];
CLOUD: string;
}

// Each of the TEST_CONFIGURATIONS define an integration test end-to-end
// It should be a combination of multiple INFRA_PHASES, order of PHASES slice is important
const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = {
STANDARD_GKE_TEST: {
CLOUD: "gcp",
DESCRIPTION: "Deploy Gitpod on GKE, with managed DNS, and run integration tests",
PHASES: [
"STANDARD_GKE_CLUSTER",
"CERT_MANAGER",
"GCP_MANAGED_DNS",
"GENERATE_KOTS_CONFIG",
"INSTALL_GITPOD",
"CHECK_INSTALLATION",
"RUN_INTEGRATION_TESTS",
"RESULTS",
"DESTROY",
],
},
STANDARD_GKE_UPGRADE_TEST: {
CLOUD: "gcp",
DESCRIPTION: `Deploy Gitpod on GKE, and test upgrade from ${version} to latest version`,
PHASES: [
"STANDARD_GKE_CLUSTER",
"CERT_MANAGER",
"GCP_MANAGED_DNS",
"GENERATE_KOTS_CONFIG",
"INSTALL_GITPOD",
"CHECK_INSTALLATION",
"KOTS_UPGRADE",
"CHECK_INSTALLATION",
"DESTROY",
],
},
STANDARD_K3S_TEST: {
CLOUD: "gcp", // the cloud provider is still GCP
DESCRIPTION:
"Deploy Gitpod on a K3s cluster, created on a GCP instance," +
" with managed DNS and run integrations tests",
PHASES: [
"STANDARD_K3S_CLUSTER_ON_GCP",
"CERT_MANAGER",
"GENERATE_KOTS_CONFIG",
"INSTALL_GITPOD",
"CHECK_INSTALLATION",
"RUN_INTEGRATION_TESTS",
"RESULTS",
"DESTROY",
],
},
STANDARD_K3S_PREVIEW: {
CLOUD: "gcp",
DESCRIPTION: "Create a SH Gitpod preview environment on a K3s cluster, created on a GCP instance",
PHASES: [
"STANDARD_K3S_CLUSTER_ON_GCP",
"CERT_MANAGER",
"GENERATE_KOTS_CONFIG",
"INSTALL_GITPOD",
"CHECK_INSTALLATION",
"RESULTS",
],
},
};

const config: TestConfig = TEST_CONFIGURATIONS[testConfig];
const cloud: string = config.CLOUD;

// `INFRA_PHASES` describe the phases that can be mixed
// and matched to form a test configuration
// Each phase should contain a `makeTarget` which
Expand All @@ -41,14 +116,22 @@ const INFRA_PHASES: { [name: string]: InfraConfig } = {
makeTarget: "managed-dns",
description: "Sets up external-dns & cloudDNS config",
},
GENERATE_KOTS_CONFIG: {
phase: "generate-kots-config",
makeTarget: `generate-kots-config storage=${randomize("storage", cloud)} registry=${randomize(
"registry",
cloud,
)} db=${randomize("db", cloud)}`,
description: `Generate KOTS Config file`,
},
INSTALL_GITPOD_IGNORE_PREFLIGHTS: {
phase: "install-gitpod-without-preflights",
makeTarget: `kots-install channel=unstable version=${version} preflights=false`, // this is a bit of a hack, for now we pass params like this
makeTarget: `kots-install channel=${channel} version=${version} preflights=false`, // this is a bit of a hack, for now we pass params like this
description: "Install gitpod using kots community edition without preflights",
},
INSTALL_GITPOD: {
phase: "install-gitpod",
makeTarget: `kots-install channel=unstable version=${version} preflights=true`,
makeTarget: `kots-install channel=${channel} version=${version} preflights=true`,
description: "Install gitpod using kots community edition",
},
CHECK_INSTALLATION: {
Expand All @@ -57,6 +140,11 @@ const INFRA_PHASES: { [name: string]: InfraConfig } = {
makeTarget: "check-gitpod-installation",
description: "Check gitpod installation",
},
KOTS_UPGRADE: {
phase: "kots-upgrade",
makeTarget: "kots-upgrade",
description: "Upgrade Gitpod installation to latest version using KOTS CLI",
},
RUN_INTEGRATION_TESTS: {
phase: "run-integration-tests",
makeTarget: "run-tests",
Expand All @@ -74,56 +162,6 @@ const INFRA_PHASES: { [name: string]: InfraConfig } = {
},
};

interface TestConfig {
DESCRIPTION: string;
PHASES: string[];
}

// Each of the TEST_CONFIGURATIONS define an integration test end-to-end
// It should be a combination of multiple INFRA_PHASES, order of PHASES slice is important
const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = {
STANDARD_GKE_TEST: {
DESCRIPTION: "Deploy Gitpod on GKE, with managed DNS, and run integration tests",
PHASES: [
"STANDARD_GKE_CLUSTER",
"CERT_MANAGER",
"GCP_MANAGED_DNS",
"INSTALL_GITPOD",
"CHECK_INSTALLATION",
"RUN_INTEGRATION_TESTS",
"RESULTS",
"DESTROY",
],
},
STANDARD_K3S_TEST: {
DESCRIPTION:
"Deploy Gitpod on a K3s cluster, created on a GCP instance," +
" with managed DNS and run integrations tests",
PHASES: [
"STANDARD_K3S_CLUSTER_ON_GCP",
"CERT_MANAGER",
"INSTALL_GITPOD_IGNORE_PREFLIGHTS",
"CHECK_INSTALLATION",
"RUN_INTEGRATION_TESTS",
"RESULTS",
"DESTROY",
],
},
STANDARD_K3S_PREVIEW: {
DESCRIPTION: "Create a SH Gitpod preview environment on a K3s cluster, created on a GCP instance",
PHASES: [
"STANDARD_K3S_CLUSTER_ON_GCP",
"GCP_MANAGED_DNS",
"INSTALL_GITPOD_IGNORE_PREFLIGHTS",
"CHECK_INSTALLATION",
"RESULTS",
],
},
};

// TODO better way to clean up
const config: TestConfig = TEST_CONFIGURATIONS[testConfig];

if (config === undefined) {
console.log(`Unknown configuration specified: "${testConfig}", Exiting...`);
process.exit(1);
Expand All @@ -135,12 +173,6 @@ installerTests(TEST_CONFIGURATIONS[testConfig]).catch((err) => {
process.exit(1);
});

function getKubeconfig() {
const ret = exec(`make -C ${makefilePath} get-kubeconfig`);
const filename = ret.stdout.toString().split("\n").slice(1, -1);
exec(`echo ${filename}`);
}

export async function installerTests(config: TestConfig) {
console.log(config.DESCRIPTION);
for (let phase of config.PHASES) {
Expand All @@ -155,9 +187,13 @@ export async function installerTests(config: TestConfig) {
}

function callMakeTargets(phase: string, description: string, makeTarget: string) {
werft.phase(phase, description);
werft.phase(phase, `${description}`);
werft.log(phase, `calling ${makeTarget}`);

const response = exec(`make -C ${makefilePath} ${makeTarget}`, { slice: "call-make-target", dontCheckRc: true });
const response = exec(`make -C ${makefilePath} ${makeTarget}`, {
slice: "call-make-target",
dontCheckRc: true,
});

if (response.code) {
console.error(`Error: ${response.stderr}`);
Expand All @@ -170,6 +206,13 @@ function callMakeTargets(phase: string, description: string, makeTarget: string)
return response.code;
}

function randomize(resource: string, platform: string): string {
// in the follow-up PR we will add `${platform}-${resource}` as an option here to
// test against resource dependencies(storage, db, registry) for each cloud platform
const options = ["incluster"];
return options[Math.floor(Math.random() * options.length)];
}

function cleanup() {
const phase = "destroy-infrastructure";
werft.phase(phase, "Destroying all the created resources");
Expand Down
Loading