diff --git a/.werft/gke-installer-tests.yaml b/.werft/gke-installer-tests.yaml new file mode 100644 index 00000000000000..bf273203ca71de --- /dev/null +++ b/.werft/gke-installer-tests.yaml @@ -0,0 +1,67 @@ +# debug using `werft run github -f -s .werft/installer-tests.ts -j .werft/gke-installer-tests.yaml -a debug=true` +pod: + serviceAccount: werft + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: dev/workload + operator: In + values: + - "builds" + securityContext: + runAsUser: 0 + volumes: + - name: sh-playground-sa-perm + secret: + secretName: sh-playground-sa-perm + - name: sh-playground-dns-perm + secret: + secretName: sh-playground-dns-perm + containers: + - name: nightly-test + image: eu.gcr.io/gitpod-core-dev/dev/dev-environment:cw-werft-cred.0 + workingDir: /workspace + imagePullPolicy: Always + volumeMounts: + - name: sh-playground-sa-perm + mountPath: /mnt/secrets/sh-playground-sa-perm + - name: sh-playground-dns-perm # this sa is used for the DNS management + mountPath: /mnt/secrets/sh-playground-dns-perm + env: + - name: WERFT_HOST + value: "werft.werft.svc.cluster.local:7777" + - name: GOOGLE_APPLICATION_CREDENTIALS + value: "/mnt/secrets/sh-playground-sa-perm/sh-sa.json" + - name: TF_VAR_sa_creds + value: "/mnt/secrets/sh-playground-sa-perm/sh-sa.json" + - name: TF_VAR_dns_sa_creds + value: "/mnt/secrets/sh-playground-dns-perm/sh-dns-sa.json" + - name: WERFT_K8S_NAMESPACE + value: "werft" + - name: WERFT_K8S_LABEL + value: "component=werft" + - name: NODENAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + command: + - bash + - -c + - | + sleep 1 + set -Eeuo pipefail + + sudo chown -R gitpod:gitpod /workspace + sudo apt update && apt install gettext-base + + export TF_VAR_TEST_ID=$(echo $RANDOM | md5sum | head -c 5; echo) + + (cd .werft && yarn install && mv node_modules ..) | werft log slice prep + printf '{{ toJson . }}' > context.json + + npx ts-node .werft/installer-tests.ts "STANDARD_GKE_TEST" +# The bit below makes this a cron job +plugins: + cron: "15 4 * * *" diff --git a/.werft/installer-tests.ts b/.werft/installer-tests.ts new file mode 100644 index 00000000000000..ba66df6bef78ff --- /dev/null +++ b/.werft/installer-tests.ts @@ -0,0 +1,204 @@ +import { join } from "path"; +import { exec } from "./util/shell"; +import { Werft } from "./util/werft"; + +const testConfig: string = process.argv.length > 2 ? process.argv[2] : "STANDARD_K3S_TEST"; +// we can provide the version of the gitpod to install (eg: 2022.4.2) +const version: string = process.argv.length > 3 ? process.argv[3] : ""; + +const makefilePath: string = join("install/tests"); + +const werft = new Werft("installer-nightly-tests"); + +interface InfraConfig { + phase: string; + makeTarget: string; + description: string; +} + +// `INFRA_PHASES` describe the phases that can be mixed +// and matched to form a test configuration +// Each phase should contain a `makeTarget` which +// corresponds to a target in the Makefile in ./nightly-tests/Makefile +const INFRA_PHASES: { [name: string]: InfraConfig } = { + STANDARD_GKE_CLUSTER: { + phase: "create-std-gke-cluster", + makeTarget: "gke-standard-cluster", + description: "Creating a GKE cluster with 1 nodepool each for workspace and server", + }, + STANDARD_K3S_CLUSTER_ON_GCP: { + phase: "create-std-k3s-cluster", + makeTarget: "k3s-standard-cluster", + description: "Creating a k3s cluster on GCP with 1 node", + }, + CERT_MANAGER: { + phase: "setup-cert-manager", + makeTarget: "cert-manager", + description: "Sets up cert-manager and optional cloud dns secret", + }, + GCP_MANAGED_DNS: { + phase: "setup-external-dns-with-cloud-dns", + makeTarget: "managed-dns", + description: "Sets up external-dns & cloudDNS config", + }, + INSTALL_GITPOD_IGNORE_PREFLIGHTS: { + phase: "install-gitpod-without-preflights", + makeTarget: `kots-install channel=unstable version=${version} preflights=false`, // this is a bit of a hack, for now we pass params like this + description: "Install gitpod using kots community edition without preflights", + }, + INSTALL_GITPOD: { + phase: "install-gitpod", + makeTarget: `kots-install channel=unstable version=${version} preflights=true`, + description: "Install gitpod using kots community edition", + }, + CHECK_INSTALLATION: { + // this is a basic test for the Gitpod setup + phase: "check-gitpod-installation", + makeTarget: "check-gitpod-installation", + description: "Check gitpod installation", + }, + RUN_INTEGRATION_TESTS: { + phase: "run-integration-tests", + makeTarget: "run-tests", + description: "Runs the existing integration tests on Gitpod", + }, + DESTROY: { + phase: "destroy", + makeTarget: "cleanup", + description: "Destroy the created infrastucture", + }, + RESULTS: { + phase: "get-results", + makeTarget: "get-results", + description: "Get the result of the setup", + }, +}; + +interface TestConfig { + DESCRIPTION: string; + PHASES: string[]; +} + +// Each of the TEST_CONFIGURATIONS define an integration test end-to-end +// It should be a combination of multiple INFRA_PHASES, order of PHASES slice is important +const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = { + STANDARD_GKE_TEST: { + DESCRIPTION: "Deploy Gitpod on GKE, with managed DNS, and run integration tests", + PHASES: [ + "STANDARD_GKE_CLUSTER", + "CERT_MANAGER", + "GCP_MANAGED_DNS", + "INSTALL_GITPOD", + "CHECK_INSTALLATION", + "RUN_INTEGRATION_TESTS", + "RESULTS", + "DESTROY", + ], + }, + STANDARD_K3S_TEST: { + DESCRIPTION: + "Deploy Gitpod on a K3s cluster, created on a GCP instance," + + " with managed DNS and run integrations tests", + PHASES: [ + "STANDARD_K3S_CLUSTER_ON_GCP", + "CERT_MANAGER", + "INSTALL_GITPOD_IGNORE_PREFLIGHTS", + "CHECK_INSTALLATION", + "RUN_INTEGRATION_TESTS", + "RESULTS", + "DESTROY", + ], + }, + STANDARD_K3S_PREVIEW: { + DESCRIPTION: "Create a SH Gitpod preview environment on a K3s cluster, created on a GCP instance", + PHASES: [ + "STANDARD_K3S_CLUSTER_ON_GCP", + "GCP_MANAGED_DNS", + "INSTALL_GITPOD_IGNORE_PREFLIGHTS", + "CHECK_INSTALLATION", + "RESULTS", + ], + }, +}; + +// TODO better way to clean up +const config: TestConfig = TEST_CONFIGURATIONS[testConfig]; + +if (config === undefined) { + console.log(`Unknown configuration specified: "${testConfig}", Exiting...`); + process.exit(1); +} + +installerTests(TEST_CONFIGURATIONS[testConfig]).catch((err) => { + cleanup(); + console.error(err); + process.exit(1); +}); + +function getKubeconfig() { + const ret = exec(`make -C ${makefilePath} get-kubeconfig`); + const filename = ret.stdout.toString().split("\n").slice(1, -1); + exec(`echo ${filename}`); +} + +export async function installerTests(config: TestConfig) { + console.log(config.DESCRIPTION); + for (let phase of config.PHASES) { + const phaseSteps = INFRA_PHASES[phase]; + const ret = callMakeTargets(phaseSteps.phase, phaseSteps.description, phaseSteps.makeTarget); + if (ret) { + // there is not point in continuing if one stage fails + // TODO: maybe add failable, phases + break; + } + } +} + +function callMakeTargets(phase: string, description: string, makeTarget: string) { + werft.phase(phase, description); + + const response = exec(`make -C ${makefilePath} ${makeTarget}`, { slice: "call-make-target", dontCheckRc: true }); + + if (response.code) { + console.error(`Error: ${response.stderr}`); + werft.fail(phase, "Operation failed"); + } else { + werft.log(phase, response.stdout.toString()); + werft.done(phase); + } + + return response.code; +} + +function cleanup() { + const phase = "destroy-infrastructure"; + werft.phase(phase, "Destroying all the created resources"); + + const response = exec(`make -C ${makefilePath} cleanup`, { slice: "run-terrafrom-destroy", dontCheckRc: true }); + + // if the destroy command fail, we check if any resources are pending to be removed + // if nothing is yet to be cleaned, we return with success + // else we list the rest of the resources to be cleaned up + if (response.code) { + console.error(`Error: ${response.stderr}`); + + const existingState = exec(`make -C ${makefilePath} list-state`, { slice: "get-uncleaned-resources" }); + if (existingState.code) { + console.error(`Error: Failed to check for the left over resources`); + } + + const itemsTobeCleaned = existingState.stdout.toString().split("\n").slice(1, -1); + + if (itemsTobeCleaned.length == 0) { + console.log("Eventhough it was not a clean run, all resources has been cleaned. Nothing to do"); + werft.done(phase); + return; + } + + console.log(`Cleanup the following resources manually: ${itemsTobeCleaned}`); + + werft.fail(phase, "Destroying of resources failed"); + } + + return response.code; +} diff --git a/.werft/k3s-installer-tests.yaml b/.werft/k3s-installer-tests.yaml new file mode 100644 index 00000000000000..e275daa4448291 --- /dev/null +++ b/.werft/k3s-installer-tests.yaml @@ -0,0 +1,69 @@ +# debug using `werft run github -f -s .werft/installer-tests.ts -j .werft/k3s-installer-tests.yaml -a debug=true` +pod: + serviceAccount: werft + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: dev/workload + operator: In + values: + - "builds" + securityContext: + runAsUser: 0 + volumes: + - name: sh-playground-sa-perm + secret: + secretName: sh-playground-sa-perm + - name: sh-playground-dns-perm + secret: + secretName: sh-playground-dns-perm + containers: + - name: nightly-test + image: eu.gcr.io/gitpod-core-dev/dev/dev-environment:cw-werft-cred.0 + workingDir: /workspace + imagePullPolicy: Always + volumeMounts: + - name: sh-playground-sa-perm + mountPath: /mnt/secrets/sh-playground-sa-perm + - name: sh-playground-dns-perm # this sa is used for the DNS management + mountPath: /mnt/secrets/sh-playground-dns-perm + env: + - name: WERFT_HOST + value: "werft.werft.svc.cluster.local:7777" + - name: GOOGLE_APPLICATION_CREDENTIALS + value: "/mnt/secrets/sh-playground-sa-perm/sh-sa.json" + - name: TF_VAR_sa_creds + value: "/mnt/secrets/sh-playground-sa-perm/sh-sa.json" + - name: TF_VAR_dns_sa_creds + value: "/mnt/secrets/sh-playground-dns-perm/sh-dns-sa.json" + - name: WERFT_K8S_NAMESPACE + value: "werft" + - name: WERFT_K8S_LABEL + value: "component=werft" + - name: NODENAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + command: + - bash + - -c + - | + sleep 1 + set -Eeuo pipefail + + sudo chown -R gitpod:gitpod /workspace + sudo apt update && apt install gettext-base + + curl -sLS https://get.k3sup.dev | sh + + export TF_VAR_TEST_ID=$(echo $RANDOM | md5sum | head -c 5; echo) + + (cd .werft && yarn install && mv node_modules ..) | werft log slice prep + printf '{{ toJson . }}' > context.json + + npx ts-node .werft/installer-tests.ts "STANDARD_K3S_TEST" +# The bit below makes this a cron job +plugins: + cron: "15 3 * * *" diff --git a/install/infra/terraform/gke/README.md b/install/infra/terraform/gke/README.md new file mode 100644 index 00000000000000..c1966509ed9d0f --- /dev/null +++ b/install/infra/terraform/gke/README.md @@ -0,0 +1,83 @@ +# GKE terraform module + +This is a terraform module currently used in the automated installation tests +for Gitpod. At successful completion, this module creates the following: + +1. A [`GKE`](https://cloud.google.com/kubernetes-engine) cluster by user + provided name `` with the following nodepools: + 1. `workspaces-` for workspace workloads + 1. `services-` for IDE and meta workloads +1. A dedicated `vpc` and `subnet` for the resources +1. `kubeconfig` data populated in a user defined `kubeconfig` variable + +## Requirements + +1. `terraform` >= `v1.1.7` + +## Providers + +1. [`google`](https://registry.terraform.io/providers/hashicorp/google/latest/docs) + +## Inputs + + +| Argument | Description | Required | Default | +|---------------------------|----------------------------------------------------------------------------------------------------------------------------------------------|----------|--------------------| +| `project` | The project ID to create the cluster. | `true` | | +| `kubeconfig` | Path to write the kubeconfig output to. | `false` | `./kubeconfig` | +| `region` | The region to create the cluster. | `true` | | +| `zone` | The zone to create the cluster in. Eg. `europe-west1-b`. If not provided, it will create a regional(high-availability) cluster| `false` | null | +| `kubernetes_version` | Kubernetes version to be setup | `false` | `1.22.8-gke.201` | +| `name` | The name of the cluster and suffix for other resources | `false` | `gitpod` | +| `workspaces_machine_type` | Type of the node compute engines for workspace nodepool. | `false` | `n2-standard-8` | +| `services_machine_type` | Type of the node compute engines for meta and IDE nodepool. | `false` | `n2-standard-4` | +| `max_count` | Maximum number of nodes in the NodePool. Must be >= min_node_count. | `false` | `50` | +| `disk_size_gb` | Disk size to provision in nodes. | `false` | `100` | +| `credentials` | Path to the JSON file storing Google service account credentials, if left empty, `tf` will look for `GOOGLE_APPLICATION_CREDENTIALS` env var | `false` | | + + +## Outputs + + +| Argument | +|-----------------------| +| `kubernetes_endpoint` | +| `client_token` | +| `ca_certificate` | +| `kubeconfig` | + + +## Usage + +Make sure you have a `GCP` account and credentials in `JSON` format to a service +account with atleast the following permissions: +1. Compute Admin +1. Compute Network Admin +1. Kubernetes Engine Admin +1. Kubernetes Engine Developer +1. Service Account User +1. Storage Admin + +Assign it to the following environment variable: + +``` sh +export GOOGLE_APPLICATION_CREDENTAILS=/path/to/service-account-json-creds +``` + +Create a `Cloud storage bucket` in the same project and region and specified +above, by name `gitpod-gke`. If you want to use a different name, edit the name +manually in the `main.tf` file. + +Run the following commands: + +``` sh +terraform init +terraform apply # provide the values that are prompted for +``` + + +## Cleanup + +``` sh +terraform destroy +``` diff --git a/install/infra/terraform/gke/main.tf b/install/infra/terraform/gke/main.tf new file mode 100644 index 00000000000000..19794a8cb8a8ad --- /dev/null +++ b/install/infra/terraform/gke/main.tf @@ -0,0 +1,168 @@ +terraform { + required_version = ">= 1.0.3" +} + +terraform { + backend "gcs" { + bucket = "gitpod-gke" + prefix = "tf-state" + } +} + +provider "google" { + project = var.project + credentials = var.credentials + region = var.region + zone = var.zone +} + +resource "google_compute_network" "vpc" { + name = "vpc-${var.name}" + auto_create_subnetworks = "false" +} + +resource "google_compute_subnetwork" "subnet" { + name = "subnet-${var.name}" + region = var.region + network = google_compute_network.vpc.name + ip_cidr_range = "10.255.0.0/16" + + secondary_ip_range { + range_name = "cluster-secondary-ip-range" + ip_cidr_range = "10.0.0.0/12" + } + + secondary_ip_range { + range_name = "services-secondary-ip-range" + ip_cidr_range = "10.64.0.0/12" + } +} + +resource "google_container_cluster" "gitpod-cluster" { + name = "c${var.name}" + location = var.zone == null ? var.region : var.zone + + cluster_autoscaling { + enabled = true + + resource_limits { + resource_type = "cpu" + minimum = 2 + maximum = 16 + } + + resource_limits { + resource_type = "memory" + minimum = 4 + maximum = 64 + } + } + + min_master_version = var.kubernetes_version + # the default nodepool is used as the services nodepool + remove_default_node_pool = false + node_config { + oauth_scopes = [ + "https://www.googleapis.com/auth/cloud-platform" + ] + + labels = { + "gitpod.io/workload_meta" = true + "gitpod.io/workload_ide" = true + } + + preemptible = false + image_type = "COS_CONTAINERD" + disk_type = "pd-standard" + disk_size_gb = var.disk_size_gb + machine_type = var.services_machine_type + tags = ["gke-node", "${var.project}-gke"] + metadata = { + disable-legacy-endpoints = "true" + } + } + + initial_node_count = 1 + release_channel { + channel = "UNSPECIFIED" + } + + ip_allocation_policy { + cluster_secondary_range_name = "cluster-secondary-ip-range" + services_secondary_range_name = "services-secondary-ip-range" + } + + network_policy { + enabled = true + provider = "CALICO" + } + + addons_config { + http_load_balancing { + disabled = false + } + + horizontal_pod_autoscaling { + disabled = false + } + } + + network = google_compute_network.vpc.name + subnetwork = google_compute_subnetwork.subnet.name +} + +resource "google_container_node_pool" "workspaces" { + name = "workspaces-${var.name}" + location = google_container_cluster.gitpod-cluster.location + cluster = google_container_cluster.gitpod-cluster.name + version = var.kubernetes_version // kubernetes version + initial_node_count = 1 + max_pods_per_node = 110 + + node_config { + oauth_scopes = [ + "https://www.googleapis.com/auth/cloud-platform" + ] + + labels = { + "gitpod.io/workload_workspace_services" = true + "gitpod.io/workload_workspace_regular" = true + "gitpod.io/workload_workspace_headless" = true + } + + preemptible = false + image_type = "UBUNTU_CONTAINERD" + disk_type = "pd-standard" + disk_size_gb = var.disk_size_gb + machine_type = var.workspaces_machine_type + tags = ["gke-node", "${var.project}-gke"] + metadata = { + disable-legacy-endpoints = "true" + } + } + + autoscaling { + min_node_count = 1 + max_node_count = var.max_count + } + + management { + auto_repair = true + auto_upgrade = false + } +} + +module "gke_auth" { + depends_on = [google_container_node_pool.workspaces] + + source = "terraform-google-modules/kubernetes-engine/google//modules/auth" + + project_id = var.project + location = google_container_cluster.gitpod-cluster.location + cluster_name = "c${var.name}" +} + +resource "local_file" "kubeconfig" { + filename = var.kubeconfig + content = module.gke_auth.kubeconfig_raw +} diff --git a/install/infra/terraform/gke/output.tf b/install/infra/terraform/gke/output.tf new file mode 100644 index 00000000000000..470911d8a97705 --- /dev/null +++ b/install/infra/terraform/gke/output.tf @@ -0,0 +1,19 @@ +output "kubernetes_endpoint" { + sensitive = true + value = module.gke_auth.host +} + +output "client_token" { + sensitive = true + value = module.gke_auth.token +} + +output "ca_certificate" { + sensitive = true + value = module.gke_auth.cluster_ca_certificate +} + +output "kubeconfig" { + sensitive = true + value = module.gke_auth.kubeconfig_raw +} diff --git a/install/infra/terraform/gke/variables.tf b/install/infra/terraform/gke/variables.tf new file mode 100644 index 00000000000000..db80ab6a9bbfcc --- /dev/null +++ b/install/infra/terraform/gke/variables.tf @@ -0,0 +1,62 @@ +variable "project" { + type = string + description = "The project ID to create the cluster." +} + +variable "kubeconfig" { + type = string + description = "Path to write the kubeconfig output to" + default = "./kubeconfig" +} + +variable "region" { + type = string + description = "The region to create the cluster." +} + +variable "zone" { + type = string + description = "The zone to create the cluster in. Eg: `europs-west1-b`. If not specified, A regional cluster(high-availability) will be created." + default = null +} + +variable "kubernetes_version" { + type = string + description = "Kubernetes version to be setup" + default = "1.22.8-gke.201" +} + +variable "name" { + type = string + description = "The name of the cluster." + default = "gitpod" +} + +variable "workspaces_machine_type" { + type = string + description = "Type of the node compute engines for workspace nodepool." + default = "n2-standard-8" +} + +variable "services_machine_type" { + type = string + description = "Type of the node compute engines for services nodepool." + default = "n2-standard-4" +} + +variable "max_count" { + type = number + description = "Maximum number of nodes in the NodePool. Must be >= 1." + default = 50 +} + +variable "disk_size_gb" { + type = number + description = "Size of the node's disk." + default = 100 +} + +variable "credentials" { + description = "Path to the JSON file storing Google service account credentials" + default = "" +} diff --git a/install/infra/terraform/k3s/README.md b/install/infra/terraform/k3s/README.md new file mode 100644 index 00000000000000..72826b984a248d --- /dev/null +++ b/install/infra/terraform/k3s/README.md @@ -0,0 +1,65 @@ +# k3s terraform module + +This is a terraform module currently used in the automated installation tests +for Gitpod. At successful completion, this module creates the following: + +1. A GCP VM instance of type `n2d-standard-4` and with `ubuntu` image, along with a service account to give auth scopes +1. A single node [`k3s`](https://k3s.io/) cluster with [`k3sup`](https://github.com/alexellis/k3sup) + +### Requirements + +1. `terraform` >= `v1.1.7` +1. [`k3sup`](https://github.com/alexellis/k3sup#download-k3sup-tldr) + ```sh + curl -sLS https://get.k3sup.dev | sh + sudo install k3sup /usr/local/bin/ + ``` + +## Providers + +1. [`google`](https://registry.terraform.io/providers/hashicorp/google/latest/docs) + +## Input + +| Argument | Description | Required | Default | +|---------------|----------------------------------------------------------------------------------------------------------------------------------------------|----------|------------------| +| `gcp_project` | The project ID to create the VM in. | `true` | | +| `kubeconfig` | Path to write the kubeconfig output to. | `false` | `./kubeconfig` | +| `gcp_region` | The region to create the VM. | `false` | `europe-west1` | +| `gcp_zone` | The GCP zone to create the VM in. | `false` | `europe-west1-b` | +| `name` | Prefix name for the nodes and firewall | `false` | `k3s` | +| `credentials` | Path to the JSON file storing Google service account credentials, if left empty, `tf` will look for `GOOGLE_APPLICATION_CREDENTIALS` env var | `false` | | + +## Output + +| Argument | +|-----------------------| +| `kubernetes_endpoint` | +| `kubeconfig` | + +## Usage + +Create a service account with at least the following access scopes: +- Compute Admin +- Create Service Accounts +- Delete Service Accounts +- Service Account User +- Storage Admin + +Get the credentials of the SA in `JSON` format and export as a variable as folows: + +``` sh +export GOOGLE_APPLICATION_CREDENTIALS=/path/to/credentials +``` +```sh +terraform init --upgrade +terraform apply # provide the values that are prompted for +``` + +You will find the `kubeconfig` populated to the path you specified or `./kubeconfig` by default. + +## Cleanup + +```sh +terraform destroy +``` diff --git a/install/infra/terraform/k3s/main.tf b/install/infra/terraform/k3s/main.tf new file mode 100644 index 00000000000000..bc6790a0b0c2ec --- /dev/null +++ b/install/infra/terraform/k3s/main.tf @@ -0,0 +1,186 @@ +terraform { + required_providers { + tls = { + source = "hashicorp/tls" + version = "3.1.0" + } + } +} + +terraform { + backend "gcs" { + bucket = "gitpod-k3s" + prefix = "tf-state" + } +} + +provider "google" { + credentials = var.credentials + project = var.gcp_project + region = var.gcp_region + zone = var.gcp_zone +} + +provider "google" { + alias = "dns" + credentials = var.dns_sa_creds +} + +resource "google_service_account" "gcp_instance" { + account_id = "gcp-k3s-compute" + display_name = "Service Account" +} + +resource "tls_private_key" "ssh" { + algorithm = "RSA" + rsa_bits = 4096 +} + +resource "local_file" "ssh_private_key_pem" { + content = tls_private_key.ssh.private_key_pem + filename = ".ssh/google_compute_engine" + file_permission = "0600" +} + +resource "google_compute_firewall" "k3s-firewall" { + name = "firewall-${var.name}" + network = "default" + + allow { + protocol = "tcp" + ports = ["6443"] + } + + allow { + protocol = "tcp" + ports = ["80"] + } + + allow { + protocol = "tcp" + ports = ["443"] + } + + target_tags = ["k3s"] + + source_ranges = ["0.0.0.0/0"] +} + +resource "google_compute_instance" "k3s_master_instance" { + name = "master-${var.name}" + machine_type = "n2d-standard-4" + tags = ["k3s", "k3s-master", "http-server", "https-server", "allow-ssh"] + + boot_disk { + initialize_params { + image = "ubuntu-2004-focal-v20220419" + size = 100 + type = "pd-ssd" + } + } + + network_interface { + network = "default" + + access_config {} + } + + depends_on = [ + google_compute_firewall.k3s-firewall, + local_file.ssh_private_key_pem, + google_service_account.gcp_instance, + ] + metadata = { + ssh-keys = "gitpod:${tls_private_key.ssh.public_key_openssh}" + } + + service_account { + # Google recommends custom service accounts that have cloud-platform scope and permissions granted via IAM Roles. + email = google_service_account.gcp_instance.email + scopes = ["cloud-platform"] + } +} + +resource "time_sleep" "wait_for_master" { + depends_on = [google_compute_instance.k3s_master_instance] + + create_duration = "60s" +} + +resource "null_resource" "k3sup_install" { + depends_on = [time_sleep.wait_for_master] + + provisioner "local-exec" { + command = < tmp_config.yml + kubectl kots install gitpod/${channel} \ + --skip-rbac-check ${version-flag} ${preflight-flag} \ + --namespace gitpod --kubeconfig=${KUBECONFIG} \ + --name gitpod --shared-password gitpod \ + --license-file ${license-file} \ + --no-port-forward \ + --config-values tmp_config.yml + +delete-cm-setup: + sleep 120 && kubectl --kubeconfig=${KUBECONFIG} delete pods --all -n cert-manager && sleep 300; + +check-kots-app: + kubectl kots get --kubeconfig=${KUBECONFIG} app gitpod -n gitpod | grep gitpod | awk '{print $$2}' | grep "ready" || { echo "Gitpod is not ready"; exit 1; } + +check-gitpod-installation: delete-cm-setup check-kots-app check-env-sub-domain + @echo "Curling http://${TF_VAR_TEST_ID}.gitpod-self-hosted.com/api/version" + curl -i -X GET http://${TF_VAR_TEST_ID}.gitpod-self-hosted.com/api/version || { echo "Curling Gitpod endpoint failed"; exit 1; } + +run-tests: + ./tests.sh ${KUBECONFIG} + +cleanup: + terraform workspace select $(TF_VAR_TEST_ID) + which ${KUBECONFIG} && terraform destroy -target=module.externaldns -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file" + which ${KUBECONFIG} && terraform destroy -target=module.certmanager -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file" + terraform destroy -target=module.gke -var kubeconfig=${KUBECONFIG} --auto-approve + terraform destroy -target=module.k3s -var kubeconfig=${KUBECONFIG} --auto-approve + +get-results: + @echo "If you have gotten this far, it means your setup succeeded" + @echo + @echo "URL of your setup is: "https://$(TF_VAR_TEST_ID).gitpod-self-hosted.com"" + @echo + @echo "Following is the KUBECONFIG you can use to connect to the cluster:" + @echo + @cat ${KUBECONFIG} + +list-state: + terraform state list +# end diff --git a/install/tests/main.tf b/install/tests/main.tf new file mode 100644 index 00000000000000..beef1a67194862 --- /dev/null +++ b/install/tests/main.tf @@ -0,0 +1,58 @@ +variable "kubeconfig" {} +variable "TEST_ID" { + default = "nightly" +} +variable "project" { + default = "sh-automated-tests" +} +variable "sa_creds" {} +variable "dns_sa_creds" {} + +terraform { + backend "gcs" { + bucket = "nightly-tests" + prefix = "tf-state" + } +} + +module "gke" { + # source = "github.com/gitpod-io/gitpod//install/infra/terraform/gke?ref=main" # we can later use tags here + source = "../infra/terraform/gke" # we can later use tags here + + name = var.TEST_ID + project = var.project + credentials = var.sa_creds + kubeconfig = var.kubeconfig + region = "europe-west1" + zone = "europe-west1-b" +} + +module "k3s" { + # source = "github.com/gitpod-io/gitpod//install/infra/terraform/k3s?ref=main" # we can later use tags here + source = "../infra/terraform/k3s" # we can later use tags here + + name = var.TEST_ID + gcp_project = var.project + credentials = var.sa_creds + kubeconfig = var.kubeconfig + dns_sa_creds = var.dns_sa_creds + dns_project = "dns-for-playgrounds" + managed_dns_zone = "gitpod-self-hosted-com" + domain_name = "${var.TEST_ID}.gitpod-self-hosted.com" +} + +module "certmanager" { + # source = "github.com/gitpod-io/gitpod//install/infra/terraform/tools/cert-manager?ref=main" + source = "../infra/terraform/tools/cert-manager" + + kubeconfig = var.kubeconfig + credentials = var.dns_sa_creds +} + +module "externaldns" { + # source = "github.com/gitpod-io/gitpod//install/infra/terraform/tools/external-dns?ref=main" + source = "../infra/terraform/tools/external-dns" + + kubeconfig = var.kubeconfig + credentials = var.dns_sa_creds +} diff --git a/install/tests/manifests/gcp-issuer.yaml b/install/tests/manifests/gcp-issuer.yaml new file mode 100644 index 00000000000000..9d13368571db9b --- /dev/null +++ b/install/tests/manifests/gcp-issuer.yaml @@ -0,0 +1,16 @@ +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: gitpod-issuer +spec: + acme: + server: https://acme-v02.api.letsencrypt.org/directory + privateKeySecretRef: + name: issuer-account-key + solvers: + - dns01: + cloudDNS: + project: "dns-for-playgrounds" + serviceAccountSecretRef: + name: "clouddns-dns01-solver" + key: "keys.json" diff --git a/install/tests/manifests/kots-config.yaml b/install/tests/manifests/kots-config.yaml new file mode 100644 index 00000000000000..7e68ea33566f06 --- /dev/null +++ b/install/tests/manifests/kots-config.yaml @@ -0,0 +1,7 @@ +apiVersion: kots.io/v1beta1 +kind: ConfigValues +spec: + values: + domain: + value: "${TF_VAR_TEST_ID}.gitpod-self-hosted.com" + data: "domain" diff --git a/install/tests/tests.sh b/install/tests/tests.sh new file mode 100755 index 00000000000000..e17ce1c61d19f9 --- /dev/null +++ b/install/tests/tests.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# Copyright (c) 2021 Gitpod GmbH. All rights reserved. +# Licensed under the GNU Affero General Public License (AGPL). +# See License-AGPL.txt in the project root for license information. + +# exclude 'e' (exit on any error) +# there are many test binaries, each can have failures + +TEST_DIR="../../test/tests" + +# shellcheck disable=SC2045 +FAILURE_COUNT=0 +CURRENT=$(pwd) +for i in $(find ${TEST_DIR} -type d -links 2 ! -empty | sort); do + # Will print */ if no directories are available + TEST_NAME=$(basename "${i}") + echo "running integration for ${TEST_NAME}" + + cd "${i}" || echo "Path invalid for ${TEST_NAME}" + go test -v ./... "-kubeconfig=$1" -namespace=gitpod 2>&0 + TEST_STATUS=$? + if [ "$TEST_STATUS" -ne "0" ]; then + FAILURE_COUNT=$((FAILURE_COUNT+1)) + echo "Test failed at $(date)" + else + echo "Test succeeded at $(date)" + fi; + + cd "${CURRENT}" || echo "Couldn't move back to test dir" +done + +if [ "$FAILURE_COUNT" -ne "0" ]; then + echo "Test suite ended with failure at $(date)" + exit $FAILURE_COUNT +fi; + +echo "Test suite ended with success at $(date)"