From b8a6f31b1bc86ef0caa3820c39c52a6c6bf44141 Mon Sep 17 00:00:00 2001 From: Nandaja Varma Date: Tue, 31 May 2022 08:57:57 +0000 Subject: [PATCH 1/4] Allowing versions --- .werft/installer-tests.ts | 10 ++- install/infra/terraform/gke/README.md | 90 +++++++++++++++++++ install/infra/terraform/gke/main.tf | 80 ++++++++--------- install/infra/terraform/gke/output.tf | 5 +- .../infra/terraform/gke/variables.auto.tfvars | 5 ++ install/infra/terraform/gke/variables.tf | 55 ++++-------- install/infra/terraform/k3s/README.md | 55 +++++++++++- install/infra/terraform/k3s/main.tf | 43 ++++----- .../infra/terraform/k3s/variables.auto.tfvars | 5 +- install/infra/terraform/k3s/variables.tf | 21 ++--- .../terraform/tools/cm-cloud-dns/README.md | 6 ++ install/tests/Makefile | 44 ++++----- install/tests/main.tf | 36 ++++---- 13 files changed, 289 insertions(+), 166 deletions(-) create mode 100644 install/infra/terraform/gke/README.md create mode 100644 install/infra/terraform/gke/variables.auto.tfvars create mode 100644 install/infra/terraform/tools/cm-cloud-dns/README.md diff --git a/.werft/installer-tests.ts b/.werft/installer-tests.ts index e864dc0e6b3a15..a67cfde849963e 100644 --- a/.werft/installer-tests.ts +++ b/.werft/installer-tests.ts @@ -2,7 +2,9 @@ import { join } from "path"; import { exec } from "./util/shell"; import { Werft } from "./util/werft"; -const testConfig: string = process.argv.length > 2 ? process.argv[2] : "gke"; +const testConfig: string = process.argv.length > 2 ? process.argv[2] : "STANDARD_K3S_TEST"; +// we can provide the version of the gitpod to install (eg: 2022.4.2) +const version: string = process.argv.length > 3 ? process.argv[3] : ""; const makefilePath: string = join("install/tests"); @@ -36,15 +38,16 @@ const INFRA_PHASES: { [name: string]: InfraConfig } = { }, INSTALL_GITPOD_IGNORE_PREFLIGHTS: { phase: "install-gitpod-without-preflights", - makeTarget: "kots-install-without-preflight-with-community-license", + makeTarget: `kots-install channel=unstable version=${version} preflights=false`, // this is a bit of a hack, for now we pass params like this description: "Install gitpod using kots community edition without preflights", }, INSTALL_GITPOD: { phase: "install-gitpod", - makeTarget: "kots-install-with-community-license", + makeTarget: `kots-install channel=unstable version=${version} preflights=true`, description: "Install gitpod using kots community edition", }, CHECK_INSTALLATION: { + // this is a basic test for the Gitpod setup phase: "check-gitpod-installation", makeTarget: "check-gitpod-installation", description: "Check gitpod installation", @@ -155,6 +158,7 @@ function callMakeTargets(phase: string, description: string, makeTarget: string) werft.fail(phase, "Operation failed"); } else { werft.log(phase, response.stdout.toString()); + werft.done(phase); } return response.code; diff --git a/install/infra/terraform/gke/README.md b/install/infra/terraform/gke/README.md new file mode 100644 index 00000000000000..8886047aaf064c --- /dev/null +++ b/install/infra/terraform/gke/README.md @@ -0,0 +1,90 @@ +# GKE terraform module + +This is a terraform module currently used in the automated installation tests +for Gitpod. At successful completion, this module creates the following: + +1. A [`GKE`](https://cloud.google.com/kubernetes-engine) cluster by user + provided name `` with the following nodepools: + 1. `workspaces-` for workspace workloads + 1. `services-` for IDE and meta workloads +1. A dedicated `vpc` and `subnet` for the resources +1. `kubeconfig` data populated in a user defined `kubeconfig` variable + +## Requirements + +1. `terraform` >= `v1.1.7` + +## Providers + +1. [`google`](https://registry.terraform.io/providers/hashicorp/google/latest/docs) + +## Inputs + + +| Argument | Description | Required | Default | +|---------------------------|----------------------------------------------------------------------------------------------------------------------------------------------|----------|--------------------| +| `project` | The project ID to create the cluster. | `true` | | +| `kubeconfig` | Path to write the kubeconfig output to. | `false` | `./kubeconfig` | +| `region` | The region to create the cluster. | `false` | `europe-west1` | +| `zone` | The zone suffix to create the cluster in. If an empty string is provided, it will create multi-zone cluster | `false` | `b` | +| `kubernetes_version` | Kubernetes version to be setup | `false` | `1.21.11-gke.1100` | +| `name` | The name of the cluster and suffix for other resources | `false` | `sh-test` | +| `workspaces_machine_type` | Type of the node compute engines for workspace nodepool. | `false` | `n2-standard-8` | +| `services_machine_type` | Type of the node compute engines for meta and IDE nodepool. | `false` | `n2-standard-4` | +| `min_count` | Minimum number of nodes in the NodePool. Must be >=0 and <= max_node_count. | `false` | `1` | +| `max_count` | Maximum number of nodes in the NodePool. Must be >= min_node_count. | `false` | `50` | +| `disk_size_gb` | Disk size to provision in nodes. | `false` | `100` | +| `initial_node_count` | The number of nodes to create in this cluster's default node pool. | `false` | `1` | +| `pre-emptible` | Set if the nodes are to be pre-emptible. | `false` | `false` | +| `credentials` | Path to the JSON file storing Google service account credentials, if left empty, `tf` will look for `GOOGLE_APPLICATION_CREDENTIALS` env var | `false` | | +| | | | | + + +## Outputs + + +| Argument | +|-----------------------| +| `kubernetes_endpoint` | +| `client_token` | +| `ca_certificate` | +| `kubeconfig` | + + +## Usage + +Make sure you have a `GCP` account and credentials in `JSON` format to a service +account with atleast the following permissions: +1. Compute Admin +1. Compute Network Admin +1. Kubernetes Engine Admin +1. Kubernetes Engine Developer +1. Service Account User +1. Storage Admin + +Assign it to the following environment variable: + +``` sh +export GOOGLE_APPLICATION_CREDENTAILS=/path/to/service-account-json-creds +``` + +Create a `Cloud storage bucket` in the same project and region and specified +above, by name `gitpod-gke`. If you want to use a different name, edit the name +manually in the `main.tf` file. + +Open the file `variables.auto.tfvars` in this directory and provide values available from the +`Inputs` list as you find fit. + +Run the following commands: + +``` sh +terraform init +terraform apply +``` + + +## Cleanup + +``` sh +terraform destroy +``` diff --git a/install/infra/terraform/gke/main.tf b/install/infra/terraform/gke/main.tf index 309b36ee17a7a6..518b5b11099933 100644 --- a/install/infra/terraform/gke/main.tf +++ b/install/infra/terraform/gke/main.tf @@ -2,11 +2,18 @@ terraform { required_version = ">= 1.0.3" } +terraform { + backend "gcs" { + bucket = "gitpod-gke" + prefix = "tf-state" + } +} + provider "google" { + project = var.project credentials = var.credentials - project = var.project - region = var.region - zone = var.zone + region = var.region + zone = var.zone } resource "google_compute_network" "vpc" { @@ -14,7 +21,6 @@ resource "google_compute_network" "vpc" { auto_create_subnetworks = "false" } -# Subnet resource "google_compute_subnetwork" "subnet" { name = "subnet-${var.name}" region = var.region @@ -40,9 +46,9 @@ resource "google_container_cluster" "gitpod-cluster" { enabled = true resource_limits { - resource_type = "cpu" - minimum = 2 - maximum = 8 + resource_type = "cpu" + minimum = 2 + maximum = 8 } resource_limits { @@ -79,27 +85,19 @@ resource "google_container_cluster" "gitpod-cluster" { horizontal_pod_autoscaling { disabled = false } - - - # only available in beta - # dns_cache_config { - # enabled = true - # } } network = google_compute_network.vpc.name subnetwork = google_compute_subnetwork.subnet.name - - } resource "google_container_node_pool" "services" { - name = "services-${var.name}" - location = google_container_cluster.gitpod-cluster.location - cluster = google_container_cluster.gitpod-cluster.name - version = var.kubernetes_version // kubernetes version + name = "services-${var.name}" + location = google_container_cluster.gitpod-cluster.location + cluster = google_container_cluster.gitpod-cluster.name + version = var.kubernetes_version // kubernetes version initial_node_count = 1 - max_pods_per_node = 110 + max_pods_per_node = 110 node_config { oauth_scopes = [ @@ -107,15 +105,15 @@ resource "google_container_node_pool" "services" { ] labels = { - "gitpod.io/workload_meta" =true - "gitpod.io/workload_ide" = true + "gitpod.io/workload_meta" = true + "gitpod.io/workload_ide" = true } preemptible = var.pre-emptible - image_type = "UBUNTU_CONTAINERD" - disk_type = "pd-ssd" - disk_size_gb = var.disk_size_gb - machine_type = var.machine_type + image_type = "UBUNTU_CONTAINERD" + disk_type = "pd-ssd" + disk_size_gb = var.disk_size_gb + machine_type = var.services_machine_type tags = ["gke-node", "${var.project}-gke"] metadata = { disable-legacy-endpoints = "true" @@ -129,18 +127,18 @@ resource "google_container_node_pool" "services" { management { - auto_repair = true + auto_repair = true auto_upgrade = false } } resource "google_container_node_pool" "workspaces" { - name = "workspaces-${var.name}" - location = google_container_cluster.gitpod-cluster.location - cluster = google_container_cluster.gitpod-cluster.name - version = var.kubernetes_version // kubernetes version + name = "workspaces-${var.name}" + location = google_container_cluster.gitpod-cluster.location + cluster = google_container_cluster.gitpod-cluster.name + version = var.kubernetes_version // kubernetes version initial_node_count = 1 - max_pods_per_node = 110 + max_pods_per_node = 110 node_config { oauth_scopes = [ @@ -148,18 +146,18 @@ resource "google_container_node_pool" "workspaces" { ] labels = { - "gitpod.io/workload_metal" =true - "gitpod.io/workload_ide" = true + "gitpod.io/workload_metal" = true + "gitpod.io/workload_ide" = true "gitpod.io/workload_workspace_services" = true - "gitpod.io/workload_workspace_regular" = true + "gitpod.io/workload_workspace_regular" = true "gitpod.io/workload_workspace_headless" = true } preemptible = var.pre-emptible - image_type = "UBUNTU_CONTAINERD" - disk_type = "pd-ssd" - disk_size_gb = var.disk_size_gb - machine_type = var.machine_type + image_type = "UBUNTU_CONTAINERD" + disk_type = "pd-ssd" + disk_size_gb = var.disk_size_gb + machine_type = var.workspaces_machine_type tags = ["gke-node", "${var.project}-gke"] metadata = { disable-legacy-endpoints = "true" @@ -172,7 +170,7 @@ resource "google_container_node_pool" "workspaces" { } management { - auto_repair = true + auto_repair = true auto_upgrade = false } } @@ -189,5 +187,5 @@ module "gke_auth" { resource "local_file" "kubeconfig" { filename = var.kubeconfig - content = module.gke_auth.kubeconfig_raw + content = module.gke_auth.kubeconfig_raw } diff --git a/install/infra/terraform/gke/output.tf b/install/infra/terraform/gke/output.tf index 29702d742affbd..470911d8a97705 100644 --- a/install/infra/terraform/gke/output.tf +++ b/install/infra/terraform/gke/output.tf @@ -10,9 +10,10 @@ output "client_token" { output "ca_certificate" { sensitive = true - value = module.gke_auth.cluster_ca_certificate + value = module.gke_auth.cluster_ca_certificate } output "kubeconfig" { - value = module.gke_auth.kubeconfig_raw + sensitive = true + value = module.gke_auth.kubeconfig_raw } diff --git a/install/infra/terraform/gke/variables.auto.tfvars b/install/infra/terraform/gke/variables.auto.tfvars new file mode 100644 index 00000000000000..d6fc1bfe33cea2 --- /dev/null +++ b/install/infra/terraform/gke/variables.auto.tfvars @@ -0,0 +1,5 @@ +kubeconfig = "./kubeconfig" + +project = "sh-automated-tests" +region = "europe-west1" +zone = "b" diff --git a/install/infra/terraform/gke/variables.tf b/install/infra/terraform/gke/variables.tf index 50197379cc53bd..4aa4c3c9349c9c 100644 --- a/install/infra/terraform/gke/variables.tf +++ b/install/infra/terraform/gke/variables.tf @@ -4,14 +4,9 @@ variable "project" { } variable "kubeconfig" { - type = string - description = "Path to write the kubeconfig output to" - default = "./kubeconfig" -} - -variable "credentials" { type = string - description = "Path to the json file storing SA credentials." + description = "Path to write the kubeconfig output to" + default = "./kubeconfig" } variable "region" { @@ -38,10 +33,16 @@ variable "name" { default = "sh-test" } -variable "machine_type" { +variable "workspaces_machine_type" { type = string - description = "Type of the node compute engines." - default = "n2d-standard-4" + description = "Type of the node compute engines for workspace nodepool." + default = "n2-standard-8" +} + +variable "services_machine_type" { + type = string + description = "Type of the node compute engines for services nodepool." + default = "n2-standard-4" } variable "min_count" { @@ -59,7 +60,7 @@ variable "max_count" { variable "disk_size_gb" { type = number description = "Size of the node's disk." - default = 100 + default = 100 } variable "initial_node_count" { @@ -68,33 +69,13 @@ variable "initial_node_count" { default = 1 } -variable "env_name" { - description = "The environment for the GKE cluster" - default = "test" -} - -variable "network" { - description = "The VPC network created to host the cluster in" - default = "gitpod-gke-network" -} - -variable "subnetwork" { - description = "The subnetwork created to host the cluster in" - default = "gitpod-gke-subnet" -} - -variable "ip_range_pods_name" { - description = "The secondary ip range to use for pods" - default = "gitpod-ip-range-pods" -} - -variable "ip_range_services_name" { - description = "The secondary ip range to use for services" - default = "gitpod-ip-range-services" -} - variable "pre-emptible" { - type = bool + type = bool description = "Set if the nodes are to be pre-emptible" default = false } + +variable "credentials" { + description = "Path to the JSON file storing Google service account credentials" + default = "" +} diff --git a/install/infra/terraform/k3s/README.md b/install/infra/terraform/k3s/README.md index 72544c364f2f18..90b611412e4c0c 100644 --- a/install/infra/terraform/k3s/README.md +++ b/install/infra/terraform/k3s/README.md @@ -1,15 +1,56 @@ -## Module to create a 1 node k3s cluster in GCP +# k3s terraform module + +This is a terraform module currently used in the automated installation tests +for Gitpod. At successful completion, this module creates the following: + +1. A GCP VM instance of type `n2d-standard-4` and with `ubuntu` image, along with a service account to give auth scopes +1. A single node [`k3s`](https://k3s.io/) cluster with [`k3sup`](https://github.com/alexellis/k3sup) ### Requirements -- `terraform` >= 1.0.0 -- [`k3sup`](https://github.com/alexellis/k3sup#download-k3sup-tldr) +1. `terraform` >= `v1.1.7` +1. [`k3sup`](https://github.com/alexellis/k3sup#download-k3sup-tldr) ```sh curl -sLS https://get.k3sup.dev | sh sudo install k3sup /usr/local/bin/ ``` -### Usage +## Providers + +1. [`google`](https://registry.terraform.io/providers/hashicorp/google/latest/docs) + +## Input + +| Argument | Description | Required | Default | +|---------------|----------------------------------------------------------------------------------------------------------------------------------------------|----------|------------------| +| `gcp_project` | The project ID to create the VM in. | `true` | | +| `kubeconfig` | Path to write the kubeconfig output to. | `false` | `./kubeconfig` | +| `gcp_region` | The region to create the VM. | `false` | `europe-west1` | +| `gcp_zone` | The GCP zone to create the VM in. | `false` | `europe-west1-b` | +| `name` | Prefix name for the nodes and firewall | `false` | `k3s` | +| `credentials` | Path to the JSON file storing Google service account credentials, if left empty, `tf` will look for `GOOGLE_APPLICATION_CREDENTIALS` env var | `false` | | + +## Output + +| Argument | +|-----------------------| +| `kubernetes_endpoint` | +| `kubeconfig` | + +## Usage + +Create a service account with at least the following access scopes: +- Compute Admin +- Create Service Accounts +- Delete Service Accounts +- Service Account User +- Storage Admin + +Get the credentials of the SA in `JSON` format and export as a variable as folows: + +``` sh +export GOOGLE_APPLICATION_CREDENTIALS=/path/to/credentials +``` Update the `tfvars` file by the name `variables.auto.tfvars` in this directory with the values you want, and then run: @@ -21,3 +62,9 @@ terraform apply You will find the `kubeconfig` populated to the path you specified in the `variables.auto.tfvars` file. + +## Cleanup + +```sh +terraform destroy +``` diff --git a/install/infra/terraform/k3s/main.tf b/install/infra/terraform/k3s/main.tf index 60876cf0c4a3d4..645a0386477d96 100644 --- a/install/infra/terraform/k3s/main.tf +++ b/install/infra/terraform/k3s/main.tf @@ -7,11 +7,18 @@ terraform { } } +terraform { + backend "gcs" { + bucket = "gitpod-k3s" + prefix = "tf-state" + } +} + provider "google" { credentials = var.credentials - project = var.gcp_project - region = var.gcp_region - zone = var.gcp_zone + project = var.gcp_project + region = var.gcp_region + zone = var.gcp_zone } resource "google_service_account" "gcp_instance" { @@ -31,15 +38,15 @@ resource "local_file" "ssh_private_key_pem" { } resource "google_compute_instance" "k3s_master_instance" { - name = "master-${var.name}" + name = "master-${var.name}" machine_type = "n2d-standard-4" - tags = ["k3s", "k3s-master", "http-server", "https-server"] + tags = ["k3s", "k3s-master", "http-server", "https-server", "allow-ssh"] boot_disk { initialize_params { image = "ubuntu-2004-focal-v20220419" - size = 100 - type = "pd-ssd" + size = 100 + type = "pd-ssd" } } @@ -63,8 +70,6 @@ resource "google_compute_instance" "k3s_master_instance" { email = google_service_account.gcp_instance.email scopes = ["cloud-platform"] } - - } resource "time_sleep" "wait_for_master" { @@ -73,7 +78,6 @@ resource "time_sleep" "wait_for_master" { create_duration = "60s" } - resource "null_resource" "k3sup_install" { depends_on = [time_sleep.wait_for_master] @@ -83,7 +87,7 @@ resource "null_resource" "k3sup_install" { --ip ${google_compute_instance.k3s_master_instance.network_interface[0].access_config[0].nat_ip} \ --ssh-key .ssh/google_compute_engine \ --context k3s \ - --user $(whoami) \ + --user gitpod \ --local-path ${var.kubeconfig} \ --k3s-extra-args=" --disable=traefik --node-label=gitpod.io/workload_meta=true --node-label=gitpod.io/workload_ide=true --node-label=gitpod.io/workload_workspace_services=true --node-label=gitpod.io/workload_workspace_regular=true --node-label=gitpod.io/workload_workspace_headless=true" \ EOT @@ -92,7 +96,7 @@ resource "null_resource" "k3sup_install" { data "local_file" "kubeconfig" { depends_on = [null_resource.k3sup_install] - filename = var.kubeconfig + filename = var.kubeconfig } resource "google_compute_firewall" "k3s-firewall" { @@ -104,21 +108,6 @@ resource "google_compute_firewall" "k3s-firewall" { ports = ["6443"] } - allow { - protocol = "tcp" - ports = ["22"] - } - - allow { - protocol = "tcp" - ports = ["443"] - } - - allow { - protocol = "tcp" - ports = ["80"] - } - target_tags = ["k3s"] source_ranges = ["0.0.0.0/0"] diff --git a/install/infra/terraform/k3s/variables.auto.tfvars b/install/infra/terraform/k3s/variables.auto.tfvars index fca6fcadfaab2c..2f7c91e8f51ed4 100644 --- a/install/infra/terraform/k3s/variables.auto.tfvars +++ b/install/infra/terraform/k3s/variables.auto.tfvars @@ -1,6 +1,5 @@ kubeconfig = "./kubeconfig" gcp_project = "sh-automated-tests" -gcp_region = "europe-west1" -gcp_zone = "europe-west1-b" -credentials = "/workspace/gcp.json" +gcp_region = "europe-west1" +gcp_zone = "europe-west1-b" diff --git a/install/infra/terraform/k3s/variables.tf b/install/infra/terraform/k3s/variables.tf index 7899623f393156..d2ed0c12f6c1a9 100644 --- a/install/infra/terraform/k3s/variables.tf +++ b/install/infra/terraform/k3s/variables.tf @@ -1,27 +1,28 @@ variable "kubeconfig" { - description = "The KUBECONFIG file path to store the resulting KUBECONFIG file to" - default = "./kubeconfig" + description = "The KUBECONFIG file path to store the resulting KUBECONFIG file to" + default = "./kubeconfig" } variable "gcp_project" { - description = "Google cloud Region to perform operations in" + description = "Google cloud Region to perform operations in" } variable "gcp_region" { - description = "Google cloud Region to perform operations in" - default = "europe-west1" + description = "Google cloud Region to perform operations in" + default = "europe-west1" } variable "gcp_zone" { - description = "Google cloud Zone to perform operations in" - default = "europe-west1-b" + description = "Google cloud Zone to perform operations in" + default = "europe-west1-b" } variable "credentials" { - description = "Path to the JSON file storing Google service account credentials" + description = "Path to the JSON file storing Google service account credentials" + default = "" } variable "name" { - description = "Prefix name for the nodes and firewall" - default = "k3s" + description = "Prefix name for the nodes and firewall" + default = "k3s" } diff --git a/install/infra/terraform/tools/cm-cloud-dns/README.md b/install/infra/terraform/tools/cm-cloud-dns/README.md new file mode 100644 index 00000000000000..a7805293f15aeb --- /dev/null +++ b/install/infra/terraform/tools/cm-cloud-dns/README.md @@ -0,0 +1,6 @@ +# Managed DNS terraform module + +This module is used in test setup, to deploy `cert-manager` and `external-dns` +on a created cluster, and to create a secret that the aforementioned deployments +can use to modify a `cloudDNS` record. This module works specifically for the +domain `gitpod-self-hosted.com`. diff --git a/install/tests/Makefile b/install/tests/Makefile index 82c60a0bb5ca58..d8b3b4a73169e4 100644 --- a/install/tests/Makefile +++ b/install/tests/Makefile @@ -5,7 +5,7 @@ PROJECTNAME := "installer-nightly-tests" TOPDIR=$(shell pwd) -KUBECONFIG := "$(TOPDIR)/kubeconfig" +KUBECONFIG := "/workspace/conf" check-env-sub-domain: ifndef TF_VAR_TEST_ID @@ -53,37 +53,39 @@ get-kubeconfig: ${KUBECONFIG} KOTS_KONFIG := "./manifests/kots-config.yaml" -CHANNEL = "gitpod/beta" -COMMUNITY_LICENSE := "../licenses/Community (Beta).yaml" +license_community_beta := "../licenses/Community (Beta).yaml" +license_community_stable := "../licenses/Community.yaml" +license_community_unstable := "../licenses/Community (Unstable).yaml" install-kots-cli: curl https://kots.io/install | bash -kots-install-without-preflight-with-community-license: install-kots-cli +preflights ?= true +channel ?= unstable +version ?= - +kots-install: version-flag = $(if $(version:-=),--app-version-label=$(version),) +kots-install: preflight-flag = $(if $(preflights:true=),--skip-preflights,) +kots-install: license-file = $(license_community_$(channel)) +kots-install: install-kots-cli envsubst < ${KOTS_KONFIG} > tmp_config.yml - kubectl kots install ${CHANNEL} \ - --skip-rbac-check --skip-preflights --namespace gitpod --kubeconfig=${KUBECONFIG} \ + kubectl kots install gitpod/${channel} \ + --skip-rbac-check ${version-flag} ${preflight-flag} \ + --namespace gitpod --kubeconfig=${KUBECONFIG} \ --name gitpod --shared-password gitpod \ - --license-file ${COMMUNITY_LICENSE} \ + --license-file ${license-file} \ --no-port-forward \ --config-values tmp_config.yml -kots-install-with-community-license: install-kots-cli - envsubst < ${KOTS_KONFIG} > tmp_config.yml - kubectl kots install ${CHANNEL} \ - --skip-rbac-check --namespace gitpod --kubeconfig=${KUBECONFIG} \ - --name gitpod --shared-password gitpod \ - --license-file ${COMMUNITY_LICENSE} \ - --no-port-forward \ - --config-values tmp_config.yml +delete-cm-setup: + sleep 120 && kubectl --kubeconfig=${KUBECONFIG} delete pods --all -n cert-manager && sleep 300; -check-gitpod-installation: - sleep 120 - kubectl --kubeconfig=${KUBECONFIG} delete pods --all -n cert-manager - sleep 300 - @kubectl kots get --kubeconfig=${KUBECONFIG} app gitpod -n gitpod | grep gitpod | awk '{print $$2}' +check-kots-app: + kubectl kots get --kubeconfig=${KUBECONFIG} app gitpod -n gitpod | grep gitpod | awk '{print $$2}' | grep "ready" || { echo "Gitpod is not ready"; exit 1; } + +check-gitpod-installation: delete-cm-setup check-kots-app check-env-sub-domain + @echo "Curling http://${TF_VAR_TEST_ID}.gitpod-self-hosted.com/api/version" + curl -i -X GET http://${TF_VAR_TEST_ID}.gitpod-self-hosted.com/api/version || { echo "Curling Gitpod endpoint failed"; exit 1; } -.PHONY: run-tests: ./tests.sh ${KUBECONFIG} diff --git a/install/tests/main.tf b/install/tests/main.tf index 3e4a691e2033c8..16408fd879440f 100644 --- a/install/tests/main.tf +++ b/install/tests/main.tf @@ -1,43 +1,43 @@ variable "kubeconfig" {} variable "TEST_ID" { - default = "nightly" + default = "nightly" } variable "project" { - default = "sh-automated-tests" + default = "sh-automated-tests" } variable "sa_creds" {} variable "dns_sa_creds" {} terraform { backend "gcs" { - bucket = "nightly-tests" - prefix = "tf-state" + bucket = "nightly-tests" + prefix = "tf-state" } } module "gke" { - source = "github.com/gitpod-io/gitpod//install/infra/terraform/gke?ref=nvn-infra-tf" # we can later use tags here + source = "github.com/gitpod-io/gitpod//install/infra/terraform/gke?ref=nvn-infra-tf" # we can later use tags here - name = var.TEST_ID - project = var.project - credentials = var.sa_creds - kubeconfig = var.kubeconfig + name = var.TEST_ID + project = var.project + credentials = var.sa_creds + kubeconfig = var.kubeconfig } module "k3s" { - source = "github.com/gitpod-io/gitpod//install/infra/terraform/k3s?ref=nvn-infra-tf" # we can later use tags here + source = "github.com/gitpod-io/gitpod//install/infra/terraform/k3s?ref=nvn-infra-tf" # we can later use tags here - name = var.TEST_ID - gcp_project = var.project - credentials = var.sa_creds - kubeconfig = var.kubeconfig + name = var.TEST_ID + gcp_project = var.project + credentials = var.sa_creds + kubeconfig = var.kubeconfig } // this module is intended to be run separately from the above two. so a separate target for apply is necessary module "tools" { - source = "github.com/gitpod-io/gitpod//install/infra/terraform/tools/cm-cloud-dns?ref=nvn-infra-tf" # we can later use tags here + source = "github.com/gitpod-io/gitpod//install/infra/terraform/tools/cm-cloud-dns?ref=nvn-infra-tf" # we can later use tags here - kubeconfig = var.kubeconfig - credentials = var.dns_sa_creds - gcp_sub_domain = var.TEST_ID + kubeconfig = var.kubeconfig + credentials = var.dns_sa_creds + gcp_sub_domain = var.TEST_ID } From 73dc14d4cb6e33a3b9758b01f242c7fee6ad9102 Mon Sep 17 00:00:00 2001 From: Nandaja Varma Date: Wed, 1 Jun 2022 08:25:34 +0000 Subject: [PATCH 2/4] adding AKS tf module --- install/infra/terraform/aks/README.md | 97 +++++++++++++++++++++++ install/infra/terraform/aks/database.tf | 50 ++++++++++++ install/infra/terraform/aks/kubernetes.tf | 95 ++++++++++++++++++++++ install/infra/terraform/aks/local.tf | 95 ++++++++++++++++++++++ install/infra/terraform/aks/main.tf | 24 ++++++ install/infra/terraform/aks/monitoring.tf | 19 +++++ install/infra/terraform/aks/networks.tf | 20 +++++ install/infra/terraform/aks/output.tf | 89 +++++++++++++++++++++ install/infra/terraform/aks/registry.tf | 25 ++++++ install/infra/terraform/aks/storage.tf | 16 ++++ install/infra/terraform/aks/variables.tf | 24 ++++++ install/tests/main.tf | 30 ++++--- 12 files changed, 574 insertions(+), 10 deletions(-) create mode 100644 install/infra/terraform/aks/README.md create mode 100644 install/infra/terraform/aks/database.tf create mode 100644 install/infra/terraform/aks/kubernetes.tf create mode 100644 install/infra/terraform/aks/local.tf create mode 100644 install/infra/terraform/aks/main.tf create mode 100644 install/infra/terraform/aks/monitoring.tf create mode 100644 install/infra/terraform/aks/networks.tf create mode 100644 install/infra/terraform/aks/output.tf create mode 100644 install/infra/terraform/aks/registry.tf create mode 100644 install/infra/terraform/aks/storage.tf create mode 100644 install/infra/terraform/aks/variables.tf diff --git a/install/infra/terraform/aks/README.md b/install/infra/terraform/aks/README.md new file mode 100644 index 00000000000000..e03e1cc8289b3b --- /dev/null +++ b/install/infra/terraform/aks/README.md @@ -0,0 +1,97 @@ +# Azure + +Azure provider for Gitpod testing + + + +- [Terraform Documentation](#terraform-documentation) + * [Requirements](#requirements) + * [Providers](#providers) + * [Modules](#modules) + * [Resources](#resources) + * [Inputs](#inputs) + * [Outputs](#outputs) + + + +# Terraform Documentation + + +## Requirements + +| Name | Version | +|------|---------| +| [azurerm](#requirement\_azurerm) | >= 3.0.0, < 4.0.0 | + +## Providers + +| Name | Version | +|------|---------| +| [azurerm](#provider\_azurerm) | >= 3.0.0, < 4.0.0 | +| [random](#provider\_random) | n/a | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [azurerm_container_registry.registry](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/container_registry) | resource | +| [azurerm_dns_zone.dns](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/dns_zone) | resource | +| [azurerm_kubernetes_cluster.k8s](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/kubernetes_cluster) | resource | +| [azurerm_kubernetes_cluster_node_pool.pools](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/kubernetes_cluster_node_pool) | resource | +| [azurerm_log_analytics_solution.monitoring](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/log_analytics_solution) | resource | +| [azurerm_log_analytics_workspace.monitoring](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/log_analytics_workspace) | resource | +| [azurerm_mysql_database.db](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/mysql_database) | resource | +| [azurerm_mysql_firewall_rule.db](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/mysql_firewall_rule) | resource | +| [azurerm_mysql_server.db](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/mysql_server) | resource | +| [azurerm_network_security_rule.k8s](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/network_security_rule) | resource | +| [azurerm_resource_group.gitpod](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/resource_group) | resource | +| [azurerm_role_assignment.k8s](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource | +| [azurerm_role_assignment.registry](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource | +| [azurerm_storage_account.storage](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/storage_account) | resource | +| [azurerm_subnet.network](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/subnet) | resource | +| [azurerm_virtual_network.network](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/virtual_network) | resource | +| [random_integer.db](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/integer) | resource | +| [random_integer.registry](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/integer) | resource | +| [random_integer.storage](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/integer) | resource | +| [random_password.db](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/password) | resource | +| [azurerm_client_config.current](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/client_config) | data source | +| [azurerm_kubernetes_service_versions.k8s](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/kubernetes_service_versions) | data source | +| [azurerm_resources.k8s](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/resources) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [dns\_enabled](#input\_dns\_enabled) | Common variables | `any` | n/a | yes | +| [domain\_name](#input\_domain\_name) | n/a | `any` | n/a | yes | +| [enable\_airgapped](#input\_enable\_airgapped) | n/a | `any` | n/a | yes | +| [enable\_external\_database](#input\_enable\_external\_database) | n/a | `any` | n/a | yes | +| [enable\_external\_registry](#input\_enable\_external\_registry) | n/a | `any` | n/a | yes | +| [enable\_external\_storage](#input\_enable\_external\_storage) | n/a | `any` | n/a | yes | +| [labels](#input\_labels) | n/a | `any` | n/a | yes | +| [location](#input\_location) | Azure-specific variables | `any` | n/a | yes | +| [name\_format](#input\_name\_format) | n/a | `any` | n/a | yes | +| [name\_format\_global](#input\_name\_format\_global) | n/a | `any` | n/a | yes | +| [workspace\_name](#input\_workspace\_name) | n/a | `any` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [cert\_manager\_issuer](#output\_cert\_manager\_issuer) | n/a | +| [cert\_manager\_secret](#output\_cert\_manager\_secret) | n/a | +| [cluster\_name](#output\_cluster\_name) | n/a | +| [database](#output\_database) | n/a | +| [domain\_nameservers](#output\_domain\_nameservers) | n/a | +| [external\_dns\_secrets](#output\_external\_dns\_secrets) | n/a | +| [external\_dns\_settings](#output\_external\_dns\_settings) | n/a | +| [k8s\_connection](#output\_k8s\_connection) | n/a | +| [kubeconfig](#output\_kubeconfig) | n/a | +| [region](#output\_region) | n/a | +| [registry](#output\_registry) | n/a | +| [storage](#output\_storage) | n/a | + diff --git a/install/infra/terraform/aks/database.tf b/install/infra/terraform/aks/database.tf new file mode 100644 index 00000000000000..50a8a21e3fa9e2 --- /dev/null +++ b/install/infra/terraform/aks/database.tf @@ -0,0 +1,50 @@ +resource "random_integer" "db" { + count = var.enable_external_database ? 1 : 0 + + min = 10000 + max = 99999 +} + +resource "random_password" "db" { + count = var.enable_external_database ? 1 : 0 + + length = 32 +} + +resource "azurerm_mysql_server" "db" { + count = var.enable_external_database ? 1 : 0 + + name = "gitpod-${random_integer.db[count.index].result}" + location = azurerm_resource_group.gitpod.location + resource_group_name = azurerm_resource_group.gitpod.name + + sku_name = local.db + storage_mb = 20480 + ssl_enforcement_enabled = false + ssl_minimal_tls_version_enforced = "TLSEnforcementDisabled" + version = "5.7" + + auto_grow_enabled = true + administrator_login = "gitpod" + administrator_login_password = random_password.db[count.index].result +} + +resource "azurerm_mysql_firewall_rule" "db" { + count = var.enable_external_database ? 1 : 0 + + name = "Azure_Resource" + resource_group_name = azurerm_resource_group.gitpod.name + server_name = azurerm_mysql_server.db[count.index].name + start_ip_address = "0.0.0.0" + end_ip_address = "0.0.0.0" +} + +resource "azurerm_mysql_database" "db" { + count = var.enable_external_database ? 1 : 0 + + name = "gitpod" + resource_group_name = azurerm_resource_group.gitpod.name + server_name = azurerm_mysql_server.db[count.index].name + charset = "utf8" + collation = "utf8_unicode_ci" +} diff --git a/install/infra/terraform/aks/kubernetes.tf b/install/infra/terraform/aks/kubernetes.tf new file mode 100644 index 00000000000000..66f6f72c8c9ad2 --- /dev/null +++ b/install/infra/terraform/aks/kubernetes.tf @@ -0,0 +1,95 @@ +data "azurerm_kubernetes_service_versions" "k8s" { + location = azurerm_resource_group.gitpod.location + include_preview = false +} + +resource "azurerm_role_assignment" "k8s" { + count = var.dns_enabled ? 1 : 0 + + principal_id = azurerm_kubernetes_cluster.k8s.kubelet_identity[count.index].object_id + role_definition_name = "DNS Zone Contributor" + scope = azurerm_dns_zone.dns[count.index].id +} + +resource "azurerm_kubernetes_cluster" "k8s" { + name = format(var.name_format, local.location, "primary") + location = azurerm_resource_group.gitpod.location + resource_group_name = azurerm_resource_group.gitpod.name + dns_prefix = "gitpod" + + kubernetes_version = data.azurerm_kubernetes_service_versions.k8s.latest_version + http_application_routing_enabled = false + + default_node_pool { + name = local.nodes.0.name + vm_size = local.machine + + enable_auto_scaling = true + min_count = 2 + max_count = 10 + orchestrator_version = data.azurerm_kubernetes_service_versions.k8s.latest_version + node_labels = local.nodes.0.labels + + type = "VirtualMachineScaleSets" + vnet_subnet_id = azurerm_subnet.network.id + } + + identity { + type = "SystemAssigned" + } + + network_profile { + network_plugin = "kubenet" + network_policy = "calico" + } + + oms_agent { + log_analytics_workspace_id = azurerm_log_analytics_workspace.monitoring.id + } +} + +resource "azurerm_kubernetes_cluster_node_pool" "pools" { + count = length(local.nodes) - 1 + + kubernetes_cluster_id = azurerm_kubernetes_cluster.k8s.id + name = local.nodes[count.index + 1].name + vm_size = local.machine + + enable_auto_scaling = true + min_count = 2 + max_count = 10 + orchestrator_version = data.azurerm_kubernetes_service_versions.k8s.latest_version + node_labels = local.nodes[count.index + 1].labels + vnet_subnet_id = azurerm_subnet.network.id +} + +data "azurerm_resources" "k8s" { + count = var.enable_airgapped ? 1 : 0 + + resource_group_name = azurerm_kubernetes_cluster.k8s.node_resource_group + type = "Microsoft.Network/networkSecurityGroups" + + depends_on = [ + azurerm_kubernetes_cluster.k8s, + azurerm_kubernetes_cluster_node_pool.pools + ] +} + +resource "azurerm_network_security_rule" "k8s" { + count = length(local.network_security_rules) + + resource_group_name = azurerm_kubernetes_cluster.k8s.node_resource_group + network_security_group_name = data.azurerm_resources.k8s.0.resources.0.name + + priority = lookup(local.network_security_rules[count.index], "priority", sum([100, count.index])) + name = local.network_security_rules[count.index].name + access = local.network_security_rules[count.index].access + direction = local.network_security_rules[count.index].direction + protocol = local.network_security_rules[count.index].protocol + + description = lookup(local.network_security_rules[count.index], "description", null) + source_port_range = lookup(local.network_security_rules[count.index], "source_port_range", null) + destination_port_range = lookup(local.network_security_rules[count.index], "destination_port_range", null) + source_address_prefix = lookup(local.network_security_rules[count.index], "source_address_prefix", null) + destination_address_prefix = lookup(local.network_security_rules[count.index], "destination_address_prefix", null) +} diff --git a/install/infra/terraform/aks/local.tf b/install/infra/terraform/aks/local.tf new file mode 100644 index 00000000000000..e11d87159206cc --- /dev/null +++ b/install/infra/terraform/aks/local.tf @@ -0,0 +1,95 @@ +locals { + labels = tomap({ + workload_meta : "gitpod.io/workload_meta" + workload_ide : "gitpod.io/workload_ide" + workspace_services : "gitpod.io/workload_workspace_services" + workspace_regular : "gitpod.io/workload_workspace_regular" + workspace_headless : "gitpod.io/workload_workspace_headless" + }) + dns_enabled = var.domain_name != null + name_format = join("-", [ + "gitpod", + "%s", # region + "%s", # name + local.workspace_name + ]) + name_format_global = join("-", [ + "gitpod", + "%s", # name + local.workspace_name + ]) + workspace_name = replace(terraform.workspace, "/[\\W\\-]/", "") # alphanumeric workspace name + db = "GP_Gen5_2" + location = substr(var.location, 0, 3) # Short code for location + machine = "Standard_D4_v3" + network_security_rules = var.enable_airgapped ? [ + { + name = "AllowContainerRegistry" + description = "Allow outgoing traffic to the container registry" + direction = "Outbound" + access = "Allow" + protocol = "*" + source_port_range = "*" + destination_port_range = "*" + source_address_prefix = "*" + destination_address_prefix = "AzureContainerRegistry" + }, + { + name = "AllowDatabase" + description = "Allow outgoing traffic to the database" + direction = "Outbound" + access = "Allow" + protocol = "*" + source_port_range = "*" + destination_port_range = "*" + source_address_prefix = "*" + destination_address_prefix = "Sql" + }, + { + name = "AllowStorage" + description = "Allow outgoing traffic to the storage" + direction = "Outbound" + access = "Allow" + protocol = "*" + source_port_range = "*" + destination_port_range = "*" + source_address_prefix = "*" + destination_address_prefix = "Storage" + }, + { + name = "AllowAzureCloud" + description = "Allow outgoing traffic to the Azure cloud" + direction = "Outbound" + access = "Allow" + protocol = "*" + source_port_range = "*" + destination_port_range = "*" + source_address_prefix = "*" + destination_address_prefix = "AzureCloud" + }, + { + name = "DenyInternetOutBound" + description = "Deny outgoing traffic to the public internet" + direction = "Outbound" + access = "Deny" + protocol = "*" + source_port_range = "*" + destination_port_range = "*" + source_address_prefix = "*" + destination_address_prefix = "Internet" + priority = 4096 + } + ] : [] + nodes = [ + { + name = "services" + labels = { + lookup(local.labels, "workload_meta") = true + lookup(local.labels, "workload_ide") = true + lookup(local.labels, "workspace_services") = true + lookup(local.labels, "workspace_regular") = true + lookup(local.labels, "workspace_headless") = true + } + } + ] +} diff --git a/install/infra/terraform/aks/main.tf b/install/infra/terraform/aks/main.tf new file mode 100644 index 00000000000000..43dd6a6e6adf63 --- /dev/null +++ b/install/infra/terraform/aks/main.tf @@ -0,0 +1,24 @@ +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = ">= 3.0.0, < 4.0.0" + } + } +} + +provider "azurerm" { + features {} +} + +data "azurerm_client_config" "current" {} + +resource "azurerm_resource_group" "gitpod" { + name = format(var.name_format_global, local.location) + location = var.location +} + +resource "local_file" "kubeconfig" { + filename = var.kubeconfig + content = azurerm_kubernetes_cluster.k8s.kube_config_raw +} diff --git a/install/infra/terraform/aks/monitoring.tf b/install/infra/terraform/aks/monitoring.tf new file mode 100644 index 00000000000000..57276917673669 --- /dev/null +++ b/install/infra/terraform/aks/monitoring.tf @@ -0,0 +1,19 @@ +resource "azurerm_log_analytics_workspace" "monitoring" { + name = format(var.name_format, var.location, "monitoring") + location = azurerm_resource_group.gitpod.location + resource_group_name = azurerm_resource_group.gitpod.name + sku = "PerGB2018" +} + +resource "azurerm_log_analytics_solution" "monitoring" { + solution_name = "ContainerInsights" + location = azurerm_resource_group.gitpod.location + resource_group_name = azurerm_resource_group.gitpod.name + workspace_name = azurerm_log_analytics_workspace.monitoring.name + workspace_resource_id = azurerm_log_analytics_workspace.monitoring.id + + plan { + publisher = "Microsoft" + product = "OMSGallery/ContainerInsights" + } +} diff --git a/install/infra/terraform/aks/networks.tf b/install/infra/terraform/aks/networks.tf new file mode 100644 index 00000000000000..3e4184ee1d1ed3 --- /dev/null +++ b/install/infra/terraform/aks/networks.tf @@ -0,0 +1,20 @@ +resource "azurerm_virtual_network" "network" { + name = format(var.name_format, local.location, "network") + location = azurerm_resource_group.gitpod.location + resource_group_name = azurerm_resource_group.gitpod.name + address_space = ["10.2.0.0/16"] +} + +resource "azurerm_subnet" "network" { + name = format(var.name_format, local.location, "network") + resource_group_name = azurerm_resource_group.gitpod.name + virtual_network_name = azurerm_virtual_network.network.name + address_prefixes = ["10.2.1.0/24"] +} + +resource "azurerm_dns_zone" "dns" { + count = var.dns_enabled ? 1 : 0 + + name = var.domain_name + resource_group_name = azurerm_resource_group.gitpod.name +} diff --git a/install/infra/terraform/aks/output.tf b/install/infra/terraform/aks/output.tf new file mode 100644 index 00000000000000..a22bd5c3efb7d2 --- /dev/null +++ b/install/infra/terraform/aks/output.tf @@ -0,0 +1,89 @@ +output "cert_manager_issuer" { + value = try([{ + dns01 = { + azureDNS = { + subscriptionID = data.azurerm_client_config.current.subscription_id + resourceGroupName = azurerm_resource_group.gitpod.name + hostedZoneName = azurerm_dns_zone.dns.0.name + managedIdentity = { + clientID = azurerm_kubernetes_cluster.k8s.kubelet_identity.0.client_id + } + } + } + }], []) +} + +output "cert_manager_secret" { + value = {} +} + +output "cluster_name" { + value = azurerm_kubernetes_cluster.k8s.name +} + +output "database" { + sensitive = true + value = try({ + host = "${azurerm_mysql_server.db.0.name}.mysql.database.azure.com" + password = azurerm_mysql_server.db.0.administrator_login_password + port = 3306 + username = "${azurerm_mysql_server.db.0.administrator_login}@${azurerm_mysql_server.db.0.name}" + }, {}) +} + +output "domain_nameservers" { + value = try(azurerm_dns_zone.dns.0.name_servers, null) +} + +output "external_dns_secrets" { + value = {} +} + +output "external_dns_settings" { + value = { + provider = "azure" + "azure.resourceGroup" = azurerm_resource_group.gitpod.name + "azure.subscriptionId" = data.azurerm_client_config.current.subscription_id + "azure.tenantId" = data.azurerm_client_config.current.tenant_id + "azure.useManagedIdentityExtension" = true + "azure.userAssignedIdentityID" = azurerm_kubernetes_cluster.k8s.kubelet_identity.0.client_id + } +} + +output "k8s_connection" { + sensitive = true + value = { + host = azurerm_kubernetes_cluster.k8s.kube_config.0.host + username = azurerm_kubernetes_cluster.k8s.kube_config.0.username + password = azurerm_kubernetes_cluster.k8s.kube_config.0.password + client_certificate = base64decode(azurerm_kubernetes_cluster.k8s.kube_config.0.client_certificate) + client_key = base64decode(azurerm_kubernetes_cluster.k8s.kube_config.0.client_key) + cluster_ca_certificate = base64decode(azurerm_kubernetes_cluster.k8s.kube_config.0.cluster_ca_certificate) + } +} + +output "kubeconfig" { + sensitive = true + value = azurerm_kubernetes_cluster.k8s.kube_config_raw +} + +output "region" { + value = var.location +} + +output "registry" { + sensitive = true + value = try({ + server = azurerm_container_registry.registry.0.login_server + password = azurerm_container_registry.registry.0.admin_password + username = azurerm_container_registry.registry.0.admin_username + }, {}) +} + +output "storage" { + sensitive = true + value = try({ + username = azurerm_storage_account.storage.0.name + password = azurerm_storage_account.storage.0.primary_access_key + }, {}) +} diff --git a/install/infra/terraform/aks/registry.tf b/install/infra/terraform/aks/registry.tf new file mode 100644 index 00000000000000..87ec560c6b5441 --- /dev/null +++ b/install/infra/terraform/aks/registry.tf @@ -0,0 +1,25 @@ +resource "random_integer" "registry" { + count = var.enable_external_registry ? 1 : 0 + + min = 10000 + max = 99999 +} + +resource "azurerm_container_registry" "registry" { + count = var.enable_external_registry ? 1 : 0 + + name = "gitpod${random_integer.registry[count.index].result}" + resource_group_name = azurerm_resource_group.gitpod.name + location = azurerm_resource_group.gitpod.location + admin_enabled = true + sku = "Premium" +} + +resource "azurerm_role_assignment" "registry" { + count = var.enable_external_registry ? 1 : 0 + + principal_id = azurerm_kubernetes_cluster.k8s.kubelet_identity[0].object_id + role_definition_name = "AcrPush" + scope = azurerm_container_registry.registry[count.index].id + skip_service_principal_aad_check = true +} diff --git a/install/infra/terraform/aks/storage.tf b/install/infra/terraform/aks/storage.tf new file mode 100644 index 00000000000000..ca4ef115dcf924 --- /dev/null +++ b/install/infra/terraform/aks/storage.tf @@ -0,0 +1,16 @@ +resource "random_integer" "storage" { + count = var.enable_external_storage ? 1 : 0 + + min = 10000 + max = 99999 +} + +resource "azurerm_storage_account" "storage" { + count = var.enable_external_storage ? 1 : 0 + + name = "gitpod${random_integer.storage[count.index].result}" + resource_group_name = azurerm_resource_group.gitpod.name + location = azurerm_resource_group.gitpod.location + account_tier = "Standard" + account_replication_type = "LRS" +} diff --git a/install/infra/terraform/aks/variables.tf b/install/infra/terraform/aks/variables.tf new file mode 100644 index 00000000000000..796f7489896181 --- /dev/null +++ b/install/infra/terraform/aks/variables.tf @@ -0,0 +1,24 @@ +// Common variables +variable "kubeconfig" { + default = "./kubeconfig" + +} +variable "dns_enabled" {} +variable "domain_name" {} +variable "enable_airgapped" {} +variable "enable_external_database" {} +variable "enable_external_registry" {} +variable "enable_external_storage" {} +variable "labels" {} +variable "name_format" { +} +variable "name_format_global" { +} +variable "workspace_name" { +} + +// Azure-specific variables +variable "location" { + default = "northeurope" + +} diff --git a/install/tests/main.tf b/install/tests/main.tf index 16408fd879440f..58f9e6407730cd 100644 --- a/install/tests/main.tf +++ b/install/tests/main.tf @@ -1,13 +1,7 @@ variable "kubeconfig" {} -variable "TEST_ID" { - default = "nightly" -} -variable "project" { - default = "sh-automated-tests" -} -variable "sa_creds" {} -variable "dns_sa_creds" {} +variable "TEST_ID" { default = "nightly" } +# We store the state always in a GCS bucket terraform { backend "gcs" { bucket = "nightly-tests" @@ -15,8 +9,12 @@ terraform { } } +variable "project" { default = "sh-automated-tests" } +variable "sa_creds" {} +variable "dns_sa_creds" {} + module "gke" { - source = "github.com/gitpod-io/gitpod//install/infra/terraform/gke?ref=nvn-infra-tf" # we can later use tags here + source = "github.com/gitpod-io/gitpod//install/infra/terraform/gke?ref=nvn/infra-tf" # we can later use tags here name = var.TEST_ID project = var.project @@ -25,7 +23,7 @@ module "gke" { } module "k3s" { - source = "github.com/gitpod-io/gitpod//install/infra/terraform/k3s?ref=nvn-infra-tf" # we can later use tags here + source = "github.com/gitpod-io/gitpod//install/infra/terraform/k3s?ref=nvn/infra-tf" # we can later use tags here name = var.TEST_ID gcp_project = var.project @@ -33,6 +31,18 @@ module "k3s" { kubeconfig = var.kubeconfig } +module "azure" { + source = "./azure" + source = "github.com/gitpod-io/gitpod//install/infra/terraform/k3s?ref=nvn/infra-aks" # we can later use tags here + + domain_name = false + enable_airgapped = false + enable_external_database = false + enable_external_registry = false + enable_external_storage = false +} + + // this module is intended to be run separately from the above two. so a separate target for apply is necessary module "tools" { source = "github.com/gitpod-io/gitpod//install/infra/terraform/tools/cm-cloud-dns?ref=nvn-infra-tf" # we can later use tags here From 61d22365a0a1bd308209aef24dbdf0e8162bc277 Mon Sep 17 00:00:00 2001 From: Nandaja Varma Date: Wed, 1 Jun 2022 17:13:38 +0000 Subject: [PATCH 3/4] adding the ts scripts --- .werft/aks-installer-tests.yaml | 89 +++++++++++++++++++++++++++++++++ .werft/installer-tests.ts | 17 +++++++ install/tests/Makefile | 11 +++- install/tests/main.tf | 36 +++++++++---- 4 files changed, 143 insertions(+), 10 deletions(-) create mode 100644 .werft/aks-installer-tests.yaml diff --git a/.werft/aks-installer-tests.yaml b/.werft/aks-installer-tests.yaml new file mode 100644 index 00000000000000..873ff6f4285017 --- /dev/null +++ b/.werft/aks-installer-tests.yaml @@ -0,0 +1,89 @@ +# debug using `werft run github -f -s .werft/installer-tests.ts -j .werft/aks-installer-tests.yaml -a debug=true` +pod: + serviceAccount: werft + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: dev/workload + operator: In + values: + - "builds" + securityContext: + runAsUser: 0 + volumes: + - name: sh-playground-sa-perm + secret: + secretName: sh-playground-sa-perm + - name: sh-playground-dns-perm + secret: + secretName: sh-playground-dns-perm + containers: + - name: nightly-test + image: eu.gcr.io/gitpod-core-dev/dev/dev-environment:cw-werft-cred.0 + workingDir: /workspace + imagePullPolicy: Always + volumeMounts: + - name: sh-playground-sa-perm + mountPath: /mnt/secrets/sh-playground-sa-perm + - name: sh-playground-dns-perm # this sa is used for the DNS management + mountPath: /mnt/secrets/sh-playground-dns-perm + env: + - name: ARM_SUBSCRIPTION_ID + valueFrom: + secretKeyRef: + name: aks-credentials + key: subscriptionid + - name: ARM_TENANT_ID + valueFrom: + secretKeyRef: + name: aks-credentials + key: tenantid + - name: ARM_CLIENT_ID + valueFrom: + secretKeyRef: + name: aks-credentials + key: clientid + - name: ARM_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: aks-credentials + key: clientsecret + - name: WERFT_HOST + value: "werft.werft.svc.cluster.local:7777" + - name: GOOGLE_APPLICATION_CREDENTIALS + value: "/mnt/secrets/sh-playground-sa-perm/sh-sa.json" + - name: TF_VAR_sa_creds + value: "/mnt/secrets/sh-playground-sa-perm/sh-sa.json" + - name: TF_VAR_dns_sa_creds + value: "/mnt/secrets/sh-playground-dns-perm/sh-dns-sa.json" + - name: WERFT_K8S_NAMESPACE + value: "werft" + - name: WERFT_K8S_LABEL + value: "component=werft" + - name: NODENAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + command: + - bash + - -c + - | + sleep 1 + set -Eeuo pipefail + + sudo chown -R gitpod:gitpod /workspace + sudo apt update && apt install gettext-base + + curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash + + export TF_VAR_TEST_ID=$(echo $RANDOM | md5sum | head -c 5; echo) + + (cd .werft && yarn install && mv node_modules ..) | werft log slice prep + printf '{{ toJson . }}' > context.json + + npx ts-node .werft/installer-tests.ts "STANDARD_AKS_TEST" +# The bit below makes this a cron job +# plugins: +# cron: "15 3 * * *" diff --git a/.werft/installer-tests.ts b/.werft/installer-tests.ts index a67cfde849963e..5ca6bd7f7ad704 100644 --- a/.werft/installer-tests.ts +++ b/.werft/installer-tests.ts @@ -31,6 +31,11 @@ const INFRA_PHASES: { [name: string]: InfraConfig } = { makeTarget: "k3s-standard-cluster", description: "Creating a k3s cluster on GCP with 1 node", }, + STANDARD_AKS_CLUSTER: { + phase: "create-std-aks-cluster", + makeTarget: "aks-standard-cluster", + description: "Creating an aks cluster(azure)", + }, GCP_MANAGED_DNS: { phase: "setup-cert-manager-with-cloud-dns", makeTarget: "managed-dns-with-cert-manager", @@ -89,6 +94,18 @@ const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = { "DESTROY", ], }, + STANDARD_AKS_TEST: { + DESCRIPTION: "Deploy Gitpod on AKS, with managed DNS, and run integration tests", + PHASES: [ + "STANDARD_AKS_CLUSTER", + "GCP_MANAGED_DNS", + "INSTALL_GITPOD", + "CHECK_INSTALLATION", + "RUN_INTEGRATION_TESTS", + "RESULTS", + "DESTROY", + ], + }, STANDARD_K3S_TEST: { DESCRIPTION: "Deploy Gitpod on a K3s cluster, created on a GCP instance," + diff --git a/install/tests/Makefile b/install/tests/Makefile index d8b3b4a73169e4..55592f0d33f388 100644 --- a/install/tests/Makefile +++ b/install/tests/Makefile @@ -5,7 +5,7 @@ PROJECTNAME := "installer-nightly-tests" TOPDIR=$(shell pwd) -KUBECONFIG := "/workspace/conf" +KUBECONFIG := "$(TOPDIR)/kubeconfig" check-env-sub-domain: ifndef TF_VAR_TEST_ID @@ -29,6 +29,14 @@ gke-standard-cluster: terraform apply -target=module.gke \ -var kubeconfig=${KUBECONFIG} --auto-approve +.PHONY: +## aks-standard-cluster: Creates an AKS cluster +aks-standard-cluster: + terraform init --upgrade && \ + terraform workspace new $(TF_VAR_TEST_ID) || terraform workspace select $(TF_VAR_TEST_ID) && \ + terraform apply -target=module.aks \ + -var kubeconfig=${KUBECONFIG} --auto-approve + .PHONY: ## k3s-standard-cluster: Creates a K3S cluster on GCP with one master and 1 worker node k3s-standard-cluster: @@ -93,6 +101,7 @@ cleanup: terraform workspace select $(TF_VAR_TEST_ID) which ${KUBECONFIG} && terraform destroy -target=module.tools -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file" terraform destroy -target=module.gke -var kubeconfig=${KUBECONFIG} --auto-approve + terraform destroy -target=module.aks -var kubeconfig=${KUBECONFIG} --auto-approve terraform destroy -target=module.k3s -var kubeconfig=${KUBECONFIG} --auto-approve get-results: diff --git a/install/tests/main.tf b/install/tests/main.tf index 58f9e6407730cd..d6d1025de01f49 100644 --- a/install/tests/main.tf +++ b/install/tests/main.tf @@ -1,4 +1,4 @@ -variable "kubeconfig" {} +variable "kubeconfig" { } variable "TEST_ID" { default = "nightly" } # We store the state always in a GCS bucket @@ -10,11 +10,11 @@ terraform { } variable "project" { default = "sh-automated-tests" } -variable "sa_creds" {} -variable "dns_sa_creds" {} +variable "sa_creds" { default = "" } +variable "dns_sa_creds" {default = "" } module "gke" { - source = "github.com/gitpod-io/gitpod//install/infra/terraform/gke?ref=nvn/infra-tf" # we can later use tags here + source = "github.com/gitpod-io/gitpod//install/infra/terraform/gke?ref=nvn-infra-tf" # we can later use tags here name = var.TEST_ID project = var.project @@ -23,7 +23,7 @@ module "gke" { } module "k3s" { - source = "github.com/gitpod-io/gitpod//install/infra/terraform/k3s?ref=nvn/infra-tf" # we can later use tags here + source = "github.com/gitpod-io/gitpod//install/infra/terraform/k3s?ref=nvn-infra-tf" # we can later use tags here name = var.TEST_ID gcp_project = var.project @@ -31,18 +31,36 @@ module "k3s" { kubeconfig = var.kubeconfig } -module "azure" { - source = "./azure" - source = "github.com/gitpod-io/gitpod//install/infra/terraform/k3s?ref=nvn/infra-aks" # we can later use tags here +module "aks" { + source = "github.com/gitpod-io/gitpod//install/infra/terraform/aks?ref=nvn-infra-aks" # we can later use tags here domain_name = false enable_airgapped = false enable_external_database = false enable_external_registry = false enable_external_storage = false + dns_enabled = false + name_format = join("-", [ + "gitpod", + "%s", # region + "%s", # name + var.TEST_ID + ]) + name_format_global = join("-", [ + "gitpod", + "%s", # name + var.TEST_ID + ]) + workspace_name = var.TEST_ID + labels = { + "gitpod.io/workload_meta" = true + "gitpod.io/workload_ide" = true + "gitpod.io/workload_workspace_services" = true + "gitpod.io/workload_workspace_regular" = true + "gitpod.io/workload_workspace_headless" = true + } } - // this module is intended to be run separately from the above two. so a separate target for apply is necessary module "tools" { source = "github.com/gitpod-io/gitpod//install/infra/terraform/tools/cm-cloud-dns?ref=nvn-infra-tf" # we can later use tags here From c6190915f65fe1668852a6d4dc0bc6284a97c6a3 Mon Sep 17 00:00:00 2001 From: Nandaja Varma Date: Thu, 2 Jun 2022 13:22:35 +0000 Subject: [PATCH 4/4] add dns mapping in k3s --- install/infra/terraform/k3s/main.tf | 17 +++++++++++++++++ install/infra/terraform/k3s/variables.tf | 20 ++++++++++++++++++++ install/tests/main.tf | 18 +++++++++++------- 3 files changed, 48 insertions(+), 7 deletions(-) diff --git a/install/infra/terraform/k3s/main.tf b/install/infra/terraform/k3s/main.tf index 645a0386477d96..5a4ca2ab515795 100644 --- a/install/infra/terraform/k3s/main.tf +++ b/install/infra/terraform/k3s/main.tf @@ -21,6 +21,11 @@ provider "google" { zone = var.gcp_zone } +provider "google" { + alias = "dns" + credentials = var.dns_sa_creds +} + resource "google_service_account" "gcp_instance" { account_id = "gcp-k3s-compute" display_name = "Service Account" @@ -94,6 +99,18 @@ resource "null_resource" "k3sup_install" { } } +resource "google_dns_record_set" "a" { + provider = google.dns + count = (var.domain_name == null) || (var.managed_dns_zone == null ) ? 0 : 1 + name = "${var.domain_name}." + managed_zone = var.managed_dns_zone + project = var.dns_project == null ? var.gcp_project : var.dns_project + type = "A" + ttl = 5 + + rrdatas = [google_compute_instance.k3s_master_instance.network_interface[0].access_config[0].nat_ip] +} + data "local_file" "kubeconfig" { depends_on = [null_resource.k3sup_install] filename = var.kubeconfig diff --git a/install/infra/terraform/k3s/variables.tf b/install/infra/terraform/k3s/variables.tf index d2ed0c12f6c1a9..26096d197bf097 100644 --- a/install/infra/terraform/k3s/variables.tf +++ b/install/infra/terraform/k3s/variables.tf @@ -26,3 +26,23 @@ variable "name" { description = "Prefix name for the nodes and firewall" default = "k3s" } + +variable "dns_sa_creds" { + description = "Credentials with DNS admin rights to the project with managed DNS record" + default = "" +} + +variable "dns_project" { + description = "Project associated with the dns maanged zone" + default = null +} + +variable "domain_name" { + description = "Domain name to add to add DNS map to" + default = null +} + +variable "managed_dns_zone" { + description = "Name of the managed DNS record" + default = null +} diff --git a/install/tests/main.tf b/install/tests/main.tf index d6d1025de01f49..b09d10aaba5df7 100644 --- a/install/tests/main.tf +++ b/install/tests/main.tf @@ -10,8 +10,8 @@ terraform { } variable "project" { default = "sh-automated-tests" } -variable "sa_creds" { default = "" } -variable "dns_sa_creds" {default = "" } +variable "sa_creds" { default = "/workspace/gcp.json" } +variable "dns_sa_creds" {default = "/workspace/gitpod/gcp.json" } module "gke" { source = "github.com/gitpod-io/gitpod//install/infra/terraform/gke?ref=nvn-infra-tf" # we can later use tags here @@ -23,12 +23,16 @@ module "gke" { } module "k3s" { - source = "github.com/gitpod-io/gitpod//install/infra/terraform/k3s?ref=nvn-infra-tf" # we can later use tags here + source = "github.com/gitpod-io/gitpod//install/infra/terraform/k3s?ref=nvn-infra-aks" # we can later use tags here - name = var.TEST_ID - gcp_project = var.project - credentials = var.sa_creds - kubeconfig = var.kubeconfig + name = var.TEST_ID + gcp_project = var.project + credentials = var.sa_creds + kubeconfig = var.kubeconfig + dns_sa_creds = var.dns_sa_creds + dns_project = "dns-for-playgrounds" + managed_dns_zone = "gitpod-self-hosted-com" + domain_name = "${var.TEST_ID}.gitpod-self-hosted.com" } module "aks" {