From b8a6f31b1bc86ef0caa3820c39c52a6c6bf44141 Mon Sep 17 00:00:00 2001
From: Nandaja Varma <nandaja.varma@gmail.com>
Date: Tue, 31 May 2022 08:57:57 +0000
Subject: [PATCH 1/4] Allowing versions

---
 .werft/installer-tests.ts                     | 10 ++-
 install/infra/terraform/gke/README.md         | 90 +++++++++++++++++++
 install/infra/terraform/gke/main.tf           | 80 ++++++++---------
 install/infra/terraform/gke/output.tf         |  5 +-
 .../infra/terraform/gke/variables.auto.tfvars |  5 ++
 install/infra/terraform/gke/variables.tf      | 55 ++++--------
 install/infra/terraform/k3s/README.md         | 55 +++++++++++-
 install/infra/terraform/k3s/main.tf           | 43 ++++-----
 .../infra/terraform/k3s/variables.auto.tfvars |  5 +-
 install/infra/terraform/k3s/variables.tf      | 21 ++---
 .../terraform/tools/cm-cloud-dns/README.md    |  6 ++
 install/tests/Makefile                        | 44 ++++-----
 install/tests/main.tf                         | 36 ++++----
 13 files changed, 289 insertions(+), 166 deletions(-)
 create mode 100644 install/infra/terraform/gke/README.md
 create mode 100644 install/infra/terraform/gke/variables.auto.tfvars
 create mode 100644 install/infra/terraform/tools/cm-cloud-dns/README.md

diff --git a/.werft/installer-tests.ts b/.werft/installer-tests.ts
index e864dc0e6b3a15..a67cfde849963e 100644
--- a/.werft/installer-tests.ts
+++ b/.werft/installer-tests.ts
@@ -2,7 +2,9 @@ import { join } from "path";
 import { exec } from "./util/shell";
 import { Werft } from "./util/werft";
 
-const testConfig: string = process.argv.length > 2 ? process.argv[2] : "gke";
+const testConfig: string = process.argv.length > 2 ? process.argv[2] : "STANDARD_K3S_TEST";
+// we can provide the version of the gitpod to install (eg: 2022.4.2)
+const version: string = process.argv.length > 3 ? process.argv[3] : "";
 
 const makefilePath: string = join("install/tests");
 
@@ -36,15 +38,16 @@ const INFRA_PHASES: { [name: string]: InfraConfig } = {
     },
     INSTALL_GITPOD_IGNORE_PREFLIGHTS: {
         phase: "install-gitpod-without-preflights",
-        makeTarget: "kots-install-without-preflight-with-community-license",
+        makeTarget: `kots-install channel=unstable version=${version} preflights=false`, // this is a bit of a hack, for now we pass params like this
         description: "Install gitpod using kots community edition without preflights",
     },
     INSTALL_GITPOD: {
         phase: "install-gitpod",
-        makeTarget: "kots-install-with-community-license",
+        makeTarget: `kots-install channel=unstable version=${version} preflights=true`,
         description: "Install gitpod using kots community edition",
     },
     CHECK_INSTALLATION: {
+        // this is a basic test for the Gitpod setup
         phase: "check-gitpod-installation",
         makeTarget: "check-gitpod-installation",
         description: "Check gitpod installation",
@@ -155,6 +158,7 @@ function callMakeTargets(phase: string, description: string, makeTarget: string)
         werft.fail(phase, "Operation failed");
     } else {
         werft.log(phase, response.stdout.toString());
+        werft.done(phase);
     }
 
     return response.code;
diff --git a/install/infra/terraform/gke/README.md b/install/infra/terraform/gke/README.md
new file mode 100644
index 00000000000000..8886047aaf064c
--- /dev/null
+++ b/install/infra/terraform/gke/README.md
@@ -0,0 +1,90 @@
+# GKE terraform module 
+
+This is a terraform module currently used in the automated installation tests
+for Gitpod. At successful completion, this module creates the following:
+
+1. A [`GKE`](https://cloud.google.com/kubernetes-engine) cluster by user
+   provided name `<name>` with the following nodepools:
+   1. `workspaces-<name>` for workspace workloads
+   1. `services-<name>` for IDE and meta workloads
+1. A dedicated `vpc` and `subnet` for the resources
+1. `kubeconfig` data populated in a user defined `kubeconfig` variable
+
+## Requirements
+
+1. `terraform` >= `v1.1.7`
+
+## Providers
+
+1. [`google`](https://registry.terraform.io/providers/hashicorp/google/latest/docs)
+
+## Inputs 
+
+
+| Argument                  | Description                                                                                                                                  | Required | Default            |
+|---------------------------|----------------------------------------------------------------------------------------------------------------------------------------------|----------|--------------------|
+| `project`                 | The project ID to create the cluster.                                                                                                        | `true`   |                    |
+| `kubeconfig`              | Path to write the kubeconfig output to.                                                                                                      | `false`  | `./kubeconfig`     |
+| `region`                  | The region to create the cluster.                                                                                                            | `false`  | `europe-west1`     |
+| `zone`                    | The zone suffix to create the cluster in. If an empty string is provided, it will create multi-zone cluster                                  | `false`  | `b`                |
+| `kubernetes_version`      | Kubernetes version to be setup                                                                                                               | `false`  | `1.21.11-gke.1100` |
+| `name`                    | The name of the cluster and suffix for other resources                                                                                       | `false`  | `sh-test`          |
+| `workspaces_machine_type` | Type of the node compute engines for workspace nodepool.                                                                                     | `false`  | `n2-standard-8`    |
+| `services_machine_type`   | Type of the node compute engines for meta and IDE nodepool.                                                                                  | `false`  | `n2-standard-4`    |
+| `min_count`               | Minimum number of nodes in the NodePool. Must be >=0 and <= max_node_count.                                                                  | `false`  | `1`                |
+| `max_count`               | Maximum number of nodes in the NodePool. Must be >= min_node_count.                                                                          | `false`  | `50`               |
+| `disk_size_gb`            | Disk size to provision in nodes.                                                                                                             | `false`  | `100`              |
+| `initial_node_count`      | The number of nodes to create in this cluster's default node pool.                                                                           | `false`  | `1`                |
+| `pre-emptible`            | Set if the nodes are to be pre-emptible.                                                                                                     | `false`  | `false`            |
+| `credentials`             | Path to the JSON file storing Google service account credentials, if left empty, `tf` will look for `GOOGLE_APPLICATION_CREDENTIALS` env var | `false`  |                    |
+|                           |                                                                                                                                              |          |                    |
+
+
+## Outputs
+
+
+| Argument              |
+|-----------------------|
+| `kubernetes_endpoint` |
+| `client_token`        |
+| `ca_certificate`      |
+| `kubeconfig`          |
+
+
+## Usage
+
+Make sure you have a `GCP` account and credentials in `JSON` format to a service
+account with atleast the following permissions:
+1. Compute Admin
+1. Compute Network Admin
+1. Kubernetes Engine Admin
+1. Kubernetes Engine Developer
+1. Service Account User
+1. Storage Admin
+
+Assign it to the following environment variable:
+
+``` sh
+export GOOGLE_APPLICATION_CREDENTAILS=/path/to/service-account-json-creds
+```
+
+Create a `Cloud storage bucket` in the same project and region and specified
+above, by name `gitpod-gke`. If you want to use a different name, edit the name
+manually in the `main.tf` file.
+
+Open the file `variables.auto.tfvars` in this directory and provide values available from the
+`Inputs` list as you find fit.
+
+Run the following commands:
+
+``` sh
+terraform init
+terraform apply
+```
+
+
+## Cleanup
+
+``` sh
+terraform destroy
+```
diff --git a/install/infra/terraform/gke/main.tf b/install/infra/terraform/gke/main.tf
index 309b36ee17a7a6..518b5b11099933 100644
--- a/install/infra/terraform/gke/main.tf
+++ b/install/infra/terraform/gke/main.tf
@@ -2,11 +2,18 @@ terraform {
   required_version = ">= 1.0.3"
 }
 
+terraform {
+  backend "gcs" {
+    bucket = "gitpod-gke"
+    prefix = "tf-state"
+  }
+}
+
 provider "google" {
+  project     = var.project
   credentials = var.credentials
-  project = var.project
-  region  = var.region
-  zone = var.zone
+  region      = var.region
+  zone        = var.zone
 }
 
 resource "google_compute_network" "vpc" {
@@ -14,7 +21,6 @@ resource "google_compute_network" "vpc" {
   auto_create_subnetworks = "false"
 }
 
-# Subnet
 resource "google_compute_subnetwork" "subnet" {
   name          = "subnet-${var.name}"
   region        = var.region
@@ -40,9 +46,9 @@ resource "google_container_cluster" "gitpod-cluster" {
     enabled = true
 
     resource_limits {
-        resource_type = "cpu"
-        minimum       = 2
-        maximum       = 8
+      resource_type = "cpu"
+      minimum       = 2
+      maximum       = 8
     }
 
     resource_limits {
@@ -79,27 +85,19 @@ resource "google_container_cluster" "gitpod-cluster" {
     horizontal_pod_autoscaling {
       disabled = false
     }
-
-
-    # only available in beta
-    # dns_cache_config {
-    #   enabled = true
-    # }
   }
 
   network    = google_compute_network.vpc.name
   subnetwork = google_compute_subnetwork.subnet.name
-
-
 }
 
 resource "google_container_node_pool" "services" {
-  name       = "services-${var.name}"
-  location   = google_container_cluster.gitpod-cluster.location
-  cluster    = google_container_cluster.gitpod-cluster.name
-  version    = var.kubernetes_version // kubernetes version
+  name               = "services-${var.name}"
+  location           = google_container_cluster.gitpod-cluster.location
+  cluster            = google_container_cluster.gitpod-cluster.name
+  version            = var.kubernetes_version // kubernetes version
   initial_node_count = 1
-  max_pods_per_node = 110
+  max_pods_per_node  = 110
 
   node_config {
     oauth_scopes = [
@@ -107,15 +105,15 @@ resource "google_container_node_pool" "services" {
     ]
 
     labels = {
-      "gitpod.io/workload_meta" =true
-      "gitpod.io/workload_ide" = true
+      "gitpod.io/workload_meta" = true
+      "gitpod.io/workload_ide"  = true
     }
 
     preemptible  = var.pre-emptible
-    image_type = "UBUNTU_CONTAINERD"
-    disk_type = "pd-ssd"
-    disk_size_gb  = var.disk_size_gb
-    machine_type = var.machine_type
+    image_type   = "UBUNTU_CONTAINERD"
+    disk_type    = "pd-ssd"
+    disk_size_gb = var.disk_size_gb
+    machine_type = var.services_machine_type
     tags         = ["gke-node", "${var.project}-gke"]
     metadata = {
       disable-legacy-endpoints = "true"
@@ -129,18 +127,18 @@ resource "google_container_node_pool" "services" {
 
 
   management {
-    auto_repair = true
+    auto_repair  = true
     auto_upgrade = false
   }
 }
 
 resource "google_container_node_pool" "workspaces" {
-  name       = "workspaces-${var.name}"
-  location     = google_container_cluster.gitpod-cluster.location
-  cluster    = google_container_cluster.gitpod-cluster.name
-  version    = var.kubernetes_version // kubernetes version
+  name               = "workspaces-${var.name}"
+  location           = google_container_cluster.gitpod-cluster.location
+  cluster            = google_container_cluster.gitpod-cluster.name
+  version            = var.kubernetes_version // kubernetes version
   initial_node_count = 1
-  max_pods_per_node = 110
+  max_pods_per_node  = 110
 
   node_config {
     oauth_scopes = [
@@ -148,18 +146,18 @@ resource "google_container_node_pool" "workspaces" {
     ]
 
     labels = {
-      "gitpod.io/workload_metal" =true
-      "gitpod.io/workload_ide" = true
+      "gitpod.io/workload_metal"              = true
+      "gitpod.io/workload_ide"                = true
       "gitpod.io/workload_workspace_services" = true
-      "gitpod.io/workload_workspace_regular" = true
+      "gitpod.io/workload_workspace_regular"  = true
       "gitpod.io/workload_workspace_headless" = true
     }
 
     preemptible  = var.pre-emptible
-    image_type = "UBUNTU_CONTAINERD"
-    disk_type = "pd-ssd"
-    disk_size_gb  = var.disk_size_gb
-    machine_type = var.machine_type
+    image_type   = "UBUNTU_CONTAINERD"
+    disk_type    = "pd-ssd"
+    disk_size_gb = var.disk_size_gb
+    machine_type = var.workspaces_machine_type
     tags         = ["gke-node", "${var.project}-gke"]
     metadata = {
       disable-legacy-endpoints = "true"
@@ -172,7 +170,7 @@ resource "google_container_node_pool" "workspaces" {
   }
 
   management {
-    auto_repair = true
+    auto_repair  = true
     auto_upgrade = false
   }
 }
@@ -189,5 +187,5 @@ module "gke_auth" {
 
 resource "local_file" "kubeconfig" {
   filename = var.kubeconfig
-  content = module.gke_auth.kubeconfig_raw
+  content  = module.gke_auth.kubeconfig_raw
 }
diff --git a/install/infra/terraform/gke/output.tf b/install/infra/terraform/gke/output.tf
index 29702d742affbd..470911d8a97705 100644
--- a/install/infra/terraform/gke/output.tf
+++ b/install/infra/terraform/gke/output.tf
@@ -10,9 +10,10 @@ output "client_token" {
 
 output "ca_certificate" {
   sensitive = true
-  value = module.gke_auth.cluster_ca_certificate
+  value     = module.gke_auth.cluster_ca_certificate
 }
 
 output "kubeconfig" {
-  value = module.gke_auth.kubeconfig_raw
+  sensitive = true
+  value     = module.gke_auth.kubeconfig_raw
 }
diff --git a/install/infra/terraform/gke/variables.auto.tfvars b/install/infra/terraform/gke/variables.auto.tfvars
new file mode 100644
index 00000000000000..d6fc1bfe33cea2
--- /dev/null
+++ b/install/infra/terraform/gke/variables.auto.tfvars
@@ -0,0 +1,5 @@
+kubeconfig = "./kubeconfig"
+
+project = "sh-automated-tests"
+region = "europe-west1"
+zone = "b"
diff --git a/install/infra/terraform/gke/variables.tf b/install/infra/terraform/gke/variables.tf
index 50197379cc53bd..4aa4c3c9349c9c 100644
--- a/install/infra/terraform/gke/variables.tf
+++ b/install/infra/terraform/gke/variables.tf
@@ -4,14 +4,9 @@ variable "project" {
 }
 
 variable "kubeconfig" {
-  type = string
-  description = "Path to write the kubeconfig output to"
-  default = "./kubeconfig"
-}
-
-variable "credentials" {
   type        = string
-  description = "Path to the json file storing SA credentials."
+  description = "Path to write the kubeconfig output to"
+  default     = "./kubeconfig"
 }
 
 variable "region" {
@@ -38,10 +33,16 @@ variable "name" {
   default     = "sh-test"
 }
 
-variable "machine_type" {
+variable "workspaces_machine_type" {
   type        = string
-  description = "Type of the node compute engines."
-  default     = "n2d-standard-4"
+  description = "Type of the node compute engines for workspace nodepool."
+  default     = "n2-standard-8"
+}
+
+variable "services_machine_type" {
+  type        = string
+  description = "Type of the node compute engines for services nodepool."
+  default     = "n2-standard-4"
 }
 
 variable "min_count" {
@@ -59,7 +60,7 @@ variable "max_count" {
 variable "disk_size_gb" {
   type        = number
   description = "Size of the node's disk."
-  default = 100
+  default     = 100
 }
 
 variable "initial_node_count" {
@@ -68,33 +69,13 @@ variable "initial_node_count" {
   default     = 1
 }
 
-variable "env_name" {
-  description = "The environment for the GKE cluster"
-  default     = "test"
-}
-
-variable "network" {
-  description = "The VPC network created to host the cluster in"
-  default     = "gitpod-gke-network"
-}
-
-variable "subnetwork" {
-  description = "The subnetwork created to host the cluster in"
-  default     = "gitpod-gke-subnet"
-}
-
-variable "ip_range_pods_name" {
-  description = "The secondary ip range to use for pods"
-  default     = "gitpod-ip-range-pods"
-}
-
-variable "ip_range_services_name" {
-  description = "The secondary ip range to use for services"
-  default     = "gitpod-ip-range-services"
-}
-
 variable "pre-emptible" {
-  type = bool
+  type        = bool
   description = "Set if the nodes are to be pre-emptible"
   default     = false
 }
+
+variable "credentials" {
+  description = "Path to the JSON file storing Google service account credentials"
+  default     = ""
+}
diff --git a/install/infra/terraform/k3s/README.md b/install/infra/terraform/k3s/README.md
index 72544c364f2f18..90b611412e4c0c 100644
--- a/install/infra/terraform/k3s/README.md
+++ b/install/infra/terraform/k3s/README.md
@@ -1,15 +1,56 @@
-## Module to create a 1 node k3s cluster in GCP
+# k3s terraform module 
+
+This is a terraform module currently used in the automated installation tests
+for Gitpod. At successful completion, this module creates the following:
+
+1. A GCP VM instance of type `n2d-standard-4` and with `ubuntu` image, along with a service account to give auth scopes
+1. A single node [`k3s`](https://k3s.io/) cluster with [`k3sup`](https://github.com/alexellis/k3sup)
 
 ### Requirements
 
-- `terraform` >= 1.0.0
-- [`k3sup`](https://github.com/alexellis/k3sup#download-k3sup-tldr)
+1. `terraform` >= `v1.1.7`
+1. [`k3sup`](https://github.com/alexellis/k3sup#download-k3sup-tldr)
     ```sh
     curl -sLS https://get.k3sup.dev | sh
     sudo install k3sup /usr/local/bin/
     ```
 
-### Usage
+## Providers
+
+1. [`google`](https://registry.terraform.io/providers/hashicorp/google/latest/docs)
+
+## Input
+
+| Argument      | Description                                                                                                                                  | Required | Default          |
+|---------------|----------------------------------------------------------------------------------------------------------------------------------------------|----------|------------------|
+| `gcp_project` | The project ID to create the VM in.                                                                                                          | `true`   |                  |
+| `kubeconfig`  | Path to write the kubeconfig output to.                                                                                                      | `false`  | `./kubeconfig`   |
+| `gcp_region`  | The region to create the VM.                                                                                                                 | `false`  | `europe-west1`   |
+| `gcp_zone`    | The GCP zone to create the VM in.                                                                                                            | `false`  | `europe-west1-b` |
+| `name`        | Prefix name for the nodes and firewall                                                                                                       | `false`  | `k3s`            |
+| `credentials` | Path to the JSON file storing Google service account credentials, if left empty, `tf` will look for `GOOGLE_APPLICATION_CREDENTIALS` env var | `false`  |                  |
+
+## Output
+
+| Argument              |
+|-----------------------|
+| `kubernetes_endpoint` |
+| `kubeconfig`          |
+
+## Usage
+
+Create a service account with at least the following access scopes:
+- Compute Admin
+- Create Service Accounts
+- Delete Service Accounts
+- Service Account User
+- Storage Admin
+
+Get the credentials of the SA in `JSON` format and export as a variable as folows:
+
+``` sh
+export GOOGLE_APPLICATION_CREDENTIALS=/path/to/credentials
+```
 
 Update the `tfvars` file by the name `variables.auto.tfvars` in this
 directory with the values you want, and then run:
@@ -21,3 +62,9 @@ terraform apply
 
 You will find the `kubeconfig` populated to the path you specified in
 the `variables.auto.tfvars` file.
+
+## Cleanup
+
+```sh
+terraform destroy
+```
diff --git a/install/infra/terraform/k3s/main.tf b/install/infra/terraform/k3s/main.tf
index 60876cf0c4a3d4..645a0386477d96 100644
--- a/install/infra/terraform/k3s/main.tf
+++ b/install/infra/terraform/k3s/main.tf
@@ -7,11 +7,18 @@ terraform {
   }
 }
 
+terraform {
+  backend "gcs" {
+    bucket = "gitpod-k3s"
+    prefix = "tf-state"
+  }
+}
+
 provider "google" {
   credentials = var.credentials
-  project = var.gcp_project
-  region  = var.gcp_region
-  zone    = var.gcp_zone
+  project     = var.gcp_project
+  region      = var.gcp_region
+  zone        = var.gcp_zone
 }
 
 resource "google_service_account" "gcp_instance" {
@@ -31,15 +38,15 @@ resource "local_file" "ssh_private_key_pem" {
 }
 
 resource "google_compute_instance" "k3s_master_instance" {
-  name    = "master-${var.name}"
+  name         = "master-${var.name}"
   machine_type = "n2d-standard-4"
-  tags         = ["k3s", "k3s-master", "http-server", "https-server"]
+  tags         = ["k3s", "k3s-master", "http-server", "https-server", "allow-ssh"]
 
   boot_disk {
     initialize_params {
       image = "ubuntu-2004-focal-v20220419"
-      size = 100
-      type = "pd-ssd"
+      size  = 100
+      type  = "pd-ssd"
     }
   }
 
@@ -63,8 +70,6 @@ resource "google_compute_instance" "k3s_master_instance" {
     email  = google_service_account.gcp_instance.email
     scopes = ["cloud-platform"]
   }
-
-
 }
 
 resource "time_sleep" "wait_for_master" {
@@ -73,7 +78,6 @@ resource "time_sleep" "wait_for_master" {
   create_duration = "60s"
 }
 
-
 resource "null_resource" "k3sup_install" {
   depends_on = [time_sleep.wait_for_master]
 
@@ -83,7 +87,7 @@ resource "null_resource" "k3sup_install" {
               --ip ${google_compute_instance.k3s_master_instance.network_interface[0].access_config[0].nat_ip} \
               --ssh-key .ssh/google_compute_engine \
               --context k3s \
-              --user $(whoami) \
+              --user gitpod \
               --local-path ${var.kubeconfig} \
               --k3s-extra-args=" --disable=traefik --node-label=gitpod.io/workload_meta=true --node-label=gitpod.io/workload_ide=true --node-label=gitpod.io/workload_workspace_services=true --node-label=gitpod.io/workload_workspace_regular=true --node-label=gitpod.io/workload_workspace_headless=true" \
           EOT
@@ -92,7 +96,7 @@ resource "null_resource" "k3sup_install" {
 
 data "local_file" "kubeconfig" {
   depends_on = [null_resource.k3sup_install]
-  filename = var.kubeconfig
+  filename   = var.kubeconfig
 }
 
 resource "google_compute_firewall" "k3s-firewall" {
@@ -104,21 +108,6 @@ resource "google_compute_firewall" "k3s-firewall" {
     ports    = ["6443"]
   }
 
-  allow {
-    protocol = "tcp"
-    ports    = ["22"]
-  }
-
-  allow {
-    protocol = "tcp"
-    ports    = ["443"]
-  }
-
-  allow {
-    protocol = "tcp"
-    ports    = ["80"]
-  }
-
   target_tags = ["k3s"]
 
   source_ranges = ["0.0.0.0/0"]
diff --git a/install/infra/terraform/k3s/variables.auto.tfvars b/install/infra/terraform/k3s/variables.auto.tfvars
index fca6fcadfaab2c..2f7c91e8f51ed4 100644
--- a/install/infra/terraform/k3s/variables.auto.tfvars
+++ b/install/infra/terraform/k3s/variables.auto.tfvars
@@ -1,6 +1,5 @@
 kubeconfig = "./kubeconfig"
 
 gcp_project = "sh-automated-tests"
-gcp_region = "europe-west1"
-gcp_zone = "europe-west1-b"
-credentials = "/workspace/gcp.json"
+gcp_region  = "europe-west1"
+gcp_zone    = "europe-west1-b"
diff --git a/install/infra/terraform/k3s/variables.tf b/install/infra/terraform/k3s/variables.tf
index 7899623f393156..d2ed0c12f6c1a9 100644
--- a/install/infra/terraform/k3s/variables.tf
+++ b/install/infra/terraform/k3s/variables.tf
@@ -1,27 +1,28 @@
 variable "kubeconfig" {
-    description = "The KUBECONFIG file path to store the resulting KUBECONFIG file to"
-    default = "./kubeconfig"
+  description = "The KUBECONFIG file path to store the resulting KUBECONFIG file to"
+  default     = "./kubeconfig"
 }
 
 variable "gcp_project" {
-    description = "Google cloud Region to perform operations in"
+  description = "Google cloud Region to perform operations in"
 }
 
 variable "gcp_region" {
-    description = "Google cloud Region to perform operations in"
-    default = "europe-west1"
+  description = "Google cloud Region to perform operations in"
+  default     = "europe-west1"
 }
 
 variable "gcp_zone" {
-    description = "Google cloud Zone to perform operations in"
-    default = "europe-west1-b"
+  description = "Google cloud Zone to perform operations in"
+  default     = "europe-west1-b"
 }
 
 variable "credentials" {
-    description = "Path to the JSON file storing Google service account credentials"
+  description = "Path to the JSON file storing Google service account credentials"
+  default     = ""
 }
 
 variable "name" {
-    description = "Prefix name for the nodes and firewall"
-    default = "k3s"
+  description = "Prefix name for the nodes and firewall"
+  default     = "k3s"
 }
diff --git a/install/infra/terraform/tools/cm-cloud-dns/README.md b/install/infra/terraform/tools/cm-cloud-dns/README.md
new file mode 100644
index 00000000000000..a7805293f15aeb
--- /dev/null
+++ b/install/infra/terraform/tools/cm-cloud-dns/README.md
@@ -0,0 +1,6 @@
+# Managed DNS terraform module
+
+This module is used in test setup, to deploy `cert-manager` and `external-dns`
+on a created cluster, and to create a secret that the aforementioned deployments
+can use to modify a `cloudDNS` record. This module works specifically for the
+domain `gitpod-self-hosted.com`.
diff --git a/install/tests/Makefile b/install/tests/Makefile
index 82c60a0bb5ca58..d8b3b4a73169e4 100644
--- a/install/tests/Makefile
+++ b/install/tests/Makefile
@@ -5,7 +5,7 @@ PROJECTNAME := "installer-nightly-tests"
 
 TOPDIR=$(shell pwd)
 
-KUBECONFIG := "$(TOPDIR)/kubeconfig"
+KUBECONFIG := "/workspace/conf"
 
 check-env-sub-domain:
 ifndef TF_VAR_TEST_ID
@@ -53,37 +53,39 @@ get-kubeconfig:
 	${KUBECONFIG}
 
 KOTS_KONFIG := "./manifests/kots-config.yaml"
-CHANNEL = "gitpod/beta"
-COMMUNITY_LICENSE := "../licenses/Community (Beta).yaml"
+license_community_beta := "../licenses/Community (Beta).yaml"
+license_community_stable := "../licenses/Community.yaml"
+license_community_unstable := "../licenses/Community (Unstable).yaml"
 
 install-kots-cli:
 	curl https://kots.io/install | bash
 
-kots-install-without-preflight-with-community-license: install-kots-cli
+preflights ?= true
+channel ?= unstable
+version ?= -
+kots-install: version-flag = $(if $(version:-=),--app-version-label=$(version),)
+kots-install: preflight-flag = $(if $(preflights:true=),--skip-preflights,)
+kots-install: license-file =  $(license_community_$(channel))
+kots-install: install-kots-cli
 	envsubst < ${KOTS_KONFIG} > tmp_config.yml
-	kubectl kots install ${CHANNEL} \
-	--skip-rbac-check --skip-preflights --namespace gitpod --kubeconfig=${KUBECONFIG} \
+	kubectl kots install gitpod/${channel} \
+	--skip-rbac-check ${version-flag} ${preflight-flag} \
+					--namespace gitpod --kubeconfig=${KUBECONFIG} \
                     --name gitpod --shared-password gitpod \
-					--license-file ${COMMUNITY_LICENSE} \
+					--license-file ${license-file} \
                     --no-port-forward \
                     --config-values tmp_config.yml
 
-kots-install-with-community-license: install-kots-cli
-	envsubst < ${KOTS_KONFIG} > tmp_config.yml
-	kubectl kots install ${CHANNEL} \
-	--skip-rbac-check --namespace gitpod --kubeconfig=${KUBECONFIG} \
-                    --name gitpod --shared-password gitpod \
-					--license-file ${COMMUNITY_LICENSE} \
-                    --no-port-forward \
-                    --config-values tmp_config.yml
+delete-cm-setup:
+	sleep 120 && kubectl --kubeconfig=${KUBECONFIG} delete pods --all -n cert-manager && sleep 300;
 
-check-gitpod-installation:
-	sleep 120
-	kubectl --kubeconfig=${KUBECONFIG} delete pods --all -n cert-manager
-	sleep 300
-	@kubectl kots get --kubeconfig=${KUBECONFIG} app gitpod -n gitpod | grep gitpod  | awk '{print $$2}'
+check-kots-app:
+	kubectl kots get --kubeconfig=${KUBECONFIG} app gitpod -n gitpod | grep gitpod  | awk '{print $$2}' | grep "ready" || { echo "Gitpod is not ready"; exit 1; }
+
+check-gitpod-installation: delete-cm-setup check-kots-app check-env-sub-domain
+	@echo "Curling http://${TF_VAR_TEST_ID}.gitpod-self-hosted.com/api/version"
+	curl -i -X GET http://${TF_VAR_TEST_ID}.gitpod-self-hosted.com/api/version || { echo "Curling Gitpod endpoint failed"; exit 1; }
 
-.PHONY:
 run-tests:
 	./tests.sh ${KUBECONFIG}
 
diff --git a/install/tests/main.tf b/install/tests/main.tf
index 3e4a691e2033c8..16408fd879440f 100644
--- a/install/tests/main.tf
+++ b/install/tests/main.tf
@@ -1,43 +1,43 @@
 variable "kubeconfig" {}
 variable "TEST_ID" {
-    default = "nightly"
+  default = "nightly"
 }
 variable "project" {
-    default = "sh-automated-tests"
+  default = "sh-automated-tests"
 }
 variable "sa_creds" {}
 variable "dns_sa_creds" {}
 
 terraform {
   backend "gcs" {
-    bucket      = "nightly-tests"
-    prefix      = "tf-state"
+    bucket = "nightly-tests"
+    prefix = "tf-state"
   }
 }
 
 module "gke" {
-    source      = "github.com/gitpod-io/gitpod//install/infra/terraform/gke?ref=nvn-infra-tf" # we can later use tags here
+  source = "github.com/gitpod-io/gitpod//install/infra/terraform/gke?ref=nvn-infra-tf" # we can later use tags here
 
-    name  = var.TEST_ID
-    project = var.project
-    credentials = var.sa_creds
-    kubeconfig  = var.kubeconfig
+  name        = var.TEST_ID
+  project     = var.project
+  credentials = var.sa_creds
+  kubeconfig  = var.kubeconfig
 }
 
 module "k3s" {
-    source      = "github.com/gitpod-io/gitpod//install/infra/terraform/k3s?ref=nvn-infra-tf" # we can later use tags here
+  source = "github.com/gitpod-io/gitpod//install/infra/terraform/k3s?ref=nvn-infra-tf" # we can later use tags here
 
-    name  = var.TEST_ID
-    gcp_project = var.project
-    credentials = var.sa_creds
-    kubeconfig  = var.kubeconfig
+  name        = var.TEST_ID
+  gcp_project = var.project
+  credentials = var.sa_creds
+  kubeconfig  = var.kubeconfig
 }
 
 // this module is intended to be run separately from the above two. so a separate target for apply is necessary
 module "tools" {
-    source          = "github.com/gitpod-io/gitpod//install/infra/terraform/tools/cm-cloud-dns?ref=nvn-infra-tf" # we can later use tags here
+  source = "github.com/gitpod-io/gitpod//install/infra/terraform/tools/cm-cloud-dns?ref=nvn-infra-tf" # we can later use tags here
 
-    kubeconfig      = var.kubeconfig
-    credentials     = var.dns_sa_creds
-    gcp_sub_domain  = var.TEST_ID
+  kubeconfig     = var.kubeconfig
+  credentials    = var.dns_sa_creds
+  gcp_sub_domain = var.TEST_ID
 }

From 73dc14d4cb6e33a3b9758b01f242c7fee6ad9102 Mon Sep 17 00:00:00 2001
From: Nandaja Varma <nandaja.varma@gmail.com>
Date: Wed, 1 Jun 2022 08:25:34 +0000
Subject: [PATCH 2/4] adding AKS tf module

---
 install/infra/terraform/aks/README.md     | 97 +++++++++++++++++++++++
 install/infra/terraform/aks/database.tf   | 50 ++++++++++++
 install/infra/terraform/aks/kubernetes.tf | 95 ++++++++++++++++++++++
 install/infra/terraform/aks/local.tf      | 95 ++++++++++++++++++++++
 install/infra/terraform/aks/main.tf       | 24 ++++++
 install/infra/terraform/aks/monitoring.tf | 19 +++++
 install/infra/terraform/aks/networks.tf   | 20 +++++
 install/infra/terraform/aks/output.tf     | 89 +++++++++++++++++++++
 install/infra/terraform/aks/registry.tf   | 25 ++++++
 install/infra/terraform/aks/storage.tf    | 16 ++++
 install/infra/terraform/aks/variables.tf  | 24 ++++++
 install/tests/main.tf                     | 30 ++++---
 12 files changed, 574 insertions(+), 10 deletions(-)
 create mode 100644 install/infra/terraform/aks/README.md
 create mode 100644 install/infra/terraform/aks/database.tf
 create mode 100644 install/infra/terraform/aks/kubernetes.tf
 create mode 100644 install/infra/terraform/aks/local.tf
 create mode 100644 install/infra/terraform/aks/main.tf
 create mode 100644 install/infra/terraform/aks/monitoring.tf
 create mode 100644 install/infra/terraform/aks/networks.tf
 create mode 100644 install/infra/terraform/aks/output.tf
 create mode 100644 install/infra/terraform/aks/registry.tf
 create mode 100644 install/infra/terraform/aks/storage.tf
 create mode 100644 install/infra/terraform/aks/variables.tf

diff --git a/install/infra/terraform/aks/README.md b/install/infra/terraform/aks/README.md
new file mode 100644
index 00000000000000..e03e1cc8289b3b
--- /dev/null
+++ b/install/infra/terraform/aks/README.md
@@ -0,0 +1,97 @@
+# Azure
+
+Azure provider for Gitpod testing
+
+<!-- toc -->
+
+- [Terraform Documentation](#terraform-documentation)
+  * [Requirements](#requirements)
+  * [Providers](#providers)
+  * [Modules](#modules)
+  * [Resources](#resources)
+  * [Inputs](#inputs)
+  * [Outputs](#outputs)
+
+<!-- tocstop -->
+
+# Terraform Documentation
+
+<!-- BEGIN_TF_DOCS -->
+## Requirements
+
+| Name | Version |
+|------|---------|
+| <a name="requirement_azurerm"></a> [azurerm](#requirement\_azurerm) | >= 3.0.0, < 4.0.0 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_azurerm"></a> [azurerm](#provider\_azurerm) | >= 3.0.0, < 4.0.0 |
+| <a name="provider_random"></a> [random](#provider\_random) | n/a |
+
+## Modules
+
+No modules.
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [azurerm_container_registry.registry](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/container_registry) | resource |
+| [azurerm_dns_zone.dns](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/dns_zone) | resource |
+| [azurerm_kubernetes_cluster.k8s](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/kubernetes_cluster) | resource |
+| [azurerm_kubernetes_cluster_node_pool.pools](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/kubernetes_cluster_node_pool) | resource |
+| [azurerm_log_analytics_solution.monitoring](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/log_analytics_solution) | resource |
+| [azurerm_log_analytics_workspace.monitoring](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/log_analytics_workspace) | resource |
+| [azurerm_mysql_database.db](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/mysql_database) | resource |
+| [azurerm_mysql_firewall_rule.db](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/mysql_firewall_rule) | resource |
+| [azurerm_mysql_server.db](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/mysql_server) | resource |
+| [azurerm_network_security_rule.k8s](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/network_security_rule) | resource |
+| [azurerm_resource_group.gitpod](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/resource_group) | resource |
+| [azurerm_role_assignment.k8s](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource |
+| [azurerm_role_assignment.registry](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource |
+| [azurerm_storage_account.storage](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/storage_account) | resource |
+| [azurerm_subnet.network](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/subnet) | resource |
+| [azurerm_virtual_network.network](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/virtual_network) | resource |
+| [random_integer.db](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/integer) | resource |
+| [random_integer.registry](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/integer) | resource |
+| [random_integer.storage](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/integer) | resource |
+| [random_password.db](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/password) | resource |
+| [azurerm_client_config.current](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/client_config) | data source |
+| [azurerm_kubernetes_service_versions.k8s](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/kubernetes_service_versions) | data source |
+| [azurerm_resources.k8s](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/resources) | data source |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_dns_enabled"></a> [dns\_enabled](#input\_dns\_enabled) | Common variables | `any` | n/a | yes |
+| <a name="input_domain_name"></a> [domain\_name](#input\_domain\_name) | n/a | `any` | n/a | yes |
+| <a name="input_enable_airgapped"></a> [enable\_airgapped](#input\_enable\_airgapped) | n/a | `any` | n/a | yes |
+| <a name="input_enable_external_database"></a> [enable\_external\_database](#input\_enable\_external\_database) | n/a | `any` | n/a | yes |
+| <a name="input_enable_external_registry"></a> [enable\_external\_registry](#input\_enable\_external\_registry) | n/a | `any` | n/a | yes |
+| <a name="input_enable_external_storage"></a> [enable\_external\_storage](#input\_enable\_external\_storage) | n/a | `any` | n/a | yes |
+| <a name="input_labels"></a> [labels](#input\_labels) | n/a | `any` | n/a | yes |
+| <a name="input_location"></a> [location](#input\_location) | Azure-specific variables | `any` | n/a | yes |
+| <a name="input_name_format"></a> [name\_format](#input\_name\_format) | n/a | `any` | n/a | yes |
+| <a name="input_name_format_global"></a> [name\_format\_global](#input\_name\_format\_global) | n/a | `any` | n/a | yes |
+| <a name="input_workspace_name"></a> [workspace\_name](#input\_workspace\_name) | n/a | `any` | n/a | yes |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| <a name="output_cert_manager_issuer"></a> [cert\_manager\_issuer](#output\_cert\_manager\_issuer) | n/a |
+| <a name="output_cert_manager_secret"></a> [cert\_manager\_secret](#output\_cert\_manager\_secret) | n/a |
+| <a name="output_cluster_name"></a> [cluster\_name](#output\_cluster\_name) | n/a |
+| <a name="output_database"></a> [database](#output\_database) | n/a |
+| <a name="output_domain_nameservers"></a> [domain\_nameservers](#output\_domain\_nameservers) | n/a |
+| <a name="output_external_dns_secrets"></a> [external\_dns\_secrets](#output\_external\_dns\_secrets) | n/a |
+| <a name="output_external_dns_settings"></a> [external\_dns\_settings](#output\_external\_dns\_settings) | n/a |
+| <a name="output_k8s_connection"></a> [k8s\_connection](#output\_k8s\_connection) | n/a |
+| <a name="output_kubeconfig"></a> [kubeconfig](#output\_kubeconfig) | n/a |
+| <a name="output_region"></a> [region](#output\_region) | n/a |
+| <a name="output_registry"></a> [registry](#output\_registry) | n/a |
+| <a name="output_storage"></a> [storage](#output\_storage) | n/a |
+<!-- END_TF_DOCS -->
diff --git a/install/infra/terraform/aks/database.tf b/install/infra/terraform/aks/database.tf
new file mode 100644
index 00000000000000..50a8a21e3fa9e2
--- /dev/null
+++ b/install/infra/terraform/aks/database.tf
@@ -0,0 +1,50 @@
+resource "random_integer" "db" {
+  count = var.enable_external_database ? 1 : 0
+
+  min = 10000
+  max = 99999
+}
+
+resource "random_password" "db" {
+  count = var.enable_external_database ? 1 : 0
+
+  length = 32
+}
+
+resource "azurerm_mysql_server" "db" {
+  count = var.enable_external_database ? 1 : 0
+
+  name                = "gitpod-${random_integer.db[count.index].result}"
+  location            = azurerm_resource_group.gitpod.location
+  resource_group_name = azurerm_resource_group.gitpod.name
+
+  sku_name                         = local.db
+  storage_mb                       = 20480
+  ssl_enforcement_enabled          = false
+  ssl_minimal_tls_version_enforced = "TLSEnforcementDisabled"
+  version                          = "5.7"
+
+  auto_grow_enabled            = true
+  administrator_login          = "gitpod"
+  administrator_login_password = random_password.db[count.index].result
+}
+
+resource "azurerm_mysql_firewall_rule" "db" {
+  count = var.enable_external_database ? 1 : 0
+
+  name                = "Azure_Resource"
+  resource_group_name = azurerm_resource_group.gitpod.name
+  server_name         = azurerm_mysql_server.db[count.index].name
+  start_ip_address    = "0.0.0.0"
+  end_ip_address      = "0.0.0.0"
+}
+
+resource "azurerm_mysql_database" "db" {
+  count = var.enable_external_database ? 1 : 0
+
+  name                = "gitpod"
+  resource_group_name = azurerm_resource_group.gitpod.name
+  server_name         = azurerm_mysql_server.db[count.index].name
+  charset             = "utf8"
+  collation           = "utf8_unicode_ci"
+}
diff --git a/install/infra/terraform/aks/kubernetes.tf b/install/infra/terraform/aks/kubernetes.tf
new file mode 100644
index 00000000000000..66f6f72c8c9ad2
--- /dev/null
+++ b/install/infra/terraform/aks/kubernetes.tf
@@ -0,0 +1,95 @@
+data "azurerm_kubernetes_service_versions" "k8s" {
+  location        = azurerm_resource_group.gitpod.location
+  include_preview = false
+}
+
+resource "azurerm_role_assignment" "k8s" {
+  count = var.dns_enabled ? 1 : 0
+
+  principal_id         = azurerm_kubernetes_cluster.k8s.kubelet_identity[count.index].object_id
+  role_definition_name = "DNS Zone Contributor"
+  scope                = azurerm_dns_zone.dns[count.index].id
+}
+
+resource "azurerm_kubernetes_cluster" "k8s" {
+  name                = format(var.name_format, local.location, "primary")
+  location            = azurerm_resource_group.gitpod.location
+  resource_group_name = azurerm_resource_group.gitpod.name
+  dns_prefix          = "gitpod"
+
+  kubernetes_version               = data.azurerm_kubernetes_service_versions.k8s.latest_version
+  http_application_routing_enabled = false
+
+  default_node_pool {
+    name    = local.nodes.0.name
+    vm_size = local.machine
+
+    enable_auto_scaling  = true
+    min_count            = 2
+    max_count            = 10
+    orchestrator_version = data.azurerm_kubernetes_service_versions.k8s.latest_version
+    node_labels          = local.nodes.0.labels
+
+    type           = "VirtualMachineScaleSets"
+    vnet_subnet_id = azurerm_subnet.network.id
+  }
+
+  identity {
+    type = "SystemAssigned"
+  }
+
+  network_profile {
+    network_plugin = "kubenet"
+    network_policy = "calico"
+  }
+
+  oms_agent {
+    log_analytics_workspace_id = azurerm_log_analytics_workspace.monitoring.id
+  }
+}
+
+resource "azurerm_kubernetes_cluster_node_pool" "pools" {
+  count = length(local.nodes) - 1
+
+  kubernetes_cluster_id = azurerm_kubernetes_cluster.k8s.id
+  name                  = local.nodes[count.index + 1].name
+  vm_size               = local.machine
+
+  enable_auto_scaling  = true
+  min_count            = 2
+  max_count            = 10
+  orchestrator_version = data.azurerm_kubernetes_service_versions.k8s.latest_version
+  node_labels          = local.nodes[count.index + 1].labels
+  vnet_subnet_id       = azurerm_subnet.network.id
+}
+
+data "azurerm_resources" "k8s" {
+  count = var.enable_airgapped ? 1 : 0
+
+  resource_group_name = azurerm_kubernetes_cluster.k8s.node_resource_group
+  type                = "Microsoft.Network/networkSecurityGroups"
+
+  depends_on = [
+    azurerm_kubernetes_cluster.k8s,
+    azurerm_kubernetes_cluster_node_pool.pools
+  ]
+}
+
+resource "azurerm_network_security_rule" "k8s" {
+  count = length(local.network_security_rules)
+
+  resource_group_name         = azurerm_kubernetes_cluster.k8s.node_resource_group
+  network_security_group_name = data.azurerm_resources.k8s.0.resources.0.name
+
+  priority  = lookup(local.network_security_rules[count.index], "priority", sum([100, count.index]))
+  name      = local.network_security_rules[count.index].name
+  access    = local.network_security_rules[count.index].access
+  direction = local.network_security_rules[count.index].direction
+  protocol  = local.network_security_rules[count.index].protocol
+
+  description                = lookup(local.network_security_rules[count.index], "description", null)
+  source_port_range          = lookup(local.network_security_rules[count.index], "source_port_range", null)
+  destination_port_range     = lookup(local.network_security_rules[count.index], "destination_port_range", null)
+  source_address_prefix      = lookup(local.network_security_rules[count.index], "source_address_prefix", null)
+  destination_address_prefix = lookup(local.network_security_rules[count.index], "destination_address_prefix", null)
+}
diff --git a/install/infra/terraform/aks/local.tf b/install/infra/terraform/aks/local.tf
new file mode 100644
index 00000000000000..e11d87159206cc
--- /dev/null
+++ b/install/infra/terraform/aks/local.tf
@@ -0,0 +1,95 @@
+locals {
+  labels = tomap({
+    workload_meta : "gitpod.io/workload_meta"
+    workload_ide : "gitpod.io/workload_ide"
+    workspace_services : "gitpod.io/workload_workspace_services"
+    workspace_regular : "gitpod.io/workload_workspace_regular"
+    workspace_headless : "gitpod.io/workload_workspace_headless"
+  })
+  dns_enabled = var.domain_name != null
+  name_format = join("-", [
+    "gitpod",
+    "%s", # region
+    "%s", # name
+    local.workspace_name
+  ])
+  name_format_global = join("-", [
+    "gitpod",
+    "%s", # name
+    local.workspace_name
+  ])
+  workspace_name = replace(terraform.workspace, "/[\\W\\-]/", "") # alphanumeric workspace name
+  db       = "GP_Gen5_2"
+  location = substr(var.location, 0, 3) # Short code for location
+  machine  = "Standard_D4_v3"
+  network_security_rules = var.enable_airgapped ? [
+    {
+      name                       = "AllowContainerRegistry"
+      description                = "Allow outgoing traffic to the container registry"
+      direction                  = "Outbound"
+      access                     = "Allow"
+      protocol                   = "*"
+      source_port_range          = "*"
+      destination_port_range     = "*"
+      source_address_prefix      = "*"
+      destination_address_prefix = "AzureContainerRegistry"
+    },
+    {
+      name                       = "AllowDatabase"
+      description                = "Allow outgoing traffic to the database"
+      direction                  = "Outbound"
+      access                     = "Allow"
+      protocol                   = "*"
+      source_port_range          = "*"
+      destination_port_range     = "*"
+      source_address_prefix      = "*"
+      destination_address_prefix = "Sql"
+    },
+    {
+      name                       = "AllowStorage"
+      description                = "Allow outgoing traffic to the storage"
+      direction                  = "Outbound"
+      access                     = "Allow"
+      protocol                   = "*"
+      source_port_range          = "*"
+      destination_port_range     = "*"
+      source_address_prefix      = "*"
+      destination_address_prefix = "Storage"
+    },
+    {
+      name                       = "AllowAzureCloud"
+      description                = "Allow outgoing traffic to the Azure cloud"
+      direction                  = "Outbound"
+      access                     = "Allow"
+      protocol                   = "*"
+      source_port_range          = "*"
+      destination_port_range     = "*"
+      source_address_prefix      = "*"
+      destination_address_prefix = "AzureCloud"
+    },
+    {
+      name                       = "DenyInternetOutBound"
+      description                = "Deny outgoing traffic to the public internet"
+      direction                  = "Outbound"
+      access                     = "Deny"
+      protocol                   = "*"
+      source_port_range          = "*"
+      destination_port_range     = "*"
+      source_address_prefix      = "*"
+      destination_address_prefix = "Internet"
+      priority                   = 4096
+    }
+  ] : []
+  nodes = [
+    {
+      name = "services"
+       labels = {
+        lookup(local.labels, "workload_meta")      = true
+        lookup(local.labels, "workload_ide")       = true
+        lookup(local.labels, "workspace_services") = true
+        lookup(local.labels, "workspace_regular")  = true
+        lookup(local.labels, "workspace_headless") = true
+      }
+    }
+  ]
+}
diff --git a/install/infra/terraform/aks/main.tf b/install/infra/terraform/aks/main.tf
new file mode 100644
index 00000000000000..43dd6a6e6adf63
--- /dev/null
+++ b/install/infra/terraform/aks/main.tf
@@ -0,0 +1,24 @@
+terraform {
+  required_providers {
+    azurerm = {
+      source  = "hashicorp/azurerm"
+      version = ">= 3.0.0, < 4.0.0"
+    }
+  }
+}
+
+provider "azurerm" {
+  features {}
+}
+
+data "azurerm_client_config" "current" {}
+
+resource "azurerm_resource_group" "gitpod" {
+  name     = format(var.name_format_global, local.location)
+  location = var.location
+}
+
+resource "local_file" "kubeconfig" {
+  filename = var.kubeconfig
+  content  = azurerm_kubernetes_cluster.k8s.kube_config_raw
+}
diff --git a/install/infra/terraform/aks/monitoring.tf b/install/infra/terraform/aks/monitoring.tf
new file mode 100644
index 00000000000000..57276917673669
--- /dev/null
+++ b/install/infra/terraform/aks/monitoring.tf
@@ -0,0 +1,19 @@
+resource "azurerm_log_analytics_workspace" "monitoring" {
+  name                = format(var.name_format, var.location, "monitoring")
+  location            = azurerm_resource_group.gitpod.location
+  resource_group_name = azurerm_resource_group.gitpod.name
+  sku                 = "PerGB2018"
+}
+
+resource "azurerm_log_analytics_solution" "monitoring" {
+  solution_name         = "ContainerInsights"
+  location              = azurerm_resource_group.gitpod.location
+  resource_group_name   = azurerm_resource_group.gitpod.name
+  workspace_name        = azurerm_log_analytics_workspace.monitoring.name
+  workspace_resource_id = azurerm_log_analytics_workspace.monitoring.id
+
+  plan {
+    publisher = "Microsoft"
+    product   = "OMSGallery/ContainerInsights"
+  }
+}
diff --git a/install/infra/terraform/aks/networks.tf b/install/infra/terraform/aks/networks.tf
new file mode 100644
index 00000000000000..3e4184ee1d1ed3
--- /dev/null
+++ b/install/infra/terraform/aks/networks.tf
@@ -0,0 +1,20 @@
+resource "azurerm_virtual_network" "network" {
+  name                = format(var.name_format, local.location, "network")
+  location            = azurerm_resource_group.gitpod.location
+  resource_group_name = azurerm_resource_group.gitpod.name
+  address_space       = ["10.2.0.0/16"]
+}
+
+resource "azurerm_subnet" "network" {
+  name                 = format(var.name_format, local.location, "network")
+  resource_group_name  = azurerm_resource_group.gitpod.name
+  virtual_network_name = azurerm_virtual_network.network.name
+  address_prefixes     = ["10.2.1.0/24"]
+}
+
+resource "azurerm_dns_zone" "dns" {
+  count = var.dns_enabled ? 1 : 0
+
+  name                = var.domain_name
+  resource_group_name = azurerm_resource_group.gitpod.name
+}
diff --git a/install/infra/terraform/aks/output.tf b/install/infra/terraform/aks/output.tf
new file mode 100644
index 00000000000000..a22bd5c3efb7d2
--- /dev/null
+++ b/install/infra/terraform/aks/output.tf
@@ -0,0 +1,89 @@
+output "cert_manager_issuer" {
+  value = try([{
+    dns01 = {
+      azureDNS = {
+        subscriptionID    = data.azurerm_client_config.current.subscription_id
+        resourceGroupName = azurerm_resource_group.gitpod.name
+        hostedZoneName    = azurerm_dns_zone.dns.0.name
+        managedIdentity = {
+          clientID = azurerm_kubernetes_cluster.k8s.kubelet_identity.0.client_id
+        }
+      }
+    }
+  }], [])
+}
+
+output "cert_manager_secret" {
+  value = {}
+}
+
+output "cluster_name" {
+  value = azurerm_kubernetes_cluster.k8s.name
+}
+
+output "database" {
+  sensitive = true
+  value = try({
+    host     = "${azurerm_mysql_server.db.0.name}.mysql.database.azure.com"
+    password = azurerm_mysql_server.db.0.administrator_login_password
+    port     = 3306
+    username = "${azurerm_mysql_server.db.0.administrator_login}@${azurerm_mysql_server.db.0.name}"
+  }, {})
+}
+
+output "domain_nameservers" {
+  value = try(azurerm_dns_zone.dns.0.name_servers, null)
+}
+
+output "external_dns_secrets" {
+  value = {}
+}
+
+output "external_dns_settings" {
+  value = {
+    provider                            = "azure"
+    "azure.resourceGroup"               = azurerm_resource_group.gitpod.name
+    "azure.subscriptionId"              = data.azurerm_client_config.current.subscription_id
+    "azure.tenantId"                    = data.azurerm_client_config.current.tenant_id
+    "azure.useManagedIdentityExtension" = true
+    "azure.userAssignedIdentityID"      = azurerm_kubernetes_cluster.k8s.kubelet_identity.0.client_id
+  }
+}
+
+output "k8s_connection" {
+  sensitive = true
+  value = {
+    host                   = azurerm_kubernetes_cluster.k8s.kube_config.0.host
+    username               = azurerm_kubernetes_cluster.k8s.kube_config.0.username
+    password               = azurerm_kubernetes_cluster.k8s.kube_config.0.password
+    client_certificate     = base64decode(azurerm_kubernetes_cluster.k8s.kube_config.0.client_certificate)
+    client_key             = base64decode(azurerm_kubernetes_cluster.k8s.kube_config.0.client_key)
+    cluster_ca_certificate = base64decode(azurerm_kubernetes_cluster.k8s.kube_config.0.cluster_ca_certificate)
+  }
+}
+
+output "kubeconfig" {
+  sensitive = true
+  value     = azurerm_kubernetes_cluster.k8s.kube_config_raw
+}
+
+output "region" {
+  value = var.location
+}
+
+output "registry" {
+  sensitive = true
+  value = try({
+    server   = azurerm_container_registry.registry.0.login_server
+    password = azurerm_container_registry.registry.0.admin_password
+    username = azurerm_container_registry.registry.0.admin_username
+  }, {})
+}
+
+output "storage" {
+  sensitive = true
+  value = try({
+    username = azurerm_storage_account.storage.0.name
+    password = azurerm_storage_account.storage.0.primary_access_key
+  }, {})
+}
diff --git a/install/infra/terraform/aks/registry.tf b/install/infra/terraform/aks/registry.tf
new file mode 100644
index 00000000000000..87ec560c6b5441
--- /dev/null
+++ b/install/infra/terraform/aks/registry.tf
@@ -0,0 +1,25 @@
+resource "random_integer" "registry" {
+  count = var.enable_external_registry ? 1 : 0
+
+  min = 10000
+  max = 99999
+}
+
+resource "azurerm_container_registry" "registry" {
+  count = var.enable_external_registry ? 1 : 0
+
+  name                = "gitpod${random_integer.registry[count.index].result}"
+  resource_group_name = azurerm_resource_group.gitpod.name
+  location            = azurerm_resource_group.gitpod.location
+  admin_enabled       = true
+  sku                 = "Premium"
+}
+
+resource "azurerm_role_assignment" "registry" {
+  count = var.enable_external_registry ? 1 : 0
+
+  principal_id                     = azurerm_kubernetes_cluster.k8s.kubelet_identity[0].object_id
+  role_definition_name             = "AcrPush"
+  scope                            = azurerm_container_registry.registry[count.index].id
+  skip_service_principal_aad_check = true
+}
diff --git a/install/infra/terraform/aks/storage.tf b/install/infra/terraform/aks/storage.tf
new file mode 100644
index 00000000000000..ca4ef115dcf924
--- /dev/null
+++ b/install/infra/terraform/aks/storage.tf
@@ -0,0 +1,16 @@
+resource "random_integer" "storage" {
+  count = var.enable_external_storage ? 1 : 0
+
+  min = 10000
+  max = 99999
+}
+
+resource "azurerm_storage_account" "storage" {
+  count = var.enable_external_storage ? 1 : 0
+
+  name                     = "gitpod${random_integer.storage[count.index].result}"
+  resource_group_name      = azurerm_resource_group.gitpod.name
+  location                 = azurerm_resource_group.gitpod.location
+  account_tier             = "Standard"
+  account_replication_type = "LRS"
+}
diff --git a/install/infra/terraform/aks/variables.tf b/install/infra/terraform/aks/variables.tf
new file mode 100644
index 00000000000000..796f7489896181
--- /dev/null
+++ b/install/infra/terraform/aks/variables.tf
@@ -0,0 +1,24 @@
+// Common variables
+variable "kubeconfig" {
+    default = "./kubeconfig"
+
+}
+variable "dns_enabled" {}
+variable "domain_name" {}
+variable "enable_airgapped" {}
+variable "enable_external_database" {}
+variable "enable_external_registry" {}
+variable "enable_external_storage" {}
+variable "labels" {}
+variable "name_format" {
+}
+variable "name_format_global" {
+}
+variable "workspace_name" {
+}
+
+// Azure-specific variables
+variable "location" {
+    default = "northeurope"
+
+}
diff --git a/install/tests/main.tf b/install/tests/main.tf
index 16408fd879440f..58f9e6407730cd 100644
--- a/install/tests/main.tf
+++ b/install/tests/main.tf
@@ -1,13 +1,7 @@
 variable "kubeconfig" {}
-variable "TEST_ID" {
-  default = "nightly"
-}
-variable "project" {
-  default = "sh-automated-tests"
-}
-variable "sa_creds" {}
-variable "dns_sa_creds" {}
+variable "TEST_ID" { default = "nightly" }
 
+# We store the state always in a GCS bucket
 terraform {
   backend "gcs" {
     bucket = "nightly-tests"
@@ -15,8 +9,12 @@ terraform {
   }
 }
 
+variable "project" { default = "sh-automated-tests" }
+variable "sa_creds" {}
+variable "dns_sa_creds" {}
+
 module "gke" {
-  source = "github.com/gitpod-io/gitpod//install/infra/terraform/gke?ref=nvn-infra-tf" # we can later use tags here
+  source = "github.com/gitpod-io/gitpod//install/infra/terraform/gke?ref=nvn/infra-tf" # we can later use tags here
 
   name        = var.TEST_ID
   project     = var.project
@@ -25,7 +23,7 @@ module "gke" {
 }
 
 module "k3s" {
-  source = "github.com/gitpod-io/gitpod//install/infra/terraform/k3s?ref=nvn-infra-tf" # we can later use tags here
+  source = "github.com/gitpod-io/gitpod//install/infra/terraform/k3s?ref=nvn/infra-tf" # we can later use tags here
 
   name        = var.TEST_ID
   gcp_project = var.project
@@ -33,6 +31,18 @@ module "k3s" {
   kubeconfig  = var.kubeconfig
 }
 
+module "azure" {
+  source = "./azure"
+  source = "github.com/gitpod-io/gitpod//install/infra/terraform/k3s?ref=nvn/infra-aks" # we can later use tags here
+
+  domain_name              = false
+  enable_airgapped         = false
+  enable_external_database = false
+  enable_external_registry = false
+  enable_external_storage  = false
+}
+
+
 // this module is intended to be run separately from the above two. so a separate target for apply is necessary
 module "tools" {
   source = "github.com/gitpod-io/gitpod//install/infra/terraform/tools/cm-cloud-dns?ref=nvn-infra-tf" # we can later use tags here

From 61d22365a0a1bd308209aef24dbdf0e8162bc277 Mon Sep 17 00:00:00 2001
From: Nandaja Varma <nandaja.varma@gmail.com>
Date: Wed, 1 Jun 2022 17:13:38 +0000
Subject: [PATCH 3/4] adding the ts scripts

---
 .werft/aks-installer-tests.yaml | 89 +++++++++++++++++++++++++++++++++
 .werft/installer-tests.ts       | 17 +++++++
 install/tests/Makefile          | 11 +++-
 install/tests/main.tf           | 36 +++++++++----
 4 files changed, 143 insertions(+), 10 deletions(-)
 create mode 100644 .werft/aks-installer-tests.yaml

diff --git a/.werft/aks-installer-tests.yaml b/.werft/aks-installer-tests.yaml
new file mode 100644
index 00000000000000..873ff6f4285017
--- /dev/null
+++ b/.werft/aks-installer-tests.yaml
@@ -0,0 +1,89 @@
+# debug using `werft run github -f -s .werft/installer-tests.ts -j .werft/aks-installer-tests.yaml -a debug=true`
+pod:
+  serviceAccount: werft
+  affinity:
+  nodeAffinity:
+    requiredDuringSchedulingIgnoredDuringExecution:
+      nodeSelectorTerms:
+      - matchExpressions:
+        - key: dev/workload
+          operator: In
+          values:
+          - "builds"
+  securityContext:
+    runAsUser: 0
+  volumes:
+  - name: sh-playground-sa-perm
+    secret:
+      secretName: sh-playground-sa-perm
+  - name: sh-playground-dns-perm
+    secret:
+      secretName: sh-playground-dns-perm
+  containers:
+  - name: nightly-test
+    image: eu.gcr.io/gitpod-core-dev/dev/dev-environment:cw-werft-cred.0
+    workingDir: /workspace
+    imagePullPolicy: Always
+    volumeMounts:
+    - name: sh-playground-sa-perm
+      mountPath: /mnt/secrets/sh-playground-sa-perm
+    - name: sh-playground-dns-perm # this sa is used for the DNS management
+      mountPath: /mnt/secrets/sh-playground-dns-perm
+    env:
+    - name: ARM_SUBSCRIPTION_ID
+      valueFrom:
+        secretKeyRef:
+          name: aks-credentials
+          key: subscriptionid
+    - name: ARM_TENANT_ID
+      valueFrom:
+        secretKeyRef:
+          name: aks-credentials
+          key: tenantid
+    - name: ARM_CLIENT_ID
+      valueFrom:
+        secretKeyRef:
+          name: aks-credentials
+          key: clientid
+    - name: ARM_CLIENT_SECRET
+      valueFrom:
+        secretKeyRef:
+          name: aks-credentials
+          key: clientsecret
+    - name: WERFT_HOST
+      value: "werft.werft.svc.cluster.local:7777"
+    - name: GOOGLE_APPLICATION_CREDENTIALS
+      value: "/mnt/secrets/sh-playground-sa-perm/sh-sa.json"
+    - name: TF_VAR_sa_creds
+      value: "/mnt/secrets/sh-playground-sa-perm/sh-sa.json"
+    - name: TF_VAR_dns_sa_creds
+      value: "/mnt/secrets/sh-playground-dns-perm/sh-dns-sa.json"
+    - name: WERFT_K8S_NAMESPACE
+      value: "werft"
+    - name: WERFT_K8S_LABEL
+      value: "component=werft"
+    - name: NODENAME
+      valueFrom:
+        fieldRef:
+          fieldPath: spec.nodeName
+    command:
+      - bash
+      - -c
+      - |
+        sleep 1
+        set -Eeuo pipefail
+
+        sudo chown -R gitpod:gitpod /workspace
+        sudo apt update && apt install gettext-base
+
+        curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
+
+        export TF_VAR_TEST_ID=$(echo $RANDOM | md5sum | head -c 5; echo)
+
+        (cd .werft && yarn install && mv node_modules ..) | werft log slice prep
+        printf '{{ toJson . }}' > context.json
+
+        npx ts-node .werft/installer-tests.ts "STANDARD_AKS_TEST"
+# The bit below makes this a cron job
+# plugins:
+#   cron: "15 3 * * *"
diff --git a/.werft/installer-tests.ts b/.werft/installer-tests.ts
index a67cfde849963e..5ca6bd7f7ad704 100644
--- a/.werft/installer-tests.ts
+++ b/.werft/installer-tests.ts
@@ -31,6 +31,11 @@ const INFRA_PHASES: { [name: string]: InfraConfig } = {
         makeTarget: "k3s-standard-cluster",
         description: "Creating a k3s cluster on GCP with 1 node",
     },
+    STANDARD_AKS_CLUSTER: {
+        phase: "create-std-aks-cluster",
+        makeTarget: "aks-standard-cluster",
+        description: "Creating an aks cluster(azure)",
+    },
     GCP_MANAGED_DNS: {
         phase: "setup-cert-manager-with-cloud-dns",
         makeTarget: "managed-dns-with-cert-manager",
@@ -89,6 +94,18 @@ const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = {
             "DESTROY",
         ],
     },
+    STANDARD_AKS_TEST: {
+        DESCRIPTION: "Deploy Gitpod on AKS, with managed DNS, and run integration tests",
+        PHASES: [
+            "STANDARD_AKS_CLUSTER",
+            "GCP_MANAGED_DNS",
+            "INSTALL_GITPOD",
+            "CHECK_INSTALLATION",
+            "RUN_INTEGRATION_TESTS",
+            "RESULTS",
+            "DESTROY",
+        ],
+    },
     STANDARD_K3S_TEST: {
         DESCRIPTION:
             "Deploy Gitpod on a K3s cluster, created on a GCP instance," +
diff --git a/install/tests/Makefile b/install/tests/Makefile
index d8b3b4a73169e4..55592f0d33f388 100644
--- a/install/tests/Makefile
+++ b/install/tests/Makefile
@@ -5,7 +5,7 @@ PROJECTNAME := "installer-nightly-tests"
 
 TOPDIR=$(shell pwd)
 
-KUBECONFIG := "/workspace/conf"
+KUBECONFIG := "$(TOPDIR)/kubeconfig"
 
 check-env-sub-domain:
 ifndef TF_VAR_TEST_ID
@@ -29,6 +29,14 @@ gke-standard-cluster:
 	terraform apply -target=module.gke \
 	-var kubeconfig=${KUBECONFIG} --auto-approve
 
+.PHONY:
+## aks-standard-cluster: Creates an AKS cluster
+aks-standard-cluster:
+	terraform init --upgrade && \
+	terraform workspace new $(TF_VAR_TEST_ID) || terraform workspace select $(TF_VAR_TEST_ID) && \
+	terraform apply -target=module.aks \
+	-var kubeconfig=${KUBECONFIG} --auto-approve
+
 .PHONY:
 ## k3s-standard-cluster: Creates a K3S cluster on GCP with one master and 1 worker node
 k3s-standard-cluster:
@@ -93,6 +101,7 @@ cleanup:
 	terraform workspace select $(TF_VAR_TEST_ID)
 	which ${KUBECONFIG} && terraform destroy -target=module.tools -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file"
 	terraform destroy -target=module.gke -var kubeconfig=${KUBECONFIG} --auto-approve
+	terraform destroy -target=module.aks -var kubeconfig=${KUBECONFIG} --auto-approve
 	terraform destroy -target=module.k3s -var kubeconfig=${KUBECONFIG} --auto-approve
 
 get-results:
diff --git a/install/tests/main.tf b/install/tests/main.tf
index 58f9e6407730cd..d6d1025de01f49 100644
--- a/install/tests/main.tf
+++ b/install/tests/main.tf
@@ -1,4 +1,4 @@
-variable "kubeconfig" {}
+variable "kubeconfig" { }
 variable "TEST_ID" { default = "nightly" }
 
 # We store the state always in a GCS bucket
@@ -10,11 +10,11 @@ terraform {
 }
 
 variable "project" { default = "sh-automated-tests" }
-variable "sa_creds" {}
-variable "dns_sa_creds" {}
+variable "sa_creds" { default = "" }
+variable "dns_sa_creds" {default = "" }
 
 module "gke" {
-  source = "github.com/gitpod-io/gitpod//install/infra/terraform/gke?ref=nvn/infra-tf" # we can later use tags here
+  source = "github.com/gitpod-io/gitpod//install/infra/terraform/gke?ref=nvn-infra-tf" # we can later use tags here
 
   name        = var.TEST_ID
   project     = var.project
@@ -23,7 +23,7 @@ module "gke" {
 }
 
 module "k3s" {
-  source = "github.com/gitpod-io/gitpod//install/infra/terraform/k3s?ref=nvn/infra-tf" # we can later use tags here
+  source = "github.com/gitpod-io/gitpod//install/infra/terraform/k3s?ref=nvn-infra-tf" # we can later use tags here
 
   name        = var.TEST_ID
   gcp_project = var.project
@@ -31,18 +31,36 @@ module "k3s" {
   kubeconfig  = var.kubeconfig
 }
 
-module "azure" {
-  source = "./azure"
-  source = "github.com/gitpod-io/gitpod//install/infra/terraform/k3s?ref=nvn/infra-aks" # we can later use tags here
+module "aks" {
+  source = "github.com/gitpod-io/gitpod//install/infra/terraform/aks?ref=nvn-infra-aks" # we can later use tags here
 
   domain_name              = false
   enable_airgapped         = false
   enable_external_database = false
   enable_external_registry = false
   enable_external_storage  = false
+  dns_enabled =  false
+  name_format = join("-", [
+    "gitpod",
+    "%s", # region
+    "%s", # name
+    var.TEST_ID
+  ])
+  name_format_global = join("-", [
+    "gitpod",
+    "%s", # name
+    var.TEST_ID
+  ])
+  workspace_name = var.TEST_ID
+  labels = {
+    "gitpod.io/workload_meta"               = true
+    "gitpod.io/workload_ide"                = true
+    "gitpod.io/workload_workspace_services" = true
+    "gitpod.io/workload_workspace_regular"  = true
+    "gitpod.io/workload_workspace_headless" = true
+  }
 }
 
-
 // this module is intended to be run separately from the above two. so a separate target for apply is necessary
 module "tools" {
   source = "github.com/gitpod-io/gitpod//install/infra/terraform/tools/cm-cloud-dns?ref=nvn-infra-tf" # we can later use tags here

From c6190915f65fe1668852a6d4dc0bc6284a97c6a3 Mon Sep 17 00:00:00 2001
From: Nandaja Varma <nandaja.varma@gmail.com>
Date: Thu, 2 Jun 2022 13:22:35 +0000
Subject: [PATCH 4/4] add dns mapping in k3s

---
 install/infra/terraform/k3s/main.tf      | 17 +++++++++++++++++
 install/infra/terraform/k3s/variables.tf | 20 ++++++++++++++++++++
 install/tests/main.tf                    | 18 +++++++++++-------
 3 files changed, 48 insertions(+), 7 deletions(-)

diff --git a/install/infra/terraform/k3s/main.tf b/install/infra/terraform/k3s/main.tf
index 645a0386477d96..5a4ca2ab515795 100644
--- a/install/infra/terraform/k3s/main.tf
+++ b/install/infra/terraform/k3s/main.tf
@@ -21,6 +21,11 @@ provider "google" {
   zone        = var.gcp_zone
 }
 
+provider "google" {
+  alias       = "dns"
+  credentials = var.dns_sa_creds
+}
+
 resource "google_service_account" "gcp_instance" {
   account_id   = "gcp-k3s-compute"
   display_name = "Service Account"
@@ -94,6 +99,18 @@ resource "null_resource" "k3sup_install" {
   }
 }
 
+resource "google_dns_record_set" "a" {
+  provider     = google.dns
+  count        = (var.domain_name == null) || (var.managed_dns_zone == null ) ? 0 : 1
+  name         = "${var.domain_name}."
+  managed_zone = var.managed_dns_zone
+  project      = var.dns_project == null ? var.gcp_project : var.dns_project
+  type         = "A"
+  ttl          = 5
+
+  rrdatas = [google_compute_instance.k3s_master_instance.network_interface[0].access_config[0].nat_ip]
+}
+
 data "local_file" "kubeconfig" {
   depends_on = [null_resource.k3sup_install]
   filename   = var.kubeconfig
diff --git a/install/infra/terraform/k3s/variables.tf b/install/infra/terraform/k3s/variables.tf
index d2ed0c12f6c1a9..26096d197bf097 100644
--- a/install/infra/terraform/k3s/variables.tf
+++ b/install/infra/terraform/k3s/variables.tf
@@ -26,3 +26,23 @@ variable "name" {
   description = "Prefix name for the nodes and firewall"
   default     = "k3s"
 }
+
+variable "dns_sa_creds" {
+  description = "Credentials with DNS admin rights to the project with managed DNS record"
+  default     = ""
+}
+
+variable "dns_project" {
+  description = "Project associated with the dns maanged zone"
+  default     = null
+}
+
+variable "domain_name" {
+  description = "Domain name to add to add DNS map to"
+  default     = null
+}
+
+variable "managed_dns_zone" {
+  description = "Name of the managed DNS record"
+  default     = null
+}
diff --git a/install/tests/main.tf b/install/tests/main.tf
index d6d1025de01f49..b09d10aaba5df7 100644
--- a/install/tests/main.tf
+++ b/install/tests/main.tf
@@ -10,8 +10,8 @@ terraform {
 }
 
 variable "project" { default = "sh-automated-tests" }
-variable "sa_creds" { default = "" }
-variable "dns_sa_creds" {default = "" }
+variable "sa_creds" { default = "/workspace/gcp.json" }
+variable "dns_sa_creds" {default = "/workspace/gitpod/gcp.json" }
 
 module "gke" {
   source = "github.com/gitpod-io/gitpod//install/infra/terraform/gke?ref=nvn-infra-tf" # we can later use tags here
@@ -23,12 +23,16 @@ module "gke" {
 }
 
 module "k3s" {
-  source = "github.com/gitpod-io/gitpod//install/infra/terraform/k3s?ref=nvn-infra-tf" # we can later use tags here
+  source = "github.com/gitpod-io/gitpod//install/infra/terraform/k3s?ref=nvn-infra-aks" # we can later use tags here
 
-  name        = var.TEST_ID
-  gcp_project = var.project
-  credentials = var.sa_creds
-  kubeconfig  = var.kubeconfig
+  name             = var.TEST_ID
+  gcp_project      = var.project
+  credentials      = var.sa_creds
+  kubeconfig       = var.kubeconfig
+  dns_sa_creds     = var.dns_sa_creds
+  dns_project      = "dns-for-playgrounds"
+  managed_dns_zone = "gitpod-self-hosted-com"
+  domain_name      = "${var.TEST_ID}.gitpod-self-hosted.com"
 }
 
 module "aks" {