Add aks automated testing setup

.werft/aks-installer-tests.yaml

@@ -0,0 +1,89 @@
+# debug using `werft run github -f -s .werft/installer-tests.ts -j .werft/aks-installer-tests.yaml -a debug=true`
+pod:
+  serviceAccount: werft
+  affinity:
+  nodeAffinity:
+    requiredDuringSchedulingIgnoredDuringExecution:
+      nodeSelectorTerms:
+      - matchExpressions:
+        - key: dev/workload
+          operator: In
+          values:
+          - "builds"
+  securityContext:
+    runAsUser: 0
+  volumes:
+  - name: sh-playground-sa-perm
+    secret:
+      secretName: sh-playground-sa-perm
+  - name: sh-playground-dns-perm
+    secret:
+      secretName: sh-playground-dns-perm
+  containers:
+  - name: nightly-test
+    image: eu.gcr.io/gitpod-core-dev/dev/dev-environment:cw-werft-cred.0
+    workingDir: /workspace
+    imagePullPolicy: Always
+    volumeMounts:
+    - name: sh-playground-sa-perm
+      mountPath: /mnt/secrets/sh-playground-sa-perm
+    - name: sh-playground-dns-perm # this sa is used for the DNS management
+      mountPath: /mnt/secrets/sh-playground-dns-perm
+    env:
+    - name: ARM_SUBSCRIPTION_ID
+      valueFrom:
+        secretKeyRef:
+          name: aks-credentials
+          key: subscriptionid
+    - name: ARM_TENANT_ID
+      valueFrom:
+        secretKeyRef:
+          name: aks-credentials
+          key: tenantid
+    - name: ARM_CLIENT_ID
+      valueFrom:
+        secretKeyRef:
+          name: aks-credentials
+          key: clientid
+    - name: ARM_CLIENT_SECRET
+      valueFrom:
+        secretKeyRef:
+          name: aks-credentials
+          key: clientsecret
+    - name: WERFT_HOST
+      value: "werft.werft.svc.cluster.local:7777"
+    - name: GOOGLE_APPLICATION_CREDENTIALS
+      value: "/mnt/secrets/sh-playground-sa-perm/sh-sa.json"
+    - name: TF_VAR_sa_creds
+      value: "/mnt/secrets/sh-playground-sa-perm/sh-sa.json"
+    - name: TF_VAR_dns_sa_creds
+      value: "/mnt/secrets/sh-playground-dns-perm/sh-dns-sa.json"
+    - name: WERFT_K8S_NAMESPACE
+      value: "werft"
+    - name: WERFT_K8S_LABEL
+      value: "component=werft"
+    - name: NODENAME
+      valueFrom:
+        fieldRef:
+          fieldPath: spec.nodeName
+    command:
+      - bash
+      - -c
+      - |
+        sleep 1
+        set -Eeuo pipefail
+
+        sudo chown -R gitpod:gitpod /workspace
+        sudo apt update && apt install gettext-base
+
+        curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
+
+        export TF_VAR_TEST_ID=$(echo $RANDOM | md5sum | head -c 5; echo)
+
+        (cd .werft && yarn install && mv node_modules ..) | werft log slice prep
+        printf '{{ toJson . }}' > context.json
+
+        npx ts-node .werft/installer-tests.ts "STANDARD_AKS_TEST"
+# The bit below makes this a cron job
+# plugins:
+#   cron: "15 3 * * *"

.werft/installer-tests.ts

@@ -2,7 +2,9 @@ import { join } from "path";
 import { exec } from "./util/shell";
 import { Werft } from "./util/werft";
 
-const testConfig: string = process.argv.length > 2 ? process.argv[2] : "gke";
+const testConfig: string = process.argv.length > 2 ? process.argv[2] : "STANDARD_K3S_TEST";
+// we can provide the version of the gitpod to install (eg: 2022.4.2)
+const version: string = process.argv.length > 3 ? process.argv[3] : "";
 
 const makefilePath: string = join("install/tests");
 
@@ -29,22 +31,28 @@ const INFRA_PHASES: { [name: string]: InfraConfig } = {
         makeTarget: "k3s-standard-cluster",
         description: "Creating a k3s cluster on GCP with 1 node",
     },
+    STANDARD_AKS_CLUSTER: {
+        phase: "create-std-aks-cluster",
+        makeTarget: "aks-standard-cluster",
+        description: "Creating an aks cluster(azure)",
+    },
     GCP_MANAGED_DNS: {
         phase: "setup-cert-manager-with-cloud-dns",
         makeTarget: "managed-dns-with-cert-manager",
         description: "Sets up cert-manager, external-dns & cloudDNS config",
     },
     INSTALL_GITPOD_IGNORE_PREFLIGHTS: {
         phase: "install-gitpod-without-preflights",
-        makeTarget: "kots-install-without-preflight-with-community-license",
+        makeTarget: `kots-install channel=unstable version=${version} preflights=false`, // this is a bit of a hack, for now we pass params like this
         description: "Install gitpod using kots community edition without preflights",
     },
     INSTALL_GITPOD: {
         phase: "install-gitpod",
-        makeTarget: "kots-install-with-community-license",
+        makeTarget: `kots-install channel=unstable version=${version} preflights=true`,
         description: "Install gitpod using kots community edition",
     },
     CHECK_INSTALLATION: {
+        // this is a basic test for the Gitpod setup
         phase: "check-gitpod-installation",
         makeTarget: "check-gitpod-installation",
         description: "Check gitpod installation",
@@ -86,6 +94,18 @@ const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = {
             "DESTROY",
         ],
     },
+    STANDARD_AKS_TEST: {
+        DESCRIPTION: "Deploy Gitpod on AKS, with managed DNS, and run integration tests",
+        PHASES: [
+            "STANDARD_AKS_CLUSTER",
+            "GCP_MANAGED_DNS",
+            "INSTALL_GITPOD",
+            "CHECK_INSTALLATION",
+            "RUN_INTEGRATION_TESTS",
+            "RESULTS",
+            "DESTROY",
+        ],
+    },
     STANDARD_K3S_TEST: {
         DESCRIPTION:
             "Deploy Gitpod on a K3s cluster, created on a GCP instance," +
@@ -155,6 +175,7 @@ function callMakeTargets(phase: string, description: string, makeTarget: string)
         werft.fail(phase, "Operation failed");
     } else {
         werft.log(phase, response.stdout.toString());
+        werft.done(phase);
     }
 
     return response.code;

install/infra/terraform/aks/README.md

@@ -0,0 +1,97 @@
+# Azure
+
+Azure provider for Gitpod testing
+
+<!-- toc -->
+
+- [Terraform Documentation](#terraform-documentation)
+  * [Requirements](#requirements)
+  * [Providers](#providers)
+  * [Modules](#modules)
+  * [Resources](#resources)
+  * [Inputs](#inputs)
+  * [Outputs](#outputs)
+
+<!-- tocstop -->
+
+# Terraform Documentation
+
+<!-- BEGIN_TF_DOCS -->
+## Requirements
+
+| Name | Version |
+|------|---------|
+| <a name="requirement_azurerm"></a> [azurerm](#requirement\_azurerm) | >= 3.0.0, < 4.0.0 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_azurerm"></a> [azurerm](#provider\_azurerm) | >= 3.0.0, < 4.0.0 |
+| <a name="provider_random"></a> [random](#provider\_random) | n/a |
+
+## Modules
+
+No modules.
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [azurerm_container_registry.registry](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/container_registry) | resource |
+| [azurerm_dns_zone.dns](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/dns_zone) | resource |
+| [azurerm_kubernetes_cluster.k8s](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/kubernetes_cluster) | resource |
+| [azurerm_kubernetes_cluster_node_pool.pools](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/kubernetes_cluster_node_pool) | resource |
+| [azurerm_log_analytics_solution.monitoring](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/log_analytics_solution) | resource |
+| [azurerm_log_analytics_workspace.monitoring](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/log_analytics_workspace) | resource |
+| [azurerm_mysql_database.db](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/mysql_database) | resource |
+| [azurerm_mysql_firewall_rule.db](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/mysql_firewall_rule) | resource |
+| [azurerm_mysql_server.db](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/mysql_server) | resource |
+| [azurerm_network_security_rule.k8s](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/network_security_rule) | resource |
+| [azurerm_resource_group.gitpod](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/resource_group) | resource |
+| [azurerm_role_assignment.k8s](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource |
+| [azurerm_role_assignment.registry](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource |
+| [azurerm_storage_account.storage](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/storage_account) | resource |
+| [azurerm_subnet.network](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/subnet) | resource |
+| [azurerm_virtual_network.network](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/virtual_network) | resource |
+| [random_integer.db](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/integer) | resource |
+| [random_integer.registry](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/integer) | resource |
+| [random_integer.storage](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/integer) | resource |
+| [random_password.db](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/password) | resource |
+| [azurerm_client_config.current](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/client_config) | data source |
+| [azurerm_kubernetes_service_versions.k8s](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/kubernetes_service_versions) | data source |
+| [azurerm_resources.k8s](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/resources) | data source |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_dns_enabled"></a> [dns\_enabled](#input\_dns\_enabled) | Common variables | `any` | n/a | yes |
+| <a name="input_domain_name"></a> [domain\_name](#input\_domain\_name) | n/a | `any` | n/a | yes |
+| <a name="input_enable_airgapped"></a> [enable\_airgapped](#input\_enable\_airgapped) | n/a | `any` | n/a | yes |
+| <a name="input_enable_external_database"></a> [enable\_external\_database](#input\_enable\_external\_database) | n/a | `any` | n/a | yes |
+| <a name="input_enable_external_registry"></a> [enable\_external\_registry](#input\_enable\_external\_registry) | n/a | `any` | n/a | yes |
+| <a name="input_enable_external_storage"></a> [enable\_external\_storage](#input\_enable\_external\_storage) | n/a | `any` | n/a | yes |
+| <a name="input_labels"></a> [labels](#input\_labels) | n/a | `any` | n/a | yes |
+| <a name="input_location"></a> [location](#input\_location) | Azure-specific variables | `any` | n/a | yes |
+| <a name="input_name_format"></a> [name\_format](#input\_name\_format) | n/a | `any` | n/a | yes |
+| <a name="input_name_format_global"></a> [name\_format\_global](#input\_name\_format\_global) | n/a | `any` | n/a | yes |
+| <a name="input_workspace_name"></a> [workspace\_name](#input\_workspace\_name) | n/a | `any` | n/a | yes |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| <a name="output_cert_manager_issuer"></a> [cert\_manager\_issuer](#output\_cert\_manager\_issuer) | n/a |
+| <a name="output_cert_manager_secret"></a> [cert\_manager\_secret](#output\_cert\_manager\_secret) | n/a |
+| <a name="output_cluster_name"></a> [cluster\_name](#output\_cluster\_name) | n/a |
+| <a name="output_database"></a> [database](#output\_database) | n/a |
+| <a name="output_domain_nameservers"></a> [domain\_nameservers](#output\_domain\_nameservers) | n/a |
+| <a name="output_external_dns_secrets"></a> [external\_dns\_secrets](#output\_external\_dns\_secrets) | n/a |
+| <a name="output_external_dns_settings"></a> [external\_dns\_settings](#output\_external\_dns\_settings) | n/a |
+| <a name="output_k8s_connection"></a> [k8s\_connection](#output\_k8s\_connection) | n/a |
+| <a name="output_kubeconfig"></a> [kubeconfig](#output\_kubeconfig) | n/a |
+| <a name="output_region"></a> [region](#output\_region) | n/a |
+| <a name="output_registry"></a> [registry](#output\_registry) | n/a |
+| <a name="output_storage"></a> [storage](#output\_storage) | n/a |
+<!-- END_TF_DOCS -->

install/infra/terraform/aks/database.tf

@@ -0,0 +1,50 @@
+resource "random_integer" "db" {
+  count = var.enable_external_database ? 1 : 0
+
+  min = 10000
+  max = 99999
+}
+
+resource "random_password" "db" {
+  count = var.enable_external_database ? 1 : 0
+
+  length = 32
+}
+
+resource "azurerm_mysql_server" "db" {
+  count = var.enable_external_database ? 1 : 0
+
+  name                = "gitpod-${random_integer.db[count.index].result}"
+  location            = azurerm_resource_group.gitpod.location
+  resource_group_name = azurerm_resource_group.gitpod.name
+
+  sku_name                         = local.db
+  storage_mb                       = 20480
+  ssl_enforcement_enabled          = false
+  ssl_minimal_tls_version_enforced = "TLSEnforcementDisabled"
+  version                          = "5.7"
+
+  auto_grow_enabled            = true
+  administrator_login          = "gitpod"
+  administrator_login_password = random_password.db[count.index].result
+}
+
+resource "azurerm_mysql_firewall_rule" "db" {
+  count = var.enable_external_database ? 1 : 0
+
+  name                = "Azure_Resource"
+  resource_group_name = azurerm_resource_group.gitpod.name
+  server_name         = azurerm_mysql_server.db[count.index].name
+  start_ip_address    = "0.0.0.0"
+  end_ip_address      = "0.0.0.0"
+}
+
+resource "azurerm_mysql_database" "db" {
+  count = var.enable_external_database ? 1 : 0
+
+  name                = "gitpod"
+  resource_group_name = azurerm_resource_group.gitpod.name
+  server_name         = azurerm_mysql_server.db[count.index].name
+  charset             = "utf8"
+  collation           = "utf8_unicode_ci"
+}