diff --git a/.werft/aks-installer-tests.yaml b/.werft/aks-installer-tests.yaml
new file mode 100644
index 00000000000000..873ff6f4285017
--- /dev/null
+++ b/.werft/aks-installer-tests.yaml
@@ -0,0 +1,89 @@
+# debug using `werft run github -f -s .werft/installer-tests.ts -j .werft/aks-installer-tests.yaml -a debug=true`
+pod:
+ serviceAccount: werft
+ affinity:
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: dev/workload
+ operator: In
+ values:
+ - "builds"
+ securityContext:
+ runAsUser: 0
+ volumes:
+ - name: sh-playground-sa-perm
+ secret:
+ secretName: sh-playground-sa-perm
+ - name: sh-playground-dns-perm
+ secret:
+ secretName: sh-playground-dns-perm
+ containers:
+ - name: nightly-test
+ image: eu.gcr.io/gitpod-core-dev/dev/dev-environment:cw-werft-cred.0
+ workingDir: /workspace
+ imagePullPolicy: Always
+ volumeMounts:
+ - name: sh-playground-sa-perm
+ mountPath: /mnt/secrets/sh-playground-sa-perm
+ - name: sh-playground-dns-perm # this sa is used for the DNS management
+ mountPath: /mnt/secrets/sh-playground-dns-perm
+ env:
+ - name: ARM_SUBSCRIPTION_ID
+ valueFrom:
+ secretKeyRef:
+ name: aks-credentials
+ key: subscriptionid
+ - name: ARM_TENANT_ID
+ valueFrom:
+ secretKeyRef:
+ name: aks-credentials
+ key: tenantid
+ - name: ARM_CLIENT_ID
+ valueFrom:
+ secretKeyRef:
+ name: aks-credentials
+ key: clientid
+ - name: ARM_CLIENT_SECRET
+ valueFrom:
+ secretKeyRef:
+ name: aks-credentials
+ key: clientsecret
+ - name: WERFT_HOST
+ value: "werft.werft.svc.cluster.local:7777"
+ - name: GOOGLE_APPLICATION_CREDENTIALS
+ value: "/mnt/secrets/sh-playground-sa-perm/sh-sa.json"
+ - name: TF_VAR_sa_creds
+ value: "/mnt/secrets/sh-playground-sa-perm/sh-sa.json"
+ - name: TF_VAR_dns_sa_creds
+ value: "/mnt/secrets/sh-playground-dns-perm/sh-dns-sa.json"
+ - name: WERFT_K8S_NAMESPACE
+ value: "werft"
+ - name: WERFT_K8S_LABEL
+ value: "component=werft"
+ - name: NODENAME
+ valueFrom:
+ fieldRef:
+ fieldPath: spec.nodeName
+ command:
+ - bash
+ - -c
+ - |
+ sleep 1
+ set -Eeuo pipefail
+
+ sudo chown -R gitpod:gitpod /workspace
+ sudo apt update && apt install gettext-base
+
+ curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
+
+ export TF_VAR_TEST_ID=$(echo $RANDOM | md5sum | head -c 5; echo)
+
+ (cd .werft && yarn install && mv node_modules ..) | werft log slice prep
+ printf '{{ toJson . }}' > context.json
+
+ npx ts-node .werft/installer-tests.ts "STANDARD_AKS_TEST"
+# The bit below makes this a cron job
+# plugins:
+# cron: "15 3 * * *"
diff --git a/.werft/installer-tests.ts b/.werft/installer-tests.ts
index e864dc0e6b3a15..5ca6bd7f7ad704 100644
--- a/.werft/installer-tests.ts
+++ b/.werft/installer-tests.ts
@@ -2,7 +2,9 @@ import { join } from "path";
import { exec } from "./util/shell";
import { Werft } from "./util/werft";
-const testConfig: string = process.argv.length > 2 ? process.argv[2] : "gke";
+const testConfig: string = process.argv.length > 2 ? process.argv[2] : "STANDARD_K3S_TEST";
+// we can provide the version of the gitpod to install (eg: 2022.4.2)
+const version: string = process.argv.length > 3 ? process.argv[3] : "";
const makefilePath: string = join("install/tests");
@@ -29,6 +31,11 @@ const INFRA_PHASES: { [name: string]: InfraConfig } = {
makeTarget: "k3s-standard-cluster",
description: "Creating a k3s cluster on GCP with 1 node",
},
+ STANDARD_AKS_CLUSTER: {
+ phase: "create-std-aks-cluster",
+ makeTarget: "aks-standard-cluster",
+ description: "Creating an aks cluster(azure)",
+ },
GCP_MANAGED_DNS: {
phase: "setup-cert-manager-with-cloud-dns",
makeTarget: "managed-dns-with-cert-manager",
@@ -36,15 +43,16 @@ const INFRA_PHASES: { [name: string]: InfraConfig } = {
},
INSTALL_GITPOD_IGNORE_PREFLIGHTS: {
phase: "install-gitpod-without-preflights",
- makeTarget: "kots-install-without-preflight-with-community-license",
+ makeTarget: `kots-install channel=unstable version=${version} preflights=false`, // this is a bit of a hack, for now we pass params like this
description: "Install gitpod using kots community edition without preflights",
},
INSTALL_GITPOD: {
phase: "install-gitpod",
- makeTarget: "kots-install-with-community-license",
+ makeTarget: `kots-install channel=unstable version=${version} preflights=true`,
description: "Install gitpod using kots community edition",
},
CHECK_INSTALLATION: {
+ // this is a basic test for the Gitpod setup
phase: "check-gitpod-installation",
makeTarget: "check-gitpod-installation",
description: "Check gitpod installation",
@@ -86,6 +94,18 @@ const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = {
"DESTROY",
],
},
+ STANDARD_AKS_TEST: {
+ DESCRIPTION: "Deploy Gitpod on AKS, with managed DNS, and run integration tests",
+ PHASES: [
+ "STANDARD_AKS_CLUSTER",
+ "GCP_MANAGED_DNS",
+ "INSTALL_GITPOD",
+ "CHECK_INSTALLATION",
+ "RUN_INTEGRATION_TESTS",
+ "RESULTS",
+ "DESTROY",
+ ],
+ },
STANDARD_K3S_TEST: {
DESCRIPTION:
"Deploy Gitpod on a K3s cluster, created on a GCP instance," +
@@ -155,6 +175,7 @@ function callMakeTargets(phase: string, description: string, makeTarget: string)
werft.fail(phase, "Operation failed");
} else {
werft.log(phase, response.stdout.toString());
+ werft.done(phase);
}
return response.code;
diff --git a/install/infra/terraform/aks/README.md b/install/infra/terraform/aks/README.md
new file mode 100644
index 00000000000000..e03e1cc8289b3b
--- /dev/null
+++ b/install/infra/terraform/aks/README.md
@@ -0,0 +1,97 @@
+# Azure
+
+Azure provider for Gitpod testing
+
+
+
+- [Terraform Documentation](#terraform-documentation)
+ * [Requirements](#requirements)
+ * [Providers](#providers)
+ * [Modules](#modules)
+ * [Resources](#resources)
+ * [Inputs](#inputs)
+ * [Outputs](#outputs)
+
+
+
+# Terraform Documentation
+
+
+## Requirements
+
+| Name | Version |
+|------|---------|
+| [azurerm](#requirement\_azurerm) | >= 3.0.0, < 4.0.0 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| [azurerm](#provider\_azurerm) | >= 3.0.0, < 4.0.0 |
+| [random](#provider\_random) | n/a |
+
+## Modules
+
+No modules.
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [azurerm_container_registry.registry](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/container_registry) | resource |
+| [azurerm_dns_zone.dns](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/dns_zone) | resource |
+| [azurerm_kubernetes_cluster.k8s](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/kubernetes_cluster) | resource |
+| [azurerm_kubernetes_cluster_node_pool.pools](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/kubernetes_cluster_node_pool) | resource |
+| [azurerm_log_analytics_solution.monitoring](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/log_analytics_solution) | resource |
+| [azurerm_log_analytics_workspace.monitoring](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/log_analytics_workspace) | resource |
+| [azurerm_mysql_database.db](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/mysql_database) | resource |
+| [azurerm_mysql_firewall_rule.db](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/mysql_firewall_rule) | resource |
+| [azurerm_mysql_server.db](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/mysql_server) | resource |
+| [azurerm_network_security_rule.k8s](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/network_security_rule) | resource |
+| [azurerm_resource_group.gitpod](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/resource_group) | resource |
+| [azurerm_role_assignment.k8s](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource |
+| [azurerm_role_assignment.registry](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource |
+| [azurerm_storage_account.storage](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/storage_account) | resource |
+| [azurerm_subnet.network](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/subnet) | resource |
+| [azurerm_virtual_network.network](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/virtual_network) | resource |
+| [random_integer.db](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/integer) | resource |
+| [random_integer.registry](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/integer) | resource |
+| [random_integer.storage](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/integer) | resource |
+| [random_password.db](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/password) | resource |
+| [azurerm_client_config.current](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/client_config) | data source |
+| [azurerm_kubernetes_service_versions.k8s](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/kubernetes_service_versions) | data source |
+| [azurerm_resources.k8s](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/resources) | data source |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| [dns\_enabled](#input\_dns\_enabled) | Common variables | `any` | n/a | yes |
+| [domain\_name](#input\_domain\_name) | n/a | `any` | n/a | yes |
+| [enable\_airgapped](#input\_enable\_airgapped) | n/a | `any` | n/a | yes |
+| [enable\_external\_database](#input\_enable\_external\_database) | n/a | `any` | n/a | yes |
+| [enable\_external\_registry](#input\_enable\_external\_registry) | n/a | `any` | n/a | yes |
+| [enable\_external\_storage](#input\_enable\_external\_storage) | n/a | `any` | n/a | yes |
+| [labels](#input\_labels) | n/a | `any` | n/a | yes |
+| [location](#input\_location) | Azure-specific variables | `any` | n/a | yes |
+| [name\_format](#input\_name\_format) | n/a | `any` | n/a | yes |
+| [name\_format\_global](#input\_name\_format\_global) | n/a | `any` | n/a | yes |
+| [workspace\_name](#input\_workspace\_name) | n/a | `any` | n/a | yes |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| [cert\_manager\_issuer](#output\_cert\_manager\_issuer) | n/a |
+| [cert\_manager\_secret](#output\_cert\_manager\_secret) | n/a |
+| [cluster\_name](#output\_cluster\_name) | n/a |
+| [database](#output\_database) | n/a |
+| [domain\_nameservers](#output\_domain\_nameservers) | n/a |
+| [external\_dns\_secrets](#output\_external\_dns\_secrets) | n/a |
+| [external\_dns\_settings](#output\_external\_dns\_settings) | n/a |
+| [k8s\_connection](#output\_k8s\_connection) | n/a |
+| [kubeconfig](#output\_kubeconfig) | n/a |
+| [region](#output\_region) | n/a |
+| [registry](#output\_registry) | n/a |
+| [storage](#output\_storage) | n/a |
+
diff --git a/install/infra/terraform/aks/database.tf b/install/infra/terraform/aks/database.tf
new file mode 100644
index 00000000000000..50a8a21e3fa9e2
--- /dev/null
+++ b/install/infra/terraform/aks/database.tf
@@ -0,0 +1,50 @@
+resource "random_integer" "db" {
+ count = var.enable_external_database ? 1 : 0
+
+ min = 10000
+ max = 99999
+}
+
+resource "random_password" "db" {
+ count = var.enable_external_database ? 1 : 0
+
+ length = 32
+}
+
+resource "azurerm_mysql_server" "db" {
+ count = var.enable_external_database ? 1 : 0
+
+ name = "gitpod-${random_integer.db[count.index].result}"
+ location = azurerm_resource_group.gitpod.location
+ resource_group_name = azurerm_resource_group.gitpod.name
+
+ sku_name = local.db
+ storage_mb = 20480
+ ssl_enforcement_enabled = false
+ ssl_minimal_tls_version_enforced = "TLSEnforcementDisabled"
+ version = "5.7"
+
+ auto_grow_enabled = true
+ administrator_login = "gitpod"
+ administrator_login_password = random_password.db[count.index].result
+}
+
+resource "azurerm_mysql_firewall_rule" "db" {
+ count = var.enable_external_database ? 1 : 0
+
+ name = "Azure_Resource"
+ resource_group_name = azurerm_resource_group.gitpod.name
+ server_name = azurerm_mysql_server.db[count.index].name
+ start_ip_address = "0.0.0.0"
+ end_ip_address = "0.0.0.0"
+}
+
+resource "azurerm_mysql_database" "db" {
+ count = var.enable_external_database ? 1 : 0
+
+ name = "gitpod"
+ resource_group_name = azurerm_resource_group.gitpod.name
+ server_name = azurerm_mysql_server.db[count.index].name
+ charset = "utf8"
+ collation = "utf8_unicode_ci"
+}
diff --git a/install/infra/terraform/aks/kubernetes.tf b/install/infra/terraform/aks/kubernetes.tf
new file mode 100644
index 00000000000000..66f6f72c8c9ad2
--- /dev/null
+++ b/install/infra/terraform/aks/kubernetes.tf
@@ -0,0 +1,95 @@
+data "azurerm_kubernetes_service_versions" "k8s" {
+ location = azurerm_resource_group.gitpod.location
+ include_preview = false
+}
+
+resource "azurerm_role_assignment" "k8s" {
+ count = var.dns_enabled ? 1 : 0
+
+ principal_id = azurerm_kubernetes_cluster.k8s.kubelet_identity[count.index].object_id
+ role_definition_name = "DNS Zone Contributor"
+ scope = azurerm_dns_zone.dns[count.index].id
+}
+
+resource "azurerm_kubernetes_cluster" "k8s" {
+ name = format(var.name_format, local.location, "primary")
+ location = azurerm_resource_group.gitpod.location
+ resource_group_name = azurerm_resource_group.gitpod.name
+ dns_prefix = "gitpod"
+
+ kubernetes_version = data.azurerm_kubernetes_service_versions.k8s.latest_version
+ http_application_routing_enabled = false
+
+ default_node_pool {
+ name = local.nodes.0.name
+ vm_size = local.machine
+
+ enable_auto_scaling = true
+ min_count = 2
+ max_count = 10
+ orchestrator_version = data.azurerm_kubernetes_service_versions.k8s.latest_version
+ node_labels = local.nodes.0.labels
+
+ type = "VirtualMachineScaleSets"
+ vnet_subnet_id = azurerm_subnet.network.id
+ }
+
+ identity {
+ type = "SystemAssigned"
+ }
+
+ network_profile {
+ network_plugin = "kubenet"
+ network_policy = "calico"
+ }
+
+ oms_agent {
+ log_analytics_workspace_id = azurerm_log_analytics_workspace.monitoring.id
+ }
+}
+
+resource "azurerm_kubernetes_cluster_node_pool" "pools" {
+ count = length(local.nodes) - 1
+
+ kubernetes_cluster_id = azurerm_kubernetes_cluster.k8s.id
+ name = local.nodes[count.index + 1].name
+ vm_size = local.machine
+
+ enable_auto_scaling = true
+ min_count = 2
+ max_count = 10
+ orchestrator_version = data.azurerm_kubernetes_service_versions.k8s.latest_version
+ node_labels = local.nodes[count.index + 1].labels
+ vnet_subnet_id = azurerm_subnet.network.id
+}
+
+data "azurerm_resources" "k8s" {
+ count = var.enable_airgapped ? 1 : 0
+
+ resource_group_name = azurerm_kubernetes_cluster.k8s.node_resource_group
+ type = "Microsoft.Network/networkSecurityGroups"
+
+ depends_on = [
+ azurerm_kubernetes_cluster.k8s,
+ azurerm_kubernetes_cluster_node_pool.pools
+ ]
+}
+
+resource "azurerm_network_security_rule" "k8s" {
+ count = length(local.network_security_rules)
+
+ resource_group_name = azurerm_kubernetes_cluster.k8s.node_resource_group
+ network_security_group_name = data.azurerm_resources.k8s.0.resources.0.name
+
+ priority = lookup(local.network_security_rules[count.index], "priority", sum([100, count.index]))
+ name = local.network_security_rules[count.index].name
+ access = local.network_security_rules[count.index].access
+ direction = local.network_security_rules[count.index].direction
+ protocol = local.network_security_rules[count.index].protocol
+
+ description = lookup(local.network_security_rules[count.index], "description", null)
+ source_port_range = lookup(local.network_security_rules[count.index], "source_port_range", null)
+ destination_port_range = lookup(local.network_security_rules[count.index], "destination_port_range", null)
+ source_address_prefix = lookup(local.network_security_rules[count.index], "source_address_prefix", null)
+ destination_address_prefix = lookup(local.network_security_rules[count.index], "destination_address_prefix", null)
+}
diff --git a/install/infra/terraform/aks/local.tf b/install/infra/terraform/aks/local.tf
new file mode 100644
index 00000000000000..e11d87159206cc
--- /dev/null
+++ b/install/infra/terraform/aks/local.tf
@@ -0,0 +1,95 @@
+locals {
+ labels = tomap({
+ workload_meta : "gitpod.io/workload_meta"
+ workload_ide : "gitpod.io/workload_ide"
+ workspace_services : "gitpod.io/workload_workspace_services"
+ workspace_regular : "gitpod.io/workload_workspace_regular"
+ workspace_headless : "gitpod.io/workload_workspace_headless"
+ })
+ dns_enabled = var.domain_name != null
+ name_format = join("-", [
+ "gitpod",
+ "%s", # region
+ "%s", # name
+ local.workspace_name
+ ])
+ name_format_global = join("-", [
+ "gitpod",
+ "%s", # name
+ local.workspace_name
+ ])
+ workspace_name = replace(terraform.workspace, "/[\\W\\-]/", "") # alphanumeric workspace name
+ db = "GP_Gen5_2"
+ location = substr(var.location, 0, 3) # Short code for location
+ machine = "Standard_D4_v3"
+ network_security_rules = var.enable_airgapped ? [
+ {
+ name = "AllowContainerRegistry"
+ description = "Allow outgoing traffic to the container registry"
+ direction = "Outbound"
+ access = "Allow"
+ protocol = "*"
+ source_port_range = "*"
+ destination_port_range = "*"
+ source_address_prefix = "*"
+ destination_address_prefix = "AzureContainerRegistry"
+ },
+ {
+ name = "AllowDatabase"
+ description = "Allow outgoing traffic to the database"
+ direction = "Outbound"
+ access = "Allow"
+ protocol = "*"
+ source_port_range = "*"
+ destination_port_range = "*"
+ source_address_prefix = "*"
+ destination_address_prefix = "Sql"
+ },
+ {
+ name = "AllowStorage"
+ description = "Allow outgoing traffic to the storage"
+ direction = "Outbound"
+ access = "Allow"
+ protocol = "*"
+ source_port_range = "*"
+ destination_port_range = "*"
+ source_address_prefix = "*"
+ destination_address_prefix = "Storage"
+ },
+ {
+ name = "AllowAzureCloud"
+ description = "Allow outgoing traffic to the Azure cloud"
+ direction = "Outbound"
+ access = "Allow"
+ protocol = "*"
+ source_port_range = "*"
+ destination_port_range = "*"
+ source_address_prefix = "*"
+ destination_address_prefix = "AzureCloud"
+ },
+ {
+ name = "DenyInternetOutBound"
+ description = "Deny outgoing traffic to the public internet"
+ direction = "Outbound"
+ access = "Deny"
+ protocol = "*"
+ source_port_range = "*"
+ destination_port_range = "*"
+ source_address_prefix = "*"
+ destination_address_prefix = "Internet"
+ priority = 4096
+ }
+ ] : []
+ nodes = [
+ {
+ name = "services"
+ labels = {
+ lookup(local.labels, "workload_meta") = true
+ lookup(local.labels, "workload_ide") = true
+ lookup(local.labels, "workspace_services") = true
+ lookup(local.labels, "workspace_regular") = true
+ lookup(local.labels, "workspace_headless") = true
+ }
+ }
+ ]
+}
diff --git a/install/infra/terraform/aks/main.tf b/install/infra/terraform/aks/main.tf
new file mode 100644
index 00000000000000..43dd6a6e6adf63
--- /dev/null
+++ b/install/infra/terraform/aks/main.tf
@@ -0,0 +1,24 @@
+terraform {
+ required_providers {
+ azurerm = {
+ source = "hashicorp/azurerm"
+ version = ">= 3.0.0, < 4.0.0"
+ }
+ }
+}
+
+provider "azurerm" {
+ features {}
+}
+
+data "azurerm_client_config" "current" {}
+
+resource "azurerm_resource_group" "gitpod" {
+ name = format(var.name_format_global, local.location)
+ location = var.location
+}
+
+resource "local_file" "kubeconfig" {
+ filename = var.kubeconfig
+ content = azurerm_kubernetes_cluster.k8s.kube_config_raw
+}
diff --git a/install/infra/terraform/aks/monitoring.tf b/install/infra/terraform/aks/monitoring.tf
new file mode 100644
index 00000000000000..57276917673669
--- /dev/null
+++ b/install/infra/terraform/aks/monitoring.tf
@@ -0,0 +1,19 @@
+resource "azurerm_log_analytics_workspace" "monitoring" {
+ name = format(var.name_format, var.location, "monitoring")
+ location = azurerm_resource_group.gitpod.location
+ resource_group_name = azurerm_resource_group.gitpod.name
+ sku = "PerGB2018"
+}
+
+resource "azurerm_log_analytics_solution" "monitoring" {
+ solution_name = "ContainerInsights"
+ location = azurerm_resource_group.gitpod.location
+ resource_group_name = azurerm_resource_group.gitpod.name
+ workspace_name = azurerm_log_analytics_workspace.monitoring.name
+ workspace_resource_id = azurerm_log_analytics_workspace.monitoring.id
+
+ plan {
+ publisher = "Microsoft"
+ product = "OMSGallery/ContainerInsights"
+ }
+}
diff --git a/install/infra/terraform/aks/networks.tf b/install/infra/terraform/aks/networks.tf
new file mode 100644
index 00000000000000..3e4184ee1d1ed3
--- /dev/null
+++ b/install/infra/terraform/aks/networks.tf
@@ -0,0 +1,20 @@
+resource "azurerm_virtual_network" "network" {
+ name = format(var.name_format, local.location, "network")
+ location = azurerm_resource_group.gitpod.location
+ resource_group_name = azurerm_resource_group.gitpod.name
+ address_space = ["10.2.0.0/16"]
+}
+
+resource "azurerm_subnet" "network" {
+ name = format(var.name_format, local.location, "network")
+ resource_group_name = azurerm_resource_group.gitpod.name
+ virtual_network_name = azurerm_virtual_network.network.name
+ address_prefixes = ["10.2.1.0/24"]
+}
+
+resource "azurerm_dns_zone" "dns" {
+ count = var.dns_enabled ? 1 : 0
+
+ name = var.domain_name
+ resource_group_name = azurerm_resource_group.gitpod.name
+}
diff --git a/install/infra/terraform/aks/output.tf b/install/infra/terraform/aks/output.tf
new file mode 100644
index 00000000000000..a22bd5c3efb7d2
--- /dev/null
+++ b/install/infra/terraform/aks/output.tf
@@ -0,0 +1,89 @@
+output "cert_manager_issuer" {
+ value = try([{
+ dns01 = {
+ azureDNS = {
+ subscriptionID = data.azurerm_client_config.current.subscription_id
+ resourceGroupName = azurerm_resource_group.gitpod.name
+ hostedZoneName = azurerm_dns_zone.dns.0.name
+ managedIdentity = {
+ clientID = azurerm_kubernetes_cluster.k8s.kubelet_identity.0.client_id
+ }
+ }
+ }
+ }], [])
+}
+
+output "cert_manager_secret" {
+ value = {}
+}
+
+output "cluster_name" {
+ value = azurerm_kubernetes_cluster.k8s.name
+}
+
+output "database" {
+ sensitive = true
+ value = try({
+ host = "${azurerm_mysql_server.db.0.name}.mysql.database.azure.com"
+ password = azurerm_mysql_server.db.0.administrator_login_password
+ port = 3306
+ username = "${azurerm_mysql_server.db.0.administrator_login}@${azurerm_mysql_server.db.0.name}"
+ }, {})
+}
+
+output "domain_nameservers" {
+ value = try(azurerm_dns_zone.dns.0.name_servers, null)
+}
+
+output "external_dns_secrets" {
+ value = {}
+}
+
+output "external_dns_settings" {
+ value = {
+ provider = "azure"
+ "azure.resourceGroup" = azurerm_resource_group.gitpod.name
+ "azure.subscriptionId" = data.azurerm_client_config.current.subscription_id
+ "azure.tenantId" = data.azurerm_client_config.current.tenant_id
+ "azure.useManagedIdentityExtension" = true
+ "azure.userAssignedIdentityID" = azurerm_kubernetes_cluster.k8s.kubelet_identity.0.client_id
+ }
+}
+
+output "k8s_connection" {
+ sensitive = true
+ value = {
+ host = azurerm_kubernetes_cluster.k8s.kube_config.0.host
+ username = azurerm_kubernetes_cluster.k8s.kube_config.0.username
+ password = azurerm_kubernetes_cluster.k8s.kube_config.0.password
+ client_certificate = base64decode(azurerm_kubernetes_cluster.k8s.kube_config.0.client_certificate)
+ client_key = base64decode(azurerm_kubernetes_cluster.k8s.kube_config.0.client_key)
+ cluster_ca_certificate = base64decode(azurerm_kubernetes_cluster.k8s.kube_config.0.cluster_ca_certificate)
+ }
+}
+
+output "kubeconfig" {
+ sensitive = true
+ value = azurerm_kubernetes_cluster.k8s.kube_config_raw
+}
+
+output "region" {
+ value = var.location
+}
+
+output "registry" {
+ sensitive = true
+ value = try({
+ server = azurerm_container_registry.registry.0.login_server
+ password = azurerm_container_registry.registry.0.admin_password
+ username = azurerm_container_registry.registry.0.admin_username
+ }, {})
+}
+
+output "storage" {
+ sensitive = true
+ value = try({
+ username = azurerm_storage_account.storage.0.name
+ password = azurerm_storage_account.storage.0.primary_access_key
+ }, {})
+}
diff --git a/install/infra/terraform/aks/registry.tf b/install/infra/terraform/aks/registry.tf
new file mode 100644
index 00000000000000..87ec560c6b5441
--- /dev/null
+++ b/install/infra/terraform/aks/registry.tf
@@ -0,0 +1,25 @@
+resource "random_integer" "registry" {
+ count = var.enable_external_registry ? 1 : 0
+
+ min = 10000
+ max = 99999
+}
+
+resource "azurerm_container_registry" "registry" {
+ count = var.enable_external_registry ? 1 : 0
+
+ name = "gitpod${random_integer.registry[count.index].result}"
+ resource_group_name = azurerm_resource_group.gitpod.name
+ location = azurerm_resource_group.gitpod.location
+ admin_enabled = true
+ sku = "Premium"
+}
+
+resource "azurerm_role_assignment" "registry" {
+ count = var.enable_external_registry ? 1 : 0
+
+ principal_id = azurerm_kubernetes_cluster.k8s.kubelet_identity[0].object_id
+ role_definition_name = "AcrPush"
+ scope = azurerm_container_registry.registry[count.index].id
+ skip_service_principal_aad_check = true
+}
diff --git a/install/infra/terraform/aks/storage.tf b/install/infra/terraform/aks/storage.tf
new file mode 100644
index 00000000000000..ca4ef115dcf924
--- /dev/null
+++ b/install/infra/terraform/aks/storage.tf
@@ -0,0 +1,16 @@
+resource "random_integer" "storage" {
+ count = var.enable_external_storage ? 1 : 0
+
+ min = 10000
+ max = 99999
+}
+
+resource "azurerm_storage_account" "storage" {
+ count = var.enable_external_storage ? 1 : 0
+
+ name = "gitpod${random_integer.storage[count.index].result}"
+ resource_group_name = azurerm_resource_group.gitpod.name
+ location = azurerm_resource_group.gitpod.location
+ account_tier = "Standard"
+ account_replication_type = "LRS"
+}
diff --git a/install/infra/terraform/aks/variables.tf b/install/infra/terraform/aks/variables.tf
new file mode 100644
index 00000000000000..796f7489896181
--- /dev/null
+++ b/install/infra/terraform/aks/variables.tf
@@ -0,0 +1,24 @@
+// Common variables
+variable "kubeconfig" {
+ default = "./kubeconfig"
+
+}
+variable "dns_enabled" {}
+variable "domain_name" {}
+variable "enable_airgapped" {}
+variable "enable_external_database" {}
+variable "enable_external_registry" {}
+variable "enable_external_storage" {}
+variable "labels" {}
+variable "name_format" {
+}
+variable "name_format_global" {
+}
+variable "workspace_name" {
+}
+
+// Azure-specific variables
+variable "location" {
+ default = "northeurope"
+
+}
diff --git a/install/infra/terraform/gke/README.md b/install/infra/terraform/gke/README.md
new file mode 100644
index 00000000000000..8886047aaf064c
--- /dev/null
+++ b/install/infra/terraform/gke/README.md
@@ -0,0 +1,90 @@
+# GKE terraform module
+
+This is a terraform module currently used in the automated installation tests
+for Gitpod. At successful completion, this module creates the following:
+
+1. A [`GKE`](https://cloud.google.com/kubernetes-engine) cluster by user
+ provided name `` with the following nodepools:
+ 1. `workspaces-` for workspace workloads
+ 1. `services-` for IDE and meta workloads
+1. A dedicated `vpc` and `subnet` for the resources
+1. `kubeconfig` data populated in a user defined `kubeconfig` variable
+
+## Requirements
+
+1. `terraform` >= `v1.1.7`
+
+## Providers
+
+1. [`google`](https://registry.terraform.io/providers/hashicorp/google/latest/docs)
+
+## Inputs
+
+
+| Argument | Description | Required | Default |
+|---------------------------|----------------------------------------------------------------------------------------------------------------------------------------------|----------|--------------------|
+| `project` | The project ID to create the cluster. | `true` | |
+| `kubeconfig` | Path to write the kubeconfig output to. | `false` | `./kubeconfig` |
+| `region` | The region to create the cluster. | `false` | `europe-west1` |
+| `zone` | The zone suffix to create the cluster in. If an empty string is provided, it will create multi-zone cluster | `false` | `b` |
+| `kubernetes_version` | Kubernetes version to be setup | `false` | `1.21.11-gke.1100` |
+| `name` | The name of the cluster and suffix for other resources | `false` | `sh-test` |
+| `workspaces_machine_type` | Type of the node compute engines for workspace nodepool. | `false` | `n2-standard-8` |
+| `services_machine_type` | Type of the node compute engines for meta and IDE nodepool. | `false` | `n2-standard-4` |
+| `min_count` | Minimum number of nodes in the NodePool. Must be >=0 and <= max_node_count. | `false` | `1` |
+| `max_count` | Maximum number of nodes in the NodePool. Must be >= min_node_count. | `false` | `50` |
+| `disk_size_gb` | Disk size to provision in nodes. | `false` | `100` |
+| `initial_node_count` | The number of nodes to create in this cluster's default node pool. | `false` | `1` |
+| `pre-emptible` | Set if the nodes are to be pre-emptible. | `false` | `false` |
+| `credentials` | Path to the JSON file storing Google service account credentials, if left empty, `tf` will look for `GOOGLE_APPLICATION_CREDENTIALS` env var | `false` | |
+| | | | |
+
+
+## Outputs
+
+
+| Argument |
+|-----------------------|
+| `kubernetes_endpoint` |
+| `client_token` |
+| `ca_certificate` |
+| `kubeconfig` |
+
+
+## Usage
+
+Make sure you have a `GCP` account and credentials in `JSON` format to a service
+account with atleast the following permissions:
+1. Compute Admin
+1. Compute Network Admin
+1. Kubernetes Engine Admin
+1. Kubernetes Engine Developer
+1. Service Account User
+1. Storage Admin
+
+Assign it to the following environment variable:
+
+``` sh
+export GOOGLE_APPLICATION_CREDENTAILS=/path/to/service-account-json-creds
+```
+
+Create a `Cloud storage bucket` in the same project and region and specified
+above, by name `gitpod-gke`. If you want to use a different name, edit the name
+manually in the `main.tf` file.
+
+Open the file `variables.auto.tfvars` in this directory and provide values available from the
+`Inputs` list as you find fit.
+
+Run the following commands:
+
+``` sh
+terraform init
+terraform apply
+```
+
+
+## Cleanup
+
+``` sh
+terraform destroy
+```
diff --git a/install/infra/terraform/gke/main.tf b/install/infra/terraform/gke/main.tf
index 309b36ee17a7a6..518b5b11099933 100644
--- a/install/infra/terraform/gke/main.tf
+++ b/install/infra/terraform/gke/main.tf
@@ -2,11 +2,18 @@ terraform {
required_version = ">= 1.0.3"
}
+terraform {
+ backend "gcs" {
+ bucket = "gitpod-gke"
+ prefix = "tf-state"
+ }
+}
+
provider "google" {
+ project = var.project
credentials = var.credentials
- project = var.project
- region = var.region
- zone = var.zone
+ region = var.region
+ zone = var.zone
}
resource "google_compute_network" "vpc" {
@@ -14,7 +21,6 @@ resource "google_compute_network" "vpc" {
auto_create_subnetworks = "false"
}
-# Subnet
resource "google_compute_subnetwork" "subnet" {
name = "subnet-${var.name}"
region = var.region
@@ -40,9 +46,9 @@ resource "google_container_cluster" "gitpod-cluster" {
enabled = true
resource_limits {
- resource_type = "cpu"
- minimum = 2
- maximum = 8
+ resource_type = "cpu"
+ minimum = 2
+ maximum = 8
}
resource_limits {
@@ -79,27 +85,19 @@ resource "google_container_cluster" "gitpod-cluster" {
horizontal_pod_autoscaling {
disabled = false
}
-
-
- # only available in beta
- # dns_cache_config {
- # enabled = true
- # }
}
network = google_compute_network.vpc.name
subnetwork = google_compute_subnetwork.subnet.name
-
-
}
resource "google_container_node_pool" "services" {
- name = "services-${var.name}"
- location = google_container_cluster.gitpod-cluster.location
- cluster = google_container_cluster.gitpod-cluster.name
- version = var.kubernetes_version // kubernetes version
+ name = "services-${var.name}"
+ location = google_container_cluster.gitpod-cluster.location
+ cluster = google_container_cluster.gitpod-cluster.name
+ version = var.kubernetes_version // kubernetes version
initial_node_count = 1
- max_pods_per_node = 110
+ max_pods_per_node = 110
node_config {
oauth_scopes = [
@@ -107,15 +105,15 @@ resource "google_container_node_pool" "services" {
]
labels = {
- "gitpod.io/workload_meta" =true
- "gitpod.io/workload_ide" = true
+ "gitpod.io/workload_meta" = true
+ "gitpod.io/workload_ide" = true
}
preemptible = var.pre-emptible
- image_type = "UBUNTU_CONTAINERD"
- disk_type = "pd-ssd"
- disk_size_gb = var.disk_size_gb
- machine_type = var.machine_type
+ image_type = "UBUNTU_CONTAINERD"
+ disk_type = "pd-ssd"
+ disk_size_gb = var.disk_size_gb
+ machine_type = var.services_machine_type
tags = ["gke-node", "${var.project}-gke"]
metadata = {
disable-legacy-endpoints = "true"
@@ -129,18 +127,18 @@ resource "google_container_node_pool" "services" {
management {
- auto_repair = true
+ auto_repair = true
auto_upgrade = false
}
}
resource "google_container_node_pool" "workspaces" {
- name = "workspaces-${var.name}"
- location = google_container_cluster.gitpod-cluster.location
- cluster = google_container_cluster.gitpod-cluster.name
- version = var.kubernetes_version // kubernetes version
+ name = "workspaces-${var.name}"
+ location = google_container_cluster.gitpod-cluster.location
+ cluster = google_container_cluster.gitpod-cluster.name
+ version = var.kubernetes_version // kubernetes version
initial_node_count = 1
- max_pods_per_node = 110
+ max_pods_per_node = 110
node_config {
oauth_scopes = [
@@ -148,18 +146,18 @@ resource "google_container_node_pool" "workspaces" {
]
labels = {
- "gitpod.io/workload_metal" =true
- "gitpod.io/workload_ide" = true
+ "gitpod.io/workload_metal" = true
+ "gitpod.io/workload_ide" = true
"gitpod.io/workload_workspace_services" = true
- "gitpod.io/workload_workspace_regular" = true
+ "gitpod.io/workload_workspace_regular" = true
"gitpod.io/workload_workspace_headless" = true
}
preemptible = var.pre-emptible
- image_type = "UBUNTU_CONTAINERD"
- disk_type = "pd-ssd"
- disk_size_gb = var.disk_size_gb
- machine_type = var.machine_type
+ image_type = "UBUNTU_CONTAINERD"
+ disk_type = "pd-ssd"
+ disk_size_gb = var.disk_size_gb
+ machine_type = var.workspaces_machine_type
tags = ["gke-node", "${var.project}-gke"]
metadata = {
disable-legacy-endpoints = "true"
@@ -172,7 +170,7 @@ resource "google_container_node_pool" "workspaces" {
}
management {
- auto_repair = true
+ auto_repair = true
auto_upgrade = false
}
}
@@ -189,5 +187,5 @@ module "gke_auth" {
resource "local_file" "kubeconfig" {
filename = var.kubeconfig
- content = module.gke_auth.kubeconfig_raw
+ content = module.gke_auth.kubeconfig_raw
}
diff --git a/install/infra/terraform/gke/output.tf b/install/infra/terraform/gke/output.tf
index 29702d742affbd..470911d8a97705 100644
--- a/install/infra/terraform/gke/output.tf
+++ b/install/infra/terraform/gke/output.tf
@@ -10,9 +10,10 @@ output "client_token" {
output "ca_certificate" {
sensitive = true
- value = module.gke_auth.cluster_ca_certificate
+ value = module.gke_auth.cluster_ca_certificate
}
output "kubeconfig" {
- value = module.gke_auth.kubeconfig_raw
+ sensitive = true
+ value = module.gke_auth.kubeconfig_raw
}
diff --git a/install/infra/terraform/gke/variables.auto.tfvars b/install/infra/terraform/gke/variables.auto.tfvars
new file mode 100644
index 00000000000000..d6fc1bfe33cea2
--- /dev/null
+++ b/install/infra/terraform/gke/variables.auto.tfvars
@@ -0,0 +1,5 @@
+kubeconfig = "./kubeconfig"
+
+project = "sh-automated-tests"
+region = "europe-west1"
+zone = "b"
diff --git a/install/infra/terraform/gke/variables.tf b/install/infra/terraform/gke/variables.tf
index 50197379cc53bd..4aa4c3c9349c9c 100644
--- a/install/infra/terraform/gke/variables.tf
+++ b/install/infra/terraform/gke/variables.tf
@@ -4,14 +4,9 @@ variable "project" {
}
variable "kubeconfig" {
- type = string
- description = "Path to write the kubeconfig output to"
- default = "./kubeconfig"
-}
-
-variable "credentials" {
type = string
- description = "Path to the json file storing SA credentials."
+ description = "Path to write the kubeconfig output to"
+ default = "./kubeconfig"
}
variable "region" {
@@ -38,10 +33,16 @@ variable "name" {
default = "sh-test"
}
-variable "machine_type" {
+variable "workspaces_machine_type" {
type = string
- description = "Type of the node compute engines."
- default = "n2d-standard-4"
+ description = "Type of the node compute engines for workspace nodepool."
+ default = "n2-standard-8"
+}
+
+variable "services_machine_type" {
+ type = string
+ description = "Type of the node compute engines for services nodepool."
+ default = "n2-standard-4"
}
variable "min_count" {
@@ -59,7 +60,7 @@ variable "max_count" {
variable "disk_size_gb" {
type = number
description = "Size of the node's disk."
- default = 100
+ default = 100
}
variable "initial_node_count" {
@@ -68,33 +69,13 @@ variable "initial_node_count" {
default = 1
}
-variable "env_name" {
- description = "The environment for the GKE cluster"
- default = "test"
-}
-
-variable "network" {
- description = "The VPC network created to host the cluster in"
- default = "gitpod-gke-network"
-}
-
-variable "subnetwork" {
- description = "The subnetwork created to host the cluster in"
- default = "gitpod-gke-subnet"
-}
-
-variable "ip_range_pods_name" {
- description = "The secondary ip range to use for pods"
- default = "gitpod-ip-range-pods"
-}
-
-variable "ip_range_services_name" {
- description = "The secondary ip range to use for services"
- default = "gitpod-ip-range-services"
-}
-
variable "pre-emptible" {
- type = bool
+ type = bool
description = "Set if the nodes are to be pre-emptible"
default = false
}
+
+variable "credentials" {
+ description = "Path to the JSON file storing Google service account credentials"
+ default = ""
+}
diff --git a/install/infra/terraform/k3s/README.md b/install/infra/terraform/k3s/README.md
index 72544c364f2f18..90b611412e4c0c 100644
--- a/install/infra/terraform/k3s/README.md
+++ b/install/infra/terraform/k3s/README.md
@@ -1,15 +1,56 @@
-## Module to create a 1 node k3s cluster in GCP
+# k3s terraform module
+
+This is a terraform module currently used in the automated installation tests
+for Gitpod. At successful completion, this module creates the following:
+
+1. A GCP VM instance of type `n2d-standard-4` and with `ubuntu` image, along with a service account to give auth scopes
+1. A single node [`k3s`](https://k3s.io/) cluster with [`k3sup`](https://github.com/alexellis/k3sup)
### Requirements
-- `terraform` >= 1.0.0
-- [`k3sup`](https://github.com/alexellis/k3sup#download-k3sup-tldr)
+1. `terraform` >= `v1.1.7`
+1. [`k3sup`](https://github.com/alexellis/k3sup#download-k3sup-tldr)
```sh
curl -sLS https://get.k3sup.dev | sh
sudo install k3sup /usr/local/bin/
```
-### Usage
+## Providers
+
+1. [`google`](https://registry.terraform.io/providers/hashicorp/google/latest/docs)
+
+## Input
+
+| Argument | Description | Required | Default |
+|---------------|----------------------------------------------------------------------------------------------------------------------------------------------|----------|------------------|
+| `gcp_project` | The project ID to create the VM in. | `true` | |
+| `kubeconfig` | Path to write the kubeconfig output to. | `false` | `./kubeconfig` |
+| `gcp_region` | The region to create the VM. | `false` | `europe-west1` |
+| `gcp_zone` | The GCP zone to create the VM in. | `false` | `europe-west1-b` |
+| `name` | Prefix name for the nodes and firewall | `false` | `k3s` |
+| `credentials` | Path to the JSON file storing Google service account credentials, if left empty, `tf` will look for `GOOGLE_APPLICATION_CREDENTIALS` env var | `false` | |
+
+## Output
+
+| Argument |
+|-----------------------|
+| `kubernetes_endpoint` |
+| `kubeconfig` |
+
+## Usage
+
+Create a service account with at least the following access scopes:
+- Compute Admin
+- Create Service Accounts
+- Delete Service Accounts
+- Service Account User
+- Storage Admin
+
+Get the credentials of the SA in `JSON` format and export as a variable as folows:
+
+``` sh
+export GOOGLE_APPLICATION_CREDENTIALS=/path/to/credentials
+```
Update the `tfvars` file by the name `variables.auto.tfvars` in this
directory with the values you want, and then run:
@@ -21,3 +62,9 @@ terraform apply
You will find the `kubeconfig` populated to the path you specified in
the `variables.auto.tfvars` file.
+
+## Cleanup
+
+```sh
+terraform destroy
+```
diff --git a/install/infra/terraform/k3s/main.tf b/install/infra/terraform/k3s/main.tf
index 60876cf0c4a3d4..5a4ca2ab515795 100644
--- a/install/infra/terraform/k3s/main.tf
+++ b/install/infra/terraform/k3s/main.tf
@@ -7,11 +7,23 @@ terraform {
}
}
+terraform {
+ backend "gcs" {
+ bucket = "gitpod-k3s"
+ prefix = "tf-state"
+ }
+}
+
provider "google" {
credentials = var.credentials
- project = var.gcp_project
- region = var.gcp_region
- zone = var.gcp_zone
+ project = var.gcp_project
+ region = var.gcp_region
+ zone = var.gcp_zone
+}
+
+provider "google" {
+ alias = "dns"
+ credentials = var.dns_sa_creds
}
resource "google_service_account" "gcp_instance" {
@@ -31,15 +43,15 @@ resource "local_file" "ssh_private_key_pem" {
}
resource "google_compute_instance" "k3s_master_instance" {
- name = "master-${var.name}"
+ name = "master-${var.name}"
machine_type = "n2d-standard-4"
- tags = ["k3s", "k3s-master", "http-server", "https-server"]
+ tags = ["k3s", "k3s-master", "http-server", "https-server", "allow-ssh"]
boot_disk {
initialize_params {
image = "ubuntu-2004-focal-v20220419"
- size = 100
- type = "pd-ssd"
+ size = 100
+ type = "pd-ssd"
}
}
@@ -63,8 +75,6 @@ resource "google_compute_instance" "k3s_master_instance" {
email = google_service_account.gcp_instance.email
scopes = ["cloud-platform"]
}
-
-
}
resource "time_sleep" "wait_for_master" {
@@ -73,7 +83,6 @@ resource "time_sleep" "wait_for_master" {
create_duration = "60s"
}
-
resource "null_resource" "k3sup_install" {
depends_on = [time_sleep.wait_for_master]
@@ -83,16 +92,28 @@ resource "null_resource" "k3sup_install" {
--ip ${google_compute_instance.k3s_master_instance.network_interface[0].access_config[0].nat_ip} \
--ssh-key .ssh/google_compute_engine \
--context k3s \
- --user $(whoami) \
+ --user gitpod \
--local-path ${var.kubeconfig} \
--k3s-extra-args=" --disable=traefik --node-label=gitpod.io/workload_meta=true --node-label=gitpod.io/workload_ide=true --node-label=gitpod.io/workload_workspace_services=true --node-label=gitpod.io/workload_workspace_regular=true --node-label=gitpod.io/workload_workspace_headless=true" \
EOT
}
}
+resource "google_dns_record_set" "a" {
+ provider = google.dns
+ count = (var.domain_name == null) || (var.managed_dns_zone == null ) ? 0 : 1
+ name = "${var.domain_name}."
+ managed_zone = var.managed_dns_zone
+ project = var.dns_project == null ? var.gcp_project : var.dns_project
+ type = "A"
+ ttl = 5
+
+ rrdatas = [google_compute_instance.k3s_master_instance.network_interface[0].access_config[0].nat_ip]
+}
+
data "local_file" "kubeconfig" {
depends_on = [null_resource.k3sup_install]
- filename = var.kubeconfig
+ filename = var.kubeconfig
}
resource "google_compute_firewall" "k3s-firewall" {
@@ -104,21 +125,6 @@ resource "google_compute_firewall" "k3s-firewall" {
ports = ["6443"]
}
- allow {
- protocol = "tcp"
- ports = ["22"]
- }
-
- allow {
- protocol = "tcp"
- ports = ["443"]
- }
-
- allow {
- protocol = "tcp"
- ports = ["80"]
- }
-
target_tags = ["k3s"]
source_ranges = ["0.0.0.0/0"]
diff --git a/install/infra/terraform/k3s/variables.auto.tfvars b/install/infra/terraform/k3s/variables.auto.tfvars
index fca6fcadfaab2c..2f7c91e8f51ed4 100644
--- a/install/infra/terraform/k3s/variables.auto.tfvars
+++ b/install/infra/terraform/k3s/variables.auto.tfvars
@@ -1,6 +1,5 @@
kubeconfig = "./kubeconfig"
gcp_project = "sh-automated-tests"
-gcp_region = "europe-west1"
-gcp_zone = "europe-west1-b"
-credentials = "/workspace/gcp.json"
+gcp_region = "europe-west1"
+gcp_zone = "europe-west1-b"
diff --git a/install/infra/terraform/k3s/variables.tf b/install/infra/terraform/k3s/variables.tf
index 7899623f393156..26096d197bf097 100644
--- a/install/infra/terraform/k3s/variables.tf
+++ b/install/infra/terraform/k3s/variables.tf
@@ -1,27 +1,48 @@
variable "kubeconfig" {
- description = "The KUBECONFIG file path to store the resulting KUBECONFIG file to"
- default = "./kubeconfig"
+ description = "The KUBECONFIG file path to store the resulting KUBECONFIG file to"
+ default = "./kubeconfig"
}
variable "gcp_project" {
- description = "Google cloud Region to perform operations in"
+ description = "Google cloud Region to perform operations in"
}
variable "gcp_region" {
- description = "Google cloud Region to perform operations in"
- default = "europe-west1"
+ description = "Google cloud Region to perform operations in"
+ default = "europe-west1"
}
variable "gcp_zone" {
- description = "Google cloud Zone to perform operations in"
- default = "europe-west1-b"
+ description = "Google cloud Zone to perform operations in"
+ default = "europe-west1-b"
}
variable "credentials" {
- description = "Path to the JSON file storing Google service account credentials"
+ description = "Path to the JSON file storing Google service account credentials"
+ default = ""
}
variable "name" {
- description = "Prefix name for the nodes and firewall"
- default = "k3s"
+ description = "Prefix name for the nodes and firewall"
+ default = "k3s"
+}
+
+variable "dns_sa_creds" {
+ description = "Credentials with DNS admin rights to the project with managed DNS record"
+ default = ""
+}
+
+variable "dns_project" {
+ description = "Project associated with the dns maanged zone"
+ default = null
+}
+
+variable "domain_name" {
+ description = "Domain name to add to add DNS map to"
+ default = null
+}
+
+variable "managed_dns_zone" {
+ description = "Name of the managed DNS record"
+ default = null
}
diff --git a/install/infra/terraform/tools/cm-cloud-dns/README.md b/install/infra/terraform/tools/cm-cloud-dns/README.md
new file mode 100644
index 00000000000000..a7805293f15aeb
--- /dev/null
+++ b/install/infra/terraform/tools/cm-cloud-dns/README.md
@@ -0,0 +1,6 @@
+# Managed DNS terraform module
+
+This module is used in test setup, to deploy `cert-manager` and `external-dns`
+on a created cluster, and to create a secret that the aforementioned deployments
+can use to modify a `cloudDNS` record. This module works specifically for the
+domain `gitpod-self-hosted.com`.
diff --git a/install/tests/Makefile b/install/tests/Makefile
index 82c60a0bb5ca58..55592f0d33f388 100644
--- a/install/tests/Makefile
+++ b/install/tests/Makefile
@@ -29,6 +29,14 @@ gke-standard-cluster:
terraform apply -target=module.gke \
-var kubeconfig=${KUBECONFIG} --auto-approve
+.PHONY:
+## aks-standard-cluster: Creates an AKS cluster
+aks-standard-cluster:
+ terraform init --upgrade && \
+ terraform workspace new $(TF_VAR_TEST_ID) || terraform workspace select $(TF_VAR_TEST_ID) && \
+ terraform apply -target=module.aks \
+ -var kubeconfig=${KUBECONFIG} --auto-approve
+
.PHONY:
## k3s-standard-cluster: Creates a K3S cluster on GCP with one master and 1 worker node
k3s-standard-cluster:
@@ -53,37 +61,39 @@ get-kubeconfig:
${KUBECONFIG}
KOTS_KONFIG := "./manifests/kots-config.yaml"
-CHANNEL = "gitpod/beta"
-COMMUNITY_LICENSE := "../licenses/Community (Beta).yaml"
+license_community_beta := "../licenses/Community (Beta).yaml"
+license_community_stable := "../licenses/Community.yaml"
+license_community_unstable := "../licenses/Community (Unstable).yaml"
install-kots-cli:
curl https://kots.io/install | bash
-kots-install-without-preflight-with-community-license: install-kots-cli
+preflights ?= true
+channel ?= unstable
+version ?= -
+kots-install: version-flag = $(if $(version:-=),--app-version-label=$(version),)
+kots-install: preflight-flag = $(if $(preflights:true=),--skip-preflights,)
+kots-install: license-file = $(license_community_$(channel))
+kots-install: install-kots-cli
envsubst < ${KOTS_KONFIG} > tmp_config.yml
- kubectl kots install ${CHANNEL} \
- --skip-rbac-check --skip-preflights --namespace gitpod --kubeconfig=${KUBECONFIG} \
+ kubectl kots install gitpod/${channel} \
+ --skip-rbac-check ${version-flag} ${preflight-flag} \
+ --namespace gitpod --kubeconfig=${KUBECONFIG} \
--name gitpod --shared-password gitpod \
- --license-file ${COMMUNITY_LICENSE} \
+ --license-file ${license-file} \
--no-port-forward \
--config-values tmp_config.yml
-kots-install-with-community-license: install-kots-cli
- envsubst < ${KOTS_KONFIG} > tmp_config.yml
- kubectl kots install ${CHANNEL} \
- --skip-rbac-check --namespace gitpod --kubeconfig=${KUBECONFIG} \
- --name gitpod --shared-password gitpod \
- --license-file ${COMMUNITY_LICENSE} \
- --no-port-forward \
- --config-values tmp_config.yml
+delete-cm-setup:
+ sleep 120 && kubectl --kubeconfig=${KUBECONFIG} delete pods --all -n cert-manager && sleep 300;
-check-gitpod-installation:
- sleep 120
- kubectl --kubeconfig=${KUBECONFIG} delete pods --all -n cert-manager
- sleep 300
- @kubectl kots get --kubeconfig=${KUBECONFIG} app gitpod -n gitpod | grep gitpod | awk '{print $$2}'
+check-kots-app:
+ kubectl kots get --kubeconfig=${KUBECONFIG} app gitpod -n gitpod | grep gitpod | awk '{print $$2}' | grep "ready" || { echo "Gitpod is not ready"; exit 1; }
+
+check-gitpod-installation: delete-cm-setup check-kots-app check-env-sub-domain
+ @echo "Curling http://${TF_VAR_TEST_ID}.gitpod-self-hosted.com/api/version"
+ curl -i -X GET http://${TF_VAR_TEST_ID}.gitpod-self-hosted.com/api/version || { echo "Curling Gitpod endpoint failed"; exit 1; }
-.PHONY:
run-tests:
./tests.sh ${KUBECONFIG}
@@ -91,6 +101,7 @@ cleanup:
terraform workspace select $(TF_VAR_TEST_ID)
which ${KUBECONFIG} && terraform destroy -target=module.tools -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file"
terraform destroy -target=module.gke -var kubeconfig=${KUBECONFIG} --auto-approve
+ terraform destroy -target=module.aks -var kubeconfig=${KUBECONFIG} --auto-approve
terraform destroy -target=module.k3s -var kubeconfig=${KUBECONFIG} --auto-approve
get-results:
diff --git a/install/tests/main.tf b/install/tests/main.tf
index 3e4a691e2033c8..b09d10aaba5df7 100644
--- a/install/tests/main.tf
+++ b/install/tests/main.tf
@@ -1,43 +1,75 @@
-variable "kubeconfig" {}
-variable "TEST_ID" {
- default = "nightly"
-}
-variable "project" {
- default = "sh-automated-tests"
-}
-variable "sa_creds" {}
-variable "dns_sa_creds" {}
+variable "kubeconfig" { }
+variable "TEST_ID" { default = "nightly" }
+# We store the state always in a GCS bucket
terraform {
backend "gcs" {
- bucket = "nightly-tests"
- prefix = "tf-state"
+ bucket = "nightly-tests"
+ prefix = "tf-state"
}
}
+variable "project" { default = "sh-automated-tests" }
+variable "sa_creds" { default = "/workspace/gcp.json" }
+variable "dns_sa_creds" {default = "/workspace/gitpod/gcp.json" }
+
module "gke" {
- source = "github.com/gitpod-io/gitpod//install/infra/terraform/gke?ref=nvn-infra-tf" # we can later use tags here
+ source = "github.com/gitpod-io/gitpod//install/infra/terraform/gke?ref=nvn-infra-tf" # we can later use tags here
- name = var.TEST_ID
- project = var.project
- credentials = var.sa_creds
- kubeconfig = var.kubeconfig
+ name = var.TEST_ID
+ project = var.project
+ credentials = var.sa_creds
+ kubeconfig = var.kubeconfig
}
module "k3s" {
- source = "github.com/gitpod-io/gitpod//install/infra/terraform/k3s?ref=nvn-infra-tf" # we can later use tags here
+ source = "github.com/gitpod-io/gitpod//install/infra/terraform/k3s?ref=nvn-infra-aks" # we can later use tags here
- name = var.TEST_ID
- gcp_project = var.project
- credentials = var.sa_creds
- kubeconfig = var.kubeconfig
+ name = var.TEST_ID
+ gcp_project = var.project
+ credentials = var.sa_creds
+ kubeconfig = var.kubeconfig
+ dns_sa_creds = var.dns_sa_creds
+ dns_project = "dns-for-playgrounds"
+ managed_dns_zone = "gitpod-self-hosted-com"
+ domain_name = "${var.TEST_ID}.gitpod-self-hosted.com"
+}
+
+module "aks" {
+ source = "github.com/gitpod-io/gitpod//install/infra/terraform/aks?ref=nvn-infra-aks" # we can later use tags here
+
+ domain_name = false
+ enable_airgapped = false
+ enable_external_database = false
+ enable_external_registry = false
+ enable_external_storage = false
+ dns_enabled = false
+ name_format = join("-", [
+ "gitpod",
+ "%s", # region
+ "%s", # name
+ var.TEST_ID
+ ])
+ name_format_global = join("-", [
+ "gitpod",
+ "%s", # name
+ var.TEST_ID
+ ])
+ workspace_name = var.TEST_ID
+ labels = {
+ "gitpod.io/workload_meta" = true
+ "gitpod.io/workload_ide" = true
+ "gitpod.io/workload_workspace_services" = true
+ "gitpod.io/workload_workspace_regular" = true
+ "gitpod.io/workload_workspace_headless" = true
+ }
}
// this module is intended to be run separately from the above two. so a separate target for apply is necessary
module "tools" {
- source = "github.com/gitpod-io/gitpod//install/infra/terraform/tools/cm-cloud-dns?ref=nvn-infra-tf" # we can later use tags here
+ source = "github.com/gitpod-io/gitpod//install/infra/terraform/tools/cm-cloud-dns?ref=nvn-infra-tf" # we can later use tags here
- kubeconfig = var.kubeconfig
- credentials = var.dns_sa_creds
- gcp_sub_domain = var.TEST_ID
+ kubeconfig = var.kubeconfig
+ credentials = var.dns_sa_creds
+ gcp_sub_domain = var.TEST_ID
}