diff --git a/install/infra/modules/eks/kubernetes.tf b/install/infra/modules/eks/kubernetes.tf index 4c3f22fc0113e8..950534dc8b6ac7 100644 --- a/install/infra/modules/eks/kubernetes.tf +++ b/install/infra/modules/eks/kubernetes.tf @@ -219,3 +219,36 @@ resource "null_resource" "kubeconfig" { create_before_destroy = true } } + +data "aws_iam_policy_document" "eks_policy" { + statement { + actions = [ + "eks:DescribeCluster", + "eks:ListClusters" + ] + resources = [ + "*", + ] + effect = "Allow" + } +} + +resource "aws_iam_policy" "eks_policy" { + name = "eks-policy-${var.cluster_name}" + description = "Gitpod ${var.cluster_name} EKS cluster access bucket policy" + policy = data.aws_iam_policy_document.eks_policy.json +} + +resource "aws_iam_user" "eks_user" { + force_destroy = true + name = "eks-user-${var.cluster_name}" +} + +resource "aws_iam_user_policy_attachment" "eks_attachment" { + user = aws_iam_user.eks_user.name + policy_arn = aws_iam_policy.eks_policy.arn +} + +resource "aws_iam_access_key" "eks_user_key" { + user = aws_iam_user.eks_user.name +} diff --git a/install/infra/modules/eks/output.tf b/install/infra/modules/eks/output.tf index 6c7815eaf2544d..75bedb45a8a65d 100644 --- a/install/infra/modules/eks/output.tf +++ b/install/infra/modules/eks/output.tf @@ -91,3 +91,13 @@ output "registry_backend" { secret_access_key = aws_iam_access_key.bucket_registry_user[0].secret }, "No s3 bucket created for registry backend.") } + +output "cluster_user" { + sensitive = true + value = { + userarn = aws_iam_user.eks_user.arn + name = aws_iam_user.eks_user.name + access_key_id = aws_iam_access_key.eks_user_key.id + secret_access_key = aws_iam_access_key.eks_user_key.secret + } +} diff --git a/install/infra/modules/gke/cluster.tf b/install/infra/modules/gke/cluster.tf index a0f89054c88501..b69fe917e2ccf3 100644 --- a/install/infra/modules/gke/cluster.tf +++ b/install/infra/modules/gke/cluster.tf @@ -149,3 +149,18 @@ resource "local_file" "kubeconfig" { filename = var.kubeconfig content = module.gke_auth.kubeconfig_raw } + +resource "google_service_account" "cluster_user_sa" { + account_id = local.gke_user_sa + display_name = "Gitpod Service Account managed by TF for GKE cluster user" +} + +resource "google_project_iam_member" "gke-user-sa-iam" { + project = var.project + role = "roles/container.developer" + member = "serviceAccount:${google_service_account.cluster_user_sa.email}" +} + +resource "google_service_account_key" "gke_sa_key" { + service_account_id = google_service_account.cluster_user_sa.name +} diff --git a/install/infra/modules/gke/database.tf b/install/infra/modules/gke/database.tf index 3065eed368f13a..8981ccffe06e4a 100644 --- a/install/infra/modules/gke/database.tf +++ b/install/infra/modules/gke/database.tf @@ -41,8 +41,7 @@ resource "random_password" "password" { count = var.enable_external_database ? 1 : 0 length = 16 - special = true - override_special = "!#$%&*()-_=+[]{}<>:?" + special = false } resource "google_sql_database" "database" { diff --git a/install/infra/modules/gke/local.tf b/install/infra/modules/gke/local.tf index 22d2861fd9ec85..c350f203731700 100644 --- a/install/infra/modules/gke/local.tf +++ b/install/infra/modules/gke/local.tf @@ -7,6 +7,8 @@ locals { "roles/container.admin" ]) + gke_user_sa = "user-${var.cluster_name}" + obj_sa = "obj-sa-${var.cluster_name}" obj_iam_roles = var.enable_external_registry ? toset([ "roles/storage.admin", diff --git a/install/infra/modules/gke/output.tf b/install/infra/modules/gke/output.tf index 0bd8f6ec582a8e..0d7a32dd69e736 100644 --- a/install/infra/modules/gke/output.tf +++ b/install/infra/modules/gke/output.tf @@ -22,6 +22,11 @@ output "kubeconfig" { value = module.gke_auth.kubeconfig_raw } +output "cluster-sa" { + sensitive = true + value = google_service_account_key.gke_sa_key.private_key +} + output "database" { sensitive = true value = try({ diff --git a/install/tests/Makefile b/install/tests/Makefile index 5122a7616e2e79..4871d0f4e02c57 100644 --- a/install/tests/Makefile +++ b/install/tests/Makefile @@ -33,12 +33,27 @@ help: Makefile @sed -n 's/^##//p' $< | column -t -s ':' | sed -e 's/^/ /' @echo +upload-gcp-cluster-creds: + export GKE_CREDS=$$(terraform output -json gke_user_key) && \ + echo $$GKE_CREDS > gcp-creds + gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS} --project=sh-automated-tests + gsutil cp gcp-creds gs://nightly-tests/tf-state/${TF_VAR_TEST_ID}-creds + +download-cluster-creds: + [[ -z $$TF_VAR_sa_creds ]] || gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS} --project=sh-automated-tests + gcloud config set project sh-automated-tests + [[ -n $$TF_VAR_sa_creds ]] || gsutil cp gs://nightly-tests/tf-state/${TF_VAR_TEST_ID}-creds gcs-creds + [[ -f gcs-creds ]] && cat gcs-creds | tr -d '"' | base64 -d > ${TF_VAR_TEST_ID}-key.json || echo "No GCP credentials" + rm -f gcs-creds + [[ -f ${TF_VAR_TEST_ID}-key.json ]] || cp ${GOOGLE_APPLICATION_CREDENTIALS} ${TF_VAR_TEST_ID}-key.json + upload-kubeconfig-to-gcp: gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS} --project=sh-automated-tests gsutil cp ${KUBECONFIG} gs://nightly-tests/tf-state/${TF_VAR_TEST_ID}-kubeconfig sync-kubeconfig: - gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS} --project=sh-automated-tests + [[ -z $$TF_VAR_sa_creds ]] || gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS} --project=sh-automated-tests + gcloud config set project sh-automated-tests gsutil cp gs://nightly-tests/tf-state/${TF_VAR_TEST_ID}-kubeconfig ${KUBECONFIG} || echo "No kubeconfig" ## k3s-kubeconfig: Get the kubeconfig configuration for GCP K3s @@ -46,22 +61,27 @@ k3s-kubeconfig: sync-kubeconfig ## gcp-kubeconfig: Get the kubeconfig configuration for GCP GKE gcp-kubeconfig: - gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS} --project=sh-automated-tests + $(MAKE) download-cluster-creds + gcloud auth activate-service-account --key-file=${TF_VAR_TEST_ID}-key.json --project=sh-automated-tests || { echo "Count not authenicate the service account"; exit 1; } export KUBECONFIG=${KUBECONFIG} && \ - gcloud container clusters get-credentials gp-${TF_VAR_TEST_ID} --zone europe-west1-d --project sh-automated-tests || $(MAKE) sync-kubeconfig || echo "No cluster present" + gcloud container clusters get-credentials gp-${TF_VAR_TEST_ID} --zone europe-west1-d --project sh-automated-tests || echo "No cluster present" + rm -f ${TF_VAR_TEST_ID}-key.json ## azure-kubeconfig: Get the kubeconfig configuration for Azure AKS azure-kubeconfig: - az login --service-principal -u $$ARM_CLIENT_ID -p $$ARM_CLIENT_SECRET --tenant $$ARM_TENANT_ID + [[ -n "$$ARM_CLIENT_SECRET" ]] && az login --service-principal -u $$ARM_CLIENT_ID -p $$ARM_CLIENT_SECRET --tenant $$ARM_TENANT_ID || { echo "Please login to azure using az login command"; exit 1; } export KUBECONFIG=${KUBECONFIG} && \ az aks get-credentials --name p$$TF_VAR_TEST_ID-cluster --resource-group p$$TF_VAR_TEST_ID --file ${KUBECONFIG} || echo "No cluster present" ## aws-kubeconfig: Get the kubeconfig configuration for AWS EKS aws-kubeconfig: - export KUBECONFIG=${KUBECONFIG} && \ + [[ -z $$TF_VAR_sa_creds ]] || gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS} --project=sh-automated-tests + gcloud config set project sh-automated-tests + [[ -n $$TF_VAR_sa_creds ]] || gsutil cp gs://nightly-tests/tf-state/${TF_VAR_TEST_ID}-creds ${TF_VAR_TEST_ID}-creds + [[ -f ${TF_VAR_TEST_ID}-creds ]] || touch ${TF_VAR_TEST_ID}-creds + source ${TF_VAR_TEST_ID}-creds; \ aws eks update-kubeconfig --name ${TF_VAR_TEST_ID} --region eu-west-1 --kubeconfig ${KUBECONFIG} || echo "No cluster present" - .PHONY: ## gke-standard-cluster: Creates a zonal GKE cluster gke-standard-cluster: check-env-cluster-version @@ -70,8 +90,22 @@ gke-standard-cluster: check-env-cluster-version rm -f ${KUBECONFIG} && \ $(MAKE) get-kubeconfig && \ [[ -f ${KUBECONFIG} ]] || terraform apply -target=module.gke -var kubeconfig=${KUBECONFIG} --auto-approve + $(MAKE) upload-gcp-cluster-creds @echo "Done creating GKE cluster" +upload-eks-user: + export AWS_CLUSTER_USER=$$(terraform output -json aws_cluster_user) && \ + export USERARN=$$(echo $$AWS_CLUSTER_USER | yq r - 'userarn') && \ + export NAME=$$(echo $$AWS_CLUSTER_USER | yq r - 'name') && \ + envsubst < ./manifests/aws-auth.yaml > tmp-aws-auth.yaml && \ + echo "export AWS_SECRET_ACCESS_KEY=$$(echo $$AWS_CLUSTER_USER | yq r - 'secret_access_key')" > ${TF_VAR_TEST_ID}-creds && \ + echo "export AWS_ACCESS_KEY_ID=$$(echo $$AWS_CLUSTER_USER | yq r - 'access_key_id')" >> ${TF_VAR_TEST_ID}-creds && \ + kubectl --kubeconfig=${KUBECONFIG} get configmap -n kube-system aws-auth -o yaml | grep -v "creationTimestamp\|resourceVersion\|selfLink\|uid" | sed '/^ annotations:/,+2 d' > /tmp/aws-auth.yaml + yq m --inplace /tmp/aws-auth.yaml tmp-aws-auth.yaml + gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS} --project=sh-automated-tests + gsutil cp ${TF_VAR_TEST_ID}-creds gs://nightly-tests/tf-state/${TF_VAR_TEST_ID}-creds + kubectl --kubeconfig=${KUBECONFIG} replace -f /tmp/aws-auth.yaml + ami_id_121 := "ami-060637af2651bc8bb" ami_id_122 := "ami-0733d755ed2c97a4d" @@ -87,6 +121,7 @@ eks-standard-cluster: check-env-cluster-version rm -f ${KUBECONFIG} && \ $(MAKE) get-kubeconfig && \ [[ -f ${KUBECONFIG} ]] || terraform apply -target=module.eks -var kubeconfig=${KUBECONFIG} -var eks_node_image_id=${ami_id} --auto-approve + $(MAKE) upload-eks-user @echo "Done creating EKS cluster" .PHONY: @@ -159,8 +194,10 @@ external-dns: check-env-cloud select-workspace .PHONY: ## get-kubeconfig: Returns KUBECONFIG of a just created cluster -get-kubeconfig: ${cloud}-kubeconfig - +get-kubeconfig: + echo "Getting kubeconfig for $$TF_VAR_TEST_ID terraform state" && \ + export provider=$$(echo "$$TF_VAR_TEST_ID" | sed 's/\(.*\)-/\1 /' | xargs | awk '{print $$2}') && \ + $(MAKE) $$provider-kubeconfig && echo "kubeconfig written to ${KUBECONFIG}" get-github-config: ifneq ($(GITHUB_SCM_OAUTH),) @@ -212,8 +249,8 @@ registry-config-azure: yq m -i tmp_config.yml tmp_2_config.yml storage-config-azure: - export PASSWORD=$$(terraform output -json azure_storage | yq r - 'account_name') && \ - export USERNAME=$$(terraform output -json azure_storage | yq r - 'account_key') && \ + export USERNAME=$$(terraform output -json azure_storage | yq r - 'account_name') && \ + export PASSWORD=$$(terraform output -json azure_storage | yq r - 'account_key') && \ export REGION=$$(terraform output -json azure_storage | yq r - 'storage_region') && \ envsubst < ./manifests/kots-config-azure-storage.yaml > tmp_2_config.yml yq m -i tmp_config.yml tmp_2_config.yml @@ -388,7 +425,7 @@ kots-upgrade: kubectl kots upstream upgrade --kubeconfig=${KUBECONFIG} gitpod -n gitpod --deploy cloud ?= cluster -cleanup: $(cloud)-kubeconfig destroy-gitpod tf-init destroy-$(cloud) destroy-workspace destroy-kubeconfig +cleanup: get-kubeconfig destroy-gitpod tf-init destroy-$(cloud) destroy-workspace destroy-kubeconfig cluster-kubeconfig: azure-kubeconfig aws-kubeconfig k3s-kubeconfig gcp-kubeconfig @@ -400,6 +437,7 @@ destroy-cluster: destroy-gcp destroy-aws destroy-azure destroy-kubeconfig: gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS} --project=sh-automated-tests gsutil rm gs://nightly-tests/tf-state/${TF_VAR_TEST_ID}-kubeconfig || echo "No kubeconfig" + gsutil rm gs://nightly-tests/tf-state/${TF_VAR_TEST_ID}-creds || echo "No credentials file" rm ${KUBECONFIG} || echo "No kubeconfig" select-workspace: diff --git a/install/tests/cleanup.sh b/install/tests/cleanup.sh index c3a93207e349a8..ed022473de1bd5 100755 --- a/install/tests/cleanup.sh +++ b/install/tests/cleanup.sh @@ -19,6 +19,7 @@ for i in $(gsutil ls gs://nightly-tests/tf-state); do [ -z "$filename" ] && continue if [[ "$filename" == *-kubeconfig ]]; then continue; fi + if [[ "$filename" == *-creds ]]; then continue; fi TF_VAR_TEST_ID=$(basename "$filename" .tfstate) diff --git a/install/tests/manifests/aws-auth.yaml b/install/tests/manifests/aws-auth.yaml new file mode 100644 index 00000000000000..cdb5a5a2672529 --- /dev/null +++ b/install/tests/manifests/aws-auth.yaml @@ -0,0 +1,6 @@ +data: + mapUsers: | + - userarn: ${USERARN} + username: ${NAME} + groups: + - system:masters diff --git a/install/tests/output.tf b/install/tests/output.tf index 4a85c3e8cbf8ed..9adf01258031dc 100644 --- a/install/tests/output.tf +++ b/install/tests/output.tf @@ -3,11 +3,21 @@ output "gke_database" { value = try(module.gke.database, null) } +output "gke_user_key" { + sensitive = true + value = try(module.gke.cluster-sa, null) +} + output "k3s_database" { sensitive = true value = try(module.k3s.database, null) } +output "aws_cluster_user" { + sensitive = true + value = try(module.eks.cluster_user, null) +} + output "aws_storage" { sensitive = true value = try(module.eks.storage, null)