From a93105861e5bfb25b83f9fb374f9dc383297c69f Mon Sep 17 00:00:00 2001 From: Carlos Santana Date: Sat, 28 Oct 2023 18:14:07 -0400 Subject: [PATCH 01/16] add getting-started Signed-off-by: Carlos Santana --- .../examples/eks/getting-started/README.md | 174 ++++++++++++ .../eks/getting-started/bootstrap/addons.yaml | 26 ++ .../getting-started/bootstrap/workloads.yaml | 34 +++ .../examples/eks/getting-started/destroy.sh | 27 ++ .../eks/getting-started/k8s/game-2048.yaml | 62 +++++ .../examples/eks/getting-started/main.tf | 250 ++++++++++++++++++ .../examples/eks/getting-started/outputs.tf | 33 +++ .../static/gitops-bridge.drawio | 1 + .../static/gitops-bridge.drawio.png | Bin 0 -> 89536 bytes .../examples/eks/getting-started/variables.tf | 76 ++++++ .../examples/eks/getting-started/versions.tf | 25 ++ 11 files changed, 708 insertions(+) create mode 100644 argocd/iac/terraform/examples/eks/getting-started/README.md create mode 100644 argocd/iac/terraform/examples/eks/getting-started/bootstrap/addons.yaml create mode 100644 argocd/iac/terraform/examples/eks/getting-started/bootstrap/workloads.yaml create mode 100755 argocd/iac/terraform/examples/eks/getting-started/destroy.sh create mode 100644 argocd/iac/terraform/examples/eks/getting-started/k8s/game-2048.yaml create mode 100644 argocd/iac/terraform/examples/eks/getting-started/main.tf create mode 100644 argocd/iac/terraform/examples/eks/getting-started/outputs.tf create mode 100644 argocd/iac/terraform/examples/eks/getting-started/static/gitops-bridge.drawio create mode 100644 argocd/iac/terraform/examples/eks/getting-started/static/gitops-bridge.drawio.png create mode 100644 argocd/iac/terraform/examples/eks/getting-started/variables.tf create mode 100644 argocd/iac/terraform/examples/eks/getting-started/versions.tf diff --git a/argocd/iac/terraform/examples/eks/getting-started/README.md b/argocd/iac/terraform/examples/eks/getting-started/README.md new file mode 100644 index 00000000..30d4792d --- /dev/null +++ b/argocd/iac/terraform/examples/eks/getting-started/README.md @@ -0,0 +1,174 @@ +# ArgoCD on Amazon EKS + +This tutorial guides you through deploying an Amazon EKS cluster with addons configured via ArgoCD, employing the [GitOps Bridge Pattern](https://github.com/gitops-bridge-dev). + + + + +The [GitOps Bridge Pattern](https://github.com/gitops-bridge-dev) enables Kubernetes administrators to utilize Infrastructure as Code (IaC) and GitOps tools for deploying Kubernetes Addons and Workloads. Addons often depend on Cloud resources that are external to the cluster. The configuration metadata for these external resources is required by the Addons' Helm charts. While IaC is used to create these cloud resources, it is not used to install the Helm charts. Instead, the IaC tool stores this metadata either within GitOps resources in the cluster or in a Git repository. The GitOps tool then extracts these metadata values and passes them to the Helm chart during the Addon installation process. This mechanism forms the bridge between IaC and GitOps, hence the term "GitOps Bridge." + +Aditonal examples available on the [GitOps Bridge Pattern](https://github.com/gitops-bridge-dev): +- [argocd-ingress](https://github.com/gitops-bridge-dev/gitops-bridge/tree/main/argocd/iac/terraform/examples/eks/argocd-ingress) +- [aws-secrets-manager](https://github.com/gitops-bridge-dev/gitops-bridge/tree/main/argocd/iac/terraform/examples/eks/aws-secrets-manager) +- [crossplane](https://github.com/gitops-bridge-dev/gitops-bridge/tree/main/argocd/iac/terraform/examples/eks/crossplane) +- [external-secrets](https://github.com/gitops-bridge-dev/gitops-bridge/tree/main/argocd/iac/terraform/examples/eks/external-secrets) +- [multi-cluster/distributed](https://github.com/gitops-bridge-dev/gitops-bridge/tree/main/argocd/iac/terraform/examples/eks/multi-cluster/distributed) +- [multi-cluster/hub-spoke](https://github.com/gitops-bridge-dev/gitops-bridge/tree/main/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke) +- [multi-cluster/hub-spoke-shared](https://github.com/gitops-bridge-dev/gitops-bridge/tree/main/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared) +- [private-git](https://github.com/gitops-bridge-dev/gitops-bridge/tree/main/argocd/iac/terraform/examples/eks/private-git) + + + +## Prerequisites +Before you begin, make sure you have the following command line tools installed: +- git +- terraform +- kubectl +- argocd + +## Fork the Git Repositories + +### Fork the Addon GitOps Repo +1. Fork the git repository for addons [here](https://github.com/gitops-bridge-dev/gitops-bridge-argocd-control-plane-template). +2. Update the following environment variables to point to your fork by changing the default values: +```shell +export TF_VAR_gitops_addons_org=https://github.com/gitops-bridge-dev +export TF_VAR_gitops_addons_repo=gitops-bridge-argocd-control-plane-template +``` + +### Fork the Workloads GitOps Repo +1. Fork the git repository for this pattern [here](https://github.com/gitops-bridge-dev/gitops-bridge) +2. Update the following environment variables to point to your fork by changing the default values: +```shell +export TF_VAR_gitops_workload_org=https://github.com/gitops-bridge-dev +export TF_VAR_gitops_workload_repo=gitops-bridge +``` + +## Deploy the EKS Cluster +Initialize Terraform and deploy the EKS cluster: +```shell +terraform init +terraform apply -auto-approve +``` +Retrieve `kubectl` config, then execute the output command: +```shell +terraform output -raw configure_kubectl +``` + +Terraform will add GitOps Bridge Metadata to the ArgoCD secret. +The annotations contain metadata for the addons' Helm charts and ArgoCD ApplicationSets. +```shell +kubectl get secret -n argocd -l argocd.argoproj.io/secret-type=cluster -o json | jq '.items[0].metadata.annotations' +``` +The output looks like the following: +```json +{ + "addons_repo_basepath": "", + "addons_repo_path": "bootstrap/control-plane/addons", + "addons_repo_revision": "main", + "addons_repo_url": "https://github.com/gitops-bridge-dev/gitops-bridge-argocd-control-plane-template", + "aws_account_id": "0123456789", + "aws_cluster_name": "getting-started-gitops", + "aws_load_balancer_controller_iam_role_arn": "arn:aws:iam::0123456789:role/alb-controller", + "aws_load_balancer_controller_namespace": "kube-system", + "aws_load_balancer_controller_service_account": "aws-load-balancer-controller-sa", + "aws_region": "us-west-2", + "aws_vpc_id": "vpc-001d3f00151bbb731", + "cluster_name": "in-cluster", + "environment": "dev", + "workload_repo_basepath": "argocd/iac/terraform/examples/eks/", + "workload_repo_path": "getting-started/k8s", + "workload_repo_revision": "main", + "workload_repo_url": "https://github.com/gitops-bridge-dev/gitops-bridge" +} +``` +The labels offer a straightforward way to enable or disable an addon in ArgoCD for the cluster. +```shell +kubectl get secret -n argocd -l argocd.argoproj.io/secret-type=cluster -o json | jq '.items[0].metadata.labels' +``` +The output looks like the following: +```json +{ + "aws_cluster_name": "getting-started-gitops", + "enable_argocd": "true", + "enable_aws_load_balancer_controller": "true", + "enable_metrics_server": "true", + "kubernetes_version": "1.28", +} +``` + +## Deploy the Addons +Bootstrap the addons using ArgoCD: +```shell +kubectl apply -f bootstrap/addons.yaml +``` + +### Monitor GitOps Progress for Addons +Wait until all the ArgoCD applications' `HEALTH STATUS` is `Healthy`. Use Crl+C to exit the `watch` command +```shell +watch kubectl get applications -n argocd +``` + +## Access ArgoCD +Access ArgoCD's UI, run the command from the output: +```shell +terraform output -raw access_argocd +``` + +### Verify the Addons +Verify that the addons are ready: +```shell +kubectl get deployment -n kube-system \ + aws-load-balancer-controller \ + metrics-server +``` + + + +## Deploy the Workloads +Deploy a sample application located in [k8s/game-2048.yaml](k8s/game-2048.yaml) using ArgoCD: +```shell +kubectl apply -f bootstrap/workloads.yaml +``` + +### Monitor GitOps Progress for Workloads +Watch until the Workloads ArgoCD Application is `Healthy` +```shell +watch kubectl get -n argocd applications workloads +``` +Wait until the ArgoCD Applications `HEALTH STATUS` is `Healthy`. Crl+C to exit the `watch` command + +### Verify the Application +Verify that the application configuration is present and the pod is running: +```shell +kubectl get -n game-2048 deployments,service,ep,ingress +``` +Wait until the Ingress/game-2048 `MESSAGE` column value is `Successfully reconciled`. Crl+C to exit the `watch` command +```shell +kubectl events -n game-2048 --for ingress/game-2048 --watch +``` + + + +### Access the Application using AWS Load Balancer +Retrieve the ingress URL for the application: +```shell +echo "Application URL: http://$(kubectl get -n game-2048 ingress game-2048 -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')" +``` +Verify the application enpoint health using `curl`: +```shell +curl -I $(kubectl get -n game-2048 ingress game-2048 -o jsonpath='{.status.loadBalancer.ingress[0].hostname}') +``` +The first line of the output should have `HTTP/1.1 200 OK`. + +### Container Metrics +Check the application's CPU and memory metrics: +```shell +kubectl top pods -n game-2048 +``` + +## Destroy the EKS Cluster +To tear down all the resources and the EKS cluster, run the following command: +```shell +./destroy.sh +``` diff --git a/argocd/iac/terraform/examples/eks/getting-started/bootstrap/addons.yaml b/argocd/iac/terraform/examples/eks/getting-started/bootstrap/addons.yaml new file mode 100644 index 00000000..e867a1c8 --- /dev/null +++ b/argocd/iac/terraform/examples/eks/getting-started/bootstrap/addons.yaml @@ -0,0 +1,26 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: cluster-addons + namespace: argocd +spec: + syncPolicy: + preserveResourcesOnDeletion: true + generators: + - clusters: {} + template: + metadata: + name: cluster-addons + spec: + project: default + source: + repoURL: '{{metadata.annotations.addons_repo_url}}' + path: '{{metadata.annotations.addons_repo_basepath}}{{metadata.annotations.addons_repo_path}}' + targetRevision: '{{metadata.annotations.addons_repo_revision}}' + directory: + recurse: true + destination: + namespace: argocd + name: '{{name}}' + syncPolicy: + automated: {} diff --git a/argocd/iac/terraform/examples/eks/getting-started/bootstrap/workloads.yaml b/argocd/iac/terraform/examples/eks/getting-started/bootstrap/workloads.yaml new file mode 100644 index 00000000..8c883ea3 --- /dev/null +++ b/argocd/iac/terraform/examples/eks/getting-started/bootstrap/workloads.yaml @@ -0,0 +1,34 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: workloads + namespace: argocd +spec: + syncPolicy: + preserveResourcesOnDeletion: true + generators: + - clusters: {} + template: + metadata: + name: workloads + finalizers: + # This finalizer is for demo purposes, in production remove apps using argocd CLI "argocd app delete workload --cascade" + # When you invoke argocd app delete with --cascade, the finalizer is added automatically. + - resources-finalizer.argocd.argoproj.io + spec: + project: default + source: + repoURL: '{{metadata.annotations.workload_repo_url}}' + path: '{{metadata.annotations.workload_repo_basepath}}{{metadata.annotations.workload_repo_path}}' + targetRevision: '{{metadata.annotations.workload_repo_revision}}' + destination: + name: '{{name}}' + syncPolicy: + automated: + allowEmpty: true + syncOptions: + - CreateNamespace=true + retry: + backoff: + duration: 1m + limit: 60 diff --git a/argocd/iac/terraform/examples/eks/getting-started/destroy.sh b/argocd/iac/terraform/examples/eks/getting-started/destroy.sh new file mode 100755 index 00000000..79d24cd4 --- /dev/null +++ b/argocd/iac/terraform/examples/eks/getting-started/destroy.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -uo pipefail + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +ROOTDIR="$(cd ${SCRIPTDIR}/../..; pwd )" +[[ -n "${DEBUG:-}" ]] && set -x + +# Delete the Ingress/SVC before removing the addons +TMPFILE=$(mktemp) +terraform -chdir=$SCRIPTDIR output -raw configure_kubectl > "$TMPFILE" +# check if TMPFILE contains the string "No outputs found" +if [[ ! $(cat $TMPFILE) == *"No outputs found"* ]]; then + source "$TMPFILE" + kubectl delete -n argocd applicationset workloads + echo "Waiting for ingress and load balancer to be deleted" + sleep 120 + kubectl delete -n argocd applicationset cluster-addons + kubectl delete -n argocd applicationset addons-argocd + kubectl delete -n argocd svc argo-cd-argocd-server +fi + +terraform destroy -target="module.gitops_bridge_bootstrap" -auto-approve +terraform destroy -target="module.eks_blueprints_addons" -auto-approve +terraform destroy -target="module.eks" -auto-approve +terraform destroy -target="module.vpc" -auto-approve +terraform destroy -auto-approve diff --git a/argocd/iac/terraform/examples/eks/getting-started/k8s/game-2048.yaml b/argocd/iac/terraform/examples/eks/getting-started/k8s/game-2048.yaml new file mode 100644 index 00000000..44e2d12e --- /dev/null +++ b/argocd/iac/terraform/examples/eks/getting-started/k8s/game-2048.yaml @@ -0,0 +1,62 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: game-2048 +spec: {} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + namespace: game-2048 + name: game-2048 +spec: + selector: + matchLabels: + app.kubernetes.io/name: game-2048 + template: + metadata: + labels: + app.kubernetes.io/name: game-2048 + spec: + containers: + - image: public.ecr.aws/l6m2t8p7/docker-2048 + name: game-2048 + ports: + - containerPort: 80 + name: http +--- +apiVersion: v1 +kind: Service +metadata: + namespace: game-2048 + name: game-2048 +spec: + ports: + - name: http + port: 80 + targetPort: http + protocol: TCP + type: ClusterIP + selector: + app.kubernetes.io/name: game-2048 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + namespace: game-2048 + name: game-2048 + annotations: + alb.ingress.kubernetes.io/scheme: internet-facing + alb.ingress.kubernetes.io/target-type: ip +spec: + ingressClassName: alb + rules: + - http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: game-2048 + port: + name: http \ No newline at end of file diff --git a/argocd/iac/terraform/examples/eks/getting-started/main.tf b/argocd/iac/terraform/examples/eks/getting-started/main.tf new file mode 100644 index 00000000..0defa86f --- /dev/null +++ b/argocd/iac/terraform/examples/eks/getting-started/main.tf @@ -0,0 +1,250 @@ +provider "aws" { + region = local.region +} +data "aws_caller_identity" "current" {} +data "aws_availability_zones" "available" {} + +provider "helm" { + kubernetes { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + # This requires the awscli to be installed locally where Terraform is executed + args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name, "--region", local.region] + } + } +} + +provider "kubernetes" { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + # This requires the awscli to be installed locally where Terraform is executed + args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name, "--region", local.region] + } +} + +locals { + name = "getting-started" + region = var.region + + cluster_version = var.kubernetes_version + + vpc_cidr = var.vpc_cidr + azs = slice(data.aws_availability_zones.available.names, 0, 3) + + gitops_addons_url = "${var.gitops_addons_org}/${var.gitops_addons_repo}" + gitops_addons_basepath = var.gitops_addons_basepath + gitops_addons_path = var.gitops_addons_path + gitops_addons_revision = var.gitops_addons_revision + + gitops_workload_url = "${var.gitops_workload_org}/${var.gitops_workload_repo}" + gitops_workload_basepath = var.gitops_workload_basepath + gitops_workload_path = var.gitops_workload_path + gitops_workload_revision = var.gitops_workload_revision + + aws_addons = { + enable_cert_manager = try(var.addons.enable_cert_manager, false) + enable_aws_efs_csi_driver = try(var.addons.enable_aws_efs_csi_driver, false) + enable_aws_fsx_csi_driver = try(var.addons.enable_aws_fsx_csi_driver, false) + enable_aws_cloudwatch_metrics = try(var.addons.enable_aws_cloudwatch_metrics, false) + enable_aws_privateca_issuer = try(var.addons.enable_aws_privateca_issuer, false) + enable_cluster_autoscaler = try(var.addons.enable_cluster_autoscaler, false) + enable_external_dns = try(var.addons.enable_external_dns, false) + enable_external_secrets = try(var.addons.enable_external_secrets, false) + enable_aws_load_balancer_controller = try(var.addons.enable_aws_load_balancer_controller, false) + enable_fargate_fluentbit = try(var.addons.enable_fargate_fluentbit, false) + enable_aws_for_fluentbit = try(var.addons.enable_aws_for_fluentbit, false) + enable_aws_node_termination_handler = try(var.addons.enable_aws_node_termination_handler, false) + enable_karpenter = try(var.addons.enable_karpenter, false) + enable_velero = try(var.addons.enable_velero, false) + enable_aws_gateway_api_controller = try(var.addons.enable_aws_gateway_api_controller, false) + enable_aws_ebs_csi_resources = try(var.addons.enable_aws_ebs_csi_resources, false) + enable_aws_secrets_store_csi_driver_provider = try(var.addons.enable_aws_secrets_store_csi_driver_provider, false) + enable_ack_apigatewayv2 = try(var.addons.enable_ack_apigatewayv2, false) + enable_ack_dynamodb = try(var.addons.enable_ack_dynamodb, false) + enable_ack_s3 = try(var.addons.enable_ack_s3, false) + enable_ack_rds = try(var.addons.enable_ack_rds, false) + enable_ack_prometheusservice = try(var.addons.enable_ack_prometheusservice, false) + enable_ack_emrcontainers = try(var.addons.enable_ack_emrcontainers, false) + enable_ack_sfn = try(var.addons.enable_ack_sfn, false) + enable_ack_eventbridge = try(var.addons.enable_ack_eventbridge, false) + } + oss_addons = { + enable_argocd = try(var.addons.enable_argocd, true) + enable_argo_rollouts = try(var.addons.enable_argo_rollouts, false) + enable_argo_events = try(var.addons.enable_argo_events, false) + enable_argo_workflows = try(var.addons.enable_argo_workflows, false) + enable_cluster_proportional_autoscaler = try(var.addons.enable_cluster_proportional_autoscaler, false) + enable_gatekeeper = try(var.addons.enable_gatekeeper, false) + enable_gpu_operator = try(var.addons.enable_gpu_operator, false) + enable_ingress_nginx = try(var.addons.enable_ingress_nginx, false) + enable_kyverno = try(var.addons.enable_kyverno, false) + enable_kube_prometheus_stack = try(var.addons.enable_kube_prometheus_stack, false) + enable_metrics_server = try(var.addons.enable_metrics_server, false) + enable_prometheus_adapter = try(var.addons.enable_prometheus_adapter, false) + enable_secrets_store_csi_driver = try(var.addons.enable_secrets_store_csi_driver, false) + enable_vpa = try(var.addons.enable_vpa, false) + } + addons = merge( + local.aws_addons, + local.oss_addons, + { kubernetes_version = local.cluster_version }, + { aws_cluster_name = module.eks.cluster_name } + ) + + addons_metadata = merge( + module.eks_blueprints_addons.gitops_metadata, + { + aws_cluster_name = module.eks.cluster_name + aws_region = local.region + aws_account_id = data.aws_caller_identity.current.account_id + aws_vpc_id = module.vpc.vpc_id + }, + { + addons_repo_url = local.gitops_addons_url + addons_repo_basepath = local.gitops_addons_basepath + addons_repo_path = local.gitops_addons_path + addons_repo_revision = local.gitops_addons_revision + }, + { + workload_repo_url = local.gitops_workload_url + workload_repo_basepath = local.gitops_workload_basepath + workload_repo_path = local.gitops_workload_path + workload_repo_revision = local.gitops_workload_revision + } + ) + + tags = { + Blueprint = local.name + GithubRepo = "github.com/aws-ia/terraform-aws-eks-blueprints" + } +} + +################################################################################ +# GitOps Bridge: Bootstrap +################################################################################ +module "gitops_bridge_bootstrap" { + source = "github.com/gitops-bridge-dev/gitops-bridge-argocd-bootstrap-terraform?ref=v2.0.0" + + cluster = { + metadata = local.addons_metadata + addons = local.addons + } +} + +################################################################################ +# EKS Blueprints Addons +################################################################################ +module "eks_blueprints_addons" { + source = "aws-ia/eks-blueprints-addons/aws" + version = "~> 1.0" + + cluster_name = module.eks.cluster_name + cluster_endpoint = module.eks.cluster_endpoint + cluster_version = module.eks.cluster_version + oidc_provider_arn = module.eks.oidc_provider_arn + + # Using GitOps Bridge + create_kubernetes_resources = false + + # EKS Blueprints Addons + enable_cert_manager = local.aws_addons.enable_cert_manager + enable_aws_efs_csi_driver = local.aws_addons.enable_aws_efs_csi_driver + enable_aws_fsx_csi_driver = local.aws_addons.enable_aws_fsx_csi_driver + enable_aws_cloudwatch_metrics = local.aws_addons.enable_aws_cloudwatch_metrics + enable_aws_privateca_issuer = local.aws_addons.enable_aws_privateca_issuer + enable_cluster_autoscaler = local.aws_addons.enable_cluster_autoscaler + enable_external_dns = local.aws_addons.enable_external_dns + enable_external_secrets = local.aws_addons.enable_external_secrets + enable_aws_load_balancer_controller = local.aws_addons.enable_aws_load_balancer_controller + enable_fargate_fluentbit = local.aws_addons.enable_fargate_fluentbit + enable_aws_for_fluentbit = local.aws_addons.enable_aws_for_fluentbit + enable_aws_node_termination_handler = local.aws_addons.enable_aws_node_termination_handler + enable_karpenter = local.aws_addons.enable_karpenter + enable_velero = local.aws_addons.enable_velero + enable_aws_gateway_api_controller = local.aws_addons.enable_aws_gateway_api_controller + + tags = local.tags +} + +################################################################################ +# EKS Cluster +################################################################################ +#tfsec:ignore:aws-eks-enable-control-plane-logging +module "eks" { + source = "terraform-aws-modules/eks/aws" + version = "~> 19.13" + + cluster_name = local.name + cluster_version = local.cluster_version + cluster_endpoint_public_access = true + + + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + + eks_managed_node_groups = { + initial = { + instance_types = ["t3.medium"] + + min_size = 1 + max_size = 3 + desired_size = 2 + } + } + # EKS Addons + cluster_addons = { + coredns = {} + kube-proxy = {} + vpc-cni = { + # Specify the VPC CNI addon should be deployed before compute to ensure + # the addon is configured before data plane compute resources are created + # See README for further details + before_compute = true + most_recent = true # To ensure access to the latest settings provided + configuration_values = jsonencode({ + env = { + # Reference docs https://docs.aws.amazon.com/eks/latest/userguide/cni-increase-ip-addresses.html + ENABLE_PREFIX_DELEGATION = "true" + WARM_PREFIX_TARGET = "1" + } + }) + } + } + tags = local.tags +} + +################################################################################ +# Supporting Resources +################################################################################ +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "~> 5.0" + + name = local.name + cidr = local.vpc_cidr + + azs = local.azs + private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 4, k)] + public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 48)] + + enable_nat_gateway = true + single_nat_gateway = true + + public_subnet_tags = { + "kubernetes.io/role/elb" = 1 + } + + private_subnet_tags = { + "kubernetes.io/role/internal-elb" = 1 + } + + tags = local.tags +} diff --git a/argocd/iac/terraform/examples/eks/getting-started/outputs.tf b/argocd/iac/terraform/examples/eks/getting-started/outputs.tf new file mode 100644 index 00000000..d4ecfbf1 --- /dev/null +++ b/argocd/iac/terraform/examples/eks/getting-started/outputs.tf @@ -0,0 +1,33 @@ +output "configure_kubectl" { + description = "Configure kubectl: make sure you're logged in with the correct AWS profile and run the following command to update your kubeconfig" + value = <<-EOT + export KUBECONFIG="/tmp/${module.eks.cluster_name}" + aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name} + EOT +} + +output "configure_argocd" { + description = "Terminal Setup" + value = <<-EOT + export KUBECONFIG="/tmp/${module.eks.cluster_name}" + aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name} + export ARGOCD_OPTS="--port-forward --port-forward-namespace argocd --grpc-web" + kubectl config set-context --current --namespace argocd + argocd login --port-forward --username admin --password $(argocd admin initial-password | head -1) + echo "ArgoCD Username: admin" + echo "ArgoCD Password: $(kubectl get secrets argocd-initial-admin-secret -n argocd --template="{{index .data.password | base64decode}}")" + echo Port Forward: http://localhost:8080 + kubectl port-forward -n argocd svc/argo-cd-argocd-server 8080:80 + EOT +} + +output "access_argocd" { + description = "ArgoCD Access" + value = <<-EOT + export KUBECONFIG="/tmp/${module.eks.cluster_name}" + aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name} + echo "ArgoCD Username: admin" + echo "ArgoCD Password: $(kubectl get secrets argocd-initial-admin-secret -n argocd --template="{{index .data.password | base64decode}}")" + echo "ArgoCD URL: https://$(kubectl get svc -n argocd argo-cd-argocd-server -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')" + EOT +} diff --git a/argocd/iac/terraform/examples/eks/getting-started/static/gitops-bridge.drawio b/argocd/iac/terraform/examples/eks/getting-started/static/gitops-bridge.drawio new file mode 100644 index 00000000..51f8e7ea --- /dev/null +++ b/argocd/iac/terraform/examples/eks/getting-started/static/gitops-bridge.drawio @@ -0,0 +1 @@ +7b3XkuNIsi36NW127sNugxaPAKEIQQhCEHjZBg2Q0JLk198IZmZ1V2W1mJkWM2M7SyShAx7uy9fyCIA/4If2Lk/xUBl9ljc/YEh2/wEXfsAwlMEZ8AuuebytIWn8bUU51dn7Tj+tONfP/H0l8r52rbN8/mrHpe+bpR6+Xpn2XZeny1fr4mnq9693K/rm66sOcZl/WnFO4+bz2qDOluptLUMiP61X8rqsPq6MIu9b2vhj5/cVcxVn/f6zVbj4A36Y+n55+9TeD3kDjfdhl7fjpF/Y+qVhU94tv+cAj+gvwv/SCb1Nzjp1bMdP0/+g76fZ4mZ9v+P31i6PDxNM/dplOTwL8gPO71W95OchTuHWHXQ6WFctbQOWUPDx/XT5tOT3X2wo+uX2gd/kfZsv0wPs8n7Ah23fPQZj35f3n+yP0+/rqp/Z/suO8Xufl19O/ZNZwId3y/wDVqI+GUnUzp/sNN/yJa3ezTT0dbe82kHy4C9o2eHtHwl2PcA1P2Lkd1Z+bx39eSX6eTfwC/3eFb5d+b119OeV6Ofd4NJHq79e+b11NPm5xd8ejX7naPSbo8FfnO/Xpam7/PAlyqGNi75bDn3TTy/74+CPBDuVL6c4q/Ovtkk0IyLEz7YJ9QROVPcd2N71E3Qsvqib5mfHCAh5QGmwfl6m/pb/bEvx+gFbsniuvkQGdPoaIIceJ3lj9XP9fvqkX5a+/dkOXFOXcMPSw+CJ35dS0Kp8+jqa4B2+YyKKfSy/exy8ZDwPb+Yo6jtsBw9AZoAb23sJ8fjHeJ+JH6d87tcpzY8pbA8PFt8+fb1Xfpv/svglvxO+5J8VvR+Q9u+Ecf+Dsl9bCcfoT1b6ss9XKIdgf5ad8E92OsaHT6YCN718bY+v46Pru/ybYHpf9cnTvw2Its4yeJnvdsDXXfSH9AH+TR/gn/sA/04X4H+apxKfegBG5S87K/rbzvrJ6H+I5Sj8a8sx7GfvJb5nuj/LeQnmk5nyDDC590WI8H3Zd3Ej/rT2G5f6aR+9h8j8Mt81X5bHOwTH69J/bdz8Xi+X98Ph5xB+BvnsbUm4/2yT8PhY6MD9Xn7aES6GP9/202GvpY/j3u4P3tSv9xqwwQvtf83P3gnNEk9lvvyaVanv+8GUN/FSb1+35I8Ph8+8C4TD/yZgaZggvfrfOMv67j8hQAic/JsD5CO//MyYZQ0YCLAnEF5l/r8JUCIAyYF5/gPMyaB/tznZT+Z0pX9HWsF8MtRfSr7IvweXf8JYBPkaZX8ErvTrQPtasvKpBhaAHvwFwn8G2h9Y/6tI/wci9gfl+E3EJvG/E7HJz1FxmPJ4yX/AqAayxmQCn0r4ifs+dENrvQTUb6DKL1JHoG7qZ5x8kUhfC3Hh1+LsvYb0fvAPXyo3P++wX3HyX45KoHtRlvoqMN9x6Xd3y/vJLXg3P9ulL4oZ+MO3/falDf98V3543M+6kpvK/iD87SD3pV8+kgHymbb/pRj3kZz+/bnnH4lIH9XY30IkHPl+b/41iER9llROHmdgDTcMDUAQWKQ55wAhPiHUB3L9fM/v7LbXwOkwBDQ3zuIl/u9ANOoX0sj76UFaRUn0qyj8H/RfQ7QvDIb5+pA/EeH+DWvfBP3b/O27ZSGUZX7Z1P9SBH1Q6J+ZSa4Xc/icu/9rKkMk8duFIeyvLAzhn0tzH/zpWzgCou4VhUP/K878N+k44lt1wn4uG31vZOeDVv7xdv1cYQC+razJfwQQ/KUkB/9cPwj66db0cQa8EPl3djvyG1sSJPXZ7bAfye9Y808DVeaTNY/dvMTwyG/5Q5dxcCAbGqWJ57lOv7bY1575wQo/GOI/VpH8Wil/Ec5/QUXy3bv+aTb5k9Si8I9a8MeUg38ocP5F/oJRX/saiXyDXW+meD/qJy/6dCIc/dZpvznRm6l+5UR/AYP6LPf5vl/Ob1XET4r/K8L9yc+/Gj7/mX9/Ncb6Qo/5a/T4ZtgWY2hSxD6Rj58P2347OP8qEsFRZ0h7vxoYfx+1/mYt+9213xu6Z9lfODH9OvpHlv0do/MQ+r/a90X6f3O4+TdJ1dv487dDxrc1yacuB2b+sX4bMB6m9/HidMr+ADhmvnFshvhdAUr8WVhMfNYBTg5uOK2b79Wtft2L/zm0fnOUnyP2W5XyH4Zs+rcwG57l29rmH4jjH4jwB+A4QhPEV46C/ZU4/j/fjA2TKAVQ4qcf7J9DdfQbZUGw5I8swdAo/fqf+RgB/t0g/436/vMxn/gsB895Ov0foP8nAvr8s577lzCdJr9y6y/z8/42TP8smH9pIOIXvPcPoN2/E8H/qCGqv4OOv6HBr8I4xWBfQ97fCuMIDcLup59v/O/3wjiGflvIoD/k478RPSd+qfb9bQj8VLv+dkvcdf3yUfn+96I62N9MdT4S4R9AdQAN+KaYjv+dQUKw/yS7Yd8m+H6VCLAfYZnvF+Lt3yFIyN+dKQ7NOkNX+vep0GCvWcF/a6YgfmcUoL9OIMC9YPg3vvPfFAP/KK//8x2fYj878l85M+dvqzUSv3f24xsL+rtGronPYxM/jUgH51c73u4V1t+7PIf9Ap9wmmBr3keHkP93dM4c3LdvAP//8cf/D27s4I75fehneLKlylv4q//hV0ax/2vG9v4H/3Y04IOf/nyI9XujUH/a6B75y6N7/yep/9Mk9dBnf4CX4t8+moD+zYKa/Kwlfjb8939u+n9u+uaSn7H0L3XTj3nhfy2peXPPj+eDf5oB8IfQFZL8F1nIL020+qaGQX6b336Bo/5R/JP5nPWAb08xIDDtdwoRw9A8vtu3/3Fz65jfUkKfSmb/6ty6P19MfDT430NMsBT1z8jivyCM/wWh/NeoDuZznodZJF2a/+6YxH5dNEBeQbNfU7J/95gkP1e2PjO1d8YQpwukVn/FI0rfyK8vcy1/Rhm+9yQX9Wepr48T/3c+ofRRZfw3ekKJ+FP4DMZ+XXUjUPrrU/zuiVfE5woeifzeCt4fVp37PCnll8rSv6zD/iOhmPrt4ZL/uIep6N/xqMEHFNft65VEvy0KG7iBj9Nb+UKjD3Wc5UW8vlzkF9438u07Q14X5D7WIh9r4KlgJRDn3hYxaehKIFtrnzedHdHksufAz+nsVaJXgk8G/E84HrgjXF+b104Fv8er2Ii27xCd+chy3/F8OgHnGqPbeJgOtl3Hju3wN5G/22Lg2CLW3Qc3qJGQdKslvhVel2TjDxh/2fTL3LjRtsXuk5n6Z9bOFH4OW6c1U1nTz6NwtY2+M5jn4+KJnu9nNlCZkh8/QaYGfsrPGJ2XY3+gaqUHiwV96APwO89T8D/F4s9JwLc1fTzPydMjwqtmVhbY0muhShqzzu3N8SBVsSjqh91QA74KzyV/2Bfh4FehLE7cPqs2V4VBVWn7LBy88hhIvW2/Dg1lqdI4cCgCDpXK8Ax22BsOnhAcCk94fh0KTgi2+GUYiD0HDrXBoeBS2m7oB4+DJwSH/m1t6QwHxfggdNVIOIaCBaMhtx/JwApyqbChzidodrnPuT6KkWvHiMueR8cVI0GLbNFC2cIH2/12RtvFb1c0GC1nPEn9VQyvqiY4sXW4OGAddXGBBpSIlOYH8NtyEdIkO41kDlgGlgHS8WtpWNWqt6QdNjtYp0rQRUMQjrRaVFp3oRDJUAdzWKYYW/Uq7H3Bpi62XLKKg8SKE+tOPHlUZ9xLsN9pyBTHm9jCjYWz+7DPXU/OwtM5Umf+HGHBLncXmbxwsWDHhYsFp93k73iGZ8UFxDbvPlK3So/OIQzuM308SgfyahSgQVQo27sM78G8MaWtHRjndhrWk9OeTPdGJgG5pPpSTOwZWYAP+4fRuZ6GDc2v2H1DrPMxGPp+Y50L0hmc6x97oUFuhl/b2daSaxgo/HOmzX0WufpQEtZlQhnr6dTKTilEc90PXelUtm1cG2abqLVH5Bt6DanTUZWOY3PRyE3DluTiIJLVtygH7qQedvqSNKjK8HnjaAjwHJI7RYJZC1qDBFfZ8Pla6AuBvGhC2avpxUdYS3jU/agThU5dGjQDHlv3zN71J569hs7N4PtGRJYuwrJrqKmWal57Clu2pPH6hpU38Y7tu6xWoB9Qs8ISA3jXvfV2udxFdZUeqzc2SNbFZn2sytmvS7uMBc5Wa2NoGAskwpcRqj5VlPsjHbVyiB2jVEjueIrk07VHtCrUpdDiCStvKttSwVWoe4lzSaZGlurow232q/m0bPoD29RbfVnplTz0LjSI1RC9pKvgBMcnWHSu91a5PA1FnqaRCs7jRbqnHnpyvevxYeynmypEjTpxucYZ0QHsAk4Fun898PYG8cjaQ4GL9ftFxbKVXugUVx/2zRKeLEbzZRw/jHJhnyluJi05u+L9lBXFHX+oh4tLXRyYmTDeMIqpeYGa1A11KPPpJXra8RJYPXsZJbuRfD/2bU075Dof6b1/KGXbMlTgu1d/XxMMXU5RY+g1sQKvACfqBirX0HgOdL60vepY6znOYtaluiL5ftUiVwB9DLxzHM+3QQMHbBNdEzkIJAnYEOGpsnQPINHcRO76uknCPyAgoI4Od1xaOls7obLbrIvusb4EAs+NGoeBwEwWijl4+AXEjWAO+Ao82eCo6C6yfH8SqOKJcTtXHg9dt62mHQtlQjxk7nG6hsgIfGdtYOwxuS6vU0206pxI9wDglcjN0gpiqLYfsWF659YHLQW4g2ZLcsaSp01oh76PQ5ZicpeAfTwk547EjbiVhiWbKnBrGLslbybeqLRTH8AwMrjkDeISsLUQUcUjcOqZ7ewI9gNgmzy8ZeiJFJuu7vGmivbYJDSKNrNcPkQQ30h5lIXUWzgEMcZ+l8NDeJu8rFTn03Vu1TKKhxX2O20+K4TOxnM7lIQhnB50BlgRP/hWgkPnOO2MKczHuUmlchdAdKSTg/jy4EtDFKTdlfHv2SW3amLQzrfyKHA0vJ0mpRt3o5B9PxMeaoLobbzhmXbaOFKLdKxEI6YBLF4U4KuI2nmEdeDq47Un81XhQW6o+lDmSsYtFmqdRog6jTc6fCqeZz4Pr7zRpgcuvobhnR+20+XKyZlmnOsYPg+Awe4mwc2N4zyUhnkFveNQN6a9AchDgxbNLw3J26HcDVhWhuahdaP5wO0WC93p3g6YBNYDfcXKAChBb3bATisAFLzQARI9MRsmgQvwr3su7TswS8ISDMD8HkRy7vsXKRJ0nF6wpFN3Q2D2hwsMeuyLBgY1dtnOlGA8QSwVPNI1iZAkEfHwNTQ3r8c9tS4jsgHTt0Ge39Kt2RKMLQlT0IZAc8dcKFIHpCMXGhln0QxcriHyg92oMRov5wEJz/XlhImv6JVudainx1ipiKBmgouOMqt+uHsab+/FNnRgjzTcZe4OsuStFDlb5uuwGx7gbAlAIc0/5/kznGNq1evYfQTxU2XGCDVKNuhrubgtpTOC7EeD3u7nbbtWON08Fr9ZkgDfJnTE2gFwrvMBZLlU6Pk8SaHbGlhhXnSSMc9cGZ4umu+sOebytDzxin54+OfFQ50mCvxuod/o1LXXFD3cz5U2sanCc69A2gbZ8YIlmPxxD4+SIYYyQfvmgBmP1gdN9ttjFgXX++LPp81SSiEaUuWwQ5gE2Mv0wMk2mi0mP04AoOzZkeMnuiUh/9MD0sOS03yR7zUZL/LV7ITuuizM814bLjpfj2clg/PPpaIoUDa7nDH3ER9UoxpPFk9ZzyBMBC6Mr2nUzmy+BHT21h84CfI4iKlncT4+5ciVuSifHmLknG4P7sKdBHjOKwjeRZWdW6I95p55O7Ip61sCQFSu0WAoLzKBywA5KoTKrAsNbOnayEm4kabrof6Zs63DpCDe4xIDREXlAfVQCzphYXXyvY8rcPPI4QHaGT+Dwx2iFeAWzKYDgJbM541yziqgu8LVSTiaunbycu6ZTRVS5+DGie7YmvQgU8qcxsdaPH2KzfEMECcWz/RIOhJa3kOw3Jq0a0CyAlTsgvJ2KsenYMaSbLoBNoExFA7Cdpgi2b2BWKFRisxi4NUSsUk6H+r6KbUPPGeLPHcWBy1oyiSgHaNzNxcFTnCjtK6GvG0f3MrC2gNHW883ELWJ1IwBI9x8ugAnpKE6h6kMeCWwDXXp6abzwY/r4Sc23y7NA02WTrnexwMgGc28BXGvAt8PfDLvhvUNmnmQSUCm4QLidPVCQSphpxTvORLELYjf5OWV4yG349QQOKvt6AG9mrzU6lirQyWS0FEV01QFyGjWqc/2qXZX0rRvEjfBMcZsGOMWRgd6uvilj8hOWQIkh3CGgni4qI+0SRaMyuYs1tNWGdhiBFzoWMB2PO79qB70IRcMx4oThym6h683td2qNZmtjd1L/LHQPbt4nqqZhlxXm9Hey2+kx6FRkQpkPIbPZtk1PjwFox8BDwTgpwCwzItixZ05V5ZkyNTDuy17yrqSoEMB7g/tM9u6a7YpywVvbmol2w8Zthsi6pMBCuEw+6URtsdHG4xoOxdXwK1mFjgNBCF8hnntkMj3NGyWfo6tCa/FuMMy+9QK4JCIY7GTWbuS7fpocQG3FAJ3t+tjfdRh7zLz9M5RGMBXUwQF3BO0cIF9hIP4zDBZcFbQzmPP2qRzrB9XxK+gE5y9Tl+wa+Ccj67heXt7Y4qr7lPMCtiqG/tnGDUTFTSyb6XqwaadIVxb5WhLq7TO7bHVwmfhTuET47Owea7ZgwPaFAOpm8dbsgoj+XwLanql1rVC1hvVeY/Su6cXiX1rcKE/bsPZa1SgbY+pEmFdR0n6RWZmBUgIT+J3B7AyQWQs7nHV+MpZAQqoh7RL2AeR5sqhurw5IUNCtgDR4rREre8GVogf/eTBJSnIr9Q2xYBAxdIeCU62vR0jbXNzi9tl0kAbONdIjKSH+mGR+pNRilHszS2rIOx4vh/8h9SQxUUHAlciz7QVgSwXZOtFa6I4kLqDieQdAnEVYN0KZDSinUFYPHcGN0VBR1LtgdkP34+CCHAz6R4Gj9V/HJQKOMENZBcK9Ci0wW3yKRLyb9mT8J1qAOhEVMQ66uLUj0qWVVsQ7ujoO3GArpM33hC5R88jUEKJ6uqggSibvN0blIvgX6VtDrV58ZULcD5whKvvg8uAaOJLZC8ZmcvBH2c0T6Cxfaa/KOrWPyOG2oESy6+w+yFHmiggNZWSQLnznDkGHW5q59J4UmFLMy8D3A2CHGKCdISfHilkolcPOWlOMD5EhE7ToqAe83R6MHlxsort6FI0rkSKHM7W9JbnYKg82Gzd3J3wLEQXbqnbOjfSiE8ym6+XdOsYohvsSxQYk9PN8tOtNd/TZpHNoL/Fp9zVkmAYAX2KkbVBtMHTsDLaxSNmXBWOUPTrFSTinl/vgiFyIrNnDCDs3l3z3BRodAA2IETcU3QFt8yc6rqvj4CX8dFTiz78NcOle+nUp/oy5JqCU5fzeAebYteJOYjV4DPLSiBSZhBXNIDXCaEs4T4bwy0UxAiHmTDtOvx+9tEcLz46i6/s+eJOJ7viTHHIigmKVJckGCyb4nGxht40nnKP83v6zm3yxkKpe2TAElpunY4Dwqvhhud9Q8tnvn+ijphX7iF8ytzzlaoRCDfxNX/n2sV2eqYotRUUTdOE4J75i5I4Fd8bfH4V91Q7SeaBI8jTIPCokmoZoNBZejjkXDFzxgT4qOqcAD0MFbMK6H59SxPg7w0pb/ao2A9m1p/izCglWJmWB75x2Bv4eFby8AFpBoA7X3pxGF2pnwNCA99P7VjeeduTQN48rXgB7Ldgk8eVrUI7yP0SQNx7y+QTSrHaIB1PQMA8RZFQDGr049lUrnsIPADQsqY80cdVl+v72SJOj6db9is90yd3yDiQoZ5vlgd/5ROdP2cxMtT5YAznkQ1uD0k/KoR37ImxeihO7x/rOx1D6suHjUwBai4Ivc5VVllfe9qEmiE1DQXk/BujDJbKWoBe0PCpdj4dQXa8zU1DbSp6UYmEDyfBn9D6DtSa++76MAGDj0o2L5DfouwF33c7xZPlTrCZfzk+jvXQNqeumFRA8tYhvjSs2blOkWtwIo+8ycSqX/dVuL+rKb7stYP6vF2shz1hT1hgi1wt0oZkgK0CPCZWngRjgqjWeHMgzAIW51xZ4y2f3z28B87H0NKzPS8IAiLgamyNUDIWyrpeJGedCfTXCrRppvgoUzTM4g+LGhbDlTmU7EXFM7YYMp5LBA/d4qAZYPVGKh6P90wFYIgEDMpcL8zwKOebV3studBsPvg3rxSPiC2Xd/EsTirMsSCZJi/5DOuJHcY+6auB3qhTqkPJv9F3JlX0+9upL0NxIdl4utA7wYnxXAmJ5Nv2mN3vV3TOF60914BbrtHRsM0ZDeWYu48hIt+wDqFxHO9WyFfhmeCrB6Vc4d3HTQ1UbGkLzJGOIl9GK0YnC7viCUTTWbivqSLsPZxsw68dTrPgFCtl1v26Dw9VGw7a+pxfap3JTakK/XiKEjWSgR67sZC4SYA+mqCz91XaQbeVZWopzXXb14cIVM/1mLCpPhRqZAI6AtlIsKxJH4a27sdCaOSonDsX4fzEXObE3Nf+6AxeGQQmOBZfdDs18cg48jyR49s71i5d1L4sal25u4Hl0CO0zMsc2DKFeHhAiLPJUu+b/nZAfQ8XIOwBgnhx41WeLwG0FyqCyuRrNIOMu+oTSMzAeyUn8Adwo4lW9zdtQ0G2TDTLkflbtF6KDR/PN9WqfOgz+pBNDKPOQqNUAdV4GGiW6XL7TQVyE1ztBD5z0qlpjwtvE5eAHiUMVkLpuONfNcijKCjuSp91rttT9jywxkmA8lSPdK0dxuQBWP1bEIKVLHt/Hu7hfBPHZjbcI4wRDLnfgZazHoDrgi7F2GIDkbherA5kK41UNBJkx2GiILPIN7e/nU9KU98jYu0GM+6mLjzvAhcJSUuvC5xsxptP0OZDxKuDCYgnHqMo9KIX8BeQ7ZXGZDya2q+KTeYeLWDqzzTQJdjIzIRlW9tfoc1V4iAm641jmEOmP3RyQW3N2tJgfpx8j/SO876GkS7f97SwAOSBDqkATjKWsN/E8gY0b24MnQENcC4JDHQYdECpnP0G8C/REe+whEl/wcB1A8qHAL2e3wPruUI8Z1qQx4AjXu3z9jirRajFIAEx1rZte3oRQnCs0rNSX3RQNlrwLgeXZFm26c6Lo3TeKYxYRpkyYYSKym983/M116WiMWmJdpUhk5YTd92q1JlpwIJsbZ6lfgWNB9ihDJBA8xvIlm+t3Mt5q45n0eXdLUHe1Y471bT5SEHvrXSOs0+3KHwpu+yMVsH8AWAgPrEkATg+DOjjKdsrTXV49i2LKiABRYoTm85uivHVALKPXPzLBcUekdF6KuWL5uNcemMbgw3PFLdeFGmh827i1GNy9qyCUKiTbQdnC1b6AOcYP5TYPB0fsP6qH/b5+LAEPMMyTWMGl2BCwUiAhpwFbjdAjrsTRKpJt0h0Um4A/HRCCUMWsjaq7XvY3qiW2SyA1fkJ6En8KFZlJBfd9E5j43q4BM8q1O0RLHEjfoHVlxQorBebG4GOUWrAgMDCWJDIbN4ubQtzxONAoIXtXe9356ZGvnYA0EMvQM7CBGt2kTf5TKLckUxyUvGYqBB7HUAjnvAhagpQ1c7qnrNy5A86RxtneVYGKt9oDNYtJCCqvejqJYHMxQLwUeWOTxdhqZjgOmawCjFNMUDUo9GadXUpYeQ0TxMceO8TRJWfsY6F+XLwxMOaOLHgALESLJDF8zZUgt2dKaTz6Dgrodga0XGEdYUCFNK864kyFdkYmnSqiEB+YvkTXg8IuUtF+IC4ux17E9K9YV81zcJK14sB2ZGNEgnCy22JxzqlOfplGU3hLvT2JZN7gxz5uml5STR5SzfiaxyjQAdOcMhsg32pS/UeKg6WXXvPLQ9LNgeq5NwSub36Ey2AW3OqSeAWBkhBNVwCOvpC1WlI90Abh+EwrsCRV3bzsQkFesEfj7sXq056BiCa66ajcOLaJrX/uBLX1fVn2JGXNN/Da2MjjJ0cwmt9D7jnySItlTa3CSJXAvKn3lQCkkO6KOwMrEuIjY/0MWiyScE6wAb3i2NEq5gTWEEdnmcgEyCWQLULo/u52wh6EgqHcg+JiwyOdosnjrTvyUET61AgpMt1l0tGgVXR9GJVzNbnSgXM7iM8x0nTrZYG+2RiUHJ559HTKk+7InmFmE2tV60tTg4+rs4e8m0pDUJjt6JdQ1z8YiiYmGFVsuu6mzyd01Kd4qfVWESYlTY6a9IBRInlDrASx9saV4TYdLVDVPZQx+NV/QxLPB4QAZk5IKd57x+Lezxmz4egcUf3QECIPNg3kIH7I6GIZB9rNb4881sIqBMsBZyWNIvkqBXK3ehFzAPEJubLVAlBjn3Mg1Z7zkE31vs49D0s3ztRxWJkK97b4g3FpNY9dVOD0UsHpznxAIPZO9A6Z0A8EkvsugOfJmKlw/GvaGKx8+lJ1Dw3AkkL3O3wFIPKE4VdRGE0WggOYEUMTVbOji0tQxx2bxLfx3AwRKnSC1wDWFYL4hmkY02T7PqgHlTUoyrGUAXuLioUxChx7lmFgKizpxJ/Awn+pTAPJWU+N6cpsf6ZGEbZcZ7HIVi743qQou0MuP7aXe8EADJbAWHZ3kHWzEEWuebRBEcL+codDeE2SXp/9RetLFx/OEkDgOol8Jc4M/EchGbfYi2I/tWwHjiShjF3wvog0vknzm6xWUNme7ynG06wiXHEQPLruLU978FhDiZPyzjW4R5eo8KKhgMSITg5nXFXm1yWt8xBHr1doIrC9U78cALygoeM9Cxej9cKocXKcPsAoMTSnzTkGW9y45JXgfbPo++/YkOo7FumVIxUEoAR+IdHwdXpyoou7IRGFo96qZS9KfTAlpCph+3pOrDx6N5vVJfSVyYCiHXSzwgU+c2QBOQat8BSD6gdHqIp2j5tJlH6IEIbIvrJ08Z1GsFt5ODe6904cO1TZrLRIAbmeIccui0NBIPSHNaa/MMgFVdkVOpp9FpRrvQQjhqSYSDBERYg0KJZtEtR4egOySmbOx91GbOEqsRA3wMi0KBTIj/mMb45NXM5drIYQSlb2kcFtKenEEUT+9tNh1e2IM82BzSYJq6o3EvfraEsspCFwHoY1GPsyiuW60i07To2B84wPa4gcaSAnDyzDa92V0ROjzTdz51RAChcL5fxxeMhIL+XLqI5v9zI6nA9ILelfuvHoLdb+XywGkEMRhRruX0+dTMIwQAa8xRswVXYxE4T703lGfztBDJXMHhUNsZtX868gc47lcaOgdOPFGgskL8ajMpMWK8qiteoYYGRF5OeA0gPJf0KhKLda2an32qAhFzXoBmeCyGkkxUcNc5WtaSOEsuh3oi+DxoyHOBQ6iNu25H05ELbmLt6wLa0z3TgICWQGA8XUiZwLXbVFc33TmZ3dL3QkB3vyAvi3XBuAGRk52aT0UE4uPAuy+OBO7EsM79V6TOtOaPA322gZGH10Of2AjU4o7lX3tlmG/G4SIA5oVDD8HQGtAWuo+3jfoAlrod30+uHUwVGQqJ+VsCtAhtTgUzVNM6XqpOEnQR8ddFd5J4uwex2DOx2UoIJREuFOoWwQZhCyW4JRoOfadTqsHMQ9CA5q+GS0tzk7vPRe4eqXPcsDImqffpmnt7QODDOE5o3CJZ10VstvRtsLz9mLTLaaqrldxo09+ZGDWlxBxACi+IgwRurgnZLgPYvnsx2jOX7UhHCteH46yRa2DOfsHwPOOQkOMhrrEFwZ3xCEntkSqVjtNZj48DiTVkYM6FZaDJP2kdrQ7/mRngFSYyx3ZmapErpKQKCIF4vzhxIbdSwrKJIQoovDPDyR9qoEfLym2mDdRyGTlRWjcPbPoBkvCJjs8IaxYviARe7G2exKWe03NUDOTawVoifnulzJs1FnpGJfyh5mnrMZLtCnz9zC8Syv+JTs+J0CYegczji5hy1Csg55VQTaX6Cl920C7Ko57HCnw6UDS6cQRHAelmy3IzqJh5vO7YzN1Z+1DQcGJ203ShkqWf25/5Eudhl9CqFfARPekRTSkI+aPVQ+anepVBNHJHcvOjNg8xAR9F8ojCtfkox2g0fV1X2GrFvzE4o+ZE6tzNvPQcJoaLpRAJFbJ9LO3VS9ZHhRbFAzynW7jLt+gVnTWGiQCfTKGm5MO7cAYECgi28aiKHK31ub2QulEKjneA8EFi6AEosQo+RclWKPE8Lc9ABmK8PxJHG9Vq7TWoah8i8voVWEGAQmwEjpVe9JDRIb5QDUF8Kv6e6aql8Gcq1gAhogVzZ417fwqsaAYeq4OOZ0Mt0/hKI6MnNv/gdTl7DV9aFUb5F1anUjKo2gKS8KE8SSHtYD7YDc14qg6+gb3SQJrPByHqZ4qC9SojRrPOcqDSux0LeCQw867NlrBjHIbIN6O0e3Z4RIMEy1eTPUDX6E+cxr9o+xLfNutYQDKPiSZqnltyS0jYUkHBTHSCZY3lMbVfikKqa/Vzvx/MjctU4afGtffJNyGFuN7SKy+VhyfCxS5ruDY4XuZfwRA/GqWPglVKT40AW0EA654Vk7PUK3i8dmajJVvkAFPoNqJZChZmKlvnzDjyYKMIGaEeospdhguqzURPCNPSOQd9qp3zg5Ve8uBO3IHLFCD0Reu09Iv9mYmsqh0EK5V9v8UeeOwLgwrAMozPs5LvhWa+Zzezc5aQu1pua90Hmv04zQWB8EkSaR/FhRZMAGdfAb6OVDufQRJbzWa6Oq/4g5m2brx3zClerRLZIffCNM2EqSQcooJ+8CNQ+4N8wT2JJ81TR2LcKntjMvChw+gKyhIlHRXDMqn6O4bCOTp8zRC4ROMFNwnHFsmgg9p+sNlhCSQTSyUXYJODXk8L3hiM7LZmZzbnewn4hVJDdsXFshzW5xXTKeTI/J+JdqeicYXfX5syKNEkrsgTCgcWM9AI0mOJid6UwDi1Qq4K9NuTq3ArlfIbI5fjKpXkMF0DE2GLD6SxdNxHomjbJVoyO8jfvNX1xRWqGmfupAsSAkk9YtkXIk48h/FOkerKgzISDXmcfnPUeaHfAs7t2P4rCa4SrgwDjEckpeokFmAQZbKx2Ni+ON2xL+pt7Sl5pE/S/CAyyl/KSwKS+TRjNshvpIoJwZz7K/c/na/SKmbe3xDl+KYcDRy8wLOp1QOWodjhcMHG538rnQLDm00Yq4X1onP5yBGJLZ/DjoyRJFgCu7kaDmBtqEdQdDhpONE09XgNisMx51AfbxloandAHlblbsSwsW4duW50PMJta8LxvdYcclhaVMzPrJ2jqwl21ITkf/HFjnwH0bDgjCUbN5XKB/jVS3PuYtX5gNv7OdmIVwiFmQEEGODmGXR2SYOKTdF4CiKvqmViTRvNblwRpftObGkcBClrK9Qln0Zn03ZTsVb6xLbVbl7oz28ECkvs1+UfXIY99aAtFsfHCWg6So2fJVC4jqktA199zK268dtDO4ZiPnCodG3WM6/pYRHcgvvSqe97Jt3l1CqxEn8GVFKdH0m5x2uEaIQvUBtu2rebd1Ui0Kjk1h6N1zi2UmXYfpWNt36pTpcIpdnBiyVbQFEW2UQswUQeesohUYXX4iB5apHjYRnifY7mv6FI95c9b9OgPZLENzzDsUVpBSYIIL9u7N+S54cJiHm+Ve2glS4vp1xCDM+OAfaDpcIo9nSyIvGfxEdxUfn8Vtk91lvReX6iRPsD8gmVTKEw2i5hX5GngAEHWwDUZdD7ccWk3T8pmXYKAsuP7nMhZ56OBn8y43jwH1VGu1eOeu7cjAuvMljHLcDYDrByGuVxNMbJIdQuZbx9cCihOgRwp5olAeEMFSNCxojdeZTq7td75cNalGjufD27/UHcyJUdAPXJtdweCMQ+q+ChsB5brMF33gH525ScJTX2R0OyMDNpRpWfGVAQ4sEudtpmWn0QEO+5QOaIjegDL1y2YkoVkHvFdnS8SyEuB7oy4ejsJNi3f5+I1aSOvjhfpmZZ7ahFQ1gOflA9lqSOSY1dSvz7nSdXOt0kMr/LlAT17Z8TKdCdwywtQoSDFAQHleLEUjBTQSOkcwhUpuCD0Rl/YUrVMpY1sB4ZBcJw8jDBOZEVBB/WCgqt4vQXJACcAAKB1S57GsR9ntmUAp5GpZ3GbkiI3hu006ESoJMVhgK9k5c/XsZlGwtzaYU4Odm4BivjQ0GR53AsDdBLJ24HUJRt+xfWnmMlOQ1LU2ZuQvjWvNdFr6nW8XlJmRpZQp3FE3KQWjicLgC4GLpT+jMbYGzmas0HlljdT+AsxEhppw0kOSUEg3gaNJAbuXmuq9gRMNEuwKBCfQQTYzVkF7Og8Js6zVl6eNuDdrBjoNgfG4SQ/86OCKYKSr52nn6HE2AA6MWcLKjVH3o6r8cjMZ7rqQL/jydafFR9lk3KY0DfQvG4UjvONPbDWggNQHcZQpiMFoeK8wCfYvR16teHIBJyRojq6ugTbBc4ARfyCfh/F3OiVyiWbujsUPheE5C1olueBR60hnoS1PAvwEQ+JGcxC1FmQyuDw6sA4lwuO+fkpOQ/eVplaJ3Gw9Xt6pxXYxVL7VlXl79sgwKGrFhWwSKsjKBEoxMu3J8EEXWoQ6b1+hsR8ei559vBHWAhmmNfsvvD6DumKqfjH6GYNV5hhL+XtLGJS42bEVgiHwyOzOjgTxvboh/W6LUimiZuqNxRTm2oV6jF/lJFrarm95zEHWGxkm7Mft7Ezi+XaEK04ZP7YOMaTDFB8kq0ubJezMc+T3T333c5B0IPwS+0WBfrgklQ7VxAdVF6TiyzaetHbgzG8JCP4R9gl/EV27xYYqMEziFWMkgZnAcljDuZY309D6V6ca8tJs18ZMCdfV/pKrvMKZXj3JNE5x2nAP8Am8SqPmpOvmFb3w6l9Jvdch96WAjx9Ei32lgDtMzTeTGSRetipzqOp2KM1WJM8hWcrHtepp3KGqP1LXY8+yxTWg7gzrxHnnvNHOGVqYFm2apk5ehyxAOa4NLdk586yi1xYSslYV+h3sLh6+OFjkPeoj/fiisE7JEHkLbIi8EDK1n2v9Ysh59ZruAWf4Y2YJL5RZq2ODu+DPNgF/ulsAtI1qFmStmc1N9tOLtvzczSJzkrxDNPrfgRId3tM603XDvNk4kJpKwOsZ9cP1VfJbG9GNr9e7iGcCYiSuVK3J/sA1MqTGwRaYW8HbTrx42ko5MBRXuMTc4d3quwYAalhpnIWF41pT9fyCfg2yRiWhd235JVV4WMMIPtfivo1Y1W5uBvFfmEaUg6/pSDKn/eu9jxWh51/ibFmOj82+Vhz9eJfFyefpNC1dAhzlmwViFu4c5GGQpyc4aCGebAZCyoEXLwjQYeQImUqGZ7RDqunXSryfWy2RAsuROuuEvWeqUynfdp8GJckQ2WQtd+86JG+avVOtt0tRG19YvYB7KkeRHxwgZ1IO/k6MkwaJs1EvGY1y69xAwwWJ3WYExQOKAgXzZ1XBXjw4+A2MGm5FZus1/ExgRXOWiY1RyRNYROvG8x+5KC5N+4oP+arTjRzc1NrrwFKeBgosL/nXK6IXR+zBuhJzAqW5gbnZOgCWsMMAUt0d/kMznsB8qwNmqfVSPP8SES58W7gBgb98fI3S6W7W92vOkqhiYNMLZ2jJ1YaYvlWF0URRV6nvMWbAblozd1CgUtmErEenLPT2fx8DJVShAzfzrA+E4EMeTajLD0+Wgb/UlGGv6yc3MKTXDvn5TIB9hj2ge8/fDPwK6h9WZJEcbzbNivW6Ck57KkEM2OCVMeztYfi7tEX0MrGvwarmuAukBGSk15K4vjOZWwXqXXtetwNU0rHJoUVgTSDvWnBubrk3XgWUGgyLNHf1IPdelbUqAa/SpEbJ9c7Qkcbg1mp/So9rdIpEvG2cqXUSmbMaxZZPHfI260gw2v2vN5maei3PCzbGE7ACq9qQYwPnA35wlu72M6Cs8bP5X3m9XdO3W6aMs+3VrscYA6GrVOtIlrNJ1BVRki1Z3DsfKzv/s2ed8zQDvvGs9jxNTetU/f0gc3VIdhQavGDcgU0TIqAGcMB+jvQUyyVH6BnGodKwJTxKZGV7Qsehr3LgkhmIOeBELLNBmHxNv+YsJqZZ7aBBG9zBmj7gCOYKHvklwfsSRpb0DiArHul1go6T6FUe6oI6eEhgjR9ew3+5EZuKlFCVsdcgcNBsOATlQZfv19a1bW7GMBZlxccR4HPbWtqhhdHvhGUfqFo8znDWUKMX8+eXcMxAXsU9+NZrIyn4Qb54LAWHHABKtOJCR9SNcW5RbZVAYvnJhx80MZpZGD1R1EfsnZHRzKTgU43WvL0lKMnYDHm2WtVwBYPfo7v+4TD0suGJ5P/uMY0nBfjnE5k2h0fdwG/oEvUAuu+qpLtmSPNqENC+nCDkwjgfMo+FvkJyGiAqNq1B+e+jY9iPRrMvTxSe1MeW5Hd0x3w+6St7W+fVBIOtDQYTGyfRZ7n2FNlANJGqEepq5xK2yWk+uqpp/DchZsAYtQR8RP0wftrBCw6lFZnIzxP+DwPpKe8X2414IkhduoGMk+xAuotHlC7leBmILhZ5vHC4bdeKWZfcnzE7PABGS/HO3UPXmNuE4XcBshyCDdw3admYc2NFNErhcTgVoFuzOd7a+DyU75xpajuTJ6p82k2R/YKwON0mk8Q8BjyyXQDfPTlApRz441NBGcYBCOebx1CG6mMJdkiO0hxiiazGYdH2imvOjWsREhUsxxMj5Xq/cJ5gGYil+v13qOWtXW3DOYMGvffq8JVQVCjaVm9Xu4ZnePd6806pxMkTekit25kk28RvMG5OFLxZOBM5bQ7PWFDQWYnG4OvVpde2idJpEVxrSjWrABSGwkcvYHx/CVLbTj8KhvpjfLDBA6RkjmqVAwAWLrK5+clvQKIlYfmubt2OMrWbA0bODE7GgH1qmi+4UEVY/ZUgmV+7CR7mc9tire6v2K7EGrIIYHz3KxABpqhHOf2yG/HyOzqMe/v537QQrHsJLNzO5FQTlpwDoEy6radOEq2xu2WQKc5OzWvqk4EkJZuYK7OhWMhSBLK3IrbalihfUJYQU4gW+lbkGpehQf0kR9boMeA5PrhbchgoxE2TUAMtHYLSxXKQOc7IttoFDQzgiez9jAoNBhRbQhPSKGIrH5GTsNz2EnuzUq3ay+b7ykFmgDiO8CQJkXlrlafHDY25rOngsMz2DqcRjiixujs5Uce6Acljil2wRJVmnzIZpnUu5w6UzifNWXfYQ8I6oBWPVVBOj/rh3KC4z3XFiQWVVMISU3uLi1bR2/xa5ArBE5wZhAeiiTtyEm4cDeIM3o2vagQQJF5fKz27TCZbLJ1O7lIcLQVVgMbIMiJPVY2koxyVWf8Pb3IdwODE4XrXBziSbSPg9irj1x5jefAeH0mDGHyZeeRXWRnD8Am8d1OoR3MxirQnnLg1OPOQVgOeN+RCzgp8vxTlV7frCcyd1s90MgFTwvFedhnxQXGOO2LFMh3lC6nBs6KxeCs1UKAGvpL9QamyZKQBUTvn89XYMEQs+rbMCag2a54n+M1APqZyXXpzhvvPl5d5V5kX6NFdjYEzypO67UWaROFk3xkKxN69m1eAGi33GxCsV0W/EGGSwwfODu2ACIBMNaNanulx+9UpjVgsYezTEy8WMVDrsJaJhwEVGnacM8apFBWsBW7R07NiFJb13VPlmZXACDJhuQbYsHMdefGGwsfshjjeoIvGpGQPj+PYhU1ZDdqqyiY7n/Hk6r/altgoF52juOst06FzxBf/AuTTdwRPufMcWfPNx2NPITHI3yAG+fTph6s+PUOi7qb8+X/wUfQAW4gxNsvSCVev+Gbwcj/7495nQbKfvnioo+H4b/zqjgU+c6XG1HEj8if9RoZ+vO3RX1+O8FPr819f1XkL78z9598a+g/8629X78FCPuDX5eL/c63ZNC/0O1/zdtw0W/fhot98/aV3/19F9982cuXl2/+Ra/RoD9/z9Yf44c/fv3Olt/ph//IO5//VD/8eCHFb7+8/G/1w/9hvnkRy8e30P3j37vyCSR/ZCmMpCiGIFiU/Xh/2b/slmBx6vvl57vDVzEafQbfwiL+/w== \ No newline at end of file diff --git a/argocd/iac/terraform/examples/eks/getting-started/static/gitops-bridge.drawio.png b/argocd/iac/terraform/examples/eks/getting-started/static/gitops-bridge.drawio.png new file mode 100644 index 0000000000000000000000000000000000000000..603b7e3e33b22c5805b38e756dc748ddd16bfeba GIT binary patch literal 89536 zcmYhjN6z(1w;pybU>JgcI|0GnACMdOfH^dobJ`>~Lq}jvn>lj?Qo|h&!~^gkJP!}R zeHXv~3KHwo-D|M9QC{`yx%=0AV^^}qbj21EabP#%Q*17+a%1;_tJP%{2!`=ly{f4>p_ zHw13*pN%AIhHYHyf1oV*t_)l81MlE4c(q*c5`upi`Y$m4U&#L*d}OBSR|G|>HQPTB z5l6|tQ34$P&zkHR`acj3eEv0E0p6q{>&Ewzd`Zr~n-Mfepufq#F>r{hm$3(pL>MCe zhW^G8jQ$&@X>jI$3{XViztO)@aLi50WNH4V$Gu}Owxs&!xKpXivpM2Wawlv>xUb09 z{yE;GSk?vz>yPOE`@`Rhw|gr7c{DGMCIdYIvJ!yE^Cqd5qz6r!f0ybzGn zEt$=X8-Rq~n+ypU!k=5)K(PM?|MMnfh=#x2l-3;$6AdP13=0mq6F4D@&YF`>V~Z(< zQBw%q2FP5@;O?7-e&|AvfvXY_JWEVw7h)gD6WRr81#pD1!HXu*H*8Q}At~_;vL#i| zONtZahcP0!0-_fcb&-2|@l~fr>BRSOgkHemZMxF;Y4}bpU0J{hj{Yb(BlQ6-bvr!i zY=%g3q{0+WM2PkdAa#}wXaNmj8g7h(g0Sh4+MUnQR zNw64M&Sv@js}mXaip)B;u7i|Y7alTFY3E~o?JP%35*AQrJ?19G`%>nZ8h51rEPup1 zaZ?IjQsKD3?J6-{hh{%IN1|;eQIxDu(O*l4HeDWb7vc$M1e)ia6^d4v8Mb@Ay--m+ zcOh+}z?XJTFc!=$8{L{#x^GvD>!xDJ*;E+gf_=2h5YE%?>5Z7oA8m%)v3)6fxcb9VP8^o)f7$>PJf(DyQa(T@q3 zZngq1;#7f`^Fa}&Br=_P51Bspvu%fKsj0}t65d|bf;;*>CuB8HcXliMh+b?|*r*46 zPyY_adv5>2*{OXE#;v|aLMPXqtUe{Ykgb2v8&E;9aFBP1ud!tlojO`M(%ZavC(Z0} zaB#s(V7I}9u=1ImZ}%$nLAJ)rN-|ap`CN1BJ*z+9)W)$TKfH^MBvL+v*8Do?`grH$ z9b$rz@bem)px~AV&kN(d5NkK{exTFF~NLe5@F}MXF4c6;s{$~{Lk5&7tq83m%!?)5KtxEQVH~3 zobm=0@>ijc?Rij&?L0zTfW=@%m!-iQ_2jH-&Cew3x5b#tE{P+f=!~*LID57im%Pf? zS;tVI_^^?Xc2~To%V%Z?Z-YID1)#U9h|o>o&Kg#w(@J z4Jox(@4#Kf7MvCp;G#r!)@Zu*U&nkV7>bZ*NykQ5dgG@Z$YleIkIe`JMh;YK4vb~X zI3-JyI&dukAgqsVx)vj~o9IC2y|HBS2c9mA6sa^&1U| za(sbTF&e6wH@jP;%ofZrYX_ib?DIR&9oZ=pP3=|3@2CXvi9)?bq1{BBv*D5~*H>Fo z3Ph!lbS=gk97dj_?s9^NQm7DbtICET#E^$qMwdhExL~>(?(`?y$&2(yy-a@pgQU$gu|9W=($t3-tx53!--r2NN zzOux7fu??=J=iKxT~_?3i9bcK$i&|O?kxIy`gzu!9-8*bueWxXH`9>rk zlqdN7xa@cOiK?E#$XKQwD6kC0hvKlyp5ocOpWu9DJ^uJe`+Y1o|tEW!d{aIQVwcK+IR{e;Qfr@SG4%*=vX&qBYqP4Yeiu zgGiJD5i__VnOKq_ftHD~~uvYnE&Dz&ATCZ^uW0)NQy`fB3nNdm{EYz1HZBc~OWuQYO|+Hgr|JFg~reQ^j2&V;%E8L@1fTC;3?VerN?<~ zD4D2qqV7q069P;m=s{CHIgqm~F7j?KHQPpR7u@KzlpLY5U;@R{ZuJSMzSv(Qg6LsV zKIZG=Ry64;*^aCkiSG$a7b}m!l9q0MdS@v@pb1oh#lSvQ&H+sq?MgM;@bpnqBJ}gM z9H;r!9yrpinwK%YEMnp{6n7Vt5XK!dI$`L~8fxqlw!yQmTo6{<7n*8dbivgYOZ3|3 z`Bu8qmWgqEy}m zJQV6uM}Nwz-(-I4z2T*=yhF9R?O8-Y%NHkgL3#{sK!OJfN#jxvK<*+IIFEfAR9<+;ghb}P<(D@4LLNTG2nfmmSRH3*}hyYSmRLv#26DC%lU)L>O zD)C{<<}9x!Cy*h6>{~;HZwguGTQLN%8Fkw*3~uJdLBLh1lvEUEig?pC^0?hE@9N(o7$L+<7Do#KPXl z_*(BufVKO)D!I%91bh!~81{joWc#b=Ej||E6<&*Zun}T2wg{7M*c=mS^_iw*`BzE_i#$v5{Yaw5e;9fJ8T`OFi>U&SOJ^XQi0eqrS` z5X9{3{>)lof}bBbV|Gk*-|o;EpCXjg5+^O{b#%bAuZzc2`}jo%a;aOg8ns~M95S!5 zCDIpa4^7&m(YPIa<&27QV%J;$ilmq;40FVrL?mhz{c^G~(Zx7txsJ)-70*9weW)~I zgS7a=FNgI(k6recXmifjOux1>M9oZg@Q(x6Kg@gWIiqEox{tDAwnfh1(2>p<0S~*e z#y@1%aQ*n+@XhADeDa4(0c^>XdB)rXxQk4t8yQoGvDpQ8${1mVO*8`!asbqsh|zRF z9e-rVM31`ozHnsxAcVbQG1v9^IYbv^-elKi4L}eBU7MJtkk;>U$Gc{_?Ey2+*;dd| zC>+KH=lc12CpY0_{ET>IpB!>$W?E0UvKGTEzpHpo6xwuVb2O;q=bYZv#veKG8@l!l zCjt!sX5!rd<`{H$c!R32$OAnwu|Sf*Xn-qXlgHWX)z)*~{j?n6i$leRSA|eNOLi@Y zBBFD+vmSQl{1k-`=t{#;_R)R>C!cbu1h*){P+Qb0-Cop(=}I0!e(pX9D7OtL@C&R| zt7BVS^>V?5NpFPHT9Q-_DiFd_>!+M98U*JkI^GLAA=GBaOJk?t3 z6`IYz{#3>B!#?kN+lrlUlB)$u#|8o!CPX$!gS1Cvt(-zcN|=k9C-!va)QhEKoj4rP zV+s;qC^2OUnm9_%mK^wrU&7*viH|la5f`MQI~lQR#G}raIE{+cp|~p7jZvA?S#!k{ zycH4Lop~+)zL)M%S&zy+EMq)}<9fpZ0yy@gbh1o1U)>sfl$GMQt-)0niI0)Uwr|u7 z8(bE7m#DJ}jCh>Txq?HN4r)Kec6r33G}6P;V6IpX*K;J%Gdb#7Mg-{wlv6jzQ@@Ng zXmu9~!ISYnsd=M#M1hhKyo=>9h=N-xUVmK4ZcbOMvcnHnO*6l47l*eCo~X}T^$Giro5d7$ zf3c!}&3Hy6(T&^pWgUc4$R&(Xu+%rFMZKnGge_o&O(?VJQ0lz?9x3{qyEJBU9d5Y$ z&BjoQtKz7syiKBj6hL~Bv)%E)J@EQ8h&V-@LYn(0YD)ceFtQeU=I~E@knR=<^cy=J z=AN+J3W+AT^xJ-*3gD~w^C|IgvM=QgU`%gs_M1{Dk__wdu;D_C?m$4iW>y&l0Ik<{ z$)(7dNqfb%hAS&Zv%X4=eF?Iwb?n%z+CJaXJTZwJu%bm z)ccrCtzP{y27$F;=qXIL2|^y|n%W=7HpPy>!?ENswli%5C6yZ9KH?ov zptzSfto2}L73av!{pA^4mO~Yp@P06ehUnY!A-h`iqz5824UF*{N%Bbem);TtVc-x#*^0t=_qNb_}oWH_Jg<2 zupS346g3o!+QeTHPoyzOd@0s7%F3Yxg)5nrxpi#(=AcJ$7Q{;Jzret;?dL=QaFVYA z%qBfMwe}rJqE#ese2%}Xjd)QyHlue&V{O>$4@x7srZ+>=w@^IU4WD9+cbwX0Vto1X z$_DQ0t>QpfY1AKV+VbnM`uEN_w`N^F-jn`}5QHtV`Q>uvlP|9sg#DC5m)&;KL@5-nH>>jChb-Dda->>2eLp+Eg=Z+ z5tc~Xc2cElSh#%1c+J%>8RZ^Lq)bA-(~7n52g2e5ZQcRd%X79 zUoS4I+W2%72w903WGUwqr?5BJ{&EaI3}6H(TLZ9^4~oN&B*ac)t(N^6c-HnUF))3p}QCLw#;i%Y3bihyJq@sR|8zhy>4yG1K%b_4*v003gwpLKxo zL{~T1YOxwF41Kq}CjK@P9qvAtBKxvHbtG#k^Qn*DTdD} zF%t^5hg!&x=L7W6;;L_^DCx~%J|T(#5;!{l_BxbQ2FyFYjLE69Qp2zjOfv4TGZ|H= zDSuvlRG84skAoBM7RTjLaa=h6vJ5OBMY^IO7G@7GJ{Gz_kNn)-N+>cm$M}Jyr=}sxGDVR|n(Iry~8w7ifGRpvcynm<>}X`HrE|`Br>y zgp1nem&I4JCwA|sgQ(h3QA6{9=!BT=%pY|Sd31gFGWy-ssm=wcKIdsd62s$asq-Ra zx<~elnh7U!jJ|^Gd;);~CBrjW<)C@cvE=YX6EQkw&+#6D{FEmH|82;?y zsv%a$RhSK*!jo-*s1YuIHu}0W)c|6EmsvF=*J8=@z+{SKJ*9SO(}L_t2P)Xu zPag&i_FF+Bn>YCTDU$*vyS~7si%iN;{Cm}1w+BG?Q4=wY`9_W3`45y#_I1`SM)`P4Xz^K0_WzcJL(f#_YZ){>LukR51uc)oSw(;oG zu6%h?$w)B_FP_RbCAb(~FRx?|| zI}R34r-zBxH;l(`drZDV0b>p;cnq9Z$rw26PY!6;zz6}fYYs7JJ1IEnj7GZw&dRTeknnUo_VCNBh4Vg#aBx~+dRABfepPbBsLOS(4nO^HRws6T)wv7bk;?%A#G@1m!nPfrecd6z8-*bZW!W&}3on`l>#)wUO_zlX_IwK4=24JVXlx z%pPBQhyoNHUP#|TAjJ5qS zS`^RCUS^gE$|SP?<+GnhE}>owr2MPe*kJIaD~Rt%wnoRV_KYfwvjin@1gvd7z*4bp zVm-l)eXgpN_c zu=4(Z^p#6;pWYetZF#5BnfB!pL<6o5*-&!h&8XV0+U8_S!Z+CFTU_3|97iD&WZd8^ ziTr9Nt9bdVu!H2R(K}!jci<7-&d92a1QSnJvZ|8+lnAR_%|CUaMxpVV`%~hmMtNZ; z#|E~ZKF%`8urKy3PSu{Ge3;8XhGNjbobEVkvq)%4A5H`C^k@o--?2AUs(V%FVEdh_ zfB^u7z%{TvNw!$4VgKvfh zsW#>(kLGd&0iZhRI=&|ty{+ z+Clld!P;-z)S~9o`0C_G(yr7MSihjTmrrtePl;vp{E=K=}n6r{f4RvN}LXA;9MVI_WF| zJnn`9>boFY_~7tmf;?2YOpu2PkxX5-G8`=CWK*G0jCkeGfvEF`JV}roj=^`SGk$H1 zRQ+hek6zst{c3N3<>|(>aADDIz?q;d04O(QaQ{v_U@Q+uu-U4}+0ukDnF?D$S7#0v~y+`=-Jxv&?2R$}G`JJ1UDuvgAFk`x3NR4hsSiq~cd z(tY0Upl_n}@Gd~^&RT$OlXQZr~0TvPRXZAwVj#!#Aw1%K`IimkeI22FQPwXX~!pwRlBZ_d&s* z71OGZBLK2QO9eFA8bd|17)I_@#5hID9tvZlA4vJ!9vLLVh2u-+_*@=wZM6Xppn?jD zztfW5Glk-1cQ}7EPPTV|`l867nEPVq7a@bldng)^h$-I%6k9*L<|Z&n%#k$WnFX@q zhH;W7Cs_CySxi;aD0f~Q#LGT!C7v})DEl-xZF#f&+gVi!cJrt;C;6%%Xm1R4+*p8? zC}Y&mcP4+gq;A_4ZCr5eo7{<2BWgd9tfW;tgNrwn3m&RNCQQbN~i&bl=!>+YZShugdtK^0>yCjlesNsf^?z z%&Of-WUYNZruTLD3)kIIW>KIm#K=xm|q?6T(5XnaLxdhj=jF4 z18$vcwH-1YAJ}_sf7!`4?ZJeb6-y0zsBII!sy7O?l&bK;EZ9hNw0T*$i{Buv=Yvq6 z5kv(});R-vTf4QJOSb}YDsKZ;tiuK(fuk7acpd9dF#&wAJX1gp`xh%C)=~PO4d&6H z$?j@`gV~Yt^Aga-D~cT?%|SiCnN)^p|FD4UYCgQ*=bIx;Z|W=w`^}Gh6%**9D|$SA zczM9LP>Pl_NBkD(6a;JUhD#W!X|1`eg40p@$v=h!rB-i~(Zthq$cX?u({?>j9$!pF zPj5q_VRE}La~?nysCAsj0C^~=F|P^7BtI)TVerr>XSJDBX4CTsJ!nF!t7YErPKe`_ z`(j1_ttV>HoSTq$KLg1hayI%%{YE`|RJkw;((sW4!~hyL$(c-^Nx%iq5pZxrT+i`ns6lijRx|>c3u~cf&x)x zMoQFjZIB%sMf8zVRRV-JLsZTC*~CzfPKRcIi`tY6+&!|nF;$-Qm@b^bVN@HToFTGO z%h^bg0T8bS@hajVd!;|;skHhH>>AGBvWB&MvwETQJ%isWxR%TmcPj?g`fMo|)5@}If%-hjpo9fXUgpm9d3>GK z#s<^*8LZ>bZPVP`LnI#D;IwN7W>_Zqc_{QnSv}N`LW~vrEdQsSv^+1X_ z^>Y$~fWysH>l`p**2tR&Ip8*IlG$z(<9Ll4X}L4i)x(M12DvQ*k`WOi+HX{36(In8 zVKsihmaAw{wDgI8_X3Y4R(+#>(9$z)9jUW`%hcxsGXG~8mpm!jG71OM2znQSzaAjp ze@1deOli{_zF`D${jyb?2W)`^j0Kmr3VLK5=;UfbLfID#_0alfk3>{Nv~qy zdk2~8I`D%}&3;PB;(smq`TR-yeO;J(xYUa}m-Ng;Ek z_ZDg$Mg7fdUjo7vq*8brE?>HN%cp6cK(!>KV)uM(9mJMX0F8aGWF`gW z>%Sg>1Qc=bmdm&{UC^$+IU)FS)6TH7v?&KrIOUOM8&R>hvSWgcRI~K`0{ftoa?r zI}wti$}5umddI-r1Pwyie0K`4c*C)w&58Wjq&pt1p>m?RWo#f7bE+0qBn?ecwqiB7NkPH47vN?uoR~9Wy}OhpbmI^Ff5@hO;rQQb~+Eu`t6R;WQaX6NZ z{y4)8o}HIJQ19vto1qaT8D8ltb4AL$nhR3K7=s#X>I0J}*xF#B(eD>+ z6`61CSVnvTV_**(QDL~wL350x0jRn8b-K+C`uYIyXW5cn*&`CNk^s~l$e#6IP&e4S z-OjF8!YE+M-8l(1oz;SERElC`vSRGd*qxa(5GjZZ#qw*ALf?W-g?cr-Q>R6!8DX5L zyCC7Qt+BPI{T^aajcWsBM)lzfHfb87YUc~}f)2ZLktD<>TC`i@_Bp!`1&+G&_maM9 z!7T5}8onTqBoi(_AEC6qK4I=-1SBm6S7YH!Y5l%5d3(TAi1sXsjcSj^qWOe9nP{^a z2JFbpO zXq(tf)2#NZQ*XHiDW;-#E+F2+J|EKhbx!d{MvX!?uNAkkIaR^?tXn;O{1^~%@Hm_8 zy|31**h&X*zW^uEK(@Qy1lWX!(ds@-p1D^Js0Np)6-&?4WxS35t|RQM8EpETsf^AM zT4v28)&ncRf>R3k&dA67Q!@_z{4P3-+*F*w;4QS-a3y9lBy2h{3_IZtsT3inWF&;@zM=1cfw^{b1@ z=_HLgb&U#?N#gpd%M}1MZf+*XW1A9y#WBZjeZ+iq^B7;_k|cp$Frdo5k0SdmucF#r z#>W^&;Fn)}gFQ3g=~im2jMtdnD3_I69z zg>D0d-+%$gd&sr46d{+1Do~8sn*PpZPqBHU@}65y2H4}QG6w@Ct^>CER zmG8cqUY*U^lZJ+vBJQczu%suIW*ixKvXeT5asJr_jCYv`qR?OXY_<(q>gtG4v|_4NZUemM${N8)giLeRFJ8S? z_AH(QN>=N`c~$!fzfkV<2#24b##4cMx(IfB?YXf0{1wT=Y%DpjVeNZ3x-b z=u&*rO1UuCz#Fi%48S>&Yd8`wz0ThK2Y^(R)daP88tm)95k;CI4wRmX-W3V@`mw(_HhYV`*IE<9q*gY((-(`E@_at-L)bLh_gY-H1G3}>01l^ypC*x1DrL4C;z5*)0>+=!;%e-Yn&az zvPwRPVXbDXnEh)!(%uASZ3oqx>AC@b3+&Y%qy$78;+6wJ4H1qFJ0OnkIKqz%C3p7-H7XUW9vy2B`;&n3(IQXG^|Y1X<+(cnx&=ia7|{8mSIyF_SLK#(f>H z{ga>t!eBh{;)S{+;l)Xp!~KR<_Pq~&cN6t#H;XAT*2I2(VY+!~BAkzaZMQ`=MeQ3UJnC!ZefK;Ce1}Ia1{#5N+HvWiY06`kItXT91q?^DVi(v1U ziMooe1jt9k6vT*LnetLi>ngF6WqBq|BT1n|2IL&f@DvmuB>w zE=eBh>3}1ZJH%llC20s+IgLXeNd!;(4lYzLC3L@lDhG^iIll(YMRLMOWdfk~1}0q0 zFC)Hx$j}ixtNvh~U=lXWLI$t7015S0r1mhP0k6p8Yj&W$&U==>>h5|YLt*QDVBSZ;kw8*?Ep>*PV4 zNH=y1@X`H~A1WNH6$G+VwJ_lM_XFK)*}7W2eLU5>%6yjD`*uILgRd@1+#*xGwI_~2 zfC1{~OtWX3f6Z;zBmj^pOuVIUL3~pB`jqPx4nf=Fa|(8QSU$zGPV^!Z_WnIT`vZWw zob!4XRHng-xN}C_lqhDVbR-WY9&E5%Mg)A$7Xn;5Kn>}l`o?IK;Tz$I?}aV#i- zutOjM`o9uW{UrtAL~Uw%!`Ys0L|%~|*h zAn~uKTIz=TGuS|LNAP{$g+(|Uys`CGu0@KPz-}DHw?T~fX2dV~t?H_@*yFC#99d&X zj4`Bqvx~!cZ4&s)_6>sK7(bRU0>0xk0wM=cxtn+dH@7M~zcSDVuR{}TXGrdVm?20`9d61LmafKy$yLM#MdOLt$;Dl?a{s zf5@;_$r*16%hx~nU`Z?HLxf;(iyN!s$QNBC%-Qs6czb=!>YW(a7mJDnQLh!A?$a+3Y%5B6VkP0(KbNJdeF5 zrN(Hk=_|DRefsa#Z3>+OXbPN{aPG<2ORHM2&v*n^h-Ju$HKb{%^%H-RG)gudnR_0~63=&PXn%dPFWhbvq!4)HM_8cf45I17NHevX zaX#jMfFV`wz*>stIgP27s3A8F>;h`c{hI=#@XG%tz+!TLa^8EFhtr_Ax}V?5C?`{; zDvX!A*VuM1B4f2W>$U^f6_9cUc5*^Qq0cOsKP7faRm>-_H&=jr<0R}&G{L@wDQaSF zjN3kvo8t@afQ8_fT3`Y2mzKi^ziI*wZM&46v1x%726nv#OErUdfK-VCWNjAM#066u z>@~U4Z>pGXsG@FEQz+(>C0=u9TzvOGv{*H%S})72;&O2bdV6iA>ncYR{G>H-q|OA! zp{VQ^BJ%eD8F(!9QuiHHAakFyQA5+8KNYjeVB#oRFDl>B-t}Whj?m z(Pa6C_()X;dh7=2kORKg09XYwqyAtp{w)5iZN6@mk3V@o72g0-7`LlrXs1bT4uQF)CK;wTh*mm!bfJo{ zp?6rp1*k^34JtFkD*IR)vO=57@r_Xw#L${7+H2JWGx&`b=kc^De}14;@GicDdPA4< z!W@{9wql;STqc7ze)WY5-FMxu)_IPcZhEVDGtzEk1NI$-ocaJ@o&m(uA?aeUWhB6y z7!3Qf7C)h`xcApOSb)laUyQnuz+3V``(RSFEKGzrD&v8`Y*hh1JFg^Gpl^{plGJkv zcdr#Xq$*ib3}te-rPqwGR5=-vp?5bhfIHP%@(5zlWzU^O#sJHHJDb_HWTW0v5D zZz|a6Za^=`+W()gH-Uz_|Ki7I#$fDY-x*6}3)y8Ej4edAkS&rmYj!b1MY1MKND`uC zDM}I}vSeSfMD``wlYRTWN6+(o&-wp;=YP(eQ(0!-%e}Ary03fhCnqDI5vd0R>gu`x zEkerDEJ)2yQYy&E@4By=(d7_=e+HO%n%-~%gSxW8zEZjY?$`9?v;%cLJ+3%P-t@aA z;cBTz5ClAR&0QTN`&Lgz#|A4jk=?bm7ki~WerO`&>epR z^bIP2FM5NvOEpVp1<#vSF4BIMV3bhziliyV8PwlESGtw7q=}xkpQXmF^KLQ(0rbjw zUw?lmeS`DrT%MA-@ryvt=pI~h{ zGjk8kK&cz@R;IE9Zsl_xq;)_9F|?Eg6>KkaBlk=4@+JxfAWI?8?TdjSK{65wQc{;C z6zKO174*sos(HzhxElbM) z@JJ)@7db#&wL%P?&lA$C8~WFcjpdC3rNG}k9V`gdOR0cMvU(s)15nYD4KWK=Fmw!X zGxC+c?xlA_$~#03eA(AQ*Fs9>mKmB$oCAM=he(}1bl`>>C_qSs5RT*~;qerpGD~ z|M!*80F5OPlKH_Q`tNtpZ^#Z0*6P3CRSw;5?R~CIe4={mXOgJp8w@qKiYO^HN6*O& z3pt(5Tx+t1@7&ij!M|sd)$fw)etT(F5VrSwe@|EN3IAOR=Co`DzhCtmjq3kwA^_~l zo>_NL?ML)^sspDEYt5V(&s^ zw@7KY9=*Q${#W;qU4)qG@$U2EgLJ#_OAJ+MR5&}l>^1t% zK<-$c`T_iKlA|(e-#ei~2QPDSxS2Re_5hAR>?#~Azb$b166-r2yx?lH`%DIwe(v-P z(fMEjn2WIs|62eW0-dpf_}DM@6`T*<_FyC{PnIhsnwL`%v8pV3bk4@=HDuB@Ni`ze zvNovV(OFO|-ujh#`6Dff;;o;kMAhT!Y@cRURtKU>`@1uVPTwnCeK$MhoDqnnp|W%t z_ranbh#cv_HESMkswQ~l?bQzp7auT*<$M09G0sKfDBqJ@`nxg+8JrK@i(QKRpGoHN zLxv*FUYWPWu}Z+>q9=kvO}kZblU->ZI4KM;3&M+E0=V&CJ{j4fh^Ov99ZxAGak zx!x9+8`&D}-7faN_4dw>7ilD(3{;*US7ce0Ln^H_Ga@TGvlUE!u%QGdPXs1;F;M`+ zzTRcFEpl|`IYk^DWf%jMu~(OZ!@u~0lYpqh?UqSn$u7Mpx)g#mK@PnJINYDTx+^8F{=9Bu9Bg2S{&&ls;2pi640zg$PGr=Ktx_xl&Xe6wh*W!b3LGAiy3b#@k zJtBtPJv+Zn8RIfr<=g$#swcWB0lG0A@b{?HJchdWuw;N$-nnGtG`ERTGlrhCCtpYLYYAsMxo zR&UN#UXBfdND@oZBfD{Fq_v+2kpn*gkY+bDTtV@_+d#qt*2cxSZ~sZw&r!1Mr=4$0 z(e6??@YtHt9whQevkP6R67?G3VOt*iU=p&k`lR?q?Gu_aY^X~R82_|^iYSx!>O{oJ zYbL6)y8V|PW7U`0WIc*D=m7MW0AtR7el+;{s(X*7M;DI^-o_qy#_yH)6JOV-8i^M< zHwL+;>nAMaYbg>O7`^O6(zjUly~0U9CeETVE>}l2)Rx>^V0gm z`L|;0m-q;0_LMHyWwQVNT={=7F_IpCBMgKUx8}r7#f}XT)1q7^7_-f-i~eh645B-W-Wt2R9 z36X@CZ)kI`f4DD%rJg)FD^KGnu}DW-i;Pr@XOl5q#%5qE?;{bYJGti$@v+l`SG~Rk zQ4OMFili>|LULir4M(K3@Y6=sV(+UR>1hH`Gmv10Ok*dCEccpQW>s{Zp&4RlBPOC@fSZL5JuAL-PXSR|KZ0q z#AN60X1A*NMA@;e?`#K;|F3~$yYOSp8vnIICXzvzUX;r1{h}pwt_!k58TuI?IC1-h z8pV_r_Wgc2Z@K4ed;yjgQNn&=|CAMq4|!tdjWBy}W{=f`HNh-6CP~d{kh=Da*f@mu zO>|pVAIeA?FB9X1t+6-5Exo@vTIkSJHDoKosd%er`ya2WC3w}&LK5G9dxBD6^=@tK%uNtFQaBJYpgRh0{|Zf3Bxqu_Y5`Lvg`M=+ngQ&grF}O9jlL zcobJ*iw-QmCYKfF$YWo_>LQke=*0^QXGE|=VLpss)5XiC)n=OG&DPO{cYJ1=o$eb* z*yAA}5Lyf95e-CVQ0g@7{%#k4WiJta7&M2KJy(C|tIS=&^6zR5z-r$wBz_Y74+AtH zSqEhF0<9g7rTnERpFxgy1o%E{&?xf*nC`bAA{?oG9uxzG(_z@u5zt{#QjupRaF}Kk zKK|3Um-qqNPI!wiRSDH(vr&-|kG~CGYvMai3`x1Pgd~-IL`fdBglV#HeLQr{wF7Qp zXN4uIzPlMCC~|3-WiCDg->J8WIe@?*g3x~*u7ot~N>@6l8aq&F7;xh00kf{hR#(nKxu zR;hrS55~)gGU=1W$srdJFij#qo7}tV0ZbIQ6-y!!t?$fB7Cqt!(Q-v}2WgWi+L|gL_a!esH(_iFVBY@A3502fs|I&4Jq{VyBCz<_@Gwv{IYS!}@ zVz@gQ0VtuORQ5GA%%lwN?;87io|yfC%T`w(+h^9Fcc z1C`tu3h<%s>#o@F?kqV&8}-Q?$PnrO&*!3LiFSt^O&&mEV*aQGtxtMrMuAMCb$R6N zT)fjK=<#HC`^)#aWqleuVBW2&&X;u(caTM_v#ztBp= za8V4mB-f$VGUN}vzm|I{vc$_=5`LtS4MUK&-oc8JA=oURYf-ISz(lco)n2v@+xQ}y z*8p(UX?5(w!-=ebAb)(JI=&*%8ZAQTCq*8>;bN>d9*0~ ziKOqe#r#)FEO^`pOL~JST9LFJzNj?+$PoQnqCd20%N;g1!{4d3@=Z5mJE&uo^S5%;u$fay5vHPdrP{y9ufvL%) zyh1Bh+oQ5#>Yfg6B8CiW|6thT%1cw-iE(W{y6Z!Ne7yL5NGTMM+=>#rdKE{dfOmIm zkP@_(g7&GTh{LpIg`Bi6C9%B%^+ zYfe?PaJ84}DP8(&;l9AatyR>Px&E{8WqA2xPDzJ5CDtE|x0gqr7n4`C$>pil0&d79 zgh=kH@ECbh5jfBw!jp!dcKG^8==ZNzWiP$$wQGs!7a;Z5ZhHSJesR7K7q$7dkgyfh zWkA^YV?~$F{2L*>A3-9IZEp2yKK|Y`UmdtM{w8eCcl6+)pdr`R+*;4GC=&P|D_rjH zsc7!6P50^N%Ic3}MRXj>E7Ha7Y>Bg^x|O8*r|dUpWdfGP3Cqr>BhX~&Q4W!RHnAH6 zk$p-Bqio;lLk;f4)QZ`}kmuLbf%4VIcA8!!vqcp9GEz4O#oebAA)3N&tzF zZhiQX^aG5jB&WJRmp@N4(ezo12d+f0ZoWjNylB{$oP5}|nZMZm_~~d%q{1Isx_?g% zLROuGf1arF@)Mi1i&5J6)Y}`KnTsS0kx)@e^D^7lBecHr9jrSp9ajW)R~k8=0Uqu% zw3tSY9H0&lbF8 zaLadKx5jHLWQhn-?bhm;CCK2M+XYzI7daGsb$atOWvVAYE;r+LqvF;oU}0&Oo+!47 z*B{cg Fu9R2B^BT$AA%0ZrttBz;Xp$)RjC8s9taRwxM=Gi52|Jpms7S_Ci%8-GM z>&&!RFx+vY$b-t_b;MY_aQ2X$dq5CrOEFXt8b*w|HgkKOEZ2sED0m*Q2r3TO7M8~Q zB!l76^vJlg7-o_S&YAuMi2yxI&o`D4(cgYc=E-!qA-FI_;o67cA})7K(B zSlFM-Ncr_nxrs-FJaPRV7yB|m&^EAmA;4U}4$J}vn)||RXZj!-*Vz1Sfk|)QKFd*|D^W^9eJfe#b z6h*z>Y5l9%pz;J3acY#W?MpVu9H8%Rbh3RoIX;+SqAJ(bR=Sg;9D4C!d!xC2z1MW2iV$LEQri8dbtQY9Ume=c+b1+A`E?&O{MJwNA; z44(HB;U76v!sO3JW_i}=Mo}^eo1SIDTr1lH=rWrVy4pL8++fImo~;z@+hN8+0XMbx zca$(E_11evtlkk6%tNq23l{r~*D?r95iDZsjg+{rYAW)S4=CrM5q1F2v&UpOsO$`40M+6A8~Td4kj+ zcAMf1D%Y1@Cm9wgR%A&=SnPFPAjnYsC&-|#!Lu32OQP`ex0eQA`y^6i?E!PL^j+u@ z3pxn@?)t*XWB3~JdK0~&&c^hYxt?blzek*(wt*|puIRO^aJZWtalH2!19i{F8s$N) z>ADn`N+=tMc!^4wld7OA_}Dzn2TX4qTc{Zs3NJkoem@@KKITYO0&&!ii?Xm`K;1&i z>tLQ?lMpXUQV1p=Ne`sR)c>SNdv=_+CuuT?WaVoq*(-lKq`Gz9d;X{#1aegycxO}n z@-%?9?d8`R|L|3F+rK>l&VkgJdR!`pGM!@AZxk_hd-G#Qfzt`EojhioisuVz{^}Za9z*g}V-*DM^3dBkP~LwbnjUxR{NW^3Sc9^skj)P6vH^;HEX+Q_*n?FNZ?31yT4-iE^7eIXKe%oFF)-bFF zu7#7qlJwtec?5Y)#`{#>V{-JpR~a+P5o#>VOZ>b_352-ccp{pnGi-6(s&SD+arYb{C=xl$Up%9A_&y~^FpgOCl8Sl*d~ zTeGaQ!lUFS!c345+Loy_tOXU&kK;KWwscdbe(h~i1}@klnDGN7!7=wCno$|$8@&G* zH%Ldxi7Prejp?Svj=G*)sDd>i&>WB^I&>~3|00J{khj)cuLWD!&X2&%4Ip;eS3r0m z=g80Q9)kGQAFrI?4?_ER&W{$$TeDBTXV_vv`gt2E-&viigyH&#h_rnk zmt4-Y4!C~i<&=PdkVi}iYcX(6;rxIXCBFSC`M1WA&w=wEgkzz8n}F&STgp68hL`Kw zfm%l0ly*5@E1pgFE;-|X8I(Od>|VU0>|wb+egS-?F9CDBH zka|BrKk>d&boISwS_g@FFXS;D5OGRNDG&iMOPO+B=ZGnoI?nF`3gLP*`tD}GBJIH> z&}H9}KJ0eCk3mCVF`1CNkwwV`9mEV&V=~_ukYh63%OKT6F9VZl z>Poos4`V}rCXjEg+=3G2J8Z zMwPu;dbKooNor#>&C-X$M`Bma&s9&tBA|cvT-3=BEZ`x?`(eKol<2$_04kn)iV8?P zMP|Gz{Aa-A(+_b@{4`4I*g^-7q0%Qn4AcHS+r}sN<5o(3L4oc<*Ap4B8IZ`CQo8gP z#*1Ekr*9$;J~1UzaRcx^rhGdNP<;}$C`UIZaEgExqyyyC-C6x1Gv^4f3!u|0^Hjk2 zTlx%aXJ9Y}q_qUuG7MZ330R9Yc9WqemT(6n(8VEH?0K{dr8_n!)mmy2I z`}Bfe-OD}+00Nuxwo6e@jO6>fn^`<;1WppmAe1*`6ENb`_tx1k_~dA(0Pvm2$>;~n z;)6?8?>)s#UR*L$1k!y?ESc!IpuKfk97T#hrV_n=eCP}3pCkr(WvztX^4~y#btc@p zl*wF>HV<2gMTI)Zl2E|d&Ge$c0?b!yH!t~{S-iawKTu|82PAs2&MdiZUN%Y~L0wD} z6mVn~7xCd%woN5EDI+942sG=qU2@vRS1aBH!TkczsF zym1ofg|tk1N?iL4{{W85X%dh7HNW0(sM-dor`W{pohM^qo1Gz4dcTl}^!rdX1|d{_ zm(`?>E-D#VJO9ITO9H3a?)n*M4dB5S%|5Y@u{qu#jTD~mVn-2*Y&6ik4Cm@gb&bHE8g33=UX<>#Ry~kz!8WJ_!NbUI+ed;3a+52?*ssbNyypc|aL?r~?s0C#MhJcJLg#X9lOkPY%C! zm-%{c-sSIQ!Vo``W#T+I+njowpGr=*GQlBCj?h!8@b?=`lKN=38 z1_QbOA_+|nKqKjMO7}BD858}I5L;e$NVn+1_>tmJ1mbLvXJ$l}qALz5Zewc*8z*3X z)@HNq-=gPr4Xipe{S>5{v{7_Ws8~h@Vxy}06P+F|C<2q`fs!N6 zWitfa2QoQ8=t%qe5~>roDSi#9x-(Py)5rI$X!^A(OVwEujsJgjEXr7CNWkheoAE# z0q9lmv*nC}B7c82<}Tz1wZpS!cfxY<7vKaGgqkRXICG8k118anl3952oM8C4e0p%M zeJ`+_1jPcEkIcm;F)VvLf;)gBGU~>y+N4vYL|MU*|6Y z*wdVTvmegV)bnqx8!Lz~La#STSEQ&Al#v(o?biql0L@Fr|2@+ly_9`_x$e0?&C#x% z_Nb#H0eL5xU;v!-bgqC%=#o^-K_B@fwNL^qUr1?>{thTu96U)xe^5}U1qa$5V zdB8@P;z~|U<4_H+>mKRdv|qJvC@tZDV~$wBx%4EZksMcFkS7P%=YrtZkd6UVUqOG4 zrBna6HC=82bm%mm=Sve@9U|q@gK`}#c|5cD+Elu_>A;8TxAZT=UR1%F1a%JFI34{) zlZYHq{Bc{}_kavkM#BqBl232Z051kaA5?)23>3+!CX?*T;?=rh4d#9}-(}r$aE#b} z`|REOsPF|>X>`PC#R>HdQLQTt=*xP&#H1A1(s$vxAU<$6C}U%#(-^FzkEc+_ zF^XfN>csrM>iRz9NO633*mb$${_IU@G>Q$)FJBoHGkY`6;C2KfZGxxO*=S~(s@KYY zMhDiW3K~mKu|Xd`Cd`HrL9s~k2q0=>;9&%gVkq#i@zkdA;_3v2@%`C`@%+h~7ta$i zv4YC95BY|A)q?Q%;2_ocTl~yCc!Sw!lB^lCzN)e_|MH3@#0wnOu`Pd}4R~a)h1PUc~%AX7oL7kZ>j&1N! z1>Ik!9N56}OvPFf7jE{cI`-F{%$Whu>fXPzCf+&)mT zyum?2z#$9<{~%8-cbBW{E)J%SX~zog-62p%eg0_ROCUe!V9rFl!+Ow24TjBtCcQiR zw%Jcykm1rFgk{VsAoVn2Y{I0>JU1FPjHOrXo1OPi!qSW4AMY)+%X_LclH!K6+L8Xx zRsP!&GZil*vSY$BWl-|l{Not|DuQP^<|teQp7|6hF+v@Ovgc%~E3f{sg6ea{y7YY1 zERE~exT_&8m@)H#he>+zg+qgC{kca`5AH2`Ep2HYa3*OyxYvfet8vE8#N*dNTj{mE zw|A~~gif>klCMNoP`^>sfWPA&HZ+OIpP3InbxUSgI33kl)UK16FGmG{UZjYE?I}^F zrd>}NRm*SPuWS`%!}!{Sd;HGYHc6huf1YtrD>tq^fhtt5SG(Akjwnb+^!Ro9a7Yhf z`k4I^w)S00XFi<1xj!tNkt&i5vD@Lc;aoy(zo<)fY+<2a)z zER4lSyl959Hhqqg?U=u+j#N5#1vbz*GM}R4WmvHQn-SXkR&a%6Gzb*lEb>{WfdWKH zwVzczAA`9T{b@z1BDJN?deh1txPnjA6oyO&mJdYZhO(BZMOn@roOl(HQ)?=AF>Rh0 ztyrpxdtc5Zr#d@gb7e9woOQu>?$i~9X?2`>QATL|Nh6TEC^UqBu#R(xXcBpKv4!j$5XJ%81hsZ9p$jnhBs!yL!W|O5j}pKL zbLDb>EPO{TWHl)t{x7r`Nw+G_2&{sbR`~tL>_%|D^V>1CC-mjC;=@V?$7~i7cQ!w7h$&;Rk<45D6G-TGxT5Ito=&j^ zA`-JHz4D{ZkcW$sBGau`2T?$}MiFY;K!1Cymu|s`aDOOX;DfxMANkM`-}xD|;o(fU z!Zf+7;0o7ypjPz5s>Po&=TdVFUZ;_qR9!CD{O;3hkA9X!mE}i$Lb((<6*tKh_W!ar=gf5TfbHASmK67_>!wm#31|C2GyDnA6k{JAk z->mHUf~4C2nihUHic36vNEGniSy{^j(hE0!jqbD|MlPtn(o}nVEwr-Kevn+A>6fCk z?fSH|&_G_<+x1~K0%%ra0Yn}MHXsm1BXF~(dV_6ZE7WjzwhY(*qOv-HQT|TT>J|u| zdnVv*i2j4^D3FkyX5L9v?X!1H+6ro;&uiFjYfVbp+l^>Jo#jaR%)HZIv)#qb6Z!^0 z665Rr@DXR7_B##fVdT$gZ~d>-6YSGge^|Gkm$f+ZpLTqrg~5)KhDWH+pjc=Dd^_yD zuv!6ldb3%<7Hqp5ewLxy5m)5W!;(fKFgqDhPw{>^)LQx{g zwoR7a-az%>&A^rL*;TOgu|K(63?K5vHHoy%DcN~&v&hyXPCglflY(d8SgX{-NdXlRf}b){x~jsHX2r zrbuuSO*M*iew@U2n!F8Ct%TU}zv}yC^CvmLT?{5y$gcF}UK`JiI@?E{r^davGBp|g zS#TmyU7Cd(BQ5Vk$++;k&*EJ|G)uX>Xo(5;DC2J>j!|Vfn}BYzb1E5iA)&Ivq2Tk7 zV)ZF|>D?5UV*Ff}z^Ic-`+FpcH|61}u`Fq3n68>aawmixESf z+AEbe#m?Tnxwl?`v$N}VVx%f*S@%Ma%8O@v8%=H4ya~NjvrNuQdFR4C6}0r(?=KUd zFAuW-{pb^%j)y5fG6)dZasQ0g?*xjS{+NYMcY@vDM0&khsUjU1sv1ZU@%8P~4fpm6 zw4-X(8tIunR3>^h$*Gu%{@h4eJ3*^X{Pq|XZD}7n!QGP^+ZT0UjWG?I_EOmGpN=~r zrHFRIf0lZ{0+ay~yn{7TM|a?PNP{QKmog zcc2V0B)Bu{7Iji=6e<_0!+m--JGTlA4kLzI7Ju)wEQS>Ir9?RIN7#oBE?HA#aRQ5J z#h_RnaNJ*rK<7n$g?fRnv?m2Ycj3@hgvF91Kb$6z9bG>AK9a{MmBEAz)BDXQ=Uks$ zZ+u4=l&coLc$r(0>P_8&_+ZVBPwyl&hQu;4eev)ZfI9~!s-va-mI}|=xR2`k7r%r# zOVfpwI^dU+S)a}X)k$TxHf$lVLA5y!uMaNv>V>)=$#Wke0-h+;yDX`|rm%NGR>%&| z>%51Q7k1?FzBV@_Y1!O4T2O&>5VK7P2GMQMGH}8F865tHi5Er;5A4da`IPr>hdiOs zHVEHsynZY{GKdoK<-~DqZR+jpdmHY0^?Mofs7s$6tVui^BU&Z1OnllmcN<6CF!5IE z0i?PvXsJ#Z#$kP8sTGUiqn_M-w)NC0hNO@akIN|c+m4CbMsZE^B%Pr-y_LJXi zKJQ?HBiYM~X|B}MQmnK4H{69fG)@WRPTjp7I@^LvuxKpF88BB z#kWf$iGX79r=z!Qv(y7nw$D&wt-KZLyVp$b&VIiC%s-Yy61};52ISQ&yqfSeFu+c< z*2x;G56d|N@`LV6Pi8pRY|F5>%GlMVUHT8IU3#i8O-I7v)0vbT_DrHQ3Cv3sL`zKV z+>YPxGBJG~pkL;7|3Mp=CJ1z7`AZP?Knpgw$fXaYcPQ{bO}UrhhQ0Vpnd7e>z(CS3 zYMm&=w(G8wZa)lT-pNv?14U8}h=wK7N;LMYTmJlR* z^%ft|qUX>*F<(6jD;4*douQOBTxof7g^U&jjS1#Vyj+5e3t2weZYz1Xco?mZ?KCz_6^lA7V z;ryR2R_$`?wng0FYXE;~s~QZ~GFI&dElfJpI8eSGgGE|G(CDJ83U=?HNFwd1nlvwY z{&rc84hGeRU&qJ}gU=6qs9ao=va6Qwnl}QaT@@(pUQ9x4)n1o|`LYm`iHsCv`MfyC zmi5&iqXnaPr!iVf9s`iI-5p(FL%Z|ObhM~1Gvxw#hS&X=K*1Gra$)?*G>xaE$`5(hI?LOapPefZR|-J6w-c;#@f6Z=QUp`-UFv>0OEsh9}< z8r_Ckqj=Y7FfqnRs!M-F6htCl17p3ejj~pXa(Tl9_j8OnvjqB8mv3~8%!KcK{8i&Mk+?y(TVYjwNY(O#1E@1X`xdp~17!F3*wSD~l6b(G|SUjVUbkC%>3Z5$A$5 zC~8c}A7G?$_852{v7X@(I^nU+_v!`u+dbpov9`v!ZhTe3!^g*G!EyLiX~rncPWwF0 z8({o&mTkR@ADT{It|u+gc;^D_*B$$wy@$M)6IH-rR8^CN?}aw`U_RXm#?cVkog4Ul z5y`ZZ8Y<^6DYe_d58teQw6@1Y4KI+g2OJiCX*hWPN6=cx+nf)VMHIevvLRJvj9&|i zt)kf&Xmkf!9Gy05K!8(B%h46!_a?{Q+!}35a&>5;%*S-VhIhKCLT^YxRc3FJeEp@Q zh8JJr%2QkTP8?Zi9k+cuqBsCa!==K*r6h25f<3i({LoqEVMHZT10cs!W1E5!n9f%@)Kh8`y4DKX6Af11EyIeGAO|rr3!h)odQF>j;Dtw_a;D>ATbn6G^3>5xlvzHE zK<+Eeqz({BhVLdB^4koqm=i4Ui96$?OI76Gr*n{BpFBfXX}yVvSN24-G;-bWH$3~} z_*w%E!3i&r-O651<Gh5c0DygXHJae-}K}6`KC(q9}Ot`*ZEzZ7^7xU}gi7kJBvB z`jUgLDstTyo7Y^umA+hCcX+qBs%*P|F_t7jESy>H52%i|fzh@C(7tzWwLrCo>1O6T z?+r?H)6G1!lG_)XmroBQIqqAdYzk=VR(s=&UAdqI-k%^2Gt;iO*VLbcMZ}Oeq1dc- z^@x~%@FoN}9qn)ZCd{mugBDi8{2OS1wc`rkz4l=`R%Cj$<=3}YbA*AjcJX?p)tW=| z22k2MhomXgf6P1OTif^ z(w+gk3uUBoW1@6e=l)8IfATcF?$AAy;*FESm)CNI8fbKlPQe{)@6{pa(Wu_SfXAM!Ywvb^eqK>MmW4&^mo%X%ddv>t$QlCz+I@?Jkn)?j;WLOfO2>>(&z zEp~%iV?HPzpAY%tQf&LF26Q`Xt3H)+Ps<_Iy9>srbf0U*&w^nXLdoc;;p!Hc-r57r zMY9PPR(rZprulBTD(D?XlgRs|wv1G@bVGQ8^(C0zj?Le$26TV>@WsH-y(Gip#d&)} zB)Gpd)3@c6Y=2_J=Tq}=;)RmG)CdJOLL*2p=st+DJpc`t7eRTM*z3z0sK~J@-YUu= zy=)KWc7B5n^>)yPDNLBlBeWL?7-aGjs$iAwLzj?zQ(#%?U|R7jIDTHxg^|uC<5n~W z`a9MOk3d&kuOjGN=kh=K6pp)p6Ddz!9fP)SSX%}yuU2waY(ZzDB2OLNHiE7Eqyh}U zAt_{RXLMm-=(KxWX~=fFZ9Tibmxu zVYm_WJrc@=#V6Y37>`532vMr?SB^gzwJZPD4Uh1zyw$C z`(Ha_eu?j7KvNuH`j0R{bi(aMbhF>c#*(UuWvTj_E0zU`G~o)Ktwyox>PJ~jlg4Fz zylVdiI}kHGscq6xHC6h|1xeMr+dVh&ZI|^8$r{McCqF4TCOY!tqr>7RgC4&Cdk78L z9}ecG4O(~-I14&v&E@WZ>%S4V97%C535=NNfZpiYHo>Q+m!WAeF!%OwsxjJG@x4Yg z_4$Bhiz>zHwYr0R(4ND_$73Y@^V_TRFwnH;I}WIkW~mEGv97SCqJ?P<9|`GiUyiUXfwWEJGq2up=+Z(OkP{G>S~@#()Z1$#?N=! z8+*lPbc(;EH#O`#ecE;6(6uJ_P;7&PDzhM1bdEJ>tUB*8EYbE@>O~F1pS20Q@Pie( zDs_oGXiY4Y%6Tz7GdB!kO<`}-ldntv0aP`&7J$DG?#H=+=Kn+dZRj~_y)@BeFf;M$ ziyA-48(A3@h74K$=jMe1-3@wc}Rf;i}j%ljFSJJeTmNC>TwJ1=RmB?@RUjm@8 zoZnVsZ2&;wiXqK~bQH{bhYZ&4ri&th*+m>x;jIXRfSJCuOr&L>wlhg2?}SF0Nn?$A zs@$(IG9ceVD#$&nr%0;KqnnAVg|htzEh9}I=!_sPLnC3^^j6lVv&>yX6x}r0GxVd)u{G#*CQSr^Gb1#oqs5FUA zy2P}Z(0Fkg?fSSViUC#t_~PNL(Nb@_%*XU0l9b^mM}ZA%CWgvp2JaeEx0Z(5gFTWI#UQRW1@vQA)WbX5RCx`W%r*@E?7ifq zE~rOANkEW)ZGnAPOV@%Y9~bZTN#Of6dY^fhpi*(`^F||C0c!EhFf5l4W+}j==%3EC ztq1PQGQTgrKzRSSpqaTsP#WXW{6Z{5EgT(AfK4GESWKEVK#t}o_yT|MRCKk}pB%p` zcHFp){OnXooF{n4BtNe6scJ-J`sN`&L51R9_ZSR@Rj6=0S##ZGn&Bih4&Xr!fV%ZS zrHnQ1L{#4vj5j3vu>FqWn`1t_ZV7U(@vZ4J+I=Su*-b2CN@#WV$xx%wpv5R5)l zGzfW`6hupgl6R%1RE8(>l znJ&x4Qz0yMlRgnQ*iEsS6h0I*FAw;*Zd7h=^y!H&2CBoaegci+#1rnx&5R>w&HHGXgzIf`=# z05I3nHq~XGf<}Fhb}T=~r5s|~`R8)o(e_nW(4cZGJ8uaZ5&Ao~z+`4AjiGzP=+Sc6 z6}|?#HF=vS)s`ZRr4==}5X5WIY~oz))AY8YxtaU%B~aLVBxIt+74nlu?cx;^)I-9kaO6M* z5}rgm4|-r9MUm2c1&zjAnxo+7u7hTJw}NudABsm=?&P*cPr`n;od=yERXRbSabzo0 z+9Lb*_Q-iKAIHd!%px|Y+~t_~uwlapwYSU$v*&n)K@Xue`0}wM_-VUh32f0|(Bw0G zAB5(@*-tU|p2UD71XFFI)b@1ubVfl(bxLKWbn)fq_ms8EV}WmDGsPt8Rp@T$zc2ET6Q^a54Tm|L;UJfV@yLo&!D9|0y~+(x1|@L~buwZWOC0bnl0 ztpNhA;I_eT{#%@=r0yj}lD3$;@Gu=zX0#)JBaI*%C5r_5<(0CW%T)T6xf$OtPjg0t zXfTXx?Ym-ky9%@unb}a=3y|~@nUk_K{_K6e@>6z6hwjugYn~L0ShPTU?Z54~JM~E| zszp0sqQ;e1Q$Bv&N?y5eoVG~+^kIr#YR_iqHn%{%*RhH1?WxCcOU$Cj>c`_%df)BF zzlsG7)6gKXi;{g}pRHa2WKj!^6K?_2vw5{g+oO*YqW8y$8n?}u9pk-_;&Aa6$?^nc zV5qf+%XKOP^3y*o98Ka3q{j2EQjGUBazfaBEh<057{qTUaTWw^OtYDKgP!FOF$a># z0vg$|UEAnKCC{cgFjCyRo`?!a8q7J{=Uxr{3~UF z#Ey!H(pkOpD0Y-^z=L6ld_a)jculAT?@mQK5uVs^^Skuz4_lzw$u&a#1AJ6@p+wvr zWdd5u80*d;I~yhG5L1{aFx9pc^@CwI^ao%9$);H=3L6QH!fbda*t{c>1UW!LbMtHi zl4{cVp3;IO^m8vE>Vz%(^5*jwLJ`F@X)-+27-!v#<*jD z7#Lx8a$}lkTLp9IF&H9xRrk5CsWq-N%=N;Vd)Rh0&uHbNC`hm9c?NxapBsvR4M06pGYPQd(Sn&ngY=h&F5@ts!woJ z&$;@P8?;l}p4|~VJ-xJWFmVcjh(r+@4ZBoN!nTA$!90$sG>e>Pwo1}QG;BVg5PjI1 zBAA%d4$do}EcBzc2ONeVnh4vBKSp~u(x_;tC2R#@qMQZvQ`o+}y76nDQ_k*18VpG_ z9NvWUTr20EBYGJ2czN`_UH5H=pa_LBEt!u#~29o1J#>yJaWB=4EAk-k@;4RoDiIYYi&9Fk`ln%l0)xKIZQ)d{uQXD)WB$f>~aWZL(r323} zV1Y1O{P}Rb?IG(?gI_TiQr^$A-<E^G0Zzk|B7G=0NzJC~%9jC>F**$qhT$WSz& z4|}6C!&ocpRAR-D`+pew@^C2sH{6-Q82i|F#xDE56dJoiRI-$1?8;8|ZLHasY)K>} zm4t|_V<&`?ecwabvqk4K-{1FlopY{p&UKxC{bAns{VdP(+|PaA&(`b*6HnJm7+Akac_0GQbhPqtR9VS&NDZ=4%ow(c8N~-c+XA zQeyC&hHcTNw{MMPdTB(gh^tiwI0)60C9uyxNGPo*?aoQw!&l>}&qXHLg>} zGrT*E0jED+$$nHTaDrqLBA(%?p`qO!bNfJv`5=qfw5I%S&Gqu8-*?a1##nU8lm;!m&ZL+!8y;0W*-2x1-PaoIP6N$hvw(1x)PZf%NP$Rf4| z_jw-BEt4k7WWPRvBcc$udyQU2+KXWC!qkSN=jWa-^sd`mg#k=um z`{A*xUQpT>a;qw8J94Ypff%T3`$MKeBJ2yNq1(1p!KsnBUjSikorL3;XQ^b$liS1<5fL?!vSQGDf#Y9@ z1zOhyg=WV^v*^%8^H_47hS+1$isXo)`=mA!^EpmZ{PPWtc-2pjf;kPc9Wp2rOoej~ zcGpx@Mv9pJFKI+!h=Rk|4CWvGsnM{H_wOj2=YlR@#(e|{B8wnQ!IX9WMN-7lab#3w ztBi;dI&aH=*WU{<-t>!BF&=;)GVOX()sf<)NJ<-aVvWeb7Sk6K)MZ4}IdAhl1536@ z6q-H1Io$W`r109LFzLh1&fH-$J#W98qYxNnRAzVsgtvQv6l`1#zCUkn5;D;ABtvZ4 zQTeU1W9I3*dYz2cE?d@*6U0Yi;mHx=gU9l-i$rH^tphv1kyf+2B64Bg7>1eZ`?t>* zlM&1Q1cq{!|MG*}M_w^^}P_XbjPn#;8!8z*`s zt^OUSY_$pREDobswUt<%5o0}UNU;@en6a&GvKKITSyy(gkwI$_xztsDx&Q!Ao5Nnb zkIK9dmYk#lBXp}+$F!_M0y|_)FGM{B85B|KD>FtS^-*4Jlixe$+U6eI#t}VVnt-Q; zcbb86^V!P4=wlt!=p$R22Da+^Es5=}=I4`~@5>kwaKU3KLT~kBXI5c?V2(|NuEgiu zwp|7kmzyf+DoV^{1e#=)*qLP(?+2`$40{E>jNijdBF(B?qUl6T(Y7oybABSR>&2{b6XyP({J zg>TwV$*igt@{xKnSWLKlD3X3Qa8uMrAG zTOE`t(Ehj>4l`opH(fqSpzQJCut|J29r453#hkAZTP zmyB`hkO}I-wxdNZM!d5ky8^O^@95}?y@9wrGp@t>B}B!K4YG4 z_s>OG<2uxU1ftxH1!>Ke1y|r)B%Ix7eYf!XRX%CX-mtCzghdLq&TsE&F>|&@7QE79 z1hqs6ck+xvu#nc2n50hyp)I4&KdJghDcQN}+{tCZR{!>DX)piS`^C?z9g3Y^^EP};aaU_Q_6El`d6T-&G z^fIY`Fu}qPDOQemm%azHto9+k!aqDH)=(3(`y+C04<`K5d2J0g>63D{ejlPpo-;96@-NbBzv>P z-Q_A98vt)w7vMWCe-FH=yYLO;Rci32p*^WwEQCLSij?9qtQf1l?CY|T_S|`ZdHt#FY_u}12zN1|50OF}mdkf2t=c`}x0Cu<+?=shH5&5xo4;KrMwZO&> z4l}e&w+tueu7R=?I!k6wYYC9~nm*w0NmMnri^mG7lr1+e;aB`Y{kT~!g0MVj*;#u- zwixq)fl#j=TrB{4x@|Q@L#XyYJljTXoe_I(}RbSG2&~lKN5KudvK(xD*G(fDJQBBbNX5wHw~PzP9qOIN(W=%9w9iYX~y|QEZbfWBSk_a zP=-3|MgPy|z!pNsuvUoU6~!-}$M%|DY{U3YpeVvHtj#o;j;dK3ieO4nvDm19$UihJ|%tBNWC^W!E!+N zKQP=9bCfXNPY$?|YXY1&!;?jXEVYeVj660BVqKt<#(l~h-}?n(5(^Oh&j2>zkk$qJ zQIcRjnisIiw~5xW)dusGWi>??YXJr$U`M?3M>@x$On~bOMOhstGxKw&xO4WB&dQx< z^|r5^QtQ2+d-1HPtE`kfBlvxj!vXv+^9;Q1V3ur`Ch=DwEt0{6E2gjXW)Tp_KayB- zX(_Wv=IJ8Bfd>aIs#mX4@={_zi)wgi@lz-l&SyJ2XIVjKUIE%N{mnARpw4K#Ltcll--{7jBS>w+mbluYp+Yl;BwCi>yJ@a4|yTYQ(Snj5z!Yji~^ zFah6zZB5dd7ci`IhTQFlQ3Ucl8P`c8yk^{oy8SOG421DUZg(GWS%{i9VJ0&3^UHr> zEQLeO8rU9<8FQRD6N3!U*U*kVABB6=C{PlYSDgbHYZ^eCv6JxTV6A@r;I&PxJHdt(Wo70#*G@a_R9Btm$3Vhjg!kcU!`(5l zcv%SDI0K!C6<}dOOIV`mn*-VrrY!PqB0wIO0d%aSU%xj42R4>s1~;o7V`(6|r5AG; z%zqc|%q7IcSP@H{zD#>w<*S&?E6ywToDM|1#}Ro+{P?b4Z^0LQ+iO#0LK^q`8qN|H z`svlrmye%JV(1MU{dW^h;lO`om`Q?G;6?`UiueH!16p+R_spsWM zzfKqG3 zZ`gXJE`?8#w(IZgeU8rDzn-ld9|%=GTbF`awlJE3Clb)-WW zsA&`GIZ(~5;Nz8c?p0M$9s|aVj_Z>BvK#H6Ji@T1*k+WlKDbdEqoiz=5Y3y1i`hX#;!8WHfqU2g zUixoLJYHE?&-Uz4eo+R9r{SAKX2^@9mcP9~a&*wQ4H`(|K-g7r+6^zcmskp=J{-=I zw2e7k_1FuDgySJPh-ySN& zJBfL<5-~bGxW^VLVJ$VlG>igS{OJThPVHCwb698>@ZBTO%?u)aYc(p?z3|$;`^zL= zAFhuwOF9ElaQi~hhU%vP#^OjPI#%A)#4_OJ(}7iB7b<_2ciHDL5E*XV6Cs?B7<7{% zQX|x}_uHmyQcNys#;gVYychsXK{lSwNPiQZ4fE}qpw_q2`=FD$c`sxW&S~4aFKv*o zBm}{US|9H$JV?T#V*l}-gowcvlGlXu8)CzUN^kNc-N$ZvvgfyYs44)L&J|ER-GQPq zoFLAGvE&4B@*!tT0A!&STnYJYoBxE+2NRVa73I$hpLs;^-`WcDo*g0ro zzIMlzmVm8%*RC?qU9*{JtNvWuzxJ^7Erd2N168>3JMx_pismNZDM1h7F;w`wf*b2Z_*o&%*4Dh)MX@8+Y9puG(WkbiYtlssk2J+}x~e%fj|It1~Kz@e|0uN6W3c79#js#r$JIqCut4 z(7?h87G!Cu`)$)Tl*dea2;`(0h;`qFeymUcpQ3}aqF&(1pXHaj*`p6Jv5pYk$l0bK zXE!fX1~jif*y(r_mgS@Y0QoHQ$kuk`PgWR=RD}M9@}Utu1@O6zchareo^$%ZrE#U% zY0$Ah(kbp_?T_6Y*#%c`nu0z~+<8oV$s_P$BEJ^wke%OuDvxd=0rQ8{RO!1sVZ^(j zy4pX$1Sw)}y#J9@&2)p`doMwwQbT@@SY3!JM6p-@1eku79!O>?Ac~zGgHP>4w*f;N0z*#bUPxCcAGZ^ z^wm+AC$LsX8&vlJwa#i&>fK_+yA32DFoJ%e@1XeRzC1`WF7mCCzo>yeu-GFbUcl6O z-tTF&U32N%SI-0|?2U7YH)=v+n;9J5QrpoP@WIyZf4@v@keZq5b$rwE*0;RL$Q5;! znzgHUv^ia7uAZ(J?S=h43S;q`FH8{IBYbK%(-O`=h>e#2<0HJ**CskdiW*mBH2xyo zCvWF#8~YHJ<*M+ccM&=^tjTpN@Iq1C>?k4(;lezZtM7pKH8O5^Mp)740z)*opM z(XaYNR>|5@Airx?ygbr5&(QqH7wkQ{izmRIt5F{8?5WGDvzn-Y*?@aVg- z@U$*jL}CtdK+V=X(#h#$Y1$4Pqk`_6yy^ehxM}@0xQ|Uh{2zX_ny8$d2L6>~#FBkJ zKh}rp1;iNsJoNOnWB22-GkJ-8)nZ7b>*~8!I)V=zgmA^gYW8Q{b@1V#y^Bx1^^A=M z$vir8|5K9R1>XP8&e-}>YcOd3$7PtoC?(KKZcB!Jw zr~+soQo0A4=|Bkec3L&y+%gf+NuXSQveqiFbdmI3w)KHM^144L8epv4riIt$5Y7a_ za7ivxRfwi@g{C*9Cc&Wib}2kHF^KC8<(?f40uX!fUwB@P59l4i^%}!4cqjvX#kn_H z+X%>=`+6B*!bg?%Wh&@>(6k^)ki^zJg57NU;q$!ol1`U`4Ir@R9vkkG_A6G1N&F(x z!XQ@Mke{Q&Jt=5vHm1$o!7KSa#=LLT`p@s|OrBbYrTT$)69Vc`YG@l+fTKR#0 zzkPcNG5!D8DA;tWfa01+)A^ceUr+AU6-i{pU7`+;fG$2Q2?X7nXXZz{2Ui=M(H*$?5LR5El{mg^0c%M_H8P0eYy zrt7qGV}@VX!DSzeq=G6=#r6#zEe3pr41s)qHQ<;)J4Q2J@ADQIhEFm|DlEd%20Cxd zk;So*ZQc+C{ECK9X6*;>RFdT^!A~#p^=vAu<9Z)8rn?3>z{ZDD32iLI)9kckYusw} zv%f9vJaxii;t}Of)?vdaQBnQ!#=Yw^)9-=P9bhc+;_ZKd0Vh}a>LnJ9n86=GTve94 zA~Zeu|DKrh#9eZlV|-Cr9+SZ}zpOsn`|*|oEJSI@bTpy&b6TFJu82*x&kB zpYVPAihGTTMN-62vJ075_}uA|dTsWh1Yc@{#u?)YaI7u{=5Kg=aD0ay-|f8xJdVt2 z9%KTJk~V{wxgg%80@#A~0=W8>F&QXAvp^zw9U$&@e~x#nWqMZO4^J&UmiDnBd;I@i zyt!{M61c?y(lr6s0dp}X0i=(O=M1S4m>exOj8up_Rhd?QwN*K-*PZp`#~8u>g|>`8^QS-bT0LyPUp1 zCm)DnVZxw~b{q^UcPYPL=4LISO~Jrj-D_uFHAgw-N5ek1%HxTH-Nf#2)`Q;ShS&(XY_opn4KtP+xs>zmL+ zy`CL_R89^o@WNRyOy!QRkst~1^a#9_P8qTMtbbAQt5=R9uR+xBW%rwYQBEv~ zFI0dzkvbmL%>mXbMsJPv0a{e7O2;LI&ZrP3dd)cMBNFm(&ocI97%;d815>c%t(vBFoYf$hm05 zeiqVtQbcshU=vzJYp1#8r#&L;d2ojXz-tfBPpm=XEi34RjC4Ye7l`Re<_jzAdTDAz z6gJB~$OQHLy*Rpxw=>86j44z-CN4@pf^iP>Vg&G;@>Q3$B?C@$IEBdUx#5$-Y1R+A z!#LLc4$Db2?)9T!-&{0n%;IYyur@Elhjx4`L3=xYqv3WQMUC7NJ^7z%hP=A4-A-~V zOPe?W$mj$KRDn46Vr!9^kd!>FTs>VCx(R;?Di3C>9`1KM4f`k-a#lIL4+2lTe%6d# z2Q8KFUh}}teQ2BzBzL3jpmCTEI0h#8*3B*{ilf?P;?@UIe9@i^Otj1xQy=U;^Zt@% zpmGvbs4AB}-_evha~Wu>Vxz$=;8xgeeLj`4RySeZ-8MZi#w6Jh%ahjiz-#}>pK1Ac zcxb*wvM6o1#?|F_Ns1F$Q?Djs;dnkI4d3;;riN=jtSSy8YHNKKl zjHgS35s~%)sPDiOkRoLHVfnP+D8?6`W?U7ZK2rOv&Ii#(fW(R4i_Q1^r=Ojgs~fMe zyk2fY>juy$I^kZXNfFYVq*6t#+ud_}if&ZYh_3-;j?bVKDT1#T2Y~4>ie+A&`;F~B z)|(nhkv7r-sG=d>ihz$eV^n;%vYhcs8QC8}%!H=_Q9Pk(%6P?78`*>u`)?d^FI9`3 zuT1_17Gho*w0#GEamTS}3z5Mc^lvM8Y|rQqplAGY_job0s^?X8`_ze@0v>9J+>F^xyr} zYVb73a+=o=A-FQiy;<=H-FOtC+=LP3MJj`BJEM`<9y>|rtGP5?kg9obWXk;>B#t`u zWm9(uX?9boWYGdsUS!K)*S@xripYO~s|{H0Nn_a}FKh)Dya1>#ytGt~z;*pbaSu3_ z@_;r)ZkLjR$2=CUrCSqm_*ih?v!zV4q0731z?2DZ0nMx{`oxg-vpzKUpgwco{%#3g z^>Hu(fE_Km2Gq!1?WsO_w$ER+Du6d;JpF6m?Eue4-x)GXwku~_uK#>dCT;GGgS`h_ zR7V3GfSNx*5|n|0-B7_ka}idpD=cToJ;%3O>&rp?{j`X^F#J+%ndPe+?C0CG21;DO zF@7Y>3Y420lOC%%j#@Jx$HZ}g*Nh-2@R(P+p|pl{Z(G7jjj~#!B7x?UzK+0DcRrx? zOPpUGT=x^x7g4U+-!uW|plp(L)&XaUW+G91^W(nd4j+p0a}CfP`2OS|*5CUeW_Fjg zZdS*=d#j-BxNiik{mTufcJl^mwTm` zLnC+D2Sch=8w+=%{>O~fm=$@!E$X2TtoW;diLDL4lqPu z>@gt#L1;{wrokn{Qi}m`nmHVK1PF$0IPez%fE|9nnz|#|y$TJJphi-+Qr~{Z>*dN{ z-658|KjOFQ(( z`s<_tKK|znJRjOwNb5aa*`*dbXN+M>V%(CBSh)Z5rQV1AUHEg*g2&gGx9_REUQ{THJ5R&(921JWD7HTz+NN_qwc z*?^7Q2j*7rf<+m{#l}WsW{>3n7g$yKp?gi7xp-XJ)V0FhL>?n-uTx8@CM%QV&0rep z;~hNFdG{BX9xw)Zi6p$<2tZU&IAPi&tPNd=hv_=x+k@IQ; ztKi*e5%W5UK1Bcv>(MPf*-UcG>za0gi5-B*jg7g}g(vZU{^dDtVa0ZP`HgOG=bdja z#G;KQPl|0bvSwt}w~|~5p9LQ)Ok}xYT}lxtDWH{cN1hQ_Zha@#^Cep21a7T-w#-a& z|GfE?hEFy7{$)#uHDS{IZl``#bt0{5?`LTvnX$M>vDdtS9rEW7gDS${a?Q$-cKi(B zR6Ag1#o}iN5V1V~x%L9~+4nqx%4=>-hsH^^QozzL=ZYW5O+O1)d97Uk1lUP3mKuN` zE>rOXaK=a2f1GZ7sK22|Sxb!xqfx~kX?wg0U9dTN%Ej0a*JZ-_AJse+wa4DVlHEOyr!Zex z!tj5ORa6(q%a$8&eDsh2RlldbJwn!Td|Dk7ZV!QFs7YX74uOgmZw|T#_HL;?#5_51 z2-X))&7-w0xs(q{z)#zG?SliIr0;j8F<=*r~q{z_@D3n2|Xw%jdB_s-Ov(YejLke2~ zJShmWyKmT+6%yzz>bFjY4Zg0eROTsq_d!3bi(9e~8ev=m<^#N={i1wi{$sAC#X&Q? zK@E5gkHkY)Vp!b_YvHC0s6(O9rN0$-(bKeVogl;9BZ@aFQ>ycvM(-E<@GmK~B=RtC zjXOAj>)FINrTy3GpDK87r-k&jVVuQ&>nv|7NF6x^rmmx;jDI5k{o2Uu?gv1mWiKF6F+dG)V>aZ-;{2Mf?rlasJ5^OJqBYfJV}r1X3_5f1xJ3Lo8lX54TUWgE#;CW7d?72srcAFe?jH5_(`386 zuBjN>u`_$N@G=m7@P(=`?1k_oR(A*#mgPq;SmUimGiR168dT)_YuH0jj8_CFO~7S% zW?IFcP5Ym1FV_9#mnIsOw#k>~qb(sbb)(mf4M|?csyW3%c{}a)E(N&MBsf$&%&;gd zIcW(l5JChctGsyv_VmaAged3s^$fum%yf7%_9Zan34X2mpNj31*Ge@QN#XZkl)hiF zgV4@GmBjZ3c*rVpqFb|cR)BzH2=fB@o&Ux&L106j{6?tEuUJeIYiLIw z_mc7(N-76GENGIEP%(P`(O_F9kPuno3txP=GdfQ`ShjojR_BcIN5DTDl$-t0)F2O| z-vCmRz;A#!C>;$lh+vRGg!h=%s1wqBvZOJqRrjyXF1kWiP-9T;8Ig4!JsyIFm)jg=kIzbp2UCb*#Y0FV|)T5so5dKJf z6$QqZRM6T$&F$s`Y2V6%W>m3|rqC_ZRZ#Vaz7gZ`7hH_lnr*t*`N4Dp9a*ui>O$SY zs*1i@hp_PMq-QCh8`XFbFMI4{#GbNaH_O?odJ?T0)rVUA*28CGl>AI*7)Raj(n0a> zh0Z!&MYT(a{b%Lr1saxL=%T}ZLI*0B5e67pED}5Ww=;EhWa_Wttg4E~fv@;O^k={`qQ0u(}M|IQeaPy-?97pn3TGkA=2e#gFKF z4dM=k3Z40MEc1?bX~xKe%D~y86NQ>r0-5V2s4y*mqE1`gvui zI!XcKQ+mbJQ94`gu$xpO^Pkwa@4Gmv1(jzLzHK&)vhgi~d{>Pv4wPzKD6l}=q54j8Rf$M7!}u>ohfuRy&7+ZEo491ty|x_I6c z1f6@&vftezz*`tg%2jA|W0?B&i*^lad%72qR+n#ID`xuEY8LyYi1({ZDuEGdZWJlCNP`Yj)Etpwz2;EH4rL@F~;?h8K8+?*62Aw5{ zUgO2#dn)-bk%q0Rg{Q+Wv+fE_0gQqG0x^W#G6!~>321^c__-G=VzeW0^Bp6W$_GI| zl>dkGK);kyJ%7Qy|F`r4O;`GRf(ob(@&Xc@YBciKc`hu7IMs%&T&vS}i0WrLBTCCZ zW>dGuRY{`m$Hf`uofI8(CP(JIJ3*j zH(#ymR}MJ`B1$*aek3$MRL$}^;0lNP=tzE-q@{84-e;yix=ahx&2!O&Nd{QKC}I+> zw1nnnnF-TAXUSnEA&1dakoyctOFED)M!LOw`yAr!KNU{SM9{cvv(MaoVL#Gvs~W+w^>NCL+2BIuY>W1B@E*=N zy^>reRVnT0PAOsg8DNf*7K8^AJ@$WNK$*;W4jjC<+i1C+JOHEhkz+KgM8|nw-O|8O zqbJF=`2|L``6EfC*73x{>Tfs54mZ=o_{cM_Xv`uZF~MBa)iyFPA*K)r6ps3Y(a>=N zpS>17zYOf=$=^CvKWELgAn*X^f;?2PN>bn(D%@g1Gyy!MoR%m2!*qEtp z5@wn5dbK>+(&r=37r&Ei{9y!6(e%a$qksUo<=NyIyhtUBFUyphX~T^eCxKc1di5Fw zl-P?@=WARX#Lh4*H1y)Q`>F#~(0Ds+?q3c(DiOjkREy4!ZWIQ$_oQ))Dz?`)IBXfS&D@}f3 zB)q_;uyb0c;OBqu5eWo}baRRJ|GILRvr+>5c?b2(k84v;?Lhv^fo{Hk{du#tPx=1+6&m&X667j>7N{KY z(`AONHO0uhUDzRY*bi8_0;m>MND$XFhBAtMIJrxECKq z-A~lkLuPKzWkx;nNpXI$!J2_hO2eTFvJ+`IUJr)+@?JnHtRE%Ds`gI!m$RWejih5G zDA{g_*RL0%SG3w!`BtbM=6CIOVMQqftoT6?9^((uK0nz$*R=wu^|Vz^yhtm`?kiu;yialq0FOROdn%s?2Avw*1;Kz+$qAr;*G25b*k z3jJ(Upd29>><-cFl=jur;a0)gSu6rVeYg*o9$(B%P@`K$OHjseHFc9iSP*;_4s?(R zS;8m@{+JhXQBJ7|r)Bb+@sjr`YzQ?yp!b&ETAe>Iox(Cz8NRyMRWiusb$@a2=GS?_ zFHgODoREY{_hhWOR|Ez&UL@hRwX`dA6PLz)lYgcp;bBK0HA@l4sor0xE@?zXcE0QGl;bwkd9$q2%*x$! z%YuXWY4bEuV{rh_CvUWsKnG&kf6J zHy|;lTa}@Mx6ANDQWsj7Gkr3|Hoq$@y$xdfHu9)OZJT-hfXqY7Dta8#HgpGbukpNX zR#d4|g`^`%g!Rz$3J9p;Sm7-izL*ZcD4-pE%Bny&PtcVT!X!bhZa*Tv3{SWIDwPCo zBm@_|Rsg4^27IJC3r9;-?TLRkle+NT!`pVjR^AF0Ft6wI9knR&_e zM=<5%3}KWyzq<%N`#(+<}{un{+zV{()d_Gnu-OH0|gTrwq}@AFIbmc z{`-C)_7xLcgMs8ouh4+TREMO0L%a6Lj)!q~WvchpQrGIQswCswPRI$i()q`CHHQk0 zgE)lX-wKvpoQA1o(0`5D?grfI49gCOW1)|X<|DW_8x$rGEOciOnbaQ2Ixb%}B-iX8 zKJkFwh`1*4z{zY+y&}XzT`zS-NIa1&D=R<0!LyRQfWZI0A5}Z)6<(I(gSc7JOlnK+ zsI}#E14=Gp8ZXiWY7TxSqDVqS1xW9sm559Ua1s88ug>Y#A^k+?ap!PoB$}XCNz%Jf zoV?q6T*33|bB)1QX??suU;4%9q9iFE$Mh!=eqEK8znuqvnvN^`5pwX9j+W!gq(c3h zD9MZJCsAth^3O z)cb!y3h5%+PGQdwJ-MG)7wmifaqY!FZ&|JU_jS{5VzODL$;_th*VCvtSh{0&|B=6cNVGpZ~>vq?$Zu`!ucE+Ib9BX_vpen>luYUJyBNn=ak}yI09mm zi493uhw1+O=dz@)Li~*Cs+I(v)s$}*y#sn7(10P-9Pu_rS!bRvlr>#7KRQ~7?A@zG zgp6%fE2M>%Uc-^XHregbGMftOopEbJ@DTpYwslpq+x|34K$_ z37Nkn+`?co)3ffw-Tvk$H#Mu!BVU4lAC?#y(5P$CrVPov6oF3h`egArdLxMFl;)vS zh@;*FsNu&e$d7` zMHrekfsKl$K$0%%X-CmA?0z ze6IbzCT8(8>Fl`x?YDw^`{{wVi@@HzT_kb)jxJNRAUh6y$?R}n4_F&;vP49&}T?zNt zc|ZNxa=gl_6eCqw#7~eD6@`g$X*wKj?YRRnq{$h!iV-OCd3I=QKXBuU*fv3yg!c3Z zm-H26T-vrT74!7Xf>)OyBqEuBD*wStcAV{rxZu=p5l9;^Uj%$F0`UYDBTsoZ>|W3 zs|L;$uZeA+u<&J6{UM5+-TZvIBI^6@c~GUTHF7U>Z+h>|{xNf|5|l=FcX^K=bdFxM zUWt?YvQGVWut6}!%H|RWz$dS&3>{hA>72T#A8QgbkL;N0ZA^}!KXlV_{xTg-=|A9{GbqlOivng#Xj-!vfgHTaB@OSCyC+UA}kr9l5k1ha;dQ;DiD66*ldo}dXLPFg2+@}x-S!$H3 zO6huov-}~|hsCM63*S_-B~GS!p)Y{*erb6QS6<=Ed>|fn()VG?y+!#??a9OU%5Z8( zG?ZPHV5!=8-~PjZ9N9Jj3&-E%Zt?2}%-#VI&kpRp?$MU|pRAp84c5F7@X7CvTYQHw z;*O^cG*prEvhM@*KeobI2~lmc7#Er^Py_S(4>+N0Jx34`2nR%vkMuA(um`3NONHq` z%t_yW)jluaIiRZ}DuqgnSe$xMg1p9O%;cz1)F|Yd7#ZJ5>rcS?_&<(Q;cUw0r=?5# zOA=qaf1$rTVPF2VbaODMnazec;1?5&a;f|;s{?i=dNT8{uhel!5z|0wH~}gh7!&S^ zmrgeu#$0lU?Q1}aBqfu4+Kc&$ZS0NmLi)qWDTby5xBNC*2H%{OrmZJ@ zO5<&?jMDDNgOO(qF?wPIF~ZxSBh&?|p%=_vG_lu^uZMB~ALTMrdHG|FqLjG)TmxY+JO-Uk0?@^1y4vsJnD!Ul}Xy(m>)CCW#m=RF)2qn)v+ez&{ z!ToBj9tTGua#$=?vmLT!H`3cWX2?< zb!~>7Iz6l-4`YVA4b!{PZCd$`2fq7e&%zeON3{?3x?1+72pIRel(Q0?N15I$4Ue*> zc#1sidaC(h?7iv&3Ar1GtISz& zKMYf3V|-JX>C_qa3c_VSK5iHtBxHH$s&y-jB^@L@EVQm~ z>e^`pH_jFEuW(4nn8XlG3V$y5H>nQX{TQj_NBMTWiSo-GCBHkGW6_6wn``KlUu^0$ z2b_6~7&W=ZhvaGN6mZ0jp49lC0#aehjbFoB(u0xk#-YZsv@}$PYDi)+A&b%XEQja9 zFya$}diB8B_;9gXKNCHKHX@+pHHw+jjO_ ziLPAiI7Tfu4jJSo)x-8>XkndY2AhO}%@YGV3yl(}!=ok&0;n@?T1PO}6N2dAH%v5( zMa-ij!qiwG+HE9J>Fe2HO)23TA4j(f<_JRK3&I`4DGvBzbcbkzq9qv5Qa*j{MzB1& zcXjtSA(uXuz=h$jO+2({%isi@22rNR%W)I{xHlohw(aAXmdY9)T64a))$>Uy`gOX< z^aHX=+5{~fZs+!}c%=07P0^)KGT-N#$8yn;&4V5+bKD@t3tVbfx?WIvwB-Mbt)A~^ z`V-OZ6=2ok%(48bFA|PljkRL3O~zR6m;mraC^Os+Jm4*F=Y|5A85?^k&nah_7|E5d z^)tyop4vr9Mdj}%Iub5*(FX(!p&Kjbb`PK&Ok(S!ZJhp`?64w%4{x@znr#z5&}sL7x`+%Y&?BXNQ6(GpZwLoiN>D4SD8$A3K^RO*BQ6PAt`PFF>aq? zhNN)_a>mNP2T^GQkS=4Kh>f0>dXDN_3}erW*4w|D7DWj=J(pVv20y=--1?aQgYP0t z_Rl==Rm1|(Xo9S@_~S%F!Hk_KB`2ITm4~$v!QR2`Hk}~)dB}L&Ks!>vBq-P8ExoZR zSA`@mxsg0Nnhjb&4e3xDDgiffa`e{EbR6$B^2%$CDeN|(%qx+hRCI!tpAdf}-B&y6 zi8&@su-dHYqO0^f8e*qz25k?A)|8wuR|^?K8wosGzd{Pe zUp?)~{Tdz8NI6pmO5#yR%J2sdyS~2xf|~o|>gyb+>k4F}1>1}zb%*e7D5`Q)M^_y-U`E9 zgT^qG=Q_@AQTg!8hA_$5(H9L5^!^qXifo+Mw?;_dGRR1vkKr1GtBmsOzepYrc)pqElp`GFV z^ERB3Rev_tLBB*PN%9A43KxU6(7-iZXwtPMtE(Vzf8`{bkFR~gMcvwV zEbmjW-Z%cd#gprz$)%|#BuLHD!HtRQO3YyC4#_-JBAVQmOWM)|t^!H59vt_)^YT;t z82jBwat{+!^6h27yVQscKdStYQ8h>PHkU)RrF^CHd<< zm}W|wTkKEJQSh;se174ibWeZ_r<`)~dm1ZQw~Mu^t9fmz|`9~L%~U*CG{r$2t( zFXBr@M-EF}H#pFMxw@;IZ0%4!)n|RAoKy0NNlQkDiCf>1i?bMiMj=F$RHZn|7p319 zU-c>Zp`+JN7Wr6LKe0MiOfBR=LPaCC;xs*TgdDF&p3X8U#Wp$@q>Ew2Fuvi2^=O3hfi&amv zn9)?x!Gjm4vThANDYj2fC2S=KZ79iHAe#W*80xLsi9)a-Y^%1}JUghvZ=!y@3hk^= zLT3erOxaW!xayL|zrK}K`aR(Pg6&j)I_Xdt^{i?i9MC8Z7ki8zN#|dR>^rc`1s})m z=W0motU{#V{xrR^ReIpBy*FQ1PxXZ^bJFxYV$(>)^=LD(RpV+~>$rNZMAFnu_VJ5?h5ah`Er>)$=W;5qN6TBa zr*tF+8S3k$?T+aoIzEFjgin8uAFdQlWqbC=82|o|-PuSexr~_(xvOlyt@rj->E50h zgs#%ay!E8tQRTxSNsceuJ3j?k1zMW*1k3Nr$@e#C-B22;U&u5Saw}e5QzZ!S;ayDR zs!(M{lb%j#)a!@#8WVXk=FlL*$TKD5HMgQs1h0|McNWLFF z>2Zr6DSAz2CMCiBTO6qPA3XA3yzU)$*8L(~Qu$=Urz6??n;edb-&DY&>j^(PxCr4h zH0e>b@cNmM|J$A|ay2upvvg6@Uhk9f@H+ulEF3t!2Kxe5=A8JpYR<(SkSzC^yEh%G z!OzOGo7eBmEIlbvi6iB^gqK{lD>Ck*ET`^`5SvujS5=eduGY4kyFxg4wasnf_|)#8 z2~Ny;6@UBH;+FeLHQp_4tjTyW*SN-*LX%Z#j#7H`Mvv$=RBTV zq{VO@^#)nnmrcjzWD!&aO>>cS}XX7_F(cb~V+K5vq{YKrUzNd3%_-5TE zzj<>k;97p%kzh^}!Relpd)g8CHM*6ZG}rd=vFTs8--Oz8)Vn|+1uW^!N>eX>Q+j`u z(X!<~TXp>4$mct9jpb>F*x6pV6kVA@*^j4aka=D#fE`b*%{JC1B7!}$_eB{Tv$NH* z?FaT#POR2cH~NqbmS^uKmQ7vf$3?>+2kXNDC4(h0kx{&B|s7~tX;)8|L)B3-uXWNNLOCM+}Mwr zVKi2FMr1K_z&N|tV-}!FgW23OEz@v^BFL@#e~dEdOFw^5XEI}(9H$asV=Sia>kN^p=7jZm@`^$FFr(9k-)vTk$fRg>@dT9+WO(`h*I8cnexIX zrwE5f^W*h3e=ulNK4}VGx=Am^vOc(R&)R3WSvzQU-*^MFxZfN-x}Ym&<$UwW&6I{L+pe@rE~aYqUo?c5sT~)ZR7$qK1r$Ret@mX6T;}>v~S609)?R zd<$S0?{=!Eds_N(s1=)$j;NAfH)HjVs&snz-d){+zVL6~78BQZvLUKR=DBInGNKNz zHRTsi9!eOOV>B?h@Y&UPCbausN*jJS7TDa3RbW7%rg`>}O_B55>46?Q0%)cWIEjJ8 zHYD#7unA$eYqz9ulPjs4{di*x8WRqm)hDw*@2G4ZFJfOh4^nC9T+kkvAegFg4wKou z>02P34#Ct*5bQ>OHvd$o9&EA%xMy1!ZA{`R3j>r2Rn66q($`-rcaDLvIF9!)eWT){ z3;Uj4n}rB=7*N36gK_epP5ays*{qt+n|~o#?125c%E&VHWt;REc$&U1 z00VnYdQ7in8b~9O6)$~~H|*DW{c-ic1B{KnzfPW29?Td2URFTNsbE=h*K-Je=DYK= zWC5Bw-u3f!7828RBgwfG0{d)eq~Bz` z{sW<3fRmF^=LPB+Rt;atuK(@jynYHcAq(x1mP&{Shm8qiiZ#L`WS6T8U}jl^#C#Vk znGCeu0rDZ_J)XnG#2pO{h=Bwv$P6;Ebv%|ijWZ>V@`wpzzyUh|+?M{+6_n(Y^1b! z0G$0q<*Ly_l}NQ6505>{XH+Sh%x3RY*W@hvFD3-JcBgTj1nwIrC7KK_uB~F3_1z0_kn`gRCvz`_z1T0 z4(^@pUuD^rQM){vP&ftTyL$wCARW&2k4<`m5ERA<6boVv9*2PL{)zDW8BDD?CK6tZ04Wrq1%jqAR9XmfQ;@#-Ooa zfIA^0oZ}|2bDd2Rx2yn%`%fYu5hc$^KO7qy&wf8jRGbfwUfu&iXFw-l;hDyyOe*!$ z)@SN&hwpz-%+aFJ0R9)E^}d$89L*dOK-mHNfo|9WPVBxm7^2u z-gX!F5VEwPaMO6wsD7a|gQu2K`2^>Z`f2H7swD(*DiAy%N@r!sD76srAgH7%I?1-# zED^4BvIn3tZ_VwH`qjDVCMk+of!GM>rCw^2|BGwGGjHQA4Y@Rwx|~^?SoCh4BP8u5 z*NJgU{Lgy;)M|{i_p<)IwPASOgjsaxZb6flLzvG#ce*Y3616QnGwL|YZ4%`8{`yab z&y(fPU;~dL{7?VTp&Mo8_^yTuj7bI?31+@-z-c0>TbL!rbGMUf*o%trebLa@VE;Sa zk9M&^h+1Uu|M6h-+0rdX$Gc;Ogk_v;2rnvv(2Y`qpV)+H@^HS^ z*b5-Y^i_P(s@NvTcLE#QfG`6QY~dsT@f3iq+8{w7Hwe3hgS@+fuY#I;M%+B^TKB~Q zy6Ix$D|SO6qx1gMJg!+W^id4q8B;Df6pZONkr$^Oimi1EtG|eQIj3Go+=UT?W*bv? zof9?x%{as0qJX{J>MUHzho$tnc|X?C0?6~QP(*YJhRSb<`da!tS=ygCT#1=>6kz#W_ z5f^?^Ru0u(o1~ip_Z`ka-N9#kI|h`}!94;MxBn@0omYh`kB}z$?AeZAch)Z9cRLO57JtDi(^Ia z^KqlakEx@B38;tg5OQFao(f+e4IS%M7^JZ^WltQ<)D;PN+#hYtwqDnuu-*s9cT)e0 zqJSH^U>cZ~p+&a{wxWR(fuRKORf+TQE=le+??cQ2pooa#2P@P#`5FFm&kB|y>^)9@ zMharOwx4-P;v=#JY&~%cbAI5!K=_@l)5Z__kn6SL_OsodQl)jy1*y5Ej{^UxrU7Q< z%i-|ys*nCw-C)e;x3UmixQ!-FQbrgTWVdIA z7>*J*ftky&PemiJze8j~^w={XiSvPm%nWeK#e?@hst+USC3x*xebh!AkA*!=V6W;# z+lYRSS}%HFYuH|I1J?)O%HLLerPeU}$+&m#1LufK_g~Dcl;L{c%@`pw?S+YIsvx(fVcmZsD;|S;xrXOBOU$(-&_#mpst)qNJu=ny)`=AlBdPc?_Bw zaW*$cIJdihVYaQ-N&b|~DwP+LB)?Y`^#Ta`bM4QMe!;k#_=cg30Q;p&*2CA|FR0Jl5 zVk3K9NRTNA$Jr8jMbOpPzN5R?{%9^0d!oZBPTYpip=D7={%*CE|7<8)|6RWebYqKg z-pn6{!s$AeLN7*-^@%ppbj2H!r2V#*%#ABsYAVw<8Gct>jCk2^>gCiwt#0gX<(d2-G!s! zZR!dkEiB$I*L>)JL@p(y69$9J_7-sR0$NtM3)Tr>zJwhRfN=RpVGBtNz2jV*f=(gZ%#0I7q=f%~|iV4OBrL!|ZST&aXd-yq6T5u*a3D zsL8HDQ6YUGTlF+Ky*R!R3{Ij|9`^tYHenqax$QK2w;TM=w}^(Ijxn)dfDMS5-W4K< z)o2jmjBemvT&rqa<=~b6<=GSdfItjAO?)uZ+PwA}I6T~PM_soSk-T2NIz+OHt8;Yi z3sYI*t(L@8OL5^D@?PYP*_5pqGGEtIg)EIfaffl>`Ed!{3bsXS^C-i+fB4?1PQ`K9 z_Xs@XYiWtobgeS;kb$tbk?Rz@dudRv036`%6<=fmKDP<%Yx=y;z~kQ-Xpo#QAT%ZF z$G|*nRmC8?1rsh&oAi6DRmmvP-)R8VjdeX_5?>=7H&SL1^_U4VilKf_DX~rv7HkAG zTp-joUXsY842Z*1C+^hC(k3;(Uf@%q24UTJH7IhugJ;7RVP!a4^|dtXk?ognT`L72JqMntyLp+5w29Qr|2rmS|94EtrsruKUnNsGaRn#_+a^Av1D+vrupT^ zB428SV`??EZ_Ialsn%k~+u7IVJ5z7WUk$R@|IB~34IKH)lQwViUwgs?Ck?Wz9KUjh zHTE9|N4*ENQDw0MJPWMG?Y4T8sO4&zmzXYYnubOA{jZUNW`eKYX+-S6S@`O>P`O3R zDCm*qMT@n3=!2LqrT++cVzSDQPTxHQ_NF&3bk#?OW%}M5SOjLipu^^L7^^Umpo2js z3);jWQuW*?nRXz>aPxZ8zPEYc8>QI-`uOjwxmUsW1x=YR1UK!4KJx}|oPgcz6I=?Au#7G{pqBj+@R4M!EUF+mX^aE@&@0F^TYl(Yg$>``s=giy72HDyi zz)J+LifjJPWFfxk9sQ*}xqKRg6SE}yWP`ySLZ~<6o0=_$9hC!9bzYDjf z?Q9EKncdzY??586zoTB7RB!OIQ-)}{k=*+|F}v*HL7Dh+M0lK3_H z4LJON*vCHb>K)KBZd%^A%59*79rlk!Y<}PB`tbU_O$lCWkbd{8b=?0}2D!yHy^&DuEq=pDx$!+E z9SPjRJFY6|qFT#}oATesv8k3$BX*+gHpYGdyJN{cx93KEh9AyXrc!(XRLvWR)? z7WbxVSM|4{^_n4sFe|rI=K6L1Yu&l;|9LX^`=?rcZc0cF-^`b48%*DY9_BZ!IAwWh z8U?*B5T5y=+UXVdJ9<5L=TI->xxo5%*)t?>0|3;$H8lQm!dpoY_UsFfFVGadj4fIJ z-oRQ`xYPggv3>6h*7#B_o9km|2}V1(S@UtJ<1euh99VG;@tSw`X5DxsOkEcP_eyD@ ztp`KtCw_LSk1?V~vNY7nTKw0{W85P-cL203TkkHoRtNf{~(kO{+T|1Gk2K<**QFLp$C)DUyp27 z{?G3|k{cTFA^7mK!7d3H=0>w5vpTLl16TV#Bk#s;x7p+dCo&c8-`s()aP6X83MnG{ zP2ax)QO5&QB4mo_6xJAHM?>tweSL?OSY3Vm0Y&CtV$lHdyaSw!)y1;G7w=ad1Y-FP zFJHzy;5cbhwVkdS(lRCA_LyI3isn}E5_{9|L2QSO=Z9FCeUqc|j3J`}Fn?qHMPB5i zOzQ|^7WA=ip4%sqqYyh9_zDUAQMD9!z zgmu2*4Ve+iA8SkGO z)=A#?TUvbsN+vH#-WeY?<1Z>QW9yy(jjF`ue!PF9V%}YYYYj+9d`}YNeOUB?4K-Wq zi1~5LGPmPThRca=tAE7#K!T$9liRC9sscv)$5WxT=y4sbC)Y^`Wk8 zFn%iR{e~(2|f+r2Gt*~x2z}c z1VCH)jMp|N5enk>E68Wgx`eHM8+qG*y8{>*0bxz`a+*HRldX#OHh-xV{`jBo03>!V zm}3v{`3U_0w8>SjgQ9HH0CrcM{9KGf-{t4S>N|>kQh)Yhd<)PF_X5V0A1EgjkH=U( zLn=35n**0-Cz#rSnQpM!WPfY8=;_SOQJ|U5~!Q zyk`!p zZqGm5{^V)7I@luraQ1ZXKx&yo4Oa@A&Itj;LT%eS_N4&^{w=K!7cWd$OX_C4fOZ!6 zJ-HG9wl-vRbC7PW@9^F79Z)Y>`QIFVfEMqa5BW=)?*Rb)_$1%}%9J?&sWxbIkd$uG zP3Nitq78r=1AkRrB(eQ^l2YG}6ag8!*G>-TW$O4ueD5$%nI;%6puwIv3*MQaRV@jP zBjk50D*EqRYE=LGmH=q=KrmA%HBRN}Mpp+JENzc35bIY*)&bdzWUWap!1Dx}1EJ@> zd@bwsyS<#n4r?vNqwpKgXT6!vqt zrJLasgOo$5t8%8!7jKe5YLrp7TylZgJ)Uv#rK;$AkFvs5KLx=?9tb`P%xFRRCC+Ne z6BhA$Nv(?jyS4_6z|3s)Jg)y)J7d)yXCFe`!?`#0yIBafU&&S3K{km+YnM=Z2Z}YF z&d?@31RTD9!8Rg52(IjbOjnZ&ATYGJy`H_LM+DzlaVCE^pVaPo6NIdauZm8#Gn`ax zrD#3>*$fO#*zE0&dkg+;@8d*l!e-!`w2JpT%7?-#0dqL-M)Z69KCbG159rR^XmVL7 z5z#}p*MtAtko@vj7NLx12++sCkxKD-T(zD%8;e||YiJ|5w)^MM;c>aYgm?FonM6Zi zP)|u8qG8%(L znzjINTl14U3Dyzu;(RNypHnsDP2R~D+q}@S0@fsqu75X0nC*es)n4)W9wCXq>EVZ4 zac=-~O}SoTllu9}(CruBeN^4<@LfBz1vj-*^I0J0jQs|<(wx18Qd0~;^w0?_v$fN7y(+HwS@9kRclTG7Wu@D&(cF^Ms^^2;)OFpXC1dBio zbXQZv2z&qa^&0ar`EJ@|nsLAko4c(EuToLGL!EF|WVytf9<}OS^#0w{>-i>ZKz2PDZG5$WmJ)gpM*r?_%k^{%rhJZM z;Z?s;Qvwi-=5;#m&932Z+ z*?pKa4Pf9w!;haQf@l3!@^xOp{MG^l<`3Bq#qM~3MY!t+>%+TEn3d9;llNws*B{po zM~8vFth{lqxM=S?@@N%i>IQQpJ5)yFGkI)ieV{=(U|kCX(43RsK6k4+i9@&& z+fSY@S~N(JTe{+p#O=-QEyKYMAT#`EB76e^jp4#UL^$29?L(>BM$`oUzO50LL$_B$ z{lY2vle0M=YA~-(n#QzE#OI`Ma_^|bKX7=AT%8FrkVtkXPiT#Hf&5YjDOJTKb&wce zV(F1+pHOZ$ql=Z~m9Q@|S#o<)*JA0%s)VI!srA{iO6#E#L`#_of(0CcbzQy2e8rO* z$W|bwoUW{gwg@CWd|=U3jz@DRj$nS}n?$x8C>P)hc6v#+CKd@FeNxR6c*LE3O#OcL z=JcQPvDwBf_52JSE=kdNID5V09vnEJlirvv9j6-r-YH}>fv-rb{COsi0;zzJ5Hv^@ zTro$-@{=ckE@Rv00QfVw!7;6LFmQz*8wd{pkz30VjQz%2KNn>3tgoEO=lOlpI7&o= zpjLls0AZkLw?jw$<;-a_)d7y_jUjNL&r58lrDB2!)kTp^;+S}?G8QZxU%rxHJi>m{ zU2h3Q72vxIMHd+W)Fp&4#0)7KW&7gK^R(1|__Fa85`F%B)_NI_g zorU51Yh_>zb+B|Jgdby5B7WWHtYgs;aEaIdV)tlqJTq5~tXLnyKhKOp#%W^05uXnO z%~Pg%wGEs*5s#_D9n|C|$-fdE8=1#GZfj=8`uodEGQ*FT=dZUUYRzA+uvG-vifqL2 z)5ea_3|G;j2V6BAz?kYQt`~hPO8ievmtn>8WVhr=B|*LbC6|(^N8`=80VV?a-GhSJ6@>wkWq|M94r-NAy$XQ5e+&Yx(x|90 zdx3DDd-~r3at?AtOuYZq;(X4;=~{jI7?lpwm?N;q;;vShesV*DXsU+gX+xpQn*gPY zf}a6Kuas@TY%4vnK(@(F;2%DmA1~=uvY{?&P+71AT(;D@r+tJU1)bCgU;sfFzO0od zb>RNO097p|JqvU1tm;`PVO&tWI;|-Q7N0f&R^`h5KrHjm9$p?qEGE2Jtp2S`q=2@d z!z1sd>~dnPw0}E$SW7Y*fHBf zK)IABrUIIG*b<+|27-SzoG768u5z1JC>ioV@2BL@C{f%8s9Mw(%{hRx{X1j4O(&=PpMthP5B)m+tdET{QY8~cg1^NQ<~2}jAdsW+cSC&g?` z&}-!4$LBI4VvcA5>L+#=Ie}*P5$a4FcQtSDK6hr~{cpkhaUUHZX!$|$!+sP*-Jo+x z(h9EA)o9m?v7-JYB8zC7-`PyO5uf|OK(aFQr`4G)^v&WP?Icds6LU7jr(h<2Bg_Ei z@eXOdkDq+XO+Un0)ZYnO3$dy;9dS(n zK?Cq)XrqwQDj_`q?$04i6Yk9WgWd=QOSvcEHxZ?sW*~8-=bqhbsvwyRpuTnG3yGF9!&ORw%~dJA ziAQ;&0;tIp`7}FyuMW~%2)3Xzf{|ye7ep3A0L9o_`{!$v@>%EkCbn;`v$H}STK>MQ+(Qe~tmuI+dUPZ0?&7|$1 z;MTqV3n1)AWa88QqfWqYyJd93=q)VS&Cxr5A47WnkDDL~DgbI&CAaNV*t=gy_>D=S zbUK70kI+uh8vYJ;31@|z7}VVBTd6J+w9**7-D`n*9FmftJ=c1WWFI2eVr?J67Wbd4 z{1Sl2i>_oOG?96r)GbP&giHcNXkeX&3FXO~?!^f9c-9?;I#HkSrw2v`t$-fhPew6o z`{3~kccOeik|H3%Z_9$}S5h_D`1halnNvq-s7!QwU`xUT<3)%Y+lAyq?my39<~mPT z!n8;T?jQE+_>$q%lj?GP{2M#i07BqGgGJNvj)O<+nT0|IHWNfB$Xfmwo4rl8SgN!C z7XpgU6a+_LVu_DqXM&B{)f;z@?x+=iM!0pC)s05Rz*oPh2UR8ka+VEdiEG&>DaHQP zAeyD>S?$L$7gd2p&IIX*n8|P_;Z~lWI!Qts5W1S`dm)Z1esPopNC%VUtdbYxdaaXA za%hTdEdZR`c}(jw7G!(&ZSULRkw4Q}KXa^9p@^ziFNG*|?eaJ7O)VJCP>8M4ZXzB# z%=;$Rib1{+IW$UcaB-f0>$Wd1T^-f4c{cJnrvX4o?{Jch+}Y4C<&ozm`c)dDvSAOY zO9ZS`+)pa@a7wPPgh_+JKLQu0++jW2E%)LKXDw#ieM~RUvoT`YHWX4{Nkemn`NLZ? zIGNrhuw{VP9^tD5!?au52h@2eE9pQGBPT5Tr^-GP(a1pJC-I)m9+LOP_fLr^NSMgK zA`-;0L37L4_Yv=4tH3$$$Y6Vvn1K-t-tEORkDS3-BS8gOzu)hj7@$Ie!vfl?JdDTA z&6uI#Y!pxPzj#kVe0rh));uc7pZii zWk7*|GUFv0n#G06fIxg0QF7vfz$H@FAmCD6l*|jiLm8?70?US=PY|v)8}$HT(lhE> z#VCJAGcAry*@En%4*SKy5g!T0ASEX`4-|I3u1-X^-zVIhQ}?` zS|Fgf9kB5OLXj~M^~gFg#%3NTP_UtTx^&D8geFJA>jp%mc{upiH1>YhD8 zJmn_>$zl+i{b;;dik@UXYI_fhZzn@qzrO)V9bnqbp(!)TN#T6hL?Of8Q0C-}_Ib1q zVV0oMuaNobcVK@I z+!{{5_Ty(H_zZ-Gwo0%O$M1e>8-4H_%LEj`6r(A^w8D%Pw5!u>Z<=Dr^_u&w^DE5{ zUjXv!uFHFoS*`2AI?%nR`(^XJE{C|Zwg-bjZD)q%(Sc&#FEf0kCbDb=hy!Yflbx^oB6J(U-?yS770MF9alnHBnM(&H0uv{>?=^xg9~u1c`vAm- zWbv1^X}hYgfe0>Wwj{fuOouD3fR}L;fGp?XrHz&|?i32M8iU~3vkVoTW+tW<5)l=C z`R^oRLj)HRtQn3PlH$^Fv=@XK*hx=>&-`cCGSn2FG-sl0>*&s{o;)!b0S{@wJKZiQ zL6_902+UbMWXo=$Yv^UIuT-UB0LjMT0BGsL?`&n{;j9PdC4xP_A;8>)wDVie>js?> zy|sms0?C*E7W>5l*429|S@Cg!SPktS=Jf}UIAYuw{TTpFBFIgm3;&|Ndm1VpZ;Ilx z!jW$fHBZzS^jLDEl7bjGF(ty#rYKW6Q>2&^Ux<0)BFMKXV-?Y|L@_r%=e)P?-qXVt zjZ$~Ymd=%!R}U5?iT;DrRWjQNfukJM%Y)jY9UKYp@-qlJzIIqdqloPw*mSv1cFhgW zBoH0fNFuHSOp&$XXN+`V9(wm}5**XnusbIjJt)_t9Trc*r9bdm0?|FV8C3iw5Ezui znTZ_wLdcpwFv8C({Kka8xLie>PSevHB!+h`CNBT3OC(!2J6ky$V2y&0?1Uep3!34& z?%5+-eIf?}Keh_JtMi9`JUHAHQ}cBCcA`u?c=E?$U8s|0>VOc^d)b9QXD-Z(>J9-g8^Lr!;zy9)B1u814t!GqZWRNv=`Z zDp?j9C94)cP=7~u`BHbsd>#F`Q*;iKOfD59G;{Yc@T{b+UpqLrAPR1X`39%}R~(t= z9c=)8|shE^Jv^;E87y^}~yw+x; zrXbf#fXeELNZ*Jl6)rglc8&4+P&Vki)!2_F&$>(O&Cgr$`?FiIFdr)$b zLVZ9Z_JmciAFrb-DV)TwhneY?G%Y7VXkPz-`m-#~hYGMggpdW7l87T+)q(~mI0Z*r zVRi$kqM1pfQkwo{I)SIHdO-chxXydj#Bc0fruZ~+>nWP+HPNnZS%fXsb0T% z1@RsJ+Z>ZWIq6V-!n*n(Y!}yfajS8ws;|x6A`O5GtzLiAG=9Py$SRyFNh6YzQ6swk zG`T^y_fhHZi@Ji^D1U2AmW<=aFR^vhQ>2sUq$d}^M0{rtsf_G^xRT-XK0o7E@dj9J zlLy*R?^pF%v$zAvN80rP??41?S9}y^k$H}7RLl2^^+I;`Nx)oDaw2(qkyoj4!Bb^Z zuV{xJe>bifz8&kj`X8%EzL`Nh%eeLJ&g{K(Ncw!Ik;Ryn@;#l`K?UyP^dF@e`h@D+ zC#XA?hdHmzNw|*SyON?vZov;f?1Hr4YJN!bcnKkuz>d>{^YiCsXr*FemD8Ng=43LH ztPxx^Xv&f4T;%FC*5fV1+<572M8H+eFgE`Oo5b@!1dDOn3v$>WUvI$b23S9&yPCdl zpJwsp$i8&m{9>;4dCDr={^j;)@v1?pu>H3^*5Bsz*R&6VLodAlUI=h9V=wmnaG;JI zJrNla0#i~%zNKhdmUB@Z)+Z~t{jU}PxMhdjj4qo?fSa6wSGJN4 zP>nMOj<9I&#ZFs?v!*EGB;pF=KH09(LaH=q3K+8ovpn0+wdVi0AiC(9#1F(sR;BRn zM5nPRCrQtwDD@$yxmZhWC%i2d(s1_kRPRmJFB(pb)@XDWYSvymPxp4K40HeXmuC5U zH7qYZ{EJ7O>%*9f`t;jLdCGF$0jbe3FS;WCm&*_Nvk7+i&!>^6=&b1oV_I!vpx$B( zfw_=mVlH$QmHzl5p6yq*xo89BV(*1{AmN^iL5_{tSWyIlRkOmS*q?7ogl)wJmamkM zo0lHHr~%u~m$rL5M$5*eUTzQwyq_5(jR*b&p{Z}y=(5&kllgGvZb2Ub2oTtlp5Ft9 zIxhiR?q=B!9<*iFqXn(;|jzY&zjgW#kDVK7h3q;^()=^fGkGeT}w>JzR8fjTP6S#AI$I~N2=lc z&dut>82IY*7;j)%S?GmA)|;*D|E{$o+o^LB7ZAK>+QROMr^EeGB%TdwvhHy3N%-`Z znb~|}%){GzQK+y+KpfbnYn(XoTNMGr3J3!(7!bHJBElS8m41HcyiykRSMyWz$M?7M z{+vfDbXdIfZ@PdxUuj6FyDCb45{%t3SMz!gaAP~nN`CA#%-^hj1+MCl^mQpEcNFIp zum~SkwTaRpZ3`ZYU^6w@q*=s+Jb0bBNunBcCJX2p50HL)6j=qp$-NGKtmp83IcRur z%@id12a#@}m&USg$?H9NZDHMiG0$WD35RCQ{b=XhTU_M(RHV-obpkx{1w#M;<86!LpcP9!Cwk4YJx{3&OGhwVG{7SN4-ymWth& z_EJ!#VZ7+PPkL|Cmg+%!#z3YAy*VGHN+R(22tEOVJ!JYh5)=(cWyc|RsrA)Bl;|5$ zq^4XYwuInzVN$u4q4H#FeMzYXzt)i+rQ=0A{@_M%RXrEz(u#cVDXO7Pz>H0l*ua z5Xjx(*`K&!*DJ62UTv~wqR~$ABtAQO;+xC46Ib=;%E@H~^(Aqq!G5+!xfX+z6z~Xl zdcha4APg>_Ie_l7K8bbodGIHe>pO|;u*Bfkk}72qpW$LSGf|5=wOb^qDW7u8(*y#d zaL-+3I_ZksU5tHGo~K*oeeEMDGHi6@F!wH`}&CRrY64I3?HzX-Dg5pd8@Uwn=y z@MppoC64l+pN~WazGLx9;>Z%v?7u_&O7@AhG_jgH)d@)XrItEKFzJSgQG_U&+W_!N zVx?{1**;edaZgaEbUL420|4~2JEuI`PcS%Df3Gd|qk(mu5VYYbnejCStYLCt`U1CQQt`6qY*|!^0&Z=5Tnx zDW?A6v~e}m3G=z;yFAw>AKby*!8kg@Ev8kQL79_s*Wx{k7N}VTSY-FTV-W{F9?f2m ztTLZ>mCIC?jAJZs)EquofMvjx%JD%_OrJhz$}cR6gv|G5P!;DEDp?_KqAVFD*$swf7|=jmPa#)(1YtC(^0PNSFFw!zLWblCXGv& zG0e6mMDh`0FrEA^DP@gHp+9|wjX@#9%Oup@md<-Z_{UBaDI^$9`5=l4F4!!B)dh96 z*Lc20V}l$5$fk5P4B}p)ibwofFeZNSz_OI}TA=_e(j~J$(@0odz&GcW$27|>4ig1y zOy9r(-9Agdoh6H;C%p>tQAAw0&ZZfd0S}O^e7+bCgpb-U6n59h;+yZU+#%lh7Z@y@ z(Zqe>Y$@B&^G&w^5>HWrQAZ%212$~xog<+kg?-PJI#W3qI5+fy+ixVdyWeJg+w`fk zU7WZ%o3)T3@8WPF?hwc5Jbg9o-;mL%!-!%UvVax&CyPHbrdI@Ea?qKQ;l|xQ|2^Jz z_CV3G|Lgw_m5R7?r_c)542$rsIAFWy6d8X1fZuB}z)tYF*{b=s7r; zMXbbs3N->Y;5Z?n{XrbygseKN5VukPM!e)o10p2&-(%H=wB0Q7H+*OlGLnxfB0Uk9z9J$ z&HfAEUVKb<>kqH+&O4UD6fxw_4JB~(SdMmm?`ring*Z9Sn!jz*e{fk@e8{ZBEXCDs zwfu#}sgsy4IB@O{09hQkJ(Wa>E$OesZ;v-9Va)z$}F%2?U9FRB>X&dtSX+%&ji>Fqe z`Ez0T#>rn7zsE{=bj{Cjy{g1(m934v0e=y-kI6Eh^Rgz=5Nx7zcMs<=P#*9_0pBm8 z@ct;`vQ6G2gd!lwS+=6bckE1ogN+0`l+DX)@sN@!4ZNuOQ8D)wK3@DYLC_v^8i(-{e(6#H5?!-ZGmW5ze0U#?6o{Wld^%k0xBb&Gc28Ky z3;SMeSsx;}>bM@zv0%(`?cnQhySU%&*ZHb0>eS;>7ZwZ)Ip0xy%vz`h;J=i>CAo-c zfRA>Gk(?QD-BENC1;Sb2PWC7989Ex6F>lJ8wi1zbTM z`)w#=e*KWnqUSdKIJnXPi>7hmUO5gHR=Jd8y%<^!7E;;a6r=d}mEEp_EB5;syz(56u>FNw=~8l z+`jU-{P+3ysBD-QMJCj}hjGmfura3xVOegH>SHKBW81K5&307lKT~d%Fq6kPgS9T1HqQBSS5+2@--S+2JNYLQ5(87g!t-? zzLs27`|n)R=ydz=E#%PVc;*=A7#0!Y!&r>Ju@*{Ld1vOA?0A0O%r;DM{WMJut0BYu zsDEfBfS~?dOcTSIp}$DryePecl+qF1+39r*<&H#AbWao;s61 zmW%PfefXj{9v8%FbnRJZoI8^T$yk9fP-@M5^;brI5V=3~?h6CB07WnJIDRQ$KMxcV z5}qL4ZzHyD2K3;O_6ljO^>u77A~V}i%*M5M`}3eR6mOlccLBd;T>wGFg!NZ45w$R7j+ZG((rI5GqR%YVv&N1Tk84x+V8xedWOFHz& zA+yWPM=uwRVY@oSF`6#1pL);wE~5M#xKNSOSfs&&2Dtola$!qg=7aqw!5dxZJucrb z4{{Y~-RvLjU;HptYU!Q&LL>O8#e&vx0#@)l#sWc-Q_unuFG;BKd>Q_fCm0Vk5A+Il zXUUI0|IG+CPst_Hi@zAOS!pf$aP&N>3-<2g;77^l(^B%{=n=6&LCUaFVglH{tqfo9 zcx%O3t-$q+`>b!fow#TVMxESr&w)Rh#JP#$?Op?KEQH z@@2fk+VpODW}DZn@hNLX`oAk)EPciNOywOBd@Xt*HC8rB)X32k)`)c3ma5&f{pv-rJIs5$;B zhhIzPEePvQxhxa;*5=0qG5h&AVN{Dnp`|Fkm}mfp*zuh%v%-_*QK@Fu*Z}KhnZUP( z-1T0OuuxVwE=>&WM|EtQgfgv?XWpBoMVz*IGa8?!{+|*0k3JqA7YC%QsD2vwCU$>j zES7`iAISZbLYCq3S6PhtD@5??!@I><`Ld>#s-cQfk9IFcI3(xok*B-GO)bxFqxIHI z^+Hds|3II-Vj2JH?*Bi})o}>q;U_Ev6Pk$xHFB?ubymKt-M@Q24dYsjFyBVny}Hl+ zv>yLy7p;6Exi|c>;sJemG0(HyLdAr%Uf+jIM!uABWs0#ysn z{z=*AE#=p)TCb~1%X);5&0TQepLu~V{XC-EzZ!oLW=XKr6vp1uc|)lft-_sC|SaZ=cK7l_LWxIZjAt=S~ypOx`Ls+Gga5fQ^t)XH3*?g zbvusa8ZtO2Z<-b0TIz5uIlj7%y!#q}MK-q$h=}JKI?GaogwKJDlqer9Dc1D=Q`T39 zMb&k0&kO_1C<8+%NDeTB2m*?f#E=3ip@1meAl)rPcdH;uC>#D9ou+ZZgENYvbHQJV-K}oJXoy{te^#i^ArEsx<7+8 zvsZ?IwEd%ZfRQys52^?3L1Q9{p}jkxLB=v*0Ci+;m%a&0cx%@gFMGVZ47gWrWj{Sf zEF)Mv^|liur0`Q8PJXn>JpM~A-Dze6+|zR;<$wjyQUh_pD5W=hIs>+{fCy1AkD>H_ z_W1rLAd+A719W|zdu-iRl2QksH6C2j2f#?k0CC_Fuo->2(q zRmO$A+%M*lVffT{eENpFDf5((p9<@g_d(mE@!Vr=U{RF%Q);zib3ZQvZP5{nJ4Qa1 ze>F(-0nf0wymrN5sKaQxIFG4nd)V%B`Hz;8hW<0JPn}iGMi-f|HiTSUp7O|b`N#5Z zT}w}CysS0%=yO~MSpyLw^N!x7#EXLxlMvIr>hYtC2F4X&RAih-V}7d`Hnr9{h${^X zg-SL2nwiED1J**=Izb)5hvf^#durQJ8vY!9OZwxC6b3tWBp3sUwp>HNGY>6#U!O%Idw-ZMP1GH7@>psO!1;?l+vVz4DBI|gbexES_%o;k}WvXzl$ z&pvwZZ;2Yz106w1Al~ltV<^8sBwO@rqRqta6Syt5ppqS;c;;ot71^nC-3+_TxSX$j zv*F28o8$BtLH_C947-IU0|9#L0 z60z|=nL95JNmJAt9>)Ulsv;!<>&DIYRNc^RQ zM|wduNNX*vO^W;KY?^tn3x?ng1f`hvy*GQWS%Wf(C6a2GjWd{C3yA+3^85&Di*hi< zcEUh*q;ayTaaZJp&*u443Z6jXk)l-wJ6V5nP+tPDSB=JMnS!Gl=jcf^EFL0cNo~Y? z)o^Z_1HA}=cTNkhQYo{89)Jjf3ZpD4B+n8EsBOW6thi#JZaZqNzOhnR(j`gK_GZN& z&NzKg4mxI%7d5Zubov~AW=JEbe}VAbhRvS${?(v;t|82o;e-LuTgRfH8i^sk`~n&V z390ObHo83pdznr?u~n3C{|re$&kuS=5r!6bl}Xf1+Xe3}v&oeYZLDyd z^Y{OKWN%T0C(zs|OPS?<9gglH7QAcRkH=p>Th>AkZA?Dv&gW zaTW2FuB@JF4e@E*gk1u>wAd(v4~n7U;uYeh-rs{jWWDzK@8o~m=FXB$bE=pO|CsIX z@a4kCfoDjDJeZ=<*lX!pQ1W;Cmfd}3myAU*lWGbn2C*!;ht2YHSAZ-ta2Z~UxF7mT z$Py2`NJAirjSL5<0fKN7dh+U97- zq(!`W=Bnm-uu@LWjhmH2VwRwt4TUP#GsP_*40?YcPQJ)>L*Qbe~K}q~5_X=0Ms;Aui{i!NOQ-GkVb; zbLYqzcX^dp_W>fs*5mH($JTYHc2NhOafe9tYcW8|=ZjB${hy3gWJ2}HSw|sowJBn* zhcd{K5U$+~AW$0R|8UA~3}M~OzVQJ5>XZ#(h&*l;G%c=7P(MR^1%VMl^)Ua*4%%lZ zuqa4Os!a4^mo=KCr)05ka)fbGGC;YDk~q;}-UBdrng#?p9l2e0&@)m!vIaVrv<-go zXLNET?DUr11GSpr1LfGK{;_uk->ZZr zBRA3bGQ1f-$f}!UtTU^SxsXVCTL(WfcDZh-OXV5cUK3h=Jc#T^9KmngMB>q5y+ss7 zAbVn6bC)DR#X29T#z_%R)^C8~vLc~{3Qgrtl?yHjexn>`&HQpQ-#z9w+-Ydi63e~e z%Bl9yD5dgN`m^WUJL}vBh6UAvQ#QA)9Li?&=Q=SA8T4-q-IbHvZd2y9%7s8oa3V?a z;`45~TK2i*B>_-k#h#h)9L+%ZtdO!pJ=1&b82d^HpO;)OZ$dWQy}m%k2hI)0Y^-byQ!!@wqxNo{0p&}C zO!8vnFhkbMB$}JEBUgB!8Z8Cyh|n7rc{j^dQP+hYxGS@xC?%(tvzt@k{M5l~$*DDe z>o+gQVk20(pI+Xh`;uwrMyvYN|7oAPgNG8rVZN5i{o5z<5AkU=a!33mo5bg7X2UFx z+=YytYVP+o))gk#Te&c6SZ=EIg}lBi;1TCxv=~LK5<*VKeb^Pfx`T3?c3*qK(ZkQ`P902-zb8$#a_Q59-&pN}*qvG|MgR3YJtTN7? z^Ae_AI}))?cJ7+bxe#XRJx>;SA!fSlNPH9}M?(;l0Efv$GpH7~aXS-xjrS<%mZ{ij z7|01I;YYJ_HJ(Fmy-U5JuJlVcp87N0Qg?8_38p)sbG&!@$I8>2KZU6C{6Czekrj8% z#Vzw?9=+Bh-Pf6k{v@}NWy$B3vfm$8H6u%-HqV5aNZmm-P!^=>N0NZ<1^XQhsV!-4 z2pFyuR^70p6KpOERZ%Jg0H(?Q1bGVfHf?SKd|{*-YVU65;3>yyxZGSc{pK5jw7FG3 zEUahx!MA>c(sk+kZ&&wx0sBSN;?eMY0`eXl6WP(GbxCn>`E!?SfeH1l=6;1l#X4~= zWh*lO4*h%4dzo@A!7_sIZ2ObFTOR{h-N;H4c8YF&tb6-5*~#n2?%VIFfs_MD2GeH~ z`-KF-O?mtPRIZWVFSI7YK|SD&MAFuF(1M=2hGB1KK)}A`Y7@Pbxlz%%9-g4H43y28 zPc3ew)TD^+slI$X_QW|=Yw>@$03PVl=Q|~%$~Tm4Gs%NU_x9LkJnT2HvS^z`Sjg}9 zMxR3@bf#H1-nkz;lOcM{mjUES2r9ZpL`0l=%&p7)IEabAt@z*}W_dG*Nq8YPC z;1mWKX==#?LmHK*{)WjnwmOslew+hmF~wUVsY_Rc4;zTyLgu@49Q7FJMRNO6@Aggg z58{K$xHQQa_4Gp+U_=t!L@qM&lS}T>M%K@#hTA26zuofXWe!by>V#t&Oqj-_MK@o9 zgHb^@FYZ_+J0)8#u;r<89~b)N&ARI)N&9_||9Pi*-*mKPLnE(3K-#427dvrEo5UuX z;d_R888X+i`LpC)GE!&Q7+U3d&1oObsOAo*7It5Q9&XW->Qc&{->)2q2P(kFotBKB zQQ+A!dF?=$8&kV_ooQ;m4JSCw+Z?EO>!a4HE-8k_M5v4ZwRH-T?7GP? zRDT#p3o&?+efy(|2;maq&uZjV)~aijGo?rwtpZ=?XnUV&yv8w3d<)cFp5F7E9*de` zI~F?$RFzjnNGAgW@e_Gs%ln9{Cwy$@RooZ(uhM!qV#PagX-E-<$I{Z{<66W|c=PM4 zoe9WM&*|4MwNx4AlsG(s$h3@-&TDf8e#V_kU12FH`wc9z^mz^ljG2nn95$~3DW>^u zgV#y1iI5RS|6_!8rNU7UC~|}~ZjHZOjh#ArRvNcybOmhys?vr)SLZA!kQ{=9PaH2e z2dtCJ?lvwvIQi-4S|mtsqwbu`@?Fe(#24`8b|y`siGjy}4!yK-P4b6ZN?cUZz{T!X zm)V{b>pt$J+@6kpr5$cyX8s9OU@2TQ@$u4SmawZti4 zN#B$qcC)-@MqubE2&L4T5sc43S>6y(n|n+MQKNbK`}tisCf1b(Yc9EP1*K@Bh3K z46rQ&@wk-}2n|XAA|RmeyYQ-n%721n2%K6;N1#IUGb`u?X${HkY(pW;h~4|zVu;rj zlu-oQk6VL29JZdlO7xaI8Sb|{L6cSM3xySC+u+IPiDbuqKpe62JCNH1dY*@C%_V=j zWo7$GiBL>i3{*M?{z?GAgg`^Q!1ghKkysl8mCB~-G)ho*dBVK^0R)d$j##)oq!~LT zG50LxUETiTb1Gsyq?tNmofT005%l!)4eHdyjdu>&ZhdT!)c!8 zaZ1VgY~*!b<1L48t6Q>UwFRbET8x zP&hmk0;0ubs8A-B%#?-@3i+uAqjNI!Kgjj401WagPOV+GjL96d=8uH%kRt4+Wlo&v z7uNlr2v42PxJ|o$#@~!KLf8G|fZ;1YV|GA38j96NWbDp_>Ui^KU-7KK^una^#EXn{D5vGrp0g=DuUz`I%b09_8vV-5% z){y!}8Hk7@ib5c*1MNeIwiQYX!*Imlg2^D!2iPf4eaZq_La1gj;WZ2OA7?W;$ZVv*T!EaJ!7-vAaAn4Kk3Oav=Ng2RXk(Lsm|t}_NO z%>dcY-Zw&gbU5`#mll1$seU&~nMAjWsAj!wfAQOT7~IZ3=d_KJ3CRs6fBSAKJ8>n> zThwL@ixg4Q*z9xv6r~JyDAh{kxPoEsyhY(=cq!wPFYruaCO~Lu*3O9yLMk5w9?u0@ zfk0pi@s#M;;YGx(ftFKOg~){HqO;-7aw;Sylb{HhfhrKDQR4xDlwQ2g0Z4% z8OT_*?^dTU%-;+lD=Fsb?30n{*w5V01ZNXyP|y?p@HUew;xaFsyzp&i`Z;E!Mb|bi zA-jm1t!=$?Ka=*#3xmUJ+FGw$AK!d+Km*BS6a!p~>i~$0GBqI|iq%YFhCuh4D&FM@ z5UA^yzCQ+d>t{-d=|pUyW{tXV;0y=JZfgCZC5#y?7*O*hH8HE4dEC*@sS%t=&QH=A z$^%pxhQ0xuG_i!B(xiTMg|30TAQb2Wi-z*TK>Js-C>YL!?-1kzz8anE4@L-u-xYLK zXmOERP}Sfgr2;8KTm;ZMF2t0SC1}X>ZCWK0P9Q!1<8Bv5)8qz48WpyKsFv?BsSH%f z<$HH?lQ*57zm})gWhTbMbsHXmZ4OHy~KJ8Ej`8>>n)U@-B>VvMFog8>E33%mP~le1v<0J zLB?`~$JUBST{+nhJW0ljOwovM_1{+TbT?|}v#eez>o8!S8@K%e#nvVapmR(2m+uB{ zUlX4ga#Zkfv54lYoCn~!W<64fW?$3HG31xtchEBTuO5tvf2>;Y{LhP*Fjg~UVZbjo z<$5KIM}zzixC7G@|Jjz)f8<11*s@y2Ymmn)HEQZ1svG1tQoiBpuHKLWl8kyj zx3*Lfs)(I(%(2*Aq^dwNpvSw?w6_KCdzdkG*7<@)yDpuD!3RzBby%}3Dv;$_e$wr4 z37M6i6UbEK8vtG)fbyzYuc&`z~i?ULc7;fMZXeNWFUYSkh9UE1sK zG9#pyO(*Mo+^tPMfz7*(WbCZtdqs^$cZMQ~q)EcUDE@Y;7pF!{Mh`hsh-710p*J?+ zLPFul1y*XL7>M(Tg)+3O{HF9)9g`9~s9u`92>P_H zJ*ytG^z(Z3%>MWBCvlY9j#-3)>Hl=Ub7G|@0 z1;o&Ft@ySstD2O89Vm29c_9iPRGCCbvc!@b&GcLmlTU99LSUfGLn!|eBwk~}zzc*y z?^M+S9EGCWp=#mvlb2?$oF)<>$U9*q~w%@a*uw|+uc~Gnfdel4 z9|6cx0Jn)%)7+5l|3)CJodYC?*uY*a}tq zrub&!g<@#ZHmfFFf=G(blTd-W_Nidf;q*sRWIh=X2fr}8e-0niNCR?A!S65`v_F}j9GNF~Ft z^2Y@8o3|-odoCt5;2VM-|Cx}Z7RgPYcJ4u0C7EK|fbPDp`=xwMlZ_Z_MyPpP7)7U&{UEkzdK;Cz8<(~LC-&ccEeVXxRPG6DiseJ`05ve-bPWAZy~(1 zQC`@(?AR*m=k_9t>91NMKDlfLusP|TzP+M~I5GB{L%0uY9t)FsJFSa1I#<6Zb#mexI49ku|Hf^5 z$S)(H&TckitgCvb%71(KV0o@ObMCGwb)pUS`uewbYqJy-L?t)xbw|U{b5O zf;IY4mRy64yl~mN%RTVqsN8Pgtxn_0f+96G=ogAEc|5z~tXr0$V-yt^<3qEvx$4xY z5Tas;P+^EKJEDP6{I2@2}cI|Z(QCDsFNt8si9Q+8hCS2!|YfZP^+zKy3?<8 z&;%9-;K`0a#$?F|P`a)xJcu&~k%80c?)lNmk|@$pr=k@uBQpEjs_Id<)g?a36WU2W zn0-1WX3?G)Jn}7y1BduTt8!1gp}E0NfKE2u;=al+iOf!wfw!oaS>S^FDVE268?{4&0xo8sa$v_BUOo+7imK7|-PS-kgY~@%nmmyDzPAOx2Uk5&FN{1?vy{ zS`fxY-jk%)`71%9q{8uM)leHqfMocvCu(j0S+Wm_RKu?VXzu;`Kww4FXwvM<8Dn`= zjP~~Cvh+7pU9}ptl#q(qLWrSzjP@ty~P;2!%LX1) z$eJnollN)5U$3(Bbi{-3Mxw&4=riKmKg{1v#n20Sy3KaxjY$U!@Ci>)qj{UdF(uCQ zQnaE&7PPd;c$s~h111iH#HlsPb_$INCvVPJW{Ey|x(7#FcLqysZ_{#9icb(3C(+)* zqd=aHn>&$z9Y`E`;@?Pte7(K(X+@G647Lf2nZLS6!%J`E{X9$uCz%UAjP6=?T1%u| zpKB$LOed=yP51hh$#>>T1mp$}t``@mD|rja7fnd7TNd7p`Fb7a6f*iP>x?tSEEH?? z9dJhNrCyMl1c#l76A|d$cU`gVYl&kJZt|G#X>66W{6eR{cQO$|W$3P1^f}g}zrYAb)klUx2$UV)LWuc^E5%Knzr@ zU79%Q?Cn&H^k%e(CW}11TqxwbX>-~yJI?BBx=sukNly*~l8B7_S?%QoSa&H;MIF+)H1T`vtq;mu7;_C+TW(>@A~l>k zSC#N0$SM>SNl-`F?iD9+h_E}nXfzHyngJvaX)&J-i)+th^7~pAOM>GEVJ;gCuTiHX z(aWrwBXdR~4JurdY|28UVgY66?T3mj<-GK0{v;?y3@Ll&>%L=lJVEh>@;2E19<3DR zmhl4(gB(&xni$%{h$aMyol!S|EI67H{S)W&>zjt`3g|g2G$rk^sC`;^Viz>`{1UH( zO%r!{p};)c=>G&=>gCCxwqe_ z%N=?oqo5sj9Tzp{{UPA{9$5EEw2DYiCl(GawZu8*$UZyvZtm-z4D0JH-==JdA4Wgg z-O(RXX2!{lvqvzS{oa_quqJ#^DZ_TjA~K_S`$0u#$vd22G{g9xxo$!4j(SaF(a#D* z9@vWxK=rs|?jLx#9-&`k5;Ow3<>b3D?VgV00jVEqUeKB19oM~4P+MpKO#8mUoMp#j z2`RDN&li^q8Y+QeUCzg^@#s*E#2eIbICbPw_+Ush^zC?6T>qSR@{Rs7)WZwlY@YF* z>35>$6%H!(AG)?+3)>KJW>;q;$vWet+jqCI6}J7NG4|YLeqE3R_(E~VU~(Pbz9mt( zFY~4mh#Ik74n1G7N}n=^!#un0AM-soa}B9TQvXNcnM1w+HJFojB|^8$*VEK0RNz$$ zm5&*tP>O0e0po401kS~Mj#?WZ-S9S;>SnuK5x6B|;@XtPV#x4^ob2Oi!sh%&tUF%1 zpWOn~DeCAkk*};@J#CVJ%3U(0_jgQfCk)mmPKzh4g>xoI^+7pn(r52(gY!WV=Zo-- zzt*jNC7N&xoYr_Hqm^jF1hQ|#l5Ix3k0z5Cb9Iv3{v<*S?y}!9Qlf-YrHXoW0)qLQ z+BkN${6Q2+rA+sjD|Ih1G#U@*f34{AGkPJS>kJ$k5pOIpb1KZ`+g%PU2N)B((U~n( zBm$TCSnzt`=uG*q)N|XV66qf#7g1X^Hw|wUV%Jnc{r9 zOu-*qv7?~ni*bKH2gL+vdE*6mLFqm^{$THrhLLtYllYdTmsCSD9kRR8A@~#MZQXMo zS5fl;(^8!vv){fr-}5}A8U(UGfiPd)7s<;9TFz6FT1YV8#VExiW_lCwsnaMG5taDb zduM}+*$JYa1_O7)E|$~0vcbL@l+R@bTi@gI+jWt4{*M8_6ZU{c-)O1rH|E$EdXw=R zmMH?O;3ZY-3?gChtVh+3QO zl^8++lJ&u^sn@S4o@2ZGaBZ;-1%Nu1Y)MPtN%x~B{@dqzN)_7XUO27Ca4sDdZcyXl=lWgm!^ zTzb5H=h;-^O$GA>=-;q+nYNtyghj~6p8;RAX=vdFH7c3_92bX2AAJ7ICT-g-|M)QbtEOt;iwaX* zMKNy;hL2r@-FVrh2Bkbirhh%Tg_?wE>e<0`Bxk&HsCLo7G@P8-BwR{ggOXM1#}R0W zZdED^FEu;fU}tW~S z9fnI?sAP7t&;6P9%^rI!pnp%Ce2f&}gu|%FfIJ!*K|>2Ry_u`5GWp z`WuuYe%wwMb+~_|h$Zm1=m$xW`FByxab9HhUuzsNVngw)41c^l3C_@Mp&&SWa8~C7 zo1HNKRRjwUVx5-Yt+>XYN;BhDW6)agAewLw9P}znj|M9EqG9C{FHzr- z6KR+imlm6vR9s3reyYeJKCKbM;00N}1kvlKonaEC$CTF4a2r4$mRzQy) zxqVAHfpz7__e6DH=!Rh_xhc78FeWCIV%G^cE+Q2FUomwYF;r!r)>Ju_L4jBI0kJ5-=DIRh%^uuxK~;zikfy{5PsspZen8L z!Y_-WWU$?gOU>N-Ggl4avEP?&$Ck?p&$UXnr!mG%DG9w5#lqU9iP_3>ux8!{=Y||u z4xoBI$faD8)5g(}_r}yvhMw1~5KF5s*fi|~;NEQ!sM6kkK-O;%B?ZGyui>)|PjTFH zcmYS?#^j#_`~r!X>C7+cH7Fi=-ij=}pn^N^Gd~6qg-jY?)n4Z=m^X29l=NhQi(geF za-mM^@@JYfnSiPQ9*(XRfUa<-?J;wpoKbG?Yq!(UGk_lZXyn1c%AZAN@P_duROqO= z7F`sXmZmlfaAU$~5^XejB{FyrLDXxLXnep3IC8#{rpt4C$WzRAW!SxF1U)(I3Wq_v zRd>)z>PfA*mQc92=`U|xOJP^dNMjIFY9LB6Cz%x6(!6xH$ zlxAhZ{3vspV_X;xz1MKVSM0tQ==Ks1&YBXpK}RjY{1`VZ*V@LU6N)V|lunQlcgmHG z!(cZ!yUit3GST?C$I|zKOzddFv}U#umpp&!RgEM5aM*J~#UI4A`fmWhRsu?nNz1unt%Bvr*?N zf=i2U9k7}6@DpIul2YJ}IQ%D-K?8ipvIaRA5I?O-!0P|2Yb>}nioe!Vb3X{lH& zY^^NEu_Y*ognZbGx0|f4WZ`r-F8QM?)f9E(IQ5QA%=EiRvQ3YT>CfC!xMj}S6oxC& zz|f!ZYe}``EGI0|{NFbrj7-`wnc?hV{D-bn7xqZn-5Rd4rN*g2UgztEo7oomQCz8pgFQJlvyma$H`cnu!F1BE~N+``6ZDtvEMl+es%9pX|Ud)o!=%+dEY(r#_ z&KX(tW6LXQnPew9XWC*+m>qT%BWvab4XgTNeRZkNkDg}0;zjZ0*Xq{4_Nz9@NuBd= z{K3N?siV-U6fhby8sQB$sj+yEExz9_-K-lA-OLBNjSf!+)P^_o1@}{gMTHdSse;X{ zF7*xXCwVnXY9hZ7VF<}_Tkp}e+LoiH(YXisJ>GEX{6l{ueMf3!c&7pB18bV_?%ISj zVr1HR#y z_5_R+swiBf+7uYy?{gq*xdHg__0Aphd|pJ2%I=MI3g$s&v#fyNQh$@8;wZTwJCV7R zp^Vgw&RLCD6^KIHy6}7f!vH73^*|v zEV=bx8^-SlzhEW5E>nGGQq&MFIz zVr8sM;(12wmxwlMRs>y^2E36aC_xh~{}_wjHR7s=&!xx)ji28DignrJk9OaLi zwruD?36e^^k>~sW$3+`-ihDuQQIKJiB^+ndAu)-4(RB*U(AKSGI zhNDpBiJQb!xwd!~fe7pvI#dxW6H0Hi^ZTcRz6BDQ%yx^%2aAgO<)%n7jMKV*a(pBZ zi*~}x3jh~3EBm50bN6~SSwUAMAsX@-*FO#PC}!V6KlF`h=?sm2l!g}uN3Hm9ZvTP8 z)Y%|4Zf8Ix8F7?-Qk}SV;~Ea1MWq^Y9-2QkM5XGXZuF4;#o~Ny6Cw(8Qfj}UL;Dq^tgU4GRO&zKWvK#74 zw<%OHF2(?S5co|lhwfznnjs($V~h=qU}H~%=quE|_+bC;1*C|_05oMVG+nwreo*s= zXA;Y;wfh`inpW!6O&q8UT$Jz&m_*2KhI~9tO(qK6$sc+4Fb=CIe_o#7H$K8!r=JH3 zltBzpvB3ZLL2JP@uPLlLz~Fo>vCHSav-(1b8PwPu}< z(L_{1IANCWI(t!ZE!tV&>kVx^m5i{=5NBd4NpP5;(GA|xuAc4-A5~tYe(kX)?(-mk z3zH8bTZU%*U1(_rMRP)W^yBF`M z9nnEy;wW;Hd9V|*5dh+3+T|rh66}#!%#Rl!f-j7D_f_Z0Lq$FL!sB_;^HmGD$Gso4 zZ-V!6BqHI`Z7JD>qDt8!0BCp^o`3&L5DuMqZXd*NMFqVUM%*Kp&<}^p9PCEK;ux=d zDV8jWqtzE1*si33C&~|HwI^wBA*^_DzvzRm0kAKEp7QvOZmM|ul%ehZ) z6`y0dK7fY?sEX}zlR_`vzl~H=0BdMl@m7}_6%Tvz(SNIjL!eHW!%$-7fLasz@@gzB!hhDrQwhH)xguf z$?U!UiQXZ>hGsPBJphSioWDL5>)zW-WLH>z2AKV=D@Dzej=O7{sMqhm{DkPRw6e3;~Fk%67{pn5|SX#O7kXk!a3j?!CX!_IUtN@%#m*Hi6|R{spTNk;jLBG)~Lz&ZHW5 zJ|+5-fuiVAKX5h(EKG`y*iSt;Ir5@Clj*y7-V5|jNEb?6`m-!RPmNK@W$?PxgazO` z+dRD~D_EQ_(7a`L`(*o0+Je@tOgp&uGm?JdrtzxzT+S?@?1ie?n0jPVN4ca0Mm)&_L=9I~$ zzGFEDLOO4Or=cpk_TOK@25E!HC-F8Mvzr2>;h=^3=Le?-#`NN4 zUKRMi87J0t3>HQ9;U>wIzq=AT3BZr08FYA2=nVk>!QGr?v{+}0sHxU-0OXrB|J4^|efs{Xs!PNoYU}fO(9#6*F6~2EDJ(?ZYn{9US zBN-oJfdgBm#p-*~cZErW@!!953jo`ie8q3?fIQzzq`cS{g;ir!!FWFU|CwYk)hZY; z&v}&(86#2JXM=j;HhK>3+$~Yc`3Exn&sspEk^j6P1$MO}I@IxD#8|zeU?$_gpTo-p z#`9(NR$dO6lPCYYEXyk8UxOo}rlF%x=AnBm7^(ArzQ>tfo8{l@j!Fja!U@Jmav?a1 zBoWV(Yg!;)#RU4T1jE?c)zdo8U4ejxG?K9wBYmtNHNNsL( z%btXeDQ#)|YhejUT_|P39r>@n{IB7tgP9Q#{;wJQ%OpTTlJdVM_P>7urb=U?ek}BY UF*Z8X3 Date: Sat, 28 Oct 2023 19:22:15 -0400 Subject: [PATCH 02/16] update readme for getting started Signed-off-by: Carlos Santana --- .../examples/eks/getting-started/README.md | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/argocd/iac/terraform/examples/eks/getting-started/README.md b/argocd/iac/terraform/examples/eks/getting-started/README.md index 30d4792d..6c1b8bdc 100644 --- a/argocd/iac/terraform/examples/eks/getting-started/README.md +++ b/argocd/iac/terraform/examples/eks/getting-started/README.md @@ -109,12 +109,6 @@ Wait until all the ArgoCD applications' `HEALTH STATUS` is `Healthy`. Use Crl+C watch kubectl get applications -n argocd ``` -## Access ArgoCD -Access ArgoCD's UI, run the command from the output: -```shell -terraform output -raw access_argocd -``` - ### Verify the Addons Verify that the addons are ready: ```shell @@ -123,6 +117,11 @@ kubectl get deployment -n kube-system \ metrics-server ``` +## Access ArgoCD +Access ArgoCD's UI, run the command from the output: +```shell +terraform output -raw access_argocd +``` ## Deploy the Workloads @@ -151,16 +150,18 @@ kubectl events -n game-2048 --for ingress/game-2048 --watch ### Access the Application using AWS Load Balancer -Retrieve the ingress URL for the application: -```shell -echo "Application URL: http://$(kubectl get -n game-2048 ingress game-2048 -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')" -``` -Verify the application enpoint health using `curl`: +Verify the application endpoint health using `curl`: ```shell curl -I $(kubectl get -n game-2048 ingress game-2048 -o jsonpath='{.status.loadBalancer.ingress[0].hostname}') ``` The first line of the output should have `HTTP/1.1 200 OK`. +Retrieve the ingress URL for the application, and access in the browser: +```shell +echo "Application URL: http://$(kubectl get -n game-2048 ingress game-2048 -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')" +``` + + ### Container Metrics Check the application's CPU and memory metrics: ```shell From 0795d2e85b3dff255fade41cfcc21c12e3abff46 Mon Sep 17 00:00:00 2001 From: Carlos Santana Date: Sat, 28 Oct 2023 19:27:34 -0400 Subject: [PATCH 03/16] remove outdated readme Signed-off-by: Carlos Santana --- README.md | 2 +- argocd/iac/terraform/examples/eks/README.md | 44 --------------------- 2 files changed, 1 insertion(+), 45 deletions(-) delete mode 100644 argocd/iac/terraform/examples/eks/README.md diff --git a/README.md b/README.md index 406230d7..c60f1e37 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ of the Kubernetes resources, any changes to these resources outside Terraform fo ### ArgoCD Status | IaC | GitOps | Status | | :--- | :----: | ---: | -| Terraform | ArgoCD | Stable [try it!](argocd/iac/terraform/examples/eks/) | +| Terraform | ArgoCD | Stable [try it!](argocd/iac/terraform/examples/eks/getting-started) | | EKSCTL | ArgoCD | | | CDK | ArgoCD | | | Crossplane | ArgoCD | | diff --git a/argocd/iac/terraform/examples/eks/README.md b/argocd/iac/terraform/examples/eks/README.md deleted file mode 100644 index 01e8dc96..00000000 --- a/argocd/iac/terraform/examples/eks/README.md +++ /dev/null @@ -1,44 +0,0 @@ -# Terraform GitOps Bridge for ArgoCD on Amazon EKS - -Install `argocd` CLI -```shell -brew install argocd -``` - -Clone the repo -```shell -git clone github.com/gitops-bridge-dev/gitops-bridge -``` - -Select an example -```shell -cd gitops-bridge/argocd/iac/terraform/examples/eks/hello-world -``` - -Run terraform -```shell -terraform init -terraform apply -``` - -Setup `kubectl`, by running the command from the `configure_kubectl` output -```shell -terraform output -raw configure_kubectl -``` - -Setup `argocd`, by running the command from the `configure_argocd` output -```shell -terraform output -raw configure_argocd -``` -Argo CD UI is available at http://localhost:8080 - -Use the `argocd`, Ctrl+C to stop the Argo CD UI -```shell -argocd app list -n argocd -argocd appset list -n argocd -``` - -Destroy Cluster -```shell -./destroy.sh -``` \ No newline at end of file From 286ed788072f6241380079d1ae59268f7f2e54eb Mon Sep 17 00:00:00 2001 From: Carlos Santana Date: Sat, 28 Oct 2023 22:21:53 -0400 Subject: [PATCH 04/16] updated argocd-ingress Signed-off-by: Carlos Santana --- .../examples/eks/argocd-ingress/README.md | 40 ++++-- .../examples/eks/argocd-ingress/main.tf | 134 ++++++++++-------- .../examples/eks/argocd-ingress/variables.tf | 24 ++++ 3 files changed, 128 insertions(+), 70 deletions(-) diff --git a/argocd/iac/terraform/examples/eks/argocd-ingress/README.md b/argocd/iac/terraform/examples/eks/argocd-ingress/README.md index 4285b802..2877d618 100644 --- a/argocd/iac/terraform/examples/eks/argocd-ingress/README.md +++ b/argocd/iac/terraform/examples/eks/argocd-ingress/README.md @@ -4,6 +4,24 @@ Example on how to deploy Amazon EKS with addons configured via ArgoCD. In this example the ArgoCD is configured with ingress using a https domain name managed on Route53 +## Prerequisites +Before you begin, make sure you have the following command line tools installed: +- git +- terraform +- kubectl +- argocd + +## Fork the Git Repositories + +### Fork the Addon GitOps Repo +1. Fork the git repository for addons [here](https://github.com/gitops-bridge-dev/gitops-bridge-argocd-control-plane-template). +2. Update the following environment variables to point to your fork by changing the default values: +```shell +export TF_VAR_gitops_addons_org=https://github.com/gitops-bridge-dev +export TF_VAR_gitops_addons_repo=gitops-bridge-argocd-control-plane-template +``` + + **Create DNS Hosted Zone in Route 53:** In this step you will delegate your registered domain DNS to Amazon Route53. You can either delegate the top level domain or a subdomain. @@ -18,24 +36,30 @@ aws route53 create-hosted-zone --name $TF_VAR_domain_name --caller-reference "$( Use the NameServers in the DelegatoinSet to update your registered domain NS records at the registrar. -After creating the Route53 zone deploy the EKS Cluster +## Deploy the EKS Cluster +Initialize Terraform and deploy the EKS cluster: ```shell terraform init -terraform apply +terraform apply -auto-approve +``` +Retrieve `kubectl` config, then execute the output command: +```shell +terraform output -raw configure_kubectl ``` -Access Terraform output to configure `kubectl` and `argocd` +### Monitor GitOps Progress for Addons +Wait until all the ArgoCD applications' `HEALTH STATUS` is `Healthy`. Use Crl+C to exit the `watch` command ```shell -terraform output +watch kubectl get applications -n argocd ``` -To access ArgoCD thru ingress https use the following command to get URL and passwords +## Access ArgoCD +Access ArgoCD's UI, run the command from the output: ```shell -echo "URL: https://$(kubectl get ing -n argocd argo-cd-argocd-server -o jsonpath='{.spec.tls[0].hosts[0]}')" -echo "Username: admin" -echo "Password: $(kubectl get secrets argocd-initial-admin-secret -n argocd --template="{{index .data.password | base64decode}}")" +terraform output -raw access_argocd ``` + Destroy EKS Cluster ```shell ./destroy.sh diff --git a/argocd/iac/terraform/examples/eks/argocd-ingress/main.tf b/argocd/iac/terraform/examples/eks/argocd-ingress/main.tf index e09d1126..0e99f78b 100644 --- a/argocd/iac/terraform/examples/eks/argocd-ingress/main.tf +++ b/argocd/iac/terraform/examples/eks/argocd-ingress/main.tf @@ -32,13 +32,12 @@ provider "kubernetes" { locals { name = "ex-${replace(basename(path.cwd), "_", "-")}" - environment = "dev" - region = "us-west-2" - cluster_version = "1.27" - gitops_addons_url = "${var.gitops_addons_org}/${var.gitops_addons_repo}" - gitops_addons_basepath = var.gitops_addons_basepath - gitops_addons_path = var.gitops_addons_path - gitops_addons_revision = var.gitops_addons_revision + region = var.region + + cluster_version = var.kubernetes_version + + vpc_cidr = var.vpc_cidr + azs = slice(data.aws_availability_zones.available.names, 0, 3) enable_ingress = true is_route53_private_zone = false @@ -49,45 +48,61 @@ locals { argocd_host = "${local.argocd_subdomain}.${local.domain_name}" route53_zone_arn = try(data.aws_route53_zone.this[0].arn, "") + gitops_addons_url = "${var.gitops_addons_org}/${var.gitops_addons_repo}" + gitops_addons_basepath = var.gitops_addons_basepath + gitops_addons_path = var.gitops_addons_path + gitops_addons_revision = var.gitops_addons_revision aws_addons = { - #enable_cert_manager = true - #enable_aws_efs_csi_driver = true - #enable_aws_fsx_csi_driver = true - #enable_aws_cloudwatch_metrics = true - #enable_aws_privateca_issuer = true - #enable_cluster_autoscaler = true - enable_external_dns = true - #enable_external_secrets = true - enable_aws_load_balancer_controller = true - #enable_fargate_fluentbit = true - #enable_aws_for_fluentbit = true - #enable_aws_node_termination_handler = true - #enable_karpenter = true - #enable_velero = true - #enable_aws_gateway_api_controller = true - #enable_aws_ebs_csi_resources = true # generate gp2 and gp3 storage classes for ebs-csi - #enable_aws_secrets_store_csi_driver_provider = true - enable_aws_argocd_ingress = true + enable_cert_manager = try(var.addons.enable_cert_manager, false) + enable_aws_efs_csi_driver = try(var.addons.enable_aws_efs_csi_driver, false) + enable_aws_fsx_csi_driver = try(var.addons.enable_aws_fsx_csi_driver, false) + enable_aws_cloudwatch_metrics = try(var.addons.enable_aws_cloudwatch_metrics, false) + enable_aws_privateca_issuer = try(var.addons.enable_aws_privateca_issuer, false) + enable_cluster_autoscaler = try(var.addons.enable_cluster_autoscaler, false) + enable_external_dns = try(var.addons.enable_external_dns, false) + enable_external_secrets = try(var.addons.enable_external_secrets, false) + enable_aws_load_balancer_controller = try(var.addons.enable_aws_load_balancer_controller, false) + enable_fargate_fluentbit = try(var.addons.enable_fargate_fluentbit, false) + enable_aws_for_fluentbit = try(var.addons.enable_aws_for_fluentbit, false) + enable_aws_node_termination_handler = try(var.addons.enable_aws_node_termination_handler, false) + enable_karpenter = try(var.addons.enable_karpenter, false) + enable_velero = try(var.addons.enable_velero, false) + enable_aws_gateway_api_controller = try(var.addons.enable_aws_gateway_api_controller, false) + enable_aws_ebs_csi_resources = try(var.addons.enable_aws_ebs_csi_resources, false) + enable_aws_secrets_store_csi_driver_provider = try(var.addons.enable_aws_secrets_store_csi_driver_provider, false) + enable_ack_apigatewayv2 = try(var.addons.enable_ack_apigatewayv2, false) + enable_ack_dynamodb = try(var.addons.enable_ack_dynamodb, false) + enable_ack_s3 = try(var.addons.enable_ack_s3, false) + enable_ack_rds = try(var.addons.enable_ack_rds, false) + enable_ack_prometheusservice = try(var.addons.enable_ack_prometheusservice, false) + enable_ack_emrcontainers = try(var.addons.enable_ack_emrcontainers, false) + enable_ack_sfn = try(var.addons.enable_ack_sfn, false) + enable_ack_eventbridge = try(var.addons.enable_ack_eventbridge, false) + enable_aws_argocd_ingress = try(var.addons.enable_aws_argocd_ingress, false) } oss_addons = { - enable_argocd = false - #enable_argo_rollouts = true - #enable_argo_events = true - #enable_argo_workflows = true - #enable_cluster_proportional_autoscaler = true - #enable_gatekeeper = true - #enable_gpu_operator = true - #enable_ingress_nginx = true - #enable_kyverno = true - #enable_kube_prometheus_stack = true - #enable_metrics_server = true - #enable_prometheus_adapter = true - #enable_secrets_store_csi_driver = true - #enable_vpa = true - #enable_foo = true # you can add any addon here, make sure to update the gitops repo with the corresponding application set + enable_argocd = try(var.addons.enable_argocd, false) + enable_argo_rollouts = try(var.addons.enable_argo_rollouts, false) + enable_argo_events = try(var.addons.enable_argo_events, false) + enable_argo_workflows = try(var.addons.enable_argo_workflows, false) + enable_cluster_proportional_autoscaler = try(var.addons.enable_cluster_proportional_autoscaler, false) + enable_gatekeeper = try(var.addons.enable_gatekeeper, false) + enable_gpu_operator = try(var.addons.enable_gpu_operator, false) + enable_ingress_nginx = try(var.addons.enable_ingress_nginx, false) + enable_kyverno = try(var.addons.enable_kyverno, false) + enable_kube_prometheus_stack = try(var.addons.enable_kube_prometheus_stack, false) + enable_metrics_server = try(var.addons.enable_metrics_server, false) + enable_prometheus_adapter = try(var.addons.enable_prometheus_adapter, false) + enable_secrets_store_csi_driver = try(var.addons.enable_secrets_store_csi_driver, false) + enable_vpa = try(var.addons.enable_vpa, false) } - addons = merge(local.aws_addons, local.oss_addons, { kubernetes_version = local.cluster_version }, { aws_cluster_name = module.eks.cluster_name }) + addons = merge( + local.aws_addons, + local.oss_addons, + { kubernetes_version = local.cluster_version }, + { aws_cluster_name = module.eks.cluster_name } + ) addons_metadata = merge( module.eks_blueprints_addons.gitops_metadata, @@ -114,12 +129,9 @@ locals { workloads = file("${path.module}/bootstrap/workloads.yaml") } - vpc_cidr = "10.0.0.0/16" - azs = slice(data.aws_availability_zones.available.names, 0, 3) - tags = { Blueprint = local.name - GithubRepo = "github.com/csantanapr/terraform-gitops-bridge" + GithubRepo = "github.com/gitops-bridge-dev/gitops-bridge" } } @@ -130,8 +142,6 @@ module "gitops_bridge_bootstrap" { source = "github.com/gitops-bridge-dev/gitops-bridge-argocd-bootstrap-terraform?ref=v2.0.0" cluster = { - cluster_name = module.eks.cluster_name - environment = local.environment metadata = local.addons_metadata addons = local.addons } @@ -154,21 +164,21 @@ module "eks_blueprints_addons" { create_kubernetes_resources = false # EKS Blueprints Addons - enable_cert_manager = try(local.aws_addons.enable_cert_manager, false) - enable_aws_efs_csi_driver = try(local.aws_addons.enable_aws_efs_csi_driver, false) - enable_aws_fsx_csi_driver = try(local.aws_addons.enable_aws_fsx_csi_driver, false) - enable_aws_cloudwatch_metrics = try(local.aws_addons.enable_aws_cloudwatch_metrics, false) - enable_aws_privateca_issuer = try(local.aws_addons.enable_aws_privateca_issuer, false) - enable_cluster_autoscaler = try(local.aws_addons.enable_cluster_autoscaler, false) - enable_external_dns = try(local.aws_addons.enable_external_dns, false) - enable_external_secrets = try(local.aws_addons.enable_external_secrets, false) - enable_aws_load_balancer_controller = try(local.aws_addons.enable_aws_load_balancer_controller, false) - enable_fargate_fluentbit = try(local.aws_addons.enable_fargate_fluentbit, false) - enable_aws_for_fluentbit = try(local.aws_addons.enable_aws_for_fluentbit, false) - enable_aws_node_termination_handler = try(local.aws_addons.enable_aws_node_termination_handler, false) - enable_karpenter = try(local.aws_addons.enable_karpenter, false) - enable_velero = try(local.aws_addons.enable_velero, false) - enable_aws_gateway_api_controller = try(local.aws_addons.enable_aws_gateway_api_controller, false) + enable_cert_manager = local.aws_addons.enable_cert_manager + enable_aws_efs_csi_driver = local.aws_addons.enable_aws_efs_csi_driver + enable_aws_fsx_csi_driver = local.aws_addons.enable_aws_fsx_csi_driver + enable_aws_cloudwatch_metrics = local.aws_addons.enable_aws_cloudwatch_metrics + enable_aws_privateca_issuer = local.aws_addons.enable_aws_privateca_issuer + enable_cluster_autoscaler = local.aws_addons.enable_cluster_autoscaler + enable_external_dns = local.aws_addons.enable_external_dns + enable_external_secrets = local.aws_addons.enable_external_secrets + enable_aws_load_balancer_controller = local.aws_addons.enable_aws_load_balancer_controller + enable_fargate_fluentbit = local.aws_addons.enable_fargate_fluentbit + enable_aws_for_fluentbit = local.aws_addons.enable_aws_for_fluentbit + enable_aws_node_termination_handler = local.aws_addons.enable_aws_node_termination_handler + enable_karpenter = local.aws_addons.enable_karpenter + enable_velero = local.aws_addons.enable_velero + enable_aws_gateway_api_controller = local.aws_addons.enable_aws_gateway_api_controller external_dns_route53_zone_arns = [local.route53_zone_arn] # ArgoCD Server and UI domain name is registered in Route 53 diff --git a/argocd/iac/terraform/examples/eks/argocd-ingress/variables.tf b/argocd/iac/terraform/examples/eks/argocd-ingress/variables.tf index e55a56ac..d928bf94 100644 --- a/argocd/iac/terraform/examples/eks/argocd-ingress/variables.tf +++ b/argocd/iac/terraform/examples/eks/argocd-ingress/variables.tf @@ -2,6 +2,30 @@ variable "domain_name" { description = "Route 53 domain name" type = string } +variable "vpc_cidr" { + description = "VPC CIDR" + type = string + default = "10.0.0.0/16" +} +variable "region" { + description = "AWS region" + type = string + default = "us-west-2" +} +variable "kubernetes_version" { + description = "Kubernetes version" + type = string + default = "1.28" +} +variable "addons" { + description = "Kubernetes addons" + type = any + default = { + enable_external_dns = true + enable_aws_load_balancer_controller = true + enable_aws_argocd_ingress = true + } +} variable "gitops_addons_org" { description = "Git repository org/user contains for addons" default = "https://github.com/gitops-bridge-dev" From 52dcb98d837abd792d5f4c482dda1ef0301f3954 Mon Sep 17 00:00:00 2001 From: Carlos Santana Date: Sat, 28 Oct 2023 22:32:31 -0400 Subject: [PATCH 05/16] update secret manager example Signed-off-by: Carlos Santana --- .../examples/eks/argocd-ingress/main.tf | 6 +- .../examples/eks/argocd-ingress/variables.tf | 20 ++- .../eks/aws-secrets-manager/README.md | 40 +++++- .../examples/eks/aws-secrets-manager/main.tf | 129 ++++++++++-------- .../eks/aws-secrets-manager/variables.tf | 37 ++++- 5 files changed, 156 insertions(+), 76 deletions(-) diff --git a/argocd/iac/terraform/examples/eks/argocd-ingress/main.tf b/argocd/iac/terraform/examples/eks/argocd-ingress/main.tf index 0e99f78b..ffb99cdb 100644 --- a/argocd/iac/terraform/examples/eks/argocd-ingress/main.tf +++ b/argocd/iac/terraform/examples/eks/argocd-ingress/main.tf @@ -31,7 +31,7 @@ provider "kubernetes" { } locals { - name = "ex-${replace(basename(path.cwd), "_", "-")}" + name = "ex-${replace(basename(path.cwd), "_", "-")}" region = var.region cluster_version = var.kubernetes_version @@ -142,8 +142,8 @@ module "gitops_bridge_bootstrap" { source = "github.com/gitops-bridge-dev/gitops-bridge-argocd-bootstrap-terraform?ref=v2.0.0" cluster = { - metadata = local.addons_metadata - addons = local.addons + metadata = local.addons_metadata + addons = local.addons } apps = local.argocd_apps } diff --git a/argocd/iac/terraform/examples/eks/argocd-ingress/variables.tf b/argocd/iac/terraform/examples/eks/argocd-ingress/variables.tf index d928bf94..e42009f1 100644 --- a/argocd/iac/terraform/examples/eks/argocd-ingress/variables.tf +++ b/argocd/iac/terraform/examples/eks/argocd-ingress/variables.tf @@ -21,28 +21,34 @@ variable "addons" { description = "Kubernetes addons" type = any default = { - enable_external_dns = true - enable_aws_load_balancer_controller = true - enable_aws_argocd_ingress = true + enable_external_dns = true + enable_aws_load_balancer_controller = true + enable_aws_argocd_ingress = true } } +# Addons Git variable "gitops_addons_org" { description = "Git repository org/user contains for addons" + type = string default = "https://github.com/gitops-bridge-dev" } variable "gitops_addons_repo" { description = "Git repository contains for addons" + type = string default = "gitops-bridge-argocd-control-plane-template" } +variable "gitops_addons_revision" { + description = "Git repository revision/branch/ref for addons" + type = string + default = "main" +} variable "gitops_addons_basepath" { description = "Git repository base path for addons" + type = string default = "" } variable "gitops_addons_path" { description = "Git repository path for addons" + type = string default = "bootstrap/control-plane/addons" } -variable "gitops_addons_revision" { - description = "Git repository revision/branch/ref for addons" - default = "HEAD" -} diff --git a/argocd/iac/terraform/examples/eks/aws-secrets-manager/README.md b/argocd/iac/terraform/examples/eks/aws-secrets-manager/README.md index bd83f6cd..e9c61019 100644 --- a/argocd/iac/terraform/examples/eks/aws-secrets-manager/README.md +++ b/argocd/iac/terraform/examples/eks/aws-secrets-manager/README.md @@ -1,24 +1,54 @@ -# Hello World ArgoCD on Amazon EKS +# AWS Secret Manager for ArgoCD Admin Password Example on how to deploy Amazon EKS with addons configured via ArgoCD. In this example the ArgoCD admin secret is stored in AWS Secret Manager -Deploy EKS Cluster +## Prerequisites +Before you begin, make sure you have the following command line tools installed: +- git +- terraform +- kubectl +- argocd + +## Fork the Git Repositories + +### Fork the Addon GitOps Repo +1. Fork the git repository for addons [here](https://github.com/gitops-bridge-dev/gitops-bridge-argocd-control-plane-template). +2. Update the following environment variables to point to your fork by changing the default values: +```shell +export TF_VAR_gitops_addons_org=https://github.com/gitops-bridge-dev +export TF_VAR_gitops_addons_repo=gitops-bridge-argocd-control-plane-template +``` + +## Deploy the EKS Cluster +Initialize Terraform and deploy the EKS cluster: ```shell terraform init -terraform apply +terraform apply -auto-approve +``` +Retrieve `kubectl` config, then execute the output command: +```shell +terraform output -raw configure_kubectl ``` -Access Terraform output to configure `kubectl` and `argocd` (it includes argocd password) +### Monitor GitOps Progress for Addons +Wait until all the ArgoCD applications' `HEALTH STATUS` is `Healthy`. Use Crl+C to exit the `watch` command ```shell -terraform output +watch kubectl get applications -n argocd ``` +## Get the ArgoCD password from AWS Secret Manager To get the argocd `admin` password stored in AWS Secret Manager ```shell aws secretsmanager get-secret-value --secret-id argocd --output json | jq -r .SecretString ``` +## Access ArgoCD +Access ArgoCD's UI, run the command from the output: +```shell +terraform output -raw access_argocd +``` + Destroy EKS Cluster ```shell cd hub diff --git a/argocd/iac/terraform/examples/eks/aws-secrets-manager/main.tf b/argocd/iac/terraform/examples/eks/aws-secrets-manager/main.tf index 94be927a..c780703d 100644 --- a/argocd/iac/terraform/examples/eks/aws-secrets-manager/main.tf +++ b/argocd/iac/terraform/examples/eks/aws-secrets-manager/main.tf @@ -33,52 +33,71 @@ provider "kubernetes" { } locals { - name = "ex-${replace(basename(path.cwd), "_", "-")}" - environment = "dev" - region = "us-west-2" - cluster_version = "1.27" + name = "ex-${replace(basename(path.cwd), "_", "-")}" + region = var.region + + cluster_version = var.kubernetes_version + + vpc_cidr = var.vpc_cidr + azs = slice(data.aws_availability_zones.available.names, 0, 3) + + + gitops_addons_url = "${var.gitops_addons_org}/${var.gitops_addons_repo}" gitops_addons_basepath = var.gitops_addons_basepath gitops_addons_path = var.gitops_addons_path gitops_addons_revision = var.gitops_addons_revision - aws_addons = { - enable_cert_manager = true - #enable_aws_efs_csi_driver = true - #enable_aws_fsx_csi_driver = true - #enable_aws_cloudwatch_metrics = true - #enable_aws_privateca_issuer = true - #enable_cluster_autoscaler = true - #enable_external_dns = true - #enable_external_secrets = true - #enable_aws_load_balancer_controller = true - #enable_fargate_fluentbit = true - #enable_aws_for_fluentbit = true - #enable_aws_node_termination_handler = true - #enable_karpenter = true - #enable_velero = true - #enable_aws_gateway_api_controller = true - #enable_aws_ebs_csi_resources = true # generate gp2 and gp3 storage classes for ebs-csi - #enable_aws_secrets_store_csi_driver_provider = true + enable_cert_manager = try(var.addons.enable_cert_manager, false) + enable_aws_efs_csi_driver = try(var.addons.enable_aws_efs_csi_driver, false) + enable_aws_fsx_csi_driver = try(var.addons.enable_aws_fsx_csi_driver, false) + enable_aws_cloudwatch_metrics = try(var.addons.enable_aws_cloudwatch_metrics, false) + enable_aws_privateca_issuer = try(var.addons.enable_aws_privateca_issuer, false) + enable_cluster_autoscaler = try(var.addons.enable_cluster_autoscaler, false) + enable_external_dns = try(var.addons.enable_external_dns, false) + enable_external_secrets = try(var.addons.enable_external_secrets, false) + enable_aws_load_balancer_controller = try(var.addons.enable_aws_load_balancer_controller, false) + enable_fargate_fluentbit = try(var.addons.enable_fargate_fluentbit, false) + enable_aws_for_fluentbit = try(var.addons.enable_aws_for_fluentbit, false) + enable_aws_node_termination_handler = try(var.addons.enable_aws_node_termination_handler, false) + enable_karpenter = try(var.addons.enable_karpenter, false) + enable_velero = try(var.addons.enable_velero, false) + enable_aws_gateway_api_controller = try(var.addons.enable_aws_gateway_api_controller, false) + enable_aws_ebs_csi_resources = try(var.addons.enable_aws_ebs_csi_resources, false) + enable_aws_secrets_store_csi_driver_provider = try(var.addons.enable_aws_secrets_store_csi_driver_provider, false) + enable_ack_apigatewayv2 = try(var.addons.enable_ack_apigatewayv2, false) + enable_ack_dynamodb = try(var.addons.enable_ack_dynamodb, false) + enable_ack_s3 = try(var.addons.enable_ack_s3, false) + enable_ack_rds = try(var.addons.enable_ack_rds, false) + enable_ack_prometheusservice = try(var.addons.enable_ack_prometheusservice, false) + enable_ack_emrcontainers = try(var.addons.enable_ack_emrcontainers, false) + enable_ack_sfn = try(var.addons.enable_ack_sfn, false) + enable_ack_eventbridge = try(var.addons.enable_ack_eventbridge, false) + enable_aws_argocd_ingress = try(var.addons.enable_aws_argocd_ingress, false) } oss_addons = { - #enable_argo_rollouts = true - #enable_argo_events = true - #enable_argo_workflows = true - #enable_cluster_proportional_autoscaler = true - #enable_gatekeeper = true - #enable_gpu_operator = true - #enable_ingress_nginx = true - #enable_kyverno = true - #enable_kube_prometheus_stack = true - enable_metrics_server = true - #enable_prometheus_adapter = true - #enable_secrets_store_csi_driver = true - #enable_vpa = true - #enable_foo = true # you can add any addon here, make sure to update the gitops repo with the corresponding application set + enable_argocd = try(var.addons.enable_argocd, true) + enable_argo_rollouts = try(var.addons.enable_argo_rollouts, false) + enable_argo_events = try(var.addons.enable_argo_events, false) + enable_argo_workflows = try(var.addons.enable_argo_workflows, false) + enable_cluster_proportional_autoscaler = try(var.addons.enable_cluster_proportional_autoscaler, false) + enable_gatekeeper = try(var.addons.enable_gatekeeper, false) + enable_gpu_operator = try(var.addons.enable_gpu_operator, false) + enable_ingress_nginx = try(var.addons.enable_ingress_nginx, false) + enable_kyverno = try(var.addons.enable_kyverno, false) + enable_kube_prometheus_stack = try(var.addons.enable_kube_prometheus_stack, false) + enable_metrics_server = try(var.addons.enable_metrics_server, false) + enable_prometheus_adapter = try(var.addons.enable_prometheus_adapter, false) + enable_secrets_store_csi_driver = try(var.addons.enable_secrets_store_csi_driver, false) + enable_vpa = try(var.addons.enable_vpa, false) } - addons = merge(local.aws_addons, local.oss_addons, { kubernetes_version = local.cluster_version }, { aws_cluster_name = module.eks.cluster_name }) + addons = merge( + local.aws_addons, + local.oss_addons, + { kubernetes_version = local.cluster_version }, + { aws_cluster_name = module.eks.cluster_name } + ) addons_metadata = merge( module.eks_blueprints_addons.gitops_metadata, @@ -101,12 +120,9 @@ locals { workloads = file("${path.module}/bootstrap/workloads.yaml") } - vpc_cidr = "10.0.0.0/16" - azs = slice(data.aws_availability_zones.available.names, 0, 3) - tags = { Blueprint = local.name - GithubRepo = "github.com/csantanapr/terraform-gitops-bridge" + GithubRepo = "github.com/gitops-bridge-dev/gitops-bridge" } } @@ -118,7 +134,6 @@ module "gitops_bridge_bootstrap" { cluster = { cluster_name = module.eks.cluster_name - environment = local.environment metadata = local.addons_metadata addons = local.addons } @@ -177,21 +192,21 @@ module "eks_blueprints_addons" { create_kubernetes_resources = false # EKS Blueprints Addons - enable_cert_manager = try(local.aws_addons.enable_cert_manager, false) - enable_aws_efs_csi_driver = try(local.aws_addons.enable_aws_efs_csi_driver, false) - enable_aws_fsx_csi_driver = try(local.aws_addons.enable_aws_fsx_csi_driver, false) - enable_aws_cloudwatch_metrics = try(local.aws_addons.enable_aws_cloudwatch_metrics, false) - enable_aws_privateca_issuer = try(local.aws_addons.enable_aws_privateca_issuer, false) - enable_cluster_autoscaler = try(local.aws_addons.enable_cluster_autoscaler, false) - enable_external_dns = try(local.aws_addons.enable_external_dns, false) - enable_external_secrets = try(local.aws_addons.enable_external_secrets, false) - enable_aws_load_balancer_controller = try(local.aws_addons.enable_aws_load_balancer_controller, false) - enable_fargate_fluentbit = try(local.aws_addons.enable_fargate_fluentbit, false) - enable_aws_for_fluentbit = try(local.aws_addons.enable_aws_for_fluentbit, false) - enable_aws_node_termination_handler = try(local.aws_addons.enable_aws_node_termination_handler, false) - enable_karpenter = try(local.aws_addons.enable_karpenter, false) - enable_velero = try(local.aws_addons.enable_velero, false) - enable_aws_gateway_api_controller = try(local.aws_addons.enable_aws_gateway_api_controller, false) + enable_cert_manager = local.aws_addons.enable_cert_manager + enable_aws_efs_csi_driver = local.aws_addons.enable_aws_efs_csi_driver + enable_aws_fsx_csi_driver = local.aws_addons.enable_aws_fsx_csi_driver + enable_aws_cloudwatch_metrics = local.aws_addons.enable_aws_cloudwatch_metrics + enable_aws_privateca_issuer = local.aws_addons.enable_aws_privateca_issuer + enable_cluster_autoscaler = local.aws_addons.enable_cluster_autoscaler + enable_external_dns = local.aws_addons.enable_external_dns + enable_external_secrets = local.aws_addons.enable_external_secrets + enable_aws_load_balancer_controller = local.aws_addons.enable_aws_load_balancer_controller + enable_fargate_fluentbit = local.aws_addons.enable_fargate_fluentbit + enable_aws_for_fluentbit = local.aws_addons.enable_aws_for_fluentbit + enable_aws_node_termination_handler = local.aws_addons.enable_aws_node_termination_handler + enable_karpenter = local.aws_addons.enable_karpenter + enable_velero = local.aws_addons.enable_velero + enable_aws_gateway_api_controller = local.aws_addons.enable_aws_gateway_api_controller tags = local.tags } diff --git a/argocd/iac/terraform/examples/eks/aws-secrets-manager/variables.tf b/argocd/iac/terraform/examples/eks/aws-secrets-manager/variables.tf index c1d456eb..1de28fa7 100644 --- a/argocd/iac/terraform/examples/eks/aws-secrets-manager/variables.tf +++ b/argocd/iac/terraform/examples/eks/aws-secrets-manager/variables.tf @@ -1,20 +1,49 @@ +variable "vpc_cidr" { + description = "VPC CIDR" + type = string + default = "10.0.0.0/16" +} +variable "region" { + description = "AWS region" + type = string + default = "us-west-2" +} +variable "kubernetes_version" { + description = "Kubernetes version" + type = string + default = "1.28" +} +variable "addons" { + description = "Kubernetes addons" + type = any + default = { + enable_aws_load_balancer_controller = true + enable_metrics_server = true + } +} +# Addons Git variable "gitops_addons_org" { description = "Git repository org/user contains for addons" + type = string default = "https://github.com/gitops-bridge-dev" } variable "gitops_addons_repo" { description = "Git repository contains for addons" + type = string default = "gitops-bridge-argocd-control-plane-template" } +variable "gitops_addons_revision" { + description = "Git repository revision/branch/ref for addons" + type = string + default = "main" +} variable "gitops_addons_basepath" { description = "Git repository base path for addons" + type = string default = "" } variable "gitops_addons_path" { description = "Git repository path for addons" + type = string default = "bootstrap/control-plane/addons" } -variable "gitops_addons_revision" { - description = "Git repository revision/branch/ref for addons" - default = "HEAD" -} From cecb9ed6c80038aa24bb19775bfb8813a14a2c78 Mon Sep 17 00:00:00 2001 From: Carlos Santana Date: Sat, 28 Oct 2023 22:48:45 -0400 Subject: [PATCH 06/16] update crossplane Signed-off-by: Carlos Santana --- .../examples/eks/aws-secrets-manager/main.tf | 2 - .../terraform/examples/eks/complete/main.tf | 125 +++++++++------- .../examples/eks/complete/variables.tf | 58 +++++++- .../terraform/examples/eks/crossplane/main.tf | 133 ++++++++++-------- .../examples/eks/crossplane/variables.tf | 38 ++++- .../examples/eks/getting-started/main.tf | 2 +- 6 files changed, 230 insertions(+), 128 deletions(-) diff --git a/argocd/iac/terraform/examples/eks/aws-secrets-manager/main.tf b/argocd/iac/terraform/examples/eks/aws-secrets-manager/main.tf index c780703d..a86d1490 100644 --- a/argocd/iac/terraform/examples/eks/aws-secrets-manager/main.tf +++ b/argocd/iac/terraform/examples/eks/aws-secrets-manager/main.tf @@ -41,8 +41,6 @@ locals { vpc_cidr = var.vpc_cidr azs = slice(data.aws_availability_zones.available.names, 0, 3) - - gitops_addons_url = "${var.gitops_addons_org}/${var.gitops_addons_repo}" gitops_addons_basepath = var.gitops_addons_basepath gitops_addons_path = var.gitops_addons_path diff --git a/argocd/iac/terraform/examples/eks/complete/main.tf b/argocd/iac/terraform/examples/eks/complete/main.tf index 39bdd1bf..11b3a21e 100644 --- a/argocd/iac/terraform/examples/eks/complete/main.tf +++ b/argocd/iac/terraform/examples/eks/complete/main.tf @@ -31,50 +31,70 @@ provider "kubernetes" { } locals { - name = "ex-${replace(basename(path.cwd), "_", "-")}" - environment = "dev" - region = "us-west-2" - cluster_version = "1.27" + name = "ex-${replace(basename(path.cwd), "_", "-")}" + environment = "prod" + region = var.region + + cluster_version = var.kubernetes_version + + vpc_cidr = var.vpc_cidr + azs = slice(data.aws_availability_zones.available.names, 0, 3) + gitops_addons_url = "${var.gitops_addons_org}/${var.gitops_addons_repo}" gitops_addons_basepath = var.gitops_addons_basepath gitops_addons_path = var.gitops_addons_path gitops_addons_revision = var.gitops_addons_revision aws_addons = { - enable_cert_manager = true - enable_aws_efs_csi_driver = true - enable_aws_fsx_csi_driver = true - enable_aws_cloudwatch_metrics = true - enable_aws_privateca_issuer = true - enable_cluster_autoscaler = true - enable_external_dns = true - enable_external_secrets = true - enable_aws_load_balancer_controller = true - enable_aws_for_fluentbit = true - enable_aws_node_termination_handler = true - enable_karpenter = true - enable_velero = true - enable_aws_gateway_api_controller = true - enable_aws_ebs_csi_resources = true # generate gp2 and gp3 storage classes for ebs-csi - enable_aws_secrets_store_csi_driver_provider = true - #enable_fargate_fluentbit = true + enable_cert_manager = try(var.addons.enable_cert_manager, false) + enable_aws_efs_csi_driver = try(var.addons.enable_aws_efs_csi_driver, false) + enable_aws_fsx_csi_driver = try(var.addons.enable_aws_fsx_csi_driver, false) + enable_aws_cloudwatch_metrics = try(var.addons.enable_aws_cloudwatch_metrics, false) + enable_aws_privateca_issuer = try(var.addons.enable_aws_privateca_issuer, false) + enable_cluster_autoscaler = try(var.addons.enable_cluster_autoscaler, false) + enable_external_dns = try(var.addons.enable_external_dns, false) + enable_external_secrets = try(var.addons.enable_external_secrets, false) + enable_aws_load_balancer_controller = try(var.addons.enable_aws_load_balancer_controller, false) + enable_fargate_fluentbit = try(var.addons.enable_fargate_fluentbit, false) + enable_aws_for_fluentbit = try(var.addons.enable_aws_for_fluentbit, false) + enable_aws_node_termination_handler = try(var.addons.enable_aws_node_termination_handler, false) + enable_karpenter = try(var.addons.enable_karpenter, false) + enable_velero = try(var.addons.enable_velero, false) + enable_aws_gateway_api_controller = try(var.addons.enable_aws_gateway_api_controller, false) + enable_aws_ebs_csi_resources = try(var.addons.enable_aws_ebs_csi_resources, false) + enable_aws_secrets_store_csi_driver_provider = try(var.addons.enable_aws_secrets_store_csi_driver_provider, false) + enable_ack_apigatewayv2 = try(var.addons.enable_ack_apigatewayv2, false) + enable_ack_dynamodb = try(var.addons.enable_ack_dynamodb, false) + enable_ack_s3 = try(var.addons.enable_ack_s3, false) + enable_ack_rds = try(var.addons.enable_ack_rds, false) + enable_ack_prometheusservice = try(var.addons.enable_ack_prometheusservice, false) + enable_ack_emrcontainers = try(var.addons.enable_ack_emrcontainers, false) + enable_ack_sfn = try(var.addons.enable_ack_sfn, false) + enable_ack_eventbridge = try(var.addons.enable_ack_eventbridge, false) + enable_aws_argocd_ingress = try(var.addons.enable_aws_argocd_ingress, false) } oss_addons = { - #enable_cluster_proportional_autoscaler = true - #enable_gatekeeper = true - #enable_kyverno = true - #enable_ingress_nginx = true - enable_argo_rollouts = true - enable_argo_workflows = true - enable_gpu_operator = true - enable_kube_prometheus_stack = true - enable_metrics_server = true - enable_prometheus_adapter = true - enable_secrets_store_csi_driver = true - enable_vpa = true - enable_foo = true # you can add any addon here, make sure to update the gitops repo with the corresponding application set + enable_argocd = try(var.addons.enable_argocd, true) + enable_argo_rollouts = try(var.addons.enable_argo_rollouts, false) + enable_argo_events = try(var.addons.enable_argo_events, false) + enable_argo_workflows = try(var.addons.enable_argo_workflows, false) + enable_cluster_proportional_autoscaler = try(var.addons.enable_cluster_proportional_autoscaler, false) + enable_gatekeeper = try(var.addons.enable_gatekeeper, false) + enable_gpu_operator = try(var.addons.enable_gpu_operator, false) + enable_ingress_nginx = try(var.addons.enable_ingress_nginx, false) + enable_kyverno = try(var.addons.enable_kyverno, false) + enable_kube_prometheus_stack = try(var.addons.enable_kube_prometheus_stack, false) + enable_metrics_server = try(var.addons.enable_metrics_server, false) + enable_prometheus_adapter = try(var.addons.enable_prometheus_adapter, false) + enable_secrets_store_csi_driver = try(var.addons.enable_secrets_store_csi_driver, false) + enable_vpa = try(var.addons.enable_vpa, false) } - addons = merge(local.aws_addons, local.oss_addons, { kubernetes_version = local.cluster_version }, { aws_cluster_name = module.eks.cluster_name }) + addons = merge( + local.aws_addons, + local.oss_addons, + { kubernetes_version = local.cluster_version }, + { aws_cluster_name = module.eks.cluster_name } + ) addons_metadata = merge( module.eks_blueprints_addons.gitops_metadata, @@ -104,12 +124,9 @@ locals { workloads = file("${path.module}/bootstrap/workloads.yaml") } - vpc_cidr = "10.0.0.0/16" - azs = slice(data.aws_availability_zones.available.names, 0, 3) - tags = { Blueprint = local.name - GithubRepo = "github.com/csantanapr/terraform-gitops-bridge" + GithubRepo = "github.com/gitops-bridge-dev/gitops-bridge" } velero_backup_s3_bucket = try(split(":", module.velero_backup_s3_bucket.s3_bucket_arn), []) @@ -149,21 +166,21 @@ module "eks_blueprints_addons" { create_kubernetes_resources = false # EKS Blueprints Addons - enable_cert_manager = try(local.aws_addons.enable_cert_manager, false) - enable_aws_efs_csi_driver = try(local.aws_addons.enable_aws_efs_csi_driver, false) - enable_aws_fsx_csi_driver = try(local.aws_addons.enable_aws_fsx_csi_driver, false) - enable_aws_cloudwatch_metrics = try(local.aws_addons.enable_aws_cloudwatch_metrics, false) - enable_aws_privateca_issuer = try(local.aws_addons.enable_aws_privateca_issuer, false) - enable_cluster_autoscaler = try(local.aws_addons.enable_cluster_autoscaler, false) - enable_external_dns = try(local.aws_addons.enable_external_dns, false) - enable_external_secrets = try(local.aws_addons.enable_external_secrets, false) - enable_aws_load_balancer_controller = try(local.aws_addons.enable_aws_load_balancer_controller, false) - enable_fargate_fluentbit = try(local.aws_addons.enable_fargate_fluentbit, false) - enable_aws_for_fluentbit = try(local.aws_addons.enable_aws_for_fluentbit, false) - enable_aws_node_termination_handler = try(local.aws_addons.enable_aws_node_termination_handler, false) - enable_karpenter = try(local.aws_addons.enable_karpenter, false) - enable_velero = try(local.aws_addons.enable_velero, false) - enable_aws_gateway_api_controller = try(local.aws_addons.enable_aws_gateway_api_controller, false) + enable_cert_manager = local.aws_addons.enable_cert_manager + enable_aws_efs_csi_driver = local.aws_addons.enable_aws_efs_csi_driver + enable_aws_fsx_csi_driver = local.aws_addons.enable_aws_fsx_csi_driver + enable_aws_cloudwatch_metrics = local.aws_addons.enable_aws_cloudwatch_metrics + enable_aws_privateca_issuer = local.aws_addons.enable_aws_privateca_issuer + enable_cluster_autoscaler = local.aws_addons.enable_cluster_autoscaler + enable_external_dns = local.aws_addons.enable_external_dns + enable_external_secrets = local.aws_addons.enable_external_secrets + enable_aws_load_balancer_controller = local.aws_addons.enable_aws_load_balancer_controller + enable_fargate_fluentbit = local.aws_addons.enable_fargate_fluentbit + enable_aws_for_fluentbit = local.aws_addons.enable_aws_for_fluentbit + enable_aws_node_termination_handler = local.aws_addons.enable_aws_node_termination_handler + enable_karpenter = local.aws_addons.enable_karpenter + enable_velero = local.aws_addons.enable_velero + enable_aws_gateway_api_controller = local.aws_addons.enable_aws_gateway_api_controller external_dns_route53_zone_arns = ["arn:aws:route53:::hostedzone/Z123456789"] # fake value for testing #external_dns_route53_zone_arns = [data.aws_route53_zone.domain_name.arn] diff --git a/argocd/iac/terraform/examples/eks/complete/variables.tf b/argocd/iac/terraform/examples/eks/complete/variables.tf index c36048e5..5f25beb5 100644 --- a/argocd/iac/terraform/examples/eks/complete/variables.tf +++ b/argocd/iac/terraform/examples/eks/complete/variables.tf @@ -1,3 +1,51 @@ +variable "vpc_cidr" { + description = "VPC CIDR" + type = string + default = "10.0.0.0/16" +} +variable "region" { + description = "AWS region" + type = string + default = "us-west-2" +} +variable "kubernetes_version" { + description = "Kubernetes version" + type = string + default = "1.28" +} + +variable "addons" { + description = "Kubernetes addons" + type = any + default = { + enable_cert_manager = true + enable_aws_efs_csi_driver = true + enable_aws_fsx_csi_driver = true + enable_aws_cloudwatch_metrics = true + enable_aws_privateca_issuer = true + enable_cluster_autoscaler = true + enable_external_dns = true + enable_external_secrets = true + enable_aws_load_balancer_controller = true + enable_aws_for_fluentbit = true + enable_aws_node_termination_handler = true + enable_karpenter = true + enable_velero = true + enable_aws_gateway_api_controller = true + enable_aws_ebs_csi_resources = true # generate gp2 and gp3 storage classes for ebs-csi + enable_aws_secrets_store_csi_driver_provider = true + enable_argo_rollouts = true + enable_argo_workflows = true + enable_gpu_operator = true + enable_kube_prometheus_stack = true + enable_metrics_server = true + enable_prometheus_adapter = true + enable_secrets_store_csi_driver = true + enable_vpa = true + enable_foo = true # you can add any addon here, make sure to update the gitops repo with the corresponding application set + } +} +# Addons Git variable "gitops_addons_org" { description = "Git repository org/user contains for addons" type = string @@ -8,6 +56,11 @@ variable "gitops_addons_repo" { type = string default = "gitops-bridge-argocd-control-plane-template" } +variable "gitops_addons_revision" { + description = "Git repository revision/branch/ref for addons" + type = string + default = "main" +} variable "gitops_addons_basepath" { description = "Git repository base path for addons" type = string @@ -18,8 +71,3 @@ variable "gitops_addons_path" { type = string default = "bootstrap/control-plane/addons" } -variable "gitops_addons_revision" { - description = "Git repository revision/branch/ref for addons" - type = string - default = "HEAD" -} diff --git a/argocd/iac/terraform/examples/eks/crossplane/main.tf b/argocd/iac/terraform/examples/eks/crossplane/main.tf index 0098ca0f..2621edc5 100644 --- a/argocd/iac/terraform/examples/eks/crossplane/main.tf +++ b/argocd/iac/terraform/examples/eks/crossplane/main.tf @@ -31,57 +31,71 @@ provider "kubernetes" { } locals { - name = "ex-${replace(basename(path.cwd), "_", "-")}" - environment = "control-plane" - region = "us-west-2" - cluster_version = "1.27" + name = "ex-${replace(basename(path.cwd), "_", "-")}" + environment = "control-plane" + region = var.region + + cluster_version = var.kubernetes_version + + vpc_cidr = var.vpc_cidr + azs = slice(data.aws_availability_zones.available.names, 0, 3) + + gitops_addons_url = "${var.gitops_addons_org}/${var.gitops_addons_repo}" gitops_addons_basepath = var.gitops_addons_basepath gitops_addons_path = var.gitops_addons_path gitops_addons_revision = var.gitops_addons_revision aws_addons = { - enable_cert_manager = true - enable_aws_crossplane = true # installs aws crossplane providers - enable_aws_crossplane_provider = false # installs aws contrib provider - enable_aws_crossplane_upbound_provider = true # installs aws upbound provider - enable_crossplane_kubernetes_provider = true # installs kubernetes provider - enable_crossplane_helm_provider = true # installs helm provider - #enable_aws_efs_csi_driver = true - #enable_aws_fsx_csi_driver = true - #enable_aws_cloudwatch_metrics = true - #enable_aws_privateca_issuer = true - #enable_cluster_autoscaler = true - #enable_external_dns = true - #enable_external_secrets = true - #enable_aws_load_balancer_controller = true - #enable_fargate_fluentbit = true - #enable_aws_for_fluentbit = true - #enable_aws_node_termination_handler = true - #enable_karpenter = true - #enable_velero = true - #enable_aws_gateway_api_controller = true - #enable_aws_ebs_csi_resources = true # generate gp2 and gp3 storage classes for ebs-csi - #enable_aws_secrets_store_csi_driver_provider = true + enable_cert_manager = try(var.addons.enable_cert_manager, false) + enable_aws_efs_csi_driver = try(var.addons.enable_aws_efs_csi_driver, false) + enable_aws_fsx_csi_driver = try(var.addons.enable_aws_fsx_csi_driver, false) + enable_aws_cloudwatch_metrics = try(var.addons.enable_aws_cloudwatch_metrics, false) + enable_aws_privateca_issuer = try(var.addons.enable_aws_privateca_issuer, false) + enable_cluster_autoscaler = try(var.addons.enable_cluster_autoscaler, false) + enable_external_dns = try(var.addons.enable_external_dns, false) + enable_external_secrets = try(var.addons.enable_external_secrets, false) + enable_aws_load_balancer_controller = try(var.addons.enable_aws_load_balancer_controller, false) + enable_fargate_fluentbit = try(var.addons.enable_fargate_fluentbit, false) + enable_aws_for_fluentbit = try(var.addons.enable_aws_for_fluentbit, false) + enable_aws_node_termination_handler = try(var.addons.enable_aws_node_termination_handler, false) + enable_karpenter = try(var.addons.enable_karpenter, false) + enable_velero = try(var.addons.enable_velero, false) + enable_aws_gateway_api_controller = try(var.addons.enable_aws_gateway_api_controller, false) + enable_aws_ebs_csi_resources = try(var.addons.enable_aws_ebs_csi_resources, false) + enable_aws_secrets_store_csi_driver_provider = try(var.addons.enable_aws_secrets_store_csi_driver_provider, false) + enable_ack_apigatewayv2 = try(var.addons.enable_ack_apigatewayv2, false) + enable_ack_dynamodb = try(var.addons.enable_ack_dynamodb, false) + enable_ack_s3 = try(var.addons.enable_ack_s3, false) + enable_ack_rds = try(var.addons.enable_ack_rds, false) + enable_ack_prometheusservice = try(var.addons.enable_ack_prometheusservice, false) + enable_ack_emrcontainers = try(var.addons.enable_ack_emrcontainers, false) + enable_ack_sfn = try(var.addons.enable_ack_sfn, false) + enable_ack_eventbridge = try(var.addons.enable_ack_eventbridge, false) + enable_aws_argocd_ingress = try(var.addons.enable_aws_argocd_ingress, false) } oss_addons = { - enable_crossplane = true # installs crossplane core - enable_metrics_server = true - #enable_argo_rollouts = true - #enable_argo_events = true - #enable_argo_workflows = true - #enable_cluster_proportional_autoscaler = true - #enable_gatekeeper = true - #enable_gpu_operator = true - #enable_ingress_nginx = true - #enable_kyverno = true - #enable_kube_prometheus_stack = true - #enable_prometheus_adapter = true - #enable_secrets_store_csi_driver = true - #enable_vpa = true - #enable_foo = true # you can add any addon here, make sure to update the gitops repo with the corresponding application set + enable_argocd = try(var.addons.enable_argocd, true) + enable_argo_rollouts = try(var.addons.enable_argo_rollouts, false) + enable_argo_events = try(var.addons.enable_argo_events, false) + enable_argo_workflows = try(var.addons.enable_argo_workflows, false) + enable_cluster_proportional_autoscaler = try(var.addons.enable_cluster_proportional_autoscaler, false) + enable_gatekeeper = try(var.addons.enable_gatekeeper, false) + enable_gpu_operator = try(var.addons.enable_gpu_operator, false) + enable_ingress_nginx = try(var.addons.enable_ingress_nginx, false) + enable_kyverno = try(var.addons.enable_kyverno, false) + enable_kube_prometheus_stack = try(var.addons.enable_kube_prometheus_stack, false) + enable_metrics_server = try(var.addons.enable_metrics_server, false) + enable_prometheus_adapter = try(var.addons.enable_prometheus_adapter, false) + enable_secrets_store_csi_driver = try(var.addons.enable_secrets_store_csi_driver, false) + enable_vpa = try(var.addons.enable_vpa, false) } - addons = merge(local.aws_addons, local.oss_addons, { kubernetes_version = local.cluster_version }, { aws_cluster_name = module.eks.cluster_name }) + addons = merge( + local.aws_addons, + local.oss_addons, + { kubernetes_version = local.cluster_version }, + { aws_cluster_name = module.eks.cluster_name } + ) addons_metadata = merge( module.eks_blueprints_addons.gitops_metadata, @@ -108,12 +122,9 @@ locals { workloads = file("${path.module}/bootstrap/workloads.yaml") } - vpc_cidr = "10.0.0.0/16" - azs = slice(data.aws_availability_zones.available.names, 0, 3) - tags = { Blueprint = local.name - GithubRepo = "github.com/csantanapr/terraform-gitops-bridge" + GithubRepo = "github.com/gitops-bridge-dev/gitops-bridge" } } @@ -179,21 +190,21 @@ module "eks_blueprints_addons" { create_kubernetes_resources = false # EKS Blueprints Addons - enable_cert_manager = try(local.aws_addons.enable_cert_manager, false) - enable_aws_efs_csi_driver = try(local.aws_addons.enable_aws_efs_csi_driver, false) - enable_aws_fsx_csi_driver = try(local.aws_addons.enable_aws_fsx_csi_driver, false) - enable_aws_cloudwatch_metrics = try(local.aws_addons.enable_aws_cloudwatch_metrics, false) - enable_aws_privateca_issuer = try(local.aws_addons.enable_aws_privateca_issuer, false) - enable_cluster_autoscaler = try(local.aws_addons.enable_cluster_autoscaler, false) - enable_external_dns = try(local.aws_addons.enable_external_dns, false) - enable_external_secrets = try(local.aws_addons.enable_external_secrets, false) - enable_aws_load_balancer_controller = try(local.aws_addons.enable_aws_load_balancer_controller, false) - enable_fargate_fluentbit = try(local.aws_addons.enable_fargate_fluentbit, false) - enable_aws_for_fluentbit = try(local.aws_addons.enable_aws_for_fluentbit, false) - enable_aws_node_termination_handler = try(local.aws_addons.enable_aws_node_termination_handler, false) - enable_karpenter = try(local.aws_addons.enable_karpenter, false) - enable_velero = try(local.aws_addons.enable_velero, false) - enable_aws_gateway_api_controller = try(local.aws_addons.enable_aws_gateway_api_controller, false) + enable_cert_manager = local.aws_addons.enable_cert_manager + enable_aws_efs_csi_driver = local.aws_addons.enable_aws_efs_csi_driver + enable_aws_fsx_csi_driver = local.aws_addons.enable_aws_fsx_csi_driver + enable_aws_cloudwatch_metrics = local.aws_addons.enable_aws_cloudwatch_metrics + enable_aws_privateca_issuer = local.aws_addons.enable_aws_privateca_issuer + enable_cluster_autoscaler = local.aws_addons.enable_cluster_autoscaler + enable_external_dns = local.aws_addons.enable_external_dns + enable_external_secrets = local.aws_addons.enable_external_secrets + enable_aws_load_balancer_controller = local.aws_addons.enable_aws_load_balancer_controller + enable_fargate_fluentbit = local.aws_addons.enable_fargate_fluentbit + enable_aws_for_fluentbit = local.aws_addons.enable_aws_for_fluentbit + enable_aws_node_termination_handler = local.aws_addons.enable_aws_node_termination_handler + enable_karpenter = local.aws_addons.enable_karpenter + enable_velero = local.aws_addons.enable_velero + enable_aws_gateway_api_controller = local.aws_addons.enable_aws_gateway_api_controller tags = local.tags } diff --git a/argocd/iac/terraform/examples/eks/crossplane/variables.tf b/argocd/iac/terraform/examples/eks/crossplane/variables.tf index c36048e5..fc76c47d 100644 --- a/argocd/iac/terraform/examples/eks/crossplane/variables.tf +++ b/argocd/iac/terraform/examples/eks/crossplane/variables.tf @@ -1,3 +1,31 @@ +variable "vpc_cidr" { + description = "VPC CIDR" + type = string + default = "10.0.0.0/16" +} +variable "region" { + description = "AWS region" + type = string + default = "us-west-2" +} +variable "kubernetes_version" { + description = "Kubernetes version" + type = string + default = "1.28" +} +variable "addons" { + description = "Kubernetes addons" + type = any + default = { + enable_aws_crossplane = true # installs aws crossplane providers + enable_aws_crossplane_provider = false # installs aws contrib provider + enable_aws_crossplane_upbound_provider = true # installs aws upbound provider + enable_crossplane_kubernetes_provider = true # installs kubernetes provider + enable_crossplane_helm_provider = true # installs helm provider + enable_crossplane = true # installs crossplane core + } +} +# Addons Git variable "gitops_addons_org" { description = "Git repository org/user contains for addons" type = string @@ -8,6 +36,11 @@ variable "gitops_addons_repo" { type = string default = "gitops-bridge-argocd-control-plane-template" } +variable "gitops_addons_revision" { + description = "Git repository revision/branch/ref for addons" + type = string + default = "main" +} variable "gitops_addons_basepath" { description = "Git repository base path for addons" type = string @@ -18,8 +51,3 @@ variable "gitops_addons_path" { type = string default = "bootstrap/control-plane/addons" } -variable "gitops_addons_revision" { - description = "Git repository revision/branch/ref for addons" - type = string - default = "HEAD" -} diff --git a/argocd/iac/terraform/examples/eks/getting-started/main.tf b/argocd/iac/terraform/examples/eks/getting-started/main.tf index 0defa86f..799c02af 100644 --- a/argocd/iac/terraform/examples/eks/getting-started/main.tf +++ b/argocd/iac/terraform/examples/eks/getting-started/main.tf @@ -123,7 +123,7 @@ locals { tags = { Blueprint = local.name - GithubRepo = "github.com/aws-ia/terraform-aws-eks-blueprints" + GithubRepo = "github.com/gitops-bridge-dev/gitops-bridge" } } From ecab8753406f712a552ddfb67d21dd24d997e1db Mon Sep 17 00:00:00 2001 From: Carlos Santana Date: Sat, 28 Oct 2023 23:02:20 -0400 Subject: [PATCH 07/16] external secrets Signed-off-by: Carlos Santana --- .../terraform/examples/eks/complete/README.md | 39 ++++- .../examples/eks/external-secrets/README.md | 56 ++++++- .../external-secrets/bootstrap/workloads.yaml | 2 +- .../examples/eks/external-secrets/main.tf | 140 ++++++++++-------- .../eks/external-secrets/variables.tf | 51 +++++-- 5 files changed, 202 insertions(+), 86 deletions(-) diff --git a/argocd/iac/terraform/examples/eks/complete/README.md b/argocd/iac/terraform/examples/eks/complete/README.md index b56d7433..34aa8ccf 100644 --- a/argocd/iac/terraform/examples/eks/complete/README.md +++ b/argocd/iac/terraform/examples/eks/complete/README.md @@ -3,17 +3,48 @@ Example on how to deploy Amazon EKS with addons configured via ArgoCD. In this example shows how to enable all the available addons -Deploy EKS Cluster +## Prerequisites +Before you begin, make sure you have the following command line tools installed: +- git +- terraform +- kubectl +- argocd + +## Fork the Git Repositories + +### Fork the Addon GitOps Repo +1. Fork the git repository for addons [here](https://github.com/gitops-bridge-dev/gitops-bridge-argocd-control-plane-template). +2. Update the following environment variables to point to your fork by changing the default values: +```shell +export TF_VAR_gitops_addons_org=https://github.com/gitops-bridge-dev +export TF_VAR_gitops_addons_repo=gitops-bridge-argocd-control-plane-template +``` + + +## Deploy the EKS Cluster +Initialize Terraform and deploy the EKS cluster: ```shell terraform init -terraform apply +terraform apply -auto-approve +``` +Retrieve `kubectl` config, then execute the output command: +```shell +terraform output -raw configure_kubectl +``` + +### Monitor GitOps Progress for Addons +Wait until all the ArgoCD applications' `HEALTH STATUS` is `Healthy`. Use Crl+C to exit the `watch` command +```shell +watch kubectl get applications -n argocd ``` -Access Terraform output to configure `kubectl` and `argocd` +## Access ArgoCD +Access ArgoCD's UI, run the command from the output: ```shell -terraform output +terraform output -raw access_argocd ``` + Destroy EKS Cluster ```shell cd hub diff --git a/argocd/iac/terraform/examples/eks/external-secrets/README.md b/argocd/iac/terraform/examples/eks/external-secrets/README.md index 3f806a40..49696adc 100644 --- a/argocd/iac/terraform/examples/eks/external-secrets/README.md +++ b/argocd/iac/terraform/examples/eks/external-secrets/README.md @@ -5,15 +5,59 @@ This example shows how to deploy Amazon EKS with addons configured via ArgoCD The example demonstrate how to use [External Secret Operator(ESO)](https://external-secrets.io) with AWS Secret Manager and AWS Systems Manager Parameter Store -Deploy EKS Cluster + + +## Prerequisites +Before you begin, make sure you have the following command line tools installed: +- git +- terraform +- kubectl +- argocd + +## Fork the Git Repositories + +### Fork the Addon GitOps Repo +1. Fork the git repository for addons [here](https://github.com/gitops-bridge-dev/gitops-bridge-argocd-control-plane-template). +2. Update the following environment variables to point to your fork by changing the default values: +```shell +export TF_VAR_gitops_addons_org=https://github.com/gitops-bridge-dev +export TF_VAR_gitops_addons_repo=gitops-bridge-argocd-control-plane-template +``` + +### Fork the Workloads GitOps Repo +1. Fork the git repository for this pattern [here](https://github.com/gitops-bridge-dev/gitops-bridge) +2. Update the following environment variables to point to your fork by changing the default values: +```shell +export TF_VAR_gitops_workload_org=https://github.com/gitops-bridge-dev +export TF_VAR_gitops_workload_repo=gitops-bridge +``` + +## Deploy the EKS Cluster +Initialize Terraform and deploy the EKS cluster: ```shell terraform init -terraform apply +terraform apply -auto-approve +``` +Retrieve `kubectl` config, then execute the output command: +```shell +terraform output -raw configure_kubectl +``` +## Deploy the Addons +Bootstrap the addons using ArgoCD: +```shell +kubectl apply -f bootstrap/addons.yaml +``` + +### Monitor GitOps Progress for Addons +Wait until all the ArgoCD applications' `HEALTH STATUS` is `Healthy`. Use Crl+C to exit the `watch` command +```shell +watch kubectl get applications -n argocd ``` -Access Terraform output to configure `kubectl` and `argocd` +## Access ArgoCD +Access ArgoCD's UI, run the command from the output: ```shell -terraform output +terraform output -raw access_argocd ``` Verify that the secrets `external-secrets-ps` and `external-secrets-sm` are present @@ -28,8 +72,8 @@ external-secrets-ps Opaque 2 1m external-secrets-sm Opaque 2 1m ``` -Destroy EKS Cluster +## Destroy the EKS Cluster +To tear down all the resources and the EKS cluster, run the following command: ```shell -cd hub ./destroy.sh ``` diff --git a/argocd/iac/terraform/examples/eks/external-secrets/bootstrap/workloads.yaml b/argocd/iac/terraform/examples/eks/external-secrets/bootstrap/workloads.yaml index 2cb95175..b8712a14 100644 --- a/argocd/iac/terraform/examples/eks/external-secrets/bootstrap/workloads.yaml +++ b/argocd/iac/terraform/examples/eks/external-secrets/bootstrap/workloads.yaml @@ -20,7 +20,7 @@ spec: project: default source: repoURL: '{{metadata.annotations.workload_repo_url}}' - path: '{{metadata.annotations.workload_repo_path}}' + path: '{{metadata.annotations.workload_repo_basepath}}{{metadata.annotations.workload_repo_path}}' targetRevision: '{{metadata.annotations.workload_repo_revision}}' helm: releaseName: 'external-secrets-example' diff --git a/argocd/iac/terraform/examples/eks/external-secrets/main.tf b/argocd/iac/terraform/examples/eks/external-secrets/main.tf index cf21cdc5..9efa6c52 100644 --- a/argocd/iac/terraform/examples/eks/external-secrets/main.tf +++ b/argocd/iac/terraform/examples/eks/external-secrets/main.tf @@ -31,62 +31,77 @@ provider "kubernetes" { } locals { - name = "ex-${replace(basename(path.cwd), "_", "-")}" - environment = "dev" - region = "us-west-2" - cluster_version = "1.27" + name = "ex-${replace(basename(path.cwd), "_", "-")}" + region = var.region + + cluster_version = var.kubernetes_version + + vpc_cidr = var.vpc_cidr + azs = slice(data.aws_availability_zones.available.names, 0, 3) + + # Secret names in AWS + workload_sm_secret = local.name + workload_pm_secret = "/${local.name}/secret" + gitops_addons_url = "${var.gitops_addons_org}/${var.gitops_addons_repo}" gitops_addons_basepath = var.gitops_addons_basepath gitops_addons_path = var.gitops_addons_path gitops_addons_revision = var.gitops_addons_revision - gitops_workload_org = var.gitops_workload_org - gitops_workload_repo = var.gitops_workload_repo + gitops_workload_url = "${var.gitops_workload_org}/${var.gitops_workload_repo}" + gitops_workload_basepath = var.gitops_workload_basepath gitops_workload_path = var.gitops_workload_path gitops_workload_revision = var.gitops_workload_revision - gitops_workload_url = "${local.gitops_workload_org}/${local.gitops_workload_repo}" - - # Secret names in AWS - workload_sm_secret = local.name - workload_pm_secret = "/${local.name}/secret" - aws_addons = { - #enable_cert_manager = true - #enable_aws_efs_csi_driver = true - #enable_aws_fsx_csi_driver = true - #enable_aws_cloudwatch_metrics = true - #enable_aws_privateca_issuer = true - #enable_cluster_autoscaler = true - #enable_external_dns = true - enable_external_secrets = true - #enable_aws_load_balancer_controller = true - #enable_fargate_fluentbit = true - #enable_aws_for_fluentbit = true - #enable_aws_node_termination_handler = true - #enable_karpenter = true - #enable_velero = true - #enable_aws_gateway_api_controller = true - #enable_aws_ebs_csi_resources = true # generate gp2 and gp3 storage classes for ebs-csi - #enable_aws_secrets_store_csi_driver_provider = true + enable_cert_manager = try(var.addons.enable_cert_manager, false) + enable_aws_efs_csi_driver = try(var.addons.enable_aws_efs_csi_driver, false) + enable_aws_fsx_csi_driver = try(var.addons.enable_aws_fsx_csi_driver, false) + enable_aws_cloudwatch_metrics = try(var.addons.enable_aws_cloudwatch_metrics, false) + enable_aws_privateca_issuer = try(var.addons.enable_aws_privateca_issuer, false) + enable_cluster_autoscaler = try(var.addons.enable_cluster_autoscaler, false) + enable_external_dns = try(var.addons.enable_external_dns, false) + enable_external_secrets = try(var.addons.enable_external_secrets, false) + enable_aws_load_balancer_controller = try(var.addons.enable_aws_load_balancer_controller, false) + enable_fargate_fluentbit = try(var.addons.enable_fargate_fluentbit, false) + enable_aws_for_fluentbit = try(var.addons.enable_aws_for_fluentbit, false) + enable_aws_node_termination_handler = try(var.addons.enable_aws_node_termination_handler, false) + enable_karpenter = try(var.addons.enable_karpenter, false) + enable_velero = try(var.addons.enable_velero, false) + enable_aws_gateway_api_controller = try(var.addons.enable_aws_gateway_api_controller, false) + enable_aws_ebs_csi_resources = try(var.addons.enable_aws_ebs_csi_resources, false) + enable_aws_secrets_store_csi_driver_provider = try(var.addons.enable_aws_secrets_store_csi_driver_provider, false) + enable_ack_apigatewayv2 = try(var.addons.enable_ack_apigatewayv2, false) + enable_ack_dynamodb = try(var.addons.enable_ack_dynamodb, false) + enable_ack_s3 = try(var.addons.enable_ack_s3, false) + enable_ack_rds = try(var.addons.enable_ack_rds, false) + enable_ack_prometheusservice = try(var.addons.enable_ack_prometheusservice, false) + enable_ack_emrcontainers = try(var.addons.enable_ack_emrcontainers, false) + enable_ack_sfn = try(var.addons.enable_ack_sfn, false) + enable_ack_eventbridge = try(var.addons.enable_ack_eventbridge, false) } oss_addons = { - #enable_argo_rollouts = true - #enable_argo_events = true - #enable_argo_workflows = true - #enable_cluster_proportional_autoscaler = true - #enable_gatekeeper = true - #enable_gpu_operator = true - #enable_ingress_nginx = true - #enable_kyverno = true - #enable_kube_prometheus_stack = true - #enable_metrics_server = true - #enable_prometheus_adapter = true - #enable_secrets_store_csi_driver = true - #enable_vpa = true - #enable_foo = true # you can add any addon here, make sure to update the gitops repo with the corresponding application set + enable_argocd = try(var.addons.enable_argocd, true) + enable_argo_rollouts = try(var.addons.enable_argo_rollouts, false) + enable_argo_events = try(var.addons.enable_argo_events, false) + enable_argo_workflows = try(var.addons.enable_argo_workflows, false) + enable_cluster_proportional_autoscaler = try(var.addons.enable_cluster_proportional_autoscaler, false) + enable_gatekeeper = try(var.addons.enable_gatekeeper, false) + enable_gpu_operator = try(var.addons.enable_gpu_operator, false) + enable_ingress_nginx = try(var.addons.enable_ingress_nginx, false) + enable_kyverno = try(var.addons.enable_kyverno, false) + enable_kube_prometheus_stack = try(var.addons.enable_kube_prometheus_stack, false) + enable_metrics_server = try(var.addons.enable_metrics_server, false) + enable_prometheus_adapter = try(var.addons.enable_prometheus_adapter, false) + enable_secrets_store_csi_driver = try(var.addons.enable_secrets_store_csi_driver, false) + enable_vpa = try(var.addons.enable_vpa, false) } - addons = merge(local.aws_addons, local.oss_addons, { kubernetes_version = local.cluster_version }, { aws_cluster_name = module.eks.cluster_name }) + addons = merge( + local.aws_addons, + local.oss_addons, + { kubernetes_version = local.cluster_version }, + { aws_cluster_name = module.eks.cluster_name } + ) addons_metadata = merge( module.eks_blueprints_addons.gitops_metadata, @@ -104,6 +119,7 @@ locals { }, { workload_repo_url = local.gitops_workload_url + workload_repo_basepath = local.gitops_workload_basepath workload_repo_path = local.gitops_workload_path workload_repo_revision = local.gitops_workload_revision }, @@ -118,12 +134,9 @@ locals { workloads = file("${path.module}/bootstrap/workloads.yaml") } - vpc_cidr = "10.0.0.0/16" - azs = slice(data.aws_availability_zones.available.names, 0, 3) - tags = { Blueprint = local.name - GithubRepo = "github.com/csantanapr/terraform-gitops-bridge" + GithubRepo = "github.com/gitops-bridge-dev/gitops-bridge" } } @@ -163,7 +176,6 @@ module "gitops_bridge_bootstrap" { cluster = { cluster_name = module.eks.cluster_name - environment = local.environment metadata = local.addons_metadata addons = local.addons } @@ -186,21 +198,21 @@ module "eks_blueprints_addons" { create_kubernetes_resources = false # EKS Blueprints Addons - enable_cert_manager = try(local.aws_addons.enable_cert_manager, false) - enable_aws_efs_csi_driver = try(local.aws_addons.enable_aws_efs_csi_driver, false) - enable_aws_fsx_csi_driver = try(local.aws_addons.enable_aws_fsx_csi_driver, false) - enable_aws_cloudwatch_metrics = try(local.aws_addons.enable_aws_cloudwatch_metrics, false) - enable_aws_privateca_issuer = try(local.aws_addons.enable_aws_privateca_issuer, false) - enable_cluster_autoscaler = try(local.aws_addons.enable_cluster_autoscaler, false) - enable_external_dns = try(local.aws_addons.enable_external_dns, false) - enable_external_secrets = try(local.aws_addons.enable_external_secrets, false) - enable_aws_load_balancer_controller = try(local.aws_addons.enable_aws_load_balancer_controller, false) - enable_fargate_fluentbit = try(local.aws_addons.enable_fargate_fluentbit, false) - enable_aws_for_fluentbit = try(local.aws_addons.enable_aws_for_fluentbit, false) - enable_aws_node_termination_handler = try(local.aws_addons.enable_aws_node_termination_handler, false) - enable_karpenter = try(local.aws_addons.enable_karpenter, false) - enable_velero = try(local.aws_addons.enable_velero, false) - enable_aws_gateway_api_controller = try(local.aws_addons.enable_aws_gateway_api_controller, false) + enable_cert_manager = local.aws_addons.enable_cert_manager + enable_aws_efs_csi_driver = local.aws_addons.enable_aws_efs_csi_driver + enable_aws_fsx_csi_driver = local.aws_addons.enable_aws_fsx_csi_driver + enable_aws_cloudwatch_metrics = local.aws_addons.enable_aws_cloudwatch_metrics + enable_aws_privateca_issuer = local.aws_addons.enable_aws_privateca_issuer + enable_cluster_autoscaler = local.aws_addons.enable_cluster_autoscaler + enable_external_dns = local.aws_addons.enable_external_dns + enable_external_secrets = local.aws_addons.enable_external_secrets + enable_aws_load_balancer_controller = local.aws_addons.enable_aws_load_balancer_controller + enable_fargate_fluentbit = local.aws_addons.enable_fargate_fluentbit + enable_aws_for_fluentbit = local.aws_addons.enable_aws_for_fluentbit + enable_aws_node_termination_handler = local.aws_addons.enable_aws_node_termination_handler + enable_karpenter = local.aws_addons.enable_karpenter + enable_velero = local.aws_addons.enable_velero + enable_aws_gateway_api_controller = local.aws_addons.enable_aws_gateway_api_controller tags = local.tags } diff --git a/argocd/iac/terraform/examples/eks/external-secrets/variables.tf b/argocd/iac/terraform/examples/eks/external-secrets/variables.tf index 371464fc..4a0687b0 100644 --- a/argocd/iac/terraform/examples/eks/external-secrets/variables.tf +++ b/argocd/iac/terraform/examples/eks/external-secrets/variables.tf @@ -1,3 +1,26 @@ +variable "vpc_cidr" { + description = "VPC CIDR" + type = string + default = "10.0.0.0/16" +} +variable "region" { + description = "AWS region" + type = string + default = "us-west-2" +} +variable "kubernetes_version" { + description = "Kubernetes version" + type = string + default = "1.28" +} +variable "addons" { + description = "Kubernetes addons" + type = any + default = { + enable_external_secrets = true + } +} +# Addons Git variable "gitops_addons_org" { description = "Git repository org/user contains for addons" type = string @@ -8,6 +31,11 @@ variable "gitops_addons_repo" { type = string default = "gitops-bridge-argocd-control-plane-template" } +variable "gitops_addons_revision" { + description = "Git repository revision/branch/ref for addons" + type = string + default = "main" +} variable "gitops_addons_basepath" { description = "Git repository base path for addons" type = string @@ -18,12 +46,8 @@ variable "gitops_addons_path" { type = string default = "bootstrap/control-plane/addons" } -variable "gitops_addons_revision" { - description = "Git repository revision/branch/ref for addons" - type = string - default = "HEAD" -} +# Workloads Git variable "gitops_workload_org" { description = "Git repository org/user contains for workload" type = string @@ -34,13 +58,18 @@ variable "gitops_workload_repo" { type = string default = "gitops-bridge" } -variable "gitops_workload_path" { - description = "Git repository path for workload" - type = string - default = "argocd/iac/terraform/examples/eks/external-secrets/k8s" -} variable "gitops_workload_revision" { description = "Git repository revision/branch/ref for workload" type = string - default = "HEAD" + default = "main" +} +variable "gitops_workload_basepath" { + description = "Git repository base path for workload" + type = string + default = "argocd/iac/terraform/examples/eks/" +} +variable "gitops_workload_path" { + description = "Git repository path for workload" + type = string + default = "external-secrets/k8s" } From e45d147b65472d8259fcc205e94eedf6ce2da938 Mon Sep 17 00:00:00 2001 From: Carlos Santana Date: Sat, 28 Oct 2023 23:03:02 -0400 Subject: [PATCH 08/16] replace hello-world Signed-off-by: Carlos Santana --- .../examples/eks/hello-world/README.md | 21 -- .../eks/hello-world/bootstrap/addons.yaml | 32 --- .../eks/hello-world/bootstrap/workloads.yaml | 20 -- .../examples/eks/hello-world/destroy.sh | 16 -- .../examples/eks/hello-world/main.tf | 231 ------------------ .../examples/eks/hello-world/outputs.tf | 33 --- .../examples/eks/hello-world/variables.tf | 20 -- .../examples/eks/hello-world/versions.tf | 25 -- 8 files changed, 398 deletions(-) delete mode 100644 argocd/iac/terraform/examples/eks/hello-world/README.md delete mode 100644 argocd/iac/terraform/examples/eks/hello-world/bootstrap/addons.yaml delete mode 100644 argocd/iac/terraform/examples/eks/hello-world/bootstrap/workloads.yaml delete mode 100755 argocd/iac/terraform/examples/eks/hello-world/destroy.sh delete mode 100644 argocd/iac/terraform/examples/eks/hello-world/main.tf delete mode 100644 argocd/iac/terraform/examples/eks/hello-world/outputs.tf delete mode 100644 argocd/iac/terraform/examples/eks/hello-world/variables.tf delete mode 100644 argocd/iac/terraform/examples/eks/hello-world/versions.tf diff --git a/argocd/iac/terraform/examples/eks/hello-world/README.md b/argocd/iac/terraform/examples/eks/hello-world/README.md deleted file mode 100644 index cb0f72fc..00000000 --- a/argocd/iac/terraform/examples/eks/hello-world/README.md +++ /dev/null @@ -1,21 +0,0 @@ -# ArgoCD on Amazon EKS - -Example on how to deploy Amazon EKS with addons configured via ArgoCD. -This example shows how to deploy Amazon EKS with addons configured via ArgoCD - -Deploy EKS Cluster -```shell -terraform init -terraform apply -``` - -Access Terraform output to configure `kubectl` and `argocd` -```shell -terraform output -``` - -Destroy EKS Cluster -```shell -cd hub -./destroy.sh -``` diff --git a/argocd/iac/terraform/examples/eks/hello-world/bootstrap/addons.yaml b/argocd/iac/terraform/examples/eks/hello-world/bootstrap/addons.yaml deleted file mode 100644 index 02535923..00000000 --- a/argocd/iac/terraform/examples/eks/hello-world/bootstrap/addons.yaml +++ /dev/null @@ -1,32 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: ApplicationSet -metadata: - name: bootstrap-addons - namespace: argocd -spec: - syncPolicy: - preserveResourcesOnDeletion: true - generators: - - clusters: - selector: - matchExpressions: - - key: akuity.io/argo-cd-cluster-name - operator: NotIn - values: [in-cluster] - template: - metadata: - name: 'bootstrap-addons' - spec: - project: default - source: - repoURL: '{{metadata.annotations.addons_repo_url}}' - path: '{{metadata.annotations.addons_repo_basepath}}{{metadata.annotations.addons_repo_path}}' - targetRevision: '{{metadata.annotations.addons_repo_revision}}' - directory: - recurse: true - exclude: exclude/* - destination: - namespace: 'argocd' - name: '{{name}}' - syncPolicy: - automated: {} diff --git a/argocd/iac/terraform/examples/eks/hello-world/bootstrap/workloads.yaml b/argocd/iac/terraform/examples/eks/hello-world/bootstrap/workloads.yaml deleted file mode 100644 index 60293af4..00000000 --- a/argocd/iac/terraform/examples/eks/hello-world/bootstrap/workloads.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: bootstrap-workloads - namespace: 'argocd' - finalizers: - - resources-finalizer.argocd.argoproj.io -spec: - destination: - server: https://kubernetes.default.svc - namespace: 'guestbook' - project: default - source: - path: helm-guestbook - repoURL: https://github.com/argoproj/argocd-example-apps - targetRevision: HEAD - syncPolicy: - automated: {} - syncOptions: - - CreateNamespace=true diff --git a/argocd/iac/terraform/examples/eks/hello-world/destroy.sh b/argocd/iac/terraform/examples/eks/hello-world/destroy.sh deleted file mode 100755 index 195f9885..00000000 --- a/argocd/iac/terraform/examples/eks/hello-world/destroy.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -set -x - -# Delete the Ingress/SVC before removing the addons -TMPFILE=$(mktemp) -terraform output -raw configure_kubectl > "$TMPFILE" -source "$TMPFILE" - -kubectl delete svc -n argocd argo-cd-argocd-server - -terraform destroy -target="module.gitops_bridge_bootstrap" -auto-approve -terraform destroy -target="module.eks_blueprints_addons" -auto-approve -terraform destroy -target="module.eks" -auto-approve -terraform destroy -target="module.vpc" -auto-approve -terraform destroy -auto-approve diff --git a/argocd/iac/terraform/examples/eks/hello-world/main.tf b/argocd/iac/terraform/examples/eks/hello-world/main.tf deleted file mode 100644 index 649c66d4..00000000 --- a/argocd/iac/terraform/examples/eks/hello-world/main.tf +++ /dev/null @@ -1,231 +0,0 @@ -provider "aws" { - region = local.region -} -data "aws_caller_identity" "current" {} -data "aws_availability_zones" "available" {} - -provider "helm" { - kubernetes { - host = module.eks.cluster_endpoint - cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) - - exec { - api_version = "client.authentication.k8s.io/v1beta1" - command = "aws" - # This requires the awscli to be installed locally where Terraform is executed - args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name, "--region", local.region] - } - } -} - -provider "kubernetes" { - host = module.eks.cluster_endpoint - cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) - - exec { - api_version = "client.authentication.k8s.io/v1beta1" - command = "aws" - # This requires the awscli to be installed locally where Terraform is executed - args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name, "--region", local.region] - } -} - -locals { - name = "ex-${replace(basename(path.cwd), "_", "-")}" - environment = "dev" - region = "us-west-2" - cluster_version = "1.27" - gitops_addons_url = "${var.gitops_addons_org}/${var.gitops_addons_repo}" - gitops_addons_basepath = var.gitops_addons_basepath - gitops_addons_path = var.gitops_addons_path - gitops_addons_revision = var.gitops_addons_revision - - aws_addons = { - enable_cert_manager = true - #enable_aws_efs_csi_driver = true - #enable_aws_fsx_csi_driver = true - #enable_aws_cloudwatch_metrics = true - #enable_aws_privateca_issuer = true - #enable_cluster_autoscaler = true - #enable_external_dns = true - #enable_external_secrets = true - #enable_aws_load_balancer_controller = true - #enable_fargate_fluentbit = true - #enable_aws_for_fluentbit = true - #enable_aws_node_termination_handler = true - #enable_karpenter = true - #enable_velero = true - #enable_aws_gateway_api_controller = true - #enable_aws_ebs_csi_resources = true # generate gp2 and gp3 storage classes for ebs-csi - #enable_aws_secrets_store_csi_driver_provider = true - } - oss_addons = { - #enable_argo_rollouts = true - #enable_argo_events = true - #enable_argo_workflows = true - #enable_cluster_proportional_autoscaler = true - #enable_gatekeeper = true - #enable_gpu_operator = true - #enable_ingress_nginx = true - #enable_kyverno = true - #enable_kube_prometheus_stack = true - enable_metrics_server = true - #enable_prometheus_adapter = true - #enable_secrets_store_csi_driver = true - #enable_vpa = true - #enable_foo = true # you can add any addon here, make sure to update the gitops repo with the corresponding application set - } - addons = merge(local.aws_addons, local.oss_addons, { kubernetes_version = local.cluster_version }, { aws_cluster_name = module.eks.cluster_name }) - - addons_metadata = merge( - module.eks_blueprints_addons.gitops_metadata, - { - aws_cluster_name = module.eks.cluster_name - aws_region = local.region - aws_account_id = data.aws_caller_identity.current.account_id - aws_vpc_id = module.vpc.vpc_id - }, - { - addons_repo_url = local.gitops_addons_url - addons_repo_basepath = local.gitops_addons_basepath - addons_repo_path = local.gitops_addons_path - addons_repo_revision = local.gitops_addons_revision - } - ) - - argocd_apps = { - addons = file("${path.module}/bootstrap/addons.yaml") - workloads = file("${path.module}/bootstrap/workloads.yaml") - } - - vpc_cidr = "10.0.0.0/16" - azs = slice(data.aws_availability_zones.available.names, 0, 3) - - tags = { - Blueprint = local.name - GithubRepo = "github.com/csantanapr/terraform-gitops-bridge" - } -} - -################################################################################ -# GitOps Bridge: Bootstrap -################################################################################ -module "gitops_bridge_bootstrap" { - source = "github.com/gitops-bridge-dev/gitops-bridge-argocd-bootstrap-terraform?ref=v2.0.0" - - cluster = { - cluster_name = module.eks.cluster_name - environment = local.environment - metadata = local.addons_metadata - addons = local.addons - } - apps = local.argocd_apps -} - -################################################################################ -# EKS Blueprints Addons -################################################################################ -module "eks_blueprints_addons" { - source = "aws-ia/eks-blueprints-addons/aws" - version = "~> 1.0" - - cluster_name = module.eks.cluster_name - cluster_endpoint = module.eks.cluster_endpoint - cluster_version = module.eks.cluster_version - oidc_provider_arn = module.eks.oidc_provider_arn - - # Using GitOps Bridge - create_kubernetes_resources = false - - # EKS Blueprints Addons - enable_cert_manager = try(local.aws_addons.enable_cert_manager, false) - enable_aws_efs_csi_driver = try(local.aws_addons.enable_aws_efs_csi_driver, false) - enable_aws_fsx_csi_driver = try(local.aws_addons.enable_aws_fsx_csi_driver, false) - enable_aws_cloudwatch_metrics = try(local.aws_addons.enable_aws_cloudwatch_metrics, false) - enable_aws_privateca_issuer = try(local.aws_addons.enable_aws_privateca_issuer, false) - enable_cluster_autoscaler = try(local.aws_addons.enable_cluster_autoscaler, false) - enable_external_dns = try(local.aws_addons.enable_external_dns, false) - enable_external_secrets = try(local.aws_addons.enable_external_secrets, false) - enable_aws_load_balancer_controller = try(local.aws_addons.enable_aws_load_balancer_controller, false) - enable_fargate_fluentbit = try(local.aws_addons.enable_fargate_fluentbit, false) - enable_aws_for_fluentbit = try(local.aws_addons.enable_aws_for_fluentbit, false) - enable_aws_node_termination_handler = try(local.aws_addons.enable_aws_node_termination_handler, false) - enable_karpenter = try(local.aws_addons.enable_karpenter, false) - enable_velero = try(local.aws_addons.enable_velero, false) - enable_aws_gateway_api_controller = try(local.aws_addons.enable_aws_gateway_api_controller, false) - - tags = local.tags -} - -################################################################################ -# EKS Cluster -################################################################################ -#tfsec:ignore:aws-eks-enable-control-plane-logging -module "eks" { - source = "terraform-aws-modules/eks/aws" - version = "~> 19.13" - - cluster_name = local.name - cluster_version = local.cluster_version - cluster_endpoint_public_access = true - - - vpc_id = module.vpc.vpc_id - subnet_ids = module.vpc.private_subnets - - eks_managed_node_groups = { - initial = { - instance_types = ["t3.medium"] - - min_size = 3 - max_size = 10 - desired_size = 3 - } - } - # EKS Addons - cluster_addons = { - vpc-cni = { - # Specify the VPC CNI addon should be deployed before compute to ensure - # the addon is configured before data plane compute resources are created - # See README for further details - before_compute = true - most_recent = true # To ensure access to the latest settings provided - configuration_values = jsonencode({ - env = { - # Reference docs https://docs.aws.amazon.com/eks/latest/userguide/cni-increase-ip-addresses.html - ENABLE_PREFIX_DELEGATION = "true" - WARM_PREFIX_TARGET = "1" - } - }) - } - } - tags = local.tags -} - -################################################################################ -# Supporting Resources -################################################################################ -module "vpc" { - source = "terraform-aws-modules/vpc/aws" - version = "~> 5.0" - - name = local.name - cidr = local.vpc_cidr - - azs = local.azs - private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 4, k)] - public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 48)] - - enable_nat_gateway = true - single_nat_gateway = true - - public_subnet_tags = { - "kubernetes.io/role/elb" = 1 - } - - private_subnet_tags = { - "kubernetes.io/role/internal-elb" = 1 - } - - tags = local.tags -} diff --git a/argocd/iac/terraform/examples/eks/hello-world/outputs.tf b/argocd/iac/terraform/examples/eks/hello-world/outputs.tf deleted file mode 100644 index 2d3b6e57..00000000 --- a/argocd/iac/terraform/examples/eks/hello-world/outputs.tf +++ /dev/null @@ -1,33 +0,0 @@ -output "configure_kubectl" { - description = "Configure kubectl: make sure you're logged in with the correct AWS profile and run the following command to update your kubeconfig" - value = <<-EOT - export KUBECONFIG="/tmp/${module.eks.cluster_name}" - aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name} - EOT -} - -output "configure_argocd" { - description = "Terminal Setup" - value = <<-EOT - export KUBECONFIG="/tmp/${module.eks.cluster_name}" - aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name} - export ARGOCD_OPTS="--port-forward --port-forward-namespace argocd --grpc-web" - kubectl config set-context --current --namespace argocd - argocd login --port-forward --username admin --password $(argocd admin initial-password | head -1) - echo "ArgoCD Username: admin" - echo "ArgoCD Password: $(kubectl get secrets argocd-initial-admin-secret -n argocd --template="{{index .data.password | base64decode}}")" - echo Port Forward: http://localhost:8080 - kubectl port-forward -n argocd svc/argo-cd-argocd-server 8080:80 - EOT -} - -output "access_argocd" { - description = "ArgoCD Access" - value = <<-EOT - export KUBECONFIG="/tmp/${module.eks.cluster_name}" - aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name} - echo "ArgoCD URL: https://$(kubectl get svc -n argocd argo-cd-argocd-server -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')" - echo "ArgoCD Username: admin" - echo "ArgoCD Password: $(kubectl get secrets argocd-initial-admin-secret -n argocd --template="{{index .data.password | base64decode}}")" - EOT -} diff --git a/argocd/iac/terraform/examples/eks/hello-world/variables.tf b/argocd/iac/terraform/examples/eks/hello-world/variables.tf deleted file mode 100644 index c1d456eb..00000000 --- a/argocd/iac/terraform/examples/eks/hello-world/variables.tf +++ /dev/null @@ -1,20 +0,0 @@ -variable "gitops_addons_org" { - description = "Git repository org/user contains for addons" - default = "https://github.com/gitops-bridge-dev" -} -variable "gitops_addons_repo" { - description = "Git repository contains for addons" - default = "gitops-bridge-argocd-control-plane-template" -} -variable "gitops_addons_basepath" { - description = "Git repository base path for addons" - default = "" -} -variable "gitops_addons_path" { - description = "Git repository path for addons" - default = "bootstrap/control-plane/addons" -} -variable "gitops_addons_revision" { - description = "Git repository revision/branch/ref for addons" - default = "HEAD" -} diff --git a/argocd/iac/terraform/examples/eks/hello-world/versions.tf b/argocd/iac/terraform/examples/eks/hello-world/versions.tf deleted file mode 100644 index 2de60d58..00000000 --- a/argocd/iac/terraform/examples/eks/hello-world/versions.tf +++ /dev/null @@ -1,25 +0,0 @@ -terraform { - required_version = ">= 1.0" - - required_providers { - aws = { - source = "hashicorp/aws" - version = ">= 4.67.0" - } - helm = { - source = "hashicorp/helm" - version = ">= 2.10.1" - } - kubernetes = { - source = "hashicorp/kubernetes" - version = "2.22.0" - } - } - - # ## Used for end-to-end testing on project; update to suit your needs - # backend "s3" { - # bucket = "terraform-ssp-github-actions-state" - # region = "us-west-2" - # key = "e2e/ipv4-prefix-delegation/terraform.tfstate" - # } -} From d53dbe599d33a5c2cb0272c4b2d96d5d8784a88e Mon Sep 17 00:00:00 2001 From: Carlos Santana Date: Sat, 28 Oct 2023 23:23:53 -0400 Subject: [PATCH 09/16] test workload Signed-off-by: Carlos Santana --- .../examples/eks/external-secrets/README.md | 6 +- .../examples/eks/private-git/README.md | 48 ++++++- .../eks/private-git/bootstrap/workloads.yaml | 2 +- .../examples/eks/private-git/main.tf | 129 ++++++++++-------- .../examples/eks/private-git/variables.tf | 56 ++++++-- 5 files changed, 160 insertions(+), 81 deletions(-) diff --git a/argocd/iac/terraform/examples/eks/external-secrets/README.md b/argocd/iac/terraform/examples/eks/external-secrets/README.md index 49696adc..e92d3bd3 100644 --- a/argocd/iac/terraform/examples/eks/external-secrets/README.md +++ b/argocd/iac/terraform/examples/eks/external-secrets/README.md @@ -42,11 +42,7 @@ Retrieve `kubectl` config, then execute the output command: ```shell terraform output -raw configure_kubectl ``` -## Deploy the Addons -Bootstrap the addons using ArgoCD: -```shell -kubectl apply -f bootstrap/addons.yaml -``` +` ### Monitor GitOps Progress for Addons Wait until all the ArgoCD applications' `HEALTH STATUS` is `Healthy`. Use Crl+C to exit the `watch` command diff --git a/argocd/iac/terraform/examples/eks/private-git/README.md b/argocd/iac/terraform/examples/eks/private-git/README.md index 3974e9ab..c8f6be90 100644 --- a/argocd/iac/terraform/examples/eks/private-git/README.md +++ b/argocd/iac/terraform/examples/eks/private-git/README.md @@ -9,19 +9,55 @@ The example reads your private ssh key, and creates two secretes to access the g ## Prerequisites - Create a Github ssh key file, example assumes the file path `~/.ssh/id_rsa`, update `main.tf` if using a different location -Deploy EKS Cluster +Before you begin, make sure you have the following command line tools installed: +- git +- terraform +- kubectl +- argocd + +### Fork the Addon GitOps Repo +1. Fork the git repository for addons [here](https://github.com/gitops-bridge-dev/gitops-bridge-argocd-control-plane-template). +2. Update the following environment variables to point to your fork by changing the default values: +```shell +export TF_VAR_gitops_addons_org=https://github.com/gitops-bridge-dev +export TF_VAR_gitops_addons_repo=gitops-bridge-argocd-control-plane-template +``` + +## Deploy the EKS Cluster +Initialize Terraform and deploy the EKS cluster: ```shell terraform init -terraform apply +terraform apply -auto-approve +``` +Retrieve `kubectl` config, then execute the output command: +```shell +terraform output -raw configure_kubectl +``` + + +### Monitor GitOps Progress for Addons +Wait until all the ArgoCD applications' `HEALTH STATUS` is `Healthy`. Use Crl+C to exit the `watch` command +```shell +watch kubectl get applications -n argocd ``` -Access Terraform output to configure `kubectl` and `argocd` +### Verify the Addons +Verify that the addons are ready: ```shell -terraform output +kubectl get deployment -n kube-system \ + aws-load-balancer-controller \ + metrics-server ``` -Destroy EKS Cluster +## Access ArgoCD +Access ArgoCD's UI, run the command from the output: +```shell +terraform output -raw access_argocd +``` + + +## Destroy the EKS Cluster +To tear down all the resources and the EKS cluster, run the following command: ```shell -cd hub ./destroy.sh ``` diff --git a/argocd/iac/terraform/examples/eks/private-git/bootstrap/workloads.yaml b/argocd/iac/terraform/examples/eks/private-git/bootstrap/workloads.yaml index 3a9e96e5..1d24c565 100644 --- a/argocd/iac/terraform/examples/eks/private-git/bootstrap/workloads.yaml +++ b/argocd/iac/terraform/examples/eks/private-git/bootstrap/workloads.yaml @@ -20,7 +20,7 @@ spec: project: default source: repoURL: '{{metadata.annotations.workload_repo_url}}' - path: '{{metadata.annotations.workload_repo_path}}' + path: '{{metadata.annotations.workload_repo_basepath}}{{metadata.annotations.workload_repo_path}}' targetRevision: '{{metadata.annotations.workload_repo_revision}}' destination: namespace: 'workload' diff --git a/argocd/iac/terraform/examples/eks/private-git/main.tf b/argocd/iac/terraform/examples/eks/private-git/main.tf index 8871afdf..47773f53 100644 --- a/argocd/iac/terraform/examples/eks/private-git/main.tf +++ b/argocd/iac/terraform/examples/eks/private-git/main.tf @@ -31,12 +31,15 @@ provider "kubernetes" { } locals { - name = "ex-${replace(basename(path.cwd), "_", "-")}" - environment = "dev" - region = "us-west-2" - cluster_version = "1.27" + name = "ex-${replace(basename(path.cwd), "_", "-")}" + region = var.region - git_private_ssh_key = "~/.ssh/id_rsa" # Update with the git ssh key to be used by ArgoCD + cluster_version = var.kubernetes_version + + vpc_cidr = var.vpc_cidr + azs = slice(data.aws_availability_zones.available.names, 0, 3) + + git_private_ssh_key = var.ssh_key_path # Update with the git ssh key to be used by ArgoCD gitops_addons_org = var.gitops_addons_org gitops_addons_url = "${var.gitops_addons_org}/${var.gitops_addons_repo}" @@ -46,47 +49,60 @@ locals { gitops_workload_org = var.gitops_workload_org gitops_workload_repo = var.gitops_workload_repo + gitops_workload_basepath = var.gitops_workload_basepath gitops_workload_path = var.gitops_workload_path gitops_workload_revision = var.gitops_workload_revision gitops_workload_url = "${local.gitops_workload_org}/${local.gitops_workload_repo}" - aws_addons = { - enable_cert_manager = true - #enable_aws_efs_csi_driver = true - #enable_aws_fsx_csi_driver = true - #enable_aws_cloudwatch_metrics = true - #enable_aws_privateca_issuer = true - #enable_cluster_autoscaler = true - #enable_external_dns = true - #enable_external_secrets = true - #enable_aws_load_balancer_controller = true - #enable_fargate_fluentbit = true - #enable_aws_for_fluentbit = true - #enable_aws_node_termination_handler = true - #enable_karpenter = true - #enable_velero = true - #enable_aws_gateway_api_controller = true - #enable_aws_ebs_csi_resources = true # generate gp2 and gp3 storage classes for ebs-csi - #enable_aws_secrets_store_csi_driver_provider = true + enable_cert_manager = try(var.addons.enable_cert_manager, false) + enable_aws_efs_csi_driver = try(var.addons.enable_aws_efs_csi_driver, false) + enable_aws_fsx_csi_driver = try(var.addons.enable_aws_fsx_csi_driver, false) + enable_aws_cloudwatch_metrics = try(var.addons.enable_aws_cloudwatch_metrics, false) + enable_aws_privateca_issuer = try(var.addons.enable_aws_privateca_issuer, false) + enable_cluster_autoscaler = try(var.addons.enable_cluster_autoscaler, false) + enable_external_dns = try(var.addons.enable_external_dns, false) + enable_external_secrets = try(var.addons.enable_external_secrets, false) + enable_aws_load_balancer_controller = try(var.addons.enable_aws_load_balancer_controller, false) + enable_fargate_fluentbit = try(var.addons.enable_fargate_fluentbit, false) + enable_aws_for_fluentbit = try(var.addons.enable_aws_for_fluentbit, false) + enable_aws_node_termination_handler = try(var.addons.enable_aws_node_termination_handler, false) + enable_karpenter = try(var.addons.enable_karpenter, false) + enable_velero = try(var.addons.enable_velero, false) + enable_aws_gateway_api_controller = try(var.addons.enable_aws_gateway_api_controller, false) + enable_aws_ebs_csi_resources = try(var.addons.enable_aws_ebs_csi_resources, false) + enable_aws_secrets_store_csi_driver_provider = try(var.addons.enable_aws_secrets_store_csi_driver_provider, false) + enable_ack_apigatewayv2 = try(var.addons.enable_ack_apigatewayv2, false) + enable_ack_dynamodb = try(var.addons.enable_ack_dynamodb, false) + enable_ack_s3 = try(var.addons.enable_ack_s3, false) + enable_ack_rds = try(var.addons.enable_ack_rds, false) + enable_ack_prometheusservice = try(var.addons.enable_ack_prometheusservice, false) + enable_ack_emrcontainers = try(var.addons.enable_ack_emrcontainers, false) + enable_ack_sfn = try(var.addons.enable_ack_sfn, false) + enable_ack_eventbridge = try(var.addons.enable_ack_eventbridge, false) } oss_addons = { - #enable_argo_rollouts = true - #enable_argo_events = true - #enable_argo_workflows = true - #enable_cluster_proportional_autoscaler = true - #enable_gatekeeper = true - #enable_gpu_operator = true - #enable_ingress_nginx = true - #enable_kyverno = true - #enable_kube_prometheus_stack = true - enable_metrics_server = true - #enable_prometheus_adapter = true - #enable_secrets_store_csi_driver = true - #enable_vpa = true - #enable_foo = true # you can add any addon here, make sure to update the gitops repo with the corresponding application set + enable_argocd = try(var.addons.enable_argocd, true) + enable_argo_rollouts = try(var.addons.enable_argo_rollouts, false) + enable_argo_events = try(var.addons.enable_argo_events, false) + enable_argo_workflows = try(var.addons.enable_argo_workflows, false) + enable_cluster_proportional_autoscaler = try(var.addons.enable_cluster_proportional_autoscaler, false) + enable_gatekeeper = try(var.addons.enable_gatekeeper, false) + enable_gpu_operator = try(var.addons.enable_gpu_operator, false) + enable_ingress_nginx = try(var.addons.enable_ingress_nginx, false) + enable_kyverno = try(var.addons.enable_kyverno, false) + enable_kube_prometheus_stack = try(var.addons.enable_kube_prometheus_stack, false) + enable_metrics_server = try(var.addons.enable_metrics_server, false) + enable_prometheus_adapter = try(var.addons.enable_prometheus_adapter, false) + enable_secrets_store_csi_driver = try(var.addons.enable_secrets_store_csi_driver, false) + enable_vpa = try(var.addons.enable_vpa, false) } - addons = merge(local.aws_addons, local.oss_addons, { kubernetes_version = local.cluster_version }, { aws_cluster_name = module.eks.cluster_name }) + addons = merge( + local.aws_addons, + local.oss_addons, + { kubernetes_version = local.cluster_version }, + { aws_cluster_name = module.eks.cluster_name } + ) addons_metadata = merge( module.eks_blueprints_addons.gitops_metadata, @@ -104,6 +120,7 @@ locals { }, { workload_repo_url = local.gitops_workload_url + workload_repo_basepath = local.gitops_workload_basepath workload_repo_path = local.gitops_workload_path workload_repo_revision = local.gitops_workload_revision } @@ -114,12 +131,9 @@ locals { workloads = file("${path.module}/bootstrap/workloads.yaml") } - vpc_cidr = "10.0.0.0/16" - azs = slice(data.aws_availability_zones.available.names, 0, 3) - tags = { Blueprint = local.name - GithubRepo = "github.com/csantanapr/terraform-gitops-bridge" + GithubRepo = "github.com/gitops-bridge-dev/gitops-bridge" } } @@ -164,7 +178,6 @@ module "gitops_bridge_bootstrap" { cluster = { cluster_name = module.eks.cluster_name - environment = local.environment metadata = local.addons_metadata addons = local.addons } @@ -190,21 +203,21 @@ module "eks_blueprints_addons" { create_kubernetes_resources = false # EKS Blueprints Addons - enable_cert_manager = try(local.aws_addons.enable_cert_manager, false) - enable_aws_efs_csi_driver = try(local.aws_addons.enable_aws_efs_csi_driver, false) - enable_aws_fsx_csi_driver = try(local.aws_addons.enable_aws_fsx_csi_driver, false) - enable_aws_cloudwatch_metrics = try(local.aws_addons.enable_aws_cloudwatch_metrics, false) - enable_aws_privateca_issuer = try(local.aws_addons.enable_aws_privateca_issuer, false) - enable_cluster_autoscaler = try(local.aws_addons.enable_cluster_autoscaler, false) - enable_external_dns = try(local.aws_addons.enable_external_dns, false) - enable_external_secrets = try(local.aws_addons.enable_external_secrets, false) - enable_aws_load_balancer_controller = try(local.aws_addons.enable_aws_load_balancer_controller, false) - enable_fargate_fluentbit = try(local.aws_addons.enable_fargate_fluentbit, false) - enable_aws_for_fluentbit = try(local.aws_addons.enable_aws_for_fluentbit, false) - enable_aws_node_termination_handler = try(local.aws_addons.enable_aws_node_termination_handler, false) - enable_karpenter = try(local.aws_addons.enable_karpenter, false) - enable_velero = try(local.aws_addons.enable_velero, false) - enable_aws_gateway_api_controller = try(local.aws_addons.enable_aws_gateway_api_controller, false) + enable_cert_manager = local.aws_addons.enable_cert_manager + enable_aws_efs_csi_driver = local.aws_addons.enable_aws_efs_csi_driver + enable_aws_fsx_csi_driver = local.aws_addons.enable_aws_fsx_csi_driver + enable_aws_cloudwatch_metrics = local.aws_addons.enable_aws_cloudwatch_metrics + enable_aws_privateca_issuer = local.aws_addons.enable_aws_privateca_issuer + enable_cluster_autoscaler = local.aws_addons.enable_cluster_autoscaler + enable_external_dns = local.aws_addons.enable_external_dns + enable_external_secrets = local.aws_addons.enable_external_secrets + enable_aws_load_balancer_controller = local.aws_addons.enable_aws_load_balancer_controller + enable_fargate_fluentbit = local.aws_addons.enable_fargate_fluentbit + enable_aws_for_fluentbit = local.aws_addons.enable_aws_for_fluentbit + enable_aws_node_termination_handler = local.aws_addons.enable_aws_node_termination_handler + enable_karpenter = local.aws_addons.enable_karpenter + enable_velero = local.aws_addons.enable_velero + enable_aws_gateway_api_controller = local.aws_addons.enable_aws_gateway_api_controller tags = local.tags } diff --git a/argocd/iac/terraform/examples/eks/private-git/variables.tf b/argocd/iac/terraform/examples/eks/private-git/variables.tf index 4b0cfdaa..06c36689 100644 --- a/argocd/iac/terraform/examples/eks/private-git/variables.tf +++ b/argocd/iac/terraform/examples/eks/private-git/variables.tf @@ -1,3 +1,32 @@ +variable "ssh_key_path" { + description = "SSH key path for git access" + type = string + default = "~/.ssh/id_rsa" +} +variable "vpc_cidr" { + description = "VPC CIDR" + type = string + default = "10.0.0.0/16" +} +variable "region" { + description = "AWS region" + type = string + default = "us-west-2" +} +variable "kubernetes_version" { + description = "Kubernetes version" + type = string + default = "1.28" +} +variable "addons" { + description = "Kubernetes addons" + type = any + default = { + enable_aws_load_balancer_controller = true + enable_metrics_server = true + } +} +# Addons Git variable "gitops_addons_org" { description = "Git repository org/user contains for addons" type = string @@ -8,6 +37,11 @@ variable "gitops_addons_repo" { type = string default = "gitops-bridge-argocd-control-plane-template" } +variable "gitops_addons_revision" { + description = "Git repository revision/branch/ref for addons" + type = string + default = "main" +} variable "gitops_addons_basepath" { description = "Git repository base path for addons" type = string @@ -18,12 +52,7 @@ variable "gitops_addons_path" { type = string default = "bootstrap/control-plane/addons" } -variable "gitops_addons_revision" { - description = "Git repository revision/branch/ref for addons" - type = string - default = "HEAD" -} - +# Workloads Git variable "gitops_workload_org" { description = "Git repository org/user contains for workload" type = string @@ -34,13 +63,18 @@ variable "gitops_workload_repo" { type = string default = "argocd-example-apps" } +variable "gitops_workload_revision" { + description = "Git repository revision/branch/ref for workload" + type = string + default = "master" +} +variable "gitops_workload_basepath" { + description = "Git repository base path for workload" + type = string + default = "" +} variable "gitops_workload_path" { description = "Git repository path for workload" type = string default = "helm-guestbook" } -variable "gitops_workload_revision" { - description = "Git repository revision/branch/ref for workload" - type = string - default = "HEAD" -} From a29a7398bef679bfba479c1959fe71bee5bbdc77 Mon Sep 17 00:00:00 2001 From: Carlos Santana Date: Sat, 28 Oct 2023 23:42:29 -0400 Subject: [PATCH 10/16] fix output Signed-off-by: Carlos Santana --- argocd/iac/terraform/examples/eks/private-git/outputs.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/argocd/iac/terraform/examples/eks/private-git/outputs.tf b/argocd/iac/terraform/examples/eks/private-git/outputs.tf index 2d3b6e57..d4ecfbf1 100644 --- a/argocd/iac/terraform/examples/eks/private-git/outputs.tf +++ b/argocd/iac/terraform/examples/eks/private-git/outputs.tf @@ -26,8 +26,8 @@ output "access_argocd" { value = <<-EOT export KUBECONFIG="/tmp/${module.eks.cluster_name}" aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name} - echo "ArgoCD URL: https://$(kubectl get svc -n argocd argo-cd-argocd-server -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')" echo "ArgoCD Username: admin" echo "ArgoCD Password: $(kubectl get secrets argocd-initial-admin-secret -n argocd --template="{{index .data.password | base64decode}}")" + echo "ArgoCD URL: https://$(kubectl get svc -n argocd argo-cd-argocd-server -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')" EOT } From df50dae82c068f80f7d2f3288ff5e3a632f19f2f Mon Sep 17 00:00:00 2001 From: Carlos Santana Date: Sun, 29 Oct 2023 08:41:01 -0400 Subject: [PATCH 11/16] update cluster-addons name Signed-off-by: Carlos Santana --- .../terraform/examples/eks/argocd-ingress/bootstrap/addons.yaml | 2 +- .../examples/eks/aws-secrets-manager/bootstrap/addons.yaml | 2 +- .../iac/terraform/examples/eks/complete/bootstrap/addons.yaml | 2 +- .../iac/terraform/examples/eks/crossplane/bootstrap/addons.yaml | 2 +- .../examples/eks/external-secrets/bootstrap/addons.yaml | 2 +- .../eks/multi-cluster/distributed/bootstrap/addons.yaml | 2 +- .../multi-cluster/hub-spoke-shared/hub/bootstrap/addons.yaml | 2 +- .../eks/multi-cluster/hub-spoke/hub/bootstrap/addons.yaml | 2 +- .../terraform/examples/eks/private-git/bootstrap/addons.yaml | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/argocd/iac/terraform/examples/eks/argocd-ingress/bootstrap/addons.yaml b/argocd/iac/terraform/examples/eks/argocd-ingress/bootstrap/addons.yaml index 02535923..edf1f795 100644 --- a/argocd/iac/terraform/examples/eks/argocd-ingress/bootstrap/addons.yaml +++ b/argocd/iac/terraform/examples/eks/argocd-ingress/bootstrap/addons.yaml @@ -1,7 +1,7 @@ apiVersion: argoproj.io/v1alpha1 kind: ApplicationSet metadata: - name: bootstrap-addons + name: cluster-addons namespace: argocd spec: syncPolicy: diff --git a/argocd/iac/terraform/examples/eks/aws-secrets-manager/bootstrap/addons.yaml b/argocd/iac/terraform/examples/eks/aws-secrets-manager/bootstrap/addons.yaml index 02535923..edf1f795 100644 --- a/argocd/iac/terraform/examples/eks/aws-secrets-manager/bootstrap/addons.yaml +++ b/argocd/iac/terraform/examples/eks/aws-secrets-manager/bootstrap/addons.yaml @@ -1,7 +1,7 @@ apiVersion: argoproj.io/v1alpha1 kind: ApplicationSet metadata: - name: bootstrap-addons + name: cluster-addons namespace: argocd spec: syncPolicy: diff --git a/argocd/iac/terraform/examples/eks/complete/bootstrap/addons.yaml b/argocd/iac/terraform/examples/eks/complete/bootstrap/addons.yaml index 02535923..edf1f795 100644 --- a/argocd/iac/terraform/examples/eks/complete/bootstrap/addons.yaml +++ b/argocd/iac/terraform/examples/eks/complete/bootstrap/addons.yaml @@ -1,7 +1,7 @@ apiVersion: argoproj.io/v1alpha1 kind: ApplicationSet metadata: - name: bootstrap-addons + name: cluster-addons namespace: argocd spec: syncPolicy: diff --git a/argocd/iac/terraform/examples/eks/crossplane/bootstrap/addons.yaml b/argocd/iac/terraform/examples/eks/crossplane/bootstrap/addons.yaml index 02535923..edf1f795 100644 --- a/argocd/iac/terraform/examples/eks/crossplane/bootstrap/addons.yaml +++ b/argocd/iac/terraform/examples/eks/crossplane/bootstrap/addons.yaml @@ -1,7 +1,7 @@ apiVersion: argoproj.io/v1alpha1 kind: ApplicationSet metadata: - name: bootstrap-addons + name: cluster-addons namespace: argocd spec: syncPolicy: diff --git a/argocd/iac/terraform/examples/eks/external-secrets/bootstrap/addons.yaml b/argocd/iac/terraform/examples/eks/external-secrets/bootstrap/addons.yaml index 02535923..edf1f795 100644 --- a/argocd/iac/terraform/examples/eks/external-secrets/bootstrap/addons.yaml +++ b/argocd/iac/terraform/examples/eks/external-secrets/bootstrap/addons.yaml @@ -1,7 +1,7 @@ apiVersion: argoproj.io/v1alpha1 kind: ApplicationSet metadata: - name: bootstrap-addons + name: cluster-addons namespace: argocd spec: syncPolicy: diff --git a/argocd/iac/terraform/examples/eks/multi-cluster/distributed/bootstrap/addons.yaml b/argocd/iac/terraform/examples/eks/multi-cluster/distributed/bootstrap/addons.yaml index 02535923..edf1f795 100644 --- a/argocd/iac/terraform/examples/eks/multi-cluster/distributed/bootstrap/addons.yaml +++ b/argocd/iac/terraform/examples/eks/multi-cluster/distributed/bootstrap/addons.yaml @@ -1,7 +1,7 @@ apiVersion: argoproj.io/v1alpha1 kind: ApplicationSet metadata: - name: bootstrap-addons + name: cluster-addons namespace: argocd spec: syncPolicy: diff --git a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/hub/bootstrap/addons.yaml b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/hub/bootstrap/addons.yaml index 02535923..edf1f795 100644 --- a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/hub/bootstrap/addons.yaml +++ b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/hub/bootstrap/addons.yaml @@ -1,7 +1,7 @@ apiVersion: argoproj.io/v1alpha1 kind: ApplicationSet metadata: - name: bootstrap-addons + name: cluster-addons namespace: argocd spec: syncPolicy: diff --git a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/hub/bootstrap/addons.yaml b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/hub/bootstrap/addons.yaml index 02535923..edf1f795 100644 --- a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/hub/bootstrap/addons.yaml +++ b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/hub/bootstrap/addons.yaml @@ -1,7 +1,7 @@ apiVersion: argoproj.io/v1alpha1 kind: ApplicationSet metadata: - name: bootstrap-addons + name: cluster-addons namespace: argocd spec: syncPolicy: diff --git a/argocd/iac/terraform/examples/eks/private-git/bootstrap/addons.yaml b/argocd/iac/terraform/examples/eks/private-git/bootstrap/addons.yaml index 02535923..edf1f795 100644 --- a/argocd/iac/terraform/examples/eks/private-git/bootstrap/addons.yaml +++ b/argocd/iac/terraform/examples/eks/private-git/bootstrap/addons.yaml @@ -1,7 +1,7 @@ apiVersion: argoproj.io/v1alpha1 kind: ApplicationSet metadata: - name: bootstrap-addons + name: cluster-addons namespace: argocd spec: syncPolicy: From 5221dea246c66398bfba7a45d2f26b9f10837294 Mon Sep 17 00:00:00 2001 From: Carlos Santana Date: Sun, 29 Oct 2023 09:47:44 -0400 Subject: [PATCH 12/16] update distributed Signed-off-by: Carlos Santana --- .../eks/multi-cluster/distributed/README.md | 65 ++++++++- .../distributed/bootstrap/workloads.yaml | 4 +- .../eks/multi-cluster/distributed/destroy.sh | 34 +++-- .../eks/multi-cluster/distributed/main.tf | 127 ++++++++++-------- .../eks/multi-cluster/distributed/outputs.tf | 3 +- .../multi-cluster/distributed/variables.tf | 57 +++++--- 6 files changed, 196 insertions(+), 94 deletions(-) diff --git a/argocd/iac/terraform/examples/eks/multi-cluster/distributed/README.md b/argocd/iac/terraform/examples/eks/multi-cluster/distributed/README.md index 557ef643..de349158 100644 --- a/argocd/iac/terraform/examples/eks/multi-cluster/distributed/README.md +++ b/argocd/iac/terraform/examples/eks/multi-cluster/distributed/README.md @@ -3,21 +3,80 @@ This example deploys argocd on each cluster, each ArgoCD instance points to the same git repository for cluster addons. Each cluster gets deployed an app of apps ArgoCD Application with the name `workloads-${env}` -To deploy run the following commands +## Prerequisites +Before you begin, make sure you have the following command line tools installed: +- git +- terraform +- kubectl +- argocd + +## Fork the Git Repositories + +### Fork the Addon GitOps Repo +1. Fork the git repository for addons [here](https://github.com/gitops-bridge-dev/gitops-bridge-argocd-control-plane-template). +2. Update the following environment variables to point to your fork by changing the default values: +```shell +export TF_VAR_gitops_addons_org=https://github.com/gitops-bridge-dev +export TF_VAR_gitops_addons_repo=gitops-bridge-argocd-control-plane-template +``` + +## Deploy the EKS Cluster +Initialize Terraform and deploy the EKS cluster: ```shell ./deploy.sh dev ./deploy.sh staging ./deploy.sh prod ``` + Each environment uses a Terraform workspace To access Terraform output run the following commands for the particular environment ```shell -terraform workspace select ${env} +terraform workspace select dev +terraform output +``` +```shell +terraform workspace select staging terraform output ``` +```shell +terraform workspace select prod +terraform output +``` + +Retrieve `kubectl` config, then execute the output command: +```shell +terraform output -raw configure_kubectl +``` + +### Monitor GitOps Progress for Addons +Wait until all the ArgoCD applications' `HEALTH STATUS` is `Healthy`. Use Crl+C to exit the `watch` command +```shell +watch kubectl get applications -n argocd +``` + +### Verify the Addons +Verify that the addons are ready: +```shell +kubectl get deployment -n kube-system \ + metrics-server +``` + +## Access ArgoCD +Access ArgoCD's UI, run the command from the output: +```shell +terraform output -raw access_argocd +``` + +### Monitor GitOps Progress for Workloads +Watch until the Workloads ArgoCD Application is `Healthy` +```shell +watch kubectl get -n argocd applications workloads-dev +``` +Wait until the ArgoCD Applications `HEALTH STATUS` is `Healthy`. Crl+C to exit the `watch` command -To destroy run the following commands +## Destroy the EKS Clusters +To tear down all the resources and the EKS cluster, run the following command: ```shell ./destroy.sh dev ./destroy.sh staging diff --git a/argocd/iac/terraform/examples/eks/multi-cluster/distributed/bootstrap/workloads.yaml b/argocd/iac/terraform/examples/eks/multi-cluster/distributed/bootstrap/workloads.yaml index dadf64d8..31b798c9 100644 --- a/argocd/iac/terraform/examples/eks/multi-cluster/distributed/bootstrap/workloads.yaml +++ b/argocd/iac/terraform/examples/eks/multi-cluster/distributed/bootstrap/workloads.yaml @@ -5,7 +5,7 @@ metadata: namespace: argocd spec: syncPolicy: - preserveResourcesOnDeletion: true + preserveResourcesOnDeletion: false generators: - clusters: selector: @@ -20,7 +20,7 @@ spec: project: default source: repoURL: '{{metadata.annotations.workload_repo_url}}' - path: '{{metadata.annotations.workload_repo_path}}' + path: '{{metadata.annotations.workload_repo_basepath}}{{metadata.annotations.workload_repo_path}}' targetRevision: '{{metadata.annotations.workload_repo_revision}}' destination: namespace: 'workload' diff --git a/argocd/iac/terraform/examples/eks/multi-cluster/distributed/destroy.sh b/argocd/iac/terraform/examples/eks/multi-cluster/distributed/destroy.sh index 3b50252f..4f8d8fb8 100755 --- a/argocd/iac/terraform/examples/eks/multi-cluster/distributed/destroy.sh +++ b/argocd/iac/terraform/examples/eks/multi-cluster/distributed/destroy.sh @@ -1,5 +1,11 @@ #!/bin/bash +set -uo pipefail + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +ROOTDIR="$(cd ${SCRIPTDIR}/../..; pwd )" +[[ -n "${DEBUG:-}" ]] && set -x + if [[ $# -eq 0 ]] ; then echo "No arguments supplied" @@ -9,20 +15,24 @@ if [[ $# -eq 0 ]] ; then fi env=$1 echo "Destroying $env ..." - -set -x - terraform workspace select $env # Delete the Ingress/SVC before removing the addons TMPFILE=$(mktemp) -terraform output -raw configure_kubectl > "$TMPFILE" -source "$TMPFILE" - -kubectl delete svc -n argocd argo-cd-argocd-server +terraform -chdir=$SCRIPTDIR output -raw configure_kubectl > "$TMPFILE" +# check if TMPFILE contains the string "No outputs found" +if [[ ! $(cat $TMPFILE) == *"No outputs found"* ]]; then + source "$TMPFILE" + kubectl delete -n argocd applicationset workloads + echo "Waiting for ingress and load balancer to be deleted" + sleep 120 + kubectl delete -n argocd applicationset cluster-addons + kubectl delete -n argocd applicationset addons-argocd + kubectl delete -n argocd svc argo-cd-argocd-server +fi -terraform destroy -target="module.gitops_bridge_bootstrap" -auto-approve -var-file="workspaces/${env}.tfvars" -terraform destroy -target="module.eks_blueprints_addons" -auto-approve -var-file="workspaces/${env}.tfvars" -terraform destroy -target="module.eks" -auto-approve -var-file="workspaces/${env}.tfvars" -terraform destroy -target="module.vpc" -auto-approve -var-file="workspaces/${env}.tfvars" -terraform destroy -auto-approve -var-file="workspaces/${env}.tfvars" +terraform destroy -target="module.gitops_bridge_bootstrap" -auto-approve +terraform destroy -target="module.eks_blueprints_addons" -auto-approve +terraform destroy -target="module.eks" -auto-approve +terraform destroy -target="module.vpc" -auto-approve +terraform destroy -auto-approve diff --git a/argocd/iac/terraform/examples/eks/multi-cluster/distributed/main.tf b/argocd/iac/terraform/examples/eks/multi-cluster/distributed/main.tf index 3c4a4a8b..bea0323e 100644 --- a/argocd/iac/terraform/examples/eks/multi-cluster/distributed/main.tf +++ b/argocd/iac/terraform/examples/eks/multi-cluster/distributed/main.tf @@ -31,11 +31,15 @@ provider "kubernetes" { } locals { - name = "multi-cluster-${terraform.workspace}" - environment = terraform.workspace - region = "us-west-2" - cluster_version = var.kubernetes_version - vpc_cidr = var.vpc_cidr + name = "multi-cluster-${terraform.workspace}" + environment = terraform.workspace + region = var.region + + cluster_version = var.kubernetes_version + + vpc_cidr = var.vpc_cidr + azs = slice(data.aws_availability_zones.available.names, 0, 3) + gitops_addons_url = "${var.gitops_addons_org}/${var.gitops_addons_repo}" gitops_addons_basepath = var.gitops_addons_basepath gitops_addons_path = var.gitops_addons_path @@ -43,46 +47,60 @@ locals { gitops_workload_org = var.gitops_workload_org gitops_workload_repo = var.gitops_workload_repo + gitops_workload_basepath = var.gitops_workload_basepath gitops_workload_path = var.gitops_workload_path gitops_workload_revision = var.gitops_workload_revision gitops_workload_url = "${local.gitops_workload_org}/${local.gitops_workload_repo}" aws_addons = { - enable_cert_manager = true - #enable_aws_efs_csi_driver = true - #enable_aws_fsx_csi_driver = true - #enable_aws_cloudwatch_metrics = true - #enable_aws_privateca_issuer = true - #enable_cluster_autoscaler = true - #enable_external_dns = true - #enable_external_secrets = true - #enable_aws_load_balancer_controller = true - #enable_fargate_fluentbit = true - #enable_aws_for_fluentbit = true - #enable_aws_node_termination_handler = true - #enable_karpenter = true - #enable_velero = true - #enable_aws_gateway_api_controller = true - #enable_aws_ebs_csi_resources = true # generate gp2 and gp3 storage classes for ebs-csi - #enable_aws_secrets_store_csi_driver_provider = true + enable_cert_manager = try(var.addons.enable_cert_manager, false) + enable_aws_efs_csi_driver = try(var.addons.enable_aws_efs_csi_driver, false) + enable_aws_fsx_csi_driver = try(var.addons.enable_aws_fsx_csi_driver, false) + enable_aws_cloudwatch_metrics = try(var.addons.enable_aws_cloudwatch_metrics, false) + enable_aws_privateca_issuer = try(var.addons.enable_aws_privateca_issuer, false) + enable_cluster_autoscaler = try(var.addons.enable_cluster_autoscaler, false) + enable_external_dns = try(var.addons.enable_external_dns, false) + enable_external_secrets = try(var.addons.enable_external_secrets, false) + enable_aws_load_balancer_controller = try(var.addons.enable_aws_load_balancer_controller, false) + enable_fargate_fluentbit = try(var.addons.enable_fargate_fluentbit, false) + enable_aws_for_fluentbit = try(var.addons.enable_aws_for_fluentbit, false) + enable_aws_node_termination_handler = try(var.addons.enable_aws_node_termination_handler, false) + enable_karpenter = try(var.addons.enable_karpenter, false) + enable_velero = try(var.addons.enable_velero, false) + enable_aws_gateway_api_controller = try(var.addons.enable_aws_gateway_api_controller, false) + enable_aws_ebs_csi_resources = try(var.addons.enable_aws_ebs_csi_resources, false) + enable_aws_secrets_store_csi_driver_provider = try(var.addons.enable_aws_secrets_store_csi_driver_provider, false) + enable_ack_apigatewayv2 = try(var.addons.enable_ack_apigatewayv2, false) + enable_ack_dynamodb = try(var.addons.enable_ack_dynamodb, false) + enable_ack_s3 = try(var.addons.enable_ack_s3, false) + enable_ack_rds = try(var.addons.enable_ack_rds, false) + enable_ack_prometheusservice = try(var.addons.enable_ack_prometheusservice, false) + enable_ack_emrcontainers = try(var.addons.enable_ack_emrcontainers, false) + enable_ack_sfn = try(var.addons.enable_ack_sfn, false) + enable_ack_eventbridge = try(var.addons.enable_ack_eventbridge, false) } oss_addons = { - #enable_argo_rollouts = true - #enable_argo_events = true - #enable_argo_workflows = true - #enable_cluster_proportional_autoscaler = true - #enable_gatekeeper = true - #enable_gpu_operator = true - #enable_ingress_nginx = true - #enable_kyverno = true - #enable_kube_prometheus_stack = true - enable_metrics_server = true - #enable_prometheus_adapter = true - #enable_secrets_store_csi_driver = true - #enable_vpa = true - #enable_foo = true # you can add any addon here, make sure to update the gitops repo with the corresponding application set + enable_argocd = try(var.addons.enable_argocd, false) + enable_argo_rollouts = try(var.addons.enable_argo_rollouts, false) + enable_argo_events = try(var.addons.enable_argo_events, false) + enable_argo_workflows = try(var.addons.enable_argo_workflows, false) + enable_cluster_proportional_autoscaler = try(var.addons.enable_cluster_proportional_autoscaler, false) + enable_gatekeeper = try(var.addons.enable_gatekeeper, false) + enable_gpu_operator = try(var.addons.enable_gpu_operator, false) + enable_ingress_nginx = try(var.addons.enable_ingress_nginx, false) + enable_kyverno = try(var.addons.enable_kyverno, false) + enable_kube_prometheus_stack = try(var.addons.enable_kube_prometheus_stack, false) + enable_metrics_server = try(var.addons.enable_metrics_server, false) + enable_prometheus_adapter = try(var.addons.enable_prometheus_adapter, false) + enable_secrets_store_csi_driver = try(var.addons.enable_secrets_store_csi_driver, false) + enable_vpa = try(var.addons.enable_vpa, false) } - addons = merge(local.aws_addons, local.oss_addons, { kubernetes_version = local.cluster_version }, { aws_cluster_name = module.eks.cluster_name }) + addons = merge( + local.aws_addons, + local.oss_addons, + { kubernetes_version = local.cluster_version }, + { aws_cluster_name = module.eks.cluster_name } + ) addons_metadata = merge( module.eks_blueprints_addons.gitops_metadata, @@ -100,6 +118,7 @@ locals { }, { workload_repo_url = local.gitops_workload_url + workload_repo_basepath = local.gitops_workload_basepath workload_repo_path = local.gitops_workload_path workload_repo_revision = local.gitops_workload_revision } @@ -110,11 +129,9 @@ locals { workloads = file("${path.module}/bootstrap/workloads.yaml") } - azs = slice(data.aws_availability_zones.available.names, 0, 3) - tags = { Blueprint = local.name - GithubRepo = "github.com/csantanapr/terraform-gitops-bridge" + GithubRepo = "github.com/gitops-bridge-dev/gitops-bridge" } } @@ -149,21 +166,21 @@ module "eks_blueprints_addons" { create_kubernetes_resources = false # EKS Blueprints Addons - enable_cert_manager = try(local.aws_addons.enable_cert_manager, false) - enable_aws_efs_csi_driver = try(local.aws_addons.enable_aws_efs_csi_driver, false) - enable_aws_fsx_csi_driver = try(local.aws_addons.enable_aws_fsx_csi_driver, false) - enable_aws_cloudwatch_metrics = try(local.aws_addons.enable_aws_cloudwatch_metrics, false) - enable_aws_privateca_issuer = try(local.aws_addons.enable_aws_privateca_issuer, false) - enable_cluster_autoscaler = try(local.aws_addons.enable_cluster_autoscaler, false) - enable_external_dns = try(local.aws_addons.enable_external_dns, false) - enable_external_secrets = try(local.aws_addons.enable_external_secrets, false) - enable_aws_load_balancer_controller = try(local.aws_addons.enable_aws_load_balancer_controller, false) - enable_fargate_fluentbit = try(local.aws_addons.enable_fargate_fluentbit, false) - enable_aws_for_fluentbit = try(local.aws_addons.enable_aws_for_fluentbit, false) - enable_aws_node_termination_handler = try(local.aws_addons.enable_aws_node_termination_handler, false) - enable_karpenter = try(local.aws_addons.enable_karpenter, false) - enable_velero = try(local.aws_addons.enable_velero, false) - enable_aws_gateway_api_controller = try(local.aws_addons.enable_aws_gateway_api_controller, false) + enable_cert_manager = local.aws_addons.enable_cert_manager + enable_aws_efs_csi_driver = local.aws_addons.enable_aws_efs_csi_driver + enable_aws_fsx_csi_driver = local.aws_addons.enable_aws_fsx_csi_driver + enable_aws_cloudwatch_metrics = local.aws_addons.enable_aws_cloudwatch_metrics + enable_aws_privateca_issuer = local.aws_addons.enable_aws_privateca_issuer + enable_cluster_autoscaler = local.aws_addons.enable_cluster_autoscaler + enable_external_dns = local.aws_addons.enable_external_dns + enable_external_secrets = local.aws_addons.enable_external_secrets + enable_aws_load_balancer_controller = local.aws_addons.enable_aws_load_balancer_controller + enable_fargate_fluentbit = local.aws_addons.enable_fargate_fluentbit + enable_aws_for_fluentbit = local.aws_addons.enable_aws_for_fluentbit + enable_aws_node_termination_handler = local.aws_addons.enable_aws_node_termination_handler + enable_karpenter = local.aws_addons.enable_karpenter + enable_velero = local.aws_addons.enable_velero + enable_aws_gateway_api_controller = local.aws_addons.enable_aws_gateway_api_controller tags = local.tags } diff --git a/argocd/iac/terraform/examples/eks/multi-cluster/distributed/outputs.tf b/argocd/iac/terraform/examples/eks/multi-cluster/distributed/outputs.tf index a9f667cb..d4ecfbf1 100644 --- a/argocd/iac/terraform/examples/eks/multi-cluster/distributed/outputs.tf +++ b/argocd/iac/terraform/examples/eks/multi-cluster/distributed/outputs.tf @@ -26,9 +26,8 @@ output "access_argocd" { value = <<-EOT export KUBECONFIG="/tmp/${module.eks.cluster_name}" aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name} - echo "ArgoCD URL: https://$(kubectl get svc -n argocd argo-cd-argocd-server -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')" echo "ArgoCD Username: admin" echo "ArgoCD Password: $(kubectl get secrets argocd-initial-admin-secret -n argocd --template="{{index .data.password | base64decode}}")" + echo "ArgoCD URL: https://$(kubectl get svc -n argocd argo-cd-argocd-server -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')" EOT } - diff --git a/argocd/iac/terraform/examples/eks/multi-cluster/distributed/variables.tf b/argocd/iac/terraform/examples/eks/multi-cluster/distributed/variables.tf index 6167e9b9..0cd492ee 100644 --- a/argocd/iac/terraform/examples/eks/multi-cluster/distributed/variables.tf +++ b/argocd/iac/terraform/examples/eks/multi-cluster/distributed/variables.tf @@ -1,3 +1,27 @@ +variable "vpc_cidr" { + description = "VPC CIDR" + type = string + default = "10.0.0.0/16" +} +variable "region" { + description = "AWS region" + type = string + default = "us-west-2" +} +variable "kubernetes_version" { + description = "Kubernetes version" + type = string + default = "1.28" +} +variable "addons" { + description = "Kubernetes addons" + type = any + default = { + enable_argocd = true + enable_metrics_server = true + } +} +# Addons Git variable "gitops_addons_org" { description = "Git repository org/user contains for addons" type = string @@ -8,6 +32,11 @@ variable "gitops_addons_repo" { type = string default = "gitops-bridge-argocd-control-plane-template" } +variable "gitops_addons_revision" { + description = "Git repository revision/branch/ref for addons" + type = string + default = "main" +} variable "gitops_addons_basepath" { description = "Git repository base path for addons" type = string @@ -18,12 +47,8 @@ variable "gitops_addons_path" { type = string default = "bootstrap/control-plane/addons" } -variable "gitops_addons_revision" { - description = "Git repository revision/branch/ref for addons" - type = string - default = "HEAD" -} +# Workloads Git variable "gitops_workload_org" { description = "Git repository org/user contains for workload" type = string @@ -34,26 +59,18 @@ variable "gitops_workload_repo" { type = string default = "argocd-example-apps" } -variable "gitops_workload_path" { - description = "Git repository path for workload" - type = string - default = "helm-guestbook" -} variable "gitops_workload_revision" { description = "Git repository revision/branch/ref for workload" type = string - default = "HEAD" + default = "master" } - - - - - -variable "vpc_cidr" { - description = "VPC CIDR" +variable "gitops_workload_basepath" { + description = "Git repository base path for workload" type = string + default = "" } -variable "kubernetes_version" { - description = "EKS version" +variable "gitops_workload_path" { + description = "Git repository path for workload" type = string + default = "helm-guestbook" } From 30b3d211383bc31258eb7cbd106208ba0a934824 Mon Sep 17 00:00:00 2001 From: Carlos Santana Date: Sun, 29 Oct 2023 11:00:45 -0400 Subject: [PATCH 13/16] multi-cluster-distrubuted Signed-off-by: Carlos Santana --- .../eks/multi-cluster/distributed/README.md | 14 +++++++++++++- .../distributed/bootstrap/workloads.yaml | 2 +- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/argocd/iac/terraform/examples/eks/multi-cluster/distributed/README.md b/argocd/iac/terraform/examples/eks/multi-cluster/distributed/README.md index de349158..68d7a64a 100644 --- a/argocd/iac/terraform/examples/eks/multi-cluster/distributed/README.md +++ b/argocd/iac/terraform/examples/eks/multi-cluster/distributed/README.md @@ -71,10 +71,22 @@ terraform output -raw access_argocd ### Monitor GitOps Progress for Workloads Watch until the Workloads ArgoCD Application is `Healthy` ```shell -watch kubectl get -n argocd applications workloads-dev +watch kubectl get -n argocd applications workload ``` Wait until the ArgoCD Applications `HEALTH STATUS` is `Healthy`. Crl+C to exit the `watch` command +### Verify the Application +Verify that the application configuration is present and the pod is running: +```shell +kubectl get all -n workload +``` + +### Container Metrics +Check the application's CPU and memory metrics: +```shell +kubectl top pods -n workload +``` + ## Destroy the EKS Clusters To tear down all the resources and the EKS cluster, run the following command: ```shell diff --git a/argocd/iac/terraform/examples/eks/multi-cluster/distributed/bootstrap/workloads.yaml b/argocd/iac/terraform/examples/eks/multi-cluster/distributed/bootstrap/workloads.yaml index 31b798c9..ace7aed0 100644 --- a/argocd/iac/terraform/examples/eks/multi-cluster/distributed/bootstrap/workloads.yaml +++ b/argocd/iac/terraform/examples/eks/multi-cluster/distributed/bootstrap/workloads.yaml @@ -15,7 +15,7 @@ spec: values: [in-cluster] template: metadata: - name: 'workload-{{metadata.labels.environment}}' + name: 'workload' spec: project: default source: From 943f8bcb4c9355e6ab9edb4b7c1e707cbca881cc Mon Sep 17 00:00:00 2001 From: Carlos Santana Date: Sun, 29 Oct 2023 15:49:09 -0400 Subject: [PATCH 14/16] update hub spoke Signed-off-by: Carlos Santana --- .../getting-started/bootstrap/workloads.yaml | 2 +- .../examples/eks/getting-started/destroy.sh | 3 +- .../examples/eks/getting-started/main.tf | 20 +++ .../examples/eks/getting-started/variables.tf | 1 + .../eks/multi-cluster/distributed/README.md | 4 +- .../eks/multi-cluster/distributed/destroy.sh | 8 +- .../eks/multi-cluster/hub-spoke/README.md | 131 ++++++++++++++++-- .../hub-spoke/hub/bootstrap/addons.yaml | 2 +- .../hub-spoke/hub/bootstrap/workloads.yaml | 4 +- .../eks/multi-cluster/hub-spoke/hub/main.tf | 128 +++++++++-------- .../multi-cluster/hub-spoke/hub/outputs.tf | 2 +- .../multi-cluster/hub-spoke/hub/variables.tf | 50 ++++--- .../multi-cluster/hub-spoke/spokes/destroy.sh | 17 ++- .../multi-cluster/hub-spoke/spokes/main.tf | 131 ++++++++++-------- .../multi-cluster/hub-spoke/spokes/outputs.tf | 1 - .../hub-spoke/spokes/variables.tf | 48 ++++--- .../hub-spoke/spokes/workspaces/dev.tfvars | 10 +- .../hub-spoke/spokes/workspaces/prod.tfvars | 10 +- .../spokes/workspaces/staging.tfvars | 10 +- 19 files changed, 397 insertions(+), 185 deletions(-) diff --git a/argocd/iac/terraform/examples/eks/getting-started/bootstrap/workloads.yaml b/argocd/iac/terraform/examples/eks/getting-started/bootstrap/workloads.yaml index 8c883ea3..e0b3be26 100644 --- a/argocd/iac/terraform/examples/eks/getting-started/bootstrap/workloads.yaml +++ b/argocd/iac/terraform/examples/eks/getting-started/bootstrap/workloads.yaml @@ -5,7 +5,7 @@ metadata: namespace: argocd spec: syncPolicy: - preserveResourcesOnDeletion: true + preserveResourcesOnDeletion: false generators: - clusters: {} template: diff --git a/argocd/iac/terraform/examples/eks/getting-started/destroy.sh b/argocd/iac/terraform/examples/eks/getting-started/destroy.sh index 79d24cd4..7089a684 100755 --- a/argocd/iac/terraform/examples/eks/getting-started/destroy.sh +++ b/argocd/iac/terraform/examples/eks/getting-started/destroy.sh @@ -13,8 +13,7 @@ terraform -chdir=$SCRIPTDIR output -raw configure_kubectl > "$TMPFILE" if [[ ! $(cat $TMPFILE) == *"No outputs found"* ]]; then source "$TMPFILE" kubectl delete -n argocd applicationset workloads - echo "Waiting for ingress and load balancer to be deleted" - sleep 120 + kubectl delete -n game-2048 ingress game-2048 kubectl delete -n argocd applicationset cluster-addons kubectl delete -n argocd applicationset addons-argocd kubectl delete -n argocd svc argo-cd-argocd-server diff --git a/argocd/iac/terraform/examples/eks/getting-started/main.tf b/argocd/iac/terraform/examples/eks/getting-started/main.tf index 799c02af..97f78b58 100644 --- a/argocd/iac/terraform/examples/eks/getting-started/main.tf +++ b/argocd/iac/terraform/examples/eks/getting-started/main.tf @@ -217,9 +217,29 @@ module "eks" { } }) } + aws-ebs-csi-driver = { + service_account_role_arn = module.ebs_csi_driver_irsa.iam_role_arn + } } tags = local.tags } +module "ebs_csi_driver_irsa" { + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" + version = "~> 5.20" + + role_name_prefix = "${module.eks.cluster_name}-ebs-csi-" + + attach_ebs_csi_policy = true + + oidc_providers = { + main = { + provider_arn = module.eks.oidc_provider_arn + namespace_service_accounts = ["kube-system:ebs-csi-controller-sa"] + } + } + + tags = local.tags +} ################################################################################ # Supporting Resources diff --git a/argocd/iac/terraform/examples/eks/getting-started/variables.tf b/argocd/iac/terraform/examples/eks/getting-started/variables.tf index e683df52..960a3d86 100644 --- a/argocd/iac/terraform/examples/eks/getting-started/variables.tf +++ b/argocd/iac/terraform/examples/eks/getting-started/variables.tf @@ -18,6 +18,7 @@ variable "addons" { type = any default = { enable_aws_load_balancer_controller = true + enable_aws_ebs_csi_resources = true # generate gp2 and gp3 storage classes for ebs-csi enable_metrics_server = true } } diff --git a/argocd/iac/terraform/examples/eks/multi-cluster/distributed/README.md b/argocd/iac/terraform/examples/eks/multi-cluster/distributed/README.md index 68d7a64a..d44b1992 100644 --- a/argocd/iac/terraform/examples/eks/multi-cluster/distributed/README.md +++ b/argocd/iac/terraform/examples/eks/multi-cluster/distributed/README.md @@ -20,8 +20,8 @@ export TF_VAR_gitops_addons_org=https://github.com/gitops-bridge-dev export TF_VAR_gitops_addons_repo=gitops-bridge-argocd-control-plane-template ``` -## Deploy the EKS Cluster -Initialize Terraform and deploy the EKS cluster: +## Deploy the EKS Clusters +Initialize Terraform and deploy the EKS clusters: ```shell ./deploy.sh dev ./deploy.sh staging diff --git a/argocd/iac/terraform/examples/eks/multi-cluster/distributed/destroy.sh b/argocd/iac/terraform/examples/eks/multi-cluster/distributed/destroy.sh index 4f8d8fb8..fec327ed 100755 --- a/argocd/iac/terraform/examples/eks/multi-cluster/distributed/destroy.sh +++ b/argocd/iac/terraform/examples/eks/multi-cluster/distributed/destroy.sh @@ -31,8 +31,8 @@ if [[ ! $(cat $TMPFILE) == *"No outputs found"* ]]; then kubectl delete -n argocd svc argo-cd-argocd-server fi -terraform destroy -target="module.gitops_bridge_bootstrap" -auto-approve -terraform destroy -target="module.eks_blueprints_addons" -auto-approve -terraform destroy -target="module.eks" -auto-approve -terraform destroy -target="module.vpc" -auto-approve +terraform destroy -auto-approve -var-file="workspaces/${env}.tfvars" -target="module.gitops_bridge_bootstrap" -auto-approve +terraform destroy -auto-approve -var-file="workspaces/${env}.tfvars" -target="module.eks_blueprints_addons" -auto-approve +terraform destroy -auto-approve -var-file="workspaces/${env}.tfvars" -target="module.eks" -auto-approve +terraform destroy -auto-approve -var-file="workspaces/${env}.tfvars" -target="module.vpc" -auto-approve terraform destroy -auto-approve diff --git a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/README.md b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/README.md index 1b6f7717..47410d7e 100644 --- a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/README.md +++ b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/README.md @@ -6,46 +6,151 @@ The ArgoCD on the Hub Cluster deploy addons and workloads to the spoke clusters Each spoke cluster gets deployed an app of apps ArgoCD Application with the name `workloads-${env}` -Deploy the Hub Cluster +## Prerequisites +Before you begin, make sure you have the following command line tools installed: +- git +- terraform +- kubectl +- argocd + +## Fork the Git Repositories + +### Fork the Addon GitOps Repo +1. Fork the git repository for addons [here](https://github.com/gitops-bridge-dev/gitops-bridge-argocd-control-plane-template). +2. Update the following environment variables to point to your fork by changing the default values: +```shell +export TF_VAR_gitops_addons_org=https://github.com/gitops-bridge-dev +export TF_VAR_gitops_addons_repo=gitops-bridge-argocd-control-plane-template +``` + +## Deploy the Hub EKS Cluster +Change Director to `hub` ```shell cd hub +``` +Initialize Terraform and deploy the EKS cluster: +```shell terraform init -terraform apply +terraform apply -auto-approve +``` +Retrieve `kubectl` config, then execute the output command: +```shell +terraform output -raw configure_kubectl ``` -Access Terraform output for Hub Cluster +### Monitor GitOps Progress for Addons +Wait until **all** the ArgoCD applications' `HEALTH STATUS` is `Healthy`. Use Crl+C to exit the `watch` command ```shell -cd hub -terraform output +watch kubectl get applications -n argocd +``` + +## Access ArgoCD on Hub Cluster +Access ArgoCD's UI, run the command from the output: +```shell +terraform output -raw access_argocd +``` + +## Verify that ArgoCD Service Accouts has the annotation for IRSA +```shell +kubectl get sa -n argocd argocd-application-controller -o json | jq '.metadata.annotations."eks.amazonaws.com/role-arn"' +kubectl get sa -n argocd argocd-server -o json | jq '.metadata.annotations."eks.amazonaws.com/role-arn"' +``` +The output should match the `arn` for the IAM Role that will assume the IAM Role in spoke/remote clusters +```text +"arn:aws:iam::0123456789:role/hub-spoke-control-plane-argocd-hub" ``` +kubectl get secret -n argocd hub-spoke-control-plane --template='{{index .data.config | base64decode}}' -Deploy the Spoke Clusters +## Deploy the Spoke EKS Cluster +Initialize Terraform and deploy the EKS clusters: ```shell -cd spokes +cd ../spokes ./deploy.sh dev ./deploy.sh staging ./deploy.sh prod ``` Each environment uses a Terraform workspace -Access Terraform output for each environment +To access Terraform output run the following commands for the particular environment +```shell +terraform workspace select dev +terraform output +``` +```shell +terraform workspace select staging +terraform output +``` ```shell -cd spokes -terraform workspace select ${env} +terraform workspace select prod terraform output ``` -Destroy Spoke Clusters +Retrieve `kubectl` config, then execute the output command: +```shell +terraform output -raw configure_kubectl +``` + +### Verify ArgoCD Cluster Secret for Spoke has the correct IAM Role to be assume by Hub Cluster +```shell +kubectl get secret -n argocd hub-spoke-dev --template='{{index .data.config | base64decode}}' +``` +Do the same for the other cluster replaced `dev` in `hub-spoke-dev` +The output have a section `awsAuthConfig` with the `clusterName` and the `roleARN` that has write access to the spoke cluster +```json +{ + "tlsClientConfig": { + "insecure": false, + "caData" : "LS0tL...." + }, + "awsAuthConfig" : { + "clusterName": "hub-spoke-dev", + "roleARN": "arn:aws:iam::0123456789:role/hub-spoke-dev-argocd-spoke" + } +} +``` + + +### Verify the Addons on Spoke Clusters +Verify that the addons are ready: +```shell +kubectl get deployment -n kube-system \ + metrics-server +``` + + +### Monitor GitOps Progress for Workloads from Hub Cluster (run on Hub Cluster context) +Watch until **all* the Workloads ArgoCD Applications are `Healthy` +```shell +watch kubectl get -n argocd applications +``` +Wait until the ArgoCD Applications `HEALTH STATUS` is `Healthy`. Crl+C to exit the `watch` command + + +### Verify the Application +Verify that the application configuration is present and the pod is running: +```shell +kubectl get all -n workload +``` + +### Container Metrics +Check the application's CPU and memory metrics: +```shell +kubectl top pods -n workload +``` + +## Destroy the Spoke EKS Clusters +To tear down all the resources and the EKS cluster, run the following command: ```shell -cd spokes ./destroy.sh dev ./destroy.sh staging ./destroy.sh prod ``` +## Destroy the Hub EKS Clusters +To tear down all the resources and the EKS cluster, run the following command: Destroy Hub Clusters ```shell -cd hub +cd ../hub ./destroy.sh ``` diff --git a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/hub/bootstrap/addons.yaml b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/hub/bootstrap/addons.yaml index edf1f795..89f36028 100644 --- a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/hub/bootstrap/addons.yaml +++ b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/hub/bootstrap/addons.yaml @@ -15,7 +15,7 @@ spec: values: [in-cluster] template: metadata: - name: 'bootstrap-addons' + name: cluster-addons spec: project: default source: diff --git a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/hub/bootstrap/workloads.yaml b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/hub/bootstrap/workloads.yaml index 70594313..c3990393 100644 --- a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/hub/bootstrap/workloads.yaml +++ b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/hub/bootstrap/workloads.yaml @@ -5,7 +5,7 @@ metadata: namespace: argocd spec: syncPolicy: - preserveResourcesOnDeletion: true + preserveResourcesOnDeletion: false generators: - clusters: selector: @@ -23,7 +23,7 @@ spec: project: default source: repoURL: '{{metadata.annotations.workload_repo_url}}' - path: '{{metadata.annotations.workload_repo_path}}' + path: '{{metadata.annotations.workload_repo_basepath}}{{metadata.annotations.workload_repo_path}}' targetRevision: '{{metadata.annotations.workload_repo_revision}}' destination: namespace: 'workload' diff --git a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/hub/main.tf b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/hub/main.tf index f1e0f90f..60710e5d 100644 --- a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/hub/main.tf +++ b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/hub/main.tf @@ -31,11 +31,15 @@ provider "kubernetes" { } locals { - name = "hub-spoke-${local.environment}" - environment = "control-plane" - region = "us-west-2" - cluster_version = var.kubernetes_version - vpc_cidr = var.vpc_cidr + name = "hub-spoke-${local.environment}" + environment = "control-plane" + region = var.region + + cluster_version = var.kubernetes_version + + vpc_cidr = var.vpc_cidr + azs = slice(data.aws_availability_zones.available.names, 0, 3) + gitops_addons_url = "${var.gitops_addons_org}/${var.gitops_addons_repo}" gitops_addons_basepath = var.gitops_addons_basepath gitops_addons_path = var.gitops_addons_path @@ -44,43 +48,55 @@ locals { argocd_namespace = "argocd" aws_addons = { - enable_cert_manager = true - #enable_aws_efs_csi_driver = true - #enable_aws_fsx_csi_driver = true - #enable_aws_cloudwatch_metrics = true - #enable_aws_privateca_issuer = true - #enable_cluster_autoscaler = true - #enable_external_dns = true - #enable_external_secrets = true - #enable_aws_load_balancer_controller = true - #enable_fargate_fluentbit = true - #enable_aws_for_fluentbit = true - #enable_aws_node_termination_handler = true - #enable_karpenter = true - #enable_velero = true - #enable_aws_gateway_api_controller = true - #enable_aws_ebs_csi_resources = true # generate gp2 and gp3 storage classes for ebs-csi - #enable_aws_secrets_store_csi_driver_provider = true - enable_aws_argocd = true + enable_cert_manager = try(var.addons.enable_cert_manager, false) + enable_aws_efs_csi_driver = try(var.addons.enable_aws_efs_csi_driver, false) + enable_aws_fsx_csi_driver = try(var.addons.enable_aws_fsx_csi_driver, false) + enable_aws_cloudwatch_metrics = try(var.addons.enable_aws_cloudwatch_metrics, false) + enable_aws_privateca_issuer = try(var.addons.enable_aws_privateca_issuer, false) + enable_cluster_autoscaler = try(var.addons.enable_cluster_autoscaler, false) + enable_external_dns = try(var.addons.enable_external_dns, false) + enable_external_secrets = try(var.addons.enable_external_secrets, false) + enable_aws_load_balancer_controller = try(var.addons.enable_aws_load_balancer_controller, false) + enable_fargate_fluentbit = try(var.addons.enable_fargate_fluentbit, false) + enable_aws_for_fluentbit = try(var.addons.enable_aws_for_fluentbit, false) + enable_aws_node_termination_handler = try(var.addons.enable_aws_node_termination_handler, false) + enable_karpenter = try(var.addons.enable_karpenter, false) + enable_velero = try(var.addons.enable_velero, false) + enable_aws_gateway_api_controller = try(var.addons.enable_aws_gateway_api_controller, false) + enable_aws_ebs_csi_resources = try(var.addons.enable_aws_ebs_csi_resources, false) + enable_aws_secrets_store_csi_driver_provider = try(var.addons.enable_aws_secrets_store_csi_driver_provider, false) + enable_ack_apigatewayv2 = try(var.addons.enable_ack_apigatewayv2, false) + enable_ack_dynamodb = try(var.addons.enable_ack_dynamodb, false) + enable_ack_s3 = try(var.addons.enable_ack_s3, false) + enable_ack_rds = try(var.addons.enable_ack_rds, false) + enable_ack_prometheusservice = try(var.addons.enable_ack_prometheusservice, false) + enable_ack_emrcontainers = try(var.addons.enable_ack_emrcontainers, false) + enable_ack_sfn = try(var.addons.enable_ack_sfn, false) + enable_ack_eventbridge = try(var.addons.enable_ack_eventbridge, false) + enable_aws_argocd = try(var.addons.enable_aws_argocd, false) } oss_addons = { - enable_argocd = false # disable default argocd application set, we enable enable_aws_argocd above - #enable_argo_rollouts = true - #enable_argo_events = true - #enable_argo_workflows = true - #enable_cluster_proportional_autoscaler = true - #enable_gatekeeper = true - #enable_gpu_operator = true - #enable_ingress_nginx = true - #enable_kyverno = true - #enable_kube_prometheus_stack = true - #enable_metrics_server = true - #enable_prometheus_adapter = true - #enable_secrets_store_csi_driver = true - #enable_vpa = true - #enable_foo = true # you can add any addon here, make sure to update the gitops repo with the corresponding application set + enable_argocd = try(var.addons.enable_argocd, false) + enable_argo_rollouts = try(var.addons.enable_argo_rollouts, false) + enable_argo_events = try(var.addons.enable_argo_events, false) + enable_argo_workflows = try(var.addons.enable_argo_workflows, false) + enable_cluster_proportional_autoscaler = try(var.addons.enable_cluster_proportional_autoscaler, false) + enable_gatekeeper = try(var.addons.enable_gatekeeper, false) + enable_gpu_operator = try(var.addons.enable_gpu_operator, false) + enable_ingress_nginx = try(var.addons.enable_ingress_nginx, false) + enable_kyverno = try(var.addons.enable_kyverno, false) + enable_kube_prometheus_stack = try(var.addons.enable_kube_prometheus_stack, false) + enable_metrics_server = try(var.addons.enable_metrics_server, false) + enable_prometheus_adapter = try(var.addons.enable_prometheus_adapter, false) + enable_secrets_store_csi_driver = try(var.addons.enable_secrets_store_csi_driver, false) + enable_vpa = try(var.addons.enable_vpa, false) } - addons = merge(local.aws_addons, local.oss_addons, { kubernetes_version = local.cluster_version }, { aws_cluster_name = module.eks.cluster_name }) + addons = merge( + local.aws_addons, + local.oss_addons, + { kubernetes_version = local.cluster_version }, + { aws_cluster_name = module.eks.cluster_name } + ) addons_metadata = merge( module.eks_blueprints_addons.gitops_metadata, @@ -107,11 +123,9 @@ locals { workloads = file("${path.module}/bootstrap/workloads.yaml") } - azs = slice(data.aws_availability_zones.available.names, 0, 3) - tags = { Blueprint = local.name - GithubRepo = "github.com/csantanapr/terraform-gitops-bridge" + GithubRepo = "github.com/gitops-bridge-dev/gitops-bridge" } } @@ -190,21 +204,21 @@ module "eks_blueprints_addons" { create_kubernetes_resources = false # EKS Blueprints Addons - enable_cert_manager = try(local.aws_addons.enable_cert_manager, false) - enable_aws_efs_csi_driver = try(local.aws_addons.enable_aws_efs_csi_driver, false) - enable_aws_fsx_csi_driver = try(local.aws_addons.enable_aws_fsx_csi_driver, false) - enable_aws_cloudwatch_metrics = try(local.aws_addons.enable_aws_cloudwatch_metrics, false) - enable_aws_privateca_issuer = try(local.aws_addons.enable_aws_privateca_issuer, false) - enable_cluster_autoscaler = try(local.aws_addons.enable_cluster_autoscaler, false) - enable_external_dns = try(local.aws_addons.enable_external_dns, false) - enable_external_secrets = try(local.aws_addons.enable_external_secrets, false) - enable_aws_load_balancer_controller = try(local.aws_addons.enable_aws_load_balancer_controller, false) - enable_fargate_fluentbit = try(local.aws_addons.enable_fargate_fluentbit, false) - enable_aws_for_fluentbit = try(local.aws_addons.enable_aws_for_fluentbit, false) - enable_aws_node_termination_handler = try(local.aws_addons.enable_aws_node_termination_handler, false) - enable_karpenter = try(local.aws_addons.enable_karpenter, false) - enable_velero = try(local.aws_addons.enable_velero, false) - enable_aws_gateway_api_controller = try(local.aws_addons.enable_aws_gateway_api_controller, false) + enable_cert_manager = local.aws_addons.enable_cert_manager + enable_aws_efs_csi_driver = local.aws_addons.enable_aws_efs_csi_driver + enable_aws_fsx_csi_driver = local.aws_addons.enable_aws_fsx_csi_driver + enable_aws_cloudwatch_metrics = local.aws_addons.enable_aws_cloudwatch_metrics + enable_aws_privateca_issuer = local.aws_addons.enable_aws_privateca_issuer + enable_cluster_autoscaler = local.aws_addons.enable_cluster_autoscaler + enable_external_dns = local.aws_addons.enable_external_dns + enable_external_secrets = local.aws_addons.enable_external_secrets + enable_aws_load_balancer_controller = local.aws_addons.enable_aws_load_balancer_controller + enable_fargate_fluentbit = local.aws_addons.enable_fargate_fluentbit + enable_aws_for_fluentbit = local.aws_addons.enable_aws_for_fluentbit + enable_aws_node_termination_handler = local.aws_addons.enable_aws_node_termination_handler + enable_karpenter = local.aws_addons.enable_karpenter + enable_velero = local.aws_addons.enable_velero + enable_aws_gateway_api_controller = local.aws_addons.enable_aws_gateway_api_controller tags = local.tags } diff --git a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/hub/outputs.tf b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/hub/outputs.tf index c9b0f3cc..c5b089f6 100644 --- a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/hub/outputs.tf +++ b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/hub/outputs.tf @@ -27,9 +27,9 @@ output "access_argocd" { value = <<-EOT export KUBECONFIG="/tmp/${module.eks.cluster_name}" aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name} - echo "ArgoCD URL: https://$(kubectl get svc -n argocd argo-cd-argocd-server -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')" echo "ArgoCD Username: admin" echo "ArgoCD Password: $(kubectl get secrets argocd-initial-admin-secret -n argocd --template="{{index .data.password | base64decode}}")" + echo "ArgoCD URL: https://$(kubectl get svc -n argocd argo-cd-argocd-server -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')" EOT } diff --git a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/hub/variables.tf b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/hub/variables.tf index 42fa5f04..2e2c8462 100644 --- a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/hub/variables.tf +++ b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/hub/variables.tf @@ -1,3 +1,31 @@ +variable "vpc_cidr" { + description = "VPC CIDR" + type = string + default = "10.0.0.0/16" +} +variable "region" { + description = "AWS region" + type = string + default = "us-west-2" +} +variable "kubernetes_version" { + description = "Kubernetes version" + type = string + default = "1.28" +} +variable "addons" { + description = "Kubernetes addons" + type = any + default = { + enable_aws_load_balancer_controller = true + enable_metrics_server = true + # Enable argocd with IRSA + enable_aws_argocd = true + # Disable argocd without IRSA + enable_argocd = false + } +} +# Addons Git variable "gitops_addons_org" { description = "Git repository org/user contains for addons" type = string @@ -8,6 +36,11 @@ variable "gitops_addons_repo" { type = string default = "gitops-bridge-argocd-control-plane-template" } +variable "gitops_addons_revision" { + description = "Git repository revision/branch/ref for addons" + type = string + default = "main" +} variable "gitops_addons_basepath" { description = "Git repository base path for addons" type = string @@ -18,20 +51,3 @@ variable "gitops_addons_path" { type = string default = "bootstrap/control-plane/addons" } -variable "gitops_addons_revision" { - description = "Git repository revision/branch/ref for addons" - type = string - default = "HEAD" -} - - -variable "vpc_cidr" { - description = "VPC CIDR" - type = string - default = "10.0.0.0/16" -} -variable "kubernetes_version" { - description = "EKS version" - type = string - default = "1.27" -} diff --git a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/spokes/destroy.sh b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/spokes/destroy.sh index 79f53c6a..88fc8f55 100755 --- a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/spokes/destroy.sh +++ b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/spokes/destroy.sh @@ -1,5 +1,11 @@ #!/bin/bash +set -uo pipefail + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +ROOTDIR="$(cd ${SCRIPTDIR}/../..; pwd )" +[[ -n "${DEBUG:-}" ]] && set -x + if [[ $# -eq 0 ]] ; then echo "No arguments supplied" @@ -9,12 +15,11 @@ if [[ $# -eq 0 ]] ; then fi env=$1 echo "Destroying $env ..." - -set -x +terraform workspace select $env terraform workspace select $env -terraform destroy -target="module.gitops_bridge_bootstrap" -auto-approve -var-file="workspaces/${env}.tfvars" -terraform destroy -target="module.eks_blueprints_addons" -auto-approve -var-file="workspaces/${env}.tfvars" -terraform destroy -target="module.eks" -auto-approve -var-file="workspaces/${env}.tfvars" -terraform destroy -target="module.vpc" -auto-approve -var-file="workspaces/${env}.tfvars" +terraform destroy -auto-approve -var-file="workspaces/${env}.tfvars" -target="module.gitops_bridge_bootstrap" -auto-approve +terraform destroy -auto-approve -var-file="workspaces/${env}.tfvars" -target="module.eks_blueprints_addons" -auto-approve +terraform destroy -auto-approve -var-file="workspaces/${env}.tfvars" -target="module.eks" -auto-approve +terraform destroy -auto-approve -var-file="workspaces/${env}.tfvars" -target="module.vpc" -auto-approve terraform destroy -auto-approve -var-file="workspaces/${env}.tfvars" diff --git a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/spokes/main.tf b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/spokes/main.tf index 16b5c61b..018a363f 100644 --- a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/spokes/main.tf +++ b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/spokes/main.tf @@ -50,11 +50,16 @@ provider "kubernetes" { locals { - name = "hub-spoke-${terraform.workspace}" - environment = terraform.workspace - region = "us-west-2" - cluster_version = var.kubernetes_version - vpc_cidr = var.vpc_cidr + name = "hub-spoke-${terraform.workspace}" + environment = terraform.workspace + region = var.region + + cluster_version = var.kubernetes_version + + vpc_cidr = var.vpc_cidr + azs = slice(data.aws_availability_zones.available.names, 0, 3) + + gitops_addons_url = "${var.gitops_addons_org}/${var.gitops_addons_repo}" gitops_addons_basepath = var.gitops_addons_basepath gitops_addons_path = var.gitops_addons_path @@ -62,47 +67,61 @@ locals { gitops_workload_org = var.gitops_workload_org gitops_workload_repo = var.gitops_workload_repo + gitops_workload_basepath = var.gitops_workload_basepath gitops_workload_path = var.gitops_workload_path gitops_workload_revision = var.gitops_workload_revision gitops_workload_url = "${local.gitops_workload_org}/${local.gitops_workload_repo}" aws_addons = { - enable_cert_manager = true - #enable_aws_efs_csi_driver = true - #enable_aws_fsx_csi_driver = true - #enable_aws_cloudwatch_metrics = true - #enable_aws_privateca_issuer = true - #enable_cluster_autoscaler = true - #enable_external_dns = true - #enable_external_secrets = true - enable_aws_load_balancer_controller = true - #enable_fargate_fluentbit = true - #enable_aws_for_fluentbit = true - #enable_aws_node_termination_handler = true - #enable_karpenter = true - #enable_velero = true - #enable_aws_gateway_api_controller = true - #enable_aws_ebs_csi_resources = true # generate gp2 and gp3 storage classes for ebs-csi - #enable_aws_secrets_store_csi_driver_provider = true + enable_cert_manager = try(var.addons.enable_cert_manager, false) + enable_aws_efs_csi_driver = try(var.addons.enable_aws_efs_csi_driver, false) + enable_aws_fsx_csi_driver = try(var.addons.enable_aws_fsx_csi_driver, false) + enable_aws_cloudwatch_metrics = try(var.addons.enable_aws_cloudwatch_metrics, false) + enable_aws_privateca_issuer = try(var.addons.enable_aws_privateca_issuer, false) + enable_cluster_autoscaler = try(var.addons.enable_cluster_autoscaler, false) + enable_external_dns = try(var.addons.enable_external_dns, false) + enable_external_secrets = try(var.addons.enable_external_secrets, false) + enable_aws_load_balancer_controller = try(var.addons.enable_aws_load_balancer_controller, false) + enable_fargate_fluentbit = try(var.addons.enable_fargate_fluentbit, false) + enable_aws_for_fluentbit = try(var.addons.enable_aws_for_fluentbit, false) + enable_aws_node_termination_handler = try(var.addons.enable_aws_node_termination_handler, false) + enable_karpenter = try(var.addons.enable_karpenter, false) + enable_velero = try(var.addons.enable_velero, false) + enable_aws_gateway_api_controller = try(var.addons.enable_aws_gateway_api_controller, false) + enable_aws_ebs_csi_resources = try(var.addons.enable_aws_ebs_csi_resources, false) + enable_aws_secrets_store_csi_driver_provider = try(var.addons.enable_aws_secrets_store_csi_driver_provider, false) + enable_ack_apigatewayv2 = try(var.addons.enable_ack_apigatewayv2, false) + enable_ack_dynamodb = try(var.addons.enable_ack_dynamodb, false) + enable_ack_s3 = try(var.addons.enable_ack_s3, false) + enable_ack_rds = try(var.addons.enable_ack_rds, false) + enable_ack_prometheusservice = try(var.addons.enable_ack_prometheusservice, false) + enable_ack_emrcontainers = try(var.addons.enable_ack_emrcontainers, false) + enable_ack_sfn = try(var.addons.enable_ack_sfn, false) + enable_ack_eventbridge = try(var.addons.enable_ack_eventbridge, false) + enable_aws_argocd = try(var.addons.enable_aws_argocd, false) } oss_addons = { - enable_argocd = false # we are not deploying argocd to spoke clusters - #enable_argo_rollouts = true - #enable_argo_events = true - #enable_argo_workflows = true - #enable_cluster_proportional_autoscaler = true - #enable_gatekeeper = true - #enable_gpu_operator = true - #enable_ingress_nginx = true - #enable_kyverno = true - #enable_kube_prometheus_stack = true - enable_metrics_server = true - #enable_prometheus_adapter = true - #enable_secrets_store_csi_driver = true - #enable_vpa = true - #enable_foo = true # you can add any addon here, make sure to update the gitops repo with the corresponding application set + enable_argocd = try(var.addons.enable_argocd, false) + enable_argo_rollouts = try(var.addons.enable_argo_rollouts, false) + enable_argo_events = try(var.addons.enable_argo_events, false) + enable_argo_workflows = try(var.addons.enable_argo_workflows, false) + enable_cluster_proportional_autoscaler = try(var.addons.enable_cluster_proportional_autoscaler, false) + enable_gatekeeper = try(var.addons.enable_gatekeeper, false) + enable_gpu_operator = try(var.addons.enable_gpu_operator, false) + enable_ingress_nginx = try(var.addons.enable_ingress_nginx, false) + enable_kyverno = try(var.addons.enable_kyverno, false) + enable_kube_prometheus_stack = try(var.addons.enable_kube_prometheus_stack, false) + enable_metrics_server = try(var.addons.enable_metrics_server, false) + enable_prometheus_adapter = try(var.addons.enable_prometheus_adapter, false) + enable_secrets_store_csi_driver = try(var.addons.enable_secrets_store_csi_driver, false) + enable_vpa = try(var.addons.enable_vpa, false) } - addons = merge(local.aws_addons, local.oss_addons, { kubernetes_version = local.cluster_version }, { aws_cluster_name = module.eks.cluster_name }) + addons = merge( + local.aws_addons, + local.oss_addons, + { kubernetes_version = local.cluster_version }, + { aws_cluster_name = module.eks.cluster_name } + ) addons_metadata = merge( module.eks_blueprints_addons.gitops_metadata, @@ -120,17 +139,15 @@ locals { }, { workload_repo_url = local.gitops_workload_url + workload_repo_basepath = local.gitops_workload_basepath workload_repo_path = local.gitops_workload_path workload_repo_revision = local.gitops_workload_revision } ) - - azs = slice(data.aws_availability_zones.available.names, 0, 3) - tags = { Blueprint = local.name - GithubRepo = "github.com/csantanapr/terraform-gitops-bridge" + GithubRepo = "github.com/gitops-bridge-dev/gitops-bridge" } } @@ -203,21 +220,21 @@ module "eks_blueprints_addons" { create_kubernetes_resources = false # EKS Blueprints Addons - enable_cert_manager = try(local.aws_addons.enable_cert_manager, false) - enable_aws_efs_csi_driver = try(local.aws_addons.enable_aws_efs_csi_driver, false) - enable_aws_fsx_csi_driver = try(local.aws_addons.enable_aws_fsx_csi_driver, false) - enable_aws_cloudwatch_metrics = try(local.aws_addons.enable_aws_cloudwatch_metrics, false) - enable_aws_privateca_issuer = try(local.aws_addons.enable_aws_privateca_issuer, false) - enable_cluster_autoscaler = try(local.aws_addons.enable_cluster_autoscaler, false) - enable_external_dns = try(local.aws_addons.enable_external_dns, false) - enable_external_secrets = try(local.aws_addons.enable_external_secrets, false) - enable_aws_load_balancer_controller = try(local.aws_addons.enable_aws_load_balancer_controller, false) - enable_fargate_fluentbit = try(local.aws_addons.enable_fargate_fluentbit, false) - enable_aws_for_fluentbit = try(local.aws_addons.enable_aws_for_fluentbit, false) - enable_aws_node_termination_handler = try(local.aws_addons.enable_aws_node_termination_handler, false) - enable_karpenter = try(local.aws_addons.enable_karpenter, false) - enable_velero = try(local.aws_addons.enable_velero, false) - enable_aws_gateway_api_controller = try(local.aws_addons.enable_aws_gateway_api_controller, false) + enable_cert_manager = local.aws_addons.enable_cert_manager + enable_aws_efs_csi_driver = local.aws_addons.enable_aws_efs_csi_driver + enable_aws_fsx_csi_driver = local.aws_addons.enable_aws_fsx_csi_driver + enable_aws_cloudwatch_metrics = local.aws_addons.enable_aws_cloudwatch_metrics + enable_aws_privateca_issuer = local.aws_addons.enable_aws_privateca_issuer + enable_cluster_autoscaler = local.aws_addons.enable_cluster_autoscaler + enable_external_dns = local.aws_addons.enable_external_dns + enable_external_secrets = local.aws_addons.enable_external_secrets + enable_aws_load_balancer_controller = local.aws_addons.enable_aws_load_balancer_controller + enable_fargate_fluentbit = local.aws_addons.enable_fargate_fluentbit + enable_aws_for_fluentbit = local.aws_addons.enable_aws_for_fluentbit + enable_aws_node_termination_handler = local.aws_addons.enable_aws_node_termination_handler + enable_karpenter = local.aws_addons.enable_karpenter + enable_velero = local.aws_addons.enable_velero + enable_aws_gateway_api_controller = local.aws_addons.enable_aws_gateway_api_controller tags = local.tags } diff --git a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/spokes/outputs.tf b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/spokes/outputs.tf index 974a0a9f..e398229a 100644 --- a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/spokes/outputs.tf +++ b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/spokes/outputs.tf @@ -5,4 +5,3 @@ output "configure_kubectl" { aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name} EOT } - diff --git a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/spokes/variables.tf b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/spokes/variables.tf index 3ef79fd7..57032511 100644 --- a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/spokes/variables.tf +++ b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/spokes/variables.tf @@ -1,3 +1,20 @@ +variable "region" { + description = "AWS region" + type = string +} +variable "vpc_cidr" { + description = "VPC CIDR" + type = string +} +variable "kubernetes_version" { + description = "EKS version" + type = string +} +variable "addons" { + description = "Kubernetes addons" + type = any +} +# Addons Git variable "gitops_addons_org" { description = "Git repository org/user contains for addons" type = string @@ -8,6 +25,11 @@ variable "gitops_addons_repo" { type = string default = "gitops-bridge-argocd-control-plane-template" } +variable "gitops_addons_revision" { + description = "Git repository revision/branch/ref for addons" + type = string + default = "main" +} variable "gitops_addons_basepath" { description = "Git repository base path for addons" type = string @@ -18,12 +40,8 @@ variable "gitops_addons_path" { type = string default = "bootstrap/control-plane/addons" } -variable "gitops_addons_revision" { - description = "Git repository revision/branch/ref for addons" - type = string - default = "HEAD" -} +# Workloads Git variable "gitops_workload_org" { description = "Git repository org/user contains for workload" type = string @@ -34,24 +52,18 @@ variable "gitops_workload_repo" { type = string default = "argocd-example-apps" } -variable "gitops_workload_path" { - description = "Git repository path for workload" - type = string - default = "helm-guestbook" -} variable "gitops_workload_revision" { description = "Git repository revision/branch/ref for workload" type = string - default = "HEAD" + default = "master" } - - - -variable "vpc_cidr" { - description = "VPC CIDR" +variable "gitops_workload_basepath" { + description = "Git repository base path for workload" type = string + default = "" } -variable "kubernetes_version" { - description = "EKS version" +variable "gitops_workload_path" { + description = "Git repository path for workload" type = string + default = "helm-guestbook" } diff --git a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/spokes/workspaces/dev.tfvars b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/spokes/workspaces/dev.tfvars index 8c1910fc..706f9b84 100644 --- a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/spokes/workspaces/dev.tfvars +++ b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/spokes/workspaces/dev.tfvars @@ -1,2 +1,10 @@ vpc_cidr = "10.1.0.0/16" -kubernetes_version = "1.27" \ No newline at end of file +region = "us-west-2" +kubernetes_version = "1.28" +addons = { + enable_aws_load_balancer_controller = true + enable_metrics_server = true + # Disable argocd on spoke clusters + enable_aws_argocd = false + enable_argocd = false +} \ No newline at end of file diff --git a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/spokes/workspaces/prod.tfvars b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/spokes/workspaces/prod.tfvars index 6301efd5..f1fdaa1c 100644 --- a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/spokes/workspaces/prod.tfvars +++ b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/spokes/workspaces/prod.tfvars @@ -1,2 +1,10 @@ vpc_cidr = "10.3.0.0/16" -kubernetes_version = "1.27" \ No newline at end of file +region = "us-west-2" +kubernetes_version = "1.28" +addons = { + enable_aws_load_balancer_controller = true + enable_metrics_server = true + # Disable argocd on spoke clusters + enable_aws_argocd = false + enable_argocd = false +} \ No newline at end of file diff --git a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/spokes/workspaces/staging.tfvars b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/spokes/workspaces/staging.tfvars index a7dc928c..5fc3593f 100644 --- a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/spokes/workspaces/staging.tfvars +++ b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/spokes/workspaces/staging.tfvars @@ -1,2 +1,10 @@ vpc_cidr = "10.2.0.0/16" -kubernetes_version = "1.27" \ No newline at end of file +region = "us-west-2" +kubernetes_version = "1.28" +addons = { + enable_aws_load_balancer_controller = true + enable_metrics_server = true + # Disable argocd on spoke clusters + enable_aws_argocd = false + enable_argocd = false +} \ No newline at end of file From 1ad0d45abf267b07a535b453ab312cd2b4aa71d3 Mon Sep 17 00:00:00 2001 From: Carlos Santana Date: Sun, 29 Oct 2023 15:50:11 -0400 Subject: [PATCH 15/16] update cluster-addons Signed-off-by: Carlos Santana --- .../eks/multi-cluster/distributed/bootstrap/addons.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/argocd/iac/terraform/examples/eks/multi-cluster/distributed/bootstrap/addons.yaml b/argocd/iac/terraform/examples/eks/multi-cluster/distributed/bootstrap/addons.yaml index edf1f795..89f36028 100644 --- a/argocd/iac/terraform/examples/eks/multi-cluster/distributed/bootstrap/addons.yaml +++ b/argocd/iac/terraform/examples/eks/multi-cluster/distributed/bootstrap/addons.yaml @@ -15,7 +15,7 @@ spec: values: [in-cluster] template: metadata: - name: 'bootstrap-addons' + name: cluster-addons spec: project: default source: From 2efcd9a3bac7914828c1942ad1230abdcc8120b5 Mon Sep 17 00:00:00 2001 From: Carlos Santana Date: Sun, 29 Oct 2023 18:30:09 -0400 Subject: [PATCH 16/16] update hub-spoke shared Signed-off-by: Carlos Santana --- README.md | 49 +++--- .../distributed/bootstrap/workloads.yaml | 2 +- .../multi-cluster/hub-spoke-shared/README.md | 140 +++++++++++++++--- .../hub-spoke-shared/hub/destroy.sh | 19 ++- .../hub-spoke-shared/hub/main.tf | 128 +++++++++------- .../hub-spoke-shared/hub/outputs.tf | 2 +- .../hub-spoke-shared/hub/variables.tf | 50 ++++--- .../spokes/bootstrap/workloads.yaml | 4 +- .../hub-spoke-shared/spokes/destroy.sh | 33 +++-- .../hub-spoke-shared/spokes/main.tf | 128 +++++++++------- .../hub-spoke-shared/spokes/variables.tf | 47 +++--- .../spokes/workspaces/dev.tfvars | 9 +- .../spokes/workspaces/prod.tfvars | 9 +- .../spokes/workspaces/staging.tfvars | 9 +- .../eks/multi-cluster/hub-spoke/README.md | 2 - .../multi-cluster/hub-spoke/hub/destroy.sh | 19 ++- .../multi-cluster/hub-spoke/spokes/destroy.sh | 1 - .../multi-cluster/hub-spoke/spokes/main.tf | 2 - 18 files changed, 435 insertions(+), 218 deletions(-) diff --git a/README.md b/README.md index c60f1e37..072756de 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,33 @@ # GitOps Bridge -The GitOps Bridge is a community project to show best practices and patterms on how to bridge the process of creating a Kubernetes Cluster to then delegate everything after that to GitOps using [ArgoCD](https://www.cncf.io/projects/argo/) or [FluxCD](https://www.cncf.io/projects/flux/) both CNCF graduated projects. +The [GitOps Bridge](https://github.com/gitops-bridge-dev/gitops-bridge) is a community project that aims to showcase best practices and patterns for bridging the process of creating a Kubernetes cluster to subsequently managing everything through GitOps. It focuses on using [ArgoCD](https://www.cncf.io/projects/argo/) or [FluxCD](https://www.cncf.io/projects/flux/), both of which are CNCF-graduated projects. -See the git repository [GitOps Control Plane](https://github.com/gitops-bridge-dev/gitops-bridge-argocd-control-plane-template) for an example template on bootstrapping ArgoCD +For an example template on bootstrapping ArgoCD, see the GitHub repository [GitOps Control Plane](https://github.com/gitops-bridge-dev/gitops-bridge-argocd-control-plane-template). -There are many tools to create Kubernetes clusters, this include roll your own like kubeadmin/minikube/kind or a cloud managed service like Amazon EKS. It should not matter how the the cluster is created in terms of GitOps, GitOps engines should be compatible with any tool that the user choose to use to create the cluster include cases using Kubernetes to create other Kubernetes clusters like CAPI/CAPA, Crossplane, ACK, or any tool running inside Kubernetes to deploy Kubernetes. +There are many tools available for creating Kubernetes clusters. These include "roll-your-own" solutions like `kubeadm`, `minikube`, and `kind`, as well as cloud-managed services like Amazon EKS. The method of cluster creation should not impact GitOps compatibility; GitOps engines should work with any tool that the user chooses for cluster creation. This includes scenarios where Kubernetes is used to create other Kubernetes clusters, such as with CAPI/CAPA, Crossplane, ACK, or any tool running inside Kubernetes to deploy Kubernetes. -The GitOps Bridge becomes extremely important for cloud managed kubernetes, this cluster have integrations with cloud services. When using GitOps to install a tool in this cases, the tool usually via helm needs to be configure with metadata about resources or workload identity (IAM) that is available as a result of running a IaC tool such terraform, cloudformation, or cloud cli. The GitOps Bridge would show patterns on how to bridge this metadata about the cluster to GitOps using features specific GitOps engine combined. +The GitOps Bridge becomes extremely important in the context of cloud-managed Kubernetes clusters, as these clusters often have integrations with cloud services. When using GitOps to install a tool in such cases, the tool—usually configured via Helm—needs to be set up with metadata about resources or workload identities (like IAM). This metadata is often available as a result of running an Infrastructure as Code (IaC) tool such as Terraform, CloudFormation, or a cloud CLI. The GitOps Bridge provides patterns for bridging this metadata to GitOps, using features specific to the GitOps engine in use. + +The GitOps Bridge should also be compatible with GitOps engines that run as SaaS and are not installed inside the cluster, such as the Akuity Platform, CodeFresh, Weaveworks, and others. + + + + + +The [GitOps Bridge Pattern](https://github.com/gitops-bridge-dev) enables Kubernetes administrators to utilize Infrastructure as Code (IaC) and GitOps tools for deploying Kubernetes Addons and Workloads. Addons often depend on Cloud resources that are external to the cluster. The configuration metadata for these external resources is required by the Addons' Helm charts. While IaC is used to create these cloud resources, it is not used to install the Helm charts. Instead, the IaC tool stores this metadata either within GitOps resources in the cluster or in a Git repository. The GitOps tool then extracts these metadata values and passes them to the Helm chart during the Addon installation process. This mechanism forms the bridge between IaC and GitOps, hence the term "GitOps Bridge." + +Try out the [Getting Started](https://github.com/gitops-bridge-dev/gitops-bridge/tree/main/argocd/iac/terraform/examples/eks/getting-started) example. + +Additional examples available on the [GitOps Bridge Pattern](https://github.com/gitops-bridge-dev): +- [argocd-ingress](https://github.com/gitops-bridge-dev/gitops-bridge/tree/main/argocd/iac/terraform/examples/eks/argocd-ingress) +- [aws-secrets-manager](https://github.com/gitops-bridge-dev/gitops-bridge/tree/main/argocd/iac/terraform/examples/eks/aws-secrets-manager) +- [crossplane](https://github.com/gitops-bridge-dev/gitops-bridge/tree/main/argocd/iac/terraform/examples/eks/crossplane) +- [external-secrets](https://github.com/gitops-bridge-dev/gitops-bridge/tree/main/argocd/iac/terraform/examples/eks/external-secrets) +- [multi-cluster/distributed](https://github.com/gitops-bridge-dev/gitops-bridge/tree/main/argocd/iac/terraform/examples/eks/multi-cluster/distributed) +- [multi-cluster/hub-spoke](https://github.com/gitops-bridge-dev/gitops-bridge/tree/main/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke) +- [multi-cluster/hub-spoke-shared](https://github.com/gitops-bridge-dev/gitops-bridge/tree/main/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared) +- [private-git](https://github.com/gitops-bridge-dev/gitops-bridge/tree/main/argocd/iac/terraform/examples/eks/private-git) -The GitOps Bridge should also be compatible with GitOps engines that run as Saas and not install inside the cluster such as Akuity Platform, CodeFresh, Weaveworks and others. ### ArgoCD @@ -28,24 +47,16 @@ of the Kubernetes resources, any changes to these resources outside Terraform fo | Crossplane | ArgoCD | | | CAPI | ArgoCD | | | Pulumi | ArgoCD | | -| ACK | ArgoCD | | -| CloudFormation | ArgoCD | | -| Kops | ArgoCD | | -| Ansible | ArgoCD | | ### FluxCD Status | IaC | GitOps | Status | | :--- | :----: | ---: | -| Terraform | FluxCD | Soon | -| EKSCTL | FluxCD | | -| CDK | FluxCD | | -| Crossplane | FluxCD | | -| CAPI | FluxCD | | -| Pulumi | FluxCD | | -| ACK | FluxCD | | -| CloudFormation | FluxCD | | -| Kops | FluxCD | | -| Ansible | FluxCD | | +| Terraform | FluxCD | [In Progress](https://github.com/gitops-bridge-dev/gitops-bridge/issues/32) | +| EKSCTL | ArgoCD | | +| CDK | ArgoCD | | +| Crossplane | ArgoCD | | +| CAPI | ArgoCD | | +| Pulumi | ArgoCD | | #### Researched Resources: - https://docs.akuity.io/tutorials/adv-gitops diff --git a/argocd/iac/terraform/examples/eks/multi-cluster/distributed/bootstrap/workloads.yaml b/argocd/iac/terraform/examples/eks/multi-cluster/distributed/bootstrap/workloads.yaml index ace7aed0..31b798c9 100644 --- a/argocd/iac/terraform/examples/eks/multi-cluster/distributed/bootstrap/workloads.yaml +++ b/argocd/iac/terraform/examples/eks/multi-cluster/distributed/bootstrap/workloads.yaml @@ -15,7 +15,7 @@ spec: values: [in-cluster] template: metadata: - name: 'workload' + name: 'workload-{{metadata.labels.environment}}' spec: project: default source: diff --git a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/README.md b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/README.md index 2a6614e0..6ffb30d6 100644 --- a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/README.md +++ b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/README.md @@ -1,56 +1,160 @@ -# Multi-Cluster centralized hub-spoke topology (shared responsability) +# Multi-Cluster centralized hub-spoke topology This example deploys ArgoCD on the Hub cluster (ie. management/control-plane cluster). The spoke clusters are registered as remote clusters in the Hub Cluster's ArgoCD -The ArgoCD on the Hub Cluster deploy addons to the spoke clusters -Each spoke cluster have ArgoCD only use for workloads, not the addons +The ArgoCD on the Hub Cluster deploy addons and workloads to the spoke clusters Each spoke cluster gets deployed an app of apps ArgoCD Application with the name `workloads-${env}` -This platform team and the application team have a **shared** responsability of the cluster's GitOps configuration with multiple instances of ArgoCD targeting the same cluster, -the platform team responsible of the cluster addons using the ArgoCD instance on the hub cluster and the application team responsible of the workloads using the ArgoCD instance on the spoke cluster. +## Prerequisites +Before you begin, make sure you have the following command line tools installed: +- git +- terraform +- kubectl +- argocd +## Fork the Git Repositories -Deploy the Hub Cluster +### Fork the Addon GitOps Repo +1. Fork the git repository for addons [here](https://github.com/gitops-bridge-dev/gitops-bridge-argocd-control-plane-template). +2. Update the following environment variables to point to your fork by changing the default values: +```shell +export TF_VAR_gitops_addons_org=https://github.com/gitops-bridge-dev +export TF_VAR_gitops_addons_repo=gitops-bridge-argocd-control-plane-template +``` + +## Deploy the Hub EKS Cluster +Change Director to `hub` ```shell cd hub +``` +Initialize Terraform and deploy the EKS cluster: +```shell terraform init -terraform apply +terraform apply -auto-approve +``` +Retrieve `kubectl` config, then execute the output command: +```shell +terraform output -raw configure_kubectl ``` -Access Terraform output for Hub Cluster +### Monitor GitOps Progress for Addons +Wait until **all** the ArgoCD applications' `HEALTH STATUS` is `Healthy`. Use Crl+C to exit the `watch` command ```shell -cd hub -terraform output +watch kubectl get applications -n argocd ``` +## Access ArgoCD on Hub Cluster +Access ArgoCD's UI, run the command from the output: +```shell +terraform output -raw access_argocd +``` -Deploy the Spoke Clusters +## Verify that ArgoCD Service Accouts has the annotation for IRSA ```shell -cd spokes +kubectl get sa -n argocd argocd-application-controller -o json | jq '.metadata.annotations."eks.amazonaws.com/role-arn"' +kubectl get sa -n argocd argocd-server -o json | jq '.metadata.annotations."eks.amazonaws.com/role-arn"' +``` +The output should match the `arn` for the IAM Role that will assume the IAM Role in spoke/remote clusters +```text +"arn:aws:iam::0123456789:role/hub-spoke-control-plane-argocd-hub" +``` + +## Deploy the Spoke EKS Cluster +Initialize Terraform and deploy the EKS clusters: +Is recommended to use a new terminal window for each cluster +```shell +cd ../spokes ./deploy.sh dev ./deploy.sh staging ./deploy.sh prod ``` Each environment uses a Terraform workspace -Access Terraform output for each environment +To access Terraform output run the following commands for the particular environment +```shell +terraform workspace select dev +terraform output +``` ```shell -cd spokes -terraform workspace select ${env} +terraform workspace select staging terraform output ``` +```shell +terraform workspace select prod +terraform output +``` + +Retrieve `kubectl` config, then execute the output command: +```shell +terraform output -raw configure_kubectl +``` + +### Verify ArgoCD Cluster Secret for Spoke has the correct IAM Role to be assume by Hub Cluster +```shell +kubectl get secret -n argocd hub-spoke-dev --template='{{index .data.config | base64decode}}' +``` +Do the same for the other cluster replaced `dev` in `hub-spoke-dev` +The output have a section `awsAuthConfig` with the `clusterName` and the `roleARN` that has write access to the spoke cluster +```json +{ + "tlsClientConfig": { + "insecure": false, + "caData" : "LS0tL...." + }, + "awsAuthConfig" : { + "clusterName": "hub-spoke-dev", + "roleARN": "arn:aws:iam::0123456789:role/hub-spoke-dev-argocd-spoke" + } +} +``` + + +### Verify the Addons on Spoke Clusters +Verify that the addons are ready: +```shell +kubectl get deployment -n kube-system \ + metrics-server +``` -Destroy Spoke Clusters + +### Monitor GitOps Progress for Workloads from Hub Cluster (run on Hub Cluster context) +Watch until **all* the Workloads ArgoCD Applications are `Healthy` +```shell +watch kubectl get -n argocd applications +``` +Wait until the ArgoCD Applications `HEALTH STATUS` is `Healthy`. Crl+C to exit the `watch` command + +### Monitor Workloads Progress on Spoke Cluster +Wait until **all** the ArgoCD applications' `HEALTH STATUS` is `Healthy`. Use Crl+C to exit the `watch` command +```shell +watch kubectl get applications -n argocd +``` + +### Verify the Application +Verify that the application configuration is present and the pod is running: +```shell +kubectl get all -n workload +``` + +### Container Metrics +Check the application's CPU and memory metrics: +```shell +kubectl top pods -n workload +``` + +## Destroy the Spoke EKS Clusters +To tear down all the resources and the EKS cluster, run the following command: ```shell -cd spokes ./destroy.sh dev ./destroy.sh staging ./destroy.sh prod ``` +## Destroy the Hub EKS Clusters +To tear down all the resources and the EKS cluster, run the following command: Destroy Hub Clusters ```shell -cd hub +cd ../hub ./destroy.sh ``` diff --git a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/hub/destroy.sh b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/hub/destroy.sh index 195f9885..c488c222 100755 --- a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/hub/destroy.sh +++ b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/hub/destroy.sh @@ -1,13 +1,22 @@ #!/bin/bash -set -x +set -uo pipefail + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +ROOTDIR="$(cd ${SCRIPTDIR}/../..; pwd )" +[[ -n "${DEBUG:-}" ]] && set -x # Delete the Ingress/SVC before removing the addons TMPFILE=$(mktemp) -terraform output -raw configure_kubectl > "$TMPFILE" -source "$TMPFILE" - -kubectl delete svc -n argocd argo-cd-argocd-server +terraform -chdir=$SCRIPTDIR output -raw configure_kubectl > "$TMPFILE" +# check if TMPFILE contains the string "No outputs found" +if [[ ! $(cat $TMPFILE) == *"No outputs found"* ]]; then + source "$TMPFILE" + #kubectl delete -n argocd applicationset workloads + kubectl delete -n argocd applicationset cluster-addons + kubectl delete -n argocd applicationset addons-argocd + kubectl delete -n argocd svc argo-cd-argocd-server +fi terraform destroy -target="module.gitops_bridge_bootstrap" -auto-approve terraform destroy -target="module.eks_blueprints_addons" -auto-approve diff --git a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/hub/main.tf b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/hub/main.tf index c1538aec..69fde499 100644 --- a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/hub/main.tf +++ b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/hub/main.tf @@ -31,11 +31,15 @@ provider "kubernetes" { } locals { - name = "hub-spoke-2-${local.environment}" - environment = "control-plane" - region = "us-west-2" - cluster_version = var.kubernetes_version - vpc_cidr = var.vpc_cidr + name = "hub-spoke-2-${local.environment}" + environment = "control-plane" + region = var.region + + cluster_version = var.kubernetes_version + + vpc_cidr = var.vpc_cidr + azs = slice(data.aws_availability_zones.available.names, 0, 3) + gitops_addons_url = "${var.gitops_addons_org}/${var.gitops_addons_repo}" gitops_addons_basepath = var.gitops_addons_basepath gitops_addons_path = var.gitops_addons_path @@ -44,43 +48,55 @@ locals { argocd_namespace = "argocd" aws_addons = { - enable_cert_manager = true - #enable_aws_efs_csi_driver = true - #enable_aws_fsx_csi_driver = true - #enable_aws_cloudwatch_metrics = true - #enable_aws_privateca_issuer = true - #enable_cluster_autoscaler = true - #enable_external_dns = true - #enable_external_secrets = true - #enable_aws_load_balancer_controller = true - #enable_fargate_fluentbit = true - #enable_aws_for_fluentbit = true - #enable_aws_node_termination_handler = true - #enable_karpenter = true - #enable_velero = true - #enable_aws_gateway_api_controller = true - #enable_aws_ebs_csi_resources = true # generate gp2 and gp3 storage classes for ebs-csi - #enable_aws_secrets_store_csi_driver_provider = true - enable_aws_argocd = true + enable_cert_manager = try(var.addons.enable_cert_manager, false) + enable_aws_efs_csi_driver = try(var.addons.enable_aws_efs_csi_driver, false) + enable_aws_fsx_csi_driver = try(var.addons.enable_aws_fsx_csi_driver, false) + enable_aws_cloudwatch_metrics = try(var.addons.enable_aws_cloudwatch_metrics, false) + enable_aws_privateca_issuer = try(var.addons.enable_aws_privateca_issuer, false) + enable_cluster_autoscaler = try(var.addons.enable_cluster_autoscaler, false) + enable_external_dns = try(var.addons.enable_external_dns, false) + enable_external_secrets = try(var.addons.enable_external_secrets, false) + enable_aws_load_balancer_controller = try(var.addons.enable_aws_load_balancer_controller, false) + enable_fargate_fluentbit = try(var.addons.enable_fargate_fluentbit, false) + enable_aws_for_fluentbit = try(var.addons.enable_aws_for_fluentbit, false) + enable_aws_node_termination_handler = try(var.addons.enable_aws_node_termination_handler, false) + enable_karpenter = try(var.addons.enable_karpenter, false) + enable_velero = try(var.addons.enable_velero, false) + enable_aws_gateway_api_controller = try(var.addons.enable_aws_gateway_api_controller, false) + enable_aws_ebs_csi_resources = try(var.addons.enable_aws_ebs_csi_resources, false) + enable_aws_secrets_store_csi_driver_provider = try(var.addons.enable_aws_secrets_store_csi_driver_provider, false) + enable_ack_apigatewayv2 = try(var.addons.enable_ack_apigatewayv2, false) + enable_ack_dynamodb = try(var.addons.enable_ack_dynamodb, false) + enable_ack_s3 = try(var.addons.enable_ack_s3, false) + enable_ack_rds = try(var.addons.enable_ack_rds, false) + enable_ack_prometheusservice = try(var.addons.enable_ack_prometheusservice, false) + enable_ack_emrcontainers = try(var.addons.enable_ack_emrcontainers, false) + enable_ack_sfn = try(var.addons.enable_ack_sfn, false) + enable_ack_eventbridge = try(var.addons.enable_ack_eventbridge, false) + enable_aws_argocd = try(var.addons.enable_aws_argocd, false) } oss_addons = { - enable_argocd = false # disable default argocd application set, we enable enable_aws_argocd above - #enable_argo_rollouts = true - #enable_argo_events = true - #enable_argo_workflows = true - #enable_cluster_proportional_autoscaler = true - #enable_gatekeeper = true - #enable_gpu_operator = true - #enable_ingress_nginx = true - #enable_kyverno = true - #enable_kube_prometheus_stack = true - #enable_metrics_server = true - #enable_prometheus_adapter = true - #enable_secrets_store_csi_driver = true - #enable_vpa = true - #enable_foo = true # you can add any addon here, make sure to update the gitops repo with the corresponding application set + enable_argocd = try(var.addons.enable_argocd, false) + enable_argo_rollouts = try(var.addons.enable_argo_rollouts, false) + enable_argo_events = try(var.addons.enable_argo_events, false) + enable_argo_workflows = try(var.addons.enable_argo_workflows, false) + enable_cluster_proportional_autoscaler = try(var.addons.enable_cluster_proportional_autoscaler, false) + enable_gatekeeper = try(var.addons.enable_gatekeeper, false) + enable_gpu_operator = try(var.addons.enable_gpu_operator, false) + enable_ingress_nginx = try(var.addons.enable_ingress_nginx, false) + enable_kyverno = try(var.addons.enable_kyverno, false) + enable_kube_prometheus_stack = try(var.addons.enable_kube_prometheus_stack, false) + enable_metrics_server = try(var.addons.enable_metrics_server, false) + enable_prometheus_adapter = try(var.addons.enable_prometheus_adapter, false) + enable_secrets_store_csi_driver = try(var.addons.enable_secrets_store_csi_driver, false) + enable_vpa = try(var.addons.enable_vpa, false) } - addons = merge(local.aws_addons, local.oss_addons, { kubernetes_version = local.cluster_version }, { aws_cluster_name = module.eks.cluster_name }) + addons = merge( + local.aws_addons, + local.oss_addons, + { kubernetes_version = local.cluster_version }, + { aws_cluster_name = module.eks.cluster_name } + ) addons_metadata = merge( module.eks_blueprints_addons.gitops_metadata, @@ -106,11 +122,9 @@ locals { addons = file("${path.module}/bootstrap/addons.yaml") } - azs = slice(data.aws_availability_zones.available.names, 0, 3) - tags = { Blueprint = local.name - GithubRepo = "github.com/csantanapr/terraform-gitops-bridge" + GithubRepo = "github.com/gitops-bridge-dev/gitops-bridge" } } @@ -189,21 +203,21 @@ module "eks_blueprints_addons" { create_kubernetes_resources = false # EKS Blueprints Addons - enable_cert_manager = try(local.aws_addons.enable_cert_manager, false) - enable_aws_efs_csi_driver = try(local.aws_addons.enable_aws_efs_csi_driver, false) - enable_aws_fsx_csi_driver = try(local.aws_addons.enable_aws_fsx_csi_driver, false) - enable_aws_cloudwatch_metrics = try(local.aws_addons.enable_aws_cloudwatch_metrics, false) - enable_aws_privateca_issuer = try(local.aws_addons.enable_aws_privateca_issuer, false) - enable_cluster_autoscaler = try(local.aws_addons.enable_cluster_autoscaler, false) - enable_external_dns = try(local.aws_addons.enable_external_dns, false) - enable_external_secrets = try(local.aws_addons.enable_external_secrets, false) - enable_aws_load_balancer_controller = try(local.aws_addons.enable_aws_load_balancer_controller, false) - enable_fargate_fluentbit = try(local.aws_addons.enable_fargate_fluentbit, false) - enable_aws_for_fluentbit = try(local.aws_addons.enable_aws_for_fluentbit, false) - enable_aws_node_termination_handler = try(local.aws_addons.enable_aws_node_termination_handler, false) - enable_karpenter = try(local.aws_addons.enable_karpenter, false) - enable_velero = try(local.aws_addons.enable_velero, false) - enable_aws_gateway_api_controller = try(local.aws_addons.enable_aws_gateway_api_controller, false) + enable_cert_manager = local.aws_addons.enable_cert_manager + enable_aws_efs_csi_driver = local.aws_addons.enable_aws_efs_csi_driver + enable_aws_fsx_csi_driver = local.aws_addons.enable_aws_fsx_csi_driver + enable_aws_cloudwatch_metrics = local.aws_addons.enable_aws_cloudwatch_metrics + enable_aws_privateca_issuer = local.aws_addons.enable_aws_privateca_issuer + enable_cluster_autoscaler = local.aws_addons.enable_cluster_autoscaler + enable_external_dns = local.aws_addons.enable_external_dns + enable_external_secrets = local.aws_addons.enable_external_secrets + enable_aws_load_balancer_controller = local.aws_addons.enable_aws_load_balancer_controller + enable_fargate_fluentbit = local.aws_addons.enable_fargate_fluentbit + enable_aws_for_fluentbit = local.aws_addons.enable_aws_for_fluentbit + enable_aws_node_termination_handler = local.aws_addons.enable_aws_node_termination_handler + enable_karpenter = local.aws_addons.enable_karpenter + enable_velero = local.aws_addons.enable_velero + enable_aws_gateway_api_controller = local.aws_addons.enable_aws_gateway_api_controller tags = local.tags } diff --git a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/hub/outputs.tf b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/hub/outputs.tf index c9b0f3cc..c5b089f6 100644 --- a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/hub/outputs.tf +++ b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/hub/outputs.tf @@ -27,9 +27,9 @@ output "access_argocd" { value = <<-EOT export KUBECONFIG="/tmp/${module.eks.cluster_name}" aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name} - echo "ArgoCD URL: https://$(kubectl get svc -n argocd argo-cd-argocd-server -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')" echo "ArgoCD Username: admin" echo "ArgoCD Password: $(kubectl get secrets argocd-initial-admin-secret -n argocd --template="{{index .data.password | base64decode}}")" + echo "ArgoCD URL: https://$(kubectl get svc -n argocd argo-cd-argocd-server -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')" EOT } diff --git a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/hub/variables.tf b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/hub/variables.tf index 77ccb5d6..2e2c8462 100644 --- a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/hub/variables.tf +++ b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/hub/variables.tf @@ -1,3 +1,31 @@ +variable "vpc_cidr" { + description = "VPC CIDR" + type = string + default = "10.0.0.0/16" +} +variable "region" { + description = "AWS region" + type = string + default = "us-west-2" +} +variable "kubernetes_version" { + description = "Kubernetes version" + type = string + default = "1.28" +} +variable "addons" { + description = "Kubernetes addons" + type = any + default = { + enable_aws_load_balancer_controller = true + enable_metrics_server = true + # Enable argocd with IRSA + enable_aws_argocd = true + # Disable argocd without IRSA + enable_argocd = false + } +} +# Addons Git variable "gitops_addons_org" { description = "Git repository org/user contains for addons" type = string @@ -8,6 +36,11 @@ variable "gitops_addons_repo" { type = string default = "gitops-bridge-argocd-control-plane-template" } +variable "gitops_addons_revision" { + description = "Git repository revision/branch/ref for addons" + type = string + default = "main" +} variable "gitops_addons_basepath" { description = "Git repository base path for addons" type = string @@ -18,20 +51,3 @@ variable "gitops_addons_path" { type = string default = "bootstrap/control-plane/addons" } -variable "gitops_addons_revision" { - description = "Git repository revision/branch/ref for addons" - type = string - default = "HEAD" -} - - -variable "vpc_cidr" { - description = "VPC CIDR" - default = "10.0.0.0/16" - type = string -} -variable "kubernetes_version" { - description = "EKS version" - default = "1.27" - type = string -} diff --git a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/spokes/bootstrap/workloads.yaml b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/spokes/bootstrap/workloads.yaml index 70594313..c3990393 100644 --- a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/spokes/bootstrap/workloads.yaml +++ b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/spokes/bootstrap/workloads.yaml @@ -5,7 +5,7 @@ metadata: namespace: argocd spec: syncPolicy: - preserveResourcesOnDeletion: true + preserveResourcesOnDeletion: false generators: - clusters: selector: @@ -23,7 +23,7 @@ spec: project: default source: repoURL: '{{metadata.annotations.workload_repo_url}}' - path: '{{metadata.annotations.workload_repo_path}}' + path: '{{metadata.annotations.workload_repo_basepath}}{{metadata.annotations.workload_repo_path}}' targetRevision: '{{metadata.annotations.workload_repo_revision}}' destination: namespace: 'workload' diff --git a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/spokes/destroy.sh b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/spokes/destroy.sh index c4c4ba39..b47fe45b 100755 --- a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/spokes/destroy.sh +++ b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/spokes/destroy.sh @@ -1,5 +1,11 @@ #!/bin/bash +set -uo pipefail + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +ROOTDIR="$(cd ${SCRIPTDIR}/../..; pwd )" +[[ -n "${DEBUG:-}" ]] && set -x + if [[ $# -eq 0 ]] ; then echo "No arguments supplied" @@ -9,21 +15,22 @@ if [[ $# -eq 0 ]] ; then fi env=$1 echo "Destroying $env ..." - -set -x - terraform workspace select $env + # Delete the Ingress/SVC before removing the addons TMPFILE=$(mktemp) -terraform output -raw configure_kubectl > "$TMPFILE" -source "$TMPFILE" - -terraform destroy -target="module.gitops_bridge_bootstrap_hub" -auto-approve -var-file="workspaces/${env}.tfvars" -kubectl delete svc -n argocd argo-cd-argocd-server - +terraform -chdir=$SCRIPTDIR output -raw configure_kubectl > "$TMPFILE" +# check if TMPFILE contains the string "No outputs found" +if [[ ! $(cat $TMPFILE) == *"No outputs found"* ]]; then + source "$TMPFILE" + kubectl delete -n argocd applicationset workloads + sleep 60 + terraform destroy -target="module.gitops_bridge_bootstrap_hub" -auto-approve -var-file="workspaces/${env}.tfvars" + kubectl delete svc -n argocd argo-cd-argocd-server +fi -terraform destroy -target="module.gitops_bridge_bootstrap" -auto-approve -var-file="workspaces/${env}.tfvars" -terraform destroy -target="module.eks_blueprints_addons" -auto-approve -var-file="workspaces/${env}.tfvars" -terraform destroy -target="module.eks" -auto-approve -var-file="workspaces/${env}.tfvars" -terraform destroy -target="module.vpc" -auto-approve -var-file="workspaces/${env}.tfvars" +terraform destroy -auto-approve -var-file="workspaces/${env}.tfvars" -target="module.gitops_bridge_bootstrap" -auto-approve +terraform destroy -auto-approve -var-file="workspaces/${env}.tfvars" -target="module.eks_blueprints_addons" -auto-approve +terraform destroy -auto-approve -var-file="workspaces/${env}.tfvars" -target="module.eks" -auto-approve +terraform destroy -auto-approve -var-file="workspaces/${env}.tfvars" -target="module.vpc" -auto-approve terraform destroy -auto-approve -var-file="workspaces/${env}.tfvars" diff --git a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/spokes/main.tf b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/spokes/main.tf index 5580a7c1..08a18583 100644 --- a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/spokes/main.tf +++ b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/spokes/main.tf @@ -64,11 +64,15 @@ provider "helm" { locals { - name = "hub-spoke-2-${terraform.workspace}" - environment = terraform.workspace - region = "us-west-2" - cluster_version = var.kubernetes_version - vpc_cidr = var.vpc_cidr + name = "hub-spoke-2-${terraform.workspace}" + environment = terraform.workspace + region = var.region + + cluster_version = var.kubernetes_version + + vpc_cidr = var.vpc_cidr + azs = slice(data.aws_availability_zones.available.names, 0, 3) + gitops_addons_url = "${var.gitops_addons_org}/${var.gitops_addons_repo}" gitops_addons_basepath = var.gitops_addons_basepath gitops_addons_path = var.gitops_addons_path @@ -76,46 +80,61 @@ locals { gitops_workload_org = var.gitops_workload_org gitops_workload_repo = var.gitops_workload_repo + gitops_workload_basepath = var.gitops_workload_basepath gitops_workload_path = var.gitops_workload_path gitops_workload_revision = var.gitops_workload_revision gitops_workload_url = "${local.gitops_workload_org}/${local.gitops_workload_repo}" aws_addons = { - enable_cert_manager = true - #enable_aws_efs_csi_driver = true - #enable_aws_fsx_csi_driver = true - #enable_aws_cloudwatch_metrics = true - #enable_aws_privateca_issuer = true - #enable_cluster_autoscaler = true - #enable_external_dns = true - #enable_external_secrets = true - enable_aws_load_balancer_controller = true - #enable_fargate_fluentbit = true - #enable_aws_for_fluentbit = true - #enable_aws_node_termination_handler = true - #enable_karpenter = true - #enable_velero = true - #enable_aws_gateway_api_controller = true - #enable_aws_ebs_csi_resources = true # generate gp2 and gp3 storage classes for ebs-csi - #enable_aws_secrets_store_csi_driver_provider = true + enable_cert_manager = try(var.addons.enable_cert_manager, false) + enable_aws_efs_csi_driver = try(var.addons.enable_aws_efs_csi_driver, false) + enable_aws_fsx_csi_driver = try(var.addons.enable_aws_fsx_csi_driver, false) + enable_aws_cloudwatch_metrics = try(var.addons.enable_aws_cloudwatch_metrics, false) + enable_aws_privateca_issuer = try(var.addons.enable_aws_privateca_issuer, false) + enable_cluster_autoscaler = try(var.addons.enable_cluster_autoscaler, false) + enable_external_dns = try(var.addons.enable_external_dns, false) + enable_external_secrets = try(var.addons.enable_external_secrets, false) + enable_aws_load_balancer_controller = try(var.addons.enable_aws_load_balancer_controller, false) + enable_fargate_fluentbit = try(var.addons.enable_fargate_fluentbit, false) + enable_aws_for_fluentbit = try(var.addons.enable_aws_for_fluentbit, false) + enable_aws_node_termination_handler = try(var.addons.enable_aws_node_termination_handler, false) + enable_karpenter = try(var.addons.enable_karpenter, false) + enable_velero = try(var.addons.enable_velero, false) + enable_aws_gateway_api_controller = try(var.addons.enable_aws_gateway_api_controller, false) + enable_aws_ebs_csi_resources = try(var.addons.enable_aws_ebs_csi_resources, false) + enable_aws_secrets_store_csi_driver_provider = try(var.addons.enable_aws_secrets_store_csi_driver_provider, false) + enable_ack_apigatewayv2 = try(var.addons.enable_ack_apigatewayv2, false) + enable_ack_dynamodb = try(var.addons.enable_ack_dynamodb, false) + enable_ack_s3 = try(var.addons.enable_ack_s3, false) + enable_ack_rds = try(var.addons.enable_ack_rds, false) + enable_ack_prometheusservice = try(var.addons.enable_ack_prometheusservice, false) + enable_ack_emrcontainers = try(var.addons.enable_ack_emrcontainers, false) + enable_ack_sfn = try(var.addons.enable_ack_sfn, false) + enable_ack_eventbridge = try(var.addons.enable_ack_eventbridge, false) + enable_aws_argocd = try(var.addons.enable_aws_argocd, false) } oss_addons = { - #enable_argo_rollouts = true - #enable_argo_events = true - #enable_argo_workflows = true - #enable_cluster_proportional_autoscaler = true - #enable_gatekeeper = true - #enable_gpu_operator = true - #enable_ingress_nginx = true - #enable_kyverno = true - #enable_kube_prometheus_stack = true - enable_metrics_server = true - #enable_prometheus_adapter = true - #enable_secrets_store_csi_driver = true - #enable_vpa = true - #enable_foo = true # you can add any addon here, make sure to update the gitops repo with the corresponding application set + enable_argocd = try(var.addons.enable_argocd, false) + enable_argo_rollouts = try(var.addons.enable_argo_rollouts, false) + enable_argo_events = try(var.addons.enable_argo_events, false) + enable_argo_workflows = try(var.addons.enable_argo_workflows, false) + enable_cluster_proportional_autoscaler = try(var.addons.enable_cluster_proportional_autoscaler, false) + enable_gatekeeper = try(var.addons.enable_gatekeeper, false) + enable_gpu_operator = try(var.addons.enable_gpu_operator, false) + enable_ingress_nginx = try(var.addons.enable_ingress_nginx, false) + enable_kyverno = try(var.addons.enable_kyverno, false) + enable_kube_prometheus_stack = try(var.addons.enable_kube_prometheus_stack, false) + enable_metrics_server = try(var.addons.enable_metrics_server, false) + enable_prometheus_adapter = try(var.addons.enable_prometheus_adapter, false) + enable_secrets_store_csi_driver = try(var.addons.enable_secrets_store_csi_driver, false) + enable_vpa = try(var.addons.enable_vpa, false) } - addons = merge(local.aws_addons, local.oss_addons, { kubernetes_version = local.cluster_version }, { aws_cluster_name = module.eks.cluster_name }) + addons = merge( + local.aws_addons, + local.oss_addons, + { kubernetes_version = local.cluster_version }, + { aws_cluster_name = module.eks.cluster_name } + ) addons_metadata = merge( module.eks_blueprints_addons.gitops_metadata, @@ -142,6 +161,7 @@ locals { }, { workload_repo_url = local.gitops_workload_url + workload_repo_basepath = local.gitops_workload_basepath workload_repo_path = local.gitops_workload_path workload_repo_revision = local.gitops_workload_revision } @@ -151,11 +171,9 @@ locals { workloads = file("${path.module}/bootstrap/workloads.yaml") } - azs = slice(data.aws_availability_zones.available.names, 0, 3) - tags = { Blueprint = local.name - GithubRepo = "github.com/csantanapr/terraform-gitops-bridge" + GithubRepo = "github.com/gitops-bridge-dev/gitops-bridge" } } @@ -255,21 +273,21 @@ module "eks_blueprints_addons" { create_kubernetes_resources = false # EKS Blueprints Addons - enable_cert_manager = try(local.aws_addons.enable_cert_manager, false) - enable_aws_efs_csi_driver = try(local.aws_addons.enable_aws_efs_csi_driver, false) - enable_aws_fsx_csi_driver = try(local.aws_addons.enable_aws_fsx_csi_driver, false) - enable_aws_cloudwatch_metrics = try(local.aws_addons.enable_aws_cloudwatch_metrics, false) - enable_aws_privateca_issuer = try(local.aws_addons.enable_aws_privateca_issuer, false) - enable_cluster_autoscaler = try(local.aws_addons.enable_cluster_autoscaler, false) - enable_external_dns = try(local.aws_addons.enable_external_dns, false) - enable_external_secrets = try(local.aws_addons.enable_external_secrets, false) - enable_aws_load_balancer_controller = try(local.aws_addons.enable_aws_load_balancer_controller, false) - enable_fargate_fluentbit = try(local.aws_addons.enable_fargate_fluentbit, false) - enable_aws_for_fluentbit = try(local.aws_addons.enable_aws_for_fluentbit, false) - enable_aws_node_termination_handler = try(local.aws_addons.enable_aws_node_termination_handler, false) - enable_karpenter = try(local.aws_addons.enable_karpenter, false) - enable_velero = try(local.aws_addons.enable_velero, false) - enable_aws_gateway_api_controller = try(local.aws_addons.enable_aws_gateway_api_controller, false) + enable_cert_manager = local.aws_addons.enable_cert_manager + enable_aws_efs_csi_driver = local.aws_addons.enable_aws_efs_csi_driver + enable_aws_fsx_csi_driver = local.aws_addons.enable_aws_fsx_csi_driver + enable_aws_cloudwatch_metrics = local.aws_addons.enable_aws_cloudwatch_metrics + enable_aws_privateca_issuer = local.aws_addons.enable_aws_privateca_issuer + enable_cluster_autoscaler = local.aws_addons.enable_cluster_autoscaler + enable_external_dns = local.aws_addons.enable_external_dns + enable_external_secrets = local.aws_addons.enable_external_secrets + enable_aws_load_balancer_controller = local.aws_addons.enable_aws_load_balancer_controller + enable_fargate_fluentbit = local.aws_addons.enable_fargate_fluentbit + enable_aws_for_fluentbit = local.aws_addons.enable_aws_for_fluentbit + enable_aws_node_termination_handler = local.aws_addons.enable_aws_node_termination_handler + enable_karpenter = local.aws_addons.enable_karpenter + enable_velero = local.aws_addons.enable_velero + enable_aws_gateway_api_controller = local.aws_addons.enable_aws_gateway_api_controller tags = local.tags } diff --git a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/spokes/variables.tf b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/spokes/variables.tf index e16ff73d..57032511 100644 --- a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/spokes/variables.tf +++ b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/spokes/variables.tf @@ -1,3 +1,20 @@ +variable "region" { + description = "AWS region" + type = string +} +variable "vpc_cidr" { + description = "VPC CIDR" + type = string +} +variable "kubernetes_version" { + description = "EKS version" + type = string +} +variable "addons" { + description = "Kubernetes addons" + type = any +} +# Addons Git variable "gitops_addons_org" { description = "Git repository org/user contains for addons" type = string @@ -8,6 +25,11 @@ variable "gitops_addons_repo" { type = string default = "gitops-bridge-argocd-control-plane-template" } +variable "gitops_addons_revision" { + description = "Git repository revision/branch/ref for addons" + type = string + default = "main" +} variable "gitops_addons_basepath" { description = "Git repository base path for addons" type = string @@ -18,12 +40,8 @@ variable "gitops_addons_path" { type = string default = "bootstrap/control-plane/addons" } -variable "gitops_addons_revision" { - description = "Git repository revision/branch/ref for addons" - type = string - default = "HEAD" -} +# Workloads Git variable "gitops_workload_org" { description = "Git repository org/user contains for workload" type = string @@ -34,23 +52,18 @@ variable "gitops_workload_repo" { type = string default = "argocd-example-apps" } -variable "gitops_workload_path" { - description = "Git repository path for workload" - type = string - default = "helm-guestbook" -} variable "gitops_workload_revision" { description = "Git repository revision/branch/ref for workload" type = string - default = "HEAD" + default = "master" } - - -variable "vpc_cidr" { - description = "VPC CIDR" +variable "gitops_workload_basepath" { + description = "Git repository base path for workload" type = string + default = "" } -variable "kubernetes_version" { - description = "EKS version" +variable "gitops_workload_path" { + description = "Git repository path for workload" type = string + default = "helm-guestbook" } diff --git a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/spokes/workspaces/dev.tfvars b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/spokes/workspaces/dev.tfvars index 8c1910fc..abb8f567 100644 --- a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/spokes/workspaces/dev.tfvars +++ b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/spokes/workspaces/dev.tfvars @@ -1,2 +1,9 @@ vpc_cidr = "10.1.0.0/16" -kubernetes_version = "1.27" \ No newline at end of file +region = "us-west-2" +kubernetes_version = "1.28" +addons = { + enable_aws_load_balancer_controller = true + enable_metrics_server = true + # Enable argocd on spoke clusters only for workloads, addons are deployed by hub cluster + enable_argocd = true +} \ No newline at end of file diff --git a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/spokes/workspaces/prod.tfvars b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/spokes/workspaces/prod.tfvars index 6301efd5..c80c60ba 100644 --- a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/spokes/workspaces/prod.tfvars +++ b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/spokes/workspaces/prod.tfvars @@ -1,2 +1,9 @@ vpc_cidr = "10.3.0.0/16" -kubernetes_version = "1.27" \ No newline at end of file +region = "us-west-2" +kubernetes_version = "1.28" +addons = { + enable_aws_load_balancer_controller = true + enable_metrics_server = true + # Enable argocd on spoke clusters only for workloads, addons are deployed by hub cluster + enable_argocd = true +} \ No newline at end of file diff --git a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/spokes/workspaces/staging.tfvars b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/spokes/workspaces/staging.tfvars index a7dc928c..0460d433 100644 --- a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/spokes/workspaces/staging.tfvars +++ b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared/spokes/workspaces/staging.tfvars @@ -1,2 +1,9 @@ vpc_cidr = "10.2.0.0/16" -kubernetes_version = "1.27" \ No newline at end of file +region = "us-west-2" +kubernetes_version = "1.28" +addons = { + enable_aws_load_balancer_controller = true + enable_metrics_server = true + # Enable argocd on spoke clusters only for workloads, addons are deployed by hub cluster + enable_argocd = true +} \ No newline at end of file diff --git a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/README.md b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/README.md index 47410d7e..9db69636 100644 --- a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/README.md +++ b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/README.md @@ -60,8 +60,6 @@ The output should match the `arn` for the IAM Role that will assume the IAM Role "arn:aws:iam::0123456789:role/hub-spoke-control-plane-argocd-hub" ``` -kubectl get secret -n argocd hub-spoke-control-plane --template='{{index .data.config | base64decode}}' - ## Deploy the Spoke EKS Cluster Initialize Terraform and deploy the EKS clusters: ```shell diff --git a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/hub/destroy.sh b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/hub/destroy.sh index 195f9885..8ad29aae 100755 --- a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/hub/destroy.sh +++ b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/hub/destroy.sh @@ -1,13 +1,22 @@ #!/bin/bash -set -x +set -uo pipefail + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +ROOTDIR="$(cd ${SCRIPTDIR}/../..; pwd )" +[[ -n "${DEBUG:-}" ]] && set -x # Delete the Ingress/SVC before removing the addons TMPFILE=$(mktemp) -terraform output -raw configure_kubectl > "$TMPFILE" -source "$TMPFILE" - -kubectl delete svc -n argocd argo-cd-argocd-server +terraform -chdir=$SCRIPTDIR output -raw configure_kubectl > "$TMPFILE" +# check if TMPFILE contains the string "No outputs found" +if [[ ! $(cat $TMPFILE) == *"No outputs found"* ]]; then + source "$TMPFILE" + kubectl delete -n argocd applicationset workloads + kubectl delete -n argocd applicationset cluster-addons + kubectl delete -n argocd applicationset addons-argocd + kubectl delete -n argocd svc argo-cd-argocd-server +fi terraform destroy -target="module.gitops_bridge_bootstrap" -auto-approve terraform destroy -target="module.eks_blueprints_addons" -auto-approve diff --git a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/spokes/destroy.sh b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/spokes/destroy.sh index 88fc8f55..cc2a333f 100755 --- a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/spokes/destroy.sh +++ b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/spokes/destroy.sh @@ -17,7 +17,6 @@ env=$1 echo "Destroying $env ..." terraform workspace select $env -terraform workspace select $env terraform destroy -auto-approve -var-file="workspaces/${env}.tfvars" -target="module.gitops_bridge_bootstrap" -auto-approve terraform destroy -auto-approve -var-file="workspaces/${env}.tfvars" -target="module.eks_blueprints_addons" -auto-approve terraform destroy -auto-approve -var-file="workspaces/${env}.tfvars" -target="module.eks" -auto-approve diff --git a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/spokes/main.tf b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/spokes/main.tf index 018a363f..28b328e5 100644 --- a/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/spokes/main.tf +++ b/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke/spokes/main.tf @@ -30,7 +30,6 @@ provider "kubernetes" { alias = "hub" } - ################################################################################ # Kubernetes Access for Spoke Cluster ################################################################################ @@ -59,7 +58,6 @@ locals { vpc_cidr = var.vpc_cidr azs = slice(data.aws_availability_zones.available.names, 0, 3) - gitops_addons_url = "${var.gitops_addons_org}/${var.gitops_addons_repo}" gitops_addons_basepath = var.gitops_addons_basepath gitops_addons_path = var.gitops_addons_path