Fix IstioD causing the Cortex cluster to no longer be reachable (#2342)

Author: RobertLucian

cli/cmd/cluster.go

@@ -358,7 +358,7 @@ var _clusterConfigureCmd = &cobra.Command{
 			exit.Error(err)
 		}
 
-		k8sClient, err := k8s.New("default", false, restConfig, scheme)
+		k8sClient, err := k8s.New(consts.DefaultNamespace, false, restConfig, scheme)
 		if err != nil {
 			exit.Error(err)
 		}
@@ -804,7 +804,7 @@ var _clusterHealthCmd = &cobra.Command{
 			exit.Error(err)
 		}
 
-		k8sClient, err := k8s.New("default", false, restConfig, scheme)
+		k8sClient, err := k8s.New(consts.DefaultNamespace, false, restConfig, scheme)
 		if err != nil {
 			exit.Error(err)
 		}

manager/install.sh

@@ -36,6 +36,7 @@ function cluster_up() {
   create_eks
 
   echo -n "￮ updating cluster configuration "
+  setup_namespaces
   setup_configmap
   echo "✓"
 
@@ -195,6 +196,12 @@ function write_kubeconfig() {
   out=$(kubectl get pods 2>&1 || true); if [[ "$out" == *"must be logged in to the server"* ]]; then echo "error: your aws iam user does not have access to this cluster; to grant access, see https://docs.cortex.dev/v/${CORTEX_VERSION_MINOR}/"; exit 1; fi
 }
 
+function setup_namespaces() {
+  # doing a patch to prevent getting the kubectl.kubernetes.io/last-applied-configuration annotation warning
+  kubectl patch namespace default -p '{"metadata": {"labels": {"istio-discovery": "enabled"}}}' >/dev/null
+  kubectl apply -f manifests/namespaces.yaml >/dev/null
+}
+
 function setup_configmap() {
   envsubst < manifests/default_cortex_cli_config.yaml > tmp_cli_config.yaml
   kubectl -n=default create configmap 'client-config' \
@@ -227,7 +234,9 @@ function setup_prometheus() {
   envsubst < manifests/prometheus-node-exporter.yaml | kubectl apply -f - >/dev/null
   envsubst < manifests/prometheus-monitoring.yaml | kubectl apply -f - >/dev/null
   python render_template.py $CORTEX_CLUSTER_CONFIG_FILE manifests/prometheus-additional-scrape-configs.yaml.j2 > prometheus-additional-scrape-configs.yaml
-  kubectl create secret generic additional-scrape-configs --from-file=prometheus-additional-scrape-configs.yaml
+  if ! kubectl get secret -n prometheus additional-scrape-configs >/dev/null 2>&1; then
+    kubectl create secret generic -n prometheus additional-scrape-configs --from-file=prometheus-additional-scrape-configs.yaml > /dev/null
+  fi
 }
 
 function setup_grafana() {
@@ -360,8 +369,6 @@ function remove_nodegroups() {
 }
 
 function setup_istio() {
-  envsubst < manifests/istio-namespace.yaml | kubectl apply -f - >/dev/null
-
   if ! grep -q "istio-customgateway-certs" <<< $(kubectl get secret -n istio-system); then
     WEBSITE=localhost
     openssl req -subj "/C=US/CN=$WEBSITE" -newkey rsa:2048 -nodes -keyout $WEBSITE.key -x509 -days 3650 -out $WEBSITE.crt >/dev/null 2>&1
@@ -530,8 +537,8 @@ function validate_cortex() {
     fi
 
     if [ "$prometheus_ready" == "" ]; then
-      readyReplicas=$(kubectl get statefulset -n default prometheus-prometheus -o jsonpath='{.status.readyReplicas}' 2> /dev/null)
-      desiredReplicas=$(kubectl get statefulset -n default prometheus-prometheus -o jsonpath='{.status.replicas}' 2> /dev/null)
+      readyReplicas=$(kubectl get statefulset -n prometheus prometheus-prometheus -o jsonpath='{.status.readyReplicas}' 2> /dev/null)
+      desiredReplicas=$(kubectl get statefulset -n prometheus prometheus-prometheus -o jsonpath='{.status.replicas}' 2> /dev/null)
 
       if [ "$readyReplicas" != "" ] && [ "$desiredReplicas" != "" ]; then
         if [ "$readyReplicas" == "$desiredReplicas" ]; then

manager/manifests/autoscaler.yaml.j2

@@ -82,7 +82,7 @@ spec:
           args:
             - "--in-cluster"
             - "--port=8000"
-            - "--prometheus-url=http://prometheus.default:9090"
+            - "--prometheus-url=http://prometheus.prometheus:9090"
             - "--namespace=default"
             - "--cluster-config=/configs/cluster/cluster.yaml"
           ports:

manager/manifests/event-exporter.yaml

@@ -15,7 +15,7 @@
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  namespace: default
+  namespace: logging
   name: event-exporter
 
 ---
@@ -30,7 +30,7 @@ roleRef:
   name: view
 subjects:
   - kind: ServiceAccount
-    namespace: default
+    namespace: logging
     name: event-exporter
 
 ---
@@ -39,7 +39,7 @@ apiVersion: v1
 kind: ConfigMap
 metadata:
   name: event-exporter-config
-  namespace: default
+  namespace: logging
 data:
   config.yaml: |
     logLevel: error
@@ -61,7 +61,7 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: event-exporter
-  namespace: default
+  namespace: logging
 spec:
   replicas: 1
   selector:

manager/manifests/fluent-bit.yaml.j2

@@ -16,7 +16,7 @@ apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: fluent-bit
-  namespace: default
+  namespace: logging
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
@@ -40,13 +40,13 @@ roleRef:
 subjects:
   - kind: ServiceAccount
     name: fluent-bit
-    namespace: default
+    namespace: logging
 ---
 apiVersion: v1
 kind: ConfigMap
 metadata:
   name: fluent-bit-config
-  namespace: default
+  namespace: logging
   labels:
     k8s-app: fluent-bit
 data:
@@ -186,7 +186,7 @@ apiVersion: apps/v1
 kind: DaemonSet
 metadata:
   name: fluent-bit
-  namespace: default
+  namespace: logging
 spec:
   selector:
     matchLabels:

manager/manifests/grafana/grafana.yaml.j2

@@ -28,7 +28,7 @@ data:
                 "name": "prometheus",
                 "orgId": 1,
                 "type": "prometheus",
-                "url": "http://prometheus.default:9090",
+                "url": "http://prometheus.prometheus:9090",
                 "version": 1,
                 "isDefault": true
             }

manager/manifests/istio.yaml.j2

@@ -18,6 +18,10 @@ spec:
   profile: minimal
   hub: {{ env['CORTEX_IMAGE_ISTIO_PROXY_HUB'] }}  # this is only used by proxy, since pilot overrides it (proxy doesn't have dedicated hub config)
   tag: {{ env['CORTEX_IMAGE_ISTIO_PROXY_TAG'] }}  # this is only used by proxy, since pilot overrides it (proxy doesn't have dedicated tag config)
+  meshConfig:
+    discoverySelectors:
+      - matchLabels:
+          istio-discovery: enabled
   components:
     pilot:  # "pilot" refers to the istiod container
       hub: {{ env['CORTEX_IMAGE_ISTIO_PILOT_HUB'] }}
@@ -26,7 +30,23 @@ spec:
         resources:
           requests:
             cpu: 100m  # default is 500m
-            memory: 200Mi  # default is 2048Mi == 2Gi
+            memory: 700Mi  # default is 2048Mi == 2Gi
+        hpaSpec:
+          minReplicas: 1
+          maxReplicas: 5
+          metrics:
+            - type: Resource
+              resource:
+                name: cpu
+                targetAverageUtilization: 90
+            - type: Resource
+              resource:
+                name: memory
+                targetAverageUtilization: 90
+          scaleTargetRef:
+            apiVersion: apps/v1
+            kind: Deployment
+            name: istiod
     cni:
       enabled: false
     ingressGateways:
@@ -71,7 +91,7 @@ spec:
           replicaCount: 1
           hpaSpec:
             minReplicas: 1
-            maxReplicas: 1  # edit autoscaleEnabled in values if increasing this
+            maxReplicas: 1
             metrics:
               - type: Resource
                 resource:
@@ -124,7 +144,7 @@ spec:
           replicaCount: 1
           hpaSpec:
             minReplicas: 1
-            maxReplicas: 100  # edit autoscaleEnabled in values if increasing this
+            maxReplicas: 100
             metrics:
               - type: Resource
                 resource:

manager/manifests/istio-namespace.yaml renamed to manager/manifests/namespaces.yaml

@@ -16,3 +16,16 @@ apiVersion: v1
 kind: Namespace
 metadata:
   name: istio-system
+---
+
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: logging
+---
+
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: prometheus
+---

manager/manifests/prometheus-dcgm-exporter.yaml

@@ -12,16 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: monitoring
----
 apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: dcgm-exporter
-  namespace: default
+  namespace: prometheus
   labels:
     app.kubernetes.io/name: dcgm-exporter
     app.kubernetes.io/instance: dcgm-exporter
@@ -31,7 +26,7 @@ apiVersion: apps/v1
 kind: DaemonSet
 metadata:
   name: dcgm-exporter
-  namespace: default
+  namespace: prometheus
   labels:
     app.kubernetes.io/name: dcgm-exporter
     app.kubernetes.io/instance: dcgm-exporter
@@ -106,7 +101,7 @@ apiVersion: monitoring.coreos.com/v1
 kind: PodMonitor
 metadata:
   name: dcgm-exporter
-  namespace: default
+  namespace: prometheus
   labels:
     monitoring.cortex.dev: dcgm-exporter
     app.kubernetes.io/name: dcgm-exporter

manager/manifests/prometheus-kube-state-metrics.yaml

@@ -17,7 +17,7 @@ metadata:
   labels:
     app.kubernetes.io/name: kube-state-metrics
   name: kube-state-metrics
-  namespace: default
+  namespace: prometheus
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
@@ -180,13 +180,13 @@ roleRef:
 subjects:
 - kind: ServiceAccount
   name: kube-state-metrics
-  namespace: default
+  namespace: prometheus
 ---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: kube-state-metrics
-  namespace: default
+  namespace: prometheus
   labels:
     app.kubernetes.io/name: kube-state-metrics
     app.kubernetes.io/version: "2.1.0"
@@ -245,7 +245,7 @@ apiVersion: monitoring.coreos.com/v1
 kind: PodMonitor
 metadata:
   name: kube-state-metrics
-  namespace: default
+  namespace: prometheus
   labels:
     name: kube-state-metrics
     monitoring.cortex.dev: kube-state-metrics

manager/manifests/prometheus-kubelet-exporter.yaml

@@ -19,7 +19,7 @@ metadata:
     k8s-app: kubelet
     monitoring.cortex.dev: kubelet-exporter
   name: kubelet
-  namespace: default
+  namespace: prometheus
 spec:
   endpoints:
   - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token

manager/manifests/prometheus-monitoring.yaml

@@ -27,6 +27,7 @@ apiVersion: monitoring.coreos.com/v1
 kind: Prometheus
 metadata:
   name: prometheus
+  namespace: prometheus
 spec:
   image: $CORTEX_IMAGE_PROMETHEUS
   serviceAccountName: prometheus
@@ -73,6 +74,7 @@ apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: prometheus
+  namespace: prometheus
 
 ---
 
@@ -114,14 +116,15 @@ roleRef:
 subjects:
   - kind: ServiceAccount
     name: prometheus
-    namespace: default
+    namespace: prometheus
 
 ---
 
 apiVersion: v1
 kind: Service
 metadata:
   name: prometheus
+  namespace: prometheus
 spec:
   type: ClusterIP
   ports:
@@ -136,6 +139,7 @@ apiVersion: monitoring.coreos.com/v1
 kind: PodMonitor
 metadata:
   name: istio-stats
+  namespace: prometheus
   labels:
     monitoring.cortex.dev: "istio"
 spec:
@@ -187,6 +191,7 @@ apiVersion: monitoring.coreos.com/v1
 kind: PodMonitor
 metadata:
   name: proxy-stats
+  namespace: prometheus
   labels:
     monitoring.cortex.dev: "proxy"
 spec:
@@ -240,6 +245,7 @@ apiVersion: monitoring.coreos.com/v1
 kind: PodMonitor
 metadata:
   name: async-stats
+  namespace: prometheus
   labels:
     monitoring.cortex.dev: "dequeuer-async"
 spec:
@@ -294,6 +300,7 @@ apiVersion: monitoring.coreos.com/v1
 kind: PodMonitor
 metadata:
   name: prometheus-statsd-exporter
+  namespace: prometheus
   labels:
     name: prometheus-statsd-exporter
     monitoring.cortex.dev: "statsd-exporter"
@@ -320,6 +327,7 @@ apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
   name: operator
+  namespace: prometheus
   labels:
     name: operator
     monitoring.cortex.dev: "operator"