Use kube-rbac-proxy to expose prometheus metric endpoint

Author: aledbf

chart/templates/_helpers.tpl

@@ -351,3 +351,29 @@ storage:
 {{ toYaml .remoteStorage | indent 2 }}
 {{- end -}}
 {{- end -}}
+
+{{- define "gitpod.kube-rbac-proxy" -}}
+- name: kube-rbac-proxy
+  image: quay.io/brancz/kube-rbac-proxy:v0.9.0
+  args:
+  - --logtostderr
+  - --insecure-listen-address=[$(IP)]:9500
+  - --upstream=http://127.0.0.1:9500/
+  env:
+  - name: IP
+    valueFrom:
+      fieldRef:
+        fieldPath: status.podIP
+  ports:
+  - containerPort: 9500
+    name: metrics
+  resources:
+    requests:
+      cpu: 1m
+      memory: 30Mi
+  securityContext:
+    runAsGroup: 65532
+    runAsNonRoot: true
+    runAsUser: 65532
+  terminationMessagePolicy: FallbackToLogsOnError
+{{- end -}}

chart/templates/blobserve-configmap.yaml

@@ -17,7 +17,7 @@ data:
     {
         {{ if .Values.components.workspace.pullSecret.secretName -}}"dockerAuth": "/mnt/pull-secret.json",{{- end }}
         "pprofAddr": ":6060",
-        "prometheusAddr": ":9500"
+        "prometheusAddr": "127.0.0.1:9500"
         , "blobserve": {
             "port": {{ $comp.ports.service.containerPort }},
             "timeout": {{ ($comp.timeout | default "5s") | quote }},

chart/templates/blobserve-deployment.yaml

@@ -71,6 +71,7 @@ spec:
           mountPath: /mnt/pull-secret.json
           subPath: .dockerconfigjson
         {{- end }}
+{{ include "gitpod.kube-rbac-proxy" $this | indent 6 }}
       volumes:
       - name: cache
         emptyDir: {}

chart/templates/blobserve-rolebinding.yaml

@@ -17,3 +17,23 @@ roleRef:
   kind: ClusterRole
   name: {{ .Release.Namespace }}-ns-psp:restricted-root-user
   apiGroup: rbac.authorization.k8s.io
+
+---
+
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: blobserve-kube-rbac-proxy
+  labels:
+    app: {{ template "gitpod.fullname" . }}
+    component: blobserve
+    kind: role-binding
+    stage: {{ .Values.installation.stage }}
+subjects:
+- kind: ServiceAccount
+  name: blobserve
+  namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name:  {{ .Release.Namespace }}-kube-rbac-proxy
+  apiGroup: rbac.authorization.k8s.io

chart/templates/blobserve-serviceaccount.yaml

@@ -10,4 +10,3 @@ metadata:
     component: blobserve
     kind: service-account
     stage: {{ .Values.installation.stage }}
-automountServiceAccountToken: false

chart/templates/image-builder-configmap.yaml

@@ -44,7 +44,7 @@ data:
             "address": ":6060"
         },
         "prometheus": {
-            "address": ":9500"
+            "address": "127.0.0.1:9500"
         },
         "service": {
             "address": ":8080"

chart/templates/image-builder-deployment.yaml

@@ -86,6 +86,7 @@ spec:
 {{- end }}
 {{ include "gitpod.container.defaultEnv" $this | indent 8 }}
 {{ include "gitpod.container.tracingEnv" $this | indent 8 }}
+{{ include "gitpod.kube-rbac-proxy" $this | indent 6 }}
       - name: service
         image: {{ template "gitpod.comp.imageFull" $this }}
         args:

chart/templates/image-builder-rolebinding.yaml

@@ -17,3 +17,23 @@ roleRef:
   kind: ClusterRole
   name: {{ .Release.Namespace }}-ns-image-builder
   apiGroup: rbac.authorization.k8s.io
+
+---
+
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: image-builder-kube-rbac-proxy
+  labels:
+    app: {{ template "gitpod.fullname" . }}
+    component: image-builder
+    kind: role-binding
+    stage: {{ .Values.installation.stage }}
+subjects:
+- kind: ServiceAccount
+  name: image-builder
+  namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name:  {{ .Release.Namespace }}-kube-rbac-proxy
+  apiGroup: rbac.authorization.k8s.io

chart/templates/image-builder-serviceaccount.yaml

@@ -10,4 +10,3 @@ metadata:
     component: image-builder
     kind: service-account
     stage: {{ .Values.installation.stage }}
-automountServiceAccountToken: false

chart/templates/kube-rbac-proxy-clusterrole.yaml

@@ -0,0 +1,16 @@
+# Copyright (c) 2021 Gitpod GmbH. All rights reserved.
+# Licensed under the MIT License. See License-MIT.txt in the project root for license information.
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ .Release.Namespace }}-kube-rbac-proxy
+rules:
+- apiGroups: ["authentication.k8s.io"]
+  resources:
+  - tokenreviews
+  verbs: ["create"]
+- apiGroups: ["authorization.k8s.io"]
+  resources:
+  - subjectaccessreviews
+  verbs: ["create"]

chart/templates/registry-facade-configmap.yaml

@@ -53,6 +53,6 @@ data:
             ]
         },
         "pprofAddr": ":6060",
-        "prometheusAddr": ":9500"
+        "prometheusAddr": "127.0.0.1:9500"
     }
 {{- end -}}

chart/templates/registry-facade-daemonset.yaml

@@ -74,6 +74,7 @@ spec:
         - name: https-certificates
           mountPath: "/mnt/certificates"
         {{- end }}
+{{ include "gitpod.kube-rbac-proxy" $this | indent 6 }}
       volumes:
       - name: cache
         emptyDir: {}

chart/templates/registry-facade-rolebinding.yaml

@@ -17,3 +17,23 @@ roleRef:
   kind: ClusterRole
   name: {{ .Release.Namespace }}-ns-registry-facade
   apiGroup: rbac.authorization.k8s.io
+
+---
+
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: registry-facade-kube-rbac-proxy
+  labels:
+    app: {{ template "gitpod.fullname" . }}
+    component: registry-facade
+    kind: role-binding
+    stage: {{ .Values.installation.stage }}
+subjects:
+- kind: ServiceAccount
+  name: registry-facade
+  namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name:  {{ .Release.Namespace }}-kube-rbac-proxy
+  apiGroup: rbac.authorization.k8s.io

chart/templates/registry-facade-serviceaccount.yaml

@@ -10,4 +10,3 @@ metadata:
     component: registry-facade
     kind: service-account
     stage: {{ .Values.installation.stage }}
-automountServiceAccountToken: false

chart/templates/ws-daemon-configmap.yaml

@@ -72,7 +72,7 @@ service:
     crt: "/certs/tls.crt"
     key: "/certs/tls.key"
 prometheus:
-  address: ":9500"
+  address: "127.0.0.1:9500"
 pprof:
   address: ":6060"
 {{ end }}

chart/templates/ws-daemon-daemonset.yaml

@@ -201,5 +201,6 @@ spec:
         securityContext:
           privileged: true
           procMount: Unmasked
+{{ include "gitpod.kube-rbac-proxy" $this | indent 6 }}
 {{ toYaml .Values.defaults | indent 6 }}
 {{ end }}

chart/templates/ws-daemon-rolebinding.yaml

@@ -17,3 +17,23 @@ roleRef:
   kind: ClusterRole
   name: {{ .Release.Namespace }}-ns-ws-daemon
   apiGroup: rbac.authorization.k8s.io
+
+---
+
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ws-daemon-rb-kube-rbac-proxy
+  labels:
+    app: {{ template "gitpod.fullname" . }}
+    component: ws-daemon
+    kind: role-binding
+    stage: {{ .Values.installation.stage }}
+subjects:
+- kind: ServiceAccount
+  name: ws-daemon
+  namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name:  {{ .Release.Namespace }}-kube-rbac-proxy
+  apiGroup: rbac.authorization.k8s.io

chart/templates/ws-manager-configmap.yaml

@@ -118,7 +118,7 @@ data:
             "addr": "localhost:6060"
         },
         "prometheus": {
-            "addr": ":9500"
+            "addr": "127.0.0.1:9500"
         }
     }
 {{- end -}}

chart/templates/ws-manager-rolebinding.yaml

@@ -17,3 +17,23 @@ roleRef:
 subjects:
 - kind: ServiceAccount
   name: ws-manager
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: ws-manager-kube-rbac-proxy
+  labels:
+    app: {{ template "gitpod.fullname" . }}
+    component: ws-manager
+    kind: role-binding
+    stage: {{ .Values.installation.stage }}
+subjects:
+- kind: ServiceAccount
+  name: ws-manager
+  namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name:  {{ .Release.Namespace }}-kube-rbac-proxy
+  apiGroup: rbac.authorization.k8s.io

chart/templates/ws-scheduler-clusterrolebinding.yaml

@@ -17,4 +17,26 @@ subjects:
 roleRef:
   kind: ClusterRole
   name: {{ .Release.Namespace }}-ns-ws-scheduler
-  apiGroup: rbac.authorization.k8s.io
+  apiGroup: rbac.authorization.k8s.io
+
+---
+
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ .Release.Namespace }}-ns-ws-scheduler-kube-rbac-proxy
+  labels:
+    app: {{ template "gitpod.fullname" . }}
+    component: ws-scheduler
+    kind: role-binding
+    stage: {{ .Values.installation.stage }}
+subjects:
+- kind: ServiceAccount
+  name: ws-scheduler
+  namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name:  {{ .Release.Namespace }}-kube-rbac-proxy
+  apiGroup: rbac.authorization.k8s.io
+
+

chart/templates/ws-scheduler-configmap.yaml

@@ -19,7 +19,7 @@ data:
             "addr": "localhost:6060"
         },
         "prometheus": {
-            "addr": ":9500"
+            "addr": "127.0.0.1:9500"
         },
         "scheduler": {
             "schedulerName": "{{ $comp.schedulerName }}",

components/image-builder/example-config.json

@@ -13,7 +13,7 @@
         "address": ":9999"
     },
     "prometheus": {
-        "address": ":9500"
+        "address": "127.0.0.1:9500"
     },
     "service": {
         "address": ":8080"

components/ws-daemon/example-config.json

@@ -28,6 +28,6 @@
       "address": ":8080"
     },
     "prometheus": {
-      "address": ":9500"
+      "address": "127.0.0.1:9500"
     }
 }