Add alert for target down

operations/observability/mixins/platform/mixin.libsonnet

@@ -3,4 +3,5 @@
  * Licensed under the MIT License. See License-MIT.txt in the project root for license information.
  */
 
-(import './dashboards.libsonnet')
+(import './dashboards.libsonnet') +
+(import './rules.libsonnet')

operations/observability/mixins/platform/rules.libsonnet

@@ -0,0 +1,6 @@
+/**
+ * Copyright (c) 2022 Gitpod GmbH. All rights reserved.
+ * Licensed under the MIT License. See License-MIT.txt in the project root for license information.
+ */
+
+(import './rules/scrape.libsonnet')

operations/observability/mixins/platform/rules/scrape.libsonnet

@@ -0,0 +1,29 @@
+/**
+ * Copyright (c) 2022 Gitpod GmbH. All rights reserved.
+ * Licensed under the MIT License. See License-MIT.txt in the project root for license information.
+ */
+
+{
+  prometheusAlerts+:: {
+    groups+: [
+      {
+        name: 'prometheus-scraping-rules',
+        rules: [
+          {
+            alert: 'TargetDownOrFailedScrape',
+            labels: {
+              severity: 'warning',
+              team: 'platform',
+            },
+            'for': '10m',
+            annotations: {
+              summary: 'Prometheus failed to scrape {{ $labels.job }}',
+              description: 'Prometheus couldn\'t scrape {{ printf "%.4g" $value }}% of the {{ $labels.job }} targets. Components could be unnavailable or we have some scraping misconfiguration.',
+            },
+            expr: '100 * (count(up{container!="workspace"} == 0) BY (job) / count(up{container!="workspace"}) BY (job)) > 0',
+          },
+        ],
+      },
+    ],
+  },
+}