diff --git a/operations/observability/mixins/platform/mixin.libsonnet b/operations/observability/mixins/platform/mixin.libsonnet index e4f7fcf7b22a85..acb307a7030d9f 100644 --- a/operations/observability/mixins/platform/mixin.libsonnet +++ b/operations/observability/mixins/platform/mixin.libsonnet @@ -3,4 +3,5 @@ * Licensed under the MIT License. See License-MIT.txt in the project root for license information. */ -(import './dashboards.libsonnet') +(import './dashboards.libsonnet') + +(import './rules.libsonnet') diff --git a/operations/observability/mixins/platform/rules.libsonnet b/operations/observability/mixins/platform/rules.libsonnet new file mode 100644 index 00000000000000..826def1bbcd0f2 --- /dev/null +++ b/operations/observability/mixins/platform/rules.libsonnet @@ -0,0 +1,6 @@ +/** + * Copyright (c) 2022 Gitpod GmbH. All rights reserved. + * Licensed under the MIT License. See License-MIT.txt in the project root for license information. + */ + +(import './rules/scrape.libsonnet') diff --git a/operations/observability/mixins/platform/rules/scrape.libsonnet b/operations/observability/mixins/platform/rules/scrape.libsonnet new file mode 100644 index 00000000000000..80f9b72b5f7067 --- /dev/null +++ b/operations/observability/mixins/platform/rules/scrape.libsonnet @@ -0,0 +1,29 @@ +/** + * Copyright (c) 2022 Gitpod GmbH. All rights reserved. + * Licensed under the MIT License. See License-MIT.txt in the project root for license information. + */ + +{ + prometheusAlerts+:: { + groups+: [ + { + name: 'prometheus-scraping-rules', + rules: [ + { + alert: 'TargetDownOrFailedScrape', + labels: { + severity: 'warning', + team: 'platform', + }, + 'for': '10m', + annotations: { + summary: 'Prometheus failed to scrape {{ $labels.job }}', + description: 'Prometheus couldn\'t scrape {{ printf "%.4g" $value }}% of the {{ $labels.job }} targets. Components could be unnavailable or we have some scraping misconfiguration.', + }, + expr: '100 * (count(up{container!="workspace"} == 0) BY (job) / count(up{container!="workspace"}) BY (job)) > 0', + }, + ], + }, + ], + }, +}