diff --git a/CHANGELOG.md b/CHANGELOG.md index 49fb8712..ac0f276a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ * [FEATURE] Added "Cortex / Rollout progress" dashboard. #289 #290 * [ENHANCEMENT] Added `newCompactorStatefulSet()` function to create a custom statefulset for the compactor. #287 * [ENHANCEMENT] Added option to configure compactor job name used in dashboards and alerts. #287 +* [ENHANCEMENT] Added `CortexCompactorHasNotSuccessfullyRunCompaction` alert. #292 #294 * [BUGFIX] Fixed `CortexCompactorRunFailed` false positives. #288 ## 1.8.0 / 2021-03-25 diff --git a/cortex-mixin/alerts/compactor.libsonnet b/cortex-mixin/alerts/compactor.libsonnet index 7a858c56..be3de8c0 100644 --- a/cortex-mixin/alerts/compactor.libsonnet +++ b/cortex-mixin/alerts/compactor.libsonnet @@ -18,17 +18,33 @@ }, }, { - // Alert if the compactor has not successfully run compaction in the last 6h. + // Alert if the compactor has not successfully run compaction in the last 24h. alert: 'CortexCompactorHasNotSuccessfullyRunCompaction', 'for': '1h', expr: ||| - time() - cortex_compactor_last_successful_run_timestamp_seconds > 60 * 60 * 6 + (time() - cortex_compactor_last_successful_run_timestamp_seconds > 60 * 60 * 24) + and + (cortex_compactor_last_successful_run_timestamp_seconds > 0) + |||, + labels: { + severity: 'critical', + }, + annotations: { + message: 'Cortex Compactor {{ $labels.namespace }}/{{ $labels.instance }} has not run compaction in the last 24 hours.', + }, + }, + { + // Alert if the compactor has not successfully run compaction in the last 24h since startup. + alert: 'CortexCompactorHasNotSuccessfullyRunCompaction', + 'for': '24h', + expr: ||| + cortex_compactor_last_successful_run_timestamp_seconds == 0 |||, labels: { severity: 'critical', }, annotations: { - message: 'Cortex Compactor {{ $labels.namespace }}/{{ $labels.instance }} has not run compaction in the last 6 hours.', + message: 'Cortex Compactor {{ $labels.namespace }}/{{ $labels.instance }} has not run compaction in the last 24 hours.', }, }, {