Merge pull request #88 from grafana/improve-compactor-alerts-and-dashboard

Improved Cortex blocks compactor alerts and dashboard

Author: pracucci

cortex-mixin/alerts/compactor.libsonnet

@@ -4,33 +4,33 @@
       name: 'cortex_compactor_alerts',
       rules: [
         {
-          // Alert if the compactor has not successfully completed a run in the last 24h.
-          alert: 'CortexCompactorHasNotSuccessfullyRun',
+          // Alert if the compactor has not successfully cleaned up blocks in the last 24h.
+          alert: 'CortexCompactorHasNotSuccessfullyCleanedUpBlocks',
           'for': '15m',
           expr: |||
-            (time() - cortex_compactor_last_successful_run_timestamp_seconds{%s} > 60 * 60 * 24)
+            (time() - cortex_compactor_block_cleanup_last_successful_run_timestamp_seconds{%s} > 60 * 60 * 24)
             and
-            (cortex_compactor_last_successful_run_timestamp_seconds{%s} > 0)
+            (cortex_compactor_block_cleanup_last_successful_run_timestamp_seconds{%s} > 0)
           ||| % [$.namespace_matcher(''), $.namespace_matcher('')],
           labels: {
             severity: 'critical',
           },
           annotations: {
-            message: 'Cortex Compactor {{ $labels.namespace }}/{{ $labels.instance }} has not successfully completed a run in the last 24 hours.',
+            message: 'Cortex Compactor {{ $labels.namespace }}/{{ $labels.instance }} has not successfully cleaned up blocks in the last 24 hours.',
           },
         },
         {
-          // Alert if the compactor has not successfully completed a run since its start.
-          alert: 'CortexCompactorHasNotSuccessfullyRunSinceStart',
+          // Alert if the compactor has not successfully cleaned up blocks since its start.
+          alert: 'CortexCompactorHasNotSuccessfullyCleanedUpBlocksSinceStart',
           'for': '24h',
           expr: |||
-            cortex_compactor_last_successful_run_timestamp_seconds{%s} == 0
+            cortex_compactor_block_cleanup_last_successful_run_timestamp_seconds{%s} == 0
           ||| % $.namespace_matcher(''),
           labels: {
             severity: 'critical',
           },
           annotations: {
-            message: 'Cortex Compactor {{ $labels.namespace }}/{{ $labels.instance }} has not successfully completed a run in the last 24 hours.',
+            message: 'Cortex Compactor {{ $labels.namespace }}/{{ $labels.instance }} has not successfully cleaned up blocks in the last 24 hours.',
           },
         },
         {

cortex-mixin/dashboards/compactor.libsonnet

@@ -9,7 +9,8 @@ local utils = import 'mixin-utils/utils.libsonnet';
       .addPanel(
         $.textPanel('', |||
           - **Per-instance runs**: number of times a compactor instance triggers a compaction across all tenants its shard manage.
-          - **Per-tenant runs**: number of times a compactor instance triggers the compaction for a single tenant's blocks.
+          - **Compacted blocks**: number of blocks generated as a result of a compaction operation.
+          - **Per-block compaction duration**: time taken to generate a single compacted block.
         |||),
       )
       .addPanel(
@@ -22,24 +23,6 @@ local utils = import 'mixin-utils/utils.libsonnet';
         $.bars +
         { yaxes: $.yaxes('ops') },
       )
-      .addPanel(
-        $.successFailurePanel(
-          'Per-tenant runs / sec',
-          'sum(rate(cortex_compactor_group_compactions_total{%s}[$__interval])) - sum(rate(cortex_compactor_group_compactions_failures_total{%s}[$__interval]))' % [$.jobMatcher('compactor'), $.jobMatcher('compactor')],
-          'sum(rate(cortex_compactor_group_compactions_failures_total{%s}[$__interval]))' % $.jobMatcher('compactor'),
-        ) +
-        $.bars +
-        { yaxes: $.yaxes('ops') },
-      )
-    )
-    .addRow(
-      $.row('')
-      .addPanel(
-        $.textPanel('', |||
-          - **Compacted blocks**: number of blocks generated as a result of a compaction operation.
-          - **Per-block compaction duration**: time taken to generate a single compacted block.
-        |||),
-      )
       .addPanel(
         $.panel('Compacted blocks / sec') +
         $.queryPanel('sum(rate(prometheus_tsdb_compactions_total{%s}[$__interval]))' % $.jobMatcher('compactor'), 'blocks') +