adjust recording rules to use kv_name (#86)

jtlisi · web-flow · commit 59dc3354a688 · 2020-06-04T09:58:26.000-04:00
Signed-off-by: Jacob Lisi &lt;jacob.t.lisi@gmail.com&gt;
diff --git a/cortex-mixin/dashboards/writes.libsonnet b/cortex-mixin/dashboards/writes.libsonnet
@@ -56,11 +56,11 @@ local utils = import 'mixin-utils/utils.libsonnet';
       $.row('KV Store (HA Dedupe)')
       .addPanel(
         $.panel('QPS') +
-        $.qpsPanel('cortex_kv_request_duration_seconds_count{%s}' % $.jobMatcher($._config.job_names.distributor))
+        $.qpsPanel('cortex_kv_request_duration_seconds_count{%s, kv_name="distributor-hatracker"}' % $.jobMatcher($._config.job_names.distributor))
       )
       .addPanel(
         $.panel('Latency') +
-        utils.latencyRecordingRulePanel('cortex_kv_request_duration_seconds', $.jobSelector($._config.job_names.distributor))
+        utils.latencyRecordingRulePanel('cortex_kv_request_duration_seconds', $.jobSelector($._config.job_names.distributor) + [utils.selector.eq('kv_name', 'distributor-hatracker')])
       )
     )
     .addRow(
@@ -78,11 +78,11 @@ local utils = import 'mixin-utils/utils.libsonnet';
       $.row('KV Store (Ring)')
       .addPanel(
         $.panel('QPS') +
-        $.qpsPanel('cortex_kv_request_duration_seconds_count{%s}' % $.jobMatcher($._config.job_names.ingester))
+        $.qpsPanel('cortex_kv_request_duration_seconds_count{%s, kv_name="ingester-lifecycler"}' % $.jobMatcher($._config.job_names.ingester))
       )
       .addPanel(
         $.panel('Latency') +
-        utils.latencyRecordingRulePanel('cortex_kv_request_duration_seconds', $.jobSelector($._config.job_names.ingester))
+        utils.latencyRecordingRulePanel('cortex_kv_request_duration_seconds', $.jobSelector($._config.job_names.ingester) + [utils.selector.eq('kv_name', 'ingester-lifecycler')])
       )
     )
     .addRowIf(
diff --git a/cortex-mixin/recording_rules.libsonnet b/cortex-mixin/recording_rules.libsonnet
@@ -29,7 +29,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
           utils.histogramRules('cortex_chunk_store_chunks_per_query', ['cluster', 'job']) +
           utils.histogramRules('cortex_database_request_duration_seconds', ['cluster', 'job', 'method']) +
           utils.histogramRules('cortex_gcs_request_duration_seconds', ['cluster', 'job', 'operation']) +
-          utils.histogramRules('cortex_kv_request_duration_seconds', ['cluster', 'job']),
+          utils.histogramRules('cortex_kv_request_duration_seconds', ['cluster', 'job', 'kv_name']),
       },
       {
         name: 'cortex_queries',

Original file line number	Diff line number	Diff line change
`@@ -56,11 +56,11 @@ local utils = import 'mixin-utils/utils.libsonnet';`
`56`	`56`	`$.row('KV Store (HA Dedupe)')`
`57`	`57`	`.addPanel(`
`58`	`58`	`$.panel('QPS') +`
`59`		`- $.qpsPanel('cortex_kv_request_duration_seconds_count{%s}' % $.jobMatcher($._config.job_names.distributor))`
	`59`	`+ $.qpsPanel('cortex_kv_request_duration_seconds_count{%s, kv_name="distributor-hatracker"}' % $.jobMatcher($._config.job_names.distributor))`
`60`	`60`	`)`
`61`	`61`	`.addPanel(`
`62`	`62`	`$.panel('Latency') +`
`63`		`- utils.latencyRecordingRulePanel('cortex_kv_request_duration_seconds', $.jobSelector($._config.job_names.distributor))`
	`63`	`+ utils.latencyRecordingRulePanel('cortex_kv_request_duration_seconds', $.jobSelector($._config.job_names.distributor) + [utils.selector.eq('kv_name', 'distributor-hatracker')])`
`64`	`64`	`)`
`65`	`65`	`)`
`66`	`66`	`.addRow(`
`@@ -78,11 +78,11 @@ local utils = import 'mixin-utils/utils.libsonnet';`
`78`	`78`	`$.row('KV Store (Ring)')`
`79`	`79`	`.addPanel(`
`80`	`80`	`$.panel('QPS') +`
`81`		`- $.qpsPanel('cortex_kv_request_duration_seconds_count{%s}' % $.jobMatcher($._config.job_names.ingester))`
	`81`	`+ $.qpsPanel('cortex_kv_request_duration_seconds_count{%s, kv_name="ingester-lifecycler"}' % $.jobMatcher($._config.job_names.ingester))`
`82`	`82`	`)`
`83`	`83`	`.addPanel(`
`84`	`84`	`$.panel('Latency') +`
`85`		`- utils.latencyRecordingRulePanel('cortex_kv_request_duration_seconds', $.jobSelector($._config.job_names.ingester))`
	`85`	`+ utils.latencyRecordingRulePanel('cortex_kv_request_duration_seconds', $.jobSelector($._config.job_names.ingester) + [utils.selector.eq('kv_name', 'ingester-lifecycler')])`
`86`	`86`	`)`
`87`	`87`	`)`
`88`	`88`	`.addRowIf(`
Original file line number	Diff line number	Diff line change
`@@ -29,7 +29,7 @@ local utils = import 'mixin-utils/utils.libsonnet';`
`29`	`29`	`utils.histogramRules('cortex_chunk_store_chunks_per_query', ['cluster', 'job']) +`
`30`	`30`	`utils.histogramRules('cortex_database_request_duration_seconds', ['cluster', 'job', 'method']) +`
`31`	`31`	`utils.histogramRules('cortex_gcs_request_duration_seconds', ['cluster', 'job', 'operation']) +`
`32`		`- utils.histogramRules('cortex_kv_request_duration_seconds', ['cluster', 'job']),`
	`32`	`+ utils.histogramRules('cortex_kv_request_duration_seconds', ['cluster', 'job', 'kv_name']),`
`33`	`33`	`},`
`34`	`34`	`{`
`35`	`35`	`name: 'cortex_queries',`