Skip to content

Commit b605588

Browse files
justinjung04justinjung04
authored
Merge branch 'master' into block-sync-improvement
Signed-off-by: Justin Jung <[email protected]>
2 parents 1257dc5 + a307fc0 commit b605588

File tree

25 files changed

+805
-335
lines changed

25 files changed

+805
-335
lines changed

‎CHANGELOG.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
# Changelog
22

33
## master / unreleased
4+
* [CHANGE] AlertManager: include reason label in cortex_alertmanager_notifications_failed_total.#5409
45
* [CHANGE] Query: Set CORS Origin headers for Query API #5388
56
* [CHANGE] Updating prometheus/alertmanager from v0.25.0 to v0.25.1-0.20230505130626-263ca5c9438e. This includes the below changes. #5276
67
- Validating new fields on the Webhook AM config, PushOver AM Config and Telegram AM Config.
@@ -24,8 +25,11 @@
2425
* [ENHANCEMENT] Improving Performance on the API Gzip Handler. #5347
2526
* [ENHANCEMENT] Dynamodb: Add `puller-sync-time` to allow different pull time for ring. #5357
2627
* [ENHANCEMENT] Emit querier `max_concurrent` as a metric. #5362
28+
* [ENHANCEMENT] Avoid sort tokens on lifecycler autoJoin. #5394
2729
* [ENHANCEMENT] Do not resync blocks in running store gateways during rollout deployment and container restart. #5363
2830
* [ENHANCEMENT] Store Gateway: Add new metrics `cortex_bucket_store_sent_chunk_size_bytes`, `cortex_bucket_store_postings_size_bytes` and `cortex_bucket_store_empty_postings_total`. #5397
31+
* [ENHANCEMENT] Add jitter to lifecycler heartbeat. #5404
32+
* [ENHANCEMENT] Store Gateway: Add config `estimated_max_series_size_bytes` and `estimated_max_chunk_size_bytes` to address data overfetch. #5401
2933
* [ENHANCEMENT] Store Gateway: Apply WaitStabilityMinDuration when syncing blocks #5406
3034
* [BUGFIX] Ruler: Validate if rule group can be safely converted back to rule group yaml from protobuf message #5265
3135
* [BUGFIX] Querier: Convert gRPC `ResourceExhausted` status code from store gateway to 422 limit error. #5286

‎go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ require (
5353
github.com/stretchr/testify v1.8.4
5454
github.com/thanos-io/objstore v0.0.0-20230201072718-11ffbc490204
5555
github.com/thanos-io/promql-engine v0.0.0-20230526105742-791d78b260ea
56-
github.com/thanos-io/thanos v0.31.1-0.20230607122802-662211055334
56+
github.com/thanos-io/thanos v0.31.1-0.20230616082957-d43026952989
5757
github.com/uber/jaeger-client-go v2.30.0+incompatible
5858
github.com/weaveworks/common v0.0.0-20221201103051-7c2720a9024d
5959
go.etcd.io/etcd/api/v3 v3.5.8

‎go.sum‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1163,8 +1163,8 @@ github.com/thanos-io/objstore v0.0.0-20230201072718-11ffbc490204 h1:W4w5Iph7j32S
11631163
github.com/thanos-io/objstore v0.0.0-20230201072718-11ffbc490204/go.mod h1:STSgpY8M6EKF2G/raUFdbIMf2U9GgYlEjAEHJxjvpAo=
11641164
github.com/thanos-io/promql-engine v0.0.0-20230526105742-791d78b260ea h1:kzK8sBn2+mo3NAxP+XjAjAqr1hwfxxFUy5CybaBkjAI=
11651165
github.com/thanos-io/promql-engine v0.0.0-20230526105742-791d78b260ea/go.mod h1:eIgPaXWgOhNAv6CPPrgu09r0AtT7byBTZy+7WkX0D18=
1166-
github.com/thanos-io/thanos v0.31.1-0.20230607122802-662211055334 h1:1pqel0J04gQRJpl3P3JX+zt6PbbTOfbUPdSww6jK8ws=
1167-
github.com/thanos-io/thanos v0.31.1-0.20230607122802-662211055334/go.mod h1:lHSiSsXIQuAv5u+6yu0LLw6cS/MC8vUQswQ6rkdxB7c=
1166+
github.com/thanos-io/thanos v0.31.1-0.20230616082957-d43026952989 h1:5prEq1YagZAt5Ah3HE876r3fhNhUhVh8JPuZLh/lJBI=
1167+
github.com/thanos-io/thanos v0.31.1-0.20230616082957-d43026952989/go.mod h1:jscDD4ecQW4A+6fpKgXLqOWOrtiTjcAEnOebEwAjXAM=
11681168
github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab h1:7ZR3hmisBWw77ZpO1/o86g+JV3VKlk3d48jopJxzTjU=
11691169
github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab/go.mod h1:eheTFp954zcWZXCU8d0AT76ftsQOTo4DTqkN/h3k1MY=
11701170
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=

‎integration/querier_test.go‎

Lines changed: 49 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -241,14 +241,14 @@ func TestQuerierWithBlocksStorageRunningInMicroservicesMode(t *testing.T) {
241241
assert.Equal(t, expectedVector3, result.(model.Vector))
242242

243243
// Check the in-memory index cache metrics (in the store-gateway).
244-
require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(7), "thanos_store_index_cache_requests_total"))
244+
require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(5+5+2), "thanos_store_index_cache_requests_total"))
245245
require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(0), "thanos_store_index_cache_hits_total")) // no cache hit cause the cache was empty
246246

247247
if testCfg.indexCacheBackend == tsdb.IndexCacheBackendInMemory {
248-
require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(2*2), "thanos_store_index_cache_items")) // 2 series both for postings and series cache
249-
require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(2*2), "thanos_store_index_cache_items_added_total")) // 2 series both for postings and series cache
248+
require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(9), "thanos_store_index_cache_items"))
249+
require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(9), "thanos_store_index_cache_items_added_total"))
250250
} else if testCfg.indexCacheBackend == tsdb.IndexCacheBackendMemcached {
251-
require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(11), "thanos_memcached_operations_total")) // 7 gets + 4 sets
251+
require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(21), "thanos_memcached_operations_total")) // 14 gets + 7 sets
252252
}
253253

254254
// Query back again the 1st series from storage. This time it should use the index cache.
@@ -257,14 +257,14 @@ func TestQuerierWithBlocksStorageRunningInMicroservicesMode(t *testing.T) {
257257
require.Equal(t, model.ValVector, result.Type())
258258
assert.Equal(t, expectedVector1, result.(model.Vector))
259259

260-
require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(7+2), "thanos_store_index_cache_requests_total"))
260+
require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(12+2), "thanos_store_index_cache_requests_total"))
261261
require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(2), "thanos_store_index_cache_hits_total")) // this time has used the index cache
262262

263263
if testCfg.indexCacheBackend == tsdb.IndexCacheBackendInMemory {
264-
require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(2*2), "thanos_store_index_cache_items")) // as before
265-
require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(2*2), "thanos_store_index_cache_items_added_total")) // as before
264+
require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(9), "thanos_store_index_cache_items")) // as before
265+
require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(9), "thanos_store_index_cache_items_added_total")) // as before
266266
} else if testCfg.indexCacheBackend == tsdb.IndexCacheBackendMemcached {
267-
require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(11+2), "thanos_memcached_operations_total")) // as before + 2 gets
267+
require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(23), "thanos_memcached_operations_total")) // as before + 2 gets
268268
}
269269

270270
// Query metadata.
@@ -298,38 +298,38 @@ func TestQuerierWithBlocksStorageRunningInSingleBinaryMode(t *testing.T) {
298298
ingesterStreamingEnabled: true,
299299
indexCacheBackend: tsdb.IndexCacheBackendInMemory,
300300
},
301-
"blocks sharding disabled, ingester gRPC streaming disabled, memcached index cache": {
302-
blocksShardingEnabled: false,
303-
ingesterStreamingEnabled: false,
304-
indexCacheBackend: tsdb.IndexCacheBackendMemcached,
305-
},
306-
"blocks sharding enabled, ingester gRPC streaming enabled, memcached index cache": {
307-
blocksShardingEnabled: true,
308-
ingesterStreamingEnabled: true,
309-
indexCacheBackend: tsdb.IndexCacheBackendMemcached,
310-
},
311-
"blocks sharding enabled, ingester gRPC streaming enabled, memcached index cache, bucket index enabled": {
312-
blocksShardingEnabled: true,
313-
ingesterStreamingEnabled: true,
314-
indexCacheBackend: tsdb.IndexCacheBackendMemcached,
315-
bucketIndexEnabled: true,
316-
},
317-
"blocks sharding disabled, ingester gRPC streaming disabled, redis index cache": {
318-
blocksShardingEnabled: false,
319-
ingesterStreamingEnabled: false,
320-
indexCacheBackend: tsdb.IndexCacheBackendRedis,
321-
},
322-
"blocks sharding enabled, ingester gRPC streaming enabled, redis index cache": {
323-
blocksShardingEnabled: true,
324-
ingesterStreamingEnabled: true,
325-
indexCacheBackend: tsdb.IndexCacheBackendRedis,
326-
},
327-
"blocks sharding enabled, ingester gRPC streaming enabled, redis index cache, bucket index enabled": {
328-
blocksShardingEnabled: true,
329-
ingesterStreamingEnabled: true,
330-
indexCacheBackend: tsdb.IndexCacheBackendRedis,
331-
bucketIndexEnabled: true,
332-
},
301+
//"blocks sharding disabled, ingester gRPC streaming disabled, memcached index cache": {
302+
// blocksShardingEnabled: false,
303+
// ingesterStreamingEnabled: false,
304+
// indexCacheBackend: tsdb.IndexCacheBackendMemcached,
305+
//},
306+
//"blocks sharding enabled, ingester gRPC streaming enabled, memcached index cache": {
307+
// blocksShardingEnabled: true,
308+
// ingesterStreamingEnabled: true,
309+
// indexCacheBackend: tsdb.IndexCacheBackendMemcached,
310+
//},
311+
//"blocks sharding enabled, ingester gRPC streaming enabled, memcached index cache, bucket index enabled": {
312+
// blocksShardingEnabled: true,
313+
// ingesterStreamingEnabled: true,
314+
// indexCacheBackend: tsdb.IndexCacheBackendMemcached,
315+
// bucketIndexEnabled: true,
316+
//},
317+
//"blocks sharding disabled, ingester gRPC streaming disabled, redis index cache": {
318+
// blocksShardingEnabled: false,
319+
// ingesterStreamingEnabled: false,
320+
// indexCacheBackend: tsdb.IndexCacheBackendRedis,
321+
//},
322+
//"blocks sharding enabled, ingester gRPC streaming enabled, redis index cache": {
323+
// blocksShardingEnabled: true,
324+
// ingesterStreamingEnabled: true,
325+
// indexCacheBackend: tsdb.IndexCacheBackendRedis,
326+
//},
327+
//"blocks sharding enabled, ingester gRPC streaming enabled, redis index cache, bucket index enabled": {
328+
// blocksShardingEnabled: true,
329+
// ingesterStreamingEnabled: true,
330+
// indexCacheBackend: tsdb.IndexCacheBackendRedis,
331+
// bucketIndexEnabled: true,
332+
//},
333333
}
334334

335335
for testName, testCfg := range tests {
@@ -475,14 +475,14 @@ func TestQuerierWithBlocksStorageRunningInSingleBinaryMode(t *testing.T) {
475475
assert.Equal(t, expectedVector3, result.(model.Vector))
476476

477477
// Check the in-memory index cache metrics (in the store-gateway).
478-
require.NoError(t, cluster.WaitSumMetrics(e2e.Equals(float64(7*seriesReplicationFactor)), "thanos_store_index_cache_requests_total"))
479-
require.NoError(t, cluster.WaitSumMetrics(e2e.Equals(0), "thanos_store_index_cache_hits_total")) // no cache hit cause the cache was empty
478+
require.NoError(t, cluster.WaitSumMetrics(e2e.Equals(float64((5+5+2)*seriesReplicationFactor)), "thanos_store_index_cache_requests_total")) // 5 for expanded postings and postings, 2 for series
479+
require.NoError(t, cluster.WaitSumMetrics(e2e.Equals(0), "thanos_store_index_cache_hits_total")) // no cache hit cause the cache was empty
480480

481481
if testCfg.indexCacheBackend == tsdb.IndexCacheBackendInMemory {
482-
require.NoError(t, cluster.WaitSumMetrics(e2e.Equals(float64(2*2*seriesReplicationFactor)), "thanos_store_index_cache_items")) // 2 series both for postings and series cache
483-
require.NoError(t, cluster.WaitSumMetrics(e2e.Equals(float64(2*2*seriesReplicationFactor)), "thanos_store_index_cache_items_added_total")) // 2 series both for postings and series cache
482+
require.NoError(t, cluster.WaitSumMetrics(e2e.Equals(float64(9*seriesReplicationFactor)), "thanos_store_index_cache_items"))
483+
require.NoError(t, cluster.WaitSumMetrics(e2e.Equals(float64(9*seriesReplicationFactor)), "thanos_store_index_cache_items_added_total"))
484484
} else if testCfg.indexCacheBackend == tsdb.IndexCacheBackendMemcached {
485-
require.NoError(t, cluster.WaitSumMetrics(e2e.Equals(float64(11*seriesReplicationFactor)), "thanos_memcached_operations_total")) // 7 gets + 4 sets
485+
require.NoError(t, cluster.WaitSumMetrics(e2e.Equals(float64(21*seriesReplicationFactor)), "thanos_memcached_operations_total")) // 14 gets + 7 sets
486486
}
487487

488488
// Query back again the 1st series from storage. This time it should use the index cache.
@@ -491,14 +491,14 @@ func TestQuerierWithBlocksStorageRunningInSingleBinaryMode(t *testing.T) {
491491
require.Equal(t, model.ValVector, result.Type())
492492
assert.Equal(t, expectedVector1, result.(model.Vector))
493493

494-
require.NoError(t, cluster.WaitSumMetrics(e2e.Equals(float64((7+2)*seriesReplicationFactor)), "thanos_store_index_cache_requests_total"))
494+
require.NoError(t, cluster.WaitSumMetrics(e2e.Equals(float64((12+2)*seriesReplicationFactor)), "thanos_store_index_cache_requests_total"))
495495
require.NoError(t, cluster.WaitSumMetrics(e2e.Equals(float64(2*seriesReplicationFactor)), "thanos_store_index_cache_hits_total")) // this time has used the index cache
496496

497497
if testCfg.indexCacheBackend == tsdb.IndexCacheBackendInMemory {
498-
require.NoError(t, cluster.WaitSumMetrics(e2e.Equals(float64(2*2*seriesReplicationFactor)), "thanos_store_index_cache_items")) // as before
499-
require.NoError(t, cluster.WaitSumMetrics(e2e.Equals(float64(2*2*seriesReplicationFactor)), "thanos_store_index_cache_items_added_total")) // as before
498+
require.NoError(t, cluster.WaitSumMetrics(e2e.Equals(float64(9*seriesReplicationFactor)), "thanos_store_index_cache_items")) // as before
499+
require.NoError(t, cluster.WaitSumMetrics(e2e.Equals(float64(9*seriesReplicationFactor)), "thanos_store_index_cache_items_added_total")) // as before
500500
} else if testCfg.indexCacheBackend == tsdb.IndexCacheBackendMemcached {
501-
require.NoError(t, cluster.WaitSumMetrics(e2e.Equals(float64((11+2)*seriesReplicationFactor)), "thanos_memcached_operations_total")) // as before + 2 gets
501+
require.NoError(t, cluster.WaitSumMetrics(e2e.Equals(float64((21+2)*seriesReplicationFactor)), "thanos_memcached_operations_total")) // as before + 2 gets
502502
}
503503

504504
// Query metadata.

‎pkg/alertmanager/alertmanager_metrics.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ func newAlertmanagerMetrics() *alertmanagerMetrics {
8484
numFailedNotifications: prometheus.NewDesc(
8585
"cortex_alertmanager_notifications_failed_total",
8686
"The total number of failed notifications.",
87-
[]string{"user", "integration"}, nil),
87+
[]string{"user", "integration", "reason"}, nil),
8888
numNotificationRequestsTotal: prometheus.NewDesc(
8989
"cortex_alertmanager_notification_requests_total",
9090
"The total number of attempted notification requests.",
@@ -292,7 +292,7 @@ func (m *alertmanagerMetrics) Collect(out chan<- prometheus.Metric) {
292292
data.SendSumOfCountersPerUser(out, m.alertsInvalid, "alertmanager_alerts_invalid_total")
293293

294294
data.SendSumOfCountersPerUserWithLabels(out, m.numNotifications, "alertmanager_notifications_total", "integration")
295-
data.SendSumOfCountersPerUserWithLabels(out, m.numFailedNotifications, "alertmanager_notifications_failed_total", "integration")
295+
data.SendSumOfCountersPerUserWithLabels(out, m.numFailedNotifications, "alertmanager_notifications_failed_total", "integration", "reason")
296296
data.SendSumOfCountersPerUserWithLabels(out, m.numNotificationRequestsTotal, "alertmanager_notification_requests_total", "integration")
297297
data.SendSumOfCountersPerUserWithLabels(out, m.numNotificationRequestsFailedTotal, "alertmanager_notification_requests_failed_total", "integration")
298298
data.SendSumOfHistograms(out, m.notificationLatencySeconds, "alertmanager_notification_latency_seconds")

0 commit comments

Comments
 (0)