Skip to content

Commit e5f60fb

Browse files
committed
test vertical sharding on binary expr
Signed-off-by: Ben Ye <[email protected]>
1 parent eedb5ab commit e5f60fb

File tree

27 files changed

+1748
-66
lines changed

27 files changed

+1748
-66
lines changed

go.mod

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,7 @@ require (
117117
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
118118
github.com/davecgh/go-spew v1.1.1 // indirect
119119
github.com/dennwc/varint v1.0.0 // indirect
120+
github.com/dgryski/go-metro v0.0.0-20200812162917-85c65e2d0165 // indirect
120121
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
121122
github.com/docker/go-units v0.5.0 // indirect
122123
github.com/edsrzf/mmap-go v1.1.0 // indirect
@@ -187,6 +188,7 @@ require (
187188
github.com/rs/cors v1.9.0 // indirect
188189
github.com/rs/xid v1.5.0 // indirect
189190
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect
191+
github.com/seiflotfy/cuckoofilter v0.0.0-20220411075957-e3b120b3f5fb // indirect
190192
github.com/sercand/kuberesolver v2.4.0+incompatible // indirect
191193
github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 // indirect
192194
github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546 // indirect
@@ -258,3 +260,5 @@ replace github.com/google/gnostic => github.com/googleapis/gnostic v0.6.9
258260
replace gopkg.in/alecthomas/kingpin.v2 => github.com/alecthomas/kingpin v1.3.8-0.20210301060133-17f40c25f497
259261

260262
replace github.com/sercand/kuberesolver => github.com/sercand/kuberesolver/v4 v4.0.0
263+
264+
replace github.com/thanos-io/thanos => github.com/yeya24/thanos v0.2.2-0.20230810215401-9ef4c021bfe4

go.sum

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -549,6 +549,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
549549
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
550550
github.com/dennwc/varint v1.0.0 h1:kGNFFSSw8ToIy3obO/kKr8U9GZYUAxQEVuix4zfDWzE=
551551
github.com/dennwc/varint v1.0.0/go.mod h1:hnItb35rvZvJrbTALZtY/iQfDs48JKRG1RPpgziApxA=
552+
github.com/dgryski/go-metro v0.0.0-20200812162917-85c65e2d0165 h1:BS21ZUJ/B5X2UVUbczfmdWH7GapPWAhxcMsDnjJTU1E=
553+
github.com/dgryski/go-metro v0.0.0-20200812162917-85c65e2d0165/go.mod h1:c9O8+fpSOX1DM8cPNSkX/qsBWdkD4yd2dpciOWQjpBw=
552554
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
553555
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
554556
github.com/dhui/dktest v0.3.16 h1:i6gq2YQEtcrjKbeJpBkWjE8MmLZPYllcjOFbTZuPDnw=
@@ -1150,6 +1152,8 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUt
11501152
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
11511153
github.com/segmentio/fasthash v1.0.3 h1:EI9+KE1EwvMLBWwjpRDc+fEM+prwxDYbslddQGtrmhM=
11521154
github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY=
1155+
github.com/seiflotfy/cuckoofilter v0.0.0-20220411075957-e3b120b3f5fb h1:XfLJSPIOUX+osiMraVgIrMR27uMXnRJWGm1+GL8/63U=
1156+
github.com/seiflotfy/cuckoofilter v0.0.0-20220411075957-e3b120b3f5fb/go.mod h1:bR6DqgcAl1zTcOX8/pE2Qkj9XO00eCNqmKb7lXP8EAg=
11531157
github.com/sercand/kuberesolver/v4 v4.0.0 h1:frL7laPDG/lFm5n98ODmWnn+cvPpzlkf3LhzuPhcHP4=
11541158
github.com/sercand/kuberesolver/v4 v4.0.0/go.mod h1:F4RGyuRmMAjeXHKL+w4P7AwUnPceEAPAhxUgXZjKgvM=
11551159
github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 h1:bUGsEnyNbVPw06Bs80sCeARAlK8lhwqGyi6UT8ymuGk=
@@ -1206,8 +1210,6 @@ github.com/thanos-io/objstore v0.0.0-20230721074820-89475d4508df h1:QEQsto338hS1
12061210
github.com/thanos-io/objstore v0.0.0-20230721074820-89475d4508df/go.mod h1:IS7Z25+0KaknyU2P5PTP/5hwY6Yr/FzbInF88Yd5auU=
12071211
github.com/thanos-io/promql-engine v0.0.0-20230526105742-791d78b260ea h1:kzK8sBn2+mo3NAxP+XjAjAqr1hwfxxFUy5CybaBkjAI=
12081212
github.com/thanos-io/promql-engine v0.0.0-20230526105742-791d78b260ea/go.mod h1:eIgPaXWgOhNAv6CPPrgu09r0AtT7byBTZy+7WkX0D18=
1209-
github.com/thanos-io/thanos v0.31.1-0.20230809185825-46339760e526 h1:QpAdI+mFTgZ1UUviL7Wzxi+h45LZEoxErHQHdYFb+Z4=
1210-
github.com/thanos-io/thanos v0.31.1-0.20230809185825-46339760e526/go.mod h1:4MiWDaeClW8CSHiOR57bnuUOID3HDYQphMthOPwtpJY=
12111213
github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab h1:7ZR3hmisBWw77ZpO1/o86g+JV3VKlk3d48jopJxzTjU=
12121214
github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab/go.mod h1:eheTFp954zcWZXCU8d0AT76ftsQOTo4DTqkN/h3k1MY=
12131215
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
@@ -1229,6 +1231,8 @@ github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+
12291231
github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
12301232
github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM=
12311233
github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
1234+
github.com/yeya24/thanos v0.2.2-0.20230810215401-9ef4c021bfe4 h1:Wao4JHvb/rZd4/seaY6KHi2ZcomMBQc3GceQykdznlY=
1235+
github.com/yeya24/thanos v0.2.2-0.20230810215401-9ef4c021bfe4/go.mod h1:FneDtvhqV1PtNWmqVrN8/Fi/YDt0mCfSk9B+yFHS1qw=
12321236
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
12331237
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
12341238
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=

pkg/querier/tripperware/test_shard_by_query_utils.go

Lines changed: 31 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -45,18 +45,6 @@ func TestQueryShardQuery(t *testing.T, instantQueryCodec Codec, shardedPrometheu
4545
name: "outer aggregation with without grouping",
4646
expression: "count(sum without (pod) (http_requests_total))",
4747
},
48-
{
49-
name: "aggregate expression with label_replace",
50-
expression: `sum by (pod) (label_replace(metric, "dst_label", "$1", "src_label", "re"))`,
51-
},
52-
{
53-
name: "aggregate without expression with label_replace",
54-
expression: `sum without (pod) (label_replace(metric, "dst_label", "$1", "src_label", "re"))`,
55-
},
56-
{
57-
name: "binary expression",
58-
expression: `http_requests_total{code="400"} / http_requests_total`,
59-
},
6048
{
6149
name: "binary expression with constant",
6250
expression: `http_requests_total{code="400"} / 4`,
@@ -69,10 +57,6 @@ func TestQueryShardQuery(t *testing.T, instantQueryCodec Codec, shardedPrometheu
6957
name: "binary aggregation with different grouping labels",
7058
expression: `sum by (pod) (http_requests_total{code="400"}) / sum by (cluster) (http_requests_total)`,
7159
},
72-
{
73-
name: "multiple binary expressions",
74-
expression: `(http_requests_total{code="400"} + http_requests_total{code="500"}) / http_requests_total`,
75-
},
7660
{
7761
name: "multiple binary expressions with empty vector matchers",
7862
expression: `
@@ -193,6 +177,11 @@ sum by (container) (
193177
expression: `sort_desc(avg(label_replace(label_replace(label_replace(count_over_time(container_memory_working_set_bytes{container!="", container!="POD", instance!="", }[1h] ), "node", "$1", "instance", "(.+)"), "container_name", "$1", "container", "(.+)"), "pod_name", "$1", "pod", "(.+)")*label_replace(label_replace(label_replace(avg_over_time(container_memory_working_set_bytes{container!="", container!="POD", instance!="", }[1h] ), "node", "$1", "instance", "(.+)"), "container_name", "$1", "container", "(.+)"), "pod_name", "$1", "pod", "(.+)")) by (namespace, container_name, pod_name, node, cluster_id))`,
194178
shardingLabels: []string{"namespace", "cluster_id"},
195179
},
180+
{
181+
name: "aggregate expression with label_replace",
182+
expression: `sum by (pod) (label_replace(metric, "dst_label", "$1", "src_label", "re"))`,
183+
shardingLabels: []string{"pod"},
184+
},
196185
}
197186

198187
// Shardable by labels instant queries with matrix response
@@ -233,7 +222,7 @@ sum by (container) (
233222
{
234223
name: "binary expression with outer without grouping",
235224
expression: `sum(http_requests_total{code="400"} * http_requests_total) without (pod)`,
236-
shardingLabels: []string{"pod"},
225+
shardingLabels: []string{model.MetricNameLabel, "pod"},
237226
},
238227
{
239228
name: "binary expression with vector matching and outer without grouping",
@@ -268,6 +257,26 @@ http_requests_total`,
268257
expression: `sum without (pod) (label_replace(metric, "dst_label", "$1", "src_label", "re"))`,
269258
shardingLabels: []string{"pod", "dst_label"},
270259
},
260+
{
261+
name: "binary expression",
262+
expression: `http_requests_total{code="400"} / http_requests_total`,
263+
shardingLabels: []string{model.MetricNameLabel},
264+
},
265+
{
266+
name: "binary expression among vector and scalar",
267+
expression: `aaaa - bbb > 1000`,
268+
shardingLabels: []string{model.MetricNameLabel},
269+
},
270+
{
271+
name: "binary expression with set operation",
272+
expression: `aaaa and bbb`,
273+
shardingLabels: []string{model.MetricNameLabel},
274+
},
275+
{
276+
name: "multiple binary expressions",
277+
expression: `(http_requests_total{code="400"} + http_requests_total{code="500"}) / http_requests_total`,
278+
shardingLabels: []string{model.MetricNameLabel},
279+
},
271280
}
272281

273282
type testCase struct {
@@ -324,6 +333,7 @@ http_requests_total`,
324333
name: fmt.Sprintf("non shardable query: %s", query.name),
325334
path: fmt.Sprintf(`/api/v1/query?time=120&query=%s`, url.QueryEscape(query.expression)),
326335
codec: instantQueryCodec,
336+
shardSize: 2,
327337
isShardable: false,
328338
responses: []string{
329339
`{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"up","job":"foo"},"value":[1,"1"]}],"stats":{"samples":{"totalQueryableSamples":10,"totalQueryableSamplesPerStep":[[1,10]]}}}}`,
@@ -334,6 +344,7 @@ http_requests_total`,
334344
name: fmt.Sprintf("non shardable query_range: %s", query.name),
335345
path: fmt.Sprintf(`/api/v1/query_range?start=1&end=2&step=1&query=%s`, url.QueryEscape(query.expression)),
336346
codec: shardedPrometheusCodec,
347+
shardSize: 2,
337348
isShardable: false,
338349
responses: []string{
339350
`{"status":"success","data":{"resultType":"matrix","result":[{"metric":{"__job__":"a","__name__":"metric"},"values":[[1,"1"],[2,"2"],[3,"3"]]}],"stats":{"samples":{"totalQueryableSamples":6,"totalQueryableSamplesPerStep":[[1,1],[2,2],[3,3]]}}}}`,
@@ -388,6 +399,9 @@ http_requests_total`,
388399
}
389400

390401
for _, tt := range tests {
402+
//if tt.name != "non shardable query: multiple binary expressions" {
403+
// continue
404+
//}
391405
t.Run(tt.name, func(t *testing.T) {
392406
//parallel testing causes data race
393407
sort.Strings(tt.shardingLabels)

vendor/github.com/dgryski/go-metro/LICENSE

Lines changed: 24 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

vendor/github.com/dgryski/go-metro/README

Lines changed: 6 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

vendor/github.com/dgryski/go-metro/metro.py

Lines changed: 199 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)