Skip to content

Commit 3a382d5

Browse files
alanprotalanprot
committed
Cache Matchers on ingesters
Signed-off-by: alanprot <[email protected]>
1 parent 006cab3 commit 3a382d5

File tree

9 files changed

+145
-75
lines changed

9 files changed

+145
-75
lines changed

‎docs/configuration/config-file-reference.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3128,6 +3128,10 @@ instance_limits:
31283128
# change by changing this option.
31293129
# CLI flag: -ingester.disable-chunk-trimming
31303130
[disable_chunk_trimming: <boolean> | default = false]
3131+
3132+
# Maximum number of entries in the matchers cache. 0 to disable.
3133+
# CLI flag: -ingester.matchers-cache-max-items
3134+
[matchers_cache_max_items: <int> | default = 0]
31313135
```
31323136

31333137
### `ingester_client_config`

‎integration/query_fuzz_test.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -347,8 +347,9 @@ func TestExpandedPostingsCacheFuzz(t *testing.T) {
347347
"-blocks-storage.bucket-store.bucket-index.enabled": "true",
348348
"-querier.query-store-for-labels-enabled": "true",
349349
// Ingester.
350-
"-ring.store": "consul",
351-
"-consul.hostname": consul1.NetworkHTTPEndpoint(),
350+
"-ring.store": "consul",
351+
"-ingester.matchers-cache-max-items": "1024",
352+
"-consul.hostname": consul1.NetworkHTTPEndpoint(),
352353
// Distributor.
353354
"-distributor.replication-factor": "1",
354355
// Store-gateway.

‎pkg/distributor/distributor_test.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3333,7 +3333,7 @@ func (i *mockIngester) Query(ctx context.Context, req *client.QueryRequest, opts
33333333
return nil, errFail
33343334
}
33353335

3336-
_, _, matchers, err := client.FromQueryRequest(req)
3336+
_, _, matchers, err := client.FromQueryRequest(req, labels.NewMatcher)
33373337
if err != nil {
33383338
return nil, err
33393339
}
@@ -3359,7 +3359,7 @@ func (i *mockIngester) QueryStream(ctx context.Context, req *client.QueryRequest
33593359
return nil, errFail
33603360
}
33613361

3362-
_, _, matchers, err := client.FromQueryRequest(req)
3362+
_, _, matchers, err := client.FromQueryRequest(req, labels.NewMatcher)
33633363
if err != nil {
33643364
return nil, err
33653365
}
@@ -3418,7 +3418,7 @@ func (i *mockIngester) MetricsForLabelMatchersStream(ctx context.Context, req *c
34183418
return nil, errFail
34193419
}
34203420

3421-
_, _, _, multiMatchers, err := client.FromMetricsForLabelMatchersRequest(req)
3421+
_, _, _, multiMatchers, err := client.FromMetricsForLabelMatchersRequest(req, labels.NewMatcher)
34223422
if err != nil {
34233423
return nil, err
34243424
}
@@ -3450,7 +3450,7 @@ func (i *mockIngester) MetricsForLabelMatchers(ctx context.Context, req *client.
34503450
return nil, errFail
34513451
}
34523452

3453-
_, _, _, multiMatchers, err := client.FromMetricsForLabelMatchersRequest(req)
3453+
_, _, _, multiMatchers, err := client.FromMetricsForLabelMatchersRequest(req, labels.NewMatcher)
34543454
if err != nil {
34553455
return nil, err
34563456
}

‎pkg/ingester/client/compat.go

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,8 @@ func ToQueryRequest(from, to model.Time, matchers []*labels.Matcher) (*QueryRequ
2626
}
2727

2828
// FromQueryRequest unpacks a QueryRequest proto.
29-
func FromQueryRequest(req *QueryRequest) (model.Time, model.Time, []*labels.Matcher, error) {
30-
matchers, err := FromLabelMatchers(req.Matchers)
29+
func FromQueryRequest(req *QueryRequest, newMatcher func(t labels.MatchType, n, v string) (*labels.Matcher, error)) (model.Time, model.Time, []*labels.Matcher, error) {
30+
matchers, err := FromLabelMatchers(req.Matchers, newMatcher)
3131
if err != nil {
3232
return 0, 0, nil, err
3333
}
@@ -55,10 +55,10 @@ func ToExemplarQueryRequest(from, to model.Time, matchers ...[]*labels.Matcher)
5555
}
5656

5757
// FromExemplarQueryRequest unpacks a ExemplarQueryRequest proto.
58-
func FromExemplarQueryRequest(req *ExemplarQueryRequest) (int64, int64, [][]*labels.Matcher, error) {
58+
func FromExemplarQueryRequest(req *ExemplarQueryRequest, newMatcher func(t labels.MatchType, n, v string) (*labels.Matcher, error)) (int64, int64, [][]*labels.Matcher, error) {
5959
var result [][]*labels.Matcher
6060
for _, m := range req.Matchers {
61-
matchers, err := FromLabelMatchers(m.Matchers)
61+
matchers, err := FromLabelMatchers(m.Matchers, newMatcher)
6262
if err != nil {
6363
return 0, 0, nil, err
6464
}
@@ -175,10 +175,10 @@ func SeriesSetToQueryResponse(s storage.SeriesSet) (*QueryResponse, error) {
175175
}
176176

177177
// FromMetricsForLabelMatchersRequest unpacks a MetricsForLabelMatchersRequest proto
178-
func FromMetricsForLabelMatchersRequest(req *MetricsForLabelMatchersRequest) (model.Time, model.Time, int, [][]*labels.Matcher, error) {
178+
func FromMetricsForLabelMatchersRequest(req *MetricsForLabelMatchersRequest, newMatcher func(t labels.MatchType, n, v string) (*labels.Matcher, error)) (model.Time, model.Time, int, [][]*labels.Matcher, error) {
179179
matchersSet := make([][]*labels.Matcher, 0, len(req.MatchersSet))
180180
for _, matchers := range req.MatchersSet {
181-
matchers, err := FromLabelMatchers(matchers.Matchers)
181+
matchers, err := FromLabelMatchers(matchers.Matchers, newMatcher)
182182
if err != nil {
183183
return 0, 0, 0, nil, err
184184
}
@@ -206,12 +206,12 @@ func ToLabelValuesRequest(labelName model.LabelName, from, to model.Time, limit
206206
}
207207

208208
// FromLabelValuesRequest unpacks a LabelValuesRequest proto
209-
func FromLabelValuesRequest(req *LabelValuesRequest) (string, int64, int64, int, []*labels.Matcher, error) {
209+
func FromLabelValuesRequest(req *LabelValuesRequest, newMatcher func(t labels.MatchType, n, v string) (*labels.Matcher, error)) (string, int64, int64, int, []*labels.Matcher, error) {
210210
var err error
211211
var matchers []*labels.Matcher
212212

213213
if req.Matchers != nil {
214-
matchers, err = FromLabelMatchers(req.Matchers.Matchers)
214+
matchers, err = FromLabelMatchers(req.Matchers.Matchers, newMatcher)
215215
if err != nil {
216216
return "", 0, 0, 0, nil, err
217217
}
@@ -236,12 +236,12 @@ func ToLabelNamesRequest(from, to model.Time, limit int, matchers []*labels.Matc
236236
}
237237

238238
// FromLabelNamesRequest unpacks a LabelNamesRequest proto
239-
func FromLabelNamesRequest(req *LabelNamesRequest) (int64, int64, int, []*labels.Matcher, error) {
239+
func FromLabelNamesRequest(req *LabelNamesRequest, newMatcher func(t labels.MatchType, n, v string) (*labels.Matcher, error)) (int64, int64, int, []*labels.Matcher, error) {
240240
var err error
241241
var matchers []*labels.Matcher
242242

243243
if req.Matchers != nil {
244-
matchers, err = FromLabelMatchers(req.Matchers.Matchers)
244+
matchers, err = FromLabelMatchers(req.Matchers.Matchers, newMatcher)
245245
if err != nil {
246246
return 0, 0, 0, nil, err
247247
}
@@ -275,7 +275,7 @@ func toLabelMatchers(matchers []*labels.Matcher) ([]*LabelMatcher, error) {
275275
return result, nil
276276
}
277277

278-
func FromLabelMatchers(matchers []*LabelMatcher) ([]*labels.Matcher, error) {
278+
func FromLabelMatchers(matchers []*LabelMatcher, newMatcher func(t labels.MatchType, n, v string) (*labels.Matcher, error)) ([]*labels.Matcher, error) {
279279
result := make([]*labels.Matcher, 0, len(matchers))
280280
for _, matcher := range matchers {
281281
var mtype labels.MatchType
@@ -291,7 +291,7 @@ func FromLabelMatchers(matchers []*LabelMatcher) ([]*labels.Matcher, error) {
291291
default:
292292
return nil, fmt.Errorf("invalid matcher type")
293293
}
294-
matcher, err := labels.NewMatcher(mtype, matcher.Name, matcher.Value)
294+
matcher, err := newMatcher(mtype, matcher.Name, matcher.Value)
295295
if err != nil {
296296
return nil, err
297297
}

‎pkg/ingester/client/compat_test.go

Lines changed: 58 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -7,53 +7,74 @@ import (
77

88
"github.com/prometheus/common/model"
99
"github.com/prometheus/prometheus/model/labels"
10+
"github.com/stretchr/testify/require"
11+
12+
"github.com/cortexproject/cortex/pkg/util"
1013
)
1114

1215
func TestQueryRequest(t *testing.T) {
13-
from, to := model.Time(int64(0)), model.Time(int64(10))
14-
matchers := []*labels.Matcher{}
15-
matcher1, err := labels.NewMatcher(labels.MatchEqual, "foo", "1")
16-
if err != nil {
17-
t.Fatal(err)
16+
c, err := util.NewMatcherCache(1024)
17+
require.NoError(t, err)
18+
19+
tc := map[string]struct {
20+
newMatcherFunc func(t labels.MatchType, n, v string) (*labels.Matcher, error)
21+
}{
22+
"no cache": {
23+
newMatcherFunc: labels.NewMatcher,
24+
},
25+
"cache": {
26+
newMatcherFunc: c.GetMatcher,
27+
},
1828
}
19-
matchers = append(matchers, matcher1)
2029

21-
matcher2, err := labels.NewMatcher(labels.MatchNotEqual, "bar", "2")
22-
if err != nil {
23-
t.Fatal(err)
24-
}
25-
matchers = append(matchers, matcher2)
30+
for name, tc := range tc {
31+
t.Run(name, func(t *testing.T) {
32+
from, to := model.Time(int64(0)), model.Time(int64(10))
33+
matchers := []*labels.Matcher{}
34+
matcher1, err := labels.NewMatcher(labels.MatchEqual, "foo", "1")
35+
if err != nil {
36+
t.Fatal(err)
37+
}
38+
matchers = append(matchers, matcher1)
2639

27-
matcher3, err := labels.NewMatcher(labels.MatchRegexp, "baz", "3")
28-
if err != nil {
29-
t.Fatal(err)
30-
}
31-
matchers = append(matchers, matcher3)
40+
matcher2, err := labels.NewMatcher(labels.MatchNotEqual, "bar", "2")
41+
if err != nil {
42+
t.Fatal(err)
43+
}
44+
matchers = append(matchers, matcher2)
3245

33-
matcher4, err := labels.NewMatcher(labels.MatchNotRegexp, "bop", "4")
34-
if err != nil {
35-
t.Fatal(err)
36-
}
37-
matchers = append(matchers, matcher4)
46+
matcher3, err := labels.NewMatcher(labels.MatchRegexp, "baz", "3")
47+
if err != nil {
48+
t.Fatal(err)
49+
}
50+
matchers = append(matchers, matcher3)
3851

39-
req, err := ToQueryRequest(from, to, matchers)
40-
if err != nil {
41-
t.Fatal(err)
42-
}
52+
matcher4, err := labels.NewMatcher(labels.MatchNotRegexp, "bop", "4")
53+
if err != nil {
54+
t.Fatal(err)
55+
}
56+
matchers = append(matchers, matcher4)
4357

44-
haveFrom, haveTo, haveMatchers, err := FromQueryRequest(req)
45-
if err != nil {
46-
t.Fatal(err)
47-
}
58+
req, err := ToQueryRequest(from, to, matchers)
59+
if err != nil {
60+
t.Fatal(err)
61+
}
4862

49-
if !reflect.DeepEqual(haveFrom, from) {
50-
t.Fatalf("Bad from FromQueryRequest(ToQueryRequest) round trip")
51-
}
52-
if !reflect.DeepEqual(haveTo, to) {
53-
t.Fatalf("Bad to FromQueryRequest(ToQueryRequest) round trip")
54-
}
55-
if !matchersEqual(haveMatchers, matchers) {
56-
t.Fatalf("Bad have FromQueryRequest(ToQueryRequest) round trip - %v != %v", haveMatchers, matchers)
63+
haveFrom, haveTo, haveMatchers, err := FromQueryRequest(req, tc.newMatcherFunc)
64+
if err != nil {
65+
t.Fatal(err)
66+
}
67+
68+
if !reflect.DeepEqual(haveFrom, from) {
69+
t.Fatalf("Bad from FromQueryRequest(ToQueryRequest) round trip")
70+
}
71+
if !reflect.DeepEqual(haveTo, to) {
72+
t.Fatalf("Bad to FromQueryRequest(ToQueryRequest) round trip")
73+
}
74+
if !matchersEqual(haveMatchers, matchers) {
75+
t.Fatalf("Bad have FromQueryRequest(ToQueryRequest) round trip - %v != %v", haveMatchers, matchers)
76+
}
77+
})
5778
}
5879
}
5980

‎pkg/ingester/ingester.go

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,8 @@ type Config struct {
142142
// When disabled, the result may contain samples outside the queried time range but Select() performances
143143
// may be improved.
144144
DisableChunkTrimming bool `yaml:"disable_chunk_trimming"`
145+
146+
MatchersCacheMaxItems int `yaml:"matchers_cache_max_items"`
145147
}
146148

147149
// RegisterFlags adds the flags required to config this to the given FlagSet
@@ -170,6 +172,7 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
170172
f.BoolVar(&cfg.LabelsStringInterningEnabled, "ingester.labels-string-interning-enabled", false, "Experimental: Enable string interning for metrics labels.")
171173

172174
f.BoolVar(&cfg.DisableChunkTrimming, "ingester.disable-chunk-trimming", false, "Disable trimming of matching series chunks based on query Start and End time. When disabled, the result may contain samples outside the queried time range but select performances may be improved. Note that certain query results might change by changing this option.")
175+
f.IntVar(&cfg.MatchersCacheMaxItems, "ingester.matchers-cache-max-items", 0, "Maximum number of entries in the matchers cache. 0 to disable.")
173176
}
174177

175178
func (cfg *Config) Validate() error {
@@ -240,6 +243,8 @@ type Ingester struct {
240243
maxInflightQueryRequests util_math.MaxTracker
241244

242245
expandedPostingsCacheFactory *cortex_tsdb.ExpandedPostingsCacheFactory
246+
247+
newMatcherFunc func(t labels.MatchType, n, v string) (*labels.Matcher, error)
243248
}
244249

245250
// Shipper interface is used to have an easy way to mock it in tests.
@@ -698,12 +703,22 @@ func New(cfg Config, limits *validation.Overrides, registerer prometheus.Registe
698703
return nil, errors.Wrap(err, "failed to create the bucket client")
699704
}
700705

706+
newMatcherFunc := labels.NewMatcher
707+
if cfg.MatchersCacheMaxItems > 0 {
708+
matcherCache, err := util.NewMatcherCache(1024)
709+
if err != nil {
710+
return nil, errors.Wrap(err, "failed to create matcher cache")
711+
}
712+
newMatcherFunc = matcherCache.GetMatcher
713+
}
714+
701715
i := &Ingester{
702716
cfg: cfg,
703717
limits: limits,
704718
usersMetadata: map[string]*userMetricsMetadata{},
705719
TSDBState: newTSDBState(bucketClient, registerer),
706720
logger: logger,
721+
newMatcherFunc: newMatcherFunc,
707722
ingestionRate: util_math.NewEWMARate(0.2, instanceIngestionRateTickInterval),
708723
expandedPostingsCacheFactory: cortex_tsdb.NewExpandedPostingsCacheFactory(cfg.BlocksStorageConfig.TSDB.PostingsCache),
709724
}
@@ -1448,7 +1463,7 @@ func (i *Ingester) QueryExemplars(ctx context.Context, req *client.ExemplarQuery
14481463
return nil, err
14491464
}
14501465

1451-
from, through, matchers, err := client.FromExemplarQueryRequest(req)
1466+
from, through, matchers, err := client.FromExemplarQueryRequest(req, i.newMatcherFunc)
14521467
if err != nil {
14531468
return nil, err
14541469
}
@@ -1538,7 +1553,7 @@ func (i *Ingester) labelsValuesCommon(ctx context.Context, req *client.LabelValu
15381553
return nil, cleanup, err
15391554
}
15401555

1541-
labelName, startTimestampMs, endTimestampMs, limit, matchers, err := client.FromLabelValuesRequest(req)
1556+
labelName, startTimestampMs, endTimestampMs, limit, matchers, err := client.FromLabelValuesRequest(req, i.newMatcherFunc)
15421557
if err != nil {
15431558
return nil, cleanup, err
15441559
}
@@ -1628,7 +1643,7 @@ func (i *Ingester) labelNamesCommon(ctx context.Context, req *client.LabelNamesR
16281643
return nil, cleanup, err
16291644
}
16301645

1631-
startTimestampMs, endTimestampMs, limit, matchers, err := client.FromLabelNamesRequest(req)
1646+
startTimestampMs, endTimestampMs, limit, matchers, err := client.FromLabelNamesRequest(req, i.newMatcherFunc)
16321647
if err != nil {
16331648
return nil, cleanup, err
16341649
}
@@ -1727,7 +1742,7 @@ func (i *Ingester) metricsForLabelMatchersCommon(ctx context.Context, req *clien
17271742
}
17281743

17291744
// Parse the request
1730-
_, _, limit, matchersSet, err := client.FromMetricsForLabelMatchersRequest(req)
1745+
_, _, limit, matchersSet, err := client.FromMetricsForLabelMatchersRequest(req, i.newMatcherFunc)
17311746
if err != nil {
17321747
return nil, cleanup, err
17331748
}
@@ -1946,7 +1961,7 @@ func (i *Ingester) QueryStream(req *client.QueryRequest, stream client.Ingester_
19461961
return err
19471962
}
19481963

1949-
from, through, matchers, err := client.FromQueryRequest(req)
1964+
from, through, matchers, err := client.FromQueryRequest(req, i.newMatcherFunc)
19501965
if err != nil {
19511966
return err
19521967
}

‎pkg/ingester/lifecycle_test.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ func defaultIngesterTestConfig(t testing.TB) Config {
4343
cfg.LifecyclerConfig.FinalSleep = 0
4444
cfg.ActiveSeriesMetricsEnabled = true
4545
cfg.LabelsStringInterningEnabled = true
46+
cfg.MatchersCacheMaxItems = 1024
4647
return cfg
4748
}
4849

‎pkg/querier/remote_read.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55

66
"github.com/go-kit/log"
77
"github.com/go-kit/log/level"
8+
"github.com/prometheus/prometheus/model/labels"
89
"github.com/prometheus/prometheus/storage"
910

1011
"github.com/cortexproject/cortex/pkg/ingester/client"
@@ -34,7 +35,7 @@ func RemoteReadHandler(q storage.Queryable, logger log.Logger) http.Handler {
3435
errors := make(chan error)
3536
for i, qr := range req.Queries {
3637
go func(i int, qr *client.QueryRequest) {
37-
from, to, matchers, err := client.FromQueryRequest(qr)
38+
from, to, matchers, err := client.FromQueryRequest(qr, labels.NewMatcher)
3839
if err != nil {
3940
errors <- err
4041
return

0 commit comments

Comments
 (0)