diff --git a/pkg/chunk/cache/instrumented.go b/pkg/chunk/cache/instrumented.go index 28b0c6fa934..67c994d1c77 100644 --- a/pkg/chunk/cache/instrumented.go +++ b/pkg/chunk/cache/instrumented.go @@ -29,31 +29,51 @@ var ( Name: "cache_hits", Help: "Total count of keys found in cache.", }, []string{"name"}) + + valueSize = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "cortex", + Name: "cache_value_size_bytes", + Help: "Size of values in the cache.", + // Cached chunks are generally in the KBs, but cached index can + // get big. Histogram goes from 1KB to 4MB. + // 1024 * 4^(7-1) = 4MB + Buckets: prometheus.ExponentialBuckets(1024, 4, 7), + }, []string{"name", "method"}) ) func init() { requestDuration.Register() prometheus.MustRegister(fetchedKeys) prometheus.MustRegister(hits) + prometheus.MustRegister(valueSize) } // Instrument returns an instrumented cache. func Instrument(name string, cache Cache) Cache { return &instrumentedCache{ - name: name, - fetchedKeys: fetchedKeys.WithLabelValues(name), - hits: hits.WithLabelValues(name), - Cache: cache, + name: name, + Cache: cache, + + fetchedKeys: fetchedKeys.WithLabelValues(name), + hits: hits.WithLabelValues(name), + storedValueSize: valueSize.WithLabelValues(name, "store"), + fetchedValueSize: valueSize.WithLabelValues(name, "fetch"), } } type instrumentedCache struct { - name string - fetchedKeys, hits prometheus.Counter + name string Cache + + fetchedKeys, hits prometheus.Counter + storedValueSize, fetchedValueSize prometheus.Observer } func (i *instrumentedCache) Store(ctx context.Context, keys []string, bufs [][]byte) { + for j := range bufs { + i.storedValueSize.Observe(float64(len(bufs[j]))) + } + method := i.name + ".store" instr.CollectedRequest(ctx, method, requestDuration, instr.ErrorCode, func(ctx context.Context) error { sp := ot.SpanFromContext(ctx) @@ -82,6 +102,10 @@ func (i *instrumentedCache) Fetch(ctx context.Context, keys []string) ([]string, i.fetchedKeys.Add(float64(len(keys))) i.hits.Add(float64(len(found))) + for j := range bufs { + i.fetchedValueSize.Observe(float64(len(bufs[j]))) + } + return found, bufs, missing } diff --git a/pkg/chunk/chunk_store_utils.go b/pkg/chunk/chunk_store_utils.go index fe98dcf6438..9a5ef76ea37 100644 --- a/pkg/chunk/chunk_store_utils.go +++ b/pkg/chunk/chunk_store_utils.go @@ -108,6 +108,7 @@ type decodeResponse struct { // NewChunkFetcher makes a new ChunkFetcher. func NewChunkFetcher(cfg cache.Config, cacheStubs bool, storage ObjectClient) (*Fetcher, error) { + cfg.Prefix = "chunks" cache, err := cache.New(cfg) if err != nil { return nil, err