From 1afc6b4698188d59c19fd517e9f589c5ab6ec227 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Tue, 25 Jun 2019 18:10:38 +0000 Subject: [PATCH] Optionally write stub entries to the chunk cache from ingesters Designed for the case where you don't want to cache every chunk written by ingesters, but you do want to take advantage of chunk write deduplication. Signed-off-by: Bryan Boreham --- docs/arguments.md | 5 +++++ pkg/chunk/cache/cache_test.go | 2 +- pkg/chunk/chunk_store.go | 6 +++++- pkg/chunk/chunk_store_utils.go | 20 +++++++++++++------- pkg/chunk/series_store.go | 2 +- 5 files changed, 25 insertions(+), 10 deletions(-) diff --git a/docs/arguments.md b/docs/arguments.md index f7e0fcd9936..9d79f5c722e 100644 --- a/docs/arguments.md +++ b/docs/arguments.md @@ -116,6 +116,11 @@ The ingester query API was improved over time, but defaults to the old behaviour When `push` requests arrive, pre-allocate this many slots to decode them. Tune this setting to reduce memory allocations and garbage. The optimum value will depend on how many labels are sent with your timeseries samples. +- `-store.chunk-cache-stubs` + + Where you don't want to cache every chunk written by ingesters, but you do want to take advantage of chunk write deduplication, this option will make ingesters write a placeholder to the cache for each chunk. + Make sure you configure ingesters with a different cache to queriers, which need the whole value. + ## Ingester, Distributor & Querier limits. Cortex implements various limits on the requests it can process, in order to prevent a single tenant overwhelming the cluster. There are various default global limits which apply to all tenants which can be set on the command line. These limits can also be overridden on a per-tenant basis, using a configuration file. Specify the filename for the override configuration file using the `-limits.per-user-override-config=` flag. The override file will be re-read every 10 seconds by default - this can also be controlled using the `-limits.per-user-override-period=10s` flag. diff --git a/pkg/chunk/cache/cache_test.go b/pkg/chunk/cache/cache_test.go index 3feb22b1ca8..8ef1d4c9331 100644 --- a/pkg/chunk/cache/cache_test.go +++ b/pkg/chunk/cache/cache_test.go @@ -96,7 +96,7 @@ func testCacheMultiple(t *testing.T, cache cache.Cache, keys []string, chunks [] func testChunkFetcher(t *testing.T, c cache.Cache, keys []string, chunks []chunk.Chunk) { fetcher, err := chunk.NewChunkFetcher(cache.Config{ Cache: c, - }, nil) + }, false, nil) require.NoError(t, err) defer fetcher.Stop() diff --git a/pkg/chunk/chunk_store.go b/pkg/chunk/chunk_store.go index 7ae86fc3de4..03e178e9b82 100644 --- a/pkg/chunk/chunk_store.go +++ b/pkg/chunk/chunk_store.go @@ -64,11 +64,15 @@ type StoreConfig struct { // Limits query start time to be greater than now() - MaxLookBackPeriod, if set. MaxLookBackPeriod time.Duration `yaml:"max_look_back_period"` + + // Not visible in yaml because the setting shouldn't be common between ingesters and queriers + chunkCacheStubs bool // don't write the full chunk to cache, just a stub entry } // RegisterFlags adds the flags required to config this to the given FlagSet func (cfg *StoreConfig) RegisterFlags(f *flag.FlagSet) { cfg.ChunkCacheConfig.RegisterFlagsWithPrefix("", "Cache config for chunks. ", f) + f.BoolVar(&cfg.chunkCacheStubs, "store.chunk-cache-stubs", false, "If true, don't write the full chunk to cache, just a stub entry.") cfg.WriteDedupeCacheConfig.RegisterFlagsWithPrefix("store.index-cache-write.", "Cache config for index entry writing. ", f) f.DurationVar(&cfg.MinChunkAge, "store.min-chunk-age", 0, "Minimum time between chunk update and being saved to the store.") @@ -92,7 +96,7 @@ type store struct { } func newStore(cfg StoreConfig, schema Schema, index IndexClient, chunks ObjectClient, limits *validation.Overrides) (Store, error) { - fetcher, err := NewChunkFetcher(cfg.ChunkCacheConfig, chunks) + fetcher, err := NewChunkFetcher(cfg.ChunkCacheConfig, cfg.chunkCacheStubs, chunks) if err != nil { return nil, err } diff --git a/pkg/chunk/chunk_store_utils.go b/pkg/chunk/chunk_store_utils.go index c5e7cade69b..ffa707eaf44 100644 --- a/pkg/chunk/chunk_store_utils.go +++ b/pkg/chunk/chunk_store_utils.go @@ -54,8 +54,9 @@ outer: // and writing back any misses to the cache. Also responsible for decoding // chunks from the cache, in parallel. type Fetcher struct { - storage ObjectClient - cache cache.Cache + storage ObjectClient + cache cache.Cache + cacheStubs bool wait sync.WaitGroup decodeRequests chan decodeRequest @@ -72,7 +73,7 @@ type decodeResponse struct { } // NewChunkFetcher makes a new ChunkFetcher. -func NewChunkFetcher(cfg cache.Config, storage ObjectClient) (*Fetcher, error) { +func NewChunkFetcher(cfg cache.Config, cacheStubs bool, storage ObjectClient) (*Fetcher, error) { cache, err := cache.New(cfg) if err != nil { return nil, err @@ -81,6 +82,7 @@ func NewChunkFetcher(cfg cache.Config, storage ObjectClient) (*Fetcher, error) { c := &Fetcher{ storage: storage, cache: cache, + cacheStubs: cacheStubs, decodeRequests: make(chan decodeRequest), } @@ -149,10 +151,14 @@ func (c *Fetcher) writeBackCache(ctx context.Context, chunks []Chunk) error { keys := make([]string, 0, len(chunks)) bufs := make([][]byte, 0, len(chunks)) for i := range chunks { - encoded, err := chunks[i].Encoded() - // TODO don't fail, just log and conitnue? - if err != nil { - return err + var encoded []byte + var err error + if !c.cacheStubs { + encoded, err = chunks[i].Encoded() + // TODO don't fail, just log and conitnue? + if err != nil { + return err + } } keys = append(keys, chunks[i].ExternalKey()) diff --git a/pkg/chunk/series_store.go b/pkg/chunk/series_store.go index 5462a0fc38f..9111d52448d 100644 --- a/pkg/chunk/series_store.go +++ b/pkg/chunk/series_store.go @@ -68,7 +68,7 @@ type seriesStore struct { } func newSeriesStore(cfg StoreConfig, schema Schema, index IndexClient, chunks ObjectClient, limits *validation.Overrides) (Store, error) { - fetcher, err := NewChunkFetcher(cfg.ChunkCacheConfig, chunks) + fetcher, err := NewChunkFetcher(cfg.ChunkCacheConfig, cfg.chunkCacheStubs, chunks) if err != nil { return nil, err }