Support filtered index cache

docs/blocks-storage/querier.md

@@ -536,6 +536,11 @@ blocks_storage:
         # CLI flag: -blocks-storage.bucket-store.index-cache.inmemory.max-size-bytes
         [max_size_bytes: <int> | default = 1073741824]
 
+        # Selectively cache index item types. Supported values are Postings,
+        # ExpandedPostings and Series
+        # CLI flag: -blocks-storage.bucket-store.index-cache.inmemory.enabled-items
+        [enabled_items: <list of string> | default = []]
+
       memcached:
         # Comma separated list of memcached addresses. Supported prefixes are:
         # dns+ (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV
@@ -583,6 +588,11 @@ blocks_storage:
         # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.auto-discovery
         [auto_discovery: <boolean> | default = false]
 
+        # Selectively cache index item types. Supported values are Postings,
+        # ExpandedPostings and Series
+        # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.enabled-items
+        [enabled_items: <list of string> | default = []]
+
       redis:
         # Comma separated list of redis addresses. Supported prefixes are: dns+
         # (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV query,
@@ -679,6 +689,11 @@ blocks_storage:
         # CLI flag: -blocks-storage.bucket-store.index-cache.redis.cache-size
         [cache_size: <int> | default = 0]
 
+        # Selectively cache index item types. Supported values are Postings,
+        # ExpandedPostings and Series
+        # CLI flag: -blocks-storage.bucket-store.index-cache.redis.enabled-items
+        [enabled_items: <list of string> | default = []]
+
     chunks_cache:
       # Backend for chunks cache, if not empty. Supported values: memcached.
       # CLI flag: -blocks-storage.bucket-store.chunks-cache.backend

docs/blocks-storage/store-gateway.md

@@ -643,6 +643,11 @@ blocks_storage:
         # CLI flag: -blocks-storage.bucket-store.index-cache.inmemory.max-size-bytes
         [max_size_bytes: <int> | default = 1073741824]
 
+        # Selectively cache index item types. Supported values are Postings,
+        # ExpandedPostings and Series
+        # CLI flag: -blocks-storage.bucket-store.index-cache.inmemory.enabled-items
+        [enabled_items: <list of string> | default = []]
+
       memcached:
         # Comma separated list of memcached addresses. Supported prefixes are:
         # dns+ (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV
@@ -690,6 +695,11 @@ blocks_storage:
         # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.auto-discovery
         [auto_discovery: <boolean> | default = false]
 
+        # Selectively cache index item types. Supported values are Postings,
+        # ExpandedPostings and Series
+        # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.enabled-items
+        [enabled_items: <list of string> | default = []]
+
       redis:
         # Comma separated list of redis addresses. Supported prefixes are: dns+
         # (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV query,
@@ -786,6 +796,11 @@ blocks_storage:
         # CLI flag: -blocks-storage.bucket-store.index-cache.redis.cache-size
         [cache_size: <int> | default = 0]
 
+        # Selectively cache index item types. Supported values are Postings,
+        # ExpandedPostings and Series
+        # CLI flag: -blocks-storage.bucket-store.index-cache.redis.enabled-items
+        [enabled_items: <list of string> | default = []]
+
     chunks_cache:
       # Backend for chunks cache, if not empty. Supported values: memcached.
       # CLI flag: -blocks-storage.bucket-store.chunks-cache.backend

docs/configuration/config-file-reference.md

@@ -1083,6 +1083,11 @@ bucket_store:
       # CLI flag: -blocks-storage.bucket-store.index-cache.inmemory.max-size-bytes
       [max_size_bytes: <int> | default = 1073741824]
 
+      # Selectively cache index item types. Supported values are Postings,
+      # ExpandedPostings and Series
+      # CLI flag: -blocks-storage.bucket-store.index-cache.inmemory.enabled-items
+      [enabled_items: <list of string> | default = []]
+
     memcached:
       # Comma separated list of memcached addresses. Supported prefixes are:
       # dns+ (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV query,
@@ -1130,6 +1135,11 @@ bucket_store:
       # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.auto-discovery
       [auto_discovery: <boolean> | default = false]
 
+      # Selectively cache index item types. Supported values are Postings,
+      # ExpandedPostings and Series
+      # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.enabled-items
+      [enabled_items: <list of string> | default = []]
+
     redis:
       # Comma separated list of redis addresses. Supported prefixes are: dns+
       # (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV query,
@@ -1226,6 +1236,11 @@ bucket_store:
       # CLI flag: -blocks-storage.bucket-store.index-cache.redis.cache-size
       [cache_size: <int> | default = 0]
 
+      # Selectively cache index item types. Supported values are Postings,
+      # ExpandedPostings and Series
+      # CLI flag: -blocks-storage.bucket-store.index-cache.redis.enabled-items
+      [enabled_items: <list of string> | default = []]
+
   chunks_cache:
     # Backend for chunks cache, if not empty. Supported values: memcached.
     # CLI flag: -blocks-storage.bucket-store.chunks-cache.backend

go.mod

@@ -53,7 +53,7 @@ require (
 	github.com/stretchr/testify v1.8.4
 	github.com/thanos-io/objstore v0.0.0-20230921130928-63a603e651ed
 	github.com/thanos-io/promql-engine v0.0.0-20230821193351-e1ae4275b96e
-	github.com/thanos-io/thanos v0.32.4-0.20231001083734-531cdb1e8ec3
+	github.com/thanos-io/thanos v0.32.5-0.20231006043659-79bbf34b4275
 	github.com/uber/jaeger-client-go v2.30.0+incompatible
 	github.com/weaveworks/common v0.0.0-20221201103051-7c2720a9024d
 	go.etcd.io/etcd/api/v3 v3.5.9

go.sum

@@ -1212,8 +1212,8 @@ github.com/thanos-io/objstore v0.0.0-20230921130928-63a603e651ed h1:iWQdY3S6DpWj
 github.com/thanos-io/objstore v0.0.0-20230921130928-63a603e651ed/go.mod h1:oJ82xgcBDzGJrEgUsjlTj6n01+ZWUMMUR8BlZzX5xDE=
 github.com/thanos-io/promql-engine v0.0.0-20230821193351-e1ae4275b96e h1:kwsFCU8eSkZehbrAN3nXPw5RdMHi/Bok/y8l2C4M+gk=
 github.com/thanos-io/promql-engine v0.0.0-20230821193351-e1ae4275b96e/go.mod h1:+T/ZYNCGybT6eTsGGvVtGb63nT1cvUmH6MjqRrcQoKw=
-github.com/thanos-io/thanos v0.32.4-0.20231001083734-531cdb1e8ec3 h1:ekD3P1XF0Hlg/u7rSNqdyLhwYE4W4RGfkMDudtepRL8=
-github.com/thanos-io/thanos v0.32.4-0.20231001083734-531cdb1e8ec3/go.mod h1:Px5Boq60s+2WwR+V4v4oxgmxfw9WHrwMwjRou6pkUNw=
+github.com/thanos-io/thanos v0.32.5-0.20231006043659-79bbf34b4275 h1:y2YPqM1XiBw7EhLg45F6A1g8bgt4yYxkaRAeQaNLWYk=
+github.com/thanos-io/thanos v0.32.5-0.20231006043659-79bbf34b4275/go.mod h1:HwiHn7u6GeES403BTACOYib/JKAJknf8dByU/uJiEr0=
 github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab h1:7ZR3hmisBWw77ZpO1/o86g+JV3VKlk3d48jopJxzTjU=
 github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab/go.mod h1:eheTFp954zcWZXCU8d0AT76ftsQOTo4DTqkN/h3k1MY=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=

pkg/storage/tsdb/index_cache.go

@@ -14,6 +14,7 @@ import (
 	storecache "github.com/thanos-io/thanos/pkg/store/cache"
 
 	"github.com/cortexproject/cortex/pkg/util"
+	"github.com/cortexproject/cortex/pkg/util/flagext"
 )
 
 const (
@@ -41,10 +42,10 @@ var (
 )
 
 type IndexCacheConfig struct {
-	Backend   string                   `yaml:"backend"`
-	InMemory  InMemoryIndexCacheConfig `yaml:"inmemory"`
-	Memcached MemcachedClientConfig    `yaml:"memcached"`
-	Redis     RedisClientConfig        `yaml:"redis"`
+	Backend   string                    `yaml:"backend"`
+	InMemory  InMemoryIndexCacheConfig  `yaml:"inmemory"`
+	Memcached MemcachedIndexCacheConfig `yaml:"memcached"`
+	Redis     RedisIndexCacheConfig     `yaml:"redis"`
 }
 
 func (cfg *IndexCacheConfig) RegisterFlags(f *flag.FlagSet) {
@@ -85,6 +86,10 @@ func (cfg *IndexCacheConfig) Validate() error {
 			if err := cfg.Redis.Validate(); err != nil {
 				return err
 			}
+		} else {
+			if err := cfg.InMemory.Validate(); err != nil {
+				return err
+			}
 		}
 
 		configuredBackends[backend] = struct{}{}
@@ -94,17 +99,63 @@ func (cfg *IndexCacheConfig) Validate() error {
 }
 
 type InMemoryIndexCacheConfig struct {
-	MaxSizeBytes uint64 `yaml:"max_size_bytes"`
+	MaxSizeBytes uint64   `yaml:"max_size_bytes"`
+	EnabledItems []string `yaml:"enabled_items"`
+}
+
+func (cfg *InMemoryIndexCacheConfig) Validate() error {
+	if err := storecache.ValidateEnabledItems(cfg.EnabledItems); err != nil {
+		return err
+	}
+	return nil
 }
 
 func (cfg *InMemoryIndexCacheConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string) {
 	f.Uint64Var(&cfg.MaxSizeBytes, prefix+"max-size-bytes", uint64(1*units.Gibibyte), "Maximum size in bytes of in-memory index cache used to speed up blocks index lookups (shared between all tenants).")
+	f.Var((*flagext.StringSlice)(&cfg.EnabledItems), prefix+"enabled-items", "Selectively cache index item types. Supported values are Postings, ExpandedPostings and Series")
+}
+
+type MemcachedIndexCacheConfig struct {
+	ClientConfig MemcachedClientConfig `yaml:",inline"`
+	EnabledItems []string              `yaml:"enabled_items"`
+}
+
+func (cfg *MemcachedIndexCacheConfig) Validate() error {
+	if err := cfg.ClientConfig.Validate(); err != nil {
+		return err
+	}
+	return storecache.ValidateEnabledItems(cfg.EnabledItems)
+}
+
+func (cfg *MemcachedIndexCacheConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string) {
+	cfg.ClientConfig.RegisterFlagsWithPrefix(f, prefix)
+	f.Var((*flagext.StringSlice)(&cfg.EnabledItems), prefix+"enabled-items", "Selectively cache index item types. Supported values are Postings, ExpandedPostings and Series")
+}
+
+type RedisIndexCacheConfig struct {
+	ClientConfig RedisClientConfig `yaml:",inline"`
+	EnabledItems []string          `yaml:"enabled_items"`
+}
+
+func (cfg *RedisIndexCacheConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string) {
+	cfg.ClientConfig.RegisterFlagsWithPrefix(f, prefix)
+	f.Var((*flagext.StringSlice)(&cfg.EnabledItems), prefix+"enabled-items", "Selectively cache index item types. Supported values are Postings, ExpandedPostings and Series")
+}
+
+func (cfg *RedisIndexCacheConfig) Validate() error {
+	if err := cfg.ClientConfig.Validate(); err != nil {
+		return err
+	}
+	return storecache.ValidateEnabledItems(cfg.EnabledItems)
 }
 
 // NewIndexCache creates a new index cache based on the input configuration.
 func NewIndexCache(cfg IndexCacheConfig, logger log.Logger, registerer prometheus.Registerer) (storecache.IndexCache, error) {
 	splitBackends := strings.Split(cfg.Backend, ",")
-	var caches []storecache.IndexCache
+	var (
+		caches       []storecache.IndexCache
+		enabledItems []string
+	)
 
 	for i, backend := range splitBackends {
 		iReg := registerer
@@ -121,8 +172,9 @@ func NewIndexCache(cfg IndexCacheConfig, logger log.Logger, registerer prometheu
 				return c, err
 			}
 			caches = append(caches, c)
+			enabledItems = cfg.InMemory.EnabledItems
 		case IndexCacheBackendMemcached:
-			c, err := newMemcachedIndexCacheClient(cfg.Memcached, logger, registerer)
+			c, err := newMemcachedIndexCacheClient(cfg.Memcached.ClientConfig, logger, registerer)
 			if err != nil {
 				return nil, err
 			}
@@ -131,8 +183,9 @@ func NewIndexCache(cfg IndexCacheConfig, logger log.Logger, registerer prometheu
 				return nil, err
 			}
 			caches = append(caches, cache)
+			enabledItems = cfg.Memcached.EnabledItems
 		case IndexCacheBackendRedis:
-			c, err := newRedisIndexCacheClient(cfg.Redis, logger, iReg)
+			c, err := newRedisIndexCacheClient(cfg.Redis.ClientConfig, logger, iReg)
 			if err != nil {
 				return nil, err
 			}
@@ -141,9 +194,15 @@ func NewIndexCache(cfg IndexCacheConfig, logger log.Logger, registerer prometheu
 				return nil, err
 			}
 			caches = append(caches, cache)
+			enabledItems = cfg.Redis.EnabledItems
 		default:
 			return nil, errUnsupportedIndexCacheBackend
 		}
+		if len(enabledItems) > 0 {
+			latestCache := caches[len(caches)-1]
+			cache := storecache.NewFilteredIndexCache(latestCache, enabledItems)
+			caches[len(caches)-1] = cache
+		}
 	}
 
 	return newMultiLevelCache(caches...), nil

pkg/storage/tsdb/index_cache_test.go

@@ -1,6 +1,7 @@
 package tsdb
 
 import (
+	"fmt"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -35,11 +36,46 @@ func TestIndexCacheConfig_Validate(t *testing.T) {
 		"one memcached address should pass": {
 			cfg: IndexCacheConfig{
 				Backend: "memcached",
-				Memcached: MemcachedClientConfig{
-					Addresses: "dns+localhost:11211",
+				Memcached: MemcachedIndexCacheConfig{
+					ClientConfig: MemcachedClientConfig{
+						Addresses: "dns+localhost:11211",
+					},
 				},
 			},
 		},
+		"invalid enabled items memcached": {
+			cfg: IndexCacheConfig{
+				Backend: "memcached",
+				Memcached: MemcachedIndexCacheConfig{
+					ClientConfig: MemcachedClientConfig{
+						Addresses: "dns+localhost:11211",
+					},
+					EnabledItems: []string{"foo", "bar"},
+				},
+			},
+			expected: fmt.Errorf("unsupported item type foo"),
+		},
+		"invalid enabled items inmemory": {
+			cfg: IndexCacheConfig{
+				Backend: "inmemory",
+				InMemory: InMemoryIndexCacheConfig{
+					EnabledItems: []string{"foo", "bar"},
+				},
+			},
+			expected: fmt.Errorf("unsupported item type foo"),
+		},
+		"invalid enabled items redis": {
+			cfg: IndexCacheConfig{
+				Backend: "redis",
+				Redis: RedisIndexCacheConfig{
+					ClientConfig: RedisClientConfig{
+						Addresses: "test",
+					},
+					EnabledItems: []string{"foo", "bar"},
+				},
+			},
+			expected: fmt.Errorf("unsupported item type foo"),
+		},
 	}
 
 	for testName, testData := range tests {