Add a query result limit to regex matcher

Signed-off-by: SungJin1212 <[email protected]>

Author: SungJin1212

CHANGELOG.md

@@ -18,6 +18,7 @@
 * [FEATURE] Distributor/Ingester: Implemented experimental feature to use gRPC stream connection for push requests. This can be enabled by setting `-distributor.use-stream-push=true`. #6580
 * [FEATURE] Compactor: Add support for percentage based sharding for compactors. #6738
 * [FEATURE] Querier: Allow choosing PromQL engine via header. #6777
+* [ENHANCEMENT] Tenant Federation: Add a # of query result limit logic when the `-tenant-federation.regex-matcher-enabled` is enabled.
 * [ENHANCEMENT] Query Frontend: Change to return 400 when the tenant resolving fail. #6715
 * [ENHANCEMENT] Querier: Support query parameters to metadata api (/api/v1/metadata) to allow user to limit metadata to return. Add a `-ingester.return-all-metadata` flag to make the metadata API run when the deployment. Please set this flag to `false` to use the metadata API with the limits later. #6681 #6744
 * [ENHANCEMENT] Ingester: Add a `cortex_ingester_active_native_histogram_series` metric to track # of active NH series. #6695

docs/configuration/config-file-reference.md

@@ -177,7 +177,9 @@ tenant_federation:
   # CLI flag: -tenant-federation.max-concurrent
   [max_concurrent: <int> | default = 16]
 
-  # A maximum number of tenants to query at once. 0 means no limit.
+  # A maximum number of tenants to query at once. 0 means no limit. If
+  # `-tenant-federation.regex-matcher-enabled` is enabled and this limit is
+  # greater than 0, the number of tenants to query is adjusted to this value.
   # CLI flag: -tenant-federation.max-tenant
   [max_tenant: <int> | default = 0]
 

pkg/cortex/modules.go

@@ -295,7 +295,7 @@ func (t *Cortex) initTenantFederation() (serv services.Service, err error) {
 				return bucket.NewClient(ctx, t.Cfg.BlocksStorage.Bucket, nil, "regex-resolver", util_log.Logger, prometheus.DefaultRegisterer)
 			}
 
-			regexResolver, err := tenantfederation.NewRegexResolver(t.Cfg.BlocksStorage.UsersScanner, prometheus.DefaultRegisterer, bucketClientFactory, t.Cfg.TenantFederation.UserSyncInterval, util_log.Logger)
+			regexResolver, err := tenantfederation.NewRegexResolver(t.Cfg.BlocksStorage.UsersScanner, t.Cfg.TenantFederation, prometheus.DefaultRegisterer, bucketClientFactory, util_log.Logger)
 			if err != nil {
 				return nil, fmt.Errorf("failed to initialize regex resolver: %v", err)
 			}

pkg/querier/tenantfederation/exemplar_merge_queryable_test.go

@@ -335,7 +335,8 @@ func Test_MergeExemplarQuerier_Select_WhenUseRegexResolver(t *testing.T) {
 	}
 
 	usersScannerConfig := cortex_tsdb.UsersScannerConfig{Strategy: cortex_tsdb.UserScanStrategyList}
-	regexResolver, err := NewRegexResolver(usersScannerConfig, reg, bucketClientFactory, time.Second, log.NewNopLogger())
+	tenantFederationConfig := Config{UserSyncInterval: time.Second}
+	regexResolver, err := NewRegexResolver(usersScannerConfig, tenantFederationConfig, reg, bucketClientFactory, log.NewNopLogger())
 	require.NoError(t, err)
 	tenant.WithDefaultResolver(regexResolver)
 	require.NoError(t, services.StartAndAwaitRunning(context.Background(), regexResolver))

pkg/querier/tenantfederation/merge_queryable_test.go

@@ -677,8 +677,8 @@ func TestMergeQueryable_Select(t *testing.T) {
 							}
 
 							usersScannerConfig := cortex_tsdb.UsersScannerConfig{Strategy: cortex_tsdb.UserScanStrategyList}
-
-							regexResolver, err := NewRegexResolver(usersScannerConfig, reg, bucketClientFactory, time.Second, log.NewNopLogger())
+							tenantFederationConfig := Config{UserSyncInterval: time.Second}
+							regexResolver, err := NewRegexResolver(usersScannerConfig, tenantFederationConfig, reg, bucketClientFactory, log.NewNopLogger())
 							require.NoError(t, err)
 
 							// set a regex tenant resolver
@@ -876,8 +876,8 @@ func TestMergeQueryable_LabelNames(t *testing.T) {
 						return bucketClient, nil
 					}
 					usersScannerConfig := cortex_tsdb.UsersScannerConfig{Strategy: cortex_tsdb.UserScanStrategyList}
-
-					regexResolver, err := NewRegexResolver(usersScannerConfig, reg, bucketClientFactory, time.Second, log.NewNopLogger())
+					tenantFederationConfig := Config{UserSyncInterval: time.Second}
+					regexResolver, err := NewRegexResolver(usersScannerConfig, tenantFederationConfig, reg, bucketClientFactory, log.NewNopLogger())
 					require.NoError(t, err)
 
 					// set a regex tenant resolver
@@ -1114,8 +1114,8 @@ func TestMergeQueryable_LabelValues(t *testing.T) {
 								return bucketClient, nil
 							}
 							usersScannerConfig := cortex_tsdb.UsersScannerConfig{Strategy: cortex_tsdb.UserScanStrategyList}
-
-							regexResolver, err := NewRegexResolver(usersScannerConfig, reg, bucketClientFactory, time.Second, log.NewNopLogger())
+							tenantFederationConfig := Config{UserSyncInterval: time.Second}
+							regexResolver, err := NewRegexResolver(usersScannerConfig, tenantFederationConfig, reg, bucketClientFactory, log.NewNopLogger())
 							require.NoError(t, err)
 
 							// set a regex tenant resolver

pkg/querier/tenantfederation/metadata_merge_querier_test.go

@@ -169,7 +169,8 @@ func Test_mergeMetadataQuerier_MetricsMetadata_WhenUseRegexResolver(t *testing.T
 	}
 
 	usersScannerConfig := cortex_tsdb.UsersScannerConfig{Strategy: cortex_tsdb.UserScanStrategyList}
-	regexResolver, err := NewRegexResolver(usersScannerConfig, reg, bucketClientFactory, time.Second, log.NewNopLogger())
+	tenantFederationConfig := Config{UserSyncInterval: time.Second}
+	regexResolver, err := NewRegexResolver(usersScannerConfig, tenantFederationConfig, reg, bucketClientFactory, log.NewNopLogger())
 	require.NoError(t, err)
 	tenant.WithDefaultResolver(regexResolver)
 	require.NoError(t, services.StartAndAwaitRunning(context.Background(), regexResolver))

pkg/querier/tenantfederation/regex_resolver.go

@@ -32,6 +32,7 @@ type RegexResolver struct {
 
 	knownUsers       []string
 	userSyncInterval time.Duration
+	maxTenant        int
 	userScanner      users.Scanner
 	logger           log.Logger
 	sync.Mutex
@@ -42,7 +43,7 @@ type RegexResolver struct {
 	discoveredUsers prometheus.Gauge
 }
 
-func NewRegexResolver(cfg tsdb.UsersScannerConfig, reg prometheus.Registerer, bucketClientFactory func(ctx context.Context) (objstore.InstrumentedBucket, error), userSyncInterval time.Duration, logger log.Logger) (*RegexResolver, error) {
+func NewRegexResolver(cfg tsdb.UsersScannerConfig, tenantFederationCfg Config, reg prometheus.Registerer, bucketClientFactory func(ctx context.Context) (objstore.InstrumentedBucket, error), logger log.Logger) (*RegexResolver, error) {
 	bucketClient, err := bucketClientFactory(context.Background())
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to create the bucket client")
@@ -54,7 +55,8 @@ func NewRegexResolver(cfg tsdb.UsersScannerConfig, reg prometheus.Registerer, bu
 	}
 
 	r := &RegexResolver{
-		userSyncInterval: userSyncInterval,
+		userSyncInterval: tenantFederationCfg.UserSyncInterval,
+		maxTenant:        tenantFederationCfg.MaxTenant,
 		userScanner:      userScanner,
 		logger:           logger,
 	}
@@ -160,6 +162,11 @@ func (r *RegexResolver) getRegexMatchedOrgIds(orgID string) ([]string, error) {
 		return []string{"fake"}, nil
 	}
 
+	if r.maxTenant > 0 && len(matched) > r.maxTenant {
+		// adjust matched tenants to maxTenant
+		matched = matched[:r.maxTenant]
+	}
+
 	return matched, nil
 }
 

pkg/querier/tenantfederation/regex_resolver_test.go

@@ -25,6 +25,7 @@ func Test_RegexResolver(t *testing.T) {
 		orgID           string
 		expectedErr     error
 		expectedOrgIDs  []string
+		maxTenants      int
 	}{
 		{
 			description:     "invalid regex",
@@ -62,6 +63,13 @@ func Test_RegexResolver(t *testing.T) {
 			orgID:           "user-1|user-2",
 			expectedOrgIDs:  []string{"user-1"},
 		},
+		{
+			description:     "adjust maxTenant",
+			existingTenants: []string{"user-1", "user-2", "user-3"},
+			orgID:           "user-.+",
+			expectedOrgIDs:  []string{"user-1", "user-2"},
+			maxTenants:      2,
+		},
 	}
 
 	for _, tc := range tests {
@@ -80,7 +88,8 @@ func Test_RegexResolver(t *testing.T) {
 			}
 
 			usersScannerConfig := cortex_tsdb.UsersScannerConfig{Strategy: cortex_tsdb.UserScanStrategyList}
-			regexResolver, err := NewRegexResolver(usersScannerConfig, reg, bucketClientFactory, time.Second, log.NewNopLogger())
+			tenantFederationConfig := Config{UserSyncInterval: time.Second, MaxTenant: tc.maxTenants}
+			regexResolver, err := NewRegexResolver(usersScannerConfig, tenantFederationConfig, reg, bucketClientFactory, log.NewNopLogger())
 			require.NoError(t, err)
 			require.NoError(t, services.StartAndAwaitRunning(context.Background(), regexResolver))
 

pkg/querier/tenantfederation/tenant_federation.go

@@ -21,7 +21,7 @@ type Config struct {
 func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
 	f.BoolVar(&cfg.Enabled, "tenant-federation.enabled", false, "If enabled on all Cortex services, queries can be federated across multiple tenants. The tenant IDs involved need to be specified separated by a `|` character in the `X-Scope-OrgID` header (experimental).")
 	f.IntVar(&cfg.MaxConcurrent, "tenant-federation.max-concurrent", defaultMaxConcurrency, "The number of workers used to process each federated query.")
-	f.IntVar(&cfg.MaxTenant, "tenant-federation.max-tenant", 0, "A maximum number of tenants to query at once. 0 means no limit.")
+	f.IntVar(&cfg.MaxTenant, "tenant-federation.max-tenant", 0, "A maximum number of tenants to query at once. 0 means no limit. If `-tenant-federation.regex-matcher-enabled` is enabled and this limit is greater than 0, the number of tenants to query is adjusted to this value.")
 	f.BoolVar(&cfg.RegexMatcherEnabled, "tenant-federation.regex-matcher-enabled", false, "[Experimental] If enabled, the `X-Scope-OrgID` header value can accept a regex and the matched tenantIDs are automatically involved. The regex matching rule follows the Prometheus, see the detail: https://prometheus.io/docs/prometheus/latest/querying/basics/#regular-expressions. The user discovery is based on scanning block storage, so new users can get queries after uploading a block (generally 2h).")
 	f.DurationVar(&cfg.UserSyncInterval, "tenant-federation.user-sync-interval", time.Minute*5, "[Experimental] If the regex matcher is enabled, it specifies how frequently to scan users. The scanned users are used to calculate matched tenantIDs. The scanning strategy depends on the `-blocks-storage.users-scanner.strategy`.")
 }