Merge branch 'master' into cache-freshness

Signed-off-by: Annanay <[email protected]>

Author: annanay25

CHANGELOG.md

@@ -34,6 +34,7 @@
 * [FEATURE] Experimental: Added support for `/api/v1/metadata` Prometheus-based endpoint. #2549
 * [FEATURE] Add ability to limit concurrent queries to Cassandra with `-cassandra.query-concurrency` flag. #2562
 * [FEATURE] TLS config options added for GRPC clients in Querier (Query-frontend client & Ingester client), Ruler, Store Gateway, as well as HTTP client in Config store client. #2502
+* [ENHANCEMENT] `query-tee` supports `/metadata`, `/alerts`, and `/rules` #2600
 * [ENHANCEMENT] Ruler: Automatically remove unhealthy rulers from the ring. #2587
 * [ENHANCEMENT] Experimental TSDB: sample ingestion errors are now reported via existing `cortex_discarded_samples_total` metric. #2370
 * [ENHANCEMENT] Failures on samples at distributors and ingesters return the first validation error as opposed to the last. #2383
@@ -82,6 +83,7 @@
   * `cortex_storegateway_blocks_last_successful_sync_timestamp_seconds`
 * [ENHANCEMENT] Experimental TSDB: added the flag `-experimental.tsdb.wal-compression-enabled` to allow to enable TSDB WAL compression. #2585
 * [ENHANCEMENT] Experimental TSDB: Querier and store-gateway components can now use so-called "caching bucket", which can currently cache fetched chunks into shared memcached server. #2572
+* [ENHANCEMENT] Experimental TSDB: added per-tenant blocks sharding support in the compactor, in order to parallelize blocks compaction for a single tenant in a single node. Sharding can be enabled via `-compactor.per-tenant-num-shards` while the parallelization can be controlled with `-compactor.per-tenant-shards-concurrency`. #2599
 * [BUGFIX] Ruler: Ensure temporary rule files with special characters are properly mapped and cleaned up. #2506
 * [BUGFIX] Fixes #2411, Ensure requests are properly routed to the prometheus api embedded in the query if `-server.path-prefix` is set. #2372
 * [BUGFIX] Experimental TSDB: fixed chunk data corruption when querying back series using the experimental blocks storage. #2400
@@ -95,6 +97,7 @@
 * [BUGFIX] Querier: Fixed a situation where querier would crash because of an unresponsive frontend instance. #2569
 * [BUGFIX] Fixed collection of tracing spans from Thanos components used internally. #2584
 * [BUGFIX] Experimental TSDB: fixed memory leak in ingesters. #2586
+* [BUGFIX] Ingester: Fix an ingester starting up in the JOINING state and staying there forever. #2565
 
 ## 1.0.1 / 2020-04-23
 

cmd/cortex/main.go

@@ -104,7 +104,7 @@ func main() {
 	util.InitEvents(eventSampleRate)
 
 	// Setting the environment variable JAEGER_AGENT_HOST enables tracing
-	if trace, err := tracing.NewFromEnv("cortex-" + cfg.Target.String()); err != nil {
+	if trace, err := tracing.NewFromEnv("cortex-" + cfg.Target); err != nil {
 		level.Error(util.Logger).Log("msg", "Failed to setup tracing", "err", err.Error())
 	} else {
 		defer trace.Close()

cmd/cortex/main_test.go

@@ -26,17 +26,6 @@ func TestFlagParsing(t *testing.T) {
 			stderrMessage: configFileOption,
 		},
 
-		// check that config file is used
-		"config with unknown target": {
-			yaml:          "target: unknown",
-			stderrMessage: "unrecognised module name: unknown",
-		},
-
-		"argument with unknown target": {
-			arguments:     []string{"-target=unknown"},
-			stderrMessage: "unrecognised module name: unknown",
-		},
-
 		"unknown flag": {
 			arguments:     []string{"-unknown.flag"},
 			stderrMessage: "-unknown.flag",
@@ -48,12 +37,6 @@ func TestFlagParsing(t *testing.T) {
 			stdoutMessage: "target: ingester",
 		},
 
-		"config with wrong argument override": {
-			yaml:          "target: ingester",
-			arguments:     []string{"-target=unknown"},
-			stderrMessage: "unrecognised module name: unknown",
-		},
-
 		"default values": {
 			stdoutMessage: "target: all\n",
 		},
@@ -63,11 +46,6 @@ func TestFlagParsing(t *testing.T) {
 			stdoutMessage: "target: ingester\n",
 		},
 
-		"config without expand-env": {
-			yaml:          "target: $TARGET",
-			stderrMessage: "Error parsing config file: unrecognised module name: $TARGET\n",
-		},
-
 		"config with expand-env": {
 			arguments:     []string{"-config.expand-env"},
 			yaml:          "target: $TARGET",

cmd/query-tee/proxy.go

@@ -99,12 +99,25 @@ func (p *Proxy) Start() error {
 		w.WriteHeader(http.StatusOK)
 	}))
 
-	// Read endpoints.
-	router.Path("/api/v1/query").Methods("GET").Handler(NewProxyEndpoint(p.backends, "api_v1_query", p.metrics, p.logger))
-	router.Path("/api/v1/query_range").Methods("GET").Handler(NewProxyEndpoint(p.backends, "api_v1_query_range", p.metrics, p.logger))
-	router.Path("/api/v1/labels").Methods("GET").Handler(NewProxyEndpoint(p.backends, "api_v1_labels", p.metrics, p.logger))
-	router.Path("/api/v1/label/{name}/values").Methods("GET").Handler(NewProxyEndpoint(p.backends, "api_v1_label_name_values", p.metrics, p.logger))
-	router.Path("/api/v1/series").Methods("GET").Handler(NewProxyEndpoint(p.backends, "api_v1_series", p.metrics, p.logger))
+	endpoints := []struct {
+		endpoint  string
+		methods   string
+		routeName string
+	}{
+		{"/api/v1/query", "GET", "api_v1_query"},
+		{"/api/v1/query_range", "GET", "api_v1_query_range"},
+		{"/api/v1/labels", "GET", "api_v1_labels"},
+		{"/api/v1/label/{name}/values", "GET", "api_v1_label_name_values"},
+		{"/api/v1/series", "GET", "api_v1_series"},
+		{"/api/v1/metadata", "GET", "api_v1_metadata"},
+		{"/api/v1/rules", "GET", "api_v1_rules"},
+		{"/api/v1/alerts", "GET", "api_v1_alerts"},
+	}
+
+	// Read endpoints
+	for _, e := range endpoints {
+		router.Path(e.endpoint).Methods(e.methods).Handler(NewProxyEndpoint(p.backends, e.routeName, p.metrics, p.logger))
+	}
 
 	p.srvListener = listener
 	p.srv = &http.Server{

docs/configuration/config-file-reference.md

@@ -2993,6 +2993,15 @@ sharding_ring:
   # the ring.
   # CLI flag: -compactor.ring.heartbeat-timeout
   [heartbeat_timeout: <duration> | default = 1m]
+
+# Number of shards a single tenant blocks should be grouped into (0 or 1 means
+# per-tenant blocks sharding is disabled).
+# CLI flag: -compactor.per-tenant-num-shards
+[per_tenant_num_shards: <int> | default = 1]
+
+# Number of concurrent shards compacted for a single tenant.
+# CLI flag: -compactor.per-tenant-shards-concurrency
+[per_tenant_shards_concurrency: <int> | default = 1]
 ```
 
 ### `store_gateway_config`

docs/operations/blocks-storage.md

@@ -613,6 +613,15 @@ compactor:
     # within the ring.
     # CLI flag: -compactor.ring.heartbeat-timeout
     [heartbeat_timeout: <duration> | default = 1m]
+
+  # Number of shards a single tenant blocks should be grouped into (0 or 1 means
+  # per-tenant blocks sharding is disabled).
+  # CLI flag: -compactor.per-tenant-num-shards
+  [per_tenant_num_shards: <int> | default = 1]
+
+  # Number of concurrent shards compacted for a single tenant.
+  # CLI flag: -compactor.per-tenant-shards-concurrency
+  [per_tenant_shards_concurrency: <int> | default = 1]
 ```
 
 ## Known issues

docs/operations/query-tee.md

@@ -35,6 +35,9 @@ The following Prometheus API endpoints are supported by `query-tee`:
 - `/api/v1/labels` (GET)
 - `/api/v1/label/{name}/values` (GET)
 - `/api/v1/series` (GET)
+- `/api/v1/metadata` (GET)
+- `/api/v1/alerts` (GET)
+- `/api/v1/rules` (GET)
 
 ### Authentication
 

go.mod

@@ -50,7 +50,7 @@ require (
 	github.com/segmentio/fasthash v0.0.0-20180216231524-a72b379d632e
 	github.com/spf13/afero v1.2.2
 	github.com/stretchr/testify v1.4.0
-	github.com/thanos-io/thanos v0.12.3-0.20200507181659-b9ff23c5c31d
+	github.com/thanos-io/thanos v0.12.3-0.20200518091645-762eb5e1dfab
 	github.com/uber/jaeger-client-go v2.20.1+incompatible
 	github.com/weaveworks/common v0.0.0-20200512154658-384f10054ec5
 	go.etcd.io/bbolt v1.3.3

go.sum

@@ -869,8 +869,8 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/thanos-io/thanos v0.8.1-0.20200109203923-552ffa4c1a0d/go.mod h1:usT/TxtJQ7DzinTt+G9kinDQmRS5sxwu0unVKZ9vdcw=
-github.com/thanos-io/thanos v0.12.3-0.20200507181659-b9ff23c5c31d h1:rNFFyjaZmsyB0EItY6EM+QN2PdDoxIeCNy8CbIbIxNo=
-github.com/thanos-io/thanos v0.12.3-0.20200507181659-b9ff23c5c31d/go.mod h1:N3stAuvPDLaWMQym7uDXAoAanu4ujieMjK7zzxJzky4=
+github.com/thanos-io/thanos v0.12.3-0.20200518091645-762eb5e1dfab h1:MDobfwpPIBYUbSaynWnGKNKECcAxYiJO9d8q/TjJm4M=
+github.com/thanos-io/thanos v0.12.3-0.20200518091645-762eb5e1dfab/go.mod h1:N3stAuvPDLaWMQym7uDXAoAanu4ujieMjK7zzxJzky4=
 github.com/tidwall/pretty v0.0.0-20180105212114-65a9db5fad51/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=

pkg/compactor/blocks_sharding_filter.go

@@ -0,0 +1,66 @@
+package compactor
+
+import (
+	"context"
+	"strconv"
+
+	"github.com/oklog/ulid"
+	"github.com/thanos-io/thanos/pkg/block/metadata"
+	"github.com/thanos-io/thanos/pkg/extprom"
+
+	"github.com/cortexproject/cortex/pkg/ingester/client"
+	cortex_tsdb "github.com/cortexproject/cortex/pkg/storage/tsdb"
+)
+
+type BlocksShardingFilter struct {
+	shards uint32
+}
+
+func NewBlocksShardingFilter(shards uint32) *BlocksShardingFilter {
+	return &BlocksShardingFilter{
+		shards: shards,
+	}
+}
+
+func (f *BlocksShardingFilter) Filter(_ context.Context, metas map[ulid.ULID]*metadata.Meta, _ *extprom.TxGaugeVec) error {
+	for _, m := range metas {
+		// Just remove the ingester ID label if sharding is disabled.
+		if f.shards <= 1 {
+			delete(m.Thanos.Labels, cortex_tsdb.IngesterIDExternalLabel)
+			continue
+		}
+
+		// Skip any block already containing the shard ID to avoid any manipulation
+		// (ie. in case the sharding algorithm changes over the time).
+		if _, ok := m.Thanos.Labels[cortex_tsdb.ShardIDExternalLabel]; ok {
+			continue
+		}
+
+		// Skip any block without the ingester ID, which means the block has been generated
+		// before the introduction of the ingester ID.
+		ingesterID, ok := m.Thanos.Labels[cortex_tsdb.IngesterIDExternalLabel]
+		if !ok {
+			continue
+		}
+
+		// Compute the shard ID and replace the ingester label with the shard label
+		// so that the compactor will group blocks based on the shard.
+		shardID := shardByIngesterID(ingesterID, f.shards)
+		delete(m.Thanos.Labels, cortex_tsdb.IngesterIDExternalLabel)
+		m.Thanos.Labels[cortex_tsdb.ShardIDExternalLabel] = shardID
+	}
+
+	return nil
+}
+
+// hashIngesterID returns a 32-bit hash of the ingester ID.
+func hashIngesterID(id string) uint32 {
+	h := client.HashNew32()
+	h = client.HashAdd32(h, id)
+	return h
+}
+
+// shardByIngesterID returns the shard given an ingester ID.
+func shardByIngesterID(id string, numShards uint32) string {
+	return strconv.Itoa(int(hashIngesterID(id) % numShards))
+}