Add shuffle sharding grouper/planner

.errcheck-exclude

@@ -1,6 +1,6 @@
 io/ioutil.WriteFile
 io/ioutil.ReadFile
-(github.com/go-kit/kit/log.Logger).Log
+(github.com/go-kit/log.Logger).Log
 io.Copy
 (github.com/opentracing/opentracing-go.Tracer).Inject
 (*github.com/cortexproject/cortex/pkg/util/spanlogger.SpanLogger).Error

.github/workflows/test-build-deploy.yml

@@ -10,7 +10,7 @@ jobs:
   lint:
     runs-on: ubuntu-20.04
     container:
-      image: quay.io/cortexproject/build-image:build-image-multiarch-1d2497ff6
+      image: quay.io/cortexproject/build-image:master-f0c5aab05
     steps:
       - name: Checkout Repo
         uses: actions/checkout@v2
@@ -34,7 +34,7 @@ jobs:
   test:
     runs-on: ubuntu-20.04
     container:
-      image: quay.io/cortexproject/build-image:build-image-multiarch-1d2497ff6
+      image: quay.io/cortexproject/build-image:master-f0c5aab05
     services:
       cassandra:
         image: cassandra:3.11
@@ -55,7 +55,7 @@ jobs:
   build:
     runs-on: ubuntu-20.04
     container:
-      image: quay.io/cortexproject/build-image:build-image-multiarch-1d2497ff6
+      image: quay.io/cortexproject/build-image:master-f0c5aab05
     steps:
       - name: Checkout Repo
         uses: actions/checkout@v2
@@ -96,13 +96,9 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: Upgrade golang
-        run: |
-          cd /tmp
-          wget https://dl.google.com/go/go1.16.3.linux-amd64.tar.gz
-          tar -zxvf go1.16.3.linux-amd64.tar.gz
-          sudo rm -fr /usr/local/go
-          sudo mv /tmp/go /usr/local/go
-          cd -
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.17.5
       - name: Checkout Repo
         uses: actions/checkout@v2
       - name: Install Docker Client
@@ -140,6 +136,7 @@ jobs:
           docker pull quay.io/cortexproject/cortex:v1.7.0
           docker pull quay.io/cortexproject/cortex:v1.8.0
           docker pull quay.io/cortexproject/cortex:v1.9.0
+          docker pull quay.io/cortexproject/cortex:v1.10.0
           docker pull shopify/bigtable-emulator:0.1.0
           docker pull rinscy/cassandra:3.11.0
           docker pull memcached:1.6.1
@@ -174,14 +171,14 @@ jobs:
         run: |
           touch build-image/.uptodate
           MIGRATIONS_DIR=$(pwd)/cmd/cortex/migrations
-          make BUILD_IMAGE=quay.io/cortexproject/build-image:build-image-multiarch-1d2497ff6 TTY='' configs-integration-test
+          make BUILD_IMAGE=quay.io/cortexproject/build-image:master-f0c5aab05 TTY='' configs-integration-test
 
   deploy_website:
     needs: [build, test]
     if: (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) && github.repository == 'cortexproject/cortex'
     runs-on: ubuntu-20.04
     container:
-      image: quay.io/cortexproject/build-image:build-image-multiarch-1d2497ff6
+      image: quay.io/cortexproject/build-image:master-f0c5aab05
     steps:
       - name: Checkout Repo
         uses: actions/checkout@v2
@@ -218,7 +215,7 @@ jobs:
     if: (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) && github.repository == 'cortexproject/cortex'
     runs-on: ubuntu-20.04
     container:
-      image: quay.io/cortexproject/build-image:build-image-multiarch-1d2497ff6
+      image: quay.io/cortexproject/build-image:master-f0c5aab05
     steps:
       - name: Checkout Repo
         uses: actions/checkout@v2

.golangci.yml

@@ -7,6 +7,7 @@ linters:
     - golint
     - gofmt
     - misspell
+    - depguard
 
 linters-settings:
   errcheck:
@@ -16,6 +17,12 @@ linters-settings:
   goimports:
     local-prefixes: "github.com/cortexproject/cortex"
 
+  depguard:
+    list-type: blacklist
+    include-go-root: true
+    packages-with-error-message:
+      - github.com/go-kit/kit/log: "Use github.com/go-kit/log instead of github.com/go-kit/kit/log"
+
 run:
   timeout: 5m
 

ADOPTERS.md

@@ -12,6 +12,7 @@ This is the list of organisations that are using Cortex in **production environm
 * [GoJek](https://www.gojek.io/)
 * [GrafanaLabs](https://grafana.com/)
 * [MayaData](https://mayadata.io/)
+* [Northflank](https://northflank.com/)
 * [Opstrace](https://opstrace.com/)
 * [Platform9](https://platform9.com/)
 * [REWE Digital](https://rewe-digital.com/)

CHANGELOG.md

@@ -2,8 +2,94 @@
 
 ## master / unreleased
 
-* [CHANGE] Querier / ruler: Change `-querier.max-fetched-chunks-per-query` configuration to limit to maximum number of chunks that can be fetched in a single query. The number of chunks fetched by ingesters AND long-term storare combined should not exceed the value configured on `-querier.max-fetched-chunks-per-query`. #4260
+* [CHANGE] Changed default for `-ingester.min-ready-duration` from 1 minute to 15 seconds. #4539
+* [CHANGE] query-frontend: Do not print anything in the logs of `query-frontend` if a in-progress query has been canceled (context canceled). #4562
+* [CHANGE] Compactor block deletion mark migration, needed when upgrading from v1.7, is now disabled by default. #4597
+* [CHANGE] The `status_code` label on gRPC client metrics has changed from '200' and '500' to '2xx', '5xx', '4xx', 'cancel' or 'error'. 4601
+* [CHANGE] Memberlist: changed probe interval from `1s` to `5s` and probe timeout from `500ms` to `2s`. #4601
+* [FEATURE] Add shuffle sharding grouper and planner within compactor to allow further work towards parallelizing compaction #4621
+* [ENHANCEMENT] Update Go version to 1.17.5. #4602 #4604
+* [ENHANCEMENT] Keep track of discarded samples due to relabel configuration in `cortex_discarded_samples_total`. #4503
+* [ENHANCEMENT] Ruler: Add `-ruler.disable-rule-group-label` to disable the `rule_group` label on exported metrics. #4571
+* [ENHANCEMENT] Query federation: improve performance in MergeQueryable by memoizing labels. #4502
+* [ENHANCEMENT] Added new ring related config `-ingester.readiness-check-ring-health` when enabled the readiness probe will succeed only after all instances are ACTIVE and healthy in the ring, this is enabled by default. #4539
+* [ENHANCEMENT] Added new ring related config `-distributor.excluded-zones` when set this will exclude the comma-separated zones from the ring, default is "". #4539
+* [ENHANCEMENT] Upgraded Docker base images to `alpine:3.14`. #4514
+* [ENHANCEMENT] Updated Prometheus to latest. Includes changes from prometheus#9239, adding 15 new functions. Multiple TSDB bugfixes prometheus#9438 & prometheus#9381. #4524
+* [ENHANCEMENT] Query Frontend: Add setting `-frontend.forward-headers-list` in frontend  to configure the set of headers from the requests to be forwarded to downstream requests. #4486
+* [ENHANCEMENT] Blocks storage: Add `-blocks-storage.azure.http.*`, `-alertmanager-storage.azure.http.*`, and `-ruler-storage.azure.http.*` to configure the Azure storage client. #4581
+* [ENHANCEMENT] Optimise memberlist receive path when used as a backing store for rings with a large number of members. #4601
+* [BUGFIX] AlertManager: remove stale template files. #4495
+* [BUGFIX] Distributor: fix bug in query-exemplar where some results would get dropped. #4582
+* [BUGFIX] Update Thanos dependency: compactor tracing support, azure blocks storage memory fix. #4585
+* [BUGFIX] Querier: Disable query scheduler SRV DNS lookup, which removes noisy log messages about "failed DNS SRV record lookup". #4601
+* [BUGFIX] Memberlist: fixed corrupted packets when sending compound messages with more than 255 messages or messages bigger than 64KB. #4601
+
+## 1.11.0 2021-11-25
+
+* [CHANGE] Memberlist: Expose default configuration values to the command line options. Note that setting these explicitly to zero will no longer cause the default to be used. If the default is desired, then do set the option. The following are affected: #4276
+  - `-memberlist.stream-timeout`
+  - `-memberlist.retransmit-factor`
+  - `-memberlist.pull-push-interval`
+  - `-memberlist.gossip-interval`
+  - `-memberlist.gossip-nodes`
+  - `-memberlist.gossip-to-dead-nodes-time`
+  - `-memberlist.dead-node-reclaim-time`
+* [CHANGE] `-querier.max-fetched-chunks-per-query` previously applied to chunks from ingesters and store separately; now the two combined should not exceed the limit. #4260
+* [CHANGE] Memberlist: the metric `memberlist_kv_store_value_bytes` has been removed due to values no longer being stored in-memory as encoded bytes. #4345
+* [CHANGE] Some files and directories created by Cortex components on local disk now have stricter permissions, and are only readable by owner, but not group or others. #4394
+* [CHANGE] The metric `cortex_deprecated_flags_inuse_total` has been renamed to `deprecated_flags_inuse_total` as part of using grafana/dskit functionality. #4443
+* [FEATURE] Ruler: Add new `-ruler.query-stats-enabled` which when enabled will report the `cortex_ruler_query_seconds_total` as a per-user metric that tracks the sum of the wall time of executing queries in the ruler in seconds. #4317
+* [FEATURE] Query Frontend: Add `cortex_query_fetched_series_total` and `cortex_query_fetched_chunks_bytes_total` per-user counters to expose the number of series and bytes fetched as part of queries. These metrics can be enabled with the `-frontend.query-stats-enabled` flag (or its respective YAML config option `query_stats_enabled`). #4343
+* [FEATURE] AlertManager: Add support for SNS Receiver. #4382
+* [FEATURE] Distributor: Add label `status` to metric `cortex_distributor_ingester_append_failures_total` #4442
+* [FEATURE] Queries: Added `present_over_time` PromQL function, also some TSDB optimisations. #4505
+* [ENHANCEMENT] Add timeout for waiting on compactor to become ACTIVE in the ring. #4262
+* [ENHANCEMENT] Reduce memory used by streaming queries, particularly in ruler. #4341
+* [ENHANCEMENT] Ring: allow experimental configuration of disabling of heartbeat timeouts by setting the relevant configuration value to zero. Applies to the following: #4342
+  * `-distributor.ring.heartbeat-timeout`
+  * `-ring.heartbeat-timeout`
+  * `-ruler.ring.heartbeat-timeout`
+  * `-alertmanager.sharding-ring.heartbeat-timeout`
+  * `-compactor.ring.heartbeat-timeout`
+  * `-store-gateway.sharding-ring.heartbeat-timeout`
+* [ENHANCEMENT] Ring: allow heartbeats to be explicitly disabled by setting the interval to zero. This is considered experimental. This applies to the following configuration options: #4344
+  * `-distributor.ring.heartbeat-period`
+  * `-ingester.heartbeat-period`
+  * `-ruler.ring.heartbeat-period`
+  * `-alertmanager.sharding-ring.heartbeat-period`
+  * `-compactor.ring.heartbeat-period`
+  * `-store-gateway.sharding-ring.heartbeat-period`
+* [ENHANCEMENT] Memberlist: optimized receive path for processing ring state updates, to help reduce CPU utilization in large clusters. #4345
+* [ENHANCEMENT] Memberlist: expose configuration of memberlist packet compression via `-memberlist.compression=enabled`. #4346
+* [ENHANCEMENT] Update Go version to 1.16.6. #4362
+* [ENHANCEMENT] Updated Prometheus to include changes from prometheus/prometheus#9083. Now whenever `/labels` API calls include matchers, blocks store is queried for `LabelNames` with matchers instead of `Series` calls which was inefficient. #4380
+* [ENHANCEMENT] Querier: performance improvements in socket and memory handling. #4429 #4377
+* [ENHANCEMENT] Exemplars are now emitted for all gRPC calls and many operations tracked by histograms. #4462
+* [ENHANCEMENT] New options `-server.http-listen-network` and `-server.grpc-listen-network` allow binding as 'tcp4' or 'tcp6'. #4462
+* [ENHANCEMENT] Rulers: Using shuffle sharding subring on GetRules API. #4466
+* [ENHANCEMENT] Support memcached auto-discovery via `auto-discovery` flag, introduced by thanos in https://github.com/thanos-io/thanos/pull/4487. Both AWS and Google Cloud memcached service support auto-discovery, which returns a list of nodes of the memcached cluster. #4412
+* [BUGFIX] Fixes a panic in the query-tee when comparing result. #4465
+* [BUGFIX] Frontend: Fixes @ modifier functions (start/end) when splitting queries by time. #4464
+* [BUGFIX] Compactor: compactor will no longer try to compact blocks that are already marked for deletion. Previously compactor would consider blocks marked for deletion within `-compactor.deletion-delay / 2` period as eligible for compaction. #4328
 * [BUGFIX] HA Tracker: when cleaning up obsolete elected replicas from KV store, tracker didn't update number of cluster per user correctly. #4336
+* [BUGFIX] Ruler: fixed counting of PromQL evaluation errors as user-errors when updating `cortex_ruler_queries_failed_total`. #4335
+* [BUGFIX] Ingester: When using block storage, prevent any reads or writes while the ingester is stopping. This will prevent accessing TSDB blocks once they have been already closed. #4304
+* [BUGFIX] Ingester: fixed ingester stuck on start up (LEAVING ring state) when `-ingester.heartbeat-period=0` and `-ingester.unregister-on-shutdown=false`. #4366
+* [BUGFIX] Ingester: panic during shutdown while fetching batches from cache. #4397
+* [BUGFIX] Querier: After query-frontend restart, querier may have lower than configured concurrency. #4417
+* [BUGFIX] Memberlist: forward only changes, not entire original message. #4419
+* [BUGFIX] Memberlist: don't accept old tombstones as incoming change, and don't forward such messages to other gossip members. #4420
+* [BUGFIX] Querier: fixed panic when querying exemplars and using `-distributor.shard-by-all-labels=false`. #4473
+* [BUGFIX] Querier: honor querier minT,maxT if `nil` SelectHints are passed to Select(). #4413
+* [BUGFIX] Compactor: fixed panic while collecting Prometheus metrics. #4483
+* [BUGFIX] Update go-kit package to fix spurious log messages #4544
+
+
+## 1.10.0 / 2021-08-03
+
+* [CHANGE] Prevent path traversal attack from users able to control the HTTP header `X-Scope-OrgID`. #4375 (CVE-2021-36157)
+  * Users only have control of the HTTP header when Cortex is not frontend by an auth proxy validating the tenant IDs
 
 ## 1.10.0-rc.0 / 2021-06-28
 
@@ -73,6 +159,7 @@
 * [BUGFIX] Store-gateway: when blocks sharding is enabled, do not load all blocks in each store-gateway in case of a cold startup, but load only blocks owned by the store-gateway replica. #4271
 * [BUGFIX] Memberlist: fix to setting the default configuration value for `-memberlist.retransmit-factor` when not provided. This should improve propagation delay of the ring state (including, but not limited to, tombstones). Note that if the configuration is already explicitly given, this fix has no effect. #4269
 * [BUGFIX] Querier: Fix issue where samples in a chunk might get skipped by batch iterator. #4218
+
 ## Blocksconvert
 
 * [ENHANCEMENT] Scanner: add support for DynamoDB (v9 schema only). #3828

MAINTAINERS

@@ -1,6 +1,7 @@
-Bryan Boreham <[email protected]> (@bboreham)
+Alvin Lin, Amazon Web Services <[email protected]> (@alvinlin123)
+Bryan Boreham, Grafana Labs <[email protected]> (@bboreham)
 Goutham Veeramachaneni, Grafana Labs <[email protected]> (@gouthamve)
-Jacob Lisi, Grafana Labs <[email protected]> (@jtlisi)
+Jacob Lisi, Google <[email protected]> (@jtlisi)
 Marco Pracucci, Grafana Labs <[email protected]> (@pracucci)
 Peter Štibraný, Grafana Labs <[email protected]> (@pstibrany)
 Tom Wilkie, Grafana Labs <[email protected]> (@tomwilkie)

Makefile

@@ -119,7 +119,7 @@ build-image/$(UPTODATE): build-image/*
 SUDO := $(shell docker info >/dev/null 2>&1 || echo "sudo -E")
 BUILD_IN_CONTAINER := true
 BUILD_IMAGE ?= $(IMAGE_PREFIX)build-image
-LATEST_BUILD_IMAGE_TAG ?= build-image-multiarch-1d2497ff6
+LATEST_BUILD_IMAGE_TAG ?= master-f0c5aab05
 
 # TTY is parameterized to allow Google Cloud Builder to run builds,
 # as it currently disallows TTY devices. This value needs to be overridden
@@ -176,7 +176,7 @@ lint:
 	# Configured via .golangci.yml.
 	golangci-lint run
 
-	# Ensure no blacklisted package is imported.
+	# Ensure no blocklisted package is imported.
 	GOFLAGS="-tags=requires_docker" faillint -paths "github.com/bmizerany/assert=github.com/stretchr/testify/assert,\
 		golang.org/x/net/context=context,\
 		sync/atomic=go.uber.org/atomic,\

README.md

@@ -132,7 +132,7 @@ For security issues see https://github.com/cortexproject/cortex/security/policy
 
 ## Community Meetings
 
-The Cortex community call happens every three weeks on Thursday at 03:30pm – 04:15pm UTC to get calendar invite join the [google groups](https://groups.google.com/forum/#!forum/cortex-monitoring).
+The Cortex community call happens every two weeks on Thursday, alternating at 1200 UTC and 1700 UTC. To get a calendar invite join the [google groups](https://groups.google.com/forum/#!forum/cortex-monitoring) or check out [the CNCF community calendar](https://www.cncf.io/calendar/).
 
 Meeting notes are held [here](https://docs.google.com/document/d/1shtXSAqp3t7fiC-9uZcKkq3mgwsItAJlH6YW6x1joZo/edit).
 

VERSION

@@ -1 +1,5 @@
-1.10.0-rc.0
+<<<<<<< HEAD
+1.10.0
+=======
+1.11.0
+>>>>>>> Merge release 1.11 to master (#4569)

build-image/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.16.3-buster
+FROM golang:1.17.5-buster
 ARG goproxyValue
 ENV GOPROXY=${goproxyValue}
 RUN apt-get update && apt-get install -y curl python-requests python-yaml file jq unzip protobuf-compiler libprotobuf-dev && \

cmd/blocksconvert/Dockerfile

@@ -1,4 +1,4 @@
-FROM       alpine:3.13
+FROM       alpine:3.14
 RUN        apk add --no-cache ca-certificates
 COPY       blocksconvert /
 ENTRYPOINT ["/blocksconvert"]

cmd/blocksconvert/main.go

@@ -7,14 +7,14 @@ import (
 	"os"
 	"strings"
 
-	"github.com/go-kit/kit/log/level"
+	"github.com/go-kit/log/level"
+	"github.com/grafana/dskit/services"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/weaveworks/common/server"
 	"github.com/weaveworks/common/signals"
 
 	"github.com/cortexproject/cortex/pkg/cortex"
 	util_log "github.com/cortexproject/cortex/pkg/util/log"
-	"github.com/cortexproject/cortex/pkg/util/services"
 	"github.com/cortexproject/cortex/tools/blocksconvert"
 	"github.com/cortexproject/cortex/tools/blocksconvert/builder"
 	"github.com/cortexproject/cortex/tools/blocksconvert/cleaner"

cmd/cortex/Dockerfile

@@ -1,4 +1,4 @@
-FROM       alpine:3.13
+FROM       alpine:3.14
 RUN        apk add --no-cache ca-certificates
 COPY       migrations /migrations/
 COPY       cortex /bin/cortex

cmd/cortex/main.go

@@ -12,7 +12,8 @@ import (
 	"strings"
 	"time"
 
-	"github.com/go-kit/kit/log/level"
+	"github.com/go-kit/log/level"
+	"github.com/grafana/dskit/flagext"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/version"
@@ -21,7 +22,6 @@ import (
 
 	"github.com/cortexproject/cortex/pkg/cortex"
 	"github.com/cortexproject/cortex/pkg/util"
-	"github.com/cortexproject/cortex/pkg/util/flagext"
 	util_log "github.com/cortexproject/cortex/pkg/util/log"
 )
 

cmd/query-tee/Dockerfile

@@ -1,4 +1,4 @@
-FROM       alpine:3.13
+FROM       alpine:3.14
 RUN        apk add --no-cache ca-certificates
 COPY       query-tee /
 ENTRYPOINT ["/query-tee"]

cmd/query-tee/main.go

@@ -4,8 +4,9 @@ import (
 	"flag"
 	"os"
 
-	"github.com/go-kit/kit/log/level"
+	"github.com/go-kit/log/level"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/collectors"
 	"github.com/weaveworks/common/logging"
 	"github.com/weaveworks/common/server"
 
@@ -35,7 +36,7 @@ func main() {
 
 	// Run the instrumentation server.
 	registry := prometheus.NewRegistry()
-	registry.MustRegister(prometheus.NewGoCollector())
+	registry.MustRegister(collectors.NewGoCollector())
 
 	i := querytee.NewInstrumentationServer(cfg.ServerMetricsPort, registry)
 	if err := i.Start(); err != nil {

cmd/test-exporter/Dockerfile

@@ -1,4 +1,4 @@
-FROM       alpine:3.13
+FROM       alpine:3.14
 RUN        apk add --no-cache ca-certificates
 COPY       test-exporter /
 ENTRYPOINT ["/test-exporter"]

cmd/test-exporter/main.go

@@ -5,13 +5,13 @@ import (
 	"math"
 	"time"
 
-	"github.com/go-kit/kit/log/level"
+	"github.com/go-kit/log/level"
+	"github.com/grafana/dskit/flagext"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/weaveworks/common/server"
 	"github.com/weaveworks/common/tracing"
 
 	"github.com/cortexproject/cortex/pkg/testexporter/correctness"
-	"github.com/cortexproject/cortex/pkg/util/flagext"
 	"github.com/cortexproject/cortex/pkg/util/log"
 )
 

cmd/thanosconvert/Dockerfile

@@ -1,4 +1,4 @@
-FROM       alpine:3.13
+FROM       alpine:3.14
 RUN        apk add --no-cache ca-certificates
 COPY       thanosconvert /
 ENTRYPOINT ["/thanosconvert"]

development/tsdb-blocks-storage-s3-gossip/dev.dockerfile

@@ -1,8 +1,8 @@
-FROM golang:1.16
+FROM golang:1.17
 ENV CGO_ENABLED=0
 RUN go get github.com/go-delve/delve/cmd/dlv
 
-FROM alpine:3.13
+FROM alpine:3.14
 
 RUN     mkdir /cortex
 WORKDIR /cortex