Partitioning compaction for Cortex

.github/workflows/scripts/install-docker.sh

@@ -4,4 +4,8 @@ set -x
 VER="20.10.19"
 curl -L -o /tmp/docker-$VER.tgz https://download.docker.com/linux/static/stable/x86_64/docker-$VER.tgz
 tar -xz -C /tmp -f /tmp/docker-$VER.tgz
+mkdir -vp ~/.docker/cli-plugins/
+curl --silent -L "https://github.com/docker/buildx/releases/download/v0.3.0/buildx-v0.3.0.linux-amd64" > ~/.docker/cli-plugins/docker-buildx
+chmod a+x ~/.docker/cli-plugins/docker-buildx
 mv /tmp/docker/* /usr/bin
+docker run --privileged --rm tonistiigi/binfmt --install all

.github/workflows/test-build-deploy.yml

@@ -11,10 +11,15 @@ jobs:
   lint:
     runs-on: ubuntu-20.04
     container:
-      image: quay.io/cortexproject/build-image:upgrade-to-go1.19.3-e8b98ddc0
+      image: quay.io/cortexproject/build-image:upgrade-to-go1.20.1-pr-5159
     steps:
       - name: Checkout Repo
         uses: actions/checkout@v2
+      - name: Setup Git safe.directory
+        run: |
+          echo "this step is needed because when running in container, actions/checkout does not set safe.directory effectively."
+          echo "See https://github.com/actions/runner/issues/2033. We should use --system instead of --global"
+          git config --system --add safe.directory $GITHUB_WORKSPACE
         # Commands in the Makefile are hardcoded with an assumed file structure of the CI container
         # Symlink ensures paths specified in previous commands don’t break
       - name: Sym Link Expected Path to Workspace
@@ -35,10 +40,15 @@ jobs:
   test:
     runs-on: ubuntu-20.04
     container:
-      image: quay.io/cortexproject/build-image:upgrade-to-go1.19.3-e8b98ddc0
+      image: quay.io/cortexproject/build-image:upgrade-to-go1.20.1-pr-5159
     steps:
       - name: Checkout Repo
         uses: actions/checkout@v2
+      - name: Setup Git safe.directory
+        run: |
+          echo "this step is needed because when running in container, actions/checkout does not set safe.directory effectively."
+          echo "See https://github.com/actions/runner/issues/2033. We should use --system instead of --global"
+          git config --system --add safe.directory $GITHUB_WORKSPACE
       - name: Sym Link Expected Path to Workspace
         run: |
           mkdir -p /go/src/github.com/cortexproject/cortex
@@ -49,10 +59,15 @@ jobs:
   build:
     runs-on: ubuntu-20.04
     container:
-      image: quay.io/cortexproject/build-image:upgrade-to-go1.19.3-e8b98ddc0
+      image: quay.io/cortexproject/build-image:upgrade-to-go1.20.1-pr-5159
     steps:
       - name: Checkout Repo
         uses: actions/checkout@v2
+      - name: Setup Git safe.directory
+        run: |
+          echo "this step is needed because when running in container, actions/checkout does not set safe.directory effectively."
+          echo "See https://github.com/actions/runner/issues/2033. We should use --system instead of --global"
+          git config --system --add safe.directory $GITHUB_WORKSPACE
       - name: Install Docker Client
         run: ./.github/workflows/scripts/install-docker.sh
       - name: Sym Link Expected Path to Workspace
@@ -88,11 +103,22 @@ jobs:
   integration:
     needs: build
     runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        tags:
+          - requires_docker
+          - integration_alertmanager
+          - integration_backward_compatibility
+          - integration_memberlist
+          - integration_querier
+          - integration_ruler
+          - integration_query_fuzz
     steps:
       - name: Upgrade golang
         uses: actions/setup-go@v2
         with:
-          go-version: 1.19.x
+          go-version: 1.20.1
       - name: Checkout Repo
         uses: actions/checkout@v2
       - name: Install Docker Client
@@ -116,35 +142,35 @@ jobs:
         # locally and the download time doesn't account in the test execution time, which is subject
         # to a timeout
         run: |
-          docker pull minio/minio:RELEASE.2019-12-30T05-45-39Z
-          docker pull amazon/dynamodb-local:1.11.477
+          docker pull minio/minio:RELEASE.2021-10-13T00-23-17Z
           docker pull consul:1.8.4
           docker pull gcr.io/etcd-development/etcd:v3.4.7
-          docker pull quay.io/cortexproject/cortex:v1.0.0
-          docker pull quay.io/cortexproject/cortex:v1.1.0
-          docker pull quay.io/cortexproject/cortex:v1.2.0
-          docker pull quay.io/cortexproject/cortex:v1.3.0
-          docker pull quay.io/cortexproject/cortex:v1.4.0
-          docker pull quay.io/cortexproject/cortex:v1.5.0
-          docker pull quay.io/cortexproject/cortex:v1.6.0
-          docker pull quay.io/cortexproject/cortex:v1.7.0
-          docker pull quay.io/cortexproject/cortex:v1.8.0
-          docker pull quay.io/cortexproject/cortex:v1.9.0
-          docker pull quay.io/cortexproject/cortex:v1.10.0
-          docker pull quay.io/cortexproject/cortex:v1.11.1
-          docker pull quay.io/cortexproject/cortex:v1.13.1
-          docker pull quay.io/cortexproject/cortex:v1.14.0
-          docker pull shopify/bigtable-emulator:0.1.0
+          if [ "$TEST_TAGS" = "integration_backward_compatibility" ]; then
+            docker pull quay.io/cortexproject/cortex:v1.6.0
+            docker pull quay.io/cortexproject/cortex:v1.7.0
+            docker pull quay.io/cortexproject/cortex:v1.8.0
+            docker pull quay.io/cortexproject/cortex:v1.9.0
+            docker pull quay.io/cortexproject/cortex:v1.10.0
+            docker pull quay.io/cortexproject/cortex:v1.11.1
+            docker pull quay.io/cortexproject/cortex:v1.13.1
+            docker pull quay.io/cortexproject/cortex:v1.13.2
+            docker pull quay.io/cortexproject/cortex:v1.14.0
+            docker pull quay.io/cortexproject/cortex:v1.14.1
+            docker pull quay.io/cortexproject/cortex:v1.15.0
+            docker pull quay.io/cortexproject/cortex:v1.15.1
+          fi
           docker pull memcached:1.6.1
-          docker pull bouncestorage/swift-aio:55ba4331
+          docker pull redis:7.0.4-alpine
+        env:
+          TEST_TAGS: ${{ matrix.tags }}
       - name: Integration Tests
         run: |
           export CORTEX_IMAGE_PREFIX="${IMAGE_PREFIX:-quay.io/cortexproject/}"
           export IMAGE_TAG=$(make image-tag)
-          export CORTEX_IMAGE="${CORTEX_IMAGE_PREFIX}cortex:$IMAGE_TAG"
+          export CORTEX_IMAGE="${CORTEX_IMAGE_PREFIX}cortex:$IMAGE_TAG-amd64"
           export CORTEX_CHECKOUT_DIR="/go/src/github.com/cortexproject/cortex"
           echo "Running integration tests with image: $CORTEX_IMAGE"
-          go test -tags=requires_docker -timeout 2400s -v -count=1 ./integration/...
+          go test -tags=integration,${{ matrix.tags }} -timeout 2400s -v -count=1 ./integration/...
         env:
           IMAGE_PREFIX: ${{ secrets.IMAGE_PREFIX }}
 
@@ -167,20 +193,25 @@ jobs:
         run: |
           touch build-image/.uptodate
           MIGRATIONS_DIR=$(pwd)/cmd/cortex/migrations
-          make BUILD_IMAGE=quay.io/cortexproject/build-image:upgrade-to-go1.19.3-e8b98ddc0 TTY='' configs-integration-test
+          make BUILD_IMAGE=quay.io/cortexproject/build-image:upgrade-to-go1.20.1-pr-5159 TTY='' configs-integration-test
 
   deploy_website:
     needs: [build, test]
     if: (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) && github.repository == 'cortexproject/cortex'
     runs-on: ubuntu-20.04
     container:
-      image: quay.io/cortexproject/build-image:upgrade-to-go1.19.3-e8b98ddc0
+      image: quay.io/cortexproject/build-image:upgrade-to-go1.20.1-pr-5159
     steps:
       - name: Checkout Repo
         uses: actions/checkout@v2
         with:
           # web-deploy script expects repo to be cloned with ssh for some commands to work
           ssh-key: ${{ secrets.WEBSITE_DEPLOY_SSH_PRIVATE_KEY }}
+      - name: Setup Git safe.directory
+        run: |
+          echo "this step is needed because when running in container, actions/checkout does not set safe.directory effectively."
+          echo "See https://github.com/actions/runner/issues/2033. We should use --system instead of --global"
+          git config --system --add safe.directory $GITHUB_WORKSPACE
       - name: Sym Link Expected Path to Workspace
         run: |
           mkdir -p /go/src/github.com/cortexproject/cortex
@@ -211,10 +242,15 @@ jobs:
     if: (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) && github.repository == 'cortexproject/cortex'
     runs-on: ubuntu-20.04
     container:
-      image: quay.io/cortexproject/build-image:upgrade-to-go1.19.3-e8b98ddc0
+      image: quay.io/cortexproject/build-image:upgrade-to-go1.20.1-pr-5159
     steps:
       - name: Checkout Repo
         uses: actions/checkout@v2
+      - name: Setup Git safe.directory
+        run: |
+          echo "this step is needed because when running in container, actions/checkout does not set safe.directory effectively."
+          echo "See https://github.com/actions/runner/issues/2033. We should use --system instead of --global"
+          git config --system --add safe.directory $GITHUB_WORKSPACE
       - name: Install Docker Client
         run: ./.github/workflows/scripts/install-docker.sh
       - name: Sym link Expected Path to Workspace

.golangci.yml

@@ -33,3 +33,10 @@ run:
   build-tags:
     - netgo
     - requires_docker
+    - integration
+    - integration_alertmanager
+    - integration_backward_compatibility
+    - integration_memberlist
+    - integration_querier
+    - integration_ruler
+    - integration_query_fuzz

CHANGELOG.md

@@ -1,16 +1,49 @@
 # Changelog
 
 ## master / unreleased
+* [CHANGE] Store gateways summary metrics have been converted to histograms `cortex_bucket_store_series_blocks_queried`, `cortex_bucket_store_series_data_fetched`, `cortex_bucket_store_series_data_size_touched_bytes`, `cortex_bucket_store_series_data_size_fetched_bytes`, `cortex_bucket_store_series_data_touched`, `cortex_bucket_store_series_result_series` #5239
+
+## 1.15.0 in progress
+
+* [CHANGE] Storage: Make Max exemplars config per tenant instead of global configuration. #5016
+* [CHANGE] Alertmanager: Local file disclosure vulnerability in OpsGenie configuration has been fixed. #5045
+* [CHANGE] Rename oltp_endpoint to otlp_endpoint to match opentelemetry spec and lib name. #5067
+* [CHANGE] Distributor/Ingester: Log warn level on push requests when they have status code 4xx. Do not log if status is 429. #5103
+* [CHANGE] Tracing: Use the default OTEL trace sampler when `-tracing.otel.exporter-type` is set to `awsxray`. #5141
+* [CHANGE] Ingester partial error log line to debug level. #5192
+* [CHANGE] Change HTTP status code from 503/422 to 499 if a request is canceled. #5220
 * [ENHANCEMENT] Update Go version to 1.19.3. #4988
 * [ENHANCEMENT] Querier: limit series query to only ingesters if `start` param is not specified. #4976
 * [ENHANCEMENT] Query-frontend/scheduler: add a new limit `frontend.max-outstanding-requests-per-tenant` for configuring queue size per tenant. Started deprecating two flags `-query-scheduler.max-outstanding-requests-per-tenant` and `-querier.max-outstanding-requests-per-tenant`, and change their value default to 0. Now if both the old flag and new flag are specified, the old flag's queue size will be picked. #5005
 * [ENHANCEMENT] Query-tee: Add `/api/v1/query_exemplars` API endpoint support. #5010
+* [ENHANCEMENT] Query Frontend/Query Scheduler: Increase upper bound to 60s for queue duration histogram metric. #5029
+* [ENHANCEMENT] Query Frontend: Log Vertical sharding information when `query_stats_enabled` is enabled. #5037
+* [ENHANCEMENT] Ingester: The metadata APIs should honour `querier.query-ingesters-within` when `querier.query-store-for-labels-enabled` is true. #5027
+* [ENHANCEMENT] Query Frontend: Skip instant query roundtripper if sharding is not applicable. #5062
+* [ENHANCEMENT] Push reduce one hash operation of Labels. #4945 #5114
+* [ENHANCEMENT] Alertmanager: Added `-alertmanager.enabled-tenants` and `-alertmanager.disabled-tenants` to explicitly enable or disable alertmanager for specific tenants. #5116
+* [ENHANCEMENT] Upgraded Docker base images to `alpine:3.17`. #5132
+* [ENHANCEMENT] Add retry logic to S3 bucket client. #5135
+* [ENHANCEMENT] Update Go version to 1.20.1. #5159
+* [ENHANCEMENT] Distributor: Reuse byte slices when serializing requests from distributors to ingesters. #5193
+* [ENHANCEMENT] Query Frontend: Add number of chunks and samples fetched in query stats. #5198
+* [ENHANCEMENT] Implement grpc.Compressor.DecompressedSize for snappy to optimize memory allocations. #5213
 * [FEATURE] Querier/Query Frontend: support Prometheus /api/v1/status/buildinfo API. #4978
 * [FEATURE] Ingester: Add active series to all_user_stats page. #4972
 * [FEATURE] Ingester: Added `-blocks-storage.tsdb.head-chunks-write-queue-size` allowing to configure the size of the in-memory queue used before flushing chunks to the disk . #5000
 * [FEATURE] Query Frontend: Log query params in query frontend even if error happens. #5005
 * [FEATURE] Ingester: Enable snapshotting of In-memory TSDB on disk during shutdown via `-blocks-storage.tsdb.memory-snapshot-on-shutdown`. #5011
+* [FEATURE] Query Frontend/Scheduler: Add a new counter metric `cortex_request_queue_requests_total` for total requests going to queue. #5030
+* [FEATURE] Build ARM docker images. #5041
+* [FEATURE] Query-frontend/Querier: Create spans to measure time to merge promql responses. #5041
+* [FEATURE] Querier/Ruler: Support the new thanos promql engine. This is an experimental feature and might change in the future. #5093
+* [FEATURE] Added zstd as an option for grpc compression #5092
+* [FEATURE] Ring: Add new kv store option `dynamodb`. #5026
+* [FEATURE] Cache: Support redis as backend for caching bucket and index cache. #5057
+* [FEATURE] Querier/Store-Gateway: Added `-blocks-storage.bucket-store.ignore-blocks-within` allowing to filter out the recently created blocks from being synced by queriers and store-gateways. #5166
+* [FEATURE] AlertManager/Ruler: Added support for  `keep_firing_for` on alerting rulers.
 * [BUGFIX] Updated `golang.org/x/net` dependency to fix CVE-2022-27664. #5008
+* [BUGFIX] Fix panic when otel and xray tracing is enabled. #5044
 
 ## 1.14.0 2022-12-02
 
@@ -44,6 +77,7 @@
     - `-ingester.chunk-age-jitter`
     - `-ingester.concurrent-flushes`
     - `-ingester.spread-flushes`
+    - `-ingester.chunk-encoding`
     - `-store.*` except `-store.engine` and `-store.max-query-length`
     - `-store.query-chunk-limit` was deprecated and replaced by `-querier.max-fetched-chunks-per-query`
   - `-deletes.*`

Makefile

@@ -9,6 +9,9 @@
 VERSION=$(shell cat "./VERSION" 2> /dev/null)
 GOPROXY_VALUE=$(shell go env GOPROXY)
 
+# ARCHS
+ARCHS       = amd64 arm64
+
 # Boiler plate for building Docker containers.
 # All this must go at top of file I'm afraid.
 IMAGE_PREFIX ?= quay.io/cortexproject/
@@ -37,8 +40,9 @@ SED ?= $(shell which gsed 2>/dev/null || which sed)
 # Dependencies (i.e. things that go in the image) still need to be explicitly
 # declared.
 %/$(UPTODATE): %/Dockerfile
-	@echo
-	$(SUDO) docker build --build-arg=revision=$(GIT_REVISION) --build-arg=goproxyValue=$(GOPROXY_VALUE) -t $(IMAGE_PREFIX)$(shell basename $(@D)) -t $(IMAGE_PREFIX)$(shell basename $(@D)):$(IMAGE_TAG) $(@D)/
+	for arch in $(ARCHS); do \
+		$(SUDO) docker buildx build --platform linux/$$arch --build-arg=revision=$(GIT_REVISION) --build-arg=goproxyValue=$(GOPROXY_VALUE) -t $(IMAGE_PREFIX)$(shell basename $(@D)) -t $(IMAGE_PREFIX)$(shell basename $(@D)):$(IMAGE_TAG)-$$arch $(@D)/ ; \
+	done
 	@echo
 	@echo Please use push-multiarch-build-image to build and push build image for all supported architectures.
 	touch $@
@@ -118,7 +122,7 @@ build-image/$(UPTODATE): build-image/*
 SUDO := $(shell docker info >/dev/null 2>&1 || echo "sudo -E")
 BUILD_IN_CONTAINER := true
 BUILD_IMAGE ?= $(IMAGE_PREFIX)build-image
-LATEST_BUILD_IMAGE_TAG ?= upgrade-to-go1.19.3-e8b98ddc0
+LATEST_BUILD_IMAGE_TAG ?= upgrade-to-go1.20.1-e6945e022
 
 # TTY is parameterized to allow Google Cloud Builder to run builds,
 # as it currently disallows TTY devices. This value needs to be overridden
@@ -160,7 +164,11 @@ else
 exes: $(EXES)
 
 $(EXES):
-	CGO_ENABLED=0 go build $(GO_FLAGS) -o $@ ./$(@D)
+	@for arch in $(ARCHS); do \
+  		echo "Building $@ for $$arch";\
+  		CGO_ENABLED=0 GOARCH=$$arch GOOS=linux go build $(GO_FLAGS) -o $@-$$arch ./$(@D); \
+  	done
+
 
 protos: $(PROTO_GOS)
 
@@ -176,7 +184,7 @@ lint:
 	golangci-lint run
 
 	# Ensure no blocklisted package is imported.
-	GOFLAGS="-tags=requires_docker" faillint -paths "github.com/bmizerany/assert=github.com/stretchr/testify/assert,\
+	GOFLAGS="-tags=requires_docker,integration,integration_alertmanager,integration_backward_compatibility,integration_memberlist,integration_querier,integration_ruler,integration_query_fuzz" faillint -paths "github.com/bmizerany/assert=github.com/stretchr/testify/assert,\
 		golang.org/x/net/context=context,\
 		sync/atomic=go.uber.org/atomic,\
 		github.com/prometheus/client_golang/prometheus.{MultiError}=github.com/prometheus/prometheus/tsdb/errors.{NewMulti},\
@@ -272,16 +280,20 @@ clean-protos:
 
 save-images:
 	@mkdir -p docker-images
-	for image_name in $(IMAGE_NAMES); do \
+	@for image_name in $(IMAGE_NAMES); do \
 		if ! echo $$image_name | grep build; then \
-			docker save $$image_name:$(IMAGE_TAG) -o docker-images/$$(echo $$image_name | tr "/" _):$(IMAGE_TAG); \
+		  	for arch in $(ARCHS); do \
+				docker save $$image_name:$(IMAGE_TAG)-$$arch -o docker-images/$$(echo $$image_name | tr "/" _):$(IMAGE_TAG)-$$arch; \
+			done;\
 		fi \
 	done
 
 load-images:
-	for image_name in $(IMAGE_NAMES); do \
+	@for image_name in $(IMAGE_NAMES); do \
 		if ! echo $$image_name | grep build; then \
-			docker load -i docker-images/$$(echo $$image_name | tr "/" _):$(IMAGE_TAG); \
+		  	for arch in $(ARCHS); do \
+				docker load -i docker-images/$$(echo $$image_name | tr "/" _):$(IMAGE_TAG)-$$arch; \
+			done;\
 		fi \
 	done
 

README.md

@@ -27,7 +27,7 @@ Cortex is primarily used as a [remote write](https://prometheus.io/docs/operatin
 
 The chunks storage is deprecated since v1.10.0. You're encouraged to use the [blocks storage](docs/blocks-storage/_index.md).
 
-Chunks storage is scheduled to be removed in release 1.14.0
+Chunks storage was removed in release 1.14.0
 
 ## Documentation
 

RELEASE.md

@@ -30,6 +30,7 @@ Our goal is to provide a new minor release every 6 weeks. This is a new process
 | v1.12.0        | 2022-02-17                                 | _Abandoned_                                 |
 | v1.13.0        | 2022-06-23                                 | Alvin Lin (@alvinlin123)                    |
 | v1.14.0        | 2022-11-15                                 | Alan Protasio (@alanprot)                   |
+| v1.15.0        | 2023-03-27                                 | Ben Ye (@yeya24)                            |
 
 ## Release shepherd responsibilities
 

build-image/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.19.3-buster
+FROM golang:1.20.1-buster
 ARG goproxyValue
 ENV GOPROXY=${goproxyValue}
 RUN apt-get update && apt-get install -y curl python-requests python-yaml file jq unzip protobuf-compiler libprotobuf-dev && \
@@ -23,7 +23,7 @@ RUN GOARCH=$(go env GOARCH) && \
 	chmod +x shfmt && \
 	mv shfmt /usr/bin
 
-RUN curl -sfL https://github.com/raw/golangci/golangci-lint/master/install.sh| sh -s -- -b /usr/bin v1.48.0
+RUN curl -sfL https://github.com/raw/golangci/golangci-lint/master/install.sh| sh -s -- -b /usr/bin v1.51.2
 
 ENV HUGO_VERSION=v0.101.0
 RUN go install github.com/client9/misspell/cmd/[email protected] &&\

cmd/cortex/Dockerfile

@@ -1,7 +1,9 @@
-FROM       alpine:3.14
+FROM       alpine:3.17
+ARG TARGETARCH
+
 RUN        apk add --no-cache ca-certificates
 COPY       migrations /migrations/
-COPY       cortex /bin/cortex
+COPY       cortex-$TARGETARCH /bin/cortex
 EXPOSE     80
 ENTRYPOINT [ "/bin/cortex" ]
 

cmd/query-tee/Dockerfile

@@ -1,6 +1,8 @@
-FROM       alpine:3.14
+FROM       alpine:3.17
+ARG TARGETARCH
+
 RUN        apk add --no-cache ca-certificates
-COPY       query-tee /
+COPY       query-tee-$TARGETARCH /query-tee
 ENTRYPOINT ["/query-tee"]
 
 ARG revision

cmd/test-exporter/Dockerfile

@@ -1,6 +1,7 @@
-FROM       alpine:3.14
+FROM       alpine:3.17
+ARG TARGETARCH
 RUN        apk add --no-cache ca-certificates
-COPY       test-exporter /
+COPY       test-exporter-$TARGETARCH /test-exporter
 ENTRYPOINT ["/test-exporter"]
 
 ARG revision