Add config file support to integration tests (#2167)

* Added query-frontend support to local tsdb-blocks-storage-s3 dev env

Signed-off-by: Marco Pracucci <[email protected]>

* Introduced CortexService to have cleaner integration tests

Signed-off-by: Marco Pracucci <[email protected]>

* Added to writeFileToSharedDir() the ability to create the path of directories

Signed-off-by: Marco Pracucci <[email protected]>

* Enhanced query-frontend tests to run with config file too

Signed-off-by: Marco Pracucci <[email protected]>

Author: pracucci

development/tsdb-blocks-storage-s3/config/cortex.yaml

@@ -66,3 +66,6 @@ compactor:
       store: consul
       consul:
         host: consul:8500
+
+frontend_worker:
+  address: "query-frontend:9007"

development/tsdb-blocks-storage-s3/docker-compose.yml

@@ -108,3 +108,17 @@ services:
       - 8006:8006
     volumes:
       - ./config:/cortex/config
+
+  query-frontend:
+    build:
+      context:    .
+      dockerfile: dev.dockerfile
+    image: cortex
+    command: ["sh", "-c", "sleep 3 && exec ./cortex -config.file=./config/cortex.yaml -target=query-frontend -server.http-listen-port=8007 -server.grpc-listen-port=9007"]
+    depends_on:
+      - consul
+      - minio
+    ports:
+      - 8007:8007
+    volumes:
+      - ./config:/cortex/config

integration/alertmanager_test.go

@@ -2,9 +2,6 @@ package main
 
 import (
 	"context"
-	"io/ioutil"
-	"os"
-	"path/filepath"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -18,16 +15,9 @@ func TestAlertmanager(t *testing.T) {
 	require.NoError(t, err)
 	defer s.Close()
 
-	alertmanagerDir := filepath.Join(s.SharedDir(), "alertmanager_configs")
-	require.NoError(t, os.Mkdir(alertmanagerDir, os.ModePerm))
+	require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs/user-1.yaml", []byte(cortexAlertmanagerUserConfigYaml)))
 
-	require.NoError(t, ioutil.WriteFile(
-		filepath.Join(alertmanagerDir, "user-1.yaml"),
-		[]byte(cortexAlertmanagerUserConfigYaml),
-		os.ModePerm),
-	)
-
-	alertmanager := e2ecortex.NewAlertmanager("alertmanager", AlertmanagerConfigs, "")
+	alertmanager := e2ecortex.NewAlertmanager("alertmanager", AlertmanagerFlags, "")
 	require.NoError(t, s.StartAndWaitReady(alertmanager))
 	require.NoError(t, alertmanager.WaitSumMetrics(e2e.Equals(1), "cortex_alertmanager_configs"))
 

integration/api_config_test.go

@@ -25,7 +25,7 @@ func TestConfigAPIEndpoint(t *testing.T) {
 		"-config.file": filepath.Join(e2e.ContainerSharedDir, cortexConfigFile),
 	}
 
-	cortex1 := e2ecortex.NewSingleBinary("cortex-1", flags, "", 9009)
+	cortex1 := e2ecortex.NewSingleBinary("cortex-1", flags, "", 9009, 9095)
 	require.NoError(t, s.StartAndWaitReady(cortex1))
 
 	// Get config from /config API endpoint.
@@ -40,6 +40,6 @@ func TestConfigAPIEndpoint(t *testing.T) {
 	// Start again Cortex in single binary with the exported config
 	// and ensure it starts (pass the readiness probe).
 	require.NoError(t, writeFileToSharedDir(s, cortexConfigFile, body))
-	cortex2 := e2ecortex.NewSingleBinary("cortex-2", flags, "", 9009)
+	cortex2 := e2ecortex.NewSingleBinary("cortex-2", flags, "", 9009, 9095)
 	require.NoError(t, s.StartAndWaitReady(cortex2))
 }

integration/backward_compatibility_test.go

@@ -31,9 +31,9 @@ func TestBackwardCompatibilityWithChunksStorage(t *testing.T) {
 
 	// Start Cortex components (ingester running on previous version).
 	require.NoError(t, writeFileToSharedDir(s, cortexSchemaConfigFile, []byte(cortexSchemaConfigYaml)))
-	tableManager := e2ecortex.NewTableManager("table-manager", ChunksStorage, previousVersionImage)
-	ingester1 := e2ecortex.NewIngester("ingester-1", consul.NetworkHTTPEndpoint(), ChunksStorage, "")
-	distributor := e2ecortex.NewDistributor("distributor", consul.NetworkHTTPEndpoint(), ChunksStorage, "")
+	tableManager := e2ecortex.NewTableManager("table-manager", ChunksStorageFlags, previousVersionImage)
+	ingester1 := e2ecortex.NewIngester("ingester-1", consul.NetworkHTTPEndpoint(), ChunksStorageFlags, "")
+	distributor := e2ecortex.NewDistributor("distributor", consul.NetworkHTTPEndpoint(), ChunksStorageFlags, "")
 	require.NoError(t, s.StartAndWaitReady(distributor, ingester1, tableManager))
 
 	// Wait until the first table-manager sync has completed, so that we're
@@ -54,7 +54,7 @@ func TestBackwardCompatibilityWithChunksStorage(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, 200, res.StatusCode)
 
-	ingester2 := e2ecortex.NewIngester("ingester-2", consul.NetworkHTTPEndpoint(), mergeFlags(ChunksStorage, map[string]string{
+	ingester2 := e2ecortex.NewIngester("ingester-2", consul.NetworkHTTPEndpoint(), mergeFlags(ChunksStorageFlags, map[string]string{
 		"-ingester.join-after": "10s",
 	}), "")
 	// Start ingester-2 on new version, to ensure the transfer is backward compatible.
@@ -66,7 +66,7 @@ func TestBackwardCompatibilityWithChunksStorage(t *testing.T) {
 
 	// Query the new ingester both with the old and the new querier.
 	for _, image := range []string{previousVersionImage, ""} {
-		querier := e2ecortex.NewQuerier("querier", consul.NetworkHTTPEndpoint(), ChunksStorage, image)
+		querier := e2ecortex.NewQuerier("querier", consul.NetworkHTTPEndpoint(), ChunksStorageFlags, image)
 		require.NoError(t, s.StartAndWaitReady(querier))
 
 		// Wait until the querier has updated the ring.

integration/configs.go

@@ -3,6 +3,8 @@ package main
 import (
 	"fmt"
 	"path/filepath"
+	"strings"
+	"text/template"
 
 	"github.com/cortexproject/cortex/integration/e2e"
 	e2edb "github.com/cortexproject/cortex/integration/e2e/db"
@@ -33,13 +35,13 @@ receivers:
 )
 
 var (
-	AlertmanagerConfigs = map[string]string{
+	AlertmanagerFlags = map[string]string{
 		"-alertmanager.storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"),
 		"-alertmanager.storage.type":       "local",
 		"-alertmanager.web.external-url":   "http://localhost/api/prom",
 	}
 
-	BlocksStorage = map[string]string{
+	BlocksStorageFlags = map[string]string{
 		"-store.engine":                                 "tsdb",
 		"-experimental.tsdb.backend":                    "s3",
 		"-experimental.tsdb.block-ranges-period":        "1m",
@@ -53,10 +55,90 @@ var (
 		"-experimental.tsdb.s3.insecure":                "true",
 	}
 
-	ChunksStorage = map[string]string{
+	BlocksStorageConfig = buildConfigFromTemplate(`
+storage:
+  engine: tsdb
+
+tsdb:
+  backend:             s3
+  block_ranges_period: ["1m"]
+  retention_period:    5m
+  ship_interval:       1m
+
+  bucket_store:
+    sync_interval: 5s
+
+  s3:
+    bucket_name:       cortex
+    access_key_id:     {{.MinioAccessKey}}
+    secret_access_key: {{.MinioSecretKey}}
+    endpoint:          {{.MinioEndpoint}}
+    insecure:          true
+`, struct {
+		MinioAccessKey string
+		MinioSecretKey string
+		MinioEndpoint  string
+	}{
+		MinioAccessKey: e2edb.MinioAccessKey,
+		MinioSecretKey: e2edb.MinioSecretKey,
+		MinioEndpoint:  fmt.Sprintf("%s-minio-9000:9000", networkName),
+	})
+
+	ChunksStorageFlags = map[string]string{
 		"-dynamodb.url":                   fmt.Sprintf("dynamodb://u:p@%s-dynamodb.:8000", networkName),
 		"-dynamodb.poll-interval":         "1m",
 		"-config-yaml":                    filepath.Join(e2e.ContainerSharedDir, cortexSchemaConfigFile),
 		"-table-manager.retention-period": "168h",
 	}
+
+	ChunksStorageConfig = buildConfigFromTemplate(`
+storage:
+  aws:
+    dynamodbconfig:
+      dynamodb: {{.DynamoDBURL}}
+
+table_manager:
+  dynamodb_poll_interval: 1m
+  retention_period:       168h
+
+schema:
+{{.SchemaConfig}}
+`, struct {
+		DynamoDBURL  string
+		SchemaConfig string
+	}{
+		DynamoDBURL:  fmt.Sprintf("dynamodb://u:p@%s-dynamodb.:8000", networkName),
+		SchemaConfig: indentConfig(cortexSchemaConfigYaml, 2),
+	})
 )
+
+func buildConfigFromTemplate(tmpl string, data interface{}) string {
+	t, err := template.New("config").Parse(tmpl)
+	if err != nil {
+		panic(err)
+	}
+
+	w := &strings.Builder{}
+	if err = t.Execute(w, data); err != nil {
+		panic(err)
+	}
+
+	return w.String()
+}
+
+func indentConfig(config string, indentation int) string {
+	output := strings.Builder{}
+
+	for _, line := range strings.Split(config, "\n") {
+		if line == "" {
+			output.WriteString("\n")
+			continue
+		}
+
+		output.WriteString(strings.Repeat(" ", indentation))
+		output.WriteString(line)
+		output.WriteString("\n")
+	}
+
+	return output.String()
+}

integration/e2e/util.go

@@ -17,6 +17,10 @@ func RunCommandAndGetOutput(name string, args ...string) ([]byte, error) {
 	return cmd.CombinedOutput()
 }
 
+func EmptyFlags() map[string]string {
+	return map[string]string{}
+}
+
 func MergeFlags(inputs ...map[string]string) map[string]string {
 	output := map[string]string{}
 

integration/e2ecortex/service.go

@@ -0,0 +1,33 @@
+package e2ecortex
+
+import "github.com/cortexproject/cortex/integration/e2e"
+
+// CortexService represents a Cortex service with at least an HTTP and GRPC port exposed.
+type CortexService struct {
+	*e2e.HTTPService
+
+	grpcPort int
+}
+
+func NewCortexService(
+	name string,
+	image string,
+	command *e2e.Command,
+	readiness *e2e.ReadinessProbe,
+	httpPort int,
+	grpcPort int,
+	otherPorts ...int,
+) *CortexService {
+	return &CortexService{
+		HTTPService: e2e.NewHTTPService(name, image, command, readiness, httpPort, otherPorts...),
+		grpcPort:    grpcPort,
+	}
+}
+
+func (s *CortexService) GRPCEndpoint() string {
+	return s.Endpoint(s.grpcPort)
+}
+
+func (s *CortexService) NetworkGRPCEndpoint() string {
+	return s.NetworkEndpoint(s.grpcPort)
+}