From 483b9af2cd081213b345952b4a788e654cb8d60f Mon Sep 17 00:00:00 2001 From: Wing924 Date: Tue, 18 Feb 2020 15:46:02 +0900 Subject: [PATCH 1/3] Add ability to override configuration settings using environment variables Signed-off-by: Wing924 --- CHANGELOG.md | 1 + cmd/cortex/main.go | 19 ++++++++++++ cmd/cortex/main_test.go | 30 +++++++++++++++++++ docs/configuration/config-file-reference.md | 27 +++++++++++++++-- .../config-file-reference.template | 27 +++++++++++++++-- 5 files changed, 100 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c05b0813405..95c6e0b5adc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ * `--experimental.distributor.user-subring-size` * [FEATURE] Added flag `-experimental.ruler.enable-api` to enable the ruler api which implements the Prometheus API `/api/v1/rules` and `/api/v1/alerts` endpoints under the configured `-http.prefix`. #1999 * [FEATURE] Added sharding support to compactor when using the experimental TSDB blocks storage. #2113 +* [FEATURE] Add ability to override configuration settings using environment variables. * [ENHANCEMENT] Add `status` label to `cortex_alertmanager_configs` metric to gauge the number of valid and invalid configs. #2125 * [ENHANCEMENT] Cassandra Authentication: added the `custom_authenticators` config option that allows users to authenticate with cassandra clusters using password authenticators that are not approved by default in [gocql](https://github.com/gocql/gocql/blob/81b8263d9fe526782a588ef94d3fa5c6148e5d67/conn.go#L27) #2093 * [ENHANCEMENT] Experimental TSDB: Export TSDB Syncer metrics from Compactor component, they are prefixed with `cortex_compactor_`. #2023 diff --git a/cmd/cortex/main.go b/cmd/cortex/main.go index bd45e3a0539..ef4adb2168e 100644 --- a/cmd/cortex/main.go +++ b/cmd/cortex/main.go @@ -7,6 +7,7 @@ import ( "math/rand" "os" "runtime" + "strings" "time" "github.com/go-kit/kit/log/level" @@ -139,6 +140,8 @@ func LoadConfig(filename string, cfg *cortex.Config) error { return errors.Wrap(err, "Error reading config file") } + buf = expandEnv(buf) + err = yaml.UnmarshalStrict(buf, cfg) if err != nil { return errors.Wrap(err, "Error parsing config file") @@ -155,3 +158,19 @@ func DumpYaml(cfg *cortex.Config) { fmt.Printf("%s\n", out) } } + +// expandEnv replaces ${var} or $var in config according to the values of the current environment variables. +// The replacement is case-sensitive. References to undefined variables are replaced by the empty string. +// A default value can be given by using the form ${var:default value}. +func expandEnv(config []byte) []byte { + return []byte(os.Expand(string(config), func(key string) string { + keyAndDefault := strings.SplitN(key, ":", 2) + key = keyAndDefault[0] + + v := os.Getenv(key) + if v == "" && len(keyAndDefault) == 2 { + v = keyAndDefault[1] // Set value to the default. + } + return v + })) +} diff --git a/cmd/cortex/main_test.go b/cmd/cortex/main_test.go index b9ba1a85c48..2bdf6b13de2 100644 --- a/cmd/cortex/main_test.go +++ b/cmd/cortex/main_test.go @@ -9,6 +9,7 @@ import ( "sync" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -167,3 +168,32 @@ func (co *capturedOutput) Done() (stdout []byte, stderr []byte) { return co.stdoutBuf.Bytes(), co.stderrBuf.Bytes() } + +func TestExpandEnv(t *testing.T) { + var tests = []struct { + in string + out string + }{ + // Environment variables can be specified as ${env} or $env. + {"x$y", "xy"}, + {"x${y}", "xy"}, + + // Environment variables are case-sensitive. Neither are replaced. + {"x$Y", "x"}, + {"x${Y}", "x"}, + + // Defaults can only be specified when using braces. + {"x${Z:D}", "xD"}, + {"x${Z:A B C D}", "xA B C D"}, // Spaces are allowed in the default. + {"x${Z:}", "x"}, + + // Defaults don't work unless braces are used. + {"x$y:D", "xy:D"}, + } + + for _, test := range tests { + os.Setenv("y", "y") + output := expandEnv([]byte(test.in)) + assert.Equal(t, test.out, string(output), "Input: %s", test.in) + } +} diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index db97324b6dd..90236d66fbd 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -11,7 +11,7 @@ Cortex can be configured using a YAML file - specified using the `-config.file` To specify which configuration file to load, pass the `-config.file` flag at the command line. The file is written in [YAML format](https://en.wikipedia.org/wiki/YAML), defined by the scheme below. Brackets indicate that a parameter is optional. -Generic placeholders are defined as follows: +### Generic placeholders * ``: a boolean that can take the values `true` or `false` * ``: any integer matching the regular expression `[1-9]+[0-9]*` @@ -20,7 +20,30 @@ Generic placeholders are defined as follows: * ``: an URL * ``: a CLI flag prefix based on the context (look at the parent configuration block to see which CLI flags prefix should be used) -Supported contents and default values of the config file: +### Use environment variables in the configuration + +You can use environment variable references in the config file to set values that need to be configurable during deployment. +To do this, use: + +``` +${VAR} +``` + +Where VAR is the name of the environment variable. + +Each variable reference is replaced at startup by the value of the environment variable. +The replacement is case-sensitive and occurs before the YAML file is parsed. +References to undefined variables are replaced by empty strings unless you specify a default value or custom error text. + +To specify a default value, use: + +``` +${VAR:default_value} +``` + +Where default_value is the value to use if the environment variable is undefined. + +### Supported contents and default values of the config file ```yaml # The Cortex service to run. Supported values are: all, distributor, ingester, diff --git a/docs/configuration/config-file-reference.template b/docs/configuration/config-file-reference.template index 3fb632789c5..f2e93275d43 100644 --- a/docs/configuration/config-file-reference.template +++ b/docs/configuration/config-file-reference.template @@ -11,7 +11,7 @@ Cortex can be configured using a YAML file - specified using the `-config.file` To specify which configuration file to load, pass the `-config.file` flag at the command line. The file is written in [YAML format](https://en.wikipedia.org/wiki/YAML), defined by the scheme below. Brackets indicate that a parameter is optional. -Generic placeholders are defined as follows: +### Generic placeholders * ``: a boolean that can take the values `true` or `false` * ``: any integer matching the regular expression `[1-9]+[0-9]*` @@ -20,5 +20,28 @@ Generic placeholders are defined as follows: * ``: an URL * ``: a CLI flag prefix based on the context (look at the parent configuration block to see which CLI flags prefix should be used) -Supported contents and default values of the config file: +### Use environment variables in the configuration + +You can use environment variable references in the config file to set values that need to be configurable during deployment. +To do this, use: + +``` +${VAR} +``` + +Where VAR is the name of the environment variable. + +Each variable reference is replaced at startup by the value of the environment variable. +The replacement is case-sensitive and occurs before the YAML file is parsed. +References to undefined variables are replaced by empty strings unless you specify a default value or custom error text. + +To specify a default value, use: + +``` +${VAR:default_value} +``` + +Where default_value is the value to use if the environment variable is undefined. + +### Supported contents and default values of the config file From 2325eed1264314245b6fe18b531d605e2449b42e Mon Sep 17 00:00:00 2001 From: Wing924 Date: Tue, 18 Feb 2020 15:54:07 +0900 Subject: [PATCH 2/3] add PR no. Signed-off-by: Wing924 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 95c6e0b5adc..ccbf8a8eae6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ * `--experimental.distributor.user-subring-size` * [FEATURE] Added flag `-experimental.ruler.enable-api` to enable the ruler api which implements the Prometheus API `/api/v1/rules` and `/api/v1/alerts` endpoints under the configured `-http.prefix`. #1999 * [FEATURE] Added sharding support to compactor when using the experimental TSDB blocks storage. #2113 -* [FEATURE] Add ability to override configuration settings using environment variables. +* [FEATURE] Add ability to override configuration settings using environment variables. #2147 * [ENHANCEMENT] Add `status` label to `cortex_alertmanager_configs` metric to gauge the number of valid and invalid configs. #2125 * [ENHANCEMENT] Cassandra Authentication: added the `custom_authenticators` config option that allows users to authenticate with cassandra clusters using password authenticators that are not approved by default in [gocql](https://github.com/gocql/gocql/blob/81b8263d9fe526782a588ef94d3fa5c6148e5d67/conn.go#L27) #2093 * [ENHANCEMENT] Experimental TSDB: Export TSDB Syncer metrics from Compactor component, they are prefixed with `cortex_compactor_`. #2023 From 4bdde7083af39fe19231744157060266a588358d Mon Sep 17 00:00:00 2001 From: Wing924 Date: Thu, 20 Feb 2020 10:51:31 +0900 Subject: [PATCH 3/3] fix Signed-off-by: Wing924 --- CHANGELOG.md | 3 +- cmd/cortex/main.go | 40 ++++++++++++++------------ cmd/cortex/main_test.go | 62 +++++++++++++++++++++++++++++++++++++++-- 3 files changed, 83 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 92d407f8675..9885e7f9a82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,8 @@ * `--experimental.distributor.user-subring-size` * [FEATURE] Added flag `-experimental.ruler.enable-api` to enable the ruler api which implements the Prometheus API `/api/v1/rules` and `/api/v1/alerts` endpoints under the configured `-http.prefix`. #1999 * [FEATURE] Added sharding support to compactor when using the experimental TSDB blocks storage. #2113 -* [FEATURE] Add ability to override configuration settings using environment variables. #2147 +* [FEATURE] Add ability to override YAML config file settings using environment variables. #2147 + * `-config.expand-env` * [ENHANCEMENT] Add `status` label to `cortex_alertmanager_configs` metric to gauge the number of valid and invalid configs. #2125 * [ENHANCEMENT] Cassandra Authentication: added the `custom_authenticators` config option that allows users to authenticate with cassandra clusters using password authenticators that are not approved by default in [gocql](https://github.com/gocql/gocql/blob/81b8263d9fe526782a588ef94d3fa5c6148e5d67/conn.go#L27) #2093 * [ENHANCEMENT] Experimental TSDB: Export TSDB Syncer metrics from Compactor component, they are prefixed with `cortex_compactor_`. #2023 diff --git a/cmd/cortex/main.go b/cmd/cortex/main.go index ef4adb2168e..24e53d8759a 100644 --- a/cmd/cortex/main.go +++ b/cmd/cortex/main.go @@ -26,7 +26,10 @@ func init() { prometheus.MustRegister(version.NewCollector("cortex")) } -const configFileOption = "config.file" +const ( + configFileOption = "config.file" + configExpandENV = "config.expand-env" +) var testMode = false @@ -38,14 +41,14 @@ func main() { mutexProfileFraction int ) - configFile := parseConfigFileParameter() + configFile, expandENV := parseConfigFileParameter(os.Args[1:]) // This sets default values from flags to the config. // It needs to be called before parsing the config file! flagext.RegisterFlags(&cfg) if configFile != "" { - if err := LoadConfig(configFile, &cfg); err != nil { + if err := LoadConfig(configFile, expandENV, &cfg); err != nil { fmt.Fprintf(os.Stderr, "error loading config from %s: %v\n", configFile, err) if testMode { return @@ -54,8 +57,10 @@ func main() { } } - // Ignore -config.file here, since it was already parsed, but it's still present on command line. + // Ignore -config.file and -config.expand-env here, since it was already parsed, but it's still present on command line. flagext.IgnoredFlag(flag.CommandLine, configFileOption, "Configuration file to load.") + flagext.IgnoredFlag(flag.CommandLine, configExpandENV, "Expands ${var} or $var in config according to the values of the environment variables.") + flag.IntVar(&eventSampleRate, "event.sample-rate", 0, "How often to sample observability events (0 = never).") flag.IntVar(&ballastBytes, "mem-ballast-size-bytes", 0, "Size of memory ballast to allocate.") flag.IntVar(&mutexProfileFraction, "debug.mutex-profile-fraction", 0, "Fraction at which mutex profile vents will be reported, 0 to disable") @@ -109,38 +114,37 @@ func main() { util.CheckFatal("initializing cortex", err) } -// Parse -config.file option via separate flag set, to avoid polluting default one and calling flag.Parse on it twice. -func parseConfigFileParameter() string { - var configFile = "" +// Parse -config.file and -config.expand-env option via separate flag set, to avoid polluting default one and calling flag.Parse on it twice. +func parseConfigFileParameter(args []string) (configFile string, expandEnv bool) { // ignore errors and any output here. Any flag errors will be reported by main flag.Parse() call. - fs := flag.NewFlagSet(os.Args[0], flag.ContinueOnError) + fs := flag.NewFlagSet("", flag.ContinueOnError) fs.SetOutput(ioutil.Discard) - fs.StringVar(&configFile, configFileOption, "", "") // usage not used in this function. - // Try to find -config.file option in the flags. As Parsing stops on the first error, eg. unknown flag, we simply + // usage not used in these functions. + fs.StringVar(&configFile, configFileOption, "", "") + fs.BoolVar(&expandEnv, configExpandENV, false, "") + + // Try to find -config.file and -config.expand-env option in the flags. As Parsing stops on the first error, eg. unknown flag, we simply // try remaining parameters until we find config flag, or there are no params left. // (ContinueOnError just means that flag.Parse doesn't call panic or os.Exit, but it returns error, which we ignore) - args := os.Args[1:] for len(args) > 0 { _ = fs.Parse(args) - if configFile != "" { - // found (!) - break - } args = args[1:] } - return configFile + return } // LoadConfig read YAML-formatted config from filename into cfg. -func LoadConfig(filename string, cfg *cortex.Config) error { +func LoadConfig(filename string, expandENV bool, cfg *cortex.Config) error { buf, err := ioutil.ReadFile(filename) if err != nil { return errors.Wrap(err, "Error reading config file") } - buf = expandEnv(buf) + if expandENV { + buf = expandEnv(buf) + } err = yaml.UnmarshalStrict(buf, cfg) if err != nil { diff --git a/cmd/cortex/main_test.go b/cmd/cortex/main_test.go index 2bdf6b13de2..e6ee1f8ee8d 100644 --- a/cmd/cortex/main_test.go +++ b/cmd/cortex/main_test.go @@ -6,6 +6,7 @@ import ( "io" "io/ioutil" "os" + "strings" "sync" "testing" @@ -62,6 +63,17 @@ func TestFlagParsing(t *testing.T) { stdoutMessage: "target: ingester\n", }, + "config without expand-env": { + yaml: "target: $TARGET", + stderrMessage: "Error parsing config file: unrecognised module name: $TARGET\n", + }, + + "config with expand-env": { + arguments: []string{"-config.expand-env"}, + yaml: "target: $TARGET", + stdoutMessage: "target: ingester\n", + }, + "config with arguments override": { yaml: "target: ingester", arguments: []string{"-target=distributor"}, @@ -71,12 +83,14 @@ func TestFlagParsing(t *testing.T) { // we cannot test the happy path, as cortex would then fully start } { t.Run(name, func(t *testing.T) { + _ = os.Setenv("TARGET", "ingester") testSingle(t, tc.arguments, tc.yaml, []byte(tc.stdoutMessage), []byte(tc.stderrMessage)) }) } } func testSingle(t *testing.T, arguments []string, yaml string, stdoutMessage, stderrMessage []byte) { + t.Helper() oldArgs, oldStdout, oldStderr, oldTestMode := os.Args, os.Stdout, os.Stderr, testMode defer func() { os.Stdout = oldStdout @@ -192,8 +206,50 @@ func TestExpandEnv(t *testing.T) { } for _, test := range tests { - os.Setenv("y", "y") - output := expandEnv([]byte(test.in)) - assert.Equal(t, test.out, string(output), "Input: %s", test.in) + test := test + t.Run(test.in, func(t *testing.T) { + _ = os.Setenv("y", "y") + output := expandEnv([]byte(test.in)) + assert.Equal(t, test.out, string(output), "Input: %s", test.in) + }) + } +} + +func TestParseConfigFileParameter(t *testing.T) { + var tests = []struct { + args string + configFile string + expandENV bool + }{ + {"", "", false}, + {"--foo", "", false}, + {"-f -a", "", false}, + + {"--config.file=foo", "foo", false}, + {"--config.file foo", "foo", false}, + {"--config.file=foo --config.expand-env", "foo", true}, + {"--config.expand-env --config.file=foo", "foo", true}, + + {"--opt1 --config.file=foo", "foo", false}, + {"--opt1 --config.file foo", "foo", false}, + {"--opt1 --config.file=foo --config.expand-env", "foo", true}, + {"--opt1 --config.expand-env --config.file=foo", "foo", true}, + + {"--config.file=foo --opt1", "foo", false}, + {"--config.file foo --opt1", "foo", false}, + {"--config.file=foo --config.expand-env --opt1", "foo", true}, + {"--config.expand-env --config.file=foo --opt1", "foo", true}, + + {"--config.file=foo --opt1 --config.expand-env", "foo", true}, + {"--config.expand-env --opt1 --config.file=foo", "foo", true}, + } + for _, test := range tests { + test := test + t.Run(test.args, func(t *testing.T) { + args := strings.Split(test.args, " ") + configFile, expandENV := parseConfigFileParameter(args) + assert.Equal(t, test.configFile, configFile) + assert.Equal(t, test.expandENV, expandENV) + }) } }