config: add low priority env source

Totktonada · Totktonada · commit 59f7371ae4a7 · 2023-07-31T22:00:17.000+03:00
The usual environment configuration source is useful for parametrized run: ``` TT_MEMTX_MEMORY=<...> tarantool --name <...> --config <...> ``` However, sometimes a user may need to set a default value, which doesn't rewrite one that is provided in the configuration. Now, it is possible to do using the environment variables with the `_DEFAULT` suffix. ``` TT_MEMTX_MEMORY_DEFAULT=<...> tarantool --name <...> --config <...> ``` This feature may be especially useful for wrappers that run tarantool internally with some default paths for data directories, socket files, pid file. We likely will use it in the `tt` tool. Part of tarantool#8862 NO_DOC=added into tarantool/doc#3544 manually
diff --git a/changelogs/unreleased/config-low-priority-env-source.md b/changelogs/unreleased/config-low-priority-env-source.md
@@ -0,0 +1,4 @@
+## feature/config
+
+* Added a low priority environment configuration source that looks through the
+  `TT_*_DEFAULT` variables. It is useful for declaring default values (gh-8862).
diff --git a/src/box/lua/config/init.lua b/src/box/lua/config/init.lua
@@ -104,12 +104,14 @@ end
 
 function methods._initialize(self)
     -- The sources are synchronized in the order of registration:
-    -- env, file, etcd (the latter is present in Tarantool EE).
+    -- env, file, etcd (present in Tarantool EE), env for
+    -- defaults.
     --
     -- The configuration values from the first source has highest
     -- priority. The menthal rule here is the following: values
     -- closer to the process are preferred: env first, then file,
-    -- then etcd (if available).
+    -- then etcd (if available). And only then the env source with
+    -- defaults.
     self:_register_source(require('internal.config.source.env').new())
 
     if self._config_file ~= nil then
@@ -128,6 +130,10 @@ function methods._initialize(self)
     if extras ~= nil then
         extras.initialize(self)
     end
+
+    self:_register_source(require('internal.config.source.env').new({
+        env_var_suffix = 'default',
+    }))
 end
 
 function methods._collect(self, opts)
diff --git a/src/box/lua/config/source/env.lua b/src/box/lua/config/source/env.lua
@@ -6,12 +6,20 @@ local mt = {
     __index = methods,
 }
 
+function methods._env_var_name(self, path_in_schema)
+    local env_var_name = 'TT_' .. table.concat(path_in_schema, '_'):upper()
+    if self._env_var_suffix ~= nil then
+        return env_var_name .. self._env_var_suffix
+    end
+    return env_var_name
+end
+
 -- Gather most actual config values.
 function methods.sync(self, _config_module, _iconfig)
     local values = {}
 
     for _, w in instance_config:pairs() do
-        local env_var_name = 'TT_' .. table.concat(w.path, '_'):upper()
+        local env_var_name = self:_env_var_name(w.path)
         local raw_value = os.getenv(env_var_name)
         local value = schema.fromenv(env_var_name, raw_value, w.schema)
         if value ~= nil then
@@ -35,11 +43,23 @@ function methods.get(self)
     return self._values
 end
 
-local function new()
+local function new(opts)
+    local opts = opts or {}
+    local env_var_suffix = opts.env_var_suffix
+
+    local name = 'env'
+
+    if env_var_suffix ~= nil then
+        name = ('%s (%s)'):format(name, env_var_suffix)
+        env_var_suffix = '_' .. env_var_suffix:upper()
+    end
+
     return setmetatable({
-        name = 'env',
+        name = name,
         type = 'instance',
+
         _values = {},
+        _env_var_suffix = env_var_suffix,
     }, mt)
 end
 
diff --git a/test/config-luatest/sources_test.lua b/test/config-luatest/sources_test.lua
@@ -1,8 +1,11 @@
 local t = require('luatest')
+local fun = require('fun')
 local json = require('json')
+local yaml = require('yaml')
 local treegen = require('test.treegen')
 local justrun = require('test.justrun')
 local source_file = require('internal.config.source.file').new()
+local server = require('test.luatest_helpers.server')
 
 local g = t.group()
 
@@ -14,6 +17,13 @@ g.after_all(function()
     treegen.clean(g)
 end)
 
+g.after_each(function(g)
+    if g.server ~= nil then
+        g.server:stop()
+        g.server = nil
+    end
+end)
+
 g.test_source_file = function()
     local config = {_config_file = 'doc/examples/config/single.yaml'}
     source_file:sync(config, {})
@@ -49,15 +59,14 @@ g.test_source_env = function()
     local dir = treegen.prepare_directory(g, {}, {})
     local script = [[
         local json = require('json')
-        local source_env = require('internal.config.source.env').new()
+        local source_env = require('internal.config.source.env').new({
+            env_var_suffix = arg[1],
+        })
         source_env:sync({}, {})
         print(json.encode(source_env:get()))
     ]]
     treegen.write_script(dir, 'main.lua', script)
 
-    local env = {TT_LOG_LEVEL = 'info', TT_MEMTX_MEMORY = 1000000}
-    local opts = {nojson = true, stderr = false}
-    local res = justrun.tarantool(dir, env, {'main.lua'}, opts)
     local exp = {
         config = {
             version = 'dev',
@@ -69,6 +78,104 @@ g.test_source_env = function()
             memory = 1000000
         },
     }
-    t.assert_equals(res.exit_code, 0)
-    t.assert_equals(json.decode(res.stdout), exp)
+
+    local cases = {
+        {
+            name = 'env',
+            env_var_suffix = nil,
+            env = {
+                TT_LOG_LEVEL = 'info',
+                TT_MEMTX_MEMORY = 1000000,
+            },
+        },
+        {
+            name = 'env default',
+            env_var_suffix = 'default',
+            env = {
+                TT_LOG_LEVEL_DEFAULT = 'info',
+                TT_MEMTX_MEMORY_DEFAULT = 1000000,
+            },
+        },
+    }
+    local opts = {nojson = true, stderr = false}
+    for _, case in ipairs(cases) do
+        local comment = ('case: %s'):format(case.name)
+        local args = {'main.lua', case.env_var_suffix}
+        local res = justrun.tarantool(dir, case.env, args, opts)
+        t.assert_equals(res.exit_code, 0, comment)
+        t.assert_equals(json.decode(res.stdout), exp, comment)
+    end
+end
+
+-- Verify priority of configuration sources.
+--
+-- 1. env (TT_*)
+-- 2. file
+-- 3. env default (TT_*_DEFAULT)
+--
+-- Several string options from the instance config are chosen for
+-- testing purposes, their meaning is irrelevant for the test.
+--
+-- The table below shows where the given option is set (which
+-- source defines it) and what we expect as a result.
+--
+-- | option              | env | file | env default | result      |
+-- | ------------------- | --- | ---- | ----------- | ----------- |
+-- | process.title       |     |  +   |      +      | file        |
+-- | log.file            |  +  |  +   |             | env         |
+-- | log.pipe            |  +  |      |      +      | env         |
+-- | log.syslog.identity |     |      |      +      | env default |
+g.test_sources_priority = function(g)
+    local dir = treegen.prepare_directory(g, {}, {})
+    local config = {
+        credentials = {
+            users = {
+                guest = {
+                    roles = {'super'},
+                },
+            },
+        },
+        iproto = {
+            listen = 'unix/:./{{ instance_name }}.iproto',
+        },
+        process = {
+            title = 'from file',
+        },
+        log = {
+            file = 'from file',
+        },
+        groups = {
+            ['group-001'] = {
+                replicasets = {
+                    ['replicaset-001'] = {
+                        instances = {
+                            ['instance-001'] = {},
+                        },
+                    },
+                },
+            },
+        },
+    }
+    local config_file = treegen.write_script(dir, 'config.yaml',
+                                             yaml.encode(config))
+    local opts = {
+        config_file = config_file,
+        chdir = dir,
+        env = {
+            TT_PROCESS_TITLE_DEFAULT = 'from env default',
+            TT_LOG_FILE = 'from env',
+            TT_LOG_PIPE = 'from env',
+            TT_LOG_PIPE_DEFAULT = 'from env default',
+            TT_LOG_SYSLOG_IDENTITY_DEFAULT = 'from env default',
+        },
+    }
+    g.server = server:new(fun.chain(opts, {alias = 'instance-001'}):tomap())
+    g.server:start()
+    g.server:exec(function()
+        local config = require('config')
+        t.assert_equals(config:get('process.title'), 'from file')
+        t.assert_equals(config:get('log.file'), 'from env')
+        t.assert_equals(config:get('log.pipe'), 'from env')
+        t.assert_equals(config:get('log.syslog.identity'), 'from env default')
+    end)
 end
