* [Tarantool-patches] [PATCH 2/2] box: set box.cfg options via environment variables
2021-04-15 6:04 [Tarantool-patches] [PATCH 0/2] Set box.cfg options using environment variables Alexander Turenko via Tarantool-patches
2021-04-15 6:04 ` [Tarantool-patches] [PATCH 1/2] cfg: provide types for logger options Alexander Turenko via Tarantool-patches
@ 2021-04-15 6:04 ` Alexander Turenko via Tarantool-patches
2021-04-15 6:04 ` [Tarantool-patches] [PATCH 0/2] Set box.cfg options using " Leonid Vasiliev via Tarantool-patches
` (2 subsequent siblings)
4 siblings, 0 replies; 6+ messages in thread
From: Alexander Turenko via Tarantool-patches @ 2021-04-15 6:04 UTC (permalink / raw)
To: Leonid Vasiliev; +Cc: tarantool-patches, Alexander Turenko
From: Roman Khabibov <roman.habibov@tarantool.org>
Add ability to set box.cfg options via environment variables. These
variables should have name `TT_<OPTION>`. When Tarantool instance is
started under tarantoolctl utility, environment variables have higher
priority than tarantoolctl configuration file.
Closes #5602
Co-authored-by: Leonid Vasiliev <lvasiliev@tarantool.org>
Co-authored-by: Alexander Turenko <alexander.turenko@tarantool.org>
@TarantoolBot document
Title: Set box.cfg options via environment variables
Now, it is possible to set box.cfg options via environment variables.
The name of variable should correspond the following pattern:
`TT_<NAME>`, where `<NAME>` is uppercase box.cfg option name. For
example: `TT_LISTEN`, `TT_READAHEAD`.
Array values are separated by comma. Example:
```sh
export TT_REPLICATION=localhost:3301,localhost:3302
```
An empty variable is the same as unset one.
---
changelogs/unreleased/environment-cfg.md | 7 ++
extra/dist/tarantoolctl.in | 19 +++-
src/box/lua/load_cfg.lua | 99 +++++++++++++++++++
.../gh-5602-environment-cfg-test-cases.lua | 71 +++++++++++++
.../gh-5602-environment-vars-cfg.result | 33 +++++++
.../gh-5602-environment-vars-cfg.test.lua | 57 +++++++++++
6 files changed, 285 insertions(+), 1 deletion(-)
create mode 100755 changelogs/unreleased/environment-cfg.md
create mode 100755 test/box-tap/gh-5602-environment-cfg-test-cases.lua
create mode 100644 test/box-tap/gh-5602-environment-vars-cfg.result
create mode 100755 test/box-tap/gh-5602-environment-vars-cfg.test.lua
diff --git a/changelogs/unreleased/environment-cfg.md b/changelogs/unreleased/environment-cfg.md
new file mode 100755
index 000000000..f1d37f6e6
--- /dev/null
+++ b/changelogs/unreleased/environment-cfg.md
@@ -0,0 +1,7 @@
+## feature/core
+
+* Now, it is possible to set box.cfg options with environment variables
+ (gh-5602).
+
+ The priority of sources of configuration options is the following (from low
+ to high): default, tarantoolctl, environment, box.cfg{}.
diff --git a/extra/dist/tarantoolctl.in b/extra/dist/tarantoolctl.in
index 0726e7f46..fc0885fbe 100755
--- a/extra/dist/tarantoolctl.in
+++ b/extra/dist/tarantoolctl.in
@@ -449,10 +449,27 @@ local function wrapper_cfg(cfg)
end
end
+ -- Collect box.cfg options from environment variables if
+ -- tarantool supports this feature.
+ local ok, env_cfg = pcall(function()
+ return box.internal.cfg and box.internal.cfg.env or {}
+ end)
+ if not ok then
+ log.error(tostring(env_cfg))
+ os.exit(1)
+ end
+
cfg = cfg or {}
for i, v in pairs(default_cfg) do
if cfg[i] == nil then
- cfg[i] = v
+ -- If an option is set using an environment variable,
+ -- prefer this value. Otherwise fallback to
+ -- tarantoolctl's default value.
+ --
+ -- If we'll not do it there, the tarantoolctl's
+ -- default will rewrite the value passed via the
+ -- environment variable.
+ cfg[i] = env_cfg[i] or v
end
end
-- force these startup options
diff --git a/src/box/lua/load_cfg.lua b/src/box/lua/load_cfg.lua
index 72e889071..11fa98586 100644
--- a/src/box/lua/load_cfg.lua
+++ b/src/box/lua/load_cfg.lua
@@ -123,6 +123,10 @@ local module_cfg = {
-- provide some API with type enumeration or
-- similar. Currently it has use for environment
-- processing only.
+--
+-- get_option_from_env() leans on the set of types
+-- in use: don't forget to update it when add a new
+-- type or a combination of types here.
local module_cfg_type = {
-- logging
log = 'string',
@@ -133,6 +137,10 @@ local module_cfg_type = {
-- types of available options
-- could be comma separated lua types or 'any' if any type is allowed
+--
+-- get_option_from_env() leans on the set of types in use: don't
+-- forget to update it when add a new type or a combination of
+-- types here.
local template_cfg = {
listen = 'string, number',
memtx_memory = 'number',
@@ -539,6 +547,18 @@ local function prepare_cfg(cfg, default_cfg, template_cfg,
return new_cfg
end
+-- Transfer options from env_cfg to cfg.
+local function apply_env_cfg(cfg, env_cfg)
+ -- Add options passed through environment variables.
+ -- Here we only add options without overloading the ones set
+ -- by the user.
+ for k, v in pairs(env_cfg) do
+ if cfg[k] == nil then
+ cfg[k] = v
+ end
+ end
+end
+
local function apply_default_cfg(cfg, default_cfg, module_cfg)
for k,v in pairs(default_cfg) do
if cfg[k] == nil then
@@ -682,6 +702,10 @@ local function load_cfg(cfg)
end
cfg = upgrade_cfg(cfg, translate_cfg)
+
+ -- Set options passed through environment variables.
+ apply_env_cfg(cfg, box.internal.cfg.env)
+
cfg = prepare_cfg(cfg, default_cfg, template_cfg,
module_cfg, modify_cfg)
apply_default_cfg(cfg, default_cfg, module_cfg);
@@ -794,6 +818,81 @@ box_load_and_execute = function(...)
end
box.execute = box_load_and_execute
+--
+-- Parse TT_* environment variable that corresponds to given
+-- option.
+--
+local function get_option_from_env(option)
+ local param_type = template_cfg[option]
+ assert(type(param_type) == 'string')
+
+ if param_type == 'module' then
+ -- Parameter from module.
+ param_type = module_cfg_type[option]
+ end
+
+ local env_var_name = 'TT_' .. option:upper()
+ local raw_value = os.getenv(env_var_name)
+
+ if raw_value == nil or raw_value == '' then
+ return nil
+ end
+
+ local err_msg_fmt = 'Environment variable %s has ' ..
+ 'incorrect value for option "%s": should be %s'
+
+ -- This code lean on the existing set of template_cfg and
+ -- module_cfg_type types for simplicity.
+ if param_type:find('table') and raw_value:find(',') then
+ assert(not param_type:find('boolean'))
+ local res = {}
+ for i, v in ipairs(raw_value:split(',')) do
+ res[i] = tonumber(v) or v
+ end
+ return res
+ elseif param_type:find('boolean') then
+ assert(param_type == 'boolean')
+ if raw_value:lower() == 'false' then
+ return false
+ elseif raw_value:lower() == 'true' then
+ return true
+ end
+ error(err_msg_fmt:format(env_var_name, option, '"true" or "false"'))
+ elseif param_type == 'number' then
+ local res = tonumber(raw_value)
+ if res == nil then
+ error(err_msg_fmt:format(env_var_name, option,
+ 'convertible to a number'))
+ end
+ return res
+ elseif param_type:find('number') then
+ assert(not param_type:find('boolean'))
+ return tonumber(raw_value) or raw_value
+ else
+ assert(param_type == 'string')
+ return raw_value
+ end
+end
+
+--
+-- Read box configuration from environment variables.
+--
+box.internal.cfg = setmetatable({}, {
+ __index = function(self, key)
+ if key == 'env' then
+ local res = {}
+ for option, _ in pairs(template_cfg) do
+ res[option] = get_option_from_env(option)
+ end
+ return res
+ end
+ assert(false)
+ end,
+ __newindex = function(self, key, value) -- luacheck: no unused args
+ error('Attempt to modify a read-only table')
+ end,
+})
+
-- gh-810:
-- hack luajit default cpath
-- commented out because we fixed luajit to build properly, see
diff --git a/test/box-tap/gh-5602-environment-cfg-test-cases.lua b/test/box-tap/gh-5602-environment-cfg-test-cases.lua
new file mode 100755
index 000000000..72031778b
--- /dev/null
+++ b/test/box-tap/gh-5602-environment-cfg-test-cases.lua
@@ -0,0 +1,71 @@
+local tap = require('tap')
+
+local test = tap.test('gh-5602')
+
+local status, err = pcall(box.cfg, {background = false, vinyl_timeout = 70.1})
+
+-- Check that environment cfg values are set correctly.
+if arg[1] == '1' then
+ test:plan(6)
+ test:ok(status, 'box.cfg is successful')
+ test:is(box.cfg['listen'], '3301', 'listen')
+ test:is(box.cfg['readahead'], 10000, 'readahead')
+ test:is(box.cfg['strip_core'], false, 'strip_core')
+ test:is(box.cfg['log_format'], 'json', 'log_format is not set')
+ test:is(box.cfg['log_nonblock'], false, 'log_nonblock')
+end
+if arg[1] == '2' then
+ test:plan(7)
+ test:ok(status, 'box.cfg is successful')
+ test:is(box.cfg['listen'], '3301', 'listen')
+ local replication = box.cfg['replication']
+ test:is(type(replication), 'table', 'replication is table')
+ test:ok(replication[1] == '0.0.0.0:12345' or
+ replication[1] == '1.1.1.1:12345', 'replication URI 1')
+ test:ok(replication[2] == '0.0.0.0:12345' or
+ replication[2] == '1.1.1.1:12345', 'replication URI 2')
+ test:is(box.cfg['replication_connect_timeout'], 0.01,
+ 'replication_connect_timeout')
+ test:is(box.cfg['replication_synchro_quorum'], '4 + 1',
+ 'replication_synchro_quorum')
+end
+
+-- Check that box.cfg{} values are more prioritized than
+-- environment cfg values.
+if arg[1] == '3' then
+ test:plan(3)
+ test:ok(status, 'box.cfg is successful')
+ test:is(box.cfg['background'], false,
+ 'box.cfg{} background value is prioritized')
+ test:is(box.cfg['vinyl_timeout'], 70.1,
+ 'box.cfg{} vinyl_timeout value is prioritized')
+end
+
+local err_msg_fmt = 'Environment variable TT_%s has incorrect value for ' ..
+ 'option "%s": should be %s'
+
+-- Check bad environment cfg values.
+if arg[1] == '4' then
+ test:plan(2)
+ test:ok(not status, 'box.cfg is not successful')
+ local exp_err = err_msg_fmt:format('SQL_CACHE_SIZE', 'sql_cache_size',
+ 'convertible to a number')
+ local err_msg = tostring(err)
+ while err_msg:find('^.-:.-: ') do
+ err_msg = err_msg:gsub('^.-:.-: ', '')
+ end
+ test:is(err_msg, exp_err, 'bad sql_cache_size value')
+end
+if arg[1] == '5' then
+ test:plan(2)
+ test:ok(not status, 'box.cfg is not successful')
+ local exp_err = err_msg_fmt:format('STRIP_CORE', 'strip_core',
+ '"true" or "false"')
+ local err_msg = tostring(err)
+ while err_msg:find('^.-:.-: ') do
+ err_msg = err_msg:gsub('^.-:.-: ', '')
+ end
+ test:is(err_msg, exp_err, 'bad strip_core value')
+end
+
+os.exit(test:check() and 0 or 1)
diff --git a/test/box-tap/gh-5602-environment-vars-cfg.result b/test/box-tap/gh-5602-environment-vars-cfg.result
new file mode 100644
index 000000000..3050d5747
--- /dev/null
+++ b/test/box-tap/gh-5602-environment-vars-cfg.result
@@ -0,0 +1,33 @@
+TAP version 13
+1..6
+ok - box.cfg is successful
+ok - listen
+ok - readahead
+ok - strip_core
+ok - log_format is not set
+ok - log_nonblock
+TAP version 13
+1..7
+ok - box.cfg is successful
+ok - listen
+ok - replication is table
+ok - replication URI 1
+ok - replication URI 2
+ok - replication_connect_timeout
+ok - replication_synchro_quorum
+TAP version 13
+1..3
+ok - box.cfg is successful
+ok - box.cfg{} background value is prioritized
+ok - box.cfg{} vinyl_timeout value is prioritized
+TAP version 13
+1..2
+ok - box.cfg is not successful
+ok - bad sql_cache_size value
+TAP version 13
+1..2
+ok - box.cfg is not successful
+ok - bad strip_core value
+TAP version 13
+1..1
+ok - exit status list
diff --git a/test/box-tap/gh-5602-environment-vars-cfg.test.lua b/test/box-tap/gh-5602-environment-vars-cfg.test.lua
new file mode 100755
index 000000000..be1e402ee
--- /dev/null
+++ b/test/box-tap/gh-5602-environment-vars-cfg.test.lua
@@ -0,0 +1,57 @@
+#!/usr/bin/env tarantool
+
+local os = require('os')
+local fio = require('fio')
+local tap = require('tap')
+
+local test = tap.test('gh-5602')
+
+-- gh-5602: Check that environment cfg variables working.
+local TARANTOOL_PATH = arg[-1]
+local script_name = 'gh-5602-environment-cfg-test-cases.lua'
+local path_to_script = fio.pathjoin(
+ os.getenv('PWD'),
+ 'box-tap',
+ script_name)
+
+-- Generate a shell command like
+-- `FOO=x BAR=y /path/to/tarantool /path/to/script.lua 42`.
+local function shell_command(case, i)
+ return ('%s %s %s %d'):format(
+ case,
+ TARANTOOL_PATH,
+ path_to_script,
+ i)
+end
+
+local cases = {
+ ('%s %s %s %s %s'):format(
+ 'TT_LISTEN=3301',
+ 'TT_READAHEAD=10000',
+ 'TT_STRIP_CORE=false',
+ 'TT_LOG_FORMAT=json',
+ 'TT_LOG_NONBLOCK=false'),
+ ('%s %s %s %s'):format(
+ 'TT_LISTEN=3301',
+ 'TT_REPLICATION=0.0.0.0:12345,1.1.1.1:12345',
+ 'TT_REPLICATION_CONNECT_TIMEOUT=0.01',
+ 'TT_REPLICATION_SYNCHRO_QUORUM=\'4 + 1\''),
+ 'TT_BACKGROUND=true TT_VINYL_TIMEOUT=60.1',
+ 'TT_SQL_CACHE_SIZE=a',
+ 'TT_STRIP_CORE=a',
+}
+
+test:plan(1)
+local exit_status_list = {}
+local exit_status_list_exp = {}
+for i, case in ipairs(cases) do
+ local tmpdir = fio.tempdir()
+ local new_path = fio.pathjoin(tmpdir, script_name)
+ fio.copyfile(path_to_script, new_path)
+ exit_status_list[i] = os.execute(shell_command(case, i))
+ exit_status_list_exp[i] = 0
+end
+
+test:is_deeply(exit_status_list, exit_status_list_exp, 'exit status list')
+
+os.exit(test:check() and 0 or 1)
--
2.30.0
^ permalink raw reply [flat|nested] 6+ messages in thread