From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from smtp41.i.mail.ru (smtp41.i.mail.ru [94.100.177.101]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by dev.tarantool.org (Postfix) with ESMTPS id C17704696C4 for ; Fri, 1 May 2020 03:16:37 +0300 (MSK) From: Vladislav Shpilevoy Date: Fri, 1 May 2020 02:16:29 +0200 Message-Id: <3a29de4882b343fbff17058f67ec59867ec25e4a.1588292014.git.v.shpilevoy@tarantool.org> In-Reply-To: References: MIME-Version: 1.0 Content-Transfer-Encoding: 8bit Subject: [Tarantool-patches] [PATCH vshard 2/7] router: introduce discovery_mode List-Id: Tarantool development patches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , To: tarantool-patches@dev.tarantool.org, olegrok@tarantool.org Discovery disabling may be useful when there are very many routers and buckets, and a user does not want to pay overhead of the automatic massive discovery. It may be expensive in big clusters. In that case users may want to turn off discovery when there is no rebalancing, and turn it back on, when it starts, to keep the routers up to date with _bucket changes. Part of #210 --- test/router/router2.result | 143 +++++++++++++++++++++++++++++++++++ test/router/router2.test.lua | 51 +++++++++++++ vshard/cfg.lua | 10 +++ vshard/router/init.lua | 27 ++++++- 4 files changed, 227 insertions(+), 4 deletions(-) create mode 100644 test/router/router2.result create mode 100644 test/router/router2.test.lua diff --git a/test/router/router2.result b/test/router/router2.result new file mode 100644 index 0000000..556f749 --- /dev/null +++ b/test/router/router2.result @@ -0,0 +1,143 @@ +-- test-run result file version 2 +test_run = require('test_run').new() + | --- + | ... +REPLICASET_1 = { 'storage_1_a', 'storage_1_b' } + | --- + | ... +REPLICASET_2 = { 'storage_2_a', 'storage_2_b' } + | --- + | ... +test_run:create_cluster(REPLICASET_1, 'router') + | --- + | ... +test_run:create_cluster(REPLICASET_2, 'router') + | --- + | ... +util = require('util') + | --- + | ... +util.wait_master(test_run, REPLICASET_1, 'storage_1_a') + | --- + | ... +util.wait_master(test_run, REPLICASET_2, 'storage_2_a') + | --- + | ... +util.map_evals(test_run, {REPLICASET_1, REPLICASET_2}, 'bootstrap_storage(\'memtx\')') + | --- + | ... +util.push_rs_filters(test_run) + | --- + | ... +_ = test_run:cmd("create server router_1 with script='router/router_1.lua'") + | --- + | ... +_ = test_run:cmd("start server router_1") + | --- + | ... + +_ = test_run:switch("router_1") + | --- + | ... +util = require('util') + | --- + | ... + +-- gh-210: router should provide API to enable/disable discovery, +-- since it is a too expensive thing in big clusters to be not +-- stoppable/controllable. + +f1 = vshard.router.static.discovery_fiber + | --- + | ... +cfg.discovery_mode = 'off' + | --- + | ... +vshard.router.cfg(cfg) + | --- + | ... +vshard.router.static.discovery_fiber + | --- + | - null + | ... +f2 = vshard.router.static.discovery_fiber + | --- + | ... + +cfg.discovery_mode = 'on' + | --- + | ... +vshard.router.cfg(cfg) + | --- + | ... +f3 = vshard.router.static.discovery_fiber + | --- + | ... +vshard.router.static.discovery_fiber:status() + | --- + | - suspended + | ... + +cfg.discovery_mode = nil + | --- + | ... +vshard.router.cfg(cfg) + | --- + | ... +f4 = vshard.router.static.discovery_fiber + | --- + | ... +vshard.router.static.discovery_fiber:status() + | --- + | - suspended + | ... + +vshard.router.discovery_set('off') + | --- + | ... +f5 = vshard.router.static.discovery_fiber + | --- + | ... +vshard.router.static.discovery_fiber + | --- + | - null + | ... +vshard.router.discovery_set('on') + | --- + | ... +f6 = vshard.router.static.discovery_fiber + | --- + | ... +vshard.router.static.discovery_fiber:status() + | --- + | - suspended + | ... + +f1:status(), f2, f3:status(), f4:status(), f5, f6:status() + | --- + | - dead + | - null + | - dead + | - dead + | - null + | - suspended + | ... + +_ = test_run:switch("default") + | --- + | ... +_ = test_run:cmd("stop server router_1") + | --- + | ... +_ = test_run:cmd("cleanup server router_1") + | --- + | ... +test_run:drop_cluster(REPLICASET_1) + | --- + | ... +test_run:drop_cluster(REPLICASET_2) + | --- + | ... +_ = test_run:cmd('clear filter') + | --- + | ... diff --git a/test/router/router2.test.lua b/test/router/router2.test.lua new file mode 100644 index 0000000..33f4d3e --- /dev/null +++ b/test/router/router2.test.lua @@ -0,0 +1,51 @@ +test_run = require('test_run').new() +REPLICASET_1 = { 'storage_1_a', 'storage_1_b' } +REPLICASET_2 = { 'storage_2_a', 'storage_2_b' } +test_run:create_cluster(REPLICASET_1, 'router') +test_run:create_cluster(REPLICASET_2, 'router') +util = require('util') +util.wait_master(test_run, REPLICASET_1, 'storage_1_a') +util.wait_master(test_run, REPLICASET_2, 'storage_2_a') +util.map_evals(test_run, {REPLICASET_1, REPLICASET_2}, 'bootstrap_storage(\'memtx\')') +util.push_rs_filters(test_run) +_ = test_run:cmd("create server router_1 with script='router/router_1.lua'") +_ = test_run:cmd("start server router_1") + +_ = test_run:switch("router_1") +util = require('util') + +-- gh-210: router should provide API to enable/disable discovery, +-- since it is a too expensive thing in big clusters to be not +-- stoppable/controllable. + +f1 = vshard.router.static.discovery_fiber +cfg.discovery_mode = 'off' +vshard.router.cfg(cfg) +vshard.router.static.discovery_fiber +f2 = vshard.router.static.discovery_fiber + +cfg.discovery_mode = 'on' +vshard.router.cfg(cfg) +f3 = vshard.router.static.discovery_fiber +vshard.router.static.discovery_fiber:status() + +cfg.discovery_mode = nil +vshard.router.cfg(cfg) +f4 = vshard.router.static.discovery_fiber +vshard.router.static.discovery_fiber:status() + +vshard.router.discovery_set('off') +f5 = vshard.router.static.discovery_fiber +vshard.router.static.discovery_fiber +vshard.router.discovery_set('on') +f6 = vshard.router.static.discovery_fiber +vshard.router.static.discovery_fiber:status() + +f1:status(), f2, f3:status(), f4:status(), f5, f6:status() + +_ = test_run:switch("default") +_ = test_run:cmd("stop server router_1") +_ = test_run:cmd("cleanup server router_1") +test_run:drop_cluster(REPLICASET_1) +test_run:drop_cluster(REPLICASET_2) +_ = test_run:cmd('clear filter') diff --git a/vshard/cfg.lua b/vshard/cfg.lua index 0ce7b34..8a3e812 100644 --- a/vshard/cfg.lua +++ b/vshard/cfg.lua @@ -151,6 +151,12 @@ local function cfg_check_weights(weights) end end +local function check_discovery_mode(value) + if value ~= 'on' and value ~= 'off' then + error("Expected 'on' or 'off' for discovery_mode") + end +end + local function check_sharding(sharding) local uuids = {} local uris = {} @@ -255,6 +261,10 @@ local cfg_template = { type = 'positive number', name = 'Failover ping timeout', is_optional = true, default = consts.DEFAULT_FAILOVER_PING_TIMEOUT }, + discovery_mode = { + type = 'string', name = 'Discovery mode: on, off', + is_optional = true, default = 'on', check = check_discovery_mode + }, } -- diff --git a/vshard/router/init.lua b/vshard/router/init.lua index e430d92..ebd8356 100644 --- a/vshard/router/init.lua +++ b/vshard/router/init.lua @@ -64,6 +64,9 @@ local ROUTER_TEMPLATE = { failover_fiber = nil, -- Fiber to discovery buckets in background. discovery_fiber = nil, + -- How discovery works. On - work infinitely. Off - no + -- discovery. + discovery_mode = nil, -- Bucket count stored on all replicasets. total_bucket_count = 0, -- Boolean lua_gc state (create periodic gc task). @@ -232,6 +235,7 @@ if util.version_is_at_least(1, 10, 0) then -- discovery_f = function(router) local module_version = M.module_version + assert(router.discovery_mode == 'on') while module_version == M.module_version do while not next(router.replicasets) do lfiber.sleep(consts.DISCOVERY_INTERVAL) @@ -329,6 +333,23 @@ local function discovery_wakeup(router) end end +local function discovery_set(router, new_mode) + local current_mode = router.discovery_mode + if current_mode == new_mode then + return + end + router.discovery_mode = new_mode + if router.discovery_fiber ~= nil then + pcall(router.discovery_fiber.cancel, router.discovery_fiber) + end + if new_mode == 'off' then + router.discovery_fiber = nil + return + end + router.discovery_fiber = util.reloadable_fiber_create( + 'vshard.discovery.' .. router.name, M, 'discovery_f', router) +end + -------------------------------------------------------------------------------- -- API -------------------------------------------------------------------------------- @@ -792,10 +813,7 @@ local function router_cfg(router, cfg, is_reload) router.failover_fiber = util.reloadable_fiber_create( 'vshard.failover.' .. router.name, M, 'failover_f', router) end - if router.discovery_fiber == nil then - router.discovery_fiber = util.reloadable_fiber_create( - 'vshard.discovery.' .. router.name, M, 'discovery_f', router) - end + discovery_set(router, vshard_cfg.discovery_mode) end -------------------------------------------------------------------------------- @@ -1154,6 +1172,7 @@ local router_mt = { bootstrap = cluster_bootstrap; bucket_discovery = bucket_discovery; discovery_wakeup = discovery_wakeup; + discovery_set = discovery_set, } } -- 2.21.1 (Apple Git-122.3)