[Tarantool-patches] [PATCH vshard 2/7] router: introduce discovery_mode

[Vladislav Shpilevoy <v.shpilevoy@tarantool.org>]

Discovery disabling may be useful when there are very many routers
and buckets, and a user does not want to pay overhead of the
automatic massive discovery. It may be expensive in big clusters.

In that case users may want to turn off discovery when there is
no rebalancing, and turn it back on, when it starts, to keep the
routers up to date with _bucket changes.

Part of #210
---
 test/router/router2.result   | 143 +++++++++++++++++++++++++++++++++++
 test/router/router2.test.lua |  51 +++++++++++++
 vshard/cfg.lua               |  10 +++
 vshard/router/init.lua       |  27 ++++++-
 4 files changed, 227 insertions(+), 4 deletions(-)
 create mode 100644 test/router/router2.result
 create mode 100644 test/router/router2.test.lua

diff --git a/test/router/router2.result b/test/router/router2.result
new file mode 100644
index 0000000..556f749
--- /dev/null
+++ b/test/router/router2.result
@@ -0,0 +1,143 @@
+-- test-run result file version 2
+test_run = require('test_run').new()
+ | ---
+ | ...
+REPLICASET_1 = { 'storage_1_a', 'storage_1_b' }
+ | ---
+ | ...
+REPLICASET_2 = { 'storage_2_a', 'storage_2_b' }
+ | ---
+ | ...
+test_run:create_cluster(REPLICASET_1, 'router')
+ | ---
+ | ...
+test_run:create_cluster(REPLICASET_2, 'router')
+ | ---
+ | ...
+util = require('util')
+ | ---
+ | ...
+util.wait_master(test_run, REPLICASET_1, 'storage_1_a')
+ | ---
+ | ...
+util.wait_master(test_run, REPLICASET_2, 'storage_2_a')
+ | ---
+ | ...
+util.map_evals(test_run, {REPLICASET_1, REPLICASET_2}, 'bootstrap_storage(\'memtx\')')
+ | ---
+ | ...
+util.push_rs_filters(test_run)
+ | ---
+ | ...
+_ = test_run:cmd("create server router_1 with script='router/router_1.lua'")
+ | ---
+ | ...
+_ = test_run:cmd("start server router_1")
+ | ---
+ | ...
+
+_ = test_run:switch("router_1")
+ | ---
+ | ...
+util = require('util')
+ | ---
+ | ...
+
+-- gh-210: router should provide API to enable/disable discovery,
+-- since it is a too expensive thing in big clusters to be not
+-- stoppable/controllable.
+
+f1 = vshard.router.static.discovery_fiber
+ | ---
+ | ...
+cfg.discovery_mode = 'off'
+ | ---
+ | ...
+vshard.router.cfg(cfg)
+ | ---
+ | ...
+vshard.router.static.discovery_fiber
+ | ---
+ | - null
+ | ...
+f2 = vshard.router.static.discovery_fiber
+ | ---
+ | ...
+
+cfg.discovery_mode = 'on'
+ | ---
+ | ...
+vshard.router.cfg(cfg)
+ | ---
+ | ...
+f3 = vshard.router.static.discovery_fiber
+ | ---
+ | ...
+vshard.router.static.discovery_fiber:status()
+ | ---
+ | - suspended
+ | ...
+
+cfg.discovery_mode = nil
+ | ---
+ | ...
+vshard.router.cfg(cfg)
+ | ---
+ | ...
+f4 = vshard.router.static.discovery_fiber
+ | ---
+ | ...
+vshard.router.static.discovery_fiber:status()
+ | ---
+ | - suspended
+ | ...
+
+vshard.router.discovery_set('off')
+ | ---
+ | ...
+f5 = vshard.router.static.discovery_fiber
+ | ---
+ | ...
+vshard.router.static.discovery_fiber
+ | ---
+ | - null
+ | ...
+vshard.router.discovery_set('on')
+ | ---
+ | ...
+f6 = vshard.router.static.discovery_fiber
+ | ---
+ | ...
+vshard.router.static.discovery_fiber:status()
+ | ---
+ | - suspended
+ | ...
+
+f1:status(), f2, f3:status(), f4:status(), f5, f6:status()
+ | ---
+ | - dead
+ | - null
+ | - dead
+ | - dead
+ | - null
+ | - suspended
+ | ...
+
+_ = test_run:switch("default")
+ | ---
+ | ...
+_ = test_run:cmd("stop server router_1")
+ | ---
+ | ...
+_ = test_run:cmd("cleanup server router_1")
+ | ---
+ | ...
+test_run:drop_cluster(REPLICASET_1)
+ | ---
+ | ...
+test_run:drop_cluster(REPLICASET_2)
+ | ---
+ | ...
+_ = test_run:cmd('clear filter')
+ | ---
+ | ...
diff --git a/test/router/router2.test.lua b/test/router/router2.test.lua
new file mode 100644
index 0000000..33f4d3e
--- /dev/null
+++ b/test/router/router2.test.lua
@@ -0,0 +1,51 @@
+test_run = require('test_run').new()
+REPLICASET_1 = { 'storage_1_a', 'storage_1_b' }
+REPLICASET_2 = { 'storage_2_a', 'storage_2_b' }
+test_run:create_cluster(REPLICASET_1, 'router')
+test_run:create_cluster(REPLICASET_2, 'router')
+util = require('util')
+util.wait_master(test_run, REPLICASET_1, 'storage_1_a')
+util.wait_master(test_run, REPLICASET_2, 'storage_2_a')
+util.map_evals(test_run, {REPLICASET_1, REPLICASET_2}, 'bootstrap_storage(\'memtx\')')
+util.push_rs_filters(test_run)
+_ = test_run:cmd("create server router_1 with script='router/router_1.lua'")
+_ = test_run:cmd("start server router_1")
+
+_ = test_run:switch("router_1")
+util = require('util')
+
+-- gh-210: router should provide API to enable/disable discovery,
+-- since it is a too expensive thing in big clusters to be not
+-- stoppable/controllable.
+
+f1 = vshard.router.static.discovery_fiber
+cfg.discovery_mode = 'off'
+vshard.router.cfg(cfg)
+vshard.router.static.discovery_fiber
+f2 = vshard.router.static.discovery_fiber
+
+cfg.discovery_mode = 'on'
+vshard.router.cfg(cfg)
+f3 = vshard.router.static.discovery_fiber
+vshard.router.static.discovery_fiber:status()
+
+cfg.discovery_mode = nil
+vshard.router.cfg(cfg)
+f4 = vshard.router.static.discovery_fiber
+vshard.router.static.discovery_fiber:status()
+
+vshard.router.discovery_set('off')
+f5 = vshard.router.static.discovery_fiber
+vshard.router.static.discovery_fiber
+vshard.router.discovery_set('on')
+f6 = vshard.router.static.discovery_fiber
+vshard.router.static.discovery_fiber:status()
+
+f1:status(), f2, f3:status(), f4:status(), f5, f6:status()
+
+_ = test_run:switch("default")
+_ = test_run:cmd("stop server router_1")
+_ = test_run:cmd("cleanup server router_1")
+test_run:drop_cluster(REPLICASET_1)
+test_run:drop_cluster(REPLICASET_2)
+_ = test_run:cmd('clear filter')
diff --git a/vshard/cfg.lua b/vshard/cfg.lua
index 0ce7b34..8a3e812 100644
--- a/vshard/cfg.lua
+++ b/vshard/cfg.lua
@@ -151,6 +151,12 @@ local function cfg_check_weights(weights)
     end
 end
 
+local function check_discovery_mode(value)
+    if value ~= 'on' and value ~= 'off' then
+        error("Expected 'on' or 'off' for discovery_mode")
+    end
+end
+
 local function check_sharding(sharding)
     local uuids = {}
     local uris = {}
@@ -255,6 +261,10 @@ local cfg_template = {
         type = 'positive number', name = 'Failover ping timeout',
         is_optional = true, default = consts.DEFAULT_FAILOVER_PING_TIMEOUT
     },
+    discovery_mode = {
+        type = 'string', name = 'Discovery mode: on, off',
+        is_optional = true, default = 'on', check = check_discovery_mode
+    },
 }
 
 --
diff --git a/vshard/router/init.lua b/vshard/router/init.lua
index e430d92..ebd8356 100644
--- a/vshard/router/init.lua
+++ b/vshard/router/init.lua
@@ -64,6 +64,9 @@ local ROUTER_TEMPLATE = {
         failover_fiber = nil,
         -- Fiber to discovery buckets in background.
         discovery_fiber = nil,
+        -- How discovery works. On - work infinitely. Off - no
+        -- discovery.
+        discovery_mode = nil,
         -- Bucket count stored on all replicasets.
         total_bucket_count = 0,
         -- Boolean lua_gc state (create periodic gc task).
@@ -232,6 +235,7 @@ if util.version_is_at_least(1, 10, 0) then
 --
 discovery_f = function(router)
     local module_version = M.module_version
+    assert(router.discovery_mode == 'on')
     while module_version == M.module_version do
         while not next(router.replicasets) do
             lfiber.sleep(consts.DISCOVERY_INTERVAL)
@@ -329,6 +333,23 @@ local function discovery_wakeup(router)
     end
 end
 
+local function discovery_set(router, new_mode)
+    local current_mode = router.discovery_mode
+    if current_mode == new_mode then
+        return
+    end
+    router.discovery_mode = new_mode
+    if router.discovery_fiber ~= nil then
+        pcall(router.discovery_fiber.cancel, router.discovery_fiber)
+    end
+    if new_mode == 'off' then
+        router.discovery_fiber = nil
+        return
+    end
+    router.discovery_fiber = util.reloadable_fiber_create(
+        'vshard.discovery.' .. router.name, M, 'discovery_f', router)
+end
+
 --------------------------------------------------------------------------------
 -- API
 --------------------------------------------------------------------------------
@@ -792,10 +813,7 @@ local function router_cfg(router, cfg, is_reload)
         router.failover_fiber = util.reloadable_fiber_create(
             'vshard.failover.' .. router.name, M, 'failover_f', router)
     end
-    if router.discovery_fiber == nil then
-        router.discovery_fiber = util.reloadable_fiber_create(
-            'vshard.discovery.' .. router.name, M, 'discovery_f', router)
-    end
+    discovery_set(router, vshard_cfg.discovery_mode)
 end
 
 --------------------------------------------------------------------------------
@@ -1154,6 +1172,7 @@ local router_mt = {
         bootstrap = cluster_bootstrap;
         bucket_discovery = bucket_discovery;
         discovery_wakeup = discovery_wakeup;
+        discovery_set = discovery_set,
     }
 }
 
-- 
2.21.1 (Apple Git-122.3)