[tarantool-patches] [PATCH v3] ctl: added functionality to detect and prune dead replicas

[Olga Arkhangelskaia <arkholga@tarantool.org>]

After restart of replicaset or after some configuration changes
replicaset may suffer from littering the system space by "dead" replicas.
To detect and throw away such replica replicaset_list_inactive(),
replica_prune(), replicaset_purge() are introduced.

replicaset_list_inactive() returns uuid table of dead replicas
replica_prune(uuid) throws away replica with given id from system space
replicaset_purge([uuid]) throws away table of dead replicas, that is
provided by admin or by replicaset_list_inactive().

Closes #3110

@TarantoolBot document
Title: Introduce replicaset_list_inactive(), replica_prune(uuid),
replicaset_purge([uuid])

After some changes in replicaset (restart, configuration change) replicaset may suffer
from leftovers or previous configurations: empty replicas with uuid, but
without any read/write activity. To delete such trash from system the space space,
replicaset_list_inactive(), replica_prune(), replicaset_purge() are introduced.
replicaset_list_inactive() lists uuid table of such "zombie" replicas.
replica_prune(uuid) throws away replica with given id from system space
replicaset_purge([uuid]) throws away table of dead replicas, that is
provided by admin or by replicaset_list_inactive().
---
Issue:
https://github.com/tarantool/tarantool/issues/3110
Branch:
https://github.com/tarantool/tarantool/tree/OKriw/gh-3110-prune-dead-replica-from-replicaset-2.1


v1:
https://www.freelists.org/post/tarantool-patches/PATCH-rfc-schema-add-possibility-to-find-and-throw-away-dead-replicas

Changes v2:
- changed the way of replicas death detection
- added special box options
- changed test
- now only dead replicas are shown
- added function to throw away any replica

v2:
https://www.freelists.org/post/tarantool-patches/PATCH-v2-02-detect-and-throw-away-dead-replicas

Changes v3:
- detect and throw away "zombie" replica, with null applier/relay
- no additional box configuration parameters 

 src/box/CMakeLists.txt         |   1 +
 src/box/lua/ctl.lua            |  56 ++++++++++
 src/box/lua/init.c             |   2 +
 test/replication/trim.lua      |  32 ++++++
 test/replication/trim.result   | 244 +++++++++++++++++++++++++++++++++++++++++
 test/replication/trim.test.lua |  90 +++++++++++++++
 test/replication/trim1.lua     |   1 +
 test/replication/trim2.lua     |   1 +
 test/replication/trim3.lua     |   1 +
 9 files changed, 428 insertions(+)
 create mode 100644 src/box/lua/ctl.lua
 create mode 100644 test/replication/trim.lua
 create mode 100644 test/replication/trim.result
 create mode 100644 test/replication/trim.test.lua
 create mode 120000 test/replication/trim1.lua
 create mode 120000 test/replication/trim2.lua
 create mode 120000 test/replication/trim3.lua

diff --git a/src/box/CMakeLists.txt b/src/box/CMakeLists.txt
index d1276472b..3a1735637 100644
--- a/src/box/CMakeLists.txt
+++ b/src/box/CMakeLists.txt
@@ -13,6 +13,7 @@ lua_source(lua_sources lua/net_box.lua)
 lua_source(lua_sources lua/upgrade.lua)
 lua_source(lua_sources lua/console.lua)
 lua_source(lua_sources lua/xlog.lua)
+lua_source(lua_sources lua/ctl.lua)
 set(bin_sources)
 bin_source(bin_sources bootstrap.snap bootstrap.h)
 
diff --git a/src/box/lua/ctl.lua b/src/box/lua/ctl.lua
new file mode 100644
index 000000000..d2991a4ee
--- /dev/null
+++ b/src/box/lua/ctl.lua
@@ -0,0 +1,56 @@
+-- ctl.lua (internal file)
+
+-- checks whether given replica is dead
+local function is_dead(replica)
+    -- self
+    if replica.uuid == box.info.uuid then
+        return false
+    end
+    -- no applier no relay
+    if replica.upstream == nil and replica.downstream == nil then
+        return true
+    end
+    -- disconnected or stopped relay
+    if replica.upstream == nil and replica.downstream ~= nil then
+        if replica.downstream.status == 'disconnected' or replica.downstream.status == 'stopped' then
+            return true
+        else
+            return false
+        end
+    end
+    -- in other cases replica is alive
+    return false
+end
+
+-- return list of replicas suspected to be dead
+function replicaset_list_inactive()
+    local inactive_list = {}
+    local replicaset = box.info.replication
+    for i, replica in pairs(replicaset) do
+        -- current replica is alive
+        if is_dead(replica) then
+            table.insert(inactive_list, replica.uuid)
+        end
+    end
+    return inactive_list
+end
+
+-- throw away any replica from system space
+function replica_prune(uuid)
+    if uuid == nil then
+        error("Usage: replica_displace([uuid])")
+    end
+    box.space._cluster.index.uuid:delete{uuid}
+end
+
+-- delete table of dead replica obtained from replicaset_list_inactive() or
+-- formed by admin
+function replicaset_purge(uuid_table)
+    if uuid_table == nil then
+        error("Usage: replicaset_trim([uuid_table])")
+    end
+   for i in pairs(uuid_table) do
+        print("Deleting replica with uuid ".. i.. " "..uuid_table[i])
+        replica_prune(uuid_table[i])
+    end
+end
diff --git a/src/box/lua/init.c b/src/box/lua/init.c
index ccb4c6a46..0339ea92e 100644
--- a/src/box/lua/init.c
+++ b/src/box/lua/init.c
@@ -63,6 +63,7 @@
 extern char session_lua[],
 	tuple_lua[],
 	schema_lua[],
+	ctl_lua[],
 	load_cfg_lua[],
 	xlog_lua[],
 	checkpoint_daemon_lua[],
@@ -82,6 +83,7 @@ static const char *lua_sources[] = {
 	"box/console", console_lua,
 	"box/load_cfg", load_cfg_lua,
 	"box/xlog", xlog_lua,
+	"box/ctl", ctl_lua,
 	NULL
 };
 
diff --git a/test/replication/trim.lua b/test/replication/trim.lua
new file mode 100644
index 000000000..f79fe6e02
--- /dev/null
+++ b/test/replication/trim.lua
@@ -0,0 +1,32 @@
+#!/usr/bin/env tarantool
+
+-- get instance name from filename (trim1.lua => trim1)
+local INSTANCE_ID = string.match(arg[0], "%d")
+local SOCKET_DIR = require('fio').cwd()
+local TIMEOUT = tonumber(arg[1])
+local CON_TIMEOUT = arg[2] and tonumber(arg[2]) or 30.0
+
+local function instance_uri(instance_id)
+    --return 'localhost:'..(3310 + instance_id)
+    return SOCKET_DIR..'/trim'..instance_id..'.sock';
+end
+
+require('console').listen(os.getenv('ADMIN'))
+
+box.cfg({
+    listen = instance_uri(INSTANCE_ID);
+    replication_timeout = TIMEOUT;
+    replication_connect_timeout = CON_TIMEOUT;
+    replication = {
+        instance_uri(1);
+        instance_uri(2);
+        instance_uri(3);
+    };
+})
+
+box.once("bootstrap", function()
+    local test_run = require('test_run').new()
+    box.schema.user.grant("guest", 'replication')
+    box.schema.space.create('test', {engine = test_run:get_cfg('engine')})
+    box.space.test:create_index('primary')
+end)
diff --git a/test/replication/trim.result b/test/replication/trim.result
new file mode 100644
index 000000000..f85b3db97
--- /dev/null
+++ b/test/replication/trim.result
@@ -0,0 +1,244 @@
+test_run = require('test_run').new()
+---
+...
+SERVERS = {'trim1', 'trim2', 'trim3'}
+---
+...
+-- Deploy cluster
+test_run:create_cluster(SERVERS, "replication", {args="0.1"})
+---
+...
+test_run:wait_fullmesh(SERVERS)
+---
+...
+test_run:cmd('switch trim1')
+---
+- true
+...
+len = box.space._cluster:len()
+---
+...
+-- no errors
+replicaset_list_inactive()
+---
+- []
+...
+replica_prune()
+---
+- error: 'builtin/box/ctl.lua:41: Usage: replica_displace([uuid])'
+...
+replicaset_purge()
+---
+- error: 'builtin/box/ctl.lua:50: Usage: replicaset_trim([uuid_table])'
+...
+-- create zombies after restart all replicas
+test_run:cmd('switch trim1')
+---
+- true
+...
+fiber = require('fiber')
+---
+...
+old_trim2 = test_run:get_param('trim2', 'id')
+---
+...
+old_trim3 = test_run:get_param('trim3', 'id')
+---
+...
+len = box.space._cluster:len()
+---
+...
+test_run:cmd('switch default')
+---
+- true
+...
+test_run:cmd('stop server trim2')
+---
+- true
+...
+test_run:cmd('cleanup server trim2')
+---
+- true
+...
+test_run:cmd('start server trim2')
+---
+- true
+...
+test_run:cmd('stop server trim3')
+---
+- true
+...
+test_run:cmd('cleanup server trim3')
+---
+- true
+...
+test_run:cmd('start server trim3')
+---
+- true
+...
+test_run:cmd('switch trim1')
+---
+- true
+...
+replicaset_list_inactive() ~= nil
+---
+- true
+...
+box.space._cluster:len() == len + #replicaset_list_inactive()
+---
+- true
+...
+-- check that we showed and throw away only dead replicas
+trim2 = test_run:get_param('trim2', 'id')
+---
+...
+trim3 = test_run:get_param('trim3', 'id')
+---
+...
+while box.info.replication[trim2[1]].upstream.status == 'follow' do fiber.sleep(0.01) end
+---
+...
+while box.info.replication[trim3[1]].upstream.status =='follow' do fiber.sleep(0.01) end
+---
+...
+box.info.replication[trim2[1]].downstream.status == nil
+---
+- true
+...
+box.info.replication[trim3[1]].downstream.status == nil
+---
+- true
+...
+box.info.replication[old_trim2[1]].upstream == nil
+---
+- true
+...
+box.info.replication[old_trim3[1]].upstream == nil
+---
+- true
+...
+box.info.replication[old_trim2[1]].downstream.status == 'stopped'
+---
+- true
+...
+box.info.replication[old_trim3[1]].downstream.status == 'stopped'
+---
+- true
+...
+--
+replicaset_list_inactive() == 2
+---
+- false
+...
+replicaset_purge(replicaset_list_inactive())
+---
+...
+#replicaset_list_inactive() == 0
+---
+- true
+...
+box.space._cluster:len() == len
+---
+- true
+...
+box.info.replication[trim2[1]] ~= nil
+---
+- true
+...
+box.info.replication[trim3[1]] ~= nil
+---
+- true
+...
+box.info.replication[trim2[1]].downstream.status == nil
+---
+- true
+...
+box.info.replication[trim3[1]].downstream.status == nil
+---
+- true
+...
+box.info.replication[old_trim2[1]] == nil
+---
+- true
+...
+box.info.replication[old_trim3[1]] == nil
+---
+- true
+...
+-- no applier no relay
+test_run:cmd('switch default')
+---
+- true
+...
+test_run:cmd('stop server trim2')
+---
+- true
+...
+test_run:cmd('cleanup server trim2')
+---
+- true
+...
+test_run:cmd('start server trim2')
+---
+- true
+...
+test_run:cmd('stop server trim3')
+---
+- true
+...
+test_run:cmd('cleanup server trim3')
+---
+- true
+...
+test_run:cmd('start server trim3')
+---
+- true
+...
+test_run:cmd('stop server trim1')
+---
+- true
+...
+test_run:cmd('cleanup server trim1')
+---
+- true
+...
+test_run:cmd('start server trim1')
+---
+- true
+...
+test_run:cmd('switch trim1')
+---
+- true
+...
+box.ctl.wait_rw(10)
+---
+- error: timed out
+...
+inactive = replicaset_list_inactive()
+---
+...
+-- prune given replica
+replica_prune(inactive[1])
+---
+- error: Can't modify data because this instance is in read-only mode.
+...
+#replicaset_list_inactive() ~= #inactive
+---
+- false
+...
+replicaset_purge(replicaset_list_inactive())
+---
+- error: Can't modify data because this instance is in read-only mode.
+...
+box.space._cluster:len() == 3
+---
+- false
+...
+-- Cleanup
+test_run:cmd('switch default')
+---
+- true
+...
+test_run:drop_cluster(SERVERS)
+---
+...
diff --git a/test/replication/trim.test.lua b/test/replication/trim.test.lua
new file mode 100644
index 000000000..2c467e313
--- /dev/null
+++ b/test/replication/trim.test.lua
@@ -0,0 +1,90 @@
+test_run = require('test_run').new()
+
+SERVERS = {'trim1', 'trim2', 'trim3'}
+
+-- Deploy cluster
+test_run:create_cluster(SERVERS, "replication", {args="0.1"})
+test_run:wait_fullmesh(SERVERS)
+
+test_run:cmd('switch trim1')
+len = box.space._cluster:len()
+
+-- no errors
+replicaset_list_inactive()
+replica_prune()
+replicaset_purge()
+
+-- create zombies after restart all replicas
+test_run:cmd('switch trim1')
+fiber = require('fiber')
+old_trim2 = test_run:get_param('trim2', 'id')
+old_trim3 = test_run:get_param('trim3', 'id')
+
+len = box.space._cluster:len()
+test_run:cmd('switch default')
+test_run:cmd('stop server trim2')
+test_run:cmd('cleanup server trim2')
+test_run:cmd('start server trim2')
+test_run:cmd('stop server trim3')
+test_run:cmd('cleanup server trim3')
+test_run:cmd('start server trim3')
+test_run:cmd('switch trim1')
+
+replicaset_list_inactive() ~= nil
+box.space._cluster:len() == len + #replicaset_list_inactive()
+
+-- check that we showed and throw away only dead replicas
+trim2 = test_run:get_param('trim2', 'id')
+trim3 = test_run:get_param('trim3', 'id')
+
+while box.info.replication[trim2[1]].upstream.status == 'follow' do fiber.sleep(0.01) end
+while box.info.replication[trim3[1]].upstream.status =='follow' do fiber.sleep(0.01) end
+box.info.replication[trim2[1]].downstream.status == nil
+box.info.replication[trim3[1]].downstream.status == nil
+
+box.info.replication[old_trim2[1]].upstream == nil
+box.info.replication[old_trim3[1]].upstream == nil
+box.info.replication[old_trim2[1]].downstream.status == 'stopped'
+box.info.replication[old_trim3[1]].downstream.status == 'stopped'
+--
+replicaset_list_inactive() == 2
+replicaset_purge(replicaset_list_inactive())
+#replicaset_list_inactive() == 0
+box.space._cluster:len() == len
+
+box.info.replication[trim2[1]] ~= nil
+box.info.replication[trim3[1]] ~= nil
+box.info.replication[trim2[1]].downstream.status == nil
+box.info.replication[trim3[1]].downstream.status == nil
+
+box.info.replication[old_trim2[1]] == nil
+box.info.replication[old_trim3[1]] == nil
+
+
+-- no applier no relay
+
+test_run:cmd('switch default')
+test_run:cmd('stop server trim2')
+test_run:cmd('cleanup server trim2')
+test_run:cmd('start server trim2')
+test_run:cmd('stop server trim3')
+test_run:cmd('cleanup server trim3')
+test_run:cmd('start server trim3')
+test_run:cmd('stop server trim1')
+test_run:cmd('cleanup server trim1')
+test_run:cmd('start server trim1')
+test_run:cmd('switch trim1')
+
+box.ctl.wait_rw(10)
+
+inactive = replicaset_list_inactive()
+
+-- prune given replica
+replica_prune(inactive[1])
+#replicaset_list_inactive() ~= #inactive
+replicaset_purge(replicaset_list_inactive())
+box.space._cluster:len() == 3
+
+-- Cleanup
+test_run:cmd('switch default')
+test_run:drop_cluster(SERVERS)
diff --git a/test/replication/trim1.lua b/test/replication/trim1.lua
new file mode 120000
index 000000000..14e98bd68
--- /dev/null
+++ b/test/replication/trim1.lua
@@ -0,0 +1 @@
+trim.lua
\ No newline at end of file
diff --git a/test/replication/trim2.lua b/test/replication/trim2.lua
new file mode 120000
index 000000000..14e98bd68
--- /dev/null
+++ b/test/replication/trim2.lua
@@ -0,0 +1 @@
+trim.lua
\ No newline at end of file
diff --git a/test/replication/trim3.lua b/test/replication/trim3.lua
new file mode 120000
index 000000000..14e98bd68
--- /dev/null
+++ b/test/replication/trim3.lua
@@ -0,0 +1 @@
+trim.lua
\ No newline at end of file
-- 
2.14.3 (Apple Git-98)