[tarantool-patches] [PATCH v3] ctl: added functionality to detect and prune dead replicas
Olga Arkhangelskaia
arkholga at tarantool.org
Mon Oct 29 22:25:15 MSK 2018
After restart of replicaset or after some configuration changes
replicaset may suffer from littering the system space by "dead" replicas.
To detect and throw away such replica replicaset_list_inactive(),
replica_prune(), replicaset_purge() are introduced.
replicaset_list_inactive() returns uuid table of dead replicas
replica_prune(uuid) throws away replica with given id from system space
replicaset_purge([uuid]) throws away table of dead replicas, that is
provided by admin or by replicaset_list_inactive().
Closes #3110
@TarantoolBot document
Title: Introduce replicaset_list_inactive(), replica_prune(uuid),
replicaset_purge([uuid])
After some changes in replicaset (restart, configuration change) replicaset may suffer
from leftovers or previous configurations: empty replicas with uuid, but
without any read/write activity. To delete such trash from system the space space,
replicaset_list_inactive(), replica_prune(), replicaset_purge() are introduced.
replicaset_list_inactive() lists uuid table of such "zombie" replicas.
replica_prune(uuid) throws away replica with given id from system space
replicaset_purge([uuid]) throws away table of dead replicas, that is
provided by admin or by replicaset_list_inactive().
---
Issue:
https://github.com/tarantool/tarantool/issues/3110
Branch:
https://github.com/tarantool/tarantool/tree/OKriw/gh-3110-prune-dead-replica-from-replicaset-2.1
v1:
https://www.freelists.org/post/tarantool-patches/PATCH-rfc-schema-add-possibility-to-find-and-throw-away-dead-replicas
Changes v2:
- changed the way of replicas death detection
- added special box options
- changed test
- now only dead replicas are shown
- added function to throw away any replica
v2:
https://www.freelists.org/post/tarantool-patches/PATCH-v2-02-detect-and-throw-away-dead-replicas
Changes v3:
- detect and throw away "zombie" replica, with null applier/relay
- no additional box configuration parameters
src/box/CMakeLists.txt | 1 +
src/box/lua/ctl.lua | 56 ++++++++++
src/box/lua/init.c | 2 +
test/replication/trim.lua | 32 ++++++
test/replication/trim.result | 244 +++++++++++++++++++++++++++++++++++++++++
test/replication/trim.test.lua | 90 +++++++++++++++
test/replication/trim1.lua | 1 +
test/replication/trim2.lua | 1 +
test/replication/trim3.lua | 1 +
9 files changed, 428 insertions(+)
create mode 100644 src/box/lua/ctl.lua
create mode 100644 test/replication/trim.lua
create mode 100644 test/replication/trim.result
create mode 100644 test/replication/trim.test.lua
create mode 120000 test/replication/trim1.lua
create mode 120000 test/replication/trim2.lua
create mode 120000 test/replication/trim3.lua
diff --git a/src/box/CMakeLists.txt b/src/box/CMakeLists.txt
index d1276472b..3a1735637 100644
--- a/src/box/CMakeLists.txt
+++ b/src/box/CMakeLists.txt
@@ -13,6 +13,7 @@ lua_source(lua_sources lua/net_box.lua)
lua_source(lua_sources lua/upgrade.lua)
lua_source(lua_sources lua/console.lua)
lua_source(lua_sources lua/xlog.lua)
+lua_source(lua_sources lua/ctl.lua)
set(bin_sources)
bin_source(bin_sources bootstrap.snap bootstrap.h)
diff --git a/src/box/lua/ctl.lua b/src/box/lua/ctl.lua
new file mode 100644
index 000000000..d2991a4ee
--- /dev/null
+++ b/src/box/lua/ctl.lua
@@ -0,0 +1,56 @@
+-- ctl.lua (internal file)
+
+-- checks whether given replica is dead
+local function is_dead(replica)
+ -- self
+ if replica.uuid == box.info.uuid then
+ return false
+ end
+ -- no applier no relay
+ if replica.upstream == nil and replica.downstream == nil then
+ return true
+ end
+ -- disconnected or stopped relay
+ if replica.upstream == nil and replica.downstream ~= nil then
+ if replica.downstream.status == 'disconnected' or replica.downstream.status == 'stopped' then
+ return true
+ else
+ return false
+ end
+ end
+ -- in other cases replica is alive
+ return false
+end
+
+-- return list of replicas suspected to be dead
+function replicaset_list_inactive()
+ local inactive_list = {}
+ local replicaset = box.info.replication
+ for i, replica in pairs(replicaset) do
+ -- current replica is alive
+ if is_dead(replica) then
+ table.insert(inactive_list, replica.uuid)
+ end
+ end
+ return inactive_list
+end
+
+-- throw away any replica from system space
+function replica_prune(uuid)
+ if uuid == nil then
+ error("Usage: replica_displace([uuid])")
+ end
+ box.space._cluster.index.uuid:delete{uuid}
+end
+
+-- delete table of dead replica obtained from replicaset_list_inactive() or
+-- formed by admin
+function replicaset_purge(uuid_table)
+ if uuid_table == nil then
+ error("Usage: replicaset_trim([uuid_table])")
+ end
+ for i in pairs(uuid_table) do
+ print("Deleting replica with uuid ".. i.. " "..uuid_table[i])
+ replica_prune(uuid_table[i])
+ end
+end
diff --git a/src/box/lua/init.c b/src/box/lua/init.c
index ccb4c6a46..0339ea92e 100644
--- a/src/box/lua/init.c
+++ b/src/box/lua/init.c
@@ -63,6 +63,7 @@
extern char session_lua[],
tuple_lua[],
schema_lua[],
+ ctl_lua[],
load_cfg_lua[],
xlog_lua[],
checkpoint_daemon_lua[],
@@ -82,6 +83,7 @@ static const char *lua_sources[] = {
"box/console", console_lua,
"box/load_cfg", load_cfg_lua,
"box/xlog", xlog_lua,
+ "box/ctl", ctl_lua,
NULL
};
diff --git a/test/replication/trim.lua b/test/replication/trim.lua
new file mode 100644
index 000000000..f79fe6e02
--- /dev/null
+++ b/test/replication/trim.lua
@@ -0,0 +1,32 @@
+#!/usr/bin/env tarantool
+
+-- get instance name from filename (trim1.lua => trim1)
+local INSTANCE_ID = string.match(arg[0], "%d")
+local SOCKET_DIR = require('fio').cwd()
+local TIMEOUT = tonumber(arg[1])
+local CON_TIMEOUT = arg[2] and tonumber(arg[2]) or 30.0
+
+local function instance_uri(instance_id)
+ --return 'localhost:'..(3310 + instance_id)
+ return SOCKET_DIR..'/trim'..instance_id..'.sock';
+end
+
+require('console').listen(os.getenv('ADMIN'))
+
+box.cfg({
+ listen = instance_uri(INSTANCE_ID);
+ replication_timeout = TIMEOUT;
+ replication_connect_timeout = CON_TIMEOUT;
+ replication = {
+ instance_uri(1);
+ instance_uri(2);
+ instance_uri(3);
+ };
+})
+
+box.once("bootstrap", function()
+ local test_run = require('test_run').new()
+ box.schema.user.grant("guest", 'replication')
+ box.schema.space.create('test', {engine = test_run:get_cfg('engine')})
+ box.space.test:create_index('primary')
+end)
diff --git a/test/replication/trim.result b/test/replication/trim.result
new file mode 100644
index 000000000..f85b3db97
--- /dev/null
+++ b/test/replication/trim.result
@@ -0,0 +1,244 @@
+test_run = require('test_run').new()
+---
+...
+SERVERS = {'trim1', 'trim2', 'trim3'}
+---
+...
+-- Deploy cluster
+test_run:create_cluster(SERVERS, "replication", {args="0.1"})
+---
+...
+test_run:wait_fullmesh(SERVERS)
+---
+...
+test_run:cmd('switch trim1')
+---
+- true
+...
+len = box.space._cluster:len()
+---
+...
+-- no errors
+replicaset_list_inactive()
+---
+- []
+...
+replica_prune()
+---
+- error: 'builtin/box/ctl.lua:41: Usage: replica_displace([uuid])'
+...
+replicaset_purge()
+---
+- error: 'builtin/box/ctl.lua:50: Usage: replicaset_trim([uuid_table])'
+...
+-- create zombies after restart all replicas
+test_run:cmd('switch trim1')
+---
+- true
+...
+fiber = require('fiber')
+---
+...
+old_trim2 = test_run:get_param('trim2', 'id')
+---
+...
+old_trim3 = test_run:get_param('trim3', 'id')
+---
+...
+len = box.space._cluster:len()
+---
+...
+test_run:cmd('switch default')
+---
+- true
+...
+test_run:cmd('stop server trim2')
+---
+- true
+...
+test_run:cmd('cleanup server trim2')
+---
+- true
+...
+test_run:cmd('start server trim2')
+---
+- true
+...
+test_run:cmd('stop server trim3')
+---
+- true
+...
+test_run:cmd('cleanup server trim3')
+---
+- true
+...
+test_run:cmd('start server trim3')
+---
+- true
+...
+test_run:cmd('switch trim1')
+---
+- true
+...
+replicaset_list_inactive() ~= nil
+---
+- true
+...
+box.space._cluster:len() == len + #replicaset_list_inactive()
+---
+- true
+...
+-- check that we showed and throw away only dead replicas
+trim2 = test_run:get_param('trim2', 'id')
+---
+...
+trim3 = test_run:get_param('trim3', 'id')
+---
+...
+while box.info.replication[trim2[1]].upstream.status == 'follow' do fiber.sleep(0.01) end
+---
+...
+while box.info.replication[trim3[1]].upstream.status =='follow' do fiber.sleep(0.01) end
+---
+...
+box.info.replication[trim2[1]].downstream.status == nil
+---
+- true
+...
+box.info.replication[trim3[1]].downstream.status == nil
+---
+- true
+...
+box.info.replication[old_trim2[1]].upstream == nil
+---
+- true
+...
+box.info.replication[old_trim3[1]].upstream == nil
+---
+- true
+...
+box.info.replication[old_trim2[1]].downstream.status == 'stopped'
+---
+- true
+...
+box.info.replication[old_trim3[1]].downstream.status == 'stopped'
+---
+- true
+...
+--
+replicaset_list_inactive() == 2
+---
+- false
+...
+replicaset_purge(replicaset_list_inactive())
+---
+...
+#replicaset_list_inactive() == 0
+---
+- true
+...
+box.space._cluster:len() == len
+---
+- true
+...
+box.info.replication[trim2[1]] ~= nil
+---
+- true
+...
+box.info.replication[trim3[1]] ~= nil
+---
+- true
+...
+box.info.replication[trim2[1]].downstream.status == nil
+---
+- true
+...
+box.info.replication[trim3[1]].downstream.status == nil
+---
+- true
+...
+box.info.replication[old_trim2[1]] == nil
+---
+- true
+...
+box.info.replication[old_trim3[1]] == nil
+---
+- true
+...
+-- no applier no relay
+test_run:cmd('switch default')
+---
+- true
+...
+test_run:cmd('stop server trim2')
+---
+- true
+...
+test_run:cmd('cleanup server trim2')
+---
+- true
+...
+test_run:cmd('start server trim2')
+---
+- true
+...
+test_run:cmd('stop server trim3')
+---
+- true
+...
+test_run:cmd('cleanup server trim3')
+---
+- true
+...
+test_run:cmd('start server trim3')
+---
+- true
+...
+test_run:cmd('stop server trim1')
+---
+- true
+...
+test_run:cmd('cleanup server trim1')
+---
+- true
+...
+test_run:cmd('start server trim1')
+---
+- true
+...
+test_run:cmd('switch trim1')
+---
+- true
+...
+box.ctl.wait_rw(10)
+---
+- error: timed out
+...
+inactive = replicaset_list_inactive()
+---
+...
+-- prune given replica
+replica_prune(inactive[1])
+---
+- error: Can't modify data because this instance is in read-only mode.
+...
+#replicaset_list_inactive() ~= #inactive
+---
+- false
+...
+replicaset_purge(replicaset_list_inactive())
+---
+- error: Can't modify data because this instance is in read-only mode.
+...
+box.space._cluster:len() == 3
+---
+- false
+...
+-- Cleanup
+test_run:cmd('switch default')
+---
+- true
+...
+test_run:drop_cluster(SERVERS)
+---
+...
diff --git a/test/replication/trim.test.lua b/test/replication/trim.test.lua
new file mode 100644
index 000000000..2c467e313
--- /dev/null
+++ b/test/replication/trim.test.lua
@@ -0,0 +1,90 @@
+test_run = require('test_run').new()
+
+SERVERS = {'trim1', 'trim2', 'trim3'}
+
+-- Deploy cluster
+test_run:create_cluster(SERVERS, "replication", {args="0.1"})
+test_run:wait_fullmesh(SERVERS)
+
+test_run:cmd('switch trim1')
+len = box.space._cluster:len()
+
+-- no errors
+replicaset_list_inactive()
+replica_prune()
+replicaset_purge()
+
+-- create zombies after restart all replicas
+test_run:cmd('switch trim1')
+fiber = require('fiber')
+old_trim2 = test_run:get_param('trim2', 'id')
+old_trim3 = test_run:get_param('trim3', 'id')
+
+len = box.space._cluster:len()
+test_run:cmd('switch default')
+test_run:cmd('stop server trim2')
+test_run:cmd('cleanup server trim2')
+test_run:cmd('start server trim2')
+test_run:cmd('stop server trim3')
+test_run:cmd('cleanup server trim3')
+test_run:cmd('start server trim3')
+test_run:cmd('switch trim1')
+
+replicaset_list_inactive() ~= nil
+box.space._cluster:len() == len + #replicaset_list_inactive()
+
+-- check that we showed and throw away only dead replicas
+trim2 = test_run:get_param('trim2', 'id')
+trim3 = test_run:get_param('trim3', 'id')
+
+while box.info.replication[trim2[1]].upstream.status == 'follow' do fiber.sleep(0.01) end
+while box.info.replication[trim3[1]].upstream.status =='follow' do fiber.sleep(0.01) end
+box.info.replication[trim2[1]].downstream.status == nil
+box.info.replication[trim3[1]].downstream.status == nil
+
+box.info.replication[old_trim2[1]].upstream == nil
+box.info.replication[old_trim3[1]].upstream == nil
+box.info.replication[old_trim2[1]].downstream.status == 'stopped'
+box.info.replication[old_trim3[1]].downstream.status == 'stopped'
+--
+replicaset_list_inactive() == 2
+replicaset_purge(replicaset_list_inactive())
+#replicaset_list_inactive() == 0
+box.space._cluster:len() == len
+
+box.info.replication[trim2[1]] ~= nil
+box.info.replication[trim3[1]] ~= nil
+box.info.replication[trim2[1]].downstream.status == nil
+box.info.replication[trim3[1]].downstream.status == nil
+
+box.info.replication[old_trim2[1]] == nil
+box.info.replication[old_trim3[1]] == nil
+
+
+-- no applier no relay
+
+test_run:cmd('switch default')
+test_run:cmd('stop server trim2')
+test_run:cmd('cleanup server trim2')
+test_run:cmd('start server trim2')
+test_run:cmd('stop server trim3')
+test_run:cmd('cleanup server trim3')
+test_run:cmd('start server trim3')
+test_run:cmd('stop server trim1')
+test_run:cmd('cleanup server trim1')
+test_run:cmd('start server trim1')
+test_run:cmd('switch trim1')
+
+box.ctl.wait_rw(10)
+
+inactive = replicaset_list_inactive()
+
+-- prune given replica
+replica_prune(inactive[1])
+#replicaset_list_inactive() ~= #inactive
+replicaset_purge(replicaset_list_inactive())
+box.space._cluster:len() == 3
+
+-- Cleanup
+test_run:cmd('switch default')
+test_run:drop_cluster(SERVERS)
diff --git a/test/replication/trim1.lua b/test/replication/trim1.lua
new file mode 120000
index 000000000..14e98bd68
--- /dev/null
+++ b/test/replication/trim1.lua
@@ -0,0 +1 @@
+trim.lua
\ No newline at end of file
diff --git a/test/replication/trim2.lua b/test/replication/trim2.lua
new file mode 120000
index 000000000..14e98bd68
--- /dev/null
+++ b/test/replication/trim2.lua
@@ -0,0 +1 @@
+trim.lua
\ No newline at end of file
diff --git a/test/replication/trim3.lua b/test/replication/trim3.lua
new file mode 120000
index 000000000..14e98bd68
--- /dev/null
+++ b/test/replication/trim3.lua
@@ -0,0 +1 @@
+trim.lua
\ No newline at end of file
--
2.14.3 (Apple Git-98)
More information about the Tarantool-patches
mailing list