From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from localhost (localhost [127.0.0.1]) by turing.freelists.org (Avenir Technologies Mail Multiplex) with ESMTP id F244D2DB93 for ; Mon, 29 Oct 2018 15:25:33 -0400 (EDT) Received: from turing.freelists.org ([127.0.0.1]) by localhost (turing.freelists.org [127.0.0.1]) (amavisd-new, port 10024) with ESMTP id IK26qDflmTH7 for ; Mon, 29 Oct 2018 15:25:33 -0400 (EDT) Received: from smtp49.i.mail.ru (smtp49.i.mail.ru [94.100.177.109]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by turing.freelists.org (Avenir Technologies Mail Multiplex) with ESMTPS id 7DB3A2DAEB for ; Mon, 29 Oct 2018 15:25:33 -0400 (EDT) From: Olga Arkhangelskaia Subject: [tarantool-patches] [PATCH v3] ctl: added functionality to detect and prune dead replicas Date: Mon, 29 Oct 2018 22:25:15 +0300 Message-Id: <20181029192515.13892-1-arkholga@tarantool.org> Sender: tarantool-patches-bounce@freelists.org Errors-to: tarantool-patches-bounce@freelists.org Reply-To: tarantool-patches@freelists.org List-help: List-unsubscribe: List-software: Ecartis version 1.0.0 List-Id: tarantool-patches List-subscribe: List-owner: List-post: List-archive: To: tarantool-patches@freelists.org Cc: Olga Arkhangelskaia After restart of replicaset or after some configuration changes replicaset may suffer from littering the system space by "dead" replicas. To detect and throw away such replica replicaset_list_inactive(), replica_prune(), replicaset_purge() are introduced. replicaset_list_inactive() returns uuid table of dead replicas replica_prune(uuid) throws away replica with given id from system space replicaset_purge([uuid]) throws away table of dead replicas, that is provided by admin or by replicaset_list_inactive(). Closes #3110 @TarantoolBot document Title: Introduce replicaset_list_inactive(), replica_prune(uuid), replicaset_purge([uuid]) After some changes in replicaset (restart, configuration change) replicaset may suffer from leftovers or previous configurations: empty replicas with uuid, but without any read/write activity. To delete such trash from system the space space, replicaset_list_inactive(), replica_prune(), replicaset_purge() are introduced. replicaset_list_inactive() lists uuid table of such "zombie" replicas. replica_prune(uuid) throws away replica with given id from system space replicaset_purge([uuid]) throws away table of dead replicas, that is provided by admin or by replicaset_list_inactive(). --- Issue: https://github.com/tarantool/tarantool/issues/3110 Branch: https://github.com/tarantool/tarantool/tree/OKriw/gh-3110-prune-dead-replica-from-replicaset-2.1 v1: https://www.freelists.org/post/tarantool-patches/PATCH-rfc-schema-add-possibility-to-find-and-throw-away-dead-replicas Changes v2: - changed the way of replicas death detection - added special box options - changed test - now only dead replicas are shown - added function to throw away any replica v2: https://www.freelists.org/post/tarantool-patches/PATCH-v2-02-detect-and-throw-away-dead-replicas Changes v3: - detect and throw away "zombie" replica, with null applier/relay - no additional box configuration parameters src/box/CMakeLists.txt | 1 + src/box/lua/ctl.lua | 56 ++++++++++ src/box/lua/init.c | 2 + test/replication/trim.lua | 32 ++++++ test/replication/trim.result | 244 +++++++++++++++++++++++++++++++++++++++++ test/replication/trim.test.lua | 90 +++++++++++++++ test/replication/trim1.lua | 1 + test/replication/trim2.lua | 1 + test/replication/trim3.lua | 1 + 9 files changed, 428 insertions(+) create mode 100644 src/box/lua/ctl.lua create mode 100644 test/replication/trim.lua create mode 100644 test/replication/trim.result create mode 100644 test/replication/trim.test.lua create mode 120000 test/replication/trim1.lua create mode 120000 test/replication/trim2.lua create mode 120000 test/replication/trim3.lua diff --git a/src/box/CMakeLists.txt b/src/box/CMakeLists.txt index d1276472b..3a1735637 100644 --- a/src/box/CMakeLists.txt +++ b/src/box/CMakeLists.txt @@ -13,6 +13,7 @@ lua_source(lua_sources lua/net_box.lua) lua_source(lua_sources lua/upgrade.lua) lua_source(lua_sources lua/console.lua) lua_source(lua_sources lua/xlog.lua) +lua_source(lua_sources lua/ctl.lua) set(bin_sources) bin_source(bin_sources bootstrap.snap bootstrap.h) diff --git a/src/box/lua/ctl.lua b/src/box/lua/ctl.lua new file mode 100644 index 000000000..d2991a4ee --- /dev/null +++ b/src/box/lua/ctl.lua @@ -0,0 +1,56 @@ +-- ctl.lua (internal file) + +-- checks whether given replica is dead +local function is_dead(replica) + -- self + if replica.uuid == box.info.uuid then + return false + end + -- no applier no relay + if replica.upstream == nil and replica.downstream == nil then + return true + end + -- disconnected or stopped relay + if replica.upstream == nil and replica.downstream ~= nil then + if replica.downstream.status == 'disconnected' or replica.downstream.status == 'stopped' then + return true + else + return false + end + end + -- in other cases replica is alive + return false +end + +-- return list of replicas suspected to be dead +function replicaset_list_inactive() + local inactive_list = {} + local replicaset = box.info.replication + for i, replica in pairs(replicaset) do + -- current replica is alive + if is_dead(replica) then + table.insert(inactive_list, replica.uuid) + end + end + return inactive_list +end + +-- throw away any replica from system space +function replica_prune(uuid) + if uuid == nil then + error("Usage: replica_displace([uuid])") + end + box.space._cluster.index.uuid:delete{uuid} +end + +-- delete table of dead replica obtained from replicaset_list_inactive() or +-- formed by admin +function replicaset_purge(uuid_table) + if uuid_table == nil then + error("Usage: replicaset_trim([uuid_table])") + end + for i in pairs(uuid_table) do + print("Deleting replica with uuid ".. i.. " "..uuid_table[i]) + replica_prune(uuid_table[i]) + end +end diff --git a/src/box/lua/init.c b/src/box/lua/init.c index ccb4c6a46..0339ea92e 100644 --- a/src/box/lua/init.c +++ b/src/box/lua/init.c @@ -63,6 +63,7 @@ extern char session_lua[], tuple_lua[], schema_lua[], + ctl_lua[], load_cfg_lua[], xlog_lua[], checkpoint_daemon_lua[], @@ -82,6 +83,7 @@ static const char *lua_sources[] = { "box/console", console_lua, "box/load_cfg", load_cfg_lua, "box/xlog", xlog_lua, + "box/ctl", ctl_lua, NULL }; diff --git a/test/replication/trim.lua b/test/replication/trim.lua new file mode 100644 index 000000000..f79fe6e02 --- /dev/null +++ b/test/replication/trim.lua @@ -0,0 +1,32 @@ +#!/usr/bin/env tarantool + +-- get instance name from filename (trim1.lua => trim1) +local INSTANCE_ID = string.match(arg[0], "%d") +local SOCKET_DIR = require('fio').cwd() +local TIMEOUT = tonumber(arg[1]) +local CON_TIMEOUT = arg[2] and tonumber(arg[2]) or 30.0 + +local function instance_uri(instance_id) + --return 'localhost:'..(3310 + instance_id) + return SOCKET_DIR..'/trim'..instance_id..'.sock'; +end + +require('console').listen(os.getenv('ADMIN')) + +box.cfg({ + listen = instance_uri(INSTANCE_ID); + replication_timeout = TIMEOUT; + replication_connect_timeout = CON_TIMEOUT; + replication = { + instance_uri(1); + instance_uri(2); + instance_uri(3); + }; +}) + +box.once("bootstrap", function() + local test_run = require('test_run').new() + box.schema.user.grant("guest", 'replication') + box.schema.space.create('test', {engine = test_run:get_cfg('engine')}) + box.space.test:create_index('primary') +end) diff --git a/test/replication/trim.result b/test/replication/trim.result new file mode 100644 index 000000000..f85b3db97 --- /dev/null +++ b/test/replication/trim.result @@ -0,0 +1,244 @@ +test_run = require('test_run').new() +--- +... +SERVERS = {'trim1', 'trim2', 'trim3'} +--- +... +-- Deploy cluster +test_run:create_cluster(SERVERS, "replication", {args="0.1"}) +--- +... +test_run:wait_fullmesh(SERVERS) +--- +... +test_run:cmd('switch trim1') +--- +- true +... +len = box.space._cluster:len() +--- +... +-- no errors +replicaset_list_inactive() +--- +- [] +... +replica_prune() +--- +- error: 'builtin/box/ctl.lua:41: Usage: replica_displace([uuid])' +... +replicaset_purge() +--- +- error: 'builtin/box/ctl.lua:50: Usage: replicaset_trim([uuid_table])' +... +-- create zombies after restart all replicas +test_run:cmd('switch trim1') +--- +- true +... +fiber = require('fiber') +--- +... +old_trim2 = test_run:get_param('trim2', 'id') +--- +... +old_trim3 = test_run:get_param('trim3', 'id') +--- +... +len = box.space._cluster:len() +--- +... +test_run:cmd('switch default') +--- +- true +... +test_run:cmd('stop server trim2') +--- +- true +... +test_run:cmd('cleanup server trim2') +--- +- true +... +test_run:cmd('start server trim2') +--- +- true +... +test_run:cmd('stop server trim3') +--- +- true +... +test_run:cmd('cleanup server trim3') +--- +- true +... +test_run:cmd('start server trim3') +--- +- true +... +test_run:cmd('switch trim1') +--- +- true +... +replicaset_list_inactive() ~= nil +--- +- true +... +box.space._cluster:len() == len + #replicaset_list_inactive() +--- +- true +... +-- check that we showed and throw away only dead replicas +trim2 = test_run:get_param('trim2', 'id') +--- +... +trim3 = test_run:get_param('trim3', 'id') +--- +... +while box.info.replication[trim2[1]].upstream.status == 'follow' do fiber.sleep(0.01) end +--- +... +while box.info.replication[trim3[1]].upstream.status =='follow' do fiber.sleep(0.01) end +--- +... +box.info.replication[trim2[1]].downstream.status == nil +--- +- true +... +box.info.replication[trim3[1]].downstream.status == nil +--- +- true +... +box.info.replication[old_trim2[1]].upstream == nil +--- +- true +... +box.info.replication[old_trim3[1]].upstream == nil +--- +- true +... +box.info.replication[old_trim2[1]].downstream.status == 'stopped' +--- +- true +... +box.info.replication[old_trim3[1]].downstream.status == 'stopped' +--- +- true +... +-- +replicaset_list_inactive() == 2 +--- +- false +... +replicaset_purge(replicaset_list_inactive()) +--- +... +#replicaset_list_inactive() == 0 +--- +- true +... +box.space._cluster:len() == len +--- +- true +... +box.info.replication[trim2[1]] ~= nil +--- +- true +... +box.info.replication[trim3[1]] ~= nil +--- +- true +... +box.info.replication[trim2[1]].downstream.status == nil +--- +- true +... +box.info.replication[trim3[1]].downstream.status == nil +--- +- true +... +box.info.replication[old_trim2[1]] == nil +--- +- true +... +box.info.replication[old_trim3[1]] == nil +--- +- true +... +-- no applier no relay +test_run:cmd('switch default') +--- +- true +... +test_run:cmd('stop server trim2') +--- +- true +... +test_run:cmd('cleanup server trim2') +--- +- true +... +test_run:cmd('start server trim2') +--- +- true +... +test_run:cmd('stop server trim3') +--- +- true +... +test_run:cmd('cleanup server trim3') +--- +- true +... +test_run:cmd('start server trim3') +--- +- true +... +test_run:cmd('stop server trim1') +--- +- true +... +test_run:cmd('cleanup server trim1') +--- +- true +... +test_run:cmd('start server trim1') +--- +- true +... +test_run:cmd('switch trim1') +--- +- true +... +box.ctl.wait_rw(10) +--- +- error: timed out +... +inactive = replicaset_list_inactive() +--- +... +-- prune given replica +replica_prune(inactive[1]) +--- +- error: Can't modify data because this instance is in read-only mode. +... +#replicaset_list_inactive() ~= #inactive +--- +- false +... +replicaset_purge(replicaset_list_inactive()) +--- +- error: Can't modify data because this instance is in read-only mode. +... +box.space._cluster:len() == 3 +--- +- false +... +-- Cleanup +test_run:cmd('switch default') +--- +- true +... +test_run:drop_cluster(SERVERS) +--- +... diff --git a/test/replication/trim.test.lua b/test/replication/trim.test.lua new file mode 100644 index 000000000..2c467e313 --- /dev/null +++ b/test/replication/trim.test.lua @@ -0,0 +1,90 @@ +test_run = require('test_run').new() + +SERVERS = {'trim1', 'trim2', 'trim3'} + +-- Deploy cluster +test_run:create_cluster(SERVERS, "replication", {args="0.1"}) +test_run:wait_fullmesh(SERVERS) + +test_run:cmd('switch trim1') +len = box.space._cluster:len() + +-- no errors +replicaset_list_inactive() +replica_prune() +replicaset_purge() + +-- create zombies after restart all replicas +test_run:cmd('switch trim1') +fiber = require('fiber') +old_trim2 = test_run:get_param('trim2', 'id') +old_trim3 = test_run:get_param('trim3', 'id') + +len = box.space._cluster:len() +test_run:cmd('switch default') +test_run:cmd('stop server trim2') +test_run:cmd('cleanup server trim2') +test_run:cmd('start server trim2') +test_run:cmd('stop server trim3') +test_run:cmd('cleanup server trim3') +test_run:cmd('start server trim3') +test_run:cmd('switch trim1') + +replicaset_list_inactive() ~= nil +box.space._cluster:len() == len + #replicaset_list_inactive() + +-- check that we showed and throw away only dead replicas +trim2 = test_run:get_param('trim2', 'id') +trim3 = test_run:get_param('trim3', 'id') + +while box.info.replication[trim2[1]].upstream.status == 'follow' do fiber.sleep(0.01) end +while box.info.replication[trim3[1]].upstream.status =='follow' do fiber.sleep(0.01) end +box.info.replication[trim2[1]].downstream.status == nil +box.info.replication[trim3[1]].downstream.status == nil + +box.info.replication[old_trim2[1]].upstream == nil +box.info.replication[old_trim3[1]].upstream == nil +box.info.replication[old_trim2[1]].downstream.status == 'stopped' +box.info.replication[old_trim3[1]].downstream.status == 'stopped' +-- +replicaset_list_inactive() == 2 +replicaset_purge(replicaset_list_inactive()) +#replicaset_list_inactive() == 0 +box.space._cluster:len() == len + +box.info.replication[trim2[1]] ~= nil +box.info.replication[trim3[1]] ~= nil +box.info.replication[trim2[1]].downstream.status == nil +box.info.replication[trim3[1]].downstream.status == nil + +box.info.replication[old_trim2[1]] == nil +box.info.replication[old_trim3[1]] == nil + + +-- no applier no relay + +test_run:cmd('switch default') +test_run:cmd('stop server trim2') +test_run:cmd('cleanup server trim2') +test_run:cmd('start server trim2') +test_run:cmd('stop server trim3') +test_run:cmd('cleanup server trim3') +test_run:cmd('start server trim3') +test_run:cmd('stop server trim1') +test_run:cmd('cleanup server trim1') +test_run:cmd('start server trim1') +test_run:cmd('switch trim1') + +box.ctl.wait_rw(10) + +inactive = replicaset_list_inactive() + +-- prune given replica +replica_prune(inactive[1]) +#replicaset_list_inactive() ~= #inactive +replicaset_purge(replicaset_list_inactive()) +box.space._cluster:len() == 3 + +-- Cleanup +test_run:cmd('switch default') +test_run:drop_cluster(SERVERS) diff --git a/test/replication/trim1.lua b/test/replication/trim1.lua new file mode 120000 index 000000000..14e98bd68 --- /dev/null +++ b/test/replication/trim1.lua @@ -0,0 +1 @@ +trim.lua \ No newline at end of file diff --git a/test/replication/trim2.lua b/test/replication/trim2.lua new file mode 120000 index 000000000..14e98bd68 --- /dev/null +++ b/test/replication/trim2.lua @@ -0,0 +1 @@ +trim.lua \ No newline at end of file diff --git a/test/replication/trim3.lua b/test/replication/trim3.lua new file mode 120000 index 000000000..14e98bd68 --- /dev/null +++ b/test/replication/trim3.lua @@ -0,0 +1 @@ +trim.lua \ No newline at end of file -- 2.14.3 (Apple Git-98)