From: Vladislav Shpilevoy <v.shpilevoy@tarantool.org>
To: tarantool-patches@dev.tarantool.org, sergepetrenko@tarantool.org,
gorcunov@gmail.com
Subject: Re: [Tarantool-patches] [PATCH v2 12/11] raft: add tests
Date: Tue, 22 Sep 2020 00:48:01 +0200 [thread overview]
Message-ID: <a9c1cbf5-6bd7-8f55-036d-58934be867b6@tarantool.org> (raw)
In-Reply-To: <5a7cf2f2-1b7c-28be-688f-d604a64d0623@tarantool.org>
A new test in a separate commit on top of the branch.
I also found that sometimes after bootstrap we have leader count != 1.
But I didn't catch it again to see what was it equal. I suspect it was
0. Probably bootstrap was too long and led to election timeout at some
point, so somebody bumped all the terms and made all nodes the followers.
Probably need to replace some things with test_run:wait_cond() here.
====================
[tosquash] raft: add re-election test
diff --git a/test/replication/raft_basic.result b/test/replication/raft_basic.result
index 2996fe3eb..3421227fb 100644
--- a/test/replication/raft_basic.result
+++ b/test/replication/raft_basic.result
@@ -140,6 +140,9 @@ test_run:wait_fullmesh(SERVERS)
is_leader_cmd = 'return box.info.raft.state == \'leader\''
| ---
| ...
+leader_id_cmd = 'return box.info.raft.leader'
+ | ---
+ | ...
is_r1_leader = test_run:eval('raft_replica1', is_leader_cmd)[1]
| ---
| ...
@@ -162,6 +165,114 @@ assert(leader_count == 1)
| ---
| - true
| ...
+-- All nodes have the same leader.
+r1_leader = test_run:eval('raft_replica1', leader_id_cmd)[1]
+ | ---
+ | ...
+r2_leader = test_run:eval('raft_replica2', leader_id_cmd)[1]
+ | ---
+ | ...
+r3_leader = test_run:eval('raft_replica3', leader_id_cmd)[1]
+ | ---
+ | ...
+assert(r1_leader ~= 0)
+ | ---
+ | - true
+ | ...
+assert(r1_leader == r2_leader)
+ | ---
+ | - true
+ | ...
+assert(r1_leader == r3_leader)
+ | ---
+ | - true
+ | ...
+
+--
+-- Leader death starts a new election.
+--
+leader_name = nil
+ | ---
+ | ...
+nonleader1_name = nil
+ | ---
+ | ...
+nonleader2_name = nil
+ | ---
+ | ...
+if is_r1_leader then \
+ leader_name = 'raft_replica1' \
+ nonleader1_name = 'raft_replica2' \
+ nonleader2_name = 'raft_replica3' \
+elseif is_r2_leader then \
+ leader_name = 'raft_replica2' \
+ nonleader1_name = 'raft_replica1' \
+ nonleader2_name = 'raft_replica3' \
+else \
+ leader_name = 'raft_replica3' \
+ nonleader1_name = 'raft_replica1' \
+ nonleader2_name = 'raft_replica2' \
+end
+ | ---
+ | ...
+-- Lower the quorum so the 2 alive nodes could elect a new leader when the third
+-- node dies.
+test_run:switch(nonleader1_name)
+ | ---
+ | - true
+ | ...
+box.cfg{replication_synchro_quorum = 2}
+ | ---
+ | ...
+-- Switch via default where the names are defined.
+test_run:switch('default')
+ | ---
+ | - true
+ | ...
+test_run:switch(nonleader2_name)
+ | ---
+ | - true
+ | ...
+box.cfg{replication_synchro_quorum = 2}
+ | ---
+ | ...
+
+test_run:switch('default')
+ | ---
+ | - true
+ | ...
+test_run:cmd(string.format('stop server %s', leader_name))
+ | ---
+ | - true
+ | ...
+test_run:wait_cond(function() \
+ is_r1_leader = test_run:eval(nonleader1_name, is_leader_cmd)[1] \
+ is_r2_leader = test_run:eval(nonleader2_name, is_leader_cmd)[1] \
+ return is_r1_leader or is_r2_leader \
+end)
+ | ---
+ | - true
+ | ...
+r1_leader = test_run:eval(nonleader1_name, leader_id_cmd)[1]
+ | ---
+ | ...
+r2_leader = test_run:eval(nonleader2_name, leader_id_cmd)[1]
+ | ---
+ | ...
+assert(r1_leader ~= 0)
+ | ---
+ | - true
+ | ...
+assert(r1_leader == r2_leader)
+ | ---
+ | - true
+ | ...
+
+test_run:cmd(string.format('start server %s', leader_name))
+ | ---
+ | - true
+ | ...
+
test_run:drop_cluster(SERVERS)
| ---
| ...
diff --git a/test/replication/raft_basic.test.lua b/test/replication/raft_basic.test.lua
index 7e7568991..b8e5a5eaf 100644
--- a/test/replication/raft_basic.test.lua
+++ b/test/replication/raft_basic.test.lua
@@ -57,6 +57,7 @@ SERVERS = {'raft_replica1', 'raft_replica2', 'raft_replica3'}
test_run:create_cluster(SERVERS, "replication")
test_run:wait_fullmesh(SERVERS)
is_leader_cmd = 'return box.info.raft.state == \'leader\''
+leader_id_cmd = 'return box.info.raft.leader'
is_r1_leader = test_run:eval('raft_replica1', is_leader_cmd)[1]
is_r2_leader = test_run:eval('raft_replica2', is_leader_cmd)[1]
is_r3_leader = test_run:eval('raft_replica3', is_leader_cmd)[1]
@@ -64,4 +65,54 @@ leader_count = is_r1_leader and 1 or 0
leader_count = leader_count + (is_r2_leader and 1 or 0)
leader_count = leader_count + (is_r3_leader and 1 or 0)
assert(leader_count == 1)
+-- All nodes have the same leader.
+r1_leader = test_run:eval('raft_replica1', leader_id_cmd)[1]
+r2_leader = test_run:eval('raft_replica2', leader_id_cmd)[1]
+r3_leader = test_run:eval('raft_replica3', leader_id_cmd)[1]
+assert(r1_leader ~= 0)
+assert(r1_leader == r2_leader)
+assert(r1_leader == r3_leader)
+
+--
+-- Leader death starts a new election.
+--
+leader_name = nil
+nonleader1_name = nil
+nonleader2_name = nil
+if is_r1_leader then \
+ leader_name = 'raft_replica1' \
+ nonleader1_name = 'raft_replica2' \
+ nonleader2_name = 'raft_replica3' \
+elseif is_r2_leader then \
+ leader_name = 'raft_replica2' \
+ nonleader1_name = 'raft_replica1' \
+ nonleader2_name = 'raft_replica3' \
+else \
+ leader_name = 'raft_replica3' \
+ nonleader1_name = 'raft_replica1' \
+ nonleader2_name = 'raft_replica2' \
+end
+-- Lower the quorum so the 2 alive nodes could elect a new leader when the third
+-- node dies.
+test_run:switch(nonleader1_name)
+box.cfg{replication_synchro_quorum = 2}
+-- Switch via default where the names are defined.
+test_run:switch('default')
+test_run:switch(nonleader2_name)
+box.cfg{replication_synchro_quorum = 2}
+
+test_run:switch('default')
+test_run:cmd(string.format('stop server %s', leader_name))
+test_run:wait_cond(function() \
+ is_r1_leader = test_run:eval(nonleader1_name, is_leader_cmd)[1] \
+ is_r2_leader = test_run:eval(nonleader2_name, is_leader_cmd)[1] \
+ return is_r1_leader or is_r2_leader \
+end)
+r1_leader = test_run:eval(nonleader1_name, leader_id_cmd)[1]
+r2_leader = test_run:eval(nonleader2_name, leader_id_cmd)[1]
+assert(r1_leader ~= 0)
+assert(r1_leader == r2_leader)
+
+test_run:cmd(string.format('start server %s', leader_name))
+
test_run:drop_cluster(SERVERS)
next prev parent reply other threads:[~2020-09-21 22:48 UTC|newest]
Thread overview: 40+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-09-09 23:16 [Tarantool-patches] [PATCH v2 00/11] dRaft Vladislav Shpilevoy
2020-09-09 23:16 ` [Tarantool-patches] [PATCH v2 01/11] applier: store instance_id in struct applier Vladislav Shpilevoy
2020-09-14 9:38 ` Serge Petrenko
2020-09-19 15:44 ` Vladislav Shpilevoy
2020-09-21 6:23 ` Serge Petrenko
2020-09-09 23:16 ` [Tarantool-patches] [PATCH v2 10/11] raft: introduce box.info.raft Vladislav Shpilevoy
2020-09-14 9:42 ` Serge Petrenko
2020-09-09 23:16 ` [Tarantool-patches] [PATCH v2 11/11] [tosquash] raft: a swarm of minor fixes Vladislav Shpilevoy
2020-09-14 10:13 ` Serge Petrenko
2020-09-09 23:16 ` [Tarantool-patches] [PATCH v2 02/11] box: introduce summary RO flag Vladislav Shpilevoy
2020-09-09 23:16 ` [Tarantool-patches] [PATCH v2 03/11] wal: don't touch box.cfg.wal_dir more than once Vladislav Shpilevoy
2020-09-09 23:16 ` [Tarantool-patches] [PATCH v2 04/11] replication: track registered replica count Vladislav Shpilevoy
2020-09-09 23:16 ` [Tarantool-patches] [PATCH v2 05/11] [wip] box: do not register outgoing connections Vladislav Shpilevoy
2020-09-09 23:16 ` [Tarantool-patches] [PATCH v2 06/11] raft: introduce persistent raft state Vladislav Shpilevoy
2020-09-09 23:16 ` [Tarantool-patches] [PATCH v2 07/11] raft: introduce box.cfg.raft_* options Vladislav Shpilevoy
2020-09-09 23:16 ` [Tarantool-patches] [PATCH v2 08/11] raft: relay status updates to followers Vladislav Shpilevoy
2020-09-20 17:17 ` Vladislav Shpilevoy
2020-09-21 7:13 ` Serge Petrenko
2020-09-21 10:50 ` Serge Petrenko
2020-09-21 22:47 ` Vladislav Shpilevoy
2020-09-22 8:48 ` Serge Petrenko
2020-09-21 22:47 ` Vladislav Shpilevoy
2020-09-22 8:47 ` Serge Petrenko
2020-09-09 23:17 ` [Tarantool-patches] [PATCH v2 09/11] raft: introduce state machine Vladislav Shpilevoy
2020-09-19 15:49 ` Vladislav Shpilevoy
2020-09-19 15:50 ` Vladislav Shpilevoy
2020-09-21 8:20 ` Serge Petrenko
2020-09-21 8:22 ` Serge Petrenko
2020-09-21 8:34 ` Serge Petrenko
2020-09-21 22:47 ` Vladislav Shpilevoy
2020-09-22 8:49 ` Serge Petrenko
2020-09-22 22:48 ` Vladislav Shpilevoy
2020-09-23 9:59 ` Serge Petrenko
2020-09-23 20:31 ` Vladislav Shpilevoy
2020-09-24 9:34 ` Serge Petrenko
2020-09-19 15:58 ` [Tarantool-patches] [PATCH v2 12/11] dRaft Vladislav Shpilevoy
2020-09-19 15:59 ` Vladislav Shpilevoy
2020-09-21 7:24 ` Serge Petrenko
2020-09-21 22:48 ` Vladislav Shpilevoy [this message]
2020-09-30 10:56 ` [Tarantool-patches] [PATCH v2 00/11] dRaft Kirill Yukhin
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=a9c1cbf5-6bd7-8f55-036d-58934be867b6@tarantool.org \
--to=v.shpilevoy@tarantool.org \
--cc=gorcunov@gmail.com \
--cc=sergepetrenko@tarantool.org \
--cc=tarantool-patches@dev.tarantool.org \
--subject='Re: [Tarantool-patches] [PATCH v2 12/11] raft: add tests' \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox