[Tarantool-patches] [PATCH] box: introduce on_election triggers
Serge Petrenko
sergepetrenko at tarantool.org
Thu Jul 22 10:20:02 MSK 2021
On_election triggers are fired asynchronously after any Raft event with
a broadcast, they are run in a worker fiber, so it's allowed to yield
inside them, unlike Raft's on_update triggers we already had.
Closes #5819
@TarantoolBot document
Title: document triggers on election state change
A new function to register triggers is added, `box.ctl.on_election()`.
Triggers registered via this function are run asynchronously every time
a visible change in `box.info.election` table appears.
No parameters are passed to the trigger, it may check what's changed by
looking at `box.info.election` and `box.info.synchro`.
---
I'm not sure whether every change in `box.info.election.vote` actually runs the
triggers. That's because I run triggers inside of box_raft_broadcast(), which
isn't triggered on vote change. I thought of adding raft_schedule_trigger as a
counterpart to raft_schedule_broadcast. Then raft_sm_schedule_new_vote() would
have raft_schedule_trigger() instead of raft_schedule_broadcast(), and the
triggers would fire on a vote change as well.
https://github.com/tarantool/tarantool/issues/5819
https://github.com/tarantool/tarantool/tree/sp/gh-5819-raft-state-triggers
.../unreleased/gh-5819-election-triggers.md | 5 +
src/box/lua/ctl.c | 8 +
src/box/raft.c | 3 +
src/box/raft.h | 7 +
.../gh-5819-election-triggers.result | 147 ++++++++++++++++++
.../gh-5819-election-triggers.test.lua | 53 +++++++
test/replication/suite.cfg | 1 +
7 files changed, 224 insertions(+)
create mode 100644 changelogs/unreleased/gh-5819-election-triggers.md
create mode 100644 test/replication/gh-5819-election-triggers.result
create mode 100644 test/replication/gh-5819-election-triggers.test.lua
diff --git a/changelogs/unreleased/gh-5819-election-triggers.md b/changelogs/unreleased/gh-5819-election-triggers.md
new file mode 100644
index 000000000..edb7be8ac
--- /dev/null
+++ b/changelogs/unreleased/gh-5819-election-triggers.md
@@ -0,0 +1,5 @@
+## feature/replication
+
+* Introduce on_election triggers. The triggers may be registered via
+`box.ctl.on_election()` interface and are run asynchronously each time
+`box.info.election` changes (gh-5819).
diff --git a/src/box/lua/ctl.c b/src/box/lua/ctl.c
index 368b9ab60..4b2a467d8 100644
--- a/src/box/lua/ctl.c
+++ b/src/box/lua/ctl.c
@@ -43,6 +43,7 @@
#include "box/schema.h"
#include "box/engine.h"
#include "box/memtx_engine.h"
+#include "box/raft.h"
static int
lbox_ctl_wait_ro(struct lua_State *L)
@@ -81,6 +82,12 @@ lbox_ctl_on_schema_init(struct lua_State *L)
return lbox_trigger_reset(L, 2, &on_schema_init, NULL, NULL);
}
+static int
+lbox_ctl_on_election(struct lua_State *L)
+{
+ return lbox_trigger_reset(L, 2, &box_raft_on_state, NULL, NULL);
+}
+
static int
lbox_ctl_promote(struct lua_State *L)
{
@@ -124,6 +131,7 @@ static const struct luaL_Reg lbox_ctl_lib[] = {
{"wait_rw", lbox_ctl_wait_rw},
{"on_shutdown", lbox_ctl_on_shutdown},
{"on_schema_init", lbox_ctl_on_schema_init},
+ {"on_election", lbox_ctl_on_election},
{"promote", lbox_ctl_promote},
/* An old alias. */
{"clear_synchro_queue", lbox_ctl_promote},
diff --git a/src/box/raft.c b/src/box/raft.c
index 7f787c0c5..c9c3ba818 100644
--- a/src/box/raft.c
+++ b/src/box/raft.c
@@ -52,6 +52,8 @@ enum election_mode box_election_mode = ELECTION_MODE_INVALID;
*/
static struct trigger box_raft_on_update;
+struct rlist box_raft_on_state = RLIST_HEAD_INITIALIZER(box_raft_on_state);
+
/**
* Worker fiber does all the asynchronous work, which may need yields and can be
* long. These are WAL writes, network broadcasts. That allows not to block the
@@ -274,6 +276,7 @@ box_raft_broadcast(struct raft *raft, const struct raft_msg *msg)
assert(raft == box_raft());
struct raft_request req;
box_raft_msg_to_request(msg, &req);
+ trigger_run(&box_raft_on_state, NULL);
replicaset_foreach(replica)
relay_push_raft(replica->relay, &req);
}
diff --git a/src/box/raft.h b/src/box/raft.h
index 6b6136510..dbe16eebf 100644
--- a/src/box/raft.h
+++ b/src/box/raft.h
@@ -30,11 +30,18 @@
* SUCH DAMAGE.
*/
#include "raft/raft.h"
+#include "small/rlist.h"
#if defined(__cplusplus)
extern "C" {
#endif
+/**
+ * A public trigger fired on Raft state change. It's allowed to yield inside
+ * it, and it's run asynchronously.
+ */
+extern struct rlist box_raft_on_state;
+
enum election_mode {
ELECTION_MODE_INVALID = -1,
ELECTION_MODE_OFF = 0,
diff --git a/test/replication/gh-5819-election-triggers.result b/test/replication/gh-5819-election-triggers.result
new file mode 100644
index 000000000..546aac8a2
--- /dev/null
+++ b/test/replication/gh-5819-election-triggers.result
@@ -0,0 +1,147 @@
+-- test-run result file version 2
+test_run = require('test_run').new()
+ | ---
+ | ...
+
+box.schema.user.grant('guest', 'replication')
+ | ---
+ | ...
+test_run:cmd('create server replica with rpl_master=default,\
+ script="replication/replica.lua"')
+ | ---
+ | - true
+ | ...
+test_run:cmd('start server replica')
+ | ---
+ | - true
+ | ...
+
+repl = test_run:eval('replica', 'return box.cfg.listen')[1]
+ | ---
+ | ...
+box.cfg{replication = repl}
+ | ---
+ | ...
+test_run:switch('replica')
+ | ---
+ | - true
+ | ...
+box.cfg{election_mode='voter'}
+ | ---
+ | ...
+test_run:switch('default')
+ | ---
+ | - true
+ | ...
+
+election_tbl = {}
+ | ---
+ | ...
+
+function trig()\
+ election_tbl[#election_tbl+1] = box.info.election\
+end
+ | ---
+ | ...
+
+_ = box.ctl.on_election(trig)
+ | ---
+ | ...
+
+box.cfg{replication_synchro_quorum=2}
+ | ---
+ | ...
+box.cfg{election_mode='candidate'}
+ | ---
+ | ...
+
+test_run:wait_cond(function() return box.info.election.state == 'leader' end)
+ | ---
+ | - true
+ | ...
+assert(#election_tbl == 3)
+ | ---
+ | - true
+ | ...
+assert(election_tbl[1].state == 'follower')
+ | ---
+ | - true
+ | ...
+assert(election_tbl[2].state == 'candidate')
+ | ---
+ | - true
+ | ...
+assert(election_tbl[2].vote == 1)
+ | ---
+ | - true
+ | ...
+assert(election_tbl[3].state == 'leader')
+ | ---
+ | - true
+ | ...
+box.cfg{election_mode='voter'}
+ | ---
+ | ...
+
+assert(#election_tbl == 4)
+ | ---
+ | - true
+ | ...
+assert(election_tbl[4].state == 'follower')
+ | ---
+ | - true
+ | ...
+
+box.cfg{election_mode='off'}
+ | ---
+ | ...
+assert(#election_tbl == 5)
+ | ---
+ | - true
+ | ...
+
+box.cfg{election_mode='manual'}
+ | ---
+ | ...
+
+assert(#election_tbl == 6)
+ | ---
+ | - true
+ | ...
+assert(election_tbl[6].state == 'follower')
+ | ---
+ | - true
+ | ...
+
+box.ctl.promote()
+ | ---
+ | ...
+
+assert(#election_tbl == 9)
+ | ---
+ | - true
+ | ...
+assert(election_tbl[7].state == 'follower')
+ | ---
+ | - true
+ | ...
+assert(election_tbl[8].state == 'candidate')
+ | ---
+ | - true
+ | ...
+assert(election_tbl[9].state == 'leader')
+ | ---
+ | - true
+ | ...
+
+test_run:cmd('stop server replica')
+ | ---
+ | - true
+ | ...
+test_run:cmd('delete server replica')
+ | ---
+ | - true
+ | ...
+box.schema.user.revoke('guest', 'replication')
+ | ---
+ | ...
diff --git a/test/replication/gh-5819-election-triggers.test.lua b/test/replication/gh-5819-election-triggers.test.lua
new file mode 100644
index 000000000..5f61a59f9
--- /dev/null
+++ b/test/replication/gh-5819-election-triggers.test.lua
@@ -0,0 +1,53 @@
+test_run = require('test_run').new()
+
+box.schema.user.grant('guest', 'replication')
+test_run:cmd('create server replica with rpl_master=default,\
+ script="replication/replica.lua"')
+test_run:cmd('start server replica')
+
+repl = test_run:eval('replica', 'return box.cfg.listen')[1]
+box.cfg{replication = repl}
+test_run:switch('replica')
+box.cfg{election_mode='voter'}
+test_run:switch('default')
+
+election_tbl = {}
+
+function trig()\
+ election_tbl[#election_tbl+1] = box.info.election\
+end
+
+_ = box.ctl.on_election(trig)
+
+box.cfg{replication_synchro_quorum=2}
+box.cfg{election_mode='candidate'}
+
+test_run:wait_cond(function() return box.info.election.state == 'leader' end)
+assert(#election_tbl == 3)
+assert(election_tbl[1].state == 'follower')
+assert(election_tbl[2].state == 'candidate')
+assert(election_tbl[2].vote == 1)
+assert(election_tbl[3].state == 'leader')
+box.cfg{election_mode='voter'}
+
+assert(#election_tbl == 4)
+assert(election_tbl[4].state == 'follower')
+
+box.cfg{election_mode='off'}
+assert(#election_tbl == 5)
+
+box.cfg{election_mode='manual'}
+
+assert(#election_tbl == 6)
+assert(election_tbl[6].state == 'follower')
+
+box.ctl.promote()
+
+assert(#election_tbl == 9)
+assert(election_tbl[7].state == 'follower')
+assert(election_tbl[8].state == 'candidate')
+assert(election_tbl[9].state == 'leader')
+
+test_run:cmd('stop server replica')
+test_run:cmd('delete server replica')
+box.schema.user.revoke('guest', 'replication')
diff --git a/test/replication/suite.cfg b/test/replication/suite.cfg
index a51a2d51a..f2af00038 100644
--- a/test/replication/suite.cfg
+++ b/test/replication/suite.cfg
@@ -46,6 +46,7 @@
"gh-5536-wal-limit.test.lua": {},
"gh-5566-final-join-synchro.test.lua": {},
"gh-5613-bootstrap-prefer-booted.test.lua": {},
+ "gh-5819-election-triggers.test.lua": {},
"gh-6027-applier-error-show.test.lua": {},
"gh-6032-promote-wal-write.test.lua": {},
"gh-6057-qsync-confirm-async-no-wal.test.lua": {},
--
2.30.1 (Apple Git-130)
More information about the Tarantool-patches
mailing list