[Tarantool-patches] [PATCH] box: introduce on_election triggers

[Serge Petrenko via Tarantool-patches <tarantool-patches@dev.tarantool.org>]

On_election triggers are fired asynchronously after any Raft event with
a broadcast, they are run in a worker fiber, so it's allowed to yield
inside them, unlike Raft's on_update triggers we already had.

Closes #5819

@TarantoolBot document
Title: document triggers on election state change

A new function to register triggers is added, `box.ctl.on_election()`.
Triggers registered via this function are run asynchronously every time
a visible change in `box.info.election` table appears.
No parameters are passed to the trigger, it may check what's changed by
looking at `box.info.election` and `box.info.synchro`.
---
I'm not sure whether every change in `box.info.election.vote` actually runs the
triggers. That's because I run triggers inside of box_raft_broadcast(), which
isn't triggered on vote change. I thought of adding raft_schedule_trigger as a
counterpart to raft_schedule_broadcast. Then raft_sm_schedule_new_vote() would
have raft_schedule_trigger() instead of raft_schedule_broadcast(), and the
triggers would fire on a vote change as well.

https://github.com/tarantool/tarantool/issues/5819
https://github.com/tarantool/tarantool/tree/sp/gh-5819-raft-state-triggers

.../unreleased/gh-5819-election-triggers.md   |   5 +
 src/box/lua/ctl.c                             |   8 +
 src/box/raft.c                                |   3 +
 src/box/raft.h                                |   7 +
 .../gh-5819-election-triggers.result          | 147 ++++++++++++++++++
 .../gh-5819-election-triggers.test.lua        |  53 +++++++
 test/replication/suite.cfg                    |   1 +
 7 files changed, 224 insertions(+)
 create mode 100644 changelogs/unreleased/gh-5819-election-triggers.md
 create mode 100644 test/replication/gh-5819-election-triggers.result
 create mode 100644 test/replication/gh-5819-election-triggers.test.lua

diff --git a/changelogs/unreleased/gh-5819-election-triggers.md b/changelogs/unreleased/gh-5819-election-triggers.md
new file mode 100644
index 000000000..edb7be8ac
--- /dev/null
+++ b/changelogs/unreleased/gh-5819-election-triggers.md
@@ -0,0 +1,5 @@
+## feature/replication
+
+* Introduce on_election triggers. The triggers may be registered via
+`box.ctl.on_election()` interface and are run asynchronously each time
+`box.info.election` changes (gh-5819).
diff --git a/src/box/lua/ctl.c b/src/box/lua/ctl.c
index 368b9ab60..4b2a467d8 100644
--- a/src/box/lua/ctl.c
+++ b/src/box/lua/ctl.c
@@ -43,6 +43,7 @@
 #include "box/schema.h"
 #include "box/engine.h"
 #include "box/memtx_engine.h"
+#include "box/raft.h"
 
 static int
 lbox_ctl_wait_ro(struct lua_State *L)
@@ -81,6 +82,12 @@ lbox_ctl_on_schema_init(struct lua_State *L)
 	return lbox_trigger_reset(L, 2, &on_schema_init, NULL, NULL);
 }
 
+static int
+lbox_ctl_on_election(struct lua_State *L)
+{
+	return lbox_trigger_reset(L, 2, &box_raft_on_state, NULL, NULL);
+}
+
 static int
 lbox_ctl_promote(struct lua_State *L)
 {
@@ -124,6 +131,7 @@ static const struct luaL_Reg lbox_ctl_lib[] = {
 	{"wait_rw", lbox_ctl_wait_rw},
 	{"on_shutdown", lbox_ctl_on_shutdown},
 	{"on_schema_init", lbox_ctl_on_schema_init},
+	{"on_election", lbox_ctl_on_election},
 	{"promote", lbox_ctl_promote},
 	/* An old alias. */
 	{"clear_synchro_queue", lbox_ctl_promote},
diff --git a/src/box/raft.c b/src/box/raft.c
index 7f787c0c5..c9c3ba818 100644
--- a/src/box/raft.c
+++ b/src/box/raft.c
@@ -52,6 +52,8 @@ enum election_mode box_election_mode = ELECTION_MODE_INVALID;
  */
 static struct trigger box_raft_on_update;
 
+struct rlist box_raft_on_state = RLIST_HEAD_INITIALIZER(box_raft_on_state);
+
 /**
  * Worker fiber does all the asynchronous work, which may need yields and can be
  * long. These are WAL writes, network broadcasts. That allows not to block the
@@ -274,6 +276,7 @@ box_raft_broadcast(struct raft *raft, const struct raft_msg *msg)
 	assert(raft == box_raft());
 	struct raft_request req;
 	box_raft_msg_to_request(msg, &req);
+	trigger_run(&box_raft_on_state, NULL);
 	replicaset_foreach(replica)
 		relay_push_raft(replica->relay, &req);
 }
diff --git a/src/box/raft.h b/src/box/raft.h
index 6b6136510..dbe16eebf 100644
--- a/src/box/raft.h
+++ b/src/box/raft.h
@@ -30,11 +30,18 @@
  * SUCH DAMAGE.
  */
 #include "raft/raft.h"
+#include "small/rlist.h"
 
 #if defined(__cplusplus)
 extern "C" {
 #endif
 
+/**
+ * A public trigger fired on Raft state change. It's allowed to yield inside
+ * it, and it's run asynchronously.
+ */
+extern struct rlist box_raft_on_state;
+
 enum election_mode {
 	ELECTION_MODE_INVALID = -1,
 	ELECTION_MODE_OFF = 0,
diff --git a/test/replication/gh-5819-election-triggers.result b/test/replication/gh-5819-election-triggers.result
new file mode 100644
index 000000000..546aac8a2
--- /dev/null
+++ b/test/replication/gh-5819-election-triggers.result
@@ -0,0 +1,147 @@
+-- test-run result file version 2
+test_run = require('test_run').new()
+ | ---
+ | ...
+
+box.schema.user.grant('guest', 'replication')
+ | ---
+ | ...
+test_run:cmd('create server replica with rpl_master=default,\
+	      script="replication/replica.lua"')
+ | ---
+ | - true
+ | ...
+test_run:cmd('start server replica')
+ | ---
+ | - true
+ | ...
+
+repl = test_run:eval('replica', 'return box.cfg.listen')[1]
+ | ---
+ | ...
+box.cfg{replication = repl}
+ | ---
+ | ...
+test_run:switch('replica')
+ | ---
+ | - true
+ | ...
+box.cfg{election_mode='voter'}
+ | ---
+ | ...
+test_run:switch('default')
+ | ---
+ | - true
+ | ...
+
+election_tbl = {}
+ | ---
+ | ...
+
+function trig()\
+    election_tbl[#election_tbl+1] = box.info.election\
+end
+ | ---
+ | ...
+
+_ = box.ctl.on_election(trig)
+ | ---
+ | ...
+
+box.cfg{replication_synchro_quorum=2}
+ | ---
+ | ...
+box.cfg{election_mode='candidate'}
+ | ---
+ | ...
+
+test_run:wait_cond(function() return box.info.election.state == 'leader' end)
+ | ---
+ | - true
+ | ...
+assert(#election_tbl == 3)
+ | ---
+ | - true
+ | ...
+assert(election_tbl[1].state == 'follower')
+ | ---
+ | - true
+ | ...
+assert(election_tbl[2].state == 'candidate')
+ | ---
+ | - true
+ | ...
+assert(election_tbl[2].vote == 1)
+ | ---
+ | - true
+ | ...
+assert(election_tbl[3].state == 'leader')
+ | ---
+ | - true
+ | ...
+box.cfg{election_mode='voter'}
+ | ---
+ | ...
+
+assert(#election_tbl == 4)
+ | ---
+ | - true
+ | ...
+assert(election_tbl[4].state == 'follower')
+ | ---
+ | - true
+ | ...
+
+box.cfg{election_mode='off'}
+ | ---
+ | ...
+assert(#election_tbl == 5)
+ | ---
+ | - true
+ | ...
+
+box.cfg{election_mode='manual'}
+ | ---
+ | ...
+
+assert(#election_tbl == 6)
+ | ---
+ | - true
+ | ...
+assert(election_tbl[6].state == 'follower')
+ | ---
+ | - true
+ | ...
+
+box.ctl.promote()
+ | ---
+ | ...
+
+assert(#election_tbl == 9)
+ | ---
+ | - true
+ | ...
+assert(election_tbl[7].state == 'follower')
+ | ---
+ | - true
+ | ...
+assert(election_tbl[8].state == 'candidate')
+ | ---
+ | - true
+ | ...
+assert(election_tbl[9].state == 'leader')
+ | ---
+ | - true
+ | ...
+
+test_run:cmd('stop server replica')
+ | ---
+ | - true
+ | ...
+test_run:cmd('delete server replica')
+ | ---
+ | - true
+ | ...
+box.schema.user.revoke('guest', 'replication')
+ | ---
+ | ...
diff --git a/test/replication/gh-5819-election-triggers.test.lua b/test/replication/gh-5819-election-triggers.test.lua
new file mode 100644
index 000000000..5f61a59f9
--- /dev/null
+++ b/test/replication/gh-5819-election-triggers.test.lua
@@ -0,0 +1,53 @@
+test_run = require('test_run').new()
+
+box.schema.user.grant('guest', 'replication')
+test_run:cmd('create server replica with rpl_master=default,\
+	      script="replication/replica.lua"')
+test_run:cmd('start server replica')
+
+repl = test_run:eval('replica', 'return box.cfg.listen')[1]
+box.cfg{replication = repl}
+test_run:switch('replica')
+box.cfg{election_mode='voter'}
+test_run:switch('default')
+
+election_tbl = {}
+
+function trig()\
+    election_tbl[#election_tbl+1] = box.info.election\
+end
+
+_ = box.ctl.on_election(trig)
+
+box.cfg{replication_synchro_quorum=2}
+box.cfg{election_mode='candidate'}
+
+test_run:wait_cond(function() return box.info.election.state == 'leader' end)
+assert(#election_tbl == 3)
+assert(election_tbl[1].state == 'follower')
+assert(election_tbl[2].state == 'candidate')
+assert(election_tbl[2].vote == 1)
+assert(election_tbl[3].state == 'leader')
+box.cfg{election_mode='voter'}
+
+assert(#election_tbl == 4)
+assert(election_tbl[4].state == 'follower')
+
+box.cfg{election_mode='off'}
+assert(#election_tbl == 5)
+
+box.cfg{election_mode='manual'}
+
+assert(#election_tbl == 6)
+assert(election_tbl[6].state == 'follower')
+
+box.ctl.promote()
+
+assert(#election_tbl == 9)
+assert(election_tbl[7].state == 'follower')
+assert(election_tbl[8].state == 'candidate')
+assert(election_tbl[9].state == 'leader')
+
+test_run:cmd('stop server replica')
+test_run:cmd('delete server replica')
+box.schema.user.revoke('guest', 'replication')
diff --git a/test/replication/suite.cfg b/test/replication/suite.cfg
index a51a2d51a..f2af00038 100644
--- a/test/replication/suite.cfg
+++ b/test/replication/suite.cfg
@@ -46,6 +46,7 @@
     "gh-5536-wal-limit.test.lua": {},
     "gh-5566-final-join-synchro.test.lua": {},
     "gh-5613-bootstrap-prefer-booted.test.lua": {},
+    "gh-5819-election-triggers.test.lua": {},
     "gh-6027-applier-error-show.test.lua": {},
     "gh-6032-promote-wal-write.test.lua": {},
     "gh-6057-qsync-confirm-async-no-wal.test.lua": {},
-- 
2.30.1 (Apple Git-130)