<HTML><BODY>Hi,<br>overall patch looks good with minor conplains (summary)<br>- unused headers<br>- is it worth it to create a separate fiber to check if read-only condition was changed?<br>- test SHUTDOWN should be implemented<br><br><br><br><blockquote style="border-left:1px solid #0857A6; margin:10px; padding:0 0 0 10px;">
Четверг, 14 июня 2018, 18:04 +03:00 от Ilya Markov <imarkov@tarantool.org>:<br><br><div id=""><div class="js-helper js-readmsg-msg"><div><div id="style_15289886900000000863_BODY">Add following cases of triggers:<br>
* System space recovery. Called on finish of bootstrap or finish of join or<br>
snapshot recovery.<br>
* Local recovery. Called on finish of bootstrap or finish of recovery.<br>
* Read_only/read_write. Called on changes of read_only state of<br>
instance.<br>
* Shutdown. Called on controlled shutdown.<br>
* Replicaset_add/replicaset_remove. Called on changes in space _cluster.<br><br>
Errors inside triggers are logged and don't influence on instance<br>
behaviour.<br><br>
Continue #3159<br>
---<br>
src/box/alter.cc | 1 +<br>
src/box/box.cc | 44 ++++++-<br>
src/box/engine.c | 14 +++<br>
src/box/lua/ctl.c | 4 +-<br>
src/box/memtx_engine.c | 2 +-<br>
src/box/replication.cc | 14 +++<br>
test/replication/master_onctl.lua | 34 +++++<br>
test/replication/onctl.result | 250 +++++++++++++++++++++++++++++++++++++<br>
test/replication/onctl.test.lua | 105 ++++++++++++++++<br>
test/replication/replica_onctl.lua | 34 +++++<br>
test/replication/suite.cfg | 1 +<br>
11 files changed, 498 insertions(+), 5 deletions(-)<br>
create mode 100644 test/replication/master_onctl.lua<br>
create mode 100644 test/replication/onctl.result<br>
create mode 100644 test/replication/onctl.test.lua<br>
create mode 100644 test/replication/replica_onctl.lua<br><br>
diff --git a/src/box/alter.cc b/src/box/alter.cc<br>
index 6f6fcb0..7ec548b 100644<br>
--- a/src/box/alter.cc<br>
+++ b/src/box/alter.cc<br>
@@ -52,6 +52,7 @@<br>
#include "identifier.h"<br>
#include "version.h"<br>
#include "sequence.h"</div></div></div></div></blockquote><br><blockquote style="border-left:1px solid #0857A6; margin:10px; padding:0 0 0 10px;"><div id=""><div class="js-helper js-readmsg-msg"><div><div id="style_15289886900000000863_BODY"><br>
+#include "ctl.h"</div></div></div></div></blockquote>Not used.<br><blockquote style="border-left:1px solid #0857A6; margin:10px; padding:0 0 0 10px;"><div id=""><div class="js-helper js-readmsg-msg"><div><div id="style_15289886900000000863_BODY"><br><br>
/**<br>
* chap-sha1 of empty string, i.e.<br>
diff --git a/src/box/box.cc b/src/box/box.cc<br>
index 26277e7..1de37d2 100644<br>
--- a/src/box/box.cc<br>
+++ b/src/box/box.cc<br>
@@ -113,6 +113,11 @@ static fiber_cond ro_cond;<br>
*/<br>
static bool is_orphan = true;<br><br>
+/**<br>
+ * Fiber used for on_ctl_event trigger.<br>
+ */<br>
+static fiber *ro_checker;<br>
+<br>
/* Use the shared instance of xstream for all appliers */<br>
static struct xstream join_stream;<br>
static struct xstream subscribe_stream;<br>
@@ -1573,6 +1578,9 @@ box_set_replicaset_uuid(const struct tt_uuid *replicaset_uuid)<br>
void<br>
box_free(void)<br>
{<br>
+ if (run_on_ctl_event_trigger_type(CTL_EVENT_SHUTDOWN) < 0)<br>
+ say_error("ctl_trigger error in shutdown: %s",<br>
+ diag_last_error(diag_get())->errmsg);<br>
/*<br>
* See gh-584 "box_free() is called even if box is not<br>
* initialized<br>
@@ -1592,7 +1600,8 @@ box_free(void)<br>
engine_shutdown();<br>
wal_thread_stop();<br>
}<br>
-<br>
+ if (!fiber_is_dead(ro_checker))<br>
+ fiber_cancel(ro_checker);<br>
fiber_cond_destroy(&ro_cond);<br>
}<br><br>
@@ -1693,6 +1702,10 @@ bootstrap_from_master(struct replica *master)<br>
engine_begin_initial_recovery_xc(NULL);<br>
applier_resume_to_state(applier, APPLIER_FINAL_JOIN, TIMEOUT_INFINITY);<br><br>
+ if (run_on_ctl_event_trigger_type(CTL_EVENT_SYSTEM_SPACE_RECOVERY) < 0)<br>
+ say_error("ctl_trigger error in system space recovery: %s",<br>
+ diag_last_error(diag_get())->errmsg);<br>
+<br>
/*<br>
* Process final data (WALs).<br>
*/<br>
@@ -1755,11 +1768,35 @@ tx_prio_cb(struct ev_loop *loop, ev_watcher *watcher, int events)<br>
cbus_process(endpoint);<br>
}<br><br>
+static int<br>
+check_ro_f(MAYBE_UNUSED va_list ap)<br>
+{<br>
+ double timeout = TIMEOUT_INFINITY;<br>
+ struct on_ctl_event_ctx ctx;<br>
+ memset(&ctx, 0, sizeof(ctx));<br>
+ while (true) {<br>
+ if (box_wait_ro(!box_is_ro(), timeout) != 0) {<br>
+ if (fiber_is_cancelled())<br>
+ break;<br>
+ else<br>
+ return -1;<br>
+ }<br>
+ if (run_on_ctl_event_trigger_type(<br>
+ box_is_ro() ? CTL_EVENT_READ_ONLY:<br>
+ CTL_EVENT_READ_WRITE) < 0)<br>
+ say_error("ctl_trigger error in %s: %s",<br>
+ box_is_ro() ? "read_only" :"read_write",<br>
+ diag_last_error(diag_get())->errmsg);<br>
+ }<br>
+ return 0;<br>
+}<br>
+</div></div></div></div></blockquote>Is it worth it to create a separate fiber? May be check it near or inside fiber_cond_broadcast(&ro_cond)?<br><br><blockquote style="border-left:1px solid #0857A6; margin:10px; padding:0 0 0 10px;"><div id=""><div class="js-helper js-readmsg-msg"><div><div id="style_15289886900000000863_BODY"><br>
void<br>
box_init(void)<br>
{<br>
fiber_cond_create(&ro_cond);<br>
-<br>
+ ro_checker = fiber_new_xc("check_ro", check_ro_f);<br>
+ fiber_start(ro_checker, NULL);<br>
user_cache_init();<br>
/*<br>
* The order is important: to initialize sessions,<br>
@@ -1885,6 +1922,9 @@ box_cfg_xc(void)<br>
*/<br>
memtx_engine_recover_snapshot_xc(memtx,<br>
&last_checkpoint_vclock);<br>
+ if (run_on_ctl_event_trigger_type(CTL_EVENT_SYSTEM_SPACE_RECOVERY) < 0)<br>
+ say_error("ctl_trigger error in system space recovery: %s",<br>
+ diag_last_error(diag_get())->errmsg);<br><br>
engine_begin_final_recovery_xc();<br>
recovery_follow_local(recovery, &wal_stream.base, "hot_standby",<br>
diff --git a/src/box/engine.c b/src/box/engine.c<br>
index 82293fd..fa78753 100644<br>
--- a/src/box/engine.c<br>
+++ b/src/box/engine.c<br>
@@ -29,10 +29,12 @@<br>
* SUCH DAMAGE.<br>
*/<br>
#include "engine.h"<br>
+#include "ctl.h"<br><br>
#include <stdint.h><br>
#include <string.h><br>
#include <small/rlist.h></div></div></div></div></blockquote><br><blockquote style="border-left:1px solid #0857A6; margin:10px; padding:0 0 0 10px;"><div id=""><div class="js-helper js-readmsg-msg"><div><div id="style_15289886900000000863_BODY"><br>
+#include <fiber.h><br></div></div></div></div></blockquote>Not used.<br><blockquote style="border-left:1px solid #0857A6; margin:10px; padding:0 0 0 10px;"><div id=""><div class="js-helper js-readmsg-msg"><div><div id="style_15289886900000000863_BODY"><br>
RLIST_HEAD(engines);<br><br>
@@ -73,6 +75,14 @@ engine_bootstrap(void)<br>
if (engine->vtab->bootstrap(engine) != 0)<br>
return -1;<br>
}<br>
+ if (run_on_ctl_event_trigger_type(CTL_EVENT_SYSTEM_SPACE_RECOVERY) < 0)<br>
+ say_error("ctl_trigger error in system space recovery: %s",<br>
+ diag_last_error(diag_get())->errmsg);<br>
+<br>
+ if (run_on_ctl_event_trigger_type(CTL_EVENT_LOCAL_RECOVERY) < 0)<br>
+ say_error("ctl_trigger error in local recovery: %s",<br>
+ diag_last_error(diag_get())->errmsg);<br>
+<br>
return 0;<br>
}<br><br>
@@ -111,6 +121,10 @@ engine_end_recovery(void)<br>
if (engine->vtab->end_recovery(engine) != 0)<br>
return -1;<br>
}<br>
+ if (run_on_ctl_event_trigger_type(CTL_EVENT_LOCAL_RECOVERY) < 0)<br>
+ say_error("ctl_trigger error in local recovery: %s",<br>
+ diag_last_error(diag_get())->errmsg);<br>
+<br>
return 0;<br>
}<br><br>
diff --git a/src/box/lua/ctl.c b/src/box/lua/ctl.c<br>
index 52f320a..5bd9be3 100644<br>
--- a/src/box/lua/ctl.c<br>
+++ b/src/box/lua/ctl.c<br>
@@ -122,8 +122,8 @@ box_lua_ctl_init(struct lua_State *L)<br>
lua_pushnumber(L, CTL_EVENT_SHUTDOWN);<br>
lua_setfield(L, -2, "SHUTDOWN");<br>
lua_pushnumber(L, CTL_EVENT_REPLICASET_ADD);<br>
- lua_setfield(L, -2, "CTL_EVENT_REPLICASET_ADD");<br>
+ lua_setfield(L, -2, "REPLICASET_ADD");<br>
lua_pushnumber(L, CTL_EVENT_REPLICASET_REMOVE);<br>
- lua_setfield(L, -2, "CTL_EVENT_REPLICASET_REMOVE");<br>
+ lua_setfield(L, -2, "REPLICASET_REMOVE");<br>
lua_pop(L, 2); /* box, ctl */<br>
}<br>
diff --git a/src/box/memtx_engine.c b/src/box/memtx_engine.c<br>
index fac84ce..e737ea3 100644<br>
--- a/src/box/memtx_engine.c<br>
+++ b/src/box/memtx_engine.c<br>
@@ -48,6 +48,7 @@<br>
#include "replication.h"<br>
#include "schema.h"<br>
#include "gc.h"</div></div></div></div></blockquote><br><blockquote style="border-left:1px solid #0857A6; margin:10px; padding:0 0 0 10px;"><div id=""><div class="js-helper js-readmsg-msg"><div><div id="style_15289886900000000863_BODY"><br>
+#include "ctl.h"<br></div></div></div></div></blockquote>Also not used.<br><blockquote style="border-left:1px solid #0857A6; margin:10px; padding:0 0 0 10px;"><div id=""><div class="js-helper js-readmsg-msg"><div><div id="style_15289886900000000863_BODY"><br>
static void<br>
txn_on_yield_or_stop(struct trigger *trigger, void *event)<br>
@@ -197,7 +198,6 @@ memtx_engine_recover_snapshot(struct memtx_engine *memtx,<br>
*/<br>
if (!xlog_cursor_is_eof(&cursor))<br>
panic("snapshot `%s' has no EOF marker", filename);<br>
-<br>
return 0;<br>
}<br><br>
diff --git a/src/box/replication.cc b/src/box/replication.cc<br>
index c1e1769..75aecd0 100644<br>
--- a/src/box/replication.cc<br>
+++ b/src/box/replication.cc<br>
@@ -41,6 +41,7 @@<br>
#include "error.h"<br>
#include "relay.h"<br>
#include "vclock.h" /* VCLOCK_MAX */<br>
+#include "ctl.h"<br><br>
uint32_t instance_id = REPLICA_ID_NIL;<br>
struct tt_uuid INSTANCE_UUID;<br>
@@ -172,6 +173,12 @@ replicaset_add(uint32_t replica_id, const struct tt_uuid *replica_uuid)<br>
replica->uuid = *replica_uuid;<br>
replica_hash_insert(&replicaset.hash, replica);<br>
replica_set_id(replica, replica_id);<br>
+ struct on_ctl_event_ctx on_ctl_ctx;<br>
+ on_ctl_ctx.type = CTL_EVENT_REPLICASET_ADD;<br>
+ on_ctl_ctx.replica_id = replica_id;<br>
+ if (run_on_ctl_event_triggers(&on_ctl_ctx) < 0)<br>
+ say_error("ctl_trigger error in replica add: %s",<br>
+ diag_last_error(diag_get())->errmsg);<br>
return replica;<br>
}<br><br>
@@ -203,12 +210,19 @@ replica_clear_id(struct replica *replica)<br>
* Some records may arrive later on due to asynchronous nature of<br>
* replication.<br>
*/<br>
+ struct on_ctl_event_ctx on_ctl_ctx;<br>
+ on_ctl_ctx.type = CTL_EVENT_REPLICASET_REMOVE;<br>
+ on_ctl_ctx.replica_id = replica->id;<br>
+<br>
replicaset.replica_by_id[replica->id] = NULL;<br>
replica->id = REPLICA_ID_NIL;<br>
if (replica_is_orphan(replica)) {<br>
replica_hash_remove(&replicaset.hash, replica);<br>
replica_delete(replica);<br>
}<br>
+ if (run_on_ctl_event_triggers(&on_ctl_ctx) < 0)<br>
+ say_error("ctl_trigger error in replica remove: %s",<br>
+ diag_last_error(diag_get())->errmsg);<br>
}<br><br>
static void<br>
diff --git a/test/replication/master_onctl.lua b/test/replication/master_onctl.lua<br>
new file mode 100644<br>
index 0000000..e0eb39a<br>
--- /dev/null<br>
+++ b/test/replication/master_onctl.lua<br>
@@ -0,0 +1,34 @@<br>
+#!/usr/bin/env tarantool<br>
+os = require('os')<br>
+<br>
+SYSTEM_SPACE_RECOVERY = 0<br>
+LOCAL_RECOVERY = 0<br>
+READ_ONLY = 0<br>
+READ_WRITE = 0<br>
+REPLICASET_ADD = {}<br>
+REPLICASET_REMOVE = {}<br>
+<br>
+local function onctl(ctx)<br>
+ if ctx.type == box.ctl.event.SYSTEM_SPACE_RECOVERY then<br>
+ SYSTEM_SPACE_RECOVERY = SYSTEM_SPACE_RECOVERY + 1<br>
+ elseif ctx.type == box.ctl.event.LOCAL_RECOVERY then<br>
+ LOCAL_RECOVERY = LOCAL_RECOVERY + 1<br>
+ elseif ctx.type == box.ctl.event.READ_ONLY then<br>
+ READ_ONLY = READ_ONLY + 1<br>
+ elseif ctx.type == box.ctl.event.READ_WRITE then<br>
+ READ_WRITE = READ_WRITE + 1<br>
+ elseif ctx.type == box.ctl.event.REPLICASET_ADD then<br>
+ table.insert(REPLICASET_ADD, ctx.replica_id)<br>
+ elseif ctx.type == box.ctl.event.REPLICASET_REMOVE then<br>
+ table.insert(REPLICASET_REMOVE, ctx.replica_id)<br>
+ end<br>
+end<br>
+<br>
+box.cfg({<br>
+ listen = os.getenv("LISTEN"),<br>
+ memtx_memory = 107374182,<br>
+ replication_connect_timeout = 0.5,<br>
+ on_ctl_event = onctl,<br>
+})<br>
+<br>
+require('console').listen(os.getenv('ADMIN'))<br>
diff --git a/test/replication/onctl.result b/test/replication/onctl.result<br>
new file mode 100644<br>
index 0000000..19b3e67<br>
--- /dev/null<br>
+++ b/test/replication/onctl.result<br>
@@ -0,0 +1,250 @@<br>
+env = require('test_run')<br>
+---<br>
+...<br>
+test_run = env.new()<br>
+---<br>
+...<br>
+test_run:cmd("create server master with script='replication/master_onctl.lua'")<br>
+---<br>
+- true<br>
+...<br>
+test_run:cmd("create server replica with rpl_master=master, script='replication/replica_onctl.lua'")<br>
+---<br>
+- true<br>
+...<br>
+test_run:cmd("start server master")<br>
+---<br>
+- true<br>
+...<br>
+test_run:cmd("switch master")<br>
+---<br>
+- true<br>
+...<br>
+box.schema.user.grant('guest', 'replication')<br>
+---<br>
+...<br>
+SYSTEM_SPACE_RECOVERY<br>
+---<br>
+- 1<br>
+...<br>
+LOCAL_RECOVERY<br>
+---<br>
+- 1<br>
+...<br>
+READ_ONLY<br>
+---<br>
+- 0<br>
+...<br>
+READ_WRITE<br>
+---<br>
+- 1<br>
+...<br>
+-- must be two entries. First from bootstrap.snap, second for current instance.<br>
+REPLICASET_ADD<br>
+---<br>
+- - 1<br>
+ - 1<br>
+...<br>
+-- must be one entry. Deletion of initial tuple in _cluster space.<br>
+REPLICASET_REMOVE<br>
+---<br>
+- - 1<br>
+...<br>
+REPLICASET_ADD = {}<br>
+---<br>
+...<br>
+REPLICASET_REMOVE = {}<br>
+---<br>
+...<br>
+new_replica_id = 0<br>
+---<br>
+...<br>
+deleted_replica_id = 0<br>
+---<br>
+...<br>
+test_run:cmd("setopt delimiter ';'")<br>
+---<br>
+- true<br>
+...<br>
+function on_ctl_new(ctx)<br>
+ if ctx.type == box.ctl.event.REPLICASET_ADD then<br>
+ new_replica_id = ctx.replica_id<br>
+ elseif ctx.type == box.ctl.event.REPLICASET_REMOVE then<br>
+ deleted_replica_id = ctx.replica_id<br>
+ end<br>
+end;<br>
+---<br>
+...<br>
+test_run:cmd("setopt delimiter ''");<br>
+---<br>
+- true<br>
+...<br>
+_ = box.ctl.on_ctl_event(on_ctl_new)<br>
+---<br>
+...<br>
+test_run:cmd("start server replica")<br>
+---<br>
+- true<br>
+...<br>
+REPLICASET_ADD<br>
+---<br>
+- - 2<br>
+...<br>
+REPLICASET_REMOVE<br>
+---<br>
+- []<br>
+...<br>
+new_replica_id<br>
+---<br>
+- 2<br>
+...<br>
+deleted_replica_id<br>
+---<br>
+- 0<br>
+...<br>
+test_run:cmd("switch replica")<br>
+---<br>
+- true<br>
+...<br>
+test_run:cmd("setopt delimiter ';'")<br>
+---<br>
+- true<br>
+...<br>
+function on_ctl_shutdown(ctx)<br>
+ if ctx.type == box.ctl.event.SHUTDOWN then<br>
+ require("log").info("test replica shutdown")<br>
+ end<br>
+end;<br>
+---<br>
+...<br>
+function on_ctl_error(ctx)<br>
+ error("trigger error")<br>
+end;<br>
+---<br>
+...<br>
+test_run:cmd("setopt delimiter ''");<br>
+---<br>
+- true<br>
+...<br>
+SYSTEM_SPACE_RECOVERY<br>
+---<br>
+- 1<br>
+...<br>
+LOCAL_RECOVERY<br>
+---<br>
+- 1<br>
+...<br>
+READ_ONLY<br>
+---<br>
+- 0<br>
+...<br>
+READ_WRITE<br>
+---<br>
+- 1<br>
+...<br>
+REPLICASET_ADD<br>
+---<br>
+- - 2<br>
+...<br>
+REPLICASET_REMOVE<br>
+---<br>
+- []<br>
+...<br>
+box.cfg{read_only = true}<br>
+---<br>
+...<br>
+fiber = require("fiber")<br>
+---<br>
+...<br>
+while READ_ONLY == 0 do fiber.sleep(0.001) end<br>
+---<br>
+...<br>
+READ_ONLY<br>
+---<br>
+- 1<br>
+...<br>
+box.cfg{on_ctl_event = on_ctl_error}<br>
+---<br>
+...<br>
+box.cfg{read_only = false}<br>
+---<br>
+...<br>
+test_run:grep_log('replica', 'ctl_trigger error')<br>
+---<br>
+- ctl_trigger error<br>
+...<br>
+box.cfg{on_ctl_event = {box.NULL, on_ctl_error}}<br>
+---<br>
+...<br>
+box.cfg{on_ctl_event = on_ctl_shutdown}<br>
+---<br>
+...<br>
+test_run:cmd("restart server replica")<br>
+-- TODO: test SHUTDOWN, when it is possible to grep logs of killed replica.<br>
+--test_run:grep_log('replica', 'test replica shutdown')<br>
+test_run:cmd("switch master")<br>
+---<br>
+- true<br>
+...<br>
+box.schema.user.revoke('guest', 'replication')<br>
+---<br>
+...<br>
+_ = box.space._cluster:delete{2}<br>
+---<br>
+...<br>
+SYSTEM_SPACE_RECOVERY<br>
+---<br>
+- 1<br>
+...<br>
+LOCAL_RECOVERY<br>
+---<br>
+- 1<br>
+...<br>
+READ_ONLY<br>
+---<br>
+- 0<br>
+...<br>
+READ_WRITE<br>
+---<br>
+- 1<br>
+...<br>
+REPLICASET_ADD<br>
+---<br>
+- - 2<br>
+...<br>
+REPLICASET_REMOVE<br>
+---<br>
+- - 2<br>
+...<br>
+new_replica_id<br>
+---<br>
+- 2<br>
+...<br>
+deleted_replica_id<br>
+---<br>
+- 2<br>
+...<br>
+box.ctl.on_ctl_event(nil, on_ctl_new)<br>
+---<br>
+...<br>
+-- cleanup<br>
+test_run:cmd("switch default")<br>
+---<br>
+- true<br>
+...<br>
+test_run:cmd("stop server master")<br>
+---<br>
+- true<br>
+...<br>
+test_run:cmd("cleanup server master")<br>
+---<br>
+- true<br>
+...<br>
+test_run:cmd("stop server replica")<br>
+---<br>
+- true<br>
+...<br>
+test_run:cmd("cleanup server replica")<br>
+---<br>
+- true<br>
+...<br>
diff --git a/test/replication/onctl.test.lua b/test/replication/onctl.test.lua<br>
new file mode 100644<br>
index 0000000..ff6a898<br>
--- /dev/null<br>
+++ b/test/replication/onctl.test.lua<br>
@@ -0,0 +1,105 @@<br>
+env = require('test_run')<br>
+test_run = env.new()<br>
+<br>
+test_run:cmd("create server master with script='replication/master_onctl.lua'")<br>
+test_run:cmd("create server replica with rpl_master=master, script='replication/replica_onctl.lua'")<br>
+<br>
+test_run:cmd("start server master")<br>
+test_run:cmd("switch master")<br>
+box.schema.user.grant('guest', 'replication')<br>
+<br>
+SYSTEM_SPACE_RECOVERY<br>
+LOCAL_RECOVERY<br>
+READ_ONLY<br>
+READ_WRITE<br>
+-- must be two entries. First from bootstrap.snap, second for current instance.<br>
+REPLICASET_ADD<br>
+-- must be one entry. Deletion of initial tuple in _cluster space.<br>
+REPLICASET_REMOVE<br>
+<br>
+REPLICASET_ADD = {}<br>
+REPLICASET_REMOVE = {}<br>
+<br>
+new_replica_id = 0<br>
+deleted_replica_id = 0<br>
+<br>
+test_run:cmd("setopt delimiter ';'")<br>
+function on_ctl_new(ctx)<br>
+ if ctx.type == box.ctl.event.REPLICASET_ADD then<br>
+ new_replica_id = ctx.replica_id<br>
+ elseif ctx.type == box.ctl.event.REPLICASET_REMOVE then<br>
+ deleted_replica_id = ctx.replica_id<br>
+ end<br>
+end;<br>
+test_run:cmd("setopt delimiter ''");<br>
+<br>
+_ = box.ctl.on_ctl_event(on_ctl_new)<br>
+<br>
+test_run:cmd("start server replica")<br>
+<br>
+REPLICASET_ADD<br>
+REPLICASET_REMOVE<br>
+<br>
+new_replica_id<br>
+deleted_replica_id<br>
+<br>
+test_run:cmd("switch replica")<br>
+<br>
+test_run:cmd("setopt delimiter ';'")<br>
+function on_ctl_shutdown(ctx)<br>
+ if ctx.type == box.ctl.event.SHUTDOWN then<br>
+ require("log").info("test replica shutdown")<br>
+ end<br>
+end;<br>
+<br>
+function on_ctl_error(ctx)<br>
+ error("trigger error")<br>
+end;<br>
+<br>
+test_run:cmd("setopt delimiter ''");<br>
+<br>
+SYSTEM_SPACE_RECOVERY<br>
+LOCAL_RECOVERY<br>
+READ_ONLY<br>
+READ_WRITE<br>
+REPLICASET_ADD<br>
+REPLICASET_REMOVE<br>
+<br>
+box.cfg{read_only = true}<br>
+fiber = require("fiber")<br>
+while READ_ONLY == 0 do fiber.sleep(0.001) end<br>
+READ_ONLY<br>
+<br>
+box.cfg{on_ctl_event = on_ctl_error}<br>
+box.cfg{read_only = false}<br>
+test_run:grep_log('replica', 'ctl_trigger error')<br>
+box.cfg{on_ctl_event = {box.NULL, on_ctl_error}}<br>
+box.cfg{on_ctl_event = on_ctl_shutdown}<br>
+<br>
+test_run:cmd("restart server replica")<br>
+-- TODO: test SHUTDOWN, when it is possible to grep logs of killed replica.<br>
+--test_run:grep_log('replica', 'test replica shutdown')</div></div></div></div></blockquote>It should be done<br><blockquote style="border-left:1px solid #0857A6; margin:10px; padding:0 0 0 10px;"><div id=""><div class="js-helper js-readmsg-msg"><div><div id="style_15289886900000000863_BODY"><br>
+<br>
+<br>
+test_run:cmd("switch master")<br>
+box.schema.user.revoke('guest', 'replication')<br>
+_ = box.space._cluster:delete{2}<br>
+<br>
+SYSTEM_SPACE_RECOVERY<br>
+LOCAL_RECOVERY<br>
+READ_ONLY<br>
+READ_WRITE<br>
+REPLICASET_ADD<br>
+REPLICASET_REMOVE<br>
+<br>
+new_replica_id<br>
+deleted_replica_id<br>
+<br>
+box.ctl.on_ctl_event(nil, on_ctl_new)<br>
+<br>
+-- cleanup<br>
+test_run:cmd("switch default")<br>
+test_run:cmd("stop server master")<br>
+test_run:cmd("cleanup server master")<br>
+test_run:cmd("stop server replica")<br>
+test_run:cmd("cleanup server replica")<br>
diff --git a/test/replication/replica_onctl.lua b/test/replication/replica_onctl.lua<br>
new file mode 100644<br>
index 0000000..d6ce73c<br>
--- /dev/null<br>
+++ b/test/replication/replica_onctl.lua<br>
@@ -0,0 +1,34 @@<br>
+#!/usr/bin/env tarantool<br>
+<br>
+SYSTEM_SPACE_RECOVERY = 0<br>
+LOCAL_RECOVERY = 0<br>
+READ_ONLY = 0<br>
+READ_WRITE = 0<br>
+REPLICASET_ADD = {}<br>
+REPLICASET_REMOVE = {}<br>
+<br>
+local function onctl(ctx)<br>
+ if ctx.type == box.ctl.event.SYSTEM_SPACE_RECOVERY then<br>
+ SYSTEM_SPACE_RECOVERY = SYSTEM_SPACE_RECOVERY + 1<br>
+ elseif ctx.type == box.ctl.event.LOCAL_RECOVERY then<br>
+ LOCAL_RECOVERY = LOCAL_RECOVERY + 1<br>
+ elseif ctx.type == box.ctl.event.READ_ONLY then<br>
+ READ_ONLY = READ_ONLY + 1<br>
+ elseif ctx.type == box.ctl.event.READ_WRITE then<br>
+ READ_WRITE = READ_WRITE + 1<br>
+ elseif ctx.type == box.ctl.event.REPLICASET_ADD then<br>
+ table.insert(REPLICASET_ADD, ctx.replica_id)<br>
+ elseif ctx.type == box.ctl.event.REPLICASET_REMOVE then<br>
+ table.insert(REPLICASET_REMOVE, ctx.replica_id)<br>
+ end<br>
+end<br>
+<br>
+box.cfg({<br>
+ listen = os.getenv("LISTEN"),<br>
+ replication = os.getenv("MASTER"),<br>
+ memtx_memory = 107374182,<br>
+ replication_connect_timeout = 0.5,<br>
+ on_ctl_event = onctl,<br>
+})<br>
+<br>
+require('console').listen(os.getenv('ADMIN'))<br>
diff --git a/test/replication/suite.cfg b/test/replication/suite.cfg<br>
index 95e94e5..365a825 100644<br>
--- a/test/replication/suite.cfg<br>
+++ b/test/replication/suite.cfg<br>
@@ -6,6 +6,7 @@<br>
"wal_off.test.lua": {},<br>
"hot_standby.test.lua": {},<br>
"rebootstrap.test.lua": {},<br>
+ "onctl.test.lua": {},<br>
"*": {<br>
"memtx": {"engine": "memtx"},<br>
"vinyl": {"engine": "vinyl"}<br>
-- <br>
2.7.4<br><br><br></div></div></div></div></blockquote>
<br>
<br>Best regards,<br>Konstantin Belyavskiy<br>k.belyavskiy@tarantool.org<br></BODY></HTML>