[PATCH 0/2] Do not enter orphan if replica is re-bootstrapping

[PATCH 0/2] Do not enter orphan if replica is re-bootstrapping

[Vladimir Davydov]

If replication quorum can't be formed, because some replicas are
re-bootstrapping, box.cfg{} will return leaving the instance in
orphan mode. This is wrong, because those replicas will be up and
running soon. Let's take into account loading replicas while syncing
a quorum.

https://github.com/tarantool/tarantool/issues/3422
https://github.com/tarantool/tarantool/commits/gh-3422-replication-dont-stop-sync-if-replica-loading

Vladimir Davydov (2):
  replication: fix log message in case of sync failure
  replication: do not stop syncing if replicas are loading

 src/box/replication.cc                | 45 +++++++++++++++++++++++------------
 src/box/replication.h                 | 10 ++++++++
 test/replication/rebootstrap.lua      | 26 ++++++++++++++++++++
 test/replication/rebootstrap.result   | 45 +++++++++++++++++++++++++++++++++++
 test/replication/rebootstrap.test.lua | 21 ++++++++++++++++
 test/replication/rebootstrap1.lua     |  1 +
 test/replication/rebootstrap2.lua     |  1 +
 test/replication/suite.cfg            |  1 +
 8 files changed, 135 insertions(+), 15 deletions(-)
 create mode 100644 test/replication/rebootstrap.lua
 create mode 100644 test/replication/rebootstrap.result
 create mode 100644 test/replication/rebootstrap.test.lua
 create mode 120000 test/replication/rebootstrap1.lua
 create mode 120000 test/replication/rebootstrap2.lua

-- 
2.11.0

[PATCH 1/2] replication: fix log message in case of sync failure

[Vladimir Davydov]

replicaset_sync() returns not only if the instance synchronized to
connected replicas, but also if some replicas have disconnected and
the quorum can't be formed any more. Nevertheless, it always prints
that sync has been completed. Fix it.

See #3422
---
 src/box/replication.cc | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/src/box/replication.cc b/src/box/replication.cc
index 7bdefccf..6d90ee5f 100644
--- a/src/box/replication.cc
+++ b/src/box/replication.cc
@@ -636,18 +636,10 @@ replicaset_sync(void)
 {
 	int quorum = replicaset_quorum();
 
-	if (quorum > 0)
-		say_verbose("synchronizing with %d replicas", quorum);
-
-	if (replicaset.applier.connected < quorum) {
-		/*
-		 * Not enough replicas connected to form a quorum.
-		 * Do not stall configuration, leave the instance
-		 * in 'orphan' state.
-		 */
-		say_crit("entering orphan mode");
+	if (quorum == 0)
 		return;
-	}
+
+	say_verbose("synchronizing with %d replicas", quorum);
 
 	/*
 	 * Wait until all connected replicas synchronize up to
@@ -657,10 +649,18 @@ replicaset_sync(void)
 	       replicaset.applier.connected >= quorum)
 		fiber_cond_wait(&replicaset.applier.cond);
 
-	if (quorum > 0) {
-		say_crit("replica set sync complete, quorum of %d "
-			 "replicas formed", quorum);
+	if (replicaset.applier.synced < quorum) {
+		/*
+		 * Not enough replicas connected to form a quorum.
+		 * Do not stall configuration, leave the instance
+		 * in 'orphan' state.
+		 */
+		say_crit("entering orphan mode");
+		return;
 	}
+
+	say_crit("replica set sync complete, quorum of %d "
+		 "replicas formed", quorum);
 }
 
 void
-- 
2.11.0

[PATCH 2/2] replication: do not stop syncing if replicas are loading

[Vladimir Davydov]

If a replica disconnects while sync is in progress, box.cfg{} may stop
syncing leaving the instance in 'orphan' mode. This will happen if not
enough replicas are connected to form a quorum. This makes sense e.g. on
network error, but not when a replica is loading, because in the latter
case it should be up and running quite soon. Let's account replicas that
disconnected because they haven't completed initial configuration yet
and continue syncing if connected + loading > quorum.

Closes #3422
---
 src/box/replication.cc                | 17 ++++++++++++-
 src/box/replication.h                 | 10 ++++++++
 test/replication/rebootstrap.lua      | 26 ++++++++++++++++++++
 test/replication/rebootstrap.result   | 45 +++++++++++++++++++++++++++++++++++
 test/replication/rebootstrap.test.lua | 21 ++++++++++++++++
 test/replication/rebootstrap1.lua     |  1 +
 test/replication/rebootstrap2.lua     |  1 +
 test/replication/suite.cfg            |  1 +
 8 files changed, 121 insertions(+), 1 deletion(-)
 create mode 100644 test/replication/rebootstrap.lua
 create mode 100644 test/replication/rebootstrap.result
 create mode 100644 test/replication/rebootstrap.test.lua
 create mode 120000 test/replication/rebootstrap1.lua
 create mode 120000 test/replication/rebootstrap2.lua

diff --git a/src/box/replication.cc b/src/box/replication.cc
index 6d90ee5f..a9fab319 100644
--- a/src/box/replication.cc
+++ b/src/box/replication.cc
@@ -138,6 +138,7 @@ replica_new(void)
 	trigger_create(&replica->on_applier_state,
 		       replica_on_applier_state_f, NULL, NULL);
 	replica->state = REPLICA_DISCONNECTED;
+	replica->is_loading = false;
 	return replica;
 }
 
@@ -234,6 +235,7 @@ replica_on_applier_connect(struct replica *replica)
 	assert(tt_uuid_is_nil(&replica->uuid));
 	assert(!tt_uuid_is_nil(&applier->uuid));
 	assert(replica->state == REPLICA_DISCONNECTED);
+	assert(!replica->is_loading);
 
 	replica->uuid = applier->uuid;
 
@@ -277,6 +279,12 @@ replica_on_applier_reconnect(struct replica *replica)
 	assert(!tt_uuid_is_nil(&applier->uuid));
 	assert(replica->state == REPLICA_DISCONNECTED);
 
+	if (replica->is_loading) {
+		assert(replicaset.applier.loading > 0);
+		replicaset.applier.loading--;
+		replica->is_loading = false;
+	}
+
 	if (!tt_uuid_is_equal(&replica->uuid, &applier->uuid)) {
 		/*
 		 * Master's UUID changed, most likely because it was
@@ -316,6 +324,10 @@ replica_on_applier_disconnect(struct replica *replica)
 	case REPLICA_CONNECTED:
 		assert(replicaset.applier.connected > 0);
 		replicaset.applier.connected--;
+		if (replica->applier->last_logged_errcode == ER_LOADING) {
+			replicaset.applier.loading++;
+			replica->is_loading = true;
+		}
 		break;
 	case REPLICA_DISCONNECTED:
 		break;
@@ -424,6 +436,7 @@ replicaset_update(struct applier **appliers, int count)
 		applier = replica->applier;
 		replica_clear_applier(replica);
 		replica->state = REPLICA_DISCONNECTED;
+		replica->is_loading = false;
 		applier_stop(applier);
 		applier_delete(applier);
 	}
@@ -439,6 +452,7 @@ replicaset_update(struct applier **appliers, int count)
 	/* Save new appliers */
 	replicaset.applier.total = count;
 	replicaset.applier.connected = 0;
+	replicaset.applier.loading = 0;
 	replicaset.applier.synced = 0;
 
 	replica_hash_foreach_safe(&uniq, replica, next) {
@@ -646,7 +660,8 @@ replicaset_sync(void)
 	 * replication_sync_lag
 	 */
 	while (replicaset.applier.synced < quorum &&
-	       replicaset.applier.connected >= quorum)
+	       replicaset.applier.connected +
+	       replicaset.applier.loading >= quorum)
 		fiber_cond_wait(&replicaset.applier.cond);
 
 	if (replicaset.applier.synced < quorum) {
diff --git a/src/box/replication.h b/src/box/replication.h
index 8a9d5754..6d83f43b 100644
--- a/src/box/replication.h
+++ b/src/box/replication.h
@@ -194,6 +194,11 @@ struct replicaset {
 		 */
 		int connected;
 		/**
+		 * Number of appliers that are disconnected,
+		 * because replica is loading.
+		 */
+		int loading;
+		/**
 		 * Number of appliers that have successfully
 		 * synchronized and hence contribute to the
 		 * quorum.
@@ -256,6 +261,11 @@ struct replica {
 	struct trigger on_applier_state;
 	/** Replica sync state. */
 	enum replica_state state;
+	/**
+	 * Set if we failed to sync to the replica, because it
+	 * hasn't finished initial configuration yet.
+	 */
+	bool is_loading;
 };
 
 enum {
diff --git a/test/replication/rebootstrap.lua b/test/replication/rebootstrap.lua
new file mode 100644
index 00000000..0b67e7b0
--- /dev/null
+++ b/test/replication/rebootstrap.lua
@@ -0,0 +1,26 @@
+#!/usr/bin/env tarantool
+
+-- get instance name from filename (quorum1.lua => quorum1)
+local INSTANCE_ID = string.match(arg[0], "%d")
+
+local SOCKET_DIR = require('fio').cwd()
+local function instance_uri(instance_id)
+    return SOCKET_DIR..'/rebootstrap'..instance_id..'.sock';
+end
+
+-- start console first
+require('console').listen(os.getenv('ADMIN'))
+
+box.cfg({
+    listen = instance_uri(INSTANCE_ID);
+    instance_uuid = '12345678-abcd-1234-abcd-123456789ef' .. INSTANCE_ID,
+    replication_timeout = 0.1;
+    replication = {
+        instance_uri(1);
+        instance_uri(2);
+    };
+})
+
+box.once("bootstrap", function()
+    box.schema.user.grant('guest', 'replication')
+end)
diff --git a/test/replication/rebootstrap.result b/test/replication/rebootstrap.result
new file mode 100644
index 00000000..afbfc8e6
--- /dev/null
+++ b/test/replication/rebootstrap.result
@@ -0,0 +1,45 @@
+test_run = require('test_run').new()
+---
+...
+SERVERS = {'rebootstrap1', 'rebootstrap2'}
+---
+...
+test_run:create_cluster(SERVERS)
+---
+...
+test_run:wait_fullmesh(SERVERS)
+---
+...
+--
+-- gh-3422: If quorum can't be formed, because some replicas are
+-- re-bootstrapping, box.cfg{} must wait for bootstrap to complete
+-- instead of stopping synchronization and leaving the instance
+-- in 'orphan' mode.
+--
+test_run:cmd('stop server rebootstrap1')
+---
+- true
+...
+test_run:cmd('restart server rebootstrap2 with cleanup=True, wait=False, wait_load=False')
+---
+- true
+...
+test_run:cmd('start server rebootstrap1')
+---
+- true
+...
+test_run:cmd('switch rebootstrap1')
+---
+- true
+...
+box.info.status -- running
+---
+- running
+...
+test_run:cmd('switch default')
+---
+- true
+...
+test_run:drop_cluster(SERVERS)
+---
+...
diff --git a/test/replication/rebootstrap.test.lua b/test/replication/rebootstrap.test.lua
new file mode 100644
index 00000000..954726dd
--- /dev/null
+++ b/test/replication/rebootstrap.test.lua
@@ -0,0 +1,21 @@
+test_run = require('test_run').new()
+
+SERVERS = {'rebootstrap1', 'rebootstrap2'}
+
+test_run:create_cluster(SERVERS)
+test_run:wait_fullmesh(SERVERS)
+
+--
+-- gh-3422: If quorum can't be formed, because some replicas are
+-- re-bootstrapping, box.cfg{} must wait for bootstrap to complete
+-- instead of stopping synchronization and leaving the instance
+-- in 'orphan' mode.
+--
+test_run:cmd('stop server rebootstrap1')
+test_run:cmd('restart server rebootstrap2 with cleanup=True, wait=False, wait_load=False')
+test_run:cmd('start server rebootstrap1')
+test_run:cmd('switch rebootstrap1')
+box.info.status -- running
+
+test_run:cmd('switch default')
+test_run:drop_cluster(SERVERS)
diff --git a/test/replication/rebootstrap1.lua b/test/replication/rebootstrap1.lua
new file mode 120000
index 00000000..0840babd
--- /dev/null
+++ b/test/replication/rebootstrap1.lua
@@ -0,0 +1 @@
+rebootstrap.lua
\ No newline at end of file
diff --git a/test/replication/rebootstrap2.lua b/test/replication/rebootstrap2.lua
new file mode 120000
index 00000000..0840babd
--- /dev/null
+++ b/test/replication/rebootstrap2.lua
@@ -0,0 +1 @@
+rebootstrap.lua
\ No newline at end of file
diff --git a/test/replication/suite.cfg b/test/replication/suite.cfg
index 7ae078f7..95e94e5a 100644
--- a/test/replication/suite.cfg
+++ b/test/replication/suite.cfg
@@ -5,6 +5,7 @@
     "status.test.lua": {},
     "wal_off.test.lua": {},
     "hot_standby.test.lua": {},
+    "rebootstrap.test.lua": {},
     "*": {
         "memtx": {"engine": "memtx"},
         "vinyl": {"engine": "vinyl"}
-- 
2.11.0