[server 5/5] replication: introduce orphan mode

[Vladimir Davydov]

On Wed, Jan 24, 2018 at 08:44:54PM +0300, Vladimir Davydov wrote:
> We assume that a replica is synchronized if its lag is not greater than
> the value of new configuration option box.cfg.replication_sync_lag.
> Otherwise a replica is considered to be syncing and has "sync" status.
> If replication_sync_lag is unset (nil) or set to TIMEOUT_INFINITY, then
> a replica skips the "sync" state and switches to "follow" immediately.
> For the sake of backward compatibility, the new option is unset by
> default.

Changed the default to 10 seconds - see on the branch.
Incremental diff follows.
---
diff --git a/src/box/applier.cc b/src/box/applier.cc
index 60d4446e..f3870a4f 100644
--- a/src/box/applier.cc
+++ b/src/box/applier.cc
@@ -355,7 +355,16 @@ applier_subscribe(struct applier *applier)
 	coio_write_xrow(coio, &row);
 
 	if (applier->state == APPLIER_READY) {
-		applier_set_state(applier, APPLIER_SYNC);
+		/*
+		 * Tarantool < 1.7.7 does not send periodic heartbeat
+		 * messages so we cannot enable applier synchronization
+		 * for it without risking getting stuck in the 'orphan'
+		 * mode until a DML operation happens on the master.
+		 */
+		if (applier->version_id >= version_id(1, 7, 7))
+			applier_set_state(applier, APPLIER_SYNC);
+		else
+			applier_set_state(applier, APPLIER_FOLLOW);
 	} else {
 		/*
 		 * Tarantool < 1.7.0 sends replica id during
@@ -419,7 +428,7 @@ applier_subscribe(struct applier *applier)
 			say_info("final data received");
 			applier_set_state(applier, APPLIER_JOINED);
 			applier_set_state(applier, APPLIER_READY);
-			applier_set_state(applier, APPLIER_SYNC);
+			applier_set_state(applier, APPLIER_FOLLOW);
 		}
 
 		if (applier->state == APPLIER_SYNC &&
diff --git a/src/box/lua/load_cfg.lua b/src/box/lua/load_cfg.lua
index 6b0ca9ce..4ac04083 100644
--- a/src/box/lua/load_cfg.lua
+++ b/src/box/lua/load_cfg.lua
@@ -56,7 +56,7 @@ local default_cfg = {
     checkpoint_count    = 2,
     worker_pool_threads = 4,
     replication_timeout = 1,
-    replication_sync_lag = nil, -- no sync
+    replication_sync_lag = 10,
     replication_connect_quorum = nil, -- connect all
 }
 
diff --git a/src/box/replication.cc b/src/box/replication.cc
index 2e106a7c..11b2dfc8 100644
--- a/src/box/replication.cc
+++ b/src/box/replication.cc
@@ -48,7 +48,7 @@ struct tt_uuid REPLICASET_UUID;
 
 double replication_timeout = 1.0; /* seconds */
 int replication_connect_quorum = REPLICATION_CONNECT_QUORUM_ALL;
-double replication_sync_lag = TIMEOUT_INFINITY;
+double replication_sync_lag = 10.0; /* seconds */
 
 struct replicaset replicaset;
 
diff --git a/test/app-tap/init_script.result b/test/app-tap/init_script.result
index 73dea399..53f87a54 100644
--- a/test/app-tap/init_script.result
+++ b/test/app-tap/init_script.result
@@ -21,27 +21,28 @@ box.cfg
 16	pid_file:box.pid
 17	read_only:false
 18	readahead:16320
-19	replication_timeout:1
-20	rows_per_wal:500000
-21	slab_alloc_factor:1.05
-22	too_long_threshold:0.5
-23	vinyl_bloom_fpr:0.05
-24	vinyl_cache:134217728
-25	vinyl_dir:.
-26	vinyl_max_tuple_size:1048576
-27	vinyl_memory:134217728
-28	vinyl_page_size:8192
-29	vinyl_range_size:1073741824
-30	vinyl_read_threads:1
-31	vinyl_run_count_per_level:2
-32	vinyl_run_size_ratio:3.5
-33	vinyl_timeout:60
-34	vinyl_write_threads:2
-35	wal_dir:.
-36	wal_dir_rescan_delay:2
-37	wal_max_size:268435456
-38	wal_mode:write
-39	worker_pool_threads:4
+19	replication_sync_lag:10
+20	replication_timeout:1
+21	rows_per_wal:500000
+22	slab_alloc_factor:1.05
+23	too_long_threshold:0.5
+24	vinyl_bloom_fpr:0.05
+25	vinyl_cache:134217728
+26	vinyl_dir:.
+27	vinyl_max_tuple_size:1048576
+28	vinyl_memory:134217728
+29	vinyl_page_size:8192
+30	vinyl_range_size:1073741824
+31	vinyl_read_threads:1
+32	vinyl_run_count_per_level:2
+33	vinyl_run_size_ratio:3.5
+34	vinyl_timeout:60
+35	vinyl_write_threads:2
+36	wal_dir:.
+37	wal_dir_rescan_delay:2
+38	wal_max_size:268435456
+39	wal_mode:write
+40	worker_pool_threads:4
 --
 -- Test insert from detached fiber
 --
diff --git a/test/box/admin.result b/test/box/admin.result
index cb1c7d4a..13e599eb 100644
--- a/test/box/admin.result
+++ b/test/box/admin.result
@@ -54,6 +54,8 @@ cfg_filter(box.cfg)
     - false
   - - readahead
     - 16320
+  - - replication_sync_lag
+    - 10
   - - replication_timeout
     - 1
   - - rows_per_wal
diff --git a/test/box/cfg.result b/test/box/cfg.result
index 7be132e3..9f0ad595 100644
--- a/test/box/cfg.result
+++ b/test/box/cfg.result
@@ -50,6 +50,8 @@ cfg_filter(box.cfg)
     - false
   - - readahead
     - 16320
+  - - replication_sync_lag
+    - 10
   - - replication_timeout
     - 1
   - - rows_per_wal
@@ -135,6 +137,8 @@ cfg_filter(box.cfg)
     - false
   - - readahead
     - 16320
+  - - replication_sync_lag
+    - 10
   - - replication_timeout
     - 1
   - - rows_per_wal