[PATCH v2.1 2/2] replication: fix a failing assert in replica_on_applier_disconnect()

[Serge Petrenko]

One possible case when two applier errors happen one after another
wasn't handled in replica_on_applier_disconnect(), which lead to
occasional test failures and crashes. Handle this case and add a
regression test.

Part of #3510
---
 src/box/replication.cc         |  4 ++++
 test/replication/er_load.lua   | 25 +++++++++++++++++++++++++
 test/replication/er_load1.lua  |  1 +
 test/replication/er_load2.lua  |  1 +
 test/replication/misc.result   | 32 ++++++++++++++++++++++++++++++++
 test/replication/misc.test.lua | 12 ++++++++++++
 6 files changed, 75 insertions(+)
 create mode 100644 test/replication/er_load.lua
 create mode 120000 test/replication/er_load1.lua
 create mode 120000 test/replication/er_load2.lua

diff --git a/src/box/replication.cc b/src/box/replication.cc
index 4270911ef..c9283bd82 100644
--- a/src/box/replication.cc
+++ b/src/box/replication.cc
@@ -358,6 +358,10 @@ replica_on_applier_disconnect(struct replica *replica)
 		assert(replicaset.applier.connected > 0);
 		replicaset.applier.connected--;
 		break;
+	case APPLIER_LOADING:
+		assert(replicaset.applier.loading > 0);
+		replicaset.applier.loading--;
+		break;
 	case APPLIER_DISCONNECTED:
 		break;
 	default:
diff --git a/test/replication/er_load.lua b/test/replication/er_load.lua
new file mode 100644
index 000000000..071fc7b02
--- /dev/null
+++ b/test/replication/er_load.lua
@@ -0,0 +1,25 @@
+#!/usr/bin/env tarantool
+
+-- get instance id from filename (er_load1.lua => 1)
+local INSTANCE_ID = string.match(arg[0], '%d')
+
+local SOCKET_DIR =  require('fio').cwd()
+local function instance_uri(instance_id)
+    return SOCKET_DIR..'/er_load'..instance_id..'.sock'
+end
+
+require('console').listen(os.getenv('ADMIN'))
+
+box.cfg{
+    listen = instance_uri(INSTANCE_ID);
+    replication = {
+        instance_uri(INSTANCE_ID),
+        instance_uri(INSTANCE_ID % 2 + 1)
+    },
+    replication_timeout = 0.01,
+    read_only = INSTANCE_ID == '2'
+}
+box.once('bootstrap', function()
+    box.schema.user.grant('guest', 'replication')
+    box.space._cluster:delete(2)
+end)
diff --git a/test/replication/er_load1.lua b/test/replication/er_load1.lua
new file mode 120000
index 000000000..18f7ffa5a
--- /dev/null
+++ b/test/replication/er_load1.lua
@@ -0,0 +1 @@
+er_load.lua
\ No newline at end of file
diff --git a/test/replication/er_load2.lua b/test/replication/er_load2.lua
new file mode 120000
index 000000000..18f7ffa5a
--- /dev/null
+++ b/test/replication/er_load2.lua
@@ -0,0 +1 @@
+er_load.lua
\ No newline at end of file
diff --git a/test/replication/misc.result b/test/replication/misc.result
index 9df2a2c4b..16b0fc362 100644
--- a/test/replication/misc.result
+++ b/test/replication/misc.result
@@ -232,3 +232,35 @@ test_run:drop_cluster(SERVERS)
 box.schema.user.revoke('guest', 'replication')
 ---
 ...
+-- gh-3510 assertion failure in replica_on_applier_disconnect()
+test_run:cmd('create server er_load1 with script="replication/er_load1.lua"')
+---
+- true
+...
+test_run:cmd('create server er_load2 with script="replication/er_load2.lua"')
+---
+- true
+...
+test_run:cmd('start server er_load1 with wait=False, wait_load=False')
+---
+- true
+...
+-- instance er_load2 will fail with error ER_READONLY. this is ok.
+-- We only test here that er_load1 doesn't assert.
+test_run:cmd('start server er_load2 with wait=True, wait_load=True, crash_expected = True')
+---
+- false
+...
+test_run:cmd('stop server er_load1')
+---
+- true
+...
+-- er_load2 exits automatically.
+test_run:cmd('cleanup server er_load1')
+---
+- true
+...
+test_run:cmd('cleanup server er_load2')
+---
+- true
+...
diff --git a/test/replication/misc.test.lua b/test/replication/misc.test.lua
index 979c5d58c..852d7374b 100644
--- a/test/replication/misc.test.lua
+++ b/test/replication/misc.test.lua
@@ -91,3 +91,15 @@ test_run:cmd("switch default")
 test_run:drop_cluster(SERVERS)
 
 box.schema.user.revoke('guest', 'replication')
+
+-- gh-3510 assertion failure in replica_on_applier_disconnect()
+test_run:cmd('create server er_load1 with script="replication/er_load1.lua"')
+test_run:cmd('create server er_load2 with script="replication/er_load2.lua"')
+test_run:cmd('start server er_load1 with wait=False, wait_load=False')
+-- instance er_load2 will fail with error ER_READONLY. this is ok.
+-- We only test here that er_load1 doesn't assert.
+test_run:cmd('start server er_load2 with wait=True, wait_load=True, crash_expected = True')
+test_run:cmd('stop server er_load1')
+-- er_load2 exits automatically.
+test_run:cmd('cleanup server er_load1')
+test_run:cmd('cleanup server er_load2')
-- 
2.15.2 (Apple Git-101.1)