[PATCH] test: enable parallel mode for replication tests

[Sergei Voronezhskii <sergw@tarantool.org>]

- need more sleeps and timeout because increasing load on i/o
- at the end of tests which create any replication config need to call
`test_run:clenup_cluster()` which clears `box.space._cluster`
- switch on `use_unix_sockets` because of 'Address already in use'
problems
- instead of just checking `box.space.test:count()` or
`#fio.glob('./master/*.xlog')` need to wait for values because of
increading load in replication process

Part of #2436, #3232
---
BRANCH: https://github.com/tarantool/tarantool/tree/sergw/enable-parallel-test-replication
 test/replication/before_replace.result      |  9 ++-
 test/replication/before_replace.test.lua    |  7 ++-
 test/replication/catch.result               | 31 ++++++----
 test/replication/catch.test.lua             | 14 +++--
 test/replication/gc.result                  | 18 +++---
 test/replication/gc.test.lua                | 12 ++--
 test/replication/local_spaces.result        |  3 +
 test/replication/local_spaces.test.lua      |  1 +
 test/replication/misc.result                | 65 +++++++++++++++------
 test/replication/misc.test.lua              | 43 ++++++++------
 test/replication/on_replace.result          | 13 +++++
 test/replication/on_replace.test.lua        |  4 ++
 test/replication/once.result                |  2 +-
 test/replication/once.test.lua              |  2 +-
 test/replication/quorum.result              |  9 ++-
 test/replication/quorum.test.lua            |  7 ++-
 test/replication/replica_rejoin.result      |  7 +++
 test/replication/replica_rejoin.test.lua    |  2 +
 test/replication/skip_conflict_row.result   |  7 +++
 test/replication/skip_conflict_row.test.lua |  2 +
 test/replication/status.result              |  7 +++
 test/replication/status.test.lua            |  2 +
 test/replication/suite.ini                  |  3 +-
 test/replication/sync.result                |  7 +++
 test/replication/sync.test.lua              |  2 +
 test/replication/wal_off.result             |  7 +++
 test/replication/wal_off.test.lua           |  2 +
 27 files changed, 209 insertions(+), 79 deletions(-)

diff --git a/test/replication/before_replace.result b/test/replication/before_replace.result
index 858a52de6..a17afb4a3 100644
--- a/test/replication/before_replace.result
+++ b/test/replication/before_replace.result
@@ -76,7 +76,7 @@ test_run:cmd("switch autobootstrap1")
 ---
 - true
 ...
-box.error.injection.set('ERRINJ_RELAY_TIMEOUT', 0.01)
+box.error.injection.set('ERRINJ_RELAY_TIMEOUT', 0.1)
 ---
 - ok
 ...
@@ -87,7 +87,7 @@ test_run:cmd("switch autobootstrap2")
 ---
 - true
 ...
-box.error.injection.set('ERRINJ_RELAY_TIMEOUT', 0.01)
+box.error.injection.set('ERRINJ_RELAY_TIMEOUT', 0.1)
 ---
 - ok
 ...
@@ -98,7 +98,7 @@ test_run:cmd("switch autobootstrap3")
 ---
 - true
 ...
-box.error.injection.set('ERRINJ_RELAY_TIMEOUT', 0.01)
+box.error.injection.set('ERRINJ_RELAY_TIMEOUT', 0.1)
 ---
 - ok
 ...
@@ -223,3 +223,6 @@ test_run:cmd("switch default")
 test_run:drop_cluster(SERVERS)
 ---
 ...
+test_run:cleanup_cluster()
+---
+...
diff --git a/test/replication/before_replace.test.lua b/test/replication/before_replace.test.lua
index f1e590703..d86f74b05 100644
--- a/test/replication/before_replace.test.lua
+++ b/test/replication/before_replace.test.lua
@@ -46,13 +46,13 @@ test_run:cmd("setopt delimiter ''");
 -- Stall replication and generate incompatible data
 -- on the replicas.
 test_run:cmd("switch autobootstrap1")
-box.error.injection.set('ERRINJ_RELAY_TIMEOUT', 0.01)
+box.error.injection.set('ERRINJ_RELAY_TIMEOUT', 0.1)
 for i = 1, 10 do box.space.test:replace{i, i % 3 == 1 and i * 10 or i} end
 test_run:cmd("switch autobootstrap2")
-box.error.injection.set('ERRINJ_RELAY_TIMEOUT', 0.01)
+box.error.injection.set('ERRINJ_RELAY_TIMEOUT', 0.1)
 for i = 1, 10 do box.space.test:replace{i, i % 3 == 2 and i * 10 or i} end
 test_run:cmd("switch autobootstrap3")
-box.error.injection.set('ERRINJ_RELAY_TIMEOUT', 0.01)
+box.error.injection.set('ERRINJ_RELAY_TIMEOUT', 0.1)
 for i = 1, 10 do box.space.test:replace{i, i % 3 == 0 and i * 10 or i} end
 
 -- Synchronize.
@@ -80,3 +80,4 @@ box.space.test:select()
 -- Cleanup.
 test_run:cmd("switch default")
 test_run:drop_cluster(SERVERS)
+test_run:cleanup_cluster()
diff --git a/test/replication/catch.result b/test/replication/catch.result
index aebba819f..c0b9bfb19 100644
--- a/test/replication/catch.result
+++ b/test/replication/catch.result
@@ -1,6 +1,9 @@
 env = require('test_run')
 ---
 ...
+fiber = require('fiber')
+---
+...
 test_run = env.new()
 ---
 ...
@@ -58,11 +61,11 @@ test_run:cmd("stop server replica")
 - true
 ...
 -- insert values on the master while replica is stopped and can't fetch them
-for i=1,100 do s:insert{i, 'this is test message12345'} end
+for i=1,1000 do s:insert{i, 'this is test message12345'} end
 ---
 ...
 -- sleep after every tuple
-errinj.set("ERRINJ_RELAY_TIMEOUT", 1000.0)
+errinj.set("ERRINJ_RELAY_TIMEOUT", 2.0)
 ---
 - ok
 ...
@@ -74,6 +77,10 @@ test_run:cmd("switch replica")
 ---
 - true
 ...
+-- After stop server got error variable 'fiber' is not declared
+fiber = require('fiber')
+---
+...
 -- Check that replica doesn't enter read-write mode before
 -- catching up with the master: to check that we inject sleep into
 -- the master relay_send function and attempt a data modifying
@@ -88,13 +95,12 @@ box.space.test ~= nil
 ---
 - true
 ...
-d = box.space.test:delete{1}
+repeat fiber.sleep(0.001) until box.space.test:get(1) ~= nil
 ---
-- error: Can't modify data because this instance is in read-only mode.
 ...
-box.space.test:get(1) ~= nil
+d = box.space.test:delete{1}
 ---
-- true
+- error: Can't modify data because this instance is in read-only mode.
 ...
 -- case #2: delete tuple by net.box
 test_run:cmd("switch default")
@@ -108,19 +114,21 @@ test_run:cmd("set variable r_uri to 'replica.listen'")
 c = net_box.connect(r_uri)
 ---
 ...
-d = c.space.test:delete{1}
+repeat fiber.sleep(0.001) until c.space.test:get(1) ~= nil
 ---
-- error: Can't modify data because this instance is in read-only mode.
 ...
-c.space.test:get(1) ~= nil
+d = c.space.test:delete{1}
 ---
-- true
+- error: Can't modify data because this instance is in read-only mode.
 ...
 -- check sync
 errinj.set("ERRINJ_RELAY_TIMEOUT", 0)
 ---
 - ok
 ...
+fiber.sleep(2.0) -- wait until release errinj sleep
+---
+...
 -- cleanup
 test_run:cmd("stop server replica")
 ---
@@ -130,6 +138,9 @@ test_run:cmd("cleanup server replica")
 ---
 - true
 ...
+test_run:cleanup_cluster()
+---
+...
 box.space.test:drop()
 ---
 ...
diff --git a/test/replication/catch.test.lua b/test/replication/catch.test.lua
index 8cc3242f7..68cca831e 100644
--- a/test/replication/catch.test.lua
+++ b/test/replication/catch.test.lua
@@ -1,4 +1,5 @@
 env = require('test_run')
+fiber = require('fiber')
 test_run = env.new()
 engine = test_run:get_cfg('engine')
 
@@ -23,13 +24,15 @@ test_run:cmd("switch default")
 test_run:cmd("stop server replica")
 
 -- insert values on the master while replica is stopped and can't fetch them
-for i=1,100 do s:insert{i, 'this is test message12345'} end
+for i=1,1000 do s:insert{i, 'this is test message12345'} end
 
 -- sleep after every tuple
-errinj.set("ERRINJ_RELAY_TIMEOUT", 1000.0)
+errinj.set("ERRINJ_RELAY_TIMEOUT", 2.0)
 
 test_run:cmd("start server replica with args='0.01'")
 test_run:cmd("switch replica")
+-- After stop server got error variable 'fiber' is not declared
+fiber = require('fiber')
 
 -- Check that replica doesn't enter read-write mode before
 -- catching up with the master: to check that we inject sleep into
@@ -42,23 +45,26 @@ test_run:cmd("switch replica")
 -- #1: delete tuple on replica
 --
 box.space.test ~= nil
+repeat fiber.sleep(0.001) until box.space.test:get(1) ~= nil
 d = box.space.test:delete{1}
-box.space.test:get(1) ~= nil
 
 -- case #2: delete tuple by net.box
 
 test_run:cmd("switch default")
 test_run:cmd("set variable r_uri to 'replica.listen'")
 c = net_box.connect(r_uri)
+repeat fiber.sleep(0.001) until c.space.test:get(1) ~= nil
 d = c.space.test:delete{1}
-c.space.test:get(1) ~= nil
 
 -- check sync
 errinj.set("ERRINJ_RELAY_TIMEOUT", 0)
+fiber.sleep(2.0) -- wait until release errinj sleep
+
 
 -- cleanup
 test_run:cmd("stop server replica")
 test_run:cmd("cleanup server replica")
+test_run:cleanup_cluster()
 box.space.test:drop()
 box.schema.user.revoke('guest', 'replication')
 
diff --git a/test/replication/gc.result b/test/replication/gc.result
index 83d0de293..2639c4cf2 100644
--- a/test/replication/gc.result
+++ b/test/replication/gc.result
@@ -1,6 +1,3 @@
-fio = require 'fio'
----
-...
 test_run = require('test_run').new()
 ---
 ...
@@ -13,6 +10,9 @@ replica_set = require('fast_replica')
 fiber = require('fiber')
 ---
 ...
+fio = require('fio')
+---
+...
 test_run:cleanup_cluster()
 ---
 ...
@@ -95,7 +95,7 @@ test_run:cmd("switch replica")
 fiber = require('fiber')
 ---
 ...
-while box.space.test:count() < 200 do fiber.sleep(0.01) end
+while box.space.test == nil or box.space.test:count() < 200 do fiber.sleep(0.01) end
 ---
 ...
 box.space.test:count()
@@ -121,7 +121,9 @@ wait_gc(1)
 ...
 -- Make sure the replica will receive data it is subscribed
 -- to long enough for us to invoke garbage collection.
-box.error.injection.set("ERRINJ_RELAY_TIMEOUT", 0.05)
+-- On parallel test_run mode we need here so big value `1.0`
+-- because replica received data faster than xlog was created
+box.error.injection.set("ERRINJ_RELAY_TIMEOUT", 1.0)
 ---
 - ok
 ...
@@ -153,9 +155,8 @@ box.snapshot()
 ---
 - true
 ...
-#fio.glob('./master/*.xlog') == 2 or fio.listdir('./master')
+while #fio.glob('./master/*.xlog') ~= 2 do fiber.sleep(0.01) end
 ---
-- true
 ...
 -- Remove the timeout injection so that the replica catches
 -- up quickly.
@@ -188,9 +189,8 @@ wait_gc(1)
 ---
 - true
 ...
-#fio.glob('./master/*.xlog') == 0 or fio.listdir('./master')
+while #fio.glob('./master/*.xlog') ~= 0 do fiber.sleep(0.01) end
 ---
-- true
 ...
 --
 -- Check that the master doesn't delete xlog files sent to the
diff --git a/test/replication/gc.test.lua b/test/replication/gc.test.lua
index eed76850c..74151a243 100644
--- a/test/replication/gc.test.lua
+++ b/test/replication/gc.test.lua
@@ -1,8 +1,8 @@
-fio = require 'fio'
 test_run = require('test_run').new()
 engine = test_run:get_cfg('engine')
 replica_set = require('fast_replica')
 fiber = require('fiber')
+fio = require('fio')
 
 test_run:cleanup_cluster()
 
@@ -52,7 +52,7 @@ test_run:cmd("start server replica")
 -- data from the master. Check it.
 test_run:cmd("switch replica")
 fiber = require('fiber')
-while box.space.test:count() < 200 do fiber.sleep(0.01) end
+while box.space.test == nil or box.space.test:count() < 200 do fiber.sleep(0.01) end
 box.space.test:count()
 test_run:cmd("switch default")
 
@@ -63,7 +63,9 @@ wait_gc(1)
 #fio.glob('./master/*.xlog') == 1 or fio.listdir('./master')
 -- Make sure the replica will receive data it is subscribed
 -- to long enough for us to invoke garbage collection.
-box.error.injection.set("ERRINJ_RELAY_TIMEOUT", 0.05)
+-- On parallel test_run mode we need here so big value `1.0`
+-- because replica received data faster than xlog was created
+box.error.injection.set("ERRINJ_RELAY_TIMEOUT", 1.0)
 
 -- Send more data to the replica.
 -- Need to do 2 snapshots here, otherwise the replica would
@@ -78,7 +80,7 @@ box.snapshot()
 -- xlogs needed by the replica.
 box.snapshot()
 #box.info.gc().checkpoints == 1 or box.info.gc()
-#fio.glob('./master/*.xlog') == 2 or fio.listdir('./master')
+while #fio.glob('./master/*.xlog') ~= 2 do fiber.sleep(0.01) end
 
 -- Remove the timeout injection so that the replica catches
 -- up quickly.
@@ -94,7 +96,7 @@ test_run:cmd("switch default")
 -- from the old checkpoint.
 wait_gc(1)
 #box.info.gc().checkpoints == 1 or box.info.gc()
-#fio.glob('./master/*.xlog') == 0 or fio.listdir('./master')
+while #fio.glob('./master/*.xlog') ~= 0 do fiber.sleep(0.01) end
 --
 -- Check that the master doesn't delete xlog files sent to the
 -- replica until it receives a confirmation that the data has
diff --git a/test/replication/local_spaces.result b/test/replication/local_spaces.result
index 151735530..4de223261 100644
--- a/test/replication/local_spaces.result
+++ b/test/replication/local_spaces.result
@@ -216,6 +216,9 @@ test_run:cmd("cleanup server replica")
 ---
 - true
 ...
+test_run:cleanup_cluster()
+---
+...
 box.schema.user.revoke('guest', 'replication')
 ---
 ...
diff --git a/test/replication/local_spaces.test.lua b/test/replication/local_spaces.test.lua
index 06e2b0bd2..633cc9f1a 100644
--- a/test/replication/local_spaces.test.lua
+++ b/test/replication/local_spaces.test.lua
@@ -76,6 +76,7 @@ box.space.test3:select()
 test_run:cmd("switch default")
 test_run:cmd("stop server replica")
 test_run:cmd("cleanup server replica")
+test_run:cleanup_cluster()
 box.schema.user.revoke('guest', 'replication')
 
 s1:select()
diff --git a/test/replication/misc.result b/test/replication/misc.result
index 0ac48ba34..31ac26e48 100644
--- a/test/replication/misc.result
+++ b/test/replication/misc.result
@@ -88,6 +88,13 @@ test_run:cmd('cleanup server test')
 box.cfg{read_only = false}
 ---
 ...
+test_run:cmd('delete server test')
+---
+- true
+...
+test_run:cleanup_cluster()
+---
+...
 -- gh-3160 - Send heartbeats if there are changes from a remote master only
 SERVERS = { 'autobootstrap1', 'autobootstrap2', 'autobootstrap3' }
 ---
@@ -106,7 +113,7 @@ test_run:cmd("switch autobootstrap1")
 test_run = require('test_run').new()
 ---
 ...
-box.cfg{replication_timeout = 0.01, replication_connect_timeout=0.01}
+box.cfg{replication_timeout = 0.2, replication_connect_timeout=0.2}
 ---
 ...
 test_run:cmd("switch autobootstrap2")
@@ -116,7 +123,7 @@ test_run:cmd("switch autobootstrap2")
 test_run = require('test_run').new()
 ---
 ...
-box.cfg{replication_timeout = 0.01, replication_connect_timeout=0.01}
+box.cfg{replication_timeout = 0.2, replication_connect_timeout=0.2}
 ---
 ...
 test_run:cmd("switch autobootstrap3")
@@ -129,7 +136,7 @@ test_run = require('test_run').new()
 fiber=require('fiber')
 ---
 ...
-box.cfg{replication_timeout = 0.01, replication_connect_timeout=0.01}
+box.cfg{replication_timeout = 0.2, replication_connect_timeout=0.2}
 ---
 ...
 _ = box.schema.space.create('test_timeout'):create_index('pk')
@@ -140,15 +147,16 @@ test_run:cmd("setopt delimiter ';'")
 - true
 ...
 function test_timeout()
+    local replicaA = box.info.replication[1].upstream or box.info.replication[2].upstream
+    local replicaB = box.info.replication[3].upstream or box.info.replication[2].upstream
     for i = 0, 99 do 
         box.space.test_timeout:replace({1})
-        fiber.sleep(0.005)
-        local rinfo = box.info.replication
-        if rinfo[1].upstream and rinfo[1].upstream.status ~= 'follow' or
-           rinfo[2].upstream and rinfo[2].upstream.status ~= 'follow' or
-           rinfo[3].upstream and rinfo[3].upstream.status ~= 'follow' then
-            return error('Replication broken')
-        end
+        local n = 200
+        repeat
+            fiber.sleep(0.001)
+            n = n - 1
+            if n == 0 then return error(box.info.replication) end
+        until replicaA.status == 'follow' and replicaB.status == 'follow'
     end
     return true
 end ;
@@ -229,6 +237,9 @@ test_run:cmd("switch default")
 test_run:drop_cluster(SERVERS)
 ---
 ...
+test_run:cleanup_cluster()
+---
+...
 -- gh-3642 - Check that socket file descriptor doesn't leak
 -- when a replica is disconnected.
 rlimit = require('rlimit')
@@ -249,15 +260,15 @@ lim.rlim_cur = 64
 rlimit.setrlimit(rlimit.RLIMIT_NOFILE, lim)
 ---
 ...
-test_run:cmd('create server sock with rpl_master=default, script="replication/replica.lua"')
+test_run:cmd('create server bork with rpl_master=default, script="replication/replica.lua"')
 ---
 - true
 ...
-test_run:cmd(string.format('start server sock'))
+test_run:cmd('start server bork')
 ---
 - true
 ...
-test_run:cmd('switch sock')
+test_run:cmd('switch bork')
 ---
 - true
 ...
@@ -299,14 +310,21 @@ lim.rlim_cur = old_fno
 rlimit.setrlimit(rlimit.RLIMIT_NOFILE, lim)
 ---
 ...
-test_run:cmd('stop server sock')
+test_run:cmd("stop server bork")
+---
+- true
+...
+test_run:cmd("cleanup server bork")
 ---
 - true
 ...
-test_run:cmd('cleanup server sock')
+test_run:cmd("delete server bork")
 ---
 - true
 ...
+test_run:cleanup_cluster()
+---
+...
 box.schema.user.revoke('guest', 'replication')
 ---
 ...
@@ -342,6 +360,17 @@ test_run:cmd('cleanup server er_load2')
 ---
 - true
 ...
+test_run:cmd('delete server er_load1')
+---
+- true
+...
+test_run:cmd('delete server er_load2')
+---
+- true
+...
+test_run:cleanup_cluster()
+---
+...
 --
 -- Test case for gh-3637. Before the fix replica would exit with
 -- an error. Now check that we don't hang and successfully connect.
@@ -349,9 +378,6 @@ test_run:cmd('cleanup server er_load2')
 fiber = require('fiber')
 ---
 ...
-test_run:cleanup_cluster()
----
-...
 test_run:cmd("create server replica_auth with rpl_master=default, script='replication/replica_auth.lua'")
 ---
 - true
@@ -391,6 +417,9 @@ test_run:cmd("delete server replica_auth")
 ---
 - true
 ...
+test_run:cleanup_cluster()
+---
+...
 box.schema.user.drop('cluster')
 ---
 ...
diff --git a/test/replication/misc.test.lua b/test/replication/misc.test.lua
index 56e1bab69..21fa1889c 100644
--- a/test/replication/misc.test.lua
+++ b/test/replication/misc.test.lua
@@ -32,6 +32,8 @@ test_run:cmd(string.format('start server test with args="%s"', replica_uuid))
 test_run:cmd('stop server test')
 test_run:cmd('cleanup server test')
 box.cfg{read_only = false}
+test_run:cmd('delete server test')
+test_run:cleanup_cluster()
 
 -- gh-3160 - Send heartbeats if there are changes from a remote master only
 SERVERS = { 'autobootstrap1', 'autobootstrap2', 'autobootstrap3' }
@@ -41,26 +43,27 @@ test_run:create_cluster(SERVERS, "replication", {args="0.1"})
 test_run:wait_fullmesh(SERVERS)
 test_run:cmd("switch autobootstrap1")
 test_run = require('test_run').new()
-box.cfg{replication_timeout = 0.01, replication_connect_timeout=0.01}
+box.cfg{replication_timeout = 0.2, replication_connect_timeout=0.2}
 test_run:cmd("switch autobootstrap2")
 test_run = require('test_run').new()
-box.cfg{replication_timeout = 0.01, replication_connect_timeout=0.01}
+box.cfg{replication_timeout = 0.2, replication_connect_timeout=0.2}
 test_run:cmd("switch autobootstrap3")
 test_run = require('test_run').new()
 fiber=require('fiber')
-box.cfg{replication_timeout = 0.01, replication_connect_timeout=0.01}
+box.cfg{replication_timeout = 0.2, replication_connect_timeout=0.2}
 _ = box.schema.space.create('test_timeout'):create_index('pk')
 test_run:cmd("setopt delimiter ';'")
 function test_timeout()
+    local replicaA = box.info.replication[1].upstream or box.info.replication[2].upstream
+    local replicaB = box.info.replication[3].upstream or box.info.replication[2].upstream
     for i = 0, 99 do 
         box.space.test_timeout:replace({1})
-        fiber.sleep(0.005)
-        local rinfo = box.info.replication
-        if rinfo[1].upstream and rinfo[1].upstream.status ~= 'follow' or
-           rinfo[2].upstream and rinfo[2].upstream.status ~= 'follow' or
-           rinfo[3].upstream and rinfo[3].upstream.status ~= 'follow' then
-            return error('Replication broken')
-        end
+        local n = 200
+        repeat
+            fiber.sleep(0.001)
+            n = n - 1
+            if n == 0 then return error(box.info.replication) end
+        until replicaA.status == 'follow' and replicaB.status == 'follow'
     end
     return true
 end ;
@@ -89,6 +92,7 @@ box.space.space1:drop()
 
 test_run:cmd("switch default")
 test_run:drop_cluster(SERVERS)
+test_run:cleanup_cluster()
 
 -- gh-3642 - Check that socket file descriptor doesn't leak
 -- when a replica is disconnected.
@@ -99,9 +103,9 @@ old_fno = lim.rlim_cur
 lim.rlim_cur = 64
 rlimit.setrlimit(rlimit.RLIMIT_NOFILE, lim)
 
-test_run:cmd('create server sock with rpl_master=default, script="replication/replica.lua"')
-test_run:cmd(string.format('start server sock'))
-test_run:cmd('switch sock')
+test_run:cmd('create server bork with rpl_master=default, script="replication/replica.lua"')
+test_run:cmd('start server bork')
+test_run:cmd('switch bork')
 test_run = require('test_run').new()
 fiber = require('fiber')
 test_run:cmd("setopt delimiter ';'")
@@ -122,8 +126,10 @@ test_run:cmd('switch default')
 lim.rlim_cur = old_fno
 rlimit.setrlimit(rlimit.RLIMIT_NOFILE, lim)
 
-test_run:cmd('stop server sock')
-test_run:cmd('cleanup server sock')
+test_run:cmd("stop server bork")
+test_run:cmd("cleanup server bork")
+test_run:cmd("delete server bork")
+test_run:cleanup_cluster()
 
 box.schema.user.revoke('guest', 'replication')
 
@@ -138,15 +144,15 @@ test_run:cmd('stop server er_load1')
 -- er_load2 exits automatically.
 test_run:cmd('cleanup server er_load1')
 test_run:cmd('cleanup server er_load2')
+test_run:cmd('delete server er_load1')
+test_run:cmd('delete server er_load2')
+test_run:cleanup_cluster()
 
 --
 -- Test case for gh-3637. Before the fix replica would exit with
 -- an error. Now check that we don't hang and successfully connect.
 --
 fiber = require('fiber')
-
-test_run:cleanup_cluster()
-
 test_run:cmd("create server replica_auth with rpl_master=default, script='replication/replica_auth.lua'")
 test_run:cmd("start server replica_auth with wait=False, wait_load=False, args='cluster:pass 0.05'")
 -- Wait a bit to make sure replica waits till user is created.
@@ -161,5 +167,6 @@ _ = test_run:wait_vclock('replica_auth', vclock)
 test_run:cmd("stop server replica_auth")
 test_run:cmd("cleanup server replica_auth")
 test_run:cmd("delete server replica_auth")
+test_run:cleanup_cluster()
 
 box.schema.user.drop('cluster')
diff --git a/test/replication/on_replace.result b/test/replication/on_replace.result
index 4ffa3b25a..2e95b90ea 100644
--- a/test/replication/on_replace.result
+++ b/test/replication/on_replace.result
@@ -63,6 +63,9 @@ test_run:cmd("switch replica")
 ---
 - true
 ...
+fiber = require('fiber')
+---
+...
 while box.space.test:count() < 2 do fiber.sleep(0.01) end
 ---
 ...
@@ -88,6 +91,13 @@ test_run:cmd("cleanup server replica")
 ---
 - true
 ...
+test_run:cmd("delete server replica")
+---
+- true
+...
+test_run:cleanup_cluster()
+---
+...
 box.space.test:drop()
 ---
 ...
@@ -177,3 +187,6 @@ _ = test_run:cmd('switch default')
 test_run:drop_cluster(SERVERS)
 ---
 ...
+test_run:cleanup_cluster()
+---
+...
diff --git a/test/replication/on_replace.test.lua b/test/replication/on_replace.test.lua
index 371b71cbd..e34832103 100644
--- a/test/replication/on_replace.test.lua
+++ b/test/replication/on_replace.test.lua
@@ -26,6 +26,7 @@ session_type
 test_run:cmd("switch default")
 box.space.test:insert{2}
 test_run:cmd("switch replica")
+fiber = require('fiber')
 while box.space.test:count() < 2 do fiber.sleep(0.01) end
 --
 -- applier
@@ -37,6 +38,8 @@ test_run:cmd("switch default")
 --
 test_run:cmd("stop server replica")
 test_run:cmd("cleanup server replica")
+test_run:cmd("delete server replica")
+test_run:cleanup_cluster()
 box.space.test:drop()
 box.schema.user.revoke('guest', 'replication')
 
@@ -73,3 +76,4 @@ box.space.s2:select()
 
 _ = test_run:cmd('switch default')
 test_run:drop_cluster(SERVERS)
+test_run:cleanup_cluster()
diff --git a/test/replication/once.result b/test/replication/once.result
index 99ac05b72..e6cda0f92 100644
--- a/test/replication/once.result
+++ b/test/replication/once.result
@@ -54,7 +54,7 @@ ch = fiber.channel(1)
 _ = fiber.create(function() box.once("ro", f, 1) ch:put(true) end)
 ---
 ...
-fiber.sleep(0.001)
+fiber.sleep(0.01)
 ---
 ...
 once -- nil
diff --git a/test/replication/once.test.lua b/test/replication/once.test.lua
index 264c63670..275776abc 100644
--- a/test/replication/once.test.lua
+++ b/test/replication/once.test.lua
@@ -19,7 +19,7 @@ once = nil
 box.cfg{read_only = true}
 ch = fiber.channel(1)
 _ = fiber.create(function() box.once("ro", f, 1) ch:put(true) end)
-fiber.sleep(0.001)
+fiber.sleep(0.01)
 once -- nil
 box.cfg{read_only = false}
 ch:get()
diff --git a/test/replication/quorum.result b/test/replication/quorum.result
index 265b099b7..2642fe8f4 100644
--- a/test/replication/quorum.result
+++ b/test/replication/quorum.result
@@ -435,18 +435,21 @@ test_run:cmd('switch default')
 ---
 - true
 ...
-test_run:cmd('stop server replica_quorum')
+test_run:cmd("stop server replica_quorum")
 ---
 - true
 ...
-test_run:cmd('cleanup server replica_quorum')
+test_run:cmd("cleanup server replica_quorum")
 ---
 - true
 ...
-test_run:cmd('delete server replica_quorum')
+test_run:cmd("delete server replica_quorum")
 ---
 - true
 ...
+test_run:cleanup_cluster()
+---
+...
 box.schema.user.revoke('guest', 'replication')
 ---
 ...
diff --git a/test/replication/quorum.test.lua b/test/replication/quorum.test.lua
index 5a43275c2..24d1b27c4 100644
--- a/test/replication/quorum.test.lua
+++ b/test/replication/quorum.test.lua
@@ -166,7 +166,8 @@ test_run:cmd('switch replica_quorum')
 box.cfg{replication={INSTANCE_URI, nonexistent_uri(1)}}
 box.info.id
 test_run:cmd('switch default')
-test_run:cmd('stop server replica_quorum')
-test_run:cmd('cleanup server replica_quorum')
-test_run:cmd('delete server replica_quorum')
+test_run:cmd("stop server replica_quorum")
+test_run:cmd("cleanup server replica_quorum")
+test_run:cmd("delete server replica_quorum")
+test_run:cleanup_cluster()
 box.schema.user.revoke('guest', 'replication')
diff --git a/test/replication/replica_rejoin.result b/test/replication/replica_rejoin.result
index 4370fae4b..37849850f 100644
--- a/test/replication/replica_rejoin.result
+++ b/test/replication/replica_rejoin.result
@@ -242,6 +242,13 @@ test_run:cmd("cleanup server replica")
 ---
 - true
 ...
+test_run:cmd("delete server replica")
+---
+- true
+...
+test_run:cleanup_cluster()
+---
+...
 box.space.test:drop()
 ---
 ...
diff --git a/test/replication/replica_rejoin.test.lua b/test/replication/replica_rejoin.test.lua
index f998f60d0..950ec7532 100644
--- a/test/replication/replica_rejoin.test.lua
+++ b/test/replication/replica_rejoin.test.lua
@@ -87,5 +87,7 @@ box.space.test:select()
 test_run:cmd("switch default")
 test_run:cmd("stop server replica")
 test_run:cmd("cleanup server replica")
+test_run:cmd("delete server replica")
+test_run:cleanup_cluster()
 box.space.test:drop()
 box.schema.user.revoke('guest', 'replication')
diff --git a/test/replication/skip_conflict_row.result b/test/replication/skip_conflict_row.result
index 29963f56a..6ca13b472 100644
--- a/test/replication/skip_conflict_row.result
+++ b/test/replication/skip_conflict_row.result
@@ -91,6 +91,13 @@ test_run:cmd("cleanup server replica")
 ---
 - true
 ...
+test_run:cmd("delete server replica")
+---
+- true
+...
+test_run:cleanup_cluster()
+---
+...
 box.space.test:drop()
 ---
 ...
diff --git a/test/replication/skip_conflict_row.test.lua b/test/replication/skip_conflict_row.test.lua
index 5f7d6ead3..4406ced95 100644
--- a/test/replication/skip_conflict_row.test.lua
+++ b/test/replication/skip_conflict_row.test.lua
@@ -31,5 +31,7 @@ box.info.status
 -- cleanup
 test_run:cmd("stop server replica")
 test_run:cmd("cleanup server replica")
+test_run:cmd("delete server replica")
+test_run:cleanup_cluster()
 box.space.test:drop()
 box.schema.user.revoke('guest', 'replication')
diff --git a/test/replication/status.result b/test/replication/status.result
index 8394b98c1..9e69f2478 100644
--- a/test/replication/status.result
+++ b/test/replication/status.result
@@ -391,3 +391,10 @@ test_run:cmd("cleanup server replica")
 ---
 - true
 ...
+test_run:cmd("delete server replica")
+---
+- true
+...
+test_run:cleanup_cluster()
+---
+...
diff --git a/test/replication/status.test.lua b/test/replication/status.test.lua
index 8bb25e0c6..cfdf6acdb 100644
--- a/test/replication/status.test.lua
+++ b/test/replication/status.test.lua
@@ -142,3 +142,5 @@ test_run:cmd('switch default')
 box.schema.user.revoke('guest', 'replication')
 test_run:cmd("stop server replica")
 test_run:cmd("cleanup server replica")
+test_run:cmd("delete server replica")
+test_run:cleanup_cluster()
diff --git a/test/replication/suite.ini b/test/replication/suite.ini
index f4abc7af1..5cbc371c2 100644
--- a/test/replication/suite.ini
+++ b/test/replication/suite.ini
@@ -6,5 +6,6 @@ disabled = consistent.test.lua
 release_disabled = catch.test.lua errinj.test.lua gc.test.lua before_replace.test.lua quorum.test.lua recover_missing_xlog.test.lua sync.test.lua
 config = suite.cfg
 lua_libs = lua/fast_replica.lua lua/rlimit.lua
+use_unix_sockets = True
 long_run = prune.test.lua
-is_parallel = False
+is_parallel = True
diff --git a/test/replication/sync.result b/test/replication/sync.result
index 81de60758..b2381ac59 100644
--- a/test/replication/sync.result
+++ b/test/replication/sync.result
@@ -303,6 +303,13 @@ test_run:cmd("cleanup server replica")
 ---
 - true
 ...
+test_run:cmd("delete server replica")
+---
+- true
+...
+test_run:cleanup_cluster()
+---
+...
 box.space.test:drop()
 ---
 ...
diff --git a/test/replication/sync.test.lua b/test/replication/sync.test.lua
index a5cfab8de..51131667d 100644
--- a/test/replication/sync.test.lua
+++ b/test/replication/sync.test.lua
@@ -145,6 +145,8 @@ test_run:grep_log('replica', 'ER_CFG.*')
 test_run:cmd("switch default")
 test_run:cmd("stop server replica")
 test_run:cmd("cleanup server replica")
+test_run:cmd("delete server replica")
+test_run:cleanup_cluster()
 
 box.space.test:drop()
 box.schema.user.revoke('guest', 'replication')
diff --git a/test/replication/wal_off.result b/test/replication/wal_off.result
index e3b5709e9..e0ae84bd7 100644
--- a/test/replication/wal_off.result
+++ b/test/replication/wal_off.result
@@ -107,6 +107,13 @@ test_run:cmd("cleanup server wal_off")
 ---
 - true
 ...
+test_run:cmd("delete server wal_off")
+---
+- true
+...
+test_run:cleanup_cluster()
+---
+...
 box.schema.user.revoke('guest', 'replication')
 ---
 ...
diff --git a/test/replication/wal_off.test.lua b/test/replication/wal_off.test.lua
index 81fcf0b33..110f2f1f7 100644
--- a/test/replication/wal_off.test.lua
+++ b/test/replication/wal_off.test.lua
@@ -37,5 +37,7 @@ box.cfg { replication = "" }
 
 test_run:cmd("stop server wal_off")
 test_run:cmd("cleanup server wal_off")
+test_run:cmd("delete server wal_off")
+test_run:cleanup_cluster()
 
 box.schema.user.revoke('guest', 'replication')
-- 
2.18.0