From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from smtp51.i.mail.ru (smtp51.i.mail.ru [94.100.177.111]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by dev.tarantool.org (Postfix) with ESMTPS id 23AC1445323 for ; Thu, 9 Jul 2020 20:17:05 +0300 (MSK) From: sergeyb@tarantool.org Date: Thu, 9 Jul 2020 20:16:46 +0300 Message-Id: In-Reply-To: MIME-Version: 1.0 Content-Transfer-Encoding: 8bit Subject: [Tarantool-patches] [PATCH 3/3] replication: add test with random leaders promotion and demotion List-Id: Tarantool development patches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , To: tarantool-patches@dev.tarantool.org, v.shpilevoy@tarantool.org, sergepetrenko@tarantool.org, gorcunov@gmail.com, lvasiliev@tarantool.org From: Sergey Bronnikov Part of #5055 --- test/replication/qsync.lua | 62 ++++++++++ test/replication/qsync1.lua | 1 + test/replication/qsync10.lua | 1 + test/replication/qsync11.lua | 1 + test/replication/qsync12.lua | 1 + test/replication/qsync13.lua | 1 + test/replication/qsync14.lua | 1 + test/replication/qsync15.lua | 1 + test/replication/qsync16.lua | 1 + test/replication/qsync17.lua | 1 + test/replication/qsync18.lua | 1 + test/replication/qsync19.lua | 1 + test/replication/qsync2.lua | 1 + test/replication/qsync20.lua | 1 + test/replication/qsync21.lua | 1 + test/replication/qsync22.lua | 1 + test/replication/qsync23.lua | 1 + test/replication/qsync24.lua | 1 + test/replication/qsync25.lua | 1 + test/replication/qsync26.lua | 1 + test/replication/qsync27.lua | 1 + test/replication/qsync28.lua | 1 + test/replication/qsync29.lua | 1 + test/replication/qsync3.lua | 1 + test/replication/qsync30.lua | 1 + test/replication/qsync31.lua | 1 + test/replication/qsync4.lua | 1 + test/replication/qsync5.lua | 1 + test/replication/qsync6.lua | 1 + test/replication/qsync7.lua | 1 + test/replication/qsync8.lua | 1 + test/replication/qsync9.lua | 1 + test/replication/qsync_random_leader.result | 117 ++++++++++++++++++ test/replication/qsync_random_leader.test.lua | 52 ++++++++ 34 files changed, 262 insertions(+) create mode 100644 test/replication/qsync.lua create mode 120000 test/replication/qsync1.lua create mode 120000 test/replication/qsync10.lua create mode 120000 test/replication/qsync11.lua create mode 120000 test/replication/qsync12.lua create mode 120000 test/replication/qsync13.lua create mode 120000 test/replication/qsync14.lua create mode 120000 test/replication/qsync15.lua create mode 120000 test/replication/qsync16.lua create mode 120000 test/replication/qsync17.lua create mode 120000 test/replication/qsync18.lua create mode 120000 test/replication/qsync19.lua create mode 120000 test/replication/qsync2.lua create mode 120000 test/replication/qsync20.lua create mode 120000 test/replication/qsync21.lua create mode 120000 test/replication/qsync22.lua create mode 120000 test/replication/qsync23.lua create mode 120000 test/replication/qsync24.lua create mode 120000 test/replication/qsync25.lua create mode 120000 test/replication/qsync26.lua create mode 120000 test/replication/qsync27.lua create mode 120000 test/replication/qsync28.lua create mode 120000 test/replication/qsync29.lua create mode 120000 test/replication/qsync3.lua create mode 120000 test/replication/qsync30.lua create mode 120000 test/replication/qsync31.lua create mode 120000 test/replication/qsync4.lua create mode 120000 test/replication/qsync5.lua create mode 120000 test/replication/qsync6.lua create mode 120000 test/replication/qsync7.lua create mode 120000 test/replication/qsync8.lua create mode 120000 test/replication/qsync9.lua create mode 100644 test/replication/qsync_random_leader.result create mode 100644 test/replication/qsync_random_leader.test.lua diff --git a/test/replication/qsync.lua b/test/replication/qsync.lua new file mode 100644 index 000000000..383aa5272 --- /dev/null +++ b/test/replication/qsync.lua @@ -0,0 +1,62 @@ +#!/usr/bin/env tarantool + +-- get instance name from filename (qsync1.lua => qsync1) +local INSTANCE_ID = string.match(arg[0], "%d") + +local SOCKET_DIR = require('fio').cwd() + +local TIMEOUT = tonumber(arg[1]) + +local function instance_uri(instance_id) + return SOCKET_DIR..'/qsync'..instance_id..'.sock'; +end + +-- start console first +require('console').listen(os.getenv('ADMIN')) + +box.cfg({ + listen = instance_uri(INSTANCE_ID); + replication_timeout = TIMEOUT; + replication_sync_lag = 0.01; + replication_connect_quorum = 3; + replication = { + instance_uri(1); + instance_uri(2); + instance_uri(3); + instance_uri(4); + instance_uri(5); + instance_uri(6); + instance_uri(7); + instance_uri(8); + instance_uri(9); + instance_uri(10); + instance_uri(11); + instance_uri(12); + instance_uri(13); + instance_uri(14); + instance_uri(15); + instance_uri(16); + instance_uri(17); + instance_uri(18); + instance_uri(19); + instance_uri(20); + instance_uri(21); + instance_uri(22); + instance_uri(23); + instance_uri(24); + instance_uri(25); + instance_uri(26); + instance_uri(27); + instance_uri(28); + instance_uri(29); + instance_uri(30); + instance_uri(31); + }; +}) + +box.once("bootstrap", function() + local test_run = require('test_run').new() + box.schema.user.grant("guest", 'replication') + box.schema.space.create('test', {engine = test_run:get_cfg('engine')}) + box.space.test:create_index('primary') +end) diff --git a/test/replication/qsync1.lua b/test/replication/qsync1.lua new file mode 120000 index 000000000..df9f3a883 --- /dev/null +++ b/test/replication/qsync1.lua @@ -0,0 +1 @@ +qsync.lua \ No newline at end of file diff --git a/test/replication/qsync10.lua b/test/replication/qsync10.lua new file mode 120000 index 000000000..df9f3a883 --- /dev/null +++ b/test/replication/qsync10.lua @@ -0,0 +1 @@ +qsync.lua \ No newline at end of file diff --git a/test/replication/qsync11.lua b/test/replication/qsync11.lua new file mode 120000 index 000000000..df9f3a883 --- /dev/null +++ b/test/replication/qsync11.lua @@ -0,0 +1 @@ +qsync.lua \ No newline at end of file diff --git a/test/replication/qsync12.lua b/test/replication/qsync12.lua new file mode 120000 index 000000000..df9f3a883 --- /dev/null +++ b/test/replication/qsync12.lua @@ -0,0 +1 @@ +qsync.lua \ No newline at end of file diff --git a/test/replication/qsync13.lua b/test/replication/qsync13.lua new file mode 120000 index 000000000..df9f3a883 --- /dev/null +++ b/test/replication/qsync13.lua @@ -0,0 +1 @@ +qsync.lua \ No newline at end of file diff --git a/test/replication/qsync14.lua b/test/replication/qsync14.lua new file mode 120000 index 000000000..df9f3a883 --- /dev/null +++ b/test/replication/qsync14.lua @@ -0,0 +1 @@ +qsync.lua \ No newline at end of file diff --git a/test/replication/qsync15.lua b/test/replication/qsync15.lua new file mode 120000 index 000000000..df9f3a883 --- /dev/null +++ b/test/replication/qsync15.lua @@ -0,0 +1 @@ +qsync.lua \ No newline at end of file diff --git a/test/replication/qsync16.lua b/test/replication/qsync16.lua new file mode 120000 index 000000000..df9f3a883 --- /dev/null +++ b/test/replication/qsync16.lua @@ -0,0 +1 @@ +qsync.lua \ No newline at end of file diff --git a/test/replication/qsync17.lua b/test/replication/qsync17.lua new file mode 120000 index 000000000..df9f3a883 --- /dev/null +++ b/test/replication/qsync17.lua @@ -0,0 +1 @@ +qsync.lua \ No newline at end of file diff --git a/test/replication/qsync18.lua b/test/replication/qsync18.lua new file mode 120000 index 000000000..df9f3a883 --- /dev/null +++ b/test/replication/qsync18.lua @@ -0,0 +1 @@ +qsync.lua \ No newline at end of file diff --git a/test/replication/qsync19.lua b/test/replication/qsync19.lua new file mode 120000 index 000000000..df9f3a883 --- /dev/null +++ b/test/replication/qsync19.lua @@ -0,0 +1 @@ +qsync.lua \ No newline at end of file diff --git a/test/replication/qsync2.lua b/test/replication/qsync2.lua new file mode 120000 index 000000000..df9f3a883 --- /dev/null +++ b/test/replication/qsync2.lua @@ -0,0 +1 @@ +qsync.lua \ No newline at end of file diff --git a/test/replication/qsync20.lua b/test/replication/qsync20.lua new file mode 120000 index 000000000..df9f3a883 --- /dev/null +++ b/test/replication/qsync20.lua @@ -0,0 +1 @@ +qsync.lua \ No newline at end of file diff --git a/test/replication/qsync21.lua b/test/replication/qsync21.lua new file mode 120000 index 000000000..df9f3a883 --- /dev/null +++ b/test/replication/qsync21.lua @@ -0,0 +1 @@ +qsync.lua \ No newline at end of file diff --git a/test/replication/qsync22.lua b/test/replication/qsync22.lua new file mode 120000 index 000000000..df9f3a883 --- /dev/null +++ b/test/replication/qsync22.lua @@ -0,0 +1 @@ +qsync.lua \ No newline at end of file diff --git a/test/replication/qsync23.lua b/test/replication/qsync23.lua new file mode 120000 index 000000000..df9f3a883 --- /dev/null +++ b/test/replication/qsync23.lua @@ -0,0 +1 @@ +qsync.lua \ No newline at end of file diff --git a/test/replication/qsync24.lua b/test/replication/qsync24.lua new file mode 120000 index 000000000..df9f3a883 --- /dev/null +++ b/test/replication/qsync24.lua @@ -0,0 +1 @@ +qsync.lua \ No newline at end of file diff --git a/test/replication/qsync25.lua b/test/replication/qsync25.lua new file mode 120000 index 000000000..df9f3a883 --- /dev/null +++ b/test/replication/qsync25.lua @@ -0,0 +1 @@ +qsync.lua \ No newline at end of file diff --git a/test/replication/qsync26.lua b/test/replication/qsync26.lua new file mode 120000 index 000000000..df9f3a883 --- /dev/null +++ b/test/replication/qsync26.lua @@ -0,0 +1 @@ +qsync.lua \ No newline at end of file diff --git a/test/replication/qsync27.lua b/test/replication/qsync27.lua new file mode 120000 index 000000000..df9f3a883 --- /dev/null +++ b/test/replication/qsync27.lua @@ -0,0 +1 @@ +qsync.lua \ No newline at end of file diff --git a/test/replication/qsync28.lua b/test/replication/qsync28.lua new file mode 120000 index 000000000..df9f3a883 --- /dev/null +++ b/test/replication/qsync28.lua @@ -0,0 +1 @@ +qsync.lua \ No newline at end of file diff --git a/test/replication/qsync29.lua b/test/replication/qsync29.lua new file mode 120000 index 000000000..df9f3a883 --- /dev/null +++ b/test/replication/qsync29.lua @@ -0,0 +1 @@ +qsync.lua \ No newline at end of file diff --git a/test/replication/qsync3.lua b/test/replication/qsync3.lua new file mode 120000 index 000000000..df9f3a883 --- /dev/null +++ b/test/replication/qsync3.lua @@ -0,0 +1 @@ +qsync.lua \ No newline at end of file diff --git a/test/replication/qsync30.lua b/test/replication/qsync30.lua new file mode 120000 index 000000000..df9f3a883 --- /dev/null +++ b/test/replication/qsync30.lua @@ -0,0 +1 @@ +qsync.lua \ No newline at end of file diff --git a/test/replication/qsync31.lua b/test/replication/qsync31.lua new file mode 120000 index 000000000..df9f3a883 --- /dev/null +++ b/test/replication/qsync31.lua @@ -0,0 +1 @@ +qsync.lua \ No newline at end of file diff --git a/test/replication/qsync4.lua b/test/replication/qsync4.lua new file mode 120000 index 000000000..df9f3a883 --- /dev/null +++ b/test/replication/qsync4.lua @@ -0,0 +1 @@ +qsync.lua \ No newline at end of file diff --git a/test/replication/qsync5.lua b/test/replication/qsync5.lua new file mode 120000 index 000000000..df9f3a883 --- /dev/null +++ b/test/replication/qsync5.lua @@ -0,0 +1 @@ +qsync.lua \ No newline at end of file diff --git a/test/replication/qsync6.lua b/test/replication/qsync6.lua new file mode 120000 index 000000000..df9f3a883 --- /dev/null +++ b/test/replication/qsync6.lua @@ -0,0 +1 @@ +qsync.lua \ No newline at end of file diff --git a/test/replication/qsync7.lua b/test/replication/qsync7.lua new file mode 120000 index 000000000..df9f3a883 --- /dev/null +++ b/test/replication/qsync7.lua @@ -0,0 +1 @@ +qsync.lua \ No newline at end of file diff --git a/test/replication/qsync8.lua b/test/replication/qsync8.lua new file mode 120000 index 000000000..df9f3a883 --- /dev/null +++ b/test/replication/qsync8.lua @@ -0,0 +1 @@ +qsync.lua \ No newline at end of file diff --git a/test/replication/qsync9.lua b/test/replication/qsync9.lua new file mode 120000 index 000000000..df9f3a883 --- /dev/null +++ b/test/replication/qsync9.lua @@ -0,0 +1 @@ +qsync.lua \ No newline at end of file diff --git a/test/replication/qsync_random_leader.result b/test/replication/qsync_random_leader.result new file mode 100644 index 000000000..5d0c5d18f --- /dev/null +++ b/test/replication/qsync_random_leader.result @@ -0,0 +1,117 @@ +-- test-run result file version 2 +env = require('test_run') + | --- + | ... +math = require('math') + | --- + | ... +fiber = require('fiber') + | --- + | ... +test_run = env.new() + | --- + | ... +engine = test_run:get_cfg('engine') + | --- + | ... + +MAX_REPLICAS = 3 + | --- + | ... +NUM_INSTANCES = MAX_REPLICAS + 1 + | --- + | ... +BROKEN_QUORUM = NUM_INSTANCES + 1 + | --- + | ... + +SERVERS = {} + | --- + | ... +test_run:cmd("setopt delimiter ';'") + | --- + | - true + | ... +for i=1,MAX_REPLICAS do + SERVERS[i] = 'qsync' .. i +end; + | --- + | ... +test_run:cmd("setopt delimiter ''"); + | --- + | - true + | ... +SERVERS -- print instance names + | --- + | - - qsync1 + | - qsync2 + | - qsync3 + | ... + +test_run:create_cluster(SERVERS, "replication", {args="0.1"}) + | --- + | ... +test_run:wait_fullmesh(SERVERS) + | --- + | ... +box.schema.user.grant('guest', 'replication') + | --- + | ... +test_run:switch('default') + | --- + | - true + | ... +box.cfg{replication_synchro_quorum=NUM_INSTANCES, replication_synchro_timeout=0.1} + | --- + | ... +_ = box.schema.space.create('sync', {is_sync=true, engine=engine}) + | --- + | ... +_ = box.space.sync:create_index('pk') + | --- + | ... + +-- Testcase body. +current_leader = 'default' + | --- + | ... +for i=1,100 do \ + new_leader_id = math.random(1, #SERVERS) \ + new_leader = SERVERS[new_leader_id] \ + test_run:switch(new_leader) \ + box.cfg{read_only=false} \ + fiber = require('fiber') \ + f = fiber.create(function() box.space.sync:delete{} end) \ + f = fiber.create(function() for i=1,10 do box.space.sync:insert{i} end end) \ + f.status() \ + test_run:switch('default') \ + test_run:switch(current_leader) \ + box.cfg{read_only=true} \ + test_run:switch('default') \ + current_leader = new_leader \ + fiber.sleep(0.1) \ +end + | --- + | ... + +-- Teardown. +test_run:switch(current_leader) + | --- + | - true + | ... +box.cfg{read_only=true} -- demote master to replica + | --- + | ... +test_run:switch('default') + | --- + | - true + | ... +box.cfg{read_only=false} -- promote replica to master + | --- + | ... +box.space.sync:drop() + | --- + | ... +test_run:drop_cluster(SERVERS) + | --- + | ... diff --git a/test/replication/qsync_random_leader.test.lua b/test/replication/qsync_random_leader.test.lua new file mode 100644 index 000000000..3bf534e23 --- /dev/null +++ b/test/replication/qsync_random_leader.test.lua @@ -0,0 +1,52 @@ +env = require('test_run') +math = require('math') +fiber = require('fiber') +test_run = env.new() +engine = test_run:get_cfg('engine') + +MAX_REPLICAS = 3 +NUM_INSTANCES = MAX_REPLICAS + 1 +BROKEN_QUORUM = NUM_INSTANCES + 1 + +SERVERS = {} +test_run:cmd("setopt delimiter ';'") +for i=1,MAX_REPLICAS do + SERVERS[i] = 'qsync' .. i +end; +test_run:cmd("setopt delimiter ''"); +SERVERS -- print instance names + +test_run:create_cluster(SERVERS, "replication", {args="0.1"}) +test_run:wait_fullmesh(SERVERS) +box.schema.user.grant('guest', 'replication') +test_run:switch('default') +box.cfg{replication_synchro_quorum=NUM_INSTANCES, replication_synchro_timeout=0.1} +_ = box.schema.space.create('sync', {is_sync=true, engine=engine}) +_ = box.space.sync:create_index('pk') + +-- Testcase body. +current_leader = 'default' +for i=1,100 do \ + new_leader_id = math.random(1, #SERVERS) \ + new_leader = SERVERS[new_leader_id] \ + test_run:switch(new_leader) \ + box.cfg{read_only=false} \ + fiber = require('fiber') \ + f = fiber.create(function() box.space.sync:delete{} end) \ + f = fiber.create(function() for i=1,10 do box.space.sync:insert{i} end end) \ + f.status() \ + test_run:switch('default') \ + test_run:switch(current_leader) \ + box.cfg{read_only=true} \ + test_run:switch('default') \ + current_leader = new_leader \ + fiber.sleep(0.1) \ +end + +-- Teardown. +test_run:switch(current_leader) +box.cfg{read_only=true} -- demote master to replica +test_run:switch('default') +box.cfg{read_only=false} -- promote replica to master +box.space.sync:drop() +test_run:drop_cluster(SERVERS) -- 2.26.2