From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from [87.239.111.99] (localhost [127.0.0.1]) by dev.tarantool.org (Postfix) with ESMTP id 10D6C6E454; Thu, 3 Mar 2022 13:09:02 +0300 (MSK) DKIM-Filter: OpenDKIM Filter v2.11.0 dev.tarantool.org 10D6C6E454 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=tarantool.org; s=dev; t=1646302142; bh=0w/QH/0Jkh/G6n10d1yqSVRX5lx5WraIZhn3PKDvKd0=; h=Date:To:References:In-Reply-To:Subject:List-Id:List-Unsubscribe: List-Archive:List-Post:List-Help:List-Subscribe:From:Reply-To:Cc: From; b=pTJcx5vYfn4d1FmkU+3ve8J7dsZjFL32IKyXZph9s54cpeSORKHzTBDMkDgnRqB1U Z3PGYq3dx4GMqeLxldrFpqmadUmxAXTBh+sOJlZTnb4baDT4PtTIuXyQrUrKXiLVtB qc83zkI+azyB5KjYTvHAUUUwNfh+YoF58Ntu91xc= Received: from smtp38.i.mail.ru (smtp38.i.mail.ru [94.100.177.98]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by dev.tarantool.org (Postfix) with ESMTPS id 087C26E454 for ; Thu, 3 Mar 2022 13:09:00 +0300 (MSK) DKIM-Filter: OpenDKIM Filter v2.11.0 dev.tarantool.org 087C26E454 Received: by smtp38.i.mail.ru with esmtpa (envelope-from ) id 1nPiOJ-0007ZT-WF; Thu, 03 Mar 2022 13:09:00 +0300 Message-ID: Date: Thu, 3 Mar 2022 13:08:59 +0300 MIME-Version: 1.0 User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:91.0) Gecko/20100101 Thunderbird/91.5.0 Content-Language: en-US To: Cyrill Gorcunov , Vladislav Shpilevoy References: <20220302202711.1003906-1-gorcunov@gmail.com> <20220302202711.1003906-4-gorcunov@gmail.com> In-Reply-To: <20220302202711.1003906-4-gorcunov@gmail.com> Content-Type: text/plain; charset=UTF-8; format=flowed Content-Transfer-Encoding: 8bit X-4EC0790: 10 X-7564579A: 646B95376F6C166E X-77F55803: 4F1203BC0FB41BD9BE929572AA3D6CA1A2AB2B581E5F725399B3871A01C2BF6400894C459B0CD1B937CA70CD801D58343F24AACBF4C76C7E159133062332A94C77B46462DFFF61CB X-7FA49CB5: FF5795518A3D127A4AD6D5ED66289B5278DA827A17800CE7CE4525FFB91B9BBCEA1F7E6F0F101C67BD4B6F7A4D31EC0BCC500DACC3FED6E28638F802B75D45FF8AA50765F7900637BC6099E116BE74F18638F802B75D45FF36EB9D2243A4F8B5A6FCA7DBDB1FC311F39EFFDF887939037866D6147AF826D85506CF76130BD77E3475702DA085848D117882F4460429724CE54428C33FAD305F5C1EE8F4F765FC8C7ADC89C2F0B2A5A471835C12D1D9774AD6D5ED66289B52BA9C0B312567BB23117882F446042972877693876707352033AC447995A7AD18618001F51B5FD3F9D2E47CDBA5A96583BA9C0B312567BB2376E601842F6C81A19E625A9149C048EE140C956E756FBB7A6136E347CC761E074AD6D5ED66289B52698AB9A7B718F8C46E0066C2D8992A16725E5C173C3A84C388CE97E0F2853DA0BA3038C0950A5D36B5C8C57E37DE458B0BC6067A898B09E46D1867E19FE14079C09775C1D3CA48CF3D321E7403792E342EB15956EA79C166A417C69337E82CC275ECD9A6C639B01B78DA827A17800CE7B2B7C64F398C7410731C566533BA786AA5CC5B56E945C8DA X-8FC586DF: 6EFBBC1D9D64D975 X-B7AD71C0: 1B70FBA5C9BEEE72C9761FC34675ADEB871C96603B655635EE9D5CB6078CC77C72CCC204D750B150135824B3FEAB4495 X-C1DE0DAB: 0D63561A33F958A5F38E128F55DC3BC94FBDC3B19D4FE8A3D1A333963695F424D59269BC5F550898D99A6476B3ADF6B47008B74DF8BB9EF7333BD3B22AA88B938A852937E12ACA752239DE2A13B0F287410CA545F18667F91A7EA1CDA0B5A7A0 X-C8649E89: 4E36BF7865823D7055A7F0CF078B5EC49A30900B95165D342B8615F5CFAD9D02464E29B401F1C72548EA9BFEEFB58B3C054E5D9B83BD2535893A6D34F84DB60F1D7E09C32AA3244CE15F5123BDC4CF4BF293706F0AD0C0E735DA7DC5AF9B58C0927AC6DF5659F194 X-D57D3AED: 3ZO7eAau8CL7WIMRKs4sN3D3tLDjz0dLbV79QFUyzQ2Ujvy7cMT6pYYqY16iZVKkSc3dCLJ7zSJH7+u4VD18S7Vl4ZUrpaVfd2+vE6kuoey4m4VkSEu530nj6fImhcD4MUrOEAnl0W826KZ9Q+tr5ycPtXkTV4k65bRjmOUUP8cvGozZ33TWg5HZplvhhXbhDGzqmQDTd6OAevLeAnq3Ra9uf7zvY2zzsIhlcp/Y7m53TZgf2aB4JOg4gkr2biojiOLg73bewXSadgNNrrQSdg== X-Mailru-Sender: 583F1D7ACE8F49BD2E491048B3B9BC06FCF12CB690C20555E132FAF508A54F8A2F2486D011A08044823C4E0A9438D55D74690CA6451351EDEC462FDC9CAD1E11B969B486931C0B990F27244EEAA5B9A50D4ABDE8C577C2ED X-Mras: Ok Subject: Re: [Tarantool-patches] [PATCH v31 3/3] test: add gh-6036-qsync-order test X-BeenThere: tarantool-patches@dev.tarantool.org X-Mailman-Version: 2.1.34 Precedence: list List-Id: Tarantool development patches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , From: Serge Petrenko via Tarantool-patches Reply-To: Serge Petrenko Cc: tml Errors-To: tarantool-patches-bounces@dev.tarantool.org Sender: "Tarantool-patches" 02.03.2022 23:27, Cyrill Gorcunov пишет: > To test that promotion requests are handled only when appropriate > write to WAL completes, because we update memory data before the > write finishes. > > Part-of #6036 > > Signed-off-by: Cyrill Gorcunov Thanks for working on this! Please consider the following cosmetic changes: 1. There's build_and_add_server as an alias for build_server + add_server 2. Use luatest.assert instead of plain assert everywhere 3. Use exec instead of eval everywhere =================================== diff --git a/test/replication-luatest/gh_6036_qsync_order_test.lua b/test/replication-luatest/gh_6036_qsync_order_test.lua index c23c7a3a1..964101571 100644 --- a/test/replication-luatest/gh_6036_qsync_order_test.lua +++ b/test/replication-luatest/gh_6036_qsync_order_test.lua @@ -22,11 +22,9 @@ g.before_all(function(cg)          log_level                   = 6,      } -    cg.r1 = cg.cluster:build_server({ alias = 'r1', box_cfg = cg.box_cfg }) -    cg.r2 = cg.cluster:build_server({ alias = 'r2', box_cfg = cg.box_cfg }) +    cg.r1 = cg.cluster:build_and_add_server({ alias = 'r1', box_cfg = cg.box_cfg }) +    cg.r2 = cg.cluster:build_and_add_server({ alias = 'r2', box_cfg = cg.box_cfg }) -    cg.cluster:add_server(cg.r1) -    cg.cluster:add_server(cg.r2)      cg.cluster:start()  end) @@ -43,8 +41,7 @@ end  -- The test requires 3rd replica to graft in.  g.before_test("test_qsync_order", function(cg)      cg.box_cfg.replication[3] = server.build_instance_uri("r3") -    cg.r3 = cg.cluster:build_server({ alias = 'r3', box_cfg = cg.box_cfg }) -    cg.cluster:add_server(cg.r3) +    cg.r3 = cg.cluster:build_and_add_server({ alias = 'r3', box_cfg = cg.box_cfg })      cg.r3:start()      cg.r1:exec(update_replication, cg.box_cfg.replication)      cg.r2:exec(update_replication, cg.box_cfg.replication) @@ -69,9 +66,9 @@ g.test_qsync_order = function(cg)      cg.r2:wait_vclock(vclock)      cg.r3:wait_vclock(vclock) -    t.assert_equals(cg.r1:eval("return box.space.test:select()"), {{1}}) -    t.assert_equals(cg.r2:eval("return box.space.test:select()"), {{1}}) -    t.assert_equals(cg.r3:eval("return box.space.test:select()"), {{1}}) +    t.assert_equals(cg.r1:exec(function() return box.space.test:select() end), {{1}}) +    t.assert_equals(cg.r2:exec(function() return box.space.test:select() end), {{1}}) +    t.assert_equals(cg.r3:exec(function() return box.space.test:select() end), {{1}})      --      -- Drop connection between r1 and r2. @@ -113,7 +110,7 @@ g.test_qsync_order = function(cg)          box.ctl.wait_rw()      end)      t.helpers.retrying({}, function() -        assert(cg.r3:exec(function() +        t.assert(cg.r3:exec(function()              return box.info.synchro.queue.busy == true          end))      end) @@ -136,12 +133,12 @@ g.test_qsync_order = function(cg)          box.error.injection.set('ERRINJ_WAL_DELAY', false)      end)      t.helpers.retrying({}, function() -        assert(cg.r3:exec(function() +        t.assert(cg.r3:exec(function()              return box.space.test:get{2} ~= nil          end))      end) -    t.assert_equals(cg.r3:eval("return box.space.test:select()"), {{1},{2}}) +    t.assert_equals(cg.r3:exec(function() return box.space.test:select() end), {{1},{2}})  end  -- @@ -189,10 +186,10 @@ g.test_promote_order = function(cg)          box.space.test:insert{4}      end)      cg.r2:exec(function() -        assert(box.info.synchro.queue.busy == true) +        require('luatest').assert(box.info.synchro.queue.busy == true)          box.error.injection.set('ERRINJ_WAL_DELAY', false)          box.ctl.wait_rw()      end) -    t.assert_equals(cg.r2:eval("return box.space.test:select()"), {{1},{2}}) +    t.assert_equals(cg.r2:exec(function() return box.space.test:select() end), {{1},{2}})  end =================================== The patchset LGTM once you apply these (or decline them). Please, proceed to Vlad's review. > --- > .../gh_6036_qsync_order_test.lua | 198 ++++++++++++++++++ > test/replication-luatest/suite.ini | 1 + > 2 files changed, 199 insertions(+) > create mode 100644 test/replication-luatest/gh_6036_qsync_order_test.lua > > diff --git a/test/replication-luatest/gh_6036_qsync_order_test.lua b/test/replication-luatest/gh_6036_qsync_order_test.lua > new file mode 100644 > index 000000000..c23c7a3a1 > --- /dev/null > +++ b/test/replication-luatest/gh_6036_qsync_order_test.lua > @@ -0,0 +1,198 @@ > +local t = require('luatest') > +local cluster = require('test.luatest_helpers.cluster') > +local server = require('test.luatest_helpers.server') > +local fiber = require('fiber') > + > +local g = t.group('gh-6036') > + > +g.before_all(function(cg) > + cg.cluster = cluster:new({}) > + > + cg.box_cfg = { > + replication = { > + server.build_instance_uri('r1'), > + server.build_instance_uri('r2'), > + }, > + replication_timeout = 0.1, > + replication_connect_quorum = 1, > + election_mode = 'manual', > + election_timeout = 0.1, > + replication_synchro_quorum = 1, > + replication_synchro_timeout = 0.1, > + log_level = 6, > + } > + > + cg.r1 = cg.cluster:build_server({ alias = 'r1', box_cfg = cg.box_cfg }) > + cg.r2 = cg.cluster:build_server({ alias = 'r2', box_cfg = cg.box_cfg }) > + > + cg.cluster:add_server(cg.r1) > + cg.cluster:add_server(cg.r2) > + cg.cluster:start() > +end) > + > +g.after_all(function(cg) > + cg.cluster:drop() > + cg.cluster.servers = nil > +end) > + > +local function update_replication(...) > + return (box.cfg{ replication = { ... } }) > +end > + > +-- > +-- The test requires 3rd replica to graft in. > +g.before_test("test_qsync_order", function(cg) > + cg.box_cfg.replication[3] = server.build_instance_uri("r3") > + cg.r3 = cg.cluster:build_server({ alias = 'r3', box_cfg = cg.box_cfg }) > + cg.cluster:add_server(cg.r3) > + cg.r3:start() > + cg.r1:exec(update_replication, cg.box_cfg.replication) > + cg.r2:exec(update_replication, cg.box_cfg.replication) > +end) > + > +g.test_qsync_order = function(cg) > + cg.cluster:wait_fullmesh() > + > + -- > + -- Create a synchro space on the r1 node and make > + -- sure the write processed just fine. > + cg.r1:exec(function() > + box.ctl.promote() > + box.ctl.wait_rw() > + local s = box.schema.create_space('test', {is_sync = true}) > + s:create_index('pk') > + s:insert{1} > + end) > + > + local vclock = cg.r1:get_vclock() > + vclock[0] = nil > + cg.r2:wait_vclock(vclock) > + cg.r3:wait_vclock(vclock) > + > + t.assert_equals(cg.r1:eval("return box.space.test:select()"), {{1}}) > + t.assert_equals(cg.r2:eval("return box.space.test:select()"), {{1}}) > + t.assert_equals(cg.r3:eval("return box.space.test:select()"), {{1}}) > + > + -- > + -- Drop connection between r1 and r2. > + cg.r1:exec(update_replication, { > + server.build_instance_uri("r1"), > + server.build_instance_uri("r3"), > + }) > + > + -- > + -- Drop connection between r2 and r1. > + cg.r2:exec(update_replication, { > + server.build_instance_uri("r2"), > + server.build_instance_uri("r3"), > + }) > + > + -- > + -- Here we have the following scheme > + -- > + -- r3 (WAL delay) > + -- / \ > + -- r1 r2 > + -- > + > + -- > + -- Initiate disk delay in a bit tricky way: the next write will > + -- fall into forever sleep. > + cg.r3:exec(function() > + box.error.injection.set('ERRINJ_WAL_DELAY', true) > + end) > + > + -- > + -- Make r2 been a leader and start writting data, the PROMOTE > + -- request get queued on r3 and not yet processed, same time > + -- the INSERT won't complete either waiting for the PROMOTE > + -- completion first. Note that we enter r3 as well just to be > + -- sure the PROMOTE has reached it via queue state test. > + cg.r2:exec(function() > + box.ctl.promote() > + box.ctl.wait_rw() > + end) > + t.helpers.retrying({}, function() > + assert(cg.r3:exec(function() > + return box.info.synchro.queue.busy == true > + end)) > + end) > + cg.r2:exec(function() > + box.space.test:insert{2} > + end) > + > + -- > + -- The r1 node has no clue that there is a new leader and continue > + -- writing data with obsolete term. Since r3 is delayed now > + -- the INSERT won't proceed yet but get queued. > + cg.r1:exec(function() > + box.space.test:insert{3} > + end) > + > + -- > + -- Finally enable r3 back. Make sure the data from new r2 leader get > + -- writing while old leader's data ignored. > + cg.r3:exec(function() > + box.error.injection.set('ERRINJ_WAL_DELAY', false) > + end) > + t.helpers.retrying({}, function() > + assert(cg.r3:exec(function() > + return box.space.test:get{2} ~= nil > + end)) > + end) > + > + t.assert_equals(cg.r3:eval("return box.space.test:select()"), {{1},{2}}) > +end > + > +-- > +-- Drop the r3 replica, since it is no longer needed for this test. > +g.after_test("test_qsync_order", function(cg) > + cg.box_cfg.replication[3] = nil > + cg.r1:exec(update_replication, cg.box_cfg.replication) > + cg.r2:exec(update_replication, cg.box_cfg.replication) > + cg.r3:stop() > + cg.r3:cleanup() > + cg.r3 = nil > +end) > + > +g.test_promote_order = function(cg) > + -- > + -- Make sure that while we're processing PROMOTE no other records > + -- get sneaked in via applier code from other replicas. For this > + -- sake initiate voting and stop inside wal thread just before > + -- PROMOTE get written. Another replica sends us new record and > + -- it should be dropped. > + cg.r1:exec(function() > + box.ctl.promote() > + box.ctl.wait_rw() > + end) > + local vclock = cg.r1:get_vclock() > + vclock[0] = nil > + cg.r2:wait_vclock(vclock) > + > + -- > + -- Drop connection between r1 and the rest of the cluster. > + -- Otherwise r1 might become Raft follower before attempting > + -- insert{4}. > + cg.r1:exec(function() box.cfg{replication=""} end) > + cg.r2:exec(function() > + box.error.injection.set('ERRINJ_WAL_DELAY_COUNTDOWN', 2) > + require('fiber').create(function() box.ctl.promote() end) > + end) > + t.helpers.retrying({}, function() > + t.assert(cg.r2:exec(function() > + return box.info.synchro.queue.busy > + end)) > + end) > + t.assert(cg.r1:exec(function() return box.info.ro == false end)) > + cg.r1:exec(function() > + box.space.test:insert{4} > + end) > + cg.r2:exec(function() > + assert(box.info.synchro.queue.busy == true) > + box.error.injection.set('ERRINJ_WAL_DELAY', false) > + box.ctl.wait_rw() > + end) > + > + t.assert_equals(cg.r2:eval("return box.space.test:select()"), {{1},{2}}) > +end > diff --git a/test/replication-luatest/suite.ini b/test/replication-luatest/suite.ini > index 374f1b87a..07ec93a52 100644 > --- a/test/replication-luatest/suite.ini > +++ b/test/replication-luatest/suite.ini > @@ -2,3 +2,4 @@ > core = luatest > description = replication luatests > is_parallel = True > +release_disabled = gh_6036_qsync_order_test.lua