From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from [87.239.111.99] (localhost [127.0.0.1]) by dev.tarantool.org (Postfix) with ESMTP id F3B796F867; Tue, 1 Feb 2022 00:57:31 +0300 (MSK) DKIM-Filter: OpenDKIM Filter v2.11.0 dev.tarantool.org F3B796F867 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=tarantool.org; s=dev; t=1643666252; bh=92+g3565isdZGSTCbJxMW/d2P8a3M8rCNBDf2laqpPE=; h=To:Date:In-Reply-To:References:Subject:List-Id:List-Unsubscribe: List-Archive:List-Post:List-Help:List-Subscribe:From:Reply-To:Cc: From; b=cGhO5+fwq12x+zqdpFCq5J/Cn1cXK1GcGfxmNQuVPIH+i6Cu+JwCvOpl/ldB+oE28 OwQLIWUpK6P6IT71g304DnRAiQjTRqkwXCis3auHTN+0d2u5kJDigGN8aeUpP2Giza /S1pi2+DsTV3f2zt32Wr7D6boTiJjbtaycrsszwU= Received: from mail-lj1-f174.google.com (mail-lj1-f174.google.com [209.85.208.174]) (using TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 (128/128 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256) (No client certificate requested) by dev.tarantool.org (Postfix) with ESMTPS id 761586F86A for ; Tue, 1 Feb 2022 00:56:34 +0300 (MSK) DKIM-Filter: OpenDKIM Filter v2.11.0 dev.tarantool.org 761586F86A Received: by mail-lj1-f174.google.com with SMTP id e17so21389793ljk.5 for ; Mon, 31 Jan 2022 13:56:34 -0800 (PST) X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=UsvRoqf2K0SXbwqEsJek8C26zQuF4oz5T4uNWnmBS44=; b=PcHf2T595pw53A8WIFZdxtnHnDAJHQ4ViJR4vk2Z00KhRmDmrCO/aFkNnu8AtMMqFT h1RDRPBM1TRf/fH2arsIzivTuqGjhTDMnmDg33+IDXTLNhNJAgGs6uTagtl6bUNXkFur WWxNH0lxqYTFeVKr/yYf3JcDrgIbbdJ7CUK3K5uwfQBWJivWw6BxmlMDCtR4/RH5qpHu pNXgplObJP1CgMkgW8ElfD1lW7UJv6lDrv1WR97pQ8wA4UdaU2KngJCeRZ8CtRfcy8ha Nfd9QztaRxcwdgyuYJV9nweyZR+9NxwOq0hy7ybxJP+MSLMVyDpiKCONnRL82BftUCyn zZ5g== X-Gm-Message-State: AOAM533p0UBc8bakWw8s2BdTmbl1rcSjFkb1OQLhmtY3Iqw9l0V44Uk7 eDm1w8czoPALxOTEW8NFPHgkz7EgvBM= X-Google-Smtp-Source: ABdhPJzKKO2PYVUYyXNzlL/lyiJNuWgU8VQR5PcarCkAEdrH9zWg8ovghecYbWt1NWHVbW/MTRddcQ== X-Received: by 2002:a2e:9dc2:: with SMTP id x2mr14223497ljj.202.1643666193438; Mon, 31 Jan 2022 13:56:33 -0800 (PST) Received: from grain.localdomain ([5.18.251.97]) by smtp.gmail.com with ESMTPSA id u14sm933584lfo.58.2022.01.31.13.56.32 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Mon, 31 Jan 2022 13:56:32 -0800 (PST) Received: by grain.localdomain (Postfix, from userid 1000) id 29EB25A0023; Tue, 1 Feb 2022 00:55:57 +0300 (MSK) To: tml Date: Tue, 1 Feb 2022 00:55:54 +0300 Message-Id: <20220131215554.1367429-4-gorcunov@gmail.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20220131215554.1367429-1-gorcunov@gmail.com> References: <20220131215554.1367429-1-gorcunov@gmail.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit Subject: [Tarantool-patches] [RFC v29 3/3] test: add gh-6036-qsync-order test X-BeenThere: tarantool-patches@dev.tarantool.org X-Mailman-Version: 2.1.34 Precedence: list List-Id: Tarantool development patches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , From: Cyrill Gorcunov via Tarantool-patches Reply-To: Cyrill Gorcunov Cc: Vladislav Shpilevoy Errors-To: tarantool-patches-bounces@dev.tarantool.org Sender: "Tarantool-patches" To test that promotion requests are handled only when appropriate write to WAL completes, because we update memory data before the write finishes. Part-of #6036 Signed-off-by: Cyrill Gorcunov --- .../gh_6036_qsync_order_test.lua | 137 ++++++++++++++++++ test/replication-luatest/suite.ini | 1 + 2 files changed, 138 insertions(+) create mode 100644 test/replication-luatest/gh_6036_qsync_order_test.lua diff --git a/test/replication-luatest/gh_6036_qsync_order_test.lua b/test/replication-luatest/gh_6036_qsync_order_test.lua new file mode 100644 index 000000000..4c0059764 --- /dev/null +++ b/test/replication-luatest/gh_6036_qsync_order_test.lua @@ -0,0 +1,137 @@ +local t = require('luatest') +local cluster = require('test.luatest_helpers.cluster') +local asserts = require('test.luatest_helpers.asserts') +local helpers = require('test.luatest_helpers') +local log = require('log') + +local g = t.group('gh-6036', {{engine = 'memtx'}, {engine = 'vinyl'}}) + +g.before_each(function(cg) + pcall(log.cfg, {level = 6}) + + local engine = cg.params.engine + + cg.cluster = cluster:new({}) + + local box_cfg = { + replication = { + helpers.instance_uri('r1'), + helpers.instance_uri('r2'), + helpers.instance_uri('r3'), + }, + replication_timeout = 0.1, + replication_connect_quorum = 1, + election_mode = 'manual', + election_timeout = 0.1, + replication_synchro_quorum = 1, + replication_synchro_timeout = 0.1, + log_level = 6, + } + + cg.r1 = cg.cluster:build_server({ alias = 'r1', + engine = engine, box_cfg = box_cfg }) + cg.r2 = cg.cluster:build_server({ alias = 'r2', + engine = engine, box_cfg = box_cfg }) + cg.r3 = cg.cluster:build_server({ alias = 'r3', + engine = engine, box_cfg = box_cfg }) + + cg.cluster:add_server(cg.r1) + cg.cluster:add_server(cg.r2) + cg.cluster:add_server(cg.r3) + cg.cluster:start() +end) + +g.after_each(function(cg) + cg.cluster:drop() + cg.cluster.servers = nil +end) + +g.test_qsync_order = function(cg) + asserts:wait_fullmesh({cg.r1, cg.r2, cg.r3}) + + -- + -- Create a synchro space on the r1 node and make + -- sure the write processed just fine. + cg.r1:exec(function() + box.ctl.promote() + box.ctl.wait_rw() + local s = box.schema.create_space('test', {is_sync = true}) + s:create_index('pk') + s:insert{1} + end) + + local vclock = cg.r1:eval("return box.info.vclock") + vclock[0] = nil + helpers:wait_vclock(cg.r2, vclock) + helpers:wait_vclock(cg.r3, vclock) + + t.assert_equals(cg.r1:eval("return box.space.test:select()"), {{1}}) + t.assert_equals(cg.r2:eval("return box.space.test:select()"), {{1}}) + t.assert_equals(cg.r3:eval("return box.space.test:select()"), {{1}}) + + local function update_replication(...) + return (box.cfg{ replication = { ... } }) + end + + -- + -- Drop connection between r1 and r2. + cg.r1:exec(update_replication, { + helpers.instance_uri("r1"), + helpers.instance_uri("r3"), + }) + + -- + -- Drop connection between r2 and r1. + cg.r2:exec(update_replication, { + helpers.instance_uri("r2"), + helpers.instance_uri("r3"), + }) + + -- + -- Here we have the following scheme + -- + -- r3 (WAL delay) + -- / \ + -- r1 r2 + -- + + -- + -- Initiate disk delay in a bit tricky way: the next write will + -- fall into forever sleep. + cg.r3:eval("box.error.injection.set('ERRINJ_WAL_DELAY', true)") + + -- + -- Make r2 been a leader and start writting data, the PROMOTE + -- request get queued on r3 and not yet processed, same time + -- the INSERT won't complete either waiting for the PROMOTE + -- completion first. Note that we enter r3 as well just to be + -- sure the PROMOTE has reached it via queue state test. + cg.r2:exec(function() + box.ctl.promote() + box.ctl.wait_rw() + end) + t.helpers.retrying({}, function() + assert(cg.r3:exec(function() + return box.info.synchro.queue.latched == true + end)) + end) + cg.r2:eval("box.space.test:insert{2}") + + -- + -- The r1 node has no clue that there is a new leader and continue + -- writing data with obsolete term. Since r3 is delayed now + -- the INSERT won't proceed yet but get queued. + cg.r1:eval("box.space.test:insert{3}") + + -- + -- Finally enable r3 back. Make sure the data from new r2 leader get + -- writing while old leader's data ignored. + cg.r3:eval("box.error.injection.set('ERRINJ_WAL_DELAY', false)") + t.helpers.retrying({}, function() + assert(cg.r3:exec(function() + return box.space.test:get{2} ~= nil + end)) + end) + + t.assert_equals(cg.r3:eval("return box.space.test:select()"), {{1},{2}}) +end diff --git a/test/replication-luatest/suite.ini b/test/replication-luatest/suite.ini index 374f1b87a..07ec93a52 100644 --- a/test/replication-luatest/suite.ini +++ b/test/replication-luatest/suite.ini @@ -2,3 +2,4 @@ core = luatest description = replication luatests is_parallel = True +release_disabled = gh_6036_qsync_order_test.lua -- 2.34.1