Tarantool development patches archive
 help / color / mirror / Atom feed
From: Cyrill Gorcunov via Tarantool-patches <tarantool-patches@dev.tarantool.org>
To: tml <tarantool-patches@dev.tarantool.org>
Cc: Vladislav Shpilevoy <v.shpilevoy@tarantool.org>
Subject: [Tarantool-patches] [RFC v29 3/3] test: add gh-6036-qsync-order test
Date: Tue,  1 Feb 2022 00:55:54 +0300	[thread overview]
Message-ID: <20220131215554.1367429-4-gorcunov@gmail.com> (raw)
In-Reply-To: <20220131215554.1367429-1-gorcunov@gmail.com>

To test that promotion requests are handled only when appropriate
write to WAL completes, because we update memory data before the
write finishes.

Part-of #6036

Signed-off-by: Cyrill Gorcunov <gorcunov@gmail.com>
---
 .../gh_6036_qsync_order_test.lua              | 137 ++++++++++++++++++
 test/replication-luatest/suite.ini            |   1 +
 2 files changed, 138 insertions(+)
 create mode 100644 test/replication-luatest/gh_6036_qsync_order_test.lua

diff --git a/test/replication-luatest/gh_6036_qsync_order_test.lua b/test/replication-luatest/gh_6036_qsync_order_test.lua
new file mode 100644
index 000000000..4c0059764
--- /dev/null
+++ b/test/replication-luatest/gh_6036_qsync_order_test.lua
@@ -0,0 +1,137 @@
+local t = require('luatest')
+local cluster = require('test.luatest_helpers.cluster')
+local asserts = require('test.luatest_helpers.asserts')
+local helpers = require('test.luatest_helpers')
+local log = require('log')
+
+local g = t.group('gh-6036', {{engine = 'memtx'}, {engine = 'vinyl'}})
+
+g.before_each(function(cg)
+    pcall(log.cfg, {level = 6})
+
+    local engine = cg.params.engine
+
+    cg.cluster = cluster:new({})
+
+    local box_cfg = {
+        replication = {
+            helpers.instance_uri('r1'),
+            helpers.instance_uri('r2'),
+            helpers.instance_uri('r3'),
+        },
+        replication_timeout         = 0.1,
+        replication_connect_quorum  = 1,
+        election_mode               = 'manual',
+        election_timeout            = 0.1,
+        replication_synchro_quorum  = 1,
+        replication_synchro_timeout = 0.1,
+        log_level                   = 6,
+    }
+
+    cg.r1 = cg.cluster:build_server({ alias = 'r1',
+        engine = engine, box_cfg = box_cfg })
+    cg.r2 = cg.cluster:build_server({ alias = 'r2',
+        engine = engine, box_cfg = box_cfg })
+    cg.r3 = cg.cluster:build_server({ alias = 'r3',
+        engine = engine, box_cfg = box_cfg })
+
+    cg.cluster:add_server(cg.r1)
+    cg.cluster:add_server(cg.r2)
+    cg.cluster:add_server(cg.r3)
+    cg.cluster:start()
+end)
+
+g.after_each(function(cg)
+    cg.cluster:drop()
+    cg.cluster.servers = nil
+end)
+
+g.test_qsync_order = function(cg)
+    asserts:wait_fullmesh({cg.r1, cg.r2, cg.r3})
+
+    --
+    -- Create a synchro space on the r1 node and make
+    -- sure the write processed just fine.
+    cg.r1:exec(function()
+        box.ctl.promote()
+        box.ctl.wait_rw()
+        local s = box.schema.create_space('test', {is_sync = true})
+        s:create_index('pk')
+        s:insert{1}
+    end)
+
+    local vclock = cg.r1:eval("return box.info.vclock")
+    vclock[0] = nil
+    helpers:wait_vclock(cg.r2, vclock)
+    helpers:wait_vclock(cg.r3, vclock)
+
+    t.assert_equals(cg.r1:eval("return box.space.test:select()"), {{1}})
+    t.assert_equals(cg.r2:eval("return box.space.test:select()"), {{1}})
+    t.assert_equals(cg.r3:eval("return box.space.test:select()"), {{1}})
+
+    local function update_replication(...)
+        return (box.cfg{ replication = { ... } })
+    end
+
+    --
+    -- Drop connection between r1 and r2.
+    cg.r1:exec(update_replication, {
+            helpers.instance_uri("r1"),
+            helpers.instance_uri("r3"),
+        })
+
+    --
+    -- Drop connection between r2 and r1.
+    cg.r2:exec(update_replication, {
+        helpers.instance_uri("r2"),
+        helpers.instance_uri("r3"),
+    })
+
+    --
+    -- Here we have the following scheme
+    --
+    --      r3 (WAL delay)
+    --      /            \
+    --    r1              r2
+    --
+
+    --
+    -- Initiate disk delay in a bit tricky way: the next write will
+    -- fall into forever sleep.
+    cg.r3:eval("box.error.injection.set('ERRINJ_WAL_DELAY', true)")
+
+    --
+    -- Make r2 been a leader and start writting data, the PROMOTE
+    -- request get queued on r3 and not yet processed, same time
+    -- the INSERT won't complete either waiting for the PROMOTE
+    -- completion first. Note that we enter r3 as well just to be
+    -- sure the PROMOTE has reached it via queue state test.
+    cg.r2:exec(function()
+        box.ctl.promote()
+        box.ctl.wait_rw()
+    end)
+    t.helpers.retrying({}, function()
+        assert(cg.r3:exec(function()
+            return box.info.synchro.queue.latched == true
+        end))
+    end)
+    cg.r2:eval("box.space.test:insert{2}")
+
+    --
+    -- The r1 node has no clue that there is a new leader and continue
+    -- writing data with obsolete term. Since r3 is delayed now
+    -- the INSERT won't proceed yet but get queued.
+    cg.r1:eval("box.space.test:insert{3}")
+
+    --
+    -- Finally enable r3 back. Make sure the data from new r2 leader get
+    -- writing while old leader's data ignored.
+    cg.r3:eval("box.error.injection.set('ERRINJ_WAL_DELAY', false)")
+    t.helpers.retrying({}, function()
+        assert(cg.r3:exec(function()
+            return box.space.test:get{2} ~= nil
+        end))
+    end)
+
+    t.assert_equals(cg.r3:eval("return box.space.test:select()"), {{1},{2}})
+end
diff --git a/test/replication-luatest/suite.ini b/test/replication-luatest/suite.ini
index 374f1b87a..07ec93a52 100644
--- a/test/replication-luatest/suite.ini
+++ b/test/replication-luatest/suite.ini
@@ -2,3 +2,4 @@
 core = luatest
 description = replication luatests
 is_parallel = True
+release_disabled = gh_6036_qsync_order_test.lua
-- 
2.34.1


  parent reply	other threads:[~2022-01-31 21:57 UTC|newest]

Thread overview: 6+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2022-01-31 21:55 [Tarantool-patches] [RFC v29 0/3] qsync: implement packet filtering (part 1) Cyrill Gorcunov via Tarantool-patches
2022-01-31 21:55 ` [Tarantool-patches] [RFC v29 1/3] latch: add latch_is_locked helper Cyrill Gorcunov via Tarantool-patches
2022-01-31 21:55 ` [Tarantool-patches] [RFC v29 2/3] qsync: order access to the limbo terms Cyrill Gorcunov via Tarantool-patches
2022-02-09  9:10   ` Serge Petrenko via Tarantool-patches
2022-01-31 21:55 ` Cyrill Gorcunov via Tarantool-patches [this message]
2022-02-09  9:11   ` [Tarantool-patches] [RFC v29 3/3] test: add gh-6036-qsync-order test Serge Petrenko via Tarantool-patches

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20220131215554.1367429-4-gorcunov@gmail.com \
    --to=tarantool-patches@dev.tarantool.org \
    --cc=gorcunov@gmail.com \
    --cc=v.shpilevoy@tarantool.org \
    --subject='Re: [Tarantool-patches] [RFC v29 3/3] test: add gh-6036-qsync-order test' \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox