From: Cyrill Gorcunov via Tarantool-patches <tarantool-patches@dev.tarantool.org> To: tml <tarantool-patches@dev.tarantool.org> Cc: Vladislav Shpilevoy <v.shpilevoy@tarantool.org>, Mons Anderson <v.perepelitsa@corp.mail.ru> Subject: [Tarantool-patches] [PATCH v2 2/3] test: add a test for wal_cleanup_delay option Date: Sat, 20 Mar 2021 16:15:20 +0300 [thread overview] Message-ID: <20210320131521.1249747-3-gorcunov@gmail.com> (raw) In-Reply-To: <20210320131521.1249747-1-gorcunov@gmail.com> Part-of #5806 Signed-off-by: Cyrill Gorcunov <gorcunov@gmail.com> --- test/replication/gh-5806-master.lua | 8 + test/replication/gh-5806-slave.lua | 8 + test/replication/gh-5806-xlog-cleanup.result | 397 ++++++++++++++++++ .../replication/gh-5806-xlog-cleanup.test.lua | 166 ++++++++ 4 files changed, 579 insertions(+) create mode 100644 test/replication/gh-5806-master.lua create mode 100644 test/replication/gh-5806-slave.lua create mode 100644 test/replication/gh-5806-xlog-cleanup.result create mode 100644 test/replication/gh-5806-xlog-cleanup.test.lua diff --git a/test/replication/gh-5806-master.lua b/test/replication/gh-5806-master.lua new file mode 100644 index 000000000..bc15dab67 --- /dev/null +++ b/test/replication/gh-5806-master.lua @@ -0,0 +1,8 @@ +#!/usr/bin/env tarantool + +require('console').listen(os.getenv('ADMIN')) + +box.cfg({ + listen = os.getenv("LISTEN"), + wal_cleanup_delay = tonumber(arg[1]) or 0, +}) diff --git a/test/replication/gh-5806-slave.lua b/test/replication/gh-5806-slave.lua new file mode 100644 index 000000000..3abb3e035 --- /dev/null +++ b/test/replication/gh-5806-slave.lua @@ -0,0 +1,8 @@ +#!/usr/bin/env tarantool + +require('console').listen(os.getenv('ADMIN')) + +box.cfg({ + listen = os.getenv("LISTEN"), + replication = os.getenv("MASTER"), +}) diff --git a/test/replication/gh-5806-xlog-cleanup.result b/test/replication/gh-5806-xlog-cleanup.result new file mode 100644 index 000000000..26e7519fc --- /dev/null +++ b/test/replication/gh-5806-xlog-cleanup.result @@ -0,0 +1,397 @@ +-- test-run result file version 2 +-- +-- gh-5806: defer xlog cleanup to keep xlogs until +-- replicas present in "_cluster" are connected. +-- Otherwise we are getting XlogGapError since +-- master might go far forwad from replica and +-- replica won't be able to connect without full +-- rebootstrap. +-- + +fiber = require('fiber') + | --- + | ... +test_run = require('test_run').new() + | --- + | ... +engine = test_run:get_cfg('engine') + | --- + | ... + +-- +-- Case 1. +-- +-- First lets make sure we're getting XlogGapError in +-- case if wal_cleanup_delay is not used. +-- + +test_run:cmd('create server master with script="replication/gh-5806-master.lua"') + | --- + | - true + | ... +test_run:cmd('start server master with wait=True, wait_load=True') + | --- + | - true + | ... + +test_run:switch('master') + | --- + | - true + | ... +box.schema.user.grant('guest', 'replication') + | --- + | ... + +-- +-- Keep small number of snaps to force cleanup +-- procedure be more intensive. +box.cfg{checkpoint_count = 1} + | --- + | ... + +engine = test_run:get_cfg('engine') + | --- + | ... +s = box.schema.space.create('test', {engine = engine}) + | --- + | ... +_ = s:create_index('pk') + | --- + | ... + +test_run:switch('default') + | --- + | - true + | ... +test_run:cmd('create server replica with rpl_master=master,\ + script="replication/gh-5806-slave.lua"') + | --- + | - true + | ... +test_run:cmd('start server replica with wait=True, wait_load=True') + | --- + | - true + | ... + +-- +-- On replica we create an own space which allows us to +-- use more complex scenario and disableds replica from +-- automatic rejoin. Otherwise XlogGapError won't happen. +test_run:switch('replica') + | --- + | - true + | ... +box.cfg{checkpoint_count = 1} + | --- + | ... +s = box.schema.space.create('testtemp', {temporary = true}) + | --- + | ... +_ = s:create_index('pk') + | --- + | ... +box.space.testtemp:insert({1}) + | --- + | - [1] + | ... +box.snapshot() + | --- + | - ok + | ... +box.space.testtemp:insert({2}) + | --- + | - [2] + | ... +box.snapshot() + | --- + | - ok + | ... + +-- +-- Stop the replica node and generate +-- first range of xlogs on the master. +test_run:switch('default') + | --- + | - true + | ... +test_run:cmd('stop server replica') + | --- + | - true + | ... + +test_run:switch('master') + | --- + | - true + | ... +box.space.test:insert({1}) + | --- + | - [1] + | ... +box.snapshot() + | --- + | - ok + | ... +box.space.test:insert({2}) + | --- + | - [2] + | ... +box.snapshot() + | --- + | - ok + | ... + +-- +-- Restart the master and generate the +-- next range of xlogs. +test_run:switch('default') + | --- + | - true + | ... +test_run:cmd('restart server master with wait_load=True') + | --- + | - true + | ... +test_run:switch('master') + | --- + | - true + | ... +box.space.test:insert({3}) + | --- + | - [3] + | ... +box.snapshot() + | --- + | - ok + | ... +box.space.test:insert({4}) + | --- + | - [4] + | ... +box.snapshot() + | --- + | - ok + | ... + +-- +-- Start replica and wait for error. +test_run:switch('default') + | --- + | - true + | ... +test_run:cmd('start server replica with wait=False, wait_load=False') + | --- + | - true + | ... + +-- +-- Wait error to appear. +test_run:wait_log('master', 'XlogGapError', 1024, 0.1) ~= nil + | --- + | - true + | ... + +-- +-- Cleanup. +test_run:cmd('stop server master') + | --- + | - true + | ... +test_run:cmd('stop server replica') + | --- + | - true + | ... +test_run:cmd('delete server master') + | --- + | - true + | ... +test_run:cmd('delete server replica') + | --- + | - true + | ... + +-- +-- Case 2. +-- +-- Lets make sure we're not getting XlogGapError in +-- case if wal_cleanup_delay is used. +-- + +test_run:cmd('create server master with script="replication/gh-5806-master.lua"') + | --- + | - true + | ... +test_run:cmd('start server master with args="3600", wait=True, wait_load=True') + | --- + | - true + | ... + +test_run:switch('master') + | --- + | - true + | ... +box.schema.user.grant('guest', 'replication') + | --- + | ... + +-- +-- Keep small number of snaps to force cleanup +-- procedure be more intensive. +box.cfg{checkpoint_count = 1} + | --- + | ... + +engine = test_run:get_cfg('engine') + | --- + | ... +s = box.schema.space.create('test', {engine = engine}) + | --- + | ... +_ = s:create_index('pk') + | --- + | ... + +test_run:switch('default') + | --- + | - true + | ... +test_run:cmd('create server replica with rpl_master=master,\ + script="replication/gh-5806-slave.lua"') + | --- + | - true + | ... +test_run:cmd('start server replica with wait=True, wait_load=True') + | --- + | - true + | ... + +test_run:switch('replica') + | --- + | - true + | ... +box.cfg{checkpoint_count = 1} + | --- + | ... +s = box.schema.space.create('testtemp', {temporary = true}) + | --- + | ... +_ = s:create_index('pk') + | --- + | ... +box.space.testtemp:insert({1}) + | --- + | - [1] + | ... +box.snapshot() + | --- + | - ok + | ... +box.space.testtemp:insert({2}) + | --- + | - [2] + | ... +box.snapshot() + | --- + | - ok + | ... + +-- +-- Stop the replica node and generate +-- first range of xlogs on the master. +test_run:switch('default') + | --- + | - true + | ... +test_run:cmd('stop server replica') + | --- + | - true + | ... + +test_run:switch('master') + | --- + | - true + | ... +box.space.test:insert({1}) + | --- + | - [1] + | ... +box.snapshot() + | --- + | - ok + | ... +box.space.test:insert({2}) + | --- + | - [2] + | ... +box.snapshot() + | --- + | - ok + | ... + +-- +-- Restart the master and generate the +-- next range of xlogs. +test_run:switch('default') + | --- + | - true + | ... +test_run:cmd('restart server master with args="3600", wait=True, wait_load=True') + | --- + | - true + | ... +test_run:switch('master') + | --- + | - true + | ... +box.space.test:insert({3}) + | --- + | - [3] + | ... +box.snapshot() + | --- + | - ok + | ... +box.space.test:insert({4}) + | --- + | - [4] + | ... +box.snapshot() + | --- + | - ok + | ... + +-- +-- Restart master node and the replica then. +test_run:switch('default') + | --- + | - true + | ... +test_run:cmd('start server replica with wait=True, wait_load=True') + | --- + | - true + | ... + +-- +-- Make sure no error happened. +assert(test_run:grep_log("master", "XlogGapError") == nil) + | --- + | - true + | ... + +-- +-- Cleanup. +test_run:cmd('stop server master') + | --- + | - true + | ... +test_run:cmd('stop server replica') + | --- + | - true + | ... +test_run:cmd('delete server master') + | --- + | - true + | ... +test_run:cmd('delete server replica') + | --- + | - true + | ... diff --git a/test/replication/gh-5806-xlog-cleanup.test.lua b/test/replication/gh-5806-xlog-cleanup.test.lua new file mode 100644 index 000000000..2ff5a4f3a --- /dev/null +++ b/test/replication/gh-5806-xlog-cleanup.test.lua @@ -0,0 +1,166 @@ +-- +-- gh-5806: defer xlog cleanup to keep xlogs until +-- replicas present in "_cluster" are connected. +-- Otherwise we are getting XlogGapError since +-- master might go far forwad from replica and +-- replica won't be able to connect without full +-- rebootstrap. +-- + +fiber = require('fiber') +test_run = require('test_run').new() +engine = test_run:get_cfg('engine') + +-- +-- Case 1. +-- +-- First lets make sure we're getting XlogGapError in +-- case if wal_cleanup_delay is not used. +-- + +test_run:cmd('create server master with script="replication/gh-5806-master.lua"') +test_run:cmd('start server master with wait=True, wait_load=True') + +test_run:switch('master') +box.schema.user.grant('guest', 'replication') + +-- +-- Keep small number of snaps to force cleanup +-- procedure be more intensive. +box.cfg{checkpoint_count = 1} + +engine = test_run:get_cfg('engine') +s = box.schema.space.create('test', {engine = engine}) +_ = s:create_index('pk') + +test_run:switch('default') +test_run:cmd('create server replica with rpl_master=master,\ + script="replication/gh-5806-slave.lua"') +test_run:cmd('start server replica with wait=True, wait_load=True') + +-- +-- On replica we create an own space which allows us to +-- use more complex scenario and disableds replica from +-- automatic rejoin. Otherwise XlogGapError won't happen. +test_run:switch('replica') +box.cfg{checkpoint_count = 1} +s = box.schema.space.create('testtemp', {temporary = true}) +_ = s:create_index('pk') +box.space.testtemp:insert({1}) +box.snapshot() +box.space.testtemp:insert({2}) +box.snapshot() + +-- +-- Stop the replica node and generate +-- first range of xlogs on the master. +test_run:switch('default') +test_run:cmd('stop server replica') + +test_run:switch('master') +box.space.test:insert({1}) +box.snapshot() +box.space.test:insert({2}) +box.snapshot() + +-- +-- Restart the master and generate the +-- next range of xlogs. +test_run:switch('default') +test_run:cmd('restart server master with wait_load=True') +test_run:switch('master') +box.space.test:insert({3}) +box.snapshot() +box.space.test:insert({4}) +box.snapshot() + +-- +-- Start replica and wait for error. +test_run:switch('default') +test_run:cmd('start server replica with wait=False, wait_load=False') + +-- +-- Wait error to appear. +test_run:wait_log('master', 'XlogGapError', 1024, 0.1) ~= nil + +-- +-- Cleanup. +test_run:cmd('stop server master') +test_run:cmd('stop server replica') +test_run:cmd('delete server master') +test_run:cmd('delete server replica') + +-- +-- Case 2. +-- +-- Lets make sure we're not getting XlogGapError in +-- case if wal_cleanup_delay is used. +-- + +test_run:cmd('create server master with script="replication/gh-5806-master.lua"') +test_run:cmd('start server master with args="3600", wait=True, wait_load=True') + +test_run:switch('master') +box.schema.user.grant('guest', 'replication') + +-- +-- Keep small number of snaps to force cleanup +-- procedure be more intensive. +box.cfg{checkpoint_count = 1} + +engine = test_run:get_cfg('engine') +s = box.schema.space.create('test', {engine = engine}) +_ = s:create_index('pk') + +test_run:switch('default') +test_run:cmd('create server replica with rpl_master=master,\ + script="replication/gh-5806-slave.lua"') +test_run:cmd('start server replica with wait=True, wait_load=True') + +test_run:switch('replica') +box.cfg{checkpoint_count = 1} +s = box.schema.space.create('testtemp', {temporary = true}) +_ = s:create_index('pk') +box.space.testtemp:insert({1}) +box.snapshot() +box.space.testtemp:insert({2}) +box.snapshot() + +-- +-- Stop the replica node and generate +-- first range of xlogs on the master. +test_run:switch('default') +test_run:cmd('stop server replica') + +test_run:switch('master') +box.space.test:insert({1}) +box.snapshot() +box.space.test:insert({2}) +box.snapshot() + +-- +-- Restart the master and generate the +-- next range of xlogs. +test_run:switch('default') +test_run:cmd('restart server master with args="3600", wait=True, wait_load=True') +test_run:switch('master') +box.space.test:insert({3}) +box.snapshot() +box.space.test:insert({4}) +box.snapshot() + +-- +-- Restart master node and the replica then. +test_run:switch('default') +test_run:cmd('start server replica with wait=True, wait_load=True') + +-- +-- Make sure no error happened. +assert(test_run:grep_log("master", "XlogGapError") == nil) + +-- +-- Cleanup. +test_run:cmd('stop server master') +test_run:cmd('stop server replica') +test_run:cmd('delete server master') +test_run:cmd('delete server replica') -- 2.30.2
next prev parent reply other threads:[~2021-03-20 13:16 UTC|newest] Thread overview: 11+ messages / expand[flat|nested] mbox.gz Atom feed top 2021-03-20 13:15 [Tarantool-patches] [PATCH v2 0/3] gc/xlog: delay xlog cleanup until relays are subscribed Cyrill Gorcunov via Tarantool-patches 2021-03-20 13:15 ` [Tarantool-patches] [PATCH v2 1/3] " Cyrill Gorcunov via Tarantool-patches 2021-03-22 8:07 ` Konstantin Osipov via Tarantool-patches 2021-03-22 8:30 ` Cyrill Gorcunov via Tarantool-patches 2021-03-22 21:40 ` Vladislav Shpilevoy via Tarantool-patches 2021-03-23 7:28 ` Cyrill Gorcunov via Tarantool-patches 2021-03-23 11:25 ` Cyrill Gorcunov via Tarantool-patches 2021-03-23 12:43 ` Cyrill Gorcunov via Tarantool-patches 2021-03-20 13:15 ` Cyrill Gorcunov via Tarantool-patches [this message] 2021-03-22 21:40 ` [Tarantool-patches] [PATCH v2 2/3] test: add a test for wal_cleanup_delay option Vladislav Shpilevoy via Tarantool-patches 2021-03-20 13:15 ` [Tarantool-patches] [PATCH v2 3/3] test: box-tap/gc -- add test for is_paused field Cyrill Gorcunov via Tarantool-patches
Reply instructions: You may reply publicly to this message via plain-text email using any one of the following methods: * Save the following mbox file, import it into your mail client, and reply-to-all from there: mbox Avoid top-posting and favor interleaved quoting: https://en.wikipedia.org/wiki/Posting_style#Interleaved_style * Reply using the --to, --cc, and --in-reply-to switches of git-send-email(1): git send-email \ --in-reply-to=20210320131521.1249747-3-gorcunov@gmail.com \ --to=tarantool-patches@dev.tarantool.org \ --cc=gorcunov@gmail.com \ --cc=v.perepelitsa@corp.mail.ru \ --cc=v.shpilevoy@tarantool.org \ --subject='Re: [Tarantool-patches] [PATCH v2 2/3] test: add a test for wal_cleanup_delay option' \ /path/to/YOUR_REPLY https://kernel.org/pub/software/scm/git/docs/git-send-email.html * If your mail client supports setting the In-Reply-To header via mailto: links, try the mailto: link
This is a public inbox, see mirroring instructions for how to clone and mirror all data and code used for this inbox