From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from localhost (localhost [127.0.0.1]) by turing.freelists.org (Avenir Technologies Mail Multiplex) with ESMTP id 03EAF2A6B7 for ; Thu, 30 Aug 2018 19:38:33 -0400 (EDT) Received: from turing.freelists.org ([127.0.0.1]) by localhost (turing.freelists.org [127.0.0.1]) (amavisd-new, port 10024) with ESMTP id DkeXEppqRvnL for ; Thu, 30 Aug 2018 19:38:32 -0400 (EDT) Received: from mail-lj1-f195.google.com (mail-lj1-f195.google.com [209.85.208.195]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by turing.freelists.org (Avenir Technologies Mail Multiplex) with ESMTPS id 85AFF2A6B5 for ; Thu, 30 Aug 2018 19:38:32 -0400 (EDT) Received: by mail-lj1-f195.google.com with SMTP id p6-v6so8658399ljc.5 for ; Thu, 30 Aug 2018 16:38:32 -0700 (PDT) From: Olga Arkhangelskaia Subject: [tarantool-patches] [PATCH v7 3/3] box: adds replication sync after cfg. update Date: Fri, 31 Aug 2018 02:38:04 +0300 Message-Id: <20180830233804.20727-3-krishtal.olja@gmail.com> In-Reply-To: <20180830233804.20727-1-krishtal.olja@gmail.com> References: <20180830233804.20727-1-krishtal.olja@gmail.com> Sender: tarantool-patches-bounce@freelists.org Errors-to: tarantool-patches-bounce@freelists.org Reply-To: tarantool-patches@freelists.org List-help: List-unsubscribe: List-software: Ecartis version 1.0.0 List-Id: tarantool-patches List-subscribe: List-owner: List-post: List-archive: To: tarantool-patches@freelists.org Cc: Olga Arkhangelskaia When replica reconnects to replica set not for the first time, we suffer from absence of synchronization. Such behavior leads to giving away outdated data. Closes #3427 @TarantoolBot document Title: Orphan status after configuration update or initial bootstrap. In case of initial bootstrap or after configuration update we can get an orphan status in two cases. If we synced up with number of replicas that is smaller than quorum or if we failed to sync up during the time specified in replication_sync_timeout. --- https://github.com/tarantool/tarantool/issues/3427 https://github.com/tarantool/tarantool/tree/OKriw/gh-3427-replication-no-sync-1.9 v1: https://www.freelists.org/post/tarantool-patches/PATCH-replication-adds-replication-sync-after-cfg-update v2: https://www.freelists.org/post/tarantool-patches/PATCH-v2-replication-adds-replication-sync-after-cfg-update v3: https://www.freelists.org/post/tarantool-patches/PATCH-v3-box-adds-replication-sync-after-cfg-update v4: https://www.freelists.org/post/tarantool-patches/PATCH-v4-22-box-adds-replication-sync-after-cfg-update v5: https://www.freelists.org/post/tarantool-patches/PATCH-v5-33-box-adds-replication-sync-after-cfg-update v6: https://www.freelists.org/post/tarantool-patches/PATCH-v6-33-box-adds-replication-sync-after-cfg-update Changes in v2: - fixed test - changed replicaset_sync Changes in v3: - now we raise the exception when sync is not successful. - fixed test - renamed test Changes in v4: - fixed test - replication_sync_lag is made dynamicall in separate patch - removed unnecessary error type - moved say_crit to another place - in case of sync error we rollback to prev. config Changes in v5: - added test case - now we don't roll back to prev. cfg Changes in v6: - set orphan - added testcases Changes in v7: - fixed test with orphan state (added error inj.) - now we enter orphan state in the end of sync - this point need to be discussed - no good check for big replication lag - need to be discussed src/box/box.cc | 17 +++++ src/box/box.h | 8 +++ src/box/replication.cc | 1 + test/replication/sync.result | 146 +++++++++++++++++++++++++++++++++++++++++ test/replication/sync.test.lua | 73 +++++++++++++++++++++ 5 files changed, 245 insertions(+) create mode 100644 test/replication/sync.result create mode 100644 test/replication/sync.test.lua diff --git a/src/box/box.cc b/src/box/box.cc index dcedfd002..ca7bb60c2 100644 --- a/src/box/box.cc +++ b/src/box/box.cc @@ -231,6 +231,19 @@ box_clear_orphan(void) title("running"); } +void +box_set_orphan(void) +{ + if (is_orphan) + return; /* nothing to do */ + + is_orphan = true; + fiber_cond_broadcast(&ro_cond); + + /* Update the title to reflect the new status. */ + title("orphan"); +} + struct wal_stream { struct xstream base; /** How many rows have been recovered so far. */ @@ -646,6 +659,10 @@ box_set_replication(void) box_sync_replication(true); /* Follow replica */ replicaset_follow(); + /* Set orphan and sync replica up to quorum. + * If we fail to sync up, replica will be left in orphan state. + */ + replicaset_sync(); } void diff --git a/src/box/box.h b/src/box/box.h index 6e1c13f59..15a305fb1 100644 --- a/src/box/box.h +++ b/src/box/box.h @@ -107,6 +107,14 @@ box_wait_ro(bool ro, double timeout); void box_clear_orphan(void); +/** + * Switch this instance from 'running' to 'orphan' state. + * Called on configuration change if this instance failed to + * synchronizes with enough replicas to form a quorum. + */ +void +box_set_orphan(void); + /** True if snapshot is in progress. */ extern bool box_checkpoint_is_in_progress; /** Incremented with each next snapshot. */ diff --git a/src/box/replication.cc b/src/box/replication.cc index 4001c86a3..ef365c17f 100644 --- a/src/box/replication.cc +++ b/src/box/replication.cc @@ -695,6 +695,7 @@ replicaset_sync(void) * in 'orphan' state. */ say_crit("entering orphan mode"); + box_set_orphan(); return; } diff --git a/test/replication/sync.result b/test/replication/sync.result new file mode 100644 index 000000000..bca3250d1 --- /dev/null +++ b/test/replication/sync.result @@ -0,0 +1,146 @@ +fiber = require('fiber') +--- +... +-- +-- gh-3427: no sync after configuration update +-- +-- +-- successful sync +-- +env = require('test_run') +--- +... +test_run = env.new() +--- +... +engine = test_run:get_cfg('engine') +--- +... +box.schema.user.grant('guest', 'replication') +--- +... +test_run:cmd("create server replica with rpl_master=default, script='replication/replica.lua'") +--- +- true +... +test_run:cmd("start server replica") +--- +- true +... +s = box.schema.space.create('test', {engine = engine}) +--- +... +index = s:create_index('primary') +--- +... +-- change replica configuration +test_run:cmd("switch replica") +--- +- true +... +replication = box.cfg.replication +--- +... +box.cfg{replication={}} +--- +... +test_run:cmd("switch default") +--- +- true +... +-- insert values on the master while replica is unconfigured +box.begin() for i = 1, 100 do box.space.test:insert{i, i} end box.commit() +--- +... +box.space.test:count() +--- +- 100 +... +test_run:cmd("switch replica") +--- +- true +... +box.cfg{replication = replication} +--- +... +box.space.test:count() == 100 +--- +- true +... +-- +-- unsuccessful sync entering orphan state +-- +box.cfg{replication={}} +--- +... +box.cfg{replication_sync_timeout = 0.000001} +--- +... +test_run:cmd("switch default") +--- +- true +... +-- insert values on the master while replica is unconfigured +box.begin() for i = 101, 200 do box.space.test:insert{i, i} end box.commit() +--- +... +test_run:cmd("switch replica") +--- +- true +... +box.cfg{replication = replication} +--- +... +status = box.info.status +--- +... +status == "orphan" +--- +- true +... +while status == "orphan" do require'fiber'.sleep(0.1) status = box.info.status end +--- +... +-- +-- replication_sync_lag is too big +-- +box.cfg{replication_sync_lag = 100} +--- +... +test_run:cmd("switch default") +--- +- true +... +function f () box.begin() for i = 201, 500 do box.space.test:insert{i, i} end box.commit(); end +--- +... +_=fiber.create(f) +--- +... +test_run:cmd("switch replica") +--- +- true +... +box.space.test:count() < 500 +--- +- true +... +test_run:cmd("switch default") +--- +- true +... +-- cleanup +test_run:cmd("stop server replica") +--- +- true +... +test_run:cmd("cleanup server replica") +--- +- true +... +box.space.test:drop() +--- +... +box.schema.user.revoke('guest', 'replication') +--- +... diff --git a/test/replication/sync.test.lua b/test/replication/sync.test.lua new file mode 100644 index 000000000..c15581a50 --- /dev/null +++ b/test/replication/sync.test.lua @@ -0,0 +1,73 @@ +fiber = require('fiber') +-- +-- gh-3427: no sync after configuration update +-- + +-- +-- successful sync +-- + +env = require('test_run') +test_run = env.new() +engine = test_run:get_cfg('engine') + +box.schema.user.grant('guest', 'replication') + +test_run:cmd("create server replica with rpl_master=default, script='replication/replica.lua'") +test_run:cmd("start server replica") + +s = box.schema.space.create('test', {engine = engine}) +index = s:create_index('primary') + +-- change replica configuration +test_run:cmd("switch replica") +replication = box.cfg.replication +box.cfg{replication={}} + +test_run:cmd("switch default") +-- insert values on the master while replica is unconfigured +box.begin() for i = 1, 100 do box.space.test:insert{i, i} end box.commit() +box.space.test:count() + +test_run:cmd("switch replica") +box.cfg{replication = replication} +box.space.test:count() == 100 + +-- +-- unsuccessful sync entering orphan state +-- +box.cfg{replication={}} +box.cfg{replication_sync_timeout = 1} + +test_run:cmd("switch default") +-- insert values on the master while replica is unconfigured +box.begin() for i = 101, 200 do box.space.test:insert{i, i} end box.commit() +box.error.injection.set("ERRINJ_RELAY_TIMEOUT", 0.01) +test_run:cmd("switch replica") + +box.cfg{replication = replication} +status = box.info.status +status == "orphan" + +while status == "orphan" do require'fiber'.sleep(0.1) status = box.info.status end + +-- +-- replication_sync_lag is too big +-- + +box.cfg{replication_sync_lag = 100} + +test_run:cmd("switch default") + +function f () box.begin() for i = 201, 500 do box.space.test:insert{i, i} end box.commit(); end +_=fiber.create(f) + +test_run:cmd("switch replica") +box.space.test:count() < 500 + +test_run:cmd("switch default") +-- cleanup +test_run:cmd("stop server replica") +test_run:cmd("cleanup server replica") +box.space.test:drop() +box.schema.user.revoke('guest', 'replication') -- 2.14.3 (Apple Git-98)