From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from localhost (localhost [127.0.0.1]) by turing.freelists.org (Avenir Technologies Mail Multiplex) with ESMTP id 6BCE726564 for ; Sat, 9 Jun 2018 13:47:33 -0400 (EDT) Received: from turing.freelists.org ([127.0.0.1]) by localhost (turing.freelists.org [127.0.0.1]) (amavisd-new, port 10024) with ESMTP id 0Kg5AwBQv0cW for ; Sat, 9 Jun 2018 13:47:33 -0400 (EDT) Received: from smtp42.i.mail.ru (smtp42.i.mail.ru [94.100.177.102]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by turing.freelists.org (Avenir Technologies Mail Multiplex) with ESMTPS id 191A926490 for ; Sat, 9 Jun 2018 13:47:33 -0400 (EDT) From: AKhatskevich Subject: [tarantool-patches] [PATCH 1/2] Add test on error during reconfigure Date: Sat, 9 Jun 2018 20:47:15 +0300 Message-Id: <489ade011c878e28236afe2792e0eddb1ded75b9.1528566184.git.avkhatskevich@tarantool.org> In-Reply-To: References: In-Reply-To: References: Sender: tarantool-patches-bounce@freelists.org Errors-to: tarantool-patches-bounce@freelists.org Reply-To: tarantool-patches@freelists.org List-help: List-unsubscribe: List-software: Ecartis version 1.0.0 List-Id: tarantool-patches List-subscribe: List-owner: List-post: List-archive: To: tarantool-patches@freelists.org, v.shpilevoy@tarantool.org In case reconfigure process fails, the node should continue work properly. --- test/lua_libs/util.lua | 16 ++++++++++++++++ test/router/router.result | 27 +++++++++++++++++++++++++++ test/router/router.test.lua | 10 ++++++++++ test/storage/storage.result | 33 +++++++++++++++++++++++++++++++++ test/storage/storage.test.lua | 12 ++++++++++++ vshard/router/init.lua | 7 +++++++ vshard/storage/init.lua | 9 +++++++++ 7 files changed, 114 insertions(+) diff --git a/test/lua_libs/util.lua b/test/lua_libs/util.lua index f2d3b48..aeb2342 100644 --- a/test/lua_libs/util.lua +++ b/test/lua_libs/util.lua @@ -69,9 +69,25 @@ local function wait_master(test_run, replicaset, master) log.info('Slaves are connected to a master "%s"', master) end +-- Check that data has at least all fields as an ethalon. +local function has_same_fields(ethalon, data) + assert(type(ethalon) == 'table' and type(data) == 'table') + local diff = {} + for k, v in pairs(ethalon) do + if v ~= data[k] then + table.insert(diff, k) + end + end + if #diff > 0 then + return false, diff + end + return true +end + return { check_error = check_error, shuffle_masters = shuffle_masters, collect_timeouts = collect_timeouts, wait_master = wait_master, + has_same_fields = has_same_fields, } diff --git a/test/router/router.result b/test/router/router.result index 2ee1bff..3ebab5d 100644 --- a/test/router/router.result +++ b/test/router/router.result @@ -1057,6 +1057,33 @@ error_messages - - Use replica:is_connected(...) instead of replica.is_connected(...) - Use replica:safe_uri(...) instead of replica.safe_uri(...) ... +-- Error during reconfigure process. +_ = vshard.router.route(1):callro('echo', {'some_data'}) +--- +... +vshard.router.internal.errinj.ERRINJ_CFG = true +--- +... +old_internal = table.copy(vshard.router.internal) +--- +... +_, err = pcall(vshard.router.cfg, cfg) +--- +... +err:match('Error injection:.*') +--- +- 'Error injection: cfg' +... +vshard.router.internal.errinj.ERRINJ_CFG = false +--- +... +util.has_same_fields(old_internal, vshard.router.internal) +--- +- true +... +_ = vshard.router.route(1):callro('echo', {'some_data'}) +--- +... _ = test_run:cmd("switch default") --- ... diff --git a/test/router/router.test.lua b/test/router/router.test.lua index fae8e24..afcdb9d 100644 --- a/test/router/router.test.lua +++ b/test/router/router.test.lua @@ -389,6 +389,16 @@ end; test_run:cmd("setopt delimiter ''"); error_messages +-- Error during reconfigure process. +_ = vshard.router.route(1):callro('echo', {'some_data'}) +vshard.router.internal.errinj.ERRINJ_CFG = true +old_internal = table.copy(vshard.router.internal) +_, err = pcall(vshard.router.cfg, cfg) +err:match('Error injection:.*') +vshard.router.internal.errinj.ERRINJ_CFG = false +util.has_same_fields(old_internal, vshard.router.internal) +_ = vshard.router.route(1):callro('echo', {'some_data'}) + _ = test_run:cmd("switch default") test_run:drop_cluster(REPLICASET_2) diff --git a/test/storage/storage.result b/test/storage/storage.result index d0bf792..8d88bf4 100644 --- a/test/storage/storage.result +++ b/test/storage/storage.result @@ -720,6 +720,39 @@ test_run:cmd("setopt delimiter ''"); --- - true ... +-- Error during reconfigure process. +_, rs = next(vshard.storage.internal.replicasets) +--- +... +_ = rs:callro('echo', {'some_data'}) +--- +... +vshard.storage.internal.errinj.ERRINJ_CFG = true +--- +... +old_internal = table.copy(vshard.storage.internal) +--- +... +_, err = pcall(vshard.storage.cfg, cfg, names.storage_1_a) +--- +... +err:match('Error injection:.*') +--- +- 'Error injection: cfg' +... +vshard.storage.internal.errinj.ERRINJ_CFG = false +--- +... +util.has_same_fields(old_internal, vshard.storage.internal) +--- +- true +... +_, rs = next(vshard.storage.internal.replicasets) +--- +... +_ = rs:callro('echo', {'some_data'}) +--- +... _ = test_run:cmd("switch default") --- ... diff --git a/test/storage/storage.test.lua b/test/storage/storage.test.lua index f4bbf0e..d215db6 100644 --- a/test/storage/storage.test.lua +++ b/test/storage/storage.test.lua @@ -177,6 +177,18 @@ for _, new_replicaset in pairs(new_replicasets) do end; test_run:cmd("setopt delimiter ''"); +-- Error during reconfigure process. +_, rs = next(vshard.storage.internal.replicasets) +_ = rs:callro('echo', {'some_data'}) +vshard.storage.internal.errinj.ERRINJ_CFG = true +old_internal = table.copy(vshard.storage.internal) +_, err = pcall(vshard.storage.cfg, cfg, names.storage_1_a) +err:match('Error injection:.*') +vshard.storage.internal.errinj.ERRINJ_CFG = false +util.has_same_fields(old_internal, vshard.storage.internal) +_, rs = next(vshard.storage.internal.replicasets) +_ = rs:callro('echo', {'some_data'}) + _ = test_run:cmd("switch default") test_run:drop_cluster(REPLICASET_2) diff --git a/vshard/router/init.lua b/vshard/router/init.lua index 21093e5..1dee80c 100644 --- a/vshard/router/init.lua +++ b/vshard/router/init.lua @@ -11,6 +11,7 @@ local M = rawget(_G, '__module_vshard_router') if not M then M = { errinj = { + ERRINJ_CFG = false, ERRINJ_FAILOVER_CHANGE_CFG = false, ERRINJ_RELOAD = false, }, @@ -473,6 +474,12 @@ local function router_cfg(cfg) end box.cfg(cfg) log.info("Box has been configured") + -- It is considered that all possible errors during cfg + -- process occur only before this place. + -- This check should be placed as late as possible. + if M.errinj.ERRINJ_CFG then + error('Error injection: cfg') + end M.total_bucket_count = total_bucket_count M.collect_lua_garbage = collect_lua_garbage -- TODO: update existing route map in-place diff --git a/vshard/storage/init.lua b/vshard/storage/init.lua index 57076e1..879c7c4 100644 --- a/vshard/storage/init.lua +++ b/vshard/storage/init.lua @@ -33,6 +33,7 @@ if not M then -- Bucket count stored on all replicasets. total_bucket_count = 0, errinj = { + ERRINJ_CFG = false, ERRINJ_BUCKET_FIND_GARBAGE_DELAY = false, ERRINJ_RELOAD = false, ERRINJ_CFG_DELAY = false, @@ -1527,6 +1528,14 @@ local function storage_cfg(cfg, this_replica_uuid) local shard_index = cfg.shard_index local collect_bucket_garbage_interval = cfg.collect_bucket_garbage_interval local collect_lua_garbage = cfg.collect_lua_garbage + + -- It is considered that all possible errors during cfg + -- process occur only before this place. + -- This check should be placed as late as possible. + if M.errinj.ERRINJ_CFG then + error('Error injection: cfg') + end + -- -- Sync timeout is a special case - it must be updated before -- all other options to allow a user to demote a master with -- 2.14.1