From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from smtpng2.m.smailru.net (smtpng2.m.smailru.net [94.100.179.3]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by dev.tarantool.org (Postfix) with ESMTPS id 9AFCF469719 for ; Sat, 19 Sep 2020 18:58:30 +0300 (MSK) From: Vladislav Shpilevoy References: Message-ID: <5a7cf2f2-1b7c-28be-688f-d604a64d0623@tarantool.org> Date: Sat, 19 Sep 2020 17:58:28 +0200 MIME-Version: 1.0 In-Reply-To: Content-Type: text/plain; charset=utf-8 Content-Language: en-US Content-Transfer-Encoding: 7bit Subject: [Tarantool-patches] [PATCH v2 12/11] dRaft List-Id: Tarantool development patches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , To: tarantool-patches@dev.tarantool.org, sergepetrenko@tarantool.org, gorcunov@gmail.com raft: add tests Part of #1146 diff --git a/test/replication/raft_basic.result b/test/replication/raft_basic.result new file mode 100644 index 000000000..cff710e24 --- /dev/null +++ b/test/replication/raft_basic.result @@ -0,0 +1,168 @@ +-- test-run result file version 2 +test_run = require('test_run').new() + | --- + | ... +-- +-- gh-1146: Raft protocol for automated leader election. +-- + +old_raft_election_timeout = box.cfg.raft_election_timeout + | --- + | ... + +-- Raft is turned off by default. +assert(not box.cfg.raft_is_enabled) + | --- + | - true + | ... +-- Is candidate by default. Although it does not matter, until Raft is +-- turned on. +assert(box.cfg.raft_is_candidate) + | --- + | - true + | ... +-- Ensure raft options are validated. +box.cfg{raft_is_enabled = 100} + | --- + | - error: 'Incorrect value for option ''raft_is_enabled'': should be of type boolean' + | ... +box.cfg{raft_is_candidate = 100} + | --- + | - error: 'Incorrect value for option ''raft_is_candidate'': should be of type boolean' + | ... +box.cfg{raft_election_timeout = -1} + | --- + | - error: 'Incorrect value for option ''raft_election_timeout'': the value must be + | a positive number' + | ... +box.cfg{raft_election_timeout = 0} + | --- + | - error: 'Incorrect value for option ''raft_election_timeout'': the value must be + | a positive number' + | ... + +-- When Raft is disabled, the instance is a follower. Does not +-- try to become a leader, and does not block write operations. +term = box.info.raft.term + | --- + | ... +assert(box.info.raft.vote == 0) + | --- + | - true + | ... +assert(box.info.raft.state == 'follower') + | --- + | - true + | ... +assert(box.info.raft.leader == 0) + | --- + | - true + | ... +assert(not box.info.ro) + | --- + | - true + | ... + +-- Turned on Raft blocks writes until the instance becomes a +-- leader. +box.cfg{raft_is_candidate = false} + | --- + | ... +box.cfg{raft_is_enabled = true} + | --- + | ... +assert(box.info.raft.state == 'follower') + | --- + | - true + | ... +assert(box.info.ro) + | --- + | - true + | ... +-- Term is not changed, because the instance can't be a candidate, +-- and therefore didn't try to vote nor to bump the term. +assert(box.info.raft.term == term) + | --- + | - true + | ... +assert(box.info.raft.vote == 0) + | --- + | - true + | ... +assert(box.info.raft.leader == 0) + | --- + | - true + | ... + +-- Candidate instance votes immediately, if sees no leader. +box.cfg{raft_election_timeout = 1000} + | --- + | ... +box.cfg{raft_is_candidate = true} + | --- + | ... +test_run:wait_cond(function() return box.info.raft.state == 'leader' end) + | --- + | - true + | ... +assert(box.info.raft.term > term) + | --- + | - true + | ... +assert(box.info.raft.vote == box.info.id) + | --- + | - true + | ... +assert(box.info.raft.leader == box.info.id) + | --- + | - true + | ... + +box.cfg{ \ + raft_is_enabled = false, \ + raft_is_candidate = true, \ + raft_election_timeout = old_raft_election_timeout \ +} + | --- + | ... + +-- +-- See if bootstrap with Raft enabled works. +-- +SERVERS = {'raft_replica1', 'raft_replica2', 'raft_replica3'} + | --- + | ... +test_run:create_cluster(SERVERS, "replication") + | --- + | ... +test_run:wait_fullmesh(SERVERS) + | --- + | ... +is_leader_cmd = 'return box.info.raft.state == \'leader\'' + | --- + | ... +is_r1_leader = test_run:eval('raft_replica1', is_leader_cmd)[1] + | --- + | ... +is_r2_leader = test_run:eval('raft_replica2', is_leader_cmd)[1] + | --- + | ... +is_r3_leader = test_run:eval('raft_replica3', is_leader_cmd)[1] + | --- + | ... +leader_count = is_r1_leader and 1 or 0 + | --- + | ... +leader_count = leader_count + (is_r2_leader and 1 or 0) + | --- + | ... +leader_count = leader_count + (is_r3_leader and 1 or 0) + | --- + | ... +assert(leader_count == 1) + | --- + | - true + | ... +test_run:drop_cluster(SERVERS) + | --- + | ... diff --git a/test/replication/raft_basic.test.lua b/test/replication/raft_basic.test.lua new file mode 100644 index 000000000..faf3ad107 --- /dev/null +++ b/test/replication/raft_basic.test.lua @@ -0,0 +1,67 @@ +test_run = require('test_run').new() +-- +-- gh-1146: Raft protocol for automated leader election. +-- + +old_raft_election_timeout = box.cfg.raft_election_timeout + +-- Raft is turned off by default. +assert(not box.cfg.raft_is_enabled) +-- Is candidate by default. Although it does not matter, until Raft is +-- turned on. +assert(box.cfg.raft_is_candidate) +-- Ensure raft options are validated. +box.cfg{raft_is_enabled = 100} +box.cfg{raft_is_candidate = 100} +box.cfg{raft_election_timeout = -1} +box.cfg{raft_election_timeout = 0} + +-- When Raft is disabled, the instance is a follower. Does not +-- try to become a leader, and does not block write operations. +term = box.info.raft.term +assert(box.info.raft.vote == 0) +assert(box.info.raft.state == 'follower') +assert(box.info.raft.leader == 0) +assert(not box.info.ro) + +-- Turned on Raft blocks writes until the instance becomes a +-- leader. +box.cfg{raft_is_candidate = false} +box.cfg{raft_is_enabled = true} +assert(box.info.raft.state == 'follower') +assert(box.info.ro) +-- Term is not changed, because the instance can't be a candidate, +-- and therefore didn't try to vote nor to bump the term. +assert(box.info.raft.term == term) +assert(box.info.raft.vote == 0) +assert(box.info.raft.leader == 0) + +-- Candidate instance votes immediately, if sees no leader. +box.cfg{raft_election_timeout = 1000} +box.cfg{raft_is_candidate = true} +test_run:wait_cond(function() return box.info.raft.state == 'leader' end) +assert(box.info.raft.term > term) +assert(box.info.raft.vote == box.info.id) +assert(box.info.raft.leader == box.info.id) + +box.cfg{ \ + raft_is_enabled = false, \ + raft_is_candidate = true, \ + raft_election_timeout = old_raft_election_timeout \ +} + +-- +-- See if bootstrap with Raft enabled works. +-- +SERVERS = {'raft_replica1', 'raft_replica2', 'raft_replica3'} +test_run:create_cluster(SERVERS, "replication") +test_run:wait_fullmesh(SERVERS) +is_leader_cmd = 'return box.info.raft.state == \'leader\'' +is_r1_leader = test_run:eval('raft_replica1', is_leader_cmd)[1] +is_r2_leader = test_run:eval('raft_replica2', is_leader_cmd)[1] +is_r3_leader = test_run:eval('raft_replica3', is_leader_cmd)[1] +leader_count = is_r1_leader and 1 or 0 +leader_count = leader_count + (is_r2_leader and 1 or 0) +leader_count = leader_count + (is_r3_leader and 1 or 0) +assert(leader_count == 1) +test_run:drop_cluster(SERVERS) diff --git a/test/replication/raft_replica.lua b/test/replication/raft_replica.lua new file mode 100644 index 000000000..8b4c740d8 --- /dev/null +++ b/test/replication/raft_replica.lua @@ -0,0 +1,30 @@ +#!/usr/bin/env tarantool + +local INSTANCE_ID = string.match(arg[0], "%d") +local SOCKET_DIR = require('fio').cwd() + +local function instance_uri(instance_id) + return SOCKET_DIR..'/autobootstrap'..instance_id..'.sock'; +end + +require('console').listen(os.getenv('ADMIN')) + +box.cfg({ + listen = instance_uri(INSTANCE_ID), + replication = { + instance_uri(1), + instance_uri(2), + instance_uri(3), + }, + replication_timeout = 0.1, + raft_is_enabled = true, + raft_is_candidate = true, + raft_election_timeout = 0.1, + replication_synchro_quorum = 3, + -- To reveal more Raft logs. + log_level = 6, +}) + +box.once("bootstrap", function() + box.schema.user.grant('guest', 'super') +end) diff --git a/test/replication/raft_replica1.lua b/test/replication/raft_replica1.lua new file mode 120000 index 000000000..d1f4d6f25 --- /dev/null +++ b/test/replication/raft_replica1.lua @@ -0,0 +1 @@ +raft_replica.lua \ No newline at end of file diff --git a/test/replication/raft_replica2.lua b/test/replication/raft_replica2.lua new file mode 120000 index 000000000..d1f4d6f25 --- /dev/null +++ b/test/replication/raft_replica2.lua @@ -0,0 +1 @@ +raft_replica.lua \ No newline at end of file diff --git a/test/replication/raft_replica3.lua b/test/replication/raft_replica3.lua new file mode 120000 index 000000000..d1f4d6f25 --- /dev/null +++ b/test/replication/raft_replica3.lua @@ -0,0 +1 @@ +raft_replica.lua \ No newline at end of file