[Tarantool-patches] [PATCH 05/12] raft: stop using instance_id
Serge Petrenko
sergepetrenko at tarantool.org
Tue Nov 17 11:59:03 MSK 2020
17.11.2020 03:02, Vladislav Shpilevoy пишет:
> Raft is being moved to a separate library in src/lib. It means,
> it can't depend on anything from box/.
>
> The patch makes raft stop using instance_id.
>
> Instead, it has a new option 'instance_id'. It is stored inside
> struct raft as 'self', and should be configured using
> raft_cfg_instance_id().
>
> The configuration is done when bootstrap ends and the instance_id
> is either recovered successfully, or the instance is anonymous.
>
> While working on this, I also considered introducing a new
> function raft_boot() instead of raft_cfg_instance_id(). Which I
> would also use to configure vclock later. Raft_boot() would be
> meant to be called only one time with non-dynamic parameters
> instance_id and vclock.
>
> But then I decided to keep adding new raft_cfg_*() functions.
> Because:
>
> - It is more consistent with the existing options;
>
> - Does not require to think about too many different functions
> like raft_create(), raft_boot(), raft_cfg_*() and in which order
> to call them;
>
> Also I was thinking to introduce a single raft_cfg() like I did
> in swim with swim_cfg(), to reduce number of raft_cfg_*()
> functions, but decided it would be even worse with so many
> options.
>
> Part of #5303
> ---
LGTM.
> src/box/box.cc | 10 +++++-----
> src/box/raftlib.c | 32 ++++++++++++++++++++------------
> src/box/raftlib.h | 9 +++++++++
> 3 files changed, 34 insertions(+), 17 deletions(-)
>
> diff --git a/src/box/box.cc b/src/box/box.cc
> index cc0d7b81d..8f5f3558e 100644
> --- a/src/box/box.cc
> +++ b/src/box/box.cc
> @@ -2763,12 +2763,12 @@ box_cfg_xc(void)
> * Fill in leader election parameters after bootstrap. Before it is not
> * possible - there may be relevant data to recover from WAL and
> * snapshot. Also until recovery is done, it is not possible to write
> - * new records into WAL. It is also totally safe, because relaying is
> - * not started until the box is configured. So it can't happen, that
> - * this election-enabled node will try to relay to another
> - * election-enabled node without election actually enabled leading to
> - * disconnect.
> + * new records into WAL. Another reason - before recovery is done,
> + * instance_id is not known, so Raft simply can't work.
> */
> + if (!replication_anon)
> + raft_cfg_instance_id(box_raft(), instance_id);
> +
> if (box_set_election_timeout() != 0)
> diag_raise();
> /*
> diff --git a/src/box/raftlib.c b/src/box/raftlib.c
> index 0657fa85a..ca1940ba6 100644
> --- a/src/box/raftlib.c
> +++ b/src/box/raftlib.c
> @@ -296,7 +296,7 @@ raft_process_msg(struct raft *raft, const struct raft_request *req,
> say_info("RAFT: message %s from %u", raft_request_to_string(req),
> source);
> assert(source > 0);
> - assert(source != instance_id);
> + assert(source != raft->self);
> if (req->term == 0 || req->state == 0) {
> diag_set(ClientError, ER_PROTOCOL, "Raft term and state can't "
> "be zero");
> @@ -337,7 +337,7 @@ raft_process_msg(struct raft *raft, const struct raft_request *req,
> raft->leader);
> break;
> }
> - if (req->vote == instance_id) {
> + if (req->vote == raft->self) {
> /*
> * This is entirely valid. This instance could
> * request a vote, then become a follower or
> @@ -373,7 +373,7 @@ raft_process_msg(struct raft *raft, const struct raft_request *req,
> break;
> case RAFT_STATE_CANDIDATE:
> /* Check if this is a vote for a competing candidate. */
> - if (req->vote != instance_id) {
> + if (req->vote != raft->self) {
> say_info("RAFT: vote request is skipped - "
> "competing candidate");
> break;
> @@ -382,7 +382,7 @@ raft_process_msg(struct raft *raft, const struct raft_request *req,
> * Vote for self was requested earlier in this round,
> * and now was answered by some other instance.
> */
> - assert(raft->volatile_vote == instance_id);
> + assert(raft->volatile_vote == raft->self);
> bool was_set = bit_set(&raft->vote_mask, source);
> raft->vote_count += !was_set;
> if (raft->vote_count < raft->election_quorum) {
> @@ -547,7 +547,7 @@ end_dump:
> } else if (raft->leader != 0) {
> /* There is a known leader. Wait until it is dead. */
> raft_sm_wait_leader_dead(raft);
> - } else if (raft->vote == instance_id) {
> + } else if (raft->vote == raft->self) {
> /* Just wrote own vote. */
> if (raft->election_quorum == 1)
> raft_sm_become_leader(raft);
> @@ -561,7 +561,7 @@ end_dump:
> raft_sm_wait_election_end(raft);
> } else {
> /* No leaders, no votes. */
> - raft_sm_schedule_new_vote(raft, instance_id);
> + raft_sm_schedule_new_vote(raft, raft->self);
> }
> } else {
> memset(&req, 0, sizeof(req));
> @@ -596,7 +596,7 @@ raft_worker_handle_broadcast(struct raft *raft)
> req.vote = raft->vote;
> req.state = raft->state;
> if (req.state == RAFT_STATE_CANDIDATE) {
> - assert(raft->vote == instance_id);
> + assert(raft->vote == raft->self);
> req.vclock = &replicaset.vclock;
> }
> replicaset_foreach(replica)
> @@ -652,7 +652,7 @@ raft_sm_become_leader(struct raft *raft)
> assert(raft->is_candidate);
> assert(!raft->is_write_in_progress);
> raft->state = RAFT_STATE_LEADER;
> - raft->leader = instance_id;
> + raft->leader = raft->self;
> ev_timer_stop(loop(), &raft->timer);
> /* Make read-write (if other subsystems allow that. */
> box_update_ro_summary();
> @@ -682,14 +682,14 @@ raft_sm_become_candidate(struct raft *raft)
> say_info("RAFT: enter candidate state with 1 self vote");
> assert(raft->state == RAFT_STATE_FOLLOWER);
> assert(raft->leader == 0);
> - assert(raft->vote == instance_id);
> + assert(raft->vote == raft->self);
> assert(raft->is_candidate);
> assert(!raft->is_write_in_progress);
> assert(raft->election_quorum > 1);
> raft->state = RAFT_STATE_CANDIDATE;
> raft->vote_count = 1;
> raft->vote_mask = 0;
> - bit_set(&raft->vote_mask, instance_id);
> + bit_set(&raft->vote_mask, raft->self);
> raft_sm_wait_election_end(raft);
> /* State is visible and it is changed - broadcast. */
> raft_schedule_broadcast(raft);
> @@ -736,7 +736,7 @@ raft_sm_schedule_new_election(struct raft *raft)
> assert(raft->is_candidate);
> /* Everyone is a follower until its vote for self is persisted. */
> raft_sm_schedule_new_term(raft, raft->term + 1);
> - raft_sm_schedule_new_vote(raft, instance_id);
> + raft_sm_schedule_new_vote(raft, raft->self);
> box_update_ro_summary();
> }
>
> @@ -783,7 +783,7 @@ raft_sm_wait_election_end(struct raft *raft)
> assert(raft->is_candidate);
> assert(raft->state == RAFT_STATE_FOLLOWER ||
> (raft->state == RAFT_STATE_CANDIDATE &&
> - raft->volatile_vote == instance_id));
> + raft->volatile_vote == raft->self));
> assert(raft->leader == 0);
> double election_timeout = raft->election_timeout +
> raft_new_random_election_shift(raft);
> @@ -979,6 +979,14 @@ raft_cfg_death_timeout(struct raft *raft, double death_timeout)
> }
> }
>
> +void
> +raft_cfg_instance_id(struct raft *raft, uint32_t instance_id)
> +{
> + assert(raft->self == 0);
> + assert(instance_id != 0);
> + raft->self = instance_id;
> +}
> +
> void
> raft_new_term(struct raft *raft)
> {
> diff --git a/src/box/raftlib.h b/src/box/raftlib.h
> index c9c13136e..f75ed2567 100644
> --- a/src/box/raftlib.h
> +++ b/src/box/raftlib.h
> @@ -95,6 +95,8 @@ const char *
> raft_state_str(uint32_t state);
>
> struct raft {
> + /** Instance ID of this node. */
> + uint32_t self;
> /** Instance ID of leader of the current term. */
> uint32_t leader;
> /** State of the instance. */
> @@ -241,6 +243,13 @@ raft_cfg_election_quorum(struct raft *raft, int election_quorum);
> void
> raft_cfg_death_timeout(struct raft *raft, double death_timeout);
>
> +/**
> + * Configure ID of the given Raft instance. The ID can't be changed after it is
> + * assigned first time.
> + */
> +void
> +raft_cfg_instance_id(struct raft *raft, uint32_t instance_id);
> +
> /**
> * Bump the term. When it is persisted, the node checks if there is a leader,
> * and if there is not, a new election is started. That said, this function can
--
Serge Petrenko
More information about the Tarantool-patches
mailing list