[Tarantool-patches] [PATCH 3/8] raft: introduce raft_ev

[Vladislav Shpilevoy <v.shpilevoy@tarantool.org>]

Raft_ev.h/.c encapsulates usage of libev, to a certain extent. All
libev functions are wrapped into raft_ev_* wrappers. Objects and
types are left intact.

This is done in order to be able to replace raft_ev.c in the
soon coming unit tests with fakeev functions. That will allow to
simulate time and catch all the raft events with 100%
reproducibility, and without actual waiting for the events in
real time.

The similar approach is used for swim unit tests.

Original raft core file is built into a new library 'raft_algo'.
It is done for the sake of code coverage in unit tests. A test
could be built by directly referencing raft.c in
unit/CMakeLists.txt, but it can't apply compilation options to it,
including gcov options.

When raft.c is built into a library right where it is defined, it
gets the gcov options, and the code covered by unit tests can be
properly shown.

Part of #5303
---
 src/lib/raft/CMakeLists.txt |  8 +++++
 src/lib/raft/raft.c         | 65 +++++++++++++++++++------------------
 src/lib/raft/raft_ev.c      | 57 ++++++++++++++++++++++++++++++++
 src/lib/raft/raft_ev.h      | 57 ++++++++++++++++++++++++++++++++
 4 files changed, 156 insertions(+), 31 deletions(-)
 create mode 100644 src/lib/raft/raft_ev.c
 create mode 100644 src/lib/raft/raft_ev.h

diff --git a/src/lib/raft/CMakeLists.txt b/src/lib/raft/CMakeLists.txt
index be14817ad..92614b365 100644
--- a/src/lib/raft/CMakeLists.txt
+++ b/src/lib/raft/CMakeLists.txt
@@ -1,7 +1,15 @@
 set(lib_sources
     raft.c
+    raft_ev.c
 )
 
 set_source_files_compile_flags(${lib_sources})
+
 add_library(raft STATIC ${lib_sources})
 target_link_libraries(raft core vclock)
+
+# Algorithm library is for unit tests, and is not self-sufficient. In order to
+# use it some other source file should define test symbols such as raft event
+# loop utilities.
+add_library(raft_algo STATIC raft.c)
+target_link_libraries(raft_algo core vclock)
diff --git a/src/lib/raft/raft.c b/src/lib/raft/raft.c
index f91e21423..4d07d37e5 100644
--- a/src/lib/raft/raft.c
+++ b/src/lib/raft/raft.c
@@ -29,7 +29,7 @@
  * SUCH DAMAGE.
  */
 #include "raft.h"
-
+#include "raft_ev.h"
 #include "exception.h"
 #include "fiber.h"
 #include "tt_static.h"
@@ -477,8 +477,8 @@ raft_process_heartbeat(struct raft *raft, uint32_t source)
 	 * anything was heard from the leader. Then in the timer callback check
 	 * the timestamp, and restart the timer, if it is fine.
 	 */
-	assert(ev_is_active(&raft->timer));
-	ev_timer_stop(loop(), &raft->timer);
+	assert(raft_ev_is_active(&raft->timer));
+	raft_ev_timer_stop(raft_loop(), &raft->timer);
 	raft_sm_wait_leader_dead(raft);
 }
 
@@ -595,7 +595,7 @@ raft_sm_pause_and_dump(struct raft *raft)
 	assert(raft->state == RAFT_STATE_FOLLOWER);
 	if (raft->is_write_in_progress)
 		return;
-	ev_timer_stop(loop(), &raft->timer);
+	raft_ev_timer_stop(raft_loop(), &raft->timer);
 	raft_schedule_async(raft);
 	raft->is_write_in_progress = true;
 }
@@ -611,7 +611,7 @@ raft_sm_become_leader(struct raft *raft)
 	assert(!raft->is_write_in_progress);
 	raft->state = RAFT_STATE_LEADER;
 	raft->leader = raft->self;
-	ev_timer_stop(loop(), &raft->timer);
+	raft_ev_timer_stop(raft_loop(), &raft->timer);
 	/* State is visible and it is changed - broadcast. */
 	raft_schedule_broadcast(raft);
 }
@@ -625,7 +625,7 @@ raft_sm_follow_leader(struct raft *raft, uint32_t leader)
 	raft->state = RAFT_STATE_FOLLOWER;
 	raft->leader = leader;
 	if (!raft->is_write_in_progress && raft->is_candidate) {
-		ev_timer_stop(loop(), &raft->timer);
+		raft_ev_timer_stop(raft_loop(), &raft->timer);
 		raft_sm_wait_leader_dead(raft);
 	}
 	/* State is visible and it is changed - broadcast. */
@@ -701,38 +701,38 @@ raft_sm_schedule_new_election_cb(struct ev_loop *loop, struct ev_timer *timer,
 	(void)events;
 	struct raft *raft = timer->data;
 	assert(timer == &raft->timer);
-	ev_timer_stop(loop, timer);
+	raft_ev_timer_stop(loop, timer);
 	raft_sm_schedule_new_election(raft);
 }
 
 static void
 raft_sm_wait_leader_dead(struct raft *raft)
 {
-	assert(!ev_is_active(&raft->timer));
+	assert(!raft_ev_is_active(&raft->timer));
 	assert(!raft->is_write_in_progress);
 	assert(raft->is_candidate);
 	assert(raft->state == RAFT_STATE_FOLLOWER);
 	assert(raft->leader != 0);
-	ev_timer_set(&raft->timer, raft->death_timeout, raft->death_timeout);
-	ev_timer_start(loop(), &raft->timer);
+	raft_ev_timer_set(&raft->timer, raft->death_timeout, raft->death_timeout);
+	raft_ev_timer_start(raft_loop(), &raft->timer);
 }
 
 static void
 raft_sm_wait_leader_found(struct raft *raft)
 {
-	assert(!ev_is_active(&raft->timer));
+	assert(!raft_ev_is_active(&raft->timer));
 	assert(!raft->is_write_in_progress);
 	assert(raft->is_candidate);
 	assert(raft->state == RAFT_STATE_FOLLOWER);
 	assert(raft->leader == 0);
-	ev_timer_set(&raft->timer, raft->death_timeout, raft->death_timeout);
-	ev_timer_start(loop(), &raft->timer);
+	raft_ev_timer_set(&raft->timer, raft->death_timeout, raft->death_timeout);
+	raft_ev_timer_start(raft_loop(), &raft->timer);
 }
 
 static void
 raft_sm_wait_election_end(struct raft *raft)
 {
-	assert(!ev_is_active(&raft->timer));
+	assert(!raft_ev_is_active(&raft->timer));
 	assert(!raft->is_write_in_progress);
 	assert(raft->is_candidate);
 	assert(raft->state == RAFT_STATE_FOLLOWER ||
@@ -741,15 +741,15 @@ raft_sm_wait_election_end(struct raft *raft)
 	assert(raft->leader == 0);
 	double election_timeout = raft->election_timeout +
 				  raft_new_random_election_shift(raft);
-	ev_timer_set(&raft->timer, election_timeout, election_timeout);
-	ev_timer_start(loop(), &raft->timer);
+	raft_ev_timer_set(&raft->timer, election_timeout, election_timeout);
+	raft_ev_timer_start(raft_loop(), &raft->timer);
 }
 
 static void
 raft_sm_start(struct raft *raft)
 {
 	say_info("RAFT: start state machine");
-	assert(!ev_is_active(&raft->timer));
+	assert(!raft_ev_is_active(&raft->timer));
 	assert(!raft->is_enabled);
 	assert(raft->state == RAFT_STATE_FOLLOWER);
 	raft->is_enabled = true;
@@ -796,7 +796,7 @@ raft_sm_stop(struct raft *raft)
 	if (raft->state == RAFT_STATE_LEADER)
 		raft->leader = 0;
 	raft->state = RAFT_STATE_FOLLOWER;
-	ev_timer_stop(loop(), &raft->timer);
+	raft_ev_timer_stop(raft_loop(), &raft->timer);
 	/* State is visible and changed - broadcast. */
 	raft_schedule_broadcast(raft);
 }
@@ -872,7 +872,7 @@ raft_cfg_is_candidate(struct raft *raft, bool is_candidate)
 	} else {
 		if (raft->state != RAFT_STATE_LEADER) {
 			/* Do not wait for anything while being a voter. */
-			ev_timer_stop(loop(), &raft->timer);
+			raft_ev_timer_stop(raft_loop(), &raft->timer);
 		}
 		if (raft->state != RAFT_STATE_FOLLOWER) {
 			if (raft->state == RAFT_STATE_LEADER)
@@ -892,12 +892,13 @@ raft_cfg_election_timeout(struct raft *raft, double timeout)
 
 	raft->election_timeout = timeout;
 	if (raft->vote != 0 && raft->leader == 0 && raft->is_candidate) {
-		assert(ev_is_active(&raft->timer));
-		double timeout = ev_timer_remaining(loop(), &raft->timer) -
+		assert(raft_ev_is_active(&raft->timer));
+		struct ev_loop *loop = raft_loop();
+		double timeout = raft_ev_timer_remaining(loop, &raft->timer) -
 				 raft->timer.at + raft->election_timeout;
-		ev_timer_stop(loop(), &raft->timer);
-		ev_timer_set(&raft->timer, timeout, timeout);
-		ev_timer_start(loop(), &raft->timer);
+		raft_ev_timer_stop(loop, &raft->timer);
+		raft_ev_timer_set(&raft->timer, timeout, timeout);
+		raft_ev_timer_start(loop, &raft->timer);
 	}
 }
 
@@ -918,12 +919,13 @@ raft_cfg_death_timeout(struct raft *raft, double death_timeout)
 	raft->death_timeout = death_timeout;
 	if (raft->state == RAFT_STATE_FOLLOWER && raft->is_candidate &&
 	    raft->leader != 0) {
-		assert(ev_is_active(&raft->timer));
-		double timeout = ev_timer_remaining(loop(), &raft->timer) -
+		assert(raft_ev_is_active(&raft->timer));
+		struct ev_loop *loop = raft_loop();
+		double timeout = raft_ev_timer_remaining(loop, &raft->timer) -
 				 raft->timer.at + raft->death_timeout;
-		ev_timer_stop(loop(), &raft->timer);
-		ev_timer_set(&raft->timer, timeout, timeout);
-		ev_timer_start(loop(), &raft->timer);
+		raft_ev_timer_stop(loop, &raft->timer);
+		raft_ev_timer_set(&raft->timer, timeout, timeout);
+		raft_ev_timer_start(loop, &raft->timer);
 	}
 }
 
@@ -980,7 +982,8 @@ raft_create(struct raft *raft, const struct raft_vtab *vtab)
 		.death_timeout = 5,
 		.vtab = vtab,
 	};
-	ev_timer_init(&raft->timer, raft_sm_schedule_new_election_cb, 0, 0);
+	raft_ev_timer_init(&raft->timer, raft_sm_schedule_new_election_cb,
+			   0, 0);
 	raft->timer.data = raft;
 	rlist_create(&raft->on_update);
 }
@@ -988,6 +991,6 @@ raft_create(struct raft *raft, const struct raft_vtab *vtab)
 void
 raft_destroy(struct raft *raft)
 {
-	ev_timer_stop(loop(), &raft->timer);
+	raft_ev_timer_stop(raft_loop(), &raft->timer);
 	trigger_destroy(&raft->on_update);
 }
diff --git a/src/lib/raft/raft_ev.c b/src/lib/raft/raft_ev.c
new file mode 100644
index 000000000..ce99b3dff
--- /dev/null
+++ b/src/lib/raft/raft_ev.c
@@ -0,0 +1,57 @@
+/*
+ * Copyright 2010-2020, Tarantool AUTHORS, please see AUTHORS file.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above
+ *    copyright notice, this list of conditions and the
+ *    following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above
+ *    copyright notice, this list of conditions and the following
+ *    disclaimer in the documentation and/or other materials
+ *    provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY <COPYRIGHT HOLDER> ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+ * <COPYRIGHT HOLDER> OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+ * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+#include "raft_ev.h"
+#include "tarantool_ev.h"
+#include "fiber.h"
+
+void
+raft_ev_timer_start(struct ev_loop *loop, struct ev_timer *watcher)
+{
+	ev_timer_start(loop, watcher);
+}
+
+double
+raft_ev_timer_remaining(struct ev_loop *loop, struct ev_timer *watcher)
+{
+	return ev_timer_remaining(loop, watcher);
+}
+
+void
+raft_ev_timer_stop(struct ev_loop *loop, struct ev_timer *watcher)
+{
+	ev_timer_stop(loop, watcher);
+}
+
+struct ev_loop *
+raft_loop(void)
+{
+	return loop();
+}
diff --git a/src/lib/raft/raft_ev.h b/src/lib/raft/raft_ev.h
new file mode 100644
index 000000000..cfa663743
--- /dev/null
+++ b/src/lib/raft/raft_ev.h
@@ -0,0 +1,57 @@
+#pragma once
+/*
+ * Copyright 2010-2020, Tarantool AUTHORS, please see AUTHORS file.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above
+ *    copyright notice, this list of conditions and the
+ *    following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above
+ *    copyright notice, this list of conditions and the following
+ *    disclaimer in the documentation and/or other materials
+ *    provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY <COPYRIGHT HOLDER> ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+ * <COPYRIGHT HOLDER> OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+ * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+struct ev_loop;
+struct ev_timer;
+
+/**
+ * A set of wrappers around libev functions. Libev is not used directly, because
+ * in unit tests these wrappers are redefined to use fakeev event loop in order
+ * to speed the tests up, and have more control over the event loop.
+ */
+
+void
+raft_ev_timer_start(struct ev_loop *loop, struct ev_timer *watcher);
+
+double
+raft_ev_timer_remaining(struct ev_loop *loop, struct ev_timer *watcher);
+
+void
+raft_ev_timer_stop(struct ev_loop *loop, struct ev_timer *watcher);
+
+struct ev_loop *
+raft_loop(void);
+
+#define raft_ev_is_active ev_is_active
+
+#define raft_ev_timer_init ev_timer_init
+
+#define raft_ev_timer_set ev_timer_set
-- 
2.24.3 (Apple Git-128)