[tarantool-patches] [PATCH 1/1] swim: keep encoded round message cached

[Vladislav Shpilevoy]

During a SWIM round a message is being handed out consisting of
at most 4 sections. Parts of the message change rarely along with
a member attribute update, or with removal of a member. So it is
possible to cache the message and send it during several round
steps in a row. Or even do not rebuild it the whole round.

Part of #3234
---
Branch: https://github.com/tarantool/tarantool/tree/gerold103/gh-3234-swim-cached-msg
Issue: https://github.com/tarantool/tarantool/issues/3234

 src/lib/swim/swim.c    | 24 ++++++++++++++++++++----
 src/lib/swim/swim_io.h |  7 +++++++
 2 files changed, 27 insertions(+), 4 deletions(-)

diff --git a/src/lib/swim/swim.c b/src/lib/swim/swim.c
index c64b8df3a..8453295a5 100644
--- a/src/lib/swim/swim.c
+++ b/src/lib/swim/swim.c
@@ -417,6 +417,13 @@ struct swim {
 	struct rlist dissemination_queue;
 };
 
+/** Reset cached round message on any change of any member. */
+static inline void
+cached_round_msg_invalidate(struct swim *swim)
+{
+	swim_packet_create(&swim->round_step_task.packet);
+}
+
 /** Put the member into a list of ACK waiters. */
 static void
 swim_wait_ack(struct swim *swim, struct swim_member *member)
@@ -456,6 +463,7 @@ static void
 swim_on_member_update(struct swim *swim, struct swim_member *member)
 {
 	member->unacknowledged_pings = 0;
+	cached_round_msg_invalidate(swim);
 	swim_register_event(swim, member);
 }
 
@@ -588,6 +596,7 @@ swim_delete_member(struct swim *swim, struct swim_member *member)
 	mh_int_t rc = mh_swim_table_find(swim->members, key, NULL);
 	assert(rc != mh_end(swim->members));
 	mh_swim_table_del(swim->members, rc, NULL);
+	cached_round_msg_invalidate(swim);
 	rlist_del_entry(member, in_round_queue);
 
 	/* Failure detection component. */
@@ -836,8 +845,11 @@ swim_encode_dissemination(struct swim *swim, struct swim_packet *packet)
 
 /** Encode SWIM components into a UDP packet. */
 static void
-swim_encode_round_msg(struct swim *swim, struct swim_packet *packet)
+swim_encode_round_msg(struct swim *swim)
 {
+	if (swim_packet_body_size(&swim->round_step_task.packet) > 0)
+		return;
+	struct swim_packet *packet = &swim->round_step_task.packet;
 	swim_packet_create(packet);
 	char *header = swim_packet_alloc(packet, 1);
 	int map_size = 0;
@@ -872,6 +884,7 @@ swim_decrease_event_ttl(struct swim *swim)
 				 tmp) {
 		if (--member->status_ttl == 0) {
 			rlist_del_entry(member, in_dissemination_queue);
+			cached_round_msg_invalidate(swim);
 			if (member->status == MEMBER_LEFT)
 				swim_delete_member(swim, member);
 		}
@@ -901,8 +914,7 @@ swim_begin_step(struct ev_loop *loop, struct ev_timer *t, int events)
 	 */
 	if (rlist_empty(&swim->round_queue))
 		return;
-
-	swim_encode_round_msg(swim, &swim->round_step_task.packet);
+	swim_encode_round_msg(swim);
 	struct swim_member *m =
 		rlist_first_entry(&swim->round_queue, struct swim_member,
 				  in_round_queue);
@@ -1559,7 +1571,6 @@ swim_delete(struct swim *swim)
 	swim_scheduler_destroy(&swim->scheduler);
 	swim_ev_timer_stop(loop(), &swim->round_tick);
 	swim_ev_timer_stop(loop(), &swim->wait_ack_tick);
-	swim_task_destroy(&swim->round_step_task);
 	mh_int_t node;
 	mh_foreach(swim->members, node) {
 		struct swim_member *m =
@@ -1570,6 +1581,11 @@ swim_delete(struct swim *swim)
 		rlist_del_entry(m, in_dissemination_queue);
 		swim_member_delete(m);
 	}
+	/*
+	 * Destroy the task after members - otherwise they would
+	 * try to invalidate the already destroyed task.
+	 */
+	swim_task_destroy(&swim->round_step_task);
 	wait_ack_heap_destroy(&swim->wait_ack_heap);
 	mh_swim_table_delete(swim->members);
 	free(swim->shuffled);
diff --git a/src/lib/swim/swim_io.h b/src/lib/swim/swim_io.h
index a6032127d..30016f904 100644
--- a/src/lib/swim/swim_io.h
+++ b/src/lib/swim/swim_io.h
@@ -126,6 +126,13 @@ swim_packet_alloc(struct swim_packet *packet, int size)
 	return res;
 }
 
+/** Size of the packet body. Meta is not counted. */
+static inline int
+swim_packet_body_size(const struct swim_packet *packet)
+{
+	return packet->pos - packet->body;
+}
+
 /** Initialize @a packet, reserve some space for meta. */
 void
 swim_packet_create(struct swim_packet *packet);
-- 
2.17.2 (Apple Git-113)