[tarantool-patches] [PATCH 0/5] SWIM bug fixes

[tarantool-patches] [PATCH 0/5] SWIM bug fixes

[Vladislav Shpilevoy]

This patchset consists of quite independent fixes related to leaks, segfaults,
and flaky tests discovered by simultaneous run of swim/swim.test.lua in 100-1000
tasks.

Branch: http://github.com/tarantool/tarantool/tree/gerold103/swim-memory-fixes

Vladislav Shpilevoy (5):
  swim: fix an obvious leak in swim_delete()
  swim: fix an obvious use-after-free
  swim: fix flaky parts in swim/swim.test.lua
  swim: be ready to idle round steps when net is slow
  swim: allow to set codec before cfg

 src/lib/swim/swim.c      | 23 +++++++++++++++--
 src/lua/swim.lua         |  1 +
 test/swim/box.lua        | 22 ++++++++++++++++
 test/swim/swim.result    | 55 +++++++++++++++++-----------------------
 test/swim/swim.test.lua  | 34 ++++++++++++-------------
 test/unit/swim.c         | 20 ++++++++++++++-
 test/unit/swim.result    |  7 ++++-
 test/unit/swim_test_ev.c | 10 ++++++--
 8 files changed, 116 insertions(+), 56 deletions(-)

-- 
2.20.1 (Apple Git-117)

[tarantool-patches] [PATCH 1/5] swim: fix an obvious leak in swim_delete()

[Vladislav Shpilevoy]

Follow up #3234
---
 src/lib/swim/swim.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/lib/swim/swim.c b/src/lib/swim/swim.c
index 1283520d3..67700027e 100644
--- a/src/lib/swim/swim.c
+++ b/src/lib/swim/swim.c
@@ -1956,6 +1956,7 @@ swim_delete(struct swim *swim)
 	wait_ack_heap_destroy(&swim->wait_ack_heap);
 	mh_swim_table_delete(swim->members);
 	free(swim->shuffled);
+	free(swim);
 }
 
 /**
-- 
2.20.1 (Apple Git-117)

[tarantool-patches] [PATCH 2/5] swim: fix an obvious use-after-free

[Vladislav Shpilevoy]

Follow up #3234
---
 src/lib/swim/swim.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lib/swim/swim.c b/src/lib/swim/swim.c
index 67700027e..eb7aa6f01 100644
--- a/src/lib/swim/swim.c
+++ b/src/lib/swim/swim.c
@@ -2014,8 +2014,8 @@ swim_quit(struct swim *swim)
 		 swim_encode_quit(swim, &task->packet);
 	assert(rc == 2);
 	mp_encode_map(header, rc);
-	swim_quit_step_complete(task, &swim->scheduler, 0);
 	swim->self->status = MEMBER_LEFT;
+	swim_quit_step_complete(task, &swim->scheduler, 0);
 }
 
 struct swim_member *
-- 
2.20.1 (Apple Git-117)

[tarantool-patches] [PATCH 3/5] swim: fix flaky parts in swim/swim.test.lua

[Vladislav Shpilevoy]

They are caused by

    * too slow network, when SWIM tests are run under high load;

    * UDP packets late arrival or drop.

Follow up #3234
---
 test/swim/swim.result   | 29 ++++++++---------------------
 test/swim/swim.test.lua | 18 ++++++++----------
 2 files changed, 16 insertions(+), 31 deletions(-)

diff --git a/test/swim/swim.result b/test/swim/swim.result
index dffacd999..1b5ae2b0f 100644
--- a/test/swim/swim.result
+++ b/test/swim/swim.result
@@ -236,17 +236,9 @@ s1:broadcast(listen_port)
 ---
 - true
 ...
-while s2:size() ~= 2 do fiber.sleep(0.01) end
+while s2:size() ~= 2 or s1:size() ~= 2 do fiber.sleep(0.01) s1:broadcast(listen_port) end
 ---
 ...
-s1:size()
----
-- 2
-...
-s2:size()
----
-- 2
-...
 s2:delete()
 ---
 ...
@@ -263,11 +255,10 @@ s1:remove_member('1234')
 ---
 - error: 'builtin/swim.lua:<line>: swim:remove_member: invalid UUID'
 ...
-s1:remove_member(uuid(2))
+s1:remove_member(uuid(2)) size = s1:size()
 ---
-- true
 ...
-s1:size()
+size
 ---
 - 1
 ...
@@ -293,9 +284,8 @@ s1:add_member({uuid = uuid(2)})
 - null
 - 'swim.add_member: URI and UUID are mandatory'
 ...
-s1:add_member({uri = listen_uri, uuid = uuid(2)})
+s1:remove_member(uuid(2)) s1:add_member({uri = listen_uri, uuid = uuid(2)})
 ---
-- true
 ...
 s1:add_member({uri = listen_uri, uuid = uuid(2)})
 ---
@@ -306,15 +296,14 @@ s1:size()
 ---
 - 2
 ...
-s1:cfg({uuid = uuid(3)})
+s1:cfg({uuid = uuid(3)}) old_self = s1:member_by_uuid(uuid(1))
 ---
-- true
 ...
 s1:self():uuid()
 ---
 - 00000000-0000-1000-8000-000000000003
 ...
-s1:member_by_uuid(uuid(1))
+old_self
 ---
 - uri: 127.0.0.1:<port>
   status: left
@@ -848,13 +837,11 @@ s2 = swim.new({uuid = uuid(2), uri = uri(), heartbeat_rate = 0.01})
 s1_self = s1:self()
 ---
 ...
-s1:add_member({uuid = s2:self():uuid(), uri = s2:self():uri()})
+_ = s1:add_member({uuid = s2:self():uuid(), uri = s2:self():uri()})
 ---
-- true
 ...
-s2:add_member({uuid = s1_self:uuid(), uri = s1_self:uri()})
+_ = s2:add_member({uuid = s1_self:uuid(), uri = s1_self:uri()})
 ---
-- true
 ...
 s1:size()
 ---
diff --git a/test/swim/swim.test.lua b/test/swim/swim.test.lua
index 081d89b3a..b344c69be 100644
--- a/test/swim/swim.test.lua
+++ b/test/swim/swim.test.lua
@@ -82,30 +82,28 @@ s1:broadcast('127.0.0.1:3333')
 s1:broadcast()
 
 s1:broadcast(listen_port)
-while s2:size() ~= 2 do fiber.sleep(0.01) end
-s1:size()
-s2:size()
+while s2:size() ~= 2 or s1:size() ~= 2 do fiber.sleep(0.01) s1:broadcast(listen_port) end
 
 s2:delete()
 
 s1.remove_member()
 s1:remove_member(100)
 s1:remove_member('1234')
-s1:remove_member(uuid(2))
-s1:size()
+s1:remove_member(uuid(2)) size = s1:size()
+size
 
 s1.add_member()
 s1:add_member(100)
 s1:add_member({uri = true})
 s1:add_member({uri = listen_uri})
 s1:add_member({uuid = uuid(2)})
-s1:add_member({uri = listen_uri, uuid = uuid(2)})
+s1:remove_member(uuid(2)) s1:add_member({uri = listen_uri, uuid = uuid(2)})
 s1:add_member({uri = listen_uri, uuid = uuid(2)})
 s1:size()
 
-s1:cfg({uuid = uuid(3)})
+s1:cfg({uuid = uuid(3)}) old_self = s1:member_by_uuid(uuid(1))
 s1:self():uuid()
-s1:member_by_uuid(uuid(1))
+old_self
 -- Can't remove self.
 s1:remove_member(uuid(3))
 -- Not existing.
@@ -274,8 +272,8 @@ s:delete()
 s1 = swim.new({uuid = uuid(1), uri = uri(listen_port), heartbeat_rate = 0.01})
 s2 = swim.new({uuid = uuid(2), uri = uri(), heartbeat_rate = 0.01})
 s1_self = s1:self()
-s1:add_member({uuid = s2:self():uuid(), uri = s2:self():uri()})
-s2:add_member({uuid = s1_self:uuid(), uri = s1_self:uri()})
+_ = s1:add_member({uuid = s2:self():uuid(), uri = s2:self():uri()})
+_ = s2:add_member({uuid = s1_self:uuid(), uri = s1_self:uri()})
 s1:size()
 s2:size()
 s1_view = s2:member_by_uuid(s1_self:uuid())
-- 
2.20.1 (Apple Git-117)

[tarantool-patches] [PATCH 4/5] swim: be ready to idle round steps when net is slow

[Vladislav Shpilevoy]

First of all, the problem in a nutshell was that ev_timer with
non-zero 'repeat' field in fact is a ev_periodic. It is restarted
*automatically*, even if a user does not write ev_timer_again()
nor ev_timer_start().

This led to a situation, that a round message send is scheduled,
and next round step timer alarm happens before the message is
actually sent. It, in turn, led to an assertion on attempt to
schedule a task twice.

This patch fixes the swim test harness to behave like ev_timer
with 'repeat' > 0, and on first idle round step stops the timer -
it will be restarted once the currently hanging task will be
finally sent.

Follow up #3234
---
 src/lib/swim/swim.c      | 20 +++++++++++++++++++-
 test/unit/swim.c         | 20 +++++++++++++++++++-
 test/unit/swim.result    |  7 ++++++-
 test/unit/swim_test_ev.c | 10 ++++++++--
 4 files changed, 52 insertions(+), 5 deletions(-)

diff --git a/src/lib/swim/swim.c b/src/lib/swim/swim.c
index eb7aa6f01..f7a885b76 100644
--- a/src/lib/swim/swim.c
+++ b/src/lib/swim/swim.c
@@ -1077,6 +1077,18 @@ swim_begin_step(struct ev_loop *loop, struct ev_timer *t, int events)
 	(void) events;
 	(void) loop;
 	struct swim *swim = (struct swim *) t->data;
+	/*
+	 * There are possible false-positive wakeups. They can
+	 * appear, when a round task was scheduled, but event
+	 * loop was too busy to send the task, and the timer
+	 * alarms again. In such a case stop it - it makes no
+	 * sense to waste time on idle wakeups. Completion
+	 * callback will restart the timer.
+	 */
+	if (swim_task_is_scheduled(&swim->round_step_task)) {
+		swim_ev_timer_stop(loop, t);
+		return;
+	}
 	if (! rlist_empty(&swim->round_queue))
 		say_verbose("SWIM %d: continue the round", swim_fd(swim));
 	else
@@ -1084,8 +1096,10 @@ swim_begin_step(struct ev_loop *loop, struct ev_timer *t, int events)
 	/*
 	 * Possibly empty, if no members but self are specified.
 	 */
-	if (rlist_empty(&swim->round_queue))
+	if (rlist_empty(&swim->round_queue)) {
+		swim_ev_timer_stop(loop, t);
 		return;
+	}
 	swim_encode_round_msg(swim);
 	struct swim_member *m =
 		rlist_first_entry(&swim->round_queue, struct swim_member,
@@ -1104,6 +1118,10 @@ swim_complete_step(struct swim_task *task,
 	(void) rc;
 	(void) task;
 	struct swim *swim = swim_by_scheduler(scheduler);
+	/*
+	 * It could be stopped by the step begin function, if the
+	 * sending was too long.
+	 */
 	swim_ev_timer_again(loop(), &swim->round_tick);
 	/*
 	 * It is possible that the original member was deleted
diff --git a/test/unit/swim.c b/test/unit/swim.c
index 6467aa35e..2ba9820d8 100644
--- a/test/unit/swim.c
+++ b/test/unit/swim.c
@@ -933,10 +933,27 @@ swim_test_encryption(void)
 	swim_finish_test();
 }
 
+static void
+swim_test_slow_net(void)
+{
+	swim_start_test(0);
+	struct swim_cluster *cluster = swim_cluster_new(2);
+	swim_cluster_interconnect(cluster, 0, 1);
+	swim_cluster_block_io(cluster, 0);
+	swim_cluster_block_io(cluster, 1);
+
+	note("slow network leads to idle round steps, they should not produce "\
+	     "a new message");
+	swim_run_for(5);
+
+	swim_cluster_delete(cluster);
+	swim_finish_test();
+}
+
 static int
 main_f(va_list ap)
 {
-	swim_start_test(19);
+	swim_start_test(20);
 
 	(void) ap;
 	swim_test_ev_init();
@@ -961,6 +978,7 @@ main_f(va_list ap)
 	swim_test_payload_refutation();
 	swim_test_indirect_ping();
 	swim_test_encryption();
+	swim_test_slow_net();
 
 	swim_test_transport_free();
 	swim_test_ev_free();
diff --git a/test/unit/swim.result b/test/unit/swim.result
index 4093ecb93..25fdb8833 100644
--- a/test/unit/swim.result
+++ b/test/unit/swim.result
@@ -1,5 +1,5 @@
 	*** main_f ***
-1..19
+1..20
 	*** swim_test_one_link ***
     1..6
     ok 1 - no rounds - no fullmesh
@@ -195,4 +195,9 @@ ok 18 - subtests
     ok 3 - cluster works after encryption has been disabled
 ok 19 - subtests
 	*** swim_test_encryption: done ***
+	*** swim_test_slow_net ***
+    1..0
+    # slow network leads to idle round steps, they should not produce a new message
+ok 20 - subtests
+	*** swim_test_slow_net: done ***
 	*** main_f: done ***
diff --git a/test/unit/swim_test_ev.c b/test/unit/swim_test_ev.c
index d415cec0a..a4ffa2fc8 100644
--- a/test/unit/swim_test_ev.c
+++ b/test/unit/swim_test_ev.c
@@ -177,18 +177,24 @@ swim_timer_event_delete(struct swim_event *e)
 	free(te);
 }
 
+/** Create a new timer event. */
+static void
+swim_timer_event_new(struct ev_watcher *watcher, double delay);
+
 /** Process a timer event and delete it. */
 static void
 swim_timer_event_process(struct swim_event *e, struct ev_loop *loop)
 {
 	assert(e->type == SWIM_EVENT_TIMER);
 	struct ev_watcher *w = ((struct swim_timer_event *) e)->watcher;
+	struct ev_timer *t = (struct ev_timer *) w;
 	swim_timer_event_delete(e);
-	((struct ev_timer *) w)->at = 0;
+	t->at = 0;
+	if (t->repeat > 0)
+		swim_timer_event_new(w, t->repeat);
 	ev_invoke(loop, w, EV_TIMER);
 }
 
-/** Create a new timer event. */
 static void
 swim_timer_event_new(struct ev_watcher *watcher, double delay)
 {
-- 
2.20.1 (Apple Git-117)

[tarantool-patches] [PATCH 5/5] swim: allow to set codec before cfg

[Vladislav Shpilevoy]

One another problem discovered with UDP broadcast test is that
it can affect other tests, even after termination. Doing
swim:broadcast() on one test a programmer can't be sure, who will
listen it, answer, and break the test scenario.

This commit reduces probability of such a problem by

    * allowance to set a codec before swim:cfg(). It allows to
      protect SWIM nodes of different tests from each other -
      they will not understand messages from other tests. By the
      way, the same problem can appear in real applications too;

    * do not binding again a URI passed by test-run into the
      test and closed here. If a test closes a URI given to it,
      it can't be sure, that next bind() will be successful -
      test-run could already reuse it.

Follow up #3234
---
 src/lua/swim.lua        |  1 +
 test/swim/box.lua       | 22 ++++++++++++++++++++++
 test/swim/swim.result   | 26 +++++++++++++++-----------
 test/swim/swim.test.lua | 16 ++++++++--------
 4 files changed, 46 insertions(+), 19 deletions(-)

diff --git a/src/lua/swim.lua b/src/lua/swim.lua
index d3d6b01e9..4893d5767 100644
--- a/src/lua/swim.lua
+++ b/src/lua/swim.lua
@@ -804,6 +804,7 @@ local swim_not_configured_mt = {
     __index = {
         delete = swim_delete,
         is_configured = swim_is_configured,
+        set_codec = swim_set_codec,
     },
     __serialize = swim_serialize
 }
diff --git a/test/swim/box.lua b/test/swim/box.lua
index b6a39575e..a603777e0 100644
--- a/test/swim/box.lua
+++ b/test/swim/box.lua
@@ -7,6 +7,28 @@ listen_port = require('uri').parse(listen_uri).service
 
 box.cfg{}
 
+--
+-- SWIM instances, using broadcast, should protect themselves
+-- with encryption. Otherwise they can accidentally discover
+-- SWIM instances from other tests.
+--
+enc_key = box.info.uuid
+enc_algo = 'aes128'
+
+--
+-- Wrap swim.new with a codec to prevent test workers affecting
+-- each other.
+--
+local original_new = swim.new
+swim.new = function(...)
+    local s, err = original_new(...)
+    if s == nil then
+        return s, err
+    end
+    assert(s:set_codec({algo = enc_algo, key = enc_key, key_size = 16}))
+    return s
+end
+
 function uuid(i)
     local min_valid_prefix = '00000000-0000-1000-8000-'
     if i < 10 then
diff --git a/test/swim/swim.result b/test/swim/swim.result
index 1b5ae2b0f..436d4e579 100644
--- a/test/swim/swim.result
+++ b/test/swim/swim.result
@@ -333,7 +333,7 @@ s1:delete()
 s1 = swim.new({uuid = uuid(1), uri = uri()})
 ---
 ...
-s2 = swim.new({uuid = uuid(2), uri = listen_uri})
+s2 = swim.new({uuid = uuid(2), uri = uri()})
 ---
 ...
 s1.probe_member()
@@ -361,7 +361,7 @@ s1:size()
 ---
 - 1
 ...
-s1:probe_member(listen_uri)
+s1:probe_member(s2:self():uri())
 ---
 - true
 ...
@@ -698,10 +698,10 @@ self:is_dropped()
 s1 = swim.new({uuid = uuid(1), uri = uri(), heartbeat_rate = 0.01})
 ---
 ...
-s2 = swim.new({uuid = uuid(2), uri = listen_port, heartbeat_rate = 0.01})
+s2 = swim.new({uuid = uuid(2), uri = uri(), heartbeat_rate = 0.01})
 ---
 ...
-s1:add_member({uuid = uuid(2), uri = listen_port})
+s1:add_member({uuid = uuid(2), uri = s2:self():uri()})
 ---
 - true
 ...
@@ -828,7 +828,7 @@ s:delete()
 --
 -- Payload caching.
 --
-s1 = swim.new({uuid = uuid(1), uri = uri(listen_port), heartbeat_rate = 0.01})
+s1 = swim.new({uuid = uuid(1), uri = uri(), heartbeat_rate = 0.01})
 ---
 ...
 s2 = swim.new({uuid = uuid(2), uri = uri(), heartbeat_rate = 0.01})
@@ -902,7 +902,7 @@ s1_view:payload()
 ...
 s1_view:incarnation()
 ---
-- 2
+- 3
 ...
 s1:cfg({heartbeat_rate = 0.01})
 ---
@@ -928,7 +928,7 @@ p
 ...
 s1_view:incarnation()
 ---
-- 2
+- 3
 ...
 s1:delete()
 ---
@@ -1099,7 +1099,7 @@ s1:set_codec(cfg)
 ---
 - true
 ...
--- S2 does not use encryption and can't decode the ping.
+-- S2 uses different encryption and can't decode the ping.
 s1:probe_member(s2:self():uri())
 ---
 - true
@@ -1199,12 +1199,16 @@ s1:probe_member(s2:self():uri())
 ---
 - true
 ...
-while s1:size() ~= 2 do fiber.sleep(0.01) end
+while s1:member_by_uuid(s2:self():uuid()) == nil do fiber.sleep(0.01) end
 ---
 ...
-s2:size()
+s2:member_by_uuid(s1:self():uuid())
 ---
-- 2
+- uri: 127.0.0.1:<port>
+  status: alive
+  incarnation: 1
+  uuid: 00000000-0000-1000-8000-000000000001
+  payload_size: 0
 ...
 s1:delete()
 ---
diff --git a/test/swim/swim.test.lua b/test/swim/swim.test.lua
index b344c69be..a3eac9b46 100644
--- a/test/swim/swim.test.lua
+++ b/test/swim/swim.test.lua
@@ -114,7 +114,7 @@ s1:remove_member(uuid(1))
 s1:delete()
 
 s1 = swim.new({uuid = uuid(1), uri = uri()})
-s2 = swim.new({uuid = uuid(2), uri = listen_uri})
+s2 = swim.new({uuid = uuid(2), uri = uri()})
 s1.probe_member()
 s1:probe_member()
 s1:probe_member(true)
@@ -122,7 +122,7 @@ s1:probe_member(true)
 s1:probe_member('127.0.0.1:1')
 fiber.yield()
 s1:size()
-s1:probe_member(listen_uri)
+s1:probe_member(s2:self():uri())
 while s1:size() ~= 2 do fiber.sleep(0.01) end
 s2:size()
 
@@ -233,8 +233,8 @@ self:is_dropped()
 -- Check payload dissemination.
 --
 s1 = swim.new({uuid = uuid(1), uri = uri(), heartbeat_rate = 0.01})
-s2 = swim.new({uuid = uuid(2), uri = listen_port, heartbeat_rate = 0.01})
-s1:add_member({uuid = uuid(2), uri = listen_port})
+s2 = swim.new({uuid = uuid(2), uri = uri(), heartbeat_rate = 0.01})
+s1:add_member({uuid = uuid(2), uri = s2:self():uri()})
 while s2:size() ~= 2 do fiber.sleep(0.01) end
 s1_view = s2:member_by_uuid(uuid(1))
 s1_view:payload()
@@ -269,7 +269,7 @@ s:delete()
 --
 -- Payload caching.
 --
-s1 = swim.new({uuid = uuid(1), uri = uri(listen_port), heartbeat_rate = 0.01})
+s1 = swim.new({uuid = uuid(1), uri = uri(), heartbeat_rate = 0.01})
 s2 = swim.new({uuid = uuid(2), uri = uri(), heartbeat_rate = 0.01})
 s1_self = s1:self()
 _ = s1:add_member({uuid = s2:self():uuid(), uri = s2:self():uri()})
@@ -371,7 +371,7 @@ cfg.key = '1234567812345678'
 cfg.key_size = 16
 s1:set_codec(cfg)
 
--- S2 does not use encryption and can't decode the ping.
+-- S2 uses different encryption and can't decode the ping.
 s1:probe_member(s2:self():uri())
 fiber.sleep(0.01)
 s1:size()
@@ -407,8 +407,8 @@ s2:size()
 s1:set_codec({algo = 'none'})
 s2:set_codec({algo = 'none'})
 s1:probe_member(s2:self():uri())
-while s1:size() ~= 2 do fiber.sleep(0.01) end
-s2:size()
+while s1:member_by_uuid(s2:self():uuid()) == nil do fiber.sleep(0.01) end
+s2:member_by_uuid(s1:self():uuid())
 
 s1:delete()
 s2:delete()
-- 
2.20.1 (Apple Git-117)

[tarantool-patches] Re: [PATCH 0/5] SWIM bug fixes

[Vladislav Shpilevoy]

On 22/05/2019 22:56, Konstantin Osipov wrote:> * Vladislav Shpilevoy <v.shpilevoy@tarantool.org> [19/05/22 22:54]:
>> This patchset consists of quite independent fixes related to leaks, segfaults,
>> and flaky tests discovered by simultaneous run of swim/swim.test.lua in 100-1000
> 
> OK to push to all patches.  
> 

Pushed to the master.

On 22/05/2019 22:52, Vladislav Shpilevoy wrote:
> This patchset consists of quite independent fixes related to leaks, segfaults,
> and flaky tests discovered by simultaneous run of swim/swim.test.lua in 100-1000
> tasks.
> 
> Branch: http://github.com/tarantool/tarantool/tree/gerold103/swim-memory-fixes
> 
> Vladislav Shpilevoy (5):
>   swim: fix an obvious leak in swim_delete()
>   swim: fix an obvious use-after-free
>   swim: fix flaky parts in swim/swim.test.lua
>   swim: be ready to idle round steps when net is slow
>   swim: allow to set codec before cfg
> 
>  src/lib/swim/swim.c      | 23 +++++++++++++++--
>  src/lua/swim.lua         |  1 +
>  test/swim/box.lua        | 22 ++++++++++++++++
>  test/swim/swim.result    | 55 +++++++++++++++++-----------------------
>  test/swim/swim.test.lua  | 34 ++++++++++++-------------
>  test/unit/swim.c         | 20 ++++++++++++++-
>  test/unit/swim.result    |  7 ++++-
>  test/unit/swim_test_ev.c | 10 ++++++--
>  8 files changed, 116 insertions(+), 56 deletions(-)
>