* [Tarantool-patches] [PATCH] replication: cancel replica joining thread at exit
@ 2019-10-15 15:50 Ilya Kosarev
2019-10-21 13:57 ` [Tarantool-patches] [tarantool-patches] " Alexander Tikhonov
` (2 more replies)
0 siblings, 3 replies; 6+ messages in thread
From: Ilya Kosarev @ 2019-10-15 15:50 UTC (permalink / raw)
To: tarantool-patches; +Cc: tarantool-patches
If a tarantool instance exits while joining replica is in progress,
the replica joining thread can access already freed data resulting
in a crash. Let's fix this the same way we did for checkpoint thread
- simply cancel the thread forcefully and wait for it to terminate.
Closes #4528
---
https://github.com/tarantool/tarantool/tree/i.kosarev/gh-4528-fix-shutdown-on-replica-join
https://github.com/tarantool/tarantool/issues/4528
src/box/memtx_engine.c | 25 ++++++++++++++++++++++++-
src/box/memtx_engine.h | 2 ++
2 files changed, 26 insertions(+), 1 deletion(-)
diff --git a/src/box/memtx_engine.c b/src/box/memtx_engine.c
index eb11346c1..325129a6e 100644
--- a/src/box/memtx_engine.c
+++ b/src/box/memtx_engine.c
@@ -55,6 +55,9 @@
static void
checkpoint_cancel(struct checkpoint *ckpt);
+static void
+replica_join_cancel(struct cord *replica_join_cord);
+
struct PACKED memtx_tuple {
/*
* sic: the header of the tuple is used
@@ -129,6 +132,8 @@ memtx_engine_shutdown(struct engine *engine)
struct memtx_engine *memtx = (struct memtx_engine *)engine;
if (memtx->checkpoint != NULL)
checkpoint_cancel(memtx->checkpoint);
+ if (memtx->replica_join_cord != NULL)
+ replica_join_cancel(memtx->replica_join_cord);
mempool_destroy(&memtx->iterator_pool);
if (mempool_is_initialized(&memtx->rtree_iterator_pool))
mempool_destroy(&memtx->rtree_iterator_pool);
@@ -528,6 +533,18 @@ checkpoint_cancel(struct checkpoint *ckpt)
checkpoint_delete(ckpt);
}
+static void
+replica_join_cancel(struct cord *replica_join_cord)
+{
+ /*
+ * Cancel the thread being used to join replica if it's
+ * running and wait for it to terminate so as to
+ * eliminate the possibility of use-after-free.
+ */
+ tt_pthread_cancel(replica_join_cord->id);
+ tt_pthread_join(replica_join_cord->id, NULL);
+}
+
static int
checkpoint_add_space(struct space *sp, void *data)
{
@@ -848,7 +865,11 @@ memtx_engine_join(struct engine *engine, void *arg, struct xstream *stream)
struct cord cord;
if (cord_costart(&cord, "initial_join", memtx_join_f, ctx) != 0)
return -1;
- return cord_cojoin(&cord);
+ struct memtx_engine *memtx = (struct memtx_engine *)engine;
+ memtx->replica_join_cord = &cord;
+ int res = cord_cojoin(&cord);
+ memtx->replica_join_cord = NULL;
+ return res;
}
static void
@@ -1030,6 +1051,8 @@ memtx_engine_new(const char *snap_dirname, bool force_recovery,
memtx->max_tuple_size = MAX_TUPLE_SIZE;
memtx->force_recovery = force_recovery;
+ memtx->replica_join_cord = NULL;
+
memtx->base.vtab = &memtx_engine_vtab;
memtx->base.name = "memtx";
diff --git a/src/box/memtx_engine.h b/src/box/memtx_engine.h
index c092f5d8e..43e16879d 100644
--- a/src/box/memtx_engine.h
+++ b/src/box/memtx_engine.h
@@ -107,6 +107,8 @@ struct memtx_engine {
uint64_t snap_io_rate_limit;
/** Skip invalid snapshot records if this flag is set. */
bool force_recovery;
+ /** cord being currently used to join replica **/
+ struct cord *replica_join_cord;
/** Common quota for tuples and indexes. */
struct quota quota;
/**
--
2.17.1
^ permalink raw reply [flat|nested] 6+ messages in thread
* Re: [Tarantool-patches] [tarantool-patches] [PATCH] replication: cancel replica joining thread at exit
2019-10-15 15:50 [Tarantool-patches] [PATCH] replication: cancel replica joining thread at exit Ilya Kosarev
@ 2019-10-21 13:57 ` Alexander Tikhonov
2019-10-22 21:28 ` Vladislav Shpilevoy
2019-10-24 5:09 ` Kirill Yukhin
2 siblings, 0 replies; 6+ messages in thread
From: Alexander Tikhonov @ 2019-10-21 13:57 UTC (permalink / raw)
To: tarantool-patches; +Cc: tarantool-patches
[-- Attachment #1: Type: text/plain, Size: 3397 bytes --]
Hi Ilya,
The patch LGTM, thanks.
Alexander
>Вторник, 15 октября 2019, 18:50 +03:00 от Ilya Kosarev <i.kosarev@tarantool.org>:
>
>If a tarantool instance exits while joining replica is in progress,
>the replica joining thread can access already freed data resulting
>in a crash. Let's fix this the same way we did for checkpoint thread
>- simply cancel the thread forcefully and wait for it to terminate.
>
>Closes #4528
>---
>https://github.com/tarantool/tarantool/tree/i.kosarev/gh-4528-fix-shutdown-on-replica-join
>https://github.com/tarantool/tarantool/issues/4528
>
> src/box/memtx_engine.c | 25 ++++++++++++++++++++++++-
> src/box/memtx_engine.h | 2 ++
> 2 files changed, 26 insertions(+), 1 deletion(-)
>
>diff --git a/src/box/memtx_engine.c b/src/box/memtx_engine.c
>index eb11346c1..325129a6e 100644
>--- a/src/box/memtx_engine.c
>+++ b/src/box/memtx_engine.c
>@@ -55,6 +55,9 @@
> static void
> checkpoint_cancel(struct checkpoint *ckpt);
>
>+static void
>+replica_join_cancel(struct cord *replica_join_cord);
>+
> struct PACKED memtx_tuple {
> /*
> * sic: the header of the tuple is used
>@@ -129,6 +132,8 @@ memtx_engine_shutdown(struct engine *engine)
> struct memtx_engine *memtx = (struct memtx_engine *)engine;
> if (memtx->checkpoint != NULL)
> checkpoint_cancel(memtx->checkpoint);
>+ if (memtx->replica_join_cord != NULL)
>+ replica_join_cancel(memtx->replica_join_cord);
> mempool_destroy(&memtx->iterator_pool);
> if (mempool_is_initialized(&memtx->rtree_iterator_pool))
> mempool_destroy(&memtx->rtree_iterator_pool);
>@@ -528,6 +533,18 @@ checkpoint_cancel(struct checkpoint *ckpt)
> checkpoint_delete(ckpt);
> }
>
>+static void
>+replica_join_cancel(struct cord *replica_join_cord)
>+{
>+ /*
>+ * Cancel the thread being used to join replica if it's
>+ * running and wait for it to terminate so as to
>+ * eliminate the possibility of use-after-free.
>+ */
>+ tt_pthread_cancel(replica_join_cord->id);
>+ tt_pthread_join(replica_join_cord->id, NULL);
>+}
>+
> static int
> checkpoint_add_space(struct space *sp, void *data)
> {
>@@ -848,7 +865,11 @@ memtx_engine_join(struct engine *engine, void *arg, struct xstream *stream)
> struct cord cord;
> if (cord_costart(&cord, "initial_join", memtx_join_f, ctx) != 0)
> return -1;
>- return cord_cojoin(&cord);
>+ struct memtx_engine *memtx = (struct memtx_engine *)engine;
>+ memtx->replica_join_cord = &cord;
>+ int res = cord_cojoin(&cord);
>+ memtx->replica_join_cord = NULL;
>+ return res;
> }
>
> static void
>@@ -1030,6 +1051,8 @@ memtx_engine_new(const char *snap_dirname, bool force_recovery,
> memtx->max_tuple_size = MAX_TUPLE_SIZE;
> memtx->force_recovery = force_recovery;
>
>+ memtx->replica_join_cord = NULL;
>+
> memtx->base.vtab = &memtx_engine_vtab;
> memtx->base.name = "memtx";
>
>diff --git a/src/box/memtx_engine.h b/src/box/memtx_engine.h
>index c092f5d8e..43e16879d 100644
>--- a/src/box/memtx_engine.h
>+++ b/src/box/memtx_engine.h
>@@ -107,6 +107,8 @@ struct memtx_engine {
> uint64_t snap_io_rate_limit;
> /** Skip invalid snapshot records if this flag is set. */
> bool force_recovery;
>+ /** cord being currently used to join replica **/
>+ struct cord *replica_join_cord;
> /** Common quota for tuples and indexes. */
> struct quota quota;
> /**
>--
>2.17.1
>
>
--
Alexander Tikhonov
[-- Attachment #2: Type: text/html, Size: 4528 bytes --]
^ permalink raw reply [flat|nested] 6+ messages in thread
* Re: [Tarantool-patches] [tarantool-patches] [PATCH] replication: cancel replica joining thread at exit
2019-10-15 15:50 [Tarantool-patches] [PATCH] replication: cancel replica joining thread at exit Ilya Kosarev
2019-10-21 13:57 ` [Tarantool-patches] [tarantool-patches] " Alexander Tikhonov
@ 2019-10-22 21:28 ` Vladislav Shpilevoy
2019-10-23 12:59 ` [Tarantool-patches] " Ilya Kosarev
2019-10-24 5:09 ` Kirill Yukhin
2 siblings, 1 reply; 6+ messages in thread
From: Vladislav Shpilevoy @ 2019-10-22 21:28 UTC (permalink / raw)
To: tarantool-patches, Ilya Kosarev, tarantool-patches
Hi! Thanks for the patch!
> diff --git a/src/box/memtx_engine.h b/src/box/memtx_engine.h
> index c092f5d8e..43e16879d 100644
> --- a/src/box/memtx_engine.h
> +++ b/src/box/memtx_engine.h
> @@ -107,6 +107,8 @@ struct memtx_engine {
> uint64_t snap_io_rate_limit;
> /** Skip invalid snapshot records if this flag is set. */
> bool force_recovery;
> + /** cord being currently used to join replica **/
Please, start the sentence from a capital letter,
and finish with a dot. Also, as you probably have seen
we use */ commend end mark, never **/.
I would also mention, that we need that pointer only
to be able to cancel the cord. Nothing else.
> + struct cord *replica_join_cord;
> /** Common quota for tuples and indexes. */
> struct quota quota;
> /**
>
^ permalink raw reply [flat|nested] 6+ messages in thread
* [Tarantool-patches] [PATCH] replication: cancel replica joining thread at exit
2019-10-22 21:28 ` Vladislav Shpilevoy
@ 2019-10-23 12:59 ` Ilya Kosarev
2019-10-23 20:55 ` Vladislav Shpilevoy
0 siblings, 1 reply; 6+ messages in thread
From: Ilya Kosarev @ 2019-10-23 12:59 UTC (permalink / raw)
To: tarantool-patches
If a tarantool instance exits while joining replica is in progress,
the replica joining thread can access already freed data resulting
in a crash. Let's fix this the same way we did for checkpoint thread
- simply cancel the thread forcefully and wait for it to terminate.
Closes #4528
---
https://github.com/tarantool/tarantool/tree/i.kosarev/gh-4528-fix-shutdown-on-replica-join
https://github.com/tarantool/tarantool/issues/4528
src/box/memtx_engine.c | 25 ++++++++++++++++++++++++-
src/box/memtx_engine.h | 5 +++++
2 files changed, 29 insertions(+), 1 deletion(-)
diff --git a/src/box/memtx_engine.c b/src/box/memtx_engine.c
index ecce3b1b6..23ccc4703 100644
--- a/src/box/memtx_engine.c
+++ b/src/box/memtx_engine.c
@@ -55,6 +55,9 @@
static void
checkpoint_cancel(struct checkpoint *ckpt);
+static void
+replica_join_cancel(struct cord *replica_join_cord);
+
struct PACKED memtx_tuple {
/*
* sic: the header of the tuple is used
@@ -129,6 +132,8 @@ memtx_engine_shutdown(struct engine *engine)
struct memtx_engine *memtx = (struct memtx_engine *)engine;
if (memtx->checkpoint != NULL)
checkpoint_cancel(memtx->checkpoint);
+ if (memtx->replica_join_cord != NULL)
+ replica_join_cancel(memtx->replica_join_cord);
mempool_destroy(&memtx->iterator_pool);
if (mempool_is_initialized(&memtx->rtree_iterator_pool))
mempool_destroy(&memtx->rtree_iterator_pool);
@@ -527,6 +532,18 @@ checkpoint_cancel(struct checkpoint *ckpt)
checkpoint_delete(ckpt);
}
+static void
+replica_join_cancel(struct cord *replica_join_cord)
+{
+ /*
+ * Cancel the thread being used to join replica if it's
+ * running and wait for it to terminate so as to
+ * eliminate the possibility of use-after-free.
+ */
+ tt_pthread_cancel(replica_join_cord->id);
+ tt_pthread_join(replica_join_cord->id, NULL);
+}
+
static int
checkpoint_add_space(struct space *sp, void *data)
{
@@ -848,7 +865,11 @@ memtx_engine_join(struct engine *engine, void *arg, struct xstream *stream)
struct cord cord;
if (cord_costart(&cord, "initial_join", memtx_join_f, ctx) != 0)
return -1;
- return cord_cojoin(&cord);
+ struct memtx_engine *memtx = (struct memtx_engine *)engine;
+ memtx->replica_join_cord = &cord;
+ int res = cord_cojoin(&cord);
+ memtx->replica_join_cord = NULL;
+ return res;
}
static void
@@ -1030,6 +1051,8 @@ memtx_engine_new(const char *snap_dirname, bool force_recovery,
memtx->max_tuple_size = MAX_TUPLE_SIZE;
memtx->force_recovery = force_recovery;
+ memtx->replica_join_cord = NULL;
+
memtx->base.vtab = &memtx_engine_vtab;
memtx->base.name = "memtx";
diff --git a/src/box/memtx_engine.h b/src/box/memtx_engine.h
index c092f5d8e..f562c66df 100644
--- a/src/box/memtx_engine.h
+++ b/src/box/memtx_engine.h
@@ -107,6 +107,11 @@ struct memtx_engine {
uint64_t snap_io_rate_limit;
/** Skip invalid snapshot records if this flag is set. */
bool force_recovery;
+ /**
+ * Cord being currently used to join replica. It is only
+ * needed to be able to cancel it on shutdown.
+ */
+ struct cord *replica_join_cord;
/** Common quota for tuples and indexes. */
struct quota quota;
/**
--
2.17.1
^ permalink raw reply [flat|nested] 6+ messages in thread
* Re: [Tarantool-patches] [PATCH] replication: cancel replica joining thread at exit
2019-10-23 12:59 ` [Tarantool-patches] " Ilya Kosarev
@ 2019-10-23 20:55 ` Vladislav Shpilevoy
0 siblings, 0 replies; 6+ messages in thread
From: Vladislav Shpilevoy @ 2019-10-23 20:55 UTC (permalink / raw)
To: Ilya Kosarev, tarantool-patches
LGTM.
On 23/10/2019 14:59, Ilya Kosarev wrote:
> If a tarantool instance exits while joining replica is in progress,
> the replica joining thread can access already freed data resulting
> in a crash. Let's fix this the same way we did for checkpoint thread
> - simply cancel the thread forcefully and wait for it to terminate.
>
> Closes #4528
> ---
> https://github.com/tarantool/tarantool/tree/i.kosarev/gh-4528-fix-shutdown-on-replica-join
> https://github.com/tarantool/tarantool/issues/4528
>
> src/box/memtx_engine.c | 25 ++++++++++++++++++++++++-
> src/box/memtx_engine.h | 5 +++++
> 2 files changed, 29 insertions(+), 1 deletion(-)
>
> diff --git a/src/box/memtx_engine.c b/src/box/memtx_engine.c
> index ecce3b1b6..23ccc4703 100644
> --- a/src/box/memtx_engine.c
> +++ b/src/box/memtx_engine.c
> @@ -55,6 +55,9 @@
> static void
> checkpoint_cancel(struct checkpoint *ckpt);
>
> +static void
> +replica_join_cancel(struct cord *replica_join_cord);
> +
> struct PACKED memtx_tuple {
> /*
> * sic: the header of the tuple is used
> @@ -129,6 +132,8 @@ memtx_engine_shutdown(struct engine *engine)
> struct memtx_engine *memtx = (struct memtx_engine *)engine;
> if (memtx->checkpoint != NULL)
> checkpoint_cancel(memtx->checkpoint);
> + if (memtx->replica_join_cord != NULL)
> + replica_join_cancel(memtx->replica_join_cord);
> mempool_destroy(&memtx->iterator_pool);
> if (mempool_is_initialized(&memtx->rtree_iterator_pool))
> mempool_destroy(&memtx->rtree_iterator_pool);
> @@ -527,6 +532,18 @@ checkpoint_cancel(struct checkpoint *ckpt)
> checkpoint_delete(ckpt);
> }
>
> +static void
> +replica_join_cancel(struct cord *replica_join_cord)
> +{
> + /*
> + * Cancel the thread being used to join replica if it's
> + * running and wait for it to terminate so as to
> + * eliminate the possibility of use-after-free.
> + */
> + tt_pthread_cancel(replica_join_cord->id);
> + tt_pthread_join(replica_join_cord->id, NULL);
> +}
> +
> static int
> checkpoint_add_space(struct space *sp, void *data)
> {
> @@ -848,7 +865,11 @@ memtx_engine_join(struct engine *engine, void *arg, struct xstream *stream)
> struct cord cord;
> if (cord_costart(&cord, "initial_join", memtx_join_f, ctx) != 0)
> return -1;
> - return cord_cojoin(&cord);
> + struct memtx_engine *memtx = (struct memtx_engine *)engine;
> + memtx->replica_join_cord = &cord;
> + int res = cord_cojoin(&cord);
> + memtx->replica_join_cord = NULL;
> + return res;
> }
>
> static void
> @@ -1030,6 +1051,8 @@ memtx_engine_new(const char *snap_dirname, bool force_recovery,
> memtx->max_tuple_size = MAX_TUPLE_SIZE;
> memtx->force_recovery = force_recovery;
>
> + memtx->replica_join_cord = NULL;
> +
> memtx->base.vtab = &memtx_engine_vtab;
> memtx->base.name = "memtx";
>
> diff --git a/src/box/memtx_engine.h b/src/box/memtx_engine.h
> index c092f5d8e..f562c66df 100644
> --- a/src/box/memtx_engine.h
> +++ b/src/box/memtx_engine.h
> @@ -107,6 +107,11 @@ struct memtx_engine {
> uint64_t snap_io_rate_limit;
> /** Skip invalid snapshot records if this flag is set. */
> bool force_recovery;
> + /**
> + * Cord being currently used to join replica. It is only
> + * needed to be able to cancel it on shutdown.
> + */
> + struct cord *replica_join_cord;
> /** Common quota for tuples and indexes. */
> struct quota quota;
> /**
>
^ permalink raw reply [flat|nested] 6+ messages in thread
* Re: [Tarantool-patches] [PATCH] replication: cancel replica joining thread at exit
2019-10-15 15:50 [Tarantool-patches] [PATCH] replication: cancel replica joining thread at exit Ilya Kosarev
2019-10-21 13:57 ` [Tarantool-patches] [tarantool-patches] " Alexander Tikhonov
2019-10-22 21:28 ` Vladislav Shpilevoy
@ 2019-10-24 5:09 ` Kirill Yukhin
2 siblings, 0 replies; 6+ messages in thread
From: Kirill Yukhin @ 2019-10-24 5:09 UTC (permalink / raw)
To: Ilya Kosarev; +Cc: tarantool-patches, tarantool-patches
Hello,
On 15 окт 18:50, Ilya Kosarev wrote:
> If a tarantool instance exits while joining replica is in progress,
> the replica joining thread can access already freed data resulting
> in a crash. Let's fix this the same way we did for checkpoint thread
> - simply cancel the thread forcefully and wait for it to terminate.
>
> Closes #4528
> ---
> https://github.com/tarantool/tarantool/tree/i.kosarev/gh-4528-fix-shutdown-on-replica-join
> https://github.com/tarantool/tarantool/issues/4528
I've checked your patch into master.
--
Regards, Kirill Yukhin
^ permalink raw reply [flat|nested] 6+ messages in thread
end of thread, other threads:[~2019-10-24 5:10 UTC | newest]
Thread overview: 6+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2019-10-15 15:50 [Tarantool-patches] [PATCH] replication: cancel replica joining thread at exit Ilya Kosarev
2019-10-21 13:57 ` [Tarantool-patches] [tarantool-patches] " Alexander Tikhonov
2019-10-22 21:28 ` Vladislav Shpilevoy
2019-10-23 12:59 ` [Tarantool-patches] " Ilya Kosarev
2019-10-23 20:55 ` Vladislav Shpilevoy
2019-10-24 5:09 ` Kirill Yukhin
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox