* [tarantool-patches] [PATCH] [replication] make join stage more informative @ 2018-10-16 10:10 Serge Petrenko 2018-10-18 12:43 ` Vladimir Davydov 0 siblings, 1 reply; 4+ messages in thread From: Serge Petrenko @ 2018-10-16 10:10 UTC (permalink / raw) To: tarantool-patches; +Cc: Serge Petrenko This patch adds logging amount of rows recieved by applier during the join stage, the same way that recovery has it. It also adds downstream: status: joining message to master's box.info.replication for a replica that is joining. Closes #3165 --- https://github.com/tarantool/tarantool/issues/3165 https://github.com/tarantool/tarantool/tree/sp/gh-3165-join-progress-report src/box/applier.cc | 7 +++++++ src/box/lua/info.c | 9 +++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/box/applier.cc b/src/box/applier.cc index 7da278e68..ba68c3d07 100644 --- a/src/box/applier.cc +++ b/src/box/applier.cc @@ -309,11 +309,15 @@ applier_join(struct applier *applier) * Receive initial data. */ assert(applier->join_stream != NULL); + uint64_t row_count = 0; while (true) { coio_read_xrow(coio, ibuf, &row); applier->last_row_time = ev_monotonic_now(loop()); if (iproto_type_is_dml(row.type)) { xstream_write_xc(applier->join_stream, &row); + ++row_count; + if (row_count % 100000 == 0) + say_info("%.1fM rows recieved", row_count / 1e6); } else if (row.type == IPROTO_OK) { if (applier->version_id < version_id(1, 7, 0)) { /* @@ -354,6 +358,9 @@ applier_join(struct applier *applier) if (iproto_type_is_dml(row.type)) { vclock_follow_xrow(&replicaset.vclock, &row); xstream_write_xc(applier->subscribe_stream, &row); + ++row_count; + if (row_count % 100000 == 0) + say_info("%.1fM rows recieved", row_count / 1e6); } else if (row.type == IPROTO_OK) { /* * Current vclock. This is not used now, diff --git a/src/box/lua/info.c b/src/box/lua/info.c index 655768ec4..4c25255c0 100644 --- a/src/box/lua/info.c +++ b/src/box/lua/info.c @@ -119,8 +119,13 @@ static void lbox_pushrelay(lua_State *L, struct relay *relay) { lua_newtable(L); - lua_pushstring(L, "vclock"); - lbox_pushvclock(L, relay_vclock(relay)); + if (relay_vclock(relay)->map == 0) { + lua_pushstring(L, "status"); + lua_pushstring(L, "joining"); + } else { + lua_pushstring(L, "vclock"); + lbox_pushvclock(L, relay_vclock(relay)); + } lua_settable(L, -3); } -- 2.17.1 (Apple Git-112) ^ permalink raw reply [flat|nested] 4+ messages in thread
* Re: [tarantool-patches] [PATCH] [replication] make join stage more informative 2018-10-16 10:10 [tarantool-patches] [PATCH] [replication] make join stage more informative Serge Petrenko @ 2018-10-18 12:43 ` Vladimir Davydov 2018-10-18 13:38 ` Serge Petrenko 0 siblings, 1 reply; 4+ messages in thread From: Vladimir Davydov @ 2018-10-18 12:43 UTC (permalink / raw) To: Serge Petrenko; +Cc: tarantool-patches On Tue, Oct 16, 2018 at 01:10:53PM +0300, Serge Petrenko wrote: > This patch adds logging amount of rows recieved by applier during the > join stage, the same way that recovery has it. It also adds > > downstream: > status: joining > > message to master's box.info.replication for a replica that is joining. Please add a test case for the new box.info.replication status. > > Closes #3165 > --- > https://github.com/tarantool/tarantool/issues/3165 > https://github.com/tarantool/tarantool/tree/sp/gh-3165-join-progress-report > > src/box/applier.cc | 7 +++++++ > src/box/lua/info.c | 9 +++++++-- > 2 files changed, 14 insertions(+), 2 deletions(-) > > diff --git a/src/box/applier.cc b/src/box/applier.cc > index 7da278e68..ba68c3d07 100644 > --- a/src/box/applier.cc > +++ b/src/box/applier.cc > @@ -309,11 +309,15 @@ applier_join(struct applier *applier) > * Receive initial data. > */ > assert(applier->join_stream != NULL); > + uint64_t row_count = 0; > while (true) { > coio_read_xrow(coio, ibuf, &row); > applier->last_row_time = ev_monotonic_now(loop()); > if (iproto_type_is_dml(row.type)) { > xstream_write_xc(applier->join_stream, &row); > + ++row_count; > + if (row_count % 100000 == 0) > + say_info("%.1fM rows recieved", row_count / 1e6); Without knowing the total number of rows to be received, this isn't very informative IMO. > } else if (row.type == IPROTO_OK) { > if (applier->version_id < version_id(1, 7, 0)) { > /* > @@ -354,6 +358,9 @@ applier_join(struct applier *applier) > if (iproto_type_is_dml(row.type)) { > vclock_follow_xrow(&replicaset.vclock, &row); > xstream_write_xc(applier->subscribe_stream, &row); > + ++row_count; > + if (row_count % 100000 == 0) > + say_info("%.1fM rows recieved", row_count / 1e6); When bootstrapping from Tarantool < 1.7, final join won't end here, it will end in applier_subscribe(). Not sure if we want to handle this case though. > } else if (row.type == IPROTO_OK) { > /* > * Current vclock. This is not used now, > diff --git a/src/box/lua/info.c b/src/box/lua/info.c > index 655768ec4..4c25255c0 100644 > --- a/src/box/lua/info.c > +++ b/src/box/lua/info.c > @@ -119,8 +119,13 @@ static void > lbox_pushrelay(lua_State *L, struct relay *relay) > { > lua_newtable(L); > - lua_pushstring(L, "vclock"); > - lbox_pushvclock(L, relay_vclock(relay)); > + if (relay_vclock(relay)->map == 0) { > + lua_pushstring(L, "status"); > + lua_pushstring(L, "joining"); > + } else { > + lua_pushstring(L, "vclock"); > + lbox_pushvclock(L, relay_vclock(relay)); > + } This wouldn't work, because struct replica doesn't exist before initial join is complete. > lua_settable(L, -3); > } ^ permalink raw reply [flat|nested] 4+ messages in thread
* Re: [tarantool-patches] [PATCH] [replication] make join stage more informative 2018-10-18 12:43 ` Vladimir Davydov @ 2018-10-18 13:38 ` Serge Petrenko 2018-10-25 16:51 ` Vladimir Davydov 0 siblings, 1 reply; 4+ messages in thread From: Serge Petrenko @ 2018-10-18 13:38 UTC (permalink / raw) To: Vladimir Davydov; +Cc: tarantool-patches [-- Attachment #1: Type: text/plain, Size: 3274 bytes --] HI! Thank you for review. The new diff is below. > 18 окт. 2018 г., в 15:43, Vladimir Davydov <vdavydov.dev@gmail.com> написал(а): > > On Tue, Oct 16, 2018 at 01:10:53PM +0300, Serge Petrenko wrote: >> >> + if (row_count % 100000 == 0) >> + say_info("%.1fM rows recieved", row_count / 1e6); > > Without knowing the total number of rows to be received, this isn't very > informative IMO. The same is done in logging during recovery. Here it at least lets you make a rough time estimation. > >> } else if (row.type == IPROTO_OK) { >> if (applier->version_id < version_id(1, 7, 0)) { >> /* >> @@ -354,6 +358,9 @@ applier_join(struct applier *applier) >> if (iproto_type_is_dml(row.type)) { >> vclock_follow_xrow(&replicaset.vclock, &row); >> xstream_write_xc(applier->subscribe_stream, &row); >> + ++row_count; >> + if (row_count % 100000 == 0) >> + say_info("%.1fM rows recieved", row_count / 1e6); > > When bootstrapping from Tarantool < 1.7, final join won't end here, it > will end in applier_subscribe(). Not sure if we want to handle this case > though. I suggest not adding logging to the subscribe stage, it is not needed IMO. > >> } else if (row.type == IPROTO_OK) { >> /* >> * Current vclock. This is not used now, >> diff --git a/src/box/lua/info.c b/src/box/lua/info.c >> index 655768ec4..4c25255c0 100644 >> --- a/src/box/lua/info.c >> +++ b/src/box/lua/info.c >> @@ -119,8 +119,13 @@ static void >> lbox_pushrelay(lua_State *L, struct relay *relay) >> { >> lua_newtable(L); >> - lua_pushstring(L, "vclock"); >> - lbox_pushvclock(L, relay_vclock(relay)); >> + if (relay_vclock(relay)->map == 0) { >> + lua_pushstring(L, "status"); >> + lua_pushstring(L, "joining"); >> + } else { >> + lua_pushstring(L, "vclock"); >> + lbox_pushvclock(L, relay_vclock(relay)); >> + } > > This wouldn't work, because struct replica doesn't exist before initial > join is complete. Yes, you’re correct. Removed this. --- src/box/applier.cc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/box/applier.cc b/src/box/applier.cc index 7da278e68..ba68c3d07 100644 --- a/src/box/applier.cc +++ b/src/box/applier.cc @@ -309,11 +309,15 @@ applier_join(struct applier *applier) * Receive initial data. */ assert(applier->join_stream != NULL); + uint64_t row_count = 0; while (true) { coio_read_xrow(coio, ibuf, &row); applier->last_row_time = ev_monotonic_now(loop()); if (iproto_type_is_dml(row.type)) { xstream_write_xc(applier->join_stream, &row); + ++row_count; + if (row_count % 100000 == 0) + say_info("%.1fM rows recieved", row_count / 1e6); } else if (row.type == IPROTO_OK) { if (applier->version_id < version_id(1, 7, 0)) { /* @@ -354,6 +358,9 @@ applier_join(struct applier *applier) if (iproto_type_is_dml(row.type)) { vclock_follow_xrow(&replicaset.vclock, &row); xstream_write_xc(applier->subscribe_stream, &row); + ++row_count; + if (row_count % 100000 == 0) + say_info("%.1fM rows recieved", row_count / 1e6); } else if (row.type == IPROTO_OK) { /* * Current vclock. This is not used now, -- 2.17.1 (Apple Git-112) [-- Attachment #2: Type: text/html, Size: 11459 bytes --] ^ permalink raw reply [flat|nested] 4+ messages in thread
* Re: [tarantool-patches] [PATCH] [replication] make join stage more informative 2018-10-18 13:38 ` Serge Petrenko @ 2018-10-25 16:51 ` Vladimir Davydov 0 siblings, 0 replies; 4+ messages in thread From: Vladimir Davydov @ 2018-10-25 16:51 UTC (permalink / raw) To: Serge Petrenko; +Cc: tarantool-patches Pushed to 1.10-features ^ permalink raw reply [flat|nested] 4+ messages in thread
end of thread, other threads:[~2018-10-25 16:51 UTC | newest] Thread overview: 4+ messages (download: mbox.gz / follow: Atom feed) -- links below jump to the message on this page -- 2018-10-16 10:10 [tarantool-patches] [PATCH] [replication] make join stage more informative Serge Petrenko 2018-10-18 12:43 ` Vladimir Davydov 2018-10-18 13:38 ` Serge Petrenko 2018-10-25 16:51 ` Vladimir Davydov
This is a public inbox, see mirroring instructions for how to clone and mirror all data and code used for this inbox