* [tarantool-patches] [PATCH] [replication] make join stage more informative
@ 2018-10-16 10:10 Serge Petrenko
2018-10-18 12:43 ` Vladimir Davydov
0 siblings, 1 reply; 4+ messages in thread
From: Serge Petrenko @ 2018-10-16 10:10 UTC (permalink / raw)
To: tarantool-patches; +Cc: Serge Petrenko
This patch adds logging amount of rows recieved by applier during the
join stage, the same way that recovery has it. It also adds
downstream:
status: joining
message to master's box.info.replication for a replica that is joining.
Closes #3165
---
https://github.com/tarantool/tarantool/issues/3165
https://github.com/tarantool/tarantool/tree/sp/gh-3165-join-progress-report
src/box/applier.cc | 7 +++++++
src/box/lua/info.c | 9 +++++++--
2 files changed, 14 insertions(+), 2 deletions(-)
diff --git a/src/box/applier.cc b/src/box/applier.cc
index 7da278e68..ba68c3d07 100644
--- a/src/box/applier.cc
+++ b/src/box/applier.cc
@@ -309,11 +309,15 @@ applier_join(struct applier *applier)
* Receive initial data.
*/
assert(applier->join_stream != NULL);
+ uint64_t row_count = 0;
while (true) {
coio_read_xrow(coio, ibuf, &row);
applier->last_row_time = ev_monotonic_now(loop());
if (iproto_type_is_dml(row.type)) {
xstream_write_xc(applier->join_stream, &row);
+ ++row_count;
+ if (row_count % 100000 == 0)
+ say_info("%.1fM rows recieved", row_count / 1e6);
} else if (row.type == IPROTO_OK) {
if (applier->version_id < version_id(1, 7, 0)) {
/*
@@ -354,6 +358,9 @@ applier_join(struct applier *applier)
if (iproto_type_is_dml(row.type)) {
vclock_follow_xrow(&replicaset.vclock, &row);
xstream_write_xc(applier->subscribe_stream, &row);
+ ++row_count;
+ if (row_count % 100000 == 0)
+ say_info("%.1fM rows recieved", row_count / 1e6);
} else if (row.type == IPROTO_OK) {
/*
* Current vclock. This is not used now,
diff --git a/src/box/lua/info.c b/src/box/lua/info.c
index 655768ec4..4c25255c0 100644
--- a/src/box/lua/info.c
+++ b/src/box/lua/info.c
@@ -119,8 +119,13 @@ static void
lbox_pushrelay(lua_State *L, struct relay *relay)
{
lua_newtable(L);
- lua_pushstring(L, "vclock");
- lbox_pushvclock(L, relay_vclock(relay));
+ if (relay_vclock(relay)->map == 0) {
+ lua_pushstring(L, "status");
+ lua_pushstring(L, "joining");
+ } else {
+ lua_pushstring(L, "vclock");
+ lbox_pushvclock(L, relay_vclock(relay));
+ }
lua_settable(L, -3);
}
--
2.17.1 (Apple Git-112)
^ permalink raw reply [flat|nested] 4+ messages in thread
* Re: [tarantool-patches] [PATCH] [replication] make join stage more informative
2018-10-16 10:10 [tarantool-patches] [PATCH] [replication] make join stage more informative Serge Petrenko
@ 2018-10-18 12:43 ` Vladimir Davydov
2018-10-18 13:38 ` Serge Petrenko
0 siblings, 1 reply; 4+ messages in thread
From: Vladimir Davydov @ 2018-10-18 12:43 UTC (permalink / raw)
To: Serge Petrenko; +Cc: tarantool-patches
On Tue, Oct 16, 2018 at 01:10:53PM +0300, Serge Petrenko wrote:
> This patch adds logging amount of rows recieved by applier during the
> join stage, the same way that recovery has it. It also adds
>
> downstream:
> status: joining
>
> message to master's box.info.replication for a replica that is joining.
Please add a test case for the new box.info.replication status.
>
> Closes #3165
> ---
> https://github.com/tarantool/tarantool/issues/3165
> https://github.com/tarantool/tarantool/tree/sp/gh-3165-join-progress-report
>
> src/box/applier.cc | 7 +++++++
> src/box/lua/info.c | 9 +++++++--
> 2 files changed, 14 insertions(+), 2 deletions(-)
>
> diff --git a/src/box/applier.cc b/src/box/applier.cc
> index 7da278e68..ba68c3d07 100644
> --- a/src/box/applier.cc
> +++ b/src/box/applier.cc
> @@ -309,11 +309,15 @@ applier_join(struct applier *applier)
> * Receive initial data.
> */
> assert(applier->join_stream != NULL);
> + uint64_t row_count = 0;
> while (true) {
> coio_read_xrow(coio, ibuf, &row);
> applier->last_row_time = ev_monotonic_now(loop());
> if (iproto_type_is_dml(row.type)) {
> xstream_write_xc(applier->join_stream, &row);
> + ++row_count;
> + if (row_count % 100000 == 0)
> + say_info("%.1fM rows recieved", row_count / 1e6);
Without knowing the total number of rows to be received, this isn't very
informative IMO.
> } else if (row.type == IPROTO_OK) {
> if (applier->version_id < version_id(1, 7, 0)) {
> /*
> @@ -354,6 +358,9 @@ applier_join(struct applier *applier)
> if (iproto_type_is_dml(row.type)) {
> vclock_follow_xrow(&replicaset.vclock, &row);
> xstream_write_xc(applier->subscribe_stream, &row);
> + ++row_count;
> + if (row_count % 100000 == 0)
> + say_info("%.1fM rows recieved", row_count / 1e6);
When bootstrapping from Tarantool < 1.7, final join won't end here, it
will end in applier_subscribe(). Not sure if we want to handle this case
though.
> } else if (row.type == IPROTO_OK) {
> /*
> * Current vclock. This is not used now,
> diff --git a/src/box/lua/info.c b/src/box/lua/info.c
> index 655768ec4..4c25255c0 100644
> --- a/src/box/lua/info.c
> +++ b/src/box/lua/info.c
> @@ -119,8 +119,13 @@ static void
> lbox_pushrelay(lua_State *L, struct relay *relay)
> {
> lua_newtable(L);
> - lua_pushstring(L, "vclock");
> - lbox_pushvclock(L, relay_vclock(relay));
> + if (relay_vclock(relay)->map == 0) {
> + lua_pushstring(L, "status");
> + lua_pushstring(L, "joining");
> + } else {
> + lua_pushstring(L, "vclock");
> + lbox_pushvclock(L, relay_vclock(relay));
> + }
This wouldn't work, because struct replica doesn't exist before initial
join is complete.
> lua_settable(L, -3);
> }
^ permalink raw reply [flat|nested] 4+ messages in thread
* Re: [tarantool-patches] [PATCH] [replication] make join stage more informative
2018-10-18 12:43 ` Vladimir Davydov
@ 2018-10-18 13:38 ` Serge Petrenko
2018-10-25 16:51 ` Vladimir Davydov
0 siblings, 1 reply; 4+ messages in thread
From: Serge Petrenko @ 2018-10-18 13:38 UTC (permalink / raw)
To: Vladimir Davydov; +Cc: tarantool-patches
[-- Attachment #1: Type: text/plain, Size: 3274 bytes --]
HI! Thank you for review.
The new diff is below.
> 18 окт. 2018 г., в 15:43, Vladimir Davydov <vdavydov.dev@gmail.com> написал(а):
>
> On Tue, Oct 16, 2018 at 01:10:53PM +0300, Serge Petrenko wrote:
>>
>> + if (row_count % 100000 == 0)
>> + say_info("%.1fM rows recieved", row_count / 1e6);
>
> Without knowing the total number of rows to be received, this isn't very
> informative IMO.
The same is done in logging during recovery. Here it at least lets you make a rough
time estimation.
>
>> } else if (row.type == IPROTO_OK) {
>> if (applier->version_id < version_id(1, 7, 0)) {
>> /*
>> @@ -354,6 +358,9 @@ applier_join(struct applier *applier)
>> if (iproto_type_is_dml(row.type)) {
>> vclock_follow_xrow(&replicaset.vclock, &row);
>> xstream_write_xc(applier->subscribe_stream, &row);
>> + ++row_count;
>> + if (row_count % 100000 == 0)
>> + say_info("%.1fM rows recieved", row_count / 1e6);
>
> When bootstrapping from Tarantool < 1.7, final join won't end here, it
> will end in applier_subscribe(). Not sure if we want to handle this case
> though.
I suggest not adding logging to the subscribe stage, it is not needed IMO.
>
>> } else if (row.type == IPROTO_OK) {
>> /*
>> * Current vclock. This is not used now,
>> diff --git a/src/box/lua/info.c b/src/box/lua/info.c
>> index 655768ec4..4c25255c0 100644
>> --- a/src/box/lua/info.c
>> +++ b/src/box/lua/info.c
>> @@ -119,8 +119,13 @@ static void
>> lbox_pushrelay(lua_State *L, struct relay *relay)
>> {
>> lua_newtable(L);
>> - lua_pushstring(L, "vclock");
>> - lbox_pushvclock(L, relay_vclock(relay));
>> + if (relay_vclock(relay)->map == 0) {
>> + lua_pushstring(L, "status");
>> + lua_pushstring(L, "joining");
>> + } else {
>> + lua_pushstring(L, "vclock");
>> + lbox_pushvclock(L, relay_vclock(relay));
>> + }
>
> This wouldn't work, because struct replica doesn't exist before initial
> join is complete.
Yes, you’re correct. Removed this.
---
src/box/applier.cc | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/src/box/applier.cc b/src/box/applier.cc
index 7da278e68..ba68c3d07 100644
--- a/src/box/applier.cc
+++ b/src/box/applier.cc
@@ -309,11 +309,15 @@ applier_join(struct applier *applier)
* Receive initial data.
*/
assert(applier->join_stream != NULL);
+ uint64_t row_count = 0;
while (true) {
coio_read_xrow(coio, ibuf, &row);
applier->last_row_time = ev_monotonic_now(loop());
if (iproto_type_is_dml(row.type)) {
xstream_write_xc(applier->join_stream, &row);
+ ++row_count;
+ if (row_count % 100000 == 0)
+ say_info("%.1fM rows recieved", row_count / 1e6);
} else if (row.type == IPROTO_OK) {
if (applier->version_id < version_id(1, 7, 0)) {
/*
@@ -354,6 +358,9 @@ applier_join(struct applier *applier)
if (iproto_type_is_dml(row.type)) {
vclock_follow_xrow(&replicaset.vclock, &row);
xstream_write_xc(applier->subscribe_stream, &row);
+ ++row_count;
+ if (row_count % 100000 == 0)
+ say_info("%.1fM rows recieved", row_count / 1e6);
} else if (row.type == IPROTO_OK) {
/*
* Current vclock. This is not used now,
--
2.17.1 (Apple Git-112)
[-- Attachment #2: Type: text/html, Size: 11459 bytes --]
^ permalink raw reply [flat|nested] 4+ messages in thread
* Re: [tarantool-patches] [PATCH] [replication] make join stage more informative
2018-10-18 13:38 ` Serge Petrenko
@ 2018-10-25 16:51 ` Vladimir Davydov
0 siblings, 0 replies; 4+ messages in thread
From: Vladimir Davydov @ 2018-10-25 16:51 UTC (permalink / raw)
To: Serge Petrenko; +Cc: tarantool-patches
Pushed to 1.10-features
^ permalink raw reply [flat|nested] 4+ messages in thread
end of thread, other threads:[~2018-10-25 16:51 UTC | newest]
Thread overview: 4+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2018-10-16 10:10 [tarantool-patches] [PATCH] [replication] make join stage more informative Serge Petrenko
2018-10-18 12:43 ` Vladimir Davydov
2018-10-18 13:38 ` Serge Petrenko
2018-10-25 16:51 ` Vladimir Davydov
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox