Re: [Tarantool-patches] [PATCH v3 07/12] box: introduce `box.ctl.demote`

[Serge Petrenko via Tarantool-patches <tarantool-patches@dev.tarantool.org>]

04.08.2021 02:25, Vladislav Shpilevoy пишет:
> Thanks for the fixes!
>
> See 3 comments below.
>
>> diff --git a/src/box/iproto.cc b/src/box/iproto.cc
>> index 3ed641eea..5cc69b77f 100644
>> --- a/src/box/iproto.cc
>> +++ b/src/box/iproto.cc
>> @@ -557,6 +558,20 @@ struct iproto_connection
>>       struct iproto_thread *iproto_thread;
>>   };
>>
>> +#ifdef NDEBUG
>> +#define iproto_write_error(fd, e, schema_version, sync)                         \
>> +    iproto_do_write_error(fd, e, schema_version, sync);
>> +#else
>> +#define iproto_write_error(fd, e, schema_version, sync) do {                    \
>> +    int flags = fcntl(fd, F_GETFL, 0);                                      \
>> +    if (flags >= 0)                                                         \
>> +        fcntl(fd, F_SETFL, flags & (~O_NONBLOCK));                      \
>> +    iproto_do_write_error(fd, e, schema_version, sync);                     \
>> +    if (flags >= 0)                                                         \
>> +        fcntl(fd, F_SETFL, flags);                                      \
>> +} while (0);
>> +#endif
> 1. Why didn't you do this right in iproto_write_error? Why do
> you need to change iproto.cc code at all?

I've found a commit way back from 2017 where you removed the
fcntl dependency from xrow. I didn't want to introduce it again.

>> diff --git a/src/box/relay.cc b/src/box/relay.cc
>> index 805b5e7ff..2947468ba 100644
>> --- a/src/box/relay.cc
>> +++ b/src/box/relay.cc
>> @@ -444,20 +444,20 @@ relay_initial_join(int fd, uint64_t sync, struct vclock *vclock,
>>        */
>>       if (replica_version_id > 0) {
>>           /* Mark the beginning of the metadata stream. */
>> -        row.type = IPROTO_JOIN_META;
>> -        coio_write_xrow(&relay->io, &row);
>> +        xrow_encode_type(&row, IPROTO_JOIN_META);
>> +        xstream_write(&relay->stream, &row);
> 2. Why do you use xtream_write? And why coio_write_xrow just a few lines
> above works just fine then?

xstream write is basically a shortcut for "row.sync = sync ; 
coio_write_xrow"

Writing row.sync = sync before every coio_write_xrow looked rather ugly 
IMO, that's
why I changed it.

I need to reassign row.sync before each despatch because xrow_encode_* 
functions usually
reset all the xrow fields to 0.

>
>>           xrow_encode_raft(&row, &fiber()->gc, &raft_req);
>> -        coio_write_xrow(&relay->io, &row);
>> +        xstream_write(&relay->stream, &row);
>>
>>           char body[XROW_SYNCHRO_BODY_LEN_MAX];
>>           xrow_encode_synchro(&row, body, &req);
>>           row.replica_id = req.replica_id;
>> -        coio_write_xrow(&relay->io, &row);
>> +        xstream_write(&relay->stream, &row);
>>
>>           /* Mark the end of the metadata stream. */
>> -        row.type = IPROTO_JOIN_SNAPSHOT;
>> -        coio_write_xrow(&relay->io, &row);
>> +        xrow_encode_type(&row, IPROTO_JOIN_SNAPSHOT);
>> +        xstream_write(&relay->stream, &row);
>>       }
>> diff --git a/src/box/raft.c b/src/box/raft.c
>> index f8d13aa32..bc69f7f1b 100644
>> --- a/src/box/raft.c
>> +++ b/src/box/raft.c
>> @@ -83,6 +83,24 @@ box_raft_request_to_msg(const struct raft_request *req, struct raft_msg *msg)
>>       };
>>   }
>>
>> +static void
>> +box_raft_update_synchro_queue(struct raft *raft)
>> +{
>> +    assert(raft == box_raft());
>> +    if (raft->state != RAFT_STATE_LEADER)
>> +        return;
>> +    int rc = 0;
>> +    uint32_t errcode = 0;
>> +    do {
>> +        rc = box_promote_qsync();
>> +        if (rc != 0) {
>> +            struct error *err = diag_last_error(diag_get());
>> +            errcode = box_error_code(err);
>> +            diag_log();
>> +        }
> 3. Could it stop being a leader during the retries? Would it
> make sense to continue them then?

Thanks for noticing!

Check out the diff:

===============================
diff --git a/src/box/box.cc b/src/box/box.cc
index 4882b76a4..c7a972992 100644
--- a/src/box/box.cc
+++ b/src/box/box.cc
@@ -1698,6 +1698,10 @@ box_promote_qsync(void)
         int64_t wait_lsn = box_wait_limbo_acked(TIMEOUT_INFINITY);
         if (wait_lsn < 0)
                 return -1;
+       if (raft->state != RAFT_STATE_LEADER) {
+               diag_set(ClientError, ER_NOT_LEADER, raft->leader);
+               return -1;
+       }
         box_issue_promote(txn_limbo.owner_id, wait_lsn);
         return 0;
  }
diff --git a/src/box/errcode.h b/src/box/errcode.h
index d2854677f..906cd28b2 100644
--- a/src/box/errcode.h
+++ b/src/box/errcode.h
@@ -278,6 +278,7 @@ struct errcode_record {
         /*223 */_(ER_INTERFERING_PROMOTE,       "Instance with replica 
id %u was promoted first") \
         /*224 */_(ER_ELECTION_DISABLED,         "Elections were turned 
off")\
         /*225 */_(ER_TXN_ROLLBACK,              "Transaction was rolled 
back") \
+       /*226 */_(ER_NOT_LEADER,                "The instance is not a 
leader. New leader is %u")\

  /*
   * !IMPORTANT! Please follow instructions at start of the file

===============================
>
>> +    } while (rc != 0 && errcode == ER_QUORUM_WAIT && !fiber_is_cancelled());
>> +}

-- 
Serge Petrenko