Re: [Tarantool-patches] [PATCH 1/1] qsync: remove polling from box_promote()

[Serge Petrenko via Tarantool-patches <tarantool-patches@dev.tarantool.org>]

13.07.2021 01:20, Vladislav Shpilevoy пишет:
> box_promote() when called manually used to wait for the existing
> transactions from a foreign limbo to end during a timeout. Giving
> them a chance to end on their terms.
>
> The waiting was done via polling like
>
>      while (!done)
>          sleep(small_timeout);
>
> Polling is almost always super bad both for execution time and
> for CPU usage. The patch replaces it with proper waiting based on
> events happening in the limbo.
>
> Closes #5190
> ---
> Branch: http://github.com/tarantool/tarantool/tree/gerold103/gh-5190-qsync-polling
> Issue: https://github.com/tarantool/tarantool/issues/5190
>
>   src/box/box.cc      |  7 +----
>   src/box/txn_limbo.c | 74 ++++++++++++++++++++++++++++++++++-----------
>   src/box/txn_limbo.h |  4 +++
>   3 files changed, 62 insertions(+), 23 deletions(-)
>
> diff --git a/src/box/box.cc b/src/box/box.cc
> index ab7d983c9..eeb57b04e 100644
> --- a/src/box/box.cc
> +++ b/src/box/box.cc
> @@ -1627,12 +1627,7 @@ box_promote(void)
>   	if (try_wait) {
>   		/* Wait until pending confirmations/rollbacks reach us. */
>   		double timeout = 2 * replication_synchro_timeout;
> -		double start_tm = fiber_clock();
> -		while (!txn_limbo_is_empty(&txn_limbo)) {
> -			if (fiber_clock() - start_tm > timeout)
> -				break;
> -			fiber_sleep(0.001);
> -		}
> +		txn_limbo_wait_empty(&txn_limbo, timeout);
>   		/*
>   		 * Our mission was to clear the limbo from former leader's
>   		 * transactions. Exit in case someone did that for us.
> diff --git a/src/box/txn_limbo.c b/src/box/txn_limbo.c
> index 51dc2a186..fdea287c7 100644
> --- a/src/box/txn_limbo.c
> +++ b/src/box/txn_limbo.c
> @@ -612,11 +612,14 @@ txn_rollback_cb(struct trigger *trigger, void *event)
>   	return 0;
>   }
>   
> -int
> -txn_limbo_wait_confirm(struct txn_limbo *limbo)
> +/**
> + * Wait until the last transaction in the limbo is finished and get its result.
> + */
> +static int
> +txn_limbo_wait_last_txn(struct txn_limbo *limbo, bool *is_rollback,
> +			double timeout)
>   {
> -	if (txn_limbo_is_empty(limbo))
> -		return 0;
> +	assert(!txn_limbo_is_empty(limbo));
>   
>   	/* initialization of a waitpoint. */
>   	struct confirm_waitpoint cwp;
> @@ -632,27 +635,42 @@ txn_limbo_wait_confirm(struct txn_limbo *limbo)
>   	struct txn_limbo_entry *tle = txn_limbo_last_entry(limbo);
>   	txn_on_commit(tle->txn, &on_complete);
>   	txn_on_rollback(tle->txn, &on_rollback);
> -	double start_time = fiber_clock();
> +	double deadline = fiber_clock() + timeout;
> +	int rc;
>   	while (true) {
> -		double deadline = start_time + replication_synchro_timeout;
> +		if (timeout < 0) {
> +			rc = -1;
> +			break;
> +		}
>   		bool cancellable = fiber_set_cancellable(false);
> -		double timeout = deadline - fiber_clock();
> -		int rc = fiber_cond_wait_timeout(&limbo->wait_cond, timeout);
> +		rc = fiber_cond_wait_timeout(&limbo->wait_cond, timeout);
>   		fiber_set_cancellable(cancellable);
> -		if (cwp.is_confirm || cwp.is_rollback)
> -			goto complete;
> +		if (cwp.is_confirm || cwp.is_rollback) {
> +			*is_rollback = cwp.is_rollback;
> +			rc = 0;
> +			break;
> +		}
>   		if (rc != 0)
> -			goto timed_out;
> +			break;
> +		timeout = deadline - fiber_clock();
>   	}
> -timed_out:
> -	/* Clear the triggers if the timeout has been reached. */
>   	trigger_clear(&on_complete);
>   	trigger_clear(&on_rollback);
> -	diag_set(ClientError, ER_SYNC_QUORUM_TIMEOUT);
> -	return -1;
> +	return rc;
> +}
>   
> -complete:
> -	if (!cwp.is_confirm) {
> +int
> +txn_limbo_wait_confirm(struct txn_limbo *limbo)
> +{
> +	if (txn_limbo_is_empty(limbo))
> +		return 0;
> +	bool is_rollback;
> +	if (txn_limbo_wait_last_txn(limbo, &is_rollback,
> +				    replication_synchro_timeout) != 0) {
> +		diag_set(ClientError, ER_SYNC_QUORUM_TIMEOUT);
> +		return -1;
> +	}
> +	if (is_rollback) {
>   		/* The transaction has been rolled back. */
>   		diag_set(ClientError, ER_SYNC_ROLLBACK);
>   		return -1;
> @@ -660,6 +678,28 @@ complete:
>   	return 0;
>   }
>   
> +int
> +txn_limbo_wait_empty(struct txn_limbo *limbo, double timeout)
> +{
> +	if (txn_limbo_is_empty(limbo))
> +		return 0;
> +	bool is_rollback;
> +	double deadline = fiber_clock() + timeout;
> +	/*
> +	 * Retry in the loop. More transactions might be added while waiting for
> +	 * the last one.
> +	 */
> +	do {
> +		if (txn_limbo_wait_last_txn(limbo, &is_rollback,
> +					    timeout) != 0) {
> +			diag_set(ClientError, ER_TIMEOUT);
> +			return -1;
> +		}
> +		timeout = deadline - fiber_clock();
> +	} while (!txn_limbo_is_empty(limbo));
> +	return 0;
> +}
> +
>   void
>   txn_limbo_process(struct txn_limbo *limbo, const struct synchro_request *req)
>   {
> diff --git a/src/box/txn_limbo.h b/src/box/txn_limbo.h
> index e409ac657..7debbc0b9 100644
> --- a/src/box/txn_limbo.h
> +++ b/src/box/txn_limbo.h
> @@ -311,6 +311,10 @@ txn_limbo_process(struct txn_limbo *limbo, const struct synchro_request *req);
>   int
>   txn_limbo_wait_confirm(struct txn_limbo *limbo);
>   
> +/** Wait until the limbo is empty. Regardless of how its transactions end. */
> +int
> +txn_limbo_wait_empty(struct txn_limbo *limbo, double timeout);
> +
>   /**
>    * Write a PROMOTE request, which has the same effect as CONFIRM(@a lsn) and
>    * ROLLBACK(@a lsn + 1) combined.

-- 
Serge Petrenko