[PATCH v2 03/11] vinyl: minor refactoring of quota methods

[Vladimir Davydov <vdavydov.dev@gmail.com>]

The refactoring is targeted at facilitating introduction of rate
limiting within the quota class. It moves code blocks around, factors
out some blocks in functions, and improves comments. No functional
changes.

Needed for #1862
---
 src/box/vy_quota.c | 131 +++++++++++++++++++++++++++++++++++++----------------
 1 file changed, 91 insertions(+), 40 deletions(-)

diff --git a/src/box/vy_quota.c b/src/box/vy_quota.c
index e6d22348..64ce56c0 100644
--- a/src/box/vy_quota.c
+++ b/src/box/vy_quota.c
@@ -31,6 +31,7 @@
 #include "vy_quota.h"
 
 #include <assert.h>
+#include <stdbool.h>
 #include <stddef.h>
 #include <tarantool_ev.h>
 
@@ -40,13 +41,55 @@
 #include "trivia/util.h"
 
 /**
- * Returns true if the quota limit is exceeded and so consumers
- * have to wait.
+ * Return true if the requested amount of memory may be consumed
+ * right now, false if consumers have to wait.
  */
 static inline bool
-vy_quota_is_exceeded(struct vy_quota *q)
+vy_quota_may_use(struct vy_quota *q, size_t size)
 {
-	return q->used > q->limit;
+	if (q->used + size > q->limit)
+		return false;
+	return true;
+}
+
+/**
+ * Consume the given amount of memory without checking the limit.
+ */
+static inline void
+vy_quota_do_use(struct vy_quota *q, size_t size)
+{
+	q->used += size;
+}
+
+/**
+ * Return the given amount of memory without waking blocked fibers.
+ * This function is an exact opposite of vy_quota_do_use().
+ */
+static inline void
+vy_quota_do_unuse(struct vy_quota *q, size_t size)
+{
+	assert(q->used >= size);
+	q->used -= size;
+}
+
+/**
+ * Invoke the registered callback in case memory usage exceeds
+ * the configured limit.
+ */
+static inline void
+vy_quota_check_limit(struct vy_quota *q)
+{
+	if (q->used > q->limit)
+		q->quota_exceeded_cb(q);
+}
+
+/**
+ * Wake up the first consumer in the line waiting for quota.
+ */
+static void
+vy_quota_signal(struct vy_quota *q)
+{
+	fiber_cond_signal(&q->cond);
 }
 
 void
@@ -70,55 +113,63 @@ void
 vy_quota_set_limit(struct vy_quota *q, size_t limit)
 {
 	q->limit = limit;
-	if (q->used >= limit)
-		q->quota_exceeded_cb(q);
-	fiber_cond_signal(&q->cond);
+	vy_quota_check_limit(q);
+	vy_quota_signal(q);
 }
 
 void
 vy_quota_force_use(struct vy_quota *q, size_t size)
 {
-	q->used += size;
-	if (q->used >= q->limit)
-		q->quota_exceeded_cb(q);
+	vy_quota_do_use(q, size);
+	vy_quota_check_limit(q);
 }
 
 void
 vy_quota_release(struct vy_quota *q, size_t size)
 {
-	assert(q->used >= size);
-	q->used -= size;
-	fiber_cond_signal(&q->cond);
+	vy_quota_do_unuse(q, size);
+	vy_quota_signal(q);
 }
 
 int
 vy_quota_use(struct vy_quota *q, size_t size, double timeout)
 {
-	q->used += size;
-	if (vy_quota_is_exceeded(q)) {
-		/* Wait for quota. */
-		double start_time = ev_monotonic_now(loop());
-		double deadline = start_time + timeout;
+	if (vy_quota_may_use(q, size)) {
+		vy_quota_do_use(q, size);
+		return 0;
+	}
 
-		do {
-			q->quota_exceeded_cb(q);
-			q->used -= size;
-			if (fiber_cond_wait_deadline(&q->cond, deadline) != 0)
-				return -1; /* timed out */
-			q->used += size;
-		} while (vy_quota_is_exceeded(q));
-
-		double wait_time = ev_monotonic_now(loop()) - start_time;
-		if (wait_time > q->too_long_threshold) {
-			say_warn("waited for %zu bytes of vinyl memory quota "
-				 "for too long: %.3f sec", size, wait_time);
-		}
+	/* Wait for quota. */
+	double wait_start = ev_monotonic_now(loop());
+	double deadline = wait_start + timeout;
+
+	do {
 		/*
-		 * Wake up the next fiber in the line waiting
-		 * for quota.
+		 * If the requested amount of memory cannot be
+		 * consumed due to the configured limit, notify
+		 * the caller before going to sleep so that it
+		 * can start memory reclaim immediately.
 		 */
-		fiber_cond_signal(&q->cond);
+		if (q->used + size > q->limit)
+			q->quota_exceeded_cb(q);
+		if (fiber_cond_wait_deadline(&q->cond, deadline) != 0)
+			return -1; /* timed out */
+	} while (!vy_quota_may_use(q, size));
+
+	double wait_time = ev_monotonic_now(loop()) - wait_start;
+	if (wait_time > q->too_long_threshold) {
+		say_warn("waited for %zu bytes of vinyl memory quota "
+			 "for too long: %.3f sec", size, wait_time);
 	}
+
+	vy_quota_do_use(q, size);
+	/*
+	 * Blocked consumers are awaken one by one to preserve
+	 * the order they were put to sleep. It's a responsibility
+	 * of a consumer that managed to acquire the requested
+	 * amount of quota to wake up the next one in the line.
+	 */
+	vy_quota_signal(q);
 	return 0;
 }
 
@@ -126,11 +177,11 @@ void
 vy_quota_adjust(struct vy_quota *q, size_t reserved, size_t used)
 {
 	if (reserved > used) {
-		size_t excess = reserved - used;
-		assert(q->used >= excess);
-		q->used -= excess;
-		fiber_cond_signal(&q->cond);
+		vy_quota_do_unuse(q, reserved - used);
+		vy_quota_signal(q);
+	}
+	if (reserved < used) {
+		vy_quota_do_use(q, used - reserved);
+		vy_quota_check_limit(q);
 	}
-	if (reserved < used)
-		vy_quota_force_use(q, used - reserved);
 }
-- 
2.11.0