[PATCH 2/2] vinyl: implement index compact method

[Vladimir Davydov <vdavydov.dev@gmail.com>]

Force major compaction of all ranges when index.compact() is called.
Note, the function only triggers compaction, it doesn't wait until
compaction is complete.

Closes #3139
---
 src/box/vinyl.c             |  10 +++-
 src/box/vy_lsm.c            |  13 +++++
 src/box/vy_lsm.h            |   6 +++
 src/box/vy_range.c          |  23 +++++++++
 src/box/vy_range.h          |  12 +++++
 src/box/vy_scheduler.c      |   9 ++++
 src/box/vy_scheduler.h      |   7 +++
 test/vinyl/compact.result   | 114 +++++++++++++++++++++++++++++++++++++++++++-
 test/vinyl/compact.test.lua |  58 +++++++++++++++++++++-
 9 files changed, 247 insertions(+), 5 deletions(-)

diff --git a/src/box/vinyl.c b/src/box/vinyl.c
index 896f8559..2e2c145a 100644
--- a/src/box/vinyl.c
+++ b/src/box/vinyl.c
@@ -1265,6 +1265,14 @@ vinyl_index_bsize(struct index *index)
 	return bsize;
 }
 
+static void
+vinyl_index_compact(struct index *index)
+{
+	struct vy_lsm *lsm = vy_lsm(index);
+	struct vy_env *env = vy_env(index->engine);
+	vy_scheduler_force_compaction(&env->scheduler, lsm);
+}
+
 /* {{{ Public API of transaction control: start/end transaction,
  * read, write data in the context of a transaction.
  */
@@ -4060,7 +4068,7 @@ static const struct index_vtab vinyl_index_vtab = {
 	/* .create_snapshot_iterator = */
 		generic_index_create_snapshot_iterator,
 	/* .info = */ vinyl_index_info,
-	/* .compact = */ generic_index_compact,
+	/* .compact = */ vinyl_index_compact,
 	/* .reset_stat = */ vinyl_index_reset_stat,
 	/* .begin_build = */ generic_index_begin_build,
 	/* .reserve = */ generic_index_reserve,
diff --git a/src/box/vy_lsm.c b/src/box/vy_lsm.c
index 4dfc0a25..edc3b1a4 100644
--- a/src/box/vy_lsm.c
+++ b/src/box/vy_lsm.c
@@ -1109,3 +1109,16 @@ fail_range:
 		  vy_lsm_name(lsm), vy_range_str(range));
 	return false;
 }
+
+void
+vy_lsm_force_compaction(struct vy_lsm *lsm)
+{
+	struct vy_range *range;
+	struct vy_range_tree_iterator it;
+
+	vy_range_tree_ifirst(lsm->tree, &it);
+	while ((range = vy_range_tree_inext(&it)) != NULL)
+		vy_range_force_compaction(range);
+
+	vy_range_heap_update_all(&lsm->range_heap);
+}
diff --git a/src/box/vy_lsm.h b/src/box/vy_lsm.h
index c7ccb7d6..3f820fac 100644
--- a/src/box/vy_lsm.h
+++ b/src/box/vy_lsm.h
@@ -466,6 +466,12 @@ bool
 vy_lsm_coalesce_range(struct vy_lsm *lsm, struct vy_range *range);
 
 /**
+ * Mark all ranges of an LSM tree for major compaction.
+ */
+void
+vy_lsm_force_compaction(struct vy_lsm *lsm);
+
+/**
  * Insert a statement into the in-memory index of an LSM tree. If
  * the region_stmt is NULL and the statement is successfully inserted
  * then the new lsregion statement is returned via @a region_stmt.
diff --git a/src/box/vy_range.c b/src/box/vy_range.c
index baecf178..6a55a018 100644
--- a/src/box/vy_range.c
+++ b/src/box/vy_range.c
@@ -262,6 +262,18 @@ vy_range_remove_slice(struct vy_range *range, struct vy_slice *slice)
 	vy_disk_stmt_counter_sub(&range->count, &slice->count);
 }
 
+void
+vy_range_force_compaction(struct vy_range *range)
+{
+	if (range->slice_count == 1) {
+		/* Already compacted. */
+		assert(!range->needs_compaction);
+		return;
+	}
+	range->needs_compaction = true;
+	range->compact_priority = range->slice_count;
+}
+
 /**
  * To reduce write amplification caused by compaction, we follow
  * the LSM tree design. Runs in each range are divided into groups
@@ -292,6 +304,17 @@ vy_range_update_compact_priority(struct vy_range *range,
 	assert(opts->run_count_per_level > 0);
 	assert(opts->run_size_ratio > 1);
 
+	if (range->slice_count == 1) {
+		/* Nothing to compact. */
+		range->compact_priority = 1;
+		range->needs_compaction = false;
+		return;
+	}
+	if (range->needs_compaction) {
+		range->compact_priority = range->slice_count;
+		return;
+	}
+
 	range->compact_priority = 0;
 
 	/* Total number of checked runs. */
diff --git a/src/box/vy_range.h b/src/box/vy_range.h
index 3a451fb4..d7031e70 100644
--- a/src/box/vy_range.h
+++ b/src/box/vy_range.h
@@ -106,6 +106,14 @@ struct vy_range {
 	 * how we  decide how many runs to compact next time.
 	 */
 	int compact_priority;
+	/**
+	 * If this flag is set, the range must be scheduled for
+	 * major compaction, i.e. its compact_priority must be
+	 * raised to max (slice_count). The flag is set by
+	 * vy_range_force_compaction() and cleared automatically
+	 * when all slices of the range have been compacted.
+	 */
+	bool needs_compaction;
 	/** Number of times the range was compacted. */
 	int n_compactions;
 	/** Link in vy_lsm->tree. */
@@ -221,6 +229,10 @@ vy_range_add_slice_before(struct vy_range *range, struct vy_slice *slice,
 void
 vy_range_remove_slice(struct vy_range *range, struct vy_slice *slice);
 
+/** Mark a range for major compaction. */
+void
+vy_range_force_compaction(struct vy_range *range);
+
 /**
  * Update compaction priority of a range.
  *
diff --git a/src/box/vy_scheduler.c b/src/box/vy_scheduler.c
index 790d818a..d1545e76 100644
--- a/src/box/vy_scheduler.c
+++ b/src/box/vy_scheduler.c
@@ -458,6 +458,15 @@ vy_scheduler_trigger_dump(struct vy_scheduler *scheduler)
 	fiber_cond_signal(&scheduler->scheduler_cond);
 }
 
+void
+vy_scheduler_force_compaction(struct vy_scheduler *scheduler,
+			      struct vy_lsm *lsm)
+{
+	vy_lsm_force_compaction(lsm);
+	vy_scheduler_update_lsm(scheduler, lsm);
+	fiber_cond_signal(&scheduler->scheduler_cond);
+}
+
 /**
  * Check whether the current dump round is complete.
  * If it is, free memory and proceed to the next dump round.
diff --git a/src/box/vy_scheduler.h b/src/box/vy_scheduler.h
index 4db86152..d3bc4c03 100644
--- a/src/box/vy_scheduler.h
+++ b/src/box/vy_scheduler.h
@@ -202,6 +202,13 @@ void
 vy_scheduler_trigger_dump(struct vy_scheduler *scheduler);
 
 /**
+ * Force major compaction of an LSM tree.
+ */
+void
+vy_scheduler_force_compaction(struct vy_scheduler *scheduler,
+			      struct vy_lsm *lsm);
+
+/**
  * Schedule a checkpoint. Please call vy_scheduler_wait_checkpoint()
  * after that.
  */
diff --git a/test/vinyl/compact.result b/test/vinyl/compact.result
index 79762f5b..758f7691 100644
--- a/test/vinyl/compact.result
+++ b/test/vinyl/compact.result
@@ -4,6 +4,9 @@ test_run = require('test_run').new()
 fiber = require('fiber')
 ---
 ...
+digest = require('digest')
+---
+...
 space = box.schema.space.create("vinyl", { engine = 'vinyl' })
 ---
 ...
@@ -94,9 +97,116 @@ test_run:grep_log('default', 'UPSERT operation failed') ~= nil
 space:drop()
 ---
 ...
-fiber = nil
+--
+-- gh-3139: index.compact forces major compaction for all ranges
+--
+s = box.schema.space.create('test', {engine = 'vinyl'})
+---
+...
+_ = s:create_index('pk', {run_count_per_level = 100, page_size = 128, range_size = 1024})
+---
+...
+test_run:cmd("setopt delimiter ';'")
+---
+- true
+...
+function dump()
+    for i = 1, 10 do
+        s:replace{i, digest.urandom(1000)}
+    end
+    box.snapshot()
+end;
+---
+...
+function info()
+    local info = s.index.pk:info()
+    return {range_count = info.range_count, run_count = info.run_count}
+end
+function compact()
+    s.index.pk:compact()
+    repeat
+        fiber.sleep(0.001)
+        local info = s.index.pk:info()
+    until info.range_count == info.run_count
+end;
+---
+...
+test_run:cmd("setopt delimiter ''");
+---
+- true
+...
+dump()
+---
+...
+dump()
+---
+...
+dump()
+---
+...
+info() -- 1 range, 3 runs
+---
+- run_count: 3
+  range_count: 1
+...
+compact()
+---
+...
+info() -- 1 range, 1 run
+---
+- run_count: 1
+  range_count: 1
+...
+compact() -- no-op
+---
+...
+dump()
+---
+...
+dump()
+---
+...
+dump()
+---
+...
+info() -- 1 range, 4 runs
+---
+- run_count: 4
+  range_count: 1
+...
+compact()
+---
+...
+info() -- 2 ranges, 2 runs
+---
+- run_count: 2
+  range_count: 2
+...
+compact() -- no-op
+---
+...
+dump()
+---
+...
+dump()
+---
+...
+dump()
+---
+...
+info() -- 2 ranges, 5 runs
+---
+- run_count: 5
+  range_count: 2
+...
+compact()
+---
+...
+info() -- 4 ranges, 4 runs
 ---
+- run_count: 4
+  range_count: 4
 ...
-test_run = nil
+s:drop()
 ---
 ...
diff --git a/test/vinyl/compact.test.lua b/test/vinyl/compact.test.lua
index 920aafbd..b2a746aa 100644
--- a/test/vinyl/compact.test.lua
+++ b/test/vinyl/compact.test.lua
@@ -1,5 +1,6 @@
 test_run = require('test_run').new()
 fiber = require('fiber')
+digest = require('digest')
 
 space = box.schema.space.create("vinyl", { engine = 'vinyl' })
 _= space:create_index('primary', { parts = { 1, 'unsigned' }, run_count_per_level = 2 })
@@ -39,5 +40,58 @@ test_run:grep_log('default', 'UPSERT operation failed') ~= nil
 
 space:drop()
 
-fiber = nil
-test_run = nil
+--
+-- gh-3139: index.compact forces major compaction for all ranges
+--
+s = box.schema.space.create('test', {engine = 'vinyl'})
+_ = s:create_index('pk', {run_count_per_level = 100, page_size = 128, range_size = 1024})
+
+test_run:cmd("setopt delimiter ';'")
+function dump()
+    for i = 1, 10 do
+        s:replace{i, digest.urandom(1000)}
+    end
+    box.snapshot()
+end;
+function info()
+    local info = s.index.pk:info()
+    return {range_count = info.range_count, run_count = info.run_count}
+end
+function compact()
+    s.index.pk:compact()
+    repeat
+        fiber.sleep(0.001)
+        local info = s.index.pk:info()
+    until info.range_count == info.run_count
+end;
+test_run:cmd("setopt delimiter ''");
+
+dump()
+dump()
+dump()
+info() -- 1 range, 3 runs
+
+compact()
+info() -- 1 range, 1 run
+
+compact() -- no-op
+
+dump()
+dump()
+dump()
+info() -- 1 range, 4 runs
+
+compact()
+info() -- 2 ranges, 2 runs
+
+compact() -- no-op
+
+dump()
+dump()
+dump()
+info() -- 2 ranges, 5 runs
+
+compact()
+info() -- 4 ranges, 4 runs
+
+s:drop()
-- 
2.11.0