[PATCH 0/2] vinyl: add knob for forcing major compaction

[PATCH 0/2] vinyl: add knob for forcing major compaction

[Vladimir Davydov]

This patch adds a new Lua method index.compact(). If called on
a vinyl index, this method triggers major compaction. For other
kinds of indexes, it is a no-op.

https://github.com/tarantool/tarantool/issues/3139
https://github.com/tarantool/tarantool/commits/gh-3139-vy-allow-to-force-compaction

Vladimir Davydov (2):
  index: add compact method
  vinyl: implement index compact method

 src/box/index.cc            |  19 +++++++-
 src/box/index.h             |  23 +++++++++
 src/box/lua/index.c         |  15 ++++++
 src/box/lua/schema.lua      |   4 ++
 src/box/memtx_bitset.c      |   1 +
 src/box/memtx_hash.c        |   1 +
 src/box/memtx_rtree.c       |   1 +
 src/box/memtx_tree.c        |   1 +
 src/box/sysview_index.c     |   1 +
 src/box/vinyl.c             |   9 ++++
 src/box/vy_lsm.c            |  13 +++++
 src/box/vy_lsm.h            |   6 +++
 src/box/vy_range.c          |  23 +++++++++
 src/box/vy_range.h          |  12 +++++
 src/box/vy_scheduler.c      |   9 ++++
 src/box/vy_scheduler.h      |   7 +++
 test/vinyl/compact.result   | 114 +++++++++++++++++++++++++++++++++++++++++++-
 test/vinyl/compact.test.lua |  58 +++++++++++++++++++++-
 18 files changed, 312 insertions(+), 5 deletions(-)

-- 
2.11.0

[PATCH 1/2] index: add compact method

[Vladimir Davydov]

This patch adds index.compact() Lua method. The new method is backed by
index_vtab::compact. Currently, it's a no-op for all kinds of indexes.
It will be used by Vinyl engine in order to trigger major compaction.

Part of #3139
---
 src/box/index.cc        | 19 ++++++++++++++++++-
 src/box/index.h         | 23 +++++++++++++++++++++++
 src/box/lua/index.c     | 15 +++++++++++++++
 src/box/lua/schema.lua  |  4 ++++
 src/box/memtx_bitset.c  |  1 +
 src/box/memtx_hash.c    |  1 +
 src/box/memtx_rtree.c   |  1 +
 src/box/memtx_tree.c    |  1 +
 src/box/sysview_index.c |  1 +
 src/box/vinyl.c         |  1 +
 10 files changed, 66 insertions(+), 1 deletion(-)

diff --git a/src/box/index.cc b/src/box/index.cc
index 6cd04833..978935b3 100644
--- a/src/box/index.cc
+++ b/src/box/index.cc
@@ -410,7 +410,7 @@ box_iterator_free(box_iterator_t *it)
 
 /* }}} */
 
-/* {{{ Introspection */
+/* {{{ Other index functions */
 
 int
 box_index_info(uint32_t space_id, uint32_t index_id,
@@ -424,6 +424,17 @@ box_index_info(uint32_t space_id, uint32_t index_id,
 	return 0;
 }
 
+int
+box_index_compact(uint32_t space_id, uint32_t index_id)
+{
+	struct space *space;
+	struct index *index;
+	if (check_index(space_id, index_id, &space, &index) != 0)
+		return -1;
+	index_compact(index);
+	return 0;
+}
+
 /* }}} */
 
 /* {{{ Internal API */
@@ -666,6 +677,12 @@ generic_index_info(struct index *index, struct info_handler *handler)
 }
 
 void
+generic_index_compact(struct index *index)
+{
+	(void)index;
+}
+
+void
 generic_index_reset_stat(struct index *index)
 {
 	(void)index;
diff --git a/src/box/index.h b/src/box/index.h
index 0aa96ade..c7384b61 100644
--- a/src/box/index.h
+++ b/src/box/index.h
@@ -224,6 +224,17 @@ int
 box_index_info(uint32_t space_id, uint32_t index_id,
 	       struct info_handler *info);
 
+/**
+ * Trigger index compaction (index:compact())
+ *
+ * \param space_id space identifier
+ * \param index_id index identifier
+ * \retval -1 on error (check box_error_last())
+ * \retval >=0 on success
+ */
+int
+box_index_compact(uint32_t space_id, uint32_t index_id);
+
 struct iterator {
 	/**
 	 * Iterate to the next tuple.
@@ -412,6 +423,11 @@ struct index_vtab {
 	struct snapshot_iterator *(*create_snapshot_iterator)(struct index *);
 	/** Introspection (index:info()) */
 	void (*info)(struct index *, struct info_handler *);
+	/**
+	 * Trigger asynchronous index compaction. What exactly
+	 * is implied under 'compaction' depends on the engine.
+	 */
+	void (*compact)(struct index *);
 	/** Reset all incremental statistic counters. */
 	void (*reset_stat)(struct index *);
 	/**
@@ -604,6 +620,12 @@ index_info(struct index *index, struct info_handler *handler)
 }
 
 static inline void
+index_compact(struct index *index)
+{
+	index->vtab->compact(index);
+}
+
+static inline void
 index_reset_stat(struct index *index)
 {
 	index->vtab->reset_stat(index);
@@ -653,6 +675,7 @@ int generic_index_replace(struct index *, struct tuple *, struct tuple *,
 			  enum dup_replace_mode, struct tuple **);
 struct snapshot_iterator *generic_index_create_snapshot_iterator(struct index *);
 void generic_index_info(struct index *, struct info_handler *);
+void generic_index_compact(struct index *);
 void generic_index_reset_stat(struct index *);
 void generic_index_begin_build(struct index *);
 int generic_index_reserve(struct index *, uint32_t);
diff --git a/src/box/lua/index.c b/src/box/lua/index.c
index 6dfa6485..5b212f0b 100644
--- a/src/box/lua/index.c
+++ b/src/box/lua/index.c
@@ -314,6 +314,20 @@ lbox_index_info(lua_State *L)
 	return 1;
 }
 
+static int
+lbox_index_compact(lua_State *L)
+{
+	if (lua_gettop(L) != 2 || !lua_isnumber(L, 1) || !lua_isnumber(L, 2))
+		return luaL_error(L, "usage index.compact(space_id, index_id)");
+
+	uint32_t space_id = lua_tonumber(L, 1);
+	uint32_t index_id = lua_tonumber(L, 2);
+
+	if (box_index_compact(space_id, index_id) != 0)
+		return luaT_error(L);
+	return 0;
+}
+
 /* }}} */
 
 void
@@ -350,6 +364,7 @@ box_lua_index_init(struct lua_State *L)
 		{"iterator_next", lbox_iterator_next},
 		{"truncate", lbox_truncate},
 		{"info", lbox_index_info},
+		{"compact", lbox_index_compact},
 		{NULL, NULL}
 	};
 
diff --git a/src/box/lua/schema.lua b/src/box/lua/schema.lua
index 1a616d56..552d4651 100644
--- a/src/box/lua/schema.lua
+++ b/src/box/lua/schema.lua
@@ -1288,6 +1288,10 @@ base_index_mt.info = function(index)
     return internal.info(index.space_id, index.id);
 end
 
+base_index_mt.compact = function(index)
+    return internal.compact(index.space_id, index.id)
+end
+
 base_index_mt.drop = function(index)
     check_index_arg(index, 'drop')
     return box.schema.index.drop(index.space_id, index.id)
diff --git a/src/box/memtx_bitset.c b/src/box/memtx_bitset.c
index 7b87560b..e0088337 100644
--- a/src/box/memtx_bitset.c
+++ b/src/box/memtx_bitset.c
@@ -476,6 +476,7 @@ static const struct index_vtab memtx_bitset_index_vtab = {
 	/* .create_snapshot_iterator = */
 		generic_index_create_snapshot_iterator,
 	/* .info = */ generic_index_info,
+	/* .compact = */ generic_index_compact,
 	/* .reset_stat = */ generic_index_reset_stat,
 	/* .begin_build = */ generic_index_begin_build,
 	/* .reserve = */ generic_index_reserve,
diff --git a/src/box/memtx_hash.c b/src/box/memtx_hash.c
index 24138f42..2748e850 100644
--- a/src/box/memtx_hash.c
+++ b/src/box/memtx_hash.c
@@ -400,6 +400,7 @@ static const struct index_vtab memtx_hash_index_vtab = {
 	/* .create_snapshot_iterator = */
 		memtx_hash_index_create_snapshot_iterator,
 	/* .info = */ generic_index_info,
+	/* .compact = */ generic_index_compact,
 	/* .reset_stat = */ generic_index_reset_stat,
 	/* .begin_build = */ generic_index_begin_build,
 	/* .reserve = */ generic_index_reserve,
diff --git a/src/box/memtx_rtree.c b/src/box/memtx_rtree.c
index 653343a7..83de921c 100644
--- a/src/box/memtx_rtree.c
+++ b/src/box/memtx_rtree.c
@@ -320,6 +320,7 @@ static const struct index_vtab memtx_rtree_index_vtab = {
 	/* .create_snapshot_iterator = */
 		generic_index_create_snapshot_iterator,
 	/* .info = */ generic_index_info,
+	/* .compact = */ generic_index_compact,
 	/* .reset_stat = */ generic_index_reset_stat,
 	/* .begin_build = */ generic_index_begin_build,
 	/* .reserve = */ generic_index_reserve,
diff --git a/src/box/memtx_tree.c b/src/box/memtx_tree.c
index 073006b4..6b77ef3f 100644
--- a/src/box/memtx_tree.c
+++ b/src/box/memtx_tree.c
@@ -609,6 +609,7 @@ static const struct index_vtab memtx_tree_index_vtab = {
 	/* .create_snapshot_iterator = */
 		memtx_tree_index_create_snapshot_iterator,
 	/* .info = */ generic_index_info,
+	/* .compact = */ generic_index_compact,
 	/* .reset_stat = */ generic_index_reset_stat,
 	/* .begin_build = */ memtx_tree_index_begin_build,
 	/* .reserve = */ memtx_tree_index_reserve,
diff --git a/src/box/sysview_index.c b/src/box/sysview_index.c
index 8bfc39b0..43b30d48 100644
--- a/src/box/sysview_index.c
+++ b/src/box/sysview_index.c
@@ -190,6 +190,7 @@ static const struct index_vtab sysview_index_vtab = {
 	/* .create_snapshot_iterator = */
 		generic_index_create_snapshot_iterator,
 	/* .info = */ generic_index_info,
+	/* .compact = */ generic_index_compact,
 	/* .reset_stat = */ generic_index_reset_stat,
 	/* .begin_build = */ generic_index_begin_build,
 	/* .reserve = */ generic_index_reserve,
diff --git a/src/box/vinyl.c b/src/box/vinyl.c
index 6c8f0df8..896f8559 100644
--- a/src/box/vinyl.c
+++ b/src/box/vinyl.c
@@ -4060,6 +4060,7 @@ static const struct index_vtab vinyl_index_vtab = {
 	/* .create_snapshot_iterator = */
 		generic_index_create_snapshot_iterator,
 	/* .info = */ vinyl_index_info,
+	/* .compact = */ generic_index_compact,
 	/* .reset_stat = */ vinyl_index_reset_stat,
 	/* .begin_build = */ generic_index_begin_build,
 	/* .reserve = */ generic_index_reserve,
-- 
2.11.0

[PATCH 2/2] vinyl: implement index compact method

[Vladimir Davydov]

Force major compaction of all ranges when index.compact() is called.
Note, the function only triggers compaction, it doesn't wait until
compaction is complete.

Closes #3139
---
 src/box/vinyl.c             |  10 +++-
 src/box/vy_lsm.c            |  13 +++++
 src/box/vy_lsm.h            |   6 +++
 src/box/vy_range.c          |  23 +++++++++
 src/box/vy_range.h          |  12 +++++
 src/box/vy_scheduler.c      |   9 ++++
 src/box/vy_scheduler.h      |   7 +++
 test/vinyl/compact.result   | 114 +++++++++++++++++++++++++++++++++++++++++++-
 test/vinyl/compact.test.lua |  58 +++++++++++++++++++++-
 9 files changed, 247 insertions(+), 5 deletions(-)

diff --git a/src/box/vinyl.c b/src/box/vinyl.c
index 896f8559..2e2c145a 100644
--- a/src/box/vinyl.c
+++ b/src/box/vinyl.c
@@ -1265,6 +1265,14 @@ vinyl_index_bsize(struct index *index)
 	return bsize;
 }
 
+static void
+vinyl_index_compact(struct index *index)
+{
+	struct vy_lsm *lsm = vy_lsm(index);
+	struct vy_env *env = vy_env(index->engine);
+	vy_scheduler_force_compaction(&env->scheduler, lsm);
+}
+
 /* {{{ Public API of transaction control: start/end transaction,
  * read, write data in the context of a transaction.
  */
@@ -4060,7 +4068,7 @@ static const struct index_vtab vinyl_index_vtab = {
 	/* .create_snapshot_iterator = */
 		generic_index_create_snapshot_iterator,
 	/* .info = */ vinyl_index_info,
-	/* .compact = */ generic_index_compact,
+	/* .compact = */ vinyl_index_compact,
 	/* .reset_stat = */ vinyl_index_reset_stat,
 	/* .begin_build = */ generic_index_begin_build,
 	/* .reserve = */ generic_index_reserve,
diff --git a/src/box/vy_lsm.c b/src/box/vy_lsm.c
index 4dfc0a25..edc3b1a4 100644
--- a/src/box/vy_lsm.c
+++ b/src/box/vy_lsm.c
@@ -1109,3 +1109,16 @@ fail_range:
 		  vy_lsm_name(lsm), vy_range_str(range));
 	return false;
 }
+
+void
+vy_lsm_force_compaction(struct vy_lsm *lsm)
+{
+	struct vy_range *range;
+	struct vy_range_tree_iterator it;
+
+	vy_range_tree_ifirst(lsm->tree, &it);
+	while ((range = vy_range_tree_inext(&it)) != NULL)
+		vy_range_force_compaction(range);
+
+	vy_range_heap_update_all(&lsm->range_heap);
+}
diff --git a/src/box/vy_lsm.h b/src/box/vy_lsm.h
index c7ccb7d6..3f820fac 100644
--- a/src/box/vy_lsm.h
+++ b/src/box/vy_lsm.h
@@ -466,6 +466,12 @@ bool
 vy_lsm_coalesce_range(struct vy_lsm *lsm, struct vy_range *range);
 
 /**
+ * Mark all ranges of an LSM tree for major compaction.
+ */
+void
+vy_lsm_force_compaction(struct vy_lsm *lsm);
+
+/**
  * Insert a statement into the in-memory index of an LSM tree. If
  * the region_stmt is NULL and the statement is successfully inserted
  * then the new lsregion statement is returned via @a region_stmt.
diff --git a/src/box/vy_range.c b/src/box/vy_range.c
index baecf178..6a55a018 100644
--- a/src/box/vy_range.c
+++ b/src/box/vy_range.c
@@ -262,6 +262,18 @@ vy_range_remove_slice(struct vy_range *range, struct vy_slice *slice)
 	vy_disk_stmt_counter_sub(&range->count, &slice->count);
 }
 
+void
+vy_range_force_compaction(struct vy_range *range)
+{
+	if (range->slice_count == 1) {
+		/* Already compacted. */
+		assert(!range->needs_compaction);
+		return;
+	}
+	range->needs_compaction = true;
+	range->compact_priority = range->slice_count;
+}
+
 /**
  * To reduce write amplification caused by compaction, we follow
  * the LSM tree design. Runs in each range are divided into groups
@@ -292,6 +304,17 @@ vy_range_update_compact_priority(struct vy_range *range,
 	assert(opts->run_count_per_level > 0);
 	assert(opts->run_size_ratio > 1);
 
+	if (range->slice_count == 1) {
+		/* Nothing to compact. */
+		range->compact_priority = 1;
+		range->needs_compaction = false;
+		return;
+	}
+	if (range->needs_compaction) {
+		range->compact_priority = range->slice_count;
+		return;
+	}
+
 	range->compact_priority = 0;
 
 	/* Total number of checked runs. */
diff --git a/src/box/vy_range.h b/src/box/vy_range.h
index 3a451fb4..d7031e70 100644
--- a/src/box/vy_range.h
+++ b/src/box/vy_range.h
@@ -106,6 +106,14 @@ struct vy_range {
 	 * how we  decide how many runs to compact next time.
 	 */
 	int compact_priority;
+	/**
+	 * If this flag is set, the range must be scheduled for
+	 * major compaction, i.e. its compact_priority must be
+	 * raised to max (slice_count). The flag is set by
+	 * vy_range_force_compaction() and cleared automatically
+	 * when all slices of the range have been compacted.
+	 */
+	bool needs_compaction;
 	/** Number of times the range was compacted. */
 	int n_compactions;
 	/** Link in vy_lsm->tree. */
@@ -221,6 +229,10 @@ vy_range_add_slice_before(struct vy_range *range, struct vy_slice *slice,
 void
 vy_range_remove_slice(struct vy_range *range, struct vy_slice *slice);
 
+/** Mark a range for major compaction. */
+void
+vy_range_force_compaction(struct vy_range *range);
+
 /**
  * Update compaction priority of a range.
  *
diff --git a/src/box/vy_scheduler.c b/src/box/vy_scheduler.c
index 790d818a..d1545e76 100644
--- a/src/box/vy_scheduler.c
+++ b/src/box/vy_scheduler.c
@@ -458,6 +458,15 @@ vy_scheduler_trigger_dump(struct vy_scheduler *scheduler)
 	fiber_cond_signal(&scheduler->scheduler_cond);
 }
 
+void
+vy_scheduler_force_compaction(struct vy_scheduler *scheduler,
+			      struct vy_lsm *lsm)
+{
+	vy_lsm_force_compaction(lsm);
+	vy_scheduler_update_lsm(scheduler, lsm);
+	fiber_cond_signal(&scheduler->scheduler_cond);
+}
+
 /**
  * Check whether the current dump round is complete.
  * If it is, free memory and proceed to the next dump round.
diff --git a/src/box/vy_scheduler.h b/src/box/vy_scheduler.h
index 4db86152..d3bc4c03 100644
--- a/src/box/vy_scheduler.h
+++ b/src/box/vy_scheduler.h
@@ -202,6 +202,13 @@ void
 vy_scheduler_trigger_dump(struct vy_scheduler *scheduler);
 
 /**
+ * Force major compaction of an LSM tree.
+ */
+void
+vy_scheduler_force_compaction(struct vy_scheduler *scheduler,
+			      struct vy_lsm *lsm);
+
+/**
  * Schedule a checkpoint. Please call vy_scheduler_wait_checkpoint()
  * after that.
  */
diff --git a/test/vinyl/compact.result b/test/vinyl/compact.result
index 79762f5b..758f7691 100644
--- a/test/vinyl/compact.result
+++ b/test/vinyl/compact.result
@@ -4,6 +4,9 @@ test_run = require('test_run').new()
 fiber = require('fiber')
 ---
 ...
+digest = require('digest')
+---
+...
 space = box.schema.space.create("vinyl", { engine = 'vinyl' })
 ---
 ...
@@ -94,9 +97,116 @@ test_run:grep_log('default', 'UPSERT operation failed') ~= nil
 space:drop()
 ---
 ...
-fiber = nil
+--
+-- gh-3139: index.compact forces major compaction for all ranges
+--
+s = box.schema.space.create('test', {engine = 'vinyl'})
+---
+...
+_ = s:create_index('pk', {run_count_per_level = 100, page_size = 128, range_size = 1024})
+---
+...
+test_run:cmd("setopt delimiter ';'")
+---
+- true
+...
+function dump()
+    for i = 1, 10 do
+        s:replace{i, digest.urandom(1000)}
+    end
+    box.snapshot()
+end;
+---
+...
+function info()
+    local info = s.index.pk:info()
+    return {range_count = info.range_count, run_count = info.run_count}
+end
+function compact()
+    s.index.pk:compact()
+    repeat
+        fiber.sleep(0.001)
+        local info = s.index.pk:info()
+    until info.range_count == info.run_count
+end;
+---
+...
+test_run:cmd("setopt delimiter ''");
+---
+- true
+...
+dump()
+---
+...
+dump()
+---
+...
+dump()
+---
+...
+info() -- 1 range, 3 runs
+---
+- run_count: 3
+  range_count: 1
+...
+compact()
+---
+...
+info() -- 1 range, 1 run
+---
+- run_count: 1
+  range_count: 1
+...
+compact() -- no-op
+---
+...
+dump()
+---
+...
+dump()
+---
+...
+dump()
+---
+...
+info() -- 1 range, 4 runs
+---
+- run_count: 4
+  range_count: 1
+...
+compact()
+---
+...
+info() -- 2 ranges, 2 runs
+---
+- run_count: 2
+  range_count: 2
+...
+compact() -- no-op
+---
+...
+dump()
+---
+...
+dump()
+---
+...
+dump()
+---
+...
+info() -- 2 ranges, 5 runs
+---
+- run_count: 5
+  range_count: 2
+...
+compact()
+---
+...
+info() -- 4 ranges, 4 runs
 ---
+- run_count: 4
+  range_count: 4
 ...
-test_run = nil
+s:drop()
 ---
 ...
diff --git a/test/vinyl/compact.test.lua b/test/vinyl/compact.test.lua
index 920aafbd..b2a746aa 100644
--- a/test/vinyl/compact.test.lua
+++ b/test/vinyl/compact.test.lua
@@ -1,5 +1,6 @@
 test_run = require('test_run').new()
 fiber = require('fiber')
+digest = require('digest')
 
 space = box.schema.space.create("vinyl", { engine = 'vinyl' })
 _= space:create_index('primary', { parts = { 1, 'unsigned' }, run_count_per_level = 2 })
@@ -39,5 +40,58 @@ test_run:grep_log('default', 'UPSERT operation failed') ~= nil
 
 space:drop()
 
-fiber = nil
-test_run = nil
+--
+-- gh-3139: index.compact forces major compaction for all ranges
+--
+s = box.schema.space.create('test', {engine = 'vinyl'})
+_ = s:create_index('pk', {run_count_per_level = 100, page_size = 128, range_size = 1024})
+
+test_run:cmd("setopt delimiter ';'")
+function dump()
+    for i = 1, 10 do
+        s:replace{i, digest.urandom(1000)}
+    end
+    box.snapshot()
+end;
+function info()
+    local info = s.index.pk:info()
+    return {range_count = info.range_count, run_count = info.run_count}
+end
+function compact()
+    s.index.pk:compact()
+    repeat
+        fiber.sleep(0.001)
+        local info = s.index.pk:info()
+    until info.range_count == info.run_count
+end;
+test_run:cmd("setopt delimiter ''");
+
+dump()
+dump()
+dump()
+info() -- 1 range, 3 runs
+
+compact()
+info() -- 1 range, 1 run
+
+compact() -- no-op
+
+dump()
+dump()
+dump()
+info() -- 1 range, 4 runs
+
+compact()
+info() -- 2 ranges, 2 runs
+
+compact() -- no-op
+
+dump()
+dump()
+dump()
+info() -- 2 ranges, 5 runs
+
+compact()
+info() -- 4 ranges, 4 runs
+
+s:drop()
-- 
2.11.0

Re: [PATCH 1/2] index: add compact method

[Konstantin Osipov]

* Vladimir Davydov <vdavydov.dev@gmail.com> [18/05/18 19:21]:
> Part of #3139

OK to push.

Please don't forget to file a documentation request, you can now
use a bot for it.


-- 
Konstantin Osipov, Moscow, Russia, +7 903 626 22 32
http://tarantool.io - www.twitter.com/kostja_osipov

Re: [PATCH 2/2] vinyl: implement index compact method

[Konstantin Osipov]

* Vladimir Davydov <vdavydov.dev@gmail.com> [18/05/18 19:21]:
> Force major compaction of all ranges when index.compact() is called.
> Note, the function only triggers compaction, it doesn't wait until
> compaction is complete.
> 
> Closes #3139

OK to push.

-- 
Konstantin Osipov, Moscow, Russia, +7 903 626 22 32
http://tarantool.io - www.twitter.com/kostja_osipov