* [PATCH 1/2] index: add compact method
2018-05-18 16:21 [PATCH 0/2] vinyl: add knob for forcing major compaction Vladimir Davydov
@ 2018-05-18 16:21 ` Vladimir Davydov
2018-05-20 21:32 ` Konstantin Osipov
2018-05-18 16:21 ` [PATCH 2/2] vinyl: implement index " Vladimir Davydov
1 sibling, 1 reply; 5+ messages in thread
From: Vladimir Davydov @ 2018-05-18 16:21 UTC (permalink / raw)
To: kostja; +Cc: tarantool-patches
This patch adds index.compact() Lua method. The new method is backed by
index_vtab::compact. Currently, it's a no-op for all kinds of indexes.
It will be used by Vinyl engine in order to trigger major compaction.
Part of #3139
---
src/box/index.cc | 19 ++++++++++++++++++-
src/box/index.h | 23 +++++++++++++++++++++++
src/box/lua/index.c | 15 +++++++++++++++
src/box/lua/schema.lua | 4 ++++
src/box/memtx_bitset.c | 1 +
src/box/memtx_hash.c | 1 +
src/box/memtx_rtree.c | 1 +
src/box/memtx_tree.c | 1 +
src/box/sysview_index.c | 1 +
src/box/vinyl.c | 1 +
10 files changed, 66 insertions(+), 1 deletion(-)
diff --git a/src/box/index.cc b/src/box/index.cc
index 6cd04833..978935b3 100644
--- a/src/box/index.cc
+++ b/src/box/index.cc
@@ -410,7 +410,7 @@ box_iterator_free(box_iterator_t *it)
/* }}} */
-/* {{{ Introspection */
+/* {{{ Other index functions */
int
box_index_info(uint32_t space_id, uint32_t index_id,
@@ -424,6 +424,17 @@ box_index_info(uint32_t space_id, uint32_t index_id,
return 0;
}
+int
+box_index_compact(uint32_t space_id, uint32_t index_id)
+{
+ struct space *space;
+ struct index *index;
+ if (check_index(space_id, index_id, &space, &index) != 0)
+ return -1;
+ index_compact(index);
+ return 0;
+}
+
/* }}} */
/* {{{ Internal API */
@@ -666,6 +677,12 @@ generic_index_info(struct index *index, struct info_handler *handler)
}
void
+generic_index_compact(struct index *index)
+{
+ (void)index;
+}
+
+void
generic_index_reset_stat(struct index *index)
{
(void)index;
diff --git a/src/box/index.h b/src/box/index.h
index 0aa96ade..c7384b61 100644
--- a/src/box/index.h
+++ b/src/box/index.h
@@ -224,6 +224,17 @@ int
box_index_info(uint32_t space_id, uint32_t index_id,
struct info_handler *info);
+/**
+ * Trigger index compaction (index:compact())
+ *
+ * \param space_id space identifier
+ * \param index_id index identifier
+ * \retval -1 on error (check box_error_last())
+ * \retval >=0 on success
+ */
+int
+box_index_compact(uint32_t space_id, uint32_t index_id);
+
struct iterator {
/**
* Iterate to the next tuple.
@@ -412,6 +423,11 @@ struct index_vtab {
struct snapshot_iterator *(*create_snapshot_iterator)(struct index *);
/** Introspection (index:info()) */
void (*info)(struct index *, struct info_handler *);
+ /**
+ * Trigger asynchronous index compaction. What exactly
+ * is implied under 'compaction' depends on the engine.
+ */
+ void (*compact)(struct index *);
/** Reset all incremental statistic counters. */
void (*reset_stat)(struct index *);
/**
@@ -604,6 +620,12 @@ index_info(struct index *index, struct info_handler *handler)
}
static inline void
+index_compact(struct index *index)
+{
+ index->vtab->compact(index);
+}
+
+static inline void
index_reset_stat(struct index *index)
{
index->vtab->reset_stat(index);
@@ -653,6 +675,7 @@ int generic_index_replace(struct index *, struct tuple *, struct tuple *,
enum dup_replace_mode, struct tuple **);
struct snapshot_iterator *generic_index_create_snapshot_iterator(struct index *);
void generic_index_info(struct index *, struct info_handler *);
+void generic_index_compact(struct index *);
void generic_index_reset_stat(struct index *);
void generic_index_begin_build(struct index *);
int generic_index_reserve(struct index *, uint32_t);
diff --git a/src/box/lua/index.c b/src/box/lua/index.c
index 6dfa6485..5b212f0b 100644
--- a/src/box/lua/index.c
+++ b/src/box/lua/index.c
@@ -314,6 +314,20 @@ lbox_index_info(lua_State *L)
return 1;
}
+static int
+lbox_index_compact(lua_State *L)
+{
+ if (lua_gettop(L) != 2 || !lua_isnumber(L, 1) || !lua_isnumber(L, 2))
+ return luaL_error(L, "usage index.compact(space_id, index_id)");
+
+ uint32_t space_id = lua_tonumber(L, 1);
+ uint32_t index_id = lua_tonumber(L, 2);
+
+ if (box_index_compact(space_id, index_id) != 0)
+ return luaT_error(L);
+ return 0;
+}
+
/* }}} */
void
@@ -350,6 +364,7 @@ box_lua_index_init(struct lua_State *L)
{"iterator_next", lbox_iterator_next},
{"truncate", lbox_truncate},
{"info", lbox_index_info},
+ {"compact", lbox_index_compact},
{NULL, NULL}
};
diff --git a/src/box/lua/schema.lua b/src/box/lua/schema.lua
index 1a616d56..552d4651 100644
--- a/src/box/lua/schema.lua
+++ b/src/box/lua/schema.lua
@@ -1288,6 +1288,10 @@ base_index_mt.info = function(index)
return internal.info(index.space_id, index.id);
end
+base_index_mt.compact = function(index)
+ return internal.compact(index.space_id, index.id)
+end
+
base_index_mt.drop = function(index)
check_index_arg(index, 'drop')
return box.schema.index.drop(index.space_id, index.id)
diff --git a/src/box/memtx_bitset.c b/src/box/memtx_bitset.c
index 7b87560b..e0088337 100644
--- a/src/box/memtx_bitset.c
+++ b/src/box/memtx_bitset.c
@@ -476,6 +476,7 @@ static const struct index_vtab memtx_bitset_index_vtab = {
/* .create_snapshot_iterator = */
generic_index_create_snapshot_iterator,
/* .info = */ generic_index_info,
+ /* .compact = */ generic_index_compact,
/* .reset_stat = */ generic_index_reset_stat,
/* .begin_build = */ generic_index_begin_build,
/* .reserve = */ generic_index_reserve,
diff --git a/src/box/memtx_hash.c b/src/box/memtx_hash.c
index 24138f42..2748e850 100644
--- a/src/box/memtx_hash.c
+++ b/src/box/memtx_hash.c
@@ -400,6 +400,7 @@ static const struct index_vtab memtx_hash_index_vtab = {
/* .create_snapshot_iterator = */
memtx_hash_index_create_snapshot_iterator,
/* .info = */ generic_index_info,
+ /* .compact = */ generic_index_compact,
/* .reset_stat = */ generic_index_reset_stat,
/* .begin_build = */ generic_index_begin_build,
/* .reserve = */ generic_index_reserve,
diff --git a/src/box/memtx_rtree.c b/src/box/memtx_rtree.c
index 653343a7..83de921c 100644
--- a/src/box/memtx_rtree.c
+++ b/src/box/memtx_rtree.c
@@ -320,6 +320,7 @@ static const struct index_vtab memtx_rtree_index_vtab = {
/* .create_snapshot_iterator = */
generic_index_create_snapshot_iterator,
/* .info = */ generic_index_info,
+ /* .compact = */ generic_index_compact,
/* .reset_stat = */ generic_index_reset_stat,
/* .begin_build = */ generic_index_begin_build,
/* .reserve = */ generic_index_reserve,
diff --git a/src/box/memtx_tree.c b/src/box/memtx_tree.c
index 073006b4..6b77ef3f 100644
--- a/src/box/memtx_tree.c
+++ b/src/box/memtx_tree.c
@@ -609,6 +609,7 @@ static const struct index_vtab memtx_tree_index_vtab = {
/* .create_snapshot_iterator = */
memtx_tree_index_create_snapshot_iterator,
/* .info = */ generic_index_info,
+ /* .compact = */ generic_index_compact,
/* .reset_stat = */ generic_index_reset_stat,
/* .begin_build = */ memtx_tree_index_begin_build,
/* .reserve = */ memtx_tree_index_reserve,
diff --git a/src/box/sysview_index.c b/src/box/sysview_index.c
index 8bfc39b0..43b30d48 100644
--- a/src/box/sysview_index.c
+++ b/src/box/sysview_index.c
@@ -190,6 +190,7 @@ static const struct index_vtab sysview_index_vtab = {
/* .create_snapshot_iterator = */
generic_index_create_snapshot_iterator,
/* .info = */ generic_index_info,
+ /* .compact = */ generic_index_compact,
/* .reset_stat = */ generic_index_reset_stat,
/* .begin_build = */ generic_index_begin_build,
/* .reserve = */ generic_index_reserve,
diff --git a/src/box/vinyl.c b/src/box/vinyl.c
index 6c8f0df8..896f8559 100644
--- a/src/box/vinyl.c
+++ b/src/box/vinyl.c
@@ -4060,6 +4060,7 @@ static const struct index_vtab vinyl_index_vtab = {
/* .create_snapshot_iterator = */
generic_index_create_snapshot_iterator,
/* .info = */ vinyl_index_info,
+ /* .compact = */ generic_index_compact,
/* .reset_stat = */ vinyl_index_reset_stat,
/* .begin_build = */ generic_index_begin_build,
/* .reserve = */ generic_index_reserve,
--
2.11.0
^ permalink raw reply [flat|nested] 5+ messages in thread* [PATCH 2/2] vinyl: implement index compact method
2018-05-18 16:21 [PATCH 0/2] vinyl: add knob for forcing major compaction Vladimir Davydov
2018-05-18 16:21 ` [PATCH 1/2] index: add compact method Vladimir Davydov
@ 2018-05-18 16:21 ` Vladimir Davydov
2018-05-20 21:33 ` Konstantin Osipov
1 sibling, 1 reply; 5+ messages in thread
From: Vladimir Davydov @ 2018-05-18 16:21 UTC (permalink / raw)
To: kostja; +Cc: tarantool-patches
Force major compaction of all ranges when index.compact() is called.
Note, the function only triggers compaction, it doesn't wait until
compaction is complete.
Closes #3139
---
src/box/vinyl.c | 10 +++-
src/box/vy_lsm.c | 13 +++++
src/box/vy_lsm.h | 6 +++
src/box/vy_range.c | 23 +++++++++
src/box/vy_range.h | 12 +++++
src/box/vy_scheduler.c | 9 ++++
src/box/vy_scheduler.h | 7 +++
test/vinyl/compact.result | 114 +++++++++++++++++++++++++++++++++++++++++++-
test/vinyl/compact.test.lua | 58 +++++++++++++++++++++-
9 files changed, 247 insertions(+), 5 deletions(-)
diff --git a/src/box/vinyl.c b/src/box/vinyl.c
index 896f8559..2e2c145a 100644
--- a/src/box/vinyl.c
+++ b/src/box/vinyl.c
@@ -1265,6 +1265,14 @@ vinyl_index_bsize(struct index *index)
return bsize;
}
+static void
+vinyl_index_compact(struct index *index)
+{
+ struct vy_lsm *lsm = vy_lsm(index);
+ struct vy_env *env = vy_env(index->engine);
+ vy_scheduler_force_compaction(&env->scheduler, lsm);
+}
+
/* {{{ Public API of transaction control: start/end transaction,
* read, write data in the context of a transaction.
*/
@@ -4060,7 +4068,7 @@ static const struct index_vtab vinyl_index_vtab = {
/* .create_snapshot_iterator = */
generic_index_create_snapshot_iterator,
/* .info = */ vinyl_index_info,
- /* .compact = */ generic_index_compact,
+ /* .compact = */ vinyl_index_compact,
/* .reset_stat = */ vinyl_index_reset_stat,
/* .begin_build = */ generic_index_begin_build,
/* .reserve = */ generic_index_reserve,
diff --git a/src/box/vy_lsm.c b/src/box/vy_lsm.c
index 4dfc0a25..edc3b1a4 100644
--- a/src/box/vy_lsm.c
+++ b/src/box/vy_lsm.c
@@ -1109,3 +1109,16 @@ fail_range:
vy_lsm_name(lsm), vy_range_str(range));
return false;
}
+
+void
+vy_lsm_force_compaction(struct vy_lsm *lsm)
+{
+ struct vy_range *range;
+ struct vy_range_tree_iterator it;
+
+ vy_range_tree_ifirst(lsm->tree, &it);
+ while ((range = vy_range_tree_inext(&it)) != NULL)
+ vy_range_force_compaction(range);
+
+ vy_range_heap_update_all(&lsm->range_heap);
+}
diff --git a/src/box/vy_lsm.h b/src/box/vy_lsm.h
index c7ccb7d6..3f820fac 100644
--- a/src/box/vy_lsm.h
+++ b/src/box/vy_lsm.h
@@ -466,6 +466,12 @@ bool
vy_lsm_coalesce_range(struct vy_lsm *lsm, struct vy_range *range);
/**
+ * Mark all ranges of an LSM tree for major compaction.
+ */
+void
+vy_lsm_force_compaction(struct vy_lsm *lsm);
+
+/**
* Insert a statement into the in-memory index of an LSM tree. If
* the region_stmt is NULL and the statement is successfully inserted
* then the new lsregion statement is returned via @a region_stmt.
diff --git a/src/box/vy_range.c b/src/box/vy_range.c
index baecf178..6a55a018 100644
--- a/src/box/vy_range.c
+++ b/src/box/vy_range.c
@@ -262,6 +262,18 @@ vy_range_remove_slice(struct vy_range *range, struct vy_slice *slice)
vy_disk_stmt_counter_sub(&range->count, &slice->count);
}
+void
+vy_range_force_compaction(struct vy_range *range)
+{
+ if (range->slice_count == 1) {
+ /* Already compacted. */
+ assert(!range->needs_compaction);
+ return;
+ }
+ range->needs_compaction = true;
+ range->compact_priority = range->slice_count;
+}
+
/**
* To reduce write amplification caused by compaction, we follow
* the LSM tree design. Runs in each range are divided into groups
@@ -292,6 +304,17 @@ vy_range_update_compact_priority(struct vy_range *range,
assert(opts->run_count_per_level > 0);
assert(opts->run_size_ratio > 1);
+ if (range->slice_count == 1) {
+ /* Nothing to compact. */
+ range->compact_priority = 1;
+ range->needs_compaction = false;
+ return;
+ }
+ if (range->needs_compaction) {
+ range->compact_priority = range->slice_count;
+ return;
+ }
+
range->compact_priority = 0;
/* Total number of checked runs. */
diff --git a/src/box/vy_range.h b/src/box/vy_range.h
index 3a451fb4..d7031e70 100644
--- a/src/box/vy_range.h
+++ b/src/box/vy_range.h
@@ -106,6 +106,14 @@ struct vy_range {
* how we decide how many runs to compact next time.
*/
int compact_priority;
+ /**
+ * If this flag is set, the range must be scheduled for
+ * major compaction, i.e. its compact_priority must be
+ * raised to max (slice_count). The flag is set by
+ * vy_range_force_compaction() and cleared automatically
+ * when all slices of the range have been compacted.
+ */
+ bool needs_compaction;
/** Number of times the range was compacted. */
int n_compactions;
/** Link in vy_lsm->tree. */
@@ -221,6 +229,10 @@ vy_range_add_slice_before(struct vy_range *range, struct vy_slice *slice,
void
vy_range_remove_slice(struct vy_range *range, struct vy_slice *slice);
+/** Mark a range for major compaction. */
+void
+vy_range_force_compaction(struct vy_range *range);
+
/**
* Update compaction priority of a range.
*
diff --git a/src/box/vy_scheduler.c b/src/box/vy_scheduler.c
index 790d818a..d1545e76 100644
--- a/src/box/vy_scheduler.c
+++ b/src/box/vy_scheduler.c
@@ -458,6 +458,15 @@ vy_scheduler_trigger_dump(struct vy_scheduler *scheduler)
fiber_cond_signal(&scheduler->scheduler_cond);
}
+void
+vy_scheduler_force_compaction(struct vy_scheduler *scheduler,
+ struct vy_lsm *lsm)
+{
+ vy_lsm_force_compaction(lsm);
+ vy_scheduler_update_lsm(scheduler, lsm);
+ fiber_cond_signal(&scheduler->scheduler_cond);
+}
+
/**
* Check whether the current dump round is complete.
* If it is, free memory and proceed to the next dump round.
diff --git a/src/box/vy_scheduler.h b/src/box/vy_scheduler.h
index 4db86152..d3bc4c03 100644
--- a/src/box/vy_scheduler.h
+++ b/src/box/vy_scheduler.h
@@ -202,6 +202,13 @@ void
vy_scheduler_trigger_dump(struct vy_scheduler *scheduler);
/**
+ * Force major compaction of an LSM tree.
+ */
+void
+vy_scheduler_force_compaction(struct vy_scheduler *scheduler,
+ struct vy_lsm *lsm);
+
+/**
* Schedule a checkpoint. Please call vy_scheduler_wait_checkpoint()
* after that.
*/
diff --git a/test/vinyl/compact.result b/test/vinyl/compact.result
index 79762f5b..758f7691 100644
--- a/test/vinyl/compact.result
+++ b/test/vinyl/compact.result
@@ -4,6 +4,9 @@ test_run = require('test_run').new()
fiber = require('fiber')
---
...
+digest = require('digest')
+---
+...
space = box.schema.space.create("vinyl", { engine = 'vinyl' })
---
...
@@ -94,9 +97,116 @@ test_run:grep_log('default', 'UPSERT operation failed') ~= nil
space:drop()
---
...
-fiber = nil
+--
+-- gh-3139: index.compact forces major compaction for all ranges
+--
+s = box.schema.space.create('test', {engine = 'vinyl'})
+---
+...
+_ = s:create_index('pk', {run_count_per_level = 100, page_size = 128, range_size = 1024})
+---
+...
+test_run:cmd("setopt delimiter ';'")
+---
+- true
+...
+function dump()
+ for i = 1, 10 do
+ s:replace{i, digest.urandom(1000)}
+ end
+ box.snapshot()
+end;
+---
+...
+function info()
+ local info = s.index.pk:info()
+ return {range_count = info.range_count, run_count = info.run_count}
+end
+function compact()
+ s.index.pk:compact()
+ repeat
+ fiber.sleep(0.001)
+ local info = s.index.pk:info()
+ until info.range_count == info.run_count
+end;
+---
+...
+test_run:cmd("setopt delimiter ''");
+---
+- true
+...
+dump()
+---
+...
+dump()
+---
+...
+dump()
+---
+...
+info() -- 1 range, 3 runs
+---
+- run_count: 3
+ range_count: 1
+...
+compact()
+---
+...
+info() -- 1 range, 1 run
+---
+- run_count: 1
+ range_count: 1
+...
+compact() -- no-op
+---
+...
+dump()
+---
+...
+dump()
+---
+...
+dump()
+---
+...
+info() -- 1 range, 4 runs
+---
+- run_count: 4
+ range_count: 1
+...
+compact()
+---
+...
+info() -- 2 ranges, 2 runs
+---
+- run_count: 2
+ range_count: 2
+...
+compact() -- no-op
+---
+...
+dump()
+---
+...
+dump()
+---
+...
+dump()
+---
+...
+info() -- 2 ranges, 5 runs
+---
+- run_count: 5
+ range_count: 2
+...
+compact()
+---
+...
+info() -- 4 ranges, 4 runs
---
+- run_count: 4
+ range_count: 4
...
-test_run = nil
+s:drop()
---
...
diff --git a/test/vinyl/compact.test.lua b/test/vinyl/compact.test.lua
index 920aafbd..b2a746aa 100644
--- a/test/vinyl/compact.test.lua
+++ b/test/vinyl/compact.test.lua
@@ -1,5 +1,6 @@
test_run = require('test_run').new()
fiber = require('fiber')
+digest = require('digest')
space = box.schema.space.create("vinyl", { engine = 'vinyl' })
_= space:create_index('primary', { parts = { 1, 'unsigned' }, run_count_per_level = 2 })
@@ -39,5 +40,58 @@ test_run:grep_log('default', 'UPSERT operation failed') ~= nil
space:drop()
-fiber = nil
-test_run = nil
+--
+-- gh-3139: index.compact forces major compaction for all ranges
+--
+s = box.schema.space.create('test', {engine = 'vinyl'})
+_ = s:create_index('pk', {run_count_per_level = 100, page_size = 128, range_size = 1024})
+
+test_run:cmd("setopt delimiter ';'")
+function dump()
+ for i = 1, 10 do
+ s:replace{i, digest.urandom(1000)}
+ end
+ box.snapshot()
+end;
+function info()
+ local info = s.index.pk:info()
+ return {range_count = info.range_count, run_count = info.run_count}
+end
+function compact()
+ s.index.pk:compact()
+ repeat
+ fiber.sleep(0.001)
+ local info = s.index.pk:info()
+ until info.range_count == info.run_count
+end;
+test_run:cmd("setopt delimiter ''");
+
+dump()
+dump()
+dump()
+info() -- 1 range, 3 runs
+
+compact()
+info() -- 1 range, 1 run
+
+compact() -- no-op
+
+dump()
+dump()
+dump()
+info() -- 1 range, 4 runs
+
+compact()
+info() -- 2 ranges, 2 runs
+
+compact() -- no-op
+
+dump()
+dump()
+dump()
+info() -- 2 ranges, 5 runs
+
+compact()
+info() -- 4 ranges, 4 runs
+
+s:drop()
--
2.11.0
^ permalink raw reply [flat|nested] 5+ messages in thread