[PATCH 2/2] vinyl: implement index compact method
Vladimir Davydov
vdavydov.dev at gmail.com
Fri May 18 19:21:34 MSK 2018
Force major compaction of all ranges when index.compact() is called.
Note, the function only triggers compaction, it doesn't wait until
compaction is complete.
Closes #3139
---
src/box/vinyl.c | 10 +++-
src/box/vy_lsm.c | 13 +++++
src/box/vy_lsm.h | 6 +++
src/box/vy_range.c | 23 +++++++++
src/box/vy_range.h | 12 +++++
src/box/vy_scheduler.c | 9 ++++
src/box/vy_scheduler.h | 7 +++
test/vinyl/compact.result | 114 +++++++++++++++++++++++++++++++++++++++++++-
test/vinyl/compact.test.lua | 58 +++++++++++++++++++++-
9 files changed, 247 insertions(+), 5 deletions(-)
diff --git a/src/box/vinyl.c b/src/box/vinyl.c
index 896f8559..2e2c145a 100644
--- a/src/box/vinyl.c
+++ b/src/box/vinyl.c
@@ -1265,6 +1265,14 @@ vinyl_index_bsize(struct index *index)
return bsize;
}
+static void
+vinyl_index_compact(struct index *index)
+{
+ struct vy_lsm *lsm = vy_lsm(index);
+ struct vy_env *env = vy_env(index->engine);
+ vy_scheduler_force_compaction(&env->scheduler, lsm);
+}
+
/* {{{ Public API of transaction control: start/end transaction,
* read, write data in the context of a transaction.
*/
@@ -4060,7 +4068,7 @@ static const struct index_vtab vinyl_index_vtab = {
/* .create_snapshot_iterator = */
generic_index_create_snapshot_iterator,
/* .info = */ vinyl_index_info,
- /* .compact = */ generic_index_compact,
+ /* .compact = */ vinyl_index_compact,
/* .reset_stat = */ vinyl_index_reset_stat,
/* .begin_build = */ generic_index_begin_build,
/* .reserve = */ generic_index_reserve,
diff --git a/src/box/vy_lsm.c b/src/box/vy_lsm.c
index 4dfc0a25..edc3b1a4 100644
--- a/src/box/vy_lsm.c
+++ b/src/box/vy_lsm.c
@@ -1109,3 +1109,16 @@ fail_range:
vy_lsm_name(lsm), vy_range_str(range));
return false;
}
+
+void
+vy_lsm_force_compaction(struct vy_lsm *lsm)
+{
+ struct vy_range *range;
+ struct vy_range_tree_iterator it;
+
+ vy_range_tree_ifirst(lsm->tree, &it);
+ while ((range = vy_range_tree_inext(&it)) != NULL)
+ vy_range_force_compaction(range);
+
+ vy_range_heap_update_all(&lsm->range_heap);
+}
diff --git a/src/box/vy_lsm.h b/src/box/vy_lsm.h
index c7ccb7d6..3f820fac 100644
--- a/src/box/vy_lsm.h
+++ b/src/box/vy_lsm.h
@@ -466,6 +466,12 @@ bool
vy_lsm_coalesce_range(struct vy_lsm *lsm, struct vy_range *range);
/**
+ * Mark all ranges of an LSM tree for major compaction.
+ */
+void
+vy_lsm_force_compaction(struct vy_lsm *lsm);
+
+/**
* Insert a statement into the in-memory index of an LSM tree. If
* the region_stmt is NULL and the statement is successfully inserted
* then the new lsregion statement is returned via @a region_stmt.
diff --git a/src/box/vy_range.c b/src/box/vy_range.c
index baecf178..6a55a018 100644
--- a/src/box/vy_range.c
+++ b/src/box/vy_range.c
@@ -262,6 +262,18 @@ vy_range_remove_slice(struct vy_range *range, struct vy_slice *slice)
vy_disk_stmt_counter_sub(&range->count, &slice->count);
}
+void
+vy_range_force_compaction(struct vy_range *range)
+{
+ if (range->slice_count == 1) {
+ /* Already compacted. */
+ assert(!range->needs_compaction);
+ return;
+ }
+ range->needs_compaction = true;
+ range->compact_priority = range->slice_count;
+}
+
/**
* To reduce write amplification caused by compaction, we follow
* the LSM tree design. Runs in each range are divided into groups
@@ -292,6 +304,17 @@ vy_range_update_compact_priority(struct vy_range *range,
assert(opts->run_count_per_level > 0);
assert(opts->run_size_ratio > 1);
+ if (range->slice_count == 1) {
+ /* Nothing to compact. */
+ range->compact_priority = 1;
+ range->needs_compaction = false;
+ return;
+ }
+ if (range->needs_compaction) {
+ range->compact_priority = range->slice_count;
+ return;
+ }
+
range->compact_priority = 0;
/* Total number of checked runs. */
diff --git a/src/box/vy_range.h b/src/box/vy_range.h
index 3a451fb4..d7031e70 100644
--- a/src/box/vy_range.h
+++ b/src/box/vy_range.h
@@ -106,6 +106,14 @@ struct vy_range {
* how we decide how many runs to compact next time.
*/
int compact_priority;
+ /**
+ * If this flag is set, the range must be scheduled for
+ * major compaction, i.e. its compact_priority must be
+ * raised to max (slice_count). The flag is set by
+ * vy_range_force_compaction() and cleared automatically
+ * when all slices of the range have been compacted.
+ */
+ bool needs_compaction;
/** Number of times the range was compacted. */
int n_compactions;
/** Link in vy_lsm->tree. */
@@ -221,6 +229,10 @@ vy_range_add_slice_before(struct vy_range *range, struct vy_slice *slice,
void
vy_range_remove_slice(struct vy_range *range, struct vy_slice *slice);
+/** Mark a range for major compaction. */
+void
+vy_range_force_compaction(struct vy_range *range);
+
/**
* Update compaction priority of a range.
*
diff --git a/src/box/vy_scheduler.c b/src/box/vy_scheduler.c
index 790d818a..d1545e76 100644
--- a/src/box/vy_scheduler.c
+++ b/src/box/vy_scheduler.c
@@ -458,6 +458,15 @@ vy_scheduler_trigger_dump(struct vy_scheduler *scheduler)
fiber_cond_signal(&scheduler->scheduler_cond);
}
+void
+vy_scheduler_force_compaction(struct vy_scheduler *scheduler,
+ struct vy_lsm *lsm)
+{
+ vy_lsm_force_compaction(lsm);
+ vy_scheduler_update_lsm(scheduler, lsm);
+ fiber_cond_signal(&scheduler->scheduler_cond);
+}
+
/**
* Check whether the current dump round is complete.
* If it is, free memory and proceed to the next dump round.
diff --git a/src/box/vy_scheduler.h b/src/box/vy_scheduler.h
index 4db86152..d3bc4c03 100644
--- a/src/box/vy_scheduler.h
+++ b/src/box/vy_scheduler.h
@@ -202,6 +202,13 @@ void
vy_scheduler_trigger_dump(struct vy_scheduler *scheduler);
/**
+ * Force major compaction of an LSM tree.
+ */
+void
+vy_scheduler_force_compaction(struct vy_scheduler *scheduler,
+ struct vy_lsm *lsm);
+
+/**
* Schedule a checkpoint. Please call vy_scheduler_wait_checkpoint()
* after that.
*/
diff --git a/test/vinyl/compact.result b/test/vinyl/compact.result
index 79762f5b..758f7691 100644
--- a/test/vinyl/compact.result
+++ b/test/vinyl/compact.result
@@ -4,6 +4,9 @@ test_run = require('test_run').new()
fiber = require('fiber')
---
...
+digest = require('digest')
+---
+...
space = box.schema.space.create("vinyl", { engine = 'vinyl' })
---
...
@@ -94,9 +97,116 @@ test_run:grep_log('default', 'UPSERT operation failed') ~= nil
space:drop()
---
...
-fiber = nil
+--
+-- gh-3139: index.compact forces major compaction for all ranges
+--
+s = box.schema.space.create('test', {engine = 'vinyl'})
+---
+...
+_ = s:create_index('pk', {run_count_per_level = 100, page_size = 128, range_size = 1024})
+---
+...
+test_run:cmd("setopt delimiter ';'")
+---
+- true
+...
+function dump()
+ for i = 1, 10 do
+ s:replace{i, digest.urandom(1000)}
+ end
+ box.snapshot()
+end;
+---
+...
+function info()
+ local info = s.index.pk:info()
+ return {range_count = info.range_count, run_count = info.run_count}
+end
+function compact()
+ s.index.pk:compact()
+ repeat
+ fiber.sleep(0.001)
+ local info = s.index.pk:info()
+ until info.range_count == info.run_count
+end;
+---
+...
+test_run:cmd("setopt delimiter ''");
+---
+- true
+...
+dump()
+---
+...
+dump()
+---
+...
+dump()
+---
+...
+info() -- 1 range, 3 runs
+---
+- run_count: 3
+ range_count: 1
+...
+compact()
+---
+...
+info() -- 1 range, 1 run
+---
+- run_count: 1
+ range_count: 1
+...
+compact() -- no-op
+---
+...
+dump()
+---
+...
+dump()
+---
+...
+dump()
+---
+...
+info() -- 1 range, 4 runs
+---
+- run_count: 4
+ range_count: 1
+...
+compact()
+---
+...
+info() -- 2 ranges, 2 runs
+---
+- run_count: 2
+ range_count: 2
+...
+compact() -- no-op
+---
+...
+dump()
+---
+...
+dump()
+---
+...
+dump()
+---
+...
+info() -- 2 ranges, 5 runs
+---
+- run_count: 5
+ range_count: 2
+...
+compact()
+---
+...
+info() -- 4 ranges, 4 runs
---
+- run_count: 4
+ range_count: 4
...
-test_run = nil
+s:drop()
---
...
diff --git a/test/vinyl/compact.test.lua b/test/vinyl/compact.test.lua
index 920aafbd..b2a746aa 100644
--- a/test/vinyl/compact.test.lua
+++ b/test/vinyl/compact.test.lua
@@ -1,5 +1,6 @@
test_run = require('test_run').new()
fiber = require('fiber')
+digest = require('digest')
space = box.schema.space.create("vinyl", { engine = 'vinyl' })
_= space:create_index('primary', { parts = { 1, 'unsigned' }, run_count_per_level = 2 })
@@ -39,5 +40,58 @@ test_run:grep_log('default', 'UPSERT operation failed') ~= nil
space:drop()
-fiber = nil
-test_run = nil
+--
+-- gh-3139: index.compact forces major compaction for all ranges
+--
+s = box.schema.space.create('test', {engine = 'vinyl'})
+_ = s:create_index('pk', {run_count_per_level = 100, page_size = 128, range_size = 1024})
+
+test_run:cmd("setopt delimiter ';'")
+function dump()
+ for i = 1, 10 do
+ s:replace{i, digest.urandom(1000)}
+ end
+ box.snapshot()
+end;
+function info()
+ local info = s.index.pk:info()
+ return {range_count = info.range_count, run_count = info.run_count}
+end
+function compact()
+ s.index.pk:compact()
+ repeat
+ fiber.sleep(0.001)
+ local info = s.index.pk:info()
+ until info.range_count == info.run_count
+end;
+test_run:cmd("setopt delimiter ''");
+
+dump()
+dump()
+dump()
+info() -- 1 range, 3 runs
+
+compact()
+info() -- 1 range, 1 run
+
+compact() -- no-op
+
+dump()
+dump()
+dump()
+info() -- 1 range, 4 runs
+
+compact()
+info() -- 2 ranges, 2 runs
+
+compact() -- no-op
+
+dump()
+dump()
+dump()
+info() -- 2 ranges, 5 runs
+
+compact()
+info() -- 4 ranges, 4 runs
+
+s:drop()
--
2.11.0
More information about the Tarantool-patches
mailing list