From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: From: Vladimir Davydov Subject: [PATCH 2/2] vinyl: implement index compact method Date: Fri, 18 May 2018 19:21:34 +0300 Message-Id: <909cc7fdf331b6131b8cd4da3fcb3bb7f7db3853.1526658945.git.vdavydov.dev@gmail.com> In-Reply-To: References: In-Reply-To: References: To: kostja@tarantool.org Cc: tarantool-patches@freelists.org List-ID: Force major compaction of all ranges when index.compact() is called. Note, the function only triggers compaction, it doesn't wait until compaction is complete. Closes #3139 --- src/box/vinyl.c | 10 +++- src/box/vy_lsm.c | 13 +++++ src/box/vy_lsm.h | 6 +++ src/box/vy_range.c | 23 +++++++++ src/box/vy_range.h | 12 +++++ src/box/vy_scheduler.c | 9 ++++ src/box/vy_scheduler.h | 7 +++ test/vinyl/compact.result | 114 +++++++++++++++++++++++++++++++++++++++++++- test/vinyl/compact.test.lua | 58 +++++++++++++++++++++- 9 files changed, 247 insertions(+), 5 deletions(-) diff --git a/src/box/vinyl.c b/src/box/vinyl.c index 896f8559..2e2c145a 100644 --- a/src/box/vinyl.c +++ b/src/box/vinyl.c @@ -1265,6 +1265,14 @@ vinyl_index_bsize(struct index *index) return bsize; } +static void +vinyl_index_compact(struct index *index) +{ + struct vy_lsm *lsm = vy_lsm(index); + struct vy_env *env = vy_env(index->engine); + vy_scheduler_force_compaction(&env->scheduler, lsm); +} + /* {{{ Public API of transaction control: start/end transaction, * read, write data in the context of a transaction. */ @@ -4060,7 +4068,7 @@ static const struct index_vtab vinyl_index_vtab = { /* .create_snapshot_iterator = */ generic_index_create_snapshot_iterator, /* .info = */ vinyl_index_info, - /* .compact = */ generic_index_compact, + /* .compact = */ vinyl_index_compact, /* .reset_stat = */ vinyl_index_reset_stat, /* .begin_build = */ generic_index_begin_build, /* .reserve = */ generic_index_reserve, diff --git a/src/box/vy_lsm.c b/src/box/vy_lsm.c index 4dfc0a25..edc3b1a4 100644 --- a/src/box/vy_lsm.c +++ b/src/box/vy_lsm.c @@ -1109,3 +1109,16 @@ fail_range: vy_lsm_name(lsm), vy_range_str(range)); return false; } + +void +vy_lsm_force_compaction(struct vy_lsm *lsm) +{ + struct vy_range *range; + struct vy_range_tree_iterator it; + + vy_range_tree_ifirst(lsm->tree, &it); + while ((range = vy_range_tree_inext(&it)) != NULL) + vy_range_force_compaction(range); + + vy_range_heap_update_all(&lsm->range_heap); +} diff --git a/src/box/vy_lsm.h b/src/box/vy_lsm.h index c7ccb7d6..3f820fac 100644 --- a/src/box/vy_lsm.h +++ b/src/box/vy_lsm.h @@ -466,6 +466,12 @@ bool vy_lsm_coalesce_range(struct vy_lsm *lsm, struct vy_range *range); /** + * Mark all ranges of an LSM tree for major compaction. + */ +void +vy_lsm_force_compaction(struct vy_lsm *lsm); + +/** * Insert a statement into the in-memory index of an LSM tree. If * the region_stmt is NULL and the statement is successfully inserted * then the new lsregion statement is returned via @a region_stmt. diff --git a/src/box/vy_range.c b/src/box/vy_range.c index baecf178..6a55a018 100644 --- a/src/box/vy_range.c +++ b/src/box/vy_range.c @@ -262,6 +262,18 @@ vy_range_remove_slice(struct vy_range *range, struct vy_slice *slice) vy_disk_stmt_counter_sub(&range->count, &slice->count); } +void +vy_range_force_compaction(struct vy_range *range) +{ + if (range->slice_count == 1) { + /* Already compacted. */ + assert(!range->needs_compaction); + return; + } + range->needs_compaction = true; + range->compact_priority = range->slice_count; +} + /** * To reduce write amplification caused by compaction, we follow * the LSM tree design. Runs in each range are divided into groups @@ -292,6 +304,17 @@ vy_range_update_compact_priority(struct vy_range *range, assert(opts->run_count_per_level > 0); assert(opts->run_size_ratio > 1); + if (range->slice_count == 1) { + /* Nothing to compact. */ + range->compact_priority = 1; + range->needs_compaction = false; + return; + } + if (range->needs_compaction) { + range->compact_priority = range->slice_count; + return; + } + range->compact_priority = 0; /* Total number of checked runs. */ diff --git a/src/box/vy_range.h b/src/box/vy_range.h index 3a451fb4..d7031e70 100644 --- a/src/box/vy_range.h +++ b/src/box/vy_range.h @@ -106,6 +106,14 @@ struct vy_range { * how we decide how many runs to compact next time. */ int compact_priority; + /** + * If this flag is set, the range must be scheduled for + * major compaction, i.e. its compact_priority must be + * raised to max (slice_count). The flag is set by + * vy_range_force_compaction() and cleared automatically + * when all slices of the range have been compacted. + */ + bool needs_compaction; /** Number of times the range was compacted. */ int n_compactions; /** Link in vy_lsm->tree. */ @@ -221,6 +229,10 @@ vy_range_add_slice_before(struct vy_range *range, struct vy_slice *slice, void vy_range_remove_slice(struct vy_range *range, struct vy_slice *slice); +/** Mark a range for major compaction. */ +void +vy_range_force_compaction(struct vy_range *range); + /** * Update compaction priority of a range. * diff --git a/src/box/vy_scheduler.c b/src/box/vy_scheduler.c index 790d818a..d1545e76 100644 --- a/src/box/vy_scheduler.c +++ b/src/box/vy_scheduler.c @@ -458,6 +458,15 @@ vy_scheduler_trigger_dump(struct vy_scheduler *scheduler) fiber_cond_signal(&scheduler->scheduler_cond); } +void +vy_scheduler_force_compaction(struct vy_scheduler *scheduler, + struct vy_lsm *lsm) +{ + vy_lsm_force_compaction(lsm); + vy_scheduler_update_lsm(scheduler, lsm); + fiber_cond_signal(&scheduler->scheduler_cond); +} + /** * Check whether the current dump round is complete. * If it is, free memory and proceed to the next dump round. diff --git a/src/box/vy_scheduler.h b/src/box/vy_scheduler.h index 4db86152..d3bc4c03 100644 --- a/src/box/vy_scheduler.h +++ b/src/box/vy_scheduler.h @@ -202,6 +202,13 @@ void vy_scheduler_trigger_dump(struct vy_scheduler *scheduler); /** + * Force major compaction of an LSM tree. + */ +void +vy_scheduler_force_compaction(struct vy_scheduler *scheduler, + struct vy_lsm *lsm); + +/** * Schedule a checkpoint. Please call vy_scheduler_wait_checkpoint() * after that. */ diff --git a/test/vinyl/compact.result b/test/vinyl/compact.result index 79762f5b..758f7691 100644 --- a/test/vinyl/compact.result +++ b/test/vinyl/compact.result @@ -4,6 +4,9 @@ test_run = require('test_run').new() fiber = require('fiber') --- ... +digest = require('digest') +--- +... space = box.schema.space.create("vinyl", { engine = 'vinyl' }) --- ... @@ -94,9 +97,116 @@ test_run:grep_log('default', 'UPSERT operation failed') ~= nil space:drop() --- ... -fiber = nil +-- +-- gh-3139: index.compact forces major compaction for all ranges +-- +s = box.schema.space.create('test', {engine = 'vinyl'}) +--- +... +_ = s:create_index('pk', {run_count_per_level = 100, page_size = 128, range_size = 1024}) +--- +... +test_run:cmd("setopt delimiter ';'") +--- +- true +... +function dump() + for i = 1, 10 do + s:replace{i, digest.urandom(1000)} + end + box.snapshot() +end; +--- +... +function info() + local info = s.index.pk:info() + return {range_count = info.range_count, run_count = info.run_count} +end +function compact() + s.index.pk:compact() + repeat + fiber.sleep(0.001) + local info = s.index.pk:info() + until info.range_count == info.run_count +end; +--- +... +test_run:cmd("setopt delimiter ''"); +--- +- true +... +dump() +--- +... +dump() +--- +... +dump() +--- +... +info() -- 1 range, 3 runs +--- +- run_count: 3 + range_count: 1 +... +compact() +--- +... +info() -- 1 range, 1 run +--- +- run_count: 1 + range_count: 1 +... +compact() -- no-op +--- +... +dump() +--- +... +dump() +--- +... +dump() +--- +... +info() -- 1 range, 4 runs +--- +- run_count: 4 + range_count: 1 +... +compact() +--- +... +info() -- 2 ranges, 2 runs +--- +- run_count: 2 + range_count: 2 +... +compact() -- no-op +--- +... +dump() +--- +... +dump() +--- +... +dump() +--- +... +info() -- 2 ranges, 5 runs +--- +- run_count: 5 + range_count: 2 +... +compact() +--- +... +info() -- 4 ranges, 4 runs --- +- run_count: 4 + range_count: 4 ... -test_run = nil +s:drop() --- ... diff --git a/test/vinyl/compact.test.lua b/test/vinyl/compact.test.lua index 920aafbd..b2a746aa 100644 --- a/test/vinyl/compact.test.lua +++ b/test/vinyl/compact.test.lua @@ -1,5 +1,6 @@ test_run = require('test_run').new() fiber = require('fiber') +digest = require('digest') space = box.schema.space.create("vinyl", { engine = 'vinyl' }) _= space:create_index('primary', { parts = { 1, 'unsigned' }, run_count_per_level = 2 }) @@ -39,5 +40,58 @@ test_run:grep_log('default', 'UPSERT operation failed') ~= nil space:drop() -fiber = nil -test_run = nil +-- +-- gh-3139: index.compact forces major compaction for all ranges +-- +s = box.schema.space.create('test', {engine = 'vinyl'}) +_ = s:create_index('pk', {run_count_per_level = 100, page_size = 128, range_size = 1024}) + +test_run:cmd("setopt delimiter ';'") +function dump() + for i = 1, 10 do + s:replace{i, digest.urandom(1000)} + end + box.snapshot() +end; +function info() + local info = s.index.pk:info() + return {range_count = info.range_count, run_count = info.run_count} +end +function compact() + s.index.pk:compact() + repeat + fiber.sleep(0.001) + local info = s.index.pk:info() + until info.range_count == info.run_count +end; +test_run:cmd("setopt delimiter ''"); + +dump() +dump() +dump() +info() -- 1 range, 3 runs + +compact() +info() -- 1 range, 1 run + +compact() -- no-op + +dump() +dump() +dump() +info() -- 1 range, 4 runs + +compact() +info() -- 2 ranges, 2 runs + +compact() -- no-op + +dump() +dump() +dump() +info() -- 2 ranges, 5 runs + +compact() +info() -- 4 ranges, 4 runs + +s:drop() -- 2.11.0