From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: From: Vladimir Davydov Subject: [PATCH v2 2/8] vinyl: fix compaction priority calculation Date: Thu, 24 Jan 2019 20:12:38 +0300 Message-Id: <6739f169ad72b9f42c74d8a2e2ac895a34237c0c.1548349067.git.vdavydov.dev@gmail.com> In-Reply-To: References: MIME-Version: 1.0 In-Reply-To: References: Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: 8bit To: tarantool-patches@freelists.org List-ID: When computing the number of runs that need to be compacted for a range to conform to the target LSM tree shape, we use the newest run size for the size of the first LSM tree level. This isn't quite correct for two reasons. First, the size of the newest run is unstable - it may vary in a relatively wide range from dump to dump. This leads to frequent changes in the target LSM tree shape and, as a result, unpredictable compaction behavior. In particular this breaks compaction randomization, which is supposed to smooth out IO load generated by compaction. Second, this can increase space amplification. We trigger compaction at the last level when there's more than one run, irrespective of the value of run_count_per_level configuration option. We expect this to keep space amplification below 2 provided run_count_per_level is not greater than (run_size_ratio - 1). However, if the newest run happens to have such a size that multiplying it by run_size_ratio several times gives us a value only slightly less than the size of the oldest run, we can accumulate up to run_count_per_level more runs that are approximately as big as the last level run without triggering compaction, thus increasing space amplification by up to run_count_per_level. To fix these problems, let's use the oldest run size for computing the size of the first LSM tree level - simply divide it by run_size_ratio until it exceeds the size of the newest run. Follow-up #3657 --- src/box/vy_range.c | 42 ++++++--- test/vinyl/errinj.result | 4 +- test/vinyl/errinj.test.lua | 3 +- test/vinyl/gc.result | 3 +- test/vinyl/gc.test.lua | 3 +- test/vinyl/layout.result | 178 +++++++++++++++++++++++++++--------- test/vinyl/layout.test.lua | 5 +- test/vinyl/replica_rejoin.result | 2 +- test/vinyl/replica_rejoin.test.lua | 2 +- test/vinyl/update_optimize.result | 76 +++++++-------- test/vinyl/update_optimize.test.lua | 56 ++++++------ 11 files changed, 241 insertions(+), 133 deletions(-) diff --git a/src/box/vy_range.c b/src/box/vy_range.c index 87c4c6b9..7211cfb2 100644 --- a/src/box/vy_range.c +++ b/src/box/vy_range.c @@ -321,21 +321,41 @@ vy_range_update_compaction_priority(struct vy_range *range, uint32_t level_run_count = 0; /* * The target (perfect) size of a run at the current level. - * For the first level, it's the size of the newest run. - * For lower levels it's computed as first level run size - * times run_size_ratio. + * Calculated recurrently: the size of the next level equals + * the size of the previous level times run_size_ratio. + * + * For the last level we want it to be slightly greater + * than the size of the last (biggest, oldest) run so that + * all newer runs are at least run_size_ratio times smaller, + * because in conjunction with the fact that we never store + * more than one run at the last level, this will keep space + * amplification below 2 provided run_count_per_level is not + * greater than (run_size_ratio - 1). + * + * So to calculate the target size of the first level, we + * divide the size of the oldest run by run_size_ratio until + * it exceeds the size of the newest run. Note, DIV_ROUND_UP + * is important here, because if we used integral division, + * then after descending to the last level we would get a + * value slightly less than the last run size, not slightly + * greater, as we wanted to, which could increase space + * amplification by run_count_per_level in the worse case + * scenario. */ - uint64_t target_run_size = 0; + uint64_t target_run_size; + uint64_t size; struct vy_slice *slice; + slice = rlist_last_entry(&range->slices, struct vy_slice, in_range); + size = slice->count.bytes; + slice = rlist_first_entry(&range->slices, struct vy_slice, in_range); + do { + target_run_size = size; + size = DIV_ROUND_UP(target_run_size, opts->run_size_ratio); + } while (size > (uint64_t)MAX(slice->count.bytes, 1)); + rlist_foreach_entry(slice, &range->slices, in_range) { - uint64_t size = slice->count.bytes; - /* - * The size of the first level is defined by - * the size of the most recent run. - */ - if (target_run_size == 0) - target_run_size = size; + size = slice->count.bytes; level_run_count++; total_run_count++; vy_disk_stmt_counter_add(&total_stmt_count, &slice->count); diff --git a/test/vinyl/errinj.result b/test/vinyl/errinj.result index 4a3df6ae..990c7e85 100644 --- a/test/vinyl/errinj.result +++ b/test/vinyl/errinj.result @@ -1016,9 +1016,9 @@ s:replace{1, 10} --- - [1, 10] ... -s:replace{10, 100} -- to prevent last-level compaction (gh-3657) +-- Some padding to prevent last-level compaction (gh-3657). +for i = 1001, 1010 do s:replace{i, i} end --- -- [10, 100] ... box.snapshot() --- diff --git a/test/vinyl/errinj.test.lua b/test/vinyl/errinj.test.lua index c9d04aaf..d374a910 100644 --- a/test/vinyl/errinj.test.lua +++ b/test/vinyl/errinj.test.lua @@ -371,7 +371,8 @@ s = box.schema.space.create('test', {engine = 'vinyl'}) _ = s:create_index('pk', {run_count_per_level = 10}) _ = s:create_index('sk', {unique = false, parts = {2, 'unsigned'}}) s:replace{1, 10} -s:replace{10, 100} -- to prevent last-level compaction (gh-3657) +-- Some padding to prevent last-level compaction (gh-3657). +for i = 1001, 1010 do s:replace{i, i} end box.snapshot() s:replace{1, 20} box.snapshot() diff --git a/test/vinyl/gc.result b/test/vinyl/gc.result index 098c17c2..11e31619 100644 --- a/test/vinyl/gc.result +++ b/test/vinyl/gc.result @@ -168,7 +168,8 @@ function count_runs() return #fio.glob(fio.pathjoin(box.cfg.vinyl_dir, s.id, s.i _ = s:replace{1} --- ... -_ = s:replace{2} -- to prevent last-level compaction (gh-3657) +-- Some padding to prevent last-level compaction (gh-3657). +for i = 1001, 1010 do s:replace{i} end --- ... box.snapshot() diff --git a/test/vinyl/gc.test.lua b/test/vinyl/gc.test.lua index b5e42d6b..02cb6d32 100644 --- a/test/vinyl/gc.test.lua +++ b/test/vinyl/gc.test.lua @@ -84,7 +84,8 @@ _ = s:create_index('pk', {run_count_per_level = 3}) function count_runs() return #fio.glob(fio.pathjoin(box.cfg.vinyl_dir, s.id, s.index.pk.id, '*.run')) end _ = s:replace{1} -_ = s:replace{2} -- to prevent last-level compaction (gh-3657) +-- Some padding to prevent last-level compaction (gh-3657). +for i = 1001, 1010 do s:replace{i} end box.snapshot() _ = s:replace{3} box.snapshot() diff --git a/test/vinyl/layout.result b/test/vinyl/layout.result index 14201c5d..3be2bb91 100644 --- a/test/vinyl/layout.result +++ b/test/vinyl/layout.result @@ -38,8 +38,6 @@ box.snapshot() space.index.sk:alter{parts = {{2, 'unsigned', is_nullable = true}}} --- ... --- Note, the first run is bigger than the second one to prevent --- last-level compaction (gh-3657). space:replace{'ЭЭЭ', box.NULL} --- - ['ЭЭЭ', null] @@ -52,9 +50,9 @@ space:replace{'ёёё', box.NULL} --- - ['ёёё', null] ... -space:replace{'ЭЮЯ', 666} +-- Some padding to prevent last-level compaction (gh-3657). +for i = 1001, 1010 do space:replace{tostring(i), i} end --- -- ['ЭЮЯ', 666] ... box.snapshot() --- @@ -132,16 +130,16 @@ test_run:cmd("push filter 'offset: .*' to 'offset: '") ... result --- -- - - 00000000000000000011.vylog +- - - 00000000000000000020.vylog - - HEADER: type: INSERT BODY: tuple: [0, {6: 512, 7: [{'field': 0, 'collation': 1, 'type': 'string'}], - 9: 11, 12: 3, 13: 7}] + 9: 20, 12: 3, 13: 7}] - HEADER: type: INSERT BODY: - tuple: [5, {2: 8, 9: 11}] + tuple: [5, {2: 8, 9: 20}] - HEADER: type: INSERT BODY: @@ -162,11 +160,11 @@ result type: INSERT BODY: tuple: [0, {0: 2, 5: 1, 6: 512, 7: [{'field': 1, 'is_nullable': true, 'type': 'unsigned'}], - 9: 11, 12: 4, 13: 7}] + 9: 20, 12: 4, 13: 7}] - HEADER: type: INSERT BODY: - tuple: [5, {0: 2, 2: 6, 9: 11}] + tuple: [5, {0: 2, 2: 6, 9: 20}] - HEADER: type: INSERT BODY: @@ -206,7 +204,7 @@ result timestamp: type: INSERT BODY: - tuple: [5, {0: 2, 2: 10, 9: 14}] + tuple: [5, {0: 2, 2: 10, 9: 23}] - HEADER: timestamp: type: INSERT @@ -216,7 +214,7 @@ result timestamp: type: INSERT BODY: - tuple: [10, {0: 2, 9: 14}] + tuple: [10, {0: 2, 9: 23}] - HEADER: timestamp: type: INSERT @@ -226,7 +224,7 @@ result timestamp: type: INSERT BODY: - tuple: [5, {2: 12, 9: 14}] + tuple: [5, {2: 12, 9: 23}] - HEADER: timestamp: type: INSERT @@ -236,29 +234,79 @@ result timestamp: type: INSERT BODY: - tuple: [10, {9: 14}] + tuple: [10, {9: 23}] - - 00000000000000000008.index - - HEADER: type: RUNINFO BODY: min_lsn: 8 bloom_filter: - max_key: ['ЭЮЯ'] + max_key: ['ЭЭЭ'] page_count: 1 - stmt_stat: {9: 0, 2: 0, 5: 0, 3: 4} - max_lsn: 11 - min_key: ['ёёё'] + stmt_stat: {9: 0, 2: 0, 5: 0, 3: 13} + max_lsn: 20 + min_key: ['1001'] - HEADER: type: PAGEINFO BODY: row_index_offset: offset: - size: 108 - unpacked_size: 89 - row_count: 4 - min_key: ['ёёё'] + size: 286 + unpacked_size: 267 + row_count: 13 + min_key: ['1001'] - - 00000000000000000008.run - - HEADER: + lsn: 11 + type: REPLACE + BODY: + tuple: ['1001', 1001] + - HEADER: + lsn: 12 + type: REPLACE + BODY: + tuple: ['1002', 1002] + - HEADER: + lsn: 13 + type: REPLACE + BODY: + tuple: ['1003', 1003] + - HEADER: + lsn: 14 + type: REPLACE + BODY: + tuple: ['1004', 1004] + - HEADER: + lsn: 15 + type: REPLACE + BODY: + tuple: ['1005', 1005] + - HEADER: + lsn: 16 + type: REPLACE + BODY: + tuple: ['1006', 1006] + - HEADER: + lsn: 17 + type: REPLACE + BODY: + tuple: ['1007', 1007] + - HEADER: + lsn: 18 + type: REPLACE + BODY: + tuple: ['1008', 1008] + - HEADER: + lsn: 19 + type: REPLACE + BODY: + tuple: ['1009', 1009] + - HEADER: + lsn: 20 + type: REPLACE + BODY: + tuple: ['1010', 1010] + - HEADER: lsn: 10 type: REPLACE BODY: @@ -274,24 +322,19 @@ result BODY: tuple: ['ЭЭЭ', null] - HEADER: - lsn: 11 - type: REPLACE - BODY: - tuple: ['ЭЮЯ', 666] - - HEADER: type: ROWINDEX BODY: - row_index: "\0\0\0\0\0\0\0\x10\0\0\0 \0\0\00" + row_index: !!binary AAAAAAAAABAAAAAgAAAAMAAAAEAAAABQAAAAYAAAAHAAAACAAAAAkAAAAKAAAACwAAAAwA== - - 00000000000000000012.index - - HEADER: type: RUNINFO BODY: - min_lsn: 12 + min_lsn: 21 bloom_filter: max_key: ['ЮЮЮ'] page_count: 1 stmt_stat: {9: 0, 2: 0, 5: 0, 3: 3} - max_lsn: 14 + max_lsn: 23 min_key: ['ёёё'] - HEADER: type: PAGEINFO @@ -304,19 +347,19 @@ result min_key: ['ёёё'] - - 00000000000000000012.run - - HEADER: - lsn: 12 + lsn: 21 type: REPLACE BODY: tuple: ['ёёё', 123] tuple_meta: {1: 1} - HEADER: - lsn: 14 + lsn: 23 type: REPLACE BODY: tuple: ['ююю', 789] tuple_meta: {1: 1} - HEADER: - lsn: 13 + lsn: 22 type: REPLACE BODY: tuple: ['ЮЮЮ', 456] @@ -331,19 +374,19 @@ result BODY: min_lsn: 8 bloom_filter: - max_key: [666, 'ЭЮЯ'] + max_key: [1010, '1010'] page_count: 1 - stmt_stat: {9: 0, 2: 0, 5: 0, 3: 4} - max_lsn: 11 + stmt_stat: {9: 0, 2: 0, 5: 0, 3: 13} + max_lsn: 20 min_key: [null, 'ёёё'] - HEADER: type: PAGEINFO BODY: row_index_offset: offset: - size: 108 - unpacked_size: 89 - row_count: 4 + size: 286 + unpacked_size: 267 + row_count: 13 min_key: [null, 'ёёё'] - - 00000000000000000006.run - - HEADER: @@ -365,21 +408,66 @@ result lsn: 11 type: REPLACE BODY: - tuple: [666, 'ЭЮЯ'] + tuple: [1001, '1001'] + - HEADER: + lsn: 12 + type: REPLACE + BODY: + tuple: [1002, '1002'] + - HEADER: + lsn: 13 + type: REPLACE + BODY: + tuple: [1003, '1003'] + - HEADER: + lsn: 14 + type: REPLACE + BODY: + tuple: [1004, '1004'] + - HEADER: + lsn: 15 + type: REPLACE + BODY: + tuple: [1005, '1005'] + - HEADER: + lsn: 16 + type: REPLACE + BODY: + tuple: [1006, '1006'] + - HEADER: + lsn: 17 + type: REPLACE + BODY: + tuple: [1007, '1007'] + - HEADER: + lsn: 18 + type: REPLACE + BODY: + tuple: [1008, '1008'] + - HEADER: + lsn: 19 + type: REPLACE + BODY: + tuple: [1009, '1009'] + - HEADER: + lsn: 20 + type: REPLACE + BODY: + tuple: [1010, '1010'] - HEADER: type: ROWINDEX BODY: - row_index: "\0\0\0\0\0\0\0\x10\0\0\0 \0\0\00" + row_index: !!binary AAAAAAAAABAAAAAgAAAAMAAAAEAAAABQAAAAYAAAAHAAAACAAAAAkAAAAKAAAACwAAAAwA== - - 00000000000000000010.index - - HEADER: type: RUNINFO BODY: - min_lsn: 12 + min_lsn: 21 bloom_filter: max_key: [789, 'ююю'] page_count: 1 stmt_stat: {9: 0, 2: 0, 5: 0, 3: 3} - max_lsn: 14 + max_lsn: 23 min_key: [123, 'ёёё'] - HEADER: type: PAGEINFO @@ -392,17 +480,17 @@ result min_key: [123, 'ёёё'] - - 00000000000000000010.run - - HEADER: - lsn: 12 + lsn: 21 type: REPLACE BODY: tuple: [123, 'ёёё'] - HEADER: - lsn: 13 + lsn: 22 type: REPLACE BODY: tuple: [456, 'ЮЮЮ'] - HEADER: - lsn: 14 + lsn: 23 type: REPLACE BODY: tuple: [789, 'ююю'] diff --git a/test/vinyl/layout.test.lua b/test/vinyl/layout.test.lua index 60f22c76..f6591027 100644 --- a/test/vinyl/layout.test.lua +++ b/test/vinyl/layout.test.lua @@ -17,12 +17,11 @@ box.snapshot() space.index.sk:alter{parts = {{2, 'unsigned', is_nullable = true}}} --- Note, the first run is bigger than the second one to prevent --- last-level compaction (gh-3657). space:replace{'ЭЭЭ', box.NULL} space:replace{'эээ', box.NULL} space:replace{'ёёё', box.NULL} -space:replace{'ЭЮЯ', 666} +-- Some padding to prevent last-level compaction (gh-3657). +for i = 1001, 1010 do space:replace{tostring(i), i} end box.snapshot() space:replace{'ёёё', 123} diff --git a/test/vinyl/replica_rejoin.result b/test/vinyl/replica_rejoin.result index d153e346..1dfcb91b 100644 --- a/test/vinyl/replica_rejoin.result +++ b/test/vinyl/replica_rejoin.result @@ -17,7 +17,7 @@ _ = box.schema.space.create('test', { id = 9000, engine = 'vinyl' }) _ = box.space.test:create_index('pk') --- ... -pad = string.rep('x', 15 * 1024) +pad = string.rep('x', 12 * 1024) --- ... for i = 1, 100 do box.space.test:replace{i, pad} end diff --git a/test/vinyl/replica_rejoin.test.lua b/test/vinyl/replica_rejoin.test.lua index 8226fb94..2c5a69e0 100644 --- a/test/vinyl/replica_rejoin.test.lua +++ b/test/vinyl/replica_rejoin.test.lua @@ -8,7 +8,7 @@ test_run = env.new() box.schema.user.grant('guest', 'replication') _ = box.schema.space.create('test', { id = 9000, engine = 'vinyl' }) _ = box.space.test:create_index('pk') -pad = string.rep('x', 15 * 1024) +pad = string.rep('x', 12 * 1024) for i = 1, 100 do box.space.test:replace{i, pad} end box.snapshot() diff --git a/test/vinyl/update_optimize.result b/test/vinyl/update_optimize.result index d8ff9fc4..09370e7d 100644 --- a/test/vinyl/update_optimize.result +++ b/test/vinyl/update_optimize.result @@ -28,10 +28,10 @@ test_run:cmd("setopt delimiter ';'") - true ... function wait_for_dump(index, old_count) - while index:stat().run_count == old_count do + while index:stat().disk.dump.count == old_count do fiber.sleep(0) end - return index:stat().run_count + return index:stat().disk.dump.count end; --- ... @@ -39,10 +39,7 @@ test_run:cmd("setopt delimiter ''"); --- - true ... -index_run_count = index:stat().run_count ---- -... -index2_run_count = index2:stat().run_count +dump_count = index:stat().disk.dump.count --- ... old_stmt_count = dumped_stmt_count() @@ -69,7 +66,7 @@ box.snapshot() - ok ... -- Wait for dump both indexes. -index_run_count = wait_for_dump(index, index_run_count) +dump_count = wait_for_dump(index, dump_count) --- ... new_stmt_count = dumped_stmt_count() @@ -93,7 +90,7 @@ box.snapshot() --- - ok ... -index_run_count = wait_for_dump(index, index_run_count) +dump_count = wait_for_dump(index, dump_count) --- ... space:update({1}, {{'!', 4, 20}}) -- move range containing index field @@ -104,7 +101,7 @@ box.snapshot() --- - ok ... -index_run_count = wait_for_dump(index, index_run_count) +dump_count = wait_for_dump(index, dump_count) --- ... space:update({1}, {{'#', 3, 1}}) -- same @@ -115,7 +112,7 @@ box.snapshot() --- - ok ... -index_run_count = wait_for_dump(index, index_run_count) +dump_count = wait_for_dump(index, dump_count) --- ... new_stmt_count = dumped_stmt_count() @@ -151,7 +148,7 @@ box.snapshot() --- - ok ... -index_run_count = wait_for_dump(index, index_run_count) +dump_count = wait_for_dump(index, dump_count) --- ... -- Move range that doesn't contain indexed fields. @@ -163,7 +160,7 @@ box.snapshot() --- - ok ... -index_run_count = wait_for_dump(index, index_run_count) +dump_count = wait_for_dump(index, dump_count) --- ... space:update({2}, {{'#', 6, 1}}) -- same @@ -174,7 +171,7 @@ box.snapshot() --- - ok ... -index_run_count = wait_for_dump(index, index_run_count) +dump_count = wait_for_dump(index, dump_count) --- ... new_stmt_count = dumped_stmt_count() @@ -224,13 +221,7 @@ box.snapshot() --- - ok ... -index_run_count = index:stat().run_count ---- -... -index2_run_count = index2:stat().run_count ---- -... -index3_run_count = index3:stat().run_count +dump_count = index:stat().run_count --- ... old_stmt_count = dumped_stmt_count() @@ -256,7 +247,7 @@ box.snapshot() --- - ok ... -index_run_count = wait_for_dump(index, index_run_count) +dump_count = wait_for_dump(index, dump_count) --- ... new_stmt_count = dumped_stmt_count() @@ -278,7 +269,7 @@ box.snapshot() --- - ok ... -index_run_count = wait_for_dump(index, index_run_count) +dump_count = wait_for_dump(index, dump_count) --- ... index:update({2}, {{'!', 3, 20}}) -- move range containing all indexes @@ -289,7 +280,7 @@ box.snapshot() --- - ok ... -index_run_count = wait_for_dump(index, index_run_count) +dump_count = wait_for_dump(index, dump_count) --- ... index:update({2}, {{'=', 7, 100}, {'+', 5, 10}, {'#', 3, 1}}) -- change two cols but then move range with all indexed fields @@ -300,7 +291,7 @@ box.snapshot() --- - ok ... -index_run_count = wait_for_dump(index, index_run_count) +dump_count = wait_for_dump(index, dump_count) --- ... new_stmt_count = dumped_stmt_count() @@ -343,7 +334,7 @@ box.snapshot() --- - ok ... -index_run_count = wait_for_dump(index, index_run_count) +dump_count = wait_for_dump(index, dump_count) --- ... new_stmt_count = dumped_stmt_count() @@ -365,7 +356,7 @@ box.snapshot() --- - ok ... -index_run_count = wait_for_dump(index, index_run_count) +dump_count = wait_for_dump(index, dump_count) --- ... new_stmt_count = dumped_stmt_count() @@ -387,7 +378,7 @@ box.snapshot() --- - ok ... -index_run_count = wait_for_dump(index, index_run_count) +dump_count = wait_for_dump(index, dump_count) --- ... new_stmt_count = dumped_stmt_count() @@ -439,7 +430,7 @@ box.snapshot() - ok ... -- Make update of not indexed field with pos > 64. -index_run_count = wait_for_dump(index, index_run_count) +dump_count = wait_for_dump(index, dump_count) --- ... old_stmt_count = dumped_stmt_count() @@ -453,7 +444,7 @@ box.snapshot() - ok ... -- Check the only primary index to be changed. -index_run_count = wait_for_dump(index, index_run_count) +dump_count = wait_for_dump(index, dump_count) --- ... new_stmt_count = dumped_stmt_count() @@ -481,7 +472,7 @@ box.snapshot() --- - ok ... -index_run_count = wait_for_dump(index, index_run_count) +dump_count = wait_for_dump(index, dump_count) --- ... new_stmt_count = dumped_stmt_count() @@ -524,7 +515,7 @@ box.snapshot() --- - ok ... -index_run_count = wait_for_dump(index, index_run_count) +dump_count = wait_for_dump(index, dump_count) --- ... old_stmt_count = dumped_stmt_count() @@ -538,7 +529,7 @@ box.snapshot() --- - ok ... -index_run_count = wait_for_dump(index, index_run_count) +dump_count = wait_for_dump(index, dump_count) --- ... new_stmt_count = dumped_stmt_count() @@ -604,7 +595,7 @@ box.snapshot() --- - ok ... -index_run_count = wait_for_dump(index, index_run_count) +dump_count = wait_for_dump(index, dump_count) --- ... old_stmt_count = dumped_stmt_count() @@ -728,9 +719,9 @@ s:insert{1, 10} --- - [1, 10] ... -s:insert{10, 100} -- to prevent last-level compaction (gh-3657) +-- Some padding to prevent last-level compaction (gh-3657). +for i = 1001, 1010 do s:replace{i, i} end --- -- [10, 100] ... box.snapshot() --- @@ -747,11 +738,11 @@ box.snapshot() --- - ok ... --- Should be 3: INSERT{10, 1} and INSERT{100, 10} in the first run +-- Should be 12: INSERT{10, 1} and INSERT[1001..1010] in the first run -- plus DELETE{10, 1} in the second run. s.index.sk:stat().rows --- -- 3 +- 12 ... s:insert{1, 20} --- @@ -760,7 +751,16 @@ s:insert{1, 20} s.index.sk:select() --- - - [1, 20] - - [10, 100] + - [1001, 1001] + - [1002, 1002] + - [1003, 1003] + - [1004, 1004] + - [1005, 1005] + - [1006, 1006] + - [1007, 1007] + - [1008, 1008] + - [1009, 1009] + - [1010, 1010] ... s:drop() --- diff --git a/test/vinyl/update_optimize.test.lua b/test/vinyl/update_optimize.test.lua index 41ff964b..a0de6e4c 100644 --- a/test/vinyl/update_optimize.test.lua +++ b/test/vinyl/update_optimize.test.lua @@ -12,15 +12,14 @@ function dumped_stmt_count() return index:stat().disk.dump.output.rows + index2: box.snapshot() test_run:cmd("setopt delimiter ';'") function wait_for_dump(index, old_count) - while index:stat().run_count == old_count do + while index:stat().disk.dump.count == old_count do fiber.sleep(0) end - return index:stat().run_count + return index:stat().disk.dump.count end; test_run:cmd("setopt delimiter ''"); -index_run_count = index:stat().run_count -index2_run_count = index2:stat().run_count +dump_count = index:stat().disk.dump.count old_stmt_count = dumped_stmt_count() space:insert({1, 2, 3, 4, 5}) space:insert({2, 3, 4, 5, 6}) @@ -28,7 +27,7 @@ space:insert({3, 4, 5, 6, 7}) space:insert({4, 5, 6, 7, 8}) box.snapshot() -- Wait for dump both indexes. -index_run_count = wait_for_dump(index, index_run_count) +dump_count = wait_for_dump(index, dump_count) new_stmt_count = dumped_stmt_count() new_stmt_count - old_stmt_count == 8 old_stmt_count = new_stmt_count @@ -39,13 +38,13 @@ space:update({1}, {{'=', 5, 10}}) -- change secondary index field -- statements in vy_write_iterator during dump. box.snapshot() -index_run_count = wait_for_dump(index, index_run_count) +dump_count = wait_for_dump(index, dump_count) space:update({1}, {{'!', 4, 20}}) -- move range containing index field box.snapshot() -index_run_count = wait_for_dump(index, index_run_count) +dump_count = wait_for_dump(index, dump_count) space:update({1}, {{'#', 3, 1}}) -- same box.snapshot() -index_run_count = wait_for_dump(index, index_run_count) +dump_count = wait_for_dump(index, dump_count) new_stmt_count = dumped_stmt_count() new_stmt_count - old_stmt_count == 9 old_stmt_count = new_stmt_count @@ -55,14 +54,14 @@ index2:select{} -- optimized updates space:update({2}, {{'=', 6, 10}}) -- change not indexed field box.snapshot() -index_run_count = wait_for_dump(index, index_run_count) +dump_count = wait_for_dump(index, dump_count) -- Move range that doesn't contain indexed fields. space:update({2}, {{'!', 7, 20}}) box.snapshot() -index_run_count = wait_for_dump(index, index_run_count) +dump_count = wait_for_dump(index, dump_count) space:update({2}, {{'#', 6, 1}}) -- same box.snapshot() -index_run_count = wait_for_dump(index, index_run_count) +dump_count = wait_for_dump(index, dump_count) new_stmt_count = dumped_stmt_count() new_stmt_count - old_stmt_count == 3 old_stmt_count = new_stmt_count @@ -78,16 +77,14 @@ index2 = space:create_index('secondary', { parts = {4, 'unsigned', 3, 'unsigned' index3 = space:create_index('third', { parts = {5, 'unsigned'}, run_count_per_level = 20 }) function dumped_stmt_count() return index:stat().disk.dump.output.rows + index2:stat().disk.dump.output.rows + index3:stat().disk.dump.output.rows end box.snapshot() -index_run_count = index:stat().run_count -index2_run_count = index2:stat().run_count -index3_run_count = index3:stat().run_count +dump_count = index:stat().run_count old_stmt_count = dumped_stmt_count() space:insert({1, 2, 3, 4, 5}) space:insert({2, 3, 4, 5, 6}) space:insert({3, 4, 5, 6, 7}) space:insert({4, 5, 6, 7, 8}) box.snapshot() -index_run_count = wait_for_dump(index, index_run_count) +dump_count = wait_for_dump(index, dump_count) new_stmt_count = dumped_stmt_count() new_stmt_count - old_stmt_count == 12 old_stmt_count = new_stmt_count @@ -95,13 +92,13 @@ old_stmt_count = new_stmt_count -- not optimizes updates index:update({2}, {{'+', 1, 10}, {'+', 3, 10}, {'+', 4, 10}, {'+', 5, 10}}) -- change all fields box.snapshot() -index_run_count = wait_for_dump(index, index_run_count) +dump_count = wait_for_dump(index, dump_count) index:update({2}, {{'!', 3, 20}}) -- move range containing all indexes box.snapshot() -index_run_count = wait_for_dump(index, index_run_count) +dump_count = wait_for_dump(index, dump_count) index:update({2}, {{'=', 7, 100}, {'+', 5, 10}, {'#', 3, 1}}) -- change two cols but then move range with all indexed fields box.snapshot() -index_run_count = wait_for_dump(index, index_run_count) +dump_count = wait_for_dump(index, dump_count) new_stmt_count = dumped_stmt_count() new_stmt_count - old_stmt_count == 15 old_stmt_count = new_stmt_count @@ -112,21 +109,21 @@ index3:select{} -- optimize one 'secondary' index update index:update({3}, {{'+', 1, 10}, {'-', 5, 2}, {'!', 6, 100}}) -- change only index 'third' box.snapshot() -index_run_count = wait_for_dump(index, index_run_count) +dump_count = wait_for_dump(index, dump_count) new_stmt_count = dumped_stmt_count() new_stmt_count - old_stmt_count == 3 old_stmt_count = new_stmt_count -- optimize one 'third' index update index:update({3}, {{'=', 1, 20}, {'+', 3, 5}, {'=', 4, 30}, {'!', 6, 110}}) -- change only index 'secondary' box.snapshot() -index_run_count = wait_for_dump(index, index_run_count) +dump_count = wait_for_dump(index, dump_count) new_stmt_count = dumped_stmt_count() new_stmt_count - old_stmt_count == 3 old_stmt_count = new_stmt_count -- optimize both indexes index:update({3}, {{'+', 1, 10}, {'#', 6, 1}}) -- don't change any indexed fields box.snapshot() -index_run_count = wait_for_dump(index, index_run_count) +dump_count = wait_for_dump(index, dump_count) new_stmt_count = dumped_stmt_count() new_stmt_count - old_stmt_count == 1 old_stmt_count = new_stmt_count @@ -144,13 +141,13 @@ _ = space:replace(long_tuple) box.snapshot() -- Make update of not indexed field with pos > 64. -index_run_count = wait_for_dump(index, index_run_count) +dump_count = wait_for_dump(index, dump_count) old_stmt_count = dumped_stmt_count() _ = index:update({2}, {{'=', 65, 1000}}) box.snapshot() -- Check the only primary index to be changed. -index_run_count = wait_for_dump(index, index_run_count) +dump_count = wait_for_dump(index, dump_count) new_stmt_count = dumped_stmt_count() new_stmt_count - old_stmt_count == 1 old_stmt_count = new_stmt_count @@ -161,7 +158,7 @@ space:get{2}[65] -- index:update({2}, {{'#', -65, 65}}) box.snapshot() -index_run_count = wait_for_dump(index, index_run_count) +dump_count = wait_for_dump(index, dump_count) new_stmt_count = dumped_stmt_count() new_stmt_count - old_stmt_count == 1 old_stmt_count = new_stmt_count @@ -172,12 +169,12 @@ index3:select{} -- Optimize index2 with negative update op. space:replace{10, 20, 30, 40, 50} box.snapshot() -index_run_count = wait_for_dump(index, index_run_count) +dump_count = wait_for_dump(index, dump_count) old_stmt_count = dumped_stmt_count() index:update({20}, {{'=', -1, 500}}) box.snapshot() -index_run_count = wait_for_dump(index, index_run_count) +dump_count = wait_for_dump(index, dump_count) new_stmt_count = dumped_stmt_count() -- 3 = REPLACE in index1 and DELETE + REPLACE in index3. new_stmt_count - old_stmt_count == 3 @@ -195,7 +192,7 @@ space:replace{20, 200, 2000, 20000, 200000, 2000000} index:update({200}, {{'=', 6, 2}}) box.commit() box.snapshot() -index_run_count = wait_for_dump(index, index_run_count) +dump_count = wait_for_dump(index, dump_count) old_stmt_count = dumped_stmt_count() index:select{} index2:select{} @@ -244,14 +241,15 @@ _ = s:create_index('pk') _ = s:create_index('sk', {parts = {2, 'unsigned'}, run_count_per_level = 10}) s:insert{1, 10} -s:insert{10, 100} -- to prevent last-level compaction (gh-3657) +-- Some padding to prevent last-level compaction (gh-3657). +for i = 1001, 1010 do s:replace{i, i} end box.snapshot() s:update(1, {{'=', 2, 10}}) s:delete(1) box.snapshot() --- Should be 3: INSERT{10, 1} and INSERT{100, 10} in the first run +-- Should be 12: INSERT{10, 1} and INSERT[1001..1010] in the first run -- plus DELETE{10, 1} in the second run. s.index.sk:stat().rows -- 2.11.0