From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: From: Vladimir Davydov Subject: [PATCH 3/7] vinyl: add global disk stats Date: Sun, 2 Sep 2018 23:18:56 +0300 Message-Id: <6a406b8aaf61ee0ae51f66b30e1f81b927476a98.1535917763.git.vdavydov.dev@gmail.com> In-Reply-To: References: In-Reply-To: References: To: kostja@tarantool.org Cc: tarantool-patches@freelists.org List-ID: This patch adds some essential disk statistics to box.stat.vinyl(). The new statistics are reported under the 'disk' section and currently include the following fields: - data_files: number of open data files (*.run). - data_size: size of data stored on disk. - index_size: size of index stored on disk. - dump_total: number of bytes written by dump tasks. - compact_total: number of bytes writted by compaction tasks. All sizes are in bytes, before compression. Dump and compaction counters are reset by box.stat.reset(). --- src/box/vinyl.c | 19 ++++++++++++++++ src/box/vy_lsm.c | 44 ++++++++++++++++++++++++++++--------- src/box/vy_lsm.h | 16 ++++++++++++++ src/box/vy_scheduler.c | 2 ++ test/vinyl/info.result | 57 ++++++++++++++++++++++++++++++++++++++++++++++-- test/vinyl/info.test.lua | 12 ++++++++++ 6 files changed, 138 insertions(+), 12 deletions(-) diff --git a/src/box/vinyl.c b/src/box/vinyl.c index 88cdbce9..edfaa824 100644 --- a/src/box/vinyl.c +++ b/src/box/vinyl.c @@ -298,6 +298,20 @@ vy_info_append_memory(struct vy_env *env, struct info_handler *h) info_table_end(h); } +static void +vy_info_append_disk(struct vy_env *env, struct info_handler *h) +{ + struct vy_lsm_env *lsm_env = &env->lsm_env; + + info_table_begin(h, "disk"); + info_append_int(h, "data_files", lsm_env->data_file_count); + info_append_int(h, "data_size", lsm_env->disk_data_size); + info_append_int(h, "index_size", lsm_env->disk_index_size); + info_append_int(h, "dump_total", lsm_env->dump_total); + info_append_int(h, "compact_total", lsm_env->compact_total); + info_table_end(h); +} + void vinyl_engine_stat(struct vinyl_engine *vinyl, struct info_handler *h) { @@ -307,6 +321,7 @@ vinyl_engine_stat(struct vinyl_engine *vinyl, struct info_handler *h) vy_info_append_quota(env, h); vy_info_append_tx(env, h); vy_info_append_memory(env, h); + vy_info_append_disk(env, h); info_end(h); } @@ -476,8 +491,12 @@ vinyl_engine_reset_stat(struct engine *engine) { struct vy_env *env = vy_env(engine); struct tx_manager *xm = env->xm; + struct vy_lsm_env *lsm_env = &env->lsm_env; memset(&xm->stat, 0, sizeof(xm->stat)); + + lsm_env->dump_total = 0; + lsm_env->compact_total = 0; } /** }}} Introspection */ diff --git a/src/box/vy_lsm.c b/src/box/vy_lsm.c index 1994525e..15592fbf 100644 --- a/src/box/vy_lsm.c +++ b/src/box/vy_lsm.c @@ -685,32 +685,56 @@ vy_lsm_compact_priority(struct vy_lsm *lsm) void vy_lsm_add_run(struct vy_lsm *lsm, struct vy_run *run) { + struct vy_lsm_env *env = lsm->env; + size_t bloom_size = vy_run_bloom_size(run); + size_t page_index_size = run->page_index_size; + assert(rlist_empty(&run->in_lsm)); rlist_add_entry(&lsm->runs, run, in_lsm); lsm->run_count++; vy_disk_stmt_counter_add(&lsm->stat.disk.count, &run->count); - lsm->bloom_size += vy_run_bloom_size(run); - lsm->page_index_size += run->page_index_size; - - lsm->env->bloom_size += vy_run_bloom_size(run); - lsm->env->page_index_size += run->page_index_size; + lsm->bloom_size += bloom_size; + lsm->page_index_size += page_index_size; + + env->bloom_size += bloom_size; + env->page_index_size += page_index_size; + env->data_file_count++; + /* Data size is consistent with space.bsize. */ + if (lsm->index_id == 0) + env->disk_data_size += run->count.bytes; + /* Index size is consistent with index.bsize. */ + env->disk_index_size += bloom_size + page_index_size; + if (lsm->index_id > 0) + env->disk_index_size += run->count.bytes; } void vy_lsm_remove_run(struct vy_lsm *lsm, struct vy_run *run) { + struct vy_lsm_env *env = lsm->env; + size_t bloom_size = vy_run_bloom_size(run); + size_t page_index_size = run->page_index_size; + assert(lsm->run_count > 0); assert(!rlist_empty(&run->in_lsm)); rlist_del_entry(run, in_lsm); lsm->run_count--; vy_disk_stmt_counter_sub(&lsm->stat.disk.count, &run->count); - lsm->bloom_size -= vy_run_bloom_size(run); - lsm->page_index_size -= run->page_index_size; - - lsm->env->bloom_size -= vy_run_bloom_size(run); - lsm->env->page_index_size -= run->page_index_size; + lsm->bloom_size -= bloom_size; + lsm->page_index_size -= page_index_size; + + env->bloom_size -= bloom_size; + env->page_index_size -= page_index_size; + env->data_file_count--; + /* Data size is consistent with space.bsize. */ + if (lsm->index_id == 0) + env->disk_data_size -= run->count.bytes; + /* Index size is consistent with index.bsize. */ + env->disk_index_size -= bloom_size + page_index_size; + if (lsm->index_id > 0) + env->disk_index_size -= run->count.bytes; } void diff --git a/src/box/vy_lsm.h b/src/box/vy_lsm.h index 6917d475..67a5c5d1 100644 --- a/src/box/vy_lsm.h +++ b/src/box/vy_lsm.h @@ -91,6 +91,22 @@ struct vy_lsm_env { size_t bloom_size; /** Size of memory used for page index. */ size_t page_index_size; + /** Total number of data files (*.run). */ + size_t data_file_count; + /** Size of data stored on disk (uncompressed). */ + size_t disk_data_size; + /** Size of index stored on disk (uncompressed). */ + size_t disk_index_size; + /** + * Total number of bytes written to disk by dump tasks + * (uncompressed). + */ + int64_t dump_total; + /** + * Total number of bytes written to disk by compaction + * tasks (uncompressed). + */ + int64_t compact_total; /** Memory pool for vy_history_node allocations. */ struct mempool history_node_pool; }; diff --git a/src/box/vy_scheduler.c b/src/box/vy_scheduler.c index e41f1ffd..4959300e 100644 --- a/src/box/vy_scheduler.c +++ b/src/box/vy_scheduler.c @@ -1117,6 +1117,7 @@ vy_task_dump_complete(struct vy_task *task) */ vy_lsm_add_run(lsm, new_run); vy_stmt_counter_add_disk(&lsm->stat.disk.dump.out, &new_run->count); + lsm->env->dump_total += new_run->count.bytes; /* Drop the reference held by the task. */ vy_run_unref(new_run); @@ -1457,6 +1458,7 @@ vy_task_compact_complete(struct vy_task *task) vy_lsm_add_run(lsm, new_run); vy_stmt_counter_add_disk(&lsm->stat.disk.compact.out, &new_run->count); + lsm->env->compact_total += new_run->count.bytes; /* Drop the reference held by the task. */ vy_run_unref(new_run); } else diff --git a/test/vinyl/info.result b/test/vinyl/info.result index a53ee3ae..36fa732c 100644 --- a/test/vinyl/info.result +++ b/test/vinyl/info.result @@ -210,7 +210,13 @@ istat() ... gstat() --- -- quota: +- disk: + dump_total: 0 + data_size: 0 + data_files: 0 + compact_total: 0 + index_size: 0 + quota: limit: 134217728 used: 0 memory: @@ -667,6 +673,15 @@ box.rollback() -- -- Global statistics. -- +-- dump and compaction totals +gstat().disk.dump_total == istat().disk.dump.out.bytes +--- +- true +... +gstat().disk.compact_total == istat().disk.compact.out.bytes +--- +- true +... -- use memory st = gstat() --- @@ -1008,7 +1023,13 @@ istat() ... gstat() --- -- quota: +- disk: + dump_total: 0 + data_size: 104300 + data_files: 2 + compact_total: 0 + index_size: 1190 + quota: limit: 134217728 used: 262583 memory: @@ -1113,6 +1134,14 @@ gst.memory.bloom_filter == 0 --- - true ... +gst.disk.data_size == 0 +--- +- true +... +gst.disk.index_size == 0 +--- +- true +... box.snapshot() --- - ok @@ -1168,6 +1197,14 @@ gst.memory.bloom_filter == st1.disk.bloom_size + st2.disk.bloom_size --- - true ... +gst.disk.data_size == s:bsize() +--- +- true +... +gst.disk.index_size == i1:bsize() + i2:bsize() +--- +- true +... for i = 1, 100, 2 do s:delete(i) end --- ... @@ -1287,6 +1324,14 @@ gst.memory.bloom_filter == st1.disk.bloom_size + st2.disk.bloom_size --- - true ... +gst.disk.data_size == s:bsize() +--- +- true +... +gst.disk.index_size == i1:bsize() + i2:bsize() +--- +- true +... s:drop() --- ... @@ -1301,6 +1346,14 @@ gst.memory.bloom_filter == 0 --- - true ... +gst.disk.data_size == 0 +--- +- true +... +gst.disk.index_size == 0 +--- +- true +... test_run:cmd('switch default') --- - true diff --git a/test/vinyl/info.test.lua b/test/vinyl/info.test.lua index 230bac1e..919dde63 100644 --- a/test/vinyl/info.test.lua +++ b/test/vinyl/info.test.lua @@ -206,6 +206,10 @@ box.rollback() -- Global statistics. -- +-- dump and compaction totals +gstat().disk.dump_total == istat().disk.dump.out.bytes +gstat().disk.compact_total == istat().disk.compact.out.bytes + -- use memory st = gstat() put(1) @@ -342,6 +346,8 @@ i1:bsize() == st1.memory.index_size i2:bsize() == st2.memory.index_size gst.memory.page_index == 0 gst.memory.bloom_filter == 0 +gst.disk.data_size == 0 +gst.disk.index_size == 0 box.snapshot() gst = gstat() @@ -357,6 +363,8 @@ i1:bsize() == st1.disk.index_size + st1.disk.bloom_size i2:bsize() == st2.disk.index_size + st2.disk.bloom_size + st2.disk.bytes gst.memory.page_index == st1.disk.index_size + st2.disk.index_size gst.memory.bloom_filter == st1.disk.bloom_size + st2.disk.bloom_size +gst.disk.data_size == s:bsize() +gst.disk.index_size == i1:bsize() + i2:bsize() for i = 1, 100, 2 do s:delete(i) end for i = 2, 100, 2 do s:replace{i, i, pad()} end @@ -392,12 +400,16 @@ i1:bsize() == st1.disk.index_size + st1.disk.bloom_size i2:bsize() == st2.disk.index_size + st2.disk.bloom_size + st2.disk.bytes gst.memory.page_index == st1.disk.index_size + st2.disk.index_size gst.memory.bloom_filter == st1.disk.bloom_size + st2.disk.bloom_size +gst.disk.data_size == s:bsize() +gst.disk.index_size == i1:bsize() + i2:bsize() s:drop() gst = gstat() gst.memory.page_index == 0 gst.memory.bloom_filter == 0 +gst.disk.data_size == 0 +gst.disk.index_size == 0 test_run:cmd('switch default') test_run:cmd('stop server test') -- 2.11.0