[PATCH 3/7] vinyl: add global disk stats
Vladimir Davydov
vdavydov.dev at gmail.com
Sun Sep 2 23:18:56 MSK 2018
This patch adds some essential disk statistics to box.stat.vinyl().
The new statistics are reported under the 'disk' section and currently
include the following fields:
- data_files: number of open data files (*.run).
- data_size: size of data stored on disk.
- index_size: size of index stored on disk.
- dump_total: number of bytes written by dump tasks.
- compact_total: number of bytes writted by compaction tasks.
All sizes are in bytes, before compression. Dump and compaction counters
are reset by box.stat.reset().
---
src/box/vinyl.c | 19 ++++++++++++++++
src/box/vy_lsm.c | 44 ++++++++++++++++++++++++++++---------
src/box/vy_lsm.h | 16 ++++++++++++++
src/box/vy_scheduler.c | 2 ++
test/vinyl/info.result | 57 ++++++++++++++++++++++++++++++++++++++++++++++--
test/vinyl/info.test.lua | 12 ++++++++++
6 files changed, 138 insertions(+), 12 deletions(-)
diff --git a/src/box/vinyl.c b/src/box/vinyl.c
index 88cdbce9..edfaa824 100644
--- a/src/box/vinyl.c
+++ b/src/box/vinyl.c
@@ -298,6 +298,20 @@ vy_info_append_memory(struct vy_env *env, struct info_handler *h)
info_table_end(h);
}
+static void
+vy_info_append_disk(struct vy_env *env, struct info_handler *h)
+{
+ struct vy_lsm_env *lsm_env = &env->lsm_env;
+
+ info_table_begin(h, "disk");
+ info_append_int(h, "data_files", lsm_env->data_file_count);
+ info_append_int(h, "data_size", lsm_env->disk_data_size);
+ info_append_int(h, "index_size", lsm_env->disk_index_size);
+ info_append_int(h, "dump_total", lsm_env->dump_total);
+ info_append_int(h, "compact_total", lsm_env->compact_total);
+ info_table_end(h);
+}
+
void
vinyl_engine_stat(struct vinyl_engine *vinyl, struct info_handler *h)
{
@@ -307,6 +321,7 @@ vinyl_engine_stat(struct vinyl_engine *vinyl, struct info_handler *h)
vy_info_append_quota(env, h);
vy_info_append_tx(env, h);
vy_info_append_memory(env, h);
+ vy_info_append_disk(env, h);
info_end(h);
}
@@ -476,8 +491,12 @@ vinyl_engine_reset_stat(struct engine *engine)
{
struct vy_env *env = vy_env(engine);
struct tx_manager *xm = env->xm;
+ struct vy_lsm_env *lsm_env = &env->lsm_env;
memset(&xm->stat, 0, sizeof(xm->stat));
+
+ lsm_env->dump_total = 0;
+ lsm_env->compact_total = 0;
}
/** }}} Introspection */
diff --git a/src/box/vy_lsm.c b/src/box/vy_lsm.c
index 1994525e..15592fbf 100644
--- a/src/box/vy_lsm.c
+++ b/src/box/vy_lsm.c
@@ -685,32 +685,56 @@ vy_lsm_compact_priority(struct vy_lsm *lsm)
void
vy_lsm_add_run(struct vy_lsm *lsm, struct vy_run *run)
{
+ struct vy_lsm_env *env = lsm->env;
+ size_t bloom_size = vy_run_bloom_size(run);
+ size_t page_index_size = run->page_index_size;
+
assert(rlist_empty(&run->in_lsm));
rlist_add_entry(&lsm->runs, run, in_lsm);
lsm->run_count++;
vy_disk_stmt_counter_add(&lsm->stat.disk.count, &run->count);
- lsm->bloom_size += vy_run_bloom_size(run);
- lsm->page_index_size += run->page_index_size;
-
- lsm->env->bloom_size += vy_run_bloom_size(run);
- lsm->env->page_index_size += run->page_index_size;
+ lsm->bloom_size += bloom_size;
+ lsm->page_index_size += page_index_size;
+
+ env->bloom_size += bloom_size;
+ env->page_index_size += page_index_size;
+ env->data_file_count++;
+ /* Data size is consistent with space.bsize. */
+ if (lsm->index_id == 0)
+ env->disk_data_size += run->count.bytes;
+ /* Index size is consistent with index.bsize. */
+ env->disk_index_size += bloom_size + page_index_size;
+ if (lsm->index_id > 0)
+ env->disk_index_size += run->count.bytes;
}
void
vy_lsm_remove_run(struct vy_lsm *lsm, struct vy_run *run)
{
+ struct vy_lsm_env *env = lsm->env;
+ size_t bloom_size = vy_run_bloom_size(run);
+ size_t page_index_size = run->page_index_size;
+
assert(lsm->run_count > 0);
assert(!rlist_empty(&run->in_lsm));
rlist_del_entry(run, in_lsm);
lsm->run_count--;
vy_disk_stmt_counter_sub(&lsm->stat.disk.count, &run->count);
- lsm->bloom_size -= vy_run_bloom_size(run);
- lsm->page_index_size -= run->page_index_size;
-
- lsm->env->bloom_size -= vy_run_bloom_size(run);
- lsm->env->page_index_size -= run->page_index_size;
+ lsm->bloom_size -= bloom_size;
+ lsm->page_index_size -= page_index_size;
+
+ env->bloom_size -= bloom_size;
+ env->page_index_size -= page_index_size;
+ env->data_file_count--;
+ /* Data size is consistent with space.bsize. */
+ if (lsm->index_id == 0)
+ env->disk_data_size -= run->count.bytes;
+ /* Index size is consistent with index.bsize. */
+ env->disk_index_size -= bloom_size + page_index_size;
+ if (lsm->index_id > 0)
+ env->disk_index_size -= run->count.bytes;
}
void
diff --git a/src/box/vy_lsm.h b/src/box/vy_lsm.h
index 6917d475..67a5c5d1 100644
--- a/src/box/vy_lsm.h
+++ b/src/box/vy_lsm.h
@@ -91,6 +91,22 @@ struct vy_lsm_env {
size_t bloom_size;
/** Size of memory used for page index. */
size_t page_index_size;
+ /** Total number of data files (*.run). */
+ size_t data_file_count;
+ /** Size of data stored on disk (uncompressed). */
+ size_t disk_data_size;
+ /** Size of index stored on disk (uncompressed). */
+ size_t disk_index_size;
+ /**
+ * Total number of bytes written to disk by dump tasks
+ * (uncompressed).
+ */
+ int64_t dump_total;
+ /**
+ * Total number of bytes written to disk by compaction
+ * tasks (uncompressed).
+ */
+ int64_t compact_total;
/** Memory pool for vy_history_node allocations. */
struct mempool history_node_pool;
};
diff --git a/src/box/vy_scheduler.c b/src/box/vy_scheduler.c
index e41f1ffd..4959300e 100644
--- a/src/box/vy_scheduler.c
+++ b/src/box/vy_scheduler.c
@@ -1117,6 +1117,7 @@ vy_task_dump_complete(struct vy_task *task)
*/
vy_lsm_add_run(lsm, new_run);
vy_stmt_counter_add_disk(&lsm->stat.disk.dump.out, &new_run->count);
+ lsm->env->dump_total += new_run->count.bytes;
/* Drop the reference held by the task. */
vy_run_unref(new_run);
@@ -1457,6 +1458,7 @@ vy_task_compact_complete(struct vy_task *task)
vy_lsm_add_run(lsm, new_run);
vy_stmt_counter_add_disk(&lsm->stat.disk.compact.out,
&new_run->count);
+ lsm->env->compact_total += new_run->count.bytes;
/* Drop the reference held by the task. */
vy_run_unref(new_run);
} else
diff --git a/test/vinyl/info.result b/test/vinyl/info.result
index a53ee3ae..36fa732c 100644
--- a/test/vinyl/info.result
+++ b/test/vinyl/info.result
@@ -210,7 +210,13 @@ istat()
...
gstat()
---
-- quota:
+- disk:
+ dump_total: 0
+ data_size: 0
+ data_files: 0
+ compact_total: 0
+ index_size: 0
+ quota:
limit: 134217728
used: 0
memory:
@@ -667,6 +673,15 @@ box.rollback()
--
-- Global statistics.
--
+-- dump and compaction totals
+gstat().disk.dump_total == istat().disk.dump.out.bytes
+---
+- true
+...
+gstat().disk.compact_total == istat().disk.compact.out.bytes
+---
+- true
+...
-- use memory
st = gstat()
---
@@ -1008,7 +1023,13 @@ istat()
...
gstat()
---
-- quota:
+- disk:
+ dump_total: 0
+ data_size: 104300
+ data_files: 2
+ compact_total: 0
+ index_size: 1190
+ quota:
limit: 134217728
used: 262583
memory:
@@ -1113,6 +1134,14 @@ gst.memory.bloom_filter == 0
---
- true
...
+gst.disk.data_size == 0
+---
+- true
+...
+gst.disk.index_size == 0
+---
+- true
+...
box.snapshot()
---
- ok
@@ -1168,6 +1197,14 @@ gst.memory.bloom_filter == st1.disk.bloom_size + st2.disk.bloom_size
---
- true
...
+gst.disk.data_size == s:bsize()
+---
+- true
+...
+gst.disk.index_size == i1:bsize() + i2:bsize()
+---
+- true
+...
for i = 1, 100, 2 do s:delete(i) end
---
...
@@ -1287,6 +1324,14 @@ gst.memory.bloom_filter == st1.disk.bloom_size + st2.disk.bloom_size
---
- true
...
+gst.disk.data_size == s:bsize()
+---
+- true
+...
+gst.disk.index_size == i1:bsize() + i2:bsize()
+---
+- true
+...
s:drop()
---
...
@@ -1301,6 +1346,14 @@ gst.memory.bloom_filter == 0
---
- true
...
+gst.disk.data_size == 0
+---
+- true
+...
+gst.disk.index_size == 0
+---
+- true
+...
test_run:cmd('switch default')
---
- true
diff --git a/test/vinyl/info.test.lua b/test/vinyl/info.test.lua
index 230bac1e..919dde63 100644
--- a/test/vinyl/info.test.lua
+++ b/test/vinyl/info.test.lua
@@ -206,6 +206,10 @@ box.rollback()
-- Global statistics.
--
+-- dump and compaction totals
+gstat().disk.dump_total == istat().disk.dump.out.bytes
+gstat().disk.compact_total == istat().disk.compact.out.bytes
+
-- use memory
st = gstat()
put(1)
@@ -342,6 +346,8 @@ i1:bsize() == st1.memory.index_size
i2:bsize() == st2.memory.index_size
gst.memory.page_index == 0
gst.memory.bloom_filter == 0
+gst.disk.data_size == 0
+gst.disk.index_size == 0
box.snapshot()
gst = gstat()
@@ -357,6 +363,8 @@ i1:bsize() == st1.disk.index_size + st1.disk.bloom_size
i2:bsize() == st2.disk.index_size + st2.disk.bloom_size + st2.disk.bytes
gst.memory.page_index == st1.disk.index_size + st2.disk.index_size
gst.memory.bloom_filter == st1.disk.bloom_size + st2.disk.bloom_size
+gst.disk.data_size == s:bsize()
+gst.disk.index_size == i1:bsize() + i2:bsize()
for i = 1, 100, 2 do s:delete(i) end
for i = 2, 100, 2 do s:replace{i, i, pad()} end
@@ -392,12 +400,16 @@ i1:bsize() == st1.disk.index_size + st1.disk.bloom_size
i2:bsize() == st2.disk.index_size + st2.disk.bloom_size + st2.disk.bytes
gst.memory.page_index == st1.disk.index_size + st2.disk.index_size
gst.memory.bloom_filter == st1.disk.bloom_size + st2.disk.bloom_size
+gst.disk.data_size == s:bsize()
+gst.disk.index_size == i1:bsize() + i2:bsize()
s:drop()
gst = gstat()
gst.memory.page_index == 0
gst.memory.bloom_filter == 0
+gst.disk.data_size == 0
+gst.disk.index_size == 0
test_run:cmd('switch default')
test_run:cmd('stop server test')
--
2.11.0
More information about the Tarantool-patches
mailing list