[PATCH 3/7] vinyl: add global disk stats

[Vladimir Davydov <vdavydov.dev@gmail.com>]

This patch adds some essential disk statistics to box.stat.vinyl().
The new statistics are reported under the 'disk' section and currently
include the following fields:

 - data_files: number of open data files (*.run).
 - data_size: size of data stored on disk.
 - index_size: size of index stored on disk.
 - dump_total: number of bytes written by dump tasks.
 - compact_total: number of bytes writted by compaction tasks.

All sizes are in bytes, before compression. Dump and compaction counters
are reset by box.stat.reset().
---
 src/box/vinyl.c          | 19 ++++++++++++++++
 src/box/vy_lsm.c         | 44 ++++++++++++++++++++++++++++---------
 src/box/vy_lsm.h         | 16 ++++++++++++++
 src/box/vy_scheduler.c   |  2 ++
 test/vinyl/info.result   | 57 ++++++++++++++++++++++++++++++++++++++++++++++--
 test/vinyl/info.test.lua | 12 ++++++++++
 6 files changed, 138 insertions(+), 12 deletions(-)

diff --git a/src/box/vinyl.c b/src/box/vinyl.c
index 88cdbce9..edfaa824 100644
--- a/src/box/vinyl.c
+++ b/src/box/vinyl.c
@@ -298,6 +298,20 @@ vy_info_append_memory(struct vy_env *env, struct info_handler *h)
 	info_table_end(h);
 }
 
+static void
+vy_info_append_disk(struct vy_env *env, struct info_handler *h)
+{
+	struct vy_lsm_env *lsm_env = &env->lsm_env;
+
+	info_table_begin(h, "disk");
+	info_append_int(h, "data_files", lsm_env->data_file_count);
+	info_append_int(h, "data_size", lsm_env->disk_data_size);
+	info_append_int(h, "index_size", lsm_env->disk_index_size);
+	info_append_int(h, "dump_total", lsm_env->dump_total);
+	info_append_int(h, "compact_total", lsm_env->compact_total);
+	info_table_end(h);
+}
+
 void
 vinyl_engine_stat(struct vinyl_engine *vinyl, struct info_handler *h)
 {
@@ -307,6 +321,7 @@ vinyl_engine_stat(struct vinyl_engine *vinyl, struct info_handler *h)
 	vy_info_append_quota(env, h);
 	vy_info_append_tx(env, h);
 	vy_info_append_memory(env, h);
+	vy_info_append_disk(env, h);
 	info_end(h);
 }
 
@@ -476,8 +491,12 @@ vinyl_engine_reset_stat(struct engine *engine)
 {
 	struct vy_env *env = vy_env(engine);
 	struct tx_manager *xm = env->xm;
+	struct vy_lsm_env *lsm_env = &env->lsm_env;
 
 	memset(&xm->stat, 0, sizeof(xm->stat));
+
+	lsm_env->dump_total = 0;
+	lsm_env->compact_total = 0;
 }
 
 /** }}} Introspection */
diff --git a/src/box/vy_lsm.c b/src/box/vy_lsm.c
index 1994525e..15592fbf 100644
--- a/src/box/vy_lsm.c
+++ b/src/box/vy_lsm.c
@@ -685,32 +685,56 @@ vy_lsm_compact_priority(struct vy_lsm *lsm)
 void
 vy_lsm_add_run(struct vy_lsm *lsm, struct vy_run *run)
 {
+	struct vy_lsm_env *env = lsm->env;
+	size_t bloom_size = vy_run_bloom_size(run);
+	size_t page_index_size = run->page_index_size;
+
 	assert(rlist_empty(&run->in_lsm));
 	rlist_add_entry(&lsm->runs, run, in_lsm);
 	lsm->run_count++;
 	vy_disk_stmt_counter_add(&lsm->stat.disk.count, &run->count);
 
-	lsm->bloom_size += vy_run_bloom_size(run);
-	lsm->page_index_size += run->page_index_size;
-
-	lsm->env->bloom_size += vy_run_bloom_size(run);
-	lsm->env->page_index_size += run->page_index_size;
+	lsm->bloom_size += bloom_size;
+	lsm->page_index_size += page_index_size;
+
+	env->bloom_size += bloom_size;
+	env->page_index_size += page_index_size;
+	env->data_file_count++;
+	/* Data size is consistent with space.bsize. */
+	if (lsm->index_id == 0)
+		env->disk_data_size += run->count.bytes;
+	/* Index size is consistent with index.bsize. */
+	env->disk_index_size += bloom_size + page_index_size;
+	if (lsm->index_id > 0)
+		env->disk_index_size += run->count.bytes;
 }
 
 void
 vy_lsm_remove_run(struct vy_lsm *lsm, struct vy_run *run)
 {
+	struct vy_lsm_env *env = lsm->env;
+	size_t bloom_size = vy_run_bloom_size(run);
+	size_t page_index_size = run->page_index_size;
+
 	assert(lsm->run_count > 0);
 	assert(!rlist_empty(&run->in_lsm));
 	rlist_del_entry(run, in_lsm);
 	lsm->run_count--;
 	vy_disk_stmt_counter_sub(&lsm->stat.disk.count, &run->count);
 
-	lsm->bloom_size -= vy_run_bloom_size(run);
-	lsm->page_index_size -= run->page_index_size;
-
-	lsm->env->bloom_size -= vy_run_bloom_size(run);
-	lsm->env->page_index_size -= run->page_index_size;
+	lsm->bloom_size -= bloom_size;
+	lsm->page_index_size -= page_index_size;
+
+	env->bloom_size -= bloom_size;
+	env->page_index_size -= page_index_size;
+	env->data_file_count--;
+	/* Data size is consistent with space.bsize. */
+	if (lsm->index_id == 0)
+		env->disk_data_size -= run->count.bytes;
+	/* Index size is consistent with index.bsize. */
+	env->disk_index_size -= bloom_size + page_index_size;
+	if (lsm->index_id > 0)
+		env->disk_index_size -= run->count.bytes;
 }
 
 void
diff --git a/src/box/vy_lsm.h b/src/box/vy_lsm.h
index 6917d475..67a5c5d1 100644
--- a/src/box/vy_lsm.h
+++ b/src/box/vy_lsm.h
@@ -91,6 +91,22 @@ struct vy_lsm_env {
 	size_t bloom_size;
 	/** Size of memory used for page index. */
 	size_t page_index_size;
+	/** Total number of data files (*.run). */
+	size_t data_file_count;
+	/** Size of data stored on disk (uncompressed). */
+	size_t disk_data_size;
+	/** Size of index stored on disk (uncompressed). */
+	size_t disk_index_size;
+	/**
+	 * Total number of bytes written to disk by dump tasks
+	 * (uncompressed).
+	 */
+	int64_t dump_total;
+	/**
+	 * Total number of bytes written to disk by compaction
+	 * tasks (uncompressed).
+	 */
+	int64_t compact_total;
 	/** Memory pool for vy_history_node allocations. */
 	struct mempool history_node_pool;
 };
diff --git a/src/box/vy_scheduler.c b/src/box/vy_scheduler.c
index e41f1ffd..4959300e 100644
--- a/src/box/vy_scheduler.c
+++ b/src/box/vy_scheduler.c
@@ -1117,6 +1117,7 @@ vy_task_dump_complete(struct vy_task *task)
 	 */
 	vy_lsm_add_run(lsm, new_run);
 	vy_stmt_counter_add_disk(&lsm->stat.disk.dump.out, &new_run->count);
+	lsm->env->dump_total += new_run->count.bytes;
 
 	/* Drop the reference held by the task. */
 	vy_run_unref(new_run);
@@ -1457,6 +1458,7 @@ vy_task_compact_complete(struct vy_task *task)
 		vy_lsm_add_run(lsm, new_run);
 		vy_stmt_counter_add_disk(&lsm->stat.disk.compact.out,
 					 &new_run->count);
+		lsm->env->compact_total += new_run->count.bytes;
 		/* Drop the reference held by the task. */
 		vy_run_unref(new_run);
 	} else
diff --git a/test/vinyl/info.result b/test/vinyl/info.result
index a53ee3ae..36fa732c 100644
--- a/test/vinyl/info.result
+++ b/test/vinyl/info.result
@@ -210,7 +210,13 @@ istat()
 ...
 gstat()
 ---
-- quota:
+- disk:
+    dump_total: 0
+    data_size: 0
+    data_files: 0
+    compact_total: 0
+    index_size: 0
+  quota:
     limit: 134217728
     used: 0
   memory:
@@ -667,6 +673,15 @@ box.rollback()
 --
 -- Global statistics.
 --
+-- dump and compaction totals
+gstat().disk.dump_total == istat().disk.dump.out.bytes
+---
+- true
+...
+gstat().disk.compact_total == istat().disk.compact.out.bytes
+---
+- true
+...
 -- use memory
 st = gstat()
 ---
@@ -1008,7 +1023,13 @@ istat()
 ...
 gstat()
 ---
-- quota:
+- disk:
+    dump_total: 0
+    data_size: 104300
+    data_files: 2
+    compact_total: 0
+    index_size: 1190
+  quota:
     limit: 134217728
     used: 262583
   memory:
@@ -1113,6 +1134,14 @@ gst.memory.bloom_filter == 0
 ---
 - true
 ...
+gst.disk.data_size == 0
+---
+- true
+...
+gst.disk.index_size == 0
+---
+- true
+...
 box.snapshot()
 ---
 - ok
@@ -1168,6 +1197,14 @@ gst.memory.bloom_filter == st1.disk.bloom_size + st2.disk.bloom_size
 ---
 - true
 ...
+gst.disk.data_size == s:bsize()
+---
+- true
+...
+gst.disk.index_size == i1:bsize() + i2:bsize()
+---
+- true
+...
 for i = 1, 100, 2 do s:delete(i) end
 ---
 ...
@@ -1287,6 +1324,14 @@ gst.memory.bloom_filter == st1.disk.bloom_size + st2.disk.bloom_size
 ---
 - true
 ...
+gst.disk.data_size == s:bsize()
+---
+- true
+...
+gst.disk.index_size == i1:bsize() + i2:bsize()
+---
+- true
+...
 s:drop()
 ---
 ...
@@ -1301,6 +1346,14 @@ gst.memory.bloom_filter == 0
 ---
 - true
 ...
+gst.disk.data_size == 0
+---
+- true
+...
+gst.disk.index_size == 0
+---
+- true
+...
 test_run:cmd('switch default')
 ---
 - true
diff --git a/test/vinyl/info.test.lua b/test/vinyl/info.test.lua
index 230bac1e..919dde63 100644
--- a/test/vinyl/info.test.lua
+++ b/test/vinyl/info.test.lua
@@ -206,6 +206,10 @@ box.rollback()
 -- Global statistics.
 --
 
+-- dump and compaction totals
+gstat().disk.dump_total == istat().disk.dump.out.bytes
+gstat().disk.compact_total == istat().disk.compact.out.bytes
+
 -- use memory
 st = gstat()
 put(1)
@@ -342,6 +346,8 @@ i1:bsize() == st1.memory.index_size
 i2:bsize() == st2.memory.index_size
 gst.memory.page_index == 0
 gst.memory.bloom_filter == 0
+gst.disk.data_size == 0
+gst.disk.index_size == 0
 
 box.snapshot()
 gst = gstat()
@@ -357,6 +363,8 @@ i1:bsize() == st1.disk.index_size + st1.disk.bloom_size
 i2:bsize() == st2.disk.index_size + st2.disk.bloom_size + st2.disk.bytes
 gst.memory.page_index == st1.disk.index_size + st2.disk.index_size
 gst.memory.bloom_filter == st1.disk.bloom_size + st2.disk.bloom_size
+gst.disk.data_size == s:bsize()
+gst.disk.index_size == i1:bsize() + i2:bsize()
 
 for i = 1, 100, 2 do s:delete(i) end
 for i = 2, 100, 2 do s:replace{i, i, pad()} end
@@ -392,12 +400,16 @@ i1:bsize() == st1.disk.index_size + st1.disk.bloom_size
 i2:bsize() == st2.disk.index_size + st2.disk.bloom_size + st2.disk.bytes
 gst.memory.page_index == st1.disk.index_size + st2.disk.index_size
 gst.memory.bloom_filter == st1.disk.bloom_size + st2.disk.bloom_size
+gst.disk.data_size == s:bsize()
+gst.disk.index_size == i1:bsize() + i2:bsize()
 
 s:drop()
 
 gst = gstat()
 gst.memory.page_index == 0
 gst.memory.bloom_filter == 0
+gst.disk.data_size == 0
+gst.disk.index_size == 0
 
 test_run:cmd('switch default')
 test_run:cmd('stop server test')
-- 
2.11.0