[PATCH 2/7] vinyl: add global memory stats

[Vladimir Davydov <vdavydov.dev@gmail.com>]

box.info.memory() gives you some insight on what memory is used for,
but it's very coarse. For vinyl we need finer grained global memory
statistics.

This patch adds such: they are reported under box.stat.vinyl().memory
and consist of the following entries:

 - level0: sum size of level-0 of all LSM trees.
 - tx: size of memory used by tx write and read sets.
 - tuple_cache: size of memory occupied by tuple cache.
 - page_index: size of memory used for storing page indexes.
 - bloom_filter: size of memory used for storing bloom filters.

It also removes box.stat.vinyl().cache, as the size of cache is now
reported under memory.tuple_cache.
---
 src/box/vinyl.c           | 42 ++++++++---------------
 src/box/vy_tx.c           | 19 +++++++++++
 src/box/vy_tx.h           |  4 +++
 test/vinyl/cache.result   | 10 +++---
 test/vinyl/cache.test.lua | 10 +++---
 test/vinyl/info.result    | 85 ++++++++++++++++++++++++++++++++++++-----------
 test/vinyl/info.test.lua  | 19 +++++++++--
 7 files changed, 129 insertions(+), 60 deletions(-)

diff --git a/src/box/vinyl.c b/src/box/vinyl.c
index 0b33b6f7..88cdbce9 100644
--- a/src/box/vinyl.c
+++ b/src/box/vinyl.c
@@ -263,23 +263,6 @@ vy_info_append_quota(struct vy_env *env, struct info_handler *h)
 }
 
 static void
-vy_info_append_cache(struct vy_env *env, struct info_handler *h)
-{
-	struct vy_cache_env *c = &env->cache_env;
-
-	info_table_begin(h, "cache");
-
-	info_append_int(h, "used", c->mem_used);
-	info_append_int(h, "limit", c->mem_quota);
-
-	struct mempool_stats mstats;
-	mempool_stats(&c->cache_entry_mempool, &mstats);
-	info_append_int(h, "tuples", mstats.objcount);
-
-	info_table_end(h);
-}
-
-static void
 vy_info_append_tx(struct vy_env *env, struct info_handler *h)
 {
 	struct tx_manager *xm = env->xm;
@@ -303,6 +286,18 @@ vy_info_append_tx(struct vy_env *env, struct info_handler *h)
 	info_table_end(h);
 }
 
+static void
+vy_info_append_memory(struct vy_env *env, struct info_handler *h)
+{
+	info_table_begin(h, "memory");
+	info_append_int(h, "tx", tx_manager_mem_used(env->xm));
+	info_append_int(h, "level0", lsregion_used(&env->mem_env.allocator));
+	info_append_int(h, "tuple_cache", env->cache_env.mem_used);
+	info_append_int(h, "page_index", env->lsm_env.page_index_size);
+	info_append_int(h, "bloom_filter", env->lsm_env.bloom_size);
+	info_table_end(h);
+}
+
 void
 vinyl_engine_stat(struct vinyl_engine *vinyl, struct info_handler *h)
 {
@@ -310,8 +305,8 @@ vinyl_engine_stat(struct vinyl_engine *vinyl, struct info_handler *h)
 
 	info_begin(h);
 	vy_info_append_quota(env, h);
-	vy_info_append_cache(env, h);
 	vy_info_append_tx(env, h);
+	vy_info_append_memory(env, h);
 	info_end(h);
 }
 
@@ -466,7 +461,6 @@ static void
 vinyl_engine_memory_stat(struct engine *engine, struct engine_memory_stat *stat)
 {
 	struct vy_env *env = vy_env(engine);
-	struct mempool_stats mstats;
 
 	stat->data += lsregion_used(&env->mem_env.allocator) -
 				env->mem_env.tree_extent_size;
@@ -474,15 +468,7 @@ vinyl_engine_memory_stat(struct engine *engine, struct engine_memory_stat *stat)
 	stat->index += env->lsm_env.bloom_size;
 	stat->index += env->lsm_env.page_index_size;
 	stat->cache += env->cache_env.mem_used;
-	stat->tx += env->xm->write_set_size + env->xm->read_set_size;
-	mempool_stats(&env->xm->tx_mempool, &mstats);
-	stat->tx += mstats.totals.used;
-	mempool_stats(&env->xm->txv_mempool, &mstats);
-	stat->tx += mstats.totals.used;
-	mempool_stats(&env->xm->read_interval_mempool, &mstats);
-	stat->tx += mstats.totals.used;
-	mempool_stats(&env->xm->read_view_mempool, &mstats);
-	stat->tx += mstats.totals.used;
+	stat->tx += tx_manager_mem_used(env->xm);
 }
 
 static void
diff --git a/src/box/vy_tx.c b/src/box/vy_tx.c
index 1e8775a0..957c87f0 100644
--- a/src/box/vy_tx.c
+++ b/src/box/vy_tx.c
@@ -137,6 +137,25 @@ tx_manager_delete(struct tx_manager *xm)
 	free(xm);
 }
 
+size_t
+tx_manager_mem_used(struct tx_manager *xm)
+{
+	struct mempool_stats mstats;
+	size_t ret = 0;
+
+	ret += xm->write_set_size + xm->read_set_size;
+	mempool_stats(&xm->tx_mempool, &mstats);
+	ret += mstats.totals.used;
+	mempool_stats(&xm->txv_mempool, &mstats);
+	ret += mstats.totals.used;
+	mempool_stats(&xm->read_interval_mempool, &mstats);
+	ret += mstats.totals.used;
+	mempool_stats(&xm->read_view_mempool, &mstats);
+	ret += mstats.totals.used;
+
+	return ret;
+}
+
 /** Create or reuse an instance of a read view. */
 static struct vy_read_view *
 tx_manager_read_view(struct tx_manager *xm)
diff --git a/src/box/vy_tx.h b/src/box/vy_tx.h
index 1d515c72..b201abd7 100644
--- a/src/box/vy_tx.h
+++ b/src/box/vy_tx.h
@@ -268,6 +268,10 @@ tx_manager_new(void);
 void
 tx_manager_delete(struct tx_manager *xm);
 
+/** Return total amount of memory used by active transactions. */
+size_t
+tx_manager_mem_used(struct tx_manager *xm);
+
 /**
  * Abort all rw transactions that affect the given LSM tree
  * and haven't reached WAL yet.
diff --git a/test/vinyl/cache.result b/test/vinyl/cache.result
index d75de263..85741604 100644
--- a/test/vinyl/cache.result
+++ b/test/vinyl/cache.result
@@ -1031,21 +1031,21 @@ for i = 1, 100 do s:replace{i, string.rep('x', 1000)} end
 for i = 1, 100 do s:get{i} end
 ---
 ...
-box.stat.vinyl().cache.used
+box.stat.vinyl().memory.tuple_cache
 ---
 - 107700
 ...
 box.cfg{vinyl_cache = 50 * 1000}
 ---
 ...
-box.stat.vinyl().cache.used
+box.stat.vinyl().memory.tuple_cache
 ---
 - 49542
 ...
 box.cfg{vinyl_cache = 0}
 ---
 ...
-box.stat.vinyl().cache.used
+box.stat.vinyl().memory.tuple_cache
 ---
 - 0
 ...
@@ -1067,7 +1067,7 @@ st2.put.rows - st1.put.rows
 ---
 - 0
 ...
-box.stat.vinyl().cache.used
+box.stat.vinyl().memory.tuple_cache
 ---
 - 0
 ...
@@ -1114,7 +1114,7 @@ s.index.i2:count()
 ---
 - 100
 ...
-box.stat.vinyl().cache.used -- should be about 200 KB
+box.stat.vinyl().memory.tuple_cache -- should be about 200 KB
 ---
 - 216800
 ...
diff --git a/test/vinyl/cache.test.lua b/test/vinyl/cache.test.lua
index 6d82249a..2fedf34b 100644
--- a/test/vinyl/cache.test.lua
+++ b/test/vinyl/cache.test.lua
@@ -366,18 +366,18 @@ s = box.schema.space.create('test', {engine = 'vinyl'})
 _ = s:create_index('pk')
 for i = 1, 100 do s:replace{i, string.rep('x', 1000)} end
 for i = 1, 100 do s:get{i} end
-box.stat.vinyl().cache.used
+box.stat.vinyl().memory.tuple_cache
 box.cfg{vinyl_cache = 50 * 1000}
-box.stat.vinyl().cache.used
+box.stat.vinyl().memory.tuple_cache
 box.cfg{vinyl_cache = 0}
-box.stat.vinyl().cache.used
+box.stat.vinyl().memory.tuple_cache
 -- Make sure cache is not populated if box.cfg.vinyl_cache is set to 0
 st1 = s.index.pk:stat().cache
 #s:select()
 for i = 1, 100 do s:get{i} end
 st2 = s.index.pk:stat().cache
 st2.put.rows - st1.put.rows
-box.stat.vinyl().cache.used
+box.stat.vinyl().memory.tuple_cache
 s:drop()
 box.cfg{vinyl_cache = vinyl_cache}
 
@@ -395,6 +395,6 @@ for i = 1, 100 do pad = string.rep(i % 10, 1000) s:replace{i, pad, pad} end
 s.index.pk:count()
 s.index.i1:count()
 s.index.i2:count()
-box.stat.vinyl().cache.used -- should be about 200 KB
+box.stat.vinyl().memory.tuple_cache -- should be about 200 KB
 s:drop()
 box.cfg{vinyl_cache = vinyl_cache}
diff --git a/test/vinyl/info.result b/test/vinyl/info.result
index 95e8cc60..a53ee3ae 100644
--- a/test/vinyl/info.result
+++ b/test/vinyl/info.result
@@ -210,10 +210,15 @@ istat()
 ...
 gstat()
 ---
-- cache:
-    limit: 15360
-    tuples: 0
+- quota:
+    limit: 134217728
     used: 0
+  memory:
+    tuple_cache: 0
+    tx: 0
+    level0: 0
+    page_index: 0
+    bloom_filter: 0
   tx:
     conflict: 0
     commit: 0
@@ -222,9 +227,6 @@ gstat()
     transactions: 0
     gap_locks: 0
     read_views: 0
-  quota:
-    limit: 134217728
-    used: 0
 ...
 --
 -- Index statistics.
@@ -665,16 +667,16 @@ box.rollback()
 --
 -- Global statistics.
 --
--- use quota
+-- use memory
 st = gstat()
 ---
 ...
 put(1)
 ---
 ...
-stat_diff(gstat(), st, 'quota')
+stat_diff(gstat(), st, 'memory.level0')
 ---
-- used: 1061
+- 1061
 ...
 -- use cache
 st = gstat()
@@ -683,10 +685,9 @@ st = gstat()
 _ = s:get(1)
 ---
 ...
-stat_diff(gstat(), st, 'cache')
+stat_diff(gstat(), st, 'memory.tuple_cache')
 ---
-- used: 1101
-  tuples: 1
+- 1101
 ...
 s:delete(1)
 ---
@@ -1007,10 +1008,15 @@ istat()
 ...
 gstat()
 ---
-- cache:
-    limit: 15360
-    tuples: 13
-    used: 14313
+- quota:
+    limit: 134217728
+    used: 262583
+  memory:
+    tuple_cache: 14313
+    tx: 0
+    level0: 262583
+    page_index: 1050
+    bloom_filter: 140
   tx:
     conflict: 0
     commit: 0
@@ -1019,9 +1025,6 @@ gstat()
     transactions: 0
     gap_locks: 0
     read_views: 0
-  quota:
-    limit: 134217728
-    used: 262583
 ...
 s:drop()
 ---
@@ -1059,6 +1062,9 @@ i1:bsize(), i2:bsize()
 for i = 1, 100, 2 do s:replace{i, i, pad()} end
 ---
 ...
+gst = gstat()
+---
+...
 st1 = i1:stat()
 ---
 ...
@@ -1099,10 +1105,21 @@ i2:bsize() == st2.memory.index_size
 ---
 - true
 ...
+gst.memory.page_index == 0
+---
+- true
+...
+gst.memory.bloom_filter == 0
+---
+- true
+...
 box.snapshot()
 ---
 - ok
 ...
+gst = gstat()
+---
+...
 st1 = i1:stat()
 ---
 ...
@@ -1143,6 +1160,14 @@ i2:bsize() == st2.disk.index_size + st2.disk.bloom_size + st2.disk.bytes
 ---
 - true
 ...
+gst.memory.page_index == st1.disk.index_size + st2.disk.index_size
+---
+- true
+...
+gst.memory.bloom_filter == st1.disk.bloom_size + st2.disk.bloom_size
+---
+- true
+...
 for i = 1, 100, 2 do s:delete(i) end
 ---
 ...
@@ -1211,6 +1236,9 @@ i2:compact()
 wait(function() return i2:stat() end, st2, 'disk.compact.count', 1)
 ---
 ...
+gst = gstat()
+---
+...
 st1 = i1:stat()
 ---
 ...
@@ -1251,9 +1279,28 @@ i2:bsize() == st2.disk.index_size + st2.disk.bloom_size + st2.disk.bytes
 ---
 - true
 ...
+gst.memory.page_index == st1.disk.index_size + st2.disk.index_size
+---
+- true
+...
+gst.memory.bloom_filter == st1.disk.bloom_size + st2.disk.bloom_size
+---
+- true
+...
 s:drop()
 ---
 ...
+gst = gstat()
+---
+...
+gst.memory.page_index == 0
+---
+- true
+...
+gst.memory.bloom_filter == 0
+---
+- true
+...
 test_run:cmd('switch default')
 ---
 - true
diff --git a/test/vinyl/info.test.lua b/test/vinyl/info.test.lua
index 5aebd0a8..230bac1e 100644
--- a/test/vinyl/info.test.lua
+++ b/test/vinyl/info.test.lua
@@ -206,15 +206,15 @@ box.rollback()
 -- Global statistics.
 --
 
--- use quota
+-- use memory
 st = gstat()
 put(1)
-stat_diff(gstat(), st, 'quota')
+stat_diff(gstat(), st, 'memory.level0')
 
 -- use cache
 st = gstat()
 _ = s:get(1)
-stat_diff(gstat(), st, 'cache')
+stat_diff(gstat(), st, 'memory.tuple_cache')
 
 s:delete(1)
 
@@ -329,6 +329,7 @@ i1:len(), i2:len()
 i1:bsize(), i2:bsize()
 
 for i = 1, 100, 2 do s:replace{i, i, pad()} end
+gst = gstat()
 st1 = i1:stat()
 st2 = i2:stat()
 s:bsize()
@@ -339,8 +340,11 @@ i1:len() == st1.memory.rows
 i2:len() == st2.memory.rows
 i1:bsize() == st1.memory.index_size
 i2:bsize() == st2.memory.index_size
+gst.memory.page_index == 0
+gst.memory.bloom_filter == 0
 
 box.snapshot()
+gst = gstat()
 st1 = i1:stat()
 st2 = i2:stat()
 s:bsize()
@@ -351,6 +355,8 @@ i1:len() == st1.disk.rows
 i2:len() == st2.disk.rows
 i1:bsize() == st1.disk.index_size + st1.disk.bloom_size
 i2:bsize() == st2.disk.index_size + st2.disk.bloom_size + st2.disk.bytes
+gst.memory.page_index == st1.disk.index_size + st2.disk.index_size
+gst.memory.bloom_filter == st1.disk.bloom_size + st2.disk.bloom_size
 
 for i = 1, 100, 2 do s:delete(i) end
 for i = 2, 100, 2 do s:replace{i, i, pad()} end
@@ -373,6 +379,7 @@ wait(function() return i1:stat() end, st1, 'disk.compact.count', 1)
 box.snapshot()
 i2:compact()
 wait(function() return i2:stat() end, st2, 'disk.compact.count', 1)
+gst = gstat()
 st1 = i1:stat()
 st2 = i2:stat()
 s:bsize()
@@ -383,9 +390,15 @@ i1:len() == st1.disk.rows
 i2:len() == st2.disk.rows
 i1:bsize() == st1.disk.index_size + st1.disk.bloom_size
 i2:bsize() == st2.disk.index_size + st2.disk.bloom_size + st2.disk.bytes
+gst.memory.page_index == st1.disk.index_size + st2.disk.index_size
+gst.memory.bloom_filter == st1.disk.bloom_size + st2.disk.bloom_size
 
 s:drop()
 
+gst = gstat()
+gst.memory.page_index == 0
+gst.memory.bloom_filter == 0
+
 test_run:cmd('switch default')
 test_run:cmd('stop server test')
 test_run:cmd('cleanup server test')
-- 
2.11.0