[PATCH 2/7] vinyl: add global memory stats
Vladimir Davydov
vdavydov.dev at gmail.com
Sun Sep 2 23:18:55 MSK 2018
box.info.memory() gives you some insight on what memory is used for,
but it's very coarse. For vinyl we need finer grained global memory
statistics.
This patch adds such: they are reported under box.stat.vinyl().memory
and consist of the following entries:
- level0: sum size of level-0 of all LSM trees.
- tx: size of memory used by tx write and read sets.
- tuple_cache: size of memory occupied by tuple cache.
- page_index: size of memory used for storing page indexes.
- bloom_filter: size of memory used for storing bloom filters.
It also removes box.stat.vinyl().cache, as the size of cache is now
reported under memory.tuple_cache.
---
src/box/vinyl.c | 42 ++++++++---------------
src/box/vy_tx.c | 19 +++++++++++
src/box/vy_tx.h | 4 +++
test/vinyl/cache.result | 10 +++---
test/vinyl/cache.test.lua | 10 +++---
test/vinyl/info.result | 85 ++++++++++++++++++++++++++++++++++++-----------
test/vinyl/info.test.lua | 19 +++++++++--
7 files changed, 129 insertions(+), 60 deletions(-)
diff --git a/src/box/vinyl.c b/src/box/vinyl.c
index 0b33b6f7..88cdbce9 100644
--- a/src/box/vinyl.c
+++ b/src/box/vinyl.c
@@ -263,23 +263,6 @@ vy_info_append_quota(struct vy_env *env, struct info_handler *h)
}
static void
-vy_info_append_cache(struct vy_env *env, struct info_handler *h)
-{
- struct vy_cache_env *c = &env->cache_env;
-
- info_table_begin(h, "cache");
-
- info_append_int(h, "used", c->mem_used);
- info_append_int(h, "limit", c->mem_quota);
-
- struct mempool_stats mstats;
- mempool_stats(&c->cache_entry_mempool, &mstats);
- info_append_int(h, "tuples", mstats.objcount);
-
- info_table_end(h);
-}
-
-static void
vy_info_append_tx(struct vy_env *env, struct info_handler *h)
{
struct tx_manager *xm = env->xm;
@@ -303,6 +286,18 @@ vy_info_append_tx(struct vy_env *env, struct info_handler *h)
info_table_end(h);
}
+static void
+vy_info_append_memory(struct vy_env *env, struct info_handler *h)
+{
+ info_table_begin(h, "memory");
+ info_append_int(h, "tx", tx_manager_mem_used(env->xm));
+ info_append_int(h, "level0", lsregion_used(&env->mem_env.allocator));
+ info_append_int(h, "tuple_cache", env->cache_env.mem_used);
+ info_append_int(h, "page_index", env->lsm_env.page_index_size);
+ info_append_int(h, "bloom_filter", env->lsm_env.bloom_size);
+ info_table_end(h);
+}
+
void
vinyl_engine_stat(struct vinyl_engine *vinyl, struct info_handler *h)
{
@@ -310,8 +305,8 @@ vinyl_engine_stat(struct vinyl_engine *vinyl, struct info_handler *h)
info_begin(h);
vy_info_append_quota(env, h);
- vy_info_append_cache(env, h);
vy_info_append_tx(env, h);
+ vy_info_append_memory(env, h);
info_end(h);
}
@@ -466,7 +461,6 @@ static void
vinyl_engine_memory_stat(struct engine *engine, struct engine_memory_stat *stat)
{
struct vy_env *env = vy_env(engine);
- struct mempool_stats mstats;
stat->data += lsregion_used(&env->mem_env.allocator) -
env->mem_env.tree_extent_size;
@@ -474,15 +468,7 @@ vinyl_engine_memory_stat(struct engine *engine, struct engine_memory_stat *stat)
stat->index += env->lsm_env.bloom_size;
stat->index += env->lsm_env.page_index_size;
stat->cache += env->cache_env.mem_used;
- stat->tx += env->xm->write_set_size + env->xm->read_set_size;
- mempool_stats(&env->xm->tx_mempool, &mstats);
- stat->tx += mstats.totals.used;
- mempool_stats(&env->xm->txv_mempool, &mstats);
- stat->tx += mstats.totals.used;
- mempool_stats(&env->xm->read_interval_mempool, &mstats);
- stat->tx += mstats.totals.used;
- mempool_stats(&env->xm->read_view_mempool, &mstats);
- stat->tx += mstats.totals.used;
+ stat->tx += tx_manager_mem_used(env->xm);
}
static void
diff --git a/src/box/vy_tx.c b/src/box/vy_tx.c
index 1e8775a0..957c87f0 100644
--- a/src/box/vy_tx.c
+++ b/src/box/vy_tx.c
@@ -137,6 +137,25 @@ tx_manager_delete(struct tx_manager *xm)
free(xm);
}
+size_t
+tx_manager_mem_used(struct tx_manager *xm)
+{
+ struct mempool_stats mstats;
+ size_t ret = 0;
+
+ ret += xm->write_set_size + xm->read_set_size;
+ mempool_stats(&xm->tx_mempool, &mstats);
+ ret += mstats.totals.used;
+ mempool_stats(&xm->txv_mempool, &mstats);
+ ret += mstats.totals.used;
+ mempool_stats(&xm->read_interval_mempool, &mstats);
+ ret += mstats.totals.used;
+ mempool_stats(&xm->read_view_mempool, &mstats);
+ ret += mstats.totals.used;
+
+ return ret;
+}
+
/** Create or reuse an instance of a read view. */
static struct vy_read_view *
tx_manager_read_view(struct tx_manager *xm)
diff --git a/src/box/vy_tx.h b/src/box/vy_tx.h
index 1d515c72..b201abd7 100644
--- a/src/box/vy_tx.h
+++ b/src/box/vy_tx.h
@@ -268,6 +268,10 @@ tx_manager_new(void);
void
tx_manager_delete(struct tx_manager *xm);
+/** Return total amount of memory used by active transactions. */
+size_t
+tx_manager_mem_used(struct tx_manager *xm);
+
/**
* Abort all rw transactions that affect the given LSM tree
* and haven't reached WAL yet.
diff --git a/test/vinyl/cache.result b/test/vinyl/cache.result
index d75de263..85741604 100644
--- a/test/vinyl/cache.result
+++ b/test/vinyl/cache.result
@@ -1031,21 +1031,21 @@ for i = 1, 100 do s:replace{i, string.rep('x', 1000)} end
for i = 1, 100 do s:get{i} end
---
...
-box.stat.vinyl().cache.used
+box.stat.vinyl().memory.tuple_cache
---
- 107700
...
box.cfg{vinyl_cache = 50 * 1000}
---
...
-box.stat.vinyl().cache.used
+box.stat.vinyl().memory.tuple_cache
---
- 49542
...
box.cfg{vinyl_cache = 0}
---
...
-box.stat.vinyl().cache.used
+box.stat.vinyl().memory.tuple_cache
---
- 0
...
@@ -1067,7 +1067,7 @@ st2.put.rows - st1.put.rows
---
- 0
...
-box.stat.vinyl().cache.used
+box.stat.vinyl().memory.tuple_cache
---
- 0
...
@@ -1114,7 +1114,7 @@ s.index.i2:count()
---
- 100
...
-box.stat.vinyl().cache.used -- should be about 200 KB
+box.stat.vinyl().memory.tuple_cache -- should be about 200 KB
---
- 216800
...
diff --git a/test/vinyl/cache.test.lua b/test/vinyl/cache.test.lua
index 6d82249a..2fedf34b 100644
--- a/test/vinyl/cache.test.lua
+++ b/test/vinyl/cache.test.lua
@@ -366,18 +366,18 @@ s = box.schema.space.create('test', {engine = 'vinyl'})
_ = s:create_index('pk')
for i = 1, 100 do s:replace{i, string.rep('x', 1000)} end
for i = 1, 100 do s:get{i} end
-box.stat.vinyl().cache.used
+box.stat.vinyl().memory.tuple_cache
box.cfg{vinyl_cache = 50 * 1000}
-box.stat.vinyl().cache.used
+box.stat.vinyl().memory.tuple_cache
box.cfg{vinyl_cache = 0}
-box.stat.vinyl().cache.used
+box.stat.vinyl().memory.tuple_cache
-- Make sure cache is not populated if box.cfg.vinyl_cache is set to 0
st1 = s.index.pk:stat().cache
#s:select()
for i = 1, 100 do s:get{i} end
st2 = s.index.pk:stat().cache
st2.put.rows - st1.put.rows
-box.stat.vinyl().cache.used
+box.stat.vinyl().memory.tuple_cache
s:drop()
box.cfg{vinyl_cache = vinyl_cache}
@@ -395,6 +395,6 @@ for i = 1, 100 do pad = string.rep(i % 10, 1000) s:replace{i, pad, pad} end
s.index.pk:count()
s.index.i1:count()
s.index.i2:count()
-box.stat.vinyl().cache.used -- should be about 200 KB
+box.stat.vinyl().memory.tuple_cache -- should be about 200 KB
s:drop()
box.cfg{vinyl_cache = vinyl_cache}
diff --git a/test/vinyl/info.result b/test/vinyl/info.result
index 95e8cc60..a53ee3ae 100644
--- a/test/vinyl/info.result
+++ b/test/vinyl/info.result
@@ -210,10 +210,15 @@ istat()
...
gstat()
---
-- cache:
- limit: 15360
- tuples: 0
+- quota:
+ limit: 134217728
used: 0
+ memory:
+ tuple_cache: 0
+ tx: 0
+ level0: 0
+ page_index: 0
+ bloom_filter: 0
tx:
conflict: 0
commit: 0
@@ -222,9 +227,6 @@ gstat()
transactions: 0
gap_locks: 0
read_views: 0
- quota:
- limit: 134217728
- used: 0
...
--
-- Index statistics.
@@ -665,16 +667,16 @@ box.rollback()
--
-- Global statistics.
--
--- use quota
+-- use memory
st = gstat()
---
...
put(1)
---
...
-stat_diff(gstat(), st, 'quota')
+stat_diff(gstat(), st, 'memory.level0')
---
-- used: 1061
+- 1061
...
-- use cache
st = gstat()
@@ -683,10 +685,9 @@ st = gstat()
_ = s:get(1)
---
...
-stat_diff(gstat(), st, 'cache')
+stat_diff(gstat(), st, 'memory.tuple_cache')
---
-- used: 1101
- tuples: 1
+- 1101
...
s:delete(1)
---
@@ -1007,10 +1008,15 @@ istat()
...
gstat()
---
-- cache:
- limit: 15360
- tuples: 13
- used: 14313
+- quota:
+ limit: 134217728
+ used: 262583
+ memory:
+ tuple_cache: 14313
+ tx: 0
+ level0: 262583
+ page_index: 1050
+ bloom_filter: 140
tx:
conflict: 0
commit: 0
@@ -1019,9 +1025,6 @@ gstat()
transactions: 0
gap_locks: 0
read_views: 0
- quota:
- limit: 134217728
- used: 262583
...
s:drop()
---
@@ -1059,6 +1062,9 @@ i1:bsize(), i2:bsize()
for i = 1, 100, 2 do s:replace{i, i, pad()} end
---
...
+gst = gstat()
+---
+...
st1 = i1:stat()
---
...
@@ -1099,10 +1105,21 @@ i2:bsize() == st2.memory.index_size
---
- true
...
+gst.memory.page_index == 0
+---
+- true
+...
+gst.memory.bloom_filter == 0
+---
+- true
+...
box.snapshot()
---
- ok
...
+gst = gstat()
+---
+...
st1 = i1:stat()
---
...
@@ -1143,6 +1160,14 @@ i2:bsize() == st2.disk.index_size + st2.disk.bloom_size + st2.disk.bytes
---
- true
...
+gst.memory.page_index == st1.disk.index_size + st2.disk.index_size
+---
+- true
+...
+gst.memory.bloom_filter == st1.disk.bloom_size + st2.disk.bloom_size
+---
+- true
+...
for i = 1, 100, 2 do s:delete(i) end
---
...
@@ -1211,6 +1236,9 @@ i2:compact()
wait(function() return i2:stat() end, st2, 'disk.compact.count', 1)
---
...
+gst = gstat()
+---
+...
st1 = i1:stat()
---
...
@@ -1251,9 +1279,28 @@ i2:bsize() == st2.disk.index_size + st2.disk.bloom_size + st2.disk.bytes
---
- true
...
+gst.memory.page_index == st1.disk.index_size + st2.disk.index_size
+---
+- true
+...
+gst.memory.bloom_filter == st1.disk.bloom_size + st2.disk.bloom_size
+---
+- true
+...
s:drop()
---
...
+gst = gstat()
+---
+...
+gst.memory.page_index == 0
+---
+- true
+...
+gst.memory.bloom_filter == 0
+---
+- true
+...
test_run:cmd('switch default')
---
- true
diff --git a/test/vinyl/info.test.lua b/test/vinyl/info.test.lua
index 5aebd0a8..230bac1e 100644
--- a/test/vinyl/info.test.lua
+++ b/test/vinyl/info.test.lua
@@ -206,15 +206,15 @@ box.rollback()
-- Global statistics.
--
--- use quota
+-- use memory
st = gstat()
put(1)
-stat_diff(gstat(), st, 'quota')
+stat_diff(gstat(), st, 'memory.level0')
-- use cache
st = gstat()
_ = s:get(1)
-stat_diff(gstat(), st, 'cache')
+stat_diff(gstat(), st, 'memory.tuple_cache')
s:delete(1)
@@ -329,6 +329,7 @@ i1:len(), i2:len()
i1:bsize(), i2:bsize()
for i = 1, 100, 2 do s:replace{i, i, pad()} end
+gst = gstat()
st1 = i1:stat()
st2 = i2:stat()
s:bsize()
@@ -339,8 +340,11 @@ i1:len() == st1.memory.rows
i2:len() == st2.memory.rows
i1:bsize() == st1.memory.index_size
i2:bsize() == st2.memory.index_size
+gst.memory.page_index == 0
+gst.memory.bloom_filter == 0
box.snapshot()
+gst = gstat()
st1 = i1:stat()
st2 = i2:stat()
s:bsize()
@@ -351,6 +355,8 @@ i1:len() == st1.disk.rows
i2:len() == st2.disk.rows
i1:bsize() == st1.disk.index_size + st1.disk.bloom_size
i2:bsize() == st2.disk.index_size + st2.disk.bloom_size + st2.disk.bytes
+gst.memory.page_index == st1.disk.index_size + st2.disk.index_size
+gst.memory.bloom_filter == st1.disk.bloom_size + st2.disk.bloom_size
for i = 1, 100, 2 do s:delete(i) end
for i = 2, 100, 2 do s:replace{i, i, pad()} end
@@ -373,6 +379,7 @@ wait(function() return i1:stat() end, st1, 'disk.compact.count', 1)
box.snapshot()
i2:compact()
wait(function() return i2:stat() end, st2, 'disk.compact.count', 1)
+gst = gstat()
st1 = i1:stat()
st2 = i2:stat()
s:bsize()
@@ -383,9 +390,15 @@ i1:len() == st1.disk.rows
i2:len() == st2.disk.rows
i1:bsize() == st1.disk.index_size + st1.disk.bloom_size
i2:bsize() == st2.disk.index_size + st2.disk.bloom_size + st2.disk.bytes
+gst.memory.page_index == st1.disk.index_size + st2.disk.index_size
+gst.memory.bloom_filter == st1.disk.bloom_size + st2.disk.bloom_size
s:drop()
+gst = gstat()
+gst.memory.page_index == 0
+gst.memory.bloom_filter == 0
+
test_run:cmd('switch default')
test_run:cmd('stop server test')
test_run:cmd('cleanup server test')
--
2.11.0
More information about the Tarantool-patches
mailing list