[PATCH v2 8/8] vinyl: add global disk stats
Vladimir Davydov
vdavydov.dev at gmail.com
Sun Sep 16 20:06:51 MSK 2018
This patch adds some essential disk statistics that are already
collected and reported on per index basis to box.stat.vinyl().
The new statistics are shown under the 'disk' section and currently
include the following fields:
- data: size of data stored on disk.
- index: size of index stored on disk.
- dump.in: size of dump input.
- dump.out: size of dump output.
- compact.in: size of compaction input.
- compact.out: size of compaction output.
- compact.queue: size of compaction queue.
All the counters are given in bytes without taking into account
disk compression. Dump/compaction in/out counters can be reset with
box.stat.reset().
---
src/box/vinyl.c | 33 ++++++++++++++++++++-
src/box/vy_lsm.c | 50 ++++++++++++++++++++++++++-----
src/box/vy_lsm.h | 2 ++
src/box/vy_stat.h | 20 +++++++++++++
test/vinyl/errinj.result | 20 +++++++++++++
test/vinyl/errinj.test.lua | 5 ++++
test/vinyl/info.result | 73 ++++++++++++++++++++++++++++++++++++++++++++--
test/vinyl/info.test.lua | 14 +++++++++
8 files changed, 206 insertions(+), 11 deletions(-)
diff --git a/src/box/vinyl.c b/src/box/vinyl.c
index 02a2b69d..b5904872 100644
--- a/src/box/vinyl.c
+++ b/src/box/vinyl.c
@@ -298,6 +298,30 @@ vy_info_append_memory(struct vy_env *env, struct info_handler *h)
info_table_end(h); /* memory */
}
+static void
+vy_info_append_disk(struct vy_env *env, struct info_handler *h)
+{
+ struct vy_disk_stat *stat = &env->lsm_env.disk_stat;
+
+ info_table_begin(h, "disk");
+
+ info_append_int(h, "data", stat->data);
+ info_append_int(h, "index", stat->index);
+
+ info_table_begin(h, "dump");
+ info_append_int(h, "in", stat->dump.in);
+ info_append_int(h, "out", stat->dump.out);
+ info_table_end(h); /* dump */
+
+ info_table_begin(h, "compact");
+ info_append_int(h, "in", stat->compact.in);
+ info_append_int(h, "out", stat->compact.out);
+ info_append_int(h, "queue", stat->compact.queue);
+ info_table_end(h); /* compact */
+
+ info_table_end(h); /* disk */
+}
+
void
vinyl_engine_stat(struct vinyl_engine *vinyl, struct info_handler *h)
{
@@ -307,6 +331,7 @@ vinyl_engine_stat(struct vinyl_engine *vinyl, struct info_handler *h)
vy_info_append_quota(env, h);
vy_info_append_tx(env, h);
vy_info_append_memory(env, h);
+ vy_info_append_disk(env, h);
info_end(h);
}
@@ -485,9 +510,15 @@ static void
vinyl_engine_reset_stat(struct engine *engine)
{
struct vy_env *env = vy_env(engine);
- struct tx_manager *xm = env->xm;
+ struct tx_manager *xm = env->xm;
memset(&xm->stat, 0, sizeof(xm->stat));
+
+ struct vy_disk_stat *disk_stat = &env->lsm_env.disk_stat;
+ disk_stat->dump.in = 0;
+ disk_stat->dump.out = 0;
+ disk_stat->compact.in = 0;
+ disk_stat->compact.out = 0;
}
/** }}} Introspection */
diff --git a/src/box/vy_lsm.c b/src/box/vy_lsm.c
index dbea2898..093bb820 100644
--- a/src/box/vy_lsm.c
+++ b/src/box/vy_lsm.c
@@ -252,6 +252,8 @@ vy_lsm_delete(struct vy_lsm *lsm)
assert(lsm->env->lsm_count > 0);
lsm->env->lsm_count--;
+ lsm->env->disk_stat.compact.queue -=
+ lsm->stat.disk.compact.queue.bytes;
if (lsm->pk != NULL)
vy_lsm_unref(lsm->pk);
@@ -685,32 +687,56 @@ vy_lsm_compact_priority(struct vy_lsm *lsm)
void
vy_lsm_add_run(struct vy_lsm *lsm, struct vy_run *run)
{
+ struct vy_lsm_env *env = lsm->env;
+ size_t bloom_size = vy_run_bloom_size(run);
+ size_t page_index_size = run->page_index_size;
+
assert(rlist_empty(&run->in_lsm));
rlist_add_entry(&lsm->runs, run, in_lsm);
lsm->run_count++;
vy_disk_stmt_counter_add(&lsm->stat.disk.count, &run->count);
- lsm->bloom_size += vy_run_bloom_size(run);
- lsm->page_index_size += run->page_index_size;
+ lsm->bloom_size += bloom_size;
+ lsm->page_index_size += page_index_size;
+
+ env->bloom_size += bloom_size;
+ env->page_index_size += page_index_size;
- lsm->env->bloom_size += vy_run_bloom_size(run);
- lsm->env->page_index_size += run->page_index_size;
+ /* Data size is consistent with space.bsize. */
+ if (lsm->index_id == 0)
+ env->disk_stat.data += run->count.bytes;
+ /* Index size is consistent with index.bsize. */
+ env->disk_stat.index += bloom_size + page_index_size;
+ if (lsm->index_id > 0)
+ env->disk_stat.index += run->count.bytes;
}
void
vy_lsm_remove_run(struct vy_lsm *lsm, struct vy_run *run)
{
+ struct vy_lsm_env *env = lsm->env;
+ size_t bloom_size = vy_run_bloom_size(run);
+ size_t page_index_size = run->page_index_size;
+
assert(lsm->run_count > 0);
assert(!rlist_empty(&run->in_lsm));
rlist_del_entry(run, in_lsm);
lsm->run_count--;
vy_disk_stmt_counter_sub(&lsm->stat.disk.count, &run->count);
- lsm->bloom_size -= vy_run_bloom_size(run);
- lsm->page_index_size -= run->page_index_size;
+ lsm->bloom_size -= bloom_size;
+ lsm->page_index_size -= page_index_size;
- lsm->env->bloom_size -= vy_run_bloom_size(run);
- lsm->env->page_index_size -= run->page_index_size;
+ env->bloom_size -= bloom_size;
+ env->page_index_size -= page_index_size;
+
+ /* Data size is consistent with space.bsize. */
+ if (lsm->index_id == 0)
+ env->disk_stat.data -= run->count.bytes;
+ /* Index size is consistent with index.bsize. */
+ env->disk_stat.index -= bloom_size + page_index_size;
+ if (lsm->index_id > 0)
+ env->disk_stat.index -= run->count.bytes;
}
void
@@ -737,6 +763,7 @@ vy_lsm_acct_range(struct vy_lsm *lsm, struct vy_range *range)
histogram_collect(lsm->run_hist, range->slice_count);
vy_disk_stmt_counter_add(&lsm->stat.disk.compact.queue,
&range->compact_queue);
+ lsm->env->disk_stat.compact.queue += range->compact_queue.bytes;
}
void
@@ -745,6 +772,7 @@ vy_lsm_unacct_range(struct vy_lsm *lsm, struct vy_range *range)
histogram_discard(lsm->run_hist, range->slice_count);
vy_disk_stmt_counter_sub(&lsm->stat.disk.compact.queue,
&range->compact_queue);
+ lsm->env->disk_stat.compact.queue -= range->compact_queue.bytes;
}
void
@@ -755,6 +783,9 @@ vy_lsm_acct_dump(struct vy_lsm *lsm,
lsm->stat.disk.dump.count++;
vy_stmt_counter_add(&lsm->stat.disk.dump.in, in);
vy_disk_stmt_counter_add(&lsm->stat.disk.dump.out, out);
+
+ lsm->env->disk_stat.dump.in += in->bytes;
+ lsm->env->disk_stat.dump.out += out->bytes;
}
void
@@ -765,6 +796,9 @@ vy_lsm_acct_compaction(struct vy_lsm *lsm,
lsm->stat.disk.compact.count++;
vy_disk_stmt_counter_add(&lsm->stat.disk.compact.in, in);
vy_disk_stmt_counter_add(&lsm->stat.disk.compact.out, out);
+
+ lsm->env->disk_stat.compact.in += in->bytes;
+ lsm->env->disk_stat.compact.out += out->bytes;
}
int
diff --git a/src/box/vy_lsm.h b/src/box/vy_lsm.h
index 19f82e34..2e026ced 100644
--- a/src/box/vy_lsm.h
+++ b/src/box/vy_lsm.h
@@ -91,6 +91,8 @@ struct vy_lsm_env {
size_t bloom_size;
/** Size of memory used for page index. */
size_t page_index_size;
+ /** Global disk statistics. */
+ struct vy_disk_stat disk_stat;
/** Memory pool for vy_history_node allocations. */
struct mempool history_node_pool;
};
diff --git a/src/box/vy_stat.h b/src/box/vy_stat.h
index c094d414..1545115a 100644
--- a/src/box/vy_stat.h
+++ b/src/box/vy_stat.h
@@ -194,6 +194,26 @@ struct vy_tx_stat {
int64_t conflict;
};
+/**
+ * Global disk statistics.
+ *
+ * Fields correspond to those of per LSM tree statistics.
+ * All counters are given in bytes, uncompressed.
+ */
+struct vy_disk_stat {
+ int64_t data;
+ int64_t index;
+ struct {
+ int64_t in;
+ int64_t out;
+ } dump;
+ struct {
+ int64_t in;
+ int64_t out;
+ int64_t queue;
+ } compact;
+};
+
static inline int
vy_lsm_stat_create(struct vy_lsm_stat *stat)
{
diff --git a/test/vinyl/errinj.result b/test/vinyl/errinj.result
index cc2287d2..8badb47a 100644
--- a/test/vinyl/errinj.result
+++ b/test/vinyl/errinj.result
@@ -2147,6 +2147,10 @@ i:stat().disk.compact.queue -- none
rows: 0
bytes: 0
...
+i:stat().disk.compact.queue.bytes == box.stat.vinyl().disk.compact.queue
+---
+- true
+...
errinj.set('ERRINJ_VY_COMPACTION_DELAY', true)
---
- ok
@@ -2161,6 +2165,10 @@ i:stat().disk.compact.queue -- 30 statements
rows: 30
bytes: 471
...
+i:stat().disk.compact.queue.bytes == box.stat.vinyl().disk.compact.queue
+---
+- true
+...
dump()
---
...
@@ -2171,6 +2179,10 @@ i:stat().disk.compact.queue -- 40 statements
rows: 40
bytes: 628
...
+i:stat().disk.compact.queue.bytes == box.stat.vinyl().disk.compact.queue
+---
+- true
+...
dump()
---
...
@@ -2181,6 +2193,10 @@ i:stat().disk.compact.queue -- 50 statements
rows: 50
bytes: 785
...
+i:stat().disk.compact.queue.bytes == box.stat.vinyl().disk.compact.queue
+---
+- true
+...
box.stat.reset() -- doesn't affect queue size
---
...
@@ -2191,6 +2207,10 @@ i:stat().disk.compact.queue -- 50 statements
rows: 50
bytes: 785
...
+i:stat().disk.compact.queue.bytes == box.stat.vinyl().disk.compact.queue
+---
+- true
+...
errinj.set('ERRINJ_VY_COMPACTION_DELAY', false)
---
- ok
diff --git a/test/vinyl/errinj.test.lua b/test/vinyl/errinj.test.lua
index 148662d8..5a47ecc4 100644
--- a/test/vinyl/errinj.test.lua
+++ b/test/vinyl/errinj.test.lua
@@ -862,15 +862,20 @@ function dump() for i = 1, 10 do s:replace{i} end box.snapshot() end
dump()
dump()
i:stat().disk.compact.queue -- none
+i:stat().disk.compact.queue.bytes == box.stat.vinyl().disk.compact.queue
errinj.set('ERRINJ_VY_COMPACTION_DELAY', true)
dump()
i:stat().disk.compact.queue -- 30 statements
+i:stat().disk.compact.queue.bytes == box.stat.vinyl().disk.compact.queue
dump()
i:stat().disk.compact.queue -- 40 statements
+i:stat().disk.compact.queue.bytes == box.stat.vinyl().disk.compact.queue
dump()
i:stat().disk.compact.queue -- 50 statements
+i:stat().disk.compact.queue.bytes == box.stat.vinyl().disk.compact.queue
box.stat.reset() -- doesn't affect queue size
i:stat().disk.compact.queue -- 50 statements
+i:stat().disk.compact.queue.bytes == box.stat.vinyl().disk.compact.queue
errinj.set('ERRINJ_VY_COMPACTION_DELAY', false)
while i:stat().disk.compact.count < 2 do fiber.sleep(0.01) end
i:stat().disk.compact.queue -- none
diff --git a/test/vinyl/info.result b/test/vinyl/info.result
index d13806de..21945b9d 100644
--- a/test/vinyl/info.result
+++ b/test/vinyl/info.result
@@ -221,7 +221,17 @@ istat()
...
gstat()
---
-- quota:
+- disk:
+ dump:
+ in: 0
+ out: 0
+ compact:
+ in: 0
+ queue: 0
+ out: 0
+ data: 0
+ index: 0
+ quota:
limit: 134217728
used: 0
memory:
@@ -686,6 +696,23 @@ box.rollback()
--
-- Global statistics.
--
+-- dump and compaction totals
+gstat().disk.dump['in'] == istat().disk.dump['in'].bytes
+---
+- true
+...
+gstat().disk.dump['out'] == istat().disk.dump['out'].bytes
+---
+- true
+...
+gstat().disk.compact['in'] == istat().disk.compact['in'].bytes
+---
+- true
+...
+gstat().disk.compact['out'] == istat().disk.compact['out'].bytes
+---
+- true
+...
-- use memory
st = gstat()
---
@@ -1038,7 +1065,17 @@ istat()
...
gstat()
---
-- quota:
+- disk:
+ dump:
+ in: 0
+ out: 0
+ compact:
+ in: 0
+ queue: 0
+ out: 0
+ data: 104300
+ index: 1190
+ quota:
limit: 134217728
used: 262583
memory:
@@ -1143,6 +1180,14 @@ gst.memory.bloom_filter == 0
---
- true
...
+gst.disk.data == 0
+---
+- true
+...
+gst.disk.index == 0
+---
+- true
+...
box.snapshot()
---
- ok
@@ -1198,6 +1243,14 @@ gst.memory.bloom_filter == st1.disk.bloom_size + st2.disk.bloom_size
---
- true
...
+gst.disk.data == s:bsize()
+---
+- true
+...
+gst.disk.index == i1:bsize() + i2:bsize()
+---
+- true
+...
for i = 1, 100, 2 do s:delete(i) end
---
...
@@ -1317,6 +1370,14 @@ gst.memory.bloom_filter == st1.disk.bloom_size + st2.disk.bloom_size
---
- true
...
+gst.disk.data == s:bsize()
+---
+- true
+...
+gst.disk.index == i1:bsize() + i2:bsize()
+---
+- true
+...
s:drop()
---
...
@@ -1331,6 +1392,14 @@ gst.memory.bloom_filter == 0
---
- true
...
+gst.disk.data == 0
+---
+- true
+...
+gst.disk.index == 0
+---
+- true
+...
test_run:cmd('switch default')
---
- true
diff --git a/test/vinyl/info.test.lua b/test/vinyl/info.test.lua
index 230bac1e..5912320c 100644
--- a/test/vinyl/info.test.lua
+++ b/test/vinyl/info.test.lua
@@ -206,6 +206,12 @@ box.rollback()
-- Global statistics.
--
+-- dump and compaction totals
+gstat().disk.dump['in'] == istat().disk.dump['in'].bytes
+gstat().disk.dump['out'] == istat().disk.dump['out'].bytes
+gstat().disk.compact['in'] == istat().disk.compact['in'].bytes
+gstat().disk.compact['out'] == istat().disk.compact['out'].bytes
+
-- use memory
st = gstat()
put(1)
@@ -342,6 +348,8 @@ i1:bsize() == st1.memory.index_size
i2:bsize() == st2.memory.index_size
gst.memory.page_index == 0
gst.memory.bloom_filter == 0
+gst.disk.data == 0
+gst.disk.index == 0
box.snapshot()
gst = gstat()
@@ -357,6 +365,8 @@ i1:bsize() == st1.disk.index_size + st1.disk.bloom_size
i2:bsize() == st2.disk.index_size + st2.disk.bloom_size + st2.disk.bytes
gst.memory.page_index == st1.disk.index_size + st2.disk.index_size
gst.memory.bloom_filter == st1.disk.bloom_size + st2.disk.bloom_size
+gst.disk.data == s:bsize()
+gst.disk.index == i1:bsize() + i2:bsize()
for i = 1, 100, 2 do s:delete(i) end
for i = 2, 100, 2 do s:replace{i, i, pad()} end
@@ -392,12 +402,16 @@ i1:bsize() == st1.disk.index_size + st1.disk.bloom_size
i2:bsize() == st2.disk.index_size + st2.disk.bloom_size + st2.disk.bytes
gst.memory.page_index == st1.disk.index_size + st2.disk.index_size
gst.memory.bloom_filter == st1.disk.bloom_size + st2.disk.bloom_size
+gst.disk.data == s:bsize()
+gst.disk.index == i1:bsize() + i2:bsize()
s:drop()
gst = gstat()
gst.memory.page_index == 0
gst.memory.bloom_filter == 0
+gst.disk.data == 0
+gst.disk.index == 0
test_run:cmd('switch default')
test_run:cmd('stop server test')
--
2.11.0
More information about the Tarantool-patches
mailing list