From: Vladimir Davydov <vdavydov.dev@gmail.com>
To: kostja@tarantool.org
Cc: tarantool-patches@freelists.org
Subject: [PATCH v2 8/8] vinyl: add global disk stats
Date: Sun, 16 Sep 2018 20:06:51 +0300 [thread overview]
Message-ID: <39398eb383490fe833a1d54461d7f51b816e4e3f.1537115208.git.vdavydov.dev@gmail.com> (raw)
In-Reply-To: <cover.1537115208.git.vdavydov.dev@gmail.com>
In-Reply-To: <cover.1537115208.git.vdavydov.dev@gmail.com>
This patch adds some essential disk statistics that are already
collected and reported on per index basis to box.stat.vinyl().
The new statistics are shown under the 'disk' section and currently
include the following fields:
- data: size of data stored on disk.
- index: size of index stored on disk.
- dump.in: size of dump input.
- dump.out: size of dump output.
- compact.in: size of compaction input.
- compact.out: size of compaction output.
- compact.queue: size of compaction queue.
All the counters are given in bytes without taking into account
disk compression. Dump/compaction in/out counters can be reset with
box.stat.reset().
---
src/box/vinyl.c | 33 ++++++++++++++++++++-
src/box/vy_lsm.c | 50 ++++++++++++++++++++++++++-----
src/box/vy_lsm.h | 2 ++
src/box/vy_stat.h | 20 +++++++++++++
test/vinyl/errinj.result | 20 +++++++++++++
test/vinyl/errinj.test.lua | 5 ++++
test/vinyl/info.result | 73 ++++++++++++++++++++++++++++++++++++++++++++--
test/vinyl/info.test.lua | 14 +++++++++
8 files changed, 206 insertions(+), 11 deletions(-)
diff --git a/src/box/vinyl.c b/src/box/vinyl.c
index 02a2b69d..b5904872 100644
--- a/src/box/vinyl.c
+++ b/src/box/vinyl.c
@@ -298,6 +298,30 @@ vy_info_append_memory(struct vy_env *env, struct info_handler *h)
info_table_end(h); /* memory */
}
+static void
+vy_info_append_disk(struct vy_env *env, struct info_handler *h)
+{
+ struct vy_disk_stat *stat = &env->lsm_env.disk_stat;
+
+ info_table_begin(h, "disk");
+
+ info_append_int(h, "data", stat->data);
+ info_append_int(h, "index", stat->index);
+
+ info_table_begin(h, "dump");
+ info_append_int(h, "in", stat->dump.in);
+ info_append_int(h, "out", stat->dump.out);
+ info_table_end(h); /* dump */
+
+ info_table_begin(h, "compact");
+ info_append_int(h, "in", stat->compact.in);
+ info_append_int(h, "out", stat->compact.out);
+ info_append_int(h, "queue", stat->compact.queue);
+ info_table_end(h); /* compact */
+
+ info_table_end(h); /* disk */
+}
+
void
vinyl_engine_stat(struct vinyl_engine *vinyl, struct info_handler *h)
{
@@ -307,6 +331,7 @@ vinyl_engine_stat(struct vinyl_engine *vinyl, struct info_handler *h)
vy_info_append_quota(env, h);
vy_info_append_tx(env, h);
vy_info_append_memory(env, h);
+ vy_info_append_disk(env, h);
info_end(h);
}
@@ -485,9 +510,15 @@ static void
vinyl_engine_reset_stat(struct engine *engine)
{
struct vy_env *env = vy_env(engine);
- struct tx_manager *xm = env->xm;
+ struct tx_manager *xm = env->xm;
memset(&xm->stat, 0, sizeof(xm->stat));
+
+ struct vy_disk_stat *disk_stat = &env->lsm_env.disk_stat;
+ disk_stat->dump.in = 0;
+ disk_stat->dump.out = 0;
+ disk_stat->compact.in = 0;
+ disk_stat->compact.out = 0;
}
/** }}} Introspection */
diff --git a/src/box/vy_lsm.c b/src/box/vy_lsm.c
index dbea2898..093bb820 100644
--- a/src/box/vy_lsm.c
+++ b/src/box/vy_lsm.c
@@ -252,6 +252,8 @@ vy_lsm_delete(struct vy_lsm *lsm)
assert(lsm->env->lsm_count > 0);
lsm->env->lsm_count--;
+ lsm->env->disk_stat.compact.queue -=
+ lsm->stat.disk.compact.queue.bytes;
if (lsm->pk != NULL)
vy_lsm_unref(lsm->pk);
@@ -685,32 +687,56 @@ vy_lsm_compact_priority(struct vy_lsm *lsm)
void
vy_lsm_add_run(struct vy_lsm *lsm, struct vy_run *run)
{
+ struct vy_lsm_env *env = lsm->env;
+ size_t bloom_size = vy_run_bloom_size(run);
+ size_t page_index_size = run->page_index_size;
+
assert(rlist_empty(&run->in_lsm));
rlist_add_entry(&lsm->runs, run, in_lsm);
lsm->run_count++;
vy_disk_stmt_counter_add(&lsm->stat.disk.count, &run->count);
- lsm->bloom_size += vy_run_bloom_size(run);
- lsm->page_index_size += run->page_index_size;
+ lsm->bloom_size += bloom_size;
+ lsm->page_index_size += page_index_size;
+
+ env->bloom_size += bloom_size;
+ env->page_index_size += page_index_size;
- lsm->env->bloom_size += vy_run_bloom_size(run);
- lsm->env->page_index_size += run->page_index_size;
+ /* Data size is consistent with space.bsize. */
+ if (lsm->index_id == 0)
+ env->disk_stat.data += run->count.bytes;
+ /* Index size is consistent with index.bsize. */
+ env->disk_stat.index += bloom_size + page_index_size;
+ if (lsm->index_id > 0)
+ env->disk_stat.index += run->count.bytes;
}
void
vy_lsm_remove_run(struct vy_lsm *lsm, struct vy_run *run)
{
+ struct vy_lsm_env *env = lsm->env;
+ size_t bloom_size = vy_run_bloom_size(run);
+ size_t page_index_size = run->page_index_size;
+
assert(lsm->run_count > 0);
assert(!rlist_empty(&run->in_lsm));
rlist_del_entry(run, in_lsm);
lsm->run_count--;
vy_disk_stmt_counter_sub(&lsm->stat.disk.count, &run->count);
- lsm->bloom_size -= vy_run_bloom_size(run);
- lsm->page_index_size -= run->page_index_size;
+ lsm->bloom_size -= bloom_size;
+ lsm->page_index_size -= page_index_size;
- lsm->env->bloom_size -= vy_run_bloom_size(run);
- lsm->env->page_index_size -= run->page_index_size;
+ env->bloom_size -= bloom_size;
+ env->page_index_size -= page_index_size;
+
+ /* Data size is consistent with space.bsize. */
+ if (lsm->index_id == 0)
+ env->disk_stat.data -= run->count.bytes;
+ /* Index size is consistent with index.bsize. */
+ env->disk_stat.index -= bloom_size + page_index_size;
+ if (lsm->index_id > 0)
+ env->disk_stat.index -= run->count.bytes;
}
void
@@ -737,6 +763,7 @@ vy_lsm_acct_range(struct vy_lsm *lsm, struct vy_range *range)
histogram_collect(lsm->run_hist, range->slice_count);
vy_disk_stmt_counter_add(&lsm->stat.disk.compact.queue,
&range->compact_queue);
+ lsm->env->disk_stat.compact.queue += range->compact_queue.bytes;
}
void
@@ -745,6 +772,7 @@ vy_lsm_unacct_range(struct vy_lsm *lsm, struct vy_range *range)
histogram_discard(lsm->run_hist, range->slice_count);
vy_disk_stmt_counter_sub(&lsm->stat.disk.compact.queue,
&range->compact_queue);
+ lsm->env->disk_stat.compact.queue -= range->compact_queue.bytes;
}
void
@@ -755,6 +783,9 @@ vy_lsm_acct_dump(struct vy_lsm *lsm,
lsm->stat.disk.dump.count++;
vy_stmt_counter_add(&lsm->stat.disk.dump.in, in);
vy_disk_stmt_counter_add(&lsm->stat.disk.dump.out, out);
+
+ lsm->env->disk_stat.dump.in += in->bytes;
+ lsm->env->disk_stat.dump.out += out->bytes;
}
void
@@ -765,6 +796,9 @@ vy_lsm_acct_compaction(struct vy_lsm *lsm,
lsm->stat.disk.compact.count++;
vy_disk_stmt_counter_add(&lsm->stat.disk.compact.in, in);
vy_disk_stmt_counter_add(&lsm->stat.disk.compact.out, out);
+
+ lsm->env->disk_stat.compact.in += in->bytes;
+ lsm->env->disk_stat.compact.out += out->bytes;
}
int
diff --git a/src/box/vy_lsm.h b/src/box/vy_lsm.h
index 19f82e34..2e026ced 100644
--- a/src/box/vy_lsm.h
+++ b/src/box/vy_lsm.h
@@ -91,6 +91,8 @@ struct vy_lsm_env {
size_t bloom_size;
/** Size of memory used for page index. */
size_t page_index_size;
+ /** Global disk statistics. */
+ struct vy_disk_stat disk_stat;
/** Memory pool for vy_history_node allocations. */
struct mempool history_node_pool;
};
diff --git a/src/box/vy_stat.h b/src/box/vy_stat.h
index c094d414..1545115a 100644
--- a/src/box/vy_stat.h
+++ b/src/box/vy_stat.h
@@ -194,6 +194,26 @@ struct vy_tx_stat {
int64_t conflict;
};
+/**
+ * Global disk statistics.
+ *
+ * Fields correspond to those of per LSM tree statistics.
+ * All counters are given in bytes, uncompressed.
+ */
+struct vy_disk_stat {
+ int64_t data;
+ int64_t index;
+ struct {
+ int64_t in;
+ int64_t out;
+ } dump;
+ struct {
+ int64_t in;
+ int64_t out;
+ int64_t queue;
+ } compact;
+};
+
static inline int
vy_lsm_stat_create(struct vy_lsm_stat *stat)
{
diff --git a/test/vinyl/errinj.result b/test/vinyl/errinj.result
index cc2287d2..8badb47a 100644
--- a/test/vinyl/errinj.result
+++ b/test/vinyl/errinj.result
@@ -2147,6 +2147,10 @@ i:stat().disk.compact.queue -- none
rows: 0
bytes: 0
...
+i:stat().disk.compact.queue.bytes == box.stat.vinyl().disk.compact.queue
+---
+- true
+...
errinj.set('ERRINJ_VY_COMPACTION_DELAY', true)
---
- ok
@@ -2161,6 +2165,10 @@ i:stat().disk.compact.queue -- 30 statements
rows: 30
bytes: 471
...
+i:stat().disk.compact.queue.bytes == box.stat.vinyl().disk.compact.queue
+---
+- true
+...
dump()
---
...
@@ -2171,6 +2179,10 @@ i:stat().disk.compact.queue -- 40 statements
rows: 40
bytes: 628
...
+i:stat().disk.compact.queue.bytes == box.stat.vinyl().disk.compact.queue
+---
+- true
+...
dump()
---
...
@@ -2181,6 +2193,10 @@ i:stat().disk.compact.queue -- 50 statements
rows: 50
bytes: 785
...
+i:stat().disk.compact.queue.bytes == box.stat.vinyl().disk.compact.queue
+---
+- true
+...
box.stat.reset() -- doesn't affect queue size
---
...
@@ -2191,6 +2207,10 @@ i:stat().disk.compact.queue -- 50 statements
rows: 50
bytes: 785
...
+i:stat().disk.compact.queue.bytes == box.stat.vinyl().disk.compact.queue
+---
+- true
+...
errinj.set('ERRINJ_VY_COMPACTION_DELAY', false)
---
- ok
diff --git a/test/vinyl/errinj.test.lua b/test/vinyl/errinj.test.lua
index 148662d8..5a47ecc4 100644
--- a/test/vinyl/errinj.test.lua
+++ b/test/vinyl/errinj.test.lua
@@ -862,15 +862,20 @@ function dump() for i = 1, 10 do s:replace{i} end box.snapshot() end
dump()
dump()
i:stat().disk.compact.queue -- none
+i:stat().disk.compact.queue.bytes == box.stat.vinyl().disk.compact.queue
errinj.set('ERRINJ_VY_COMPACTION_DELAY', true)
dump()
i:stat().disk.compact.queue -- 30 statements
+i:stat().disk.compact.queue.bytes == box.stat.vinyl().disk.compact.queue
dump()
i:stat().disk.compact.queue -- 40 statements
+i:stat().disk.compact.queue.bytes == box.stat.vinyl().disk.compact.queue
dump()
i:stat().disk.compact.queue -- 50 statements
+i:stat().disk.compact.queue.bytes == box.stat.vinyl().disk.compact.queue
box.stat.reset() -- doesn't affect queue size
i:stat().disk.compact.queue -- 50 statements
+i:stat().disk.compact.queue.bytes == box.stat.vinyl().disk.compact.queue
errinj.set('ERRINJ_VY_COMPACTION_DELAY', false)
while i:stat().disk.compact.count < 2 do fiber.sleep(0.01) end
i:stat().disk.compact.queue -- none
diff --git a/test/vinyl/info.result b/test/vinyl/info.result
index d13806de..21945b9d 100644
--- a/test/vinyl/info.result
+++ b/test/vinyl/info.result
@@ -221,7 +221,17 @@ istat()
...
gstat()
---
-- quota:
+- disk:
+ dump:
+ in: 0
+ out: 0
+ compact:
+ in: 0
+ queue: 0
+ out: 0
+ data: 0
+ index: 0
+ quota:
limit: 134217728
used: 0
memory:
@@ -686,6 +696,23 @@ box.rollback()
--
-- Global statistics.
--
+-- dump and compaction totals
+gstat().disk.dump['in'] == istat().disk.dump['in'].bytes
+---
+- true
+...
+gstat().disk.dump['out'] == istat().disk.dump['out'].bytes
+---
+- true
+...
+gstat().disk.compact['in'] == istat().disk.compact['in'].bytes
+---
+- true
+...
+gstat().disk.compact['out'] == istat().disk.compact['out'].bytes
+---
+- true
+...
-- use memory
st = gstat()
---
@@ -1038,7 +1065,17 @@ istat()
...
gstat()
---
-- quota:
+- disk:
+ dump:
+ in: 0
+ out: 0
+ compact:
+ in: 0
+ queue: 0
+ out: 0
+ data: 104300
+ index: 1190
+ quota:
limit: 134217728
used: 262583
memory:
@@ -1143,6 +1180,14 @@ gst.memory.bloom_filter == 0
---
- true
...
+gst.disk.data == 0
+---
+- true
+...
+gst.disk.index == 0
+---
+- true
+...
box.snapshot()
---
- ok
@@ -1198,6 +1243,14 @@ gst.memory.bloom_filter == st1.disk.bloom_size + st2.disk.bloom_size
---
- true
...
+gst.disk.data == s:bsize()
+---
+- true
+...
+gst.disk.index == i1:bsize() + i2:bsize()
+---
+- true
+...
for i = 1, 100, 2 do s:delete(i) end
---
...
@@ -1317,6 +1370,14 @@ gst.memory.bloom_filter == st1.disk.bloom_size + st2.disk.bloom_size
---
- true
...
+gst.disk.data == s:bsize()
+---
+- true
+...
+gst.disk.index == i1:bsize() + i2:bsize()
+---
+- true
+...
s:drop()
---
...
@@ -1331,6 +1392,14 @@ gst.memory.bloom_filter == 0
---
- true
...
+gst.disk.data == 0
+---
+- true
+...
+gst.disk.index == 0
+---
+- true
+...
test_run:cmd('switch default')
---
- true
diff --git a/test/vinyl/info.test.lua b/test/vinyl/info.test.lua
index 230bac1e..5912320c 100644
--- a/test/vinyl/info.test.lua
+++ b/test/vinyl/info.test.lua
@@ -206,6 +206,12 @@ box.rollback()
-- Global statistics.
--
+-- dump and compaction totals
+gstat().disk.dump['in'] == istat().disk.dump['in'].bytes
+gstat().disk.dump['out'] == istat().disk.dump['out'].bytes
+gstat().disk.compact['in'] == istat().disk.compact['in'].bytes
+gstat().disk.compact['out'] == istat().disk.compact['out'].bytes
+
-- use memory
st = gstat()
put(1)
@@ -342,6 +348,8 @@ i1:bsize() == st1.memory.index_size
i2:bsize() == st2.memory.index_size
gst.memory.page_index == 0
gst.memory.bloom_filter == 0
+gst.disk.data == 0
+gst.disk.index == 0
box.snapshot()
gst = gstat()
@@ -357,6 +365,8 @@ i1:bsize() == st1.disk.index_size + st1.disk.bloom_size
i2:bsize() == st2.disk.index_size + st2.disk.bloom_size + st2.disk.bytes
gst.memory.page_index == st1.disk.index_size + st2.disk.index_size
gst.memory.bloom_filter == st1.disk.bloom_size + st2.disk.bloom_size
+gst.disk.data == s:bsize()
+gst.disk.index == i1:bsize() + i2:bsize()
for i = 1, 100, 2 do s:delete(i) end
for i = 2, 100, 2 do s:replace{i, i, pad()} end
@@ -392,12 +402,16 @@ i1:bsize() == st1.disk.index_size + st1.disk.bloom_size
i2:bsize() == st2.disk.index_size + st2.disk.bloom_size + st2.disk.bytes
gst.memory.page_index == st1.disk.index_size + st2.disk.index_size
gst.memory.bloom_filter == st1.disk.bloom_size + st2.disk.bloom_size
+gst.disk.data == s:bsize()
+gst.disk.index == i1:bsize() + i2:bsize()
s:drop()
gst = gstat()
gst.memory.page_index == 0
gst.memory.bloom_filter == 0
+gst.disk.data == 0
+gst.disk.index == 0
test_run:cmd('switch default')
test_run:cmd('stop server test')
--
2.11.0
next prev parent reply other threads:[~2018-09-16 17:06 UTC|newest]
Thread overview: 18+ messages / expand[flat|nested] mbox.gz Atom feed top
2018-09-16 17:06 [PATCH v2 0/8] vinyl: improve stats for throttling Vladimir Davydov
2018-09-16 17:06 ` [PATCH v2 1/8] vinyl: fix force compaction logic Vladimir Davydov
2018-09-19 1:43 ` Konstantin Osipov
2018-09-16 17:06 ` [PATCH v2 2/8] vinyl: update compact priority usual way on range split/coalesce Vladimir Davydov
2018-09-19 1:46 ` Konstantin Osipov
2018-09-16 17:06 ` [PATCH v2 3/8] vinyl: annotate info_table_end with comment Vladimir Davydov
2018-09-19 1:47 ` Konstantin Osipov
2018-09-16 17:06 ` [PATCH v2 4/8] vinyl: report pages and bytes_compressed in dump/compact in/out stats Vladimir Davydov
2018-09-19 1:48 ` Konstantin Osipov
2018-09-16 17:06 ` [PATCH v2 5/8] vinyl: add helpers for resetting statement counters Vladimir Davydov
2018-09-19 1:49 ` Konstantin Osipov
2018-09-16 17:06 ` [PATCH v2 6/8] vinyl: keep track of compaction queue length Vladimir Davydov
2018-09-19 1:53 ` Konstantin Osipov
2018-09-16 17:06 ` [PATCH v2 7/8] vinyl: factor out helpers for accounting dump/compaction Vladimir Davydov
2018-09-19 1:53 ` Konstantin Osipov
2018-09-16 17:06 ` Vladimir Davydov [this message]
2018-09-19 1:56 ` [PATCH v2 8/8] vinyl: add global disk stats Konstantin Osipov
2018-09-19 9:59 ` [PATCH v2 0/8] vinyl: improve stats for throttling Vladimir Davydov
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=39398eb383490fe833a1d54461d7f51b816e4e3f.1537115208.git.vdavydov.dev@gmail.com \
--to=vdavydov.dev@gmail.com \
--cc=kostja@tarantool.org \
--cc=tarantool-patches@freelists.org \
--subject='Re: [PATCH v2 8/8] vinyl: add global disk stats' \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox