From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: From: Vladimir Davydov Subject: [PATCH] box: make gc info public Date: Thu, 31 May 2018 16:29:03 +0300 Message-Id: To: kostja@tarantool.org Cc: tarantool-patches@freelists.org List-ID: Information about garbage collection internal state is quite useful when it comes to answering the question why my xlog files are not removed so let's move it from box.inernal.gc.info() to box.info.gc. --- This patch is based on top of 1.10 https://github.com/tarantool/tarantool/commits/box-make-gc-info-public src/box/lua/checkpoint_daemon.lua | 2 +- src/box/lua/info.c | 55 ++++++++++++++++++++++++++++++++++ src/box/lua/init.c | 62 --------------------------------------- test/box/info.result | 1 + test/replication/gc.result | 16 +++++----- test/replication/gc.test.lua | 16 +++++----- test/vinyl/gc.result | 2 +- test/vinyl/gc.test.lua | 2 +- 8 files changed, 75 insertions(+), 81 deletions(-) diff --git a/src/box/lua/checkpoint_daemon.lua b/src/box/lua/checkpoint_daemon.lua index e7ef05fd..939dfa59 100644 --- a/src/box/lua/checkpoint_daemon.lua +++ b/src/box/lua/checkpoint_daemon.lua @@ -42,7 +42,7 @@ local function process(self) return false end - local checkpoints = box.internal.gc.info().checkpoints + local checkpoints = box.info.gc.checkpoints local last_checkpoint = checkpoints[#checkpoints] local last_snap = fio.pathjoin(box.cfg.memtx_dir, diff --git a/src/box/lua/info.c b/src/box/lua/info.c index 7dc2b83a..d836bb8e 100644 --- a/src/box/lua/info.c +++ b/src/box/lua/info.c @@ -46,6 +46,8 @@ #include "box/wal.h" #include "box/replication.h" #include "box/info.h" +#include "box/gc.h" +#include "box/checkpoint.h" #include "box/engine.h" #include "box/vinyl.h" #include "main.h" @@ -352,6 +354,58 @@ lbox_info_memory(struct lua_State *L) return 1; } +static int +lbox_info_gc(struct lua_State *L) +{ + int count; + const struct vclock *vclock; + + lua_newtable(L); + + lua_pushstring(L, "checkpoints"); + lua_newtable(L); + + struct checkpoint_iterator checkpoints; + checkpoint_iterator_init(&checkpoints); + + count = 0; + while ((vclock = checkpoint_iterator_next(&checkpoints)) != NULL) { + lua_createtable(L, 0, 1); + + lua_pushstring(L, "signature"); + luaL_pushint64(L, vclock_sum(vclock)); + lua_settable(L, -3); + + lua_rawseti(L, -2, ++count); + } + lua_settable(L, -3); + + lua_pushstring(L, "consumers"); + lua_newtable(L); + + struct gc_consumer_iterator consumers; + gc_consumer_iterator_init(&consumers); + + count = 0; + struct gc_consumer *consumer; + while ((consumer = gc_consumer_iterator_next(&consumers)) != NULL) { + lua_createtable(L, 0, 2); + + lua_pushstring(L, "name"); + lua_pushstring(L, gc_consumer_name(consumer)); + lua_settable(L, -3); + + lua_pushstring(L, "signature"); + luaL_pushint64(L, gc_consumer_signature(consumer)); + lua_settable(L, -3); + + lua_rawseti(L, -2, ++count); + } + lua_settable(L, -3); + + return 1; +} + static void luaT_info_begin(struct info_handler *info) { @@ -467,6 +521,7 @@ static const struct luaL_Reg lbox_info_dynamic_meta[] = { {"pid", lbox_info_pid}, {"cluster", lbox_info_cluster}, {"memory", lbox_info_memory}, + {"gc", lbox_info_gc}, {"vinyl", lbox_info_vinyl}, {NULL, NULL} }; diff --git a/src/box/lua/init.c b/src/box/lua/init.c index d4b5788f..23b83a83 100644 --- a/src/box/lua/init.c +++ b/src/box/lua/init.c @@ -38,8 +38,6 @@ #include "box/box.h" #include "box/txn.h" -#include "box/gc.h" -#include "box/checkpoint.h" #include "box/vclock.h" #include "box/lua/error.h" @@ -112,58 +110,6 @@ lbox_snapshot(struct lua_State *L) return luaT_error(L); } -static int -lbox_gc_info(struct lua_State *L) -{ - int count; - const struct vclock *vclock; - - lua_newtable(L); - - lua_pushstring(L, "checkpoints"); - lua_newtable(L); - - struct checkpoint_iterator checkpoints; - checkpoint_iterator_init(&checkpoints); - - count = 0; - while ((vclock = checkpoint_iterator_next(&checkpoints)) != NULL) { - lua_createtable(L, 0, 1); - - lua_pushstring(L, "signature"); - luaL_pushint64(L, vclock_sum(vclock)); - lua_settable(L, -3); - - lua_rawseti(L, -2, ++count); - } - lua_settable(L, -3); - - lua_pushstring(L, "consumers"); - lua_newtable(L); - - struct gc_consumer_iterator consumers; - gc_consumer_iterator_init(&consumers); - - count = 0; - struct gc_consumer *consumer; - while ((consumer = gc_consumer_iterator_next(&consumers)) != NULL) { - lua_createtable(L, 0, 2); - - lua_pushstring(L, "name"); - lua_pushstring(L, gc_consumer_name(consumer)); - lua_settable(L, -3); - - lua_pushstring(L, "signature"); - luaL_pushint64(L, gc_consumer_signature(consumer)); - lua_settable(L, -3); - - lua_rawseti(L, -2, ++count); - } - lua_settable(L, -3); - - return 1; -} - /** Argument passed to lbox_backup_fn(). */ struct lbox_backup_arg { /** Lua state. */ @@ -209,11 +155,6 @@ static const struct luaL_Reg boxlib[] = { {NULL, NULL} }; -static const struct luaL_Reg boxlib_gc[] = { - {"info", lbox_gc_info}, - {NULL, NULL} -}; - static const struct luaL_Reg boxlib_backup[] = { {"start", lbox_backup_start}, {"stop", lbox_backup_stop}, @@ -229,9 +170,6 @@ box_lua_init(struct lua_State *L) luaL_register(L, "box", boxlib); lua_pop(L, 1); - luaL_register(L, "box.internal.gc", boxlib_gc); - lua_pop(L, 1); - luaL_register(L, "box.backup", boxlib_backup); lua_pop(L, 1); diff --git a/test/box/info.result b/test/box/info.result index 88ec15bc..33846315 100644 --- a/test/box/info.result +++ b/test/box/info.result @@ -75,6 +75,7 @@ table.sort(t) t --- - - cluster + - gc - id - lsn - memory diff --git a/test/replication/gc.result b/test/replication/gc.result index fec9d819..61540921 100644 --- a/test/replication/gc.result +++ b/test/replication/gc.result @@ -20,7 +20,7 @@ default_checkpoint_count = box.cfg.checkpoint_count box.cfg{checkpoint_count = 1} --- ... -function wait_gc(n) while #box.internal.gc.info().checkpoints > n do fiber.sleep(0.01) end end +function wait_gc(n) while #box.info.gc.checkpoints > n do fiber.sleep(0.01) end end --- ... -- Grant permissions needed for replication. @@ -111,7 +111,7 @@ test_run:cmd("switch default") wait_gc(1) --- ... -#box.internal.gc.info().checkpoints == 1 or box.internal.gc.info() +#box.info.gc.checkpoints == 1 or box.info.gc --- - true ... @@ -131,7 +131,7 @@ box.snapshot() --- - ok ... -#box.internal.gc.info().checkpoints == 2 or box.internal.gc.info() +#box.info.gc.checkpoints == 2 or box.info.gc --- - true ... @@ -162,7 +162,7 @@ test_run:cmd("switch default") wait_gc(1) --- ... -#box.internal.gc.info().checkpoints == 1 or box.internal.gc.info() +#box.info.gc.checkpoints == 1 or box.info.gc --- - true ... @@ -201,7 +201,7 @@ fiber.sleep(0.1) -- wait for master to relay data -- Garbage collection must not delete the old xlog file -- (and the corresponding snapshot), because it is still -- needed by the replica. -#box.internal.gc.info().checkpoints == 2 or box.internal.gc.info() +#box.info.gc.checkpoints == 2 or box.info.gc --- - true ... @@ -261,7 +261,7 @@ test_run:cmd("switch default") wait_gc(1) --- ... -#box.internal.gc.info().checkpoints == 1 or box.internal.gc.info() +#box.info.gc.checkpoints == 1 or box.info.gc --- - true ... @@ -283,7 +283,7 @@ box.snapshot() --- - ok ... -#box.internal.gc.info().checkpoints == 2 or box.internal.gc.info() +#box.info.gc.checkpoints == 2 or box.info.gc --- - true ... @@ -292,7 +292,7 @@ box.snapshot() test_run:cleanup_cluster() --- ... -#box.internal.gc.info().checkpoints == 1 or box.internal.gc.info() +#box.info.gc.checkpoints == 1 or box.info.gc --- - true ... diff --git a/test/replication/gc.test.lua b/test/replication/gc.test.lua index 8d5a36dd..a031eae0 100644 --- a/test/replication/gc.test.lua +++ b/test/replication/gc.test.lua @@ -9,7 +9,7 @@ test_run:cleanup_cluster() default_checkpoint_count = box.cfg.checkpoint_count box.cfg{checkpoint_count = 1} -function wait_gc(n) while #box.internal.gc.info().checkpoints > n do fiber.sleep(0.01) end end +function wait_gc(n) while #box.info.gc.checkpoints > n do fiber.sleep(0.01) end end -- Grant permissions needed for replication. box.schema.user.grant('guest', 'read,write,execute', 'universe') @@ -59,7 +59,7 @@ test_run:cmd("switch default") -- Check that garbage collection removed the snapshot once -- the replica released the corresponding checkpoint. wait_gc(1) -#box.internal.gc.info().checkpoints == 1 or box.internal.gc.info() +#box.info.gc.checkpoints == 1 or box.info.gc -- Make sure the replica will receive data it is subscribed -- to long enough for us to invoke garbage collection. @@ -71,7 +71,7 @@ for i = 1, 100 do s:auto_increment{} end -- Invoke garbage collection. Check that it doesn't remove -- xlogs needed by the replica. box.snapshot() -#box.internal.gc.info().checkpoints == 2 or box.internal.gc.info() +#box.info.gc.checkpoints == 2 or box.info.gc -- Remove the timeout injection so that the replica catches -- up quickly. @@ -86,7 +86,7 @@ test_run:cmd("switch default") -- Now garbage collection should resume and delete files left -- from the old checkpoint. wait_gc(1) -#box.internal.gc.info().checkpoints == 1 or box.internal.gc.info() +#box.info.gc.checkpoints == 1 or box.info.gc -- -- Check that the master doesn't delete xlog files sent to the @@ -105,7 +105,7 @@ fiber.sleep(0.1) -- wait for master to relay data -- Garbage collection must not delete the old xlog file -- (and the corresponding snapshot), because it is still -- needed by the replica. -#box.internal.gc.info().checkpoints == 2 or box.internal.gc.info() +#box.info.gc.checkpoints == 2 or box.info.gc test_run:cmd("switch replica") -- Unblock the replica and make it fail to apply a row. box.info.replication[1].upstream.message == nil @@ -124,7 +124,7 @@ box.space.test:count() test_run:cmd("switch default") -- Now it's safe to drop the old xlog. wait_gc(1) -#box.internal.gc.info().checkpoints == 1 or box.internal.gc.info() +#box.info.gc.checkpoints == 1 or box.info.gc -- Stop the replica. test_run:cmd("stop server replica") @@ -134,12 +134,12 @@ test_run:cmd("cleanup server replica") -- the checkpoint last used by the replica. _ = s:auto_increment{} box.snapshot() -#box.internal.gc.info().checkpoints == 2 or box.internal.gc.info() +#box.info.gc.checkpoints == 2 or box.info.gc -- The checkpoint should only be deleted after the replica -- is unregistered. test_run:cleanup_cluster() -#box.internal.gc.info().checkpoints == 1 or box.internal.gc.info() +#box.info.gc.checkpoints == 1 or box.info.gc -- -- Test that concurrent invocation of the garbage collector works fine. diff --git a/test/vinyl/gc.result b/test/vinyl/gc.result index bab6f31b..80bec609 100644 --- a/test/vinyl/gc.result +++ b/test/vinyl/gc.result @@ -39,7 +39,7 @@ function ls_data() return fio.glob(fio.pathjoin(path, '*')) end function ls_vylog() return fio.glob(fio.pathjoin(box.cfg.vinyl_dir, '*.vylog')) end --- ... -function gc_info() return box.internal.gc.info() end +function gc_info() return box.info.gc end --- ... function gc() temp:auto_increment{} box.snapshot() end diff --git a/test/vinyl/gc.test.lua b/test/vinyl/gc.test.lua index eb56a1c6..bde9e369 100644 --- a/test/vinyl/gc.test.lua +++ b/test/vinyl/gc.test.lua @@ -19,7 +19,7 @@ path = fio.pathjoin(box.cfg.vinyl_dir, tostring(s.id), tostring(s.index.pk.id)) function ls_data() return fio.glob(fio.pathjoin(path, '*')) end function ls_vylog() return fio.glob(fio.pathjoin(box.cfg.vinyl_dir, '*.vylog')) end -function gc_info() return box.internal.gc.info() end +function gc_info() return box.info.gc end function gc() temp:auto_increment{} box.snapshot() end -- Check that run files are deleted by gc. -- 2.11.0