From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from smtp54.i.mail.ru (smtp54.i.mail.ru [217.69.128.34]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by dev.tarantool.org (Postfix) with ESMTPS id 2B20144643B for ; Mon, 5 Oct 2020 09:30:42 +0300 (MSK) From: Sergey Kaplun Date: Mon, 5 Oct 2020 09:30:11 +0300 Message-Id: <87bf84f266e16cd9401394c8276b1cd97f91a82d.1601878708.git.skaplun@tarantool.org> In-Reply-To: References: MIME-Version: 1.0 Content-Transfer-Encoding: 8bit Subject: [Tarantool-patches] [PATCH v4 2/2] misc: add C and Lua API for platform metrics List-Id: Tarantool development patches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , To: Igor Munkin , Sergey Ostanevich Cc: tarantool-patches@dev.tarantool.org This patch introduces both C and Lua API for LuaJIT platform metrics implemented in scope of the previous patch. New header provides C interface that fills the given structure with the platform metrics. Additionally module is loaded to Lua space and provides method that yields the corresponding metrics via table. Part of tarantool/tarantool#5187 --- Makefile | 2 +- src/Makefile | 5 +- src/Makefile.dep | 14 +- src/lib_init.c | 2 + src/lib_misc.c | 72 +++ src/lj_mapi.c | 61 +++ src/ljamalg.c | 2 + src/lmisclib.h | 64 +++ src/luaconf.h | 1 + test/clib-misclib-getmetrics.test.lua | 174 ++++++++ test/clib-misclib-getmetrics/CMakeLists.txt | 1 + test/clib-misclib-getmetrics/testgetmetrics.c | 242 ++++++++++ test/lib-misc-getmetrics.test.lua | 418 ++++++++++++++++++ 13 files changed, 1050 insertions(+), 8 deletions(-) create mode 100644 src/lib_misc.c create mode 100644 src/lj_mapi.c create mode 100644 src/lmisclib.h create mode 100755 test/clib-misclib-getmetrics.test.lua create mode 100644 test/clib-misclib-getmetrics/CMakeLists.txt create mode 100644 test/clib-misclib-getmetrics/testgetmetrics.c create mode 100755 test/lib-misc-getmetrics.test.lua diff --git a/Makefile b/Makefile index 0f93308..4a56917 100644 --- a/Makefile +++ b/Makefile @@ -84,7 +84,7 @@ FILE_A= libluajit.a FILE_SO= libluajit.so FILE_MAN= luajit.1 FILE_PC= luajit.pc -FILES_INC= lua.h lualib.h lauxlib.h luaconf.h lua.hpp luajit.h +FILES_INC= lua.h lualib.h lauxlib.h luaconf.h lua.hpp luajit.h lmisclib.h FILES_JITLIB= bc.lua bcsave.lua dump.lua p.lua v.lua zone.lua \ dis_x86.lua dis_x64.lua dis_arm.lua dis_arm64.lua \ dis_arm64be.lua dis_ppc.lua dis_mips.lua dis_mipsel.lua \ diff --git a/src/Makefile b/src/Makefile index 827d4a4..2786348 100644 --- a/src/Makefile +++ b/src/Makefile @@ -480,13 +480,14 @@ LJVM_BOUT= $(LJVM_S) LJVM_MODE= elfasm LJLIB_O= lib_base.o lib_math.o lib_bit.o lib_string.o lib_table.o \ - lib_io.o lib_os.o lib_package.o lib_debug.o lib_jit.o lib_ffi.o + lib_io.o lib_os.o lib_package.o lib_debug.o lib_jit.o lib_ffi.o \ + lib_misc.o LJLIB_C= $(LJLIB_O:.o=.c) LJCORE_O= lj_gc.o lj_err.o lj_char.o lj_bc.o lj_obj.o lj_buf.o \ lj_str.o lj_tab.o lj_func.o lj_udata.o lj_meta.o lj_debug.o \ lj_state.o lj_dispatch.o lj_vmevent.o lj_vmmath.o lj_strscan.o \ - lj_strfmt.o lj_strfmt_num.o lj_api.o lj_profile.o \ + lj_strfmt.o lj_strfmt_num.o lj_api.o lj_mapi.o lj_profile.o \ lj_lex.o lj_parse.o lj_bcread.o lj_bcwrite.o lj_load.o \ lj_ir.o lj_opt_mem.o lj_opt_fold.o lj_opt_narrow.o \ lj_opt_dce.o lj_opt_loop.o lj_opt_split.o lj_opt_sink.o \ diff --git a/src/Makefile.dep b/src/Makefile.dep index 2b1cb5e..556314e 100644 --- a/src/Makefile.dep +++ b/src/Makefile.dep @@ -18,7 +18,7 @@ lib_ffi.o: lib_ffi.c lua.h luaconf.h lauxlib.h lualib.h lj_obj.h lj_def.h \ lj_ctype.h lj_cparse.h lj_cdata.h lj_cconv.h lj_carith.h lj_ccall.h \ lj_ccallback.h lj_clib.h lj_strfmt.h lj_ff.h lj_ffdef.h lj_lib.h \ lj_libdef.h -lib_init.o: lib_init.c lua.h luaconf.h lauxlib.h lualib.h lj_arch.h +lib_init.o: lib_init.c lua.h luaconf.h lauxlib.h lualib.h lmisclib.h lj_arch.h lib_io.o: lib_io.c lua.h luaconf.h lauxlib.h lualib.h lj_obj.h lj_def.h \ lj_arch.h lj_gc.h lj_err.h lj_errmsg.h lj_buf.h lj_str.h lj_state.h \ lj_strfmt.h lj_ff.h lj_ffdef.h lj_lib.h lj_libdef.h @@ -29,6 +29,8 @@ lib_jit.o: lib_jit.c lua.h luaconf.h lauxlib.h lualib.h lj_obj.h lj_def.h \ lj_vm.h lj_vmevent.h lj_lib.h luajit.h lj_libdef.h lib_math.o: lib_math.c lua.h luaconf.h lauxlib.h lualib.h lj_obj.h \ lj_def.h lj_arch.h lj_lib.h lj_vm.h lj_libdef.h +lib_misc.o: lib_misc.c lua.h luaconf.h lmisclib.h lj_obj.h lj_def.h lj_arch.h \ + lj_str.h lj_tab.h lj_lib.h lj_libdef.h lib_os.o: lib_os.c lua.h luaconf.h lauxlib.h lualib.h lj_obj.h lj_def.h \ lj_arch.h lj_gc.h lj_err.h lj_errmsg.h lj_buf.h lj_str.h lj_lib.h \ lj_libdef.h @@ -137,6 +139,8 @@ lj_lib.o: lj_lib.c lauxlib.h lua.h luaconf.h lj_obj.h lj_def.h lj_arch.h \ lj_load.o: lj_load.c lua.h luaconf.h lauxlib.h lj_obj.h lj_def.h \ lj_arch.h lj_gc.h lj_err.h lj_errmsg.h lj_buf.h lj_str.h lj_func.h \ lj_frame.h lj_bc.h lj_vm.h lj_lex.h lj_bcdump.h lj_parse.h +lj_mapi.o: lj_mapi.c lua.h luaconf.h lmisclib.h lj_obj.h lj_def.h lj_arch.h \ + lj_dispatch.h lj_bc.h lj_jit.h lj_ir.h lj_mcode.o: lj_mcode.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \ lj_gc.h lj_err.h lj_errmsg.h lj_jit.h lj_ir.h lj_mcode.h lj_trace.h \ lj_dispatch.h lj_bc.h lj_traceerr.h lj_vm.h @@ -215,9 +219,9 @@ ljamalg.o: ljamalg.c lua.h luaconf.h lauxlib.h lj_gc.c lj_obj.h lj_def.h \ lj_func.c lj_udata.c lj_meta.c lj_strscan.h lj_lib.h lj_debug.c \ lj_state.c lj_lex.h lj_alloc.h luajit.h lj_dispatch.c lj_ccallback.h \ lj_profile.h lj_vmevent.c lj_vmevent.h lj_vmmath.c lj_strscan.c \ - lj_strfmt.c lj_strfmt_num.c lj_api.c lj_profile.c lj_lex.c lualib.h \ - lj_parse.h lj_parse.c lj_bcread.c lj_bcdump.h lj_bcwrite.c lj_load.c \ - lj_ctype.c lj_cdata.c lj_cconv.h lj_cconv.c lj_ccall.c lj_ccall.h \ + lj_strfmt.c lj_strfmt_num.c lj_api.c lj_mapi.c lmisclib.h lj_profile.c \ + lj_lex.c lualib.h lj_parse.h lj_parse.c lj_bcread.c lj_bcdump.h lj_bcwrite.c \ + lj_load.c lj_ctype.c lj_cdata.c lj_cconv.h lj_cconv.c lj_ccall.c lj_ccall.h \ lj_ccallback.c lj_target.h lj_target_*.h lj_mcode.h lj_carith.c \ lj_carith.h lj_clib.c lj_clib.h lj_cparse.c lj_cparse.h lj_lib.c lj_ir.c \ lj_ircall.h lj_iropt.h lj_opt_mem.c lj_opt_fold.c lj_folddef.h \ @@ -227,7 +231,7 @@ ljamalg.o: ljamalg.c lua.h luaconf.h lauxlib.h lj_gc.c lj_obj.h lj_def.h \ lj_emit_*.h lj_asm_*.h lj_trace.c lj_gdbjit.h lj_gdbjit.c lj_alloc.c \ lib_aux.c lib_base.c lj_libdef.h lib_math.c lib_string.c lib_table.c \ lib_io.c lib_os.c lib_package.c lib_debug.c lib_bit.c lib_jit.c \ - lib_ffi.c lib_init.c + lib_ffi.c lib_misc.c lib_init.c luajit.o: luajit.c lua.h luaconf.h lauxlib.h lualib.h luajit.h lj_arch.h host/buildvm.o: host/buildvm.c host/buildvm.h lj_def.h lua.h luaconf.h \ lj_arch.h lj_obj.h lj_def.h lj_arch.h lj_gc.h lj_obj.h lj_bc.h lj_ir.h \ diff --git a/src/lib_init.c b/src/lib_init.c index 2ed370e..664aa7d 100644 --- a/src/lib_init.c +++ b/src/lib_init.c @@ -12,6 +12,7 @@ #include "lua.h" #include "lauxlib.h" #include "lualib.h" +#include "lmisclib.h" #include "lj_arch.h" @@ -26,6 +27,7 @@ static const luaL_Reg lj_lib_load[] = { { LUA_DBLIBNAME, luaopen_debug }, { LUA_BITLIBNAME, luaopen_bit }, { LUA_JITLIBNAME, luaopen_jit }, + { LUAM_MISCLIBNAME, luaopen_misc }, { NULL, NULL } }; diff --git a/src/lib_misc.c b/src/lib_misc.c new file mode 100644 index 0000000..ef11237 --- /dev/null +++ b/src/lib_misc.c @@ -0,0 +1,72 @@ +/* +** Miscellaneous Lua extensions library. +** +** Major portions taken verbatim or adapted from the LuaVela interpreter. +** Copyright (C) 2015-2019 IPONWEB Ltd. +*/ + +#define lib_misc_c +#define LUA_LIB + +#include "lua.h" +#include "lmisclib.h" + +#include "lj_obj.h" +#include "lj_str.h" +#include "lj_tab.h" +#include "lj_lib.h" + +/* ------------------------------------------------------------------------ */ + +static LJ_AINLINE void setnumfield(struct lua_State *L, GCtab *t, + const char *name, int64_t val) +{ + setnumV(lj_tab_setstr(L, t, lj_str_newz(L, name)), (double)val); +} + +#define LJLIB_MODULE_misc + +LJLIB_CF(misc_getmetrics) +{ + struct luam_Metrics metrics; + lua_createtable(L, 0, 22); + GCtab *m = tabV(L->top - 1); + + luaM_metrics(L, &metrics); + + setnumfield(L, m, "strhash_hit", metrics.strhash_hit); + setnumfield(L, m, "strhash_miss", metrics.strhash_miss); + + setnumfield(L, m, "gc_strnum", metrics.gc_strnum); + setnumfield(L, m, "gc_tabnum", metrics.gc_tabnum); + setnumfield(L, m, "gc_udatanum", metrics.gc_udatanum); + setnumfield(L, m, "gc_cdatanum", metrics.gc_cdatanum); + + setnumfield(L, m, "gc_total", metrics.gc_total); + setnumfield(L, m, "gc_freed", metrics.gc_freed); + setnumfield(L, m, "gc_allocated", metrics.gc_allocated); + + setnumfield(L, m, "gc_steps_pause", metrics.gc_steps_pause); + setnumfield(L, m, "gc_steps_propagate", metrics.gc_steps_propagate); + setnumfield(L, m, "gc_steps_atomic", metrics.gc_steps_atomic); + setnumfield(L, m, "gc_steps_sweepstring", metrics.gc_steps_sweepstring); + setnumfield(L, m, "gc_steps_sweep", metrics.gc_steps_sweep); + setnumfield(L, m, "gc_steps_finalize", metrics.gc_steps_finalize); + + setnumfield(L, m, "jit_snap_restore", metrics.jit_snap_restore); + setnumfield(L, m, "jit_trace_abort", metrics.jit_trace_abort); + setnumfield(L, m, "jit_mcode_size", metrics.jit_mcode_size); + setnumfield(L, m, "jit_trace_num", metrics.jit_trace_num); + + return 1; +} + +/* ------------------------------------------------------------------------ */ + +#include "lj_libdef.h" + +LUALIB_API int luaopen_misc(struct lua_State *L) +{ + LJ_LIB_REG(L, LUAM_MISCLIBNAME, misc); + return 1; +} diff --git a/src/lj_mapi.c b/src/lj_mapi.c new file mode 100644 index 0000000..7645a44 --- /dev/null +++ b/src/lj_mapi.c @@ -0,0 +1,61 @@ +/* +** Miscellaneous public C API extensions. +** +** Major portions taken verbatim or adapted from the LuaVela. +** Copyright (C) 2015-2019 IPONWEB Ltd. +*/ + +#include "lua.h" +#include "lmisclib.h" + +#include "lj_obj.h" +#include "lj_dispatch.h" + +#if LJ_HASJIT +#include "lj_jit.h" +#endif + +LUAMISC_API void luaM_metrics(lua_State *L, struct luam_Metrics *metrics) +{ + lua_assert(metrics != NULL); + global_State *g = G(L); + GCState *gc = &g->gc; +#if LJ_HASJIT + jit_State *J = G2J(g); +#endif + + metrics->strhash_hit = g->strhash_hit; + metrics->strhash_miss = g->strhash_miss; + + metrics->gc_strnum = g->strnum; + metrics->gc_tabnum = gc->tabnum; + metrics->gc_udatanum = gc->udatanum; +#if LJ_HASFFI + metrics->gc_cdatanum = gc->cdatanum; +#else + metrics->gc_cdatanum = 0; +#endif + + metrics->gc_total = gc->total; + metrics->gc_freed = gc->freed; + metrics->gc_allocated = gc->allocated; + + metrics->gc_steps_pause = gc->state_count[GCSpause]; + metrics->gc_steps_propagate = gc->state_count[GCSpropagate]; + metrics->gc_steps_atomic = gc->state_count[GCSatomic]; + metrics->gc_steps_sweepstring = gc->state_count[GCSsweepstring]; + metrics->gc_steps_sweep = gc->state_count[GCSsweep]; + metrics->gc_steps_finalize = gc->state_count[GCSfinalize]; + +#if LJ_HASJIT + metrics->jit_snap_restore = J->nsnaprestore; + metrics->jit_trace_abort = J->ntraceabort; + metrics->jit_mcode_size = J->szallmcarea; + metrics->jit_trace_num = J->tracenum; +#else + metrics->jit_snap_restore = 0; + metrics->jit_trace_abort = 0; + metrics->jit_mcode_size = 0; + metrics->jit_trace_num = 0; +#endif +} diff --git a/src/ljamalg.c b/src/ljamalg.c index f1f2862..371bbb6 100644 --- a/src/ljamalg.c +++ b/src/ljamalg.c @@ -48,6 +48,7 @@ #include "lj_strfmt.c" #include "lj_strfmt_num.c" #include "lj_api.c" +#include "lj_mapi.c" #include "lj_profile.c" #include "lj_lex.c" #include "lj_parse.c" @@ -93,5 +94,6 @@ #include "lib_bit.c" #include "lib_jit.c" #include "lib_ffi.c" +#include "lib_misc.c" #include "lib_init.c" diff --git a/src/lmisclib.h b/src/lmisclib.h new file mode 100644 index 0000000..e2d1909 --- /dev/null +++ b/src/lmisclib.h @@ -0,0 +1,64 @@ +/* +** Miscellaneous public C API extensions. +** +** Major portions taken verbatim or adapted from the LuaVela. +** Copyright (C) 2015-2019 IPONWEB Ltd. +*/ + +#ifndef _LMISCLIB_H +#define _LMISCLIB_H + +#include "lua.h" + +/* API for obtaining various platform metrics. */ + +struct luam_Metrics { + /* Strings amount found in string hash instead of allocation of new one. */ + size_t strhash_hit; + /* Strings amount allocated and put into string hash. */ + size_t strhash_miss; + + /* Amount of allocated string objects. */ + size_t gc_strnum; + /* Amount of allocated table objects. */ + size_t gc_tabnum; + /* Amount of allocated udata objects. */ + size_t gc_udatanum; + /* Amount of allocated cdata objects. */ + size_t gc_cdatanum; + + /* Memory currently allocated. */ + size_t gc_total; + /* Total amount of freed memory. */ + size_t gc_freed; + /* Total amount of allocated memory. */ + size_t gc_allocated; + + /* Count of incremental GC steps per state. */ + size_t gc_steps_pause; + size_t gc_steps_propagate; + size_t gc_steps_atomic; + size_t gc_steps_sweepstring; + size_t gc_steps_sweep; + size_t gc_steps_finalize; + + /* + ** Overall number of snap restores (amount of guard assertions + ** leading to stopping trace executions and trace exits, + ** that are not stitching with other traces). + */ + size_t jit_snap_restore; + /* Overall number of abort traces. */ + size_t jit_trace_abort; + /* Total size of all allocated machine code areas. */ + size_t jit_mcode_size; + /* Amount of JIT traces. */ + unsigned int jit_trace_num; +}; + +LUAMISC_API void luaM_metrics(lua_State *L, struct luam_Metrics *metrics); + +#define LUAM_MISCLIBNAME "misc" +LUALIB_API int luaopen_misc(lua_State *L); + +#endif /* _LMISCLIB_H */ diff --git a/src/luaconf.h b/src/luaconf.h index 60cb928..8029040 100644 --- a/src/luaconf.h +++ b/src/luaconf.h @@ -144,6 +144,7 @@ #endif #define LUALIB_API LUA_API +#define LUAMISC_API LUA_API /* Support for internal assertions. */ #if defined(LUA_USE_ASSERT) || defined(LUA_USE_APICHECK) diff --git a/test/clib-misclib-getmetrics.test.lua b/test/clib-misclib-getmetrics.test.lua new file mode 100755 index 0000000..34adaba --- /dev/null +++ b/test/clib-misclib-getmetrics.test.lua @@ -0,0 +1,174 @@ +#!/usr/bin/env tarantool + +local file = debug.getinfo(1, "S").source:sub(2) +local filepath = file:match("(.*/)") +local soext = jit.os == "OSX" and "dylib" or "so" +package.cpath = filepath..'clib-misclib-getmetrics/?.'..soext..";" + ..package.cpath + +local jit_opt_default_hotloop = 56 +local jit_opt_default_hotexit = 10 +local jit_opt_default_level = 3 +local jit_opt_default_minstitch = 0 + +local tap = require('tap') + +local test = tap.test("clib-misc-getmetrics") +test:plan(10) + +local testgetmetrics = require("testgetmetrics") + +test:ok(testgetmetrics.base()) +test:ok(testgetmetrics.gc_allocated_freed()) +test:ok(testgetmetrics.gc_steps()) + +test:ok(testgetmetrics.objcount(function() + local table_new = require("table.new") + local ffi = require("ffi") + + jit.opt.start(0) + + local placeholder = { + str = {}, + tab = {}, + udata = {}, + cdata = {}, + } + + -- Separate objects creations to separate jit traces. + for i = 1, 1000 do + table.insert(placeholder.str, tostring(i)) + end + + for i = 1, 1000 do + table.insert(placeholder.tab, table_new(i, 0)) + end + + for i = 1, 1000 do + table.insert(placeholder.udata, newproxy()) + end + + for i = 1, 1000 do + -- Check counting of VLA/VLS/aligned cdata. + table.insert(placeholder.cdata, ffi.new("char[?]", 4)) + end + + for i = 1, 1000 do + -- Check counting of non-VLA/VLS/aligned cdata. + table.insert(placeholder.cdata, ffi.new("uint64_t", i)) + end + + placeholder = nil + -- Restore default jit settings. + jit.opt.start(jit_opt_default_level) +end)) + +-- Compiled loop with a direct exit to the interpreter. +test:ok(testgetmetrics.snap_restores(function() + jit.opt.start(0, "hotloop=2") + + local old_metrics = misc.getmetrics() + + local sum = 0 + for i = 1, 20 do + sum = sum + i + end + + local new_metrics = misc.getmetrics() + + -- Restore default jit settings. + jit.opt.start(jit_opt_default_level, "hotloop="..jit_opt_default_hotloop) + + -- A single snapshot restoration happened on loop finish. + return 1 +end)) + +-- Compiled loop with a side exit which does not get compiled. +test:ok(testgetmetrics.snap_restores(function() + jit.opt.start(0, "hotloop=2", "hotexit=2", "minstitch=15") + + local function foo(i) + -- math.fmod is not yet compiled! + return i <= 5 and i or math.fmod(i, 11) + end + + local sum = 0 + for i = 1, 10 do + sum = sum + foo(i) + end + + -- Restore default jit settings. + jit.opt.start(jit_opt_default_level, "hotloop="..jit_opt_default_hotloop, + "hotexit="..jit_opt_default_hotexit, + "minstitch="..jit_opt_default_minstitch) + + -- Side exits from the root trace could not get compiled. + return 5 +end)) + +-- Compiled loop with a side exit which gets compiled. +test:ok(testgetmetrics.snap_restores(function() + -- Optimization level is important here as `loop` optimization + -- may unroll the loop body and insert +1 side exit. + jit.opt.start(0, "hotloop=5", "hotexit=5") + + local function foo(i) + return i <= 10 and i or tostring(i) + end + + local sum = 0 + for i = 1, 20 do + sum = sum + foo(i) + end + + -- Restore default jit settings. + jit.opt.start(jit_opt_default_level, "hotloop="..jit_opt_default_hotloop, + "hotexit="..jit_opt_default_hotexit) + + -- 5 side exits to the interpreter before trace gets hot + -- and compiled + -- 1 side exit on loop end + return 6 +end)) + +-- Compiled scalar trace with a direct exit to the interpreter. +test:ok(testgetmetrics.snap_restores(function() + -- For calls it will be 2 * hotloop (see lj_dispatch.{c,h} + -- and hotcall@vm_*.dasc). + jit.opt.start(3, "hotloop=2", "hotexit=3") + + local function foo(i) + return i <= 15 and i or tostring(i) + end + + foo(1) -- interp only + foo(2) -- interp only + foo(3) -- interp only + foo(4) -- compile trace during this call + foo(5) -- follow the trace + foo(6) -- follow the trace + foo(7) -- follow the trace + foo(8) -- follow the trace + foo(9) -- follow the trace + foo(10) -- follow the trace + + -- Simply 2 side exits from the trace: + foo(20) + foo(21) + + -- Restore default jit settings. + jit.opt.start(jit_opt_default_level, "hotloop="..jit_opt_default_hotloop, + "hotexit="..jit_opt_default_hotexit) + return 2 +end)) + +test:ok(testgetmetrics.strhash()) + +test:ok(testgetmetrics.tracenum_base(function() + local sum = 0 + for i = 1, 200 do + sum = sum + i + end + -- Compiled only 1 loop as new trace. + return 1 +end)) diff --git a/test/clib-misclib-getmetrics/CMakeLists.txt b/test/clib-misclib-getmetrics/CMakeLists.txt new file mode 100644 index 0000000..e7cc8f8 --- /dev/null +++ b/test/clib-misclib-getmetrics/CMakeLists.txt @@ -0,0 +1 @@ +build_lualib(testgetmetrics testgetmetrics.c) diff --git a/test/clib-misclib-getmetrics/testgetmetrics.c b/test/clib-misclib-getmetrics/testgetmetrics.c new file mode 100644 index 0000000..32802d2 --- /dev/null +++ b/test/clib-misclib-getmetrics/testgetmetrics.c @@ -0,0 +1,242 @@ +#include +#include +#include + +#include + +#include + +static int base(lua_State *L) +{ + struct luam_Metrics metrics; + luaM_metrics(L, &metrics); + + /* Just check API. */ + assert((ssize_t)metrics.strhash_hit >= 0); + assert((ssize_t)metrics.strhash_miss >= 0); + + assert((ssize_t)metrics.gc_strnum >= 0); + assert((ssize_t)metrics.gc_tabnum >= 0); + assert((ssize_t)metrics.gc_udatanum >= 0); + assert((ssize_t)metrics.gc_cdatanum >= 0); + + assert((ssize_t)metrics.gc_total >= 0); + assert((ssize_t)metrics.gc_freed >= 0); + assert((ssize_t)metrics.gc_allocated >= 0); + + assert((ssize_t)metrics.gc_steps_pause >= 0); + assert((ssize_t)metrics.gc_steps_propagate >= 0); + assert((ssize_t)metrics.gc_steps_atomic >= 0); + assert((ssize_t)metrics.gc_steps_sweepstring >= 0); + assert((ssize_t)metrics.gc_steps_sweep >= 0); + assert((ssize_t)metrics.gc_steps_finalize >= 0); + + assert((ssize_t)metrics.jit_snap_restore >= 0); + assert((ssize_t)metrics.jit_trace_abort >= 0); + assert((ssize_t)metrics.jit_mcode_size >= 0); + assert((ssize_t)metrics.jit_trace_num >= 0); + + lua_pushboolean(L, 1); + return 1; +} + +static int gc_allocated_freed(lua_State *L) +{ + struct luam_Metrics oldm, newm; + /* Force up garbage collect all dead objects. */ + lua_gc(L, LUA_GCCOLLECT, 0); + + luaM_metrics(L, &oldm); + /* Simple garbage generation. */ + if (luaL_dostring(L, "local i = 0 for j = 1, 10 do i = i + j end")) + luaL_error(L, "failed to translate Lua code snippet"); + lua_gc(L, LUA_GCCOLLECT, 0); + luaM_metrics(L, &newm); + assert(newm.gc_allocated - oldm.gc_allocated > 0); + assert(newm.gc_freed - oldm.gc_freed > 0); + + lua_pushboolean(L, 1); + return 1; +} + +static int gc_steps(lua_State *L) +{ + struct luam_Metrics oldm, newm; + /* + * Some garbage has already happened before the next line, + * i.e. during fronted processing lua test chunk. + * Let's put a full garbage collection cycle on top + * of that, and confirm that non-null values are reported + * (we are not yet interested in actual numbers): + */ + luaM_metrics(L, &oldm); + lua_gc(L, LUA_GCCOLLECT, 0); + assert(oldm.gc_steps_pause > 0); + assert(oldm.gc_steps_propagate > 0); + assert(oldm.gc_steps_atomic > 0); + assert(oldm.gc_steps_sweepstring > 0); + assert(oldm.gc_steps_sweep > 0); + /* Nothing to finalize, skipped. */ + assert(oldm.gc_steps_finalize == 0); + + /* + * As long as we don't create new Lua objects + * consequent call should return the same values: + */ + luaM_metrics(L, &newm); + assert(newm.gc_steps_pause - oldm.gc_steps_pause > 0); + assert(newm.gc_steps_propagate - oldm.gc_steps_propagate > 0); + assert(newm.gc_steps_atomic - oldm.gc_steps_atomic > 0); + assert(newm.gc_steps_sweepstring - oldm.gc_steps_sweepstring > 0); + assert(newm.gc_steps_sweep - oldm.gc_steps_sweep > 0); + /* Nothing to finalize, skipped. */ + assert(newm.gc_steps_finalize == 0); + oldm = newm; + + /* + * Now the last phase: run full GC once and make sure that + * everything is being reported as expected: + */ + lua_gc(L, LUA_GCCOLLECT, 0); + luaM_metrics(L, &newm); + assert(newm.gc_steps_pause - oldm.gc_steps_pause == 1); + assert(newm.gc_steps_propagate - oldm.gc_steps_propagate >= 1); + assert(newm.gc_steps_atomic - oldm.gc_steps_atomic == 1); + assert(newm.gc_steps_sweepstring - oldm.gc_steps_sweepstring >= 1); + assert(newm.gc_steps_sweep - oldm.gc_steps_sweep >= 1); + /* Nothing to finalize, skipped. */ + assert(newm.gc_steps_finalize == 0); + oldm = newm; + + /* + * Now let's run three GC cycles to ensure that + * zero-to-one transition was not a lucky coincidence. + */ + lua_gc(L, LUA_GCCOLLECT, 0); + lua_gc(L, LUA_GCCOLLECT, 0); + lua_gc(L, LUA_GCCOLLECT, 0); + luaM_metrics(L, &newm); + assert(newm.gc_steps_pause - oldm.gc_steps_pause == 3); + assert(newm.gc_steps_propagate - oldm.gc_steps_propagate >= 3); + assert(newm.gc_steps_atomic - oldm.gc_steps_atomic == 3); + assert(newm.gc_steps_sweepstring - oldm.gc_steps_sweepstring >= 3); + assert(newm.gc_steps_sweep - oldm.gc_steps_sweep >= 3); + /* Nothing to finalize, skipped. */ + assert(newm.gc_steps_finalize == 0); + + lua_pushboolean(L, 1); + return 1; +} + +static int objcount(lua_State *L) +{ + struct luam_Metrics oldm, newm; + int n = lua_gettop(L); + if (n != 1 || !lua_isfunction(L, 1)) + luaL_error(L, "incorrect argument: 1 function is required"); + + /* Force up garbage collect all dead objects. */ + lua_gc(L, LUA_GCCOLLECT, 0); + + luaM_metrics(L, &oldm); + /* Generate garbage. */ + lua_call(L, 0, 0); + lua_gc(L, LUA_GCCOLLECT, 0); + luaM_metrics(L, &newm); + assert(newm.gc_strnum - oldm.gc_strnum == 0); + assert(newm.gc_tabnum - oldm.gc_tabnum == 0); + assert(newm.gc_udatanum - oldm.gc_udatanum == 0); + assert(newm.gc_cdatanum - oldm.gc_cdatanum == 0); + + lua_pushboolean(L, 1); + return 1; +} + +static int snap_restores(lua_State *L) +{ + struct luam_Metrics oldm, newm; + int n = lua_gettop(L); + if (n != 1 || !lua_isfunction(L, 1)) + luaL_error(L, "incorrect arguments: 1 function is required"); + + luaM_metrics(L, &oldm); + /* Generate snapshots. */ + lua_call(L, 0, 1); + n = lua_gettop(L); + if (n != 1 || !lua_isnumber(L, 1)) + luaL_error(L, "incorrect return value: 1 number is required"); + size_t snap_restores = lua_tonumber(L, 1); + luaM_metrics(L, &newm); + assert(newm.jit_snap_restore - oldm.jit_snap_restore == snap_restores); + + lua_pushboolean(L, 1); + return 1; +} + +static int strhash(lua_State *L) +{ + struct luam_Metrics oldm, newm; + lua_pushstring(L, "strhash_hit"); + luaM_metrics(L, &oldm); + lua_pushstring(L, "strhash_hit"); + lua_pushstring(L, "new_str"); + luaM_metrics(L, &newm); + assert(newm.strhash_hit - oldm.strhash_hit == 1); + assert(newm.strhash_miss - oldm.strhash_miss == 1); + lua_pop(L, 3); + lua_pushboolean(L, 1); + return 1; +} + +static int tracenum_base(lua_State *L) +{ + struct luam_Metrics metrics; + int n = lua_gettop(L); + if (n != 1 || !lua_isfunction(L, 1)) + luaL_error(L, "incorrect arguments: 1 function is required"); + + luaJIT_setmode(L, 0, LUAJIT_MODE_OFF); + luaJIT_setmode(L, 0, LUAJIT_MODE_FLUSH); + /* Force up garbage collect all dead objects. */ + lua_gc(L, LUA_GCCOLLECT, 0); + + luaM_metrics(L, &metrics); + assert(metrics.jit_trace_num == 0); + + luaJIT_setmode(L, 0, LUAJIT_MODE_ON); + /* Generate traces. */ + lua_call(L, 0, 1); + n = lua_gettop(L); + if (n != 1 || !lua_isnumber(L, 1)) + luaL_error(L, "incorrect return value: 1 number is required"); + size_t jit_trace_num = lua_tonumber(L, 1); + luaM_metrics(L, &metrics); + assert(metrics.jit_trace_num == jit_trace_num); + + luaJIT_setmode(L, 0, LUAJIT_MODE_FLUSH); + /* Force up garbage collect all dead objects. */ + lua_gc(L, LUA_GCCOLLECT, 0); + luaM_metrics(L, &metrics); + assert(metrics.jit_trace_num == 0); + + luaJIT_setmode(L, 0, LUAJIT_MODE_ON); + lua_pushboolean(L, 1); + return 1; +} + +static const struct luaL_Reg testgetmetrics[] = { + {"base", base}, + {"gc_allocated_freed", gc_allocated_freed}, + {"gc_steps", gc_steps}, + {"objcount", objcount}, + {"snap_restores", snap_restores}, + {"strhash", strhash}, + {"tracenum_base", tracenum_base}, + {NULL, NULL} +}; + +LUA_API int luaopen_testgetmetrics(lua_State *L) +{ + luaL_register(L, "testgetmetrics", testgetmetrics); + return 1; +} diff --git a/test/lib-misc-getmetrics.test.lua b/test/lib-misc-getmetrics.test.lua new file mode 100755 index 0000000..2859e26 --- /dev/null +++ b/test/lib-misc-getmetrics.test.lua @@ -0,0 +1,418 @@ +#!/usr/bin/env tarantool + +-- This is a part of tarantool/luajit testing suite. +-- Major portions taken verbatim or adapted from the LuaVela testing suit. +-- Copyright (C) 2015-2019 IPONWEB Ltd. + +local tap = require('tap') + +local test = tap.test("lib-misc-getmetrics") +test:plan(10) + +local jit_opt_default_hotloop = 56 +local jit_opt_default_hotexit = 10 +local jit_opt_default_level = 3 +local jit_opt_default_minstitch = 0 + +-- Test Lua API. +test:test("base", function(subtest) + subtest:plan(19) + local metrics = misc.getmetrics() + subtest:ok(metrics.strhash_hit >= 0) + subtest:ok(metrics.strhash_miss >= 0) + + subtest:ok(metrics.gc_strnum >= 0) + subtest:ok(metrics.gc_tabnum >= 0) + subtest:ok(metrics.gc_udatanum >= 0) + subtest:ok(metrics.gc_cdatanum >= 0) + + subtest:ok(metrics.gc_total >= 0) + subtest:ok(metrics.gc_freed >= 0) + subtest:ok(metrics.gc_allocated >= 0) + + subtest:ok(metrics.gc_steps_pause >= 0) + subtest:ok(metrics.gc_steps_propagate >= 0) + subtest:ok(metrics.gc_steps_atomic >= 0) + subtest:ok(metrics.gc_steps_sweepstring >= 0) + subtest:ok(metrics.gc_steps_sweep >= 0) + subtest:ok(metrics.gc_steps_finalize >= 0) + + subtest:ok(metrics.jit_snap_restore >= 0) + subtest:ok(metrics.jit_trace_abort >= 0) + subtest:ok(metrics.jit_mcode_size >= 0) + subtest:ok(metrics.jit_trace_num >= 0) +end) + +test:test("gc-allocated-freed", function(subtest) + subtest:plan(1) + + -- Force up garbage collect all dead objects. + collectgarbage("collect") + + -- Bump getmetrincs table and string keys allocation. + local old_metrics = misc.getmetrics() + + -- Remember allocated size for getmetrics table. + old_metrics = misc.getmetrics() + + collectgarbage("collect") + + local new_metrics = misc.getmetrics() + + local diff_alloc = new_metrics.gc_allocated - old_metrics.gc_allocated + local getmetrics_alloc = diff_alloc + + -- Do not use test:ok to avoid extra allocated/freed objects. + assert(getmetrics_alloc > 0, "count allocated table for getmetrics") + old_metrics = new_metrics + + -- NB: Avoid operations that use internal global string buffer + -- (such as concatenation, string.format, table.concat) + -- while creating the string. Otherwise gc_freed/gc_allocated + -- relations will not be so straightforward. + local str = string.sub("Hello, world", 1, 5) + collectgarbage("collect") + + new_metrics = misc.getmetrics() + + diff_alloc = new_metrics.gc_allocated - old_metrics.gc_allocated + local diff_freed = new_metrics.gc_freed - old_metrics.gc_freed + + assert(diff_alloc > getmetrics_alloc, + "allocated str 'Hello' and table for getmetrics") + assert(diff_freed == getmetrics_alloc, + "freed old old_metrics") + old_metrics = new_metrics + + str = string.sub("Hello, world", 8, -1) + + new_metrics = misc.getmetrics() + + diff_alloc = new_metrics.gc_allocated - old_metrics.gc_allocated + diff_freed = new_metrics.gc_freed - old_metrics.gc_freed + + assert(diff_alloc > getmetrics_alloc, + "allocated str 'world' and table for getmetrics") + assert(diff_freed == 0, "nothing to free without collectgarbage") + old_metrics = new_metrics + collectgarbage("collect") + + new_metrics = misc.getmetrics() + + diff_alloc = new_metrics.gc_allocated - old_metrics.gc_allocated + diff_freed = new_metrics.gc_freed - old_metrics.gc_freed + + assert(diff_alloc == getmetrics_alloc, + "allocated last one table for getmetrics") + assert(diff_freed > 2 * getmetrics_alloc, + "freed str 'Hello' and 2 tables for getmetrics") + subtest:ok(true, "no assetion failed") +end) + +test:test("gc-steps", function(subtest) + subtest:plan(24) + + -- Some garbage has already created before the next line, + -- i.e. during fronted processing this chunk. + -- Let's put a full garbage collection cycle on top of that, + -- and confirm that non-null values are reported (we are not + -- yet interested in actual numbers): + collectgarbage("collect") + collectgarbage("stop") + local oldm = misc.getmetrics() + subtest:ok(oldm.gc_steps_pause > 0) + subtest:ok(oldm.gc_steps_propagate > 0) + subtest:ok(oldm.gc_steps_atomic > 0) + subtest:ok(oldm.gc_steps_sweepstring > 0) + subtest:ok(oldm.gc_steps_sweep > 0) + -- Nothing to finalize, skipped. + subtest:is(oldm.gc_steps_finalize, 0) + + -- As long as we stopped the GC, consequent call + -- should return the same values: + local newm = misc.getmetrics() + subtest:is(newm.gc_steps_pause - oldm.gc_steps_pause, 0) + subtest:is(newm.gc_steps_propagate - oldm.gc_steps_propagate, 0) + subtest:is(newm.gc_steps_atomic - oldm.gc_steps_atomic, 0) + subtest:is(newm.gc_steps_sweepstring - oldm.gc_steps_sweepstring, 0) + subtest:is(newm.gc_steps_sweep - oldm.gc_steps_sweep, 0) + -- Nothing to finalize, skipped. + subtest:is(newm.gc_steps_finalize, 0) + oldm = newm + + -- Now the last phase: run full GC once and make sure that + -- everything is being reported as expected: + collectgarbage("collect") + collectgarbage("stop") + newm = misc.getmetrics() + subtest:ok(newm.gc_steps_pause - oldm.gc_steps_pause == 1) + subtest:ok(newm.gc_steps_propagate - oldm.gc_steps_propagate >= 1) + subtest:ok(newm.gc_steps_atomic - oldm.gc_steps_atomic == 1) + subtest:ok(newm.gc_steps_sweepstring - oldm.gc_steps_sweepstring >= 1) + subtest:ok(newm.gc_steps_sweep - oldm.gc_steps_sweep >= 1) + -- Nothing to finalize, skipped. + subtest:is(newm.gc_steps_finalize, 0) + oldm = newm + + -- Now let's run three GC cycles to ensure that zero-to-one + -- transition was not a lucky coincidence. + collectgarbage("collect") + collectgarbage("collect") + collectgarbage("collect") + collectgarbage("stop") + newm = misc.getmetrics() + subtest:ok(newm.gc_steps_pause - oldm.gc_steps_pause == 3) + subtest:ok(newm.gc_steps_propagate - oldm.gc_steps_propagate >= 3) + subtest:ok(newm.gc_steps_atomic - oldm.gc_steps_atomic == 3) + subtest:ok(newm.gc_steps_sweepstring - oldm.gc_steps_sweepstring >= 3) + subtest:ok(newm.gc_steps_sweep - oldm.gc_steps_sweep >= 3) + -- Nothing to finalize, skipped. + subtest:is(newm.gc_steps_finalize, 0) +end) + +test:test("objcount", function(subtest) + subtest:plan(4) + local table_new = require("table.new") + local ffi = require("ffi") + + jit.opt.start(0) + + -- Remove all dead objects. + collectgarbage("collect") + + -- Bump strings and table creation. + local old_metrics = misc.getmetrics() + old_metrics = misc.getmetrics() + + local placeholder = { + str = {}, + tab = {}, + udata = {}, + cdata = {}, + } + + -- Separate objects creations to separate jit traces. + for i = 1, 1000 do + table.insert(placeholder.str, tostring(i)) + end + + for i = 1, 1000 do + table.insert(placeholder.tab, table_new(i, 0)) + end + + for i = 1, 1000 do + table.insert(placeholder.udata, newproxy()) + end + + for i = 1, 1000 do + -- Check counting of VLA/VLS/aligned cdata. + table.insert(placeholder.cdata, ffi.new("char[?]", 4)) + end + + for i = 1, 1000 do + -- Check counting of non-VLA/VLS/aligned cdata. + table.insert(placeholder.cdata, ffi.new("uint64_t", i)) + end + + placeholder = nil + collectgarbage("collect") + local new_metrics = misc.getmetrics() + + -- Check that amount of objects not increased. + subtest:is(new_metrics.gc_strnum, old_metrics.gc_strnum, + "strnum don't change") + subtest:is(new_metrics.gc_tabnum, old_metrics.gc_tabnum, + "tabnum don't change") + subtest:is(new_metrics.gc_udatanum, old_metrics.gc_udatanum, + "udatanum don't change") + subtest:is(new_metrics.gc_cdatanum, old_metrics.gc_cdatanum, + "cdatanum don't change") + + -- Restore default jit settings. + jit.opt.start(jit_opt_default_level) +end) + +test:test("snap-restores-direct-loop", function(subtest) + -- Compiled loop with a direct exit to the interpreter. + subtest:plan(1) + + jit.opt.start(0, "hotloop=2") + + local old_metrics = misc.getmetrics() + + local sum = 0 + for i = 1, 20 do + sum = sum + i + end + + local new_metrics = misc.getmetrics() + + -- A single snapshot restoration happened on loop finish: + subtest:is(new_metrics.jit_snap_restore - old_metrics.jit_snap_restore, 1) + + -- Restore default jit settings. + jit.opt.start(jit_opt_default_level, "hotloop="..jit_opt_default_hotloop) +end) + +test:test("snap-restores-loop-side-exit-non-compiled", function(subtest) + -- Compiled loop with a side exit which does not get compiled. + subtest:plan(1) + + jit.opt.start(0, "hotloop=2", "hotexit=2", "minstitch=15") + + local function foo(i) + -- math.fmod is not yet compiled! + return i <= 5 and i or math.fmod(i, 11) + end + + local old_metrics = misc.getmetrics() + local sum = 0 + for i = 1, 10 do + sum = sum + foo(i) + end + + local new_metrics = misc.getmetrics() + + -- Side exits from the root trace could not get compiled. + subtest:is(new_metrics.jit_snap_restore - old_metrics.jit_snap_restore, 5) + + -- Restore default jit settings. + jit.opt.start(jit_opt_default_level, "hotloop="..jit_opt_default_hotloop, + "hotexit="..jit_opt_default_hotexit, + "minstitch="..jit_opt_default_minstitch) +end) + +test:test("snap-restores-loop-side-exit", function(subtest) + -- Compiled loop with a side exit which gets compiled. + subtest:plan(1) + + -- Optimization level is important here as `loop` optimization + -- may unroll the loop body and insert +1 side exit. + jit.opt.start(0, "hotloop=5", "hotexit=5") + + local function foo(i) + return i <= 10 and i or tostring(i) + end + + local old_metrics = misc.getmetrics() + local sum = 0 + for i = 1, 20 do + sum = sum + foo(i) + end + + local new_metrics = misc.getmetrics() + + -- 5 side exits to the interpreter before trace gets hot + -- and compiled + -- 1 side exit on loop end + subtest:is(new_metrics.jit_snap_restore - old_metrics.jit_snap_restore, 6) + + -- Restore default jit settings. + jit.opt.start(jit_opt_default_level, "hotloop="..jit_opt_default_hotloop, + "hotexit="..jit_opt_default_hotexit) +end) + +test:test("snap-restores-scalar", function(subtest) + -- Compiled scalar trace with a direct exit to the interpreter. + subtest:plan(2) + + -- For calls it will be 2 * hotloop (see lj_dispatch.{c,h} + -- and hotcall@vm_*.dasc). + jit.opt.start(3, "hotloop=2", "hotexit=3") + + local function foo(i) + return i <= 15 and i or tostring(i) + end + + local old_metrics = misc.getmetrics() + + foo(1) -- interp only + foo(2) -- interp only + foo(3) -- interp only + foo(4) -- compile trace during this call + foo(5) -- follow the trace + foo(6) -- follow the trace + foo(7) -- follow the trace + foo(8) -- follow the trace + foo(9) -- follow the trace + foo(10) -- follow the trace + + local new_metrics = misc.getmetrics() + + -- No exits triggering snap restore so far: snapshot + -- restoration was inlined into the machine code. + subtest:is(new_metrics.jit_snap_restore - old_metrics.jit_snap_restore, 0) + old_metrics = new_metrics + + -- Simply 2 side exits from the trace: + foo(20) + foo(21) + + new_metrics = misc.getmetrics() + subtest:is(new_metrics.jit_snap_restore - old_metrics.jit_snap_restore, 2) + + -- Restore default jit settings. + jit.opt.start(jit_opt_default_level, "hotloop="..jit_opt_default_hotloop, + "hotexit="..jit_opt_default_hotexit) +end) + +test:test("strhash", function(subtest) + subtest:plan(1) + + local old_metrics = misc.getmetrics() + + local new_metrics = misc.getmetrics() + -- Do not use test:ok to avoid extra strhash hits/misses. + assert(new_metrics.strhash_hit - old_metrics.strhash_hit == 19) + assert(new_metrics.strhash_miss - old_metrics.strhash_miss == 0) + old_metrics = new_metrics + + local str1 = "strhash".."_hit" + + new_metrics = misc.getmetrics() + assert(new_metrics.strhash_hit - old_metrics.strhash_hit == 20) + assert(new_metrics.strhash_miss - old_metrics.strhash_miss == 0) + old_metrics = new_metrics + + new_metrics = misc.getmetrics() + assert(new_metrics.strhash_hit - old_metrics.strhash_hit == 19) + assert(new_metrics.strhash_miss - old_metrics.strhash_miss == 0) + old_metrics = new_metrics + + local str2 = "new".."string" + + new_metrics = misc.getmetrics() + assert(new_metrics.strhash_hit - old_metrics.strhash_hit == 19) + assert(new_metrics.strhash_miss - old_metrics.strhash_miss == 1) + subtest:ok(true, "no assertion failed") +end) + +test:test("tracenum-base", function(subtest) + subtest:plan(3) + + jit.off() + jit.flush() + collectgarbage("collect") + local metrics = misc.getmetrics() + subtest:is(metrics.jit_trace_num, 0) + + jit.on() + local sum = 0 + for i = 1, 100 do + sum = sum + i + end + + metrics = misc.getmetrics() + subtest:is(metrics.jit_trace_num, 1) + + jit.off() + jit.flush() + collectgarbage("collect") + + metrics = misc.getmetrics() + subtest:is(metrics.jit_trace_num, 0) + + jit.on() +end) + +os.exit(test:check() and 0 or 1) -- 2.28.0