From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from smtp48.i.mail.ru (smtp48.i.mail.ru [94.100.177.108]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by dev.tarantool.org (Postfix) with ESMTPS id 28120446439 for ; Sun, 20 Sep 2020 20:13:30 +0300 (MSK) From: Sergey Kaplun Date: Sun, 20 Sep 2020 20:12:48 +0300 Message-Id: <6d5e5a28558cfb9f42e4a0bfe0000f67054c9b64.1600615976.git.skaplun@tarantool.org> In-Reply-To: References: MIME-Version: 1.0 Content-Transfer-Encoding: 8bit Subject: [Tarantool-patches] [PATCH v3 1/2] core: introduce various platform metrics List-Id: Tarantool development patches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , To: Igor Munkin , Sergey Ostanevich Cc: tarantool-patches@dev.tarantool.org This patch introduces the following counters: - overall amount of allocated tables, cdata and udata objects - number of incremental GC steps grouped by GC state - number of white, gray, black objects - number of string hashes hits and misses - amount of allocated and freed memory - number of trace aborts and restored snapshots Interfaces to obtain these metrics via both Lua and C API are introduced in the next patch. Part of tarantool/tarantool#5187 --- src/lj_asm.c | 2 + src/lj_asm_arm.h | 19 +++++++++ src/lj_asm_arm64.h | 19 +++++++++ src/lj_asm_mips.h | 10 +++++ src/lj_asm_ppc.h | 9 +++++ src/lj_asm_x86.h | 10 +++++ src/lj_cdata.c | 9 +++++ src/lj_cdata.h | 2 + src/lj_gc.c | 99 ++++++++++++++++++++++++++++++++++++---------- src/lj_gc.h | 33 +++++++++++----- src/lj_jit.h | 3 ++ src/lj_obj.h | 31 +++++++++++++-- src/lj_snap.c | 1 + src/lj_state.c | 7 +++- src/lj_str.c | 5 +++ src/lj_tab.c | 2 + src/lj_trace.c | 6 ++- src/lj_udata.c | 2 + src/vm_arm.dasc | 6 +++ src/vm_arm64.dasc | 6 +++ src/vm_mips.dasc | 6 +++ src/vm_mips64.dasc | 6 +++ src/vm_ppc.dasc | 6 +++ src/vm_x64.dasc | 2 + src/vm_x86.dasc | 2 + 25 files changed, 268 insertions(+), 35 deletions(-) diff --git a/src/lj_asm.c b/src/lj_asm.c index c2cf5a9..8fb3ccd 100644 --- a/src/lj_asm.c +++ b/src/lj_asm.c @@ -2273,6 +2273,7 @@ void lj_asm_trace(jit_State *J, GCtrace *T) as->J = J; as->T = T; J->curfinal = lj_trace_alloc(J->L, T); /* This copies the IR, too. */ + J->tracenum++; as->flags = J->flags; as->loopref = J->loopref; as->realign = NULL; @@ -2379,6 +2380,7 @@ void lj_asm_trace(jit_State *J, GCtrace *T) lj_trace_free(J2G(J), J->curfinal); J->curfinal = NULL; /* In case lj_trace_alloc() OOMs. */ J->curfinal = lj_trace_alloc(J->L, T); + J->tracenum++; as->realign = NULL; } diff --git a/src/lj_asm_arm.h b/src/lj_asm_arm.h index 37bfa40..62f269e 100644 --- a/src/lj_asm_arm.h +++ b/src/lj_asm_arm.h @@ -1257,9 +1257,16 @@ static void asm_cnew(ASMState *as, IRIns *ir) { uint32_t k = emit_isk12(ARMI_MOV, id); Reg r = k ? RID_R1 : ra_allock(as, id, allow); + allow = rset_exclude(allow, r); + Reg gr = ra_allock(as, i32ptr(J2G(as->J)), allow); emit_lso(as, ARMI_STRB, RID_TMP, RID_RET, offsetof(GCcdata, gct)); emit_lsox(as, ARMI_STRH, r, RID_RET, offsetof(GCcdata, ctypeid)); emit_d(as, ARMI_MOV|ARMI_K12|~LJ_TCDATA, RID_TMP); + emit_lso(as, ARMI_STR, RID_TMP, gr, + (int32_t)offsetof(global_State, gc.cdatanum)); + emit_dn(as, ARMI_ADD|ARMI_K12|1, RID_TMP, RID_TMP); + emit_lso(as, ARMI_LDR, RID_TMP, gr, + (int32_t)offsetof(global_State, gc.cdatanum)); if (k) emit_d(as, ARMI_MOV^k, RID_R1); } args[0] = ASMREF_L; /* lua_State *L */ @@ -1282,6 +1289,18 @@ static void asm_tbar(ASMState *as, IRIns *ir) rset_exclude(rset_exclude(RSET_GPR, tab), link)); Reg mark = RID_TMP; MCLabel l_end = emit_label(as); + emit_lso(as, ARMI_STR, RID_TMP, gr, + (int32_t)offsetof(global_State, gc.graynum)); + emit_dn(as, ARMI_ADD|ARMI_K12|1, RID_TMP, RID_TMP); + emit_lso(as, ARMI_LDR, RID_TMP, gr, + (int32_t)offsetof(global_State, gc.graynum)); + + emit_lso(as, ARMI_STR, RID_TMP, gr, + (int32_t)offsetof(global_State, gc.blacknum)); + emit_dn(as, ARMI_SUB|ARMI_K12|1, RID_TMP, RID_TMP); + emit_lso(as, ARMI_LDR, RID_TMP, gr, + (int32_t)offsetof(global_State, gc.blacknum)); + emit_lso(as, ARMI_STR, link, tab, (int32_t)offsetof(GCtab, gclist)); emit_lso(as, ARMI_STRB, mark, tab, (int32_t)offsetof(GCtab, marked)); emit_lso(as, ARMI_STR, tab, gr, diff --git a/src/lj_asm_arm64.h b/src/lj_asm_arm64.h index 8fd92e7..6be912e 100644 --- a/src/lj_asm_arm64.h +++ b/src/lj_asm_arm64.h @@ -1220,9 +1220,16 @@ static void asm_cnew(ASMState *as, IRIns *ir) /* Initialize gct and ctypeid. lj_mem_newgco() already sets marked. */ { Reg r = (id < 65536) ? RID_X1 : ra_allock(as, id, allow); + allow = rset_exclude(allow, r); + Reg gr = ra_allock(as, i64ptr(J2G(as->J)), allow); emit_lso(as, A64I_STRB, RID_TMP, RID_RET, offsetof(GCcdata, gct)); emit_lso(as, A64I_STRH, r, RID_RET, offsetof(GCcdata, ctypeid)); emit_d(as, A64I_MOVZw | A64F_U16(~LJ_TCDATA), RID_TMP); + emit_lso(as, A64I_STRx, RID_TMP, gr, + (int32_t)offsetof(global_State, gc.cdatanum)); + emit_dn(as, (A64I_ADDx^A64I_K12) | A64F_U12(1), RID_TMP, RID_TMP); + emit_lso(as, A64I_LDRx, RID_TMP, gr, + (int32_t)offsetof(global_State, gc.cdatanum)); if (id < 65536) emit_d(as, A64I_MOVZw | A64F_U16(id), RID_X1); } args[0] = ASMREF_L; /* lua_State *L */ @@ -1245,6 +1252,18 @@ static void asm_tbar(ASMState *as, IRIns *ir) rset_exclude(rset_exclude(RSET_GPR, tab), link)); Reg mark = RID_TMP; MCLabel l_end = emit_label(as); + emit_lso(as, A64I_STRx, RID_TMP, gr, + (int32_t)offsetof(global_State, gc.graynum)); + emit_dn(as, (A64I_ADDx^A64I_K12) | A64F_U12(1), RID_TMP, RID_TMP); + emit_lso(as, A64I_LDRx, RID_TMP, gr, + (int32_t)offsetof(global_State, gc.graynum)); + + emit_lso(as, A64I_STRx, RID_TMP, gr, + (int32_t)offsetof(global_State, gc.blacknum)); + emit_dn(as, (A64I_SUBx^A64I_K12) | A64F_U12(1), RID_TMP, RID_TMP); + emit_lso(as, A64I_LDRx, RID_TMP, gr, + (int32_t)offsetof(global_State, gc.blacknum)); + emit_lso(as, A64I_STRx, link, tab, (int32_t)offsetof(GCtab, gclist)); emit_lso(as, A64I_STRB, mark, tab, (int32_t)offsetof(GCtab, marked)); emit_lso(as, A64I_STRx, tab, gr, diff --git a/src/lj_asm_mips.h b/src/lj_asm_mips.h index affe7d8..0a1900f 100644 --- a/src/lj_asm_mips.h +++ b/src/lj_asm_mips.h @@ -1473,11 +1473,15 @@ static void asm_cnew(ASMState *as, IRIns *ir) return; } + /* Code incrementing cdatanum is sparse to avoid mips data hazards. */ + emit_setgl(as, RID_RET+2, gc.cdatanum); /* Initialize gct and ctypeid. lj_mem_newgco() already sets marked. */ emit_tsi(as, MIPSI_SB, RID_RET+1, RID_RET, offsetof(GCcdata, gct)); emit_tsi(as, MIPSI_SH, RID_TMP, RID_RET, offsetof(GCcdata, ctypeid)); + emit_tsi(as, MIPSI_AADDIU, RID_RET+2, RID_RET+2, 1); emit_ti(as, MIPSI_LI, RID_RET+1, ~LJ_TCDATA); emit_ti(as, MIPSI_LI, RID_TMP, id); /* Lower 16 bit used. Sign-ext ok. */ + emit_getgl(as, RID_RET+2, gc.cdatanum); args[0] = ASMREF_L; /* lua_State *L */ args[1] = ASMREF_TMP1; /* MSize size */ asm_gencall(as, ci, args); @@ -1500,6 +1504,12 @@ static void asm_tbar(ASMState *as, IRIns *ir) emit_tsi(as, MIPSI_SB, mark, tab, (int32_t)offsetof(GCtab, marked)); emit_setgl(as, tab, gc.grayagain); emit_getgl(as, link, gc.grayagain); + emit_setgl(as, RID_TMP, gc.graynum); + emit_tsi(as, MIPSI_AADDIU, RID_TMP, RID_TMP, 1); + emit_getgl(as, RID_TMP, gc.graynum); + emit_setgl(as, RID_TMP, gc.blacknum); + emit_tsi(as, MIPSI_AADDIU, RID_TMP, RID_TMP, -1); + emit_getgl(as, RID_TMP, gc.blacknum); emit_dst(as, MIPSI_XOR, mark, mark, RID_TMP); /* Clear black bit. */ emit_branch(as, MIPSI_BEQ, RID_TMP, RID_ZERO, l_end); emit_tsi(as, MIPSI_ANDI, RID_TMP, mark, LJ_GC_BLACK); diff --git a/src/lj_asm_ppc.h b/src/lj_asm_ppc.h index 6daa861..be4b872 100644 --- a/src/lj_asm_ppc.h +++ b/src/lj_asm_ppc.h @@ -1057,6 +1057,9 @@ static void asm_cnew(ASMState *as, IRIns *ir) return; } + emit_setgl(as, RID_TMP, gc.cdatanum); + emit_tai(as, PPCI_ADDIC, RID_TMP, RID_TMP, 1); + emit_getgl(as, RID_TMP, gc.cdatanum); /* Initialize gct and ctypeid. lj_mem_newgco() already sets marked. */ emit_tai(as, PPCI_STB, RID_RET+1, RID_RET, offsetof(GCcdata, gct)); emit_tai(as, PPCI_STH, RID_TMP, RID_RET, offsetof(GCcdata, ctypeid)); @@ -1086,6 +1089,12 @@ static void asm_tbar(ASMState *as, IRIns *ir) lua_assert(LJ_GC_BLACK == 0x04); emit_rot(as, PPCI_RLWINM, mark, mark, 0, 30, 28); /* Clear black bit. */ emit_getgl(as, link, gc.grayagain); + emit_setgl(as, RID_TMP, gc.graynum); + emit_tai(as, PPCI_ADDIC, RID_TMP, RID_TMP, 1); + emit_getgl(as, RID_TMP, gc.graynum); + emit_setgl(as, RID_TMP, gc.blacknum); + emit_tai(as, PPCI_ADDIC, RID_TMP, RID_TMP, -1); + emit_getgl(as, RID_TMP, gc.blacknum); emit_condbranch(as, PPCI_BC|PPCF_Y, CC_EQ, l_end); emit_asi(as, PPCI_ANDIDOT, RID_TMP, mark, LJ_GC_BLACK); emit_tai(as, PPCI_LBZ, mark, tab, (int32_t)offsetof(GCtab, marked)); diff --git a/src/lj_asm_x86.h b/src/lj_asm_x86.h index 3e189b1..8c9cdaa 100644 --- a/src/lj_asm_x86.h +++ b/src/lj_asm_x86.h @@ -1835,6 +1835,10 @@ static void asm_cnew(ASMState *as, IRIns *ir) return; } + /* Increment cdatanum counter by address directly. */ + emit_i8(as, 1); + emit_rmro(as, XO_ARITHi8, XOg_ADD, RID_NONE, + ptr2addr(&J2G(as->J)->gc.cdatanum)); /* Combine initialization of marked, gct and ctypeid. */ emit_movtomro(as, RID_ECX, RID_RET, offsetof(GCcdata, marked)); emit_gri(as, XG_ARITHi(XOg_OR), RID_ECX, @@ -1861,6 +1865,12 @@ static void asm_tbar(ASMState *as, IRIns *ir) emit_movtomro(as, tmp|REX_GC64, tab, offsetof(GCtab, gclist)); emit_setgl(as, tab, gc.grayagain); emit_getgl(as, tmp, gc.grayagain); + emit_i8(as, 1); + emit_rmro(as, XO_ARITHi8, XOg_ADD, RID_NONE, + ptr2addr(&J2G(as->J)->gc.graynum)); + emit_i8(as, 1); + emit_rmro(as, XO_ARITHi8, XOg_SUB, RID_NONE, + ptr2addr(&J2G(as->J)->gc.blacknum)); emit_i8(as, ~LJ_GC_BLACK); emit_rmro(as, XO_ARITHib, XOg_AND, tab, offsetof(GCtab, marked)); emit_sjcc(as, CC_Z, l_end); diff --git a/src/lj_cdata.c b/src/lj_cdata.c index 68e16d7..5870ac4 100644 --- a/src/lj_cdata.c +++ b/src/lj_cdata.c @@ -46,6 +46,7 @@ GCcdata *lj_cdata_newv(lua_State *L, CTypeID id, CTSize sz, CTSize align) cd->marked |= 0x80; cd->gct = ~LJ_TCDATA; cd->ctypeid = id; + g->gc.cdatanum++; return cd; } @@ -63,6 +64,13 @@ void LJ_FASTCALL lj_cdata_free(global_State *g, GCcdata *cd) { if (LJ_UNLIKELY(cd->marked & LJ_GC_CDATA_FIN)) { GCobj *root; + /* Restore old value of color incremented before free. */ + if (iswhite(obj2gco(cd))) + g->gc.whitenum++; + else if (isblack(obj2gco(cd))) + g->gc.blacknum++; + else + g->gc.graynum++; makewhite(g, obj2gco(cd)); markfinalized(obj2gco(cd)); if ((root = gcref(g->gc.mmudata)) != NULL) { @@ -82,6 +90,7 @@ void LJ_FASTCALL lj_cdata_free(global_State *g, GCcdata *cd) } else { lj_mem_free(g, memcdatav(cd), sizecdatav(cd)); } + g->gc.cdatanum--; } void lj_cdata_setfin(lua_State *L, GCcdata *cd, GCobj *obj, uint32_t it) diff --git a/src/lj_cdata.h b/src/lj_cdata.h index 5bb0f5d..66b023b 100644 --- a/src/lj_cdata.h +++ b/src/lj_cdata.h @@ -45,6 +45,7 @@ static LJ_AINLINE GCcdata *lj_cdata_new(CTState *cts, CTypeID id, CTSize sz) cd = (GCcdata *)lj_mem_newgco(cts->L, sizeof(GCcdata) + sz); cd->gct = ~LJ_TCDATA; cd->ctypeid = ctype_check(cts, id); + G(cts->L)->gc.cdatanum++; return cd; } @@ -54,6 +55,7 @@ static LJ_AINLINE GCcdata *lj_cdata_new_(lua_State *L, CTypeID id, CTSize sz) GCcdata *cd = (GCcdata *)lj_mem_newgco(L, sizeof(GCcdata) + sz); cd->gct = ~LJ_TCDATA; cd->ctypeid = id; + G(L)->gc.cdatanum++; return cd; } diff --git a/src/lj_gc.c b/src/lj_gc.c index 1c8e6ce..c7616f6 100644 --- a/src/lj_gc.c +++ b/src/lj_gc.c @@ -33,8 +33,18 @@ #define GCFINALIZECOST 100 /* Macros to set GCobj colors and flags. */ -#define white2gray(x) ((x)->gch.marked &= (uint8_t)~LJ_GC_WHITES) -#define gray2black(x) ((x)->gch.marked |= LJ_GC_BLACK) +#define white2gray(g, x) \ + ( (g)->gc.whitenum--, \ + (g)->gc.graynum++, \ + lua_assert((ssize_t)(g)->gc.whitenum >=2), \ + (x)->gch.marked &= (uint8_t)~LJ_GC_WHITES ) + +#define gray2black(g, x) \ + ( (g)->gc.graynum--, \ + (g)->gc.blacknum++, \ + lua_assert((ssize_t)(g)->gc.graynum >=0), \ + (x)->gch.marked |= LJ_GC_BLACK ) + #define isfinalized(u) ((u)->marked & LJ_GC_FINALIZED) /* -- Mark phase ---------------------------------------------------------- */ @@ -49,24 +59,30 @@ { if (iswhite(obj2gco(o))) gc_mark(g, obj2gco(o)); } /* Mark a string object. */ -#define gc_mark_str(s) ((s)->marked &= (uint8_t)~LJ_GC_WHITES) +#define gc_mark_str(g, s) \ + { if ((s)->marked & LJ_GC_WHITES) { \ + (g)->gc.whitenum--; \ + (g)->gc.graynum++; \ + lua_assert((ssize_t)(g)->gc.whitenum >=2); \ + }; \ + (s)->marked &= (uint8_t)~LJ_GC_WHITES; } /* Mark a white GCobj. */ static void gc_mark(global_State *g, GCobj *o) { int gct = o->gch.gct; lua_assert(iswhite(o) && !isdead(g, o)); - white2gray(o); + white2gray(g, o); if (LJ_UNLIKELY(gct == ~LJ_TUDATA)) { GCtab *mt = tabref(gco2ud(o)->metatable); - gray2black(o); /* Userdata are never gray. */ + gray2black(g, o); /* Userdata are never gray. */ if (mt) gc_markobj(g, mt); gc_markobj(g, tabref(gco2ud(o)->env)); } else if (LJ_UNLIKELY(gct == ~LJ_TUPVAL)) { GCupval *uv = gco2uv(o); gc_marktv(g, uvval(uv)); if (uv->closed) - gray2black(o); /* Closed upvalues are never gray. */ + gray2black(g, o); /* Closed upvalues are never gray. */ } else if (gct != ~LJ_TSTR && gct != ~LJ_TCDATA) { lua_assert(gct == ~LJ_TFUNC || gct == ~LJ_TTAB || gct == ~LJ_TTHREAD || gct == ~LJ_TPROTO || gct == ~LJ_TTRACE); @@ -230,7 +246,7 @@ static void gc_marktrace(global_State *g, TraceNo traceno) GCobj *o = obj2gco(traceref(G2J(g), traceno)); lua_assert(traceno != G2J(g)->cur.traceno); if (iswhite(o)) { - white2gray(o); + white2gray(g, o); setgcrefr(o->gch.gclist, g->gc.gray); setgcref(g->gc.gray, o); } @@ -264,7 +280,7 @@ static void gc_traverse_trace(global_State *g, GCtrace *T) static void gc_traverse_proto(global_State *g, GCproto *pt) { ptrdiff_t i; - gc_mark_str(proto_chunkname(pt)); + gc_mark_str(g, proto_chunkname(pt)); for (i = -(ptrdiff_t)pt->sizekgc; i < 0; i++) /* Mark collectable consts. */ gc_markobj(g, proto_kgc(pt, i)); #if LJ_HASJIT @@ -310,12 +326,12 @@ static size_t propagatemark(global_State *g) GCobj *o = gcref(g->gc.gray); int gct = o->gch.gct; lua_assert(isgray(o)); - gray2black(o); + gray2black(g, o); setgcrefr(g->gc.gray, o->gch.gclist); /* Remove from gray list. */ if (LJ_LIKELY(gct == ~LJ_TTAB)) { GCtab *t = gco2tab(o); if (gc_traverse_tab(g, t) > 0) - black2gray(o); /* Keep weak tables gray. */ + black2gray(g, o); /* Keep weak tables gray. */ return sizeof(GCtab) + sizeof(TValue) * t->asize + (t->hmask ? sizeof(Node) * (t->hmask + 1) : 0); } else if (LJ_LIKELY(gct == ~LJ_TFUNC)) { @@ -331,7 +347,7 @@ static size_t propagatemark(global_State *g) lua_State *th = gco2th(o); setgcrefr(th->gclist, g->gc.grayagain); setgcref(g->gc.grayagain, o); - black2gray(o); /* Threads are never black. */ + black2gray(g, o); /* Threads are never black. */ gc_traverse_thread(g, th); return sizeof(lua_State) + sizeof(TValue) * th->stacksize; } else { @@ -403,6 +419,16 @@ static GCRef *gc_sweep(global_State *g, GCRef *p, uint32_t lim) setgcrefr(*p, o->gch.nextgc); if (o == gcref(g->gc.root)) setgcrefr(g->gc.root, o->gch.nextgc); /* Adjust list anchor. */ + if (iswhite(o)) { + g->gc.whitenum--; + lua_assert((ssize_t)g->gc.whitenum >= 2); + } else if (isblack(o)) { + g->gc.blacknum--; + lua_assert((ssize_t)g->gc.blacknum >= 0); + } else { + g->gc.graynum--; + lua_assert((ssize_t)g->gc.graynum >= 0); + } gc_freefunc[o->gch.gct - ~LJ_TSTR](g, o); } } @@ -430,6 +456,16 @@ static GCRef *gc_sweep_str_chain(global_State *g, GCRef *p) } else { /* Otherwise value is dead, free it. */ lua_assert(isdead(g, o) || ow == LJ_GC_SFIXED); setgcrefr(*p, o->gch.nextgc); + if (iswhite(o)) { + g->gc.whitenum--; + lua_assert((ssize_t)g->gc.whitenum >= 2); + } else if (isblack(o)) { + g->gc.blacknum--; + lua_assert((ssize_t)g->gc.blacknum >= 0); + } else { + g->gc.graynum--; + lua_assert((ssize_t)g->gc.graynum >= 0); + } lj_str_free(g, &o->str); } } @@ -437,11 +473,11 @@ static GCRef *gc_sweep_str_chain(global_State *g, GCRef *p) } /* Check whether we can clear a key or a value slot from a table. */ -static int gc_mayclear(cTValue *o, int val) +static int gc_mayclear(global_State *g, cTValue *o, int val) { if (tvisgcv(o)) { /* Only collectable objects can be weak references. */ if (tvisstr(o)) { /* But strings cannot be used as weak references. */ - gc_mark_str(strV(o)); /* And need to be marked. */ + gc_mark_str(g, strV(o)); /* And need to be marked. */ return 0; } if (iswhite(gcV(o))) @@ -453,7 +489,7 @@ static int gc_mayclear(cTValue *o, int val) } /* Clear collected entries from weak tables. */ -static void gc_clearweak(GCobj *o) +static void gc_clearweak(global_State *g, GCobj *o) { while (o) { GCtab *t = gco2tab(o); @@ -463,7 +499,7 @@ static void gc_clearweak(GCobj *o) for (i = 0; i < asize; i++) { /* Clear array slot when value is about to be collected. */ TValue *tv = arrayslot(t, i); - if (gc_mayclear(tv, 1)) + if (gc_mayclear(g, tv, 1)) setnilV(tv); } } @@ -473,8 +509,8 @@ static void gc_clearweak(GCobj *o) for (i = 0; i <= hmask; i++) { Node *n = &node[i]; /* Clear hash slot when key or value is about to be collected. */ - if (!tvisnil(&n->val) && (gc_mayclear(&n->key, 0) || - gc_mayclear(&n->val, 1))) + if (!tvisnil(&n->val) && (gc_mayclear(g, &n->key, 0) || + gc_mayclear(g, &n->val, 1))) setnilV(&n->val); } } @@ -619,12 +655,21 @@ static void atomic(global_State *g, lua_State *L) udsize += gc_propagate_gray(g); /* And propagate the marks. */ /* All marking done, clear weak tables. */ - gc_clearweak(gcref(g->gc.weak)); + gc_clearweak(g, gcref(g->gc.weak)); lj_buf_shrink(L, &g->tmpbuf); /* Shrink temp buffer. */ /* Prepare for sweep phase. */ g->gc.currentwhite = (uint8_t)otherwhite(g); /* Flip current white. */ + if (g->strempty.marked & (uint8_t)LJ_GC_BLACK) { + g->gc.blacknum--; + g->gc.whitenum++; + lua_assert((ssize_t)g->gc.blacknum >=0); + } else if ((g->strempty.marked & (uint8_t)LJ_GC_WHITES) == 0) { + g->gc.graynum--; + g->gc.whitenum++; + lua_assert((ssize_t)g->gc.graynum >=0); + } g->strempty.marked = g->gc.currentwhite; setmref(g->gc.sweep, &g->gc.root); g->gc.estimate = g->gc.total - (GCSize)udsize; /* Initial estimate. */ @@ -634,6 +679,7 @@ static void atomic(global_State *g, lua_State *L) static size_t gc_onestep(lua_State *L) { global_State *g = G(L); + g->gc.state_count[g->gc.state]++; switch (g->gc.state) { case GCSpause: gc_mark_start(g); /* Start a new GC cycle by marking all GC roots. */ @@ -808,8 +854,18 @@ void LJ_FASTCALL lj_gc_barrieruv(global_State *g, TValue *tv) (*((uint8_t *)(x) - offsetof(GCupval, tv) + offsetof(GCupval, marked))) if (g->gc.state == GCSpropagate || g->gc.state == GCSatomic) gc_mark(g, gcV(tv)); - else + else { + if (TV2MARKED(tv) & (uint8_t)LJ_GC_BLACK) { + g->gc.blacknum--; + g->gc.whitenum++; + lua_assert((ssize_t)g->gc.blacknum >=0); + } else if ((TV2MARKED(tv) & (uint8_t)LJ_GC_WHITES) == 0) { + g->gc.graynum--; + g->gc.whitenum++; + lua_assert((ssize_t)g->gc.graynum >=0); + } TV2MARKED(tv) = (TV2MARKED(tv) & (uint8_t)~LJ_GC_COLORS) | curwhite(g); + } #undef TV2MARKED } @@ -825,7 +881,7 @@ void lj_gc_closeuv(global_State *g, GCupval *uv) setgcref(g->gc.root, o); if (isgray(o)) { /* A closed upvalue is never gray, so fix this. */ if (g->gc.state == GCSpropagate || g->gc.state == GCSatomic) { - gray2black(o); /* Make it black and preserve invariant. */ + gray2black(g, o); /* Make it black and preserve invariant. */ if (tviswhite(&uv->tv)) lj_gc_barrierf(g, o, gcV(&uv->tv)); } else { @@ -857,6 +913,8 @@ void *lj_mem_realloc(lua_State *L, void *p, GCSize osz, GCSize nsz) lua_assert((nsz == 0) == (p == NULL)); lua_assert(checkptrGC(p)); g->gc.total = (g->gc.total - osz) + nsz; + g->gc.allocated += nsz; + g->gc.freed += osz; return p; } @@ -869,6 +927,7 @@ void * LJ_FASTCALL lj_mem_newgco(lua_State *L, GCSize size) lj_err_mem(L); lua_assert(checkptrGC(o)); g->gc.total += size; + g->gc.allocated += size; setgcrefr(o->gch.nextgc, g->gc.root); setgcref(g->gc.root, o); newwhite(g, o); diff --git a/src/lj_gc.h b/src/lj_gc.h index 669bbe9..25e319a 100644 --- a/src/lj_gc.h +++ b/src/lj_gc.h @@ -8,11 +8,6 @@ #include "lj_obj.h" -/* Garbage collector states. Order matters. */ -enum { - GCSpause, GCSpropagate, GCSatomic, GCSsweepstring, GCSsweep, GCSfinalize -}; - /* Bitmasks for marked field of GCobj. */ #define LJ_GC_WHITE0 0x01 #define LJ_GC_WHITE1 0x02 @@ -37,11 +32,28 @@ enum { #define isdead(g, v) ((v)->gch.marked & otherwhite(g) & LJ_GC_WHITES) #define curwhite(g) ((g)->gc.currentwhite & LJ_GC_WHITES) -#define newwhite(g, x) (obj2gco(x)->gch.marked = (uint8_t)curwhite(g)) +#define newwhite(g, x) \ + ( (g)->gc.whitenum++, \ + obj2gco(x)->gch.marked = (uint8_t)curwhite(g) ) + #define makewhite(g, x) \ - ((x)->gch.marked = ((x)->gch.marked & (uint8_t)~LJ_GC_COLORS) | curwhite(g)) -#define flipwhite(x) ((x)->gch.marked ^= LJ_GC_WHITES) -#define black2gray(x) ((x)->gch.marked &= (uint8_t)~LJ_GC_BLACK) + ( iswhite(x) ? /* Nothing to do */ : \ + isblack(x) ? ((g)->gc.blacknum--, (g)->gc.whitenum++) : \ + (lua_assert(isgray(x)), (g)->gc.graynum--, (g)->gc.whitenum++), \ + lua_assert((ssize_t)(g)->gc.graynum >=0), \ + lua_assert((ssize_t)(g)->gc.blacknum >=0), \ + (x)->gch.marked = ((x)->gch.marked & (uint8_t)~LJ_GC_COLORS) | curwhite(g) ) + +#define flipwhite(x) \ + ( lua_assert(!isgray(x) && ! isblack(x)), \ + (x)->gch.marked ^= LJ_GC_WHITES ) + +#define black2gray(g, x) \ + ( (g)->gc.blacknum--, \ + (g)->gc.graynum++, \ + lua_assert((ssize_t)(g)->gc.blacknum >=0), \ + (x)->gch.marked &= (uint8_t)~LJ_GC_BLACK ) + #define fixstring(s) ((s)->marked |= LJ_GC_FIXED) #define markfinalized(x) ((x)->gch.marked |= LJ_GC_FINALIZED) @@ -83,7 +95,7 @@ static LJ_AINLINE void lj_gc_barrierback(global_State *g, GCtab *t) GCobj *o = obj2gco(t); lua_assert(isblack(o) && !isdead(g, o)); lua_assert(g->gc.state != GCSfinalize && g->gc.state != GCSpause); - black2gray(o); + black2gray(g, o); setgcrefr(t->gclist, g->gc.grayagain); setgcref(g->gc.grayagain, o); } @@ -117,6 +129,7 @@ LJ_FUNC void *lj_mem_grow(lua_State *L, void *p, static LJ_AINLINE void lj_mem_free(global_State *g, void *p, size_t osize) { g->gc.total -= (GCSize)osize; + g->gc.freed += osize; g->allocf(g->allocd, p, osize, 0); } diff --git a/src/lj_jit.h b/src/lj_jit.h index 7eb3d2a..d82292f 100644 --- a/src/lj_jit.h +++ b/src/lj_jit.h @@ -474,6 +474,9 @@ typedef struct jit_State { MCode *mcbot; /* Bottom of current mcode area. */ size_t szmcarea; /* Size of current mcode area. */ size_t szallmcarea; /* Total size of all allocated mcode areas. */ + size_t tracenum; /* Overall number of traces. */ + size_t nsnaprestore; /* Overall number of snap restores. */ + size_t ntraceabort; /* Overall number of abort traces. */ TValue errinfo; /* Additional info element for trace errors. */ diff --git a/src/lj_obj.h b/src/lj_obj.h index f368578..c9a6904 100644 --- a/src/lj_obj.h +++ b/src/lj_obj.h @@ -571,6 +571,17 @@ typedef enum { #define basemt_obj(g, o) ((g)->gcroot[GCROOT_BASEMT+itypemap(o)]) #define mmname_str(g, mm) (strref((g)->gcroot[GCROOT_MMNAME+(mm)])) +/* Garbage collector states. Order matters. */ +enum { + GCSpause, /* Start a GC cycle and mark the root set.*/ + GCSpropagate, /* One gray object is processed. */ + GCSatomic, /* Atomic transition from mark to sweep phase. */ + GCSsweepstring, /* Sweep one chain of strings. */ + GCSsweep, /* Sweep few objects from root. */ + GCSfinalize, /* Finalize one userdata or cdata object. */ + GCSmax +}; + typedef struct GCState { GCSize total; /* Memory currently allocated. */ GCSize threshold; /* Memory threshold. */ @@ -589,6 +600,18 @@ typedef struct GCState { GCSize estimate; /* Estimate of memory actually in use. */ MSize stepmul; /* Incremental GC step granularity. */ MSize pause; /* Pause between successive GC cycles. */ + + size_t freed; /* Total amount of freed memory. */ + size_t allocated; /* Total amount of allocated memory. */ + size_t state_count[GCSmax]; /* Count of incremental GC steps per state. */ + size_t tabnum; /* Amount of allocated table objects. */ + size_t udatanum; /* Amount of allocated udata objects. */ +#ifdef LJ_HASFFI + size_t cdatanum; /* Amount of allocated cdata objects. */ +#endif + size_t whitenum; /* Amount of white objects. */ + size_t graynum; /* Amount of grey objects. */ + size_t blacknum; /* Amount of black objects. */ } GCState; /* Global state, shared by all threads of a Lua universe. */ @@ -602,22 +625,24 @@ typedef struct global_State { BloomFilter next[2]; } strbloom; #endif + size_t strhash_hit; /* Strings amount found in string hash. */ + size_t strhash_miss; /* Strings amount allocated and put into string hash. */ lua_Alloc allocf; /* Memory allocator. */ void *allocd; /* Memory allocator data. */ GCState gc; /* Garbage collector. */ - volatile int32_t vmstate; /* VM state or current JIT code trace number. */ + GCRef mainthref; /* Link to main thread. */ SBuf tmpbuf; /* Temporary string buffer. */ GCstr strempty; /* Empty string. */ uint8_t stremptyz; /* Zero terminator of empty string. */ uint8_t hookmask; /* Hook mask. */ uint8_t dispatchmode; /* Dispatch mode. */ uint8_t vmevmask; /* VM event mask. */ - GCRef mainthref; /* Link to main thread. */ + int32_t hookcount; /* Instruction hook countdown. */ TValue registrytv; /* Anchor for registry. */ TValue tmptv, tmptv2; /* Temporary TValues. */ Node nilnode; /* Fallback 1-element hash part (nil key and value). */ GCupval uvhead; /* Head of double-linked list of all open upvalues. */ - int32_t hookcount; /* Instruction hook countdown. */ + volatile int32_t vmstate; /* VM state or current JIT code trace number. */ int32_t hookcstart; /* Start count for instruction hook counter. */ lua_Hook hookf; /* Hook function. */ lua_CFunction wrapf; /* Wrapper for C function calls. */ diff --git a/src/lj_snap.c b/src/lj_snap.c index 7554caf..4cf27e7 100644 --- a/src/lj_snap.c +++ b/src/lj_snap.c @@ -904,6 +904,7 @@ const BCIns *lj_snap_restore(jit_State *J, void *exptr) L->top = frame + snap->nslots; break; } + J->nsnaprestore++; return pc; } diff --git a/src/lj_state.c b/src/lj_state.c index 632dd07..d85a02a 100644 --- a/src/lj_state.c +++ b/src/lj_state.c @@ -164,6 +164,10 @@ static void close_state(lua_State *L) lj_gc_freeall(g); lua_assert(gcref(g->gc.root) == obj2gco(L)); lua_assert(g->strnum == 0); + lua_assert(g->gc.blacknum == 0); + lua_assert(g->gc.graynum == 0); + /* At least lua_State, strempty and state enviroment. */ + lua_assert(g->gc.whitenum >= 2); lj_trace_freestate(g); #if LJ_HASFFI lj_ctype_freestate(g); @@ -214,7 +218,8 @@ LUA_API lua_State *lua_newstate(lua_Alloc f, void *ud) g->gc.state = GCSpause; setgcref(g->gc.root, obj2gco(L)); setmref(g->gc.sweep, &g->gc.root); - g->gc.total = sizeof(GG_State); + g->gc.allocated = g->gc.total = sizeof(GG_State); + g->gc.whitenum = 2; /* State and strempty. */ g->gc.pause = LUAI_GCPAUSE; g->gc.stepmul = LUAI_GCMUL; lj_dispatch_init((GG_State *)L); diff --git a/src/lj_str.c b/src/lj_str.c index 24e90ca..8ff955e 100644 --- a/src/lj_str.c +++ b/src/lj_str.c @@ -222,6 +222,7 @@ GCstr *lj_str_new(lua_State *L, const char *str, size_t lenx) str_fastcmp(str, strdata(sx), len) == 0) { /* Resurrect if dead. Can only happen with fixstring() (keywords). */ if (isdead(g, o)) flipwhite(o); + g->strhash_hit++; return sx; /* Return existing string. */ } o = gcnext(o); @@ -234,6 +235,7 @@ GCstr *lj_str_new(lua_State *L, const char *str, size_t lenx) memcmp(str, strdata(sx), len) == 0) { /* Resurrect if dead. Can only happen with fixstring() (keywords). */ if (isdead(g, o)) flipwhite(o); + g->strhash_hit++; return sx; /* Return existing string. */ } o = gcnext(o); @@ -266,6 +268,7 @@ GCstr *lj_str_new(lua_State *L, const char *str, size_t lenx) if (sx->hash == fh && sx->len == len && str_fastcmp(str, strdata(sx), len) == 0) { /* Resurrect if dead. Can only happen with fixstring() (keywords). */ if (isdead(g, o)) flipwhite(o); + g->strhash_hit++; return sx; /* Return existing string. */ } o = gcnext(o); @@ -276,6 +279,7 @@ GCstr *lj_str_new(lua_State *L, const char *str, size_t lenx) if (sx->hash == fh && sx->len == len && memcmp(str, strdata(sx), len) == 0) { /* Resurrect if dead. Can only happen with fixstring() (keywords). */ if (isdead(g, o)) flipwhite(o); + g->strhash_hit++; return sx; /* Return existing string. */ } o = gcnext(o); @@ -293,6 +297,7 @@ GCstr *lj_str_new(lua_State *L, const char *str, size_t lenx) } } #endif + g->strhash_miss++; /* Nope, create a new string. */ s = lj_mem_newt(L, sizeof(GCstr)+len+1, GCstr); newwhite(g, s); diff --git a/src/lj_tab.c b/src/lj_tab.c index ff216f3..c5f358e 100644 --- a/src/lj_tab.c +++ b/src/lj_tab.c @@ -141,6 +141,7 @@ static GCtab *newtab(lua_State *L, uint32_t asize, uint32_t hbits) } if (hbits) newhpart(L, t, hbits); + G(L)->gc.tabnum++; return t; } @@ -240,6 +241,7 @@ void LJ_FASTCALL lj_tab_free(global_State *g, GCtab *t) lj_mem_free(g, t, sizetabcolo((uint32_t)t->colo & 0x7f)); else lj_mem_freet(g, t); + g->gc.tabnum--; } /* -- Table resizing ------------------------------------------------------ */ diff --git a/src/lj_trace.c b/src/lj_trace.c index d85b47f..9bcbce6 100644 --- a/src/lj_trace.c +++ b/src/lj_trace.c @@ -176,6 +176,7 @@ void LJ_FASTCALL lj_trace_free(global_State *g, GCtrace *T) lj_mem_free(g, T, ((sizeof(GCtrace)+7)&~7) + (T->nins-T->nk)*sizeof(IRIns) + T->nsnap*sizeof(SnapShot) + T->nsnapmap*sizeof(SnapEntry)); + J->tracenum--; } /* Re-enable compiling a prototype by unpatching any modified bytecode. */ @@ -538,8 +539,10 @@ static int trace_downrec(jit_State *J) /* Restart recording at the return instruction. */ lua_assert(J->pt != NULL); lua_assert(bc_isret(bc_op(*J->pc))); - if (bc_op(*J->pc) == BC_RETM) + if (bc_op(*J->pc) == BC_RETM) { + J->ntraceabort++; return 0; /* NYI: down-recursion with RETM. */ + } J->parent = 0; J->exitno = 0; J->state = LJ_TRACE_RECORD; @@ -616,6 +619,7 @@ static int trace_abort(jit_State *J) return trace_downrec(J); else if (e == LJ_TRERR_MCODEAL) lj_trace_flushall(L); + J->ntraceabort++; return 0; } diff --git a/src/lj_udata.c b/src/lj_udata.c index bd0321b..70c722a 100644 --- a/src/lj_udata.c +++ b/src/lj_udata.c @@ -24,11 +24,13 @@ GCudata *lj_udata_new(lua_State *L, MSize sz, GCtab *env) /* Chain to userdata list (after main thread). */ setgcrefr(ud->nextgc, mainthread(g)->nextgc); setgcref(mainthread(g)->nextgc, obj2gco(ud)); + g->gc.udatanum++; return ud; } void LJ_FASTCALL lj_udata_free(global_State *g, GCudata *ud) { + g->gc.udatanum--; lj_mem_free(g, ud, sizeudata(ud)); } diff --git a/src/vm_arm.dasc b/src/vm_arm.dasc index d4cdaf5..ea3bf31 100644 --- a/src/vm_arm.dasc +++ b/src/vm_arm.dasc @@ -227,6 +227,12 @@ | |// Move table write barrier back. Overwrites mark and tmp. |.macro barrierback, tab, mark, tmp +| ldr tmp, [DISPATCH, #DISPATCH_GL(gc.graynum)] +| add tmp, tmp, #1 +| str tmp, [DISPATCH, #DISPATCH_GL(gc.graynum)] +| ldr tmp, [DISPATCH, #DISPATCH_GL(gc.blacknum)] +| sub tmp, tmp, #1 +| str tmp, [DISPATCH, #DISPATCH_GL(gc.blacknum)] | ldr tmp, [DISPATCH, #DISPATCH_GL(gc.grayagain)] | bic mark, mark, #LJ_GC_BLACK // black2gray(tab) | str tab, [DISPATCH, #DISPATCH_GL(gc.grayagain)] diff --git a/src/vm_arm64.dasc b/src/vm_arm64.dasc index 3eaf376..a182f35 100644 --- a/src/vm_arm64.dasc +++ b/src/vm_arm64.dasc @@ -280,6 +280,12 @@ | |// Move table write barrier back. Overwrites mark and tmp. |.macro barrierback, tab, mark, tmp +| ldr tmp, GL->gc.blacknum +| sub tmp, tmp, #1 +| str tmp, GL->gc.blacknum +| ldr tmp, GL->gc.graynum +| add tmp, tmp, #1 +| str tmp, GL->gc.graynum | ldr tmp, GL->gc.grayagain | and mark, mark, #~LJ_GC_BLACK // black2gray(tab) | str tab, GL->gc.grayagain diff --git a/src/vm_mips.dasc b/src/vm_mips.dasc index 1afd611..7de8beb 100644 --- a/src/vm_mips.dasc +++ b/src/vm_mips.dasc @@ -351,6 +351,12 @@ | |// Move table write barrier back. Overwrites mark and tmp. |.macro barrierback, tab, mark, tmp, target +| lw tmp, DISPATCH_GL(gc.blacknum)(DISPATCH) +| addiu tmp, tmp, -1 +| sw tmp, DISPATCH_GL(gc.blacknum)(DISPATCH) +| lw tmp, DISPATCH_GL(gc.graynum)(DISPATCH) +| addiu tmp, tmp, 1 +| sw tmp, DISPATCH_GL(gc.graynum)(DISPATCH) | lw tmp, DISPATCH_GL(gc.grayagain)(DISPATCH) | andi mark, mark, ~LJ_GC_BLACK & 255 // black2gray(tab) | sw tab, DISPATCH_GL(gc.grayagain)(DISPATCH) diff --git a/src/vm_mips64.dasc b/src/vm_mips64.dasc index c06270a..30cab6c 100644 --- a/src/vm_mips64.dasc +++ b/src/vm_mips64.dasc @@ -350,6 +350,12 @@ | |// Move table write barrier back. Overwrites mark and tmp. |.macro barrierback, tab, mark, tmp, target +| ld tmp, DISPATCH_GL(gc.blacknum)(DISPATCH) +| addiu tmp, tmp, -1 +| sd tmp, DISPATCH_GL(gc.blacknum)(DISPATCH) +| ld tmp, DISPATCH_GL(gc.graynum)(DISPATCH) +| addiu tmp, tmp, 1 +| sd tmp, DISPATCH_GL(gc.graynum)(DISPATCH) | ld tmp, DISPATCH_GL(gc.grayagain)(DISPATCH) | andi mark, mark, ~LJ_GC_BLACK & 255 // black2gray(tab) | sd tab, DISPATCH_GL(gc.grayagain)(DISPATCH) diff --git a/src/vm_ppc.dasc b/src/vm_ppc.dasc index b4260eb..3015869 100644 --- a/src/vm_ppc.dasc +++ b/src/vm_ppc.dasc @@ -474,6 +474,12 @@ | |// Move table write barrier back. Overwrites mark and tmp. |.macro barrierback, tab, mark, tmp +| lwz tmp, DISPATCH_GL(gc.graynum)(DISPATCH) +| addic tmp, tmp, 1 +| stw tmp, DISPATCH_GL(gc.graynum)(DISPATCH) +| lwz tmp, DISPATCH_GL(gc.blacknum)(DISPATCH) +| subic tmp, tmp, 1 +| stw tmp, DISPATCH_GL(gc.blacknum)(DISPATCH) | lwz tmp, DISPATCH_GL(gc.grayagain)(DISPATCH) | // Assumes LJ_GC_BLACK is 0x04. | rlwinm mark, mark, 0, 30, 28 // black2gray(tab) diff --git a/src/vm_x64.dasc b/src/vm_x64.dasc index 80753e0..d37b310 100644 --- a/src/vm_x64.dasc +++ b/src/vm_x64.dasc @@ -372,6 +372,8 @@ |// Move table write barrier back. Overwrites reg. |.macro barrierback, tab, reg | and byte tab->marked, (uint8_t)~LJ_GC_BLACK // black2gray(tab) +| sub qword [DISPATCH+DISPATCH_GL(gc.blacknum)], 1 +| add qword [DISPATCH+DISPATCH_GL(gc.graynum)], 1 | mov reg, [DISPATCH+DISPATCH_GL(gc.grayagain)] | mov [DISPATCH+DISPATCH_GL(gc.grayagain)], tab | mov tab->gclist, reg diff --git a/src/vm_x86.dasc b/src/vm_x86.dasc index 56bee14..3e35cc6 100644 --- a/src/vm_x86.dasc +++ b/src/vm_x86.dasc @@ -477,6 +477,8 @@ |// Move table write barrier back. Overwrites reg. |.macro barrierback, tab, reg | and byte tab->marked, (uint8_t)~LJ_GC_BLACK // black2gray(tab) +| sub dword [DISPATCH+DISPATCH_GL(gc.blacknum)], 1 +| add dword [DISPATCH+DISPATCH_GL(gc.graynum)], 1 | mov reg, [DISPATCH+DISPATCH_GL(gc.grayagain)] | mov [DISPATCH+DISPATCH_GL(gc.grayagain)], tab | mov tab->gclist, reg -- 2.28.0