[Tarantool-patches] [PATCH luajit v2 2/3] memprof: enrich symtab when meeting new prototype

Mikhail Shishatskiy m.shishatskiy at tarantool.org
Thu Dec 2 14:03:28 MSK 2021


Previously, the profiler dumped all the function prototypes only
at the start of profiling. It was enough for most cases, but
sometimes we may want to investigate the memory profile of module
loading. In such a case, we expect memprof to dump new prototypes
from parsed source code.

This patch extends memprof's streaming format with entries providing
extra information about allocation source's symbols if they were not
streamed at the start of profiling. This event is called <event-symtab>
and precedes the allocation-related event. The format is the following:

| event-symtab := event-header sym?
| sym          := sym-lua
| sym-lua      := sym-addr sym-chunk sym-line

The `sym-addr`, `sym-chunk` and `sym-line` are the prototype's address,
its chunk name and line where it was defined.

To be able determine, if the allocation source is present in the symtab
or not, the `prof_epoch` counter added to the `GCproto` structure, which
can be compared to the `prof_epoch` added to the VM global state.
If object's epoch is less than global, the memprof will dymp symtab
entry with <event-symtab> and symchronize the object's counter with
the global.

Also, the profiler parser is adjusted to recognize entries described
above and enrich the symtab on the fly while parsing events. For this
reason, the <utils/symtab.lua> API has been changed: the function
`parse_sym_lfunc` became public to be available for the parser module.

Resolves tarantool/tarantool#5815
---

Issue: https://github.com/tarantool/tarantool/issues/5815
Branch: https://github.com/tarantool/luajit/tree/shishqa/gh-5815-enrich-symtab-when-prototype-is-allocated-v2
Tarantool branch: https://github.com/tarantool/tarantool/tree/shishqa/gh-5815-enrich-symtab-when-prototype-is-allocated

 src/lj_bcread.c                               |  3 ++
 src/lj_memprof.c                              | 35 +++++++++++----
 src/lj_memprof.h                              | 13 +++++-
 src/lj_obj.h                                  | 10 +++++
 src/lj_parse.c                                |  3 ++
 src/lj_state.c                                |  3 ++
 .../misclib-memprof-lapi.test.lua             | 43 ++++++++++++++++---
 tools/memprof/parse.lua                       | 10 ++++-
 tools/utils/symtab.lua                        |  4 +-
 9 files changed, 105 insertions(+), 19 deletions(-)

diff --git a/src/lj_bcread.c b/src/lj_bcread.c
index 48c5e7c7..4e2c141b 100644
--- a/src/lj_bcread.c
+++ b/src/lj_bcread.c
@@ -353,6 +353,9 @@ GCproto *lj_bcread_proto(LexState *ls)
   pt->sizeuv = (uint8_t)sizeuv;
   pt->flags = (uint8_t)flags;
   pt->trace = 0;
+#if LJ_HASMEMPROF
+  pt->prof_epoch = 0;
+#endif
   setgcref(pt->chunkname, obj2gco(ls->chunkname));
 
   /* Close potentially uninitialized gap between bc and kgc. */
diff --git a/src/lj_memprof.c b/src/lj_memprof.c
index 2d779983..c154de93 100644
--- a/src/lj_memprof.c
+++ b/src/lj_memprof.c
@@ -66,10 +66,19 @@ static void dump_symtab_trace(struct lj_wbuf *out, const GCtrace *trace)
 
 #endif
 
+static void dump_symtab_proto(struct lj_wbuf *out, GCproto *pt,
+			      const global_State *g)
+{
+  lj_wbuf_addu64(out, (uintptr_t)pt);
+  lj_wbuf_addstring(out, proto_chunknamestr(pt));
+  lj_wbuf_addu64(out, (uint64_t)pt->firstline);
+  pt->prof_epoch = g->prof_epoch;
+}
+
 static void dump_symtab(struct lj_wbuf *out, const struct global_State *g)
 {
   const GCRef *iter = &g->gc.root;
-  const GCobj *o;
+  GCobj *o;
   const size_t ljs_header_len = sizeof(ljs_header) / sizeof(ljs_header[0]);
 
   /* Write prologue. */
@@ -78,11 +87,9 @@ static void dump_symtab(struct lj_wbuf *out, const struct global_State *g)
   while ((o = gcref(*iter)) != NULL) {
     switch (o->gch.gct) {
     case (~LJ_TPROTO): {
-      const GCproto *pt = gco2pt(o);
+      GCproto *pt = gco2pt(o);
       lj_wbuf_addbyte(out, SYMTAB_LFUNC);
-      lj_wbuf_addu64(out, (uintptr_t)pt);
-      lj_wbuf_addstring(out, proto_chunknamestr(pt));
-      lj_wbuf_addu64(out, (uint64_t)pt->firstline);
+      dump_symtab_proto(out, pt, g);
       break;
     }
     case (~LJ_TTRACE): {
@@ -140,6 +147,7 @@ static void memprof_write_lfunc(struct lj_wbuf *out, uint8_t aevent,
   ** -DLUAJIT_DISABLE_DEBUGINFO flag.
   */
   const BCLine line = lj_debug_frameline(L, fn, nextframe);
+  GCproto *pt = funcproto(fn);
 
   if (line < 0) {
     /*
@@ -151,11 +159,17 @@ static void memprof_write_lfunc(struct lj_wbuf *out, uint8_t aevent,
     ** We report such allocations as internal in order not to confuse users.
     */
     lj_wbuf_addbyte(out, aevent | ASOURCE_INT);
-  } else {
-    lj_wbuf_addbyte(out, aevent | ASOURCE_LFUNC);
-    lj_wbuf_addu64(out, (uintptr_t)funcproto(fn));
-    lj_wbuf_addu64(out, (uint64_t)line);
+    return;
   }
+
+  if (LJ_UNLIKELY(pt->prof_epoch != memprof.g->prof_epoch)) {
+    lj_wbuf_addbyte(out, AEVENT_SYMTAB | ASOURCE_LFUNC);
+    dump_symtab_proto(out, pt, memprof.g);
+  }
+
+  lj_wbuf_addbyte(out, aevent | ASOURCE_LFUNC);
+  lj_wbuf_addu64(out, (uintptr_t)pt);
+  lj_wbuf_addu64(out, (uint64_t)line);
 }
 
 static void memprof_write_cfunc(struct lj_wbuf *out, uint8_t aevent,
@@ -329,6 +343,9 @@ int lj_memprof_start(struct lua_State *L, const struct lj_memprof_options *opt)
   mp->g = G(L);
   mp->state = MPS_PROFILE;
 
+  /* Increment the profiling epoch. */
+  mp->g->prof_epoch++;
+
   /* Init output. */
   lj_wbuf_init(&mp->out, mp_opt->writer, mp_opt->ctx, mp_opt->buf, mp_opt->len);
   dump_symtab(&mp->out, mp->g);
diff --git a/src/lj_memprof.h b/src/lj_memprof.h
index 395fb429..150e6b32 100644
--- a/src/lj_memprof.h
+++ b/src/lj_memprof.h
@@ -57,7 +57,7 @@
 #define SYMTAB_TRACE ((uint8_t)1)
 #define SYMTAB_FINAL ((uint8_t)0x80)
 
-#define LJM_CURRENT_FORMAT_VERSION 0x02
+#define LJM_CURRENT_FORMAT_VERSION 0x03
 
 /*
 ** Event stream format:
@@ -68,16 +68,21 @@
 ** prologue       := 'l' 'j' 'm' version reserved
 ** version        := <BYTE>
 ** reserved       := <BYTE> <BYTE> <BYTE>
-** event          := event-alloc | event-realloc | event-free
+** event          := event-alloc | event-realloc | event-free | event-symtab
 ** event-alloc    := event-header loc? naddr nsize
 ** event-realloc  := event-header loc? oaddr osize naddr nsize
 ** event-free     := event-header loc? oaddr osize
+** event-symtab   := event-header sym?
 ** event-header   := <BYTE>
 ** loc            := loc-lua | loc-c | loc-trace
 ** loc-lua        := sym-addr line-no
 ** loc-c          := sym-addr
 ** loc-trace      := trace-no trace-addr
+** sym            := sym-lua
+** sym-lua        := sym-addr sym-chunk sym-line
 ** sym-addr       := <ULEB128>
+** sym-chunk      := string
+** sym-line       := <ULEB128>
 ** line-no        := <ULEB128>
 ** trace-no       := <ULEB128>
 ** trace-addr     := <ULEB128>
@@ -85,6 +90,9 @@
 ** naddr          := <ULEB128>
 ** osize          := <ULEB128>
 ** nsize          := <ULEB128>
+** string         := string-len string-payload
+** string-len     := <ULEB128>
+** string-payload := <BYTE> {string-len}
 ** epilogue       := event-header
 **
 ** <BYTE>   :  A single byte (no surprises here)
@@ -104,6 +112,7 @@
 */
 
 /* Allocation events. */
+#define AEVENT_SYMTAB  ((uint8_t)0)
 #define AEVENT_ALLOC   ((uint8_t)1)
 #define AEVENT_FREE    ((uint8_t)2)
 #define AEVENT_REALLOC ((uint8_t)(AEVENT_ALLOC | AEVENT_FREE))
diff --git a/src/lj_obj.h b/src/lj_obj.h
index d26e60be..283f30f6 100644
--- a/src/lj_obj.h
+++ b/src/lj_obj.h
@@ -385,6 +385,13 @@ typedef struct GCproto {
   MRef lineinfo;	/* Compressed map from bytecode ins. to source line. */
   MRef uvinfo;		/* Upvalue names. */
   MRef varinfo;		/* Names and compressed extents of local variables. */
+#if LJ_HASMEMPROF
+  /*
+  ** Epoch indicating if this proto was dumped to the symbol table for the
+  ** current profiling session.
+  */
+  uint8_t prof_epoch;
+#endif
 } GCproto;
 
 /* Flags for prototype. */
@@ -674,6 +681,9 @@ typedef struct global_State {
   MRef jit_base;	/* Current JIT code L->base or NULL. */
   MRef ctype_state;	/* Pointer to C type state. */
   GCRef gcroot[GCROOT_MAX];  /* GC roots. */
+#if LJ_HASMEMPROF
+  uint8_t prof_epoch;	/* Current profiling epoch for this VM. */
+#endif
 } global_State;
 
 #define mainthread(g)	(&gcref(g->mainthref)->th)
diff --git a/src/lj_parse.c b/src/lj_parse.c
index a6325a76..9415665a 100644
--- a/src/lj_parse.c
+++ b/src/lj_parse.c
@@ -1577,6 +1577,9 @@ static GCproto *fs_finish(LexState *ls, BCLine line)
   pt->flags = (uint8_t)(fs->flags & ~(PROTO_HAS_RETURN|PROTO_FIXUP_RETURN));
   pt->numparams = fs->numparams;
   pt->framesize = fs->framesize;
+#if LJ_HASMEMPROF
+  pt->prof_epoch = 0;
+#endif
   setgcref(pt->chunkname, obj2gco(ls->chunkname));
 
   /* Close potentially uninitialized gap between bc and kgc. */
diff --git a/src/lj_state.c b/src/lj_state.c
index f82b1b5b..b4bfd573 100644
--- a/src/lj_state.c
+++ b/src/lj_state.c
@@ -221,6 +221,9 @@ LUA_API lua_State *lua_newstate(lua_Alloc f, void *ud)
   g->strempty.gct = ~LJ_TSTR;
   g->allocf = f;
   g->allocd = ud;
+#if LJ_HASMEMPROF
+  g->prof_epoch = 0;
+#endif
   setgcref(g->mainthref, obj2gco(L));
   setgcref(g->uvhead.prev, obj2gco(&g->uvhead));
   setgcref(g->uvhead.next, obj2gco(&g->uvhead));
diff --git a/test/tarantool-tests/misclib-memprof-lapi.test.lua b/test/tarantool-tests/misclib-memprof-lapi.test.lua
index dd973f2a..1c74b4d7 100644
--- a/test/tarantool-tests/misclib-memprof-lapi.test.lua
+++ b/test/tarantool-tests/misclib-memprof-lapi.test.lua
@@ -7,7 +7,7 @@ require("utils").skipcond(
 local tap = require("tap")
 
 local test = tap.test("misc-memprof-lapi")
-test:plan(4)
+test:plan(5)
 
 local jit_opt_default = {
     3, -- level
@@ -117,8 +117,9 @@ local function fill_ev_type(events, symbols, event_type)
   return ev_type
 end
 
-local function form_source_line(line)
-  return string.format("@%s:%d", arg[0], line)
+local function form_source_line(line, source)
+  source = source or ("@"..arg[0])
+  return string.format("%s:%d", source, line)
 end
 
 local function check_alloc_report(alloc, location, nevents)
@@ -126,10 +127,10 @@ local function check_alloc_report(alloc, location, nevents)
   local traceno = location.traceno
   if traceno then
     expected_name = string.format("TRACE [%d] ", traceno)..
-                    form_source_line(location.line)
+                    form_source_line(location.line, location.source)
     event = alloc.trace[traceno]
   else
-    expected_name = form_source_line(location.linedefined)
+    expected_name = form_source_line(location.linedefined, location.source)
     event = alloc.line[location.line]
   end
   assert(expected_name == event.name, ("got='%s', expected='%s'"):format(
@@ -223,6 +224,38 @@ test:test("stack-resize", function(subtest)
   misc.memprof.stop()
 end)
 
+-- Test for extending symtab with function prototypes
+-- while profiler is running.
+test:test("symtab-enriching", function(subtest)
+  subtest:plan(2)
+
+  local payload_str = [[
+  local M = {
+    tmp = string.rep("1", 100)  -- line 2.
+  }
+
+  function M.payload()
+    local str = string.rep("42", 100)  -- line 6.
+  end
+
+  return M
+  ]]
+
+  local symbols, events = generate_parsed_output(function()
+    local str_chunk = assert(loadstring(payload_str, 'str_chunk'))()
+    str_chunk.payload()
+  end)
+
+  local alloc = fill_ev_type(events, symbols, "alloc")
+
+  subtest:ok(check_alloc_report(
+    alloc, { source = 'str_chunk', line = 6, linedefined = 5 }, 1)
+  )
+  subtest:ok(check_alloc_report(
+    alloc, { source = 'str_chunk', line = 2, linedefined = 0 }, 1)
+  )
+end)
+
 -- Test profiler with enabled JIT.
 jit.on()
 
diff --git a/tools/memprof/parse.lua b/tools/memprof/parse.lua
index be5844a4..38f76f00 100644
--- a/tools/memprof/parse.lua
+++ b/tools/memprof/parse.lua
@@ -13,10 +13,11 @@ local string_format = string.format
 local symtab = require "utils.symtab"
 
 local LJM_MAGIC = "ljm"
-local LJM_CURRENT_VERSION = 0x02
+local LJM_CURRENT_VERSION = 0x03
 
 local LJM_EPILOGUE_HEADER = 0x80
 
+local AEVENT_SYMTAB = 0
 local AEVENT_ALLOC = 1
 local AEVENT_FREE = 2
 local AEVENT_REALLOC = 3
@@ -140,7 +141,14 @@ local function parse_free(reader, asource, events, heap, symbols)
   heap[oaddr] = nil
 end
 
+local function parse_symtab(reader, asource, _, _, symbols)
+  if asource == ASOURCE_LFUNC then
+    symtab.parse_sym_lfunc(reader, symbols)
+  end
+end
+
 local parsers = {
+  [AEVENT_SYMTAB] = {evname = "symtab", parse = parse_symtab},
   [AEVENT_ALLOC] = {evname = "alloc", parse = parse_alloc},
   [AEVENT_FREE] = {evname = "free", parse = parse_free},
   [AEVENT_REALLOC] = {evname = "realloc", parse = parse_realloc},
diff --git a/tools/utils/symtab.lua b/tools/utils/symtab.lua
index c758b67e..00bab03a 100644
--- a/tools/utils/symtab.lua
+++ b/tools/utils/symtab.lua
@@ -37,7 +37,7 @@ function M.new_loc(symtab, addr, line, traceno)
 end
 
 -- Parse a single entry in a symtab: lfunc symbol.
-local function parse_sym_lfunc(reader, symtab)
+function M.parse_sym_lfunc(reader, symtab)
   local sym_addr = reader:read_uleb128()
   local sym_chunk = reader:read_string()
   local sym_line = reader:read_uleb128()
@@ -69,7 +69,7 @@ local function parse_sym_trace(reader, symtab)
 end
 
 local parsers = {
-  [SYMTAB_LFUNC] = parse_sym_lfunc,
+  [SYMTAB_LFUNC] = M.parse_sym_lfunc,
   [SYMTAB_TRACE] = parse_sym_trace,
 }
 
-- 
2.33.1



More information about the Tarantool-patches mailing list