Re: [Tarantool-patches] [PATCH luajit v2 2/3] memprof: enrich symtab when meeting new prototype

[Mikhail Shishatskiy via Tarantool-patches <tarantool-patches@dev.tarantool.org>]

Updated patch in line with what we discussed with Sergey offline:

New commit message:
===========================================================
memprof: enrich symtab when meeting new prototype

Previously, the profiler dumped all the function prototypes only
at the start of profiling. It was enough for most cases, but
sometimes we may want to investigate the memory profile of module
loading. In such a case, we expect memprof to dump new prototypes
from parsed source code.

This patch extends memprof's streaming format with entries providing
extra information about allocation source's symbols if they were not
streamed at the start of profiling. This event is called <event-symtab>
and precedes the allocation-related event. The format is the following:

| event-symtab := event-header sym?
| sym          := sym-lua
| sym-lua      := sym-addr sym-chunk sym-line

The `sym-addr`, `sym-chunk` and `sym-line` are the prototype's address,
its chunk name and line where it was defined.

The profiler dumps new prototypes to the symbol table right after the
creation of a GCproto object.

Also, the profiler parser is adjusted to recognize entries described
above and enrich the symtab on the fly while parsing events. For this
reason, the <utils/symtab.lua> API has been changed: the function
`parse_sym_lfunc` became public to be available for the parser module.

Resolves tarantool/tarantool#5815
===========================================================

New diff:
===========================================================
diff --git a/src/Makefile.dep.original b/src/Makefile.dep.original
index faa44a0b..82f34315 100644
--- a/src/Makefile.dep.original
+++ b/src/Makefile.dep.original
@@ -59,7 +59,7 @@ lj_bc.o: lj_bc.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_bc.h \
  lj_bcread.o: lj_bcread.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
   lj_gc.h lj_err.h lj_errmsg.h lj_buf.h lj_str.h lj_tab.h lj_bc.h \
   lj_ctype.h lj_cdata.h lualib.h lj_lex.h lj_bcdump.h lj_state.h \
- lj_strfmt.h
+ lj_strfmt.h lj_memprof.h
  lj_bcwrite.o: lj_bcwrite.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
   lj_gc.h lj_buf.h lj_str.h lj_bc.h lj_ctype.h lj_dispatch.h lj_jit.h \
   lj_ir.h lj_strfmt.h lj_bcdump.h lj_lex.h lj_err.h lj_errmsg.h lj_vm.h
@@ -175,7 +175,7 @@ lj_opt_split.o: lj_opt_split.c lj_obj.h lua.h luaconf.h lj_def.h \
  lj_parse.o: lj_parse.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
   lj_gc.h lj_err.h lj_errmsg.h lj_debug.h lj_buf.h lj_str.h lj_tab.h \
   lj_func.h lj_state.h lj_bc.h lj_ctype.h lj_strfmt.h lj_lex.h lj_parse.h \
- lj_vm.h lj_vmevent.h
+ lj_vm.h lj_vmevent.h lj_memprof.h
  lj_profile.o: lj_profile.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
   lj_buf.h lj_gc.h lj_str.h lj_frame.h lj_bc.h lj_debug.h lj_dispatch.h \
   lj_jit.h lj_ir.h lj_trace.h lj_traceerr.h lj_profile.h luajit.h
diff --git a/src/lj_bcread.c b/src/lj_bcread.c
index 48c5e7c7..cb08599d 100644
--- a/src/lj_bcread.c
+++ b/src/lj_bcread.c
@@ -22,6 +22,9 @@
  #include "lj_bcdump.h"
  #include "lj_state.h"
  #include "lj_strfmt.h"
+#if LJ_HASMEMPROF
+#include "lj_memprof.h"
+#endif

  /* Reuse some lexer fields for our own purposes. */
  #define bcread_flags(ls)	ls->level
@@ -381,6 +384,12 @@ GCproto *lj_bcread_proto(LexState *ls)
      setmref(pt->uvinfo, NULL);
      setmref(pt->varinfo, NULL);
    }
+
+  /* Add a new prototype to the profiler. */
+#if LJ_HASMEMPROF
+  lj_memprof_add_proto(pt);
+#endif
+
    return pt;
  }

diff --git a/src/lj_memprof.c b/src/lj_memprof.c
index 2d779983..9bb2483e 100644
--- a/src/lj_memprof.c
+++ b/src/lj_memprof.c
@@ -66,6 +66,14 @@ static void dump_symtab_trace(struct lj_wbuf *out, const GCtrace *trace)

  #endif

+static void dump_symtab_proto(struct lj_wbuf *out, const GCproto *pt,
+			      const global_State *g)
+{
+  lj_wbuf_addu64(out, (uintptr_t)pt);
+  lj_wbuf_addstring(out, proto_chunknamestr(pt));
+  lj_wbuf_addu64(out, (uint64_t)pt->firstline);
+}
+
  static void dump_symtab(struct lj_wbuf *out, const struct global_State *g)
  {
    const GCRef *iter = &g->gc.root;
@@ -80,9 +88,7 @@ static void dump_symtab(struct lj_wbuf *out, const struct global_State *g)
      case (~LJ_TPROTO): {
        const GCproto *pt = gco2pt(o);
        lj_wbuf_addbyte(out, SYMTAB_LFUNC);
-      lj_wbuf_addu64(out, (uintptr_t)pt);
-      lj_wbuf_addstring(out, proto_chunknamestr(pt));
-      lj_wbuf_addu64(out, (uint64_t)pt->firstline);
+      dump_symtab_proto(out, pt, g);
        break;
      }
      case (~LJ_TTRACE): {
@@ -140,6 +146,7 @@ static void memprof_write_lfunc(struct lj_wbuf *out, uint8_t aevent,
    ** -DLUAJIT_DISABLE_DEBUGINFO flag.
    */
    const BCLine line = lj_debug_frameline(L, fn, nextframe);
+  const GCproto *pt = funcproto(fn);

    if (line < 0) {
      /*
@@ -151,11 +158,17 @@ static void memprof_write_lfunc(struct lj_wbuf *out, uint8_t aevent,
      ** We report such allocations as internal in order not to confuse users.
      */
      lj_wbuf_addbyte(out, aevent | ASOURCE_INT);
-  } else {
-    lj_wbuf_addbyte(out, aevent | ASOURCE_LFUNC);
-    lj_wbuf_addu64(out, (uintptr_t)funcproto(fn));
-    lj_wbuf_addu64(out, (uint64_t)line);
+    return;
    }
+
+  /*
+  ** As a prototype is a source of an allocation, it has
+  ** already been inserted into the symtab: on the start
+  ** of the profiling or right after the creation.
+  */
+  lj_wbuf_addbyte(out, aevent | ASOURCE_LFUNC);
+  lj_wbuf_addu64(out, (uintptr_t)pt);
+  lj_wbuf_addu64(out, (uint64_t)line);
  }

  static void memprof_write_cfunc(struct lj_wbuf *out, uint8_t aevent,
@@ -414,6 +427,17 @@ errio:
    return PROFILE_ERRIO;
  }

+void lj_memprof_add_proto(const struct GCproto *pt)
+{
+  struct memprof *mp = &memprof;
+
+  if (mp->state != MPS_PROFILE)
+    return;
+
+  lj_wbuf_addbyte(&mp->out, AEVENT_SYMTAB | ASOURCE_LFUNC);
+  dump_symtab_proto(&mp->out, pt, mp->g);
+}
+
  #else /* LJ_HASMEMPROF */

  int lj_memprof_start(struct lua_State *L, const struct lj_memprof_options *opt)
@@ -430,4 +454,9 @@ int lj_memprof_stop(struct lua_State *L)
    return PROFILE_ERRUSE;
  }

+void lj_memprof_add_proto(const struct GCproto *pt)
+{
+  UNUSED(pt);
+}
+
  #endif /* LJ_HASMEMPROF */
diff --git a/src/lj_memprof.h b/src/lj_memprof.h
index 395fb429..eda8ca0c 100644
--- a/src/lj_memprof.h
+++ b/src/lj_memprof.h
@@ -57,7 +57,7 @@
  #define SYMTAB_TRACE ((uint8_t)1)
  #define SYMTAB_FINAL ((uint8_t)0x80)

-#define LJM_CURRENT_FORMAT_VERSION 0x02
+#define LJM_CURRENT_FORMAT_VERSION 0x03

  /*
  ** Event stream format:
@@ -68,16 +68,21 @@
  ** prologue       := 'l' 'j' 'm' version reserved
  ** version        := <BYTE>
  ** reserved       := <BYTE> <BYTE> <BYTE>
-** event          := event-alloc | event-realloc | event-free
+** event          := event-alloc | event-realloc | event-free | event-symtab
  ** event-alloc    := event-header loc? naddr nsize
  ** event-realloc  := event-header loc? oaddr osize naddr nsize
  ** event-free     := event-header loc? oaddr osize
+** event-symtab   := event-header sym?
  ** event-header   := <BYTE>
  ** loc            := loc-lua | loc-c | loc-trace
  ** loc-lua        := sym-addr line-no
  ** loc-c          := sym-addr
  ** loc-trace      := trace-no trace-addr
+** sym            := sym-lua
+** sym-lua        := sym-addr sym-chunk sym-line
  ** sym-addr       := <ULEB128>
+** sym-chunk      := string
+** sym-line       := <ULEB128>
  ** line-no        := <ULEB128>
  ** trace-no       := <ULEB128>
  ** trace-addr     := <ULEB128>
@@ -85,6 +90,9 @@
  ** naddr          := <ULEB128>
  ** osize          := <ULEB128>
  ** nsize          := <ULEB128>
+** string         := string-len string-payload
+** string-len     := <ULEB128>
+** string-payload := <BYTE> {string-len}
  ** epilogue       := event-header
  **
  ** <BYTE>   :  A single byte (no surprises here)
@@ -104,6 +112,7 @@
  */

  /* Allocation events. */
+#define AEVENT_SYMTAB  ((uint8_t)0)
  #define AEVENT_ALLOC   ((uint8_t)1)
  #define AEVENT_FREE    ((uint8_t)2)
  #define AEVENT_REALLOC ((uint8_t)(AEVENT_ALLOC | AEVENT_FREE))
@@ -148,6 +157,7 @@ struct lj_memprof_options {

  /* Avoid to provide additional interfaces described in other headers. */
  struct lua_State;
+struct GCproto;

  /*
  ** Starts profiling. Returns PROFILE_SUCCESS on success and one of
@@ -164,4 +174,10 @@ int lj_memprof_start(struct lua_State *L, const struct lj_memprof_options *opt);
  */
  int lj_memprof_stop(struct lua_State *L);

+/*
+** Enriches profiler symbol table with a new proto, if profiler
+** is running.
+*/
+void lj_memprof_add_proto(const struct GCproto *pt);
+
  #endif
diff --git a/src/lj_parse.c b/src/lj_parse.c
index a6325a76..30b0caa0 100644
--- a/src/lj_parse.c
+++ b/src/lj_parse.c
@@ -27,6 +27,9 @@
  #include "lj_parse.h"
  #include "lj_vm.h"
  #include "lj_vmevent.h"
+#if LJ_HASMEMPROF
+#include "lj_memprof.h"
+#endif

  /* -- Parser structures and definitions ----------------------------------- */

@@ -1591,6 +1594,11 @@ static GCproto *fs_finish(LexState *ls, BCLine line)
      setprotoV(L, L->top++, pt);
    );

+  /* Add a new prototype to the profiler. */
+#if LJ_HASMEMPROF
+  lj_memprof_add_proto(pt);
+#endif
+
    L->top--;  /* Pop table of constants. */
    ls->vtop = fs->vbase;  /* Reset variable stack. */
    ls->fs = fs->prev;
diff --git a/test/tarantool-tests/misclib-memprof-lapi.test.lua b/test/tarantool-tests/misclib-memprof-lapi.test.lua
index dd973f2a..1c74b4d7 100644
--- a/test/tarantool-tests/misclib-memprof-lapi.test.lua
+++ b/test/tarantool-tests/misclib-memprof-lapi.test.lua
@@ -7,7 +7,7 @@ require("utils").skipcond(
  local tap = require("tap")

  local test = tap.test("misc-memprof-lapi")
-test:plan(4)
+test:plan(5)

  local jit_opt_default = {
      3, -- level
@@ -117,8 +117,9 @@ local function fill_ev_type(events, symbols, event_type)
    return ev_type
  end

-local function form_source_line(line)
-  return string.format("@%s:%d", arg[0], line)
+local function form_source_line(line, source)
+  source = source or ("@"..arg[0])
+  return string.format("%s:%d", source, line)
  end

  local function check_alloc_report(alloc, location, nevents)
@@ -126,10 +127,10 @@ local function check_alloc_report(alloc, location, nevents)
    local traceno = location.traceno
    if traceno then
      expected_name = string.format("TRACE [%d] ", traceno)..
-                    form_source_line(location.line)
+                    form_source_line(location.line, location.source)
      event = alloc.trace[traceno]
    else
-    expected_name = form_source_line(location.linedefined)
+    expected_name = form_source_line(location.linedefined, location.source)
      event = alloc.line[location.line]
    end
    assert(expected_name == event.name, ("got='%s', expected='%s'"):format(
@@ -223,6 +224,38 @@ test:test("stack-resize", function(subtest)
    misc.memprof.stop()
  end)

+-- Test for extending symtab with function prototypes
+-- while profiler is running.
+test:test("symtab-enriching", function(subtest)
+  subtest:plan(2)
+
+  local payload_str = [[
+  local M = {
+    tmp = string.rep("1", 100)  -- line 2.
+  }
+
+  function M.payload()
+    local str = string.rep("42", 100)  -- line 6.
+  end
+
+  return M
+  ]]
+
+  local symbols, events = generate_parsed_output(function()
+    local str_chunk = assert(loadstring(payload_str, 'str_chunk'))()
+    str_chunk.payload()
+  end)
+
+  local alloc = fill_ev_type(events, symbols, "alloc")
+
+  subtest:ok(check_alloc_report(
+    alloc, { source = 'str_chunk', line = 6, linedefined = 5 }, 1)
+  )
+  subtest:ok(check_alloc_report(
+    alloc, { source = 'str_chunk', line = 2, linedefined = 0 }, 1)
+  )
+end)
+
  -- Test profiler with enabled JIT.
  jit.on()

diff --git a/tools/memprof/parse.lua b/tools/memprof/parse.lua
index be5844a4..38f76f00 100644
--- a/tools/memprof/parse.lua
+++ b/tools/memprof/parse.lua
@@ -13,10 +13,11 @@ local string_format = string.format
  local symtab = require "utils.symtab"

  local LJM_MAGIC = "ljm"
-local LJM_CURRENT_VERSION = 0x02
+local LJM_CURRENT_VERSION = 0x03

  local LJM_EPILOGUE_HEADER = 0x80

+local AEVENT_SYMTAB = 0
  local AEVENT_ALLOC = 1
  local AEVENT_FREE = 2
  local AEVENT_REALLOC = 3
@@ -140,7 +141,14 @@ local function parse_free(reader, asource, events, heap, symbols)
    heap[oaddr] = nil
  end

+local function parse_symtab(reader, asource, _, _, symbols)
+  if asource == ASOURCE_LFUNC then
+    symtab.parse_sym_lfunc(reader, symbols)
+  end
+end
+
  local parsers = {
+  [AEVENT_SYMTAB] = {evname = "symtab", parse = parse_symtab},
    [AEVENT_ALLOC] = {evname = "alloc", parse = parse_alloc},
    [AEVENT_FREE] = {evname = "free", parse = parse_free},
    [AEVENT_REALLOC] = {evname = "realloc", parse = parse_realloc},
diff --git a/tools/utils/symtab.lua b/tools/utils/symtab.lua
index c758b67e..00bab03a 100644
--- a/tools/utils/symtab.lua
+++ b/tools/utils/symtab.lua
@@ -37,7 +37,7 @@ function M.new_loc(symtab, addr, line, traceno)
  end

  -- Parse a single entry in a symtab: lfunc symbol.
-local function parse_sym_lfunc(reader, symtab)
+function M.parse_sym_lfunc(reader, symtab)
    local sym_addr = reader:read_uleb128()
    local sym_chunk = reader:read_string()
    local sym_line = reader:read_uleb128()
@@ -69,7 +69,7 @@ local function parse_sym_trace(reader, symtab)
  end

  local parsers = {
-  [SYMTAB_LFUNC] = parse_sym_lfunc,
+  [SYMTAB_LFUNC] = M.parse_sym_lfunc,
    [SYMTAB_TRACE] = parse_sym_trace,
  }
===========================================================

On 02.12.2021 14:03, Mikhail Shishatskiy wrote:
>Previously, the profiler dumped all the function prototypes only
>at the start of profiling. It was enough for most cases, but
>sometimes we may want to investigate the memory profile of module
>loading. In such a case, we expect memprof to dump new prototypes
>from parsed source code.
>
>This patch extends memprof's streaming format with entries providing
>extra information about allocation source's symbols if they were not
>streamed at the start of profiling. This event is called <event-symtab>
>and precedes the allocation-related event. The format is the following:
>
>| event-symtab := event-header sym?
>| sym          := sym-lua
>| sym-lua      := sym-addr sym-chunk sym-line
>
>The `sym-addr`, `sym-chunk` and `sym-line` are the prototype's address,
>its chunk name and line where it was defined.
>
>To be able determine, if the allocation source is present in the symtab
>or not, the `prof_epoch` counter added to the `GCproto` structure, which
>can be compared to the `prof_epoch` added to the VM global state.
>If object's epoch is less than global, the memprof will dymp symtab
>entry with <event-symtab> and symchronize the object's counter with
>the global.
>
>Also, the profiler parser is adjusted to recognize entries described
>above and enrich the symtab on the fly while parsing events. For this
>reason, the <utils/symtab.lua> API has been changed: the function
>`parse_sym_lfunc` became public to be available for the parser module.
>
>Resolves tarantool/tarantool#5815
>---
>
>Issue: https://github.com/tarantool/tarantool/issues/5815
>Branch: https://github.com/tarantool/luajit/tree/shishqa/gh-5815-enrich-symtab-when-prototype-is-allocated-v2
>Tarantool branch: https://github.com/tarantool/tarantool/tree/shishqa/gh-5815-enrich-symtab-when-prototype-is-allocated
>
> src/lj_bcread.c                               |  3 ++
> src/lj_memprof.c                              | 35 +++++++++++----
> src/lj_memprof.h                              | 13 +++++-
> src/lj_obj.h                                  | 10 +++++
> src/lj_parse.c                                |  3 ++
> src/lj_state.c                                |  3 ++
> .../misclib-memprof-lapi.test.lua             | 43 ++++++++++++++++---
> tools/memprof/parse.lua                       | 10 ++++-
> tools/utils/symtab.lua                        |  4 +-
> 9 files changed, 105 insertions(+), 19 deletions(-)
>
>diff --git a/src/lj_bcread.c b/src/lj_bcread.c
>index 48c5e7c7..4e2c141b 100644
>--- a/src/lj_bcread.c
>+++ b/src/lj_bcread.c
>@@ -353,6 +353,9 @@ GCproto *lj_bcread_proto(LexState *ls)
>   pt->sizeuv = (uint8_t)sizeuv;
>   pt->flags = (uint8_t)flags;
>   pt->trace = 0;
>+#if LJ_HASMEMPROF
>+  pt->prof_epoch = 0;
>+#endif
>   setgcref(pt->chunkname, obj2gco(ls->chunkname));
>
>   /* Close potentially uninitialized gap between bc and kgc. */
>diff --git a/src/lj_memprof.c b/src/lj_memprof.c
>index 2d779983..c154de93 100644
>--- a/src/lj_memprof.c
>+++ b/src/lj_memprof.c
>@@ -66,10 +66,19 @@ static void dump_symtab_trace(struct lj_wbuf *out, const GCtrace *trace)
>
> #endif
>
>+static void dump_symtab_proto(struct lj_wbuf *out, GCproto *pt,
>+			      const global_State *g)
>+{
>+  lj_wbuf_addu64(out, (uintptr_t)pt);
>+  lj_wbuf_addstring(out, proto_chunknamestr(pt));
>+  lj_wbuf_addu64(out, (uint64_t)pt->firstline);
>+  pt->prof_epoch = g->prof_epoch;
>+}
>+
> static void dump_symtab(struct lj_wbuf *out, const struct global_State *g)
> {
>   const GCRef *iter = &g->gc.root;
>-  const GCobj *o;
>+  GCobj *o;
>   const size_t ljs_header_len = sizeof(ljs_header) / sizeof(ljs_header[0]);
>
>   /* Write prologue. */
>@@ -78,11 +87,9 @@ static void dump_symtab(struct lj_wbuf *out, const struct global_State *g)
>   while ((o = gcref(*iter)) != NULL) {
>     switch (o->gch.gct) {
>     case (~LJ_TPROTO): {
>-      const GCproto *pt = gco2pt(o);
>+      GCproto *pt = gco2pt(o);
>       lj_wbuf_addbyte(out, SYMTAB_LFUNC);
>-      lj_wbuf_addu64(out, (uintptr_t)pt);
>-      lj_wbuf_addstring(out, proto_chunknamestr(pt));
>-      lj_wbuf_addu64(out, (uint64_t)pt->firstline);
>+      dump_symtab_proto(out, pt, g);
>       break;
>     }
>     case (~LJ_TTRACE): {
>@@ -140,6 +147,7 @@ static void memprof_write_lfunc(struct lj_wbuf *out, uint8_t aevent,
>   ** -DLUAJIT_DISABLE_DEBUGINFO flag.
>   */
>   const BCLine line = lj_debug_frameline(L, fn, nextframe);
>+  GCproto *pt = funcproto(fn);
>
>   if (line < 0) {
>     /*
>@@ -151,11 +159,17 @@ static void memprof_write_lfunc(struct lj_wbuf *out, uint8_t aevent,
>     ** We report such allocations as internal in order not to confuse users.
>     */
>     lj_wbuf_addbyte(out, aevent | ASOURCE_INT);
>-  } else {
>-    lj_wbuf_addbyte(out, aevent | ASOURCE_LFUNC);
>-    lj_wbuf_addu64(out, (uintptr_t)funcproto(fn));
>-    lj_wbuf_addu64(out, (uint64_t)line);
>+    return;
>   }
>+
>+  if (LJ_UNLIKELY(pt->prof_epoch != memprof.g->prof_epoch)) {
>+    lj_wbuf_addbyte(out, AEVENT_SYMTAB | ASOURCE_LFUNC);
>+    dump_symtab_proto(out, pt, memprof.g);
>+  }
>+
>+  lj_wbuf_addbyte(out, aevent | ASOURCE_LFUNC);
>+  lj_wbuf_addu64(out, (uintptr_t)pt);
>+  lj_wbuf_addu64(out, (uint64_t)line);
> }
>
> static void memprof_write_cfunc(struct lj_wbuf *out, uint8_t aevent,
>@@ -329,6 +343,9 @@ int lj_memprof_start(struct lua_State *L, const struct lj_memprof_options *opt)
>   mp->g = G(L);
>   mp->state = MPS_PROFILE;
>
>+  /* Increment the profiling epoch. */
>+  mp->g->prof_epoch++;
>+
>   /* Init output. */
>   lj_wbuf_init(&mp->out, mp_opt->writer, mp_opt->ctx, mp_opt->buf, mp_opt->len);
>   dump_symtab(&mp->out, mp->g);
>diff --git a/src/lj_memprof.h b/src/lj_memprof.h
>index 395fb429..150e6b32 100644
>--- a/src/lj_memprof.h
>+++ b/src/lj_memprof.h
>@@ -57,7 +57,7 @@
> #define SYMTAB_TRACE ((uint8_t)1)
> #define SYMTAB_FINAL ((uint8_t)0x80)
>
>-#define LJM_CURRENT_FORMAT_VERSION 0x02
>+#define LJM_CURRENT_FORMAT_VERSION 0x03
>
> /*
> ** Event stream format:
>@@ -68,16 +68,21 @@
> ** prologue       := 'l' 'j' 'm' version reserved
> ** version        := <BYTE>
> ** reserved       := <BYTE> <BYTE> <BYTE>
>-** event          := event-alloc | event-realloc | event-free
>+** event          := event-alloc | event-realloc | event-free | event-symtab
> ** event-alloc    := event-header loc? naddr nsize
> ** event-realloc  := event-header loc? oaddr osize naddr nsize
> ** event-free     := event-header loc? oaddr osize
>+** event-symtab   := event-header sym?
> ** event-header   := <BYTE>
> ** loc            := loc-lua | loc-c | loc-trace
> ** loc-lua        := sym-addr line-no
> ** loc-c          := sym-addr
> ** loc-trace      := trace-no trace-addr
>+** sym            := sym-lua
>+** sym-lua        := sym-addr sym-chunk sym-line
> ** sym-addr       := <ULEB128>
>+** sym-chunk      := string
>+** sym-line       := <ULEB128>
> ** line-no        := <ULEB128>
> ** trace-no       := <ULEB128>
> ** trace-addr     := <ULEB128>
>@@ -85,6 +90,9 @@
> ** naddr          := <ULEB128>
> ** osize          := <ULEB128>
> ** nsize          := <ULEB128>
>+** string         := string-len string-payload
>+** string-len     := <ULEB128>
>+** string-payload := <BYTE> {string-len}
> ** epilogue       := event-header
> **
> ** <BYTE>   :  A single byte (no surprises here)
>@@ -104,6 +112,7 @@
> */
>
> /* Allocation events. */
>+#define AEVENT_SYMTAB  ((uint8_t)0)
> #define AEVENT_ALLOC   ((uint8_t)1)
> #define AEVENT_FREE    ((uint8_t)2)
> #define AEVENT_REALLOC ((uint8_t)(AEVENT_ALLOC | AEVENT_FREE))
>diff --git a/src/lj_obj.h b/src/lj_obj.h
>index d26e60be..283f30f6 100644
>--- a/src/lj_obj.h
>+++ b/src/lj_obj.h
>@@ -385,6 +385,13 @@ typedef struct GCproto {
>   MRef lineinfo;	/* Compressed map from bytecode ins. to source line. */
>   MRef uvinfo;		/* Upvalue names. */
>   MRef varinfo;		/* Names and compressed extents of local variables. */
>+#if LJ_HASMEMPROF
>+  /*
>+  ** Epoch indicating if this proto was dumped to the symbol table for the
>+  ** current profiling session.
>+  */
>+  uint8_t prof_epoch;
>+#endif
> } GCproto;
>
> /* Flags for prototype. */
>@@ -674,6 +681,9 @@ typedef struct global_State {
>   MRef jit_base;	/* Current JIT code L->base or NULL. */
>   MRef ctype_state;	/* Pointer to C type state. */
>   GCRef gcroot[GCROOT_MAX];  /* GC roots. */
>+#if LJ_HASMEMPROF
>+  uint8_t prof_epoch;	/* Current profiling epoch for this VM. */
>+#endif
> } global_State;
>
> #define mainthread(g)	(&gcref(g->mainthref)->th)
>diff --git a/src/lj_parse.c b/src/lj_parse.c
>index a6325a76..9415665a 100644
>--- a/src/lj_parse.c
>+++ b/src/lj_parse.c
>@@ -1577,6 +1577,9 @@ static GCproto *fs_finish(LexState *ls, BCLine line)
>   pt->flags = (uint8_t)(fs->flags & ~(PROTO_HAS_RETURN|PROTO_FIXUP_RETURN));
>   pt->numparams = fs->numparams;
>   pt->framesize = fs->framesize;
>+#if LJ_HASMEMPROF
>+  pt->prof_epoch = 0;
>+#endif
>   setgcref(pt->chunkname, obj2gco(ls->chunkname));
>
>   /* Close potentially uninitialized gap between bc and kgc. */
>diff --git a/src/lj_state.c b/src/lj_state.c
>index f82b1b5b..b4bfd573 100644
>--- a/src/lj_state.c
>+++ b/src/lj_state.c
>@@ -221,6 +221,9 @@ LUA_API lua_State *lua_newstate(lua_Alloc f, void *ud)
>   g->strempty.gct = ~LJ_TSTR;
>   g->allocf = f;
>   g->allocd = ud;
>+#if LJ_HASMEMPROF
>+  g->prof_epoch = 0;
>+#endif
>   setgcref(g->mainthref, obj2gco(L));
>   setgcref(g->uvhead.prev, obj2gco(&g->uvhead));
>   setgcref(g->uvhead.next, obj2gco(&g->uvhead));
>diff --git a/test/tarantool-tests/misclib-memprof-lapi.test.lua b/test/tarantool-tests/misclib-memprof-lapi.test.lua
>index dd973f2a..1c74b4d7 100644
>--- a/test/tarantool-tests/misclib-memprof-lapi.test.lua
>+++ b/test/tarantool-tests/misclib-memprof-lapi.test.lua
>@@ -7,7 +7,7 @@ require("utils").skipcond(
> local tap = require("tap")
>
> local test = tap.test("misc-memprof-lapi")
>-test:plan(4)
>+test:plan(5)
>
> local jit_opt_default = {
>     3, -- level
>@@ -117,8 +117,9 @@ local function fill_ev_type(events, symbols, event_type)
>   return ev_type
> end
>
>-local function form_source_line(line)
>-  return string.format("@%s:%d", arg[0], line)
>+local function form_source_line(line, source)
>+  source = source or ("@"..arg[0])
>+  return string.format("%s:%d", source, line)
> end
>
> local function check_alloc_report(alloc, location, nevents)
>@@ -126,10 +127,10 @@ local function check_alloc_report(alloc, location, nevents)
>   local traceno = location.traceno
>   if traceno then
>     expected_name = string.format("TRACE [%d] ", traceno)..
>-                    form_source_line(location.line)
>+                    form_source_line(location.line, location.source)
>     event = alloc.trace[traceno]
>   else
>-    expected_name = form_source_line(location.linedefined)
>+    expected_name = form_source_line(location.linedefined, location.source)
>     event = alloc.line[location.line]
>   end
>   assert(expected_name == event.name, ("got='%s', expected='%s'"):format(
>@@ -223,6 +224,38 @@ test:test("stack-resize", function(subtest)
>   misc.memprof.stop()
> end)
>
>+-- Test for extending symtab with function prototypes
>+-- while profiler is running.
>+test:test("symtab-enriching", function(subtest)
>+  subtest:plan(2)
>+
>+  local payload_str = [[
>+  local M = {
>+    tmp = string.rep("1", 100)  -- line 2.
>+  }
>+
>+  function M.payload()
>+    local str = string.rep("42", 100)  -- line 6.
>+  end
>+
>+  return M
>+  ]]
>+
>+  local symbols, events = generate_parsed_output(function()
>+    local str_chunk = assert(loadstring(payload_str, 'str_chunk'))()
>+    str_chunk.payload()
>+  end)
>+
>+  local alloc = fill_ev_type(events, symbols, "alloc")
>+
>+  subtest:ok(check_alloc_report(
>+    alloc, { source = 'str_chunk', line = 6, linedefined = 5 }, 1)
>+  )
>+  subtest:ok(check_alloc_report(
>+    alloc, { source = 'str_chunk', line = 2, linedefined = 0 }, 1)
>+  )
>+end)
>+
> -- Test profiler with enabled JIT.
> jit.on()
>
>diff --git a/tools/memprof/parse.lua b/tools/memprof/parse.lua
>index be5844a4..38f76f00 100644
>--- a/tools/memprof/parse.lua
>+++ b/tools/memprof/parse.lua
>@@ -13,10 +13,11 @@ local string_format = string.format
> local symtab = require "utils.symtab"
>
> local LJM_MAGIC = "ljm"
>-local LJM_CURRENT_VERSION = 0x02
>+local LJM_CURRENT_VERSION = 0x03
>
> local LJM_EPILOGUE_HEADER = 0x80
>
>+local AEVENT_SYMTAB = 0
> local AEVENT_ALLOC = 1
> local AEVENT_FREE = 2
> local AEVENT_REALLOC = 3
>@@ -140,7 +141,14 @@ local function parse_free(reader, asource, events, heap, symbols)
>   heap[oaddr] = nil
> end
>
>+local function parse_symtab(reader, asource, _, _, symbols)
>+  if asource == ASOURCE_LFUNC then
>+    symtab.parse_sym_lfunc(reader, symbols)
>+  end
>+end
>+
> local parsers = {
>+  [AEVENT_SYMTAB] = {evname = "symtab", parse = parse_symtab},
>   [AEVENT_ALLOC] = {evname = "alloc", parse = parse_alloc},
>   [AEVENT_FREE] = {evname = "free", parse = parse_free},
>   [AEVENT_REALLOC] = {evname = "realloc", parse = parse_realloc},
>diff --git a/tools/utils/symtab.lua b/tools/utils/symtab.lua
>index c758b67e..00bab03a 100644
>--- a/tools/utils/symtab.lua
>+++ b/tools/utils/symtab.lua
>@@ -37,7 +37,7 @@ function M.new_loc(symtab, addr, line, traceno)
> end
>
> -- Parse a single entry in a symtab: lfunc symbol.
>-local function parse_sym_lfunc(reader, symtab)
>+function M.parse_sym_lfunc(reader, symtab)
>   local sym_addr = reader:read_uleb128()
>   local sym_chunk = reader:read_string()
>   local sym_line = reader:read_uleb128()
>@@ -69,7 +69,7 @@ local function parse_sym_trace(reader, symtab)
> end
>
> local parsers = {
>-  [SYMTAB_LFUNC] = parse_sym_lfunc,
>+  [SYMTAB_LFUNC] = M.parse_sym_lfunc,
>   [SYMTAB_TRACE] = parse_sym_trace,
> }
>
>-- 
>2.33.1
>