* [Tarantool-patches] [PATCH luajit v2] memprof: enrich symtab when new trace is allocated
@ 2021-12-08 17:22 Mikhail Shishatskiy via Tarantool-patches
2022-04-12 14:29 ` Igor Munkin via Tarantool-patches
0 siblings, 1 reply; 2+ messages in thread
From: Mikhail Shishatskiy via Tarantool-patches @ 2021-12-08 17:22 UTC (permalink / raw)
To: tarantool-patches, imun, skaplun
Since symtab can be enriched with new prototypes, it
can be enriched with new traces as well. This feature
can help to investigate trace-generation-heavy apps
with memprof.
This patch introduces the functionality described above by
adding a `prof_epoch` counter to the `GCtrace` structure.
If the profiler meets allocation from a trace, it checks if
the trace was already streamed to the symtab. If not,
a symtab entry is dumped inside an <event-symtab>:
| event-symtab := event-header sym?
| sym := sym-lua | sym-trace
| sym-trace := trace-no sym-addr line-no
Since this patch, traces are no longer identified by address,
as all the name collisions are resolved with symbols epochs.
Also, the API of <utils/symtab.lua> changed: function `parse_sym_trace`
is now public in order to use it from the <memrprof/parser.lua> module.
Follows up tarantool/tarantool#5815
---
CI: https://github.com/tarantool/tarantool/tree/shishqa/enrich-symtab-when-trace-is-allocated-v2
branch: https://github.com/tarantool/luajit/tree/shishqa/enrich-symtab-when-trace-is-allocated-v2
src/lj_jit.h | 8 +++++++
src/lj_memprof.c | 22 +++++++++++++------
src/lj_memprof.h | 5 +++--
src/lj_trace.c | 3 +++
.../misclib-memprof-lapi.test.lua | 13 +++++------
tools/memprof/parse.lua | 3 ++-
tools/utils/symtab.lua | 14 ++++--------
7 files changed, 41 insertions(+), 27 deletions(-)
diff --git a/src/lj_jit.h b/src/lj_jit.h
index d82292f8..766892aa 100644
--- a/src/lj_jit.h
+++ b/src/lj_jit.h
@@ -254,7 +254,15 @@ typedef struct GCtrace {
uint8_t sinktags; /* Trace has SINK tags. */
uint8_t topslot; /* Top stack slot already checked to be allocated. */
uint8_t linktype; /* Type of link. */
+#if LJ_HASMEMPROF
+ /*
+ ** Epoch indicating if this trace was dumped to the symbol table for the
+ ** current profiling session.
+ */
+ uint8_t prof_epoch;
+#else
uint8_t unused1;
+#endif
#ifdef LUAJIT_USE_GDBJIT
void *gdbjit_entry; /* GDB JIT entry. */
#endif
diff --git a/src/lj_memprof.c b/src/lj_memprof.c
index c154de93..367fbe2b 100644
--- a/src/lj_memprof.c
+++ b/src/lj_memprof.c
@@ -30,7 +30,8 @@ static const unsigned char ljs_header[] = {'l', 'j', 's', LJS_CURRENT_VERSION,
#if LJ_HASJIT
-static void dump_symtab_trace(struct lj_wbuf *out, const GCtrace *trace)
+static void dump_symtab_trace(struct lj_wbuf *out, GCtrace *trace,
+ const global_State *g)
{
GCproto *pt = &gcref(trace->startpt)->pt;
BCLine lineno = 0;
@@ -41,9 +42,7 @@ static void dump_symtab_trace(struct lj_wbuf *out, const GCtrace *trace)
lineno = lj_debug_line(pt, proto_bcpos(pt, startpc));
- lj_wbuf_addbyte(out, SYMTAB_TRACE);
lj_wbuf_addu64(out, (uint64_t)trace->traceno);
- lj_wbuf_addu64(out, (uint64_t)trace->mcode);
/*
** The information about the prototype, associated with the
** trace's start has already been dumped, as it is anchored
@@ -53,11 +52,14 @@ static void dump_symtab_trace(struct lj_wbuf *out, const GCtrace *trace)
*/
lj_wbuf_addu64(out, (uintptr_t)pt);
lj_wbuf_addu64(out, (uint64_t)lineno);
+
+ trace->prof_epoch = g->prof_epoch;
}
#else
-static void dump_symtab_trace(struct lj_wbuf *out, const GCtrace *trace)
+static void dump_symtab_trace(struct lj_wbuf *out, const GCtrace *trace,
+ const global_State *g)
{
UNUSED(out);
UNUSED(trace);
@@ -93,7 +95,8 @@ static void dump_symtab(struct lj_wbuf *out, const struct global_State *g)
break;
}
case (~LJ_TTRACE): {
- dump_symtab_trace(out, gco2trace(o));
+ lj_wbuf_addbyte(out, SYMTAB_TRACE);
+ dump_symtab_trace(out, gco2trace(o), g);
break;
}
default:
@@ -222,10 +225,15 @@ static void memprof_write_trace(struct memprof *mp, uint8_t aevent)
const global_State *g = mp->g;
const jit_State *J = G2J(g);
const TraceNo traceno = g->vmstate;
- const GCtrace *trace = traceref(J, traceno);
+ GCtrace *trace = traceref(J, traceno);
+
+ if (LJ_UNLIKELY(trace->prof_epoch != g->prof_epoch)) {
+ lj_wbuf_addbyte(out, AEVENT_SYMTAB | ASOURCE_TRACE);
+ dump_symtab_trace(out, trace, g);
+ }
+
lj_wbuf_addbyte(out, aevent | ASOURCE_TRACE);
lj_wbuf_addu64(out, (uint64_t)traceno);
- lj_wbuf_addu64(out, (uintptr_t)trace->mcode);
}
#else
diff --git a/src/lj_memprof.h b/src/lj_memprof.h
index 150e6b32..0b06eb5b 100644
--- a/src/lj_memprof.h
+++ b/src/lj_memprof.h
@@ -27,7 +27,7 @@
** reserved := <BYTE> <BYTE> <BYTE>
** sym := sym-lua | sym-trace | sym-final
** sym-lua := sym-header sym-addr sym-chunk sym-line
-** sym-trace := sym-header trace-no trace-addr sym-addr sym-line
+** sym-trace := sym-header trace-no sym-addr sym-line
** sym-header := <BYTE>
** sym-addr := <ULEB128>
** sym-chunk := string
@@ -78,8 +78,9 @@
** loc-lua := sym-addr line-no
** loc-c := sym-addr
** loc-trace := trace-no trace-addr
-** sym := sym-lua
+** sym := sym-lua | sym-trace
** sym-lua := sym-addr sym-chunk sym-line
+** sym-trace := trace-no sym-addr line-no
** sym-addr := <ULEB128>
** sym-chunk := string
** sym-line := <ULEB128>
diff --git a/src/lj_trace.c b/src/lj_trace.c
index 86563cdb..3fc6fc14 100644
--- a/src/lj_trace.c
+++ b/src/lj_trace.c
@@ -163,6 +163,9 @@ static void trace_save(jit_State *J, GCtrace *T)
#ifdef LUAJIT_USE_PERFTOOLS
perftools_addtrace(T);
#endif
+#if LJ_HASMEMPROF
+ T->prof_epoch = 0;
+#endif
}
void LJ_FASTCALL lj_trace_free(global_State *g, GCtrace *T)
diff --git a/test/tarantool-tests/misclib-memprof-lapi.test.lua b/test/tarantool-tests/misclib-memprof-lapi.test.lua
index 1c74b4d7..b7822563 100644
--- a/test/tarantool-tests/misclib-memprof-lapi.test.lua
+++ b/test/tarantool-tests/misclib-memprof-lapi.test.lua
@@ -267,7 +267,7 @@ test:test("jit-output", function(subtest)
return
end
- subtest:plan(3)
+ subtest:plan(4)
jit.opt.start(3, "hotloop=10")
jit.flush()
@@ -282,13 +282,12 @@ test:test("jit-output", function(subtest)
-- See also https://github.com/tarantool/tarantool/issues/5679.
subtest:ok(alloc[0] == nil)
- -- Run already generated traces.
- symbols, events = generate_parsed_output(default_payload)
-
- alloc = fill_ev_type(events, symbols, "alloc")
-
-- We expect, that loop will be compiled into a trace.
- subtest:ok(check_alloc_report(alloc, { traceno = 1, line = 37 }, 20))
+ -- 10 allocations in interpreter mode, 1 allocation for a trace
+ -- recording and assembling and next 9 allocations will happen
+ -- inside the trace.
+ subtest:ok(check_alloc_report(alloc, { line = 39, linedefined = 32 }, 11))
+ subtest:ok(check_alloc_report(alloc, { traceno = 1, line = 37 }, 9))
-- See same checks with jit.off().
subtest:ok(check_alloc_report(alloc, { line = 34, linedefined = 32 }, 2))
diff --git a/tools/memprof/parse.lua b/tools/memprof/parse.lua
index 38f76f00..fb4fcf3c 100644
--- a/tools/memprof/parse.lua
+++ b/tools/memprof/parse.lua
@@ -77,7 +77,6 @@ local function parse_location(reader, asource, symbols)
line = reader:read_uleb128()
elseif asource == ASOURCE_TRACE then
traceno = reader:read_uleb128()
- addr = reader:read_uleb128()
else
error("Unknown asource "..asource)
end
@@ -144,6 +143,8 @@ end
local function parse_symtab(reader, asource, _, _, symbols)
if asource == ASOURCE_LFUNC then
symtab.parse_sym_lfunc(reader, symbols)
+ elseif asource == ASOURCE_TRACE then
+ symtab.parse_sym_trace(reader, symbols)
end
end
diff --git a/tools/utils/symtab.lua b/tools/utils/symtab.lua
index 133a0fc7..fced4c20 100644
--- a/tools/utils/symtab.lua
+++ b/tools/utils/symtab.lua
@@ -59,9 +59,8 @@ function M.parse_sym_lfunc(reader, symtab)
})
end
-local function parse_sym_trace(reader, symtab)
+function M.parse_sym_trace(reader, symtab)
local traceno = reader:read_uleb128()
- local trace_addr = reader:read_uleb128()
local sym_addr = reader:read_uleb128()
local sym_line = reader:read_uleb128()
@@ -70,14 +69,13 @@ local function parse_sym_trace(reader, symtab)
end
table.insert(symtab.trace[traceno], {
- addr = trace_addr,
start = M.new_loc(symtab, sym_addr, sym_line, 0)
})
end
local parsers = {
[SYMTAB_LFUNC] = M.parse_sym_lfunc,
- [SYMTAB_TRACE] = parse_sym_trace,
+ [SYMTAB_TRACE] = M.parse_sym_trace,
}
function M.parse(reader)
@@ -131,18 +129,14 @@ end
local function demangle_trace(symtab, loc)
local traceno = loc.traceno
- local addr = loc.addr
assert(traceno ~= 0, "Location is a trace")
- local trace_str = string_format("TRACE [%d] %#x", traceno, addr)
+ local trace_str = string_format("TRACE [%d]", traceno)
local epochs = symtab.trace[traceno]
local trace = epochs and epochs[loc.epoch]
- -- If trace, which was remembered in the symtab, has not
- -- been flushed, associate it with a proto, where trace
- -- recording started.
- if trace and trace.addr == addr then
+ if trace then
assert(trace.start.traceno == 0, "Trace start is not a trace")
return trace_str.." started at "..M.demangle(symtab, trace.start)
end
--
2.33.1
^ permalink raw reply [flat|nested] 2+ messages in thread
* Re: [Tarantool-patches] [PATCH luajit v2] memprof: enrich symtab when new trace is allocated
2021-12-08 17:22 [Tarantool-patches] [PATCH luajit v2] memprof: enrich symtab when new trace is allocated Mikhail Shishatskiy via Tarantool-patches
@ 2022-04-12 14:29 ` Igor Munkin via Tarantool-patches
0 siblings, 0 replies; 2+ messages in thread
From: Igor Munkin via Tarantool-patches @ 2022-04-12 14:29 UTC (permalink / raw)
To: Mikhail Shishatskiy; +Cc: tarantool-patches
Misha,
Thanks for the patch!
I've fixed few issues I've found by myself, LGTM otherwise. CI is
green[1], so I've I've checked the patch into tarantool branch in
tarantool/luajit and bumped a new version in master.
On 08.12.21, Mikhail Shishatskiy wrote:
> Since symtab can be enriched with new prototypes, it
> can be enriched with new traces as well. This feature
> can help to investigate trace-generation-heavy apps
> with memprof.
>
> This patch introduces the functionality described above by
> adding a `prof_epoch` counter to the `GCtrace` structure.
> If the profiler meets allocation from a trace, it checks if
> the trace was already streamed to the symtab. If not,
> a symtab entry is dumped inside an <event-symtab>:
>
> | event-symtab := event-header sym?
> | sym := sym-lua | sym-trace
> | sym-trace := trace-no sym-addr line-no
>
> Since this patch, traces are no longer identified by address,
> as all the name collisions are resolved with symbols epochs.
>
> Also, the API of <utils/symtab.lua> changed: function `parse_sym_trace`
> is now public in order to use it from the <memrprof/parser.lua> module.
>
> Follows up tarantool/tarantool#5815
> ---
>
> CI: https://github.com/tarantool/tarantool/tree/shishqa/enrich-symtab-when-trace-is-allocated-v2
> branch: https://github.com/tarantool/luajit/tree/shishqa/enrich-symtab-when-trace-is-allocated-v2
>
> src/lj_jit.h | 8 +++++++
> src/lj_memprof.c | 22 +++++++++++++------
> src/lj_memprof.h | 5 +++--
> src/lj_trace.c | 3 +++
> .../misclib-memprof-lapi.test.lua | 13 +++++------
> tools/memprof/parse.lua | 3 ++-
> tools/utils/symtab.lua | 14 ++++--------
> 7 files changed, 41 insertions(+), 27 deletions(-)
>
> --
> 2.33.1
>
[1]: https://github.com/tarantool/luajit/commit/11ab4c4
--
Best regards,
IM
^ permalink raw reply [flat|nested] 2+ messages in thread
end of thread, other threads:[~2022-04-12 14:33 UTC | newest]
Thread overview: 2+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2021-12-08 17:22 [Tarantool-patches] [PATCH luajit v2] memprof: enrich symtab when new trace is allocated Mikhail Shishatskiy via Tarantool-patches
2022-04-12 14:29 ` Igor Munkin via Tarantool-patches
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox