From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from [87.239.111.99] (localhost [127.0.0.1]) by dev.tarantool.org (Postfix) with ESMTP id BE7366FC87; Wed, 29 Sep 2021 23:10:03 +0300 (MSK) DKIM-Filter: OpenDKIM Filter v2.11.0 dev.tarantool.org BE7366FC87 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=tarantool.org; s=dev; t=1632946203; bh=vTsv/jbbowd/aP58x9/iWjXixHgeap997TspFfMorlw=; h=To:Date:In-Reply-To:References:Subject:List-Id:List-Unsubscribe: List-Archive:List-Post:List-Help:List-Subscribe:From:Reply-To: From; b=r1BuRR+pPnLrx+gpajXt70VnUZEHrcBROslnw/xYcFx9ZUzWlh4p6aXY8ZMIMy1nq xi3Dkrnzj5Nt4Iu+WH3fpO1mOOKLkn+zRzwFc1oRWgww1s8tV8Cv1egE6h0eE1Pk2a gf06/0zQYAOvFN1kmb9BmQ2qZqivCffAav/WNtpg= Received: from smtp59.i.mail.ru (smtp59.i.mail.ru [217.69.128.39]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by dev.tarantool.org (Postfix) with ESMTPS id 6F3A66DB09 for ; Wed, 29 Sep 2021 23:08:06 +0300 (MSK) DKIM-Filter: OpenDKIM Filter v2.11.0 dev.tarantool.org 6F3A66DB09 Received: by smtp59.i.mail.ru with esmtpa (envelope-from ) id 1mVfs5-0000Jo-Jo; Wed, 29 Sep 2021 23:08:06 +0300 To: tarantool-patches@dev.tarantool.org, imun@tarantool.org, skaplun@tarantool.org Date: Wed, 29 Sep 2021 23:07:58 +0300 Message-Id: <20210929200758.149446-5-m.shishatskiy@tarantool.org> X-Mailer: git-send-email 2.33.0 In-Reply-To: <20210929200758.149446-1-m.shishatskiy@tarantool.org> References: <20210929200758.149446-1-m.shishatskiy@tarantool.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-7564579A: 646B95376F6C166E X-77F55803: 4F1203BC0FB41BD96A58C36AA2E996498302BFE8288A953FFF2204EAAF1B1FDF182A05F538085040F526064D5CBD2122597BBBDB3C0D692241F5FB89198BC79AF7FAC471D6385158 X-7FA49CB5: FF5795518A3D127A4AD6D5ED66289B5278DA827A17800CE735102D6D0BF494F7EA1F7E6F0F101C67BD4B6F7A4D31EC0BCC500DACC3FED6E28638F802B75D45FF8AA50765F79006379763315F766189898638F802B75D45FF36EB9D2243A4F8B5A6FCA7DBDB1FC311F39EFFDF887939037866D6147AF826D844E0F08660DD5163F8855F6F45C0578F117882F4460429724CE54428C33FAD305F5C1EE8F4F765FCAA867293B0326636D2E47CDBA5A96583BD4B6F7A4D31EC0BC014FD901B82EE079FA2833FD35BB23D27C277FBC8AE2E8BAA867293B0326636D2E47CDBA5A96583BA9C0B312567BB2376E601842F6C81A19E625A9149C048EEC24E1E72F37C03A028F6BDBBAB179F4ED8FC6C240DEA7642DBF02ECDB25306B2B78CF848AE20165D0A6AB1C7CE11FEE31F9513A7CA91E5552D242C3BD2E3F4C6C4224003CC836476EA7A3FFF5B025636E2021AF6380DFAD1A18204E546F3947CB11811A4A51E3B096D1867E19FE1407959CC434672EE6371089D37D7C0E48F6C8AA50765F79006373BC478629CBEC79DEFF80C71ABB335746BA297DBC24807EABDAD6C7F3747799A X-B7AD71C0: AC4F5C86D027EB782CDD5689AFBDA7A213B5FB47DCBC3458F0AFF96BAACF4158235E5A14AD4A4A4625E192CAD1D9E79DB194B0D77246B8677B98900BD2A12CF1 X-C1DE0DAB: C20DE7B7AB408E4181F030C43753B8186998911F362727C414F749A5E30D975C5D501480F5414410A8AD913F7D0017635B9AB40A60E31B049C2B6934AE262D3EE7EAB7254005DCED7532B743992DF240BDC6A1CF3F042BAD6DF99611D93F60EF9EAAB76869E07C3E699F904B3F4130E343918A1A30D5E7FCCB5012B2E24CD356 X-C8649E89: 4E36BF7865823D7055A7F0CF078B5EC49A30900B95165D3407FE5477D6A8AF08594DB69D1C81177859DD45D7F8568922B01D4BE85AE2BD60C4BBEA446B6E2F0C1D7E09C32AA3244CE8135A1381454FB7F076B5CFC2D5233BC3B3ADDA61883BB5927AC6DF5659F194 X-D57D3AED: 3ZO7eAau8CL7WIMRKs4sN3D3tLDjz0dLbV79QFUyzQ2Ujvy7cMT6pYYqY16iZVKkSc3dCLJ7zSJH7+u4VD18S7Vl4ZUrpaVfd2+vE6kuoey4m4VkSEu530nj6fImhcD4MUrOEAnl0W826KZ9Q+tr5ycPtXkTV4k65bRjmOUUP8cvGozZ33TWg5HZplvhhXbhDGzqmQDTd6OAevLeAnq3Ra9uf7zvY2zzsIhlcp/Y7m53TZgf2aB4JOg4gkr2biojsAIehEB+Jdkv8rgMmRZHuQ== X-Mailru-Sender: EFA0F3A8419EF21635BFE795C6CB22C9AF4107F71016B8A5597BBBDB3C0D692238066306E4938E9F2376072A51849BFFE66B5C1DBFD5D09D5E022D45988A037B448E0EA96F20AB367402F9BA4338D657ED14614B50AE0675 X-Mras: Ok Subject: [Tarantool-patches] [PATCH luajit v4 4/4] memprof: add info about trace start to symtab X-BeenThere: tarantool-patches@dev.tarantool.org X-Mailman-Version: 2.1.34 Precedence: list List-Id: Tarantool development patches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , From: Mikhail Shishatskiy via Tarantool-patches Reply-To: Mikhail Shishatskiy Errors-To: tarantool-patches-bounces@dev.tarantool.org Sender: "Tarantool-patches" Trace allocation sources, recorded by the memory profiler, were reported as | TRACE [] This approach is not descriptive enough to understand, where exactly allocation took place, as we do not know the code chunk, associated with the trace. This patch fixes the problem described above by extending the symbol table with entries, consisting of a trace's mcode starting address, trace number, address of function proto, and line, where trace recording started: | sym-trace := sym-header trace-no trace-addr sym-addr sym-line | trace-no := | trace-addr := The memory profiler parser is adjusted to recognize the entries mentioned above. On top of that, the API of changed: now table with symbols contains two tables: `lfunc` for Lua functions symbols and `trace` for trace entries. The demangler module has not changed, but the function `describe_location` is added to the module, which allows one to get a description of the trace location in the format described below: | TRACE [] started at @: Follows up tarantool/tarantool#5814 --- Issue: https://github.com/tarantool/tarantool/issues/5814 Branch: https://github.com/tarantool/luajit/tree/shishqa/gh-5814-group-allocations-on-trace-by-trace-number CI: https://github.com/tarantool/tarantool/tree/shishqa/gh-5814-group-allocations-on-trace-by-trace-number src/lj_memprof.c | 43 +++++++++++++++++++ src/lj_memprof.h | 8 +++- .../misclib-memprof-lapi.test.lua | 15 ++++--- tools/memprof.lua | 4 +- tools/memprof/humanize.lua | 30 ++++++++++--- tools/memprof/process.lua | 9 ++-- tools/utils/symtab.lua | 31 ++++++++++--- 7 files changed, 118 insertions(+), 22 deletions(-) diff --git a/src/lj_memprof.c b/src/lj_memprof.c index 8702557f..e8b2ebbc 100644 --- a/src/lj_memprof.c +++ b/src/lj_memprof.c @@ -28,6 +28,45 @@ static const unsigned char ljs_header[] = {'l', 'j', 's', LJS_CURRENT_VERSION, 0x0, 0x0, 0x0}; +#if LJ_HASJIT + +static void dump_symtab_trace(struct lj_wbuf *out, const GCtrace *trace) +{ + GCproto *pt = &gcref(trace->startpt)->pt; + BCLine lineno = 0; + + const BCIns *startpc = mref(trace->startpc, const BCIns); + lua_assert(startpc >= proto_bc(pt) && + startpc < proto_bc(pt) + pt->sizebc); + + lineno = lj_debug_line(pt, proto_bcpos(pt, startpc)); + lua_assert(lineno >= 0); + + lj_wbuf_addbyte(out, SYMTAB_TRACE); + lj_wbuf_addu64(out, (uint64_t)trace->traceno); + lj_wbuf_addu64(out, (uint64_t)trace->mcode); + /* + ** The information about the prototype, associated with the + ** trace's start has already been dumped, as it is anchored + ** via the trace and is not collected while the trace is alive. + ** For this reason, we do not need to repeat dumping the chunk + ** name for the prototype. + */ + lj_wbuf_addu64(out, (uintptr_t)pt); + lj_wbuf_addu64(out, (uint64_t)lineno); +} + +#else + +static void dump_symtab_trace(struct lj_wbuf *out, const GCtrace *trace) +{ + UNUSED(out); + UNUSED(trace); + lua_assert(0); +} + +#endif + static void dump_symtab(struct lj_wbuf *out, const struct global_State *g) { const GCRef *iter = &g->gc.root; @@ -47,6 +86,10 @@ static void dump_symtab(struct lj_wbuf *out, const struct global_State *g) lj_wbuf_addu64(out, (uint64_t)pt->firstline); break; } + case (~LJ_TTRACE): { + dump_symtab_trace(out, gco2trace(o)); + break; + } default: break; } diff --git a/src/lj_memprof.h b/src/lj_memprof.h index 47474a51..395fb429 100644 --- a/src/lj_memprof.h +++ b/src/lj_memprof.h @@ -16,7 +16,7 @@ #include "lj_def.h" #include "lj_wbuf.h" -#define LJS_CURRENT_VERSION 0x1 +#define LJS_CURRENT_VERSION 0x2 /* ** symtab format: @@ -25,13 +25,16 @@ ** prologue := 'l' 'j' 's' version reserved ** version := ** reserved := -** sym := sym-lua | sym-final +** sym := sym-lua | sym-trace | sym-final ** sym-lua := sym-header sym-addr sym-chunk sym-line +** sym-trace := sym-header trace-no trace-addr sym-addr sym-line ** sym-header := ** sym-addr := ** sym-chunk := string ** sym-line := ** sym-final := sym-header +** trace-no := +** trace-addr := ** string := string-len string-payload ** string-len := ** string-payload := {string-len} @@ -51,6 +54,7 @@ */ #define SYMTAB_LFUNC ((uint8_t)0) +#define SYMTAB_TRACE ((uint8_t)1) #define SYMTAB_FINAL ((uint8_t)0x80) #define LJM_CURRENT_FORMAT_VERSION 0x02 diff --git a/test/tarantool-tests/misclib-memprof-lapi.test.lua b/test/tarantool-tests/misclib-memprof-lapi.test.lua index 3f4ffea0..b9edb80d 100644 --- a/test/tarantool-tests/misclib-memprof-lapi.test.lua +++ b/test/tarantool-tests/misclib-memprof-lapi.test.lua @@ -87,9 +87,13 @@ local function fill_ev_type(events, symbols, event_type) local addr = event.loc.addr local traceno = event.loc.traceno - if traceno ~= 0 then + if traceno ~= 0 and symbols.trace[traceno] then + local trace_loc = symbols.trace[traceno].start + addr = trace_loc.addr ev_type.trace[traceno] = { - name = string.format("TRACE [%d]", traceno), + name = string.format("TRACE [%d] %s:%d", + traceno, symbols.lfunc[addr].source, symbols.lfunc[addr].linedefined + ), num = event.num, } elseif addr == 0 then @@ -97,10 +101,10 @@ local function fill_ev_type(events, symbols, event_type) name = "INTERNAL", num = event.num, } - elseif symbols[addr] then + elseif symbols.lfunc[addr] then ev_type.line[event.loc.line] = { name = string.format( - "%s:%d", symbols[addr].source, symbols[addr].linedefined + "%s:%d", symbols.lfunc[addr].source, symbols.lfunc[addr].linedefined ), num = event.num, } @@ -116,7 +120,8 @@ end local function check_alloc_report(alloc, traceno, line, function_line, nevents) local expected_name, event if traceno ~= 0 then - expected_name = string.format("TRACE [%d]", traceno) + expected_name = string.format("TRACE [%d] ", traceno).. + form_source_line(function_line) event = alloc.trace[traceno] else expected_name = form_source_line(function_line) diff --git a/tools/memprof.lua b/tools/memprof.lua index 18b44fdd..760122fc 100644 --- a/tools/memprof.lua +++ b/tools/memprof.lua @@ -104,8 +104,8 @@ local function dump(inputfile) if not leak_only then view.profile_info(events, symbols) end - local dheap = process.form_heap_delta(events, symbols) - view.leak_info(dheap) + local dheap = process.form_heap_delta(events) + view.leak_info(dheap, symbols) os.exit(0) end diff --git a/tools/memprof/humanize.lua b/tools/memprof/humanize.lua index 7771005d..7d30f976 100644 --- a/tools/memprof/humanize.lua +++ b/tools/memprof/humanize.lua @@ -7,6 +7,23 @@ local symtab = require "utils.symtab" local M = {} +function M.describe_location(symbols, loc) + if loc.traceno == 0 then + return symtab.demangle(symbols, loc) + end + + local trace = symbols.trace[loc.traceno] + + -- If trace, which was remembered in the symtab, has not + -- been flushed, assotiate it with a proto, where trace + -- recording started. + if trace and trace.addr == loc.addr then + return symtab.demangle(symbols, loc).." started at ".. + symtab.demangle(symbols, trace.start) + end + return symtab.demangle(symbols, loc) +end + function M.render(events, symbols) local ids = {} @@ -21,7 +38,7 @@ function M.render(events, symbols) for i = 1, #ids do local event = events[ids[i]] print(string.format("%s: %d events\t+%d bytes\t-%d bytes", - symtab.demangle(symbols, event.loc), + M.describe_location(symbols, event.loc), event.num, event.alloc, event.free @@ -29,7 +46,7 @@ function M.render(events, symbols) local prim_loc = {} for _, heap_chunk in pairs(event.primary) do - table.insert(prim_loc, symtab.demangle(symbols, heap_chunk.loc)) + table.insert(prim_loc, M.describe_location(symbols, heap_chunk.loc)) end if #prim_loc ~= 0 then table.sort(prim_loc) @@ -56,13 +73,16 @@ function M.profile_info(events, symbols) print("") end -function M.leak_info(dheap) +function M.leak_info(dheap, symbols) local leaks = {} - for line, info in pairs(dheap) do + for _, info in pairs(dheap) do -- Report "INTERNAL" events inconsistencies for profiling -- with enabled jit. if info.dbytes > 0 then - table.insert(leaks, {line = line, dbytes = info.dbytes}) + table.insert(leaks, { + line = M.describe_location(symbols, info.loc), + dbytes = info.dbytes + }) end end diff --git a/tools/memprof/process.lua b/tools/memprof/process.lua index 0bcb965b..360f6cc4 100644 --- a/tools/memprof/process.lua +++ b/tools/memprof/process.lua @@ -4,7 +4,7 @@ local M = {} local symtab = require "utils.symtab" -function M.form_heap_delta(events, symbols) +function M.form_heap_delta(events) -- Auto resurrects source event lines for counting/reporting. local dheap = setmetatable({}, {__index = function(t, line) rawset(t, line, { @@ -17,11 +17,12 @@ function M.form_heap_delta(events, symbols) for _, event in pairs(events.alloc) do if event.loc then - local ev_line = symtab.demangle(symbols, event.loc) + local ev_line = symtab.id(event.loc) if (event.alloc > 0) then dheap[ev_line].dbytes = dheap[ev_line].dbytes + event.alloc dheap[ev_line].nalloc = dheap[ev_line].nalloc + event.num + dheap[ev_line].loc = event.loc end end end @@ -37,16 +38,18 @@ function M.form_heap_delta(events, symbols) -- that references the table with memory changed -- (may be empty). for _, heap_chunk in pairs(event.primary) do - local ev_line = symtab.demangle(symbols, heap_chunk.loc) + local ev_line = symtab.id(heap_chunk.loc) if (heap_chunk.alloced > 0) then dheap[ev_line].dbytes = dheap[ev_line].dbytes + heap_chunk.alloced dheap[ev_line].nalloc = dheap[ev_line].nalloc + heap_chunk.count + dheap[ev_line].loc = heap_chunk.loc end if (heap_chunk.freed > 0) then dheap[ev_line].dbytes = dheap[ev_line].dbytes - heap_chunk.freed dheap[ev_line].nfree = dheap[ev_line].nfree + heap_chunk.count + dheap[ev_line].loc = heap_chunk.loc end end end diff --git a/tools/utils/symtab.lua b/tools/utils/symtab.lua index 85945fb2..496d8480 100644 --- a/tools/utils/symtab.lua +++ b/tools/utils/symtab.lua @@ -10,11 +10,12 @@ local band = bit.band local string_format = string.format local LJS_MAGIC = "ljs" -local LJS_CURRENT_VERSION = 1 +local LJS_CURRENT_VERSION = 0x2 local LJS_EPILOGUE_HEADER = 0x80 local LJS_SYMTYPE_MASK = 0x03 local SYMTAB_LFUNC = 0 +local SYMTAB_TRACE = 1 local M = {} @@ -24,18 +25,38 @@ local function parse_sym_lfunc(reader, symtab) local sym_chunk = reader:read_string() local sym_line = reader:read_uleb128() - symtab[sym_addr] = { + symtab.lfunc[sym_addr] = { source = sym_chunk, linedefined = sym_line, } end +local function parse_sym_trace(reader, symtab) + local traceno = reader:read_uleb128() + local trace_addr = reader:read_uleb128() + local sym_addr = reader:read_uleb128() + local sym_line = reader:read_uleb128() + + symtab.trace[traceno] = { + addr = trace_addr, + start = { + addr = sym_addr, + line = sym_line, + traceno = 0, + }, + } +end + local parsers = { [SYMTAB_LFUNC] = parse_sym_lfunc, + [SYMTAB_TRACE] = parse_sym_trace, } function M.parse(reader) - local symtab = {} + local symtab = { + lfunc = {}, + trace = {}, + } local magic = reader:read_octets(3) local version = reader:read_octets(1) @@ -82,8 +103,8 @@ local function demangle_lfunc(symtab, loc) if addr == 0 then return "INTERNAL" - elseif symtab[addr] then - return string_format("%s:%d", symtab[addr].source, loc.line) + elseif symtab.lfunc[addr] then + return string_format("%s:%d", symtab.lfunc[addr].source, loc.line) end return string_format("CFUNC %#x", addr) end -- 2.33.0