[Tarantool-patches] [PATCH luajit v1 11/11] profile: introduce profile parser

Sergey Kaplun skaplun at tarantool.org
Wed Dec 16 22:13:46 MSK 2020


This patch adds parser for profiler dumped binary data.
It provides a script that parses given binary file. It parses symtab using
ffi first and after map memory events with this symtab. Finally, it
renders the data in human-readable format.

Part of tarantool/tarantool#5442
Part of tarantool/tarantool#5490
---
 test/misclib-memprof-lapi.test.lua    | 125 +++++++++++++++++
 tools/luajit-parse-memprof            |  20 +++
 tools/parse_memprof/bufread.lua       | 143 +++++++++++++++++++
 tools/parse_memprof/main.lua          | 104 ++++++++++++++
 tools/parse_memprof/parse_memprof.lua | 195 ++++++++++++++++++++++++++
 tools/parse_memprof/parse_symtab.lua  |  88 ++++++++++++
 tools/parse_memprof/view_plain.lua    |  45 ++++++
 7 files changed, 720 insertions(+)
 create mode 100755 test/misclib-memprof-lapi.test.lua
 create mode 100755 tools/luajit-parse-memprof
 create mode 100644 tools/parse_memprof/bufread.lua
 create mode 100644 tools/parse_memprof/main.lua
 create mode 100644 tools/parse_memprof/parse_memprof.lua
 create mode 100644 tools/parse_memprof/parse_symtab.lua
 create mode 100644 tools/parse_memprof/view_plain.lua

diff --git a/test/misclib-memprof-lapi.test.lua b/test/misclib-memprof-lapi.test.lua
new file mode 100755
index 0000000..e3663bb
--- /dev/null
+++ b/test/misclib-memprof-lapi.test.lua
@@ -0,0 +1,125 @@
+#!/usr/bin/env tarantool
+
+jit.off()
+jit.flush()
+
+local path = arg[0]:gsub('/[^/]+%.test%.lua', '')
+local parse_suffix = '../tools/parse_memprof/?.lua;'
+package.path = ('%s/%s;'):format(path, parse_suffix)..package.path
+
+local table_new = require "table.new"
+
+local bufread = require "bufread"
+local memprof = require "parse_memprof"
+local symtab  = require "parse_symtab"
+
+local TMP_BINFILE = arg[0]:gsub('[^/]+%.test%.lua', '%.%1.memprofdata.tmp.bin')
+local BAD_PATH    = arg[0]:gsub('[^/]+%.test%.lua', '%1/memprofdata.tmp.bin')
+
+local function payload()
+  -- Preallocate table to avoid array part reallocations.
+  local _ = table_new(100, 0)
+
+  -- Want too see 100 objects here.
+  for i = 1, 100 do
+    -- Shift to avoid crossing with "test" module objects.
+    _[i] = tostring(i + 100)
+  end
+
+  _ = nil
+  -- VMSTATE == GC, reported as INTERNAL.
+  collectgarbage()
+end
+
+local tap = require('tap')
+
+local test = tap.test("misc-memprof-lapi")
+test:plan(6)
+
+local function generate_output(filename)
+  -- Clean up all garbage to avoid polution of free.
+  collectgarbage()
+
+  local res, err = misc.memprof.start(filename)
+  -- Should start succesfully.
+  assert(res, err)
+
+  payload()
+
+  res, err = misc.memprof.stop()
+  -- Should stop succesfully.
+  assert(res, err)
+end
+
+local function fill_ev_type(events, symbols, event_type)
+  local ev_type = {}
+  for _, event in pairs(events[event_type]) do
+    local addr = event.loc.addr
+    if addr == 0 then
+      ev_type.INTERNAL = {
+        name = "INTERNAL",
+        num = event.num,
+    }
+    elseif symbols[addr] then
+      ev_type[event.loc.line] = {
+        name = symbols[addr].name,
+        num = event.num,
+      }
+    end
+  end
+  return ev_type
+end
+
+local function check_alloc_report(alloc, line, function_line, nevents)
+  assert(string.format("@%s:%d", arg[0], function_line) == alloc[line].name)
+  print(nevents, alloc[line].num)
+  assert(alloc[line].num == nevents)
+  return true
+end
+
+-- Not a directory.
+local res, err = misc.memprof.start(BAD_PATH)
+test:ok(res == nil and err:match("Not a directory"))
+
+-- Profiler is running.
+res, err = misc.memprof.start(TMP_BINFILE)
+assert(res, err)
+res, err = misc.memprof.start(TMP_BINFILE)
+test:ok(res == nil and err:match("profiler is running already"))
+
+res, err = misc.memprof.stop()
+assert(res, err)
+
+-- Profiler is not running.
+res, err = misc.memprof.stop()
+test:ok(res == nil and err:match("profiler is not running"))
+
+-- Test profiler output and parse.
+res, err = pcall(generate_output, TMP_BINFILE)
+
+-- Want to cleanup carefully if something went wrong.
+if not res then
+  os.remove(TMP_BINFILE)
+  error(err)
+end
+
+local reader  = bufread.new(TMP_BINFILE)
+local symbols = symtab.parse(reader)
+local events  = memprof.parse(reader, symbols)
+
+-- We don't need it any more.
+os.remove(TMP_BINFILE)
+
+local alloc = fill_ev_type(events, symbols, "alloc")
+local free = fill_ev_type(events, symbols, "free")
+
+-- 1 event -- alocation of table by itself + 1 allocation
+-- of array part as far it bigger then LJ_MAX_COLOSIZE (16).
+test:ok(check_alloc_report(alloc, 21, 19, 2))
+-- 100 strings allocations.
+test:ok(check_alloc_report(alloc, 26, 19, 100))
+
+-- Collect all previous allocated objects.
+test:ok(free.INTERNAL.num == 102)
+
+os.exit(test:check() and 0 or 1)
diff --git a/tools/luajit-parse-memprof b/tools/luajit-parse-memprof
new file mode 100755
index 0000000..b9b16d7
--- /dev/null
+++ b/tools/luajit-parse-memprof
@@ -0,0 +1,20 @@
+#!/bin/bash
+#
+# Launcher for memprof parser.
+# Copyright (C) 2015-2019 IPONWEB Ltd. See Copyright Notice in COPYRIGHT
+
+
+LAUNCHER_DIR=$(dirname `readlink -f $0`)
+# Assume that we are launched from the source tree for now.
+LUAJIT_BIN=$LAUNCHER_DIR/../src/luajit
+LUAJIT_TOOLS_PREFIX=$LAUNCHER_DIR
+
+if [[ ! -x "$LUAJIT_BIN" ]]; then
+    echo "FATAL: Unable to find LuaJIT at $LUAJIT_BIN. Is it built?"
+    exit 1
+fi
+
+TOOL_DIR=$LUAJIT_TOOLS_PREFIX/parse_memprof
+
+LUA_PATH="$TOOL_DIR/?.lua;;" $LUAJIT_BIN $TOOL_DIR/main.lua $@
+
diff --git a/tools/parse_memprof/bufread.lua b/tools/parse_memprof/bufread.lua
new file mode 100644
index 0000000..d48d6e8
--- /dev/null
+++ b/tools/parse_memprof/bufread.lua
@@ -0,0 +1,143 @@
+-- An implementation of buffered reading data from
+-- an arbitrary binary file.
+--
+-- Major portions taken verbatim or adapted from the LuaVela.
+-- Copyright (C) 2015-2019 IPONWEB Ltd.
+
+local assert = assert
+
+local ffi = require 'ffi'
+local bit = require 'bit'
+
+local ffi_C  = ffi.C
+local band   = bit.band
+
+local BUFFER_SIZE = 10 * 1024 * 1024 -- 10 megabytes
+
+local M = {}
+
+ffi.cdef[[
+  void *memcpy(void *, const void *, size_t);
+
+  typedef struct FILE_ FILE;
+  FILE *fopen(const char *, const char *);
+  size_t fread(void *, size_t, size_t, FILE *);
+  int feof(FILE *);
+  int fclose(FILE *);
+]]
+
+local function _read_stream(reader, n)
+  local free_size
+  local tail_size = reader._end - reader._pos
+
+  if tail_size >= n then
+    -- Enough data to satisfy the request of n bytes...
+    return true
+  end
+
+  -- ...otherwise carry tail_size bytes from the end of the buffer
+  -- to the start and fill up free_size bytes with fresh data.
+  -- tail_size < n <= free_size (see assert below) ensures that
+  -- we don't copy overlapping memory regions.
+  -- reader._pos == 0 means filling buffer for the first time.
+
+  free_size = reader._pos > 0 and reader._pos or n
+
+  assert(n <= free_size, 'Internal buffer is large enough')
+
+  if tail_size ~= 0 then
+    ffi_C.memcpy(reader._buf, reader._buf + reader._pos, tail_size)
+  end
+
+  local bytes_read = ffi_C.fread(
+    reader._buf + tail_size, 1, free_size, reader._file
+  )
+
+  reader._pos = 0
+  reader._end = tail_size + bytes_read
+
+  return reader._end - reader._pos >= n
+end
+
+function M.read_octet(reader)
+  if not _read_stream(reader, 1) then
+    return nil
+  end
+
+  local oct = reader._buf[reader._pos]
+  reader._pos = reader._pos + 1
+  return oct
+end
+
+function M.read_octets(reader, n)
+  if not _read_stream(reader, n) then
+    return nil
+  end
+
+  local octets = ffi.string(reader._buf + reader._pos, n)
+  reader._pos = reader._pos + n
+  return octets
+end
+
+function M.read_uleb128(reader)
+  local value = ffi.new('uint64_t', 0)
+  local shift = 0
+
+  repeat
+    local oct = M.read_octet(reader)
+
+    if oct == nil then
+      break
+    end
+
+    -- Alas, bit library works only with 32-bit arguments:
+    local oct_u64 = ffi.new('uint64_t', band(oct, 0x7f))
+    value = value + oct_u64 * (2 ^ shift)
+    shift = shift + 7
+
+  until band(oct, 0x80) == 0
+
+  return tonumber(value)
+end
+
+function M.read_string(reader)
+  local len = M.read_uleb128(reader)
+  return M.read_octets(reader, len)
+end
+
+function M.eof(reader)
+  local sys_feof = ffi_C.feof(reader._file)
+  if sys_feof == 0 then
+    return false
+  end
+  -- ...otherwise return true only we have reached
+  -- the end of the buffer:
+  return reader._pos == reader._end
+end
+
+function M.new(fname)
+  local file = ffi_C.fopen(fname, 'rb')
+  if file == nil then
+    error(string.format('fopen, errno: %d', ffi.errno()))
+  end
+
+  local finalizer = function(f)
+    if ffi_C.fclose(f) ~= 0 then
+      error(string.format('fclose, errno: %d', ffi.errno()))
+    end
+    ffi.gc(f, nil)
+  end
+
+  local reader = setmetatable({
+    _file = ffi.gc(file, finalizer),
+    _buf  = ffi.new('uint8_t[?]', BUFFER_SIZE),
+    _pos  = 0,
+    _end  = 0,
+  }, {__index = M})
+
+  _read_stream(reader, BUFFER_SIZE)
+
+  return reader
+end
+
+return M
diff --git a/tools/parse_memprof/main.lua b/tools/parse_memprof/main.lua
new file mode 100644
index 0000000..9a161b1
--- /dev/null
+++ b/tools/parse_memprof/main.lua
@@ -0,0 +1,104 @@
+-- A tool for parsing and visualisation of LuaJIT's memory
+-- profiler output.
+--
+-- TODO:
+-- * Think about callgraph memory profiling for complex
+--   table reallocations
+-- * Nicer output, probably an HTML view
+-- * Demangling of C symbols
+--
+-- Major portions taken verbatim or adapted from the LuaVela.
+-- Copyright (C) 2015-2019 IPONWEB Ltd.
+
+local bufread = require "bufread"
+local memprof = require "parse_memprof"
+local symtab  = require "parse_symtab"
+local view    = require "view_plain"
+
+local stdout, stderr = io.stdout, io.stderr
+local _s = string
+local match, gmatch = _s.match, _s.gmatch
+
+-- Program options.
+local opt_map = {}
+
+function opt_map.help()
+  stdout:write [[
+luajit-parse-memprof - parser of the memory usage profile collected
+                       with LuaJIT's memprof.
+
+SYNOPSIS
+
+luajit-parse-memprof [options] memprof.bin
+
+Supported options are:
+
+  --help                            Show this help and exit
+]]
+  os.exit(0)
+end
+
+-- Print error and exit with error status.
+local function opterror(...)
+  stderr:write("luajit-parse-memprof.lua: ERROR: ", ...)
+  stderr:write("\n")
+  os.exit(1)
+end
+
+-- Parse single option.
+local function parseopt(opt, args)
+  local opt_current = #opt == 1 and "-"..opt or "--"..opt
+  local f = opt_map[opt]
+  if not f then
+    opterror("unrecognized option `", opt_current, "'. Try `--help'.\n")
+  end
+  f(args)
+end
+
+-- Parse arguments.
+local function parseargs(args)
+  -- Process all option arguments.
+  args.argn = 1
+  repeat
+    local a = args[args.argn]
+    if not a then break end
+    local lopt, opt = match(a, "^%-(%-?)(.+)")
+    if not opt then break end
+    args.argn = args.argn + 1
+    if lopt == "" then
+      -- Loop through short options.
+      for o in gmatch(opt, ".") do parseopt(o, args) end
+    else
+      -- Long option.
+      parseopt(opt, args)
+    end
+  until false
+
+  -- Check for proper number of arguments.
+  local nargs = #args - args.argn + 1
+  if nargs ~= 1 then
+    opt_map.help()
+  end
+
+  -- Translate a single input file.
+  -- TODO: Handle multiple files?
+  return args[args.argn]
+end
+
+local inputfile = parseargs{...}
+
+local reader  = bufread.new(inputfile)
+local symbols = symtab.parse(reader)
+local events  = memprof.parse(reader, symbols)
+
+stdout:write("ALLOCATIONS", "\n")
+view.render(events.alloc, symbols)
+stdout:write("\n")
+
+stdout:write("REALLOCATIONS", "\n")
+view.render(events.realloc, symbols)
+stdout:write("\n")
+
+stdout:write("DEALLOCATIONS", "\n")
+view.render(events.free, symbols)
+stdout:write("\n")
diff --git a/tools/parse_memprof/parse_memprof.lua b/tools/parse_memprof/parse_memprof.lua
new file mode 100644
index 0000000..dc56fed
--- /dev/null
+++ b/tools/parse_memprof/parse_memprof.lua
@@ -0,0 +1,195 @@
+-- Parser of LuaJIT's memprof binary stream.
+-- The format spec can be found in src/profile/ljp_memprof.h.
+--
+-- Major portions taken verbatim or adapted from the LuaVela.
+-- Copyright (C) 2015-2019 IPONWEB Ltd.
+
+local bit    = require 'bit'
+local band   = bit.band
+local lshift = bit.lshift
+
+local string_format = string.format
+
+local LJM_MAGIC           = 'ljm'
+local LJM_CURRENT_VERSION = 2
+
+local LJM_EPILOGUE_HEADER = 0x80
+
+local AEVENT_ALLOC   = 1
+local AEVENT_FREE    = 2
+local AEVENT_REALLOC = 3
+
+local ASOURCE_INT   = lshift(1, 2)
+local ASOURCE_LFUNC = lshift(2, 2)
+local ASOURCE_CFUNC = lshift(3, 2)
+
+local M = {}
+
+local function new_event(loc)
+  return {
+    loc     = loc,
+    num     = 0,
+    free    = 0,
+    alloc   = 0,
+    primary = {},
+  }
+end
+
+local function link_to_previous(heap, e, oaddr)
+  -- memory at oaddr was allocated before we started tracking:
+  if heap[oaddr] then
+    e.primary[heap[oaddr][2]] = heap[oaddr][3]
+  end
+end
+
+local function parse_location(reader, asource)
+  if asource == ASOURCE_INT then
+    return 'f0l0', {
+      addr = 0, -- INTERNAL
+      line = 0,
+    }
+  elseif asource == ASOURCE_CFUNC then
+    local addr = reader:read_uleb128()
+    return string_format('f%#xl%d', addr, 0), {
+      addr = addr,
+      line = 0,
+    }
+  elseif asource == ASOURCE_LFUNC then
+    local addr = reader:read_uleb128()
+    local line = reader:read_uleb128()
+    return string_format('f%#xl%d', addr, line), {
+      addr = addr,
+      line = line,
+    }
+  end
+  error('Unknown asource '..asource)
+end
+
+local function parse_alloc(reader, asource, events, heap)
+  local id, loc = parse_location(reader, asource)
+
+  local naddr = reader:read_uleb128()
+  local nsize = reader:read_uleb128()
+
+  if not events[id] then
+    events[id] = new_event(loc)
+  end
+  local e = events[id]
+  e.num   = e.num + 1
+  e.alloc = e.alloc + nsize
+
+  heap[naddr] = {nsize, id, loc}
+end
+
+local function parse_realloc(reader, asource, events, heap)
+  local id, loc = parse_location(reader, asource)
+
+  local oaddr = reader:read_uleb128()
+  local osize = reader:read_uleb128()
+  local naddr = reader:read_uleb128()
+  local nsize = reader:read_uleb128()
+
+  if not events[id] then
+    events[id] = new_event(loc)
+  end
+  local e = events[id]
+  e.num   = e.num + 1
+  e.free  = e.free + osize
+  e.alloc = e.alloc + nsize
+
+  link_to_previous(heap, e, oaddr)
+
+  heap[oaddr] = nil
+  heap[naddr] = {nsize, id, loc}
+end
+
+local function parse_free(reader, asource, events, heap)
+  local id, loc = parse_location(reader, asource)
+
+  local oaddr = reader:read_uleb128()
+  local osize = reader:read_uleb128()
+
+  if not events[id] then
+    events[id] = new_event(loc)
+  end
+  local e = events[id]
+  e.num   = e.num + 1
+  e.free  = e.free + osize
+
+  link_to_previous(heap, e, oaddr)
+
+  heap[oaddr] = nil
+end
+
+local parsers = {
+  [AEVENT_ALLOC]   = {evname =   'alloc', parse = parse_alloc},
+  [AEVENT_REALLOC] = {evname = 'realloc', parse = parse_realloc},
+  [AEVENT_FREE]    = {evname =    'free', parse = parse_free},
+}
+
+local function ev_header_is_valid(evh)
+  return evh <= 0x0f or evh == LJM_EPILOGUE_HEADER
+end
+
+local function ev_header_is_epilogue(evh)
+  return evh == LJM_EPILOGUE_HEADER
+end
+
+-- Splits event header into event type (aka aevent = allocation
+-- event) and event source (aka asource = allocation source).
+local function ev_header_split(evh)
+  return band(evh, 0x3), band(evh, lshift(0x3, 2))
+end
+
+local function parse_event(reader, events)
+  local ev_header = reader:read_octet()
+
+  assert(ev_header_is_valid(ev_header), 'Bad ev_header '..ev_header)
+
+  if ev_header_is_epilogue(ev_header) then
+    return false
+  end
+
+  local aevent, asource = ev_header_split(ev_header)
+  local parser = parsers[aevent]
+
+  assert(parser, 'Bad aevent '..aevent)
+
+  parser.parse(reader, asource, events[parser.evname], events.heap)
+
+  return true
+end
+
+function M.parse(reader)
+  local events = {
+    alloc   = {},
+    realloc = {},
+    free    = {},
+    heap    = {},
+  }
+
+  local magic   = reader:read_octets(3)
+  local version = reader:read_octets(1)
+  -- dummy-consume reserved bytes
+  local _       = reader:read_octets(3)
+
+  if magic ~= LJM_MAGIC then
+    error('Bad LJM format prologue: '..magic)
+  end
+
+  if string.byte(version) ~= LJM_CURRENT_VERSION then
+    error(string_format(
+      'LJM format version mismatch: the tool expects %d, but your data is %d',
+      LJM_CURRENT_VERSION,
+      string.byte(version)
+    ))
+  end
+
+  while parse_event(reader, events) do
+    -- empty body
+  end
+
+  return events
+end
+
+return M
diff --git a/tools/parse_memprof/parse_symtab.lua b/tools/parse_memprof/parse_symtab.lua
new file mode 100644
index 0000000..54e9337
--- /dev/null
+++ b/tools/parse_memprof/parse_symtab.lua
@@ -0,0 +1,88 @@
+-- Parser of LuaJIT's symtab binary stream.
+-- The format spec can be found in src/profile/ljp_symtab.h.
+--
+-- Major portions taken verbatim or adapted from the LuaVela.
+-- Copyright (C) 2015-2019 IPONWEB Ltd.
+
+local bit = require 'bit'
+
+local band          = bit.band
+local string_format = string.format
+
+local LJS_MAGIC           = 'ljs'
+local LJS_CURRENT_VERSION = 2
+local LJS_EPILOGUE_HEADER = 0x80
+local LJS_SYMTYPE_MASK    = 0x03
+
+local SYMTAB_LFUNC = 0
+
+local M = {}
+
+-- Parse a single entry in a symtab: lfunc symbol
+local function parse_sym_lfunc(reader, symtab)
+  local sym_addr  = reader:read_uleb128()
+  local sym_chunk = reader:read_string()
+  local sym_line  = reader:read_uleb128()
+
+  symtab[sym_addr] = {
+    name = string_format('%s:%d', sym_chunk, sym_line),
+  }
+end
+
+local parsers = {
+  [SYMTAB_LFUNC] = parse_sym_lfunc,
+}
+
+function M.parse(reader)
+  local symtab   = {}
+  local magic    = reader:read_octets(3)
+  local version  = reader:read_octets(1)
+
+  local _ = reader:read_octets(3) -- dummy-consume reserved bytes
+
+  if magic ~= LJS_MAGIC then
+    error("Bad LJS format prologue: "..magic)
+  end
+
+  if string.byte(version) ~= LJS_CURRENT_VERSION then
+    error(string_format(
+         "LJS format version mismatch:"..
+         "the tool expects %d, but your data is %d",
+         LJS_CURRENT_VERSION,
+         string.byte(version)
+    ))
+
+  end
+
+  while not reader:eof() do
+    local header   = reader:read_octet()
+    local is_final = band(header, LJS_EPILOGUE_HEADER) ~= 0
+
+    if is_final then
+      break
+    end
+
+    local sym_type = band(header, LJS_SYMTYPE_MASK)
+    if parsers[sym_type] then
+      parsers[sym_type](reader, symtab)
+    end
+  end
+
+  return symtab
+end
+
+function M.demangle(symtab, loc)
+  local addr = loc.addr
+
+  if addr == 0 then
+    return 'INTERNAL'
+  end
+
+  if symtab[addr] then
+    return string_format('%s, line %d', symtab[addr].name, loc.line)
+  end
+
+  return string_format('CFUNC %#x', addr)
+end
+
+return M
diff --git a/tools/parse_memprof/view_plain.lua b/tools/parse_memprof/view_plain.lua
new file mode 100644
index 0000000..089bc73
--- /dev/null
+++ b/tools/parse_memprof/view_plain.lua
@@ -0,0 +1,45 @@
+-- Simple human-readable renderer of LuaJIT's memprof profile.
+--
+-- Major portions taken verbatim or adapted from the LuaVela.
+-- Copyright (C) 2015-2019 IPONWEB Ltd.
+
+local symtab = require 'parse_symtab'
+
+local M = {}
+
+function M.render(events, symbols)
+  local ids = {}
+
+  for id, _ in pairs(events) do
+    table.insert(ids, id)
+  end
+
+  table.sort(ids, function(id1, id2)
+    return events[id1].num > events[id2].num
+  end)
+
+  for i = 1, #ids do
+    local event = events[ids[i]]
+    print(string.format('%s: %d\t%d\t%d',
+      symtab.demangle(symbols, event.loc),
+      event.num,
+      event.alloc,
+      event.free
+    ))
+
+    local prim_loc = {}
+    for _, loc in pairs(event.primary) do
+      table.insert(prim_loc, symtab.demangle(symbols, loc))
+    end
+    if #prim_loc ~= 0 then
+      table.sort(prim_loc)
+      print('\tOverrides:')
+      for j = 1, #prim_loc do
+        print(string.format('\t\t%s', prim_loc[j]))
+      end
+      print('')
+    end
+  end
+end
+
+return M
-- 
2.28.0



More information about the Tarantool-patches mailing list