From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from [87.239.111.99] (localhost [127.0.0.1]) by dev.tarantool.org (Postfix) with ESMTP id A3E4D6ECE8; Wed, 13 Apr 2022 23:39:53 +0300 (MSK) DKIM-Filter: OpenDKIM Filter v2.11.0 dev.tarantool.org A3E4D6ECE8 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=tarantool.org; s=dev; t=1649882393; bh=Nk1xXvZ/uAh+wAESPcQrK1X7+8ayPkotKUJ+t2VNzUw=; h=To:Date:In-Reply-To:References:Subject:List-Id:List-Unsubscribe: List-Archive:List-Post:List-Help:List-Subscribe:From:Reply-To: From; b=fXZY7/KIBsU5oIRWN3xcA39APZUfVh5m7O8QCbwk4LLnf9EWNtFsJ/rfzbd9Lc8kq 7cwSpbr45xHk3Ua/HgQldVUWR5grxmfx4TobOKwXz88ZsD599DDhI7rrTgo9OqwueZ T/R5dkZObBCmLJEeXv+T5wE/UcZJXV94h46D2W8k= Received: from mail-lj1-f174.google.com (mail-lj1-f174.google.com [209.85.208.174]) (using TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 (128/128 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256) (No client certificate requested) by dev.tarantool.org (Postfix) with ESMTPS id 9BD716ECE8 for ; Wed, 13 Apr 2022 23:36:26 +0300 (MSK) DKIM-Filter: OpenDKIM Filter v2.11.0 dev.tarantool.org 9BD716ECE8 Received: by mail-lj1-f174.google.com with SMTP id h11so3647718ljb.2 for ; Wed, 13 Apr 2022 13:36:26 -0700 (PDT) X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=kSrpxOvwpz/sb8B5YijExgVUIv5X14cCHy7Q5z5kRPE=; b=hDJBvMFiJA2sQZRxSHGAkD36UrP05tHJRA3OZVsMmnwvujCD9nd2//mZqlzJa00Xbh V0FEgQ2ZemkRb2YmOpE/OP9ud4sKlyQ8caZ4lJK2g+RIcSm/63cjwEbHi3/nj21WrSiI 7b7qCq/Q3RI8NZMSpMsZSvo9L3cv1Ee5Lnntwk9Usnls9ocPT7CIJRJ5AjdgAc/t1oBH d3BVVmSQEkeZSy4yobvhRfgHHCsW3HkLop2bflJ0aAgc1/s8P7Liv15FLJgb/tvs5edQ 6teYYEb4LjgCBk/Zetsb65oqLaXURJuPQVdjiK1L1euT3mk9vfD70A0elfVkSz8f2tCG i3hw== X-Gm-Message-State: AOAM532ck3uB/eP0U1mwbHwsIjs992TkDukT38CHYxvxaop1ynaSmpUA +Z6PmY4BuQ5VCkh1WrKByTeld6cYpOyDKA== X-Google-Smtp-Source: ABdhPJxm0RisTmzcYb9QWFanCiExh2UD/o6dMAxqkitY0PlCHTPjZMDdZmFiG5Mf6TL5WNiOnq2aMQ== X-Received: by 2002:a2e:bc26:0:b0:24b:7124:1d5c with SMTP id b38-20020a2ebc26000000b0024b71241d5cmr5959078ljf.151.1649882185882; Wed, 13 Apr 2022 13:36:25 -0700 (PDT) Received: from localhost.localdomain ([93.175.11.199]) by smtp.gmail.com with ESMTPSA id e15-20020a19500f000000b0046bb76678bcsm2389lfb.131.2022.04.13.13.36.25 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 13 Apr 2022 13:36:25 -0700 (PDT) X-Google-Original-From: Maxim Kokryashkin To: tarantool-patches@dev.tarantool.org, imun@tarantool.org, skaplun@tarantool.org Date: Wed, 13 Apr 2022 23:36:13 +0300 Message-Id: <1bfa54e6d68ec9ca65cca2a15addb91d88cc359e.1649881981.git.m.kokryashkin@tarantool.org> X-Mailer: git-send-email 2.35.1 In-Reply-To: References: MIME-Version: 1.0 Content-Transfer-Encoding: 8bit Subject: [Tarantool-patches] [PATCH luajit v4 7/7] tools: introduce parsers for sysprof X-BeenThere: tarantool-patches@dev.tarantool.org X-Mailman-Version: 2.1.34 Precedence: list List-Id: Tarantool development patches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , From: Maxim Kokryashkin via Tarantool-patches Reply-To: Maxim Kokryashkin Errors-To: tarantool-patches-bounces@dev.tarantool.org Sender: "Tarantool-patches" Since the sysprof's binary output is not human-readable, so there is a demand to create a parser. The parser, which this commit provides, converts sysprof's event stream into the format that the flamegraph.pl can process. The sysprof parser machinery uses the same symtab module as memprof's parser since the format is the same. Part of tarantool/tarantool#781 --- .gitignore | 1 + .../misclib-sysprof-lapi.test.lua | 2 + tools/CMakeLists.txt | 83 ++++++++ tools/luajit-parse-sysprof.in | 6 + tools/sysprof.lua | 119 +++++++++++ tools/sysprof/collapse.lua | 113 +++++++++++ tools/sysprof/parse.lua | 188 ++++++++++++++++++ tools/utils/symtab.lua | 2 +- 8 files changed, 513 insertions(+), 1 deletion(-) create mode 100644 tools/luajit-parse-sysprof.in create mode 100644 tools/sysprof.lua create mode 100755 tools/sysprof/collapse.lua create mode 100755 tools/sysprof/parse.lua diff --git a/.gitignore b/.gitignore index 2103a30f..099df060 100644 --- a/.gitignore +++ b/.gitignore @@ -19,4 +19,5 @@ cmake_uninstall.cmake compile_commands.json install_manifest.txt luajit-parse-memprof +luajit-parse-sysprof luajit.pc diff --git a/test/tarantool-tests/misclib-sysprof-lapi.test.lua b/test/tarantool-tests/misclib-sysprof-lapi.test.lua index 625bdf55..30e54882 100644 --- a/test/tarantool-tests/misclib-sysprof-lapi.test.lua +++ b/test/tarantool-tests/misclib-sysprof-lapi.test.lua @@ -14,6 +14,7 @@ jit.flush() local bufread = require("utils.bufread") local symtab = require("utils.symtab") +local sysprof = require("sysprof.parse") local TMP_BINFILE = arg[0]:gsub(".+/([^/]+)%.test%.lua$", "%.%1.sysprofdata.tmp.bin") local BAD_PATH = arg[0]:gsub(".+/([^/]+)%.test%.lua$", "%1/sysprofdata.tmp.bin") @@ -51,6 +52,7 @@ local function check_mode(mode, interval) local reader = bufread.new(TMP_BINFILE) symtab.parse(reader) + sysprof.parse(reader) end -- GENERAL diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 61830e44..93a2f763 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -87,4 +87,87 @@ else() ) endif() + + +if(LUAJIT_DISABLE_SYSPROF) + message(STATUS "LuaJIT system profiler support is disabled") +else() + # XXX: Can use genex here since the value need to be evaluated + # at the configuration phase. Fortunately, we know the exact + # path where LuaJIT binary is located. + set(LUAJIT_TOOLS_BIN ${LUAJIT_BINARY_DIR}/${LUAJIT_CLI_NAME}) + set(LUAJIT_TOOLS_DIR ${CMAKE_CURRENT_SOURCE_DIR}) + # XXX: Unfortunately, there is no convenient way to set + # particular permissions to the output file via CMake. + # Furthermore, I even failed to copy the given file to the same + # path to change its permissions. After looking at the docs, I + # realized that the valid solution would be too monstrous for + # such a simple task. As a result I've made the template itself + # executable, so the issue is resolved. + configure_file(luajit-parse-sysprof.in luajit-parse-sysprof @ONLY ESCAPE_QUOTES) + + + add_custom_target(tools-parse-sysprof EXCLUDE_FROM_ALL DEPENDS + luajit-parse-sysprof + sysprof/parse.lua + sysprof/collapse.lua + sysprof.lua + utils/bufread.lua + utils/symtab.lua + ) + list(APPEND LUAJIT_TOOLS_DEPS tools-parse-sysprof) + + install(FILES + ${CMAKE_CURRENT_SOURCE_DIR}/sysprof/parse.lua + ${CMAKE_CURRENT_SOURCE_DIR}/sysprof/collapse.lua + DESTINATION ${LUAJIT_DATAROOTDIR}/sysprof + PERMISSIONS + OWNER_READ OWNER_WRITE + GROUP_READ + WORLD_READ + COMPONENT tools-parse-sysprof + ) + install(FILES + ${CMAKE_CURRENT_SOURCE_DIR}/utils/bufread.lua + ${CMAKE_CURRENT_SOURCE_DIR}/utils/symtab.lua + DESTINATION ${LUAJIT_DATAROOTDIR}/utils + PERMISSIONS + OWNER_READ OWNER_WRITE + GROUP_READ + WORLD_READ + COMPONENT tools-parse-sysprof + ) + install(FILES + ${CMAKE_CURRENT_SOURCE_DIR}/sysprof.lua + DESTINATION ${LUAJIT_DATAROOTDIR} + PERMISSIONS + OWNER_READ OWNER_WRITE + GROUP_READ + WORLD_READ + COMPONENT tools-parse-sysprof + ) + install(CODE + # XXX: The auxiliary script needs to be configured for to be + # used in repository directly. Furthermore, it needs to be + # reconfigured prior to its installation. The temporary + # output is stored to the project build + # directory and removed later after being installed. This + # script will have gone as a result of the issue: + # https://github.com/tarantool/tarantool/issues/5688. + " + set(LUAJIT_TOOLS_BIN ${CMAKE_INSTALL_PREFIX}/bin/${LUAJIT_CLI_NAME}) + set(LUAJIT_TOOLS_DIR ${CMAKE_INSTALL_PREFIX}/${LUAJIT_DATAROOTDIR}) + configure_file(${CMAKE_CURRENT_SOURCE_DIR}/luajit-parse-sysprof.in + ${PROJECT_BINARY_DIR}/luajit-parse-sysprof @ONLY ESCAPE_QUOTES) + file(INSTALL ${PROJECT_BINARY_DIR}/luajit-parse-sysprof + DESTINATION ${CMAKE_INSTALL_PREFIX}/bin + USE_SOURCE_PERMISSIONS + ) + file(REMOVE ${PROJECT_BINARY_DIR}/luajit-parse-sysprof) + " + COMPONENT tools-parse-sysprof + ) +endif() + add_custom_target(LuaJIT-tools DEPENDS ${LUAJIT_TOOLS_DEPS}) + diff --git a/tools/luajit-parse-sysprof.in b/tools/luajit-parse-sysprof.in new file mode 100644 index 00000000..2be25eb3 --- /dev/null +++ b/tools/luajit-parse-sysprof.in @@ -0,0 +1,6 @@ +#!/bin/bash +# +# Launcher for sysprof parser. + +LUA_PATH="@LUAJIT_TOOLS_DIR@/?.lua;;" \ + @LUAJIT_TOOLS_BIN@ @LUAJIT_TOOLS_DIR@/sysprof.lua $@ diff --git a/tools/sysprof.lua b/tools/sysprof.lua new file mode 100644 index 00000000..e6d8cc34 --- /dev/null +++ b/tools/sysprof.lua @@ -0,0 +1,119 @@ +local bufread = require "utils.bufread" +local sysprof = require "sysprof.parse" +local symtab = require "utils.symtab" +local misc = require "sysprof.collapse" + +local stdout, stderr = io.stdout, io.stderr +local match, gmatch = string.match, string.gmatch + +local split_by_vmstate = false + +-- Program options. +local opt_map = {} + +function opt_map.help() + stdout:write [[ +luajit-parse-sysprof - parser of the profile collected + with LuaJIT's sysprof. + +SYNOPSIS + +luajit-parse-sysprof [options] sysprof.bin + +Supported options are: + + --help Show this help and exit + --split Split callchains by vmstate +]] + os.exit(0) +end + +function opt_map.split() + split_by_vmstate = true +end + +-- Print error and exit with error status. +local function opterror(...) + stderr:write("luajit-parse-sysprof.lua: ERROR: ", ...) + stderr:write("\n") + os.exit(1) +end + +-- Parse single option. +local function parseopt(opt, args) + local opt_current = #opt == 1 and "-"..opt or "--"..opt + local f = opt_map[opt] + if not f then + opterror("unrecognized option `", opt_current, "'. Try `--help'.\n") + end + f(args) +end + +-- Parse arguments. +local function parseargs(args) + -- Process all option arguments. + args.argn = 1 + repeat + local a = args[args.argn] + if not a then + break + end + local lopt, opt = match(a, "^%-(%-?)(.+)") + if not opt then + break + end + args.argn = args.argn + 1 + if lopt == "" then + -- Loop through short options. + for o in gmatch(opt, ".") do + parseopt(o, args) + end + else + -- Long option. + parseopt(opt, args) + end + until false + + -- Check for proper number of arguments. + local nargs = #args - args.argn + 1 + if nargs ~= 1 then + opt_map.help() + end + + -- Translate a single input file. + -- TODO: Handle multiple files? + return args[args.argn] +end + +local function traverse_calltree(node, prefix) + if node.is_leaf then + print(prefix..' '..node.count) + end + + local sep_prefix = #prefix == 0 and prefix or prefix..';' + + for name,child in pairs(node.children) do + traverse_calltree(child, sep_prefix..name) + end +end + +local function dump(inputfile) + local reader = bufread.new(inputfile) + + local symbols = symtab.parse(reader) + + local events = sysprof.parse(reader) + local calltree = misc.collapse(events, symbols, split_by_vmstate) + + traverse_calltree(calltree, '') + + os.exit(0) +end + +-- FIXME: this script should be application-independent. +local args = {...} +if #args == 1 and args[1] == "sysprof" then + return dump +else + dump(parseargs(args)) +end diff --git a/tools/sysprof/collapse.lua b/tools/sysprof/collapse.lua new file mode 100755 index 00000000..a123e6bd --- /dev/null +++ b/tools/sysprof/collapse.lua @@ -0,0 +1,113 @@ +local parse = require "sysprof.parse" +local vmdef = require "jit.vmdef" +local symtab = require "utils.symtab" + +local VMST_NAMES = { + [parse.VMST.INTERP] = "VMST_INTERP", + [parse.VMST.LFUNC] = "VMST_LFUNC", + [parse.VMST.FFUNC] = "VMST_FFUNC", + [parse.VMST.CFUNC] = "VMST_CFUNC", + [parse.VMST.GC] = "VMST_GC", + [parse.VMST.EXIT] = "VMST_EXIT", + [parse.VMST.RECORD] = "VMST_RECORD", + [parse.VMST.OPT] = "VMST_OPT", + [parse.VMST.ASM] = "VMST_ASM", + [parse.VMST.TRACE] = "VMST_TRACE", +} + +local M = {} + +local function new_node(name, is_leaf) + return { + name = name, + count = 0, + is_leaf = is_leaf, + children = {} + } +end + +-- insert new child into a node (or increase counter in existing one) +local function insert(name, node, is_leaf) + if node.children[name] == nil then + node.children[name] = new_node(name, is_leaf) + end + + local child = node.children[name] + child.count = child.count + 1 + + return child +end + +local function insert_lua_callchain(chain, lua, symbols) + for _,fr in pairs(lua.callchain) do + local name_lua + + if fr.type == parse.FRAME.FFUNC then + name_lua = vmdef.ffnames[fr.ffid] + else + name_lua = symtab.demangle(symbols, { + addr = fr.addr, + line = fr.line + }) + if lua.trace.id ~= nil and lua.trace.addr == fr.addr and + lua.trace.line == fr.line then + name_lua = symtab.demangle(symbols, { + addr = fr.addr, + traceno = lua.trace.id + }) + end + end + + table.insert(chain, { name = name_lua }) + end +end + +-- merge lua and host callchains into one callchain representing +-- transfer of control +local function merge(event, symbols, sep_vmst) + local cc = {} + local lua_inserted = false + + for _,h_fr in pairs(event.host.callchain) do + local name_host = symtab.demangle(symbols, { addr = h_fr.addr }) + + -- We assume that usually the transfer of control + -- looks like: + -- HOST -> LUA -> HOST + -- so for now, lua callchain starts from lua_pcall() call + if name_host == 'lua_pcall' then + insert_lua_callchain(cc, event.lua, symbols) + lua_inserted = true + end + + table.insert(cc, { name = name_host }) + end + + if lua_inserted == false then + insert_lua_callchain(cc, event.lua, symbols) + end + + if sep_vmst == true then + table.insert(cc, { name = VMST_NAMES[event.lua.vmstate] }) + end + + return cc +end + +-- Collapse all the events into call tree +function M.collapse(events, symbols, sep_vmst) + local root = new_node('root', false) + + for _,ev in pairs(events) do + local callchain = merge(ev, symbols, sep_vmst) + local curr_node = root + for i=#callchain,1,-1 do + curr_node = insert(callchain[i].name, curr_node, false) + end + insert('', curr_node, true) + end + + return root +end + +return M diff --git a/tools/sysprof/parse.lua b/tools/sysprof/parse.lua new file mode 100755 index 00000000..766b0c99 --- /dev/null +++ b/tools/sysprof/parse.lua @@ -0,0 +1,188 @@ +-- Parser of LuaJIT's sysprof binary stream. +-- The format spec can be found in . + +local string_format = string.format + +local LJP_MAGIC = "ljp" +local LJP_CURRENT_VERSION = 1 + +local M = {} + +M.VMST = { + INTERP = 0, + LFUNC = 1, + FFUNC = 2, + CFUNC = 3, + GC = 4, + EXIT = 5, + RECORD = 6, + OPT = 7, + ASM = 8, + TRACE = 9, +} + + +M.FRAME = { + LFUNC = 1, + CFUNC = 2, + FFUNC = 3, + BOTTOM = 0x80 +} + +local STREAM_END = 0x80 + +local function new_event() + return { + lua = { + vmstate = 0, + callchain = {}, + trace = { + id = nil, + addr = 0, + line = 0 + } + }, + host = { + callchain = {} + } + } +end + +local function parse_lfunc(reader, event) + local addr = reader:read_uleb128() + local line = reader:read_uleb128() + table.insert(event.lua.callchain, { + type = M.FRAME.LFUNC, + addr = addr, + line = line + }) +end + +local function parse_ffunc(reader, event) + local ffid = reader:read_uleb128() + table.insert(event.lua.callchain, { + type = M.FRAME.FFUNC, + ffid = ffid, + }) +end + +local function parse_cfunc(reader, event) + local addr = reader:read_uleb128() + table.insert(event.lua.callchain, { + type = M.FRAME.CFUNC, + addr = addr + }) +end + +local frame_parsers = { + [M.FRAME.LFUNC] = parse_lfunc, + [M.FRAME.FFUNC] = parse_ffunc, + [M.FRAME.CFUNC] = parse_cfunc +} + +local function parse_lua_callchain(reader, event) + while true do + local frame_header = reader:read_octet() + if frame_header == M.FRAME.BOTTOM then + break + end + frame_parsers[frame_header](reader, event) + end +end + +--~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~-- + +local function parse_host_callchain(reader, event) + local addr = reader:read_uleb128() + + while addr ~= 0 do + table.insert(event.host.callchain, { + addr = addr + }) + addr = reader:read_uleb128() + end +end + +--~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~-- + +local function parse_trace_callchain(reader, event) + event.lua.trace.id = reader:read_uleb128() + event.lua.trace.addr = reader:read_uleb128() + event.lua.trace.line = reader:read_uleb128() +end + +--~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~-- + +local function parse_host_only(reader, event) + parse_host_callchain(reader, event) +end + +local function parse_lua_host(reader, event) + parse_lua_callchain(reader, event) + parse_host_callchain(reader, event) +end + +local function parse_trace(reader, event) + parse_trace_callchain(reader, event) + -- parse_lua_callchain(reader, event) +end + +local event_parsers = { + [M.VMST.INTERP] = parse_host_only, + [M.VMST.LFUNC] = parse_lua_host, + [M.VMST.FFUNC] = parse_lua_host, + [M.VMST.CFUNC] = parse_lua_host, + [M.VMST.GC] = parse_host_only, + [M.VMST.EXIT] = parse_host_only, + [M.VMST.RECORD] = parse_host_only, + [M.VMST.OPT] = parse_host_only, + [M.VMST.ASM] = parse_host_only, + [M.VMST.TRACE] = parse_trace +} + +local function parse_event(reader, events) + local event = new_event() + + local vmstate = reader:read_octet() + if vmstate == STREAM_END then + -- TODO: samples & overruns + return false + end + + assert(0 <= vmstate and vmstate <= 9, "Vmstate "..vmstate.." is not valid") + event.lua.vmstate = vmstate + + event_parsers[vmstate](reader, event) + + table.insert(events, event) + return true +end + +function M.parse(reader) + local events = {} + + local magic = reader:read_octets(3) + local version = reader:read_octets(1) + -- Dummy-consume reserved bytes. + local _ = reader:read_octets(3) + + if magic ~= LJP_MAGIC then + error("Bad LJP format prologue: "..magic) + end + + if string.byte(version) ~= LJP_CURRENT_VERSION then + error(string_format( + "LJP format version mismatch: the tool expects %d, but your data is %d", + LJP_CURRENT_VERSION, + string.byte(version) + )) + end + + while parse_event(reader, events) do + -- Empty body. + end + + return events +end + +return M diff --git a/tools/utils/symtab.lua b/tools/utils/symtab.lua index c7fcf77c..cf27d70d 100644 --- a/tools/utils/symtab.lua +++ b/tools/utils/symtab.lua @@ -121,7 +121,7 @@ local function demangle_trace(symtab, loc) end function M.demangle(symtab, loc) - if loc.traceno ~= 0 then + if loc.traceno ~= 0 and loc.traceno ~= nil then return demangle_trace(symtab, loc) end -- 2.35.1