[Tarantool-patches] [PATCH luajit] Handle all stack layouts in (delayed) TRACE vmevent.

[Tarantool-patches] [PATCH luajit] Handle all stack layouts in (delayed) TRACE vmevent.

[Sergey Bronnikov via Tarantool-patches]

From: Mike Pall <mike>

Thanks to Sergey Bronnikov and Peter Cawley.

(cherry picked from commit b138ccfa918518a152bc830fef3d53cd0a922e36)

When recording a trace using a Lua code like
`repeat until `a >= 'b' > 'b'` a Lua error is encountered
(`attempt to compare string with nil`), which (along with
raising the error) causes an asynchronous trace abort.
The trace abort remains pending until the call of
`lua_pcall(L, 2, 0, 0)` that actually calls `jit.attach(trace_cb, nil)`.

On handling abort LuaJIT is searching for the topmost Lua frame
on the stack, that was active when the trace abort happened,
it is needed to generate a better error message.
Unfortunately, because the abort was due to an error, and
the error was caught by a `lua_pcall` with unspecified
error function (4th argument), the Lua frame that caused the abort
was already removed as part of error processing, so the search
cannot find it. Furthermore, in this particular case, there are
no Lua frames on the stack, which isn't something the search code
had considered possible.

Sergey Bronnikov:
* added the description and the test for the problem

Part of tarantool/tarantool#10709
---
Branch: https://github.com/tarantool/luajit/tree/ligurio/lj-1087-vm-handler-call

Related issues:
- https://github.com/LuaJIT/LuaJIT/issues/1087
- https://github.com/tarantool/tarantool/issues/10709

 src/lj_trace.c                                |  26 +--
 test/tarantool-c-tests/CMakeLists.txt         |   4 +
 .../lj-1087-vm-handler-call.test.c            | 170 ++++++++++++++++++
 3 files changed, 190 insertions(+), 10 deletions(-)
 create mode 100644 test/tarantool-c-tests/lj-1087-vm-handler-call.test.c

diff --git a/src/lj_trace.c b/src/lj_trace.c
index 94cb27e5..6b97cc13 100644
--- a/src/lj_trace.c
+++ b/src/lj_trace.c
@@ -616,21 +616,27 @@ static int trace_abort(jit_State *J)
     J->cur.link = 0;
     J->cur.linktype = LJ_TRLINK_NONE;
     lj_vmevent_send(L, TRACE,
-      TValue *frame;
+      cTValue *bot = tvref(L->stack)+LJ_FR2;
+      cTValue *frame;
       const BCIns *pc;
-      GCfunc *fn;
+      BCPos pos = 0;
       setstrV(L, L->top++, lj_str_newlit(L, "abort"));
       setintV(L->top++, traceno);
       /* Find original Lua function call to generate a better error message. */
-      frame = J->L->base-1;
-      pc = J->pc;
-      while (!isluafunc(frame_func(frame))) {
-	pc = (frame_iscont(frame) ? frame_contpc(frame) : frame_pc(frame)) - 1;
-	frame = frame_prev(frame);
+      for (frame = J->L->base-1, pc = J->pc; ; frame = frame_prev(frame)) {
+	if (isluafunc(frame_func(frame))) {
+	  pos = proto_bcpos(funcproto(frame_func(frame)), pc);
+	  break;
+	} else if (frame_prev(frame) <= bot) {
+	  break;
+	} else if (frame_iscont(frame)) {
+	  pc = frame_contpc(frame) - 1;
+	} else {
+	  pc = frame_pc(frame) - 1;
+	}
       }
-      fn = frame_func(frame);
-      setfuncV(L, L->top++, fn);
-      setintV(L->top++, proto_bcpos(funcproto(fn), pc));
+      setfuncV(L, L->top++, frame_func(frame));
+      setintV(L->top++, pos);
       copyTV(L, L->top++, restorestack(L, errobj));
       copyTV(L, L->top++, &J->errinfo);
     );
diff --git a/test/tarantool-c-tests/CMakeLists.txt b/test/tarantool-c-tests/CMakeLists.txt
index c4a402d0..0e7c4698 100644
--- a/test/tarantool-c-tests/CMakeLists.txt
+++ b/test/tarantool-c-tests/CMakeLists.txt
@@ -36,6 +36,10 @@ add_test_suite_target(tarantool-c-tests
   DEPENDS libluajit libtest tarantool-c-tests-build
 )
 
+if(NOT LUAJIT_DISABLE_JIT)
+  AppendFlags(TESTS_C_FLAGS "-DLJ_HASJIT")
+endif(NOT LUAJIT_DISABLE_JIT)
+
 set(CTEST_SRC_SUFFIX ".test.c")
 file(GLOB tests "${CMAKE_CURRENT_SOURCE_DIR}/*${CTEST_SRC_SUFFIX}")
 foreach(test_source ${tests})
diff --git a/test/tarantool-c-tests/lj-1087-vm-handler-call.test.c b/test/tarantool-c-tests/lj-1087-vm-handler-call.test.c
new file mode 100644
index 00000000..b7db9857
--- /dev/null
+++ b/test/tarantool-c-tests/lj-1087-vm-handler-call.test.c
@@ -0,0 +1,170 @@
+#include <assert.h>
+#include <stdbool.h>
+
+#include "lua.h"
+#include "lauxlib.h"
+
+#include "test.h"
+#include "utils.h"
+
+/*
+ * Test file to demonstrate a segmentation fault under
+ * AddressSanitizer, when C function is used as a VM handler
+ * in LuaJIT:
+ *
+ * Program received signal SIGSEGV, Segmentation fault.
+ * 0x000055555557e77d in trace_abort (J=0x7ffff7f9b6b8) at lj_trace.c:615
+ * 615         lj_vmevent_send(L, TRACE,
+ * (gdb) bt
+ *
+ * See details in https://github.com/LuaJIT/LuaJIT/issues/1087.
+ */
+
+#define UNUSED(x) ((void)(x))
+
+static void jit_attach(lua_State *L, void *cb, const char *event)
+{
+	lua_getglobal(L, "jit");
+	lua_getfield(L, -1, "attach");
+	lua_pushcfunction(L, (lua_CFunction)cb);
+	if (event != NULL) {
+		lua_pushstring(L, event);
+	} else {
+		lua_pushnil(L);
+	}
+	lua_pcall(L, 2, 0, 0);
+}
+
+static int trace_cb(lua_State *L) {
+	UNUSED(L);
+	return 0;
+}
+
+static int handle_luafunc_frame(void *test_state)
+{
+	/* Setup. */
+	lua_State *L = test_state;
+	jit_attach(L, (void *)trace_cb, "trace");
+
+	/* Loading and executing of a broken Lua code. */
+	int rc = luaL_dostring(L, "repeat until nil > 1");
+	assert(rc == 1);
+
+	/* The Lua chunk generates a Lua frame. */
+	rc = luaL_dostring(L, "return function() end");
+	assert(rc == 0);
+
+	/* Teardown. */
+	lua_settop(L, 0);
+
+	return TEST_EXIT_SUCCESS;
+}
+
+static int nop(lua_State *L)
+{
+	return 0;
+}
+
+static int cframe(lua_State *L)
+{
+	int rc = luaL_dostring(L, "repeat until nil > 1");
+	assert(rc == 1);
+	lua_pop(L, 1);		/* Remove errmsg. */
+
+	lua_pushcfunction(L, nop);
+	lua_call(L, 0, 0);
+
+	return 0;
+}
+
+static int handle_c_frame(void *test_state)
+{
+	/* Setup. */
+	lua_State *L = test_state;
+	jit_attach(L, (void *)trace_cb, "trace");
+
+	lua_pushcfunction(L, cframe);
+	lua_call(L, 0, 0);
+
+	/* Teardown. */
+	lua_settop(L, 0);
+
+	return TEST_EXIT_SUCCESS;
+}
+
+static int global_f(lua_State* L)
+{
+	lua_pushstring(L, "М");
+	lua_pushstring(L, "И");
+	lua_pushstring(L, "Р");
+	lua_concat(L, 3);
+
+	return 1;
+}
+
+static int handle_cont_frame(void *test_state)
+{
+	const char lua_chunk[] =
+		"local t = setmetatable({}, { __index = global_f })"
+		"for i = 1, 4 do"
+		"    _ = t[1]"
+		"end";
+
+	/* Setup. */
+	lua_State *L = test_state;
+	jit_attach(L, (void *)trace_cb, "trace");
+	int res = luaL_dostring(L, "jit.opt.start('minstitch=16')");
+	assert(res == 0);
+	lua_pushcfunction(L, global_f);
+	lua_setglobal(L, "global_f");
+
+	res = luaL_loadstring(L, lua_chunk);
+	assert(res == 0);
+	lua_pcall(L, 0, LUA_MULTRET, 0);
+
+	/* Teardown. */
+	lua_settop(L, 0);
+
+	return TEST_EXIT_SUCCESS;
+}
+
+static int handle_bottom_frame(void *test_state)
+{
+	lua_State *L = test_state;
+
+	/* Attach VM call handler. */
+	jit_attach(L, (void *)trace_cb, "trace");
+
+	/* Load a Lua code that generate a trace abort. */
+	int rc = luaL_dostring(L, "repeat until nil > 1");
+	assert(rc == 1);
+
+	/* Triggers segmentation fault. */
+	jit_attach(L, (void *)trace_cb, NULL);
+
+	/* Teardown. */
+	lua_settop(L, 0);
+
+	return TEST_EXIT_SUCCESS;
+}
+
+int main(void)
+{
+#if !defined(LJ_HASJIT)
+	return skip_all("JIT is disabled");
+#else /* LJ_HASJIT */
+	lua_State *L = utils_lua_init();
+	const struct test_unit tgroup[] = {
+		test_unit_def(handle_luafunc_frame),
+		test_unit_def(handle_bottom_frame),
+		test_unit_def(handle_cont_frame),
+		test_unit_def(handle_c_frame),
+	};
+	luaL_openlibs(L);
+	int res = luaL_dostring(L, "jit.opt.start('hotloop=1')");
+	assert(res == 0);
+	const int test_result = test_run_group(tgroup, L);
+	utils_lua_close(L);
+	return test_result;
+#endif /* LJ_HASJIT */
+}
-- 
2.34.1