[Tarantool-patches] [PATCH luajit 5/6] FFI: Fix dangling reference to CType. Improve checks.

[Sergey Kaplun via Tarantool-patches <tarantool-patches@dev.tarantool.org>]

From: Mike Pall <mike>

Reported by elmknot.

(cherry-picked from commit cc96ab9d513582703f8663a8775a935b56db32b7)

During the recording of an arithmetic operation with a cdata function
object and some cdata value in `recff_cdata_arith()`, reallocation of
`cts->tab` in `lj_ctype_intern()` may occur. In that case, the reference
to the first `CType` object (`s[0]`) becomes invalid. This patch saves
the `CTypeID` of this object and gets its `CType` again after possible
reallocation.

Also, this commit fills `cts->tab` memory with zeros before being freed
when `-DLUAJIT_CTYPE_CHECK_ANCHOR` is defined. It helps to observe
assertion failures in case this memory is used after free.

Sergey Kaplun:
* added the description and the test for the problem

Part of tarantool/tarantool#9145
---
 src/lj_crecord.c                              |  4 +
 src/lj_ctype.c                                | 12 +++
 ...0-fix-dangling-reference-to-ctype.test.lua | 77 +++++++++++++++++++
 3 files changed, 93 insertions(+)
 create mode 100644 test/tarantool-tests/lj-920-fix-dangling-reference-to-ctype.test.lua

diff --git a/src/lj_crecord.c b/src/lj_crecord.c
index 10d1dc70..e17e512f 100644
--- a/src/lj_crecord.c
+++ b/src/lj_crecord.c
@@ -1502,9 +1502,13 @@ void LJ_FASTCALL recff_cdata_arith(jit_State *J, RecordFFData *rd)
 	if (ctype_isenum(ct->info)) ct = ctype_child(cts, ct);
 	goto ok;
       } else if (ctype_isfunc(ct->info)) {
+	CTypeID id0 = i ? ctype_typeid(cts, s[0]) : 0;
 	tr = emitir(IRT(IR_FLOAD, IRT_PTR), tr, IRFL_CDATA_PTR);
 	ct = ctype_get(cts,
 	  lj_ctype_intern(cts, CTINFO(CT_PTR, CTALIGN_PTR|id), CTSIZE_PTR));
+	if (i) {
+	  s[0] = ctype_get(cts, id0);  /* cts->tab may have been reallocated. */
+	}
 	goto ok;
       } else {
 	tr = emitir(IRT(IR_ADD, IRT_PTR), tr, lj_ir_kintp(J, sizeof(GCcdata)));
diff --git a/src/lj_ctype.c b/src/lj_ctype.c
index a42e3d60..0874fa61 100644
--- a/src/lj_ctype.c
+++ b/src/lj_ctype.c
@@ -191,8 +191,20 @@ CTypeID lj_ctype_intern(CTState *cts, CTInfo info, CTSize size)
   }
   id = cts->top;
   if (LJ_UNLIKELY(id >= cts->sizetab)) {
+#ifdef LUAJIT_CTYPE_CHECK_ANCHOR
+    CType *ct;
+#endif
     if (id >= CTID_MAX) lj_err_msg(cts->L, LJ_ERR_TABOV);
+#ifdef LUAJIT_CTYPE_CHECK_ANCHOR
+    ct = lj_mem_newvec(cts->L, id+1, CType);
+    memcpy(ct, cts->tab, id*sizeof(CType));
+    memset(cts->tab, 0, id*sizeof(CType));
+    lj_mem_freevec(cts->g, cts->tab, cts->sizetab, CType);
+    cts->tab = ct;
+    cts->sizetab = id+1;
+#else
     lj_mem_growvec(cts->L, cts->tab, cts->sizetab, CTID_MAX, CType);
+#endif
   }
   cts->top = id+1;
   cts->tab[id].info = info;
diff --git a/test/tarantool-tests/lj-920-fix-dangling-reference-to-ctype.test.lua b/test/tarantool-tests/lj-920-fix-dangling-reference-to-ctype.test.lua
new file mode 100644
index 00000000..a7e35888
--- /dev/null
+++ b/test/tarantool-tests/lj-920-fix-dangling-reference-to-ctype.test.lua
@@ -0,0 +1,77 @@
+local tap = require('tap')
+local ffi = require('ffi')
+local test = tap.test('lj-920-fix-dangling-reference-to-ctype'):skipcond({
+  ['Test requires JIT enabled'] = not jit.status(),
+  -- luacheck: no global
+  ['Impossible to predict the value of cts->top'] = _TARANTOOL,
+})
+
+test:plan(1)
+
+-- This test demonstrates LuaJIT's incorrect behaviour when the
+-- reallocation of `cts->tab` strikes during the recording of the
+-- cdata metamethod arithmetic.
+-- The test fails under ASAN.
+
+-- XXX: Just some C functions to be casted. There is no need to
+-- declare their prototypes correctly.
+ffi.cdef[[
+  int malloc(void);
+  int fprintf(void);
+  int printf(void);
+  int memset(void);
+  int memcpy(void);
+  int memmove(void);
+  int getppid(void);
+]]
+
+local cfunc_type = ffi.metatype(ffi.typeof('struct {int a;}'), {
+  -- Just some metatable with reloaded arithmetic operator.
+  __add = function(o1, _) return o1 end
+})
+-- Just some cdata with metamethod.
+local test_value = cfunc_type(1)
+
+-- XXX: structure to set `cts->top` to 112.
+local _ = ffi.new('struct {int a; long b; float c; double d;}', 0)
+
+-- Anchor table to prevent cdata objects from being collected.
+local anchor = {}
+-- Each call to this function grows `cts->top` by 3.
+local function save_new_func(func)
+  anchor[#anchor + 1] = ffi.cast('void (*)(void)', func)
+end
+
+save_new_func(ffi.C.fprintf) -- `cts->top` = 112
+save_new_func(ffi.C.printf)  -- `cts->top` = 115
+save_new_func(ffi.C.memset)  -- `cts->top` = 118
+save_new_func(ffi.C.memcpy)  -- `cts->top` = 121
+save_new_func(ffi.C.malloc)  -- `cts->top` = 124
+
+-- Assertions to check the `cts->top` value and step between
+-- calls.
+assert(ffi.typeinfo(124), 'cts->top >= 124')
+assert(not ffi.typeinfo(125), 'cts->top < 125')
+
+save_new_func(ffi.C.memmove) -- `cts->top` = 127
+
+jit.opt.start('hotloop=1')
+
+-- Just some function to record trace and cause reallocation.
+local function recorded_func(func_arg)
+  local res = test_value + func_arg
+  return res
+end
+recorded_func(ffi.C.malloc)
+
+assert(ffi.typeinfo(127), 'cts->top >= 127')
+assert(not ffi.typeinfo(128), 'cts->top < 128')
+
+-- Last call to grow `cts->top` up to 129, so this causes
+-- `cts->tab` reallocation. Need to use different functions as
+-- an argument.
+recorded_func(ffi.C.getppid)
+
+test:ok(true, 'no heap-use-after-free in recff_cdata_arith')
+
+test:done(true)
-- 
2.42.0