[Tarantool-patches] [PATCH luajit] Check frame size limit before returning to a lower frame.

[Sergey Kaplun via Tarantool-patches <tarantool-patches@dev.tarantool.org>]

From: Mike Pall <mike>

Thanks to Sergey Kaplun.

(cherry picked from commit 302366a33853b730f1b7eb61d792abc4f84f0caa)

When compiling a stitched (or side) trace, there is no check for the
frame size of the current prototype during recording. Hence, when we
return (for example, after stitching) to the lower frame with a maximum
possible frame size (249), the 251 = `baseslot` (2) + `maxslot` (249)
slot for GC64 mode may be used. This leads to the corresponding assertion
failure in `rec_check_slots()`.

This patch adds the corresponding check.

Sergey Kaplun:
* added the description and the test for the problem

Part of tarantool/tarantool#9595
---

Branch: https://github.com/tarantool/luajit/tree/skaplun/lj-1173-frame-limit-lower-frame
Tarantool PR: https://github.com/tarantool/tarantool/pull/9791
Related issues:
* https://github.com/tarantool/tarantool/issues/9595
* https://github.com/LuaJIT/LuaJIT/issues/1173

 src/lj_record.c                               |  2 +
 .../lj-1173-frame-limit-lower-frame.test.lua  | 83 +++++++++++++++++++
 2 files changed, 85 insertions(+)
 create mode 100644 test/tarantool-tests/lj-1173-frame-limit-lower-frame.test.lua

diff --git a/src/lj_record.c b/src/lj_record.c
index c01c1f0b..e3590b1a 100644
--- a/src/lj_record.c
+++ b/src/lj_record.c
@@ -886,6 +886,8 @@ void lj_record_ret(jit_State *J, BCReg rbase, ptrdiff_t gotresults)
       lj_trace_err(J, LJ_TRERR_LLEAVE);
     } else if (J->needsnap) {  /* Tailcalled to ff with side-effects. */
       lj_trace_err(J, LJ_TRERR_NYIRETL);  /* No way to insert snapshot here. */
+    } else if (1 + pt->framesize >= LJ_MAX_JSLOTS) {
+      lj_trace_err(J, LJ_TRERR_STACKOV);
     } else {  /* Return to lower frame. Guard for the target we return to. */
       TRef trpt = lj_ir_kgc(J, obj2gco(pt), IRT_PROTO);
       TRef trpc = lj_ir_kptr(J, (void *)frame_pc(frame));
diff --git a/test/tarantool-tests/lj-1173-frame-limit-lower-frame.test.lua b/test/tarantool-tests/lj-1173-frame-limit-lower-frame.test.lua
new file mode 100644
index 00000000..91e2c603
--- /dev/null
+++ b/test/tarantool-tests/lj-1173-frame-limit-lower-frame.test.lua
@@ -0,0 +1,83 @@
+local tap = require('tap')
+
+-- Test file to demonstrate LuaJIT assertion failure during
+-- recording of side trace in GC64 mode with return to lower
+-- frame, which has the maximum possible frame size.
+-- See also: https://github.com/LuaJIT/LuaJIT/issues/1173.
+
+local test = tap.test('lj-1173-frame-limit-lower-frame'):skipcond({
+  ['Test requires JIT enabled'] = not jit.status(),
+})
+
+test:plan(1)
+
+-- Parent trace with stitching and returning to a lower frame.
+local function retf()
+  math.modf(1)
+end
+_G.retf = retf
+
+local LJ_MAX_JSLOTS = 250
+
+-- Generate the following function:
+-- | local uv = {key = 1}
+-- | return function()
+-- |   local r = retf()
+-- |   uv.key, uv.key, --[[124 times in total ...]] uv.key = nil
+-- | end
+-- It will have the following bytecode:
+-- | 0001    GGET     0   0      ; "retf"
+-- | 0002    CALL     0   2   1
+-- | 0003    UGET     1   0      ; uv
+-- | ...
+-- | 0126    UGET   124   0      ; uv
+-- | 0127    KNIL   125 248
+-- | 0128    TSETS  248 124   1  ; "key"
+-- | ...
+-- | 0251    TSETS  125   1   1  ; "key"
+-- | 0252    RET0     0   1
+-- As you can see, the 249 slots (from 0 to 248) are occupied in
+-- total.
+-- When we return to the lower frame for the side trace, we may
+-- hit the slot limit mentioned above: 2 slots are occupied
+-- by the frame (`baseslot`) and `KNIL` bytecode recording sets
+-- `maxslot` (the first free slot) to 249. Hence, the JIT slots
+-- are overflowing.
+
+local chunk = 'local uv = {key = 1}\n'
+chunk = chunk .. 'return function()\n'
+chunk = chunk .. 'local r = retf()\n'
+
+-- Each `UGET` occupies 1 slot, `KNIL` occupies the same amount.
+-- 1 slot is reserved (`r` variable), 1 pair is set outside the
+-- cycle. 249 slots (the maximum available amount, see
+-- <src/lj_parse.c>, `bcreg_bump()` for details) are occupied in
+-- total.
+for _ = 1, LJ_MAX_JSLOTS / 2 - 2 do
+  chunk = chunk .. ('uv.key, ')
+end
+chunk = chunk .. 'uv.key = nil\n'
+chunk = chunk .. 'end\n'
+
+local get_func = assert(loadstring(chunk))
+local function_max_framesize = get_func()
+
+jit.opt.start('hotloop=1', 'hotexit=1')
+
+-- Compile the parent trace first.
+retf()
+retf()
+
+-- Try to compile the side trace with a return to a lower frame
+-- with a huge frame size.
+function_max_framesize()
+function_max_framesize()
+
+-- XXX: The limit check is OK with default defines for non-GC64
+-- mode, the trace is compiled for it. The test fails only with
+-- GC64 mode enabled. Still run the test for non-GC64 mode to
+-- avoid regressions.
+
+test:ok(true, 'no assertion failure during recording')
+
+test:done(true)
-- 
2.44.0