[Tarantool-patches] [PATCH luajit] Fix recording of __concat metamethod.

[Sergey Kaplun via Tarantool-patches <tarantool-patches@dev.tarantool.org>]

From: Mike Pall <mike>

Thanks to Sergey Kaplun.

(cherry picked from commit bcc5125a9188179d23c223b8865ed94951fb91b3)

`lj_record_ret()` uses `J->L->base` as a base for recording
concatenation with remaining operands, even if there is a vararg frame
or pcall frame (so it uses the wrong Lua stack slots from a frame
above). This leads to the corresponding assertion failure in the
`lj_record_mm_lookup()`.

This patch adds the calculation of the base adjusment necessary for
correct frame usage.

Sergey Kaplun:
* added the description and the test for the problem

Part of tarantool/tarantool#9595
---

Branch: https://github.com/tarantool/luajit/tree/skaplun/lj-1164-record-meta-concat-varg-pcall
Tarantool PR: https://github.com/tarantool/tarantool/pull/9788
Related issues:
* https://github.com/tarantool/tarantool/issues/9595
* https://github.com/LuaJIT/LuaJIT/issues/1164

 src/lj_record.c                               |  8 ++-
 ...164-record-meta-concat-varg-pcall.test.lua | 49 +++++++++++++++++++
 2 files changed, 55 insertions(+), 2 deletions(-)
 create mode 100644 test/tarantool-tests/lj-1164-record-meta-concat-varg-pcall.test.lua

diff --git a/src/lj_record.c b/src/lj_record.c
index c01c1f0b..77818fec 100644
--- a/src/lj_record.c
+++ b/src/lj_record.c
@@ -819,6 +819,7 @@ void lj_record_ret(jit_State *J, BCReg rbase, ptrdiff_t gotresults)
 {
   TValue *frame = J->L->base - 1;
   ptrdiff_t i;
+  BCReg baseadj = 0;
   for (i = 0; i < gotresults; i++)
     (void)getslot(J, rbase+i);  /* Ensure all results have a reference. */
   while (frame_ispcall(frame)) {  /* Immediately resolve pcall() returns. */
@@ -827,6 +828,7 @@ void lj_record_ret(jit_State *J, BCReg rbase, ptrdiff_t gotresults)
       lj_trace_err(J, LJ_TRERR_NYIRETL);
     lj_assertJ(J->baseslot > 1+LJ_FR2, "bad baseslot for return");
     gotresults++;
+    baseadj += cbase;
     rbase += cbase;
     J->baseslot -= (BCReg)cbase;
     J->base -= cbase;
@@ -851,6 +853,7 @@ void lj_record_ret(jit_State *J, BCReg rbase, ptrdiff_t gotresults)
     if (--J->framedepth < 0)  /* NYI: return of vararg func to lower frame. */
       lj_trace_err(J, LJ_TRERR_NYIRETL);
     lj_assertJ(J->baseslot > 1+LJ_FR2, "bad baseslot for return");
+    baseadj += cbase;
     rbase += cbase;
     J->baseslot -= (BCReg)cbase;
     J->base -= cbase;
@@ -918,7 +921,8 @@ void lj_record_ret(jit_State *J, BCReg rbase, ptrdiff_t gotresults)
       BCReg bslot = bc_b(*(frame_contpc(frame)-1));
       TRef tr = gotresults ? J->base[cbase+rbase] : TREF_NIL;
       if (bslot != J->maxslot) {  /* Concatenate the remainder. */
-	TValue *b = J->L->base, save;  /* Simulate lower frame and result. */
+	/* Simulate lower frame and result. */
+	TValue *b = J->L->base - baseadj, save;
 	/* Can't handle MM_concat + CALLT + fast func side-effects. */
 	if (J->postproc != LJ_POST_NONE)
 	  lj_trace_err(J, LJ_TRERR_NYIRETL);
@@ -931,7 +935,7 @@ void lj_record_ret(jit_State *J, BCReg rbase, ptrdiff_t gotresults)
 	J->L->base = b - cbase;
 	tr = rec_cat(J, bslot, cbase-(2<<LJ_FR2));
 	b = J->L->base + cbase;  /* Undo. */
-	J->L->base = b;
+	J->L->base = b + baseadj;
 	copyTV(J->L, b-(2<<LJ_FR2), &save);
       }
       if (tr) {  /* Store final result. */
diff --git a/test/tarantool-tests/lj-1164-record-meta-concat-varg-pcall.test.lua b/test/tarantool-tests/lj-1164-record-meta-concat-varg-pcall.test.lua
new file mode 100644
index 00000000..987e49a0
--- /dev/null
+++ b/test/tarantool-tests/lj-1164-record-meta-concat-varg-pcall.test.lua
@@ -0,0 +1,49 @@
+local tap = require('tap')
+
+-- Test file to demonstrate the incorrect recording of the
+-- `__concat` metamethod with the VARG or protected frame for the
+-- function used in the `__concat`, when we have to record more
+-- than 2 objects.
+-- See also: https://github.com/LuaJIT/LuaJIT/issues/1164.
+
+local test = tap.test('lj-1164-record-meta-concat-varg-pcall'):skipcond({
+  ['Test requires JIT enabled'] = not jit.status(),
+})
+
+test:plan(2)
+
+jit.opt.start('hotloop=1')
+
+-- Table with the simplest varg metamethod to call.
+local tab_varg = setmetatable({}, {
+  __concat = function(...)
+    local _, tab_obj = ...
+    return tab_obj
+  end,
+})
+
+local result
+for _ = 1, 4 do
+  -- The concatenation has right to left associativity.
+  result = '' .. '' .. tab_varg
+end
+
+test:is(result, tab_varg, 'varg frame is recorded correctly')
+
+-- Table with the simplest protected metamethod.
+local tab_pcall = setmetatable({}, {
+  -- Need to use `__call` metamethod to make the object callable.
+  __call = function(tab_obj)
+    return tab_obj
+  end,
+  __concat = pcall,
+})
+
+for _ = 1, 4 do
+  -- The concatenation has right to left associativity.
+  result = tab_pcall .. tab_pcall .. ''
+end
+
+test:is(result, true, 'protected frame is recorded correctly')
+
+test:done(true)
-- 
2.44.0