[Tarantool-patches] [PATCH luajit 2/2] Fix edge cases when recording string.byte/sub.

[Sergey Kaplun via Tarantool-patches <tarantool-patches@dev.tarantool.org>]

From: Mike Pall <mike>

Thanks to Sergey Kaplun.

(cherry picked from commit 89f268b3f745dba80da6350d3cbbb0964f3fdbee)

It is possible that the `len` (`end - start`) will underflow and become
positive in the `recff_string_range()` when the `end` is negative. For
`string.sub()` this is not crucial, since the trace will be valid
anyway. But for `string.byte()` it may lead to the assertion failure in
the `rec_check_slots()`.

This patch fixes those underflows by the correct comparison.

Sergey Kaplun:
* added the description and the test for the problem

Part of tarantool/tarantool#12134
---
 src/lj_ffrecord.c                             |  6 ++---
 .../lj-1443-stirng-byte-underflow.test.lua    | 25 +++++++++++++++++++
 2 files changed, 28 insertions(+), 3 deletions(-)
 create mode 100644 test/tarantool-tests/lj-1443-stirng-byte-underflow.test.lua

diff --git a/src/lj_ffrecord.c b/src/lj_ffrecord.c
index d888e83e..aad1bd87 100644
--- a/src/lj_ffrecord.c
+++ b/src/lj_ffrecord.c
@@ -810,7 +810,7 @@ static void LJ_FASTCALL recff_string_range(jit_State *J, RecordFFData *rd)
   }
   trstart = recff_string_start(J, str, &start, trstart, trlen, tr0);
   if (rd->data) {  /* Return string.sub result. */
-    if (end - start >= 0) {
+    if (start <= end) {
       /* Also handle empty range here, to avoid extra traces. */
       TRef trptr, trslen = emitir(IRTGI(IR_SUBOV), trend, trstart);
       emitir(IRTGI(IR_GE), trslen, tr0);
@@ -821,8 +821,8 @@ static void LJ_FASTCALL recff_string_range(jit_State *J, RecordFFData *rd)
       J->base[0] = lj_ir_kstr(J, &J2G(J)->strempty);
     }
   } else {  /* Return string.byte result(s). */
-    ptrdiff_t i, len = end - start;
-    if (len > 0) {
+    if (start < end) {
+      ptrdiff_t i, len = end - start;
       TRef trslen = emitir(IRTGI(IR_SUBOV), trend, trstart);
       emitir(IRTGI(IR_EQ), trslen, lj_ir_kint(J, (int32_t)len));
       if (J->baseslot + len > LJ_MAX_JSLOTS)
diff --git a/test/tarantool-tests/lj-1443-stirng-byte-underflow.test.lua b/test/tarantool-tests/lj-1443-stirng-byte-underflow.test.lua
new file mode 100644
index 00000000..9f91718c
--- /dev/null
+++ b/test/tarantool-tests/lj-1443-stirng-byte-underflow.test.lua
@@ -0,0 +1,25 @@
+local tap = require('tap')
+
+-- The test file to demonstrate integer underflow during recording
+-- for the `string.byte()` built-in.
+-- See also https://github.com/LuaJIT/LuaJIT/issues/1443.
+
+local test = tap.test('lj-1443-stirng-byte-underflow'):skipcond({
+  ['Test requires JIT enabled'] = not jit.status(),
+})
+
+test:plan(1)
+
+jit.opt.start('hotloop=1')
+
+local result
+local str = 'xxx'
+for _ = 1, 4 do
+  -- Failed assertion in `rec_check_slots()` due to incorrect
+  -- number of results after underflow.
+  result = (str):byte(0X7FFFFFFF, -0X7FFFFFFF)
+end
+
+test:is(result, nil, 'correct result on trace')
+
+test:done(true)
-- 
2.53.0