This patch prevents aforementioned error by comparing the unsigned
amount of values to unpack with `LUAI_MAXCSTACK` instead of 0.

Sergey Kaplun:
* added the description and the test for the problem

Part of tarantool/tarantool#7230
---

Issue/PR:
* https://github.com/LuaJIT/LuaJIT/pull/574
* https://github.com/tarantool/tarantool/issues/7230
Branch: https://github.com/tarantool/luajit/tree/skaplun/lj-574-overflow-unpack-full-ci
PR: https://github.com/tarantool/tarantool/pull/7596

src/lib_base.c | 6 ++++--
test/tarantool-tests/lj-574-overflow-unpack.test.lua | 12 ++++++++++++
2 files changed, 16 insertions(+), 2 deletions(-)
create mode 100644 test/tarantool-tests/lj-574-overflow-unpack.test.lua

diff --git a/src/lib_base.c b/src/lib_base.c
index 613a1859..cf57b4f2 100644
--- a/src/lib_base.c
+++ b/src/lib_base.c
@@ -224,9 +224,11 @@ LJLIB_CF(unpack)
   int32_t n, i = lj_lib_optint(L, 2, 1);
   int32_t e = (L->base+3-1 < L->top && !tvisnil(L->base+3-1)) ?
lj_lib_checkint(L, 3) : (int32_t)lj_tab_len(t);
+ uint32_t nu;
   if (i > e) return 0;
- n = e - i + 1;
- if (n <= 0 || !lua_checkstack(L, n))
+ nu = (uint32_t)e - (uint32_t)i;
+ n = (int32_t)(nu+1);
+ if (nu >= LUAI_MAXCSTACK || !lua_checkstack(L, n))
     lj_err_caller(L, LJ_ERR_UNPACK);
   do {
     cTValue *tv = lj_tab_getint(t, i);
diff --git a/test/tarantool-tests/lj-574-overflow-unpack.test.lua b/test/tarantool-tests/lj-574-overflow-unpack.test.lua
new file mode 100644
index 00000000..6715d947
--- /dev/null
+++ b/test/tarantool-tests/lj-574-overflow-unpack.test.lua
@@ -0,0 +1,12 @@
+local tap = require('tap')
+
+-- Test file to demonstrate integer overflow in the `unpack()`
+-- function due to compiler optimization.
+-- See also https://github.com/LuaJIT/LuaJIT/pull/574.
+local test = tap.test('lj-574-overflow-unpack')
+test:plan(1)
+
+local r, e = pcall(unpack, {}, 0, 2^31 - 1)
+test:ok(not r and e == 'too many results to unpack', 'overflow check in unpack')
+
+os.exit(test:check() and 0 or 1)
--
2.34.1