[Tarantool-patches] [PATCH luajit] Prevent sanitizer warnings for lj_tab_new*() and table.new().

[Sergey Bronnikov via Tarantool-patches <tarantool-patches@dev.tarantool.org>]

From: Mike Pall <mike>

Reported by Sergey Bronnikov.

(cherry picked from commit 8f421c81ec6aaae0bcd80e01f4353de200afbbc5)

The Undefined Behaviour Sanitizer [1] produce a warning because
the function `lua_createtable()` takes signed integer arguments,
but the `lj_tab_new_ah()` was not properly validating or converting
these signed values before using them in unsigned arithmetic.

The fix changes the signature of `lj_tab_new_ah()` to accept
uint32_t directly, and adjusts `lua_createtable()` to cast the
incoming signed int values to uint32_t before passing them.

[1]: https://clang.llvm.org/docs/UndefinedBehaviorSanitizer.html

Sergey Bronnikov:
* added the description and the test for the problem

Part of tarantool/tarantool#12480
---
Branch: https://github.com/tarantool/luajit/tree/ligurio/lj-1458-ub-lj_tab_new

Related issues:
- https://github.com/tarantool/tarantool/issues/12480
- https://github.com/LuaJIT/LuaJIT/issues/1458


 src/lj_api.c                                  |  2 +-
 src/lj_tab.c                                  |  4 +--
 src/lj_tab.h                                  |  2 +-
 .../lj-1458-ub-table.new.test.lua             | 30 +++++++++++++++++++
 4 files changed, 34 insertions(+), 4 deletions(-)
 create mode 100644 test/tarantool-tests/lj-1458-ub-table.new.test.lua

diff --git a/src/lj_api.c b/src/lj_api.c
index 16a1da0e..fc7a589f 100644
--- a/src/lj_api.c
+++ b/src/lj_api.c
@@ -746,7 +746,7 @@ LUA_API void lua_pushlightuserdata(lua_State *L, void *p)
 LUA_API void lua_createtable(lua_State *L, int narray, int nrec)
 {
   lj_gc_check(L);
-  settabV(L, L->top, lj_tab_new_ah(L, narray, nrec));
+  settabV(L, L->top, lj_tab_new_ah(L, (uint32_t)narray, (uint32_t)nrec));
   incr_top(L);
 }
 
diff --git a/src/lj_tab.c b/src/lj_tab.c
index 1d6a4b7f..9e253d03 100644
--- a/src/lj_tab.c
+++ b/src/lj_tab.c
@@ -165,9 +165,9 @@ GCtab *lj_tab_new(lua_State *L, uint32_t asize, uint32_t hbits)
 }
 
 /* The API of this function conforms to lua_createtable(). */
-GCtab *lj_tab_new_ah(lua_State *L, int32_t a, int32_t h)
+GCtab *lj_tab_new_ah(lua_State *L, uint32_t a, uint32_t h)
 {
-  return lj_tab_new(L, (uint32_t)(a > 0 ? a+1 : 0), hsize2hbits(h));
+  return lj_tab_new(L, a ? a+1 : 0, hsize2hbits(h));
 }
 
 #if LJ_HASJIT
diff --git a/src/lj_tab.h b/src/lj_tab.h
index 71e34945..77b08678 100644
--- a/src/lj_tab.h
+++ b/src/lj_tab.h
@@ -34,7 +34,7 @@ static LJ_AINLINE uint32_t hashrot(uint32_t lo, uint32_t hi)
 #define hsize2hbits(s)	((s) ? ((s)==1 ? 1 : 1+lj_fls((uint32_t)((s)-1))) : 0)
 
 LJ_FUNCA GCtab *lj_tab_new(lua_State *L, uint32_t asize, uint32_t hbits);
-LJ_FUNC GCtab *lj_tab_new_ah(lua_State *L, int32_t a, int32_t h);
+LJ_FUNC GCtab *lj_tab_new_ah(lua_State *L, uint32_t a, uint32_t h);
 #if LJ_HASJIT
 LJ_FUNC GCtab * LJ_FASTCALL lj_tab_new1(lua_State *L, uint32_t ahsize);
 #endif
diff --git a/test/tarantool-tests/lj-1458-ub-table.new.test.lua b/test/tarantool-tests/lj-1458-ub-table.new.test.lua
new file mode 100644
index 00000000..d0cf6ff5
--- /dev/null
+++ b/test/tarantool-tests/lj-1458-ub-table.new.test.lua
@@ -0,0 +1,30 @@
+local tap = require('tap')
+
+-- The test file to demonstrate UBSan warning for `table.new()`
+-- with a minimal and maximum array and hash parts values.
+-- See also: https://github.com/LuaJIT/LuaJIT/issues/1458.
+local test = tap.test('lj-1458-ub-table-new')
+
+test:plan(8)
+
+local table_new = require('table.new')
+
+local INT_MAX =  2 ^ 31 - 1
+local INT_MIN = -2 ^ 31
+
+local table_sizes = {
+  { 0, INT_MIN },
+  { 0, INT_MAX },
+  { INT_MIN, 0 },
+  { INT_MAX, 0 },
+}
+
+for _, case in ipairs(table_sizes) do
+  local apart, hpart = unpack(case)
+  local ok, err = pcall(table_new, apart, hpart)
+  local message = ('table.new(%d, %d)'):format(apart, hpart)
+  test:is(ok, false, message .. ' is failed')
+  test:ok(err:match('table overflow'), message .. ' correct error message')
+end
+
+test:done(true)
-- 
2.43.0