[tarantool-patches] [PATCH v1 1/1] lua: fix strange behaviour of tonumber64

[Kirill Shcherbatov <kshcherbatov@tarantool.org>]

Function tonumber64 has worked incorrectly with values less
than LLONG_MIN.
Now it works in the interval [LLONG_MIN, ULLONG_MAX] returning
nil otherwise.

Closes #3466.
---
Branch: https://github.com/tarantool/tarantool/compare/kshch/gh-3466-tonumber64-strange-behaviour
Issue: https://github.com/tarantool/tarantool/issues/3466

 src/lua/init.c         |  6 +++++-
 test/box/misc.result   | 19 +++++++++++++++++++
 test/box/misc.test.lua |  8 ++++++++
 3 files changed, 32 insertions(+), 1 deletion(-)

diff --git a/src/lua/init.c b/src/lua/init.c
index 9a96030..4b5285d 100644
--- a/src/lua/init.c
+++ b/src/lua/init.c
@@ -222,7 +222,11 @@ lbox_tonumber64(struct lua_State *L)
 			if (argl == 0) {
 				lua_pushnil(L);
 			} else if (negative) {
-				luaL_pushint64(L, -1 * (long long )result);
+				if (result > LLONG_MAX) {
+					lua_pushnil(L);
+				} else {
+					luaL_pushint64(L, -1 * result);
+				}
 			} else {
 				luaL_pushuint64(L, result);
 			}
diff --git a/test/box/misc.result b/test/box/misc.result
index f332a8c..dcd08d0 100644
--- a/test/box/misc.result
+++ b/test/box/misc.result
@@ -640,6 +640,25 @@ tostring(tonumber64('1234567890123456')) == '1234567890123456ULL'
 ---
 - true
 ...
+--
+-- gh-3466: Strange behaviour of tonumber64 function
+--
+tostring(tonumber64('18446744073709551615')) == '18446744073709551615ULL'
+---
+- true
+...
+tonumber64('18446744073709551616') == nil
+---
+- true
+...
+tostring(tonumber64('-9223372036854775807')) == '-9223372036854775807LL'
+---
+- true
+...
+tonumber64('-9223372036854775808') == nil
+---
+- true
+...
 tonumber64('0x12') == 18
 ---
 - true
diff --git a/test/box/misc.test.lua b/test/box/misc.test.lua
index e24228a..4816235 100644
--- a/test/box/misc.test.lua
+++ b/test/box/misc.test.lua
@@ -165,6 +165,14 @@ tostring(tonumber64('12345678901234')) == '12345678901234'
 tostring(tonumber64('123456789012345')) == '123456789012345ULL'
 tostring(tonumber64('1234567890123456')) == '1234567890123456ULL'
 
+--
+-- gh-3466: Strange behaviour of tonumber64 function
+--
+tostring(tonumber64('18446744073709551615')) == '18446744073709551615ULL'
+tonumber64('18446744073709551616') == nil
+tostring(tonumber64('-9223372036854775807')) == '-9223372036854775807LL'
+tonumber64('-9223372036854775808') == nil
+
 tonumber64('0x12') == 18
 tonumber64('0x12', 16) == 18
 tonumber64('0x12', 17) == nil
-- 
2.7.4