[Tarantool-patches] [PATCH v1 2/3] sql: fix cast of small negative DECIMAL to INTEGER

[imeevma at tarantool.org]

This patch fixes an assertion when casting DECIMAL value less than 0 and
greater than -1 to INTEGER.

Part of #6485
---
 src/box/sql/mem.c                             |  9 +++----
 test/sql-tap/gh-6485-bugs-in-decimal.test.lua | 24 ++++++++++++++++++-
 2 files changed, 28 insertions(+), 5 deletions(-)

diff --git a/src/box/sql/mem.c b/src/box/sql/mem.c
index b0eba303e..bc6f9b35a 100644
--- a/src/box/sql/mem.c
+++ b/src/box/sql/mem.c
@@ -1106,9 +1106,9 @@ dec_to_int(struct Mem *mem)
 		int64_t i;
 		if (decimal_to_int64(&mem->u.d, &i) == NULL)
 			return -1;
-		assert(i < 0);
+		assert(i <= 0);
 		mem->u.i = i;
-		mem->type = MEM_TYPE_INT;
+		mem->type = i == 0 ? MEM_TYPE_UINT : MEM_TYPE_INT;
 		mem->flags = 0;
 		return 0;
 	}
@@ -1137,14 +1137,15 @@ dec_to_int_forced(struct Mem *mem)
 	bool is_dec_int = decimal_is_int(&mem->u.d);
 	if (decimal_is_neg(&mem->u.d)) {
 		int64_t i;
-		mem->type = MEM_TYPE_INT;
 		mem->flags = 0;
 		if (decimal_to_int64(&mem->u.d, &i) == NULL) {
 			mem->u.i = INT64_MIN;
+			mem->type = MEM_TYPE_INT;
 			return -1;
 		}
-		assert(i < 0);
+		assert(i <= 0);
 		mem->u.i = i;
+		mem->type = i == 0 ? MEM_TYPE_UINT : MEM_TYPE_INT;
 		/*
 		 * Decimal is floored when cast to int, which means that after
 		 * cast it becomes bigger if it was not integer.
diff --git a/test/sql-tap/gh-6485-bugs-in-decimal.test.lua b/test/sql-tap/gh-6485-bugs-in-decimal.test.lua
index 3f63f2b76..0b9b2ea0a 100755
--- a/test/sql-tap/gh-6485-bugs-in-decimal.test.lua
+++ b/test/sql-tap/gh-6485-bugs-in-decimal.test.lua
@@ -1,6 +1,6 @@
 #!/usr/bin/env tarantool
 local test = require("sqltester")
-test:plan(1)
+test:plan(3)
 
 -- Make sure DECIMAL is not truncated when used in an index.
 test:do_execsql_test(
@@ -13,4 +13,26 @@ test:do_execsql_test(
     ]], {
     })
 
+--
+-- Make sure that DECIMAL greater than -1 and less than 0 are correctly cast to
+-- INTEGER.
+--
+test:do_execsql_test(
+    "gh-6485-2",
+    [[
+        SELECT CAST(CAST(-0.5 AS DECIMAL) AS INTEGER);
+    ]], {
+        0
+    })
+
+test:do_execsql_test(
+    "gh-6485-3",
+    [[
+        CREATE TABLE t(i INTEGER PRIMARY KEY);
+        INSERT INTO t VALUES(1);
+        SELECT i FROM t WHERE i IN (CAST(-0.1 AS DECIMAL), CAST(2 AS DECIMAL));
+        DROP TABLE t;
+    ]], {
+    })
+
 test:finish_test()
-- 
2.25.1