[Tarantool-patches] [PATCH v1 1/1] sql: fix comparison between big DOUBLE and DECIMAL

[Mergen Imeev via Tarantool-patches <tarantool-patches@dev.tarantool.org>]

This patch fixes comparison between DOUBLE values greater or equal to
1e38 or less or equal to -1e38 and DECIMAL value. Now any DOUBLE value
greater or equal to 1e38 is more than any DECIMAL value and DOUBLE
value less or equal to -1e38 is less than any DECIMAL value.

Closes #6376
---
https://github.com/tarantool/tarantool/issues/6376
https://github.com/tarantool/tarantool/tree/imeevma/gh-6376-fix-cmp-between-big-double-and-dec

 .../gh-6376-fix-incorrect-dec-inf-cmp.md      |  5 +++
 src/box/sql/mem.c                             |  4 +-
 test/sql-tap/engine.cfg                       |  1 +
 .../gh-6376-wrong-double-to-dec-cmp.test.lua  | 41 +++++++++++++++++++
 4 files changed, 49 insertions(+), 2 deletions(-)
 create mode 100644 changelogs/unreleased/gh-6376-fix-incorrect-dec-inf-cmp.md
 create mode 100755 test/sql-tap/gh-6376-wrong-double-to-dec-cmp.test.lua

diff --git a/changelogs/unreleased/gh-6376-fix-incorrect-dec-inf-cmp.md b/changelogs/unreleased/gh-6376-fix-incorrect-dec-inf-cmp.md
new file mode 100644
index 000000000..102b6b9ab
--- /dev/null
+++ b/changelogs/unreleased/gh-6376-fix-incorrect-dec-inf-cmp.md
@@ -0,0 +1,5 @@
+## bugfix/sql
+
+* Fixed wrong comparison between inf and -inf DOUBLE values and DECIMAL
+  values (gh-6376).
+
diff --git a/src/box/sql/mem.c b/src/box/sql/mem.c
index 0aca76112..fbfe81038 100644
--- a/src/box/sql/mem.c
+++ b/src/box/sql/mem.c
@@ -2451,9 +2451,9 @@ mem_cmp_num(const struct Mem *a, const struct Mem *b)
 		}
 		case MEM_TYPE_DOUBLE: {
 			if (b->u.r >= 1e38)
-				return 1;
-			if (b->u.r <= -1e38)
 				return -1;
+			if (b->u.r <= -1e38)
+				return 1;
 			decimal_t dec;
 			decimal_t *d = decimal_from_double(&dec, b->u.r);
 			assert(d != NULL && d == &dec);
diff --git a/test/sql-tap/engine.cfg b/test/sql-tap/engine.cfg
index 35754f769..26d85b674 100644
--- a/test/sql-tap/engine.cfg
+++ b/test/sql-tap/engine.cfg
@@ -35,6 +35,7 @@
     "built-in-functions.test.lua": {
         "memtx": {"engine": "memtx"}
     },
+    "gh-6376-wrong-double-to-dec-cmp.test.lua": {},
     "gh-4077-iproto-execute-no-bind.test.lua": {},
     "*": {
         "memtx": {"engine": "memtx"},
diff --git a/test/sql-tap/gh-6376-wrong-double-to-dec-cmp.test.lua b/test/sql-tap/gh-6376-wrong-double-to-dec-cmp.test.lua
new file mode 100755
index 000000000..8066e365b
--- /dev/null
+++ b/test/sql-tap/gh-6376-wrong-double-to-dec-cmp.test.lua
@@ -0,0 +1,41 @@
+#!/usr/bin/env tarantool
+local test = require("sqltester")
+test:plan(4)
+
+--
+-- Make sure that the comparison between too large or too small DOUBLE and
+-- DECIMAL is correct.
+--
+test:do_execsql_test(
+    "gh-6376-1",
+    [[
+        SELECT -1e40 > CAST(1 AS DECIMAL);
+    ]], {
+        false
+    })
+
+test:do_execsql_test(
+    "gh-6376-2",
+    [[
+        SELECT -1e400 < CAST(-1 AS DECIMAL);
+    ]], {
+        true
+    })
+
+test:do_execsql_test(
+    "gh-6376-3",
+    [[
+        SELECT 1e40 >= CAST(1 AS DECIMAL);
+    ]], {
+        true
+    })
+
+test:do_execsql_test(
+    "gh-6376-4",
+    [[
+        SELECT 1e400 <= CAST(1 AS DECIMAL);
+    ]], {
+        false
+    })
+
+test:finish_test()
-- 
2.25.1