[PATCH 10/13] sql: fixes an error in sqlSubInt64

[Stanislav Zudin <szudin@tarantool.org>]

---
 src/box/sql/util.c | 54 +++++++++++++++++++++++++---------------------
 1 file changed, 29 insertions(+), 25 deletions(-)

diff --git a/src/box/sql/util.c b/src/box/sql/util.c
index 3786c5083..980932f3b 100644
--- a/src/box/sql/util.c
+++ b/src/box/sql/util.c
@@ -1260,6 +1260,25 @@ static u64 mod64(i64 v, bool is_signed)
 		return (u64)v;
 }
 
+static enum arithmetic_result
+apply_sign(i64* pOut, u64 value, bool is_neg)
+{
+	if (is_neg) {
+		if (value > INT64_MIN_MOD)
+			return ATHR_OVERFLOW;
+		else if (value == INT64_MIN_MOD)
+			*pOut = (i64)value;
+		else
+			*pOut = -(i64)value;
+
+		return ATHR_SIGNED;
+	}
+
+	*pOut = (i64) value;
+	return (value > INT64_MAX) ? ATHR_UNSIGNED
+				   : ATHR_SIGNED;
+}
+
 /*
  * Attempt to add, substract, or multiply the 64-bit value iB against
  * the other 64-bit integer at *pA and store the result in *pA.
@@ -1351,32 +1370,17 @@ sqlSubInt64(i64 * pA, bool is_signedA, i64 iB, bool is_signedB)
 		return sqlAddInt64(pA, is_signedA, uB, is_signedB);
 	} else {
 		/* Both iA & iB are positive */
-		if ((u64)iA < (u64)iB)
-			return  ATHR_OVERFLOW;
-		u64 val = (u64)iA - (u64)iB;
-		*pA = (i64)val;
-		return (val > INT64_MAX) ? ATHR_UNSIGNED
-					 : ATHR_SIGNED;
-	}
-}
-
-static enum arithmetic_result
-apply_sign(i64* pOut, u64 value, bool is_neg)
-{
-	if (is_neg) {
-		if (value > INT64_MIN_MOD)
-			return ATHR_OVERFLOW;
-		else if (value == INT64_MIN_MOD)
-			*pOut = (i64)value;
-		else
-			*pOut = -(i64)value;
-
-		return ATHR_SIGNED;
+		if ((u64)iA < (u64)iB) {
+			/* subtract with sign changing */
+			u64 val = (u64)iB - (u64)iA;
+			return apply_sign(pA, val, true);
+		} else {
+			u64 val = (u64)iA - (u64)iB;
+			*pA = (i64)val;
+			return (val > INT64_MAX) ? ATHR_UNSIGNED
+						 : ATHR_SIGNED;
+		}
 	}
-
-	*pOut = (i64) value;
-	return (value > INT64_MAX) ? ATHR_UNSIGNED
-			  	   : ATHR_SIGNED;
 }
 
 enum arithmetic_result
-- 
2.17.1