From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: From: Serge Petrenko Subject: [PATCH 3/5] lua: allow to encode and decode decimals as msgpack Date: Wed, 17 Jul 2019 18:33:44 +0300 Message-Id: In-Reply-To: References: MIME-Version: 1.0 Content-Transfer-Encoding: 8bit To: vdavydov.dev@gmail.com Cc: tarantool-patches@freelists.org, Serge Petrenko List-ID: It is now possible to insert decimals into spaces, but only into unindexed fields. Part-of #4333 --- extra/exports | 4 ++ src/lib/core/decimal.c | 10 +++- src/lib/core/mp_user_types.h | 3 +- src/lib/core/mpstream.c | 11 ++++ src/lib/core/mpstream.h | 4 ++ src/lua/decimal.c | 10 ++-- src/lua/decimal.h | 5 ++ src/lua/msgpack.c | 52 +++++++++++++---- src/lua/msgpackffi.lua | 33 +++++++++++ src/lua/utils.c | 16 ++++- src/lua/utils.h | 8 ++- test/app/decimal.result | 106 +++++++++++++++++----------------- test/app/msgpack.result | 41 +++++++++++++ test/app/msgpack.test.lua | 15 +++++ third_party/lua-yaml/lyaml.cc | 6 ++ 15 files changed, 248 insertions(+), 76 deletions(-) diff --git a/extra/exports b/extra/exports index b8c42c0df..7b84a1452 100644 --- a/extra/exports +++ b/extra/exports @@ -62,8 +62,12 @@ PMurHash32_Result crc32_calc mp_encode_double mp_encode_float +mp_encode_decimal mp_decode_double mp_decode_float +mp_decode_extl +mp_sizeof_decimal +decimal_unpack log_type say_set_log_level diff --git a/src/lib/core/decimal.c b/src/lib/core/decimal.c index 1423ae418..2c69a773c 100644 --- a/src/lib/core/decimal.c +++ b/src/lib/core/decimal.c @@ -33,6 +33,7 @@ #include "third_party/decNumber/decContext.h" #include "third_party/decNumber/decPacked.h" #include "lib/core/tt_static.h" +#include "lib/msgpuck/msgpuck.h" #include #include #include /* DBL_DIG */ @@ -312,12 +313,15 @@ char * decimal_pack(char *data, const decimal_t *dec) { uint32_t len = decimal_len(dec); - *data++ = decimal_scale(dec); + /* reserve space for resulting scale */ + char *svp = data++; len--; int32_t scale; char *tmp = (char *)decPackedFromNumber((uint8_t *)data, len, &scale, dec); assert(tmp == data); - assert(scale == (int32_t)decimal_scale(dec)); + /* scale may be negative, when exponent is > 0 */ + assert(scale == (int32_t)decimal_scale(dec) || scale < 0); + mp_store_u8(svp, (int8_t)scale); (void)tmp; data += len; return data; @@ -326,7 +330,7 @@ decimal_pack(char *data, const decimal_t *dec) decimal_t * decimal_unpack(const char **data, uint32_t len, decimal_t *dec) { - int32_t scale = *((*data)++); + int32_t scale = (int8_t)mp_load_u8(data); len--; decimal_t *res = decPackedToNumber((uint8_t *)*data, len, &scale, dec); if (res) diff --git a/src/lib/core/mp_user_types.h b/src/lib/core/mp_user_types.h index 9158b40d3..8211e3e79 100644 --- a/src/lib/core/mp_user_types.h +++ b/src/lib/core/mp_user_types.h @@ -32,7 +32,8 @@ */ enum mp_user_type { - MP_DECIMAL = 0 + MP_UNKNOWN = 0, + MP_DECIMAL = 1 }; #endif diff --git a/src/lib/core/mpstream.c b/src/lib/core/mpstream.c index 8b7276ab1..a46b7962b 100644 --- a/src/lib/core/mpstream.c +++ b/src/lib/core/mpstream.c @@ -33,6 +33,7 @@ #include #include #include "msgpuck.h" +#include "mp_decimal.h" void mpstream_reserve_slow(struct mpstream *stream, size_t size) @@ -186,6 +187,16 @@ mpstream_encode_binl(struct mpstream *stream, uint32_t len) mpstream_advance(stream, pos - data); } +void +mpstream_encode_decimal(struct mpstream *stream, decimal_t *val) +{ + char *data = mpstream_reserve(stream, mp_sizeof_decimal(val)); + if (data == NULL) + return; + char *pos = mp_encode_decimal(data, val); + mpstream_advance(stream, pos - data); +} + void mpstream_memcpy(struct mpstream *stream, const void *src, uint32_t n) { diff --git a/src/lib/core/mpstream.h b/src/lib/core/mpstream.h index 69fa76f7f..44af28cb5 100644 --- a/src/lib/core/mpstream.h +++ b/src/lib/core/mpstream.h @@ -32,6 +32,7 @@ */ #include "diag.h" +#include "decimal.h" #if defined(__cplusplus) extern "C" { @@ -136,6 +137,9 @@ mpstream_encode_bool(struct mpstream *stream, bool val); void mpstream_encode_binl(struct mpstream *stream, uint32_t len); +void +mpstream_encode_decimal(struct mpstream *stream, decimal_t *val); + /** Copies n bytes from memory area src to stream. */ void mpstream_memcpy(struct mpstream *stream, const void *src, uint32_t n); diff --git a/src/lua/decimal.c b/src/lua/decimal.c index e548cdb9d..ab8d85f75 100644 --- a/src/lua/decimal.c +++ b/src/lua/decimal.c @@ -31,7 +31,7 @@ #include "lua/decimal.h" #include "lib/core/decimal.h" -#include "lua/utils.h" +#include "lua/utils.h" /* CTID_DECIMAL, ... */ #include #include @@ -69,16 +69,18 @@ ldecimal_##name(struct lua_State *L) { \ static int \ ldecimal_##name(struct lua_State *L) { \ assert(lua_gettop(L) == 2); \ + if (lua_isnil(L, 1) || lua_isnil(L, 2)) { \ + lua_pushboolean(L, false); \ + return 1; \ + } \ decimal_t *lhs = lua_todecimal(L, 1); \ decimal_t *rhs = lua_todecimal(L, 2); \ lua_pushboolean(L, decimal_compare(lhs, rhs) cmp 0); \ return 1; \ } -static uint32_t CTID_DECIMAL; - /** Push a new decimal on the stack and return a pointer to it. */ -static decimal_t * +decimal_t * lua_pushdecimal(struct lua_State *L) { decimal_t *res = luaL_pushcdata(L, CTID_DECIMAL); diff --git a/src/lua/decimal.h b/src/lua/decimal.h index 0485d11ef..b5c0e54b4 100644 --- a/src/lua/decimal.h +++ b/src/lua/decimal.h @@ -31,12 +31,17 @@ #ifndef TARANTOOL_LUA_DECIMAL_H_INCLUDED #define TARANTOOL_LUA_DECIMAL_H_INCLUDED +#include "lib/core/decimal.h" + #if defined(__cplusplus) extern "C" { #endif /* defined(__cplusplus) */ struct lua_State; +decimal_t * +lua_pushdecimal(struct lua_State *L); + void tarantool_lua_decimal_init(struct lua_State *L); diff --git a/src/lua/msgpack.c b/src/lua/msgpack.c index 2126988eb..3cb7d7dcd 100644 --- a/src/lua/msgpack.c +++ b/src/lua/msgpack.c @@ -41,6 +41,10 @@ #include #include +#include "lua/decimal.h" +#include "lib/core/decimal.h" +#include "lib/core/mp_user_types.h" + #include void @@ -175,16 +179,23 @@ restart: /* used by MP_EXT */ assert(lua_gettop(L) == top); return MP_ARRAY; case MP_EXT: - /* Run trigger if type can't be encoded */ - type = luamp_encode_extension(L, top, stream); - if (type != MP_EXT) - return type; /* Value has been packed by the trigger */ - /* Try to convert value to serializable type */ - luaL_convertfield(L, cfg, top, field); - /* handled by luaL_convertfield */ - assert(field->type != MP_EXT); - assert(lua_gettop(L) == top); - goto restart; + switch (field->ext_type) + { + case MP_DECIMAL: + mpstream_encode_decimal(stream, field->decval); + return MP_EXT; + case MP_UNKNOWN: + /* Run trigger if type can't be encoded */ + type = luamp_encode_extension(L, top, stream); + if (type != MP_EXT) + return type; /* Value has been packed by the trigger */ + /* Try to convert value to serializable type */ + luaL_convertfield(L, cfg, top, field); + /* handled by luaL_convertfield */ + assert(field->type != MP_EXT || field->ext_type != MP_UNKNOWN); + assert(lua_gettop(L) == top); + goto restart; + } } return MP_EXT; } @@ -283,9 +294,28 @@ luamp_decode(struct lua_State *L, struct luaL_serializer *cfg, return; } case MP_EXT: - luamp_decode_extension(L, data); + { + uint32_t len; + int8_t type; + len = mp_decode_extl(data, &type); + switch (type) { + case MP_DECIMAL: + { + decimal_t *dec = lua_pushdecimal(L); + dec = decimal_unpack(data, len, dec); + if (dec == NULL) { + lua_pop(L, -1); + luaL_error(L, "msgpack.decode: " + "invalid MsgPack."); + } + return; + } + default: + luamp_decode_extension(L, data); + } break; } + } } diff --git a/src/lua/msgpackffi.lua b/src/lua/msgpackffi.lua index bfeedbc4b..4c799eed0 100644 --- a/src/lua/msgpackffi.lua +++ b/src/lua/msgpackffi.lua @@ -17,10 +17,18 @@ char * mp_encode_float(char *data, float num); char * mp_encode_double(char *data, double num); +char * +mp_encode_decimal(char *data, decimal_t *dec); +uint32_t +mp_sizeof_decimal(const decimal_t *dec); float mp_decode_float(const char **data); double mp_decode_double(const char **data); +uint32_t +mp_decode_extl(const char **data, int8_t *type); +decimal_t * +decimal_unpack(const char **data, uint32_t len, decimal_t *dec); ]]) local strict_alignment = (jit.arch == 'arm') @@ -117,6 +125,11 @@ local function encode_double(buf, num) builtin.mp_encode_double(p, num) end +local function encode_decimal(buf, num) + local p = buf:alloc(builtin.mp_sizeof_decimal(num)) + builtin.mp_encode_decimal(p, num) +end + local function encode_int(buf, num) if num >= 0 then if num <= 0x7f then @@ -294,6 +307,7 @@ on_encode(ffi.typeof('const unsigned char'), encode_int) on_encode(ffi.typeof('bool'), encode_bool_cdata) on_encode(ffi.typeof('float'), encode_float) on_encode(ffi.typeof('double'), encode_double) +on_encode(ffi.typeof('decimal_t'), encode_decimal) -------------------------------------------------------------------------------- -- Decoder @@ -473,6 +487,23 @@ local function decode_map(data, size) return setmetatable(map, msgpack.map_mt) end +local function decode_ext(data) + local t = ffi.new("int8_t[1]") + -- mp_decode_extl and mp_decode_decimal + -- need type code + data[0] = data[0] - 1 + local old_data = data[0] + local len = builtin.mp_decode_extl(data, t) + --MP_DECIMAL + if t[0] == 1 then + local num = ffi.new("decimal_t") + builtin.decimal_unpack(data, len, num) + return num + else + error("Unsupported extension type") + end +end + local decoder_hint = { --[[{{{ MP_BIN]] [0xc4] = function(data) return decode_str(data, decode_u8(data)) end; @@ -528,6 +559,8 @@ decode_r = function(data) return false elseif c == 0xc3 then return true + elseif c >= 0xd4 and c <= 0xd8 or c >= 0xc7 and c <= 0xc9 then + return decode_ext(data) else local fun = decoder_hint[c]; assert (type(fun) == "function") diff --git a/src/lua/utils.c b/src/lua/utils.c index 0a4bcf517..47cf030ab 100644 --- a/src/lua/utils.c +++ b/src/lua/utils.c @@ -45,6 +45,8 @@ static uint32_t CTID_STRUCT_IBUF; static uint32_t CTID_STRUCT_IBUF_PTR; uint32_t CTID_CHAR_PTR; uint32_t CTID_CONST_CHAR_PTR; +uint32_t CTID_DECIMAL; + void * luaL_pushcdata(struct lua_State *L, uint32_t ctypeid) @@ -723,6 +725,12 @@ luaL_tofield(struct lua_State *L, struct luaL_serializer *cfg, int index, /* Fall through */ default: field->type = MP_EXT; + if (cd->ctypeid == CTID_DECIMAL) { + field->ext_type = MP_DECIMAL; + field->decval = (decimal_t *) cdata; + } else { + field->ext_type = MP_UNKNOWN; + } } return 0; } @@ -754,6 +762,7 @@ luaL_tofield(struct lua_State *L, struct luaL_serializer *cfg, int index, /* Fall through */ default: field->type = MP_EXT; + field->ext_type = MP_UNKNOWN; } #undef CHECK_NUMBER return 0; @@ -765,7 +774,7 @@ luaL_convertfield(struct lua_State *L, struct luaL_serializer *cfg, int idx, { if (idx < 0) idx = lua_gettop(L) + idx + 1; - assert(field->type == MP_EXT); /* must be called after tofield() */ + assert(field->type == MP_EXT && field->ext_type == MP_UNKNOWN); /* must be called after tofield() */ if (cfg->encode_load_metatables) { int type = lua_type(L, idx); @@ -782,10 +791,11 @@ luaL_convertfield(struct lua_State *L, struct luaL_serializer *cfg, int idx, } } - if (field->type == MP_EXT && cfg->encode_use_tostring) + if (field->type == MP_EXT && field->ext_type == MP_UNKNOWN && + cfg->encode_use_tostring) lua_field_tostring(L, cfg, idx, field); - if (field->type != MP_EXT) + if (field->type != MP_EXT || field->ext_type != MP_UNKNOWN) return; if (cfg->encode_invalid_as_nil) { diff --git a/src/lua/utils.h b/src/lua/utils.h index 7e7cdc0c6..3ca43292b 100644 --- a/src/lua/utils.h +++ b/src/lua/utils.h @@ -54,6 +54,8 @@ extern "C" { #include #include "lua/error.h" +#include "lib/core/mp_user_types.h" +#include "lib/core/decimal.h" /* decimal_t */ struct lua_State; struct ibuf; @@ -69,6 +71,8 @@ extern struct ibuf *tarantool_lua_ibuf; extern uint32_t CTID_CONST_CHAR_PTR; extern uint32_t CTID_CHAR_PTR; +extern uint32_t CTID_DECIMAL; + /** \cond public */ @@ -286,8 +290,10 @@ struct luaL_field { bool bval; /* Array or map. */ uint32_t size; + decimal_t *decval; }; enum mp_type type; + enum mp_user_type ext_type; bool compact; /* a flag used by YAML serializer */ }; @@ -373,7 +379,7 @@ luaL_checkfield(struct lua_State *L, struct luaL_serializer *cfg, int idx, { if (luaL_tofield(L, cfg, idx, field) < 0) luaT_error(L); - if (field->type != MP_EXT) + if (field->type != MP_EXT || field->ext_type != MP_UNKNOWN) return; luaL_convertfield(L, cfg, idx, field); } diff --git a/test/app/decimal.result b/test/app/decimal.result index 53ec73edc..334c9fbab 100644 --- a/test/app/decimal.result +++ b/test/app/decimal.result @@ -12,77 +12,77 @@ ffi = require('ffi') -- check various constructors decimal.new('1234.5678') | --- - | - '1234.5678' + | - 1234.5678 | ... decimal.new('1e6') | --- - | - '1000000' + | - 1000000 | ... decimal.new('-6.234612e2') | --- - | - '-623.4612' + | - -623.4612 | ... -- check (u)int16/32/64_t decimal.new(2ULL ^ 63) | --- - | - '9223372036854775808' + | - 9223372036854775808 | ... decimal.new(123456789123456789ULL) | --- - | - '123456789123456789' + | - 123456789123456789 | ... decimal.new(-123456789123456789LL) | --- - | - '-123456789123456789' + | - -123456789123456789 | ... decimal.new(ffi.new('uint8_t', 231)) | --- - | - '231' + | - 231 | ... decimal.new(ffi.new('int8_t', -113)) | --- - | - '-113' + | - -113 | ... decimal.new(ffi.new('uint16_t', 65535)) | --- - | - '65535' + | - 65535 | ... decimal.new(ffi.new('int16_t', -31263)) | --- - | - '-31263' + | - -31263 | ... decimal.new(ffi.new('uint32_t', 4123123123)) | --- - | - '4123123123' + | - 4123123123 | ... decimal.new(ffi.new('int32_t', -2123123123)) | --- - | - '-2123123123' + | - -2123123123 | ... decimal.new(ffi.new('float', 128.5)) | --- - | - '128.5' + | - 128.5 | ... decimal.new(ffi.new('double', 128.5)) | --- - | - '128.5' + | - 128.5 | ... decimal.new(1) | --- - | - '1' + | - 1 | ... decimal.new(-1) | --- - | - '-1' + | - -1 | ... decimal.new(2^64) | --- - | - '18446744073709600000' + | - 18446744073709600000 | ... decimal.new(2^(-20)) | --- - | - '0.00000095367431640625' + | - 0.00000095367431640625 | ... -- incorrect constructions @@ -128,38 +128,38 @@ a = decimal.new('10') | ... a | --- - | - '10' + | - 10 | ... b = decimal.new('0.1') | --- | ... b | --- - | - '0.1' + | - 0.1 | ... a + b | --- - | - '10.1' + | - 10.1 | ... a - b | --- - | - '9.9' + | - 9.9 | ... a * b | --- - | - '1.0' + | - 1.0 | ... a / b | --- - | - '100' + | - 100 | ... a ^ b | --- - | - '1.2589254117941672104239541063958006061' + | - 1.2589254117941672104239541063958006061 | ... b ^ a | --- - | - '0.0000000001' + | - 0.0000000001 | ... -a + -b == -(a + b) | --- @@ -167,11 +167,11 @@ b ^ a | ... a | --- - | - '10' + | - 10 | ... b | --- - | - '0.1' + | - 0.1 | ... a < b @@ -216,28 +216,28 @@ a ~= b | ... a | --- - | - '10' + | - 10 | ... b | --- - | - '0.1' + | - 0.1 | ... decimal.sqrt(a) | --- - | - '3.1622776601683793319988935444327185337' + | - 3.1622776601683793319988935444327185337 | ... decimal.ln(a) | --- - | - '2.3025850929940456840179914546843642076' + | - 2.3025850929940456840179914546843642076 | ... decimal.log10(a) | --- - | - '1' + | - 1 | ... decimal.exp(a) | --- - | - '22026.465794806716516957900645284244366' + | - 22026.465794806716516957900645284244366 | ... a == decimal.ln(decimal.exp(a)) | --- @@ -261,7 +261,7 @@ a + -a == 0 | ... a | --- - | - '10' + | - 10 | ... a = decimal.new('1.1234567891234567891234567891234567891') @@ -269,7 +269,7 @@ a = decimal.new('1.1234567891234567891234567891234567891') | ... a | --- - | - '1.1234567891234567891234567891234567891' + | - 1.1234567891234567891234567891234567891 | ... decimal.precision(a) | --- @@ -285,7 +285,7 @@ decimal.round(a, 37) == a | ... a | --- - | - '1.1234567891234567891234567891234567891' + | - 1.1234567891234567891234567891234567891 | ... a = decimal.round(a, 36) | --- @@ -309,19 +309,19 @@ decimal.round(a, -5) == a | ... decimal.round(a, 7) | --- - | - '1.1234568' + | - 1.1234568 | ... decimal.round(a, 3) | --- - | - '1.123' + | - 1.123 | ... decimal.round(a, 0) | --- - | - '1' + | - 1 | ... a | --- - | - '1.123456789123456789123456789123456789' + | - 1.123456789123456789123456789123456789 | ... decimal.ln(0) @@ -334,7 +334,7 @@ decimal.ln(-1) | ... decimal.ln(1) | --- - | - '0' + | - 0 | ... decimal.log10(0) | --- @@ -346,7 +346,7 @@ decimal.log10(-1) | ... decimal.log10(1) | --- - | - '0' + | - 0 | ... decimal.exp(88) | --- @@ -354,7 +354,7 @@ decimal.exp(88) | ... decimal.exp(87) | --- - | - '60760302250568721495223289381302760753' + | - 60760302250568721495223289381302760753 | ... decimal.sqrt(-5) | --- @@ -362,7 +362,7 @@ decimal.sqrt(-5) | ... decimal.sqrt(5) | --- - | - '2.2360679774997896964091736687312762354' + | - 2.2360679774997896964091736687312762354 | ... -- various incorrect operands @@ -408,11 +408,11 @@ a ^ 2 | ... a ^ 1.9 | --- - | - '1258925411794167210423954106395800606.1' + | - 1258925411794167210423954106395800606.1 | ... a * '1e18' | --- - | - '10000000000000000000000000000000000000' + | - 10000000000000000000000000000000000000 | ... a = decimal.new(string.rep('9', 38)) | --- @@ -435,7 +435,7 @@ a + '0.5' | ... a + '0.4' | --- - | - '99999999999999999999999999999999999999' + | - 99999999999999999999999999999999999999 | ... a / 0.5 | --- @@ -451,7 +451,7 @@ a = decimal.new('-13') | ... a ^ 2 | --- - | - '169' + | - 169 | ... -- fractional powers are allowed only for positive numbers a ^ 2.5 @@ -462,17 +462,17 @@ a ^ 2.5 -- check correct rounding when scale = 0 decimal.round(decimal.new(0.9), 0) | --- - | - '1' + | - 1 | ... decimal.round(decimal.new(9.9), 0) | --- - | - '10' + | - 10 | ... decimal.round(decimal.new(99.9), 0) | --- - | - '100' + | - 100 | ... decimal.round(decimal.new(99.4), 0) | --- - | - '99' + | - 99 | ... diff --git a/test/app/msgpack.result b/test/app/msgpack.result index a67c05d38..4b5aec784 100644 --- a/test/app/msgpack.result +++ b/test/app/msgpack.result @@ -252,3 +252,44 @@ msgpack.decode(ffi.cast('char *', '\x04\x05\x06'), -1) --- - error: 'msgpack.decode: size can''t be negative' ... +-- +-- gh-4333: msgpack encode/decode decimals. +-- +decimal = require('decimal') +--- +... +a = decimal.new('1e37') +--- +... +b = decimal.new('1e-38') +--- +... +c = decimal.new('1') +--- +... +d = decimal.new('0.1234567') +--- +... +e = decimal.new('123.4567') +--- +... +msgpack.decode(msgpack.encode(a)) == a +--- +- true +... +msgpack.decode(msgpack.encode(b)) == b +--- +- true +... +msgpack.decode(msgpack.encode(c)) == c +--- +- true +... +msgpack.decode(msgpack.encode(d)) == d +--- +- true +... +msgpack.decode(msgpack.encode(e)) == e +--- +- true +... diff --git a/test/app/msgpack.test.lua b/test/app/msgpack.test.lua index e0880ac22..9224d870a 100644 --- a/test/app/msgpack.test.lua +++ b/test/app/msgpack.test.lua @@ -84,3 +84,18 @@ size = msgpack.encode(100, buf) -- is not negative. -- msgpack.decode(ffi.cast('char *', '\x04\x05\x06'), -1) + +-- +-- gh-4333: msgpack encode/decode decimals. +-- +decimal = require('decimal') +a = decimal.new('1e37') +b = decimal.new('1e-38') +c = decimal.new('1') +d = decimal.new('0.1234567') +e = decimal.new('123.4567') +msgpack.decode(msgpack.encode(a)) == a +msgpack.decode(msgpack.encode(b)) == b +msgpack.decode(msgpack.encode(c)) == c +msgpack.decode(msgpack.encode(d)) == d +msgpack.decode(msgpack.encode(e)) == e diff --git a/third_party/lua-yaml/lyaml.cc b/third_party/lua-yaml/lyaml.cc index 46c98bde1..8ff2867e3 100644 --- a/third_party/lua-yaml/lyaml.cc +++ b/third_party/lua-yaml/lyaml.cc @@ -49,6 +49,7 @@ extern "C" { #include "b64.h" } /* extern "C" */ #include "lua/utils.h" +#include "lib/core/decimal.h" #define LUAYAML_TAG_PREFIX "tag:yaml.org,2002:" @@ -693,6 +694,11 @@ static int dump_node(struct lua_yaml_dumper *dumper) len = 4; break; case MP_EXT: + if (field.ext_type == MP_DECIMAL) { + str = decimal_to_string(field.decval); + len = strlen(str); + break; + } assert(0); /* checked by luaL_checkfield() */ break; } -- 2.20.1 (Apple Git-117)