From: Vladislav Shpilevoy <v.shpilevoy@tarantool.org> To: tarantool-patches@freelists.org Cc: alexander.turenko@tarantool.org Subject: [tarantool-patches] [PATCH v2 4/4] app: allow to raise an error on too nested tables Date: Mon, 9 Sep 2019 21:00:10 +0200 [thread overview] Message-ID: <c9a04a6bdf8752dce051c264edfe097ba85d2bda.1568055477.git.v.shpilevoy@tarantool.org> (raw) In-Reply-To: <cover.1568055477.git.v.shpilevoy@tarantool.org> Closes #4434 Follow-up #4366 @TarantoolBot document Title: json/msgpack.cfg.encode_crop_too_deep option Tarantool has several so called serializers to convert data between Lua and another format: YAML, JSON, msgpack. YAML is a crazy serializer without depth restrictions. But for JSON and msgpack a user could set encode_max_depth option. That option led to crop of a table when it had too many nested levels. Sometimes such behaviour is undesirable. Now a user can choose to raise an error instead of data corruption: msgpack.cfg({encode_crop_too_deep = false}) t = nil for i = 1, 100 do t = {t} end msgpack.encode(t) -- Here an exception is thrown. Option encode_crop_too_deep works for JSON and msgpack modules, and is false by default. It means, that now if some existing users have cropping, even intentional, they will get the exception. --- src/lua/msgpack.c | 4 ++++ src/lua/msgpackffi.lua | 3 +++ src/lua/utils.c | 1 + src/lua/utils.h | 6 ++++++ test/app-tap/lua/serializer_test.lua | 20 +++++++++++++++++++- test/app-tap/msgpackffi.test.lua | 7 ++++++- test/box/tuple.result | 8 +++++++- test/box/tuple.test.lua | 4 +++- third_party/lua-cjson/lua_cjson.c | 10 ++++++++-- 9 files changed, 57 insertions(+), 6 deletions(-) diff --git a/src/lua/msgpack.c b/src/lua/msgpack.c index c2be0b3e8..b1354776d 100644 --- a/src/lua/msgpack.c +++ b/src/lua/msgpack.c @@ -143,6 +143,8 @@ restart: /* used by MP_EXT */ case MP_MAP: /* Map */ if (level >= cfg->encode_max_depth) { + if (! cfg->encode_crop_too_deep) + return luaL_error(L, "Too high nest level"); mpstream_encode_nil(stream); /* Limit nested maps */ return MP_NIL; } @@ -164,6 +166,8 @@ restart: /* used by MP_EXT */ case MP_ARRAY: /* Array */ if (level >= cfg->encode_max_depth) { + if (! cfg->encode_crop_too_deep) + return luaL_error(L, "Too high nest level"); mpstream_encode_nil(stream); /* Limit nested arrays */ return MP_NIL; } diff --git a/src/lua/msgpackffi.lua b/src/lua/msgpackffi.lua index 65c57aa6e..73d0d6fe2 100644 --- a/src/lua/msgpackffi.lua +++ b/src/lua/msgpackffi.lua @@ -219,6 +219,9 @@ local function encode_r(buf, obj, level) encode_str(buf, obj) elseif type(obj) == "table" then if level >= msgpack.cfg.encode_max_depth then + if not msgpack.cfg.encode_crop_too_deep then + error('Too high nest level') + end encode_nil(buf) return end diff --git a/src/lua/utils.c b/src/lua/utils.c index b5cefd07f..c338e8dfa 100644 --- a/src/lua/utils.c +++ b/src/lua/utils.c @@ -237,6 +237,7 @@ static struct { OPTION(LUA_TNUMBER, encode_sparse_ratio, 2), OPTION(LUA_TNUMBER, encode_sparse_safe, 10), OPTION(LUA_TNUMBER, encode_max_depth, 32), + OPTION(LUA_TBOOLEAN, encode_crop_too_deep, 0), OPTION(LUA_TBOOLEAN, encode_invalid_numbers, 1), OPTION(LUA_TNUMBER, encode_number_precision, 14), OPTION(LUA_TBOOLEAN, encode_load_metatables, 1), diff --git a/src/lua/utils.h b/src/lua/utils.h index e9c316d22..d581e47fd 100644 --- a/src/lua/utils.h +++ b/src/lua/utils.h @@ -213,6 +213,12 @@ struct luaL_serializer { int encode_sparse_safe; /** Max recursion depth for encoding (MsgPack, CJSON only) */ int encode_max_depth; + /** + * A flag whether a table with too high nest level should + * be cropped. If not set, than this is considered an + * error. + */ + int encode_crop_too_deep; /** Enables encoding of NaN and Inf numbers */ int encode_invalid_numbers; /** Floating point numbers precision (YAML, CJSON only) */ diff --git a/test/app-tap/lua/serializer_test.lua b/test/app-tap/lua/serializer_test.lua index a44247d70..4fa2924cf 100644 --- a/test/app-tap/lua/serializer_test.lua +++ b/test/app-tap/lua/serializer_test.lua @@ -403,7 +403,7 @@ local function test_ucdata(test, s) end local function test_depth(test, s) - test:plan(1) + test:plan(3) -- -- gh-4434: serializer update should be reflected in Lua. -- @@ -412,6 +412,24 @@ local function test_depth(test, s) test:is(s.cfg.encode_max_depth, max_depth + 5, "cfg({<name> = value}) is reflected in cfg.<name>") s.cfg({encode_max_depth = max_depth}) + + -- + -- gh-4434 (yes, the same issue): let users choose whether + -- they want to raise an error on tables with too high nest + -- level. + -- + s.cfg({encode_crop_too_deep = false}) + + local t = nil + for i = 1, max_depth + 1 do t = {t} end + local ok, err = pcall(s.encode, t) + test:ok(not ok, "too deep encode depth") + + s.cfg({encode_max_depth = max_depth + 1}) + ok, err = pcall(s.encode, t) + test:ok(ok, "no throw in a corner case") + + s.cfg({encode_crop_too_deep = true, encode_max_depth = max_depth}) end return { diff --git a/test/app-tap/msgpackffi.test.lua b/test/app-tap/msgpackffi.test.lua index e26247625..d80978939 100755 --- a/test/app-tap/msgpackffi.test.lua +++ b/test/app-tap/msgpackffi.test.lua @@ -36,7 +36,7 @@ local function test_offsets(test, s) end local function test_other(test, s) - test:plan(21) + test:plan(22) local buf = string.char(0x93, 0x6e, 0xcb, 0x42, 0x2b, 0xed, 0x30, 0x47, 0x6f, 0xff, 0xff, 0xac, 0x77, 0x6b, 0x61, 0x71, 0x66, 0x7a, 0x73, 0x7a, 0x75, 0x71, 0x71, 0x78) @@ -82,6 +82,7 @@ local function test_other(test, s) return level end local msgpack = require('msgpack') + msgpack.cfg({encode_crop_too_deep = true}) local max_depth = msgpack.cfg.encode_max_depth local result_depth = check_depth(max_depth + 5) test:is(result_depth, max_depth, @@ -91,6 +92,10 @@ local function test_other(test, s) result_depth = check_depth(max_depth + 5) test:is(result_depth, max_depth + 5, "and uses it dynamically") + msgpack.cfg({encode_crop_too_deep = false}) + local ok = pcall(check_depth, max_depth + 6) + test:ok(not ok, "exception is thrown when crop is not allowed") + msgpack.cfg({encode_max_depth = max_depth}) end diff --git a/test/box/tuple.result b/test/box/tuple.result index 83f74d111..088400276 100644 --- a/test/box/tuple.result +++ b/test/box/tuple.result @@ -1401,6 +1401,12 @@ d:update{{'-', 1, dec.new('1e37')}} max_depth = msgpack.cfg.encode_max_depth --- ... +crop_too_deep = msgpack.cfg.encode_crop_too_deep +--- +... +msgpack.cfg({encode_crop_too_deep = true}) +--- +... t = nil --- ... @@ -1438,6 +1444,6 @@ level == max_depth + 5 or {level, max_depth} --- - true ... -msgpack.cfg({encode_max_depth = max_depth}) +msgpack.cfg({encode_max_depth = max_depth, encode_crop_too_deep = crop_too_deep}) --- ... diff --git a/test/box/tuple.test.lua b/test/box/tuple.test.lua index c8a0c03c5..7660f651c 100644 --- a/test/box/tuple.test.lua +++ b/test/box/tuple.test.lua @@ -478,6 +478,8 @@ d:update{{'-', 1, dec.new('1e37')}} -- gh-4434: tuple should use global msgpack serializer. -- max_depth = msgpack.cfg.encode_max_depth +crop_too_deep = msgpack.cfg.encode_crop_too_deep +msgpack.cfg({encode_crop_too_deep = true}) t = nil for i = 1, max_depth + 5 do t = {t} end tuple = box.tuple.new(t):totable() @@ -493,4 +495,4 @@ while tuple ~= nil do level = level + 1 tuple = tuple[1] end -- serializer allows deeper tables. level == max_depth + 5 or {level, max_depth} -msgpack.cfg({encode_max_depth = max_depth}) +msgpack.cfg({encode_max_depth = max_depth, encode_crop_too_deep = crop_too_deep}) diff --git a/third_party/lua-cjson/lua_cjson.c b/third_party/lua-cjson/lua_cjson.c index 6b03e391a..0c325d03c 100644 --- a/third_party/lua-cjson/lua_cjson.c +++ b/third_party/lua-cjson/lua_cjson.c @@ -400,14 +400,20 @@ static void json_append_data(lua_State *l, struct luaL_serializer *cfg, json_append_nil(cfg, json); break; case MP_MAP: - if (current_depth >= cfg->encode_max_depth) + if (current_depth >= cfg->encode_max_depth) { + if (! cfg->encode_crop_too_deep) + luaL_error(l, "Too high nest level"); return json_append_nil(cfg, json); /* Limit nested maps */ + } json_append_object(l, cfg, current_depth + 1, json); return; case MP_ARRAY: /* Array */ - if (current_depth >= cfg->encode_max_depth) + if (current_depth >= cfg->encode_max_depth) { + if (! cfg->encode_crop_too_deep) + luaL_error(l, "Too high nest level"); return json_append_nil(cfg, json); /* Limit nested arrays */ + } json_append_array(l, cfg, current_depth + 1, json, field.size); return; case MP_EXT: -- 2.20.1 (Apple Git-117)
next prev parent reply other threads:[~2019-09-09 18:56 UTC|newest] Thread overview: 16+ messages / expand[flat|nested] mbox.gz Atom feed top 2019-09-10 20:24 [tarantool-patches] [PATCH v2 0/4] Serializer bugs Vladislav Shpilevoy 2019-09-09 19:00 ` [tarantool-patches] [PATCH v2 1/4] app: serializers update now is reflected in Lua Vladislav Shpilevoy 2019-09-12 23:22 ` [tarantool-patches] " Alexander Turenko 2019-09-13 22:32 ` Vladislav Shpilevoy 2019-09-09 19:00 ` [tarantool-patches] [PATCH v2 2/4] msgpack: make msgpackffi use encode_max_depth option Vladislav Shpilevoy 2019-09-12 23:24 ` [tarantool-patches] " Alexander Turenko 2019-09-13 22:32 ` Vladislav Shpilevoy 2019-09-09 19:00 ` [tarantool-patches] [PATCH v2 3/4] tuple: use global msgpack serializer in Lua tuple Vladislav Shpilevoy 2019-09-12 23:27 ` [tarantool-patches] " Alexander Turenko 2019-09-13 22:32 ` Vladislav Shpilevoy 2019-09-09 19:00 ` Vladislav Shpilevoy [this message] 2019-09-12 23:32 ` [tarantool-patches] Re: [PATCH v2 4/4] app: allow to raise an error on too nested tables Alexander Turenko 2019-09-13 22:32 ` Vladislav Shpilevoy 2019-09-10 20:25 ` [tarantool-patches] Re: [PATCH v2 0/4] Serializer bugs Vladislav Shpilevoy 2019-09-12 23:44 ` Alexander Turenko 2019-09-13 22:32 ` Vladislav Shpilevoy
Reply instructions: You may reply publicly to this message via plain-text email using any one of the following methods: * Save the following mbox file, import it into your mail client, and reply-to-all from there: mbox Avoid top-posting and favor interleaved quoting: https://en.wikipedia.org/wiki/Posting_style#Interleaved_style * Reply using the --to, --cc, and --in-reply-to switches of git-send-email(1): git send-email \ --in-reply-to=c9a04a6bdf8752dce051c264edfe097ba85d2bda.1568055477.git.v.shpilevoy@tarantool.org \ --to=v.shpilevoy@tarantool.org \ --cc=alexander.turenko@tarantool.org \ --cc=tarantool-patches@freelists.org \ --subject='Re: [tarantool-patches] [PATCH v2 4/4] app: allow to raise an error on too nested tables' \ /path/to/YOUR_REPLY https://kernel.org/pub/software/scm/git/docs/git-send-email.html * If your mail client supports setting the In-Reply-To header via mailto: links, try the mailto: link
This is a public inbox, see mirroring instructions for how to clone and mirror all data and code used for this inbox