From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from [87.239.111.99] (localhost [127.0.0.1]) by dev.tarantool.org (Postfix) with ESMTP id 65E206EC5F; Tue, 2 Mar 2021 11:12:14 +0300 (MSK) DKIM-Filter: OpenDKIM Filter v2.11.0 dev.tarantool.org 65E206EC5F DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=tarantool.org; s=dev; t=1614672734; bh=TcHFnVPWM5l2+uAeL1M7ijS7UtHvMVc34bS7wITDhq8=; h=To:Date:Subject:List-Id:List-Unsubscribe:List-Archive:List-Post: List-Help:List-Subscribe:From:Reply-To:From; b=Q1U4ns3pF2bWmcVxV+PI2u6dVF0Ne9p5yOw1HzIxKx6CoE8K9ZzLsEvJfTyAeBK7n DgAxbpN3ukVIrmgf0E/dhn4yY74Ww5Oxs+FHEqlwir5m1u9WeyriIJPQDOJX0HoxRJ 8WSiYYb4h3LHaHAn4HJxFN7iY+OL6YQA01NxGf80= Received: from smtp42.i.mail.ru (smtp42.i.mail.ru [94.100.177.102]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by dev.tarantool.org (Postfix) with ESMTPS id 25A416EC5F for ; Tue, 2 Mar 2021 11:12:13 +0300 (MSK) DKIM-Filter: OpenDKIM Filter v2.11.0 dev.tarantool.org 25A416EC5F Received: by smtp42.i.mail.ru with esmtpa (envelope-from ) id 1lH08a-00023Z-4c; Tue, 02 Mar 2021 11:12:12 +0300 To: tarantool-patches@dev.tarantool.org Date: Tue, 2 Mar 2021 11:12:11 +0300 Message-Id: <20210302081211.84024-1-roman.habibov@tarantool.org> X-Mailer: git-send-email 2.24.3 (Apple Git-128) MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-7564579A: 646B95376F6C166E X-77F55803: 4F1203BC0FB41BD92A98208ECBDD29F5BCF62610559114B730DD7337B29CB406182A05F538085040DA157A659111FB5BCDF1A6802D8FA934CDB0DBFF0A1B6D45772ECE78B9CF978F X-7FA49CB5: FF5795518A3D127A4AD6D5ED66289B5278DA827A17800CE78C6616F30072131EEA1F7E6F0F101C67BD4B6F7A4D31EC0BCC500DACC3FED6E28638F802B75D45FF8AA50765F790063790B55F3E386DB9B28638F802B75D45FF5571747095F342E8C7A0BC55FA0FE5FC2EC80735919255E1674EB2EF3BDD21C1C73527A80A0FAB94389733CBF5DBD5E913377AFFFEAFD269176DF2183F8FC7C07E7E81EEA8A9722B8941B15DA834481FCF19DD082D7633A0EF3E4896CB9E6436389733CBF5DBD5E9D5E8D9A59859A8B6D082881546D93491CC7F00164DA146DA6F5DAA56C3B73B237318B6A418E8EAB8D32BA5DBAC0009BE9E8FC8737B5C22494BA1F928175D6EAC76E601842F6C81A12EF20D2F80756B5F7E9C4E3C761E06A776E601842F6C81A127C277FBC8AE2E8BB963701C3D06BE773AA81AA40904B5D9DBF02ECDB25306B2B25CBF701D1BE8734AD6D5ED66289B5278DA827A17800CE7FBF6ADB5090A71F767F23339F89546C5A8DF7F3B2552694A6FED454B719173D6725E5C173C3A84C3CF76CA422449501235872C767BF85DA2F004C906525384306FED454B719173D6462275124DF8B9C9DE2850DD75B2526BE5BFE6E7EFDEDCD789D4C264860C145E X-B7AD71C0: AC4F5C86D027EB782CDD5689AFBDA7A24A6D60772A99906F8E1CD14B953EB46D5B9E472490351971355D89D7DBCDD132 X-C1DE0DAB: C20DE7B7AB408E4181F030C43753B8186998911F362727C414F749A5E30D975CDB1C6D2319E1F167C1C14D27CF0514AE533CEC263AB2E6E59C2B6934AE262D3EE7EAB7254005DCEDE285C580305385EE1E0A4E2319210D9B64D260DF9561598F01A9E91200F654B0CCA28C6D779E2CD78E8E86DC7131B365E7726E8460B7C23C X-C8649E89: 4E36BF7865823D7055A7F0CF078B5EC49A30900B95165D34EE6683D9E546CC6257C376634B69DCA37368505FD5746D22C596483AFE82F91C2649490838248FDE1D7E09C32AA3244C40B1B9C0B2F563118807BE78E03421E3A8CE788DE6831205729B2BEF169E0186 X-D57D3AED: 3ZO7eAau8CL7WIMRKs4sN3D3tLDjz0dLbV79QFUyzQ2Ujvy7cMT6pYYqY16iZVKkSc3dCLJ7zSJH7+u4VD18S7Vl4ZUrpaVfd2+vE6kuoey4m4VkSEu530nj6fImhcD4MUrOEAnl0W826KZ9Q+tr5ycPtXkTV4k65bRjmOUUP8cvGozZ33TWg5HZplvhhXbhDGzqmQDTd6OAevLeAnq3Ra9uf7zvY2zzsIhlcp/Y7m53TZgf2aB4JOg4gkr2biojbL9S8ysBdXh6p/YHSd0EqxP29rZi6Cxc X-Mailru-Sender: ED747E36EB90C325A4602F7CA2CC58C12008CE76BD8ECF301054984D7C6CEBB43FB1763410276AF21EC3B765AEBF8DFDC77752E0C033A69EF2501F26BCC01020D1D7FFF4A7B59B3C6C18EFA0BB12DBB0 X-Mras: Ok Subject: [Tarantool-patches] [PATCH] yaml: serialize recursive structures X-BeenThere: tarantool-patches@dev.tarantool.org X-Mailman-Version: 2.1.34 Precedence: list List-Id: Tarantool development patches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , From: Roman Khabibov via Tarantool-patches Reply-To: Roman Khabibov Errors-To: tarantool-patches-bounces@dev.tarantool.org Sender: "Tarantool-patches" Fix bug with bus error during serializing of recursive structures. Closes #3228 --- Branch: https://github.com/tarantool/tarantool/tree/romanhabibov/gh-3228-visited-set Issue: https://github.com/tarantool/tarantool/issues/3228 src/box/lua/call.c | 6 +- src/box/lua/execute.c | 2 +- src/box/lua/serialize_lua.c | 91 ++++------ src/box/lua/tuple.c | 2 +- src/box/sql/func.c | 2 +- src/lua/msgpack.c | 10 +- src/lua/pickle.c | 2 +- src/lua/utils.c | 165 ++++++++++++++++-- src/lua/utils.h | 31 +++- ...-3228-serializer-look-for-recursion.result | 67 +++++++ ...228-serializer-look-for-recursion.test.lua | 17 ++ test/swim/swim.result | 18 +- third_party/lua-cjson/lua_cjson.c | 4 +- third_party/lua-yaml/lyaml.cc | 93 +++++----- 14 files changed, 362 insertions(+), 148 deletions(-) create mode 100644 test/app/gh-3228-serializer-look-for-recursion.result create mode 100644 test/app/gh-3228-serializer-look-for-recursion.test.lua diff --git a/src/box/lua/call.c b/src/box/lua/call.c index 0315e720c..e4907a3f7 100644 --- a/src/box/lua/call.c +++ b/src/box/lua/call.c @@ -193,7 +193,7 @@ luamp_encode_call_16(lua_State *L, struct luaL_serializer *cfg, */ for (int i = 1; i <= nrets; ++i) { struct luaL_field field; - if (luaL_tofield(L, cfg, NULL, i, &field) < 0) + if (luaL_tofield(L, cfg, 0, NULL, i, &field) < 0) return luaT_error(L); struct tuple *tuple; if (field.type == MP_EXT && @@ -222,7 +222,7 @@ luamp_encode_call_16(lua_State *L, struct luaL_serializer *cfg, * Inspect the first result */ struct luaL_field root; - if (luaL_tofield(L, cfg, NULL, 1, &root) < 0) + if (luaL_tofield(L, cfg, 0, NULL, 1, &root) < 0) return luaT_error(L); struct tuple *tuple; if (root.type == MP_EXT && (tuple = luaT_istuple(L, 1)) != NULL) { @@ -252,7 +252,7 @@ luamp_encode_call_16(lua_State *L, struct luaL_serializer *cfg, for (uint32_t t = 1; t <= root.size; t++) { lua_rawgeti(L, 1, t); struct luaL_field field; - if (luaL_tofield(L, cfg, NULL, -1, &field) < 0) + if (luaL_tofield(L, cfg, 0, NULL, -1, &field) < 0) return luaT_error(L); if (field.type == MP_EXT && (tuple = luaT_istuple(L, -1))) { tuple_to_mpstream(tuple, stream); diff --git a/src/box/lua/execute.c b/src/box/lua/execute.c index 926a0a61c..76d271f5c 100644 --- a/src/box/lua/execute.c +++ b/src/box/lua/execute.c @@ -328,7 +328,7 @@ lua_sql_bind_decode(struct lua_State *L, struct sql_bind *bind, int idx, int i) bind->name = NULL; bind->name_len = 0; } - if (luaL_tofield(L, luaL_msgpack_default, NULL, -1, &field) < 0) + if (luaL_tofield(L, luaL_msgpack_default, 0, NULL, -1, &field) < 0) return -1; switch (field.type) { case MP_UINT: diff --git a/src/box/lua/serialize_lua.c b/src/box/lua/serialize_lua.c index caa08a60f..fe858d59c 100644 --- a/src/box/lua/serialize_lua.c +++ b/src/box/lua/serialize_lua.c @@ -215,7 +215,7 @@ trace_node(struct lua_dumper *d) struct luaL_field field; memset(&field, 0, sizeof(field)); - luaL_checkfield(d->L, d->cfg, lua_gettop(d->L), &field); + luaL_checkfield(d->L, d->cfg, 0, lua_gettop(d->L), &field); if (field.type < lengthof(mp_type_names)) { if (field.type == MP_EXT) { @@ -234,7 +234,7 @@ trace_node(struct lua_dumper *d) memset(&field, 0, sizeof(field)); - luaL_checkfield(d->L, d->cfg, top, &field); + luaL_checkfield(d->L, d->cfg, 0, top, &field); say_info("serializer-trace: node :\tfield type %s (%d)", type_str, field.type); } @@ -340,28 +340,47 @@ static const char * get_lua_anchor(struct lua_dumper *d) { const char *s = ""; - lua_pushvalue(d->L, -1); lua_rawget(d->L, d->anchortable_index); - if (!lua_toboolean(d->L, -1)) { + if (lua_isnil(d->L, -1) == 1) { lua_pop(d->L, 1); return NULL; } + lua_pushstring(d->L, "before"); + lua_rawget(d->L, -2); + const char *str = NULL; + if (lua_type(d->L, -1) == LUA_TSTRING) { + str = lua_tostring(d->L, -1); + } + lua_Integer number_before = lua_tointeger(d->L, -1); + lua_pop(d->L, 1); - if (lua_isboolean(d->L, -1)) { - /* - * This element is referenced more - * than once but has not been named. - */ + lua_pushstring(d->L, "after"); + lua_rawget(d->L, -2); + if (lua_type(d->L, -1) == LUA_TSTRING) + str = lua_tostring(d->L, -1); + lua_Integer number_after = lua_tointeger(d->L, -1); + lua_pop(d->L, 1); + + if ((number_before > 1 || number_after > 1) && str == NULL) { + /* this element is referenced more than once but has not been named */ char buf[32]; snprintf(buf, sizeof(buf), "%u", d->anchor_number++); lua_pop(d->L, 1); lua_pushvalue(d->L, -1); + + lua_pushvalue(d->L, -1); + lua_rawget(d->L, d->anchortable_index); + if (number_before > 1) + lua_pushstring(d->L, "before"); + else + lua_pushstring(d->L, "after"); lua_pushstring(d->L, buf); s = lua_tostring(d->L, -1); + lua_rawset(d->L, -3); lua_rawset(d->L, d->anchortable_index); trace_anchor(s, false); - } else { + } else if (str != NULL) { /* * An aliased element. * @@ -372,6 +391,9 @@ get_lua_anchor(struct lua_dumper *d) const char *str = lua_tostring(d->L, -1); trace_anchor(str, true); lua_pop(d->L, 1); + } else { + lua_pop(d->L, 1); + return NULL; } return s; } @@ -783,7 +805,7 @@ dump_node(struct lua_dumper *d, struct node *nd, int indent) return -1; memset(field, 0, sizeof(*field)); - luaL_checkfield(d->L, d->cfg, lua_gettop(d->L), field); + luaL_checkfield(d->L, d->cfg, 0, lua_gettop(d->L), field); switch (field->type) { case MP_NIL: @@ -881,49 +903,6 @@ dump_node(struct lua_dumper *d, struct node *nd, int indent) return emit_node(d, nd, indent, str, len); } -/** - * Find references to tables, we use it - * to find self references in tables. - */ -static void -find_references(struct lua_dumper *d) -{ - int newval; - - if (lua_type(d->L, -1) != LUA_TTABLE) - return; - - /* Copy of a table for self refs */ - lua_pushvalue(d->L, -1); - lua_rawget(d->L, d->anchortable_index); - if (lua_isnil(d->L, -1)) - newval = 0; - else if (!lua_toboolean(d->L, -1)) - newval = 1; - else - newval = -1; - lua_pop(d->L, 1); - - if (newval != -1) { - lua_pushvalue(d->L, -1); - lua_pushboolean(d->L, newval); - lua_rawset(d->L, d->anchortable_index); - } - - if (newval != 0) - return; - - /* - * Other values and keys in the table - */ - lua_pushnil(d->L); - while (lua_next(d->L, -2) != 0) { - find_references(d); - lua_pop(d->L, 1); - find_references(d); - } -} - /** * Dump recursively from the root node. */ @@ -935,7 +914,7 @@ dump_root(struct lua_dumper *d) }; int ret; - luaL_checkfield(d->L, d->cfg, lua_gettop(d->L), &nd.field); + luaL_checkfield(d->L, d->cfg, 0, lua_gettop(d->L), &nd.field); if (nd.field.type != MP_ARRAY || nd.field.size != 1) { d->err = EINVAL; @@ -984,7 +963,7 @@ lua_encode(lua_State *L, struct luaL_serializer *serializer, /* Push copy of arg we're processing */ lua_pushvalue(L, 1); - find_references(&dumper); + find_references(dumper.L, dumper.anchortable_index, "before"); if (dump_root(&dumper) != 0) goto out; diff --git a/src/box/lua/tuple.c b/src/box/lua/tuple.c index 3e6f043b4..4d417ec34 100644 --- a/src/box/lua/tuple.c +++ b/src/box/lua/tuple.c @@ -411,7 +411,7 @@ luamp_convert_key(struct lua_State *L, struct luaL_serializer *cfg, return tuple_to_mpstream(tuple, stream); struct luaL_field field; - if (luaL_tofield(L, cfg, NULL, index, &field) < 0) + if (luaL_tofield(L, cfg, 0, NULL, index, &field) < 0) luaT_error(L); if (field.type == MP_ARRAY) { lua_pushvalue(L, index); diff --git a/src/box/sql/func.c b/src/box/sql/func.c index f15d27051..d4a7064f0 100644 --- a/src/box/sql/func.c +++ b/src/box/sql/func.c @@ -262,7 +262,7 @@ port_lua_get_vdbemem(struct port *base, uint32_t *size) return NULL; for (int i = 0; i < argc; i++) { struct luaL_field field; - if (luaL_tofield(L, luaL_msgpack_default, + if (luaL_tofield(L, luaL_msgpack_default, 0, NULL, -1 - i, &field) < 0) { goto error; } diff --git a/src/lua/msgpack.c b/src/lua/msgpack.c index 24b0d2ccd..412736572 100644 --- a/src/lua/msgpack.c +++ b/src/lua/msgpack.c @@ -156,11 +156,11 @@ restart: /* used by MP_EXT of unidentified subtype */ lua_pushnil(L); /* first key */ while (lua_next(L, top) != 0) { lua_pushvalue(L, -2); /* push a copy of key to top */ - if (luaL_tofield(L, cfg, opts, lua_gettop(L), field) < 0) + if (luaL_tofield(L, cfg, 0, opts, lua_gettop(L), field) < 0) return luaT_error(L); luamp_encode_r(L, cfg, opts, stream, field, level + 1); lua_pop(L, 1); /* pop a copy of key */ - if (luaL_tofield(L, cfg, opts, lua_gettop(L), field) < 0) + if (luaL_tofield(L, cfg, 0, opts, lua_gettop(L), field) < 0) return luaT_error(L); luamp_encode_r(L, cfg, opts, stream, field, level + 1); lua_pop(L, 1); /* pop value */ @@ -181,7 +181,7 @@ restart: /* used by MP_EXT of unidentified subtype */ mpstream_encode_array(stream, size); for (uint32_t i = 0; i < size; i++) { lua_rawgeti(L, top, i + 1); - if (luaL_tofield(L, cfg, opts, top + 1, field) < 0) + if (luaL_tofield(L, cfg, 0, opts, top + 1, field) < 0) return luaT_error(L); luamp_encode_r(L, cfg, opts, stream, field, level + 1); lua_pop(L, 1); @@ -204,7 +204,7 @@ restart: /* used by MP_EXT of unidentified subtype */ if (type != MP_EXT) return type; /* Value has been packed by the trigger */ /* Try to convert value to serializable type */ - luaL_convertfield(L, cfg, top, field); + luaL_convertfield(L, cfg, 0, top, field); /* handled by luaL_convertfield */ assert(field->type != MP_EXT); assert(lua_gettop(L) == top); @@ -229,7 +229,7 @@ luamp_encode(struct lua_State *L, struct luaL_serializer *cfg, } struct luaL_field field; - if (luaL_tofield(L, cfg, opts, lua_gettop(L), &field) < 0) + if (luaL_tofield(L, cfg, 0, opts, lua_gettop(L), &field) < 0) return luaT_error(L); enum mp_type top_type = luamp_encode_r(L, cfg, opts, stream, &field, 0); diff --git a/src/lua/pickle.c b/src/lua/pickle.c index 65208b5b3..86f0f77e8 100644 --- a/src/lua/pickle.c +++ b/src/lua/pickle.c @@ -80,7 +80,7 @@ lbox_pack(struct lua_State *L) if (i > nargs) luaL_error(L, "pickle.pack: argument count does not match " "the format"); - luaL_checkfield(L, luaL_msgpack_default, i, &field); + luaL_checkfield(L, luaL_msgpack_default, 0, i, &field); switch (*format) { case 'B': case 'b': diff --git a/src/lua/utils.c b/src/lua/utils.c index b5a6ca5b7..da4cfaebc 100644 --- a/src/lua/utils.c +++ b/src/lua/utils.c @@ -439,13 +439,61 @@ lua_gettable_wrapper(lua_State *L) return 1; } +/** + * Check if node with @a idx serialized. + */ +static int +is_serialized(struct lua_State *L, int anchortable_index, int idx) +{ + if (anchortable_index == 0) + return 0; + lua_pushvalue(L, idx); + lua_rawget(L, anchortable_index); + int is_serialized = 0; + if (lua_isnil(L, -1) == 0) { + lua_pushstring(L, "serialized"); + lua_rawget(L, -2); + is_serialized = lua_toboolean(L, -1); + lua_pop(L, 1); + } + lua_pop(L, 1); + return is_serialized; +} + +/** + * Mark node with @a idx as serialized. + */ +static void +mark_as_serialized(struct lua_State *L, int anchortable_index, int idx) +{ + if (anchortable_index == 0) + return; + /* Push copy of table. */ + lua_pushvalue(L, idx); + lua_pushvalue(L, idx); + lua_rawget(L, anchortable_index); + if (lua_isnil(L, -1) == 1) { + lua_pop(L, 1); + lua_newtable(L); + } + int flags_index = lua_gettop(L); + lua_pushstring(L, "serialized"); + lua_pushboolean(L, true); + lua_rawset(L, flags_index); + lua_rawset(L, anchortable_index); +} + static void lua_field_inspect_ucdata(struct lua_State *L, struct luaL_serializer *cfg, - int idx, struct luaL_field *field) + int anchortable_index, int idx, + struct luaL_field *field) { if (!cfg->encode_load_metatables) return; + if (is_serialized(L, anchortable_index, idx) == 1) + return; + /* * Try to call LUAL_SERIALIZE method on udata/cdata * LuaJIT specific: lua_getfield/lua_gettable raises exception on @@ -454,6 +502,7 @@ lua_field_inspect_ucdata(struct lua_State *L, struct luaL_serializer *cfg, int top = lua_gettop(L); lua_pushcfunction(L, lua_gettable_wrapper); lua_pushvalue(L, idx); + mark_as_serialized(L, anchortable_index, idx); lua_pushliteral(L, LUAL_SERIALIZE); if (lua_pcall(L, 2, 1, 0) == 0 && !lua_isnil(L, -1)) { if (!lua_isfunction(L, -1)) @@ -461,14 +510,85 @@ lua_field_inspect_ucdata(struct lua_State *L, struct luaL_serializer *cfg, /* copy object itself */ lua_pushvalue(L, idx); lua_pcall(L, 1, 1, 0); + find_references(L, anchortable_index, "after"); /* replace obj with the unpacked value */ lua_replace(L, idx); - if (luaL_tofield(L, cfg, NULL, idx, field) < 0) + if (luaL_tofield(L, cfg, 0, NULL, idx, field) < 0) luaT_error(L); } /* else ignore lua_gettable exceptions */ lua_settop(L, top); /* remove temporary objects */ } +void +find_references(struct lua_State *L, int anchortable_index, const char *mode) +{ + int type = lua_type(L, -1); + if (type != LUA_TTABLE || anchortable_index == 0) + return; + + int node_index = lua_gettop(L); + lua_pushvalue(L, node_index); + + /* Get value with flags from anchor table. */ + lua_pushvalue(L, node_index); + lua_rawget(L, anchortable_index); + if (lua_isnil(L, -1) == 1) { + lua_pop(L, 1); + lua_newtable(L); + } + int flags_index = lua_gettop(L); + + /* Get and increment counter. */ + lua_pushstring(L, mode); + lua_rawget(L, flags_index); + + int new_count = 0; + if (lua_isnil(L, -1) == 1) { + new_count = 1; + lua_pop(L, 1); + } else { + lua_Integer number = lua_tointeger(L, -1); + lua_pop(L, 1); + if (number < 2) + new_count = ++number; + else + new_count = -1; + } + + /* Push new count to value with flags. */ + if (new_count == -1) { + lua_pop(L, 2); + assert(node_index == lua_gettop(L)); + return; + } + lua_pushstring(L, mode); + lua_pushinteger(L, new_count); + lua_rawset(L, flags_index); + + /* Check if node is already serialized. */ + bool already_serialized = false; + if (strcmp(mode, "after") == 0) { + lua_pushstring(L, "serialized"); + lua_rawget(L, flags_index); + already_serialized = lua_toboolean(L, -1); + lua_pop(L, 1); + } + lua_rawset(L, anchortable_index); + assert(node_index == lua_gettop(L)); + if (already_serialized == true) + return; + + /* Recursively process other table values. */ + lua_pushnil(L); + while (lua_next(L, -2) != 0) { + /* Find references on value. */ + find_references(L, anchortable_index, mode); + lua_pop(L, 1); + /* Find references on key. */ + find_references(L, anchortable_index, mode); + } +} + /** * Call __serialize method of a table object by index * if the former exists. @@ -496,11 +616,13 @@ lua_field_inspect_ucdata(struct lua_State *L, struct luaL_serializer *cfg, * proceed with default table encoding. */ static int -lua_field_try_serialize(struct lua_State *L, struct luaL_serializer *cfg, - int idx, struct luaL_field *field) +lua_field_try_serialize(struct lua_State *L, int anchortable_index, + struct luaL_serializer *cfg, int idx, + struct luaL_field *field) { if (luaL_getmetafield(L, idx, LUAL_SERIALIZE) == 0) return 1; + mark_as_serialized(L, anchortable_index, idx); if (lua_isfunction(L, -1)) { /* copy object itself */ lua_pushvalue(L, idx); @@ -508,7 +630,9 @@ lua_field_try_serialize(struct lua_State *L, struct luaL_serializer *cfg, diag_set(LuajitError, lua_tostring(L, -1)); return -1; } - if (luaL_tofield(L, cfg, NULL, -1, field) != 0) + find_references(L, anchortable_index, "after"); + if (luaL_tofield(L, cfg, anchortable_index, NULL, -1, + field) != 0) return -1; lua_replace(L, idx); return 0; @@ -541,16 +665,19 @@ lua_field_try_serialize(struct lua_State *L, struct luaL_serializer *cfg, } static int -lua_field_inspect_table(struct lua_State *L, struct luaL_serializer *cfg, - int idx, struct luaL_field *field) +lua_field_inspect_table(struct lua_State *L, int anchortable_index, + struct luaL_serializer *cfg, int idx, + struct luaL_field *field) { assert(lua_type(L, idx) == LUA_TTABLE); uint32_t size = 0; uint32_t max = 0; - if (cfg->encode_load_metatables) { + if (cfg->encode_load_metatables && + is_serialized(L, anchortable_index, idx) != 1) { int top = lua_gettop(L); - int res = lua_field_try_serialize(L, cfg, idx, field); + int res = lua_field_try_serialize(L, anchortable_index, cfg, + idx, field); if (res == -1) return -1; assert(lua_gettop(L) == top); @@ -612,14 +739,14 @@ lua_field_tostring(struct lua_State *L, struct luaL_serializer *cfg, int idx, lua_call(L, 1, 1); lua_replace(L, idx); lua_settop(L, top); - if (luaL_tofield(L, cfg, NULL, idx, field) < 0) + if (luaL_tofield(L, cfg, 0, NULL, idx, field) < 0) luaT_error(L); } int luaL_tofield(struct lua_State *L, struct luaL_serializer *cfg, - const struct serializer_opts *opts, int index, - struct luaL_field *field) + int anchortable_index, const struct serializer_opts *opts, + int index, struct luaL_field *field) { if (index < 0) index = lua_gettop(L) + index + 1; @@ -753,7 +880,8 @@ luaL_tofield(struct lua_State *L, struct luaL_serializer *cfg, case LUA_TTABLE: { field->compact = false; - return lua_field_inspect_table(L, cfg, index, field); + return lua_field_inspect_table(L, anchortable_index, cfg, + index, field); } case LUA_TLIGHTUSERDATA: case LUA_TUSERDATA: @@ -773,8 +901,8 @@ luaL_tofield(struct lua_State *L, struct luaL_serializer *cfg, } void -luaL_convertfield(struct lua_State *L, struct luaL_serializer *cfg, int idx, - struct luaL_field *field) +luaL_convertfield(struct lua_State *L, struct luaL_serializer *cfg, + int anchortable_index, int idx, struct luaL_field *field) { if (idx < 0) idx = lua_gettop(L) + idx + 1; @@ -789,9 +917,12 @@ luaL_convertfield(struct lua_State *L, struct luaL_serializer *cfg, int idx, */ GCcdata *cd = cdataV(L->base + idx - 1); if (cd->ctypeid > CTID_CTYPEID) - lua_field_inspect_ucdata(L, cfg, idx, field); + lua_field_inspect_ucdata(L, cfg, + anchortable_index, idx, + field); } else if (type == LUA_TUSERDATA) { - lua_field_inspect_ucdata(L, cfg, idx, field); + lua_field_inspect_ucdata(L, cfg, anchortable_index, idx, + field); } } diff --git a/src/lua/utils.h b/src/lua/utils.h index 37531676d..529e7474b 100644 --- a/src/lua/utils.h +++ b/src/lua/utils.h @@ -354,6 +354,17 @@ struct luaL_field { bool compact; /* a flag used by YAML serializer */ }; +/** + * Find references to tables to look up self references in tables. + * + * @param L Lua stack. + * @param anchortable_index Index of anchor table on stack. + * @param mode When the function is called before or after dump + function. + */ +void +find_references(struct lua_State *L, int anchortable_index, const char *mode); + /** * @brief Convert a value from the Lua stack to a lua_field structure. * This function is designed for use with Lua bindings and data @@ -388,6 +399,7 @@ struct luaL_field { * * @param L stack * @param cfg configuration + * @param anchortable_index Index of anchor table on stack. * @param opts the Lua serializer additional options. * @param index stack index * @param field conversion result @@ -397,8 +409,8 @@ struct luaL_field { */ int luaL_tofield(struct lua_State *L, struct luaL_serializer *cfg, - const struct serializer_opts *opts, int index, - struct luaL_field *field); + int anchortable_index, const struct serializer_opts *opts, + int index, struct luaL_field *field); /** * @brief Try to convert userdata/cdata values using defined conversion logic. @@ -406,18 +418,21 @@ luaL_tofield(struct lua_State *L, struct luaL_serializer *cfg, * * @param L stack * @param cfg configuration + * @param anchortable_index Index of anchor table on stack. * @param idx stack index * @param field conversion result */ void -luaL_convertfield(struct lua_State *L, struct luaL_serializer *cfg, int idx, - struct luaL_field *field); +luaL_convertfield(struct lua_State *L, struct luaL_serializer *cfg, + int anchortable_index, int idx, struct luaL_field *field); /** * @brief A wrapper for luaL_tofield() and luaL_convertfield() that * tries to convert value or raise an error. * @param L stack * @param cfg configuration + * @param anchortable_index Stack index of table with node visit + and serialization info. * @param idx stack index * @param field conversion result * @sa lua_tofield() @@ -433,14 +448,14 @@ luaL_convertfield(struct lua_State *L, struct luaL_serializer *cfg, int idx, * (tostring) -> (nil) -> exception */ static inline void -luaL_checkfield(struct lua_State *L, struct luaL_serializer *cfg, int idx, - struct luaL_field *field) +luaL_checkfield(struct lua_State *L, struct luaL_serializer *cfg, + int anchortable_index, int idx, struct luaL_field *field) { - if (luaL_tofield(L, cfg, NULL, idx, field) < 0) + if (luaL_tofield(L, cfg, anchortable_index, NULL, idx, field) < 0) luaT_error(L); if (field->type != MP_EXT || field->ext_type != MP_UNKNOWN_EXTENSION) return; - luaL_convertfield(L, cfg, idx, field); + luaL_convertfield(L, cfg, anchortable_index, idx, field); } void diff --git a/test/app/gh-3228-serializer-look-for-recursion.result b/test/app/gh-3228-serializer-look-for-recursion.result new file mode 100644 index 000000000..773470c8e --- /dev/null +++ b/test/app/gh-3228-serializer-look-for-recursion.result @@ -0,0 +1,67 @@ +-- test-run result file version 2 +test_run = require('test_run').new() + | --- + | ... + +-- +-- gh-3228: Check the error message in the case of a __serialize +-- function generating infinite recursion. +-- +setmetatable({}, {__serialize = function(a) return a end}) + | --- + | - [] + | ... +setmetatable({}, {__serialize = function(a) return {a} end}) + | --- + | - - [] + | ... +setmetatable({}, {__serialize = function(a) return {a, a} end}) + | --- + | - - &0 [] + | - *0 + | ... +setmetatable({}, {__serialize = function(a) return {a, a, a} end}) + | --- + | - - &0 [] + | - *0 + | - *0 + | ... +setmetatable({}, {__serialize = function(a) return {{a, a}, a} end}) + | --- + | - - - &0 [] + | - *0 + | - *0 + | ... +setmetatable({}, {__serialize = function(a) return {a, 1} end}) + | --- + | - - [] + | - 1 + | ... +setmetatable({}, {__serialize = function(a) return {{{{a}}}} end}) + | --- + | - - - - - [] + | ... +setmetatable({}, {__serialize = function(a) return {{{{1}}}} end}) + | --- + | - - - - - 1 + | ... +b = {} + | --- + | ... +setmetatable({b}, {__serialize = function(a) return {a_1 = a, a_2 = a, b_1 = b, b_2 = b} end}) + | --- + | - b_2: &0 [] + | a_2: &1 + | - *0 + | a_1: *1 + | b_1: *0 + | ... +setmetatable({b}, {__serialize = function(a) return {a_1 = a, a_2 = {a, b}, b = b} end}) + | --- + | - b: &0 [] + | a_2: + | - &1 + | - *0 + | - *0 + | a_1: *1 + | ... diff --git a/test/app/gh-3228-serializer-look-for-recursion.test.lua b/test/app/gh-3228-serializer-look-for-recursion.test.lua new file mode 100644 index 000000000..1c9b0375f --- /dev/null +++ b/test/app/gh-3228-serializer-look-for-recursion.test.lua @@ -0,0 +1,17 @@ +test_run = require('test_run').new() + +-- +-- gh-3228: Check the error message in the case of a __serialize +-- function generating infinite recursion. +-- +setmetatable({}, {__serialize = function(a) return a end}) +setmetatable({}, {__serialize = function(a) return {a} end}) +setmetatable({}, {__serialize = function(a) return {a, a} end}) +setmetatable({}, {__serialize = function(a) return {a, a, a} end}) +setmetatable({}, {__serialize = function(a) return {{a, a}, a} end}) +setmetatable({}, {__serialize = function(a) return {a, 1} end}) +setmetatable({}, {__serialize = function(a) return {{{{a}}}} end}) +setmetatable({}, {__serialize = function(a) return {{{{1}}}} end}) +b = {} +setmetatable({b}, {__serialize = function(a) return {a_1 = a, a_2 = a, b_1 = b, b_2 = b} end}) +setmetatable({b}, {__serialize = function(a) return {a_1 = a, a_2 = {a, b}, b = b} end}) diff --git a/test/swim/swim.result b/test/swim/swim.result index bfc83c143..539131677 100644 --- a/test/swim/swim.result +++ b/test/swim/swim.result @@ -1322,16 +1322,13 @@ m_list incarnation: cdata {generation = 0ULL, version = 1ULL} uuid: 00000000-0000-1000-8000-000000000002 payload_size: 0 - - uri: 127.0.0.1: - status: alive - incarnation: cdata {generation = 0ULL, version = 2ULL} - uuid: 00000000-0000-1000-8000-000000000001 - payload_size: 8 - - uri: 127.0.0.1: + - &0 + uri: 127.0.0.1: status: alive incarnation: cdata {generation = 0ULL, version = 2ULL} uuid: 00000000-0000-1000-8000-000000000001 payload_size: 8 + - *0 ... e_list --- @@ -1374,16 +1371,13 @@ fiber.sleep(0) -- Two events - status update to 'left', and 'drop'. m_list --- -- - uri: 127.0.0.1: - status: left - incarnation: cdata {generation = 0ULL, version = 1ULL} - uuid: 00000000-0000-1000-8000-000000000002 - payload_size: 0 - - uri: 127.0.0.1: +- - &0 + uri: 127.0.0.1: status: left incarnation: cdata {generation = 0ULL, version = 1ULL} uuid: 00000000-0000-1000-8000-000000000002 payload_size: 0 + - *0 ... e_list --- diff --git a/third_party/lua-cjson/lua_cjson.c b/third_party/lua-cjson/lua_cjson.c index 38e999870..67a9762bc 100644 --- a/third_party/lua-cjson/lua_cjson.c +++ b/third_party/lua-cjson/lua_cjson.c @@ -354,7 +354,7 @@ static void json_append_object(lua_State *l, struct luaL_serializer *cfg, comma = 1; struct luaL_field field; - luaL_checkfield(l, cfg, -2, &field); + luaL_checkfield(l, cfg, 0, -2, &field); if (field.type == MP_UINT) { strbuf_append_char(json, '"'); json_append_uint(cfg, json, field.ival); @@ -384,7 +384,7 @@ static void json_append_data(lua_State *l, struct luaL_serializer *cfg, int current_depth, strbuf_t *json) { struct luaL_field field; - luaL_checkfield(l, cfg, -1, &field); + luaL_checkfield(l, cfg, 0, -1, &field); switch (field.type) { case MP_UINT: return json_append_uint(cfg, json, field.ival); diff --git a/third_party/lua-yaml/lyaml.cc b/third_party/lua-yaml/lyaml.cc index 9c3a4a646..1643193d5 100644 --- a/third_party/lua-yaml/lyaml.cc +++ b/third_party/lua-yaml/lyaml.cc @@ -503,35 +503,63 @@ static yaml_char_t *get_yaml_anchor(struct lua_yaml_dumper *dumper) { const char *s = ""; lua_pushvalue(dumper->L, -1); lua_rawget(dumper->L, dumper->anchortable_index); - if (!lua_toboolean(dumper->L, -1)) { + if (lua_isnil(dumper->L, -1) == 1) { lua_pop(dumper->L, 1); return NULL; } + lua_pushstring(dumper->L, "before"); + lua_rawget(dumper->L, -2); + const char *str = NULL; + if (lua_type(dumper->L, -1) == LUA_TSTRING) { + str = lua_tostring(dumper->L, -1); + } + lua_Integer number_before = lua_tointeger(dumper->L, -1); + lua_pop(dumper->L, 1); + + lua_pushstring(dumper->L, "after"); + lua_rawget(dumper->L, -2); + if (lua_type(dumper->L, -1) == LUA_TSTRING) { + str = lua_tostring(dumper->L, -1); + } + lua_Integer number_after = lua_tointeger(dumper->L, -1); + lua_pop(dumper->L, 1); - if (lua_isboolean(dumper->L, -1)) { + if ((number_before > 1 || number_after > 1) && str == NULL) { /* this element is referenced more than once but has not been named */ char buf[32]; snprintf(buf, sizeof(buf), "%u", dumper->anchor_number++); lua_pop(dumper->L, 1); lua_pushvalue(dumper->L, -1); + + lua_pushvalue(dumper->L, -1); + lua_rawget(dumper->L, dumper->anchortable_index); + if (number_before > 1) + lua_pushstring(dumper->L, "before"); + else + lua_pushstring(dumper->L, "after"); lua_pushstring(dumper->L, buf); s = lua_tostring(dumper->L, -1); + lua_rawset(dumper->L, -3); lua_rawset(dumper->L, dumper->anchortable_index); - } else { + } else if (str != NULL) { /* this is an aliased element */ yaml_event_t ev; - const char *str = lua_tostring(dumper->L, -1); if (!yaml_alias_event_initialize(&ev, (yaml_char_t *) str) || !yaml_emitter_emit(&dumper->emitter, &ev)) luaL_error(dumper->L, OOM_ERRMSG); lua_pop(dumper->L, 1); + } else { + lua_pop(dumper->L, 1); + return NULL; } return (yaml_char_t *)s; } -static int dump_table(struct lua_yaml_dumper *dumper, struct luaL_field *field){ +static int dump_table(struct lua_yaml_dumper *dumper, struct luaL_field *field, + yaml_char_t *anchor) { yaml_event_t ev; - yaml_char_t *anchor = get_yaml_anchor(dumper); + if (anchor == NULL) + anchor = get_yaml_anchor(dumper); if (anchor && !*anchor) return 1; @@ -556,10 +584,12 @@ static int dump_table(struct lua_yaml_dumper *dumper, struct luaL_field *field){ yaml_emitter_emit(&dumper->emitter, &ev) != 0 ? 1 : 0; } -static int dump_array(struct lua_yaml_dumper *dumper, struct luaL_field *field){ +static int dump_array(struct lua_yaml_dumper *dumper, struct luaL_field *field, + yaml_char_t *anchor) { unsigned i; yaml_event_t ev; - yaml_char_t *anchor = get_yaml_anchor(dumper); + if (anchor == NULL) + anchor = get_yaml_anchor(dumper); if (anchor && !*anchor) return 1; @@ -621,8 +651,18 @@ static int dump_node(struct lua_yaml_dumper *dumper) bool unused; (void) unused; + yaml_char_t *anchor = NULL; + + /* + * Keep anchor to print it if luaL_checkfield() push new value + * on stack after serialization. + */ + if (lua_istable(dumper->L, -1) == 1) + anchor = get_yaml_anchor(dumper); + int top = lua_gettop(dumper->L); - luaL_checkfield(dumper->L, dumper->cfg, top, &field); + luaL_checkfield(dumper->L, dumper->cfg, dumper->anchortable_index, top, + &field); switch(field.type) { case MP_UINT: snprintf(buf, sizeof(buf) - 1, "%" PRIu64, field.ival); @@ -647,9 +687,9 @@ static int dump_node(struct lua_yaml_dumper *dumper) len = strlen(buf); break; case MP_ARRAY: - return dump_array(dumper, &field); + return dump_array(dumper, &field, anchor); case MP_MAP: - return dump_table(dumper, &field); + return dump_table(dumper, &field, anchor); case MP_STR: str = lua_tolstring(dumper->L, -1, &len); if (yaml_is_null(str, len) || yaml_is_bool(str, len, &unused) || @@ -745,35 +785,6 @@ static int append_output(void *arg, unsigned char *buf, size_t len) { return 1; } -static void find_references(struct lua_yaml_dumper *dumper) { - int newval = -1, type = lua_type(dumper->L, -1); - if (type != LUA_TTABLE) - return; - - lua_pushvalue(dumper->L, -1); /* push copy of table */ - lua_rawget(dumper->L, dumper->anchortable_index); - if (lua_isnil(dumper->L, -1)) - newval = 0; - else if (!lua_toboolean(dumper->L, -1)) - newval = 1; - lua_pop(dumper->L, 1); - if (newval != -1) { - lua_pushvalue(dumper->L, -1); - lua_pushboolean(dumper->L, newval); - lua_rawset(dumper->L, dumper->anchortable_index); - } - if (newval) - return; - - /* recursively process other table values */ - lua_pushnil(dumper->L); - while (lua_next(dumper->L, -2) != 0) { - find_references(dumper); /* find references on value */ - lua_pop(dumper->L, 1); - find_references(dumper); /* find references on key */ - } -} - int lua_yaml_encode(lua_State *L, struct luaL_serializer *serializer, const char *tag_handle, const char *tag_prefix) @@ -819,7 +830,7 @@ lua_yaml_encode(lua_State *L, struct luaL_serializer *serializer, dumper.anchortable_index = lua_gettop(L); dumper.anchor_number = 0; lua_pushvalue(L, 1); /* push copy of arg we're processing */ - find_references(&dumper); + find_references(L, dumper.anchortable_index, "before"); dump_document(&dumper); if (dumper.error) goto error; -- 2.24.3 (Apple Git-128)