From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from smtp41.i.mail.ru (smtp41.i.mail.ru [94.100.177.101]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by dev.tarantool.org (Postfix) with ESMTPS id 0BF8D469710 for ; Mon, 18 May 2020 15:18:41 +0300 (MSK) References: <20200512135052.221379-1-gorcunov@gmail.com> <20200512135052.221379-6-gorcunov@gmail.com> From: Oleg Babin Message-ID: <883cac83-0ce4-8bfb-6454-b14cfa7c6e65@tarantool.org> Date: Mon, 18 May 2020 15:18:40 +0300 MIME-Version: 1.0 In-Reply-To: <20200512135052.221379-6-gorcunov@gmail.com> Content-Type: text/plain; charset=utf-8; format=flowed Content-Language: en-GB Content-Transfer-Encoding: 7bit Subject: Re: [Tarantool-patches] [PATCH 5/7] box/console: implement lua serializer List-Id: Tarantool development patches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , To: Cyrill Gorcunov , tml I've checked your changes. That's ok. There are several issues that are still relevant but I hope it will be done in the scope of separate patches. LGTM. On 12/05/2020 16:50, Cyrill Gorcunov wrote: > When we print output in console (especially in Lua mode) > it is impossible to find out the internal type an entry > represents, which in turn leads to inability to encode > "ULL" entries properly. Moreover we always introduce new > types (for example decimals, uuids) and the serpent module > we currently use for encodings has no clue about them. > But why does yaml serializer recognize such types and lua doesn't do that? > Thus lets implement own lua serializer similarly to the > yaml encoder. This allows us to properly detect every > field and encode it accordingly. > > Part-of #4682 > > Signed-off-by: Cyrill Gorcunov > --- > src/box/CMakeLists.txt | 1 + > src/box/lua/serialize_lua.c | 1059 +++++++++++++++++++++++++++++++++++ > src/box/lua/serialize_lua.h | 67 +++ > 3 files changed, 1127 insertions(+) > create mode 100644 src/box/lua/serialize_lua.c > create mode 100644 src/box/lua/serialize_lua.h > > diff --git a/src/box/CMakeLists.txt b/src/box/CMakeLists.txt > index c931ecdfe..08ffe7177 100644 > --- a/src/box/CMakeLists.txt > +++ b/src/box/CMakeLists.txt > @@ -148,6 +148,7 @@ add_library(box STATIC > lua/call.c > lua/cfg.cc > lua/console.c > + lua/serialize_lua.c > lua/tuple.c > lua/slab.c > lua/index.c > diff --git a/src/box/lua/serialize_lua.c b/src/box/lua/serialize_lua.c > new file mode 100644 > index 000000000..caa08a60f > --- /dev/null > +++ b/src/box/lua/serialize_lua.c > @@ -0,0 +1,1059 @@ > +/* > + * Copyright 2010-2020, Tarantool AUTHORS, please see AUTHORS file. > + * > + * Redistribution and use in source and binary forms, with or > + * without modification, are permitted provided that the following > + * conditions are met: > + * > + * 1. Redistributions of source code must retain the above > + * copyright notice, this list of conditions and the > + * following disclaimer. > + * > + * 2. Redistributions in binary form must reproduce the above > + * copyright notice, this list of conditions and the following > + * disclaimer in the documentation and/or other materials > + * provided with the distribution. > + * > + * THIS SOFTWARE IS PROVIDED BY ``AS IS'' AND > + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED > + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL > + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, > + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL > + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF > + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR > + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF > + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF > + * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF > + * SUCH DAMAGE. > + */ > + > +#include > +#include > + > +#include "trivia/util.h" > +#include "lua/utils.h" > +#include "say.h" > + > +#include "lib/core/decimal.h" > +#include "mp_extension_types.h" > +#include "uuid/tt_uuid.h" > + > +#include "lua-yaml/b64.h" > + > +#include "serialize_lua.h" > + > +#if 0 > +# define SERIALIZER_TRACE > +#endif > + > +/* Serializer for Lua output mode */ > +static struct luaL_serializer *serializer_lua; > + > +enum { > + NODE_NONE_BIT = 0, > + NODE_ROOT_BIT = 1, > + NODE_RAW_BIT = 2, > + NODE_LVALUE_BIT = 3, > + NODE_RVALUE_BIT = 4, > + NODE_MAP_KEY_BIT = 5, > + NODE_MAP_VALUE_BIT = 6, > + NODE_EMBRACE_BIT = 7, > + NODE_QUOTE_BIT = 8, > + > + NODE_MAX > +}; > + > +enum { > + NODE_NONE = (1u << NODE_NONE_BIT), > + NODE_ROOT = (1u << NODE_ROOT_BIT), > + NODE_RAW = (1u << NODE_RAW_BIT), > + NODE_LVALUE = (1u << NODE_LVALUE_BIT), > + NODE_RVALUE = (1u << NODE_RVALUE_BIT), > + NODE_MAP_KEY = (1u << NODE_MAP_KEY_BIT), > + NODE_MAP_VALUE = (1u << NODE_MAP_VALUE_BIT), > + NODE_EMBRACE = (1u << NODE_EMBRACE_BIT), > + NODE_QUOTE = (1u << NODE_QUOTE_BIT), > +}; > + > +struct node { > + /** Link with previous node or key */ > + union { > + struct node *prev; > + struct node *key; > + }; > + > + /** The field data we're paring */ > + struct luaL_field field; > + > + /** Node mask NODE_x */ > + int mask; > + > + /** Node index in a map */ > + int index; > +}; > + > +/** > + * Serializer context. > + */ > +struct lua_dumper { > + /** Lua state to fetch data from */ > + lua_State *L; > + > + /** General configure options */ > + struct luaL_serializer *cfg; > + > + /** Lua dumper options */ > + lua_dumper_opts_t *opts; > + > + /** Output state */ > + lua_State *outputL; > + /** Output buffer */ > + luaL_Buffer luabuf; > + > + /** Anchors for self references */ > + int anchortable_index; > + unsigned int anchor_number; > + > + /** Error message buffer */ > + char err_msg[256]; > + int err; > + > + /** Indentation buffer */ > + char indent_buf[256]; > + > + /** Output suffix */ > + char suffix_buf[32]; > + int suffix_len; > + > + /** Previous node mask */ > + int prev_nd_mask; > + > + /** Ignore indents */ > + bool noindent; > +}; > + > +#ifdef SERIALIZER_TRACE > + > +#define __gen_mp_name(__v) [__v] = # __v > +static const char *mp_type_names[] = { > + __gen_mp_name(MP_NIL), > + __gen_mp_name(MP_UINT), > + __gen_mp_name(MP_INT), > + __gen_mp_name(MP_STR), > + __gen_mp_name(MP_BIN), > + __gen_mp_name(MP_ARRAY), > + __gen_mp_name(MP_MAP), > + __gen_mp_name(MP_BOOL), > + __gen_mp_name(MP_FLOAT), > + __gen_mp_name(MP_DOUBLE), > + __gen_mp_name(MP_EXT), > +}; > + > +static const char *mp_ext_type_names[] = { > + __gen_mp_name(MP_DECIMAL), > + __gen_mp_name(MP_UUID), > + __gen_mp_name(MP_ERROR), > +}; > +#undef __gen_mp_name > + > +#define __gen_nd_name(__v) [__v ##_BIT] = # __v > +static const char *nd_type_names[] = { > + __gen_nd_name(NODE_NONE), > + __gen_nd_name(NODE_ROOT), > + __gen_nd_name(NODE_RAW), > + __gen_nd_name(NODE_LVALUE), > + __gen_nd_name(NODE_RVALUE), > + __gen_nd_name(NODE_MAP_KEY), > + __gen_nd_name(NODE_MAP_VALUE), > + __gen_nd_name(NODE_EMBRACE), > + __gen_nd_name(NODE_QUOTE), > +}; > +#undef __gen_nd_name > + > +static char * > +trace_nd_mask_str(unsigned int nd_mask) > +{ > + static char mask_str[256]; > + int left = sizeof(mask_str); > + int pos = 0; > + > + for (int i = 0; i < NODE_MAX; i++) { > + if (!(nd_mask & (1u << i))) > + continue; > + > + int nd_len = strlen(nd_type_names[i]); > + if (left >= nd_len + 1) { > + strcpy(&mask_str[pos], nd_type_names[i]); > + pos += nd_len; > + mask_str[pos++] = '|'; > + left = sizeof(mask_str) - pos; > + } > + } > + > + if (pos != 0) > + mask_str[--pos] = '\0'; > + else > + strcpy(mask_str, "UNKNOWN"); > + > + return mask_str; > +} > + > +static void > +trace_node(struct lua_dumper *d) > +{ > + int ltype = lua_type(d->L, -1); > + say_info("serializer-trace: node : lua type %d -> %s", > + ltype, lua_typename(d->L, ltype)); > + > + if (d->err != 0) > + return; > + > + char mp_type[64], *type_str = mp_type; > + int top = lua_gettop(d->L); > + struct luaL_field field; > + > + memset(&field, 0, sizeof(field)); > + luaL_checkfield(d->L, d->cfg, lua_gettop(d->L), &field); > + > + if (field.type < lengthof(mp_type_names)) { > + if (field.type == MP_EXT) { > + size_t max_ext = lengthof(mp_ext_type_names); > + snprintf(mp_type, sizeof(mp_type), "%s/%s", > + mp_type_names[field.type], > + field.ext_type < max_ext ? > + mp_ext_type_names[field.ext_type] : > + "UNKNOWN"); > + } else { > + type_str = (char *)mp_type_names[field.type]; > + } > + } else { > + type_str = "UNKNOWN"; > + } > + > + memset(&field, 0, sizeof(field)); > + > + luaL_checkfield(d->L, d->cfg, top, &field); > + say_info("serializer-trace: node :\tfield type %s (%d)", > + type_str, field.type); > +} > + > +static char * > +trace_string(const char *src, size_t len) > +{ > + static char buf[128]; > + size_t pos = 0; > + > + if (len > sizeof(buf)-1) > + len = sizeof(buf)-1; > + > + while (pos < len) { > + if (src[pos] == '\n') { > + buf[pos++] = '\\'; > + buf[pos++] = 'n'; > + continue; > + } > + buf[pos] = src[pos]; > + pos++; > + } > + buf[pos] = '\0'; > + return buf; > +} > + > +static void > +trace_emit(struct lua_dumper *d, int nd_mask, int indent, > + const char *str, size_t len) > +{ > + if (d->suffix_len) { > + say_info("serializer-trace: emit-sfx: \"%s\"", > + trace_string(d->suffix_buf, > + d->suffix_len)); > + } > + > + static_assert(NODE_MAX < sizeof(int) * 8, > + "NODE_MAX is too big"); > + > + char *names = trace_nd_mask_str(nd_mask); > + > + say_info("serializer-trace: emit : type %s (0x%x) " > + "indent %d val \"%s\" len %zu", > + names, nd_mask, indent, > + trace_string(str, len), len); > +} > + > +static void > +trace_anchor(const char *s, bool alias) > +{ > + say_info("serializer-trace: anchor : alias %d name %s", > + alias, s); > +} > + > +#else /* SERIALIZER_TRACE */ > + > +static void > +trace_node(struct lua_dumper *d) > +{ > + (void)d; > +} > + > +static void > +trace_emit(struct lua_dumper *d, int nd_mask, int indent, > + const char *str, size_t len) > +{ > + (void)d; > + (void)nd_mask; > + (void)indent; > + (void)str; > + (void)len; > +} > + > +static void > +trace_anchor(const char *s, bool alias) > +{ > + (void)s; > + (void)alias; > +} > + > +#endif /* SERIALIZER_TRACE */ > + > +static const char *lua_keywords[] = { > + "and", "break", "do", "else", > + "elseif", "end", "false", "for", > + "function", "if", "in", "local", > + "nil", "not", "or", "repeat", > + "return", "then", "true", "until", > + "while", "and", > +}; > + > +static int > +dump_node(struct lua_dumper *d, struct node *nd, int indent); > + > +static int > +emit_node(struct lua_dumper *d, struct node *nd, int indent, > + const char *str, size_t len); > + > +/** > + * Generate anchor numbers for self references. > + */ > +static const char * > +get_lua_anchor(struct lua_dumper *d) > +{ > + const char *s = ""; > + > + lua_pushvalue(d->L, -1); > + lua_rawget(d->L, d->anchortable_index); > + if (!lua_toboolean(d->L, -1)) { > + lua_pop(d->L, 1); > + return NULL; > + } > + > + if (lua_isboolean(d->L, -1)) { > + /* > + * This element is referenced more > + * than once but has not been named. > + */ > + char buf[32]; > + snprintf(buf, sizeof(buf), "%u", d->anchor_number++); > + lua_pop(d->L, 1); > + lua_pushvalue(d->L, -1); > + lua_pushstring(d->L, buf); > + s = lua_tostring(d->L, -1); > + lua_rawset(d->L, d->anchortable_index); > + trace_anchor(s, false); > + } else { > + /* > + * An aliased element. > + * > + * FIXME: Need an example to use. > + * > + * const char *str = lua_tostring(d->L, -1); > + */ > + const char *str = lua_tostring(d->L, -1); > + trace_anchor(str, true); > + lua_pop(d->L, 1); > + } > + return s; > +} > + > +static void > +suffix_append(struct lua_dumper *d, const char *str, int len) > +{ > + int left = (int)sizeof(d->suffix_buf) - d->suffix_len; > + if (len < left) { > + memcpy(&d->suffix_buf[d->suffix_len], str, len); > + d->suffix_len += len; > + d->suffix_buf[d->suffix_len] = '\0'; > + } > +} > + > +static inline void > +suffix_reset(struct lua_dumper *d) > +{ > + d->suffix_len = 0; > +} > + > +static void > +suffix_flush(struct lua_dumper *d) > +{ > + if (d->suffix_len) { > + luaL_addlstring(&d->luabuf, d->suffix_buf, d->suffix_len); > + suffix_reset(d); > + } > +} > + > +static int > +gen_indent(struct lua_dumper *d, int indent) > +{ > + static_assert(sizeof(d->indent_buf) > 0, > + "indent buffer is too small"); > + > + if (indent > 0 && d->opts->block_mode && !d->noindent) { > + snprintf(d->indent_buf, sizeof(d->indent_buf), > + "%*s", indent, ""); > + size_t len = strlen(d->indent_buf); > + d->indent_buf[len] = '\0'; > + return len; > + } > + > + return 0; > +} > + > +static void > +emit_hex_char(struct lua_dumper *d, unsigned char c) > +{ > + luaL_addchar(&d->luabuf, '\\'); > + luaL_addchar(&d->luabuf, 'x'); > + > +#define __emit_hex(v) \ > + do { \ > + if (v <= 9) \ > + luaL_addchar(&d->luabuf, '0' + v); \ > + else \ > + luaL_addchar(&d->luabuf, v - 10 + 'a'); \ > + } while (0) > + > + __emit_hex((c >> 4)); > + __emit_hex((c & 0xf)); > +#undef __emit_hex > +} > + > +/** > + * Emit the string with escapes if needed. > + * > + * FIXME: Probably should to revisit and make > + * sure we've not miss anything here (octal numbers > + * are missed for now and etc...). > + */ > +static void > +emit_string(struct lua_dumper *d, const char *str, size_t len) > +{ > + for (size_t i = 0; i < len; i++) { > + if ((str[i]) == '\'' || str[i] == '\"') { > + luaL_addchar(&d->luabuf, '\\'); > + luaL_addchar(&d->luabuf, str[i]); > + } else if (str[i] == '\0') { > + luaL_addchar(&d->luabuf, '\\'); > + luaL_addchar(&d->luabuf, '0'); > + } else if (str[i] == '\a') { > + luaL_addchar(&d->luabuf, '\\'); > + luaL_addchar(&d->luabuf, 'a'); > + } else if (str[i] == '\b') { > + luaL_addchar(&d->luabuf, '\\'); > + luaL_addchar(&d->luabuf, 'b'); > + } else if (str[i] == '\f') { > + luaL_addchar(&d->luabuf, '\\'); > + luaL_addchar(&d->luabuf, 'f'); > + } else if (str[i] == '\v') { > + luaL_addchar(&d->luabuf, '\\'); > + luaL_addchar(&d->luabuf, 'v'); > + } else if (str[i] == '\r') { > + luaL_addchar(&d->luabuf, '\\'); > + luaL_addchar(&d->luabuf, 'r'); > + } else if (str[i] == '\n') { > + luaL_addchar(&d->luabuf, '\\'); > + luaL_addchar(&d->luabuf, 'n'); > + } else if (str[i] == '\t') { > + luaL_addchar(&d->luabuf, '\\'); > + luaL_addchar(&d->luabuf, 't'); > + } else if (str[i] == '\xef') { > + if (i < len-1 && i < len-2 && > + str[i+1] == '\xbb' && > + str[i+2] == '\xbf') { > + emit_hex_char(d, 0xef); > + emit_hex_char(d, 0xbb); > + emit_hex_char(d, 0xbf); > + } else { > + emit_hex_char(d, str[i]); > + } > + } else if (isprint(str[i]) == 0) { > + emit_hex_char(d, str[i]); > + } else { > + luaL_addchar(&d->luabuf, str[i]); > + } > + } > +} > + > +/** > + * Emit value into output buffer. > + */ > +static void > +emit_value(struct lua_dumper *d, struct node *nd, > + int indent, const char *str, size_t len) > +{ > + trace_emit(d, nd->mask, indent, str, len); > + > + /* > + * There might be previous closing symbols > + * in the suffix queue. Since we're about > + * to emit new values don't forget to prepend > + * ending ones. > + */ > + suffix_flush(d); > + > + luaL_addlstring(&d->luabuf, d->indent_buf, > + gen_indent(d, indent)); > + > + if (nd->mask & NODE_EMBRACE) > + luaL_addlstring(&d->luabuf, "[", 1); > + if (nd->mask & NODE_QUOTE) > + luaL_addlstring(&d->luabuf, "\"", 1); > + > + if (nd->field.type == MP_STR) { > + emit_string(d, str, len); > + } else { > + luaL_addlstring(&d->luabuf, str, len); > + } > + > + if (nd->mask & NODE_QUOTE) > + luaL_addlstring(&d->luabuf, "\"", 1); > + if (nd->mask & NODE_EMBRACE) > + luaL_addlstring(&d->luabuf, "]", 1); > +} > + > +/** > + * Emit a raw string into output. > + */ > +static void > +emit_raw_value(struct lua_dumper *d, int indent, > + const char *str, size_t len) > +{ > + struct node node = { > + .mask = NODE_RAW, > + }; > + > + emit_value(d, &node, indent, str, len); > +} > + > +/** > + * Put an opening brace into the output. > + */ > +static int > +emit_brace_open(struct lua_dumper *d, int indent) > +{ > + if (d->opts->block_mode) { > + int _indent; > + if (d->noindent) > + _indent = 0; > + else > + _indent = indent; > + > + emit_raw_value(d, _indent, "{\n", 2); > + if (d->noindent && d->prev_nd_mask & NODE_LVALUE) > + d->noindent = false; > + } else { > + emit_raw_value(d, indent, "{", 1); > + } > + > + return indent + d->opts->indent_lvl; > +} > + > +/** > + * Put a closing brace into the output. > + */ > +static void > +emit_brace_close(struct lua_dumper *d, int indent) > +{ > + suffix_reset(d); > + > + if (d->opts->block_mode) > + emit_raw_value(d, 0, "\n", 1); > + > + indent -= d->opts->indent_lvl; > + emit_raw_value(d, indent, "}", 1); > + > + if (d->opts->block_mode) > + suffix_append(d, ",\n", 2); > + else > + suffix_append(d, ", ", 2); > +} > + > +/** > + * Handling self references. It is yaml specific > + * and I don't think we might even need it. Still > + * better to get noticed if something went in > + * an unexpected way. > + */ > +static bool > +emit_anchor(struct lua_dumper *d, struct node *nd, int indent) > +{ > + const char *anchor = get_lua_anchor(d); > + if (anchor && !*anchor) { > + emit_node(d, nd, indent, "nil", 3); > + return true; > + } > + return false; > +} > + > +/** > + * Dump an array entry. > + */ > +static void > +dump_array(struct lua_dumper *d, struct node *nd, int indent) > +{ > + indent = emit_brace_open(d, indent); > + if (emit_anchor(d, nd, indent)) > + goto out; > + > + for (int i = 0; i < (int)nd->field.size; i++) { > + lua_rawgeti(d->L, -1, i + 1); > + struct node node = { > + .prev = nd, > + .mask = NODE_RVALUE, > + }; > + dump_node(d, &node, indent); > + lua_pop(d->L, 1); > + } > +out: > + emit_brace_close(d, indent); > +} > + > +/** > + * Dump a map entry. > + */ > +static void > +dump_table(struct lua_dumper *d, struct node *nd, int indent) > +{ > + int index = 0; > + > + indent = emit_brace_open(d, indent); > + if (emit_anchor(d, nd, indent)) > + goto out; > + > + /* > + * In sake of speed we don't sort > + * keys but provide them as is. Thus > + * simply walk over keys and their > + * values. > + */ > + lua_pushnil(d->L); > + while (lua_next(d->L, -2)) { > + lua_pushvalue(d->L, -2); > + struct node node_key = { > + .prev = nd, > + .mask = NODE_LVALUE | NODE_MAP_KEY, > + .index = index++, > + }; > + dump_node(d, &node_key, indent); > + lua_pop(d->L, 1); > + > + struct node node_val = { > + .key = &node_key, > + .mask = NODE_RVALUE | NODE_MAP_VALUE, > + }; > + dump_node(d, &node_val, indent); > + lua_pop(d->L, 1); > + } > +out: > + emit_brace_close(d, indent); > +} > + > +/** > + * Figure out if we need to decorate a map key > + * with square braces and quotes or can leave > + * it as a plain value. > + */ > +static void > +decorate_key(struct node *nd, const char *str, size_t len) > +{ > + assert(nd->field.type == MP_STR); > + assert(nd->mask & NODE_MAP_KEY); > + > + /* > + * We might need to put string keys > + * to quotes and embrace them due to > + * limitation of how to declare map keys > + * (the output from serializer should be > + * parsable if pasted back to a console). > + */ > + for (size_t i = 0; i < lengthof(lua_keywords); i++) { > + const char *k = lua_keywords[i]; > + if (strcmp(k, str) == 0) { > + nd->mask |= NODE_EMBRACE | NODE_QUOTE; > + return; > + } > + } > + > + /* > + * Plain keys may be alphanumerics with underscopes. > + */ > + for (size_t i = 0; i < len; i++) { > + if (isalnum(str[i]) != 0 || str[i] == '_') > + continue; > + nd->mask |= NODE_EMBRACE | NODE_QUOTE; > + return; > + } > + > + nd->mask &= ~NODE_QUOTE; > +} > + > +static int > +emit_node(struct lua_dumper *d, struct node *nd, int indent, > + const char *str, size_t len) > +{ > + struct luaL_field *field = &nd->field; > + > + if (str == NULL) { > + d->prev_nd_mask = nd->mask; > + return 0; > + } > + > + if (nd->mask & NODE_MAP_KEY) { > + /* > + * In case if key is integer and matching > + * the current position in the table we > + * can simply skip it and print value only. > + */ > + if (nd->field.type == MP_INT || > + nd->field.type == MP_UINT) { > + if (nd->index == (int)field->ival) { > + d->noindent = false; > + return 0; > + } else { > + nd->mask |= NODE_EMBRACE; > + } > + } else if (nd->field.type == MP_STR) { > + decorate_key(nd, str, len); > + } > + } > + > + d->prev_nd_mask = nd->mask; > + emit_value(d, nd, indent, str, len); > + > + /* > + * In sake of speed we do not lookahead > + * for next lua nodes, instead just remember > + * closing symbol in suffix buffer which we > + * will flush on next emit. > + */ > + if (nd->mask & NODE_RVALUE) { > + if (d->opts->block_mode) > + suffix_append(d, ",\n", 2); > + else > + suffix_append(d, ", ", 2); > + d->noindent = false; > + } else if (nd->mask & NODE_LVALUE) { > + suffix_append(d, " = ", 3); > + d->noindent = true; > + } > + > + return 0; > +} > + > +/** > + * Dump a node. > + */ > +static int > +dump_node(struct lua_dumper *d, struct node *nd, int indent) > +{ > + struct luaL_field *field = &nd->field; > + char buf[FPCONV_G_FMT_BUFSIZE]; > + int ltype = lua_type(d->L, -1); > + const char *str = NULL; > + size_t len = 0; > + > + trace_node(d); > + > + /* > + * We can exit early if an error > + * already happened, no need to > + * continue parsing. > + */ > + if (d->err != 0) > + return -1; > + > + memset(field, 0, sizeof(*field)); > + luaL_checkfield(d->L, d->cfg, lua_gettop(d->L), field); > + > + switch (field->type) { > + case MP_NIL: > + if (ltype == LUA_TNIL) { > + static const char str_nil[] = "nil"; > + str = str_nil; > + len = strlen(str_nil); > + } else { > + static const char str_null[] = "box.NULL"; > + str = str_null; > + len = strlen(str_null); > + } > + break; > + case MP_UINT: > + snprintf(buf, sizeof(buf), "%" PRIu64, field->ival); > + len = strlen(buf); > + str = buf; > + break; > + case MP_INT: > + snprintf(buf, sizeof(buf), "%" PRIi64, field->ival); > + len = strlen(buf); > + str = buf; > + break; > + case MP_STR: > + nd->mask |= NODE_QUOTE; > + str = lua_tolstring(d->L, -1, &len); > + if (utf8_check_printable(str, len) == 1) > + break; > + /* fallthrough */ > + case MP_BIN: > + nd->mask |= NODE_QUOTE; > + tobase64(d->L, -1); > + str = lua_tolstring(d->L, -1, &len); > + lua_pop(d->L, 1); > + break; > + case MP_ARRAY: > + dump_array(d, nd, indent); > + break; > + case MP_MAP: > + dump_table(d, nd, indent); > + break; > + case MP_BOOL: > + if (field->bval) { > + static const char str_true[] = "true"; > + len = strlen(str_true); > + str = str_true; > + } else { > + static const char str_false[] = "false"; > + len = strlen(str_false); > + str = str_false; > + } > + break; > + case MP_FLOAT: > + fpconv_g_fmt(buf, field->fval, > + d->cfg->encode_number_precision); > + len = strlen(buf); > + str = buf; > + break; > + case MP_DOUBLE: > + fpconv_g_fmt(buf, field->dval, > + d->cfg->encode_number_precision); > + len = strlen(buf); > + str = buf; > + break; > + case MP_EXT: > + switch (field->ext_type) { > + case MP_DECIMAL: > + nd->mask |= NODE_QUOTE; > + str = decimal_to_string(field->decval); > + len = strlen(str); > + break; > + case MP_UUID: > + nd->mask |= NODE_QUOTE; > + str = tt_uuid_str(field->uuidval); > + len = UUID_STR_LEN; > + break; > + default: > + d->err = EINVAL; > + snprintf(d->err_msg, sizeof(d->err_msg), > + "serializer: Unknown field MP_EXT:%d type", > + field->ext_type); > + len = strlen(d->err_msg); > + return -1; > + } > + break; > + default: > + d->err = EINVAL; > + snprintf(d->err_msg, sizeof(d->err_msg), > + "serializer: Unknown field %d type", > + field->type); > + len = strlen(d->err_msg); > + return -1; > + } > + > + return emit_node(d, nd, indent, str, len); > +} > + > +/** > + * Find references to tables, we use it > + * to find self references in tables. > + */ > +static void > +find_references(struct lua_dumper *d) > +{ > + int newval; > + > + if (lua_type(d->L, -1) != LUA_TTABLE) > + return; > + > + /* Copy of a table for self refs */ > + lua_pushvalue(d->L, -1); > + lua_rawget(d->L, d->anchortable_index); > + if (lua_isnil(d->L, -1)) > + newval = 0; > + else if (!lua_toboolean(d->L, -1)) > + newval = 1; > + else > + newval = -1; > + lua_pop(d->L, 1); > + > + if (newval != -1) { > + lua_pushvalue(d->L, -1); > + lua_pushboolean(d->L, newval); > + lua_rawset(d->L, d->anchortable_index); > + } > + > + if (newval != 0) > + return; > + > + /* > + * Other values and keys in the table > + */ > + lua_pushnil(d->L); > + while (lua_next(d->L, -2) != 0) { > + find_references(d); > + lua_pop(d->L, 1); > + find_references(d); > + } > +} > + > +/** > + * Dump recursively from the root node. > + */ > +static int > +dump_root(struct lua_dumper *d) > +{ > + struct node nd = { > + .mask = NODE_ROOT, > + }; > + int ret; > + > + luaL_checkfield(d->L, d->cfg, lua_gettop(d->L), &nd.field); > + > + if (nd.field.type != MP_ARRAY || nd.field.size != 1) { > + d->err = EINVAL; > + snprintf(d->err_msg, sizeof(d->err_msg), > + "serializer: unexpected data " > + "(nd.field.size %d nd.field.type %d)", > + nd.field.size, nd.field.type); > + return -1; > + } > + > + /* > + * We don't need to show the newly generated > + * table, instead dump the nested one which > + * is the real value. > + */ > + lua_rawgeti(d->L, -1, 1); > + ret = dump_node(d, &nd, 0); > + lua_pop(d->L, 1); > + > + return (ret || d->err) ? -1 : 0; > +} > + > +/** > + * Encode data to Lua compatible form. > + */ > +int > +lua_encode(lua_State *L, struct luaL_serializer *serializer, > + lua_dumper_opts_t *opts) > +{ > + struct lua_dumper dumper = { > + .L = L, > + .cfg = serializer, > + .outputL= luaT_newthread(L), > + .opts = opts, > + }; > + > + if (!dumper.outputL) > + return luaL_error(L, "serializer: No free memory"); > + > + luaL_buffinit(dumper.outputL, &dumper.luabuf); > + > + lua_newtable(L); > + > + dumper.anchortable_index = lua_gettop(L); > + dumper.anchor_number = 0; > + > + /* Push copy of arg we're processing */ > + lua_pushvalue(L, 1); > + find_references(&dumper); > + > + if (dump_root(&dumper) != 0) > + goto out; > + > + /* Pop copied arg and anchor table */ > + lua_pop(L, 2); > + > + luaL_pushresult(&dumper.luabuf); > + > + /* Move buffer to original thread */ > + lua_xmove(dumper.outputL, L, 1); > + return 1; > + > +out: > + errno = dumper.err; > + lua_pushnil(L); > + lua_pushstring(L, dumper.err_msg); > + return 2; > +} > + > +/** > + * Parse serializer options. > + */ > +void > +lua_parse_opts(lua_State *L, lua_dumper_opts_t *opts) > +{ > + if (lua_gettop(L) < 2 || lua_type(L, -2) != LUA_TTABLE) > + luaL_error(L, "serializer: Wrong options format"); > + > + memset(opts, 0, sizeof(*opts)); > + > + lua_getfield(L, -2, "block"); > + if (lua_isboolean(L, -1)) > + opts->block_mode = lua_toboolean(L, -1); > + lua_pop(L, 1); > + > + lua_getfield(L, -2, "indent"); > + if (lua_isnumber(L, -1)) > + opts->indent_lvl = (int)lua_tonumber(L, -1); > + lua_pop(L, 1); > +} > + > +/** > + * Initialize Lua serializer. > + */ > +void > +lua_serializer_init(struct lua_State *L) > +{ > + /* > + * We don't export it as a module > + * for a while, so the library > + * is kept empty. > + */ > + static const luaL_Reg lualib[] = { > + { > + .name = NULL, > + }, > + }; > + > + serializer_lua = luaL_newserializer(L, NULL, lualib); > + serializer_lua->has_compact = 1; > + serializer_lua->encode_invalid_numbers = 1; > + serializer_lua->encode_load_metatables = 1; > + serializer_lua->encode_use_tostring = 1; > + serializer_lua->encode_invalid_as_nil = 1; > + > + /* > + * Keep a reference to this module so it > + * won't be unloaded. > + */ > + lua_setfield(L, -2, "formatter_lua"); > +} > diff --git a/src/box/lua/serialize_lua.h b/src/box/lua/serialize_lua.h > new file mode 100644 > index 000000000..923e13be9 > --- /dev/null > +++ b/src/box/lua/serialize_lua.h > @@ -0,0 +1,67 @@ > +#ifndef INCLUDES_TARANTOOL_SERLIALIZE_LUA_H > +#define INCLUDES_TARANTOOL_SERLIALIZE_LUA_H > + > +/* > + * Copyright 2010-2020, Tarantool AUTHORS, please see AUTHORS file. > + * > + * Redistribution and use in source and binary forms, with or > + * without modification, are permitted provided that the following > + * conditions are met: > + * > + * 1. Redistributions of source code must retain the above > + * copyright notice, this list of conditions and the > + * following disclaimer. > + * > + * 2. Redistributions in binary form must reproduce the above > + * copyright notice, this list of conditions and the following > + * disclaimer in the documentation and/or other materials > + * provided with the distribution. > + * > + * THIS SOFTWARE IS PROVIDED BY ``AS IS'' AND > + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED > + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL > + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, > + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL > + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF > + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR > + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF > + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF > + * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF > + * SUCH DAMAGE. > + */ > + > +#if defined(__cplusplus) > +extern "C" { > +#endif /* defined(__cplusplus) */ > + > +struct luaL_serializer; > +struct lua_State; > + > +/** > + * Serializer options. > + */ > +typedef struct { > + /** Number of indentation spaces. */ > + unsigned int indent_lvl; > + > + /** Block mode output. */ > + bool block_mode; > +} lua_dumper_opts_t; > + > +void > +lua_serializer_init(struct lua_State *L); > + > +int > +lua_encode(lua_State *L, struct luaL_serializer *serializer, > + lua_dumper_opts_t *opts); > + > +void > +lua_parse_opts(lua_State *L, lua_dumper_opts_t *opts); > + > +#if defined(__cplusplus) > +} /* extern "C" */ > +#endif /* defined(__cplusplus) */ > + > +#endif /* INCLUDES_TARANTOOL_SERLIALIZE_LUA_H */ >