* [Tarantool-patches] [PATCH 5/7] box/console: implement lua serializer
2020-05-08 11:47 [Tarantool-patches] [PATCH 0/7] box/console: add support for internal types Cyrill Gorcunov
` (3 preceding siblings ...)
2020-05-08 11:47 ` [Tarantool-patches] [PATCH 4/7] box/console: rename luaL_yaml_default to serializer_yaml Cyrill Gorcunov
@ 2020-05-08 11:47 ` Cyrill Gorcunov
2020-05-08 11:47 ` [Tarantool-patches] [PATCH 6/7] box/console: switch to new " Cyrill Gorcunov
` (3 subsequent siblings)
8 siblings, 0 replies; 16+ messages in thread
From: Cyrill Gorcunov @ 2020-05-08 11:47 UTC (permalink / raw)
To: tml
When we print output in console (especially in Lua mode)
it is impossible to find out the internal type an entry
represents, which in turn leads to inability to encode
"ULL" entries properly. Moreover we always introduce new
types (for example decimals, uuids) and the serpent module
we currently use for encodings has no clue about them.
Thus lets implement own lua serializer similarly to the
yaml encoder. This allows us to properly detect every
field and encode it accordingly.
Part-of #4682
Signed-off-by: Cyrill Gorcunov <gorcunov@gmail.com>
---
src/box/CMakeLists.txt | 1 +
src/box/lua/serialize_lua.c | 1021 +++++++++++++++++++++++++++++++++++
src/box/lua/serialize_lua.h | 67 +++
3 files changed, 1089 insertions(+)
create mode 100644 src/box/lua/serialize_lua.c
create mode 100644 src/box/lua/serialize_lua.h
diff --git a/src/box/CMakeLists.txt b/src/box/CMakeLists.txt
index c931ecdfe..08ffe7177 100644
--- a/src/box/CMakeLists.txt
+++ b/src/box/CMakeLists.txt
@@ -148,6 +148,7 @@ add_library(box STATIC
lua/call.c
lua/cfg.cc
lua/console.c
+ lua/serialize_lua.c
lua/tuple.c
lua/slab.c
lua/index.c
diff --git a/src/box/lua/serialize_lua.c b/src/box/lua/serialize_lua.c
new file mode 100644
index 000000000..8ab9fab76
--- /dev/null
+++ b/src/box/lua/serialize_lua.c
@@ -0,0 +1,1021 @@
+/*
+ * Copyright 2010-2020, Tarantool AUTHORS, please see AUTHORS file.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the
+ * following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY <COPYRIGHT HOLDER> ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+ * <COPYRIGHT HOLDER> OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+ * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <ctype.h>
+#include <lua.h>
+
+#include "trivia/util.h"
+#include "lua/utils.h"
+#include "say.h"
+
+#include "lib/core/decimal.h"
+#include "mp_extension_types.h"
+#include "uuid/tt_uuid.h"
+
+#include "lua-yaml/b64.h"
+
+#include "serialize_lua.h"
+
+#if 0
+# define SERIALIZER_TRACE
+#endif
+
+/* Serializer for Lua output mode */
+static struct luaL_serializer *serializer_lua;
+
+enum {
+ NODE_NONE_BIT = 0,
+ NODE_ROOT_BIT = 1,
+ NODE_RAW_BIT = 2,
+ NODE_LVALUE_BIT = 3,
+ NODE_RVALUE_BIT = 4,
+ NODE_MAP_KEY_BIT = 5,
+ NODE_MAP_VALUE_BIT = 6,
+ NODE_EMBRACE_BIT = 7,
+ NODE_QUOTE_BIT = 8,
+
+ NODE_MAX
+};
+
+enum {
+ NODE_NONE = (1u << NODE_NONE_BIT),
+ NODE_ROOT = (1u << NODE_ROOT_BIT),
+ NODE_RAW = (1u << NODE_RAW_BIT),
+ NODE_LVALUE = (1u << NODE_LVALUE_BIT),
+ NODE_RVALUE = (1u << NODE_RVALUE_BIT),
+ NODE_MAP_KEY = (1u << NODE_MAP_KEY_BIT),
+ NODE_MAP_VALUE = (1u << NODE_MAP_VALUE_BIT),
+ NODE_EMBRACE = (1u << NODE_EMBRACE_BIT),
+ NODE_QUOTE = (1u << NODE_QUOTE_BIT),
+};
+
+struct node {
+ /** Link with previous node or key */
+ union {
+ struct node *prev;
+ struct node *key;
+ };
+
+ /** The field data we're paring */
+ struct luaL_field field;
+
+ /** Node mask NODE_x */
+ int mask;
+
+ /** Node index in a map */
+ int index;
+};
+
+/**
+ * Serializer context.
+ */
+struct lua_dumper {
+ /** Lua state to fetch data from */
+ lua_State *L;
+
+ /** General configure options */
+ struct luaL_serializer *cfg;
+
+ /** Lua dumper options */
+ lua_dumper_opts_t *opts;
+
+ /** Output state */
+ lua_State *outputL;
+ /** Output buffer */
+ luaL_Buffer luabuf;
+
+ /** Anchors for self references */
+ int anchortable_index;
+ unsigned int anchor_number;
+
+ /** Error message buffer */
+ char err_msg[256];
+ int err;
+
+ /** Indentation buffer */
+ char indent_buf[256];
+
+ /** Output suffix */
+ char suffix_buf[32];
+ int suffix_len;
+
+ /** Previous node mask */
+ int prev_nd_mask;
+
+ /** Ignore indents */
+ bool noindent;
+};
+
+#ifdef SERIALIZER_TRACE
+
+#define __gen_mp_name(__v) [__v] = # __v
+static const char *mp_type_names[] = {
+ __gen_mp_name(MP_NIL),
+ __gen_mp_name(MP_UINT),
+ __gen_mp_name(MP_INT),
+ __gen_mp_name(MP_STR),
+ __gen_mp_name(MP_BIN),
+ __gen_mp_name(MP_ARRAY),
+ __gen_mp_name(MP_MAP),
+ __gen_mp_name(MP_BOOL),
+ __gen_mp_name(MP_FLOAT),
+ __gen_mp_name(MP_DOUBLE),
+ __gen_mp_name(MP_EXT),
+};
+
+static const char *mp_ext_type_names[] = {
+ __gen_mp_name(MP_DECIMAL),
+ __gen_mp_name(MP_UUID),
+ __gen_mp_name(MP_ERROR),
+};
+#undef __gen_mp_name
+
+#define __gen_nd_name(__v) [__v ##_BIT] = # __v
+static const char *nd_type_names[] = {
+ __gen_nd_name(NODE_NONE),
+ __gen_nd_name(NODE_ROOT),
+ __gen_nd_name(NODE_RAW),
+ __gen_nd_name(NODE_LVALUE),
+ __gen_nd_name(NODE_RVALUE),
+ __gen_nd_name(NODE_MAP_KEY),
+ __gen_nd_name(NODE_MAP_VALUE),
+ __gen_nd_name(NODE_EMBRACE),
+ __gen_nd_name(NODE_QUOTE),
+};
+#undef __gen_nd_name
+
+static void
+trace_node(struct lua_dumper *d)
+{
+ int ltype = lua_type(d->L, -1);
+ say_info("serializer-trace: node : lua type %d -> %s",
+ ltype, lua_typename(d->L, ltype));
+
+ if (d->err != 0)
+ return;
+
+ char mp_type[64], *type_str = mp_type;
+ int top = lua_gettop(d->L);
+ struct luaL_field field;
+
+ memset(&field, 0, sizeof(field));
+ luaL_checkfield(d->L, d->cfg, lua_gettop(d->L), &field);
+
+ if (field.type < lengthof(mp_type_names)) {
+ if (field.type == MP_EXT) {
+ size_t max_ext = lengthof(mp_ext_type_names);
+ snprintf(mp_type, sizeof(mp_type), "%s/%s",
+ mp_type_names[field.type],
+ field.ext_type < max_ext ?
+ mp_ext_type_names[field.ext_type] :
+ "UNKNOWN");
+ } else {
+ type_str = (char *)mp_type_names[field.type];
+ }
+ } else {
+ type_str = "UNKNOWN";
+ }
+
+ memset(&field, 0, sizeof(field));
+
+ luaL_checkfield(d->L, d->cfg, top, &field);
+ say_info("serializer-trace: node :\tfield type %s (%d)",
+ type_str, field.type);
+}
+
+static char *
+trace_string(const char *src, size_t len)
+{
+ static char buf[128];
+ size_t pos = 0;
+
+ if (len > sizeof(buf)-1)
+ len = sizeof(buf)-1;
+
+ while (pos < len) {
+ if (src[pos] == '\n') {
+ buf[pos++] = '\\';
+ buf[pos++] = 'n';
+ continue;
+ }
+ buf[pos] = src[pos];
+ pos++;
+ }
+ buf[pos] = '\0';
+ return buf;
+}
+
+static void
+trace_emit(struct lua_dumper *d, int nd_mask, int indent,
+ const char *str, size_t len)
+{
+ char buf[128], *names = buf;
+ int left = sizeof(buf);
+ int pos = 0;
+
+ if (d->suffix_len) {
+ say_info("serializer-trace: emit-sfx: \"%s\"",
+ trace_string(d->suffix_buf,
+ d->suffix_len));
+ }
+
+ static_assert(NODE_MAX < sizeof(int) * 8,
+ "NODE_MAX is too big");
+
+ for (int i = 0; i < NODE_MAX; i++) {
+ if (!(nd_mask & (1u << i)))
+ continue;
+
+ int nd_len = strlen(nd_type_names[i]);
+ if (left >= nd_len + 1) {
+ strcpy(&buf[pos], nd_type_names[i]);
+ pos += nd_len;
+ buf[pos++] = '|';
+ left = sizeof(buf) - pos;
+ }
+ }
+
+ if (pos != 0)
+ buf[--pos] = '\0';
+ else
+ names = "UNKNOWN";
+
+ say_info("serializer-trace: emit : type %s (0x%x) "
+ "indent %d val \"%s\" len %zu",
+ names, nd_mask, indent,
+ trace_string(str, len), len);
+}
+
+#else /* SERIALIZER_TRACE */
+
+static void
+trace_node(struct lua_dumper *d)
+{
+ (void)d;
+}
+
+static void
+trace_emit(struct lua_dumper *d, int nd_mask, int indent,
+ const char *str, size_t len)
+{
+ (void)d;
+ (void)nd_mask;
+ (void)indent;
+ (void)str;
+ (void)len;
+}
+
+#endif /* SERIALIZER_TRACE */
+
+static const char *lua_keywords[] = {
+ "and", "break", "do", "else",
+ "elseif", "end", "false", "for",
+ "function", "if", "in", "local",
+ "nil", "not", "or", "repeat",
+ "return", "then", "true", "until",
+ "while", "and",
+};
+
+static int
+dump_node(struct lua_dumper *d, struct node *nd, int indent);
+
+/**
+ * Generate anchor numbers for self references.
+ */
+static const char *
+get_anchor(struct lua_dumper *d)
+{
+ const char *s = "";
+
+ lua_pushvalue(d->L, -1);
+ lua_rawget(d->L, d->anchortable_index);
+ if (!lua_toboolean(d->L, -1)) {
+ lua_pop(d->L, 1);
+ return NULL;
+ }
+
+ if (lua_isboolean(d->L, -1)) {
+ /*
+ * This element is referenced more
+ * than once but has not been named.
+ */
+ char buf[32];
+ snprintf(buf, sizeof(buf), "%u", d->anchor_number++);
+ lua_pop(d->L, 1);
+ lua_pushvalue(d->L, -1);
+ lua_pushstring(d->L, buf);
+ s = lua_tostring(d->L, -1);
+ lua_rawset(d->L, d->anchortable_index);
+ } else {
+ /* An aliased element */
+ const char *str = lua_tostring(d->L, -1);
+ luaL_addstring(&d->luabuf, str);
+ luaL_addstring(&d->luabuf, "\n");
+ lua_pop(d->L, 1);
+ (void)str;
+ }
+ return s;
+}
+
+static void
+suffix_append(struct lua_dumper *d, const char *str, int len)
+{
+ int left = (int)sizeof(d->suffix_buf) - d->suffix_len;
+ if (len < left) {
+ memcpy(&d->suffix_buf[d->suffix_len], str, len);
+ d->suffix_len += len;
+ d->suffix_buf[d->suffix_len] = '\0';
+ }
+}
+
+static inline void
+suffix_reset(struct lua_dumper *d)
+{
+ d->suffix_len = 0;
+}
+
+static void
+suffix_flush(struct lua_dumper *d)
+{
+ if (d->suffix_len) {
+ luaL_addlstring(&d->luabuf, d->suffix_buf, d->suffix_len);
+ suffix_reset(d);
+ }
+}
+
+static int
+gen_indent(struct lua_dumper *d, int indent)
+{
+ static_assert(sizeof(d->indent_buf) > 0,
+ "indent buffer is too small");
+
+ if (indent > 0 && d->opts->block_mode && !d->noindent) {
+ snprintf(d->indent_buf, sizeof(d->indent_buf),
+ "%*s", indent, "");
+ size_t len = strlen(d->indent_buf);
+ d->indent_buf[len] = '\0';
+ return len;
+ }
+
+ return 0;
+}
+
+static void
+emit_hex_char(struct lua_dumper *d, unsigned char c)
+{
+ luaL_addchar(&d->luabuf, '\\');
+ luaL_addchar(&d->luabuf, 'x');
+
+#define __emit_hex(v) \
+ do { \
+ if (v <= 9) \
+ luaL_addchar(&d->luabuf, '0' + v); \
+ else \
+ luaL_addchar(&d->luabuf, v - 10 + 'a'); \
+ } while (0)
+
+ __emit_hex((c >> 4));
+ __emit_hex((c & 0xf));
+#undef __emit_hex
+}
+
+/**
+ * Emit the string with escapes if needed.
+ *
+ * FIXME: Probably should to revisit and make
+ * sure we've not miss anything here (octal numbers
+ * are missed for now and etc...).
+ */
+static void
+emit_string(struct lua_dumper *d, const char *str, size_t len)
+{
+ for (size_t i = 0; i < len; i++) {
+ if ((str[i]) == '\'' || str[i] == '\"') {
+ luaL_addchar(&d->luabuf, '\\');
+ luaL_addchar(&d->luabuf, str[i]);
+ } else if (str[i] == '\0') {
+ luaL_addchar(&d->luabuf, '\\');
+ luaL_addchar(&d->luabuf, '0');
+ } else if (str[i] == '\a') {
+ luaL_addchar(&d->luabuf, '\\');
+ luaL_addchar(&d->luabuf, 'a');
+ } else if (str[i] == '\b') {
+ luaL_addchar(&d->luabuf, '\\');
+ luaL_addchar(&d->luabuf, 'b');
+ } else if (str[i] == '\f') {
+ luaL_addchar(&d->luabuf, '\\');
+ luaL_addchar(&d->luabuf, 'f');
+ } else if (str[i] == '\v') {
+ luaL_addchar(&d->luabuf, '\\');
+ luaL_addchar(&d->luabuf, 'v');
+ } else if (str[i] == '\r') {
+ luaL_addchar(&d->luabuf, '\\');
+ luaL_addchar(&d->luabuf, 'r');
+ } else if (str[i] == '\n') {
+ luaL_addchar(&d->luabuf, '\\');
+ luaL_addchar(&d->luabuf, 'n');
+ } else if (str[i] == '\t') {
+ luaL_addchar(&d->luabuf, '\\');
+ luaL_addchar(&d->luabuf, 't');
+ } else if (str[i] == '\xef') {
+ if (i < len-1 && i < len-2 &&
+ str[i+1] == '\xbb' &&
+ str[i+2] == '\xbf') {
+ emit_hex_char(d, 0xef);
+ emit_hex_char(d, 0xbb);
+ emit_hex_char(d, 0xbf);
+ } else {
+ emit_hex_char(d, str[i]);
+ }
+ } else if (isprint(str[i]) == 0) {
+ emit_hex_char(d, str[i]);
+ } else {
+ luaL_addchar(&d->luabuf, str[i]);
+ }
+ }
+}
+
+/**
+ * Emit value into output buffer.
+ */
+static void
+emit_value(struct lua_dumper *d, struct node *nd,
+ int indent, const char *str, size_t len)
+{
+ trace_emit(d, nd->mask, indent, str, len);
+
+ /*
+ * There might be previous closing symbols
+ * in the suffix queue. Since we're about
+ * to emit new values don't forget to prepend
+ * ending ones.
+ */
+ suffix_flush(d);
+
+ luaL_addlstring(&d->luabuf, d->indent_buf,
+ gen_indent(d, indent));
+
+ if (nd->mask & NODE_EMBRACE)
+ luaL_addlstring(&d->luabuf, "[", 1);
+ if (nd->mask & NODE_QUOTE)
+ luaL_addlstring(&d->luabuf, "\"", 1);
+
+ if (nd->field.type == MP_STR) {
+ emit_string(d, str, len);
+ } else {
+ luaL_addlstring(&d->luabuf, str, len);
+ }
+
+ if (nd->mask & NODE_QUOTE)
+ luaL_addlstring(&d->luabuf, "\"", 1);
+ if (nd->mask & NODE_EMBRACE)
+ luaL_addlstring(&d->luabuf, "]", 1);
+}
+
+/**
+ * Emit a raw string into output.
+ */
+static void
+emit_raw_value(struct lua_dumper *d, int indent,
+ const char *str, size_t len)
+{
+ struct node node = {
+ .mask = NODE_RAW,
+ };
+
+ emit_value(d, &node, indent, str, len);
+}
+
+/**
+ * Handling self references. It is yaml specific
+ * and I don't think we might even need it. Still
+ * better to get noticed if something went in
+ * an unexpected way.
+ */
+static bool
+emit_anchor(struct lua_dumper *d)
+{
+ const char *anchor = get_anchor(d);
+ if (anchor && !*anchor) {
+ emit_raw_value(d, 0, "!!!*anchor", 10);
+ return true;
+ }
+ return false;
+}
+
+/**
+ * Put an opening brace into the output.
+ */
+static int
+emit_brace_open(struct lua_dumper *d, int indent)
+{
+ if (d->opts->block_mode) {
+ int _indent;
+ if (d->prev_nd_mask & NODE_LVALUE)
+ _indent = 0;
+ else
+ _indent = indent;
+
+ emit_raw_value(d, _indent, "{\n", 2);
+ if (d->prev_nd_mask & NODE_LVALUE)
+ d->noindent = false;
+ } else {
+ emit_raw_value(d, indent, "{", 1);
+ }
+
+ return indent + d->opts->indent_lvl;
+}
+
+/**
+ * Put a closing brace into the output.
+ */
+static void
+emit_brace_close(struct lua_dumper *d, int indent)
+{
+ suffix_reset(d);
+
+ if (d->opts->block_mode)
+ emit_raw_value(d, 0, "\n", 1);
+
+ indent -= d->opts->indent_lvl;
+ emit_raw_value(d, indent, "}", 1);
+
+ if (d->opts->block_mode)
+ suffix_append(d, ",\n", 2);
+ else
+ suffix_append(d, ", ", 2);
+}
+
+/**
+ * Dump an array entry.
+ */
+static void
+dump_array(struct lua_dumper *d, struct node *nd, int indent)
+{
+ struct node node = {
+ .prev = nd,
+ .mask = NODE_RVALUE,
+ };
+ if (emit_anchor(d))
+ return;
+
+ indent = emit_brace_open(d, indent);
+
+ for (int i = 0; i < (int)nd->field.size; i++) {
+ lua_rawgeti(d->L, -1, i + 1);
+ dump_node(d, &node, indent);
+ lua_pop(d->L, 1);
+ }
+
+ emit_brace_close(d, indent);
+}
+
+/**
+ * Dump a map entry.
+ */
+static void
+dump_table(struct lua_dumper *d, struct node *nd, int indent)
+{
+ struct node node_key = {
+ .prev = nd,
+ .mask = NODE_LVALUE | NODE_MAP_KEY,
+ };
+
+ struct node node_val = {
+ .key = &node_key,
+ .mask = NODE_RVALUE | NODE_MAP_VALUE,
+ };
+
+ if (emit_anchor(d))
+ return;
+
+ indent = emit_brace_open(d, indent);
+
+ /*
+ * In sake of speed we don't sort
+ * keys but provide them as is. Thus
+ * simply walk over keys and their
+ * values.
+ */
+ lua_pushnil(d->L);
+ while (lua_next(d->L, -2)) {
+ lua_pushvalue(d->L, -2);
+ node_key.index++;
+ dump_node(d, &node_key, indent);
+ lua_pop(d->L, 1);
+
+ dump_node(d, &node_val, indent);
+ lua_pop(d->L, 1);
+ }
+
+ emit_brace_close(d, indent);
+}
+
+/**
+ * Figure out if we need to decorate a map key
+ * with square braces and quotes or can leave
+ * it as a plain value.
+ */
+static void
+decorate_key(struct node *nd, const char *str, size_t len)
+{
+ assert(nd->field.type == MP_STR);
+ assert(nd->mask & NODE_MAP_KEY);
+
+ /*
+ * We might need to put string keys
+ * to quotes and embrace them due to
+ * limitation of how to declare map keys
+ * (the output from serializer should be
+ * parsable if pasted back to a console).
+ */
+ for (size_t i = 0; i < lengthof(lua_keywords); i++) {
+ const char *k = lua_keywords[i];
+ if (strcmp(k, str) == 0) {
+ nd->mask |= NODE_EMBRACE | NODE_QUOTE;
+ return;
+ }
+ }
+
+ /*
+ * Plain keys may be alphanumerics with underscopes.
+ */
+ for (size_t i = 0; i < len; i++) {
+ if (isalnum(str[i]) || str[i] == '_')
+ continue;
+ nd->mask |= NODE_EMBRACE | NODE_QUOTE;
+ return;
+ }
+
+ nd->mask &= ~NODE_QUOTE;
+}
+
+/**
+ * Dump a node.
+ */
+static int
+dump_node(struct lua_dumper *d, struct node *nd, int indent)
+{
+ struct luaL_field *field = &nd->field;
+ char buf[FPCONV_G_FMT_BUFSIZE];
+ int ltype = lua_type(d->L, -1);
+ const char *str = NULL;
+ size_t len = 0;
+
+ trace_node(d);
+
+ /*
+ * We can exit early if an error
+ * already happened, no need to
+ * continue parsing.
+ */
+ if (d->err != 0)
+ return -1;
+
+ memset(field, 0, sizeof(*field));
+ luaL_checkfield(d->L, d->cfg, lua_gettop(d->L), field);
+
+ switch (field->type) {
+ case MP_NIL:
+ if (ltype == LUA_TNIL) {
+ static const char str_nil[] = "nil";
+ str = str_nil;
+ len = strlen(str_nil);
+ } else {
+ static const char str_null[] = "box.NULL";
+ str = str_null;
+ len = strlen(str_null);
+ }
+ break;
+ case MP_UINT:
+ snprintf(buf, sizeof(buf), "%" PRIu64, field->ival);
+ len = strlen(buf);
+ str = buf;
+ break;
+ case MP_INT:
+ snprintf(buf, sizeof(buf), "%" PRIi64, field->ival);
+ len = strlen(buf);
+ str = buf;
+ break;
+ case MP_STR:
+ nd->mask |= NODE_QUOTE;
+ str = lua_tolstring(d->L, -1, &len);
+ if (utf8_check_printable(str, len) == 1)
+ break;
+ /* fallthrough */
+ case MP_BIN:
+ nd->mask |= NODE_QUOTE;
+ tobase64(d->L, -1);
+ str = lua_tolstring(d->L, -1, &len);
+ lua_pop(d->L, 1);
+ break;
+ case MP_ARRAY:
+ dump_array(d, nd, indent);
+ break;
+ case MP_MAP:
+ dump_table(d, nd, indent);
+ break;
+ case MP_BOOL:
+ if (field->bval) {
+ static const char str_true[] = "true";
+ len = strlen(str_true);
+ str = str_true;
+ } else {
+ static const char str_false[] = "false";
+ len = strlen(str_false);
+ str = str_false;
+ }
+ break;
+ case MP_FLOAT:
+ fpconv_g_fmt(buf, field->fval,
+ d->cfg->encode_number_precision);
+ len = strlen(buf);
+ str = buf;
+ break;
+ case MP_DOUBLE:
+ fpconv_g_fmt(buf, field->dval,
+ d->cfg->encode_number_precision);
+ len = strlen(buf);
+ str = buf;
+ break;
+ case MP_EXT:
+ switch (field->ext_type) {
+ case MP_DECIMAL:
+ nd->mask |= NODE_QUOTE;
+ str = decimal_to_string(field->decval);
+ len = strlen(str);
+ break;
+ case MP_UUID:
+ nd->mask |= NODE_QUOTE;
+ str = tt_uuid_str(field->uuidval);
+ len = UUID_STR_LEN;
+ break;
+ default:
+ d->err = EINVAL;
+ snprintf(d->err_msg, sizeof(d->err_msg),
+ "serializer: Unknown field MP_EXT:%d type",
+ field->ext_type);
+ len = strlen(d->err_msg);
+ return -1;
+ }
+ break;
+ default:
+ d->err = EINVAL;
+ snprintf(d->err_msg, sizeof(d->err_msg),
+ "serializer: Unknown field %d type",
+ field->type);
+ len = strlen(d->err_msg);
+ return -1;
+ }
+
+ if (str == NULL) {
+ d->prev_nd_mask = nd->mask;
+ return 0;
+ }
+
+ if (nd->mask & NODE_MAP_KEY) {
+ /*
+ * In case if key is integer and matching
+ * the current position in the table we
+ * can simply skip it and print value only.
+ */
+ if (nd->field.type == MP_INT ||
+ nd->field.type == MP_UINT) {
+ if (nd->index == (int)field->ival) {
+ d->noindent = false;
+ return 0;
+ } else {
+ nd->mask |= NODE_EMBRACE;
+ }
+ } else if (nd->field.type == MP_STR) {
+ decorate_key(nd, str, len);
+ }
+ }
+
+ d->prev_nd_mask = nd->mask;
+ emit_value(d, nd, indent, str, len);
+
+ /*
+ * In sake of speed we do not lookahead
+ * for next lua nodes, instead just remember
+ * closing symbol in suffix buffer which we
+ * will flush on next emit.
+ */
+ if (nd->mask & NODE_RVALUE) {
+ if (d->opts->block_mode)
+ suffix_append(d, ",\n", 2);
+ else
+ suffix_append(d, ", ", 2);
+ d->noindent = false;
+ } else if (nd->mask & NODE_LVALUE) {
+ suffix_append(d, " = ", 3);
+ d->noindent = true;
+ }
+
+ return 0;
+}
+
+/**
+ * Find references to tables, we use it
+ * to find self references in tables.
+ */
+static void
+find_references(struct lua_dumper *d)
+{
+ int newval;
+
+ if (lua_type(d->L, -1) != LUA_TTABLE)
+ return;
+
+ /* Copy of a table for self refs */
+ lua_pushvalue(d->L, -1);
+ lua_rawget(d->L, d->anchortable_index);
+ if (lua_isnil(d->L, -1))
+ newval = 0;
+ else if (!lua_toboolean(d->L, -1))
+ newval = 1;
+ else
+ newval = -1;
+ lua_pop(d->L, 1);
+
+ if (newval != -1) {
+ lua_pushvalue(d->L, -1);
+ lua_pushboolean(d->L, newval);
+ lua_rawset(d->L, d->anchortable_index);
+ }
+
+ if (newval != 0)
+ return;
+
+ /*
+ * Other values and keys in the table
+ */
+ lua_pushnil(d->L);
+ while (lua_next(d->L, -2) != 0) {
+ find_references(d);
+ lua_pop(d->L, 1);
+ find_references(d);
+ }
+}
+
+/**
+ * Dump recursively from the root node.
+ */
+static int
+dump_root(struct lua_dumper *d)
+{
+ struct node nd = {
+ .mask = NODE_ROOT,
+ };
+ int ret;
+
+ luaL_checkfield(d->L, d->cfg, lua_gettop(d->L), &nd.field);
+
+ if (nd.field.type != MP_ARRAY || nd.field.size != 1) {
+ d->err = EINVAL;
+ snprintf(d->err_msg, sizeof(d->err_msg),
+ "serializer: unexpected data "
+ "(nd.field.size %d nd.field.type %d)",
+ nd.field.size, nd.field.type);
+ return -1;
+ }
+
+ /*
+ * We don't need to show the newly generated
+ * table, instead dump the nested one which
+ * is the real value.
+ */
+ lua_rawgeti(d->L, -1, 1);
+ ret = dump_node(d, &nd, 0);
+ lua_pop(d->L, 1);
+
+ return (ret || d->err) ? -1 : 0;
+}
+
+/**
+ * Encode data to Lua compatible form.
+ */
+int
+lua_encode(lua_State *L, struct luaL_serializer *serializer,
+ lua_dumper_opts_t *opts)
+{
+ struct lua_dumper dumper = {
+ .L = L,
+ .cfg = serializer,
+ .outputL= luaT_newthread(L),
+ .opts = opts,
+ };
+
+ if (!dumper.outputL)
+ return luaL_error(L, "serializer: No free memory");
+
+ luaL_buffinit(dumper.outputL, &dumper.luabuf);
+
+ lua_newtable(L);
+
+ dumper.anchortable_index = lua_gettop(L);
+ dumper.anchor_number = 0;
+
+ /* Push copy of arg we're processing */
+ lua_pushvalue(L, 1);
+ find_references(&dumper);
+
+ if (dump_root(&dumper) != 0)
+ goto out;
+
+ /* Pop copied arg and anchor table */
+ lua_pop(L, 2);
+
+ luaL_pushresult(&dumper.luabuf);
+
+ /* Move buffer to original thread */
+ lua_xmove(dumper.outputL, L, 1);
+ return 1;
+
+out:
+ errno = dumper.err;
+ lua_pushnil(L);
+ lua_pushstring(L, dumper.err_msg);
+ return 2;
+}
+
+/**
+ * Parse serializer options.
+ */
+void
+lua_parse_opts(lua_State *L, lua_dumper_opts_t *opts)
+{
+ if (lua_gettop(L) < 2 || lua_type(L, -2) != LUA_TTABLE)
+ luaL_error(L, "serializer: Wrong options format");
+
+ memset(opts, 0, sizeof(*opts));
+
+ lua_getfield(L, -2, "block");
+ if (lua_isboolean(L, -1))
+ opts->block_mode = lua_toboolean(L, -1);
+ lua_pop(L, 1);
+
+ lua_getfield(L, -2, "indent");
+ if (lua_isnumber(L, -1))
+ opts->indent_lvl = (int)lua_tonumber(L, -1);
+ lua_pop(L, 1);
+}
+
+/**
+ * Initialize Lua serializer.
+ */
+void
+lua_serializer_init(struct lua_State *L)
+{
+ /*
+ * We don't export it as a module
+ * for a while, so the library
+ * is kept empty.
+ */
+ static const luaL_Reg lualib[] = {
+ {
+ .name = NULL,
+ },
+ };
+
+ serializer_lua = luaL_newserializer(L, NULL, lualib);
+ serializer_lua->has_compact = 1;
+ serializer_lua->encode_invalid_numbers = 1;
+ serializer_lua->encode_load_metatables = 1;
+ serializer_lua->encode_use_tostring = 1;
+ serializer_lua->encode_invalid_as_nil = 1;
+
+ /*
+ * Keep a reference to this module so it
+ * won't be unloaded.
+ */
+ lua_setfield(L, -2, "formatter_lua");
+}
diff --git a/src/box/lua/serialize_lua.h b/src/box/lua/serialize_lua.h
new file mode 100644
index 000000000..923e13be9
--- /dev/null
+++ b/src/box/lua/serialize_lua.h
@@ -0,0 +1,67 @@
+#ifndef INCLUDES_TARANTOOL_SERLIALIZE_LUA_H
+#define INCLUDES_TARANTOOL_SERLIALIZE_LUA_H
+
+/*
+ * Copyright 2010-2020, Tarantool AUTHORS, please see AUTHORS file.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the
+ * following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY <COPYRIGHT HOLDER> ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+ * <COPYRIGHT HOLDER> OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+ * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#if defined(__cplusplus)
+extern "C" {
+#endif /* defined(__cplusplus) */
+
+struct luaL_serializer;
+struct lua_State;
+
+/**
+ * Serializer options.
+ */
+typedef struct {
+ /** Number of indentation spaces. */
+ unsigned int indent_lvl;
+
+ /** Block mode output. */
+ bool block_mode;
+} lua_dumper_opts_t;
+
+void
+lua_serializer_init(struct lua_State *L);
+
+int
+lua_encode(lua_State *L, struct luaL_serializer *serializer,
+ lua_dumper_opts_t *opts);
+
+void
+lua_parse_opts(lua_State *L, lua_dumper_opts_t *opts);
+
+#if defined(__cplusplus)
+} /* extern "C" */
+#endif /* defined(__cplusplus) */
+
+#endif /* INCLUDES_TARANTOOL_SERLIALIZE_LUA_H */
--
2.26.2
^ permalink raw reply [flat|nested] 16+ messages in thread* [Tarantool-patches] [PATCH 6/7] box/console: switch to new lua serializer
2020-05-08 11:47 [Tarantool-patches] [PATCH 0/7] box/console: add support for internal types Cyrill Gorcunov
` (4 preceding siblings ...)
2020-05-08 11:47 ` [Tarantool-patches] [PATCH 5/7] box/console: implement lua serializer Cyrill Gorcunov
@ 2020-05-08 11:47 ` Cyrill Gorcunov
2020-05-08 11:47 ` [Tarantool-patches] [PATCH 7/7] test: extend console lua test Cyrill Gorcunov
` (2 subsequent siblings)
8 siblings, 0 replies; 16+ messages in thread
From: Cyrill Gorcunov @ 2020-05-08 11:47 UTC (permalink / raw)
To: tml
I do not drop the serpent module for a while
since I need to make sure that everything work
as expected as we've not break backward compatibility
significantly (though we didn't claim the lua mode
output is stable enough).
Fixes #4682
Signed-off-by: Cyrill Gorcunov <gorcunov@gmail.com>
---
src/box/lua/console.c | 77 +++++++++++++++++++++++++++++++
src/box/lua/console.lua | 71 ++++++++++++++--------------
test/app-tap/console_lua.test.lua | 18 ++++----
3 files changed, 120 insertions(+), 46 deletions(-)
diff --git a/src/box/lua/console.c b/src/box/lua/console.c
index c8e0dea58..d1bf7949c 100644
--- a/src/box/lua/console.c
+++ b/src/box/lua/console.c
@@ -39,6 +39,7 @@
#include "coio.h"
#include "lua/msgpack.h"
#include "lua-yaml/lyaml.h"
+#include "serialize_lua.h"
#include <lua.h>
#include <lauxlib.h>
#include <lualib.h>
@@ -50,6 +51,7 @@
extern char serpent_lua[];
static struct luaL_serializer *serializer_yaml;
+static struct luaL_serializer *serializer_lua;
/*
* Completion engine (Mike Paul's).
@@ -68,6 +70,54 @@ lua_rl_complete(lua_State *L, const char *text, int start, int end);
*/
static struct lua_State *readline_L;
+/**
+ * Encode Lua object into Lua form.
+ */
+static int
+lbox_console_format_lua(struct lua_State *L)
+{
+ lua_dumper_opts_t opts;
+ int arg_count;
+
+ /* Parse options and remove them */
+ lua_parse_opts(L, &opts);
+ lua_remove(L, 1);
+
+ arg_count = lua_gettop(L);
+
+ /* If nothing to process just exit early */
+ if (arg_count == 0)
+ return 0;
+
+ /*
+ * When processing arguments we might
+ * need to modify reference (for example
+ * when __index references to object itself)
+ * thus make a copy of incoming data.
+ *
+ * Note that in yaml there is no map for
+ * Lua's "nil" value so for yaml encoder
+ * we do
+ *
+ * if (lua_isnil(L, i + 1))
+ * luaL_pushnull(L);
+ * else
+ * lua_pushvalue(L, i + 1);
+ *
+ *
+ * For lua mode we have to preserve "nil".
+ */
+ lua_createtable(L, arg_count, 0);
+ for (int i = 0; i < arg_count; ++i) {
+ lua_pushvalue(L, i + 1);
+ lua_rawseti(L, -2, i + 1);
+ }
+
+ lua_replace(L, 1);
+ lua_settop(L, 1);
+ return lua_encode(L, serializer_lua, &opts);
+}
+
/*
* console_completion_handler()
* Called by readline to collect plausible completions;
@@ -557,6 +607,7 @@ tarantool_lua_console_init(struct lua_State *L)
{"add_history", lbox_console_add_history},
{"completion_handler", lbox_console_completion_handler},
{"format_yaml", lbox_console_format_yaml},
+ {"format_lua", lbox_console_format_lua},
{NULL, NULL}
};
luaL_register_module(L, "console", consolelib);
@@ -581,6 +632,32 @@ tarantool_lua_console_init(struct lua_State *L)
* load_history work the same way.
*/
lua_setfield(L, -2, "formatter");
+
+ /*
+ * We don't export it as a module
+ * for a while, so the library
+ * is kept empty.
+ */
+ static const luaL_Reg lualib[] = {
+ { },
+ };
+
+ serializer_lua = luaL_newserializer(L, NULL, lualib);
+ serializer_lua->has_compact = 1;
+ serializer_lua->encode_invalid_numbers = 1;
+ serializer_lua->encode_load_metatables = 1;
+ serializer_lua->encode_use_tostring = 1;
+ serializer_lua->encode_invalid_as_nil = 1;
+
+ /*
+ * Keep a reference to this module so it
+ * won't be unloaded.
+ */
+ lua_setfield(L, -2, "formatter_lua");
+
+ /* Output formatter in Lua mode */
+ lua_serializer_init(L);
+
struct session_vtab console_session_vtab = {
.push = console_session_push,
.fd = console_session_fd,
diff --git a/src/box/lua/console.lua b/src/box/lua/console.lua
index 5642ca956..e6e471a7d 100644
--- a/src/box/lua/console.lua
+++ b/src/box/lua/console.lua
@@ -69,47 +69,35 @@ output_handlers["yaml"] = function(status, opts, ...)
return internal.format_yaml({ error = err })
end
--- A map for internal symbols in case if they
--- are not inside tables and serpent won't be
--- able to handle them properly.
-local lua_map_direct_symbols = {
- [box.NULL] = 'box.NULL',
-}
-
--- A map for internal symbols in case if they
--- are coming from tables and we need to depict
--- them into user known values.
-local lua_map_table_symbols = {
- ['"cdata<void %*>: NULL"'] = 'box.NULL'
-}
-
--
--- Map internal symbols which serpent doesn't
--- know about to a known representation.
-local serpent_map_symbols = function(tag, head, body, tail, level)
- for k,v in pairs(lua_map_table_symbols) do
- body = body:gsub(k, v)
+-- Format a Lua value.
+local function format_lua_value(status, internal_opts, value)
+ local err
+ if status then
+ status, err = pcall(internal.format_lua, internal_opts, value)
+ if status then
+ return err
+ else
+ local m = 'console: exception while formatting output: "%s"'
+ err = m:format(tostring(err))
+ end
+ else
+ err = value
+ if err == nil then
+ err = box.NULL
+ end
end
- return tag..head..body..tail
+ return internal.format_lua(internal_opts, { error = err })
end
--
--- Format a Lua value.
-local function format_lua_value(value, opts)
- for k,v in pairs(lua_map_direct_symbols) do
- if k == value then
- return v
- end
- end
- local serpent_opts = {
- custom = serpent_map_symbols,
- comment = false,
- nocode = true,
- }
+-- Convert options from user form to the internal format.
+local function gen_lua_opts(opts)
if opts == "block" then
- return serpent.block(value, serpent_opts)
+ return {block=true, indent=2}
+ else
+ return {block=false, indent=2}
end
- return serpent.line(value, serpent_opts)
end
output_handlers["lua"] = function(status, opts, ...)
@@ -121,8 +109,17 @@ output_handlers["lua"] = function(status, opts, ...)
if not ... then
return output_eos["lua"]
end
+ --
+ -- Convert options to internal dictionary.
+ local internal_opts = gen_lua_opts(opts)
for i = 1, select('#', ...) do
- collect[i] = format_lua_value(select(i, ...), opts)
+ collect[i] = format_lua_value(status, internal_opts, select(i, ...))
+ if collect[i] == nil then
+ --
+ -- table.concat doesn't work for nil
+ -- so convert it explicitly.
+ collect[i] = "nil"
+ end
end
return table.concat(collect, ', ') .. output_eos["lua"]
end
@@ -210,8 +207,8 @@ end
-- Used by console_session_push.
box_internal.format_lua_push = function(value)
- local opts = current_output()["opts"]
- value = format_lua_value(value, opts)
+ local internal_opts = gen_lua_opts(current_output()["opts"])
+ value = format_lua_value(true, internal_opts, value)
return '-- Push\n' .. value .. ';'
end
diff --git a/test/app-tap/console_lua.test.lua b/test/app-tap/console_lua.test.lua
index 3ed6aad97..8bb6eb0b3 100755
--- a/test/app-tap/console_lua.test.lua
+++ b/test/app-tap/console_lua.test.lua
@@ -102,29 +102,29 @@ local cases = {
prepare = 'a = {1, 2, 3}',
opts = {block = false},
input = '1, 2, nil, a',
- expected = '1, 2, box.NULL, {1, 2, 3}',
+ expected = '1, 2, nil, {1, 2, 3}',
cleanup = 'a = nil',
}, {
name = 'multireturn block mode',
prepare = 'a = {1, 2, 3}',
opts = {block = true},
input = '1, 2, nil, a',
- expected = '1, 2, box.NULL, {\n 1,\n 2,\n 3\n}',
+ expected = '1, 2, nil, {\n 1,\n 2,\n 3\n}',
cleanup = 'a = nil',
}, {
- name = 'trailing nils, line mode',
+ name = 'trailing nils, box.NULL, line mode',
opts = {block = false},
- input = '1, nil, nil, nil',
- expected = '1, box.NULL, box.NULL, box.NULL',
+ input = '1, nil, box.NULL, nil',
+ expected = '1, nil, box.NULL, nil',
}, {
- name = 'trailing nils, block mode',
+ name = 'trailing nils, box.NULL, block mode',
opts = {block = true},
- input = '1, nil, nil, nil',
- expected = '1, box.NULL, box.NULL, box.NULL',
+ input = '1, nil, box.NULL, nil',
+ expected = '1, nil, box.NULL, nil',
}, {
name = 'empty output',
input = '\\set output',
- expected = '"Specify output format: lua or yaml."',
+ expected = '{error = \"Specify output format: lua or yaml.\"}',
}
}
--
2.26.2
^ permalink raw reply [flat|nested] 16+ messages in thread