[PATCH 2/5] lua: port console yaml formatting to C

[Vladislav Shpilevoy <v.shpilevoy@tarantool.org>]

Box.session.push() will be implemented in C lbox_session_push()
function, which will use port to encapsulate different session
types (binary, text) push() logic. And push() must be able to
either encode an argument into message pack, or format it as a
string using yaml. This formatting can not be done in Lua before
push() call, since it breaks push() virtualization.

Needed for #2677

Signed-off-by: Vladislav Shpilevoy <v.shpilevoy@tarantool.org>
---
 src/box/lua/console.c         | 42 ++++++++++++++++++++++++++++++++++++++++++
 src/box/lua/console.lua       | 34 ++++++++--------------------------
 third_party/lua-yaml/lyaml.cc |  9 +++------
 third_party/lua-yaml/lyaml.h  |  3 +++
 4 files changed, 56 insertions(+), 32 deletions(-)

diff --git a/src/box/lua/console.c b/src/box/lua/console.c
index d27d7ecac..450745c90 100644
--- a/src/box/lua/console.c
+++ b/src/box/lua/console.c
@@ -32,6 +32,7 @@
 #include "box/lua/console.h"
 #include "lua/utils.h"
 #include "lua/fiber.h"
+#include "lua-yaml/lyaml.h"
 #include "fiber.h"
 #include "coio.h"
 #include <lua.h>
@@ -328,6 +329,32 @@ lbox_console_add_history(struct lua_State *L)
 	return 0;
 }
 
+static int
+lbox_console_format(struct lua_State *L)
+{
+	int arg_count = lua_gettop(L);
+	if (arg_count == 0) {
+		lua_pushstring(L, "---\n...\n");
+		return 1;
+	}
+	lua_getfield(L, LUA_REGISTRYINDEX, "_LOADED");
+	lua_getfield(L, -1, "console");
+	lua_getfield(L, -1, "formatter");
+	lua_getfield(L, -1, "encode");
+	lua_createtable(L, arg_count, 0);
+	for (int i = 0; i < arg_count; ++i) {
+		if (lua_isnil(L, i + 1))
+			lua_getfield(L, -3, "NULL");
+		else
+			lua_pushvalue(L, i + 1);
+		lua_rawseti(L, -2, i + 1);
+	}
+	lua_call(L, 1, 1);
+	lua_insert(L, -4);
+	lua_pop(L, 3);
+	return 1;
+}
+
 void
 tarantool_lua_console_init(struct lua_State *L)
 {
@@ -336,6 +363,7 @@ tarantool_lua_console_init(struct lua_State *L)
 		{"save_history",       lbox_console_save_history},
 		{"add_history",        lbox_console_add_history},
 		{"completion_handler", lbox_console_completion_handler},
+		{"format",             lbox_console_format},
 		{NULL, NULL}
 	};
 	luaL_register_module(L, "console", consolelib);
@@ -344,6 +372,20 @@ tarantool_lua_console_init(struct lua_State *L)
 	lua_getfield(L, -1, "completion_handler");
 	lua_pushcclosure(L, lbox_console_readline, 1);
 	lua_setfield(L, -2, "readline");
+
+	lua_yaml_new_formatter(L);
+	lua_getfield(L, -1, "cfg");
+	lua_createtable(L, 0, 4);
+	lua_pushboolean(L, true);
+	lua_setfield(L, -2, "encode_invalid_numbers");
+	lua_pushboolean(L, true);
+	lua_setfield(L, -2, "encode_load_metatables");
+	lua_pushboolean(L, true);
+	lua_setfield(L, -2, "encode_use_tostring");
+	lua_pushboolean(L, true);
+	lua_setfield(L, -2, "encode_invalid_as_nil");
+	lua_call(L, 1, 0);
+	lua_setfield(L, -2, "formatter");
 }
 
 /*
diff --git a/src/box/lua/console.lua b/src/box/lua/console.lua
index d49cf42be..b4199ef85 100644
--- a/src/box/lua/console.lua
+++ b/src/box/lua/console.lua
@@ -9,34 +9,11 @@ local errno = require('errno')
 local urilib = require('uri')
 local yaml = require('yaml')
 
--- admin formatter must be able to encode any Lua variable
-local formatter = yaml.new()
-formatter.cfg{
-    encode_invalid_numbers = true;
-    encode_load_metatables = true;
-    encode_use_tostring    = true;
-    encode_invalid_as_nil  = true;
-}
-
 local function format(status, ...)
-    -- When storing a nil in a Lua table, there is no way to
-    -- distinguish nil value from no value. This is a trick to
-    -- make sure yaml converter correctly
-    local function wrapnull(v)
-        return v == nil and formatter.NULL or v
-    end
     local err
     if status then
-        local count = select('#', ...)
-        if count == 0 then
-            return "---\n...\n"
-        end
-        local res = {}
-        for i=1,count,1 do
-            table.insert(res, wrapnull(select(i, ...)))
-        end
         -- serializer can raise an exception
-        status, err = pcall(formatter.encode, res)
+        status, err = pcall(internal.format, ...)
         if status then
             return err
         else
@@ -44,9 +21,12 @@ local function format(status, ...)
                 tostring(err)
         end
     else
-        err = wrapnull(...)
+        err = ...
+        if err == nil then
+            err = box.NULL
+        end
     end
-    return formatter.encode({{error = err }})
+    return internal.format({ error = err })
 end
 
 --
@@ -395,4 +375,6 @@ package.loaded['console'] = {
     on_start = on_start;
     on_client_disconnect = on_client_disconnect;
     completion_handler = internal.completion_handler;
+    formatter = internal.formatter;
+    format = internal.format;
 }
diff --git a/third_party/lua-yaml/lyaml.cc b/third_party/lua-yaml/lyaml.cc
index 4d875fab4..25d29e01c 100644
--- a/third_party/lua-yaml/lyaml.cc
+++ b/third_party/lua-yaml/lyaml.cc
@@ -679,18 +679,15 @@ static int l_dump(lua_State *L) {
    return 1;
 }
 
-static int
-l_new(lua_State *L);
-
 static const luaL_Reg yamllib[] = {
    { "encode", l_dump },
    { "decode", l_load },
-   { "new",    l_new },
+   { "new",    lua_yaml_new_formatter },
    { NULL, NULL}
 };
 
-static int
-l_new(lua_State *L)
+int
+lua_yaml_new_formatter(lua_State *L)
 {
    struct luaL_serializer *s = luaL_newserializer(L, NULL, yamllib);
    s->has_compact = 1;
diff --git a/third_party/lua-yaml/lyaml.h b/third_party/lua-yaml/lyaml.h
index 650a2d747..9f9989f0c 100644
--- a/third_party/lua-yaml/lyaml.h
+++ b/third_party/lua-yaml/lyaml.h
@@ -10,6 +10,9 @@ extern "C" {
 LUALIB_API int
 luaopen_yaml(lua_State *L);
 
+int
+lua_yaml_new_formatter(lua_State *L);
+
 #ifdef __cplusplus
 }
 #endif
-- 
2.14.3 (Apple Git-98)