From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from localhost (localhost [127.0.0.1]) by turing.freelists.org (Avenir Technologies Mail Multiplex) with ESMTP id ED2223157C for ; Thu, 20 Jun 2019 17:54:12 -0400 (EDT) Received: from turing.freelists.org ([127.0.0.1]) by localhost (turing.freelists.org [127.0.0.1]) (amavisd-new, port 10024) with ESMTP id 3JooRvDMoKOn for ; Thu, 20 Jun 2019 17:54:12 -0400 (EDT) Received: from mail-lj1-f175.google.com (mail-lj1-f175.google.com [209.85.208.175]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by turing.freelists.org (Avenir Technologies Mail Multiplex) with ESMTPS id 639753157B for ; Thu, 20 Jun 2019 17:54:10 -0400 (EDT) Received: by mail-lj1-f175.google.com with SMTP id x25so4123653ljh.2 for ; Thu, 20 Jun 2019 14:54:10 -0700 (PDT) From: Cyrill Gorcunov Subject: [tarantool-patches] [RFC] box/lua/console: Add console.fmt module Date: Fri, 21 Jun 2019 00:54:05 +0300 Message-Id: <20190620215405.9338-1-gorcunov@gmail.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit Sender: tarantool-patches-bounce@freelists.org Errors-to: tarantool-patches-bounce@freelists.org Reply-To: tarantool-patches@freelists.org List-Help: List-Unsubscribe: List-software: Ecartis version 1.0.0 List-Id: tarantool-patches List-Subscribe: List-Owner: List-post: List-Archive: To: tml Cc: Alexander Turenko , Cyrill Gorcunov The rationale of the module is to provide a way to configure console output format, in particular one might need to copy results of eval and paste it into some other command. To switch console into that named lua mode one need to type > require('console.fmt').set_format('lua') To set it back to yaml mode just type > require('console.fmt').set_format('yaml') At the moment these two modes are supprted only. Some details on internal structure: general console code sits in src/box/lua/[console.c|console.lua] files; the C file is resposible for readline library api and yaml serializer. In turn the "formatter" now moved into src/box/lua/console_fmt.lua and coupled with general console code. General console code simply calls for formatter and provides output depending on active engine (by default we continue use yaml backend in a sake of backward compatibility). There is at least one explicit problem remains: console text protocol (console.lua:wrap_text_socket()) it waits the other peer to reply in yaml format and tries to decode it. According to repo history the text protocol is deprecated by now so I think we might safely remove the code completely. Please review carefully, any comments/complains are highly appreciated. Of course I will need to provide some more test but first would like to hear early feedback on the code design. https://github.com/tarantool/tarantool/issues/3834 --- src/box/CMakeLists.txt | 1 + src/box/lua/console.lua | 41 ++--- src/box/lua/console_fmt.lua | 295 ++++++++++++++++++++++++++++++++++++ src/box/lua/init.c | 2 + 4 files changed, 309 insertions(+), 30 deletions(-) create mode 100644 src/box/lua/console_fmt.lua diff --git a/src/box/CMakeLists.txt b/src/box/CMakeLists.txt index 481842a39..2f7e90e5e 100644 --- a/src/box/CMakeLists.txt +++ b/src/box/CMakeLists.txt @@ -11,6 +11,7 @@ lua_source(lua_sources lua/feedback_daemon.lua) lua_source(lua_sources lua/net_box.lua) lua_source(lua_sources lua/upgrade.lua) lua_source(lua_sources lua/console.lua) +lua_source(lua_sources lua/console_fmt.lua) lua_source(lua_sources lua/xlog.lua) lua_source(lua_sources lua/key_def.lua) lua_source(lua_sources lua/merger.lua) diff --git a/src/box/lua/console.lua b/src/box/lua/console.lua index f922f0320..9298ad44b 100644 --- a/src/box/lua/console.lua +++ b/src/box/lua/console.lua @@ -1,6 +1,7 @@ -- console.lua -- internal file local internal = require('console') +local cfmt = require('console.fmt') local session_internal = require('box.internal.session') local fiber = require('fiber') local socket = require('socket') @@ -13,26 +14,6 @@ local net_box = require('net.box') local YAML_TERM = '\n...\n' local PUSH_TAG_HANDLE = '!push!' -local function format(status, ...) - local err - if status then - -- serializer can raise an exception - status, err = pcall(internal.format, ...) - if status then - return err - else - err = 'console: an exception occurred when formatting the output: '.. - tostring(err) - end - else - err = ... - if err == nil then - err = box.NULL - end - end - return internal.format({ error = err }) -end - -- -- Set delimiter -- @@ -81,16 +62,16 @@ local function set_param(storage, func, param, value) d = set_delimiter } if param == nil then - return format(false, 'Invalid set syntax, type \\help for help') + return cfmt.format(false, 'Invalid set syntax, type \\help for help') end if params[param] == nil then - return format(false, 'Unknown parameter: ' .. tostring(param)) + return cfmt.format(false, 'Unknown parameter: ' .. tostring(param)) end - return format(pcall(params[param], storage, value)) + return cfmt.format(pcall(params[param], storage, value)) end local function help_wrapper(storage) - return format(true, help()) -- defined in help.lua + return cfmt.format(true, help()) -- defined in help.lua end local function quit(storage) @@ -119,7 +100,7 @@ local function preprocess(storage, line) end if operators[items[1]] == nil then local msg = "Invalid command \\%s. Type \\help for help." - return format(false, msg:format(items[1])) + return cfmt.format(false, msg:format(items[1])) end return operators[items[1]](storage, unpack(items)) end @@ -135,7 +116,7 @@ local function local_eval(storage, line) return preprocess(storage, line:sub(2)) end if storage.language == 'sql' then - return format(pcall(box.execute, line)) + return cfmt.format(pcall(box.execute, line)) end -- -- Attempt to append 'return ' before the chunk: if the chunk is @@ -148,9 +129,9 @@ local function local_eval(storage, line) fun, errmsg = loadstring(line) end if not fun then - return format(false, errmsg) + return cfmt.format(false, errmsg) end - return format(pcall(fun)) + return cfmt.format(pcall(fun)) end local function eval(line) @@ -258,7 +239,7 @@ local function remote_eval(self, line) if line and self.remote.state == 'active' then local ok, res = pcall(self.remote.eval, self.remote, line) if self.remote.state == 'active' then - return ok and res or format(false, res) + return ok and res or cfmt.format(false, res) end end local err = self.remote.error @@ -268,7 +249,7 @@ local function remote_eval(self, line) self.prompt = nil self.completion = nil pcall(self.on_client_disconnect, self) - return (err and format(false, err)) or '' + return (err and cfmt.format(false, err)) or '' end local function local_check_lua(buf) diff --git a/src/box/lua/console_fmt.lua b/src/box/lua/console_fmt.lua new file mode 100644 index 000000000..f860d76fb --- /dev/null +++ b/src/box/lua/console_fmt.lua @@ -0,0 +1,295 @@ +-- # vim: ts=4 sw=4 et + +local internal = require('console') + +local console_formats = { + ["lua"] = nil, + ["yaml"] = nil +} + +--- +--- Serializer in Lua syntax, which represent data +--- in a format suitable for Lua REPL. +--- + +local lua_keyword = { + ["and"] = true, ["break"] = true, ["do"] = true, + ["else"] = true, ["elseif"] = true, ["end"] = true, + ["false"] = true, ["for"] = true, ["function"] = true, + ["if"] = true, ["in"] = true, ["local"] = true, + ["nil"] = true, ["not"] = true, ["or"] = true, + ["repeat"] = true, ["return"] = true, ["then"] = true, + ["true"] = true, ["until"] = true, ["while"] = true +} + +local function has_lquote(s) + local lstring_pat = '([%[%]])(=*)%1' + local equals, new_equals, _ + local finish = 1 + repeat + _, finish, _, new_equals = s:find(lstring_pat, finish) + if new_equals then + equals = math.max(equals or 0, #new_equals) + end + until not new_equals + + return equals +end + +local function is_identifier(s) + return type(s) == 'string' and s:find('^[%a_][%w_]*$') and not lua_keyword[s] +end + +local function quote_string(s) + if type(s) ~= 'string' then + error("quote_string: s should be a string") + end + + -- Find out if there are any embedded long-quote sequences that may cause issues. + -- This is important when strings are embedded within strings, like when serializing. + -- Append a closing bracket to catch unfinished long-quote sequences at the end of the string. + local equal_signs = has_lquote(s .. "]") + + -- Note that strings containing "\r" can't be quoted using long brackets + -- as Lua lexer converts all newlines to "\n" within long strings. + if (s:find("\n") or equal_signs) and not s:find("\r") then + -- If there is an embedded sequence that matches a long quote, then + -- find the one with the maximum number of = signs and add one to that number. + equal_signs = ("="):rep((equal_signs or -1) + 1) + -- Long strings strip out leading newline. We want to retain that, when quoting. + if s:find("^\n") then s = "\n" .. s end + local lbracket, rbracket = + "[" .. equal_signs .. "[", + "]" .. equal_signs .. "]" + s = lbracket .. s .. rbracket + else + -- Escape funny stuff. Lua 5.1 does not handle "\r" correctly. + s = ("%q"):format(s):gsub("\r", "\\r") + end + return s +end + +local function quote_if_necessary(v) + if not v then + return '' + else + if v:find ' ' then + v = quote_string(v) + end + end + return v +end + +local function quote(s) + if type(s) == 'table' then + return console_formats["lua"](s, '') + else + return quote_string(s) + end +end + +local function index(numkey, key) + if not numkey then + key = quote(key) + key = key:find("^%[") and (" " .. key .. " ") or key + end + return '[' .. key .. ']' +end + +--- Create a string representation of a Lua table. +-- +-- This function never fails, but may complain by returning an +-- extra value. Normally puts out one item per line, using +-- the provided indent; set the second parameter to an empty string +-- if you want output on one line. +-- +-- @tab tbl Table to serialize to a string. +-- @string[opt] space The indent to use. +-- Defaults to two spaces; pass an empty string for no indentation. +-- @bool[opt] not_clever Pass `true` for plain output, e.g `{['key']=1}`. +-- Defaults to `false`. +-- +-- @return a string +console_formats["lua"] = function(tbl, space, not_clever) + if type(tbl) ~= 'table' then + if type(tbl) == 'string' then + return quote(tbl) + end + return tostring(tbl) + end + + local set = ' = ' + if space == '' then set = '=' end + space = space or ' ' + local lines = {} + local line = '' + local tables = {} + + local function put(s) + if #s > 0 then + line = line .. s + end + end + + local function putln(s) + if #line > 0 then + line = line .. s + table.insert(lines, line) + line = '' + else + table.insert(lines, s) + end + end + + local function eat_last_comma() + local n = #lines + local lastch = lines[n]:sub(-1,-1) + if lastch == ',' then + lines[n] = lines[n]:sub(1,-2) + end + end + + local function preprocess(t) + local specials = { + [box.NULL] = 'box.NULL' + } + if specials[t] then + return true, specials[t] + end + return false, nil + end + + local function writeit(t, oldindent, indent) + local pp, pps = preprocess(t) + local tp = type(t) + if pp then + putln(pps .. ',') + elseif tp ~= 'string' and tp ~= 'table' then + putln(quote_if_necessary(tostring(t)) .. ',') + elseif tp == 'string' then + putln(quote_string(t) .. ',') + elseif tp == 'table' then + if tables[t] then + putln(',') + return + end + + tables[t] = true + local newindent = indent .. space + putln('{') + local used = {} + if not not_clever then + for i,val in ipairs(t) do + put(indent) + writeit(val, indent, newindent) + used[i] = true + end + end + for key,val in pairs(t) do + local tkey = type(key) + local numkey = tkey == 'number' + if not_clever then + key = tostring(key) + put(indent .. index(numkey, key) .. set) + writeit(val, indent, newindent) + else + if not numkey or not used[key] then -- non-array indices + if tkey ~= 'string' then + key = tostring(key) + end + if numkey or not is_identifier(key) then + key = index(numkey, key) + end + put(indent .. key .. set) + writeit(val, indent, newindent) + end + end + end + tables[t] = nil + eat_last_comma() + putln(oldindent .. '},') + else + putln(tostring(t) .. ',') + end + end + writeit(tbl, '', space) + eat_last_comma() + return table.concat(lines, #space > 0 and '\n' or '') +end + +--- +--- Serializer in YAML format, basically we call for +--- external serializer which lives in C code. +--- + +console_formats["yaml"] = function(status, ...) + local err + if status then + -- serializer can raise an exception + status, err = pcall(internal.format, ...) + if status then + return err + else + err = 'console.fmt: an exception in output ' .. tostring(err) + end + else + err = ... + if err == nil then + err = box.NULL + end + return internal.format({ error = err }) + end +end + +-- +-- Default console format +-- +console_format = console_formats["yaml"] + +--- +--- Package API +--- + +-- +-- Setup requested format +local function set_format(name) + if console_formats[name] ~= nil then + console_format = console_formats[name] + else + error(string.format("console.fmt: Unsupported console format %s", name)) + end +end + +-- +-- Report currently used format +local function get_format() + for k,v in pairs(console_formats) do + if console_format == v then + return k + end + end +end + +-- +-- Main message formatter, choose a proper +-- format engine and pass arguments into. +local function format(status, ...) + if console_format == console_formats["lua"] then + local data = ... + if data == nil then + -- In case if there is no data just + -- return empty string to not spam + -- with "nil" message. + return "" + end + return console_format(..., '') + end + return console_format(status, ...) +end + +package.loaded['console.fmt'] = { + formats_map = console_formats, + set_format = set_format, + get_format = get_format, + format = format +} diff --git a/src/box/lua/init.c b/src/box/lua/init.c index 76b987b4b..aff84c87a 100644 --- a/src/box/lua/init.c +++ b/src/box/lua/init.c @@ -71,6 +71,7 @@ extern char session_lua[], feedback_daemon_lua[], net_box_lua[], upgrade_lua[], + console_fmt_lua[], console_lua[], merger_lua[]; @@ -81,6 +82,7 @@ static const char *lua_sources[] = { "box/feedback_daemon", feedback_daemon_lua, "box/upgrade", upgrade_lua, "box/net_box", net_box_lua, + "box/console_fmt", console_fmt_lua, "box/console", console_lua, "box/load_cfg", load_cfg_lua, "box/xlog", xlog_lua, -- 2.20.1