From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: From: Kirill Shcherbatov Subject: [PATCH v2 8/9] box: implement lua_port dump to region and to Lua Date: Thu, 6 Jun 2019 15:04:04 +0300 Message-Id: In-Reply-To: References: MIME-Version: 1.0 Content-Transfer-Encoding: 8bit To: tarantool-patches@freelists.org, vdavydov.dev@gmail.com Cc: Kirill Shcherbatov List-ID: Refactored port_lua class to reuse an existent machinery to dump info not only for obuf, but to region, that is also mpstream-compatible. We need this feature in scope of multikey indexes to work with keys produced with functional index extractor in memory. Also introduce a tiny method .dump_lua for port_lua. It is necessary to export registered on-board functions call endpoints. Class implementation is moved to a new file port_lua.c. Part of #4182 Needed for #1260 --- src/box/CMakeLists.txt | 1 + src/box/execute.c | 1 + src/box/lua/call.c | 254 +------------------------------- src/box/lua/port_lua.c | 318 +++++++++++++++++++++++++++++++++++++++++ src/box/port.h | 2 +- src/lib/core/port.h | 16 +++ 6 files changed, 338 insertions(+), 254 deletions(-) create mode 100644 src/box/lua/port_lua.c diff --git a/src/box/CMakeLists.txt b/src/box/CMakeLists.txt index 0864c3433..5f095f1f0 100644 --- a/src/box/CMakeLists.txt +++ b/src/box/CMakeLists.txt @@ -146,6 +146,7 @@ add_library(box STATIC lua/execute.c lua/key_def.c lua/merger.c + lua/port_lua.c ${bin_sources}) target_link_libraries(box box_error tuple stat xrow xlog vclock crc32 scramble diff --git a/src/box/execute.c b/src/box/execute.c index a3d4a92b8..2e27b6a60 100644 --- a/src/box/execute.c +++ b/src/box/execute.c @@ -106,6 +106,7 @@ port_sql_destroy(struct port *base) const struct port_vtab port_sql_vtab = { /* .dump_msgpack = */ port_sql_dump_msgpack, /* .dump_msgpack_16 = */ NULL, + /* .dump_msgpack_region = */ NULL, /* .dump_lua = */ port_sql_dump_lua, /* .dump_plain = */ NULL, /* .destroy = */ port_sql_destroy, diff --git a/src/box/lua/call.c b/src/box/lua/call.c index 4d4521363..e40c9a3f7 100644 --- a/src/box/lua/call.c +++ b/src/box/lua/call.c @@ -66,161 +66,6 @@ lbox_call_loadproc(struct lua_State *L) return count; } -/* - * Encode CALL_16 result. - * - * To allow clients to understand a complex return from - * a procedure, we are compatible with SELECT protocol, - * and return the number of return values first, and - * then each return value as a tuple. - * - * The following conversion rules apply: - * - * If a Lua stack contains at least one scalar, each - * value on the stack is converted to a tuple. A stack - * containing a single Lua table with scalars is converted to - * a tuple with multiple fields. - * - * If the stack is a Lua table, each member of which is - * not scalar, each member of the table is converted to - * a tuple. This way very large lists of return values can - * be used, since Lua stack size is limited by 8000 elements, - * while Lua table size is pretty much unlimited. - * - * Please read gh-291 carefully before "fixing" this code. - */ -static inline uint32_t -luamp_encode_call_16(lua_State *L, struct luaL_serializer *cfg, - struct mpstream *stream) -{ - int nrets = lua_gettop(L); - if (nrets == 0) { - return 0; - } else if (nrets > 1) { - /* - * Multireturn: - * `return 1, box.tuple.new(...), array, 3, ...` - */ - for (int i = 1; i <= nrets; ++i) { - struct luaL_field field; - if (luaL_tofield(L, cfg, i, &field) < 0) - return luaT_error(L); - struct tuple *tuple; - if (field.type == MP_EXT && - (tuple = luaT_istuple(L, i)) != NULL) { - /* `return ..., box.tuple.new(...), ...` */ - tuple_to_mpstream(tuple, stream); - } else if (field.type != MP_ARRAY) { - /* - * `return ..., scalar, ... => - * ..., { scalar }, ...` - */ - lua_pushvalue(L, i); - mpstream_encode_array(stream, 1); - luamp_encode_r(L, cfg, stream, &field, 0); - lua_pop(L, 1); - } else { - /* `return ..., array, ...` */ - luamp_encode(L, cfg, stream, i); - } - } - return nrets; - } - assert(nrets == 1); - - /* - * Inspect the first result - */ - struct luaL_field root; - if (luaL_tofield(L, cfg, 1, &root) < 0) - return luaT_error(L); - struct tuple *tuple; - if (root.type == MP_EXT && (tuple = luaT_istuple(L, 1)) != NULL) { - /* `return box.tuple()` */ - tuple_to_mpstream(tuple, stream); - return 1; - } else if (root.type != MP_ARRAY) { - /* - * `return scalar` - * `return map` - */ - mpstream_encode_array(stream, 1); - assert(lua_gettop(L) == 1); - luamp_encode_r(L, cfg, stream, &root, 0); - return 1; - } - - assert(root.type == MP_ARRAY); - if (root.size == 0) { - /* `return {}` => `{ box.tuple() }` */ - mpstream_encode_array(stream, 0); - return 1; - } - - /* `return { tuple, scalar, tuple }` */ - assert(root.type == MP_ARRAY && root.size > 0); - for (uint32_t t = 1; t <= root.size; t++) { - lua_rawgeti(L, 1, t); - struct luaL_field field; - if (luaL_tofield(L, cfg, -1, &field) < 0) - return luaT_error(L); - if (field.type == MP_EXT && (tuple = luaT_istuple(L, -1))) { - tuple_to_mpstream(tuple, stream); - } else if (field.type != MP_ARRAY) { - /* The first member of root table is not tuple/array */ - if (t == 1) { - /* - * `return { scalar, ... } => - * box.tuple.new(scalar, ...)` - */ - mpstream_encode_array(stream, root.size); - /* - * Encode the first field of tuple using - * existing information from luaL_tofield - */ - luamp_encode_r(L, cfg, stream, &field, 0); - lua_pop(L, 1); - assert(lua_gettop(L) == 1); - /* Encode remaining fields as usual */ - for (uint32_t f = 2; f <= root.size; f++) { - lua_rawgeti(L, 1, f); - luamp_encode(L, cfg, stream, -1); - lua_pop(L, 1); - } - return 1; - } - /* - * `return { tuple/array, ..., scalar, ... } => - * { tuple/array, ..., { scalar }, ... }` - */ - mpstream_encode_array(stream, 1); - luamp_encode_r(L, cfg, stream, &field, 0); - } else { - /* `return { tuple/array, ..., tuple/array, ... }` */ - luamp_encode_r(L, cfg, stream, &field, 0); - } - lua_pop(L, 1); - assert(lua_gettop(L) == 1); - } - return root.size; -} - -static const struct port_vtab port_lua_vtab; - -void -port_lua_create(struct port *port, struct lua_State *L) -{ - struct port_lua *port_lua = (struct port_lua *) port; - memset(port_lua, 0, sizeof(*port_lua)); - port_lua->vtab = &port_lua_vtab; - port_lua->L = L; - /* - * Allow to destroy the port even if no ref. - * @Sa luaL_unref. - */ - port_lua->ref = -1; -} - static int execute_lua_eval(lua_State *L) { @@ -248,103 +93,6 @@ execute_lua_eval(lua_State *L) return lua_gettop(L); } -static int -encode_lua_call(lua_State *L) -{ - struct port_lua *port = (struct port_lua *) lua_topointer(L, -1); - /* - * Add all elements from Lua stack to the buffer. - * - * TODO: forbid explicit yield from __serialize or __index here - */ - struct mpstream stream; - mpstream_init(&stream, port->out, obuf_reserve_cb, obuf_alloc_cb, - luamp_error, port->L); - - struct luaL_serializer *cfg = luaL_msgpack_default; - int size = lua_gettop(port->L); - for (int i = 1; i <= size; ++i) - luamp_encode(port->L, cfg, &stream, i); - port->size = size; - mpstream_flush(&stream); - return 0; -} - -static int -encode_lua_call_16(lua_State *L) -{ - struct port_lua *port = (struct port_lua *) lua_topointer(L, -1); - /* - * Add all elements from Lua stack to the buffer. - * - * TODO: forbid explicit yield from __serialize or __index here - */ - struct mpstream stream; - mpstream_init(&stream, port->out, obuf_reserve_cb, obuf_alloc_cb, - luamp_error, port->L); - - struct luaL_serializer *cfg = luaL_msgpack_default; - port->size = luamp_encode_call_16(port->L, cfg, &stream); - mpstream_flush(&stream); - return 0; -} - -static inline int -port_lua_do_dump(struct port *base, struct obuf *out, lua_CFunction handler) -{ - struct port_lua *port = (struct port_lua *)base; - assert(port->vtab == &port_lua_vtab); - /* Use port to pass arguments to encoder quickly. */ - port->out = out; - /* - * Use the same global state, assuming the encoder doesn't - * yield. - */ - struct lua_State *L = tarantool_L; - int top = lua_gettop(L); - if (lua_cpcall(L, handler, port) != 0) { - luaT_toerror(port->L); - return -1; - } - lua_settop(L, top); - return port->size; -} - -static int -port_lua_dump(struct port *base, struct obuf *out) -{ - return port_lua_do_dump(base, out, encode_lua_call); -} - -static int -port_lua_dump_16(struct port *base, struct obuf *out) -{ - return port_lua_do_dump(base, out, encode_lua_call_16); -} - -static void -port_lua_destroy(struct port *base) -{ - struct port_lua *port = (struct port_lua *)base; - assert(port->vtab == &port_lua_vtab); - luaL_unref(tarantool_L, LUA_REGISTRYINDEX, port->ref); -} - -/** - * Dump port lua as a YAML document. It is extern since depends on - * lyaml module. - */ -extern const char * -port_lua_dump_plain(struct port *port, uint32_t *size); - -static const struct port_vtab port_lua_vtab = { - .dump_msgpack = port_lua_dump, - .dump_msgpack_16 = port_lua_dump_16, - .dump_lua = NULL, - .dump_plain = port_lua_dump_plain, - .destroy = port_lua_destroy, -}; - static inline int box_process_lua(struct call_request *request, struct port *base, lua_CFunction handler) @@ -357,7 +105,7 @@ box_process_lua(struct call_request *request, struct port *base, lua_pushcfunction(L, handler); lua_pushlightuserdata(L, request); if (luaT_call(L, 1, LUA_MULTRET) != 0) { - port_lua_destroy(base); + port_destroy(base); return -1; } return 0; diff --git a/src/box/lua/port_lua.c b/src/box/lua/port_lua.c new file mode 100644 index 000000000..087650281 --- /dev/null +++ b/src/box/lua/port_lua.c @@ -0,0 +1,318 @@ +/* + * Copyright 2010-2019, Tarantool AUTHORS, please see AUTHORS file. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * 1. Redistributions of source code must retain the above + * copyright notice, this list of conditions and the + * following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF + * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ +#include "mpstream.h" +#include "box/func.h" +#include "box/tuple.h" +#include "box/lua/tuple.h" +#include "box/lua/misc.h" +#include "small/region.h" +#include "lua/utils.h" +#include "lua/msgpack.h" +#include "box/port.h" +#include "small/obuf.h" +#include + +static const struct port_vtab port_lua_vtab; + +/* + * Encode CALL_16 result. + * + * To allow clients to understand a complex return from + * a procedure, we are compatible with SELECT protocol, + * and return the number of return values first, and + * then each return value as a tuple. + * + * The following conversion rules apply: + * + * If a Lua stack contains at least one scalar, each + * value on the stack is converted to a tuple. A stack + * containing a single Lua table with scalars is converted to + * a tuple with multiple fields. + * + * If the stack is a Lua table, each member of which is + * not scalar, each member of the table is converted to + * a tuple. This way very large lists of return values can + * be used, since Lua stack size is limited by 8000 elements, + * while Lua table size is pretty much unlimited. + * + * Please read gh-291 carefully before "fixing" this code. + */ +static inline uint32_t +luamp_encode_call_16(lua_State *L, struct luaL_serializer *cfg, + struct mpstream *stream) +{ + int nrets = lua_gettop(L); + if (nrets == 0) { + return 0; + } else if (nrets > 1) { + /* + * Multireturn: + * `return 1, box.tuple.new(...), array, 3, ...` + */ + for (int i = 1; i <= nrets; ++i) { + struct luaL_field field; + if (luaL_tofield(L, cfg, i, &field) < 0) + return luaT_error(L); + struct tuple *tuple; + if (field.type == MP_EXT && + (tuple = luaT_istuple(L, i)) != NULL) { + /* `return ..., box.tuple.new(...), ...` */ + tuple_to_mpstream(tuple, stream); + } else if (field.type != MP_ARRAY) { + /* + * `return ..., scalar, ... => + * ..., { scalar }, ...` + */ + lua_pushvalue(L, i); + mpstream_encode_array(stream, 1); + luamp_encode_r(L, cfg, stream, &field, 0); + lua_pop(L, 1); + } else { + /* `return ..., array, ...` */ + luamp_encode(L, cfg, stream, i); + } + } + return nrets; + } + assert(nrets == 1); + + /* + * Inspect the first result + */ + struct luaL_field root; + if (luaL_tofield(L, cfg, 1, &root) < 0) + return luaT_error(L); + struct tuple *tuple; + if (root.type == MP_EXT && (tuple = luaT_istuple(L, 1)) != NULL) { + /* `return box.tuple()` */ + tuple_to_mpstream(tuple, stream); + return 1; + } else if (root.type != MP_ARRAY) { + /* + * `return scalar` + * `return map` + */ + mpstream_encode_array(stream, 1); + assert(lua_gettop(L) == 1); + luamp_encode_r(L, cfg, stream, &root, 0); + return 1; + } + + assert(root.type == MP_ARRAY); + if (root.size == 0) { + /* `return {}` => `{ box.tuple() }` */ + mpstream_encode_array(stream, 0); + return 1; + } + + /* `return { tuple, scalar, tuple }` */ + assert(root.type == MP_ARRAY && root.size > 0); + for (uint32_t t = 1; t <= root.size; t++) { + lua_rawgeti(L, 1, t); + struct luaL_field field; + if (luaL_tofield(L, cfg, -1, &field) < 0) + return luaT_error(L); + if (field.type == MP_EXT && (tuple = luaT_istuple(L, -1))) { + tuple_to_mpstream(tuple, stream); + } else if (field.type != MP_ARRAY) { + /* The first member of root table is not tuple/array */ + if (t == 1) { + /* + * `return { scalar, ... } => + * box.tuple.new(scalar, ...)` + */ + mpstream_encode_array(stream, root.size); + /* + * Encode the first field of tuple using + * existing information from luaL_tofield + */ + luamp_encode_r(L, cfg, stream, &field, 0); + lua_pop(L, 1); + assert(lua_gettop(L) == 1); + /* Encode remaining fields as usual */ + for (uint32_t f = 2; f <= root.size; f++) { + lua_rawgeti(L, 1, f); + luamp_encode(L, cfg, stream, -1); + lua_pop(L, 1); + } + return 1; + } + /* + * `return { tuple/array, ..., scalar, ... } => + * { tuple/array, ..., { scalar }, ... }` + */ + mpstream_encode_array(stream, 1); + luamp_encode_r(L, cfg, stream, &field, 0); + } else { + /* `return { tuple/array, ..., tuple/array, ... }` */ + luamp_encode_r(L, cfg, stream, &field, 0); + } + lua_pop(L, 1); + assert(lua_gettop(L) == 1); + } + return root.size; +} + +static inline int +port_lua_do_dump(struct port *base, struct mpstream *stream, + lua_CFunction handler) +{ + struct port_lua *port = (struct port_lua *)base; + assert(port->vtab == &port_lua_vtab); + /* Use port to pass arguments to encoder quickly. */ + port->stream = stream; + /* + * Use the same global state, assuming the encoder doesn't + * yield. + */ + struct lua_State *L = tarantool_L; + int top = lua_gettop(L); + if (lua_cpcall(L, handler, port) != 0) { + luaT_toerror(port->L); + return -1; + } + lua_settop(L, top); + return port->size; +} + +static int +encode_lua_call(lua_State *L) +{ + struct port_lua *port = (struct port_lua *) lua_topointer(L, -1); + /* + * Add all elements from Lua stack to the buffer. + * + * TODO: forbid explicit yield from __serialize or __index here + */ + struct luaL_serializer *cfg = luaL_msgpack_default; + int size = lua_gettop(port->L); + for (int i = 1; i <= size; ++i) + luamp_encode(port->L, cfg, port->stream, i); + port->size = size; + mpstream_flush(port->stream); + return 0; +} + +static int +encode_lua_call_16(lua_State *L) +{ + struct port_lua *port = (struct port_lua *) lua_topointer(L, -1); + /* + * Add all elements from Lua stack to the buffer. + * + * TODO: forbid explicit yield from __serialize or __index here + */ + struct luaL_serializer *cfg = luaL_msgpack_default; + port->size = luamp_encode_call_16(port->L, cfg, port->stream); + mpstream_flush(port->stream); + return 0; +} + +static int +port_lua_dump(struct port *base, struct obuf *obuf) +{ + struct port_lua *port = (struct port_lua *)base; + struct mpstream stream; + mpstream_init(&stream, obuf, obuf_reserve_cb, obuf_alloc_cb, + luamp_error, port->L); + return port_lua_do_dump(base, &stream, encode_lua_call); +} + +static int +port_lua_dump_16(struct port *base, struct obuf *obuf) +{ + struct port_lua *port = (struct port_lua *)base; + struct mpstream stream; + mpstream_init(&stream, obuf, obuf_reserve_cb, obuf_alloc_cb, + luamp_error, port->L); + return port_lua_do_dump(base, &stream, encode_lua_call_16); +} + +static int +port_lua_dump_region(struct port *base, struct region *region) +{ + struct port_lua *port = (struct port_lua *)base; + struct mpstream stream; + mpstream_init(&stream, region, region_reserve_cb, region_alloc_cb, + luamp_error, port->L); + return port_lua_do_dump(base, &stream, encode_lua_call); +} + +static void +port_lua_dump_lua(struct port *base, struct lua_State *L) +{ + struct port_lua *port = (struct port_lua *) base; + uint32_t size = lua_gettop(port->L); + lua_createtable(L, size, 0); + for (uint32_t i = 0; i < size; i++) { + lua_xmove(port->L, L, 1); + lua_rawseti(L, -2, i + 1); + } + port->size = 1; +} + +static void +port_lua_destroy(struct port *base) +{ + struct port_lua *port = (struct port_lua *)base; + assert(port->vtab == &port_lua_vtab); + luaL_unref(tarantool_L, LUA_REGISTRYINDEX, port->ref); +} + +/** + * Dump port lua as a YAML document. It is extern since depends on + * lyaml module. + */ +extern const char * +port_lua_dump_plain(struct port *port, uint32_t *size); + +static const struct port_vtab port_lua_vtab = { + .dump_msgpack = port_lua_dump, + .dump_msgpack_16 = port_lua_dump_16, + .dump_msgpack_region = port_lua_dump_region, + .dump_lua = port_lua_dump_lua, + .dump_plain = port_lua_dump_plain, + .destroy = port_lua_destroy, +}; + +void +port_lua_create(struct port *port, struct lua_State *L) +{ + struct port_lua *port_lua = (struct port_lua *) port; + memset(port_lua, 0, sizeof(*port_lua)); + port_lua->vtab = &port_lua_vtab; + port_lua->L = L; + /* + * Allow to destroy the port even if no ref. + * @Sa luaL_unref. + */ + port_lua->ref = LUA_REFNIL; +} diff --git a/src/box/port.h b/src/box/port.h index f18803660..28fc16ce9 100644 --- a/src/box/port.h +++ b/src/box/port.h @@ -88,7 +88,7 @@ struct port_lua { /** Reference to L in tarantool_L. */ int ref; /** The argument of port_dump */ - struct obuf *out; + struct mpstream *stream; /** Number of entries dumped to the port. */ int size; }; diff --git a/src/lib/core/port.h b/src/lib/core/port.h index 8ace40fc5..e2f657ce0 100644 --- a/src/lib/core/port.h +++ b/src/lib/core/port.h @@ -37,6 +37,7 @@ extern "C" { #endif /* defined(__cplusplus) */ struct obuf; +struct region; struct lua_State; struct port; @@ -62,6 +63,15 @@ struct port_vtab { * header. Used by the legacy Tarantool 1.6 format. */ int (*dump_msgpack_16)(struct port *port, struct obuf *out); + /** + * Dump the content of a port to a region. + * @param port Port to dump. + * @param region Region to dump to. + * + * @retval >= 0 Number of entries dumped. + * @retval < 0 Error. + */ + int (*dump_msgpack_region)(struct port *port, struct region *region); /** Dump the content of a port to Lua stack. */ void (*dump_lua)(struct port *port, struct lua_State *L); /** @@ -108,6 +118,12 @@ port_dump_msgpack_16(struct port *port, struct obuf *out) return port->vtab->dump_msgpack_16(port, out); } +static inline int +port_dump_msgpack_region(struct port *port, struct region *region) +{ + return port->vtab->dump_msgpack_region(port, region); +} + static inline void port_dump_lua(struct port *port, struct lua_State *L) { -- 2.21.0