[PATCH v2 8/9] box: implement lua_port dump to region and to Lua
Kirill Shcherbatov
kshcherbatov at tarantool.org
Thu Jun 6 15:04:04 MSK 2019
Refactored port_lua class to reuse an existent machinery to
dump info not only for obuf, but to region, that is also
mpstream-compatible. We need this feature in scope of multikey
indexes to work with keys produced with functional index
extractor in memory.
Also introduce a tiny method .dump_lua for port_lua. It is
necessary to export registered on-board functions call endpoints.
Class implementation is moved to a new file port_lua.c.
Part of #4182
Needed for #1260
---
src/box/CMakeLists.txt | 1 +
src/box/execute.c | 1 +
src/box/lua/call.c | 254 +-------------------------------
src/box/lua/port_lua.c | 318 +++++++++++++++++++++++++++++++++++++++++
src/box/port.h | 2 +-
src/lib/core/port.h | 16 +++
6 files changed, 338 insertions(+), 254 deletions(-)
create mode 100644 src/box/lua/port_lua.c
diff --git a/src/box/CMakeLists.txt b/src/box/CMakeLists.txt
index 0864c3433..5f095f1f0 100644
--- a/src/box/CMakeLists.txt
+++ b/src/box/CMakeLists.txt
@@ -146,6 +146,7 @@ add_library(box STATIC
lua/execute.c
lua/key_def.c
lua/merger.c
+ lua/port_lua.c
${bin_sources})
target_link_libraries(box box_error tuple stat xrow xlog vclock crc32 scramble
diff --git a/src/box/execute.c b/src/box/execute.c
index a3d4a92b8..2e27b6a60 100644
--- a/src/box/execute.c
+++ b/src/box/execute.c
@@ -106,6 +106,7 @@ port_sql_destroy(struct port *base)
const struct port_vtab port_sql_vtab = {
/* .dump_msgpack = */ port_sql_dump_msgpack,
/* .dump_msgpack_16 = */ NULL,
+ /* .dump_msgpack_region = */ NULL,
/* .dump_lua = */ port_sql_dump_lua,
/* .dump_plain = */ NULL,
/* .destroy = */ port_sql_destroy,
diff --git a/src/box/lua/call.c b/src/box/lua/call.c
index 4d4521363..e40c9a3f7 100644
--- a/src/box/lua/call.c
+++ b/src/box/lua/call.c
@@ -66,161 +66,6 @@ lbox_call_loadproc(struct lua_State *L)
return count;
}
-/*
- * Encode CALL_16 result.
- *
- * To allow clients to understand a complex return from
- * a procedure, we are compatible with SELECT protocol,
- * and return the number of return values first, and
- * then each return value as a tuple.
- *
- * The following conversion rules apply:
- *
- * If a Lua stack contains at least one scalar, each
- * value on the stack is converted to a tuple. A stack
- * containing a single Lua table with scalars is converted to
- * a tuple with multiple fields.
- *
- * If the stack is a Lua table, each member of which is
- * not scalar, each member of the table is converted to
- * a tuple. This way very large lists of return values can
- * be used, since Lua stack size is limited by 8000 elements,
- * while Lua table size is pretty much unlimited.
- *
- * Please read gh-291 carefully before "fixing" this code.
- */
-static inline uint32_t
-luamp_encode_call_16(lua_State *L, struct luaL_serializer *cfg,
- struct mpstream *stream)
-{
- int nrets = lua_gettop(L);
- if (nrets == 0) {
- return 0;
- } else if (nrets > 1) {
- /*
- * Multireturn:
- * `return 1, box.tuple.new(...), array, 3, ...`
- */
- for (int i = 1; i <= nrets; ++i) {
- struct luaL_field field;
- if (luaL_tofield(L, cfg, i, &field) < 0)
- return luaT_error(L);
- struct tuple *tuple;
- if (field.type == MP_EXT &&
- (tuple = luaT_istuple(L, i)) != NULL) {
- /* `return ..., box.tuple.new(...), ...` */
- tuple_to_mpstream(tuple, stream);
- } else if (field.type != MP_ARRAY) {
- /*
- * `return ..., scalar, ... =>
- * ..., { scalar }, ...`
- */
- lua_pushvalue(L, i);
- mpstream_encode_array(stream, 1);
- luamp_encode_r(L, cfg, stream, &field, 0);
- lua_pop(L, 1);
- } else {
- /* `return ..., array, ...` */
- luamp_encode(L, cfg, stream, i);
- }
- }
- return nrets;
- }
- assert(nrets == 1);
-
- /*
- * Inspect the first result
- */
- struct luaL_field root;
- if (luaL_tofield(L, cfg, 1, &root) < 0)
- return luaT_error(L);
- struct tuple *tuple;
- if (root.type == MP_EXT && (tuple = luaT_istuple(L, 1)) != NULL) {
- /* `return box.tuple()` */
- tuple_to_mpstream(tuple, stream);
- return 1;
- } else if (root.type != MP_ARRAY) {
- /*
- * `return scalar`
- * `return map`
- */
- mpstream_encode_array(stream, 1);
- assert(lua_gettop(L) == 1);
- luamp_encode_r(L, cfg, stream, &root, 0);
- return 1;
- }
-
- assert(root.type == MP_ARRAY);
- if (root.size == 0) {
- /* `return {}` => `{ box.tuple() }` */
- mpstream_encode_array(stream, 0);
- return 1;
- }
-
- /* `return { tuple, scalar, tuple }` */
- assert(root.type == MP_ARRAY && root.size > 0);
- for (uint32_t t = 1; t <= root.size; t++) {
- lua_rawgeti(L, 1, t);
- struct luaL_field field;
- if (luaL_tofield(L, cfg, -1, &field) < 0)
- return luaT_error(L);
- if (field.type == MP_EXT && (tuple = luaT_istuple(L, -1))) {
- tuple_to_mpstream(tuple, stream);
- } else if (field.type != MP_ARRAY) {
- /* The first member of root table is not tuple/array */
- if (t == 1) {
- /*
- * `return { scalar, ... } =>
- * box.tuple.new(scalar, ...)`
- */
- mpstream_encode_array(stream, root.size);
- /*
- * Encode the first field of tuple using
- * existing information from luaL_tofield
- */
- luamp_encode_r(L, cfg, stream, &field, 0);
- lua_pop(L, 1);
- assert(lua_gettop(L) == 1);
- /* Encode remaining fields as usual */
- for (uint32_t f = 2; f <= root.size; f++) {
- lua_rawgeti(L, 1, f);
- luamp_encode(L, cfg, stream, -1);
- lua_pop(L, 1);
- }
- return 1;
- }
- /*
- * `return { tuple/array, ..., scalar, ... } =>
- * { tuple/array, ..., { scalar }, ... }`
- */
- mpstream_encode_array(stream, 1);
- luamp_encode_r(L, cfg, stream, &field, 0);
- } else {
- /* `return { tuple/array, ..., tuple/array, ... }` */
- luamp_encode_r(L, cfg, stream, &field, 0);
- }
- lua_pop(L, 1);
- assert(lua_gettop(L) == 1);
- }
- return root.size;
-}
-
-static const struct port_vtab port_lua_vtab;
-
-void
-port_lua_create(struct port *port, struct lua_State *L)
-{
- struct port_lua *port_lua = (struct port_lua *) port;
- memset(port_lua, 0, sizeof(*port_lua));
- port_lua->vtab = &port_lua_vtab;
- port_lua->L = L;
- /*
- * Allow to destroy the port even if no ref.
- * @Sa luaL_unref.
- */
- port_lua->ref = -1;
-}
-
static int
execute_lua_eval(lua_State *L)
{
@@ -248,103 +93,6 @@ execute_lua_eval(lua_State *L)
return lua_gettop(L);
}
-static int
-encode_lua_call(lua_State *L)
-{
- struct port_lua *port = (struct port_lua *) lua_topointer(L, -1);
- /*
- * Add all elements from Lua stack to the buffer.
- *
- * TODO: forbid explicit yield from __serialize or __index here
- */
- struct mpstream stream;
- mpstream_init(&stream, port->out, obuf_reserve_cb, obuf_alloc_cb,
- luamp_error, port->L);
-
- struct luaL_serializer *cfg = luaL_msgpack_default;
- int size = lua_gettop(port->L);
- for (int i = 1; i <= size; ++i)
- luamp_encode(port->L, cfg, &stream, i);
- port->size = size;
- mpstream_flush(&stream);
- return 0;
-}
-
-static int
-encode_lua_call_16(lua_State *L)
-{
- struct port_lua *port = (struct port_lua *) lua_topointer(L, -1);
- /*
- * Add all elements from Lua stack to the buffer.
- *
- * TODO: forbid explicit yield from __serialize or __index here
- */
- struct mpstream stream;
- mpstream_init(&stream, port->out, obuf_reserve_cb, obuf_alloc_cb,
- luamp_error, port->L);
-
- struct luaL_serializer *cfg = luaL_msgpack_default;
- port->size = luamp_encode_call_16(port->L, cfg, &stream);
- mpstream_flush(&stream);
- return 0;
-}
-
-static inline int
-port_lua_do_dump(struct port *base, struct obuf *out, lua_CFunction handler)
-{
- struct port_lua *port = (struct port_lua *)base;
- assert(port->vtab == &port_lua_vtab);
- /* Use port to pass arguments to encoder quickly. */
- port->out = out;
- /*
- * Use the same global state, assuming the encoder doesn't
- * yield.
- */
- struct lua_State *L = tarantool_L;
- int top = lua_gettop(L);
- if (lua_cpcall(L, handler, port) != 0) {
- luaT_toerror(port->L);
- return -1;
- }
- lua_settop(L, top);
- return port->size;
-}
-
-static int
-port_lua_dump(struct port *base, struct obuf *out)
-{
- return port_lua_do_dump(base, out, encode_lua_call);
-}
-
-static int
-port_lua_dump_16(struct port *base, struct obuf *out)
-{
- return port_lua_do_dump(base, out, encode_lua_call_16);
-}
-
-static void
-port_lua_destroy(struct port *base)
-{
- struct port_lua *port = (struct port_lua *)base;
- assert(port->vtab == &port_lua_vtab);
- luaL_unref(tarantool_L, LUA_REGISTRYINDEX, port->ref);
-}
-
-/**
- * Dump port lua as a YAML document. It is extern since depends on
- * lyaml module.
- */
-extern const char *
-port_lua_dump_plain(struct port *port, uint32_t *size);
-
-static const struct port_vtab port_lua_vtab = {
- .dump_msgpack = port_lua_dump,
- .dump_msgpack_16 = port_lua_dump_16,
- .dump_lua = NULL,
- .dump_plain = port_lua_dump_plain,
- .destroy = port_lua_destroy,
-};
-
static inline int
box_process_lua(struct call_request *request, struct port *base,
lua_CFunction handler)
@@ -357,7 +105,7 @@ box_process_lua(struct call_request *request, struct port *base,
lua_pushcfunction(L, handler);
lua_pushlightuserdata(L, request);
if (luaT_call(L, 1, LUA_MULTRET) != 0) {
- port_lua_destroy(base);
+ port_destroy(base);
return -1;
}
return 0;
diff --git a/src/box/lua/port_lua.c b/src/box/lua/port_lua.c
new file mode 100644
index 000000000..087650281
--- /dev/null
+++ b/src/box/lua/port_lua.c
@@ -0,0 +1,318 @@
+/*
+ * Copyright 2010-2019, Tarantool AUTHORS, please see AUTHORS file.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the
+ * following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY <COPYRIGHT HOLDER> ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+ * <COPYRIGHT HOLDER> OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+ * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+#include "mpstream.h"
+#include "box/func.h"
+#include "box/tuple.h"
+#include "box/lua/tuple.h"
+#include "box/lua/misc.h"
+#include "small/region.h"
+#include "lua/utils.h"
+#include "lua/msgpack.h"
+#include "box/port.h"
+#include "small/obuf.h"
+#include <assert.h>
+
+static const struct port_vtab port_lua_vtab;
+
+/*
+ * Encode CALL_16 result.
+ *
+ * To allow clients to understand a complex return from
+ * a procedure, we are compatible with SELECT protocol,
+ * and return the number of return values first, and
+ * then each return value as a tuple.
+ *
+ * The following conversion rules apply:
+ *
+ * If a Lua stack contains at least one scalar, each
+ * value on the stack is converted to a tuple. A stack
+ * containing a single Lua table with scalars is converted to
+ * a tuple with multiple fields.
+ *
+ * If the stack is a Lua table, each member of which is
+ * not scalar, each member of the table is converted to
+ * a tuple. This way very large lists of return values can
+ * be used, since Lua stack size is limited by 8000 elements,
+ * while Lua table size is pretty much unlimited.
+ *
+ * Please read gh-291 carefully before "fixing" this code.
+ */
+static inline uint32_t
+luamp_encode_call_16(lua_State *L, struct luaL_serializer *cfg,
+ struct mpstream *stream)
+{
+ int nrets = lua_gettop(L);
+ if (nrets == 0) {
+ return 0;
+ } else if (nrets > 1) {
+ /*
+ * Multireturn:
+ * `return 1, box.tuple.new(...), array, 3, ...`
+ */
+ for (int i = 1; i <= nrets; ++i) {
+ struct luaL_field field;
+ if (luaL_tofield(L, cfg, i, &field) < 0)
+ return luaT_error(L);
+ struct tuple *tuple;
+ if (field.type == MP_EXT &&
+ (tuple = luaT_istuple(L, i)) != NULL) {
+ /* `return ..., box.tuple.new(...), ...` */
+ tuple_to_mpstream(tuple, stream);
+ } else if (field.type != MP_ARRAY) {
+ /*
+ * `return ..., scalar, ... =>
+ * ..., { scalar }, ...`
+ */
+ lua_pushvalue(L, i);
+ mpstream_encode_array(stream, 1);
+ luamp_encode_r(L, cfg, stream, &field, 0);
+ lua_pop(L, 1);
+ } else {
+ /* `return ..., array, ...` */
+ luamp_encode(L, cfg, stream, i);
+ }
+ }
+ return nrets;
+ }
+ assert(nrets == 1);
+
+ /*
+ * Inspect the first result
+ */
+ struct luaL_field root;
+ if (luaL_tofield(L, cfg, 1, &root) < 0)
+ return luaT_error(L);
+ struct tuple *tuple;
+ if (root.type == MP_EXT && (tuple = luaT_istuple(L, 1)) != NULL) {
+ /* `return box.tuple()` */
+ tuple_to_mpstream(tuple, stream);
+ return 1;
+ } else if (root.type != MP_ARRAY) {
+ /*
+ * `return scalar`
+ * `return map`
+ */
+ mpstream_encode_array(stream, 1);
+ assert(lua_gettop(L) == 1);
+ luamp_encode_r(L, cfg, stream, &root, 0);
+ return 1;
+ }
+
+ assert(root.type == MP_ARRAY);
+ if (root.size == 0) {
+ /* `return {}` => `{ box.tuple() }` */
+ mpstream_encode_array(stream, 0);
+ return 1;
+ }
+
+ /* `return { tuple, scalar, tuple }` */
+ assert(root.type == MP_ARRAY && root.size > 0);
+ for (uint32_t t = 1; t <= root.size; t++) {
+ lua_rawgeti(L, 1, t);
+ struct luaL_field field;
+ if (luaL_tofield(L, cfg, -1, &field) < 0)
+ return luaT_error(L);
+ if (field.type == MP_EXT && (tuple = luaT_istuple(L, -1))) {
+ tuple_to_mpstream(tuple, stream);
+ } else if (field.type != MP_ARRAY) {
+ /* The first member of root table is not tuple/array */
+ if (t == 1) {
+ /*
+ * `return { scalar, ... } =>
+ * box.tuple.new(scalar, ...)`
+ */
+ mpstream_encode_array(stream, root.size);
+ /*
+ * Encode the first field of tuple using
+ * existing information from luaL_tofield
+ */
+ luamp_encode_r(L, cfg, stream, &field, 0);
+ lua_pop(L, 1);
+ assert(lua_gettop(L) == 1);
+ /* Encode remaining fields as usual */
+ for (uint32_t f = 2; f <= root.size; f++) {
+ lua_rawgeti(L, 1, f);
+ luamp_encode(L, cfg, stream, -1);
+ lua_pop(L, 1);
+ }
+ return 1;
+ }
+ /*
+ * `return { tuple/array, ..., scalar, ... } =>
+ * { tuple/array, ..., { scalar }, ... }`
+ */
+ mpstream_encode_array(stream, 1);
+ luamp_encode_r(L, cfg, stream, &field, 0);
+ } else {
+ /* `return { tuple/array, ..., tuple/array, ... }` */
+ luamp_encode_r(L, cfg, stream, &field, 0);
+ }
+ lua_pop(L, 1);
+ assert(lua_gettop(L) == 1);
+ }
+ return root.size;
+}
+
+static inline int
+port_lua_do_dump(struct port *base, struct mpstream *stream,
+ lua_CFunction handler)
+{
+ struct port_lua *port = (struct port_lua *)base;
+ assert(port->vtab == &port_lua_vtab);
+ /* Use port to pass arguments to encoder quickly. */
+ port->stream = stream;
+ /*
+ * Use the same global state, assuming the encoder doesn't
+ * yield.
+ */
+ struct lua_State *L = tarantool_L;
+ int top = lua_gettop(L);
+ if (lua_cpcall(L, handler, port) != 0) {
+ luaT_toerror(port->L);
+ return -1;
+ }
+ lua_settop(L, top);
+ return port->size;
+}
+
+static int
+encode_lua_call(lua_State *L)
+{
+ struct port_lua *port = (struct port_lua *) lua_topointer(L, -1);
+ /*
+ * Add all elements from Lua stack to the buffer.
+ *
+ * TODO: forbid explicit yield from __serialize or __index here
+ */
+ struct luaL_serializer *cfg = luaL_msgpack_default;
+ int size = lua_gettop(port->L);
+ for (int i = 1; i <= size; ++i)
+ luamp_encode(port->L, cfg, port->stream, i);
+ port->size = size;
+ mpstream_flush(port->stream);
+ return 0;
+}
+
+static int
+encode_lua_call_16(lua_State *L)
+{
+ struct port_lua *port = (struct port_lua *) lua_topointer(L, -1);
+ /*
+ * Add all elements from Lua stack to the buffer.
+ *
+ * TODO: forbid explicit yield from __serialize or __index here
+ */
+ struct luaL_serializer *cfg = luaL_msgpack_default;
+ port->size = luamp_encode_call_16(port->L, cfg, port->stream);
+ mpstream_flush(port->stream);
+ return 0;
+}
+
+static int
+port_lua_dump(struct port *base, struct obuf *obuf)
+{
+ struct port_lua *port = (struct port_lua *)base;
+ struct mpstream stream;
+ mpstream_init(&stream, obuf, obuf_reserve_cb, obuf_alloc_cb,
+ luamp_error, port->L);
+ return port_lua_do_dump(base, &stream, encode_lua_call);
+}
+
+static int
+port_lua_dump_16(struct port *base, struct obuf *obuf)
+{
+ struct port_lua *port = (struct port_lua *)base;
+ struct mpstream stream;
+ mpstream_init(&stream, obuf, obuf_reserve_cb, obuf_alloc_cb,
+ luamp_error, port->L);
+ return port_lua_do_dump(base, &stream, encode_lua_call_16);
+}
+
+static int
+port_lua_dump_region(struct port *base, struct region *region)
+{
+ struct port_lua *port = (struct port_lua *)base;
+ struct mpstream stream;
+ mpstream_init(&stream, region, region_reserve_cb, region_alloc_cb,
+ luamp_error, port->L);
+ return port_lua_do_dump(base, &stream, encode_lua_call);
+}
+
+static void
+port_lua_dump_lua(struct port *base, struct lua_State *L)
+{
+ struct port_lua *port = (struct port_lua *) base;
+ uint32_t size = lua_gettop(port->L);
+ lua_createtable(L, size, 0);
+ for (uint32_t i = 0; i < size; i++) {
+ lua_xmove(port->L, L, 1);
+ lua_rawseti(L, -2, i + 1);
+ }
+ port->size = 1;
+}
+
+static void
+port_lua_destroy(struct port *base)
+{
+ struct port_lua *port = (struct port_lua *)base;
+ assert(port->vtab == &port_lua_vtab);
+ luaL_unref(tarantool_L, LUA_REGISTRYINDEX, port->ref);
+}
+
+/**
+ * Dump port lua as a YAML document. It is extern since depends on
+ * lyaml module.
+ */
+extern const char *
+port_lua_dump_plain(struct port *port, uint32_t *size);
+
+static const struct port_vtab port_lua_vtab = {
+ .dump_msgpack = port_lua_dump,
+ .dump_msgpack_16 = port_lua_dump_16,
+ .dump_msgpack_region = port_lua_dump_region,
+ .dump_lua = port_lua_dump_lua,
+ .dump_plain = port_lua_dump_plain,
+ .destroy = port_lua_destroy,
+};
+
+void
+port_lua_create(struct port *port, struct lua_State *L)
+{
+ struct port_lua *port_lua = (struct port_lua *) port;
+ memset(port_lua, 0, sizeof(*port_lua));
+ port_lua->vtab = &port_lua_vtab;
+ port_lua->L = L;
+ /*
+ * Allow to destroy the port even if no ref.
+ * @Sa luaL_unref.
+ */
+ port_lua->ref = LUA_REFNIL;
+}
diff --git a/src/box/port.h b/src/box/port.h
index f18803660..28fc16ce9 100644
--- a/src/box/port.h
+++ b/src/box/port.h
@@ -88,7 +88,7 @@ struct port_lua {
/** Reference to L in tarantool_L. */
int ref;
/** The argument of port_dump */
- struct obuf *out;
+ struct mpstream *stream;
/** Number of entries dumped to the port. */
int size;
};
diff --git a/src/lib/core/port.h b/src/lib/core/port.h
index 8ace40fc5..e2f657ce0 100644
--- a/src/lib/core/port.h
+++ b/src/lib/core/port.h
@@ -37,6 +37,7 @@ extern "C" {
#endif /* defined(__cplusplus) */
struct obuf;
+struct region;
struct lua_State;
struct port;
@@ -62,6 +63,15 @@ struct port_vtab {
* header. Used by the legacy Tarantool 1.6 format.
*/
int (*dump_msgpack_16)(struct port *port, struct obuf *out);
+ /**
+ * Dump the content of a port to a region.
+ * @param port Port to dump.
+ * @param region Region to dump to.
+ *
+ * @retval >= 0 Number of entries dumped.
+ * @retval < 0 Error.
+ */
+ int (*dump_msgpack_region)(struct port *port, struct region *region);
/** Dump the content of a port to Lua stack. */
void (*dump_lua)(struct port *port, struct lua_State *L);
/**
@@ -108,6 +118,12 @@ port_dump_msgpack_16(struct port *port, struct obuf *out)
return port->vtab->dump_msgpack_16(port, out);
}
+static inline int
+port_dump_msgpack_region(struct port *port, struct region *region)
+{
+ return port->vtab->dump_msgpack_region(port, region);
+}
+
static inline void
port_dump_lua(struct port *port, struct lua_State *L)
{
--
2.21.0
More information about the Tarantool-patches
mailing list