From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: From: Kirill Shcherbatov Subject: [PATCH v3 4/6] box: export registered functions in box.func folder Date: Thu, 13 Jun 2019 17:08:24 +0300 Message-Id: <81d7293c993369d7d4e984391c8854bdf5b16818.1560433806.git.kshcherbatov@tarantool.org> In-Reply-To: References: MIME-Version: 1.0 Content-Transfer-Encoding: 8bit To: tarantool-patches@freelists.org, vdavydov.dev@gmail.com Cc: Kirill Shcherbatov List-ID: Needed for #4182, #1260 @TarantoolBot document Title: Export registered functions to box.func folder Now all registered with box.schema.func.create functions are exported in box.func folder. Each function have :call and :drop method. The :drop method just a shortcut for box.schema.func.drop interface. The :call method is similar to net.box connection:call method and allows to call a registered function directly. All access checks are performed on each function call. Example: function sum(a, b) return a + b end box.schema.func.create('sum') box.func.sum --- - language: LUA setuid: false name: sum id: 2 ... box.func.sum:call({1, 3}) --- - 4 ... box.func.sum:drop() --- src/box/alter.cc | 2 + src/box/func.h | 1 + src/box/lua/call.c | 224 ++++++++++++++++++++++++++++++++---- src/box/lua/init.c | 1 + src/box/lua/misc.cc | 7 +- src/box/lua/schema.lua | 29 +++++ src/box/port.h | 4 +- src/box/schema.cc | 1 + src/box/schema.h | 5 + test/box/function1.c | 33 ++++++ test/box/function1.result | 205 +++++++++++++++++++++++++++++++++ test/box/function1.test.lua | 66 +++++++++++ test/box/misc.result | 1 + 13 files changed, 554 insertions(+), 25 deletions(-) diff --git a/src/box/alter.cc b/src/box/alter.cc index 7b0eb3334..081786820 100644 --- a/src/box/alter.cc +++ b/src/box/alter.cc @@ -2579,6 +2579,7 @@ func_cache_remove_func(struct trigger *trigger, void * /* event */) { struct func *old_func = (struct func *) trigger->data; func_cache_delete(old_func->def->fid); + trigger_run_xc(&on_alter_func, old_func); func_delete(old_func); } @@ -2612,6 +2613,7 @@ on_replace_dd_func(struct trigger * /* trigger */, void *event) }); def_guard.is_active = false; func_cache_insert(func); + trigger_run_xc(&on_alter_func, func); func_guard.is_active = false; struct trigger *on_rollback = txn_alter_trigger_new(func_cache_remove_func, func); diff --git a/src/box/func.h b/src/box/func.h index 1271bde67..f7081eccd 100644 --- a/src/box/func.h +++ b/src/box/func.h @@ -44,6 +44,7 @@ extern "C" { struct port; struct func; +struct lua_State; struct box_function_ctx { struct port *port; diff --git a/src/box/lua/call.c b/src/box/lua/call.c index 8d0328ef7..23eb29eb8 100644 --- a/src/box/lua/call.c +++ b/src/box/lua/call.c @@ -33,10 +33,13 @@ #include "box/error.h" #include "box/func.h" #include "box/func_def.h" +#include "box/schema.h" #include "fiber.h" +#include "tt_static.h" #include "lua/utils.h" #include "lua/msgpack.h" +#include "lua/trigger.h" #include "box/xrow.h" #include "box/port.h" @@ -397,16 +400,12 @@ encode_lua_call(lua_State *L) * * TODO: forbid explicit yield from __serialize or __index here */ - struct mpstream stream; - mpstream_init(&stream, port->out, obuf_reserve_cb, obuf_alloc_cb, - luamp_error, port->L); - struct luaL_serializer *cfg = luaL_msgpack_default; int size = lua_gettop(port->L); for (int i = 1; i <= size; ++i) - luamp_encode(port->L, cfg, &stream, i); + luamp_encode(port->L, cfg, port->stream, i); port->size = size; - mpstream_flush(&stream); + mpstream_flush(port->stream); return 0; } @@ -419,23 +418,20 @@ encode_lua_call_16(lua_State *L) * * TODO: forbid explicit yield from __serialize or __index here */ - struct mpstream stream; - mpstream_init(&stream, port->out, obuf_reserve_cb, obuf_alloc_cb, - luamp_error, port->L); - struct luaL_serializer *cfg = luaL_msgpack_default; - port->size = luamp_encode_call_16(port->L, cfg, &stream); - mpstream_flush(&stream); + port->size = luamp_encode_call_16(port->L, cfg, port->stream); + mpstream_flush(port->stream); return 0; } static inline int -port_lua_do_dump(struct port *base, struct obuf *out, lua_CFunction handler) +port_lua_do_dump(struct port *base, struct mpstream *stream, + lua_CFunction handler) { - struct port_lua *port = (struct port_lua *)base; + struct port_lua *port = (struct port_lua *) base; assert(port->vtab == &port_lua_vtab); /* Use port to pass arguments to encoder quickly. */ - port->out = out; + port->stream = stream; /* * Use the same global state, assuming the encoder doesn't * yield. @@ -453,13 +449,56 @@ port_lua_do_dump(struct port *base, struct obuf *out, lua_CFunction handler) static int port_lua_dump(struct port *base, struct obuf *out) { - return port_lua_do_dump(base, out, encode_lua_call); + struct port_lua *port = (struct port_lua *) base; + struct mpstream stream; + mpstream_init(&stream, out, obuf_reserve_cb, obuf_alloc_cb, + luamp_error, port->L); + return port_lua_do_dump(base, &stream, encode_lua_call); } static int port_lua_dump_16(struct port *base, struct obuf *out) { - return port_lua_do_dump(base, out, encode_lua_call_16); + struct port_lua *port = (struct port_lua *)base; + struct mpstream stream; + mpstream_init(&stream, out, obuf_reserve_cb, obuf_alloc_cb, + luamp_error, port->L); + return port_lua_do_dump(base, &stream, encode_lua_call_16); +} + +static void +port_lua_dump_lua(struct port *base, struct lua_State *L, bool is_flat) +{ + assert(is_flat == true); + struct port_lua *port = (struct port_lua *) base; + uint32_t size = lua_gettop(port->L); + lua_xmove(port->L, L, size); + port->size = size; +} + +static const char * +port_lua_get_msgpack(struct port *base, uint32_t *size) +{ + struct port_lua *port = (struct port_lua *) base; + struct region *region = &fiber()->gc; + uint32_t region_svp = region_used(region); + struct mpstream stream; + mpstream_init(&stream, region, region_reserve_cb, region_alloc_cb, + luamp_error, port->L); + mpstream_encode_array(&stream, lua_gettop(port->L)); + int rc = port_lua_do_dump(base, &stream, encode_lua_call); + if (rc < 0) { + region_truncate(region, region_svp); + return NULL; + } + *size = region_used(region) - region_svp; + const char *data = region_join(region, *size); + if (data == NULL) { + diag_set(OutOfMemory, *size, "region", "data"); + region_truncate(region, region_svp); + return NULL; + } + return data; } static void @@ -480,9 +519,9 @@ port_lua_dump_plain(struct port *port, uint32_t *size); static const struct port_vtab port_lua_vtab = { .dump_msgpack = port_lua_dump, .dump_msgpack_16 = port_lua_dump_16, - .dump_lua = NULL, + .dump_lua = port_lua_dump_lua, .dump_plain = port_lua_dump_plain, - .get_msgpack = NULL, + .get_msgpack = port_lua_get_msgpack, .destroy = port_lua_destroy, }; @@ -578,10 +617,150 @@ lbox_module_reload(lua_State *L) return 0; } +int +lbox_func_call(struct lua_State *L) +{ + if (lua_gettop(L) < 2 || !lua_isstring(L, 1)) + return luaL_error(L, "Use func:call(...)"); + + size_t name_len; + const char *name = lua_tolstring(L, 1, &name_len); + struct func *func = func_by_name(name, name_len); + if (func == NULL) { + diag_set(ClientError, ER_NO_SUCH_FUNCTION, + tt_cstr(name, name_len)); + return luaT_error(L); + } + + /* + * Prepare a new Lua stack for input arguments + * before the function call to pass it into the + * pcall-sandboxed tarantool_L handler. + */ + lua_State *args_L = lua_newthread(tarantool_L); + int coro_ref = luaL_ref(tarantool_L, LUA_REGISTRYINDEX); + lua_xmove(L, args_L, lua_gettop(L) - 1); + struct port in_port; + port_lua_create(&in_port, args_L); + ((struct port_lua *) &in_port)->ref = coro_ref; + + struct port out_port; + if (func_call(func, &in_port, &out_port) != 0) { + port_destroy(&in_port); + return luaT_error(L); + } + + int top = lua_gettop(L); + port_dump_lua(&out_port, L, true); + int cnt = lua_gettop(L) - top; + + port_destroy(&out_port); + port_destroy(&in_port); + return cnt; +} + +static void +lbox_func_new(struct lua_State *L, struct func *func) +{ + lua_getfield(L, LUA_GLOBALSINDEX, "box"); + lua_getfield(L, -1, "func"); + if (!lua_istable(L, -1)) { + lua_pop(L, 1); /* pop nil */ + lua_newtable(L); + lua_setfield(L, -2, "func"); + lua_getfield(L, -1, "func"); + } + lua_rawgeti(L, -1, func->def->fid); + if (lua_isnil(L, -1)) { + /* + * If the function already exists, modify it, + * rather than create a new one -- to not + * invalidate Lua variable references to old func + * outside the box.schema.func[]. + */ + lua_pop(L, 1); + lua_newtable(L); + lua_rawseti(L, -2, func->def->fid); + lua_rawgeti(L, -1, func->def->fid); + } else { + /* Clear the reference to old func by old name. */ + lua_getfield(L, -1, "name"); + lua_pushnil(L); + lua_settable(L, -4); + } + int top = lua_gettop(L); + lua_pushstring(L, "id"); + lua_pushnumber(L, func->def->fid); + lua_settable(L, top); + lua_pushstring(L, "name"); + lua_pushstring(L, func->def->name); + lua_settable(L, top); + lua_pushstring(L, "setuid"); + lua_pushboolean(L, func->def->setuid); + lua_settable(L, top); + lua_pushstring(L, "language"); + lua_pushstring(L, func_language_strs[func->def->language]); + lua_settable(L, top); + + /* Bless func object. */ + lua_getfield(L, LUA_GLOBALSINDEX, "box"); + lua_pushstring(L, "schema"); + lua_gettable(L, -2); + lua_pushstring(L, "func"); + lua_gettable(L, -2); + lua_pushstring(L, "bless"); + lua_gettable(L, -2); + + lua_pushvalue(L, top); + lua_call(L, 1, 0); + lua_pop(L, 3); + + lua_setfield(L, -2, func->def->name); + + lua_pop(L, 2); +} + +static void +lbox_func_delete(struct lua_State *L, struct func *func) +{ + uint32_t fid = func->def->fid; + lua_getfield(L, LUA_GLOBALSINDEX, "box"); + lua_getfield(L, -1, "func"); + assert(!lua_isnil(L, -1)); + lua_rawgeti(L, -1, fid); + if (!lua_isnil(L, -1)) { + lua_getfield(L, -1, "name"); + lua_pushnil(L); + lua_rawset(L, -4); + lua_pop(L, 1); /* pop func */ + lua_pushnil(L); + lua_rawseti(L, -2, fid); + } else { + lua_pop(L, 1); + } + lua_pop(L, 2); /* box, func */ +} + +static void +lbox_func_new_or_delete(struct trigger *trigger, void *event) +{ + struct lua_State *L = (struct lua_State *) trigger->data; + struct func *func = (struct func *)event; + if (func != NULL) + lbox_func_new(L, func); + else + lbox_func_delete(L, func); +} + +static struct trigger on_alter_func_in_lua = { + RLIST_LINK_INITIALIZER, lbox_func_new_or_delete, NULL, NULL +}; + static const struct luaL_Reg boxlib_internal[] = { {"call_loadproc", lbox_call_loadproc}, {"sql_create_function", lbox_sql_create_function}, {"module_reload", lbox_module_reload}, + {"func_call", lbox_func_call}, {NULL, NULL} }; @@ -590,7 +769,12 @@ box_lua_call_init(struct lua_State *L) { luaL_register(L, "box.internal", boxlib_internal); lua_pop(L, 1); - + /* + * Register the trigger that will push persistent + * Lua functions objects to Lua. + */ + on_alter_func_in_lua.data = L; + trigger_add(&on_alter_func, &on_alter_func_in_lua); #if 0 /* Get CTypeID for `struct port *' */ int rc = luaL_cdef(L, "struct port;"); diff --git a/src/box/lua/init.c b/src/box/lua/init.c index 76b987b4b..7ffed409d 100644 --- a/src/box/lua/init.c +++ b/src/box/lua/init.c @@ -39,6 +39,7 @@ #include "box/box.h" #include "box/txn.h" +#include "box/func.h" #include "box/vclock.h" #include "box/lua/error.h" diff --git a/src/box/lua/misc.cc b/src/box/lua/misc.cc index 15681c377..3d894b09f 100644 --- a/src/box/lua/misc.cc +++ b/src/box/lua/misc.cc @@ -63,13 +63,14 @@ lbox_encode_tuple_on_gc(lua_State *L, int idx, size_t *p_len) extern "C" void port_tuple_dump_lua(struct port *base, struct lua_State *L, bool is_flat) { - assert(is_flat == false); struct port_tuple *port = port_tuple(base); - lua_createtable(L, port->size, 0); + if (!is_flat) + lua_createtable(L, port->size, 0); struct port_tuple_entry *pe = port->first; for (int i = 0; pe != NULL; pe = pe->next) { luaT_pushtuple(L, pe->tuple); - lua_rawseti(L, -2, ++i); + if (!is_flat) + lua_rawseti(L, -2, ++i); } } diff --git a/src/box/lua/schema.lua b/src/box/lua/schema.lua index 3f080eced..9c3ee063c 100644 --- a/src/box/lua/schema.lua +++ b/src/box/lua/schema.lua @@ -2190,7 +2190,36 @@ function box.schema.func.exists(name_or_id) return tuple ~= nil end +-- Helper function to check func:method() usage +local function check_func_arg(func, method) + if type(func) ~= 'table' or func.name == nil then + local fmt = 'Use func:%s(...) instead of func.%s(...)' + error(string.format(fmt, method, method)) + end +end + +local func_mt = {} + +func_mt.drop = function(func, opts) + check_func_arg(func, 'drop') + box.schema.func.drop(func.name, opts) +end + +func_mt.call = function(func, args) + check_func_arg(func, 'call') + args = args or {} + if type(args) ~= 'table' then + error('Use func:call(table)') + end + return box.schema.func.call(func.name, unpack(args)) +end + +function box.schema.func.bless(func) + setmetatable(func, {__index = func_mt}) +end + box.schema.func.reload = internal.module_reload +box.schema.func.call = internal.func_call box.internal.collation = {} box.internal.collation.create = function(name, coll_type, locale, opts) diff --git a/src/box/port.h b/src/box/port.h index 4c0c3cf7a..8765327c7 100644 --- a/src/box/port.h +++ b/src/box/port.h @@ -102,8 +102,8 @@ struct port_lua { struct lua_State *L; /** Reference to L in tarantool_L. */ int ref; - /** The argument of port_dump */ - struct obuf *out; + /** The argument to dump a port. */ + struct mpstream *stream; /** Number of entries dumped to the port. */ int size; }; diff --git a/src/box/schema.cc b/src/box/schema.cc index f834610a5..ead6079b5 100644 --- a/src/box/schema.cc +++ b/src/box/schema.cc @@ -72,6 +72,7 @@ uint32_t space_cache_version = 0; struct rlist on_schema_init = RLIST_HEAD_INITIALIZER(on_schema_init); struct rlist on_alter_space = RLIST_HEAD_INITIALIZER(on_alter_space); struct rlist on_alter_sequence = RLIST_HEAD_INITIALIZER(on_alter_sequence); +struct rlist on_alter_func = RLIST_HEAD_INITIALIZER(on_alter_func); /** * Lock of scheme modification diff --git a/src/box/schema.h b/src/box/schema.h index d7b3c4c2d..9ff4ffa1c 100644 --- a/src/box/schema.h +++ b/src/box/schema.h @@ -238,6 +238,11 @@ extern struct rlist on_alter_sequence; */ extern struct rlist on_access_denied; +/** + * Triggers fired after committing a change in _func space. + */ +extern struct rlist on_alter_func; + /** * Context passed to on_access_denied trigger. */ diff --git a/test/box/function1.c b/test/box/function1.c index 053e4fe91..ee5a422b5 100644 --- a/test/box/function1.c +++ b/test/box/function1.c @@ -42,6 +42,39 @@ args(box_function_ctx_t *ctx, const char *args, const char *args_end) return box_return_tuple(ctx, tuple); } +int +divide(box_function_ctx_t *ctx, const char *args, const char *args_end) +{ + uint32_t arg_count = mp_decode_array(&args); + if (arg_count < 2) + goto error; + + if (mp_typeof(*args) != MP_UINT) + goto error; + double a = mp_decode_uint(&args); + if (mp_typeof(*args) != MP_UINT) + goto error; + double b = mp_decode_uint(&args); + if (b == 0) + goto error; + + char tuple_buf[512]; + char *d = tuple_buf; + d = mp_encode_array(d, 1); + d = mp_encode_double(d, a / b); + assert(d <= tuple_buf + sizeof(tuple_buf)); + + box_tuple_format_t *fmt = box_tuple_format_default(); + box_tuple_t *tuple = box_tuple_new(fmt, tuple_buf, d); + if (tuple == NULL) + return -1; + return box_return_tuple(ctx, tuple); +error: + return box_error_set(__FILE__, __LINE__, ER_PROC_C, "%s", + "invalid argument"); +} + + /* * For each UINT key in arguments create or increment counter in * box.space.test space. diff --git a/test/box/function1.result b/test/box/function1.result index cadeb0467..331bd466a 100644 --- a/test/box/function1.result +++ b/test/box/function1.result @@ -57,6 +57,25 @@ c:call('function1.args', { 15 }) --- - [[15, 'hello']] ... +box.func["function1.args"] +--- +- language: C + setuid: false + name: function1.args + id: 2 +... +box.func["function1.args"]:call() +--- +- error: Use func:call(...) +... +box.func["function1.args"]:call({ "xx" }) +--- +- error: first tuple field must be uint +... +box.func["function1.args"]:call({ 15 }) +--- +- [15, 'hello'] +... box.schema.func.drop("function1.args") --- ... @@ -299,3 +318,189 @@ test_run:cmd("setopt delimiter ''"); c:close() --- ... +-- Test registered functions interface. +function divide(a, b) return a / b end +--- +... +box.schema.func.create("divide") +--- +... +func = box.func.divide +--- +... +func.call({4, 2}) +--- +- error: 'builtin/box/schema.lua:2197: Use func:call(...) instead of func.call(...)' +... +func:call(4, 2) +--- +- error: 'builtin/box/schema.lua:2212: Use func:call(table)' +... +func:call() +--- +- error: Use func:call(...) +... +func:call({}) +--- +- error: Use func:call(...) +... +func:call({4}) +--- +- error: '[string "function divide(a, b) return a / b end "]:1: attempt to perform + arithmetic on local ''b'' (a nil value)' +... +func:call({4, 2}) +--- +- 2 +... +func:call({4, 2, 1}) +--- +- 2 +... +func:drop() +--- +... +func +--- +- language: LUA + setuid: false + name: divide + id: 2 +... +func.drop() +--- +- error: 'builtin/box/schema.lua:2197: Use func:drop(...) instead of func.drop(...)' +... +func:drop() +--- +- error: Function 'divide' does not exist +... +func:call({4, 2}) +--- +- error: Function 'divide' does not exist +... +box.internal.func_call('divide', 4, 2) +--- +- error: Function 'divide' does not exist +... +box.schema.func.create("function1.divide", {language = 'C'}) +--- +... +func = box.func["function1.divide"] +--- +... +func:call(4, 2) +--- +- error: 'builtin/box/schema.lua:2212: Use func:call(table)' +... +func:call() +--- +- error: Use func:call(...) +... +func:call({}) +--- +- error: Use func:call(...) +... +func:call({4}) +--- +- error: invalid argument +... +func:call({4, 2}) +--- +- [2] +... +func:call({4, 2, 1}) +--- +- [2] +... +func:drop() +--- +... +func +--- +- language: C + setuid: false + name: function1.divide + id: 2 +... +func:drop() +--- +- error: Function 'function1.divide' does not exist +... +func:call({4, 2}) +--- +- error: Function 'function1.divide' does not exist +... +box.internal.func_call('function1.divide', 4, 2) +--- +- error: Function 'function1.divide' does not exist +... +test_run:cmd("setopt delimiter ';'") +--- +- true +... +function minmax(array) + local min = 999 + local max = -1 + for _, v in pairs(array) do + min = math.min(min, v) + max = math.max(max, v) + end + return min, max +end +test_run:cmd("setopt delimiter ''"); +--- +... +box.schema.func.create("minmax") +--- +... +func = box.func.minmax +--- +... +func:call({{1, 2, 99, 3, -1}}) +--- +- -1 +- 99 +... +func:drop() +--- +... +-- Test access checks for registered functions. +function secret() return 1 end +--- +... +box.schema.func.create("secret") +--- +... +box.func.secret:call({}) +--- +- error: Use func:call(...) +... +function secret_leak() return box.func.secret:call() end +--- +... +box.schema.func.create('secret_leak') +--- +... +box.schema.user.grant('guest', 'execute', 'function', 'secret_leak') +--- +... +conn = net.connect(box.cfg.listen) +--- +... +conn:call('secret_leak') +--- +- error: Use func:call(...) +... +conn:close() +--- +... +box.schema.user.revoke('guest', 'execute', 'function', 'secret_leak') +--- +... +box.schema.func.drop('secret_leak') +--- +... +box.schema.func.drop('secret') +--- +... diff --git a/test/box/function1.test.lua b/test/box/function1.test.lua index e983495b6..cf4a71979 100644 --- a/test/box/function1.test.lua +++ b/test/box/function1.test.lua @@ -21,6 +21,10 @@ box.schema.user.grant('guest', 'execute', 'function', 'function1.args') c:call('function1.args') c:call('function1.args', { "xx" }) c:call('function1.args', { 15 }) +box.func["function1.args"] +box.func["function1.args"]:call() +box.func["function1.args"]:call({ "xx" }) +box.func["function1.args"]:call({ 15 }) box.schema.func.drop("function1.args") box.schema.func.create('function1.multi_inc', {language = "C"}) @@ -112,3 +116,65 @@ identifier.run_test( ); test_run:cmd("setopt delimiter ''"); c:close() + +-- Test registered functions interface. +function divide(a, b) return a / b end +box.schema.func.create("divide") +func = box.func.divide +func.call({4, 2}) +func:call(4, 2) +func:call() +func:call({}) +func:call({4}) +func:call({4, 2}) +func:call({4, 2, 1}) +func:drop() +func +func.drop() +func:drop() +func:call({4, 2}) +box.internal.func_call('divide', 4, 2) + +box.schema.func.create("function1.divide", {language = 'C'}) +func = box.func["function1.divide"] +func:call(4, 2) +func:call() +func:call({}) +func:call({4}) +func:call({4, 2}) +func:call({4, 2, 1}) +func:drop() +func +func:drop() +func:call({4, 2}) +box.internal.func_call('function1.divide', 4, 2) + +test_run:cmd("setopt delimiter ';'") +function minmax(array) + local min = 999 + local max = -1 + for _, v in pairs(array) do + min = math.min(min, v) + max = math.max(max, v) + end + return min, max +end +test_run:cmd("setopt delimiter ''"); +box.schema.func.create("minmax") +func = box.func.minmax +func:call({{1, 2, 99, 3, -1}}) +func:drop() + +-- Test access checks for registered functions. +function secret() return 1 end +box.schema.func.create("secret") +box.func.secret:call({}) +function secret_leak() return box.func.secret:call() end +box.schema.func.create('secret_leak') +box.schema.user.grant('guest', 'execute', 'function', 'secret_leak') +conn = net.connect(box.cfg.listen) +conn:call('secret_leak') +conn:close() +box.schema.user.revoke('guest', 'execute', 'function', 'secret_leak') +box.schema.func.drop('secret_leak') +box.schema.func.drop('secret') diff --git a/test/box/misc.result b/test/box/misc.result index 43b5a4a15..030c87c55 100644 --- a/test/box/misc.result +++ b/test/box/misc.result @@ -65,6 +65,7 @@ t - error - execute - feedback + - func - index - info - internal -- 2.21.0