From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: From: Kirill Shcherbatov Subject: [PATCH v1 4/8] box: load persistent Lua functions on creation Date: Thu, 30 May 2019 13:45:31 +0300 Message-Id: <9ed730e31d6c261f64239f2f4124b98716770156.1559212747.git.kshcherbatov@tarantool.org> In-Reply-To: References: MIME-Version: 1.0 Content-Transfer-Encoding: 8bit To: tarantool-patches@freelists.org, vdavydov.dev@gmail.com Cc: Kirill Shcherbatov List-ID: This patch proceed persistent Lua function load on function object creation. Each persistent function may use limited amount of Lua functions and modules: -assert -error -pairs -ipairs -next -pcall -xpcall -type -print -select -string -tonumber -tostring -unpack -math -utf8 Global variables are forbidden in persistent Lua functions. Part of #4182 Needed for #1260 --- src/box/func.c | 69 ++++++++++++++++++++++ src/box/func.h | 5 ++ src/box/func_def.h | 6 ++ src/lua/utils.c | 67 ++++++++++++++++++++++ src/lua/utils.h | 8 +++ test/box/persistent_func.result | 95 +++++++++++++++++++++++++++++++ test/box/persistent_func.test.lua | 47 +++++++++++++++ 7 files changed, 297 insertions(+) create mode 100644 test/box/persistent_func.result create mode 100644 test/box/persistent_func.test.lua diff --git a/src/box/func.c b/src/box/func.c index f7465be7e..31db7b477 100644 --- a/src/box/func.c +++ b/src/box/func.c @@ -34,6 +34,7 @@ #include "lua/utils.h" #include "error.h" #include "diag.h" +#include "fiber.h" #include /** @@ -355,6 +356,63 @@ restore: return -1; } +/* + * Assemble a Lua function object using luaL_loadstring of the + * special 'return FUNCTION_BODY' expression and calling it. + * Set default sandbox to make function use only a limited number + * of functions and modules. + */ +static int +execute_func_lua_load(struct lua_State *L) +{ + struct func *func = (struct func *) lua_topointer(L, 1); + assert(func->lua_func_ref == LUA_REFNIL); + lua_settop(L, 0); + + struct region *region = &fiber()->gc; + size_t region_svp = region_used(region); + const char *load_pref = "return "; + uint32_t load_str_sz = strlen(load_pref) + strlen(func->def->body) + 1; + char *load_str = region_alloc(region, load_str_sz); + if (load_str == NULL) { + diag_set(OutOfMemory, load_str_sz, "region", "load_str"); + goto error; + } + sprintf(load_str, "%s%s", load_pref, func->def->body); + if (luaL_loadstring(L, load_str) != 0 || + luaT_call(L, 0, 1) != 0 || !lua_isfunction(L, -1) || + luaT_get_sandbox(L) != 0) { + diag_set(ClientError, ER_LOAD_FUNCTION, func->def->name, + func->def->body); + goto error; + } + lua_setfenv(L, -2); + func->lua_func_ref = luaL_ref(L, LUA_REGISTRYINDEX); + region_truncate(region, region_svp); + return lua_gettop(L); +error: + region_truncate(region, region_svp); + return luaT_error(L); +} + +/** + * Perform persistent Lua function loading for given function + * object. + * Returns 0 in case of success, -1 otherwise and sets the diag + * message. + */ +static int +func_lua_load(struct func *func) +{ + lua_State *L = lua_newthread(tarantool_L); + int coro_ref = luaL_ref(tarantool_L, LUA_REGISTRYINDEX); + lua_pushcfunction(L, execute_func_lua_load); + lua_pushlightuserdata(L, func); + int rc = luaT_call(L, 1, 1); + luaL_unref(tarantool_L, LUA_REGISTRYINDEX, coro_ref); + return rc; +} + struct func * func_new(struct func_def *def) { @@ -380,6 +438,14 @@ func_new(struct func_def *def) func->owner_credentials.auth_token = BOX_USER_MAX; /* invalid value */ func->func = NULL; func->module = NULL; + func->lua_func_ref = LUA_REFNIL; + if (func_def_is_persistent(func->def)) { + if (func_lua_load(func) != 0) { + free(func); + return NULL; + } + assert(func->lua_func_ref != LUA_REFNIL); + } return func; } @@ -395,8 +461,11 @@ func_unload(struct func *func) } module_gc(func->module); } + if (func->lua_func_ref != LUA_REFNIL) + luaL_unref(tarantool_L, LUA_REGISTRYINDEX, func->lua_func_ref); func->module = NULL; func->func = NULL; + func->lua_func_ref = LUA_REFNIL; } /** diff --git a/src/box/func.h b/src/box/func.h index a4a758b58..7c3e81c51 100644 --- a/src/box/func.h +++ b/src/box/func.h @@ -65,6 +65,11 @@ struct func { * Anchor for module membership. */ struct rlist item; + /** + * The reference index of Lua function object. + * Is equal to LUA_REFNIL when undefined. + */ + int lua_func_ref; /** * For C functions, the body of the function. */ diff --git a/src/box/func_def.h b/src/box/func_def.h index 78fef9d22..2cbaddd1a 100644 --- a/src/box/func_def.h +++ b/src/box/func_def.h @@ -87,6 +87,12 @@ func_def_sizeof(uint32_t name_len, uint32_t body_len) return sz; } +static inline bool +func_def_is_persistent(struct func_def *def) +{ + return def->body != NULL; +} + /** * API of C stored function. */ diff --git a/src/lua/utils.c b/src/lua/utils.c index 27ff6b396..0d1cca423 100644 --- a/src/lua/utils.c +++ b/src/lua/utils.c @@ -46,6 +46,14 @@ static uint32_t CTID_STRUCT_IBUF_PTR; uint32_t CTID_CHAR_PTR; uint32_t CTID_CONST_CHAR_PTR; +static const char *default_sandbox_exports[] = + {"assert", "error", "ipairs", "math", "next", "pairs", "pcall", + "print", "select", "string", "table", "tonumber", "tostring", + "type", "unpack", "xpcall", "utf8"}; + +static int luaL_deepcopy_func_ref = LUA_REFNIL; +static int luaL_default_sandbox_ref = LUA_REFNIL; + void * luaL_pushcdata(struct lua_State *L, uint32_t ctypeid) { @@ -1248,6 +1256,65 @@ luaT_func_find(struct lua_State *L, const char *name, const char *name_end, return 0; } +/** + * Assemble a new sandbox with given exports table on top of the + * Lua stack. All modules in exports list are copying deeply + * to ensure the immutablility of this system object. + */ +static int +luaT_prepare_sandbox(struct lua_State *L, const char *exports[], + uint32_t exports_count) +{ + assert(luaL_deepcopy_func_ref != LUA_REFNIL); + lua_createtable(L, exports_count, 0); + for (unsigned i = 0; i < exports_count; i++) { + int count; + uint32_t name_len = strlen(exports[i]); + if (luaT_func_find(L, exports[i], exports[i] + name_len, + &count) != 0) + return -1; + switch (lua_type(L, -1)) { + case LUA_TTABLE: + lua_rawgeti(L, LUA_REGISTRYINDEX, + luaL_deepcopy_func_ref); + lua_insert(L, -2); + lua_call(L, 1, LUA_MULTRET); + FALLTHROUGH; + case LUA_TFUNCTION: + break; + default: + unreachable(); + } + lua_setfield(L, -2, exports[i]); + } + return 0; +} + +int +luaT_get_sandbox(struct lua_State *L) +{ + if (luaL_deepcopy_func_ref == LUA_REFNIL) { + int count; + const char *deepcopy = "table.deepcopy"; + if (luaT_func_find(L, deepcopy, deepcopy + strlen(deepcopy), + &count) != 0) + return -1; + luaL_deepcopy_func_ref = luaL_ref(L, LUA_REGISTRYINDEX); + assert(luaL_deepcopy_func_ref != LUA_REFNIL); + } + if (luaL_default_sandbox_ref == LUA_REFNIL) { + if (luaT_prepare_sandbox(L, default_sandbox_exports, + nelem(default_sandbox_exports)) != 0) + return -1; + luaL_default_sandbox_ref = luaL_ref(L, LUA_REGISTRYINDEX); + assert(luaL_default_sandbox_ref != LUA_REFNIL); + } + lua_rawgeti(L, LUA_REGISTRYINDEX, luaL_deepcopy_func_ref); + lua_rawgeti(L, LUA_REGISTRYINDEX, luaL_default_sandbox_ref); + lua_call(L, 1, LUA_MULTRET); + return 0; +} + int tarantool_lua_utils_init(struct lua_State *L) { diff --git a/src/lua/utils.h b/src/lua/utils.h index 81e936bee..36bb2e53b 100644 --- a/src/lua/utils.h +++ b/src/lua/utils.h @@ -603,6 +603,14 @@ int luaT_func_find(struct lua_State *L, const char *name, const char *name_end, int *count); +/** + * Prepare a new 'default' sandbox table on the top of the Lua + * stack. The 'default' sandbox consists of a minimum set of + * functions that are sufficient to serve persistent functions. + */ +int +luaT_get_sandbox(struct lua_State *L); + int tarantool_lua_utils_init(struct lua_State *L); diff --git a/test/box/persistent_func.result b/test/box/persistent_func.result new file mode 100644 index 000000000..0644de7fe --- /dev/null +++ b/test/box/persistent_func.result @@ -0,0 +1,95 @@ +env = require('test_run') +--- +... +test_run = env.new() +--- +... +-- +-- gh-4182: Add persistent LUA functions. +-- +-- Test valid function. +test_run:cmd("setopt delimiter ';'") +--- +- true +... +body = [[function(tuple) + if type(tuple.address) ~= 'string' then return nil, 'Invalid field type' end + local t = tuple.address:lower():split() + for k,v in pairs(t) do t[k] = {v} end + return t +end +]] +test_run:cmd("setopt delimiter ''"); +--- +... +box.schema.func.create('test', {body = body, language = "C"}) +--- +- error: 'Failed to create function ''test'': function body may be specified only + for Lua language' +... +box.schema.func.create('test', {body = body}) +--- +... +box.schema.func.exists('test') +--- +- true +... +box.schema.func.create('test2', {body = body, is_deterministic = true}) +--- +... +-- Test function with spell error - case 1. +test_run:cmd("setopt delimiter ';'") +--- +- true +... +body_bad2 = [[function(tuple) + ret tuple +end +]] +test_run:cmd("setopt delimiter ''"); +--- +... +box.schema.func.create('body_bad2', {body = body_bad2}) +--- +- error: "Failed to dynamically load function 'body_bad2': function(tuple) \tret tuple + end " +... +-- Test function with spell error - case 2. +test_run:cmd("setopt delimiter ';'") +--- +- true +... +body_bad3 = [[func(tuple) + return tuple +end +]] +test_run:cmd("setopt delimiter ''"); +--- +... +box.schema.func.create('body_bad3', {body = body_bad3}) +--- +- error: "Failed to dynamically load function 'body_bad3': func(tuple) \treturn tuple + end " +... +-- Restart server. +test_run:cmd("restart server default") +net = require('net.box') +--- +... +test_run = require('test_run').new() +--- +... +box.schema.func.exists('test') +--- +- true +... +box.schema.func.drop('test') +--- +... +box.schema.func.exists('test') +--- +- false +... +box.schema.func.drop('test2') +--- +... diff --git a/test/box/persistent_func.test.lua b/test/box/persistent_func.test.lua new file mode 100644 index 000000000..37a761d32 --- /dev/null +++ b/test/box/persistent_func.test.lua @@ -0,0 +1,47 @@ +env = require('test_run') +test_run = env.new() + +-- +-- gh-4182: Add persistent LUA functions. +-- +-- Test valid function. +test_run:cmd("setopt delimiter ';'") +body = [[function(tuple) + if type(tuple.address) ~= 'string' then return nil, 'Invalid field type' end + local t = tuple.address:lower():split() + for k,v in pairs(t) do t[k] = {v} end + return t +end +]] +test_run:cmd("setopt delimiter ''"); +box.schema.func.create('test', {body = body, language = "C"}) +box.schema.func.create('test', {body = body}) +box.schema.func.exists('test') +box.schema.func.create('test2', {body = body, is_deterministic = true}) + +-- Test function with spell error - case 1. +test_run:cmd("setopt delimiter ';'") +body_bad2 = [[function(tuple) + ret tuple +end +]] +test_run:cmd("setopt delimiter ''"); +box.schema.func.create('body_bad2', {body = body_bad2}) + +-- Test function with spell error - case 2. +test_run:cmd("setopt delimiter ';'") +body_bad3 = [[func(tuple) + return tuple +end +]] +test_run:cmd("setopt delimiter ''"); +box.schema.func.create('body_bad3', {body = body_bad3}) + +-- Restart server. +test_run:cmd("restart server default") +net = require('net.box') +test_run = require('test_run').new() +box.schema.func.exists('test') +box.schema.func.drop('test') +box.schema.func.exists('test') +box.schema.func.drop('test2') -- 2.21.0