[Tarantool-patches] [PATCH v8 3/4] box/cbox: implement cbox Lua module

[Cyrill Gorcunov <gorcunov@gmail.com>]

Currently to run "C" function from some external module
one have to register it first in "_func" system space. This
is a problem if node is in read-only mode (replica).

Still people would like to have a way to run such functions
even in ro mode. For this sake we implement "cbox" lua module.

Fixes #4692

Signed-off-by: Cyrill Gorcunov <gorcunov@gmail.com>

@TarantoolBot document
Title: cbox module

Overview
========

`cbox` module provides a way to create, delete and execute
`C` procedures. Unlinke `box.schema.func` functionality this
the functions created with `cbox` help are not persistent and
live purely in memory. Once a node get turned off they are
vanished. An initial purpose for them is to execute them on
nodes which are running in read-only mode.

Module functions
================

`cbox.func.load([dso.]name) -> obj | nil, err`
----------------------------------------------

Loads a new function with name `name` from module `dso.`.
The module name is optional and if not provided implies
to be the same as `name`.

The `load` call must be paired with `unload` and these
calls are accounted. Until coupled `unload` is called
the instance is present in memory. Any `load` calls
followed by another `load` with same name simply
increase a reference to the existing function.

Possible errors:
 - IllegalParams: function name is either not supplied
   or not a string.
 - IllegalParams: function name is too long.
 - IllegalParams: function references limit exceeded.
 - OutOfMemory: unable to allocate a function.

On success a new callable object is returned,
otherwise `nil, error` pair.

Example:

``` Lua
f, err = require('cbox').func.load('func')
if not f then
    print(err)
end
```

Once function is loaded it is possible to execute it
in a traditional Lua way, ie to call it as a function.

``` Lua
-- call with agruments arg1 and arg2
f(arg1, arg2)
```

`cbox.func.unload([dso.]name) -> true | nil, err`
-------------------------------------------------

Unload a function with name `[dso.]name`. Since function
instances are accounted the function is not unloaded until
number of `unload` calls matched to the number of `load`
calls.

Possible errors:
 - IllegalParams: function name is either not supplied
   or not a string.
 - IllegalParams: the function does not exist.

On success `true` is returned, otherwise `nil, error` pair.

Example:

``` Lua
ok, err = require('cbox').func.unload('func')
if not ok then
    print(err)
end
```

`cbox.module.reload(name) -> true | nil, err`
---------------------------------------------

Reloads a module with name `name` and all functions which
were associated for the module. Each module keeps a list of
functions belonging to the module and reload procedure cause
the bound function to update their addresses such that
function execution will be routed via a new library.

Modules are loaded with that named local binding which means
that reload of module symbols won't affect the functions which
are started execution already, only new calls will be rerouted.

Possible errors:
 - IllegalParams: module name is either not supplied
   or not a string.
 - ClientError: a module with the name provided does
   not exist.

On success `true` is returned, otherwise `nil,error` pair.

Example:

``` Lua
ok, err = require('cbox').module.reload('func')
if not ok then
    print(err)
end
```
---
 src/box/CMakeLists.txt |   1 +
 src/box/box.cc         |   5 +
 src/box/lua/cbox.c     | 486 +++++++++++++++++++++++++++++++++++++++++
 src/box/lua/cbox.h     |  39 ++++
 src/box/lua/init.c     |   2 +
 5 files changed, 533 insertions(+)
 create mode 100644 src/box/lua/cbox.c
 create mode 100644 src/box/lua/cbox.h

diff --git a/src/box/CMakeLists.txt b/src/box/CMakeLists.txt
index 91d85c6db..1202697c5 100644
--- a/src/box/CMakeLists.txt
+++ b/src/box/CMakeLists.txt
@@ -197,6 +197,7 @@ add_library(box STATIC
     lua/init.c
     lua/call.c
     lua/cfg.cc
+    lua/cbox.c
     lua/console.c
     lua/serialize_lua.c
     lua/tuple.c
diff --git a/src/box/box.cc b/src/box/box.cc
index 2485b79f3..f20761e8f 100644
--- a/src/box/box.cc
+++ b/src/box/box.cc
@@ -75,6 +75,7 @@
 #include "systemd.h"
 #include "call.h"
 #include "module_cache.h"
+#include "lua/cbox.h"
 #include "sequence.h"
 #include "sql_stmt_cache.h"
 #include "msgpack.h"
@@ -2246,6 +2247,7 @@ box_free(void)
 		tuple_free();
 		port_free();
 #endif
+		cbox_free();
 		iproto_free();
 		replication_free();
 		sequence_free();
@@ -2647,6 +2649,9 @@ box_init(void)
 	if (module_init() != 0)
 		diag_raise();
 
+	if (cbox_init() != 0)
+		diag_raise();
+
 	if (tuple_init(lua_hash) != 0)
 		diag_raise();
 
diff --git a/src/box/lua/cbox.c b/src/box/lua/cbox.c
new file mode 100644
index 000000000..0d8208024
--- /dev/null
+++ b/src/box/lua/cbox.c
@@ -0,0 +1,486 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright 2010-2020, Tarantool AUTHORS, please see AUTHORS file.
+ */
+
+#include <string.h>
+#include <lua.h>
+
+#define RB_COMPACT 1
+#include <small/rb.h>
+
+#include "diag.h"
+
+#include "box/module_cache.h"
+#include "box/error.h"
+#include "box/port.h"
+
+#include "trivia/util.h"
+#include "lua/utils.h"
+
+/**
+ * Function descriptor.
+ */
+struct cbox_func {
+	/**
+	 * Gather functions into rbtree.
+	 */
+	rb_node(struct cbox_func) nd;
+
+	/**
+	 * Symbol descriptor for the function in
+	 * an associated module.
+	 */
+	struct module_sym mod_sym;
+
+	/**
+	 * Number of references to the function
+	 * instance.
+	 */
+	ssize_t ref;
+
+	/** Function name. */
+	const char *name;
+
+	/** Function name length. */
+	size_t name_len;
+
+	/** Function name keeper. */
+	char inplace[0];
+};
+
+/**
+ * A tree to lookup functions by name.
+ */
+typedef rb_tree(struct cbox_func) func_rb_t;
+static func_rb_t func_rb_root;
+
+static int
+cbox_func_cmp(const struct cbox_func *a, const struct cbox_func *b)
+{
+	ssize_t len = (ssize_t)a->name_len - (ssize_t)b->name_len;
+	if (len == 0)
+		return strcmp(a->name, b->name);
+	return len < 0 ? -1 : 1;
+}
+
+rb_gen(MAYBE_UNUSED static, func_rb_, func_rb_t,
+       struct cbox_func, nd, cbox_func_cmp);
+
+/**
+ * Find function in a tree.
+ */
+struct cbox_func *
+cbox_func_find(const char *name, size_t name_len)
+{
+	struct cbox_func v = {
+		.name		= name,
+		.name_len	= name_len,
+	};
+	return func_rb_search(&func_rb_root, &v);
+}
+
+/**
+ * Unreference a function instance.
+ */
+static void
+cbox_func_unref(struct cbox_func *cf)
+{
+	assert(cf->ref > 0);
+	if (cf->ref-- == 1)
+		func_rb_remove(&func_rb_root, cf);
+}
+
+/**
+ * Reference a function instance.
+ */
+static bool
+cbox_func_ref(struct cbox_func *cf)
+{
+	assert(cf->ref >= 0);
+
+	/*
+	 * Hardly to ever happen but just
+	 * to make sure.
+	 */
+	if (cf->ref == SSIZE_MAX) {
+		const char *fmt =
+			"Too many function references (max %zd)";
+		diag_set(IllegalParams, fmt, SSIZE_MAX);
+		return false;
+	}
+
+	if (cf->ref++ == 0)
+		func_rb_insert(&func_rb_root, cf);
+
+	return true;
+}
+
+/**
+ * Allocate a new function instance.
+ */
+static struct cbox_func *
+cbox_func_new(const char *name, size_t name_len)
+{
+	const ssize_t cf_size = sizeof(struct cbox_func);
+	ssize_t size = cf_size + name_len + 1;
+	if (size < 0) {
+		const size_t max_len = SSIZE_MAX - cf_size - 1;
+		const char *fmt = "Function name is too long (max %zd)";
+		diag_set(IllegalParams, fmt, max_len);
+		return NULL;
+	}
+
+	struct cbox_func *cf = malloc(size);
+	if (cf == NULL) {
+		diag_set(OutOfMemory, size, "malloc", "cf");
+		return NULL;
+	}
+
+	cf->mod_sym.addr	= NULL;
+	cf->mod_sym.module	= NULL;
+	cf->ref			= 0;
+	cf->mod_sym.name	= cf->inplace;
+	cf->name		= cf->inplace;
+	cf->name_len		= name_len;
+
+	memcpy(cf->inplace, name, name_len);
+	cf->inplace[name_len] = '\0';
+
+	memset(&cf->nd, 0, sizeof(cf->nd));
+	return cf;
+}
+
+/**
+ * Fetch a function instance from the Lua object.
+ */
+static struct cbox_func *
+lcbox_func_get_handler(struct lua_State *L)
+{
+	struct cbox_func *cf = NULL;
+	int top = lua_gettop(L);
+
+	if (lua_getmetatable(L, -top) != 0) {
+		lua_getfield(L, -1, "__cbox_func");
+		if (lua_isuserdata(L, -1)) {
+			struct cbox_func **pcf = lua_touserdata(L, -1);
+			cf = pcf[0];
+		}
+		lua_pop(L, 2);
+	}
+	return cf;
+}
+
+/**
+ * Free a function instance.
+ *
+ * It is called by Lua itself when a variable has no more reference.
+ * Since we associate a function instance for each variable we
+ * can't just free the memory immediately, instead we must be sure
+ * the final unload() is called and there are no more instances left
+ * in the tree thus next load() will have to allocate a new instance.
+ */
+static int
+lcbox_func_gc(struct lua_State *L)
+{
+	struct cbox_func *cf = lcbox_func_get_handler(L);
+	if (cf->ref == 0) {
+		TRASH(cf);
+		free(cf);
+	}
+	return 0;
+}
+
+/**
+ * Call a function by its name from the Lua code.
+ */
+static int
+lcbox_func_call(struct lua_State *L)
+{
+	struct cbox_func *cf = lcbox_func_get_handler(L);
+	if (cf == NULL) {
+		/*
+		 * How can this happen? Someone screwed
+		 * internal object data intentionally?
+		 * Whatever, the pointer is ruined we
+		 * can't do anything.
+		 */
+		const char *fmt = "Function is corrupted";
+		diag_set(IllegalParams, fmt);
+		return luaT_push_nil_and_error(L);
+	}
+
+	/*
+	 * FIXME: We should get rid of luaT_newthread but this
+	 * requires serious modifications. In particular
+	 * port_lua_do_dump uses tarantool_L reference and
+	 * coro_ref must be valid as well.
+	 */
+	lua_State *args_L = luaT_newthread(tarantool_L);
+	if (args_L == NULL)
+		return luaT_push_nil_and_error(L);
+
+	int coro_ref = luaL_ref(tarantool_L, LUA_REGISTRYINDEX);
+	lua_xmove(L, args_L, lua_gettop(L) - 1);
+
+	struct port args;
+	port_lua_create(&args, args_L);
+	((struct port_lua *)&args)->ref = coro_ref;
+
+	struct port ret;
+	if (module_sym_call(&cf->mod_sym, &args, &ret) != 0) {
+		port_destroy(&args);
+		return luaT_push_nil_and_error(L);
+	}
+
+	int top = lua_gettop(L);
+	lua_pushboolean(L, true);
+	port_dump_lua(&ret, L, true);
+	int cnt = lua_gettop(L) - top;
+
+	port_destroy(&ret);
+	port_destroy(&args);
+
+	return cnt;
+}
+
+/**
+ * Load a new function.
+ *
+ * This function takes a function name from the caller
+ * stack @a L and creates a new function object. If
+ * the function is already loaded we simply return
+ * a reference to existing one.
+ *
+ * Possible errors:
+ *
+ * - IllegalParams: function name is either not supplied
+ *   or not a string.
+ * - IllegalParams: function references limit exceeded.
+ * - OutOfMemory: unable to allocate a function.
+ *
+ * @returns function object on success or {nil,error} on error,
+ * the error is set to the diagnostics area.
+ */
+static int
+lcbox_func_load(struct lua_State *L)
+{
+	const char *method = "cbox.func.load";
+	struct cbox_func *cf = NULL;
+
+	if (lua_gettop(L) != 1 || !lua_isstring(L, 1)) {
+		const char *fmt =
+			"Expects %s(\'name\') but no name passed";
+		diag_set(IllegalParams, fmt, method);
+		return luaT_push_nil_and_error(L);
+	}
+
+	size_t name_len;
+	const char *name = lua_tolstring(L, 1, &name_len);
+
+	cf = cbox_func_find(name, name_len);
+	if (cf == NULL) {
+		cf = cbox_func_new(name, name_len);
+		if (cf == NULL)
+			return luaT_push_nil_and_error(L);
+	}
+	if (!cbox_func_ref(cf))
+		return luaT_push_nil_and_error(L);
+
+	lua_newtable(L);
+
+	lua_pushstring(L, "name");
+	lua_pushstring(L, cf->name);
+	lua_settable(L, -3);
+
+	lua_newtable(L);
+
+	/*
+	 * A new variable should be callable for
+	 * convenient use in Lua.
+	 */
+	lua_pushstring(L, "__call");
+	lua_pushcfunction(L, lcbox_func_call);
+	lua_settable(L, -3);
+
+	/*
+	 * We will release the memory associated
+	 * with the objet if only no active refs
+	 * are left.
+	 */
+	lua_pushstring(L, "__gc");
+	lua_pushcfunction(L, lcbox_func_gc);
+	lua_settable(L, -3);
+
+	/*
+	 * Carry the pointer to the function so
+	 * we won't need to run a lookup when
+	 * calling.
+	 */
+	lua_pushstring(L, "__cbox_func");
+	*(struct cbox_func **)lua_newuserdata(L, sizeof(cf)) = cf;
+	lua_settable(L, -3);
+
+	lua_setmetatable(L, -2);
+	return 1;
+}
+
+/**
+ * Unload a function.
+ *
+ * This function takes a function name from the caller
+ * stack @a L and unloads a function object.
+ *
+ * Possible errors:
+ *
+ * - IllegalParams: function name is either not supplied
+ *   or not a string.
+ * - IllegalParams: the function does not exist.
+ *
+ * @returns true on success or {nil,error} on error,
+ * the error is set to the diagnostics area.
+ */
+static int
+lcbox_func_unload(struct lua_State *L)
+{
+	const char *method = "cbox.func.unload";
+	const char *name = NULL;
+
+	if (lua_gettop(L) != 1 || !lua_isstring(L, 1)) {
+		const char *fmt =
+			"Expects %s(\'name\') but no name passed";
+		diag_set(IllegalParams, fmt, method);
+		return luaT_push_nil_and_error(L);
+	}
+
+	size_t name_len;
+	name = lua_tolstring(L, 1, &name_len);
+
+	struct cbox_func *cf = cbox_func_find(name, name_len);
+	if (cf == NULL) {
+		const char *fmt = tnt_errcode_desc(ER_NO_SUCH_FUNCTION);
+		diag_set(IllegalParams, fmt, name);
+		return luaT_push_nil_and_error(L);
+	}
+
+	cbox_func_unref(cf);
+	lua_pushboolean(L, true);
+	return 1;
+}
+
+/**
+ * Reload a module.
+ *
+ * This function takes a module name from the caller
+ * stack @a L and reloads all functions associated with
+ * the module.
+ *
+ * Possible errors:
+ *
+ * - IllegalParams: module name is either not supplied
+ *   or not a string.
+ * - IllegalParams: the function does not exist.
+ * - ClientError: a module with the name provided does
+ *   not exist.
+ *
+ * @returns true on success or {nil,error} on error,
+ * the error is set to the diagnostics area.
+ */
+static int
+lcbox_module_reload(struct lua_State *L)
+{
+	const char *method = "cbox.module.reload";
+	const char *fmt = "Expects %s(\'name\') but no name passed";
+
+	if (lua_gettop(L) != 1 || !lua_isstring(L, 1)) {
+		diag_set(IllegalParams, fmt, method);
+		return luaT_push_nil_and_error(L);
+	}
+
+	size_t name_len;
+	const char *name = lua_tolstring(L, 1, &name_len);
+	if (name == NULL || name_len < 1) {
+		diag_set(IllegalParams, fmt, method);
+		return luaT_push_nil_and_error(L);
+	}
+
+	struct module *module = NULL;
+	if (module_reload(name, &name[name_len], &module) == 0) {
+		if (module != NULL) {
+			lua_pushboolean(L, true);
+			return 1;
+		}
+		diag_set(ClientError, ER_NO_SUCH_MODULE, name);
+	}
+	return luaT_push_nil_and_error(L);
+}
+
+/**
+ * Initialize cbox Lua module.
+ *
+ * @param L Lua state where to register the cbox module.
+ */
+void
+box_lua_cbox_init(struct lua_State *L)
+{
+	static const struct luaL_Reg cbox_methods[] = {
+		{ NULL, NULL }
+	};
+	luaL_register_module(L, "cbox", cbox_methods);
+
+	/* func table */
+	static const struct luaL_Reg func_table[] = {
+		{ "load",	lcbox_func_load },
+		{ "unload",	lcbox_func_unload },
+	};
+
+	lua_newtable(L);
+	for (size_t i = 0; i < lengthof(func_table); i++) {
+		lua_pushstring(L, func_table[i].name);
+		lua_pushcfunction(L, func_table[i].func);
+		lua_settable(L, -3);
+	}
+	lua_setfield(L, -2, "func");
+
+	/* module table */
+	static const struct luaL_Reg module_table[] = {
+		{ "reload",	lcbox_module_reload },
+	};
+
+	lua_newtable(L);
+	for (size_t i = 0; i < lengthof(module_table); i++) {
+		lua_pushstring(L, module_table[i].name);
+		lua_pushcfunction(L, module_table[i].func);
+		lua_settable(L, -3);
+	}
+	lua_setfield(L, -2, "module");
+
+	lua_pop(L, 1);
+}
+
+/**
+ * Initialize cbox module.
+ *
+ * @return 0 on success, -1 on error (diag is set).	
+ */
+int
+cbox_init(void)
+{
+	func_rb_new(&func_rb_root);
+	return 0;
+}
+
+/**
+ * Free cbox module.
+ */
+void
+cbox_free(void)
+{
+	struct cbox_func *cf = func_rb_first(&func_rb_root);
+	while (cf != NULL) {
+		func_rb_remove(&func_rb_root, cf);
+		cf = func_rb_first(&func_rb_root);
+	}
+	func_rb_new(&func_rb_root);
+}
diff --git a/src/box/lua/cbox.h b/src/box/lua/cbox.h
new file mode 100644
index 000000000..1e6cd9a9b
--- /dev/null
+++ b/src/box/lua/cbox.h
@@ -0,0 +1,39 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright 2010-2020, Tarantool AUTHORS, please see AUTHORS file.
+ */
+
+#pragma once
+
+#if defined(__cplusplus)
+extern "C" {
+#endif /* defined(__cplusplus) */
+
+struct lua_State;
+
+/**
+ * Initialize cbox Lua module.
+ *
+ * @param L Lua state where to register the cbox module.
+ */
+void
+box_lua_cbox_init(struct lua_State *L);
+
+/**
+ * Initialize cbox module.
+ *
+ * @return 0 on success, -1 on error (diag is set).	
+ */
+int
+cbox_init(void);
+
+/**
+ * Free cbox module.
+ */
+void
+cbox_free(void);
+
+#if defined(__cplusplus)
+}
+#endif /* defined(__plusplus) */
diff --git a/src/box/lua/init.c b/src/box/lua/init.c
index d0316ef86..b37aa284a 100644
--- a/src/box/lua/init.c
+++ b/src/box/lua/init.c
@@ -61,6 +61,7 @@
 #include "box/lua/cfg.h"
 #include "box/lua/xlog.h"
 #include "box/lua/console.h"
+#include "box/lua/cbox.h"
 #include "box/lua/tuple.h"
 #include "box/lua/execute.h"
 #include "box/lua/key_def.h"
@@ -466,6 +467,7 @@ box_lua_init(struct lua_State *L)
 	box_lua_tuple_init(L);
 	box_lua_call_init(L);
 	box_lua_cfg_init(L);
+	box_lua_cbox_init(L);
 	box_lua_slab_init(L);
 	box_lua_index_init(L);
 	box_lua_space_init(L);
-- 
2.26.2