[tarantool-patches] [PATCH v1 2/2] box: functional and multikey indexes
Kirill Shcherbatov
kshcherbatov at tarantool.org
Sat Nov 17 16:16:55 MSK 2018
Tarantool functional and multikey indexes allows you to create
an index based on user-specified function. In a nutshell, this
capability allows you to have case insensitive searches or sorts,
search on complex equations, and extend the SQL language
efficiently by implementing your own functions and operators and
then searching on them.
Feature use hidden space _i_{index_name}_{space_name} containing
tuples of structure [{extractor_format}{pk_format}] and redefines
on_replace trigger for target space to fill this space. Index
object metamethods are monkeypatched to make indirect lookups in
ispace to retrieve required tuple primary key.
Closes #1260
---
src/box/CMakeLists.txt | 1 +
src/box/alter.cc | 61 ++++++-
src/box/index.cc | 9 +
src/box/index.h | 4 +
src/box/index_def.c | 64 +++++++-
src/box/index_def.h | 14 ++
src/box/lua/call.c | 43 +++++
src/box/lua/call.h | 18 ++
src/box/lua/func_idx.lua | 319 ++++++++++++++++++++++++++++++++++++
src/box/lua/schema.lua | 49 +++++-
src/box/lua/space.cc | 21 +++
src/box/memtx_engine.c | 6 +
src/box/vinyl.c | 6 +
src/lua/init.c | 2 +
test/engine/functional_idx.result | 120 ++++++++++++++
test/engine/functional_idx.test.lua | 35 ++++
16 files changed, 751 insertions(+), 21 deletions(-)
create mode 100644 src/box/lua/func_idx.lua
create mode 100644 test/engine/functional_idx.result
create mode 100644 test/engine/functional_idx.test.lua
diff --git a/src/box/CMakeLists.txt b/src/box/CMakeLists.txt
index d127647..fd244db 100644
--- a/src/box/CMakeLists.txt
+++ b/src/box/CMakeLists.txt
@@ -13,6 +13,7 @@ lua_source(lua_sources lua/net_box.lua)
lua_source(lua_sources lua/upgrade.lua)
lua_source(lua_sources lua/console.lua)
lua_source(lua_sources lua/xlog.lua)
+lua_source(lua_sources lua/func_idx.lua)
set(bin_sources)
bin_source(bin_sources bootstrap.snap bootstrap.h)
diff --git a/src/box/alter.cc b/src/box/alter.cc
index 6d2c59b..fc5a27c 100644
--- a/src/box/alter.cc
+++ b/src/box/alter.cc
@@ -54,6 +54,7 @@
#include "version.h"
#include "sequence.h"
#include "sql.h"
+#include "lua/call.h"
/**
* chap-sha1 of empty string, i.e.
@@ -279,7 +280,13 @@ index_def_new_from_tuple(struct tuple *tuple, struct space *space)
space->def->fields,
space->def->field_count) != 0)
diag_raise();
- key_def = key_def_new(part_def, part_count);
+ if (opts.func_code != NULL) {
+ struct index *index = space_index(space, 0);
+ assert(index != NULL);
+ key_def = key_def_dup(index->def->key_def);
+ } else {
+ key_def = key_def_new(part_def, part_count);
+ }
if (key_def == NULL)
diag_raise();
struct index_def *index_def =
@@ -1380,6 +1387,46 @@ UpdateSchemaVersion::alter(struct alter_space *alter)
++schema_version;
}
+class PrepareFunctionalIndex: public AlterSpaceOp
+{
+public:
+ PrepareFunctionalIndex(struct alter_space * alter, uint32_t iid)
+ :AlterSpaceOp(alter), iid(iid), func_ref(0) {}
+ virtual ~PrepareFunctionalIndex();
+ virtual void prepare(struct alter_space *alter);
+ virtual void commit(struct alter_space *alter, int64_t signature);
+ /** New index id. */
+ uint32_t iid;
+ /** Functional index extractor routine reference. */
+ int32_t func_ref;
+};
+
+PrepareFunctionalIndex::~PrepareFunctionalIndex()
+{
+ if (func_ref != 0)
+ lua_func_delete(func_ref);
+}
+
+void
+PrepareFunctionalIndex::prepare(struct alter_space *alter)
+{
+ struct index *new_index = space_index(alter->new_space, iid);
+ assert(index_is_functional(new_index->def));
+ if (lua_func_new(new_index->def->opts.func_code, &func_ref) != 0)
+ diag_raise();
+}
+
+void
+PrepareFunctionalIndex::commit(struct alter_space *alter, int64_t signature)
+{
+ (void)signature;
+ struct index *index = space_index(alter->new_space, iid);
+ index->func_ref = func_ref;
+ lua_func_idx_trigger_set(space_name(alter->new_space),
+ index->def->name, &index->func_trigger_ref);
+ func_ref = 0;
+}
+
/* }}} */
/**
@@ -2009,12 +2056,15 @@ on_replace_dd_index(struct trigger * /* trigger */, void *event)
}
/* Case 2: create an index, if it is simply created. */
if (old_index == NULL && new_tuple != NULL) {
+ struct index_def *index_def =
+ index_def_new_from_tuple(new_tuple, old_space);
alter_space_move_indexes(alter, 0, iid);
CreateIndex *create_index = new CreateIndex(alter);
- create_index->new_index_def =
- index_def_new_from_tuple(new_tuple, old_space);
+ create_index->new_index_def = index_def;
index_def_update_optionality(create_index->new_index_def,
alter->new_min_field_count);
+ if (index_is_functional(index_def))
+ (void)new PrepareFunctionalIndex(alter, iid);
}
/* Case 3 and 4: check if we need to rebuild index data. */
if (old_index != NULL && new_tuple != NULL) {
@@ -2022,6 +2072,11 @@ on_replace_dd_index(struct trigger * /* trigger */, void *event)
index_def = index_def_new_from_tuple(new_tuple, old_space);
auto index_def_guard =
make_scoped_guard([=] { index_def_delete(index_def); });
+ if (index_is_functional(index_def)) {
+ tnt_raise(ClientError, ER_ALTER_SPACE,
+ space_name(old_space),
+ "can not alter functional index");
+ }
/*
* To detect which key parts are optional,
* min_field_count is required. But
diff --git a/src/box/index.cc b/src/box/index.cc
index cf81eca..47fc8c7 100644
--- a/src/box/index.cc
+++ b/src/box/index.cc
@@ -28,6 +28,7 @@
* THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
+#include "lua/call.h"
#include "index.h"
#include "tuple.h"
#include "say.h"
@@ -493,6 +494,8 @@ index_create(struct index *index, struct engine *engine,
index->engine = engine;
index->def = def;
index->space_cache_version = space_cache_version;
+ index->func_ref = 0;
+ index->func_trigger_ref = 0;
return 0;
}
@@ -505,7 +508,13 @@ index_delete(struct index *index)
* the index is primary or secondary.
*/
struct index_def *def = index->def;
+ int32_t func_ref = index->func_ref;
+ int32_t func_trigger_ref = index->func_trigger_ref;
index->vtab->destroy(index);
+ if (func_ref != 0)
+ lua_func_delete(func_ref);
+ if (func_trigger_ref != 0)
+ lua_func_delete(func_trigger_ref);
index_def_delete(def);
}
diff --git a/src/box/index.h b/src/box/index.h
index 0a1ac61..d5677de 100644
--- a/src/box/index.h
+++ b/src/box/index.h
@@ -454,6 +454,10 @@ struct index {
struct engine *engine;
/* Description of a possibly multipart key. */
struct index_def *def;
+ /* Functional index extractor routine reference. */
+ int32_t func_ref;
+ /* Functional index trap trigger routine reference. */
+ int32_t func_trigger_ref;
/* Space cache version at the time of construction. */
uint32_t space_cache_version;
};
diff --git a/src/box/index_def.c b/src/box/index_def.c
index 45c74d9..2b8bff8 100644
--- a/src/box/index_def.c
+++ b/src/box/index_def.c
@@ -48,8 +48,24 @@ const struct index_opts index_opts_default = {
/* .lsn = */ 0,
/* .sql = */ NULL,
/* .stat = */ NULL,
+ /* .func_code = */ NULL,
+ /* .pkey_offset = */ 0,
};
+static int
+pkey_offset_decode(const char **str, uint32_t len, char *opt, uint32_t errcode,
+ uint32_t field_no)
+{
+ (void)errcode;
+ (void)field_no;
+ (void)str;
+ (void)opt;
+ *(uint32_t *)opt = len;
+ for (uint32_t i = 0; i < len; i++)
+ mp_next(str);
+ return 0;
+}
+
const struct opt_def index_opts_reg[] = {
OPT_DEF("unique", OPT_BOOL, struct index_opts, is_unique),
OPT_DEF("dimension", OPT_INT64, struct index_opts, dimension),
@@ -62,6 +78,9 @@ const struct opt_def index_opts_reg[] = {
OPT_DEF("bloom_fpr", OPT_FLOAT, struct index_opts, bloom_fpr),
OPT_DEF("lsn", OPT_INT64, struct index_opts, lsn),
OPT_DEF("sql", OPT_STRPTR, struct index_opts, sql),
+ OPT_DEF("func_code", OPT_STRPTR, struct index_opts, func_code),
+ OPT_DEF_ARRAY("func_format", struct index_opts, pkey_offset,
+ pkey_offset_decode),
OPT_END,
};
@@ -114,13 +133,23 @@ index_def_new(uint32_t space_id, uint32_t iid, const char *name,
if (def->opts.sql == NULL) {
diag_set(OutOfMemory, strlen(opts->sql) + 1, "strdup",
"def->opts.sql");
- index_def_delete(def);
- return NULL;
+ goto error;
+ }
+ }
+ if (opts->func_code != NULL) {
+ def->opts.func_code = strdup(opts->func_code);
+ if (def->opts.func_code == NULL) {
+ diag_set(OutOfMemory, strlen(opts->func_code) + 1,
+ "strdup", "def->opts.func_code");
+ goto error;
}
}
/* Statistics are initialized separately. */
assert(opts->stat == NULL);
return def;
+error:
+ index_def_delete(def);
+ return NULL;
}
struct index_def *
@@ -153,18 +182,26 @@ index_def_dup(const struct index_def *def)
if (dup->opts.sql == NULL) {
diag_set(OutOfMemory, strlen(def->opts.sql) + 1,
"strdup", "dup->opts.sql");
- index_def_delete(dup);
- return NULL;
+ goto error;
+ }
+ }
+ if (def->opts.func_code != NULL) {
+ dup->opts.func_code = strdup(def->opts.func_code);
+ if (def->opts.func_code == NULL) {
+ diag_set(OutOfMemory, strlen(def->opts.func_code) + 1,
+ "strdup", "def->opts.func_code");
+ goto error;
}
}
if (def->opts.stat != NULL) {
dup->opts.stat = index_stat_dup(def->opts.stat);
- if (dup->opts.stat == NULL) {
- index_def_delete(dup);
- return NULL;
- }
+ if (dup->opts.stat == NULL)
+ goto error;
}
return dup;
+error:
+ index_def_delete(dup);
+ return NULL;
}
size_t
@@ -258,6 +295,14 @@ index_def_cmp(const struct index_def *key1, const struct index_def *key2)
if (index_opts_cmp(&key1->opts, &key2->opts))
return index_opts_cmp(&key1->opts, &key2->opts);
+ int rc = 0;
+ if ((rc = !!index_is_functional(key1) -
+ !!index_is_functional(key2)) != 0)
+ return rc;
+ else if (index_is_functional(key1) &&
+ (rc = strcmp(key1->opts.func_code, key2->opts.func_code)) != 0)
+ return rc;
+
return key_part_cmp(key1->key_def->parts, key1->key_def->part_count,
key2->key_def->parts, key2->key_def->part_count);
}
@@ -276,7 +321,8 @@ index_def_is_valid(struct index_def *index_def, const char *space_name)
space_name, "primary key must be unique");
return false;
}
- if (index_def->key_def->part_count == 0) {
+ if (index_def->key_def->part_count == 0 &&
+ !index_is_functional(index_def)) {
diag_set(ClientError, ER_MODIFY_INDEX, index_def->name,
space_name, "part count must be positive");
return false;
diff --git a/src/box/index_def.h b/src/box/index_def.h
index 273b8cb..358b321 100644
--- a/src/box/index_def.h
+++ b/src/box/index_def.h
@@ -167,6 +167,13 @@ struct index_opts {
* filled after running ANALYZE command.
*/
struct index_stat *stat;
+ /** Extractor function LUA code string. */
+ char *func_code;
+ /**
+ * Offset of primary key in functional index ispace
+ * tuples.
+ */
+ uint32_t pkey_offset;
};
extern const struct index_opts index_opts_default;
@@ -187,6 +194,7 @@ index_opts_create(struct index_opts *opts)
static inline void
index_opts_destroy(struct index_opts *opts)
{
+ free(opts->func_code);
free(opts->sql);
free(opts->stat);
TRASH(opts);
@@ -329,6 +337,12 @@ index_def_list_add(struct rlist *index_def_list, struct index_def *index_def)
rlist_add_tail_entry(index_def_list, index_def, link);
}
+static inline bool
+index_is_functional(const struct index_def *index_def)
+{
+ return index_def->opts.func_code != NULL;
+}
+
/**
* Create a new index definition definition.
*
diff --git a/src/box/lua/call.c b/src/box/lua/call.c
index 1f20426..cbd8870 100644
--- a/src/box/lua/call.c
+++ b/src/box/lua/call.c
@@ -467,6 +467,49 @@ lbox_module_reload(lua_State *L)
return 0;
}
+
+void
+lua_func_idx_trigger_set(const char *space_name, const char *idx_name,
+ int32_t *trigger_ref)
+{
+ lua_State *L = lua_newthread(tarantool_L);
+ int coro_ref = luaL_ref(tarantool_L, LUA_REGISTRYINDEX);
+ lua_getglobal(L, "func_idx_trigger_set");
+ lua_pushstring(L, space_name);
+ lua_pushstring(L, idx_name);
+ lua_call(L, 2, 1);
+ *trigger_ref = luaL_ref(L, LUA_REGISTRYINDEX);
+ luaL_unref(tarantool_L, LUA_REGISTRYINDEX, coro_ref);
+}
+
+int
+lua_func_new(const char *func_code, int32_t *func_ref)
+{
+ lua_State *L = lua_newthread(tarantool_L);
+ int coro_ref = luaL_ref(tarantool_L, LUA_REGISTRYINDEX);
+ int rc = 0;
+ if ((rc = luaL_loadstring(L, func_code)) != 0) {
+ const char *err_descr =
+ rc == LUA_ERRSYNTAX ?
+ "syntax error during pre-compilation" :
+ rc == LUA_ERRMEM ? "memory allocation error" :
+ "unknow";
+ diag_set(ClientError, ER_CREATE_FUNCTION, "user-defined",
+ err_descr);
+ return -1;
+ }
+ lua_call(L, 0, 1);
+ *func_ref = luaL_ref(L, LUA_REGISTRYINDEX);
+ luaL_unref(tarantool_L, LUA_REGISTRYINDEX, coro_ref);
+ return 0;
+}
+
+void
+lua_func_delete(uint32_t func_ref)
+{
+ luaL_unref(tarantool_L, LUA_REGISTRYINDEX, func_ref);
+}
+
static const struct luaL_Reg boxlib_internal[] = {
{"call_loadproc", lbox_call_loadproc},
{"sql_create_function", lbox_sql_create_function},
diff --git a/src/box/lua/call.h b/src/box/lua/call.h
index 0542123..88bfb8b 100644
--- a/src/box/lua/call.h
+++ b/src/box/lua/call.h
@@ -35,6 +35,8 @@
extern "C" {
#endif /* defined(__cplusplus) */
+#include <inttypes.h>
+
struct lua_State;
void
@@ -53,6 +55,22 @@ box_lua_call(struct call_request *request, struct port *port);
int
box_lua_eval(struct call_request *request, struct port *port);
+/** Set functional index trap trigger on space. */
+void
+lua_func_idx_trigger_set(const char *space_name, const char *idx_name,
+ int32_t *trigger_ref);
+
+/**
+ * Make and register a new LUA function object by func_code
+ * string.
+ */
+int
+lua_func_new(const char *func_code, int32_t *func_ref);
+
+/** Release registered LUA function object. */
+void
+lua_func_delete(uint32_t func_ref);
+
#if defined(__cplusplus)
} /* extern "C" */
#endif /* defined(__cplusplus) */
diff --git a/src/box/lua/func_idx.lua b/src/box/lua/func_idx.lua
new file mode 100644
index 0000000..4cf8e4f
--- /dev/null
+++ b/src/box/lua/func_idx.lua
@@ -0,0 +1,319 @@
+-- func_idx.lua (internal file)
+--
+
+local function func_idx_space_name(space_name, idx_name)
+ return '_i_' .. idx_name .. '_' .. space_name
+end
+
+local function test_a_in_b(a, b)
+ local test = {}
+ for _, v in pairs(b) do
+ test[v] = true
+ end
+ for _, v in pairs(a) do
+ if test[v] == nil then return false end
+ end
+ return true
+end
+
+local function func_idx_keys_equal(a, b)
+ return test_a_in_b(a, b) and test_a_in_b(b, a)
+end
+
+local function func_idx_key_in_set(key, set)
+ for _, r in pairs(set) do
+ if func_idx_keys_equal(key, r) then return true end
+ end
+ return false
+end
+
+local function func_idx_select_do(index, key, iterator, func, arg)
+ local space = box.space[index.space_id]
+ local ispace_name = func_idx_space_name(space.name, index.name)
+ local ispace = box.space[ispace_name]
+ local pkey_offset = index.functional.pkey_offset
+ local unique_filter = {}
+ for _, k in ispace:pairs(key, {iterator = iterator}) do
+ local pkey = {}
+ for _, p in pairs(space.index[0].parts) do
+ table.insert(pkey, k[#pkey + pkey_offset + 1])
+ end
+ if func_idx_key_in_set(pkey, unique_filter) == false then
+ table.insert(unique_filter, pkey)
+ if func(arg, space, pkey) ~= 0 then break end
+ end
+ end
+end
+
+local function func_idx_select(index, key, opts)
+ box.internal.check_index_arg(index, 'select')
+ key = key or {}
+ local iterator, offset, limit =
+ box.internal.check_select_opts(opts, #key < index.functional.pkey_offset)
+ local res = {}
+ local curr_offset, done = 0, 0
+ func_idx_select_do(index, key, iterator,
+ function(container, space, pkey)
+ if done >= limit then return 1 end
+ if curr_offset >= offset then
+ local tuple = space:get(pkey)
+ table.insert(container, tuple)
+ done = done + 1
+ end
+ curr_offset = curr_offset + 1
+ return 0
+ end, res)
+ return res
+end
+
+local function func_idx_get(index, key)
+ box.internal.check_index_arg(index, 'get')
+ if not index.unique then
+ box.error(box.error.MORE_THAN_ONE_TUPLE, "")
+ end
+ ret = func_idx_select(index, key, {limit = 1})[1]
+ if ret ~= nil then
+ return ret
+ end
+end
+
+local function func_idx_bsize(index)
+ box.internal.check_index_arg(index, 'bsize')
+ local space = box.space[index.space_id]
+ local ispace = box.space[func_idx_space_name(space.name, index.name)]
+ local ispace_pk = ispace.index[0]
+ return ispace_pk:bsize()
+end
+
+local function func_idx_random(index, seed)
+ box.internal.check_index_arg(index, 'random')
+ local space = box.space[index.space_id]
+ return space.index[0]:random(seed)
+end
+
+local function func_idx_count(index, key, opts)
+ box.internal.check_index_arg(index, 'count')
+ key = key or {}
+ local iterator, offset, limit =
+ box.internal.check_select_opts(opts, #key < index.functional.pkey_offset)
+ local res = {count = 0}
+ func_idx_select_do(index, key, iterator,
+ function(container, space, pkey)
+ container.count = container.count + 1
+ return 0
+ end, res)
+ return res.count
+end
+
+local function func_idx_rename(index, name)
+ box.internal.check_index_arg(index, 'rename')
+ local space = box.space[index.space_id]
+ local ispace = box.space[func_idx_space_name(space.name, index.name)]
+ local new_ispace_name = func_idx_space_name(space.name, name)
+ ispace:rename(new_ispace_name)
+ return box.schema.index.rename(space.id, index.id, name)
+end
+
+local function func_idx_min(index, key)
+ box.internal.check_index_arg(index, 'min')
+ return func_idx_select(index, key, {limit = 1, iterator = 'EQ'})
+end
+
+local function func_idx_max(index, key)
+ box.internal.check_index_arg(index, 'max')
+ return func_idx_select(index, key, {limit = 1, iterator = 'REQ'})
+end
+
+local function func_idx_drop(index, key)
+ box.internal.check_index_arg(index, 'drop')
+ local space = box.space[index.space_id]
+ space:on_replace(nil, index.functional.trigger)
+ local ispace = box.space[func_idx_space_name(space.name, index.name)]
+ ispace:drop()
+ return box.schema.index.drop(space.id, index.id)
+end
+
+local function func_idx_delete(index, key)
+ box.internal.check_index_arg(index, 'delete')
+ if not index.unique then
+ box.error(box.error.MORE_THAN_ONE_TUPLE, "")
+ end
+ key = key or {}
+ local iterator, offset, limit = box.internal.check_select_opts({}, false)
+ local ret = {}
+ func_idx_select_do(index, key, iterator,
+ function(container, space, pkey)
+ local tuple = space:delete(pkey)
+ if tuple ~= nil then
+ table.insert(container, tuple)
+ end
+ return 1
+ end, ret)
+ if ret[1] ~= nil then
+ return ret[1]
+ end
+end
+
+local function func_idx_update(index, key, ops)
+ box.internal.check_index_arg(index, 'update')
+ if not index.unique then
+ box.error(box.error.MORE_THAN_ONE_TUPLE, "")
+ end
+ local iterator, offset, limit = box.internal.check_select_opts({}, false)
+ local ret = {}
+ func_idx_select_do(index, key, iterator,
+ function(container, space, pkey)
+ local tuple = space:update(pkey, ops)
+ if tuple ~= nil then
+ table.insert(container, tuple)
+ end
+ return 1
+ end, ret)
+ if ret[1] ~= nil then
+ return ret[1]
+ end
+end
+
+local function func_idx_pairs(index, key, opts)
+ box.internal.check_index_arg(index, 'pairs')
+ local space = box.space[index.space_id]
+ local ispace_name = func_idx_space_name(space.name, index.name)
+ local ispace = box.space[ispace_name]
+ key = key or {}
+ local iterator, offset, limit =
+ box.internal.check_select_opts(opts, #key < index.functional.pkey_offset)
+ local obj, param, state = ispace:pairs(key, {iterator = iterator})
+ local gen = obj.gen
+ obj.gen = function(param, state)
+ local state, tuple = gen(param, state)
+ if state == nil then
+ return nil
+ else
+ local pkey_offset = index.functional.pkey_offset
+ local pkey = {}
+ for _, p in pairs(space.index[0].parts) do
+ table.insert(pkey, tuple[#pkey + pkey_offset + 1])
+ end
+ return state, space:get(pkey)
+ end
+ end
+ return obj, param, state
+end
+
+local function func_idx_unique_filter(keys)
+ local ret = {}
+ for _, k in pairs(keys) do
+ if func_idx_key_in_set(k, ret) then goto continue end
+ table.insert(ret, k)
+ ::continue::
+ end
+ return ret
+end
+
+local func_idx = {}
+
+func_idx.space_trigger_set = function (space_name, idx_name)
+ local space = box.space[space_name]
+ assert(space ~= nil)
+ local ispace = box.space[func_idx_space_name(space_name, idx_name)]
+ assert(ispace ~= nil)
+ local trigger = function (old, new)
+ if box.info.status ~= 'running' then
+ return
+ end
+ local findex = space.index[idx_name]
+ local func = findex.functional.func
+ local pkey = {}
+ local tuple = old or new
+ for _, p in pairs(space.index[0].parts) do
+ table.insert(pkey, tuple[p.fieldno])
+ end
+ if old ~= nil then
+ local fkeys = func_idx_unique_filter(func(old))
+ for _, key in pairs(fkeys) do
+ if findex.unique == false then
+ for _, v in pairs(pkey) do
+ table.insert(key, v)
+ end
+ end
+ ispace:delete(key)
+ end
+ end
+ if new ~= nil then
+ local fkeys = func_idx_unique_filter(func(new))
+ for _, key in pairs(fkeys) do
+ for _, v in pairs(pkey) do
+ table.insert(key, v)
+ end
+ ispace:insert(key)
+ end
+ end
+ end
+ space:on_replace(trigger)
+ return trigger
+end
+
+func_idx.ispace_create = function (space, name, func_code, func_format,
+ is_unique)
+ local func, err = loadstring(func_code)
+ if func == nil then
+ box.error(box.error.ILLEGAL_PARAMS,
+ "functional index extractor routine is invalid: "..err)
+ end
+ func = func()
+ local iformat = {}
+ local iparts = {}
+ for _, f in pairs(func_format) do
+ table.insert(iformat, {name = 'i' .. tostring(#iformat), type = f.type})
+ table.insert(iparts, {#iparts + 1, f.type,
+ is_nullable = f.is_nullable,
+ collation = f.collation})
+ end
+ local pkey_offset = #func_format
+ local pkey = box.internal.check_primary_index(space)
+ for _, p in pairs(pkey.parts) do
+ table.insert(iformat, {name = 'i' .. tostring(#iformat), type = p.type})
+ if is_unique == false then
+ table.insert(iparts, {#iparts + 1, p.type})
+ end
+ end
+ local ispace = box.schema.space.create('_i_' .. name .. '_' .. space.name,
+ {engine = space.engine})
+ ispace:format(iformat)
+ ispace:create_index('pk', {parts = iparts})
+
+ for _, tuple in space:pairs() do
+ local pkey = {}
+ for _, p in pairs(space.index[0].parts) do
+ table.insert(pkey, tuple[p.fieldno])
+ end
+ local fkeys = func_idx_unique_filter(func(tuple))
+ for _, key in pairs(fkeys) do
+ for _, v in pairs(pkey) do
+ table.insert(key, v)
+ end
+ ispace:insert(key)
+ end
+ end
+end
+
+func_idx.monkeypatch = function (index)
+ local meta = getmetatable(index)
+ meta.select = func_idx_select
+ meta.get = func_idx_get
+ meta.bsize = func_idx_bsize
+ meta.random = func_idx_random
+ meta.count = func_idx_count
+ meta.rename = func_idx_rename
+ meta.min = func_idx_min
+ meta.max = func_idx_max
+ meta.drop = func_idx_drop
+ meta.delete = func_idx_delete
+ meta.update = func_idx_update
+ meta.pairs = func_idx_pairs
+ meta.alter = nil
+ meta.compact = nil
+ meta.stat = nil
+end
+
+return func_idx
diff --git a/src/box/lua/schema.lua b/src/box/lua/schema.lua
index b6693b1..7d10d01 100644
--- a/src/box/lua/schema.lua
+++ b/src/box/lua/schema.lua
@@ -7,6 +7,7 @@ local fun = require('fun')
local log = require('log')
local fio = require('fio')
local json = require('json')
+local func_idx = require('func_idx')
local session = box.session
local internal = require('box.internal')
local function setmap(table)
@@ -281,6 +282,10 @@ local function check_param_table(table, template)
end
end
+function func_idx_trigger_set(space_name, idx_name)
+ return func_idx.space_trigger_set(space_name, idx_name)
+end
+
--[[
@brief Common function to check type parameter (of function)
Calls box.error(box.error.ILLEGAL_PARAMS, ) on error
@@ -700,6 +705,8 @@ local index_options = {
range_size = 'number',
page_size = 'number',
bloom_fpr = 'number',
+ func_code = 'string',
+ func_format = 'table'
}
--
@@ -745,16 +752,26 @@ box.schema.index.create = function(space_id, name, options)
}
options_defaults = type_dependent_defaults[options.type]
or type_dependent_defaults.other
- if not options.parts then
- local fieldno = options_defaults.parts[1]
- if #format >= fieldno then
- local t = format[fieldno].type
- if t ~= 'any' then
- options.parts = {{fieldno, format[fieldno].type}}
+ if not options.func_code then
+ if not options.parts then
+ local fieldno = options_defaults.parts[1]
+ if #format >= fieldno then
+ local t = format[fieldno].type
+ if t ~= 'any' then
+ options.parts = {{fieldno, format[fieldno].type}}
+ end
end
end
+ options = update_param_table(options, options_defaults)
+ elseif options.parts ~= nil then
+ box.error(box.error.ILLEGAL_PARAMS,
+ "options.parts: parts can't be set for functional index")
+ end
+ if (options.func_code == nil) ~= (options.func_format == nil) then
+ box.error(box.error.ILLEGAL_PARAMS,
+ "options.func_: bouth of parameters func_code and "..
+ "func_format should be specified for functional index")
end
- options = update_param_table(options, options_defaults)
if space.engine == 'vinyl' then
options_defaults = {
page_size = box.cfg.vinyl_page_size,
@@ -792,8 +809,11 @@ box.schema.index.create = function(space_id, name, options)
end
end
end
- local parts, parts_can_be_simplified =
- update_index_parts(format, options.parts)
+ local parts, parts_can_be_simplified = {}, false
+ if not options.func_code then
+ parts, parts_can_be_simplified =
+ update_index_parts(format, options.parts)
+ end
-- create_index() options contains type, parts, etc,
-- stored separately. Remove these members from index_opts
local index_opts = {
@@ -805,6 +825,8 @@ box.schema.index.create = function(space_id, name, options)
run_count_per_level = options.run_count_per_level,
run_size_ratio = options.run_size_ratio,
bloom_fpr = options.bloom_fpr,
+ func_code = options.func_code,
+ func_format = options.func_format,
}
local field_type_aliases = {
num = 'unsigned'; -- Deprecated since 1.7.2
@@ -849,6 +871,10 @@ box.schema.index.create = function(space_id, name, options)
if parts_can_be_simplified then
parts = simplify_index_parts(parts)
end
+ if options.func_code ~= nil then
+ func_idx.ispace_create(space, name, options.func_code,
+ options.func_format, options.unique)
+ end
_index:insert{space_id, iid, name, options.type, index_opts, parts}
if sequence ~= nil then
_space_sequence:insert{space_id, sequence, sequence_is_generated}
@@ -1319,6 +1345,7 @@ local function check_select_opts(opts, key_is_nil)
end
return iterator, offset, limit
end
+box.internal.check_select_opts = check_select_opts
base_index_mt.select_ffi = function(index, key, opts)
check_index_arg(index, 'select')
@@ -1546,6 +1573,7 @@ local function wrap_schema_object_mt(name)
return mt
end
+
function box.schema.space.bless(space)
local index_mt_name
if space.engine == 'vinyl' then
@@ -1560,6 +1588,9 @@ function box.schema.space.bless(space)
for j, index in pairs(space.index) do
if type(j) == 'number' then
setmetatable(index, wrap_schema_object_mt(index_mt_name))
+ if index.functional ~= nil then
+ func_idx.monkeypatch(index)
+ end
end
end
end
diff --git a/src/box/lua/space.cc b/src/box/lua/space.cc
index 7cae436..c80dde8 100644
--- a/src/box/lua/space.cc
+++ b/src/box/lua/space.cc
@@ -312,6 +312,27 @@ lbox_fillspace(struct lua_State *L, struct space *space, int i)
lua_settable(L, -3); /* space.index[k].parts */
+ if (index_is_functional(index_def)) {
+ lua_pushstring(L, "functional");
+ lua_newtable(L);
+
+ lua_pushstring(L, index_def->opts.func_code);
+ lua_setfield(L, -2, "func_code");
+
+ lua_pushinteger(L, index_def->opts.pkey_offset);
+ lua_setfield(L, -2, "pkey_offset");
+
+ lua_rawgeti(L, LUA_REGISTRYINDEX, index->func_ref);
+ assert(lua_isfunction(L, -1));
+ lua_setfield(L, -2, "func");
+
+ lua_rawgeti(L, LUA_REGISTRYINDEX, index->func_trigger_ref);
+ assert(lua_isfunction(L, -1));
+ lua_setfield(L, -2, "trigger");
+
+ lua_settable(L, -3);
+ }
+
lua_pushstring(L, "sequence_id");
if (k == 0 && space->sequence != NULL) {
lua_pushnumber(L, space->sequence->def->id);
diff --git a/src/box/memtx_engine.c b/src/box/memtx_engine.c
index 6a91d0b..6369c02 100644
--- a/src/box/memtx_engine.c
+++ b/src/box/memtx_engine.c
@@ -1293,6 +1293,12 @@ memtx_index_def_change_requires_rebuild(struct index *index,
if (!old_def->opts.is_unique && new_def->opts.is_unique)
return true;
+ if (index_is_functional(old_def) != index_is_functional(new_def))
+ return true;
+ else if (index_is_functional(old_def) &&
+ strcmp(old_def->opts.func_code, new_def->opts.func_code) != 0)
+ return true;
+
const struct key_def *old_cmp_def, *new_cmp_def;
if (index_depends_on_pk(index)) {
old_cmp_def = old_def->cmp_def;
diff --git a/src/box/vinyl.c b/src/box/vinyl.c
index b09b6ad..7aa370d 100644
--- a/src/box/vinyl.c
+++ b/src/box/vinyl.c
@@ -955,6 +955,12 @@ vinyl_index_def_change_requires_rebuild(struct index *index,
if (!old_def->opts.is_unique && new_def->opts.is_unique)
return true;
+ if (index_is_functional(old_def) != index_is_functional(new_def))
+ return true;
+ else if (index_is_functional(old_def) &&
+ strcmp(old_def->opts.func_code, new_def->opts.func_code) != 0)
+ return true;
+
assert(index_depends_on_pk(index));
const struct key_def *old_cmp_def = old_def->cmp_def;
const struct key_def *new_cmp_def = new_def->cmp_def;
diff --git a/src/lua/init.c b/src/lua/init.c
index 3728a57..3d58876 100644
--- a/src/lua/init.c
+++ b/src/lua/init.c
@@ -81,6 +81,7 @@ bool start_loop = true;
extern char strict_lua[],
uuid_lua[],
msgpackffi_lua[],
+ func_idx_lua[],
fun_lua[],
crypto_lua[],
digest_lua[],
@@ -120,6 +121,7 @@ extern char strict_lua[],
static const char *lua_modules[] = {
/* Make it first to affect load of all other modules */
"strict", strict_lua,
+ "func_idx", func_idx_lua,
"fun", fun_lua,
"tarantool", init_lua,
"errno", errno_lua,
diff --git a/test/engine/functional_idx.result b/test/engine/functional_idx.result
new file mode 100644
index 0000000..984b5e5
--- /dev/null
+++ b/test/engine/functional_idx.result
@@ -0,0 +1,120 @@
+msgpack = require('msgpack')
+---
+...
+env = require('test_run')
+---
+...
+test_run = env.new()
+---
+...
+engine = test_run:get_cfg('engine')
+---
+...
+format = {{'id','unsigned'}, {'name','string'},{'address', 'string'}}
+---
+...
+s = box.schema.space.create('clients', {engine = engine, format=format})
+---
+...
+addr_extractor = [[return function(tuple) if not type(tuple.address) == 'string' then return nil, 'Invalid field type' end local t = tuple.address:lower():split() for k,v in pairs(t) do t[k] = {v} end return t end]]
+---
+...
+addr_extractor_format = {{type='str', collation='unicode_ci'}}
+---
+...
+s:create_index('address', {func_code = addr_extractor, func_format = addr_extractor_format, unique=true})
+---
+- error: 'No index #0 is defined in space ''clients'''
+...
+pk = s:create_index('pk')
+---
+...
+s:insert({1, 'Vasya Pupkin', 'Russia Moscow Dolgoprudny Moscow Street 9Ð'})
+---
+- [1, 'Vasya Pupkin', 'Russia Moscow Dolgoprudny Moscow Street 9Ð']
+...
+s:create_index('address', {func_code = addr_extractor, unique=true})
+---
+- error: 'Illegal parameters, options.func_: bouth of parameters func_code and func_format
+ should be specified for functional index'
+...
+s:create_index('address', {func_format = addr_extractor_format, unique=true})
+---
+- error: 'Illegal parameters, options.func_: bouth of parameters func_code and func_format
+ should be specified for functional index'
+...
+s:create_index('address', {func_code = addr_extractor..'err', func_format = addr_extractor_format, unique=true})
+---
+- error: 'Illegal parameters, functional index extractor routine is invalid: [string
+ "return function(tuple) if not type(tuple.addr..."]:1: ''end'' expected near ''enderr'''
+...
+addr_low = s:create_index('address', {func_code = addr_extractor, func_format = addr_extractor_format, unique=true})
+---
+...
+assert(box.space.clients:on_replace()[1] == addr_low.functional.trigger)
+---
+- true
+...
+s:insert({2, 'James Bond', 'GB London Mi6'})
+---
+- [2, 'James Bond', 'GB London Mi6']
+...
+s:insert({3, 'Kirill Alekseevich', 'Russia Moscow Dolgoprudny RocketBuilders 1'})
+---
+- error: Duplicate key exists in unique index 'pk' in space '_i_address_clients'
+...
+s:insert({4, 'Jack London', 'GB'})
+---
+- error: Duplicate key exists in unique index 'pk' in space '_i_address_clients'
+...
+addr_low:count()
+---
+- 2
+...
+addr_low:count({'moscow'})
+---
+- 1
+...
+addr_low:count({'gb'})
+---
+- 1
+...
+addr_extractor = [[return function(tuple) if not type(tuple.address) == 'string' then return nil, 'Invalid field type' end local t = tuple.address:upper():split() for k,v in pairs(t) do t[k] = {v} end return t end]]
+---
+...
+addr_high = s:create_index('address2', {func_code = addr_extractor, func_format = addr_extractor_format, unique=true})
+---
+...
+addr_high:select()
+---
+- - [1, 'Vasya Pupkin', 'Russia Moscow Dolgoprudny Moscow Street 9Ð']
+ - [2, 'James Bond', 'GB London Mi6']
+...
+addr_high:select({}, {limit = 1})
+---
+- - [1, 'Vasya Pupkin', 'Russia Moscow Dolgoprudny Moscow Street 9Ð']
+...
+addr_high:select({}, {limit = 1, offset=1})
+---
+- - [2, 'James Bond', 'GB London Mi6']
+...
+addr_high:select({'Moscow'})
+---
+- - [1, 'Vasya Pupkin', 'Russia Moscow Dolgoprudny Moscow Street 9Ð']
+...
+for _, v in addr_high:pairs() do print(v.name) end
+---
+...
+_ = addr_low:delete({'moscow'})
+---
+...
+addr_high:select({'MOSCOW'})
+---
+- []
+...
+addr_high:drop()
+---
+...
+s:drop()
+---
+...
diff --git a/test/engine/functional_idx.test.lua b/test/engine/functional_idx.test.lua
new file mode 100644
index 0000000..3773b45
--- /dev/null
+++ b/test/engine/functional_idx.test.lua
@@ -0,0 +1,35 @@
+msgpack = require('msgpack')
+env = require('test_run')
+test_run = env.new()
+engine = test_run:get_cfg('engine')
+
+format = {{'id','unsigned'}, {'name','string'},{'address', 'string'}}
+s = box.schema.space.create('clients', {engine = engine, format=format})
+addr_extractor = [[return function(tuple) if not type(tuple.address) == 'string' then return nil, 'Invalid field type' end local t = tuple.address:lower():split() for k,v in pairs(t) do t[k] = {v} end return t end]]
+addr_extractor_format = {{type='str', collation='unicode_ci'}}
+s:create_index('address', {func_code = addr_extractor, func_format = addr_extractor_format, unique=true})
+pk = s:create_index('pk')
+s:insert({1, 'Vasya Pupkin', 'Russia Moscow Dolgoprudny Moscow Street 9Ð'})
+s:create_index('address', {func_code = addr_extractor, unique=true})
+s:create_index('address', {func_format = addr_extractor_format, unique=true})
+s:create_index('address', {func_code = addr_extractor..'err', func_format = addr_extractor_format, unique=true})
+addr_low = s:create_index('address', {func_code = addr_extractor, func_format = addr_extractor_format, unique=true})
+assert(box.space.clients:on_replace()[1] == addr_low.functional.trigger)
+s:insert({2, 'James Bond', 'GB London Mi6'})
+s:insert({3, 'Kirill Alekseevich', 'Russia Moscow Dolgoprudny RocketBuilders 1'})
+s:insert({4, 'Jack London', 'GB'})
+addr_low:count()
+addr_low:count({'moscow'})
+addr_low:count({'gb'})
+addr_extractor = [[return function(tuple) if not type(tuple.address) == 'string' then return nil, 'Invalid field type' end local t = tuple.address:upper():split() for k,v in pairs(t) do t[k] = {v} end return t end]]
+addr_high = s:create_index('address2', {func_code = addr_extractor, func_format = addr_extractor_format, unique=true})
+addr_high:select()
+addr_high:select({}, {limit = 1})
+addr_high:select({}, {limit = 1, offset=1})
+addr_high:select({'Moscow'})
+for _, v in addr_high:pairs() do print(v.name) end
+_ = addr_low:delete({'moscow'})
+addr_high:select({'MOSCOW'})
+addr_high:drop()
+
+s:drop()
--
2.7.4
More information about the Tarantool-patches
mailing list