From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: From: Kirill Shcherbatov Subject: [PATCH v5 3/3] box: introduce func_index Date: Fri, 26 Jul 2019 12:43:39 +0300 Message-Id: <38c49e4a186f40139488ad743fb0c37e458d5a65.1564134114.git.kshcherbatov@tarantool.org> In-Reply-To: References: MIME-Version: 1.0 Content-Transfer-Encoding: 8bit To: tarantool-patches@freelists.org, vdavydov.dev@gmail.com, kostja@tarantool.org Cc: Kirill Shcherbatov List-ID: Closes #1260 @TarantoolBot document Title: introduce func indexes in memtx Now you can define a func_index using a registered persistent function. There are restrictions for function and key definition for a func_index: - the referenced function must be persistent, deterministic and must return a scalar type or an array. - you must define key parts which describe the function return value - the function must return data which types match the defined key parts - the function may return multiple keys; this would be a multikey func_index; each key entry is indexed separately; - for multikey func_indexes, the key definition should start with part 1 and cover all returned key parts - key parts can't use JSON paths. - the function used for the func_index can not access tuple fields by name, only by index. Functional index can't be primary. It is not possible to change the used function after a func_index is defined on it. The index must be dropped first. Each key returned by func_index function (even when it is a single scalar) must be returned as a table i.e. {1} and must match the key definition. To define a multikey func_index, create a function with opts = {is_multikey = true} and return a table of keys. Example: s = box.schema.space.create('withdata') s:format({{name = 'name', type = 'string'}, {name = 'address', type = 'string'}}) pk = s:create_index('name', {parts = {1, 'string'}}) lua_code = [[function(tuple) local address = string.split(tuple[2]) local ret = {} for _, v in pairs(address) do table.insert(ret, {utf8.upper(v)}) end return ret end]] box.schema.func.create('address', {body = lua_code, is_deterministic = true, is_sandboxed = true, opts = {is_multikey = true}}) idx = s:create_index('addr', {unique = false, func = 'address', parts = {{1, 'string', collation = 'unicode_ci'}}}) s:insert({"James", "SIS Building Lambeth London UK"}) s:insert({"Sherlock", "221B Baker St Marylebone London NW1 6XE UK"}) idx:select('Uk') --- - - ['James', 'SIS Building Lambeth London UK'] - ['Sherlock', '221B Baker St Marylebone London NW1 6XE UK'] ... --- src/box/alter.h | 1 + src/box/errcode.h | 3 + src/box/index.h | 9 + src/box/index_def.h | 23 + src/box/key_def.h | 15 +- src/box/key_list.h | 105 +++++ src/box/schema_def.h | 9 + src/box/index_def.c | 7 + src/box/key_def.c | 47 +- src/box/key_list.c | 177 +++++++ src/box/lua/key_def.c | 2 +- src/box/memtx_engine.c | 2 + src/box/memtx_space.c | 18 + src/box/memtx_tree.c | 331 ++++++++++++- src/box/sql.c | 2 +- src/box/sql/build.c | 2 +- src/box/sql/select.c | 2 +- src/box/sql/where.c | 2 +- src/box/tuple_format.c | 4 + src/box/vinyl.c | 9 +- test/unit/luaT_tuple_new.c | 2 +- test/unit/merger.test.c | 4 +- src/box/CMakeLists.txt | 1 + src/box/alter.cc | 91 +++- src/box/bootstrap.snap | Bin 5863 -> 5914 bytes src/box/index.cc | 28 ++ src/box/lua/schema.lua | 33 ++ src/box/lua/space.cc | 20 + src/box/lua/upgrade.lua | 18 + src/box/schema.cc | 12 +- src/box/tuple_compare.cc | 117 ++++- src/box/tuple_extract_key.cc | 29 +- src/box/tuple_hash.cc | 1 + test/app-tap/tarantoolctl.test.lua | 4 +- test/box-py/bootstrap.result | 5 + test/box/access.result | 3 + test/box/access.test.lua | 1 + test/box/access_misc.result | 132 +++--- test/box/access_sysview.result | 6 +- test/box/alter.result | 7 +- test/box/bitset.result | 24 + test/box/bitset.test.lua | 9 + test/box/hash.result | 24 + test/box/hash.test.lua | 9 + test/box/misc.result | 3 + test/box/rtree_misc.result | 24 + test/box/rtree_misc.test.lua | 9 + test/engine/engine.cfg | 5 +- test/engine/func_index.result | 734 +++++++++++++++++++++++++++++ test/engine/func_index.test.lua | 250 ++++++++++ test/vinyl/misc.result | 23 + test/vinyl/misc.test.lua | 9 + test/wal_off/alter.result | 2 +- 53 files changed, 2300 insertions(+), 109 deletions(-) create mode 100644 src/box/key_list.h create mode 100644 src/box/key_list.c create mode 100644 test/engine/func_index.result create mode 100644 test/engine/func_index.test.lua diff --git a/src/box/alter.h b/src/box/alter.h index c339ccea6..1bc837359 100644 --- a/src/box/alter.h +++ b/src/box/alter.h @@ -47,5 +47,6 @@ extern struct trigger on_replace_space_sequence; extern struct trigger on_replace_trigger; extern struct trigger on_replace_fk_constraint; extern struct trigger on_replace_ck_constraint; +extern struct trigger on_replace_func_index; #endif /* INCLUDES_TARANTOOL_BOX_ALTER_H */ diff --git a/src/box/errcode.h b/src/box/errcode.h index 4496f353e..2b2381db6 100644 --- a/src/box/errcode.h +++ b/src/box/errcode.h @@ -250,6 +250,9 @@ struct errcode_record { /*195 */_(ER_CREATE_CK_CONSTRAINT, "Failed to create check constraint '%s': %s") \ /*196 */_(ER_CK_CONSTRAINT_FAILED, "Check constraint failed '%s': %s") \ /*197 */_(ER_SQL_COLUMN_COUNT, "Unequal number of entries in row expression: left side has %u, but right side - %u") \ + /*198 */_(ER_FUNC_INDEX_FUNC, "Failed to build a key for functional index '%s' of the space '%s': %s") \ + /*199 */_(ER_FUNC_INDEX_FORMAT, "Key format doesn't match on defined in the functional index '%s' of the space '%s': %s") \ + /*200 */_(ER_FUNC_INDEX_PARTS, "Wrong functional index definition: %s") \ /* * !IMPORTANT! Please follow instructions at start of the file diff --git a/src/box/index.h b/src/box/index.h index 97d600c96..2b1d0104b 100644 --- a/src/box/index.h +++ b/src/box/index.h @@ -685,8 +685,17 @@ void generic_index_compact(struct index *); void generic_index_reset_stat(struct index *); void generic_index_begin_build(struct index *); int generic_index_reserve(struct index *, uint32_t); +struct iterator * +generic_index_create_iterator(struct index *base, enum iterator_type type, + const char *key, uint32_t part_count); int generic_index_build_next(struct index *, struct tuple *); void generic_index_end_build(struct index *); +int +disabled_index_build_next(struct index *index, struct tuple *tuple); +int +disabled_index_replace(struct index *index, struct tuple *old_tuple, + struct tuple *new_tuple, enum dup_replace_mode mode, + struct tuple **result); #if defined(__cplusplus) } /* extern "C" */ diff --git a/src/box/index_def.h b/src/box/index_def.h index 6dac28377..1d60c982b 100644 --- a/src/box/index_def.h +++ b/src/box/index_def.h @@ -163,6 +163,8 @@ struct index_opts { * filled after running ANALYZE command. */ struct index_stat *stat; + /** Identifier of the functional index function. */ + uint32_t func_id; }; extern const struct index_opts index_opts_default; @@ -207,6 +209,8 @@ index_opts_cmp(const struct index_opts *o1, const struct index_opts *o2) return o1->run_size_ratio < o2->run_size_ratio ? -1 : 1; if (o1->bloom_fpr != o2->bloom_fpr) return o1->bloom_fpr < o2->bloom_fpr ? -1 : 1; + if (o1->func_id != o2->func_id) + return o1->func_id - o2->func_id; return 0; } @@ -298,6 +302,25 @@ index_def_update_optionality(struct index_def *def, uint32_t min_field_count) key_def_update_optionality(def->cmp_def, min_field_count); } +/** + * Update func pointer for a functional index key definition. + * @param def Index def, containing key definitions to update. + * @param func The func_index function pointer. + */ +static inline void +index_def_set_func(struct index_def *def, struct func *func) +{ + assert(def->opts.func_id > 0 && + def->key_def->for_func_index && def->key_def->for_func_index); + /* + * Set func_index_func for functional index key + * definition. It is used in key_list module to extract + * a key for given tuple. + */ + def->key_def->func_index_func = func; + def->cmp_def->func_index_func = NULL; +} + /** * Add an index definition to a list, preserving the * first position of the primary key. diff --git a/src/box/key_def.h b/src/box/key_def.h index 73aefb9a7..3d3affea0 100644 --- a/src/box/key_def.h +++ b/src/box/key_def.h @@ -198,6 +198,8 @@ struct key_def { bool has_json_paths; /** True if it is multikey key definition. */ bool is_multikey; + /** True if it is functional index key definition. */ + bool for_func_index; /** * True, if some key parts can be absent in a tuple. These * fields assumed to be MP_NIL. @@ -205,6 +207,16 @@ struct key_def { bool has_optional_parts; /** Key fields mask. @sa column_mask.h for details. */ uint64_t column_mask; + /** + * A pointer to a functional index function. + * It is initialized externally when possible and key + * definiton object doesn't take a (semantics) reference + * on functional index function object. For example, it + * is not possible to define this pointer during recovery. + * Thus functional index key definition may have this + * field uninitialized (NULL). + */ + struct func *func_index_func; /** * In case of the multikey index, a pointer to the * JSON path string, the path to the root node of @@ -330,7 +342,8 @@ key_def_sizeof(uint32_t part_count, uint32_t path_pool_size) * and initialize its parts. */ struct key_def * -key_def_new(const struct key_part_def *parts, uint32_t part_count); +key_def_new(const struct key_part_def *parts, uint32_t part_count, + bool for_func_index); /** * Dump part definitions of the given key def. diff --git a/src/box/key_list.h b/src/box/key_list.h new file mode 100644 index 000000000..673d08421 --- /dev/null +++ b/src/box/key_list.h @@ -0,0 +1,105 @@ +#ifndef TARANTOOL_BOX_KEY_LIST_H_INCLUDED +#define TARANTOOL_BOX_KEY_LIST_H_INCLUDED +/* + * Copyright 2010-2019, Tarantool AUTHORS, please see AUTHORS file. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * 1. Redistributions of source code must retain the above + * copyright notice, this list of conditions and the + * following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF + * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +struct index_def; +struct tuple; + +/** + * Function to prepare a value returned by + * key_list_iterator_next method. + */ +typedef const char *(*key_list_allocator_t)(struct tuple *tuple, const char *key, + uint32_t key_sz); + +/** + * An iterator to iterate over the key_data returned by function + * and validate it with given key definition (when required). + */ +struct key_list_iterator { + /** The ancestor tuple. */ + struct tuple *tuple; + /** + * The sequential functional index key definition that + * describes a format of functional index function keys. + */ + struct index_def *index_def; + /** The pointer to currently processed key. */ + const char *data; + /** The pointer to the end of extracted key_data. */ + const char *data_end; + /** Whether iterator must validate processed keys. */ + bool validate; + /** The method to allocate a key to be returned. */ + key_list_allocator_t key_allocator; +}; + +/** + * Initialize a new functional index function returned + * keys iterator. + * Execute a function specified in a given functional index key + * definition (a functional index function) and initialize a new + * iterator on MsgPack array of with keys. Each key is a MsgPack + * array as well. + * + * When validate flag is specified, processed keys are validated + * to match given functional index key definition. + * + * Returns 0 in case of success, -1 otherwise. + * Uses fiber region to allocate memory. + */ +int +key_list_iterator_create(struct key_list_iterator *it, struct tuple *tuple, + struct index_def *index_def, bool validate, + key_list_allocator_t key_allocator); + +/** + * Perform key iterator step and update iterator state. + * Update key pointer with an actual key. + * + * Returns 0 on success. In case of error returns -1 and sets + * the corresponding diag message. + */ +int +key_list_iterator_next(struct key_list_iterator *it, const char **value); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* TARANTOOL_BOX_KEY_LIST_H_INCLUDED */ diff --git a/src/box/schema_def.h b/src/box/schema_def.h index a97b6d531..85f652d52 100644 --- a/src/box/schema_def.h +++ b/src/box/schema_def.h @@ -112,6 +112,8 @@ enum { BOX_FK_CONSTRAINT_ID = 356, /** Space id of _ck_contraint. */ BOX_CK_CONSTRAINT_ID = 364, + /** Space id of _func_index. */ + BOX_FUNC_INDEX_ID = 372, /** End of the reserved range of system spaces. */ BOX_SYSTEM_ID_MAX = 511, BOX_ID_NIL = 2147483647 @@ -267,6 +269,13 @@ enum { BOX_CK_CONSTRAINT_FIELD_CODE = 4, }; +/** _func_index fields. */ +enum { + BOX_FUNC_INDEX_FIELD_SPACE_ID = 0, + BOX_FUNC_INDEX_FIELD_INDEX_ID = 1, + BOX_FUNC_INDEX_FUNCTION_ID = 2, +}; + /* * Different objects which can be subject to access * control. diff --git a/src/box/index_def.c b/src/box/index_def.c index eb309a30c..85128b1a5 100644 --- a/src/box/index_def.c +++ b/src/box/index_def.c @@ -50,6 +50,7 @@ const struct index_opts index_opts_default = { /* .bloom_fpr = */ 0.05, /* .lsn = */ 0, /* .stat = */ NULL, + /* .func = */ 0, }; const struct opt_def index_opts_reg[] = { @@ -63,6 +64,7 @@ const struct opt_def index_opts_reg[] = { OPT_DEF("run_size_ratio", OPT_FLOAT, struct index_opts, run_size_ratio), OPT_DEF("bloom_fpr", OPT_FLOAT, struct index_opts, bloom_fpr), OPT_DEF("lsn", OPT_INT64, struct index_opts, lsn), + OPT_DEF("func", OPT_UINT32, struct index_opts, func_id), OPT_DEF_LEGACY("sql"), OPT_END, }; @@ -296,6 +298,11 @@ index_def_is_valid(struct index_def *index_def, const char *space_name) space_name, "primary key cannot be multikey"); return false; } + if (index_def->iid == 0 && index_def->key_def->for_func_index) { + diag_set(ClientError, ER_MODIFY_INDEX, index_def->name, + space_name, "primary key can not use a function"); + return false; + } for (uint32_t i = 0; i < index_def->key_def->part_count; i++) { assert(index_def->key_def->parts[i].type < field_type_MAX); if (index_def->key_def->parts[i].fieldno > BOX_INDEX_FIELD_MAX) { diff --git a/src/box/key_def.c b/src/box/key_def.c index a842ef1ec..3e3782163 100644 --- a/src/box/key_def.c +++ b/src/box/key_def.c @@ -253,7 +253,8 @@ key_def_set_part(struct key_def *def, uint32_t part_no, uint32_t fieldno, } struct key_def * -key_def_new(const struct key_part_def *parts, uint32_t part_count) +key_def_new(const struct key_part_def *parts, uint32_t part_count, + bool for_func_index) { size_t sz = 0; for (uint32_t i = 0; i < part_count; i++) @@ -267,7 +268,7 @@ key_def_new(const struct key_part_def *parts, uint32_t part_count) def->part_count = part_count; def->unique_part_count = part_count; - + def->for_func_index = for_func_index; /* A pointer to the JSON paths data in the new key_def. */ char *path_pool = (char *)def + key_def_sizeof(part_count, 0); for (uint32_t i = 0; i < part_count; i++) { @@ -278,8 +279,7 @@ key_def_new(const struct key_part_def *parts, uint32_t part_count) if (coll_id == NULL) { diag_set(ClientError, ER_WRONG_INDEX_OPTIONS, i + 1, "collation was not found by ID"); - key_def_delete(def); - return NULL; + goto error; } coll = coll_id->coll; } @@ -288,14 +288,28 @@ key_def_new(const struct key_part_def *parts, uint32_t part_count) part->nullable_action, coll, part->coll_id, part->sort_order, part->path, path_len, &path_pool, TUPLE_OFFSET_SLOT_NIL, - 0) != 0) { - key_def_delete(def); - return NULL; + 0) != 0) + goto error; + } + if (for_func_index) { + if (def->has_json_paths) { + diag_set(ClientError, ER_UNSUPPORTED, + "Functional index", "json paths"); + goto error; + } + if(!key_def_is_sequential(def) || parts->fieldno != 0) { + diag_set(ClientError, ER_FUNC_INDEX_PARTS, + "key part numbers must be sequential and " + "first part number must be 1"); + goto error; } } assert(path_pool == (char *)def + sz); key_def_set_func(def); return def; +error: + key_def_delete(def); + return NULL; } int @@ -704,6 +718,12 @@ key_def_find(const struct key_def *key_def, const struct key_part *to_find) bool key_def_contains(const struct key_def *first, const struct key_def *second) { + /* + * Func index definitions cannot be contained in + * each other. + */ + if (first->for_func_index || second->for_func_index) + return false; const struct key_part *part = second->parts; const struct key_part *end = part + second->part_count; for (; part != end; part++) { @@ -720,6 +740,14 @@ static bool key_def_can_merge(const struct key_def *key_def, const struct key_part *to_merge) { + if (key_def->for_func_index) { + /* + * Nothing can be omitted in functional index + * key definition, everything should be merged. + */ + return true; + } + const struct key_part *part = key_def_find(key_def, to_merge); if (part == NULL) return true; @@ -734,6 +762,7 @@ key_def_can_merge(const struct key_def *key_def, struct key_def * key_def_merge(const struct key_def *first, const struct key_def *second) { + assert(!second->for_func_index); uint32_t new_part_count = first->part_count + second->part_count; /* * Find and remove part duplicates, i.e. parts counted @@ -766,6 +795,8 @@ key_def_merge(const struct key_def *first, const struct key_def *second) new_def->has_optional_parts = first->has_optional_parts || second->has_optional_parts; new_def->is_multikey = first->is_multikey || second->is_multikey; + new_def->for_func_index = first->for_func_index; + new_def->func_index_func = first->func_index_func; /* JSON paths data in the new key_def. */ char *path_pool = (char *)new_def + key_def_sizeof(new_part_count, 0); @@ -838,7 +869,7 @@ key_def_find_pk_in_cmp_def(const struct key_def *cmp_def, } /* Finally, allocate the new key definition. */ - extracted_def = key_def_new(parts, pk_def->part_count); + extracted_def = key_def_new(parts, pk_def->part_count, false); out: region_truncate(region, region_svp); return extracted_def; diff --git a/src/box/key_list.c b/src/box/key_list.c new file mode 100644 index 000000000..b4650b4af --- /dev/null +++ b/src/box/key_list.c @@ -0,0 +1,177 @@ +/* + * Copyright 2010-2016, Tarantool AUTHORS, please see AUTHORS file. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * 1. Redistributions of source code must retain the above + * copyright notice, this list of conditions and the + * following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF + * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ +#include "key_list.h" + +#include "errcode.h" +#include "diag.h" +#include "index_def.h" +#include "func.h" +#include "func_def.h" +#include "fiber.h" +#include "key_def.h" +#include "port.h" +#include "schema.h" +#include "tt_static.h" +#include "tuple.h" +#include "tuple_compare.h" + +int +key_list_iterator_create(struct key_list_iterator *it, struct tuple *tuple, + struct index_def *index_def, bool validate, + key_list_allocator_t key_allocator) +{ + it->index_def = index_def; + it->validate = validate; + it->tuple = tuple; + it->key_allocator = key_allocator; + + struct region *region = &fiber()->gc; + size_t region_svp = region_used(region); + struct func *func = index_def->key_def->func_index_func; + + struct port out_port, in_port; + port_tuple_create(&in_port); + port_tuple_add(&in_port, tuple); + int rc = func_call(func, &in_port, &out_port); + port_destroy(&in_port); + if (rc != 0) { + /* Can't evaluate function. */ + diag_set(ClientError, ER_FUNC_INDEX_FUNC, index_def->name, + space_by_id(index_def->space_id)->def->name, + diag_last_error(diag_get())->errmsg); + return -1; + } + uint32_t key_data_sz; + const char *key_data = port_get_msgpack(&out_port, &key_data_sz); + port_destroy(&out_port); + if (key_data == NULL) { + /* Can't get a result returned by function . */ + diag_set(ClientError, ER_FUNC_INDEX_FUNC, index_def->name, + space_by_id(index_def->space_id)->def->name, + diag_last_error(diag_get())->errmsg); + return -1; + } + + it->data_end = key_data + key_data_sz; + assert(mp_typeof(*key_data) == MP_ARRAY); + if (mp_decode_array(&key_data) != 1) { + /* + * Function return doesn't follow the + * convention: to many values were returned. + * i.e. return 1, 2 + */ + region_truncate(region, region_svp); + diag_set(ClientError, ER_FUNC_INDEX_FORMAT, index_def->name, + space_by_id(index_def->space_id)->def->name, + "to many values were returned"); + return -1; + } + if (func->def->opts.is_multikey) { + if (mp_typeof(*key_data) != MP_ARRAY) { + /* + * Multikey function must return an array + * of keys. + */ + region_truncate(region, region_svp); + diag_set(ClientError, ER_FUNC_INDEX_FORMAT, + index_def->name, + space_by_id(index_def->space_id)->def->name, + "multikey function mustn't return scalar"); + return -1; + } + (void)mp_decode_array(&key_data); + } + it->data = key_data; + return 0; +} + +int +key_list_iterator_next(struct key_list_iterator *it, const char **value) +{ + assert(it->data <= it->data_end); + if (it->data == it->data_end) { + *value = NULL; + return 0; + } + const char *key = it->data; + if (!it->validate) { + /* + * Valid key is a MP_ARRAY, so just go to the + * next key via mp_next(). + */ + mp_next(&it->data); + assert(it->data <= it->data_end); + *value = it->key_allocator(it->tuple, key, it->data - key); + return *value != NULL ? 0 : -1; + } + + if (mp_typeof(*key) != MP_ARRAY) { + /* + * A value returned by func_index function is + * not a valid key, i.e. {1}. + */ + diag_set(ClientError, ER_FUNC_INDEX_FORMAT, it->index_def->name, + space_by_id(it->index_def->space_id)->def->name, + tt_sprintf("supplied key type is invalid: expected %s", + field_type_strs[MP_ARRAY])); + return -1; + } + struct key_def *key_def = it->index_def->key_def; + const char *rptr = key; + uint32_t part_count = mp_decode_array(&rptr); + if (part_count != key_def->part_count) { + /* + * The key must have exact functional index + * definition's part_count(s). + */ + diag_set(ClientError, ER_FUNC_INDEX_FORMAT, it->index_def->name, + space_by_id(it->index_def->space_id)->def->name, + tt_sprintf(tnt_errcode_desc(ER_EXACT_MATCH), + key_def->part_count, part_count)); + return -1; + } + const char *key_end; + if (key_validate_parts(key_def, rptr, part_count, true, + &key_end) != 0) { + /* + * The key doesn't follow functional index key + * definition. + */ + diag_set(ClientError, ER_FUNC_INDEX_FORMAT, it->index_def->name, + space_by_id(it->index_def->space_id)->def->name, + diag_last_error(diag_get())->errmsg); + return -1; + } + + it->data = key_end; + *value = it->key_allocator(it->tuple, key, key_end - key); + return *value != NULL ? 0 : -1; +} diff --git a/src/box/lua/key_def.c b/src/box/lua/key_def.c index 041b5ec98..3a3a5ec0c 100644 --- a/src/box/lua/key_def.c +++ b/src/box/lua/key_def.c @@ -445,7 +445,7 @@ lbox_key_def_new(struct lua_State *L) lua_pop(L, 1); } - struct key_def *key_def = key_def_new(parts, part_count); + struct key_def *key_def = key_def_new(parts, part_count, false); region_truncate(region, region_svp); if (key_def == NULL) return luaT_error(L); diff --git a/src/box/memtx_engine.c b/src/box/memtx_engine.c index 91c0a7b17..8bf90b601 100644 --- a/src/box/memtx_engine.c +++ b/src/box/memtx_engine.c @@ -1251,6 +1251,8 @@ memtx_index_def_change_requires_rebuild(struct index *index, return true; if (!old_def->opts.is_unique && new_def->opts.is_unique) return true; + if (old_def->opts.func_id != new_def->opts.func_id) + return true; const struct key_def *old_cmp_def, *new_cmp_def; if (index_depends_on_pk(index)) { diff --git a/src/box/memtx_space.c b/src/box/memtx_space.c index a287eedb1..cf29cf328 100644 --- a/src/box/memtx_space.c +++ b/src/box/memtx_space.c @@ -659,6 +659,12 @@ memtx_space_check_index_def(struct space *space, struct index_def *index_def) "HASH index cannot be multikey"); return -1; } + if (index_def->key_def->for_func_index) { + diag_set(ClientError, ER_MODIFY_INDEX, + index_def->name, space_name(space), + "HASH index can not use a function"); + return -1; + } break; case TREE: /* TREE index has no limitations. */ @@ -688,6 +694,12 @@ memtx_space_check_index_def(struct space *space, struct index_def *index_def) "RTREE index cannot be multikey"); return -1; } + if (index_def->key_def->for_func_index) { + diag_set(ClientError, ER_MODIFY_INDEX, + index_def->name, space_name(space), + "RTREE index can not use a function"); + return -1; + } /* no furter checks of parts needed */ return 0; case BITSET: @@ -716,6 +728,12 @@ memtx_space_check_index_def(struct space *space, struct index_def *index_def) "BITSET index cannot be multikey"); return -1; } + if (index_def->key_def->for_func_index) { + diag_set(ClientError, ER_MODIFY_INDEX, + index_def->name, space_name(space), + "BITSET index can not use a function"); + return -1; + } /* no furter checks of parts needed */ return 0; default: diff --git a/src/box/memtx_tree.c b/src/box/memtx_tree.c index a9c1871db..b49a086ad 100644 --- a/src/box/memtx_tree.c +++ b/src/box/memtx_tree.c @@ -35,6 +35,7 @@ #include "errinj.h" #include "memory.h" #include "fiber.h" +#include "key_list.h" #include "tuple.h" #include #include @@ -769,6 +770,203 @@ memtx_tree_index_replace_multikey(struct index *base, struct tuple *old_tuple, return 0; } +/** + * Dummy functional index hint allocator doesn't allocates memory, + * just returning a given key value. + */ +static const char * +func_index_key_dummy_alloc(struct tuple *tuple, const char *key, + uint32_t key_sz) +{ + (void)tuple; + (void)key_sz; + return (void *)key; +} + +/** + * The entry for multikey functional index replace operation + * is required to rollback an incomplete action, restore the + * original key_hint(s) hints both as to commit a completed + * replace action and destruct useless key_hint(s). +*/ +struct func_key_undo { + /** A link to organize entries in list. */ + struct rlist link; + /** An inserted record copy. */ + struct memtx_tree_data key; +}; + +/** Allocate a new func_key_undo on given region. */ +struct func_key_undo * +func_key_undo_new(struct region *region) +{ + struct func_key_undo *undo = + (struct func_key_undo *)region_alloc(region, sizeof(*undo)); + if (undo == NULL) { + diag_set(OutOfMemory, sizeof(*undo), "region", "undo"); + return NULL; + } + return undo; +} + +/** + * Rollback a sequence of memtx_tree_index_replace_multikey_one + * insertions for functional index. Routine uses given list to + * return a given index object in it's original state. + */ +static void +memtx_tree_func_index_replace_rollback(struct memtx_tree_index *index, + struct rlist *old_keys, + struct rlist *new_keys) +{ + struct func_key_undo *entry; + rlist_foreach_entry(entry, new_keys, link) { + memtx_tree_delete_value(&index->tree, entry->key, NULL); + tuple_chunk_delete(entry->key.tuple, + (const char *)entry->key.hint); + } + rlist_foreach_entry(entry, old_keys, link) + memtx_tree_insert(&index->tree, entry->key, NULL); +} + +/** + * @sa memtx_tree_index_replace_multikey(). + * Use functional index function from the key definition + * to build a key list. Then each returned key is reallocated in + * engine's memory as key_hint object and is used as comparison + * hint. + * To control key_hint(s) life cycle in case of functional index + * we use a tiny list object is allocated on region. + * It allows to restore original nodes with their original + * key_hint(s) pointers in case of failure and release + * useless hints of replaced items in case of success. + */ +static int +memtx_tree_func_index_replace(struct index *base, struct tuple *old_tuple, + struct tuple *new_tuple, enum dup_replace_mode mode, + struct tuple **result) +{ + struct memtx_tree_index *index = (struct memtx_tree_index *)base; + struct index_def *index_def = index->base.def; + assert(index_def->key_def->for_func_index); + + int rc = -1; + struct region *region = &fiber()->gc; + size_t region_svp = region_used(region); + + *result = NULL; + struct key_list_iterator it; + if (new_tuple != NULL) { + struct rlist old_keys, new_keys; + rlist_create(&old_keys); + rlist_create(&new_keys); + if (key_list_iterator_create(&it, new_tuple, index_def, true, + tuple_chunk_new) != 0) + goto end; + int err = 0; + const char *key; + struct func_key_undo *undo; + while ((err = key_list_iterator_next(&it, &key)) == 0 && + key != NULL) { + /* Perform insertion, log it in list. */ + undo = func_key_undo_new(region); + if (undo == NULL) { + tuple_chunk_delete(new_tuple, key); + err = -1; + break; + } + undo->key.tuple = new_tuple; + undo->key.hint = (hint_t)key; + rlist_add(&new_keys, &undo->link); + bool is_multikey_conflict; + struct memtx_tree_data old_data; + old_data.tuple = NULL; + err = memtx_tree_index_replace_multikey_one(index, + old_tuple, new_tuple, + mode, (hint_t)key, &old_data, + &is_multikey_conflict); + if (err != 0) + break; + if (old_data.tuple != NULL && !is_multikey_conflict) { + undo = func_key_undo_new(region); + if (undo == NULL) { + /* + * Can't append this + * operation in rollback + * journal. Rollback it + * manually. + */ + memtx_tree_insert(&index->tree, + old_data, NULL); + err = -1; + break; + } + undo->key = old_data; + rlist_add(&old_keys, &undo->link); + *result = old_data.tuple; + } else if (old_data.tuple != NULL && + is_multikey_conflict) { + /* + * Remove replaced undo from + * undo list + */ + tuple_chunk_delete(new_tuple, + (const char *)old_data.hint); + rlist_foreach_entry(undo, &new_keys, link) { + if (undo->key.hint == old_data.hint) { + rlist_del(&undo->link); + break; + } + } + } + } + if (key != NULL || err != 0) { + memtx_tree_func_index_replace_rollback(index, + &old_keys, &new_keys); + goto end; + } + if (*result != NULL) { + assert(old_tuple == NULL || old_tuple == *result); + old_tuple = *result; + } + /* + * Commit changes: release hints for + * replaced entries. + */ + rlist_foreach_entry(undo, &old_keys, link) { + tuple_chunk_delete(undo->key.tuple, + (const char *)undo->key.hint); + } + } + if (old_tuple != NULL) { + if (key_list_iterator_create(&it, old_tuple, index_def, false, + func_index_key_dummy_alloc) != 0) + goto end; + struct memtx_tree_data data, deleted_data; + data.tuple = old_tuple; + const char *key; + while (key_list_iterator_next(&it, &key) == 0 && key != NULL) { + data.hint = (hint_t) key; + deleted_data.tuple = NULL; + memtx_tree_delete_value(&index->tree, data, + &deleted_data); + if (deleted_data.tuple != NULL) { + /* + * Release related hint on + * successfull node deletion. + */ + tuple_chunk_delete(deleted_data.tuple, + (const char *)deleted_data.hint); + } + } + assert(key == NULL); + } + rc = 0; +end: + region_truncate(region, region_svp); + return rc; +} + static struct iterator * memtx_tree_index_create_iterator(struct index *base, enum iterator_type type, const char *key, uint32_t part_count) @@ -900,13 +1098,48 @@ memtx_tree_index_build_next_multikey(struct index *base, struct tuple *tuple) return 0; } +static int +memtx_tree_func_index_build_next(struct index *base, struct tuple *tuple) +{ + struct memtx_tree_index *index = (struct memtx_tree_index *)base; + struct index_def *index_def = index->base.def; + assert(index_def->key_def->for_func_index); + + struct region *region = &fiber()->gc; + size_t region_svp = region_used(region); + + struct key_list_iterator it; + if (key_list_iterator_create(&it, tuple, index_def, false, + tuple_chunk_new) != 0) + return -1; + + const char *key; + uint32_t insert_idx = index->build_array_size; + while (key_list_iterator_next(&it, &key) == 0 && key != NULL) { + if (memtx_tree_index_build_array_append(index, tuple, + (hint_t)key) != 0) + goto error; + } + assert(key == NULL); + region_truncate(region, region_svp); + return 0; +error: + for (uint32_t i = insert_idx; i < index->build_array_size; i++) { + tuple_chunk_delete(index->build_array[i].tuple, + (const char *)index->build_array[i].hint); + } + region_truncate(region, region_svp); + return -1; +} + /** * Process build_array of specified index and remove duplicates * of equal tuples (in terms of index's cmp_def and have same * tuple pointer). The build_array is expected to be sorted. */ static void -memtx_tree_index_build_array_deduplicate(struct memtx_tree_index *index) +memtx_tree_index_build_array_deduplicate(struct memtx_tree_index *index, + void (*destroy)(struct tuple *tuple, const char *hint)) { if (index->build_array_size == 0) return; @@ -923,10 +1156,19 @@ memtx_tree_index_build_array_deduplicate(struct memtx_tree_index *index) /* Do not override the element itself. */ if (++w_idx == r_idx) continue; - index->build_array[w_idx] = index->build_array[r_idx]; + SWAP(index->build_array[w_idx], + index->build_array[r_idx]); } r_idx++; } + if (destroy != NULL) { + /* Destroy deduplicated entries. */ + for (r_idx = w_idx + 1; + r_idx < index->build_array_size; r_idx++) { + destroy(index->build_array[r_idx].tuple, + (const char *)index->build_array[r_idx].hint); + } + } index->build_array_size = w_idx + 1; } @@ -945,7 +1187,10 @@ memtx_tree_index_end_build(struct index *base) * the following memtx_tree_build assumes that * all keys are unique. */ - memtx_tree_index_build_array_deduplicate(index); + memtx_tree_index_build_array_deduplicate(index, NULL); + } else if (cmp_def->for_func_index) { + memtx_tree_index_build_array_deduplicate(index, + tuple_chunk_delete); } memtx_tree_build(&index->tree, index->build_array, index->build_array_size); @@ -1072,6 +1317,72 @@ static const struct index_vtab memtx_tree_index_multikey_vtab = { /* .end_build = */ memtx_tree_index_end_build, }; +static const struct index_vtab memtx_tree_func_index_vtab = { + /* .destroy = */ memtx_tree_index_destroy, + /* .commit_create = */ generic_index_commit_create, + /* .abort_create = */ generic_index_abort_create, + /* .commit_modify = */ generic_index_commit_modify, + /* .commit_drop = */ generic_index_commit_drop, + /* .update_def = */ memtx_tree_index_update_def, + /* .depends_on_pk = */ memtx_tree_index_depends_on_pk, + /* .def_change_requires_rebuild = */ + memtx_index_def_change_requires_rebuild, + /* .size = */ memtx_tree_index_size, + /* .bsize = */ memtx_tree_index_bsize, + /* .min = */ generic_index_min, + /* .max = */ generic_index_max, + /* .random = */ memtx_tree_index_random, + /* .count = */ memtx_tree_index_count, + /* .get = */ memtx_tree_index_get, + /* .replace = */ memtx_tree_func_index_replace, + /* .create_iterator = */ memtx_tree_index_create_iterator, + /* .create_snapshot_iterator = */ + memtx_tree_index_create_snapshot_iterator, + /* .stat = */ generic_index_stat, + /* .compact = */ generic_index_compact, + /* .reset_stat = */ generic_index_reset_stat, + /* .begin_build = */ memtx_tree_index_begin_build, + /* .reserve = */ memtx_tree_index_reserve, + /* .build_next = */ memtx_tree_func_index_build_next, + /* .end_build = */ memtx_tree_index_end_build, +}; + +/** + * A disabled index vtab provides safe dummy methods for + * 'inactive' index. It is required to perform a fault-tolerant + * recovery from snapshoot in case of func_index (because + * key defintion is not completely initialized at that moment). + */ +static const struct index_vtab memtx_tree_disabled_index_vtab = { + /* .destroy = */ memtx_tree_index_destroy, + /* .commit_create = */ generic_index_commit_create, + /* .abort_create = */ generic_index_abort_create, + /* .commit_modify = */ generic_index_commit_modify, + /* .commit_drop = */ generic_index_commit_drop, + /* .update_def = */ generic_index_update_def, + /* .depends_on_pk = */ generic_index_depends_on_pk, + /* .def_change_requires_rebuild = */ + generic_index_def_change_requires_rebuild, + /* .size = */ generic_index_size, + /* .bsize = */ generic_index_bsize, + /* .min = */ generic_index_min, + /* .max = */ generic_index_max, + /* .random = */ generic_index_random, + /* .count = */ generic_index_count, + /* .get = */ generic_index_get, + /* .replace = */ disabled_index_replace, + /* .create_iterator = */ generic_index_create_iterator, + /* .create_snapshot_iterator = */ + generic_index_create_snapshot_iterator, + /* .stat = */ generic_index_stat, + /* .compact = */ generic_index_compact, + /* .reset_stat = */ generic_index_reset_stat, + /* .begin_build = */ generic_index_begin_build, + /* .reserve = */ generic_index_reserve, + /* .build_next = */ disabled_index_build_next, + /* .end_build = */ generic_index_end_build, +}; + struct index * memtx_tree_index_new(struct memtx_engine *memtx, struct index_def *def) { @@ -1082,9 +1393,17 @@ memtx_tree_index_new(struct memtx_engine *memtx, struct index_def *def) "malloc", "struct memtx_tree_index"); return NULL; } - const struct index_vtab *vtab = def->key_def->is_multikey ? - &memtx_tree_index_multikey_vtab : - &memtx_tree_index_vtab; + const struct index_vtab *vtab; + if (def->key_def->for_func_index) { + if (def->key_def->func_index_func == NULL) + vtab = &memtx_tree_disabled_index_vtab; + else + vtab = &memtx_tree_func_index_vtab; + } else if (def->key_def->is_multikey) { + vtab = &memtx_tree_index_multikey_vtab; + } else { + vtab = &memtx_tree_index_vtab; + } if (index_create(&index->base, (struct engine *)memtx, vtab, def) != 0) { free(index); diff --git a/src/box/sql.c b/src/box/sql.c index 4c9a4c15b..0ab3a506f 100644 --- a/src/box/sql.c +++ b/src/box/sql.c @@ -347,7 +347,7 @@ sql_ephemeral_space_create(uint32_t field_count, struct sql_key_info *key_info) } } struct key_def *ephemer_key_def = key_def_new(ephemer_key_parts, - field_count); + field_count, false); if (ephemer_key_def == NULL) return NULL; diff --git a/src/box/sql/build.c b/src/box/sql/build.c index ec9a474ca..0a6759e41 100644 --- a/src/box/sql/build.c +++ b/src/box/sql/build.c @@ -2338,7 +2338,7 @@ index_fill_def(struct Parse *parse, struct index *index, part->coll_id = coll_id; part->path = NULL; } - key_def = key_def_new(key_parts, expr_list->nExpr); + key_def = key_def_new(key_parts, expr_list->nExpr, false); if (key_def == NULL) goto tnt_error; /* diff --git a/src/box/sql/select.c b/src/box/sql/select.c index bf0410b7e..c312f61f1 100644 --- a/src/box/sql/select.c +++ b/src/box/sql/select.c @@ -1438,7 +1438,7 @@ sql_key_info_to_key_def(struct sql_key_info *key_info) { if (key_info->key_def == NULL) { key_info->key_def = key_def_new(key_info->parts, - key_info->part_count); + key_info->part_count, false); } return key_info->key_def; } diff --git a/src/box/sql/where.c b/src/box/sql/where.c index 5458c6a75..ed507bf4d 100644 --- a/src/box/sql/where.c +++ b/src/box/sql/where.c @@ -2775,7 +2775,7 @@ whereLoopAddBtree(WhereLoopBuilder * pBuilder, /* WHERE clause information */ part.coll_id = COLL_NONE; part.path = NULL; - struct key_def *key_def = key_def_new(&part, 1); + struct key_def *key_def = key_def_new(&part, 1, false); if (key_def == NULL) { tnt_error: pWInfo->pParse->is_aborted = true; diff --git a/src/box/tuple_format.c b/src/box/tuple_format.c index 02fadf1cf..514d5d9c0 100644 --- a/src/box/tuple_format.c +++ b/src/box/tuple_format.c @@ -462,6 +462,8 @@ tuple_format_create(struct tuple_format *format, struct key_def * const *keys, /* extract field type info */ for (uint16_t key_no = 0; key_no < key_count; ++key_no) { const struct key_def *key_def = keys[key_no]; + if (key_def->for_func_index) + continue; bool is_sequential = key_def_is_sequential(key_def); const struct key_part *part = key_def->parts; const struct key_part *parts_end = part + key_def->part_count; @@ -615,6 +617,8 @@ tuple_format_alloc(struct key_def * const *keys, uint16_t key_count, /* find max max field no */ for (uint16_t key_no = 0; key_no < key_count; ++key_no) { const struct key_def *key_def = keys[key_no]; + if (key_def->for_func_index) + continue; const struct key_part *part = key_def->parts; const struct key_part *pend = part + key_def->part_count; for (; part < pend; part++) { diff --git a/src/box/vinyl.c b/src/box/vinyl.c index 04fe1c6fb..cd009c1c2 100644 --- a/src/box/vinyl.c +++ b/src/box/vinyl.c @@ -680,6 +680,11 @@ vinyl_space_check_index_def(struct space *space, struct index_def *index_def) return -1; } } + if (index_def->key_def->for_func_index) { + diag_set(ClientError, ER_UNSUPPORTED, "Vinyl", + "functional index"); + return -1; + } return 0; } @@ -974,6 +979,8 @@ vinyl_index_def_change_requires_rebuild(struct index *index, if (!old_def->opts.is_unique && new_def->opts.is_unique) return true; + if (old_def->opts.func_id != new_def->opts.func_id) + return true; assert(index_depends_on_pk(index)); const struct key_def *old_cmp_def = old_def->cmp_def; @@ -3150,7 +3157,7 @@ vy_send_lsm(struct vy_join_ctx *ctx, struct vy_lsm_recovery_info *lsm_info) /* Create key definition and tuple format. */ ctx->key_def = key_def_new(lsm_info->key_parts, - lsm_info->key_part_count); + lsm_info->key_part_count, false); if (ctx->key_def == NULL) goto out; ctx->format = vy_stmt_format_new(&ctx->env->stmt_env, &ctx->key_def, 1, diff --git a/test/unit/luaT_tuple_new.c b/test/unit/luaT_tuple_new.c index 0a16fa039..609d64e45 100644 --- a/test/unit/luaT_tuple_new.c +++ b/test/unit/luaT_tuple_new.c @@ -124,7 +124,7 @@ test_basic(struct lua_State *L) part.nullable_action = ON_CONFLICT_ACTION_DEFAULT; part.sort_order = SORT_ORDER_ASC; part.path = NULL; - struct key_def *key_def = key_def_new(&part, 1); + struct key_def *key_def = key_def_new(&part, 1, false); box_tuple_format_t *another_format = box_tuple_format_new(&key_def, 1); key_def_delete(key_def); diff --git a/test/unit/merger.test.c b/test/unit/merger.test.c index b4a989a20..345a2364e 100644 --- a/test/unit/merger.test.c +++ b/test/unit/merger.test.c @@ -214,7 +214,7 @@ test_merger(struct tuple_format *format) merge_source_array_new(true), }; - struct key_def *key_def = key_def_new(&key_part_unsigned, 1); + struct key_def *key_def = key_def_new(&key_part_unsigned, 1, false); struct merge_source *merger = merger_new(key_def, sources, source_count, false); key_def_delete(key_def); @@ -252,7 +252,7 @@ test_basic() plan(4); header(); - struct key_def *key_def = key_def_new(&key_part_integer, 1); + struct key_def *key_def = key_def_new(&key_part_integer, 1, false); struct tuple_format *format = box_tuple_format_new(&key_def, 1); assert(format != NULL); diff --git a/src/box/CMakeLists.txt b/src/box/CMakeLists.txt index eba23a0f6..9bba37bcb 100644 --- a/src/box/CMakeLists.txt +++ b/src/box/CMakeLists.txt @@ -102,6 +102,7 @@ add_library(box STATIC fk_constraint.c func.c func_def.c + key_list.c alter.cc schema.cc schema_def.c diff --git a/src/box/alter.cc b/src/box/alter.cc index 9b064286c..3caeac3bb 100644 --- a/src/box/alter.cc +++ b/src/box/alter.cc @@ -285,7 +285,7 @@ index_def_new_from_tuple(struct tuple *tuple, struct space *space) space->def->fields, space->def->field_count, &fiber()->gc) != 0) diag_raise(); - key_def = key_def_new(part_def, part_count); + key_def = key_def_new(part_def, part_count, opts.func_id > 0); if (key_def == NULL) diag_raise(); struct index_def *index_def = @@ -1368,6 +1368,27 @@ RebuildIndex::~RebuildIndex() index_def_delete(new_index_def); } +/** + * RebuildFuncIndex - prepare func index definition, + * drop the old index data and rebuild index from by reading the + * primary key. + */ +class RebuildFuncIndex: public RebuildIndex +{ + struct index_def * + func_index_def_new(struct index_def *index_def, struct func *func) + { + struct index_def *new_index_def = index_def_dup_xc(index_def); + index_def_set_func(new_index_def, func); + return new_index_def; + } +public: + RebuildFuncIndex(struct alter_space *alter, + struct index_def *old_index_def_arg, struct func *func) : + RebuildIndex(alter, func_index_def_new(old_index_def_arg, func), + old_index_def_arg) {} +}; + /** TruncateIndex - truncate an index. */ class TruncateIndex: public AlterSpaceOp { @@ -2844,6 +2865,12 @@ on_replace_dd_func(struct trigger * /* trigger */, void *event) (unsigned) old_func->def->uid, "function has grants"); } + if (old_func != NULL && + space_has_data(BOX_FUNC_INDEX_ID, 1, old_func->def->fid)) { + tnt_raise(ClientError, ER_DROP_FUNCTION, + (unsigned) old_func->def->uid, + "function has references"); + } struct trigger *on_commit = txn_alter_trigger_new(on_drop_func_commit, old_func); struct trigger *on_rollback = @@ -4692,6 +4719,64 @@ on_replace_dd_ck_constraint(struct trigger * /* trigger*/, void *event) trigger_run_xc(&on_alter_space, space); } +/** A trigger invoked on replace in the _func_index space. */ +static void +on_replace_dd_func_index(struct trigger *trigger, void *event) +{ + (void) trigger; + struct txn *txn = (struct txn *) event; + struct txn_stmt *stmt = txn_current_stmt(txn); + struct tuple *old_tuple = stmt->old_tuple; + struct tuple *new_tuple = stmt->new_tuple; + + struct alter_space *alter = NULL; + struct func *func = NULL; + struct index *index; + struct space *space; + if (old_tuple == NULL && new_tuple != NULL) { + uint32_t space_id = tuple_field_u32_xc(new_tuple, + BOX_FUNC_INDEX_FIELD_SPACE_ID); + uint32_t index_id = tuple_field_u32_xc(new_tuple, + BOX_FUNC_INDEX_FIELD_INDEX_ID); + uint32_t fid = tuple_field_u32_xc(new_tuple, + BOX_FUNC_INDEX_FUNCTION_ID); + space = space_cache_find_xc(space_id); + index = index_find_xc(space, index_id); + func = func_cache_find(fid); + if (func->def->language != FUNC_LANGUAGE_LUA || + func->def->body == NULL || !func->def->is_deterministic || + !func->def->is_sandboxed) { + tnt_raise(ClientError, ER_WRONG_INDEX_OPTIONS, 0, + "referenced function doesn't satisfy " + "functional index function constraints"); + } + } else if (old_tuple != NULL && new_tuple == NULL) { + uint32_t space_id = tuple_field_u32_xc(old_tuple, + BOX_FUNC_INDEX_FIELD_SPACE_ID); + uint32_t index_id = tuple_field_u32_xc(old_tuple, + BOX_FUNC_INDEX_FIELD_INDEX_ID); + space = space_cache_find_xc(space_id); + index = index_find_xc(space, index_id); + func = NULL; + } else { + assert(old_tuple != NULL && new_tuple != NULL); + tnt_raise(ClientError, ER_UNSUPPORTED, + "functional index", "alter"); + } + + alter = alter_space_new(space); + auto scoped_guard = make_scoped_guard([=] {alter_space_delete(alter);}); + alter_space_move_indexes(alter, 0, index->def->iid); + (void) new RebuildFuncIndex(alter, index->def, func); + alter_space_move_indexes(alter, index->def->iid + 1, + space->index_id_max + 1); + (void) new MoveCkConstraints(alter); + (void) new UpdateSchemaVersion(alter); + alter_space_do(txn, alter); + + scoped_guard.is_active = false; +} + struct trigger alter_space_on_replace_space = { RLIST_LINK_INITIALIZER, on_replace_dd_space, NULL, NULL }; @@ -4752,4 +4837,8 @@ struct trigger on_replace_ck_constraint = { RLIST_LINK_INITIALIZER, on_replace_dd_ck_constraint, NULL, NULL }; +struct trigger on_replace_func_index = { + RLIST_LINK_INITIALIZER, on_replace_dd_func_index, NULL, NULL +}; + /* vim: set foldmethod=marker */ diff --git a/src/box/index.cc b/src/box/index.cc index 843d0e73d..00a1b502e 100644 --- a/src/box/index.cc +++ b/src/box/index.cc @@ -679,6 +679,16 @@ generic_index_replace(struct index *index, struct tuple *old_tuple, return -1; } +struct iterator * +generic_index_create_iterator(struct index *base, enum iterator_type type, + const char *key, uint32_t part_count) +{ + (void) type; (void) key; (void) part_count; + diag_set(UnsupportedIndexFeature, base->def, "read view"); + return NULL; +} + + struct snapshot_iterator * generic_index_create_snapshot_iterator(struct index *index) { @@ -729,4 +739,22 @@ generic_index_end_build(struct index *) { } +int +disabled_index_build_next(struct index *index, struct tuple *tuple) +{ + (void) index; (void) tuple; + return 0; +} + +int +disabled_index_replace(struct index *index, struct tuple *old_tuple, + struct tuple *new_tuple, enum dup_replace_mode mode, + struct tuple **result) +{ + (void) old_tuple; (void) new_tuple; (void) mode; + (void) index; + *result = NULL; + return 0; +} + /* }}} */ diff --git a/src/box/lua/schema.lua b/src/box/lua/schema.lua index 334f49d51..61790c1c3 100644 --- a/src/box/lua/schema.lua +++ b/src/box/lua/schema.lua @@ -515,6 +515,7 @@ box.schema.space.drop = function(space_id, space_name, opts) local _space_sequence = box.space[box.schema.SPACE_SEQUENCE_ID] local _fk_constraint = box.space[box.schema.FK_CONSTRAINT_ID] local _ck_constraint = box.space[box.schema.CK_CONSTRAINT_ID] + local _func_index = box.space[box.schema.FUNC_INDEX_ID] local sequence_tuple = _space_sequence:delete{space_id} if sequence_tuple ~= nil and sequence_tuple.is_generated == true then -- Delete automatically generated sequence. @@ -529,6 +530,9 @@ box.schema.space.drop = function(space_id, space_name, opts) for _, t in _ck_constraint.index.primary:pairs({space_id}) do _ck_constraint:delete({space_id, t.name}) end + for _, t in _func_index.index.primary:pairs({space_id}) do + _func_index:delete({space_id, t.index_id, t.func_id}) + end local keys = _vindex:select(space_id) for i = #keys, 1, -1 do local v = keys[i] @@ -956,6 +960,7 @@ local index_options = { range_size = 'number', page_size = 'number', bloom_fpr = 'number', + func = 'number, string', } -- @@ -980,6 +985,15 @@ end local create_index_template = table.deepcopy(alter_index_template) create_index_template.if_not_exists = "boolean" +-- Find a function id by given function name +local function func_id_by_name(func_name) + local func = box.space._func.index.name:get(func_name) + if func == nil then + box.error(box.error.NO_SUCH_FUNCTION, func_name) + end + return func.id +end + box.schema.index.create = function(space_id, name, options) check_param(space_id, 'space_id', 'number') check_param(name, 'name', 'string') @@ -1061,6 +1075,7 @@ box.schema.index.create = function(space_id, name, options) run_count_per_level = options.run_count_per_level, run_size_ratio = options.run_size_ratio, bloom_fpr = options.bloom_fpr, + func = options.func, } local field_type_aliases = { num = 'unsigned'; -- Deprecated since 1.7.2 @@ -1081,11 +1096,18 @@ box.schema.index.create = function(space_id, name, options) if parts_can_be_simplified then parts = simplify_index_parts(parts) end + if index_opts.func ~= nil and type(index_opts.func) == 'string' then + index_opts.func = func_id_by_name(index_opts.func) + end local sequence_proxy = space_sequence_alter_prepare(format, parts, options, space_id, iid, space.name, name) _index:insert{space_id, iid, name, options.type, index_opts, parts} space_sequence_alter_commit(sequence_proxy) + if index_opts.func ~= nil then + local _func_index = box.space[box.schema.FUNC_INDEX_ID] + _func_index:insert{space_id, iid, index_opts.func} + end return space.index[name] end @@ -1101,6 +1123,10 @@ box.schema.index.drop = function(space_id, index_id) end end local _index = box.space[box.schema.INDEX_ID] + local _func_index = box.space[box.schema.FUNC_INDEX_ID] + for _, v in box.space._func_index:pairs{space_id, index_id} do + _func_index:delete(v) + end _index:delete{space_id, index_id} end @@ -1197,11 +1223,18 @@ box.schema.index.alter = function(space_id, index_id, options) parts = simplify_index_parts(parts) end end + if index_opts.func ~= nil and type(index_opts.func) == 'string' then + index_opts.func = func_id_by_name(index_opts.func) + end local sequence_proxy = space_sequence_alter_prepare(format, parts, options, space_id, index_id, space.name, options.name) _index:replace{space_id, index_id, options.name, options.type, index_opts, parts} + if index_opts.func ~= nil then + local _func_index = box.space[box.schema.FUNC_INDEX_ID] + _func_index:insert{space_id, iid, index_opts.func} + end space_sequence_alter_commit(sequence_proxy) end diff --git a/src/box/lua/space.cc b/src/box/lua/space.cc index 18039fd6a..d0a7e7815 100644 --- a/src/box/lua/space.cc +++ b/src/box/lua/space.cc @@ -42,6 +42,8 @@ extern "C" { #include } /* extern "C" */ +#include "box/func.h" +#include "box/func_def.h" #include "box/space.h" #include "box/schema.h" #include "box/user_def.h" @@ -335,6 +337,22 @@ lbox_fillspace(struct lua_State *L, struct space *space, int i) lua_setfield(L, -2, "dimension"); } + if (index_opts->func_id > 0) { + lua_pushstring(L, "func"); + lua_newtable(L); + + lua_pushnumber(L, index_opts->func_id); + lua_setfield(L, -2, "fid"); + + struct func *func = func_by_id(index_opts->func_id); + if (func != NULL) { + lua_pushstring(L, func->def->name); + lua_setfield(L, -2, "name"); + } + + lua_settable(L, -3); + } + lua_pushstring(L, index_type_strs[index_def->type]); lua_setfield(L, -2, "type"); @@ -629,6 +647,8 @@ box_lua_space_init(struct lua_State *L) lua_setfield(L, -2, "VSEQUENCE_ID"); lua_pushnumber(L, BOX_SPACE_SEQUENCE_ID); lua_setfield(L, -2, "SPACE_SEQUENCE_ID"); + lua_pushnumber(L, BOX_FUNC_INDEX_ID); + lua_setfield(L, -2, "FUNC_INDEX_ID"); lua_pushnumber(L, BOX_SYSTEM_ID_MIN); lua_setfield(L, -2, "SYSTEM_ID_MIN"); lua_pushnumber(L, BOX_SYSTEM_ID_MAX); diff --git a/src/box/lua/upgrade.lua b/src/box/lua/upgrade.lua index 02c1cb0ff..f570a1c08 100644 --- a/src/box/lua/upgrade.lua +++ b/src/box/lua/upgrade.lua @@ -885,11 +885,29 @@ local function upgrade_func_to_2_2_1() collation = 'unicode_ci'}}}) end +local function create_func_index() + log.info("Create _func_index space") + local _func_index = box.space[box.schema.FUNC_INDEX_ID] + local _space = box.space._space + local _index = box.space._index + local format = {{name='space_id', type='unsigned'}, + {name='index_id', type='unsigned'}, + {name='func_id', type='unsigned'}} + _space:insert{_func_index.id, ADMIN, '_func_index', 'memtx', 0, + setmap({}), format} + _index:insert{_func_index.id, 0, 'primary', 'tree', {unique = true}, + {{0, 'unsigned'}, {1, 'unsigned'}, {2, 'unsigned'}}} + _index:insert{_func_index.id, 1, 'fid', 'tree', {unique = false}, + {{2, 'unsigned'}}} + +end + local function upgrade_to_2_2_1() upgrade_sequence_to_2_2_1() upgrade_ck_constraint_to_2_2_1() create_vcollation_space() upgrade_func_to_2_2_1() + create_func_index() end -------------------------------------------------------------------------------- diff --git a/src/box/schema.cc b/src/box/schema.cc index 20666386d..5d4a3ff00 100644 --- a/src/box/schema.cc +++ b/src/box/schema.cc @@ -266,7 +266,7 @@ sc_space_new(uint32_t id, const char *name, uint32_t key_part_count, struct trigger *replace_trigger) { - struct key_def *key_def = key_def_new(key_parts, key_part_count); + struct key_def *key_def = key_def_new(key_parts, key_part_count, false); if (key_def == NULL) diag_raise(); auto key_def_guard = @@ -462,6 +462,16 @@ schema_init() sc_space_new(BOX_CK_CONSTRAINT_ID, "_ck_constraint", key_parts, 2, &on_replace_ck_constraint); + /* _func_index - check constraints. */ + key_parts[0].fieldno = 0; /* space id */ + key_parts[0].type = FIELD_TYPE_UNSIGNED; + key_parts[1].fieldno = 1; /* index id */ + key_parts[1].type = FIELD_TYPE_UNSIGNED; + key_parts[2].fieldno = 2; /* function id */ + key_parts[2].type = FIELD_TYPE_UNSIGNED; + sc_space_new(BOX_FUNC_INDEX_ID, "_func_index", key_parts, 3, + &on_replace_func_index); + /* * _vinyl_deferred_delete - blackhole that is needed * for writing deferred DELETE statements generated by diff --git a/src/box/tuple_compare.cc b/src/box/tuple_compare.cc index c03b584d0..ca0bb7913 100644 --- a/src/box/tuple_compare.cc +++ b/src/box/tuple_compare.cc @@ -1271,6 +1271,93 @@ static const comparator_with_key_signature cmp_wk_arr[] = { KEY_COMPARATOR(1, FIELD_TYPE_STRING , 2, FIELD_TYPE_STRING) }; +/** + * The following compare method is valid for func_index: + * tuple_a_hint and tuple_b_hint are expected to be a valid + * pointers to extracted key memory. Thouse keys had been already + * validated and have a format of MsgPack array having exact + * func_index_part_count parts, while a given cmp_def has + * part_count > func_index_part_count. The cmp_def had been + * produced with key_def_merge call and it last unique parts are + * taken from primary index key definition. + */ +template +static inline int +func_index_compare(struct tuple *tuple_a, hint_t tuple_a_hint, + struct tuple *tuple_b, hint_t tuple_b_hint, + struct key_def *cmp_def) +{ + assert(cmp_def->for_func_index); + assert(is_nullable == cmp_def->is_nullable); + + const char *key_a = (const char *)tuple_a_hint; + const char *key_b = (const char *)tuple_b_hint; + assert(mp_typeof(*key_a) == MP_ARRAY); + uint32_t part_count_a = mp_decode_array(&key_a); + assert(mp_typeof(*key_b) == MP_ARRAY); + uint32_t part_count_b = mp_decode_array(&key_b); + + uint32_t key_part_count = MIN(part_count_a, part_count_b); + int rc = key_compare_parts(key_a, key_b, key_part_count, + cmp_def); + if (rc != 0) + return rc; + /* + * Primary index definiton key compare. + * It cannot contain nullable parts so the code is + * simplified correspondingly. + */ + const char *tuple_a_raw = tuple_data(tuple_a); + const char *tuple_b_raw = tuple_data(tuple_b); + struct tuple_format *format_a = tuple_format(tuple_a); + struct tuple_format *format_b = tuple_format(tuple_b); + const uint32_t *field_map_a = tuple_field_map(tuple_a); + const uint32_t *field_map_b = tuple_field_map(tuple_b); + const char *field_a, *field_b; + for (uint32_t i = key_part_count; i < cmp_def->part_count; i++) { + struct key_part *part = &cmp_def->parts[i]; + field_a = tuple_field_raw_by_part(format_a, tuple_a_raw, + field_map_a, part, + MULTIKEY_NONE); + field_b = tuple_field_raw_by_part(format_b, tuple_b_raw, + field_map_b, part, + MULTIKEY_NONE); + assert(field_a != NULL && field_b != NULL); + rc = tuple_compare_field(field_a, field_b, part->type, + part->coll); + if (rc != 0) + return rc; + else + continue; + } + return 0; +} + +/** + * The following compare with key method is valid for func_index: + * tuple_hint is expected to be a valid pointer to + * extracted key memory to be compared with given key by + * func_index key definition. + */ +template +static inline int +func_index_compare_with_key(struct tuple *tuple, hint_t tuple_hint, + const char *key, uint32_t part_count, + hint_t key_hint, struct key_def *key_def) +{ + (void)tuple; (void)key_hint; + assert(key_def->for_func_index); + assert(is_nullable == key_def->is_nullable); + const char *tuple_key = (const char *)tuple_hint; + assert(mp_typeof(*tuple_key) == MP_ARRAY); + + uint32_t tuple_key_count = mp_decode_array(&tuple_key); + part_count = MIN(part_count, tuple_key_count); + part_count = MIN(part_count, key_def->part_count); + return key_compare_parts(tuple_key, key, part_count, + key_def); +} + #undef KEY_COMPARATOR /* }}} tuple_compare_with_key */ @@ -1592,7 +1679,7 @@ tuple_hint(struct tuple *tuple, struct key_def *key_def) } static hint_t -key_hint_multikey(const char *key, uint32_t part_count, struct key_def *key_def) +key_hint_stub(const char *key, uint32_t part_count, struct key_def *key_def) { (void) key; (void) part_count; @@ -1600,19 +1687,19 @@ key_hint_multikey(const char *key, uint32_t part_count, struct key_def *key_def) /* * Multikey hint for tuple is an index of the key in * array, it always must be defined. While - * tuple_hint_multikey assumes that it must be + * key_hint_stub assumes that it must be * initialized manually (so it mustn't be called), * the virtual method for a key makes sense. Overriding * this method such way, we extend existend code to * do nothing on key hint calculation an it is valid * because it is never used(unlike tuple hint). */ - assert(key_def->is_multikey); + assert(key_def->is_multikey || key_def->for_func_index); return HINT_NONE; } static hint_t -tuple_hint_multikey(struct tuple *tuple, struct key_def *key_def) +key_hint_stub(struct tuple *tuple, struct key_def *key_def) { (void) tuple; (void) key_def; @@ -1641,9 +1728,9 @@ key_def_set_hint_func(struct key_def *def) static void key_def_set_hint_func(struct key_def *def) { - if (def->is_multikey) { - def->key_hint = key_hint_multikey; - def->tuple_hint = tuple_hint_multikey; + if (def->is_multikey || def->for_func_index) { + def->key_hint = key_hint_stub; + def->tuple_hint = key_hint_stub; return; } switch (def->parts->type) { @@ -1769,10 +1856,24 @@ key_def_set_compare_func_json(struct key_def *def) } } +template +static void +key_def_set_compare_func_for_func_index(struct key_def *def) +{ + assert(def->for_func_index); + def->tuple_compare = func_index_compare; + def->tuple_compare_with_key = func_index_compare_with_key; +} + void key_def_set_compare_func(struct key_def *def) { - if (!key_def_has_collation(def) && + if (def->for_func_index) { + if (def->is_nullable) + key_def_set_compare_func_for_func_index(def); + else + key_def_set_compare_func_for_func_index(def); + } else if (!key_def_has_collation(def) && !def->is_nullable && !def->has_json_paths) { key_def_set_compare_func_fast(def); } else if (!def->has_json_paths) { diff --git a/src/box/tuple_extract_key.cc b/src/box/tuple_extract_key.cc index 471c7df80..c1ad3929e 100644 --- a/src/box/tuple_extract_key.cc +++ b/src/box/tuple_extract_key.cc @@ -120,6 +120,7 @@ tuple_extract_key_slowpath(struct tuple *tuple, struct key_def *key_def, key_def_contains_sequential_parts(key_def)); assert(is_multikey == key_def->is_multikey); assert(!key_def->is_multikey || multikey_idx != MULTIKEY_NONE); + assert(!key_def->for_func_index); assert(mp_sizeof_nil() == 1); const char *data = tuple_data(tuple); uint32_t part_count = key_def->part_count; @@ -251,6 +252,7 @@ tuple_extract_key_slowpath_raw(const char *data, const char *data_end, assert(!has_optional_parts || key_def->is_nullable); assert(has_optional_parts == key_def->has_optional_parts); assert(!key_def->is_multikey || multikey_idx != MULTIKEY_NONE); + assert(!key_def->for_func_index); assert(mp_sizeof_nil() == 1); /* allocate buffer with maximal possible size */ char *key = (char *) region_alloc(&fiber()->gc, data_end - data); @@ -367,6 +369,7 @@ key_def_set_extract_func_plain(struct key_def *def) { assert(!def->has_json_paths); assert(!def->is_multikey); + assert(!def->for_func_index); if (key_def_is_sequential(def)) { assert(contains_sequential_parts || def->part_count == 1); def->tuple_extract_key = tuple_extract_key_sequential @@ -387,6 +390,7 @@ static void key_def_set_extract_func_json(struct key_def *def) { assert(def->has_json_paths); + assert(!def->for_func_index); if (def->is_multikey) { def->tuple_extract_key = tuple_extract_key_slowpath ; } +static char * +tuple_extract_key_stub(struct tuple *tuple, struct key_def *key_def, + int multikey_idx, uint32_t *key_size) +{ + (void)tuple; (void)key_def; (void)multikey_idx; (void)key_size; + unreachable(); + return NULL; +} + +static char * +tuple_extract_key_raw_stub(const char *data, const char *data_end, + struct key_def *key_def, int multikey_idx, + uint32_t *key_size) +{ + (void)data; (void)data_end; + (void)key_def; (void)multikey_idx; (void)key_size; + unreachable(); + return NULL; +} + void key_def_set_extract_func(struct key_def *key_def) { bool contains_sequential_parts = key_def_contains_sequential_parts(key_def); bool has_optional_parts = key_def->has_optional_parts; - if (!key_def->has_json_paths) { + if (key_def->for_func_index) { + key_def->tuple_extract_key = tuple_extract_key_stub; + key_def->tuple_extract_key_raw = tuple_extract_key_raw_stub; + } else if (!key_def->has_json_paths) { if (!contains_sequential_parts && !has_optional_parts) { key_def_set_extract_func_plain(key_def); } else if (!contains_sequential_parts && has_optional_parts) { diff --git a/src/box/tuple_hash.cc b/src/box/tuple_hash.cc index 780e3d053..39f89a659 100644 --- a/src/box/tuple_hash.cc +++ b/src/box/tuple_hash.cc @@ -365,6 +365,7 @@ tuple_hash_slowpath(struct tuple *tuple, struct key_def *key_def) assert(has_json_paths == key_def->has_json_paths); assert(has_optional_parts == key_def->has_optional_parts); assert(!key_def->is_multikey); + assert(!key_def->for_func_index); uint32_t h = HASH_SEED; uint32_t carry = 0; uint32_t total_size = 0; diff --git a/test/app-tap/tarantoolctl.test.lua b/test/app-tap/tarantoolctl.test.lua index 957b883f4..df2ee377f 100755 --- a/test/app-tap/tarantoolctl.test.lua +++ b/test/app-tap/tarantoolctl.test.lua @@ -405,8 +405,8 @@ do check_ctlcat_xlog(test_i, dir, "--from=3 --to=6 --format=json --show-system --replica 1", "\n", 3) check_ctlcat_xlog(test_i, dir, "--from=3 --to=6 --format=json --show-system --replica 1 --replica 2", "\n", 3) check_ctlcat_xlog(test_i, dir, "--from=3 --to=6 --format=json --show-system --replica 2", "\n", 0) - check_ctlcat_snap(test_i, dir, "--space=280", "---\n", 23) - check_ctlcat_snap(test_i, dir, "--space=288", "---\n", 50) + check_ctlcat_snap(test_i, dir, "--space=280", "---\n", 24) + check_ctlcat_snap(test_i, dir, "--space=288", "---\n", 52) end) end) diff --git a/test/box-py/bootstrap.result b/test/box-py/bootstrap.result index f2d1f46fb..a5d645df8 100644 --- a/test/box-py/bootstrap.result +++ b/test/box-py/bootstrap.result @@ -93,6 +93,8 @@ box.space._space:select{} - [364, 1, '_ck_constraint', 'memtx', 0, {}, [{'name': 'space_id', 'type': 'unsigned'}, {'name': 'name', 'type': 'string'}, {'name': 'is_deferred', 'type': 'boolean'}, {'name': 'language', 'type': 'str'}, {'name': 'code', 'type': 'str'}]] + - [372, 1, '_func_index', 'memtx', 0, {}, [{'name': 'space_id', 'type': 'unsigned'}, + {'name': 'index_id', 'type': 'unsigned'}, {'name': 'func_id', 'type': 'unsigned'}]] ... box.space._index:select{} --- @@ -148,6 +150,9 @@ box.space._index:select{} - [356, 0, 'primary', 'tree', {'unique': true}, [[0, 'string'], [1, 'unsigned']]] - [356, 1, 'child_id', 'tree', {'unique': false}, [[1, 'unsigned']]] - [364, 0, 'primary', 'tree', {'unique': true}, [[0, 'unsigned'], [1, 'string']]] + - [372, 0, 'primary', 'tree', {'unique': true}, [[0, 'unsigned'], [1, 'unsigned'], + [2, 'unsigned']]] + - [372, 1, 'fid', 'tree', {'unique': false}, [[2, 'unsigned']]] ... box.space._user:select{} --- diff --git a/test/box/access.result b/test/box/access.result index 5ee92a443..ba72b5f74 100644 --- a/test/box/access.result +++ b/test/box/access.result @@ -1496,6 +1496,9 @@ box.schema.user.grant('tester', 'read', 'space', '_fk_constraint') box.schema.user.grant('tester', 'read', 'space', '_ck_constraint') --- ... +box.schema.user.grant('tester', 'read', 'space', '_func_index') +--- +... box.session.su("tester") --- ... diff --git a/test/box/access.test.lua b/test/box/access.test.lua index 79340b0f5..219cdb04a 100644 --- a/test/box/access.test.lua +++ b/test/box/access.test.lua @@ -557,6 +557,7 @@ box.schema.user.grant('tester', 'read', 'space', '_space_sequence') box.schema.user.grant('tester', 'read', 'space', '_trigger') box.schema.user.grant('tester', 'read', 'space', '_fk_constraint') box.schema.user.grant('tester', 'read', 'space', '_ck_constraint') +box.schema.user.grant('tester', 'read', 'space', '_func_index') box.session.su("tester") -- successful create s1 = box.schema.space.create("test_space") diff --git a/test/box/access_misc.result b/test/box/access_misc.result index c69cf0283..31b935914 100644 --- a/test/box/access_misc.result +++ b/test/box/access_misc.result @@ -833,140 +833,142 @@ box.space._space:select() - [364, 1, '_ck_constraint', 'memtx', 0, {}, [{'name': 'space_id', 'type': 'unsigned'}, {'name': 'name', 'type': 'string'}, {'name': 'is_deferred', 'type': 'boolean'}, {'name': 'language', 'type': 'str'}, {'name': 'code', 'type': 'str'}]] + - [372, 1, '_func_index', 'memtx', 0, {}, [{'name': 'space_id', 'type': 'unsigned'}, + {'name': 'index_id', 'type': 'unsigned'}, {'name': 'func_id', 'type': 'unsigned'}]] ... box.space._func:select() --- ... session = nil --- diff --git a/test/box/access_sysview.result b/test/box/access_sysview.result index d65aa37ae..a82127ebb 100644 --- a/test/box/access_sysview.result +++ b/test/box/access_sysview.result @@ -246,11 +246,11 @@ box.session.su('guest') ... #box.space._vspace:select{} --- -- 24 +- 25 ... #box.space._vindex:select{} --- -- 51 +- 53 ... #box.space._vuser:select{} --- @@ -282,7 +282,7 @@ box.session.su('guest') ... #box.space._vindex:select{} --- -- 51 +- 53 ... #box.space._vuser:select{} --- diff --git a/test/box/alter.result b/test/box/alter.result index a6db011ff..91a239bbc 100644 --- a/test/box/alter.result +++ b/test/box/alter.result @@ -92,7 +92,7 @@ space = box.space[t[1]] ... space.id --- -- 365 +- 373 ... space.field_count --- @@ -137,7 +137,7 @@ space_deleted ... space:replace{0} --- -- error: Space '365' does not exist +- error: Space '373' does not exist ... _index:insert{_space.id, 0, 'primary', 'tree', {unique=true}, {{0, 'unsigned'}}} --- @@ -218,6 +218,9 @@ _index:select{} - [356, 0, 'primary', 'tree', {'unique': true}, [[0, 'string'], [1, 'unsigned']]] - [356, 1, 'child_id', 'tree', {'unique': false}, [[1, 'unsigned']]] - [364, 0, 'primary', 'tree', {'unique': true}, [[0, 'unsigned'], [1, 'string']]] + - [372, 0, 'primary', 'tree', {'unique': true}, [[0, 'unsigned'], [1, 'unsigned'], + [2, 'unsigned']]] + - [372, 1, 'fid', 'tree', {'unique': false}, [[2, 'unsigned']]] ... -- modify indexes of a system space _index:delete{_index.id, 0} diff --git a/test/box/bitset.result b/test/box/bitset.result index 78f74ec37..bf44773ef 100644 --- a/test/box/bitset.result +++ b/test/box/bitset.result @@ -1996,3 +1996,27 @@ _ = s:create_index('bitset', {type = 'bitset', parts = {{'[2][*]', 'unsigned'}}} s:drop() --- ... +-- Bitset index can not use function. +s = box.schema.space.create('withdata') +--- +... +lua_code = [[function(tuple) return tuple[1] + tuple[2] end]] +--- +... +box.schema.func.create('s', {body = lua_code, is_deterministic = true, is_sandboxed = true}) +--- +... +_ = s:create_index('pk') +--- +... +_ = s:create_index('idx', {type = 'bitset', func = box.func.s.id, parts = {{1, 'unsigned'}}}) +--- +- error: 'Can''t create or modify index ''idx'' in space ''withdata'': BITSET index + can not use a function' +... +s:drop() +--- +... +box.schema.func.drop('s') +--- +... diff --git a/test/box/bitset.test.lua b/test/box/bitset.test.lua index eb013a1c0..d644d34e0 100644 --- a/test/box/bitset.test.lua +++ b/test/box/bitset.test.lua @@ -153,3 +153,12 @@ s = box.schema.space.create('test') _ = s:create_index('primary') _ = s:create_index('bitset', {type = 'bitset', parts = {{'[2][*]', 'unsigned'}}}) s:drop() + +-- Bitset index can not use function. +s = box.schema.space.create('withdata') +lua_code = [[function(tuple) return tuple[1] + tuple[2] end]] +box.schema.func.create('s', {body = lua_code, is_deterministic = true, is_sandboxed = true}) +_ = s:create_index('pk') +_ = s:create_index('idx', {type = 'bitset', func = box.func.s.id, parts = {{1, 'unsigned'}}}) +s:drop() +box.schema.func.drop('s') diff --git a/test/box/hash.result b/test/box/hash.result index 9f08c49b8..5e1441ecc 100644 --- a/test/box/hash.result +++ b/test/box/hash.result @@ -847,3 +847,27 @@ _ = s:create_index('hash', {type = 'hash', parts = {{'[2][*]', 'unsigned'}}}) s:drop() --- ... +-- Hash index can not use function. +s = box.schema.space.create('withdata') +--- +... +lua_code = [[function(tuple) return tuple[1] + tuple[2] end]] +--- +... +box.schema.func.create('s', {body = lua_code, is_deterministic = true, is_sandboxed = true}) +--- +... +_ = s:create_index('pk') +--- +... +_ = s:create_index('idx', {type = 'hash', func = box.func.s.id, parts = {{1, 'unsigned'}}}) +--- +- error: 'Can''t create or modify index ''idx'' in space ''withdata'': HASH index + can not use a function' +... +s:drop() +--- +... +box.schema.func.drop('s') +--- +... diff --git a/test/box/hash.test.lua b/test/box/hash.test.lua index 9801873c4..78c831f77 100644 --- a/test/box/hash.test.lua +++ b/test/box/hash.test.lua @@ -353,3 +353,12 @@ s = box.schema.space.create('test') _ = s:create_index('primary') _ = s:create_index('hash', {type = 'hash', parts = {{'[2][*]', 'unsigned'}}}) s:drop() + +-- Hash index can not use function. +s = box.schema.space.create('withdata') +lua_code = [[function(tuple) return tuple[1] + tuple[2] end]] +box.schema.func.create('s', {body = lua_code, is_deterministic = true, is_sandboxed = true}) +_ = s:create_index('pk') +_ = s:create_index('idx', {type = 'hash', func = box.func.s.id, parts = {{1, 'unsigned'}}}) +s:drop() +box.schema.func.drop('s') diff --git a/test/box/misc.result b/test/box/misc.result index 791730935..7a15dabf0 100644 --- a/test/box/misc.result +++ b/test/box/misc.result @@ -526,6 +526,9 @@ t; 195: box.error.CREATE_CK_CONSTRAINT 196: box.error.CK_CONSTRAINT_FAILED 197: box.error.SQL_COLUMN_COUNT + 198: box.error.FUNC_INDEX_FUNC + 199: box.error.FUNC_INDEX_FORMAT + 200: box.error.FUNC_INDEX_PARTS ... test_run:cmd("setopt delimiter ''"); --- diff --git a/test/box/rtree_misc.result b/test/box/rtree_misc.result index 6e48bacc7..a2e7db1e3 100644 --- a/test/box/rtree_misc.result +++ b/test/box/rtree_misc.result @@ -682,3 +682,27 @@ _ = s:create_index('rtree', {type = 'rtree', parts = {{'[2][*]', 'array'}}}) s:drop() --- ... +-- Rtree index can not use function. +s = box.schema.space.create('withdata') +--- +... +lua_code = [[function(tuple) return {tuple[1] + tuple[2], tuple[1] - tuple[2]} end]] +--- +... +box.schema.func.create('fextract', {body = lua_code, is_deterministic = true, is_sandboxed = true}) +--- +... +_ = s:create_index('pk') +--- +... +_ = s:create_index('idx', {type = 'rtree', func = box.func.fextract.id, parts = {{1, 'array'}}}) +--- +- error: 'Can''t create or modify index ''idx'' in space ''withdata'': RTREE index + can not use a function' +... +s:drop() +--- +... +box.schema.func.drop('fextract') +--- +... diff --git a/test/box/rtree_misc.test.lua b/test/box/rtree_misc.test.lua index 000a928e8..992fb5ef9 100644 --- a/test/box/rtree_misc.test.lua +++ b/test/box/rtree_misc.test.lua @@ -243,3 +243,12 @@ s = box.schema.space.create('test') _ = s:create_index('primary') _ = s:create_index('rtree', {type = 'rtree', parts = {{'[2][*]', 'array'}}}) s:drop() + +-- Rtree index can not use function. +s = box.schema.space.create('withdata') +lua_code = [[function(tuple) return {tuple[1] + tuple[2], tuple[1] - tuple[2]} end]] +box.schema.func.create('fextract', {body = lua_code, is_deterministic = true, is_sandboxed = true}) +_ = s:create_index('pk') +_ = s:create_index('idx', {type = 'rtree', func = box.func.fextract.id, parts = {{1, 'array'}}}) +s:drop() +box.schema.func.drop('fextract') diff --git a/test/engine/engine.cfg b/test/engine/engine.cfg index 9f07629b4..f1e7de274 100644 --- a/test/engine/engine.cfg +++ b/test/engine/engine.cfg @@ -2,7 +2,10 @@ "*": { "memtx": {"engine": "memtx"}, "vinyl": {"engine": "vinyl"} - } + }, + "func_index.test.lua": { + "memtx": {"engine": "memtx"} + } } diff --git a/test/engine/func_index.result b/test/engine/func_index.result new file mode 100644 index 000000000..7dd9b4f47 --- /dev/null +++ b/test/engine/func_index.result @@ -0,0 +1,734 @@ +-- test-run result file version 2 +test_run = require('test_run').new() + | --- + | ... +engine = test_run:get_cfg('engine') + | --- + | ... + +-- +-- gh-1260: Func index. +-- +s = box.schema.space.create('withdata', {engine = engine}) + | --- + | ... +lua_code = [[function(tuple) return {tuple[1] + tuple[2]} end]] + | --- + | ... +lua_code2 = [[function(tuple) return {tuple[1] + tuple[2], 2 * tuple[1] + tuple[2]} end]] + | --- + | ... +box.schema.func.create('s_nonpersistent') + | --- + | ... +box.schema.func.create('s_ivaliddef1', {body = lua_code, is_deterministic = false, is_sandboxed = true}) + | --- + | ... +box.schema.func.create('s_ivaliddef2', {body = lua_code, is_deterministic = true, is_sandboxed = false}) + | --- + | ... + +box.schema.func.create('s', {body = lua_code, is_deterministic = true, is_sandboxed = true}) + | --- + | ... +box.schema.func.create('ss', {body = lua_code2, is_deterministic = true, is_sandboxed = true}) + | --- + | ... + +-- Func index can't be primary. +_ = s:create_index('idx', {func = box.func.s.id, parts = {{1, 'unsigned'}}}) + | --- + | - error: 'Can''t create or modify index ''idx'' in space ''withdata'': primary key + | can not use a function' + | ... +pk = s:create_index('pk') + | --- + | ... +-- Invalid fid. +_ = s:create_index('idx', {func = 6666, parts = {{1, 'unsigned'}}}) + | --- + | - error: Function '6666' does not exist + | ... +s.index.idx:drop() + | --- + | ... +-- Can't use non-persistent function in functional index. +_ = s:create_index('idx', {func = box.func.s_nonpersistent.id, parts = {{1, 'unsigned'}}}) + | --- + | - error: 'Wrong index options (field 0): referenced function doesn''t satisfy functional + | index function constraints' + | ... +s.index.idx:drop() + | --- + | ... +-- Can't use non-deterministic function in functional index. +_ = s:create_index('idx', {func = box.func.s_ivaliddef1.id, parts = {{1, 'unsigned'}}}) + | --- + | - error: 'Wrong index options (field 0): referenced function doesn''t satisfy functional + | index function constraints' + | ... +s.index.idx:drop() + | --- + | ... +-- Can't use non-sandboxed function in functional index. +_ = s:create_index('idx', {func = box.func.s_ivaliddef2.id, parts = {{1, 'unsigned'}}}) + | --- + | - error: 'Wrong index options (field 0): referenced function doesn''t satisfy functional + | index function constraints' + | ... +s.index.idx:drop() + | --- + | ... +-- Can't use non-sequential parts in returned key definition. +_ = s:create_index('idx', {func = box.func.ss.id, parts = {{1, 'unsigned'}, {3, 'unsigned'}}}) + | --- + | - error: 'Wrong functional index definition: key part numbers must be sequential and + | first part number must be 1' + | ... +-- Can't use parts started not by 1 field. +_ = s:create_index('idx', {func = box.func.ss.id, parts = {{2, 'unsigned'}, {3, 'unsigned'}}}) + | --- + | - error: 'Wrong functional index definition: key part numbers must be sequential and + | first part number must be 1' + | ... +-- Can't use JSON paths in returned key definiton. +_ = s:create_index('idx', {func = box.func.ss.id, parts = {{"[1]data", 'unsigned'}}}) + | --- + | - error: Functional index does not support json paths + | ... + +-- Can't drop a function referenced by functional index. +idx = s:create_index('idx', {unique = true, func = box.func.s.id, parts = {{1, 'unsigned'}}}) + | --- + | ... +box.schema.func.drop('s') + | --- + | - error: 'Can''t drop function 1: function has references' + | ... +box.snapshot() + | --- + | - ok + | ... +test_run:cmd("restart server default") + | +box.schema.func.drop('s') + | --- + | - error: 'Can''t drop function 1: function has references' + | ... +s = box.space.withdata + | --- + | ... +idx = s.index.idx + | --- + | ... +idx:drop() + | --- + | ... +box.schema.func.drop('s') + | --- + | ... + +test_run = require('test_run').new() + | --- + | ... +engine = test_run:get_cfg('engine') + | --- + | ... + +-- Invalid functional index extractor routine return: the extractor must return keys. +lua_code = [[function(tuple) return "hello" end]] + | --- + | ... +box.schema.func.create('invalidreturn0', {body = lua_code, is_deterministic = true, is_sandboxed = true}) + | --- + | ... +idx = s:create_index('idx', {func = box.func.invalidreturn0.id, parts = {{1, 'unsigned'}}}) + | --- + | ... +s:insert({1}) + | --- + | - error: 'Key format doesn''t match on defined in the functional index ''idx'' of + | the space ''withdata'': supplied key type is invalid: expected boolean' + | ... +idx:drop() + | --- + | ... + +-- Invalid functional index extractor routine return: a stirng instead of unsigned +lua_code = [[function(tuple) return {"hello"} end]] + | --- + | ... +box.schema.func.create('invalidreturn1', {body = lua_code, is_deterministic = true, is_sandboxed = true}) + | --- + | ... +idx = s:create_index('idx', {func = box.func.invalidreturn1.id, parts = {{1, 'unsigned'}}}) + | --- + | ... +s:insert({1}) + | --- + | - error: 'Key format doesn''t match on defined in the functional index ''idx'' of + | the space ''withdata'': Supplied key type of part 0 does not match index part + | type: expected unsigned' + | ... +idx:drop() + | --- + | ... + +-- Invalid functional index extractor routine return: invalid return format for multikey index. +lua_code = [[function(tuple) return {"hello", "world"}, {1, 2} end]] + | --- + | ... +box.schema.func.create('invalidreturn2', {body = lua_code, is_deterministic = true, is_sandboxed = true, opts = {is_multikey = true}}) + | --- + | ... +idx = s:create_index('idx', {func = box.func.invalidreturn2.id, parts = {{1, 'unsigned'}, {2, 'unsigned'}}}) + | --- + | ... +s:insert({1}) + | --- + | - error: 'Key format doesn''t match on defined in the functional index ''idx'' of + | the space ''withdata'': to many values were returned' + | ... +idx:drop() + | --- + | ... + +-- Invalid functional index extractor routine return: the second returned key invalid. +lua_code = [[function(tuple) return {{"hello", "world"}, {1, 2}} end]] + | --- + | ... +box.schema.func.create('invalidreturn3', {body = lua_code, is_deterministic = true, is_sandboxed = true, opts = {is_multikey = true}}) + | --- + | ... +idx = s:create_index('idx', {func = box.func.invalidreturn3.id, parts = {{1, 'unsigned'}, {2, 'unsigned'}}}) + | --- + | ... +s:insert({1}) + | --- + | - error: 'Key format doesn''t match on defined in the functional index ''idx'' of + | the space ''withdata'': Supplied key type of part 0 does not match index part + | type: expected unsigned' + | ... +idx:drop() + | --- + | ... + +-- Invalid functional index extractor routine return: multikey return in case of regular index. +lua_code = [[function(tuple) return {{"hello", "world"}, {1, 2}} end]] + | --- + | ... +box.schema.func.create('invalidreturn4', {body = lua_code, is_deterministic = true, is_sandboxed = true}) + | --- + | ... +idx = s:create_index('idx', {func = box.func.invalidreturn4.id, parts = {{1, 'unsigned'}, {2, 'unsigned'}}}) + | --- + | ... +s:insert({1}) + | --- + | - error: 'Key format doesn''t match on defined in the functional index ''idx'' of + | the space ''withdata'': Supplied key type of part 0 does not match index part + | type: expected unsigned' + | ... +idx:drop() + | --- + | ... + +-- Invalid functional index extractor routine return: invalid return format for multikey index 2. +lua_code = [[function(tuple) return "hello" end]] + | --- + | ... +box.schema.func.create('invalidreturn5', {body = lua_code, is_deterministic = true, is_sandboxed = true, opts = {is_multikey = true}}) + | --- + | ... +idx = s:create_index('idx', {func = box.func.invalidreturn5.id, parts = {{1, 'unsigned'}, {2, 'unsigned'}}}) + | --- + | ... +s:insert({1}) + | --- + | - error: 'Key format doesn''t match on defined in the functional index ''idx'' of + | the space ''withdata'': multikey function mustn''t return scalar' + | ... +idx:drop() + | --- + | ... + +-- Invalid function: runtime extractor error +test_run:cmd("setopt delimiter ';'") + | --- + | - true + | ... +lua_code = [[function(tuple) + local json = require('json') + return json.encode(tuple) + end]] +test_run:cmd("setopt delimiter ''"); + | --- + | ... +box.schema.func.create('runtimeerror', {body = lua_code, is_deterministic = true, is_sandboxed = true}) + | --- + | ... +idx = s:create_index('idx', {func = box.func.runtimeerror.id, parts = {{1, 'string'}}}) + | --- + | ... +s:insert({1}) + | --- + | - error: 'Failed to build a key for functional index ''idx'' of the space ''withdata'': + | [string "return function(tuple) local ..."]:1: attempt to call + | global ''require'' (a nil value)' + | ... +idx:drop() + | --- + | ... + +-- Remove old persistent functions +for _, v in pairs(box.func) do if v.is_persistent then box.schema.func.drop(v.name) end end + | --- + | ... +s:drop() + | --- + | ... + +-- Func index test cases. +s = box.schema.space.create('withdata', {engine = engine}) + | --- + | ... +lua_code = [[function(tuple) return {tuple[1] + tuple[2]} end]] + | --- + | ... +box.schema.func.create('extr', {body = lua_code, is_deterministic = true, is_sandboxed = true}) + | --- + | ... +pk = s:create_index('pk') + | --- + | ... +s:insert({1, 2}) + | --- + | - [1, 2] + | ... +idx = s:create_index('idx', {unique = true, func = 'extr', parts = {{1, 'integer'}}}) + | --- + | ... +s:insert({2, 1}) + | --- + | - error: Duplicate key exists in unique index 'idx' in space 'withdata' + | ... +idx:get(3) + | --- + | - [1, 2] + | ... +idx:delete(3) + | --- + | - [1, 2] + | ... +s:select() + | --- + | - [] + | ... +s:insert({2, 1}) + | --- + | - [2, 1] + | ... +idx:get(3) + | --- + | - [2, 1] + | ... +s:drop() + | --- + | ... +box.schema.func.drop('extr') + | --- + | ... + +-- Multikey functional index. +s = box.schema.space.create('withdata', {engine = engine}) + | --- + | ... +lua_code = [[function(tuple) return {{tuple[1] + tuple[2]}, {tuple[1] + tuple[2]}, {tuple[1]}} end]] + | --- + | ... +box.schema.func.create('extr', {body = lua_code, is_deterministic = true, is_sandboxed = true, opts = {is_multikey = true}}) + | --- + | ... +pk = s:create_index('pk') + | --- + | ... +idx = s:create_index('idx', {unique = true, func = box.func.extr.id, parts = {{1, 'integer'}}}) + | --- + | ... +s:insert({1, 2}) + | --- + | - [1, 2] + | ... +s:insert({3, 5}) + | --- + | - error: Duplicate key exists in unique index 'idx' in space 'withdata' + | ... +s:insert({5, 3}) + | --- + | - [5, 3] + | ... +idx:select() + | --- + | - - [1, 2] + | - [1, 2] + | - [5, 3] + | - [5, 3] + | ... +idx:get(8) + | --- + | - [5, 3] + | ... +idx:get(3) + | --- + | - [1, 2] + | ... +idx:get(1) + | --- + | - [1, 2] + | ... +idx:get(5) + | --- + | - [5, 3] + | ... +s:drop() + | --- + | ... +box.schema.func.drop('extr') + | --- + | ... + +-- Multikey multipart functional index. +s = box.schema.space.create('withdata', {engine = engine}) + | --- + | ... +lua_code = [[function(tuple) return {{600 + tuple[1], 600 + tuple[2]}, {500 + tuple[1], 500 + tuple[2]}} end]] + | --- + | ... +box.schema.func.create('extr', {body = lua_code, is_deterministic = true, is_sandboxed = true, opts = {is_multikey = true}}) + | --- + | ... +pk = s:create_index('pk') + | --- + | ... +idx = s:create_index('idx', {unique = true, func = box.func.extr.id, parts = {{1, 'integer'}, {2, 'integer'}}}) + | --- + | ... +s:insert({1, 2}) + | --- + | - [1, 2] + | ... +s:insert({2, 1}) + | --- + | - [2, 1] + | ... +s:insert({3, 3}) + | --- + | - [3, 3] + | ... +idx:select({600}, {iterator = "GE"}) + | --- + | - - [1, 2] + | - [2, 1] + | - [3, 3] + | ... +idx:get({603, 603}) + | --- + | - [3, 3] + | ... +idx:select({503}, {iterator = "LE"}) + | --- + | - - [3, 3] + | - [2, 1] + | - [1, 2] + | ... +s:drop() + | --- + | ... +box.schema.func.drop('extr') + | --- + | ... + +-- Multikey non-unique functional index. +s = box.schema.space.create('withdata', {engine = engine}) + | --- + | ... +lua_code = [[function(tuple) return {{500 + tuple[1]}, {500 + tuple[2]}, {500 + tuple[2]}, {500 + tuple[2]}} end]] + | --- + | ... +box.schema.func.create('extr', {body = lua_code, is_deterministic = true, is_sandboxed = true, opts = {is_multikey = true}}) + | --- + | ... +pk = s:create_index('pk') + | --- + | ... +idx = s:create_index('idx', {unique = false, func = box.func.extr.id, parts = {{1, 'integer'}}}) + | --- + | ... +s:insert({1, 2}) + | --- + | - [1, 2] + | ... +s:insert({2, 1}) + | --- + | - [2, 1] + | ... +idx:select({501}) + | --- + | - - [1, 2] + | - [2, 1] + | ... +idx:select({502}) + | --- + | - - [1, 2] + | - [2, 1] + | ... +s:replace({1, 3}) + | --- + | - [1, 3] + | ... +idx:select({501}) + | --- + | - - [1, 3] + | - [2, 1] + | ... +idx:select({502}) + | --- + | - - [2, 1] + | ... +idx:select({503}) + | --- + | - - [1, 3] + | ... +box.snapshot() + | --- + | - ok + | ... +test_run:cmd("restart server default") + | +s = box.space.withdata + | --- + | ... +idx = s.index.idx + | --- + | ... +idx:select({501}) + | --- + | - - [1, 3] + | - [2, 1] + | ... +idx:select({502}) + | --- + | - - [2, 1] + | ... +idx:select({503}) + | --- + | - - [1, 3] + | ... +s:replace({1, 2}) + | --- + | - [1, 2] + | ... +idx:select({501}) + | --- + | - - [1, 2] + | - [2, 1] + | ... +idx:select({502}) + | --- + | - - [1, 2] + | - [2, 1] + | ... +idx:select({503}) + | --- + | - [] + | ... +s:drop() + | --- + | ... +box.schema.func.drop('extr') + | --- + | ... + +-- Multikey UTF-8 address extractor +test_run = require('test_run').new() + | --- + | ... +engine = test_run:get_cfg('engine') + | --- + | ... +s = box.schema.space.create('withdata', {engine = engine}) + | --- + | ... +pk = s:create_index('name', {parts = {1, 'string'}}) + | --- + | ... +s:insert({"James", "SIS Building Lambeth London UK"}) + | --- + | - ['James', 'SIS Building Lambeth London UK'] + | ... +s:insert({"Sherlock", "221B Baker St Marylebone London NW1 6XE UK"}) + | --- + | - ['Sherlock', '221B Baker St Marylebone London NW1 6XE UK'] + | ... +-- Create functional index on space with data +test_run:cmd("setopt delimiter ';'") + | --- + | - true + | ... +lua_code = [[function(tuple) + local address = string.split(tuple[2]) + local ret = {} + for _, v in pairs(address) do table.insert(ret, {utf8.upper(v)}) end + return ret + end]] +test_run:cmd("setopt delimiter ''"); + | --- + | ... +box.schema.func.create('addr_extractor', {body = lua_code, is_deterministic = true, is_sandboxed = true, opts = {is_multikey = true}}) + | --- + | ... +idx = s:create_index('addr', {unique = false, func = box.func.addr_extractor.id, parts = {{1, 'string', collation = 'unicode_ci'}}}) + | --- + | ... +idx:select('uk') + | --- + | - - ['James', 'SIS Building Lambeth London UK'] + | - ['Sherlock', '221B Baker St Marylebone London NW1 6XE UK'] + | ... +idx:select('Sis') + | --- + | - - ['James', 'SIS Building Lambeth London UK'] + | ... +s:drop() + | --- + | ... +box.schema.func.drop('addr_extractor') + | --- + | ... + +-- Partial index with functional index extractor +s = box.schema.space.create('withdata', {engine = engine}) + | --- + | ... +pk = s:create_index('pk') + | --- + | ... +lua_code = [[function(tuple) if tuple[1] % 2 == 1 then return {{tuple[1]}} else return {} end end]] + | --- + | ... +box.schema.func.create('extr', {body = lua_code, is_deterministic = true, is_sandboxed = true, opts = {is_multikey = true}}) + | --- + | ... +idx = s:create_index('idx', {unique = true, func = box.func.extr.id, parts = {{1, 'integer'}}}) + | --- + | ... +s:insert({1}) + | --- + | - [1] + | ... +s:insert({2}) + | --- + | - [2] + | ... +s:insert({3}) + | --- + | - [3] + | ... +s:insert({4}) + | --- + | - [4] + | ... +idx:select() + | --- + | - - [1] + | - [3] + | ... +s:drop() + | --- + | ... +box.schema.func.drop('extr') + | --- + | ... + +-- Return nil from functional index extractor. +s = box.schema.space.create('withdata', {engine = engine}) + | --- + | ... +pk = s:create_index('pk') + | --- + | ... +lua_code = [[function(tuple) return {nil} end]] + | --- + | ... +box.schema.func.create('extr', {body = lua_code, is_deterministic = true, is_sandboxed = true}) + | --- + | ... +idx = s:create_index('idx', {unique = false, func = box.func.extr.id, parts = {{1, 'integer', is_nullable = true}}}) + | --- + | ... +s:insert({1}) + | --- + | - error: 'Key format doesn''t match on defined in the functional index ''idx'' of + | the space ''withdata'': Invalid key part count in an exact match (expected 1, + | got 0)' + | ... +s:drop() + | --- + | ... +box.schema.func.drop('extr') + | --- + | ... + +-- Multiple functional indexes. +s = box.schema.space.create('withdata', {engine = engine}) + | --- + | ... +lua_code = [[function(tuple) return {tuple[1] + tuple[2]} end]] + | --- + | ... +box.schema.func.create('s', {body = lua_code, is_deterministic = true, is_sandboxed = true}) + | --- + | ... +lua_code = [[function(tuple) return {tuple[1] - tuple[2]} end]] + | --- + | ... +box.schema.func.create('sub', {body = lua_code, is_deterministic = true, is_sandboxed = true}) + | --- + | ... +pk = s:create_index('pk') + | --- + | ... +idx1 = s:create_index('s_idx', {unique = true, func = box.func.s.id, parts = {{1, 'integer'}}}) + | --- + | ... +idx2 = s:create_index('sub_idx', {unique = true, func = box.func.sub.id, parts = {{1, 'integer'}}}) + | --- + | ... +s:insert({4, 1}) + | --- + | - [4, 1] + | ... +idx1:get(5) + | --- + | - [4, 1] + | ... +idx2:get(3) + | --- + | - [4, 1] + | ... +idx1:drop() + | --- + | ... +idx2:get(3) + | --- + | - [4, 1] + | ... +s:drop() + | --- + | ... +box.schema.func.drop('s') + | --- + | ... +box.schema.func.drop('sub') + | --- + | ... diff --git a/test/engine/func_index.test.lua b/test/engine/func_index.test.lua new file mode 100644 index 000000000..372ec800d --- /dev/null +++ b/test/engine/func_index.test.lua @@ -0,0 +1,250 @@ +test_run = require('test_run').new() +engine = test_run:get_cfg('engine') + +-- +-- gh-1260: Func index. +-- +s = box.schema.space.create('withdata', {engine = engine}) +lua_code = [[function(tuple) return {tuple[1] + tuple[2]} end]] +lua_code2 = [[function(tuple) return {tuple[1] + tuple[2], 2 * tuple[1] + tuple[2]} end]] +box.schema.func.create('s_nonpersistent') +box.schema.func.create('s_ivaliddef1', {body = lua_code, is_deterministic = false, is_sandboxed = true}) +box.schema.func.create('s_ivaliddef2', {body = lua_code, is_deterministic = true, is_sandboxed = false}) + +box.schema.func.create('s', {body = lua_code, is_deterministic = true, is_sandboxed = true}) +box.schema.func.create('ss', {body = lua_code2, is_deterministic = true, is_sandboxed = true}) + +-- Func index can't be primary. +_ = s:create_index('idx', {func = box.func.s.id, parts = {{1, 'unsigned'}}}) +pk = s:create_index('pk') +-- Invalid fid. +_ = s:create_index('idx', {func = 6666, parts = {{1, 'unsigned'}}}) +s.index.idx:drop() +-- Can't use non-persistent function in functional index. +_ = s:create_index('idx', {func = box.func.s_nonpersistent.id, parts = {{1, 'unsigned'}}}) +s.index.idx:drop() +-- Can't use non-deterministic function in functional index. +_ = s:create_index('idx', {func = box.func.s_ivaliddef1.id, parts = {{1, 'unsigned'}}}) +s.index.idx:drop() +-- Can't use non-sandboxed function in functional index. +_ = s:create_index('idx', {func = box.func.s_ivaliddef2.id, parts = {{1, 'unsigned'}}}) +s.index.idx:drop() +-- Can't use non-sequential parts in returned key definition. +_ = s:create_index('idx', {func = box.func.ss.id, parts = {{1, 'unsigned'}, {3, 'unsigned'}}}) +-- Can't use parts started not by 1 field. +_ = s:create_index('idx', {func = box.func.ss.id, parts = {{2, 'unsigned'}, {3, 'unsigned'}}}) +-- Can't use JSON paths in returned key definiton. +_ = s:create_index('idx', {func = box.func.ss.id, parts = {{"[1]data", 'unsigned'}}}) + +-- Can't drop a function referenced by functional index. +idx = s:create_index('idx', {unique = true, func = box.func.s.id, parts = {{1, 'unsigned'}}}) +box.schema.func.drop('s') +box.snapshot() +test_run:cmd("restart server default") +box.schema.func.drop('s') +s = box.space.withdata +idx = s.index.idx +idx:drop() +box.schema.func.drop('s') + +test_run = require('test_run').new() +engine = test_run:get_cfg('engine') + +-- Invalid functional index extractor routine return: the extractor must return keys. +lua_code = [[function(tuple) return "hello" end]] +box.schema.func.create('invalidreturn0', {body = lua_code, is_deterministic = true, is_sandboxed = true}) +idx = s:create_index('idx', {func = box.func.invalidreturn0.id, parts = {{1, 'unsigned'}}}) +s:insert({1}) +idx:drop() + +-- Invalid functional index extractor routine return: a stirng instead of unsigned +lua_code = [[function(tuple) return {"hello"} end]] +box.schema.func.create('invalidreturn1', {body = lua_code, is_deterministic = true, is_sandboxed = true}) +idx = s:create_index('idx', {func = box.func.invalidreturn1.id, parts = {{1, 'unsigned'}}}) +s:insert({1}) +idx:drop() + +-- Invalid functional index extractor routine return: invalid return format for multikey index. +lua_code = [[function(tuple) return {"hello", "world"}, {1, 2} end]] +box.schema.func.create('invalidreturn2', {body = lua_code, is_deterministic = true, is_sandboxed = true, opts = {is_multikey = true}}) +idx = s:create_index('idx', {func = box.func.invalidreturn2.id, parts = {{1, 'unsigned'}, {2, 'unsigned'}}}) +s:insert({1}) +idx:drop() + +-- Invalid functional index extractor routine return: the second returned key invalid. +lua_code = [[function(tuple) return {{"hello", "world"}, {1, 2}} end]] +box.schema.func.create('invalidreturn3', {body = lua_code, is_deterministic = true, is_sandboxed = true, opts = {is_multikey = true}}) +idx = s:create_index('idx', {func = box.func.invalidreturn3.id, parts = {{1, 'unsigned'}, {2, 'unsigned'}}}) +s:insert({1}) +idx:drop() + +-- Invalid functional index extractor routine return: multikey return in case of regular index. +lua_code = [[function(tuple) return {{"hello", "world"}, {1, 2}} end]] +box.schema.func.create('invalidreturn4', {body = lua_code, is_deterministic = true, is_sandboxed = true}) +idx = s:create_index('idx', {func = box.func.invalidreturn4.id, parts = {{1, 'unsigned'}, {2, 'unsigned'}}}) +s:insert({1}) +idx:drop() + +-- Invalid functional index extractor routine return: invalid return format for multikey index 2. +lua_code = [[function(tuple) return "hello" end]] +box.schema.func.create('invalidreturn5', {body = lua_code, is_deterministic = true, is_sandboxed = true, opts = {is_multikey = true}}) +idx = s:create_index('idx', {func = box.func.invalidreturn5.id, parts = {{1, 'unsigned'}, {2, 'unsigned'}}}) +s:insert({1}) +idx:drop() + +-- Invalid function: runtime extractor error +test_run:cmd("setopt delimiter ';'") +lua_code = [[function(tuple) + local json = require('json') + return json.encode(tuple) + end]] +test_run:cmd("setopt delimiter ''"); +box.schema.func.create('runtimeerror', {body = lua_code, is_deterministic = true, is_sandboxed = true}) +idx = s:create_index('idx', {func = box.func.runtimeerror.id, parts = {{1, 'string'}}}) +s:insert({1}) +idx:drop() + +-- Remove old persistent functions +for _, v in pairs(box.func) do if v.is_persistent then box.schema.func.drop(v.name) end end +s:drop() + +-- Func index test cases. +s = box.schema.space.create('withdata', {engine = engine}) +lua_code = [[function(tuple) return {tuple[1] + tuple[2]} end]] +box.schema.func.create('extr', {body = lua_code, is_deterministic = true, is_sandboxed = true}) +pk = s:create_index('pk') +s:insert({1, 2}) +idx = s:create_index('idx', {unique = true, func = 'extr', parts = {{1, 'integer'}}}) +s:insert({2, 1}) +idx:get(3) +idx:delete(3) +s:select() +s:insert({2, 1}) +idx:get(3) +s:drop() +box.schema.func.drop('extr') + +-- Multikey functional index. +s = box.schema.space.create('withdata', {engine = engine}) +lua_code = [[function(tuple) return {{tuple[1] + tuple[2]}, {tuple[1] + tuple[2]}, {tuple[1]}} end]] +box.schema.func.create('extr', {body = lua_code, is_deterministic = true, is_sandboxed = true, opts = {is_multikey = true}}) +pk = s:create_index('pk') +idx = s:create_index('idx', {unique = true, func = box.func.extr.id, parts = {{1, 'integer'}}}) +s:insert({1, 2}) +s:insert({3, 5}) +s:insert({5, 3}) +idx:select() +idx:get(8) +idx:get(3) +idx:get(1) +idx:get(5) +s:drop() +box.schema.func.drop('extr') + +-- Multikey multipart functional index. +s = box.schema.space.create('withdata', {engine = engine}) +lua_code = [[function(tuple) return {{600 + tuple[1], 600 + tuple[2]}, {500 + tuple[1], 500 + tuple[2]}} end]] +box.schema.func.create('extr', {body = lua_code, is_deterministic = true, is_sandboxed = true, opts = {is_multikey = true}}) +pk = s:create_index('pk') +idx = s:create_index('idx', {unique = true, func = box.func.extr.id, parts = {{1, 'integer'}, {2, 'integer'}}}) +s:insert({1, 2}) +s:insert({2, 1}) +s:insert({3, 3}) +idx:select({600}, {iterator = "GE"}) +idx:get({603, 603}) +idx:select({503}, {iterator = "LE"}) +s:drop() +box.schema.func.drop('extr') + +-- Multikey non-unique functional index. +s = box.schema.space.create('withdata', {engine = engine}) +lua_code = [[function(tuple) return {{500 + tuple[1]}, {500 + tuple[2]}, {500 + tuple[2]}, {500 + tuple[2]}} end]] +box.schema.func.create('extr', {body = lua_code, is_deterministic = true, is_sandboxed = true, opts = {is_multikey = true}}) +pk = s:create_index('pk') +idx = s:create_index('idx', {unique = false, func = box.func.extr.id, parts = {{1, 'integer'}}}) +s:insert({1, 2}) +s:insert({2, 1}) +idx:select({501}) +idx:select({502}) +s:replace({1, 3}) +idx:select({501}) +idx:select({502}) +idx:select({503}) +box.snapshot() +test_run:cmd("restart server default") +s = box.space.withdata +idx = s.index.idx +idx:select({501}) +idx:select({502}) +idx:select({503}) +s:replace({1, 2}) +idx:select({501}) +idx:select({502}) +idx:select({503}) +s:drop() +box.schema.func.drop('extr') + +-- Multikey UTF-8 address extractor +test_run = require('test_run').new() +engine = test_run:get_cfg('engine') +s = box.schema.space.create('withdata', {engine = engine}) +pk = s:create_index('name', {parts = {1, 'string'}}) +s:insert({"James", "SIS Building Lambeth London UK"}) +s:insert({"Sherlock", "221B Baker St Marylebone London NW1 6XE UK"}) +-- Create functional index on space with data +test_run:cmd("setopt delimiter ';'") +lua_code = [[function(tuple) + local address = string.split(tuple[2]) + local ret = {} + for _, v in pairs(address) do table.insert(ret, {utf8.upper(v)}) end + return ret + end]] +test_run:cmd("setopt delimiter ''"); +box.schema.func.create('addr_extractor', {body = lua_code, is_deterministic = true, is_sandboxed = true, opts = {is_multikey = true}}) +idx = s:create_index('addr', {unique = false, func = box.func.addr_extractor.id, parts = {{1, 'string', collation = 'unicode_ci'}}}) +idx:select('uk') +idx:select('Sis') +s:drop() +box.schema.func.drop('addr_extractor') + +-- Partial index with functional index extractor +s = box.schema.space.create('withdata', {engine = engine}) +pk = s:create_index('pk') +lua_code = [[function(tuple) if tuple[1] % 2 == 1 then return {{tuple[1]}} else return {} end end]] +box.schema.func.create('extr', {body = lua_code, is_deterministic = true, is_sandboxed = true, opts = {is_multikey = true}}) +idx = s:create_index('idx', {unique = true, func = box.func.extr.id, parts = {{1, 'integer'}}}) +s:insert({1}) +s:insert({2}) +s:insert({3}) +s:insert({4}) +idx:select() +s:drop() +box.schema.func.drop('extr') + +-- Return nil from functional index extractor. +s = box.schema.space.create('withdata', {engine = engine}) +pk = s:create_index('pk') +lua_code = [[function(tuple) return {nil} end]] +box.schema.func.create('extr', {body = lua_code, is_deterministic = true, is_sandboxed = true}) +idx = s:create_index('idx', {unique = false, func = box.func.extr.id, parts = {{1, 'integer', is_nullable = true}}}) +s:insert({1}) +s:drop() +box.schema.func.drop('extr') + +-- Multiple functional indexes. +s = box.schema.space.create('withdata', {engine = engine}) +lua_code = [[function(tuple) return {tuple[1] + tuple[2]} end]] +box.schema.func.create('s', {body = lua_code, is_deterministic = true, is_sandboxed = true}) +lua_code = [[function(tuple) return {tuple[1] - tuple[2]} end]] +box.schema.func.create('sub', {body = lua_code, is_deterministic = true, is_sandboxed = true}) +pk = s:create_index('pk') +idx1 = s:create_index('s_idx', {unique = true, func = box.func.s.id, parts = {{1, 'integer'}}}) +idx2 = s:create_index('sub_idx', {unique = true, func = box.func.sub.id, parts = {{1, 'integer'}}}) +s:insert({4, 1}) +idx1:get(5) +idx2:get(3) +idx1:drop() +idx2:get(3) +s:drop() +box.schema.func.drop('s') +box.schema.func.drop('sub') diff --git a/test/vinyl/misc.result b/test/vinyl/misc.result index b2aacdc55..e647b93c3 100644 --- a/test/vinyl/misc.result +++ b/test/vinyl/misc.result @@ -432,3 +432,26 @@ stat.bytes_compressed < stat.bytes / 10 s:drop() --- ... +-- Vinyl doesn't support functional index. +s = box.schema.space.create('withdata', {engine = 'vinyl'}) +--- +... +lua_code = [[function(tuple) return tuple[1] + tuple[2] end]] +--- +... +box.schema.func.create('s', {body = lua_code, is_deterministic = true, is_sandboxed = true}) +--- +... +_ = s:create_index('pk') +--- +... +_ = s:create_index('idx', {func = box.func.s.id, parts = {{1, 'unsigned'}}}) +--- +- error: Vinyl does not support functional index +... +s:drop() +--- +... +box.schema.func.drop('s') +--- +... diff --git a/test/vinyl/misc.test.lua b/test/vinyl/misc.test.lua index f8da578d0..0a7c7fc99 100644 --- a/test/vinyl/misc.test.lua +++ b/test/vinyl/misc.test.lua @@ -182,3 +182,12 @@ test_run:wait_cond(function() return i:stat().disk.compaction.count > 0 end) stat = i:stat().disk stat.bytes_compressed < stat.bytes / 10 s:drop() + +-- Vinyl doesn't support functional index. +s = box.schema.space.create('withdata', {engine = 'vinyl'}) +lua_code = [[function(tuple) return tuple[1] + tuple[2] end]] +box.schema.func.create('s', {body = lua_code, is_deterministic = true, is_sandboxed = true}) +_ = s:create_index('pk') +_ = s:create_index('idx', {func = box.func.s.id, parts = {{1, 'unsigned'}}}) +s:drop() +box.schema.func.drop('s') diff --git a/test/wal_off/alter.result b/test/wal_off/alter.result index bce15711d..62cb11db7 100644 --- a/test/wal_off/alter.result +++ b/test/wal_off/alter.result @@ -28,7 +28,7 @@ end; ... #spaces; --- -- 65503 +- 65502 ... -- cleanup for k, v in pairs(spaces) do -- 2.22.0