* [PATCH v2 1/5] box: introduce tuple_extra infrastructure
[not found] <cover.1561384554.git.kshcherbatov@tarantool.org>
@ 2019-06-24 14:27 ` Kirill Shcherbatov
2019-06-24 14:27 ` [PATCH v2 2/5] box: introduce key_def->is_multikey flag Kirill Shcherbatov
` (3 subsequent siblings)
4 siblings, 0 replies; 5+ messages in thread
From: Kirill Shcherbatov @ 2019-06-24 14:27 UTC (permalink / raw)
To: tarantool-patches, vdavydov.dev; +Cc: Kirill Shcherbatov
Introduced a new object tuple_extra: a memory allocation is
associated with given tuple and chunk_id. Extended tuple_format's
vtab with few new methods to manage tuple_extras lifecycle.
Supported tuple_extra for memtx engine: a memory
chunks are allocated with memtx's smalloc allocator and are
registered in tuple_exta_cache: an engine-independent hashtable
<tuple *, chunk_id> -> <tuple_extra *>.
Needed for #1260
---
src/box/memtx_engine.c | 43 +++++++++++++++++++++++++
src/box/tuple.c | 65 ++++++++++++++++++++++++++++++++++++++
src/box/tuple.h | 71 ++++++++++++++++++++++++++++++++++++++++++
src/box/tuple_format.h | 22 +++++++++++++
src/box/vy_stmt.c | 3 ++
5 files changed, 204 insertions(+)
diff --git a/src/box/memtx_engine.c b/src/box/memtx_engine.c
index cd763e547..294f4d73d 100644
--- a/src/box/memtx_engine.c
+++ b/src/box/memtx_engine.c
@@ -1199,9 +1199,52 @@ memtx_tuple_delete(struct tuple_format *format, struct tuple *tuple)
smfree_delayed(&memtx->alloc, memtx_tuple, total);
}
+void
+metmx_tuple_extra_delete(struct tuple_format *format,
+ struct tuple_extra *tuple_extra)
+{
+ struct memtx_engine *memtx = (struct memtx_engine *)format->engine;
+ tuple_extra_cache_unregister(tuple_extra);
+ uint32_t sz = tuple_extra_sz(tuple_extra->data_sz);
+ smfree(&memtx->alloc, tuple_extra, sz);
+}
+
+struct tuple_extra *
+memtx_tuple_extra_new(struct tuple_format *format, struct tuple *tuple,
+ uint32_t chunk_id, uint32_t data_sz)
+{
+ struct memtx_engine *memtx = (struct memtx_engine *)format->engine;
+ uint32_t sz = tuple_extra_sz(data_sz);
+ struct tuple_extra *tuple_extra =
+ (struct tuple_extra *) smalloc(&memtx->alloc, sz);
+ if (tuple == NULL) {
+ diag_set(OutOfMemory, sz, "smalloc", "tuple");
+ return NULL;
+ }
+ tuple_extra->tuple = tuple;
+ tuple_extra->chunk_id = chunk_id;
+ tuple_extra->data_sz = data_sz;
+ if (tuple_extra_cache_register(tuple_extra) != 0) {
+ smfree(&memtx->alloc, tuple_extra, sz);
+ return NULL;
+ }
+ return tuple_extra;
+}
+
+static inline struct tuple_extra *
+memtx_tuple_extra_get(struct tuple_format *format, struct tuple *tuple,
+ uint32_t chunk_id)
+{
+ (void) format;
+ return tuple_extra_cache_find(tuple, chunk_id);
+}
+
struct tuple_format_vtab memtx_tuple_format_vtab = {
memtx_tuple_delete,
memtx_tuple_new,
+ metmx_tuple_extra_delete,
+ memtx_tuple_extra_new,
+ memtx_tuple_extra_get,
};
/**
diff --git a/src/box/tuple.c b/src/box/tuple.c
index a7ef332b2..9512543d6 100644
--- a/src/box/tuple.c
+++ b/src/box/tuple.c
@@ -42,6 +42,21 @@
static struct mempool tuple_iterator_pool;
static struct small_alloc runtime_alloc;
+/** Map: (struct tuple *) => (struct tuple_extra *). */
+#define MH_SOURCE 1
+#define mh_name _tuple_extra
+#define mh_key_t struct tuple *
+#define mh_node_t struct tuple_extra *
+#define mh_arg_t uint32_t
+#define mh_hash(a, arg) ((uintptr_t)(*(a))->tuple)
+#define mh_hash_key(a, arg) ((uintptr_t) a)
+#define mh_cmp(a, b, arg) ((*(a))->tuple != (*(b))->tuple || \
+ (*(a))->chunk_id != (*(b))->chunk_id)
+#define mh_cmp_key(a, b, arg) ((a) != (*(b))->tuple || (arg) != (*(b))->chunk_id)
+#include "salad/mhash.h"
+
+static struct mh_tuple_extra_t *tuple_extra_cache;
+
enum {
/** Lowest allowed slab_alloc_minimal */
OBJSIZE_MIN = 16,
@@ -67,6 +82,9 @@ runtime_tuple_new(struct tuple_format *format, const char *data, const char *end
static struct tuple_format_vtab tuple_format_runtime_vtab = {
runtime_tuple_delete,
runtime_tuple_new,
+ NULL,
+ NULL,
+ NULL,
};
static struct tuple *
@@ -301,6 +319,10 @@ tuple_init(field_name_hash_f hash)
if (tuple_format_runtime == NULL)
return -1;
+ tuple_extra_cache = mh_tuple_extra_new();
+ if (tuple_extra_cache == NULL)
+ return -1;
+
/* Make sure this one stays around. */
tuple_format_ref(tuple_format_runtime);
@@ -374,6 +396,9 @@ tuple_free(void)
mempool_destroy(&tuple_iterator_pool);
small_alloc_destroy(&runtime_alloc);
+ assert(mh_size(tuple_extra_cache) == 0);
+ mh_tuple_extra_delete(tuple_extra_cache);
+
tuple_format_free();
coll_id_cache_destroy();
@@ -785,3 +810,43 @@ mp_str(const char *data)
return "<failed to format message pack>";
return buf;
}
+
+uint32_t
+tuple_extra_sz(uint32_t data_sz)
+{
+ return sizeof(struct tuple_extra) + data_sz;
+}
+
+int
+tuple_extra_cache_register(struct tuple_extra *tuple_extra)
+{
+ mh_int_t id = mh_tuple_extra_put(tuple_extra_cache,
+ (const struct tuple_extra **)&tuple_extra,
+ NULL, 0);
+ if (id == mh_end(tuple_extra_cache))
+ return -1;
+ return 0;
+}
+
+void
+tuple_extra_cache_unregister(struct tuple_extra *tuple_extra)
+{
+ struct tuple *tuple = tuple_extra->tuple;
+ uint32_t chunk_id = tuple_extra->chunk_id;
+ mh_int_t id = mh_tuple_extra_find(tuple_extra_cache,
+ tuple, chunk_id);
+ assert(id != mh_end(tuple_extra_cache));
+ mh_tuple_extra_del(tuple_extra_cache, id, chunk_id);
+}
+
+struct tuple_extra *
+tuple_extra_cache_find(struct tuple *tuple, uint32_t chunk_id)
+{
+ mh_int_t id = mh_tuple_extra_find(tuple_extra_cache, tuple, chunk_id);
+ if (id == mh_end(tuple_extra_cache))
+ return NULL;
+ struct tuple_extra **tuple_extra_ptr =
+ mh_tuple_extra_node(tuple_extra_cache, id);
+ assert(tuple_extra_ptr != NULL && *tuple_extra_ptr != NULL);
+ return *tuple_extra_ptr;
+}
diff --git a/src/box/tuple.h b/src/box/tuple.h
index 99dfeb82d..3504c1d95 100644
--- a/src/box/tuple.h
+++ b/src/box/tuple.h
@@ -447,6 +447,77 @@ tuple_delete(struct tuple *tuple)
format->vtab.tuple_delete(format, tuple);
}
+/** Tuple metadata hashtable entry. */
+struct tuple_extra {
+ /** Tuple pointer is used as a hash key. */
+ struct tuple *tuple;
+ /** An extention identifier. */
+ uint32_t chunk_id;
+ /** The payload size. Needed to perform memory release.*/
+ uint32_t data_sz;
+ /** Metadata object payload. */
+ char data[0];
+};
+
+/** Calculate the size of tuple_extra object by given data_sz. */
+uint32_t
+tuple_extra_sz(uint32_t data_sz);
+
+/**
+ * Lookup for tuple_extra by given tuple and chunk_id in the tuple
+ * extra hashtable.
+ */
+struct tuple_extra *
+tuple_extra_cache_find(struct tuple *tuple, uint32_t chunk_id);
+
+/**
+ * Register a new tuple_extra extention in the tuple extra
+ * hashtable.
+ */
+int
+tuple_extra_cache_register(struct tuple_extra *tuple_extra);
+
+/**
+ * Unregister a given tuple_extra extention in the tuple extra
+ * hashtable.
+ */
+void
+tuple_extra_cache_unregister(struct tuple_extra *tuple_extra);
+
+/**
+ * Allocate a new extra allocation for given tuple and
+ * unique chunk_id identifier.
+ */
+static inline struct tuple_extra *
+tuple_extra_new(struct tuple *tuple, uint32_t chunk_id, uint32_t data_sz)
+{
+ struct tuple_format *format = tuple_format(tuple);
+ return format->vtab.tuple_extra_new(format, tuple, chunk_id, data_sz);
+}
+
+/**
+ * Free allocated tuple extra for given tuple and unique
+ * chunk_id identifier.
+ */
+static inline void
+tuple_extra_delete(struct tuple_extra *tuple_extra)
+{
+ struct tuple_format *format =
+ tuple_format(tuple_extra->tuple);
+ format->vtab.tuple_extra_delete(format, tuple_extra);
+}
+
+/**
+ * Get an existent extra allocation for given tuple and
+ * unique chunk_id identifier.
+ */
+static inline struct tuple_extra *
+tuple_extra_get(struct tuple *tuple, uint32_t chunk_id)
+{
+ struct tuple_format *format = tuple_format(tuple);
+ return format->vtab.tuple_extra_get(format, tuple, chunk_id);
+}
+
/**
* Check tuple data correspondence to space format.
* Actually, checks everything that is checked by
diff --git a/src/box/tuple_format.h b/src/box/tuple_format.h
index e4f5f0018..8b37a48d7 100644
--- a/src/box/tuple_format.h
+++ b/src/box/tuple_format.h
@@ -64,6 +64,7 @@ enum { TUPLE_INDEX_BASE = 1 };
enum { TUPLE_OFFSET_SLOT_NIL = INT32_MAX };
struct tuple;
+struct tuple_extra;
struct tuple_format;
struct coll;
@@ -82,6 +83,27 @@ struct tuple_format_vtab {
struct tuple*
(*tuple_new)(struct tuple_format *format, const char *data,
const char *end);
+ /**
+ * Free allocated tuple extra for given tuple and unique
+ * chunk_id identifier.
+ */
+ void
+ (*tuple_extra_delete)(struct tuple_format *format,
+ struct tuple_extra *tuple_extra);
+ /**
+ * Allocate a new extra allocation for given tuple and
+ * unique chunk_id identifier.
+ */
+ struct tuple_extra *
+ (*tuple_extra_new)(struct tuple_format *format, struct tuple *tuple,
+ uint32_t chunk_id, uint32_t data_sz);
+ /**
+ * Get an existent extra allocation for given tuple and
+ * unique chunk_id identifier.
+ */
+ struct tuple_extra *
+ (*tuple_extra_get)(struct tuple_format *format, struct tuple *tuple,
+ uint32_t chunk_id);
};
/** Tuple field meta information for tuple_format. */
diff --git a/src/box/vy_stmt.c b/src/box/vy_stmt.c
index f936cd61f..5c79eac41 100644
--- a/src/box/vy_stmt.c
+++ b/src/box/vy_stmt.c
@@ -118,6 +118,9 @@ vy_stmt_env_create(struct vy_stmt_env *env)
{
env->tuple_format_vtab.tuple_new = vy_tuple_new;
env->tuple_format_vtab.tuple_delete = vy_tuple_delete;
+ env->tuple_format_vtab.tuple_extra_new = NULL;
+ env->tuple_format_vtab.tuple_extra_delete = NULL;
+ env->tuple_format_vtab.tuple_extra_get = NULL;
env->max_tuple_size = 1024 * 1024;
env->key_format = vy_stmt_format_new(env, NULL, 0, NULL, 0, 0, NULL);
if (env->key_format == NULL)
--
2.21.0
^ permalink raw reply [flat|nested] 5+ messages in thread
* [PATCH v2 2/5] box: introduce key_def->is_multikey flag
[not found] <cover.1561384554.git.kshcherbatov@tarantool.org>
2019-06-24 14:27 ` [PATCH v2 1/5] box: introduce tuple_extra infrastructure Kirill Shcherbatov
@ 2019-06-24 14:27 ` Kirill Shcherbatov
2019-06-24 14:27 ` [PATCH v2 3/5] box: refactor key_validate_parts to return key_end Kirill Shcherbatov
` (2 subsequent siblings)
4 siblings, 0 replies; 5+ messages in thread
From: Kirill Shcherbatov @ 2019-06-24 14:27 UTC (permalink / raw)
To: tarantool-patches, vdavydov.dev; +Cc: Kirill Shcherbatov
Previously only key definitions that have JSON paths were able
to define multikey index. We used to check multikey_path != NULL
test to determine whether given key definition is multikey.
In further patches with functional indexes this rule becomes
outdated. Functional index extracted key definition may be
multikey, but has no JSON paths.
So an explicit is_multikey flag was introduced.
Needed for #1260
---
src/box/index_def.c | 2 +-
src/box/key_def.c | 2 ++
src/box/key_def.h | 12 ++++--------
src/box/memtx_bitset.c | 2 +-
src/box/memtx_engine.c | 3 +--
src/box/memtx_rtree.c | 2 +-
src/box/memtx_space.c | 6 +++---
src/box/memtx_tree.c | 4 ++--
src/box/tuple.c | 2 +-
src/box/tuple_bloom.c | 4 ++--
src/box/tuple_compare.cc | 22 +++++++++++-----------
| 12 ++++++------
src/box/tuple_hash.cc | 6 +++---
src/box/vinyl.c | 5 ++---
src/box/vy_stmt.h | 6 +++---
15 files changed, 43 insertions(+), 47 deletions(-)
diff --git a/src/box/index_def.c b/src/box/index_def.c
index 28de89274..eb309a30c 100644
--- a/src/box/index_def.c
+++ b/src/box/index_def.c
@@ -291,7 +291,7 @@ index_def_is_valid(struct index_def *index_def, const char *space_name)
space_name, "too many key parts");
return false;
}
- if (index_def->iid == 0 && key_def_is_multikey(index_def->key_def)) {
+ if (index_def->iid == 0 && index_def->key_def->is_multikey) {
diag_set(ClientError, ER_MODIFY_INDEX, index_def->name,
space_name, "primary key cannot be multikey");
return false;
diff --git a/src/box/key_def.c b/src/box/key_def.c
index eebfb7fe4..2aa095091 100644
--- a/src/box/key_def.c
+++ b/src/box/key_def.c
@@ -181,6 +181,7 @@ key_def_set_part_path(struct key_def *def, uint32_t part_no, const char *path,
def->multikey_path = part->path;
def->multikey_fieldno = part->fieldno;
def->multikey_path_len = (uint32_t) multikey_path_len;
+ def->is_multikey = true;
} else if (def->multikey_fieldno != part->fieldno ||
json_path_cmp(path, multikey_path_len, def->multikey_path,
def->multikey_path_len,
@@ -752,6 +753,7 @@ key_def_merge(const struct key_def *first, const struct key_def *second)
new_def->is_nullable = first->is_nullable || second->is_nullable;
new_def->has_optional_parts = first->has_optional_parts ||
second->has_optional_parts;
+ new_def->is_multikey = first->is_multikey || second->is_multikey;
/* JSON paths data in the new key_def. */
char *path_pool = (char *)new_def + key_def_sizeof(new_part_count, 0);
diff --git a/src/box/key_def.h b/src/box/key_def.h
index f4a1a8fd1..ab4b7c087 100644
--- a/src/box/key_def.h
+++ b/src/box/key_def.h
@@ -196,6 +196,8 @@ struct key_def {
bool is_nullable;
/** True if some key part has JSON path. */
bool has_json_paths;
+ /** True if it is multikey key definition. */
+ bool is_multikey;
/**
* True, if some key parts can be absent in a tuple. These
* fields assumed to be MP_NIL.
@@ -220,14 +222,14 @@ struct key_def {
const char *multikey_path;
/**
* The length of the key_def::multikey_path.
- * Valid when key_def_is_multikey(key_def) is true,
+ * Valid when key_def->is_multikey is true,
* undefined otherwise.
*/
uint32_t multikey_path_len;
/**
* The index of the root field of the multikey JSON
* path index key_def::multikey_path.
- * Valid when key_def_is_multikey(key_def) is true,
+ * Valid when key_def->is_multikey is true,
* undefined otherwise.
*/
uint32_t multikey_fieldno;
@@ -469,12 +471,6 @@ key_def_is_sequential(const struct key_def *key_def)
return true;
}
-static inline bool
-key_def_is_multikey(const struct key_def *key_def)
-{
- return key_def->multikey_path != NULL;
-}
-
/**
* Return true if @a key_def defines has fields that requires
* special collation comparison.
diff --git a/src/box/memtx_bitset.c b/src/box/memtx_bitset.c
index 8c626684c..59bc96427 100644
--- a/src/box/memtx_bitset.c
+++ b/src/box/memtx_bitset.c
@@ -276,7 +276,7 @@ memtx_bitset_index_replace(struct index *base, struct tuple *old_tuple,
struct memtx_bitset_index *index = (struct memtx_bitset_index *)base;
assert(!base->def->opts.is_unique);
- assert(!key_def_is_multikey(base->def->key_def));
+ assert(!base->def->key_def->is_multikey);
assert(old_tuple != NULL || new_tuple != NULL);
(void) mode;
diff --git a/src/box/memtx_engine.c b/src/box/memtx_engine.c
index 294f4d73d..3dd8d7f4d 100644
--- a/src/box/memtx_engine.c
+++ b/src/box/memtx_engine.c
@@ -1367,7 +1367,6 @@ memtx_index_def_change_requires_rebuild(struct index *index,
TUPLE_INDEX_BASE) != 0)
return true;
}
- assert(key_def_is_multikey(old_cmp_def) ==
- key_def_is_multikey(new_cmp_def));
+ assert(old_cmp_def->is_multikey == new_cmp_def->is_multikey);
return false;
}
diff --git a/src/box/memtx_rtree.c b/src/box/memtx_rtree.c
index b91a405d7..8badad797 100644
--- a/src/box/memtx_rtree.c
+++ b/src/box/memtx_rtree.c
@@ -120,7 +120,7 @@ extract_rectangle(struct rtree_rect *rect, struct tuple *tuple,
struct index_def *index_def)
{
assert(index_def->key_def->part_count == 1);
- assert(!key_def_is_multikey(index_def->key_def));
+ assert(!index_def->key_def->is_multikey);
const char *elems = tuple_field_by_part(tuple,
index_def->key_def->parts, MULTIKEY_NONE);
unsigned dimension = index_def->opts.dimension;
diff --git a/src/box/memtx_space.c b/src/box/memtx_space.c
index 78a0059a0..01091dcdc 100644
--- a/src/box/memtx_space.c
+++ b/src/box/memtx_space.c
@@ -653,7 +653,7 @@ memtx_space_check_index_def(struct space *space, struct index_def *index_def)
"HASH index must be unique");
return -1;
}
- if (key_def_is_multikey(index_def->key_def)) {
+ if (index_def->key_def->is_multikey) {
diag_set(ClientError, ER_MODIFY_INDEX,
index_def->name, space_name(space),
"HASH index cannot be multikey");
@@ -682,7 +682,7 @@ memtx_space_check_index_def(struct space *space, struct index_def *index_def)
"RTREE index field type must be ARRAY");
return -1;
}
- if (key_def_is_multikey(index_def->key_def)) {
+ if (index_def->key_def->is_multikey) {
diag_set(ClientError, ER_MODIFY_INDEX,
index_def->name, space_name(space),
"RTREE index cannot be multikey");
@@ -710,7 +710,7 @@ memtx_space_check_index_def(struct space *space, struct index_def *index_def)
"BITSET index field type must be NUM or STR");
return -1;
}
- if (key_def_is_multikey(index_def->key_def)) {
+ if (index_def->key_def->is_multikey) {
diag_set(ClientError, ER_MODIFY_INDEX,
index_def->name, space_name(space),
"BITSET index cannot be multikey");
diff --git a/src/box/memtx_tree.c b/src/box/memtx_tree.c
index 268ce4050..3edf94044 100644
--- a/src/box/memtx_tree.c
+++ b/src/box/memtx_tree.c
@@ -882,7 +882,7 @@ memtx_tree_index_end_build(struct index *base)
struct key_def *cmp_def = memtx_tree_cmp_def(&index->tree);
qsort_arg(index->build_array, index->build_array_size,
sizeof(index->build_array[0]), memtx_tree_qcompare, cmp_def);
- if (key_def_is_multikey(cmp_def)) {
+ if (cmp_def->is_multikey) {
/*
* Multikey index may have equal(in terms of
* cmp_def) keys inserted by different multikey
@@ -1027,7 +1027,7 @@ memtx_tree_index_new(struct memtx_engine *memtx, struct index_def *def)
"malloc", "struct memtx_tree_index");
return NULL;
}
- const struct index_vtab *vtab = key_def_is_multikey(def->key_def) ?
+ const struct index_vtab *vtab = def->key_def->is_multikey ?
&memtx_tree_index_multikey_vtab :
&memtx_tree_index_vtab;
if (index_create(&index->base, (struct engine *)memtx,
diff --git a/src/box/tuple.c b/src/box/tuple.c
index 9512543d6..5fff868ac 100644
--- a/src/box/tuple.c
+++ b/src/box/tuple.c
@@ -578,7 +578,7 @@ tuple_raw_multikey_count(struct tuple_format *format, const char *data,
const uint32_t *field_map,
struct key_def *key_def)
{
- assert(key_def_is_multikey(key_def));
+ assert(key_def->is_multikey);
const char *array_raw =
tuple_field_raw_by_path(format, data, field_map,
key_def->multikey_fieldno,
diff --git a/src/box/tuple_bloom.c b/src/box/tuple_bloom.c
index 8b74da22c..420a7c60e 100644
--- a/src/box/tuple_bloom.c
+++ b/src/box/tuple_bloom.c
@@ -109,7 +109,7 @@ tuple_bloom_builder_add(struct tuple_bloom_builder *builder,
int multikey_idx)
{
assert(builder->part_count == key_def->part_count);
- assert(!key_def_is_multikey(key_def) || multikey_idx != MULTIKEY_NONE);
+ assert(!key_def->is_multikey || multikey_idx != MULTIKEY_NONE);
uint32_t h = HASH_SEED;
uint32_t carry = 0;
@@ -204,7 +204,7 @@ bool
tuple_bloom_maybe_has(const struct tuple_bloom *bloom, struct tuple *tuple,
struct key_def *key_def, int multikey_idx)
{
- assert(!key_def_is_multikey(key_def) || multikey_idx != MULTIKEY_NONE);
+ assert(!key_def->is_multikey || multikey_idx != MULTIKEY_NONE);
if (bloom->is_legacy) {
return bloom_maybe_has(&bloom->parts[0],
diff --git a/src/box/tuple_compare.cc b/src/box/tuple_compare.cc
index c1a70a087..3e0d22dec 100644
--- a/src/box/tuple_compare.cc
+++ b/src/box/tuple_compare.cc
@@ -456,9 +456,9 @@ tuple_compare_slowpath(struct tuple *tuple_a, hint_t tuple_a_hint,
assert(!has_optional_parts || is_nullable);
assert(is_nullable == key_def->is_nullable);
assert(has_optional_parts == key_def->has_optional_parts);
- assert(key_def_is_multikey(key_def) == is_multikey);
- assert(!is_multikey || (is_multikey == has_json_paths &&
- tuple_a_hint != HINT_NONE && tuple_b_hint != HINT_NONE));
+ assert(key_def->is_multikey == is_multikey);
+ assert(!is_multikey || (tuple_a_hint != HINT_NONE &&
+ tuple_b_hint != HINT_NONE));
int rc = 0;
if (!is_multikey && (rc = hint_cmp(tuple_a_hint, tuple_b_hint)) != 0)
return rc;
@@ -615,9 +615,9 @@ tuple_compare_with_key_slowpath(struct tuple *tuple, hint_t tuple_hint,
assert(has_optional_parts == key_def->has_optional_parts);
assert(key != NULL || part_count == 0);
assert(part_count <= key_def->part_count);
- assert(key_def_is_multikey(key_def) == is_multikey);
- assert(!is_multikey || (is_multikey == has_json_paths &&
- tuple_hint != HINT_NONE && key_hint == HINT_NONE));
+ assert(key_def->is_multikey == is_multikey);
+ assert(!is_multikey || (tuple_hint != HINT_NONE &&
+ key_hint == HINT_NONE));
int rc = 0;
if (!is_multikey && (rc = hint_cmp(tuple_hint, key_hint)) != 0)
return rc;
@@ -1559,7 +1559,7 @@ template <enum field_type type, bool is_nullable>
static hint_t
key_hint(const char *key, uint32_t part_count, struct key_def *key_def)
{
- assert(!key_def_is_multikey(key_def));
+ assert(!key_def->is_multikey);
if (part_count == 0)
return HINT_NONE;
return field_hint<type, is_nullable>(key, key_def->parts->coll);
@@ -1569,7 +1569,7 @@ template <enum field_type type, bool is_nullable>
static hint_t
tuple_hint(struct tuple *tuple, struct key_def *key_def)
{
- assert(!key_def_is_multikey(key_def));
+ assert(!key_def->is_multikey);
const char *field = tuple_field_by_part(tuple, key_def->parts,
MULTIKEY_NONE);
if (is_nullable && field == NULL)
@@ -1593,7 +1593,7 @@ key_hint_multikey(const char *key, uint32_t part_count, struct key_def *key_def)
* do nothing on key hint calculation an it is valid
* because it is never used(unlike tuple hint).
*/
- assert(key_def_is_multikey(key_def));
+ assert(key_def->is_multikey);
return HINT_NONE;
}
@@ -1627,7 +1627,7 @@ key_def_set_hint_func(struct key_def *def)
static void
key_def_set_hint_func(struct key_def *def)
{
- if (key_def_is_multikey(def)) {
+ if (def->is_multikey) {
def->key_hint = key_hint_multikey;
def->tuple_hint = tuple_hint_multikey;
return;
@@ -1739,7 +1739,7 @@ static void
key_def_set_compare_func_json(struct key_def *def)
{
assert(def->has_json_paths);
- if (key_def_is_multikey(def)) {
+ if (def->is_multikey) {
def->tuple_compare = tuple_compare_slowpath
<is_nullable, has_optional_parts, true, true>;
def->tuple_compare_with_key = tuple_compare_with_key_slowpath
--git a/src/box/tuple_extract_key.cc b/src/box/tuple_extract_key.cc
index 3bd3cde70..471c7df80 100644
--- a/src/box/tuple_extract_key.cc
+++ b/src/box/tuple_extract_key.cc
@@ -118,8 +118,8 @@ tuple_extract_key_slowpath(struct tuple *tuple, struct key_def *key_def,
assert(has_optional_parts == key_def->has_optional_parts);
assert(contains_sequential_parts ==
key_def_contains_sequential_parts(key_def));
- assert(is_multikey == key_def_is_multikey(key_def));
- assert(!key_def_is_multikey(key_def) || multikey_idx != MULTIKEY_NONE);
+ assert(is_multikey == key_def->is_multikey);
+ assert(!key_def->is_multikey || multikey_idx != MULTIKEY_NONE);
assert(mp_sizeof_nil() == 1);
const char *data = tuple_data(tuple);
uint32_t part_count = key_def->part_count;
@@ -250,7 +250,7 @@ tuple_extract_key_slowpath_raw(const char *data, const char *data_end,
assert(has_json_paths == key_def->has_json_paths);
assert(!has_optional_parts || key_def->is_nullable);
assert(has_optional_parts == key_def->has_optional_parts);
- assert(!key_def_is_multikey(key_def) || multikey_idx != MULTIKEY_NONE);
+ assert(!key_def->is_multikey || multikey_idx != MULTIKEY_NONE);
assert(mp_sizeof_nil() == 1);
/* allocate buffer with maximal possible size */
char *key = (char *) region_alloc(&fiber()->gc, data_end - data);
@@ -366,7 +366,7 @@ static void
key_def_set_extract_func_plain(struct key_def *def)
{
assert(!def->has_json_paths);
- assert(!key_def_is_multikey(def));
+ assert(!def->is_multikey);
if (key_def_is_sequential(def)) {
assert(contains_sequential_parts || def->part_count == 1);
def->tuple_extract_key = tuple_extract_key_sequential
@@ -387,7 +387,7 @@ static void
key_def_set_extract_func_json(struct key_def *def)
{
assert(def->has_json_paths);
- if (key_def_is_multikey(def)) {
+ if (def->is_multikey) {
def->tuple_extract_key = tuple_extract_key_slowpath
<contains_sequential_parts,
has_optional_parts, true, true>;
@@ -452,7 +452,7 @@ tuple_key_contains_null(struct tuple *tuple, struct key_def *def,
int
tuple_validate_key_parts(struct key_def *key_def, struct tuple *tuple)
{
- assert(!key_def_is_multikey(key_def));
+ assert(!key_def->is_multikey);
for (uint32_t idx = 0; idx < key_def->part_count; idx++) {
struct key_part *part = &key_def->parts[idx];
const char *field = tuple_field_by_part(tuple, part,
diff --git a/src/box/tuple_hash.cc b/src/box/tuple_hash.cc
index 223c5f6f4..780e3d053 100644
--- a/src/box/tuple_hash.cc
+++ b/src/box/tuple_hash.cc
@@ -155,7 +155,7 @@ struct TupleHash
{
static uint32_t hash(struct tuple *tuple, struct key_def *key_def)
{
- assert(!key_def_is_multikey(key_def));
+ assert(!key_def->is_multikey);
uint32_t h = HASH_SEED;
uint32_t carry = 0;
uint32_t total_size = 0;
@@ -172,7 +172,7 @@ template <>
struct TupleHash<FIELD_TYPE_UNSIGNED> {
static uint32_t hash(struct tuple *tuple, struct key_def *key_def)
{
- assert(!key_def_is_multikey(key_def));
+ assert(!key_def->is_multikey);
const char *field = tuple_field_by_part(tuple,
key_def->parts,
MULTIKEY_NONE);
@@ -364,7 +364,7 @@ tuple_hash_slowpath(struct tuple *tuple, struct key_def *key_def)
{
assert(has_json_paths == key_def->has_json_paths);
assert(has_optional_parts == key_def->has_optional_parts);
- assert(!key_def_is_multikey(key_def));
+ assert(!key_def->is_multikey);
uint32_t h = HASH_SEED;
uint32_t carry = 0;
uint32_t total_size = 0;
diff --git a/src/box/vinyl.c b/src/box/vinyl.c
index 814325da5..3e686b080 100644
--- a/src/box/vinyl.c
+++ b/src/box/vinyl.c
@@ -1018,8 +1018,7 @@ vinyl_index_def_change_requires_rebuild(struct index *index,
TUPLE_INDEX_BASE) != 0)
return true;
}
- assert(key_def_is_multikey(old_cmp_def) ==
- key_def_is_multikey(new_cmp_def));
+ assert(old_cmp_def->is_multikey == new_cmp_def->is_multikey);
return false;
}
@@ -1640,7 +1639,7 @@ vy_check_is_unique_secondary(struct vy_tx *tx, const struct vy_read_view **rv,
{
if (!lsm->check_is_unique)
return 0;
- if (!key_def_is_multikey(lsm->cmp_def)) {
+ if (!lsm->cmp_def->is_multikey) {
return vy_check_is_unique_secondary_one(tx, rv,
space_name, index_name, lsm, stmt,
MULTIKEY_NONE);
diff --git a/src/box/vy_stmt.h b/src/box/vy_stmt.h
index a80456add..25219230d 100644
--- a/src/box/vy_stmt.h
+++ b/src/box/vy_stmt.h
@@ -696,7 +696,7 @@ vy_stmt_str(struct tuple *stmt);
static inline int
vy_entry_multikey_idx(struct vy_entry entry, struct key_def *key_def)
{
- if (!key_def_is_multikey(key_def) || vy_stmt_is_key(entry.stmt))
+ if (!key_def->is_multikey || vy_stmt_is_key(entry.stmt))
return MULTIKEY_NONE;
assert(entry.hint != HINT_NONE);
return (int)entry.hint;
@@ -766,11 +766,11 @@ vy_entry_compare_with_raw_key(struct vy_entry entry,
*/
#define vy_stmt_foreach_entry(entry, src_stmt, key_def) \
for (uint32_t multikey_idx = 0, \
- multikey_count = !key_def_is_multikey((key_def)) ? 1 : \
+ multikey_count = !(key_def)->is_multikey ? 1 : \
tuple_multikey_count((src_stmt), (key_def)); \
multikey_idx < multikey_count && \
(((entry).stmt = (src_stmt)), \
- ((entry).hint = !key_def_is_multikey((key_def)) ? \
+ ((entry).hint = !(key_def)->is_multikey ? \
vy_stmt_hint((src_stmt), (key_def)) : \
multikey_idx), true); \
++multikey_idx)
--
2.21.0
^ permalink raw reply [flat|nested] 5+ messages in thread
* [PATCH v2 3/5] box: refactor key_validate_parts to return key_end
[not found] <cover.1561384554.git.kshcherbatov@tarantool.org>
2019-06-24 14:27 ` [PATCH v2 1/5] box: introduce tuple_extra infrastructure Kirill Shcherbatov
2019-06-24 14:27 ` [PATCH v2 2/5] box: introduce key_def->is_multikey flag Kirill Shcherbatov
@ 2019-06-24 14:27 ` Kirill Shcherbatov
2019-06-24 14:27 ` [PATCH v2 4/5] box: move the function hash to the tuple library Kirill Shcherbatov
2019-06-24 14:27 ` [PATCH v2 5/5] box: introduce functional indexes in memtx Kirill Shcherbatov
4 siblings, 0 replies; 5+ messages in thread
From: Kirill Shcherbatov @ 2019-06-24 14:27 UTC (permalink / raw)
To: tarantool-patches, vdavydov.dev; +Cc: Kirill Shcherbatov
The key_validate_parts helper is refactored to return a pointer
to the end of a given key argument in case of success.
This is required to effectively validate a sequence of keys in
scope of functional multikey indexes.
Needed for #1260
---
src/box/index.cc | 10 ++++++----
src/box/index.h | 3 ++-
src/box/key_def.c | 4 +++-
src/box/key_def.h | 4 +++-
src/box/lua/key_def.c | 5 +++--
src/box/memtx_space.c | 10 ++++++----
src/box/space.c | 4 ++--
src/box/sysview.c | 4 +++-
src/box/vinyl.c | 4 +++-
9 files changed, 31 insertions(+), 17 deletions(-)
diff --git a/src/box/index.cc b/src/box/index.cc
index 4a444e5d0..b5419cc70 100644
--- a/src/box/index.cc
+++ b/src/box/index.cc
@@ -126,8 +126,9 @@ key_validate(const struct index_def *index_def, enum iterator_type type,
part_count);
return -1;
}
+ const char *key_end;
if (key_validate_parts(index_def->key_def, key,
- part_count, true) != 0)
+ part_count, true, &key_end) != 0)
return -1;
}
return 0;
@@ -135,7 +136,7 @@ key_validate(const struct index_def *index_def, enum iterator_type type,
int
exact_key_validate(struct key_def *key_def, const char *key,
- uint32_t part_count)
+ uint32_t part_count, const char **key_end)
{
assert(key != NULL || part_count == 0);
if (key_def->part_count != part_count) {
@@ -143,7 +144,7 @@ exact_key_validate(struct key_def *key_def, const char *key,
part_count);
return -1;
}
- return key_validate_parts(key_def, key, part_count, false);
+ return key_validate_parts(key_def, key, part_count, false, key_end);
}
char *
@@ -233,7 +234,8 @@ box_index_get(uint32_t space_id, uint32_t index_id, const char *key,
return -1;
}
uint32_t part_count = mp_decode_array(&key);
- if (exact_key_validate(index->def->key_def, key, part_count))
+ if (exact_key_validate(index->def->key_def, key, part_count,
+ &key_end))
return -1;
/* Start transaction in the engine. */
struct txn *txn;
diff --git a/src/box/index.h b/src/box/index.h
index 97d600c96..0d7ddc16a 100644
--- a/src/box/index.h
+++ b/src/box/index.h
@@ -319,12 +319,13 @@ key_validate(const struct index_def *index_def, enum iterator_type type,
/**
* Check that the supplied key is valid for a search in a unique
* index (i.e. the key must be fully specified).
+ * Update the pointer key_end to the end of the validated key.
* @retval 0 The key is valid.
* @retval -1 The key is invalid.
*/
int
exact_key_validate(struct key_def *key_def, const char *key,
- uint32_t part_count);
+ uint32_t part_count, const char **key_end);
/**
* The manner in which replace in a unique index must treat
diff --git a/src/box/key_def.c b/src/box/key_def.c
index 2aa095091..ee758eefa 100644
--- a/src/box/key_def.c
+++ b/src/box/key_def.c
@@ -834,7 +834,8 @@ out:
int
key_validate_parts(const struct key_def *key_def, const char *key,
- uint32_t part_count, bool allow_nullable)
+ uint32_t part_count, bool allow_nullable,
+ const char **key_end)
{
for (uint32_t i = 0; i < part_count; i++) {
const struct key_part *part = &key_def->parts[i];
@@ -844,5 +845,6 @@ key_validate_parts(const struct key_def *key_def, const char *key,
return -1;
mp_next(&key);
}
+ *key_end = key;
return 0;
}
diff --git a/src/box/key_def.h b/src/box/key_def.h
index ab4b7c087..df83d055c 100644
--- a/src/box/key_def.h
+++ b/src/box/key_def.h
@@ -443,13 +443,15 @@ key_def_find_pk_in_cmp_def(const struct key_def *cmp_def,
* @param key MessagePack'ed data for matching.
* @param part_count Field count in the key.
* @param allow_nullable True if nullable parts are allowed.
+ * @param key_end[out] The end of the validated key.
*
* @retval 0 The key is valid.
* @retval -1 The key is invalid.
*/
int
key_validate_parts(const struct key_def *key_def, const char *key,
- uint32_t part_count, bool allow_nullable);
+ uint32_t part_count, bool allow_nullable,
+ const char **key_end);
/**
* Return true if @a index_def defines a sequential key without
diff --git a/src/box/lua/key_def.c b/src/box/lua/key_def.c
index dfcc89442..810ce5375 100644
--- a/src/box/lua/key_def.c
+++ b/src/box/lua/key_def.c
@@ -337,9 +337,10 @@ lbox_key_def_compare_with_key(struct lua_State *L)
struct region *region = &fiber()->gc;
size_t region_svp = region_used(region);
size_t key_len;
- const char *key = lbox_encode_tuple_on_gc(L, 3, &key_len);
+ const char *key_end, *key = lbox_encode_tuple_on_gc(L, 3, &key_len);
uint32_t part_count = mp_decode_array(&key);
- if (key_validate_parts(key_def, key, part_count, true) != 0) {
+ if (key_validate_parts(key_def, key, part_count, true,
+ &key_end) != 0) {
region_truncate(region, region_svp);
tuple_unref(tuple);
return luaT_error(L);
diff --git a/src/box/memtx_space.c b/src/box/memtx_space.c
index 01091dcdc..15492a09f 100644
--- a/src/box/memtx_space.c
+++ b/src/box/memtx_space.c
@@ -376,9 +376,10 @@ memtx_space_execute_delete(struct space *space, struct txn *txn,
struct index *pk = index_find_unique(space, request->index_id);
if (pk == NULL)
return -1;
- const char *key = request->key;
+ const char *key_end, *key = request->key;
uint32_t part_count = mp_decode_array(&key);
- if (exact_key_validate(pk->def->key_def, key, part_count) != 0)
+ if (exact_key_validate(pk->def->key_def, key, part_count,
+ &key_end) != 0)
return -1;
struct tuple *old_tuple;
if (index_get(pk, key, part_count, &old_tuple) != 0)
@@ -402,9 +403,10 @@ memtx_space_execute_update(struct space *space, struct txn *txn,
struct index *pk = index_find_unique(space, request->index_id);
if (pk == NULL)
return -1;
- const char *key = request->key;
+ const char *key_end, *key = request->key;
uint32_t part_count = mp_decode_array(&key);
- if (exact_key_validate(pk->def->key_def, key, part_count) != 0)
+ if (exact_key_validate(pk->def->key_def, key, part_count,
+ &key_end) != 0)
return -1;
struct tuple *old_tuple;
if (index_get(pk, key, part_count, &old_tuple) != 0)
diff --git a/src/box/space.c b/src/box/space.c
index b6ad87bf7..e9196284f 100644
--- a/src/box/space.c
+++ b/src/box/space.c
@@ -298,7 +298,7 @@ space_before_replace(struct space *space, struct txn *txn,
enum iproto_type type = request->type;
struct index *pk = space_index(space, 0);
- const char *key = NULL;
+ const char *key_end, *key = NULL;
uint32_t part_count = 0;
struct index *index = NULL;
@@ -314,7 +314,7 @@ space_before_replace(struct space *space, struct txn *txn,
key = request->key;
part_count = mp_decode_array(&key);
if (exact_key_validate(index->def->key_def,
- key, part_count) != 0)
+ key, part_count, &key_end) != 0)
return -1;
break;
case IPROTO_INSERT:
diff --git a/src/box/sysview.c b/src/box/sysview.c
index 46cf1e13f..9c9a2bc89 100644
--- a/src/box/sysview.c
+++ b/src/box/sysview.c
@@ -164,7 +164,9 @@ sysview_index_get(struct index *base, const char *key,
diag_set(ClientError, ER_MORE_THAN_ONE_TUPLE);
return -1;
}
- if (exact_key_validate(pk->def->key_def, key, part_count) != 0)
+ const char *key_end;
+ if (exact_key_validate(pk->def->key_def, key, part_count,
+ &key_end) != 0)
return -1;
struct tuple *tuple;
if (index_get(pk, key, part_count, &tuple) != 0)
diff --git a/src/box/vinyl.c b/src/box/vinyl.c
index 3e686b080..a03132310 100644
--- a/src/box/vinyl.c
+++ b/src/box/vinyl.c
@@ -1748,7 +1748,9 @@ vy_unique_key_validate(struct vy_lsm *lsm, const char *key,
original_part_count, part_count);
return -1;
}
- return key_validate_parts(lsm->cmp_def, key, part_count, false);
+ const char *key_end;
+ return key_validate_parts(lsm->cmp_def, key, part_count, false,
+ &key_end);
}
/**
--
2.21.0
^ permalink raw reply [flat|nested] 5+ messages in thread
* [PATCH v2 4/5] box: move the function hash to the tuple library
[not found] <cover.1561384554.git.kshcherbatov@tarantool.org>
` (2 preceding siblings ...)
2019-06-24 14:27 ` [PATCH v2 3/5] box: refactor key_validate_parts to return key_end Kirill Shcherbatov
@ 2019-06-24 14:27 ` Kirill Shcherbatov
2019-06-24 14:27 ` [PATCH v2 5/5] box: introduce functional indexes in memtx Kirill Shcherbatov
4 siblings, 0 replies; 5+ messages in thread
From: Kirill Shcherbatov @ 2019-06-24 14:27 UTC (permalink / raw)
To: tarantool-patches, vdavydov.dev; +Cc: Kirill Shcherbatov
The functions hashtable moved to the tuple library. It is
required in scope of functional indexes to perform a lookup
function by it's identifier in key_def module that is a part
of tuple library.
Needed for #1260
---
src/box/CMakeLists.txt | 1 +
src/box/func_cache.c | 124 +++++++++++++++++++++++++++++++++++++++++
src/box/func_cache.h | 78 ++++++++++++++++++++++++++
src/box/schema.cc | 69 -----------------------
src/box/schema.h | 19 +------
src/box/tuple.c | 6 ++
6 files changed, 210 insertions(+), 87 deletions(-)
create mode 100644 src/box/func_cache.c
create mode 100644 src/box/func_cache.h
diff --git a/src/box/CMakeLists.txt b/src/box/CMakeLists.txt
index 481842a39..7da2d6cff 100644
--- a/src/box/CMakeLists.txt
+++ b/src/box/CMakeLists.txt
@@ -45,6 +45,7 @@ add_library(tuple STATIC
tuple_hash.cc
tuple_bloom.c
tuple_dictionary.c
+ func_cache.c
key_def.c
coll_id_def.c
coll_id.c
diff --git a/src/box/func_cache.c b/src/box/func_cache.c
new file mode 100644
index 000000000..05545eb6b
--- /dev/null
+++ b/src/box/func_cache.c
@@ -0,0 +1,124 @@
+/*
+ * Copyright 2010-2019, Tarantool AUTHORS, please see AUTHORS file.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the
+ * following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY <COPYRIGHT HOLDER> ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+ * <COPYRIGHT HOLDER> OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+ * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+#include "func_cache.h"
+#include "assoc.h"
+#include "diag.h"
+#include "errcode.h"
+#include "error.h"
+#include "func.h"
+#include "schema_def.h"
+
+/** mhash table (name, len -> collation) */
+static struct mh_i32ptr_t *func_id_cache;
+/** mhash table (id -> collation) */
+static struct mh_strnptr_t *func_name_cache;
+
+int
+func_cache_init(void)
+{
+ func_id_cache = mh_i32ptr_new();
+ if (func_id_cache == NULL) {
+ diag_set(OutOfMemory, sizeof(*func_id_cache), "malloc",
+ "func_id_cache");
+ return -1;
+ }
+ func_name_cache = mh_strnptr_new();
+ if (func_name_cache == NULL) {
+ diag_set(OutOfMemory, sizeof(*func_name_cache), "malloc",
+ "func_name_cache");
+ mh_i32ptr_delete(func_id_cache);
+ return -1;
+ }
+ return 0;
+}
+
+void
+func_cache_destroy(void)
+{
+ mh_strnptr_delete(func_name_cache);
+ mh_i32ptr_delete(func_id_cache);
+}
+
+void
+func_cache_insert(struct func *func)
+{
+ assert(func_by_id(func->def->fid) == NULL);
+ assert(func_by_name(func->def->name, strlen(func->def->name)) == NULL);
+ const struct mh_i32ptr_node_t node = { func->def->fid, func };
+ mh_int_t k1 = mh_i32ptr_put(func_id_cache, &node, NULL, NULL);
+ if (k1 == mh_end(func_id_cache)) {
+error:
+ panic_syserror("Out of memory for the data "
+ "dictionary cache (stored function).");
+ }
+ size_t def_name_len = strlen(func->def->name);
+ uint32_t name_hash = mh_strn_hash(func->def->name, def_name_len);
+ const struct mh_strnptr_node_t strnode = {
+ func->def->name, def_name_len, name_hash, func };
+ mh_int_t k2 = mh_strnptr_put(func_name_cache, &strnode, NULL, NULL);
+ if (k2 == mh_end(func_name_cache)) {
+ mh_i32ptr_del(func_id_cache, k1, NULL);
+ goto error;
+ }
+}
+
+void
+func_cache_delete(uint32_t fid)
+{
+ mh_int_t k = mh_i32ptr_find(func_id_cache, fid, NULL);
+ if (k == mh_end(func_id_cache))
+ return;
+ struct func *func = (struct func *)
+ mh_i32ptr_node(func_id_cache, k)->val;
+ mh_i32ptr_del(func_id_cache, k, NULL);
+ k = mh_strnptr_find_inp(func_name_cache, func->def->name,
+ strlen(func->def->name));
+ if (k != mh_end(func_id_cache))
+ mh_strnptr_del(func_name_cache, k, NULL);
+}
+
+struct func *
+func_by_id(uint32_t fid)
+{
+ mh_int_t func = mh_i32ptr_find(func_id_cache, fid, NULL);
+ if (func == mh_end(func_id_cache))
+ return NULL;
+ return (struct func *) mh_i32ptr_node(func_id_cache, func)->val;
+}
+
+struct func *
+func_by_name(const char *name, uint32_t name_len)
+{
+ mh_int_t func = mh_strnptr_find_inp(func_name_cache, name, name_len);
+ if (func == mh_end(func_name_cache))
+ return NULL;
+ return (struct func *) mh_strnptr_node(func_name_cache, func)->val;
+}
diff --git a/src/box/func_cache.h b/src/box/func_cache.h
new file mode 100644
index 000000000..9a8bba76d
--- /dev/null
+++ b/src/box/func_cache.h
@@ -0,0 +1,78 @@
+#ifndef TARANTOOL_BOX_FUNC_CACHE_H_INCLUDED
+#define TARANTOOL_BOX_FUNC_CACHE_H_INCLUDED
+/*
+ * Copyright 2010-2019, Tarantool AUTHORS, please see AUTHORS file.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the
+ * following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY <COPYRIGHT HOLDER> ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+ * <COPYRIGHT HOLDER> OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+ * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+#include <stdint.h>
+
+#if defined(__cplusplus)
+extern "C" {
+#endif /* defined(__cplusplus) */
+
+struct func;
+
+/**
+ * Create global function hash tables.
+ * @return - 0 on success, -1 on memory error.
+ */
+int
+func_cache_init(void);
+
+/** Delete global function hash tables. */
+void
+func_cache_destroy(void);
+
+/**
+ * Insert a new function object in the function cache.
+ * @param func Function object to insert.
+ */
+void
+func_cache_insert(struct func *func);
+
+/**
+ * Delete a function object from function cache.
+ * @param fid Function to delete identifier.
+ */
+void
+func_cache_delete(uint32_t fid);
+
+/** Find a function object by its id. */
+struct func *
+func_by_id(uint32_t fid);
+
+/** Find a function object by its name. */
+struct func *
+func_by_name(const char *name, uint32_t name_len);
+
+#if defined(__cplusplus)
+} /* extern "C" */
+#endif /* defined(__cplusplus) */
+
+#endif /* TARANTOOL_BOX_FUNC_CACHE_H_INCLUDED */
diff --git a/src/box/schema.cc b/src/box/schema.cc
index 64412fac6..4427668a1 100644
--- a/src/box/schema.cc
+++ b/src/box/schema.cc
@@ -56,8 +56,6 @@
/** All existing spaces. */
static struct mh_i32ptr_t *spaces;
static struct mh_strnptr_t *spaces_by_name;
-static struct mh_i32ptr_t *funcs;
-static struct mh_strnptr_t *funcs_by_name;
static struct mh_i32ptr_t *sequences;
/** Public change counter. On its update clients need to fetch
* new space data from the instance. */
@@ -377,8 +375,6 @@ schema_init()
/* Initialize the space cache. */
spaces = mh_i32ptr_new();
spaces_by_name = mh_strnptr_new();
- funcs = mh_i32ptr_new();
- funcs_by_name = mh_strnptr_new();
sequences = mh_i32ptr_new();
/*
* Create surrogate space objects for the mandatory system
@@ -523,15 +519,6 @@ schema_free(void)
}
mh_i32ptr_delete(spaces);
mh_strnptr_delete(spaces_by_name);
- while (mh_size(funcs) > 0) {
- mh_int_t i = mh_first(funcs);
-
- struct func *func = ((struct func *)
- mh_i32ptr_node(funcs, i)->val);
- func_cache_delete(func->def->fid);
- func_delete(func);
- }
- mh_i32ptr_delete(funcs);
while (mh_size(sequences) > 0) {
mh_int_t i = mh_first(sequences);
@@ -542,62 +529,6 @@ schema_free(void)
mh_i32ptr_delete(sequences);
}
-void
-func_cache_insert(struct func *func)
-{
- assert(func_by_id(func->def->fid) == NULL);
- assert(func_by_name(func->def->name, strlen(func->def->name)) == NULL);
- const struct mh_i32ptr_node_t node = { func->def->fid, func };
- mh_int_t k1 = mh_i32ptr_put(funcs, &node, NULL, NULL);
- if (k1 == mh_end(funcs)) {
-error:
- panic_syserror("Out of memory for the data "
- "dictionary cache (stored function).");
- }
- size_t def_name_len = strlen(func->def->name);
- uint32_t name_hash = mh_strn_hash(func->def->name, def_name_len);
- const struct mh_strnptr_node_t strnode = {
- func->def->name, def_name_len, name_hash, func };
- mh_int_t k2 = mh_strnptr_put(funcs_by_name, &strnode, NULL, NULL);
- if (k2 == mh_end(funcs_by_name)) {
- mh_i32ptr_del(funcs, k1, NULL);
- goto error;
- }
-}
-
-void
-func_cache_delete(uint32_t fid)
-{
- mh_int_t k = mh_i32ptr_find(funcs, fid, NULL);
- if (k == mh_end(funcs))
- return;
- struct func *func = (struct func *)
- mh_i32ptr_node(funcs, k)->val;
- mh_i32ptr_del(funcs, k, NULL);
- k = mh_strnptr_find_inp(funcs_by_name, func->def->name,
- strlen(func->def->name));
- if (k != mh_end(funcs))
- mh_strnptr_del(funcs_by_name, k, NULL);
-}
-
-struct func *
-func_by_id(uint32_t fid)
-{
- mh_int_t func = mh_i32ptr_find(funcs, fid, NULL);
- if (func == mh_end(funcs))
- return NULL;
- return (struct func *) mh_i32ptr_node(funcs, func)->val;
-}
-
-struct func *
-func_by_name(const char *name, uint32_t name_len)
-{
- mh_int_t func = mh_strnptr_find_inp(funcs_by_name, name, name_len);
- if (func == mh_end(funcs_by_name))
- return NULL;
- return (struct func *) mh_strnptr_node(funcs_by_name, func)->val;
-}
-
bool
schema_find_grants(const char *type, uint32_t id)
{
diff --git a/src/box/schema.h b/src/box/schema.h
index f0039b29d..181a0d2db 100644
--- a/src/box/schema.h
+++ b/src/box/schema.h
@@ -36,6 +36,7 @@
#include "error.h"
#include "space.h"
#include "latch.h"
+#include "func_cache.h"
#if defined(__cplusplus)
extern "C" {
@@ -89,9 +90,6 @@ space_cache_find(uint32_t id)
return NULL;
}
-struct func *
-func_by_name(const char *name, uint32_t name_len);
-
/** Call a visitor function on every space in the space cache. */
int
space_foreach(int (*func)(struct space *sp, void *udata), void *udata);
@@ -162,21 +160,6 @@ schema_free();
struct space *schema_space(uint32_t id);
-/**
- * Insert a new function object in the function cache.
- * @param func Function object to insert.
- */
-void
-func_cache_insert(struct func *func);
-
-void
-func_cache_delete(uint32_t fid);
-
-struct func;
-
-struct func *
-func_by_id(uint32_t fid);
-
static inline struct func *
func_cache_find(uint32_t fid)
{
diff --git a/src/box/tuple.c b/src/box/tuple.c
index 5fff868ac..e98cdb829 100644
--- a/src/box/tuple.c
+++ b/src/box/tuple.c
@@ -38,6 +38,7 @@
#include "tuple_update.h"
#include "coll_id_cache.h"
+#include "func_cache.h"
static struct mempool tuple_iterator_pool;
static struct small_alloc runtime_alloc;
@@ -339,6 +340,9 @@ tuple_init(field_name_hash_f hash)
if (coll_id_cache_init() != 0)
return -1;
+ if (func_cache_init() != 0)
+ return -1;
+
return 0;
}
@@ -403,6 +407,8 @@ tuple_free(void)
coll_id_cache_destroy();
+ func_cache_destroy();
+
bigref_list_destroy();
}
--
2.21.0
^ permalink raw reply [flat|nested] 5+ messages in thread
* [PATCH v2 5/5] box: introduce functional indexes in memtx
[not found] <cover.1561384554.git.kshcherbatov@tarantool.org>
` (3 preceding siblings ...)
2019-06-24 14:27 ` [PATCH v2 4/5] box: move the function hash to the tuple library Kirill Shcherbatov
@ 2019-06-24 14:27 ` Kirill Shcherbatov
4 siblings, 0 replies; 5+ messages in thread
From: Kirill Shcherbatov @ 2019-06-24 14:27 UTC (permalink / raw)
To: tarantool-patches, vdavydov.dev; +Cc: Kirill Shcherbatov
Closes #1260
@TarantoolBot document
Title: introduce functional indexes in memtx
Now you can define a functional index using registered function
as a key extractor.
There is some restrictions for function and key definition for
functional index:
- referenced extractor function must be persistent, deterministic,
sandboxed.
- key parts describe the referenced routine's returned key
(when the routine returned value type is array in meaning of
multikey index key): all parts must be sequential and the
first part's fieldno must be 1.
- key parts can't use JSON paths.
Functional index can't be primary.
You are not allowed to change functional index extractor function
while there are some functional indexes depends of it.
Example:
s = box.schema.space.create('withdata')
s:format({'a', 'b'})
_ = s:create_index('pk')
lua_code = [[function(tuple) return {tuple.a + tuple.b} end]]
box.schema.func.create('sum', {body = lua_code, is_deterministic = true,
is_sandboxed = true})
idx1 = s:create_index('sum', {unique = true, functional = {fid =
box.func.sum.id}, parts = {{1, 'integer'}}})
_ = s:insert({2, 3})
idx1:get(5)
---
- [2, 3]
...
---
src/box/CMakeLists.txt | 3 +-
src/box/alter.cc | 23 +-
src/box/box.cc | 3 +-
src/box/errcode.h | 1 +
src/box/func.c | 2 +
src/box/func.h | 15 +
src/box/functional_key.c | 271 +++++++++++++
src/box/functional_key.h | 87 ++++
src/box/index_def.c | 53 +++
src/box/index_def.h | 20 +
src/box/key_def.c | 66 ++-
src/box/key_def.h | 33 +-
src/box/lua/key_def.c | 2 +-
src/box/lua/schema.lua | 2 +
src/box/lua/space.cc | 14 +
src/box/memtx_bitset.c | 3 +-
src/box/memtx_engine.c | 36 ++
src/box/memtx_rtree.c | 3 +-
src/box/memtx_space.c | 18 +
src/box/opt_def.c | 11 +-
src/box/opt_def.h | 20 +-
src/box/schema.cc | 2 +-
src/box/sql.c | 2 +-
src/box/sql/build.c | 2 +-
src/box/sql/select.c | 2 +-
src/box/sql/where.c | 2 +-
src/box/tuple.c | 9 +-
src/box/tuple.h | 56 ++-
src/box/tuple_compare.cc | 152 ++++---
| 11 +-
src/box/tuple_format.c | 71 +++-
src/box/tuple_format.h | 29 ++
src/box/tuple_hash.cc | 8 +-
src/box/vinyl.c | 10 +-
test/box/bitset.result | 24 ++
test/box/bitset.test.lua | 9 +
test/box/hash.result | 24 ++
test/box/hash.test.lua | 9 +
test/box/misc.result | 1 +
test/box/rtree_misc.result | 24 ++
test/box/rtree_misc.test.lua | 9 +
test/engine/engine.cfg | 5 +-
test/engine/functional.result | 689 ++++++++++++++++++++++++++++++++
test/engine/functional.test.lua | 241 +++++++++++
test/unit/luaT_tuple_new.c | 2 +-
test/unit/merger.test.c | 6 +-
test/unit/tuple_bigref.c | 2 +-
test/unit/vy_iterators_helper.c | 2 +-
test/vinyl/misc.result | 23 ++
test/vinyl/misc.test.lua | 9 +
50 files changed, 2005 insertions(+), 116 deletions(-)
create mode 100644 src/box/functional_key.c
create mode 100644 src/box/functional_key.h
create mode 100644 test/engine/functional.result
create mode 100644 test/engine/functional.test.lua
diff --git a/src/box/CMakeLists.txt b/src/box/CMakeLists.txt
index 7da2d6cff..e6712d8c3 100644
--- a/src/box/CMakeLists.txt
+++ b/src/box/CMakeLists.txt
@@ -53,7 +53,7 @@ add_library(tuple STATIC
field_def.c
opt_def.c
)
-target_link_libraries(tuple json box_error core ${MSGPUCK_LIBRARIES} ${ICU_LIBRARIES} misc bit)
+target_link_libraries(tuple json box_error core coll ${MSGPUCK_LIBRARIES} ${ICU_LIBRARIES} misc bit)
add_library(xlog STATIC xlog.c)
target_link_libraries(xlog core box_error crc32 ${ZSTD_LIBRARIES})
@@ -126,6 +126,7 @@ add_library(box STATIC
wal.c
call.c
merger.c
+ functional_key.c
${lua_sources}
lua/init.c
lua/call.c
diff --git a/src/box/alter.cc b/src/box/alter.cc
index ff6f5c4b8..eb7cac19f 100644
--- a/src/box/alter.cc
+++ b/src/box/alter.cc
@@ -29,6 +29,7 @@
* SUCH DAMAGE.
*/
#include "alter.h"
+#include "box.h"
#include "ck_constraint.h"
#include "column_mask.h"
#include "schema.h"
@@ -233,6 +234,20 @@ index_opts_decode(struct index_opts *opts, const char *map,
"bloom_fpr must be greater than 0 and "
"less than or equal to 1");
}
+ /**
+ * Can't verify functional index extractor routine
+ * reference on load because the function object
+ * had not been registered in Tarantool yet.
+ */
+ if (opts->functional_def != NULL &&
+ strcmp(box_status(), "loading") != 0) {
+ struct func *func = func_by_id(opts->functional_def->fid);
+ if (func == NULL) {
+ tnt_raise(ClientError, ER_WRONG_INDEX_OPTIONS,
+ BOX_INDEX_FIELD_OPTS,
+ "referenced function doesn't exists");
+ }
+ }
}
/**
@@ -246,6 +261,7 @@ index_opts_decode(struct index_opts *opts, const char *map,
* - there are parts for the specified part count
* - types of parts in the parts array are known to the system
* - fieldno of each part in the parts array is within limits
+ * - referenced functional index extractor routine is valid
*/
static struct index_def *
index_def_new_from_tuple(struct tuple *tuple, struct space *space)
@@ -289,7 +305,7 @@ index_def_new_from_tuple(struct tuple *tuple, struct space *space)
space->def->fields,
space->def->field_count, &fiber()->gc) != 0)
diag_raise();
- key_def = key_def_new(part_def, part_count);
+ key_def = key_def_new(part_def, part_count, opts.functional_def);
if (key_def == NULL)
diag_raise();
struct index_def *index_def =
@@ -2696,6 +2712,11 @@ on_replace_dd_func(struct trigger * /* trigger */, void *event)
(unsigned) old_func->def->uid,
"function has grants");
}
+ if (old_func != NULL && old_func->refs != 0) {
+ tnt_raise(ClientError, ER_DROP_FUNCTION,
+ (unsigned) old_func->def->uid,
+ "function has references");
+ }
struct trigger *on_commit =
txn_alter_trigger_new(func_cache_remove_func, old_func);
txn_on_commit(txn, on_commit);
diff --git a/src/box/box.cc b/src/box/box.cc
index 57419ee01..612d9134a 100644
--- a/src/box/box.cc
+++ b/src/box/box.cc
@@ -74,6 +74,7 @@
#include "call.h"
#include "func.h"
#include "sequence.h"
+#include "functional_key.h"
static char status[64] = "unknown";
@@ -2042,7 +2043,7 @@ box_init(void)
if (module_init() != 0)
diag_raise();
- if (tuple_init(lua_hash) != 0)
+ if (tuple_init(lua_hash, functional_key_get) != 0)
diag_raise();
sequence_init();
diff --git a/src/box/errcode.h b/src/box/errcode.h
index 55299b735..93e651e8f 100644
--- a/src/box/errcode.h
+++ b/src/box/errcode.h
@@ -249,6 +249,7 @@ struct errcode_record {
/*194 */_(ER_MULTIKEY_INDEX_MISMATCH, "Field %s is used as multikey in one index and as single key in another") \
/*195 */_(ER_CREATE_CK_CONSTRAINT, "Failed to create check constraint '%s': %s") \
/*196 */_(ER_CK_CONSTRAINT_FAILED, "Check constraint failed '%s': %s") \
+ /*196 */_(ER_FUNCTIONAL_EXTRACTOR, "Functional index extractor '%s' error: %s") \
/*
* !IMPORTANT! Please follow instructions at start of the file
diff --git a/src/box/func.c b/src/box/func.c
index 88cf8cdc9..51bfd137e 100644
--- a/src/box/func.c
+++ b/src/box/func.c
@@ -394,6 +394,7 @@ func_new(struct func_def *def)
if (func == NULL)
return NULL;
func->def = def;
+ func->refs = 0;
/** Nobody has access to the function but the owner. */
memset(func->access, 0, sizeof(func->access));
/*
@@ -541,6 +542,7 @@ static struct func_vtab func_c_vtab = {
void
func_delete(struct func *func)
{
+ assert(func->refs == 0);
struct func_def *def = func->def;
func->vtab->destroy(func);
free(def);
diff --git a/src/box/func.h b/src/box/func.h
index 2236fd873..b58a62e8e 100644
--- a/src/box/func.h
+++ b/src/box/func.h
@@ -82,6 +82,8 @@ struct func {
* Cached runtime access information.
*/
struct access access[BOX_USER_MAX];
+ /** Reference counter. */
+ uint16_t refs;
};
/**
@@ -102,6 +104,19 @@ func_new(struct func_def *def);
void
func_delete(struct func *func);
+static inline void
+func_ref(struct func *func)
+{
+ ++func->refs;
+}
+
+static inline void
+func_unref(struct func *func)
+{
+ assert(func->refs > 0);
+ --func->refs;
+}
+
/**
* Call function with arguments represented with given args.
*/
diff --git a/src/box/functional_key.c b/src/box/functional_key.c
new file mode 100644
index 000000000..b3946fc65
--- /dev/null
+++ b/src/box/functional_key.c
@@ -0,0 +1,271 @@
+/*
+ * Copyright 2010-2019, Tarantool AUTHORS, please see AUTHORS file.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the
+ * following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY <COPYRIGHT HOLDER> ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+ * <COPYRIGHT HOLDER> OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+ * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+#include "functional_key.h"
+#include "index.h"
+#include "box.h"
+#include "fiber.h"
+#include "func.h"
+#include "func_cache.h"
+#include "port.h"
+#include "stdbool.h"
+#include "tuple.h"
+
+/**
+ * Execute a given functional index extractor function and
+ * return an extracted key_data and key_data_sz.
+ */
+static const char *
+functional_key_extract(struct func *func, struct port *in_port,
+ uint32_t *key_data_sz)
+{
+ struct port out_port;
+ int rc = func_call(func, in_port, &out_port);
+ if (rc != 0)
+ goto error;
+ const char *key_data = port_get_msgpack(&out_port, key_data_sz);
+ port_destroy(&out_port);
+ if (key_data == NULL)
+ goto error;
+
+ return key_data;
+error:
+ diag_set(ClientError, ER_FUNCTIONAL_EXTRACTOR, func->def->name,
+ diag_last_error(diag_get())->errmsg);
+ return NULL;
+}
+
+/**
+ * Process a given data and initialize a key_map allocation.
+ * Perform key validation if validate == true.
+ */
+static int
+functional_key_map_create(struct func *func, struct key_def *key_def,
+ const char *data, uint32_t key_count,
+ uint32_t *key_map, bool validate)
+{
+ const char *key = data;
+ for (uint32_t key_idx = 0; key_idx < key_count; key_idx++) {
+ if (key_map != NULL)
+ key_map[key_idx] = key - data;
+ if (validate && mp_typeof(*key) != MP_ARRAY) {
+ diag_set(ClientError, ER_FUNCTIONAL_EXTRACTOR,
+ func->def->name,
+ "returned key type is invalid");
+ return -1;
+ }
+
+ const char *key_end;
+ uint32_t part_count = mp_decode_array(&key);
+ if (!validate) {
+ key_end = key;
+ mp_next(&key_end);
+ } else if (exact_key_validate(key_def, key, part_count,
+ &key_end) != 0) {
+ diag_set(ClientError, ER_FUNCTIONAL_EXTRACTOR,
+ func->def->name,
+ diag_last_error(diag_get())->errmsg);
+ return -1;
+ }
+ key = key_end;
+ }
+ if (key_map != NULL)
+ key_map[0] = key_count;
+ return 0;
+}
+
+/**
+ * Process a given raw functional index key data returned by
+ * functional index extractor routine to form a key used in
+ * comparators and initialize tuple_extra extention
+ * (when enabled) and corresponding key_map.
+ * Perform key validation if validate == true.
+ */
+static const char *
+functional_key_prepare(struct func *func, struct key_def *key_def,
+ struct tuple *tuple, const char *key_data,
+ uint32_t key_data_sz, bool validate,
+ uint32_t *key_count, uint32_t **key_map)
+{
+ *key_count = mp_decode_array(&key_data);
+ key_data_sz -= mp_sizeof_array(*key_count);
+ if (validate && (!key_def->is_multikey && *key_count > 1)) {
+ diag_set(ClientError, ER_FUNCTIONAL_EXTRACTOR,
+ func->def->name, "to many keys were returned");
+ return NULL;
+ }
+
+#ifndef FUNCTIONAL_KEY_HASH_IS_DISABLED
+ uint32_t key_map_sz =
+ functional_key_map_sz(*key_count);
+ struct tuple_extra *tuple_extra =
+ tuple_extra_new(tuple, func->def->fid,
+ key_data_sz + key_map_sz);
+ if (tuple_extra == NULL)
+ return NULL;
+
+ memcpy(tuple_extra->data + key_map_sz, key_data,
+ key_data_sz);
+ *key_map = (uint32_t *) tuple_extra->data;
+ key_data = tuple_extra->data + key_map_sz;
+#else
+ (void) tuple;
+ *key_map = NULL;
+#endif /* FUNCTIONAL_KEY_HASH_IS_DISABLED */
+
+ if (functional_key_map_create(func, key_def, key_data, *key_count,
+ *key_map, validate) != 0) {
+#ifndef FUNCTIONAL_KEY_HASH_IS_DISABLED
+ tuple_extra_delete(tuple_extra);
+#endif
+ return NULL;
+ }
+ return key_data;
+}
+
+const char *
+functional_key_get(struct tuple *tuple, uint32_t functional_fid,
+ uint32_t *key_count, uint32_t **key_map)
+{
+#ifndef FUNCTIONAL_KEY_HASH_IS_DISABLED
+ struct tuple_extra *tuple_extra =
+ tuple_extra_get(tuple, functional_fid);
+ if (likely(tuple_extra != NULL)) {
+ assert(tuple_extra != NULL);
+ *key_map = (uint32_t *) tuple_extra->data;
+ *key_count = (*key_map)[0];
+ return tuple_extra->data + *key_count * sizeof(uint32_t);
+ }
+#endif /* FUNCTIONAL_KEY_HASH_IS_DISABLED */
+
+ /** Index may be created on space with data. */
+ struct func *func = func_by_id(functional_fid);
+ assert(func != NULL);
+
+ struct port in_port;
+ port_tuple_create(&in_port);
+ port_tuple_add(&in_port, tuple);
+ uint32_t key_data_sz;
+ const char *key_data =
+ functional_key_extract(func, &in_port, &key_data_sz);
+ port_destroy(&in_port);
+ if (key_data == NULL)
+ goto error;
+
+ key_data = functional_key_prepare(func, NULL, tuple, key_data,
+ key_data_sz, false, key_count, key_map);
+ if (key_data == NULL)
+ goto error;
+ return key_data;
+error:
+ panic_syserror("Functional index runtime exception: %s",
+ diag_last_error(diag_get())->errmsg);
+}
+
+int
+functional_keys_materialize(struct tuple_format *format, struct tuple *tuple)
+{
+ assert(!rlist_empty(&format->functional_handle));
+ struct region *region = &fiber()->gc;
+ uint32_t region_svp = region_used(region);
+
+ struct port in_port;
+ port_tuple_create(&in_port);
+ port_tuple_add(&in_port, tuple);
+ int extent_cnt = 0;
+ struct functional_handle *handle;
+ rlist_foreach_entry(handle, &format->functional_handle, link) {
+ assert(tuple_extra_get(tuple,
+ handle->key_def->functional_fid) == NULL);
+ if (unlikely(handle->func == NULL)) {
+ /**
+ * The functional handle function pointer
+ * initialization had been delayed during
+ * recovery. Initialize it.
+ */
+ assert(strcmp(box_status(), "loading") == 0);
+ handle->func =
+ func_by_id(handle->key_def->functional_fid);
+ assert(handle->func != NULL);
+ func_ref(handle->func);
+ }
+ struct key_def *key_def = handle->key_def;
+ struct func *func = handle->func;
+ uint32_t key_data_sz;
+ const char *key_data =
+ functional_key_extract(func, &in_port,
+ &key_data_sz);
+ if (key_data == NULL)
+ goto error;
+
+ uint32_t *key_map, key_count;
+ key_data = functional_key_prepare(func, key_def, tuple,
+ key_data, key_data_sz,
+ true, &key_count, &key_map);
+ if (key_data == NULL)
+ goto error;
+
+ region_truncate(region, region_svp);
+ extent_cnt++;
+ }
+ port_destroy(&in_port);
+ region_truncate(region, region_svp);
+ return 0;
+error:
+ port_destroy(&in_port);
+ region_truncate(region, region_svp);
+#ifndef FUNCTIONAL_KEY_HASH_IS_DISABLED
+ rlist_foreach_entry(handle, &format->functional_handle, link) {
+ if (extent_cnt-- == 0)
+ break;
+ struct tuple_extra *tuple_extra =
+ tuple_extra_get(tuple, handle->key_def->functional_fid);
+ assert(tuple_extra != NULL);
+ tuple_extra_delete(tuple_extra);
+ }
+#endif /* FUNCTIONAL_KEY_HASH_IS_DISABLED */
+ return -1;
+}
+
+void
+functional_keys_terminate(struct tuple_format *format, struct tuple *tuple)
+{
+ assert(!rlist_empty(&format->functional_handle));
+
+ struct functional_handle *handle;
+ rlist_foreach_entry(handle, &format->functional_handle, link) {
+ struct tuple_extra *tuple_extra =
+ tuple_extra_get(tuple, handle->key_def->functional_fid);
+ if (tuple_extra == NULL)
+ continue;
+ assert(tuple_extra != NULL);
+ tuple_extra_delete(tuple_extra);
+ }
+}
diff --git a/src/box/functional_key.h b/src/box/functional_key.h
new file mode 100644
index 000000000..ab7469289
--- /dev/null
+++ b/src/box/functional_key.h
@@ -0,0 +1,87 @@
+#ifndef TARANTOOL_BOX_FUNCTIONAL_KEY_H_INCLUDED
+#define TARANTOOL_BOX_FUNCTIONAL_KEY_H_INCLUDED
+/*
+ * Copyright 2010-2019, Tarantool AUTHORS, please see AUTHORS file.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the
+ * following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY <COPYRIGHT HOLDER> ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+ * <COPYRIGHT HOLDER> OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+ * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "inttypes.h"
+
+struct tuple;
+struct tuple_format;
+
+/**
+ * Functional key map is auxilary memroy allocation having the
+ * following layout:
+ *
+ * 4b 4b 4b 4b
+ * +-----------+-----------+-----------+-----------+ +------+----+
+ * | key_count |key2_offset| ... |keyN_offset| |header|data|
+ * +-----------+-----------+-----------+-----------+ +------+----+
+ * | key1
+ *
+ * The functional key map is a part of tuple_extra allocation
+ * representing initialized functional key, when tuple_extra cache
+ * is enabled.
+ */
+static inline uint32_t
+functional_key_map_sz(uint32_t key_count)
+{
+ return sizeof(uint32_t) * key_count;
+}
+
+/**
+ * Process all functional index handles are associated with given
+ * tuple format, evaluate the corresponding extractors with given
+ * tuple, validate extracted keys (when validate == true) and
+ * register functional keys in tuple_extra cache (when enabled).
+ */
+int
+functional_keys_materialize(struct tuple_format *format, struct tuple *tuple);
+
+/** Terminate all registered functional index keys. */
+void
+functional_keys_terminate(struct tuple_format *format, struct tuple *tuple);
+
+/**
+ * Get functional index key by given tuple and function
+ * identifier.
+ */
+const char *
+functional_key_get(struct tuple *tuple, uint32_t functional_fid,
+ uint32_t *key_count, uint32_t **key_map);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* TARANTOOL_BOX_FUNCTIONAL_KEY_H_INCLUDED */
diff --git a/src/box/index_def.c b/src/box/index_def.c
index eb309a30c..021af64ed 100644
--- a/src/box/index_def.c
+++ b/src/box/index_def.c
@@ -50,8 +50,13 @@ const struct index_opts index_opts_default = {
/* .bloom_fpr = */ 0.05,
/* .lsn = */ 0,
/* .stat = */ NULL,
+ /* .functional_def = */ NULL,
};
+int
+functional_def_decode(const char **map, char *opt, uint32_t errcode,
+ uint32_t field_no);
+
const struct opt_def index_opts_reg[] = {
OPT_DEF("unique", OPT_BOOL, struct index_opts, is_unique),
OPT_DEF("dimension", OPT_INT64, struct index_opts, dimension),
@@ -63,6 +68,8 @@ const struct opt_def index_opts_reg[] = {
OPT_DEF("run_size_ratio", OPT_FLOAT, struct index_opts, run_size_ratio),
OPT_DEF("bloom_fpr", OPT_FLOAT, struct index_opts, bloom_fpr),
OPT_DEF("lsn", OPT_INT64, struct index_opts, lsn),
+ OPT_DEF_OPTS("functional", struct index_opts, functional_def,
+ functional_def_decode),
OPT_DEF_LEGACY("sql"),
OPT_END,
};
@@ -148,6 +155,18 @@ index_def_dup(const struct index_def *def)
return NULL;
}
}
+ if (def->opts.functional_def != NULL) {
+ dup->opts.functional_def =
+ malloc(sizeof(*dup->opts.functional_def));
+ if (dup->opts.functional_def == NULL) {
+ diag_set(OutOfMemory, sizeof(*dup->opts.functional_def),
+ "malloc", "functional_def");
+ index_def_delete(dup);
+ return NULL;
+ }
+ memcpy(dup->opts.functional_def, def->opts.functional_def,
+ sizeof(*def->opts.functional_def));
+ }
return dup;
}
@@ -296,6 +315,11 @@ index_def_is_valid(struct index_def *index_def, const char *space_name)
space_name, "primary key cannot be multikey");
return false;
}
+ if (index_def->iid == 0 && index_def->key_def->functional_fid > 0) {
+ diag_set(ClientError, ER_MODIFY_INDEX, index_def->name,
+ space_name, "primary key cannot be functional");
+ return false;
+ }
for (uint32_t i = 0; i < index_def->key_def->part_count; i++) {
assert(index_def->key_def->parts[i].type < field_type_MAX);
if (index_def->key_def->parts[i].fieldno > BOX_INDEX_FIELD_MAX) {
@@ -323,3 +347,32 @@ index_def_is_valid(struct index_def *index_def, const char *space_name)
}
return true;
}
+
+static const struct opt_def functional_def_reg[] = {
+ OPT_DEF("fid", OPT_UINT32, struct functional_def, fid),
+ OPT_DEF("is_multikey", OPT_BOOL, struct functional_def,
+ is_multikey),
+ OPT_END,
+};
+
+int
+functional_def_decode(const char **map, char *opt, uint32_t errcode,
+ uint32_t field_no)
+{
+ assert(mp_typeof(**map) == MP_MAP);
+ struct functional_def *functional_def = malloc(sizeof(*functional_def));
+ if (functional_def == NULL) {
+ diag_set(OutOfMemory, sizeof(*functional_def), "malloc",
+ "functional_def");
+ return -1;
+ }
+ functional_def->fid = 0;
+ functional_def->is_multikey = false;
+ if (opts_decode(functional_def, functional_def_reg, map,
+ errcode, field_no, NULL) != 0) {
+ free(functional_def);
+ return -1;
+ }
+ *(struct functional_def **)opt = functional_def;
+ return 0;
+}
diff --git a/src/box/index_def.h b/src/box/index_def.h
index 6dac28377..122a16eaa 100644
--- a/src/box/index_def.h
+++ b/src/box/index_def.h
@@ -163,6 +163,8 @@ struct index_opts {
* filled after running ANALYZE command.
*/
struct index_stat *stat;
+ /** Functional index definition descriptor. */
+ struct functional_def *functional_def;
};
extern const struct index_opts index_opts_default;
@@ -183,10 +185,26 @@ index_opts_create(struct index_opts *opts)
static inline void
index_opts_destroy(struct index_opts *opts)
{
+ free(opts->functional_def);
free(opts->stat);
TRASH(opts);
}
+static inline int
+functional_def_cmp(const struct functional_def *o1,
+ const struct functional_def *o2)
+{
+ if (o1 == NULL && o2 == NULL)
+ return 0;
+ if (o1 == NULL || o2 == NULL)
+ return (int)(o1 - o2);
+ if (o1->fid != o2->fid)
+ return o1->fid - o2->fid;
+ if (o1->is_multikey != o2->is_multikey)
+ return o1->is_multikey - o2->is_multikey;
+ return 0;
+}
+
static inline int
index_opts_cmp(const struct index_opts *o1, const struct index_opts *o2)
{
@@ -207,6 +225,8 @@ index_opts_cmp(const struct index_opts *o1, const struct index_opts *o2)
return o1->run_size_ratio < o2->run_size_ratio ? -1 : 1;
if (o1->bloom_fpr != o2->bloom_fpr)
return o1->bloom_fpr < o2->bloom_fpr ? -1 : 1;
+ if (functional_def_cmp(o1->functional_def, o2->functional_def) != 0)
+ return functional_def_cmp(o1->functional_def, o2->functional_def);
return 0;
}
diff --git a/src/box/key_def.c b/src/box/key_def.c
index ee758eefa..b3325a3b4 100644
--- a/src/box/key_def.c
+++ b/src/box/key_def.c
@@ -38,6 +38,8 @@
#include "schema_def.h"
#include "coll_id_cache.h"
#include "small/region.h"
+#include "func_cache.h"
+#include "func.h"
#include "coll/coll.h"
const char *sort_order_strs[] = { "asc", "desc", "undef" };
@@ -222,7 +224,7 @@ key_def_set_part(struct key_def *def, uint32_t part_no, uint32_t fieldno,
struct coll *coll, uint32_t coll_id,
enum sort_order sort_order, const char *path,
uint32_t path_len, char **path_pool, int32_t offset_slot,
- uint64_t format_epoch)
+ uint64_t format_epoch, uint32_t functional_fid)
{
assert(part_no < def->part_count);
assert(type < field_type_MAX);
@@ -236,12 +238,14 @@ key_def_set_part(struct key_def *def, uint32_t part_no, uint32_t fieldno,
def->parts[part_no].sort_order = sort_order;
def->parts[part_no].offset_slot_cache = offset_slot;
def->parts[part_no].format_epoch = format_epoch;
+ def->parts[part_no].functional_fid = functional_fid;
column_mask_set_fieldno(&def->column_mask, fieldno);
return key_def_set_part_path(def, part_no, path, path_len, path_pool);
}
struct key_def *
-key_def_new(const struct key_part_def *parts, uint32_t part_count)
+key_def_new(const struct key_part_def *parts, uint32_t part_count,
+ struct functional_def *functional_def)
{
size_t sz = 0;
for (uint32_t i = 0; i < part_count; i++)
@@ -255,6 +259,29 @@ key_def_new(const struct key_part_def *parts, uint32_t part_count)
def->part_count = part_count;
def->unique_part_count = part_count;
+ struct func *func = NULL;
+ if (functional_def != NULL) {
+ /**
+ * Ensure that a given function definition is a
+ * valid functional index extractor routine: only
+ * a persistent deterministic sandboxed Lua
+ * function may be used in functional index
+ * definition.
+ * Function may be not registered yet during
+ * recovery.
+ */
+ func = func_by_id(functional_def->fid);
+ if (func != NULL && (func->def->language != FUNC_LANGUAGE_LUA ||
+ func->def->body == NULL || !func->def->is_deterministic ||
+ !func->def->is_sandboxed)) {
+ diag_set(ClientError, ER_WRONG_INDEX_OPTIONS, 0,
+ "referenced function doesn't satisfy "
+ "functional index constraints");
+ goto error;
+ }
+ def->functional_fid = functional_def->fid;
+ def->is_multikey = functional_def->is_multikey;
+ }
/* A pointer to the JSON paths data in the new key_def. */
char *path_pool = (char *)def + key_def_sizeof(part_count, 0);
@@ -266,8 +293,7 @@ key_def_new(const struct key_part_def *parts, uint32_t part_count)
if (coll_id == NULL) {
diag_set(ClientError, ER_WRONG_INDEX_OPTIONS,
i + 1, "collation was not found by ID");
- key_def_delete(def);
- return NULL;
+ goto error;
}
coll = coll_id->coll;
}
@@ -276,14 +302,24 @@ key_def_new(const struct key_part_def *parts, uint32_t part_count)
part->nullable_action, coll, part->coll_id,
part->sort_order, part->path, path_len,
&path_pool, TUPLE_OFFSET_SLOT_NIL,
- 0) != 0) {
- key_def_delete(def);
- return NULL;
+ 0, def->functional_fid) != 0) {
+ goto error;
+ }
+ }
+ if (functional_def != NULL) {
+ if (!key_def_is_sequential(def) || parts->fieldno != 0 ||
+ def->has_json_paths) {
+ diag_set(ClientError, ER_WRONG_INDEX_OPTIONS,
+ 0, "invalid functional key definition");
+ goto error;
}
}
assert(path_pool == (char *)def + sz);
key_def_set_func(def);
return def;
+error:
+ key_def_delete(def);
+ return NULL;
}
int
@@ -333,7 +369,7 @@ box_key_def_new(uint32_t *fields, uint32_t *types, uint32_t part_count)
(enum field_type)types[item],
ON_CONFLICT_ACTION_DEFAULT, NULL,
COLL_NONE, SORT_ORDER_ASC, NULL, 0, NULL,
- TUPLE_OFFSET_SLOT_NIL, 0) != 0) {
+ TUPLE_OFFSET_SLOT_NIL, 0, 0) != 0) {
key_def_delete(key_def);
return NULL;
}
@@ -681,6 +717,7 @@ key_def_find(const struct key_def *key_def, const struct key_part *to_find)
const struct key_part *end = part + key_def->part_count;
for (; part != end; part++) {
if (part->fieldno == to_find->fieldno &&
+ part->functional_fid == to_find->functional_fid &&
json_path_cmp(part->path, part->path_len,
to_find->path, to_find->path_len,
TUPLE_INDEX_BASE) == 0)
@@ -708,6 +745,9 @@ static bool
key_def_can_merge(const struct key_def *key_def,
const struct key_part *to_merge)
{
+ if (key_def->functional_fid > 0)
+ return true;
+
const struct key_part *part = key_def_find(key_def, to_merge);
if (part == NULL)
return true;
@@ -722,6 +762,7 @@ key_def_can_merge(const struct key_def *key_def,
struct key_def *
key_def_merge(const struct key_def *first, const struct key_def *second)
{
+ assert(second->functional_fid == 0);
uint32_t new_part_count = first->part_count + second->part_count;
/*
* Find and remove part duplicates, i.e. parts counted
@@ -754,6 +795,7 @@ key_def_merge(const struct key_def *first, const struct key_def *second)
new_def->has_optional_parts = first->has_optional_parts ||
second->has_optional_parts;
new_def->is_multikey = first->is_multikey || second->is_multikey;
+ new_def->functional_fid = first->functional_fid;
/* JSON paths data in the new key_def. */
char *path_pool = (char *)new_def + key_def_sizeof(new_part_count, 0);
@@ -768,7 +810,8 @@ key_def_merge(const struct key_def *first, const struct key_def *second)
part->coll_id, part->sort_order,
part->path, part->path_len, &path_pool,
part->offset_slot_cache,
- part->format_epoch) != 0) {
+ part->format_epoch,
+ part->functional_fid) != 0) {
key_def_delete(new_def);
return NULL;
}
@@ -785,7 +828,8 @@ key_def_merge(const struct key_def *first, const struct key_def *second)
part->coll_id, part->sort_order,
part->path, part->path_len, &path_pool,
part->offset_slot_cache,
- part->format_epoch) != 0) {
+ part->format_epoch,
+ part->functional_fid) != 0) {
key_def_delete(new_def);
return NULL;
}
@@ -826,7 +870,7 @@ key_def_find_pk_in_cmp_def(const struct key_def *cmp_def,
}
/* Finally, allocate the new key definition. */
- extracted_def = key_def_new(parts, pk_def->part_count);
+ extracted_def = key_def_new(parts, pk_def->part_count, NULL);
out:
region_truncate(region, region_svp);
return extracted_def;
diff --git a/src/box/key_def.h b/src/box/key_def.h
index df83d055c..3f7d7ae2d 100644
--- a/src/box/key_def.h
+++ b/src/box/key_def.h
@@ -75,6 +75,8 @@ struct key_part_def {
extern const struct key_part_def key_part_def_default;
+struct func;
+
/** Descriptor of a single part in a multipart key. */
struct key_part {
/** Tuple field index for this part */
@@ -98,6 +100,11 @@ struct key_part {
char *path;
/** The length of JSON path. */
uint32_t path_len;
+ /**
+ * The functional index extractor routine identifier.
+ * != 0 iff this is a functional index key definition.
+ */
+ uint32_t functional_fid;
/**
* Epoch of the tuple format the offset slot cached in
* this part is valid for, see tuple_format::epoch.
@@ -166,6 +173,18 @@ typedef hint_t (*tuple_hint_t)(struct tuple *tuple,
typedef hint_t (*key_hint_t)(const char *key, uint32_t part_count,
struct key_def *key_def);
+struct functional_def {
+ /**
+ * Index of the function used for functional index.
+ * The value > 0 for functional index, and 0 otherwise.
+ */
+ uint32_t fid;
+ /**
+ * Whether this functional index is multikey.
+ */
+ bool is_multikey;
+};
+
/* Definition of a multipart key. */
struct key_def {
/** @see tuple_compare() */
@@ -233,6 +252,12 @@ struct key_def {
* undefined otherwise.
*/
uint32_t multikey_fieldno;
+ /**
+ * Identifier of the functional index extractor
+ * routine.
+ * != 0 iff this is functional index key definition.
+ */
+ uint32_t functional_fid;
/** The size of the 'parts' array. */
uint32_t part_count;
/** Description of parts of a multipart index. */
@@ -326,11 +351,13 @@ key_def_sizeof(uint32_t part_count, uint32_t path_pool_size)
}
/**
- * Allocate a new key_def with the given part count
- * and initialize its parts.
+ * Allocate a new key_def with given valid functional index
+ * definition (or NULL) and the given part count and
+ * initialize its parts.
*/
struct key_def *
-key_def_new(const struct key_part_def *parts, uint32_t part_count);
+key_def_new(const struct key_part_def *parts, uint32_t part_count,
+ struct functional_def *functional);
/**
* Dump part definitions of the given key def.
diff --git a/src/box/lua/key_def.c b/src/box/lua/key_def.c
index 810ce5375..9bcbc14be 100644
--- a/src/box/lua/key_def.c
+++ b/src/box/lua/key_def.c
@@ -444,7 +444,7 @@ lbox_key_def_new(struct lua_State *L)
lua_pop(L, 1);
}
- struct key_def *key_def = key_def_new(parts, part_count);
+ struct key_def *key_def = key_def_new(parts, part_count, NULL);
region_truncate(region, region_svp);
if (key_def == NULL)
return luaT_error(L);
diff --git a/src/box/lua/schema.lua b/src/box/lua/schema.lua
index 1ab97440c..841bcd34b 100644
--- a/src/box/lua/schema.lua
+++ b/src/box/lua/schema.lua
@@ -968,6 +968,7 @@ local alter_index_template = {
type = 'string',
parts = 'table',
sequence = 'boolean, number, string, table',
+ functional = 'table',
}
for k, v in pairs(index_options) do
alter_index_template[k] = v
@@ -1061,6 +1062,7 @@ box.schema.index.create = function(space_id, name, options)
run_count_per_level = options.run_count_per_level,
run_size_ratio = options.run_size_ratio,
bloom_fpr = options.bloom_fpr,
+ functional = options.functional,
}
local field_type_aliases = {
num = 'unsigned'; -- Deprecated since 1.7.2
diff --git a/src/box/lua/space.cc b/src/box/lua/space.cc
index 18039fd6a..d8f128dd3 100644
--- a/src/box/lua/space.cc
+++ b/src/box/lua/space.cc
@@ -335,6 +335,20 @@ lbox_fillspace(struct lua_State *L, struct space *space, int i)
lua_setfield(L, -2, "dimension");
}
+ if (index_opts->functional_def != NULL) {
+ lua_pushstring(L, "functional");
+ lua_newtable(L);
+
+ lua_pushnumber(L, index_opts->functional_def->fid);
+ lua_setfield(L, -2, "fid");
+
+ lua_pushboolean(L,
+ index_opts->functional_def->is_multikey);
+ lua_setfield(L, -2, "is_multikey");
+
+ lua_settable(L, -3);
+ }
+
lua_pushstring(L, index_type_strs[index_def->type]);
lua_setfield(L, -2, "type");
diff --git a/src/box/memtx_bitset.c b/src/box/memtx_bitset.c
index 59bc96427..5ce1c279a 100644
--- a/src/box/memtx_bitset.c
+++ b/src/box/memtx_bitset.c
@@ -302,7 +302,8 @@ memtx_bitset_index_replace(struct index *base, struct tuple *old_tuple,
if (new_tuple != NULL) {
const char *field = tuple_field_by_part(new_tuple,
- base->def->key_def->parts, MULTIKEY_NONE);
+ base->def->key_def->parts, MULTIKEY_NONE,
+ false);
uint32_t key_len;
const void *key = make_key(field, &key_len);
#ifndef OLD_GOOD_BITSET
diff --git a/src/box/memtx_engine.c b/src/box/memtx_engine.c
index 3dd8d7f4d..054202994 100644
--- a/src/box/memtx_engine.c
+++ b/src/box/memtx_engine.c
@@ -36,6 +36,8 @@
#include <small/mempool.h>
#include "fiber.h"
+#include "func.h"
+#include "functional_key.h"
#include "errinj.h"
#include "coio_file.h"
#include "tuple.h"
@@ -133,6 +135,22 @@ memtx_end_build_primary_key(struct space *space, void *param)
memtx_space->replace == memtx_space_replace_all_keys)
return 0;
+ struct functional_handle *handle;
+ rlist_foreach_entry(handle, &space->format->functional_handle, link) {
+ /**
+ * The functional handle function pointer
+ * initialization had been delayed during
+ * recovery. Now it is possible to initialize it.
+ * When space is empty, the functional handle
+ * didn't been initialized yet, so the function
+ * object has no reference and may be dropped.
+ */
+ if (likely(handle->func != NULL))
+ continue;
+ handle->func = func_by_id(handle->key_def->functional_fid);
+ assert(handle->func != NULL);
+ func_ref(handle->func);
+ }
index_end_build(space->index[0]);
memtx_space->replace = memtx_space_replace_primary_key;
return 0;
@@ -1175,6 +1193,19 @@ memtx_tuple_new(struct tuple_format *format, const char *data, const char *end)
char *raw = (char *) tuple + tuple->data_offset;
field_map_build(&builder, raw - field_map_size);
memcpy(raw, data, tuple_len);
+ if (!rlist_empty(&format->functional_handle)) {
+ if (functional_keys_materialize(format, tuple) != 0) {
+ if (tuple->refs == 0) {
+ memtx_tuple_delete(format, tuple);
+ } else {
+ /**
+ * The garbage collector must
+ * remove the tuple later.
+ */
+ }
+ return NULL;
+ }
+ }
say_debug("%s(%zu) = %p", __func__, tuple_len, memtx_tuple);
end:
region_truncate(region, region_svp);
@@ -1187,6 +1218,8 @@ memtx_tuple_delete(struct tuple_format *format, struct tuple *tuple)
struct memtx_engine *memtx = (struct memtx_engine *)format->engine;
say_debug("%s(%p)", __func__, tuple);
assert(tuple->refs == 0);
+ if (!rlist_empty(&format->functional_handle))
+ functional_keys_terminate(format, tuple);
tuple_format_unref(format);
struct memtx_tuple *memtx_tuple =
container_of(tuple, struct memtx_tuple, base);
@@ -1337,6 +1370,9 @@ memtx_index_def_change_requires_rebuild(struct index *index,
return true;
if (!old_def->opts.is_unique && new_def->opts.is_unique)
return true;
+ if (functional_def_cmp(old_def->opts.functional_def,
+ new_def->opts.functional_def) != 0)
+ return true;
const struct key_def *old_cmp_def, *new_cmp_def;
if (index_depends_on_pk(index)) {
diff --git a/src/box/memtx_rtree.c b/src/box/memtx_rtree.c
index 8badad797..98de05ef8 100644
--- a/src/box/memtx_rtree.c
+++ b/src/box/memtx_rtree.c
@@ -122,7 +122,8 @@ extract_rectangle(struct rtree_rect *rect, struct tuple *tuple,
assert(index_def->key_def->part_count == 1);
assert(!index_def->key_def->is_multikey);
const char *elems = tuple_field_by_part(tuple,
- index_def->key_def->parts, MULTIKEY_NONE);
+ index_def->key_def->parts, MULTIKEY_NONE,
+ false);
unsigned dimension = index_def->opts.dimension;
uint32_t count = mp_decode_array(&elems);
return mp_decode_rect(rect, dimension, elems, count, "Field");
diff --git a/src/box/memtx_space.c b/src/box/memtx_space.c
index 15492a09f..724f29081 100644
--- a/src/box/memtx_space.c
+++ b/src/box/memtx_space.c
@@ -661,6 +661,12 @@ memtx_space_check_index_def(struct space *space, struct index_def *index_def)
"HASH index cannot be multikey");
return -1;
}
+ if (index_def->key_def->functional_fid > 0) {
+ diag_set(ClientError, ER_MODIFY_INDEX,
+ index_def->name, space_name(space),
+ "HASH index cannot be functional");
+ return -1;
+ }
break;
case TREE:
/* TREE index has no limitations. */
@@ -690,6 +696,12 @@ memtx_space_check_index_def(struct space *space, struct index_def *index_def)
"RTREE index cannot be multikey");
return -1;
}
+ if (index_def->key_def->functional_fid > 0) {
+ diag_set(ClientError, ER_MODIFY_INDEX,
+ index_def->name, space_name(space),
+ "RTREE index cannot be functional");
+ return -1;
+ }
/* no furter checks of parts needed */
return 0;
case BITSET:
@@ -718,6 +730,12 @@ memtx_space_check_index_def(struct space *space, struct index_def *index_def)
"BITSET index cannot be multikey");
return -1;
}
+ if (index_def->key_def->functional_fid > 0) {
+ diag_set(ClientError, ER_MODIFY_INDEX,
+ index_def->name, space_name(space),
+ "BITSET index cannot be functional");
+ return -1;
+ }
/* no furter checks of parts needed */
return 0;
default:
diff --git a/src/box/opt_def.c b/src/box/opt_def.c
index e2820853b..ae5bc21bc 100644
--- a/src/box/opt_def.c
+++ b/src/box/opt_def.c
@@ -45,7 +45,7 @@ const char *opt_type_strs[] = {
/* [OPT_STR] = */ "string",
/* [OPT_STRPTR] = */ "string",
/* [OPT_ENUM] = */ "enum",
- /* [OPT_ARRAY] = */ "array",
+ /* [OPT_OPTS] = */ "map",
/* [OPT_LEGACY] = */ "legacy",
};
@@ -139,12 +139,11 @@ opt_set(void *opts, const struct opt_def *def, const char **val,
unreachable();
};
break;
- case OPT_ARRAY:
- if (mp_typeof(**val) != MP_ARRAY)
+ case OPT_OPTS:
+ if (mp_typeof(**val) != MP_MAP)
goto type_mismatch_err;
- ival = mp_decode_array(val);
- assert(def->to_array != NULL);
- if (def->to_array(val, ival, opt, errcode, field_no) != 0)
+ assert(def->to_opts != NULL);
+ if (def->to_opts(val, opt, errcode, field_no) != 0)
return -1;
break;
case OPT_LEGACY:
diff --git a/src/box/opt_def.h b/src/box/opt_def.h
index 21544412c..579368d2e 100644
--- a/src/box/opt_def.h
+++ b/src/box/opt_def.h
@@ -47,7 +47,7 @@ enum opt_type {
OPT_STR, /* char[] */
OPT_STRPTR, /* char* */
OPT_ENUM, /* enum */
- OPT_ARRAY, /* array */
+ OPT_OPTS, /* opts */
OPT_LEGACY, /* any type, skipped */
opt_type_MAX,
};
@@ -63,19 +63,19 @@ extern const char *opt_type_strs[];
typedef int64_t (*opt_def_to_enum_cb)(const char *str, uint32_t len);
/**
- * Decode MsgPack array callback.
- * All memory allocations returned by opt_def_to_array_cb with opt
+ * Decode MsgPack map callback.
+ * All memory allocations returned by opt_def_to_map_cb with opt
* [out] argument should be managed manually.
* @param str encoded data pointer (next to MsgPack ARRAY header).
- * @param len array length (items count).
+ * @param len map length (items count).
* @param [out] opt pointer to store resulting value.
* @param errcode Code of error to set if something is wrong.
* @param field_no Field number of an option in a parent element.
* @retval 0 on success.
* @retval -1 on error.
*/
-typedef int (*opt_def_to_array_cb)(const char **str, uint32_t len, char *opt,
- uint32_t errcode, uint32_t field_no);
+typedef int (*opt_def_to_opts_cb)(const char **map, char *opt,
+ uint32_t errcode, uint32_t field_no);
struct opt_def {
const char *name;
@@ -90,7 +90,7 @@ struct opt_def {
/** MsgPack data decode callbacks. */
union {
opt_def_to_enum_cb to_enum;
- opt_def_to_array_cb to_array;
+ opt_def_to_opts_cb to_opts;
};
};
@@ -103,9 +103,9 @@ struct opt_def {
sizeof(enum enum_name), enum_name##_strs, enum_name##_MAX, \
{(void *)to_enum} }
-#define OPT_DEF_ARRAY(key, opts, field, to_array) \
- { key, OPT_ARRAY, offsetof(opts, field), sizeof(((opts *)0)->field), \
- NULL, 0, NULL, 0, {(void *)to_array} }
+#define OPT_DEF_OPTS(key, opts, field, to_opts) \
+ { key, OPT_OPTS, offsetof(opts, field), sizeof(((opts *)0)->field), \
+ NULL, 0, NULL, 0, {(void *)to_opts} }
#define OPT_DEF_LEGACY(key) \
{ key, OPT_LEGACY, 0, 0, NULL, 0, NULL, 0, {NULL} }
diff --git a/src/box/schema.cc b/src/box/schema.cc
index 4427668a1..69773f851 100644
--- a/src/box/schema.cc
+++ b/src/box/schema.cc
@@ -269,7 +269,7 @@ sc_space_new(uint32_t id, const char *name,
struct trigger *replace_trigger,
struct trigger *stmt_begin_trigger)
{
- struct key_def *key_def = key_def_new(key_parts, key_part_count);
+ struct key_def *key_def = key_def_new(key_parts, key_part_count, NULL);
if (key_def == NULL)
diag_raise();
auto key_def_guard =
diff --git a/src/box/sql.c b/src/box/sql.c
index a0350da6b..ba62e2b73 100644
--- a/src/box/sql.c
+++ b/src/box/sql.c
@@ -347,7 +347,7 @@ sql_ephemeral_space_create(uint32_t field_count, struct sql_key_info *key_info)
}
}
struct key_def *ephemer_key_def = key_def_new(ephemer_key_parts,
- field_count);
+ field_count, NULL);
if (ephemer_key_def == NULL)
return NULL;
diff --git a/src/box/sql/build.c b/src/box/sql/build.c
index 292168f88..a28135e53 100644
--- a/src/box/sql/build.c
+++ b/src/box/sql/build.c
@@ -2287,7 +2287,7 @@ index_fill_def(struct Parse *parse, struct index *index,
part->coll_id = coll_id;
part->path = NULL;
}
- key_def = key_def_new(key_parts, expr_list->nExpr);
+ key_def = key_def_new(key_parts, expr_list->nExpr, NULL);
if (key_def == NULL)
goto tnt_error;
/*
diff --git a/src/box/sql/select.c b/src/box/sql/select.c
index a257e7204..09599091e 100644
--- a/src/box/sql/select.c
+++ b/src/box/sql/select.c
@@ -1438,7 +1438,7 @@ sql_key_info_to_key_def(struct sql_key_info *key_info)
{
if (key_info->key_def == NULL) {
key_info->key_def = key_def_new(key_info->parts,
- key_info->part_count);
+ key_info->part_count, NULL);
}
return key_info->key_def;
}
diff --git a/src/box/sql/where.c b/src/box/sql/where.c
index 5458c6a75..a61527611 100644
--- a/src/box/sql/where.c
+++ b/src/box/sql/where.c
@@ -2775,7 +2775,7 @@ whereLoopAddBtree(WhereLoopBuilder * pBuilder, /* WHERE clause information */
part.coll_id = COLL_NONE;
part.path = NULL;
- struct key_def *key_def = key_def_new(&part, 1);
+ struct key_def *key_def = key_def_new(&part, 1, NULL);
if (key_def == NULL) {
tnt_error:
pWInfo->pParse->is_aborted = true;
diff --git a/src/box/tuple.c b/src/box/tuple.c
index e98cdb829..569e038a1 100644
--- a/src/box/tuple.c
+++ b/src/box/tuple.c
@@ -73,6 +73,8 @@ struct tuple *box_tuple_last;
struct tuple_format *tuple_format_runtime;
+functional_key_extractor box_functional_key_get = NULL;
+
static void
runtime_tuple_delete(struct tuple_format *format, struct tuple *tuple);
@@ -139,7 +141,8 @@ runtime_tuple_delete(struct tuple_format *format, struct tuple *tuple)
int
tuple_validate_raw(struct tuple_format *format, const char *tuple)
{
- if (tuple_format_field_count(format) == 0)
+ if (tuple_format_field_count(format) == 0 &&
+ rlist_empty(&format->functional_handle))
return 0; /* Nothing to check */
struct region *region = &fiber()->gc;
@@ -305,7 +308,7 @@ tuple_unref_slow(struct tuple *tuple)
/* }}} Bigref */
int
-tuple_init(field_name_hash_f hash)
+tuple_init(field_name_hash_f hash, functional_key_extractor functional_key_get)
{
if (tuple_format_init() != 0)
return -1;
@@ -343,6 +346,8 @@ tuple_init(field_name_hash_f hash)
if (func_cache_init() != 0)
return -1;
+ box_functional_key_get = functional_key_get;
+
return 0;
}
diff --git a/src/box/tuple.h b/src/box/tuple.h
index 3504c1d95..0204ae608 100644
--- a/src/box/tuple.h
+++ b/src/box/tuple.h
@@ -45,6 +45,14 @@ extern "C" {
struct slab_arena;
struct quota;
struct key_part;
+struct func;
+struct tuple;
+
+typedef const char *
+(*functional_key_extractor)(struct tuple *tuple, uint32_t functional_fid,
+ uint32_t *key_count, uint32_t **key_map);
+
+extern functional_key_extractor box_functional_key_get;
/**
* A format for standalone tuples allocated on runtime arena.
@@ -54,7 +62,7 @@ extern struct tuple_format *tuple_format_runtime;
/** Initialize tuple library */
int
-tuple_init(field_name_hash_f hash);
+tuple_init(field_name_hash_f hash, functional_key_extractor functional_key_get);
/** Cleanup tuple library */
void
@@ -729,6 +737,7 @@ tuple_field_raw_by_part(struct tuple_format *format, const char *data,
const uint32_t *field_map,
struct key_part *part, int multikey_idx)
{
+ assert(part->functional_fid == 0);
if (unlikely(part->format_epoch != format->epoch)) {
assert(format->epoch != 0);
part->format_epoch = format->epoch;
@@ -752,11 +761,37 @@ tuple_field_raw_by_part(struct tuple_format *format, const char *data,
*/
static inline const char *
tuple_field_by_part(struct tuple *tuple, struct key_part *part,
- int multikey_idx)
+ int multikey_idx, bool is_functional)
{
- return tuple_field_raw_by_part(tuple_format(tuple), tuple_data(tuple),
- tuple_field_map(tuple), part,
- multikey_idx);
+ if (is_functional && part->functional_fid > 0) {
+ uint32_t key_count, *key_map;
+ const char *key =
+ box_functional_key_get(tuple, part->functional_fid,
+ &key_count, &key_map);
+ if (multikey_idx != MULTIKEY_NONE && multikey_idx > 0) {
+ if (unlikely((uint32_t) multikey_idx > key_count))
+ return NULL;
+#ifndef FUNCTIONAL_KEY_HASH_IS_DISABLED
+ key += key_map[multikey_idx];
+#else
+ for (int k = 0; k < multikey_idx; k++)
+ mp_next(&key);
+#endif /* FUNCTIONAL_KEY_HASH_IS_DISABLED */
+ }
+ assert(mp_typeof(*key) == MP_ARRAY);
+ uint32_t part_count = mp_decode_array(&key);
+ if (unlikely(part->fieldno >= part_count))
+ return NULL;
+ for (uint32_t k = 0; k < part->fieldno; k++)
+ mp_next(&key);
+ return key;
+ } else {
+ assert(part->functional_fid == 0);
+ return tuple_field_raw_by_part(tuple_format(tuple),
+ tuple_data(tuple),
+ tuple_field_map(tuple), part,
+ multikey_idx);
+ }
}
/**
@@ -782,8 +817,17 @@ tuple_raw_multikey_count(struct tuple_format *format, const char *data,
static inline uint32_t
tuple_multikey_count(struct tuple *tuple, struct key_def *key_def)
{
- return tuple_raw_multikey_count(tuple_format(tuple), tuple_data(tuple),
+ if (key_def->functional_fid > 0) {
+ assert(key_def->is_multikey);
+ uint32_t key_count, *key_map;
+ (void)box_functional_key_get(tuple, key_def->functional_fid,
+ &key_count, &key_map);
+ return key_count;
+ } else {
+ return tuple_raw_multikey_count(tuple_format(tuple),
+ tuple_data(tuple),
tuple_field_map(tuple), key_def);
+ }
}
/**
diff --git a/src/box/tuple_compare.cc b/src/box/tuple_compare.cc
index 3e0d22dec..eb7b0736a 100644
--- a/src/box/tuple_compare.cc
+++ b/src/box/tuple_compare.cc
@@ -446,13 +446,14 @@ tuple_compare_field_with_type(const char *field_a, enum mp_type a_type,
}
template<bool is_nullable, bool has_optional_parts, bool has_json_paths,
- bool is_multikey>
+ bool is_multikey, bool is_functional>
static inline int
tuple_compare_slowpath(struct tuple *tuple_a, hint_t tuple_a_hint,
struct tuple *tuple_b, hint_t tuple_b_hint,
struct key_def *key_def)
{
assert(has_json_paths == key_def->has_json_paths);
+ assert(is_functional == key_def->functional_fid > 0);
assert(!has_optional_parts || is_nullable);
assert(is_nullable == key_def->is_nullable);
assert(has_optional_parts == key_def->has_optional_parts);
@@ -466,7 +467,7 @@ tuple_compare_slowpath(struct tuple *tuple_a, hint_t tuple_a_hint,
const char *tuple_a_raw = tuple_data(tuple_a);
const char *tuple_b_raw = tuple_data(tuple_b);
if (key_def->part_count == 1 && part->fieldno == 0 &&
- (!has_json_paths || part->path == NULL)) {
+ (!has_json_paths || part->path == NULL) && !is_functional) {
/*
* First field can not be optional - empty tuples
* can not exist.
@@ -503,20 +504,24 @@ tuple_compare_slowpath(struct tuple *tuple_a, hint_t tuple_a_hint,
end = part + key_def->part_count;
for (; part < end; part++) {
- if (is_multikey) {
- field_a = tuple_field_raw_by_part(format_a, tuple_a_raw,
- field_map_a, part,
- (int)tuple_a_hint);
- field_b = tuple_field_raw_by_part(format_b, tuple_b_raw,
- field_map_b, part,
- (int)tuple_b_hint);
+ if (is_functional) {
+ field_a = tuple_field_by_part(tuple_a, part,
+ is_multikey ?
+ (int)tuple_a_hint : MULTIKEY_NONE,
+ true);
+ field_b = tuple_field_by_part(tuple_b, part,
+ is_multikey ?
+ (int)tuple_b_hint : MULTIKEY_NONE,
+ true);
} else if (has_json_paths) {
field_a = tuple_field_raw_by_part(format_a, tuple_a_raw,
- field_map_a, part,
- MULTIKEY_NONE);
+ field_map_a, part,
+ is_multikey ?
+ (int)tuple_a_hint : MULTIKEY_NONE);
field_b = tuple_field_raw_by_part(format_b, tuple_b_raw,
- field_map_b, part,
- MULTIKEY_NONE);
+ field_map_b, part,
+ is_multikey ?
+ (int)tuple_b_hint : MULTIKEY_NONE);
} else {
field_a = tuple_field_raw(format_a, tuple_a_raw,
field_map_a, part->fieldno);
@@ -569,20 +574,24 @@ tuple_compare_slowpath(struct tuple *tuple_a, hint_t tuple_a_hint,
*/
end = key_def->parts + key_def->part_count;
for (; part < end; ++part) {
- if (is_multikey) {
- field_a = tuple_field_raw_by_part(format_a, tuple_a_raw,
- field_map_a, part,
- (int)tuple_a_hint);
- field_b = tuple_field_raw_by_part(format_b, tuple_b_raw,
- field_map_b, part,
- (int)tuple_b_hint);
+ if (is_functional) {
+ field_a = tuple_field_by_part(tuple_a, part,
+ is_multikey ?
+ (int)tuple_a_hint : MULTIKEY_NONE,
+ true);
+ field_b = tuple_field_by_part(tuple_b, part,
+ is_multikey ?
+ (int)tuple_b_hint : MULTIKEY_NONE,
+ true);
} else if (has_json_paths) {
field_a = tuple_field_raw_by_part(format_a, tuple_a_raw,
- field_map_a, part,
- MULTIKEY_NONE);
+ field_map_a, part,
+ is_multikey ?
+ (int)tuple_a_hint : MULTIKEY_NONE);
field_b = tuple_field_raw_by_part(format_b, tuple_b_raw,
- field_map_b, part,
- MULTIKEY_NONE);
+ field_map_b, part,
+ is_multikey ?
+ (int)tuple_b_hint : MULTIKEY_NONE);
} else {
field_a = tuple_field_raw(format_a, tuple_a_raw,
field_map_a, part->fieldno);
@@ -603,13 +612,14 @@ tuple_compare_slowpath(struct tuple *tuple_a, hint_t tuple_a_hint,
}
template<bool is_nullable, bool has_optional_parts, bool has_json_paths,
- bool is_multikey>
+ bool is_multikey, bool is_functional>
static inline int
tuple_compare_with_key_slowpath(struct tuple *tuple, hint_t tuple_hint,
const char *key, uint32_t part_count,
hint_t key_hint, struct key_def *key_def)
{
assert(has_json_paths == key_def->has_json_paths);
+ assert(is_functional == key_def->functional_fid > 0);
assert(!has_optional_parts || is_nullable);
assert(is_nullable == key_def->is_nullable);
assert(has_optional_parts == key_def->has_optional_parts);
@@ -628,14 +638,16 @@ tuple_compare_with_key_slowpath(struct tuple *tuple, hint_t tuple_hint,
enum mp_type a_type, b_type;
if (likely(part_count == 1)) {
const char *field;
- if (is_multikey) {
- field = tuple_field_raw_by_part(format, tuple_raw,
- field_map, part,
- (int)tuple_hint);
+ if (is_functional) {
+ field = tuple_field_by_part(tuple, part,
+ is_multikey ?
+ (int)tuple_hint : MULTIKEY_NONE,
+ true);
} else if (has_json_paths) {
field = tuple_field_raw_by_part(format, tuple_raw,
- field_map, part,
- MULTIKEY_NONE);
+ field_map, part,
+ is_multikey ?
+ (int)tuple_hint : MULTIKEY_NONE);
} else {
field = tuple_field_raw(format, tuple_raw, field_map,
part->fieldno);
@@ -663,14 +675,16 @@ tuple_compare_with_key_slowpath(struct tuple *tuple, hint_t tuple_hint,
struct key_part *end = part + part_count;
for (; part < end; ++part, mp_next(&key)) {
const char *field;
- if (is_multikey) {
- field = tuple_field_raw_by_part(format, tuple_raw,
- field_map, part,
- (int)tuple_hint);
+ if (is_functional) {
+ field = tuple_field_by_part(tuple, part,
+ is_multikey ?
+ (int)tuple_hint : MULTIKEY_NONE,
+ true);
} else if (has_json_paths) {
field = tuple_field_raw_by_part(format, tuple_raw,
- field_map, part,
- MULTIKEY_NONE);
+ field_map, part,
+ is_multikey ?
+ (int)tuple_hint : MULTIKEY_NONE);
} else {
field = tuple_field_raw(format, tuple_raw, field_map,
part->fieldno);
@@ -1565,13 +1579,13 @@ key_hint(const char *key, uint32_t part_count, struct key_def *key_def)
return field_hint<type, is_nullable>(key, key_def->parts->coll);
}
-template <enum field_type type, bool is_nullable>
+template <enum field_type type, bool is_nullable, bool is_functional>
static hint_t
tuple_hint(struct tuple *tuple, struct key_def *key_def)
{
assert(!key_def->is_multikey);
const char *field = tuple_field_by_part(tuple, key_def->parts,
- MULTIKEY_NONE);
+ MULTIKEY_NONE, is_functional);
if (is_nullable && field == NULL)
return hint_nil();
return field_hint<type, is_nullable>(field, key_def->parts->coll);
@@ -1611,7 +1625,10 @@ static void
key_def_set_hint_func(struct key_def *def)
{
def->key_hint = key_hint<type, is_nullable>;
- def->tuple_hint = tuple_hint<type, is_nullable>;
+ if (def->functional_fid > 0)
+ def->tuple_hint = tuple_hint<type, is_nullable, true>;
+ else
+ def->tuple_hint = tuple_hint<type, is_nullable, false>;
}
template<enum field_type type>
@@ -1703,13 +1720,14 @@ key_def_set_compare_func_fast(struct key_def *def)
if (cmp == NULL) {
cmp = is_sequential ?
tuple_compare_sequential<false, false> :
- tuple_compare_slowpath<false, false, false, false>;
+ tuple_compare_slowpath<false, false, false,
+ false, false>;
}
if (cmp_wk == NULL) {
cmp_wk = is_sequential ?
tuple_compare_with_key_sequential<false, false> :
tuple_compare_with_key_slowpath<false, false,
- false, false>;
+ false, false, false>;
}
def->tuple_compare = cmp;
@@ -1728,9 +1746,11 @@ key_def_set_compare_func_plain(struct key_def *def)
<is_nullable, has_optional_parts>;
} else {
def->tuple_compare = tuple_compare_slowpath
- <is_nullable, has_optional_parts, false, false>;
+ <is_nullable, has_optional_parts, false,
+ false, false>;
def->tuple_compare_with_key = tuple_compare_with_key_slowpath
- <is_nullable, has_optional_parts, false, false>;
+ <is_nullable, has_optional_parts, false,
+ false, false>;
}
}
@@ -1741,14 +1761,36 @@ key_def_set_compare_func_json(struct key_def *def)
assert(def->has_json_paths);
if (def->is_multikey) {
def->tuple_compare = tuple_compare_slowpath
- <is_nullable, has_optional_parts, true, true>;
+ <is_nullable, has_optional_parts, true,
+ true, false>;
+ def->tuple_compare_with_key = tuple_compare_with_key_slowpath
+ <is_nullable, has_optional_parts, true,
+ true, false>;
+ } else {
+ def->tuple_compare = tuple_compare_slowpath
+ <is_nullable, has_optional_parts, true,
+ false, false>;
+ def->tuple_compare_with_key = tuple_compare_with_key_slowpath
+ <is_nullable, has_optional_parts, true,
+ false, false>;
+ }
+}
+
+template<bool is_nullable>
+static void
+key_def_set_compare_func_functional(struct key_def *def)
+{
+ assert(def->functional_fid > 0);
+ if (def->is_multikey) {
+ def->tuple_compare = tuple_compare_slowpath
+ <is_nullable, false, false, true, true>;
def->tuple_compare_with_key = tuple_compare_with_key_slowpath
- <is_nullable, has_optional_parts, true, true>;
+ <is_nullable, false, false, true, true>;
} else {
def->tuple_compare = tuple_compare_slowpath
- <is_nullable, has_optional_parts, true, false>;
+ <is_nullable, false, false, false, true>;
def->tuple_compare_with_key = tuple_compare_with_key_slowpath
- <is_nullable, has_optional_parts, true, false>;
+ <is_nullable, false, false, false, true>;
}
}
@@ -1756,9 +1798,10 @@ void
key_def_set_compare_func(struct key_def *def)
{
if (!key_def_has_collation(def) &&
- !def->is_nullable && !def->has_json_paths) {
+ !def->is_nullable && !def->has_json_paths &&
+ def->functional_fid == 0) {
key_def_set_compare_func_fast(def);
- } else if (!def->has_json_paths) {
+ } else if (!def->has_json_paths && def->functional_fid == 0) {
if (def->is_nullable && def->has_optional_parts) {
key_def_set_compare_func_plain<true, true>(def);
} else if (def->is_nullable && !def->has_optional_parts) {
@@ -1767,7 +1810,16 @@ key_def_set_compare_func(struct key_def *def)
assert(!def->is_nullable && !def->has_optional_parts);
key_def_set_compare_func_plain<false, false>(def);
}
+ } else if (def->functional_fid > 0) {
+ assert(!def->has_json_paths);
+ assert(!def->has_optional_parts);
+ if (def->is_nullable) {
+ key_def_set_compare_func_functional<true>(def);
+ } else {
+ key_def_set_compare_func_functional<false>(def);
+ }
} else {
+ assert(def->has_json_paths);
if (def->is_nullable && def->has_optional_parts) {
key_def_set_compare_func_json<true, true>(def);
} else if (def->is_nullable && !def->has_optional_parts) {
--git a/src/box/tuple_extract_key.cc b/src/box/tuple_extract_key.cc
index 471c7df80..ea7657465 100644
--- a/src/box/tuple_extract_key.cc
+++ b/src/box/tuple_extract_key.cc
@@ -120,6 +120,7 @@ tuple_extract_key_slowpath(struct tuple *tuple, struct key_def *key_def,
key_def_contains_sequential_parts(key_def));
assert(is_multikey == key_def->is_multikey);
assert(!key_def->is_multikey || multikey_idx != MULTIKEY_NONE);
+ assert(key_def->functional_fid == 0);
assert(mp_sizeof_nil() == 1);
const char *data = tuple_data(tuple);
uint32_t part_count = key_def->part_count;
@@ -251,6 +252,7 @@ tuple_extract_key_slowpath_raw(const char *data, const char *data_end,
assert(!has_optional_parts || key_def->is_nullable);
assert(has_optional_parts == key_def->has_optional_parts);
assert(!key_def->is_multikey || multikey_idx != MULTIKEY_NONE);
+ assert(key_def->functional_fid == 0);
assert(mp_sizeof_nil() == 1);
/* allocate buffer with maximal possible size */
char *key = (char *) region_alloc(&fiber()->gc, data_end - data);
@@ -367,6 +369,7 @@ key_def_set_extract_func_plain(struct key_def *def)
{
assert(!def->has_json_paths);
assert(!def->is_multikey);
+ assert(def->functional_fid == 0);
if (key_def_is_sequential(def)) {
assert(contains_sequential_parts || def->part_count == 1);
def->tuple_extract_key = tuple_extract_key_sequential
@@ -387,6 +390,7 @@ static void
key_def_set_extract_func_json(struct key_def *def)
{
assert(def->has_json_paths);
+ assert(def->functional_fid == 0);
if (def->is_multikey) {
def->tuple_extract_key = tuple_extract_key_slowpath
<contains_sequential_parts,
@@ -406,7 +410,10 @@ key_def_set_extract_func(struct key_def *key_def)
bool contains_sequential_parts =
key_def_contains_sequential_parts(key_def);
bool has_optional_parts = key_def->has_optional_parts;
- if (!key_def->has_json_paths) {
+ if (key_def->functional_fid > 0) {
+ key_def->tuple_extract_key = NULL;
+ key_def->tuple_extract_key_raw = NULL;
+ } else if (!key_def->has_json_paths) {
if (!contains_sequential_parts && !has_optional_parts) {
key_def_set_extract_func_plain<false, false>(key_def);
} else if (!contains_sequential_parts && has_optional_parts) {
@@ -456,7 +463,7 @@ tuple_validate_key_parts(struct key_def *key_def, struct tuple *tuple)
for (uint32_t idx = 0; idx < key_def->part_count; idx++) {
struct key_part *part = &key_def->parts[idx];
const char *field = tuple_field_by_part(tuple, part,
- MULTIKEY_NONE);
+ MULTIKEY_NONE, false);
if (field == NULL) {
if (key_part_is_nullable(part))
continue;
diff --git a/src/box/tuple_format.c b/src/box/tuple_format.c
index 02fadf1cf..2bdfc2189 100644
--- a/src/box/tuple_format.c
+++ b/src/box/tuple_format.c
@@ -30,10 +30,15 @@
*/
#include "bit/bit.h"
#include "fiber.h"
+#include "schema.h"
+#include "port.h"
#include "json/json.h"
#include "tuple_format.h"
#include "coll_id_cache.h"
#include "tt_static.h"
+#include "func.h"
+#include "memtx_engine.h"
+#include "functional_key.h"
#include "third_party/PMurHash.h"
@@ -462,6 +467,12 @@ tuple_format_create(struct tuple_format *format, struct key_def * const *keys,
/* extract field type info */
for (uint16_t key_no = 0; key_no < key_count; ++key_no) {
const struct key_def *key_def = keys[key_no];
+ /*
+ * Functional key definitions are not the part
+ * of the space format.
+ */
+ if (key_def->functional_fid > 0)
+ continue;
bool is_sequential = key_def_is_sequential(key_def);
const struct key_part *part = key_def->parts;
const struct key_part *parts_end = part + key_def->part_count;
@@ -605,6 +616,41 @@ tuple_format_destroy_fields(struct tuple_format *format)
json_tree_destroy(&format->fields);
}
+static struct functional_handle *
+functional_handle_new(const struct key_def *key_def)
+{
+ assert(key_def->functional_fid > 0);
+ struct functional_handle *handle = malloc(sizeof(*handle));
+ if (handle == NULL) {
+ diag_set(OutOfMemory, sizeof(*handle), "malloc", "handle");
+ return NULL;
+ }
+ handle->key_def = key_def_dup(key_def);
+ if (handle->key_def == NULL) {
+ free(handle);
+ return NULL;
+ }
+ /**
+ * The function pointer initialization may be delayed
+ * during recovery because functional index extractor
+ * didn't exists when _index space had been created.
+ */
+ handle->func = func_by_id(key_def->functional_fid);
+ if (handle->func != NULL)
+ func_ref(handle->func);
+ rlist_create(&handle->link);
+ return handle;
+}
+
+static void
+functional_handle_delete(struct functional_handle *handle)
+{
+ if (handle->func != NULL)
+ func_unref(handle->func);
+ key_def_delete(handle->key_def);
+ free(handle);
+}
+
static struct tuple_format *
tuple_format_alloc(struct key_def * const *keys, uint16_t key_count,
uint32_t space_field_count, struct tuple_dictionary *dict)
@@ -613,8 +659,18 @@ tuple_format_alloc(struct key_def * const *keys, uint16_t key_count,
uint32_t path_pool_size = 0;
uint32_t index_field_count = 0;
/* find max max field no */
+ struct rlist functional_handle;
+ rlist_create(&functional_handle);
for (uint16_t key_no = 0; key_no < key_count; ++key_no) {
const struct key_def *key_def = keys[key_no];
+ if (key_def->functional_fid > 0) {
+ struct functional_handle *handle =
+ functional_handle_new(key_def);
+ if (handle == NULL)
+ goto error;
+ rlist_add(&functional_handle, &handle->link);
+ continue;
+ }
const struct key_part *part = key_def->parts;
const struct key_part *pend = part + key_def->part_count;
for (; part < pend; part++) {
@@ -630,13 +686,12 @@ tuple_format_alloc(struct key_def * const *keys, uint16_t key_count,
if (format == NULL) {
diag_set(OutOfMemory, allocation_size, "malloc",
"tuple format");
- return NULL;
+ goto error;
}
if (json_tree_create(&format->fields) != 0) {
diag_set(OutOfMemory, 0, "json_lexer_create",
"tuple field tree");
- free(format);
- return NULL;
+ goto error;
}
for (uint32_t fieldno = 0; fieldno < field_count; fieldno++) {
struct tuple_field *field = tuple_field_new();
@@ -662,6 +717,8 @@ tuple_format_alloc(struct key_def * const *keys, uint16_t key_count,
format->dict = dict;
tuple_dictionary_ref(dict);
}
+ rlist_create(&format->functional_handle);
+ rlist_swap(&format->functional_handle, &functional_handle);
format->total_field_count = field_count;
format->required_fields = NULL;
format->fields_depth = 1;
@@ -672,7 +729,10 @@ tuple_format_alloc(struct key_def * const *keys, uint16_t key_count,
format->min_field_count = 0;
format->epoch = 0;
return format;
-error:
+error:;
+ struct functional_handle *handle, *tmp;
+ rlist_foreach_entry_safe(handle, &functional_handle, link, tmp)
+ functional_handle_delete(handle);
tuple_format_destroy_fields(format);
free(format);
return NULL;
@@ -683,6 +743,9 @@ static inline void
tuple_format_destroy(struct tuple_format *format)
{
free(format->required_fields);
+ struct functional_handle *handle, *tmp;
+ rlist_foreach_entry_safe(handle, &format->functional_handle, link, tmp)
+ functional_handle_delete(handle);
tuple_format_destroy_fields(format);
tuple_dictionary_unref(format->dict);
}
diff --git a/src/box/tuple_format.h b/src/box/tuple_format.h
index 8b37a48d7..9ca6fa53f 100644
--- a/src/box/tuple_format.h
+++ b/src/box/tuple_format.h
@@ -67,6 +67,32 @@ struct tuple;
struct tuple_extra;
struct tuple_format;
struct coll;
+struct func;
+
+/**
+ * A functional handle is a tuple_format extention used when
+ * functional index key is defined. The functional handle
+ * represents a single functional index reference and allows
+ * to perform a validation of the functional index extracted key.
+ */
+struct functional_handle {
+ struct rlist link;
+ /**
+ * A functional index extractor routine pointer.
+ * The functional_handle takes a reference for this
+ * function object when initialized.
+ * This field may be NULL during recovery, becuase
+ * the functions may be unregistered during format
+ * construction. Therefore, the initialization of
+ * functional_handle is delayed.
+ */
+ struct func *func;
+ /**
+ * The key definition that describe keys produced by
+ * functional index extractor.
+ */
+ struct key_def *key_def;
+};
/** Engine-specific tuple format methods. */
struct tuple_format_vtab {
@@ -248,6 +274,8 @@ struct tuple_format {
* tuple_field::token.
*/
struct json_tree fields;
+ /** Functional index handle. */
+ struct rlist functional_handle;
};
/**
@@ -351,6 +379,7 @@ tuple_format_new(struct tuple_format_vtab *vtab, void *engine,
struct tuple_dictionary *dict, bool is_temporary,
bool is_ephemeral);
+
/**
* Check, if @a format1 can store any tuples of @a format2. For
* example, if a field is not nullable in format1 and the same
diff --git a/src/box/tuple_hash.cc b/src/box/tuple_hash.cc
index 780e3d053..bddc572df 100644
--- a/src/box/tuple_hash.cc
+++ b/src/box/tuple_hash.cc
@@ -161,7 +161,7 @@ struct TupleHash
uint32_t total_size = 0;
const char *field = tuple_field_by_part(tuple,
key_def->parts,
- MULTIKEY_NONE);
+ MULTIKEY_NONE, false);
TupleFieldHash<TYPE, MORE_TYPES...>::
hash(&field, &h, &carry, &total_size);
return PMurHash32_Result(h, carry, total_size);
@@ -175,7 +175,7 @@ struct TupleHash<FIELD_TYPE_UNSIGNED> {
assert(!key_def->is_multikey);
const char *field = tuple_field_by_part(tuple,
key_def->parts,
- MULTIKEY_NONE);
+ MULTIKEY_NONE, false);
uint64_t val = mp_decode_uint(&field);
if (likely(val <= UINT32_MAX))
return val;
@@ -352,7 +352,8 @@ uint32_t
tuple_hash_key_part(uint32_t *ph1, uint32_t *pcarry, struct tuple *tuple,
struct key_part *part, int multikey_idx)
{
- const char *field = tuple_field_by_part(tuple, part, multikey_idx);
+ const char *field =
+ tuple_field_by_part(tuple, part, multikey_idx, false);
if (field == NULL)
return tuple_hash_null(ph1, pcarry);
return tuple_hash_field(ph1, pcarry, &field, part->coll);
@@ -365,6 +366,7 @@ tuple_hash_slowpath(struct tuple *tuple, struct key_def *key_def)
assert(has_json_paths == key_def->has_json_paths);
assert(has_optional_parts == key_def->has_optional_parts);
assert(!key_def->is_multikey);
+ assert(key_def->functional_fid == 0);
uint32_t h = HASH_SEED;
uint32_t carry = 0;
uint32_t total_size = 0;
diff --git a/src/box/vinyl.c b/src/box/vinyl.c
index a03132310..0d84247d0 100644
--- a/src/box/vinyl.c
+++ b/src/box/vinyl.c
@@ -701,6 +701,11 @@ vinyl_space_check_index_def(struct space *space, struct index_def *index_def)
return -1;
}
}
+ if (index_def->key_def->functional_fid > 0) {
+ diag_set(ClientError, ER_UNSUPPORTED, "Vinyl",
+ "functional indexes");
+ return -1;
+ }
return 0;
}
@@ -985,6 +990,9 @@ vinyl_index_def_change_requires_rebuild(struct index *index,
if (!old_def->opts.is_unique && new_def->opts.is_unique)
return true;
+ if (functional_def_cmp(old_def->opts.functional_def,
+ new_def->opts.functional_def) != 0)
+ return true;
assert(index_depends_on_pk(index));
const struct key_def *old_cmp_def = old_def->cmp_def;
@@ -3162,7 +3170,7 @@ vy_send_lsm(struct vy_join_ctx *ctx, struct vy_lsm_recovery_info *lsm_info)
/* Create key definition and tuple format. */
ctx->key_def = key_def_new(lsm_info->key_parts,
- lsm_info->key_part_count);
+ lsm_info->key_part_count, NULL);
if (ctx->key_def == NULL)
goto out;
ctx->format = vy_stmt_format_new(&ctx->env->stmt_env, &ctx->key_def, 1,
diff --git a/test/box/bitset.result b/test/box/bitset.result
index 78f74ec37..e8a99de71 100644
--- a/test/box/bitset.result
+++ b/test/box/bitset.result
@@ -1996,3 +1996,27 @@ _ = s:create_index('bitset', {type = 'bitset', parts = {{'[2][*]', 'unsigned'}}}
s:drop()
---
...
+-- Bitset index cannot be functional.
+s = box.schema.space.create('withdata')
+---
+...
+lua_code = [[function(tuple) return tuple[1] + tuple[2] end]]
+---
+...
+box.schema.func.create('sum', {body = lua_code, is_deterministic = true, is_sandboxed = true})
+---
+...
+_ = s:create_index('pk')
+---
+...
+_ = s:create_index('idx', {type = 'bitset', functional = {fid = box.func.sum.id}, parts = {{1, 'unsigned'}}})
+---
+- error: 'Can''t create or modify index ''idx'' in space ''withdata'': BITSET index
+ cannot be functional'
+...
+s:drop()
+---
+...
+box.schema.func.drop('sum')
+---
+...
diff --git a/test/box/bitset.test.lua b/test/box/bitset.test.lua
index eb013a1c0..5a3158d82 100644
--- a/test/box/bitset.test.lua
+++ b/test/box/bitset.test.lua
@@ -153,3 +153,12 @@ s = box.schema.space.create('test')
_ = s:create_index('primary')
_ = s:create_index('bitset', {type = 'bitset', parts = {{'[2][*]', 'unsigned'}}})
s:drop()
+
+-- Bitset index cannot be functional.
+s = box.schema.space.create('withdata')
+lua_code = [[function(tuple) return tuple[1] + tuple[2] end]]
+box.schema.func.create('sum', {body = lua_code, is_deterministic = true, is_sandboxed = true})
+_ = s:create_index('pk')
+_ = s:create_index('idx', {type = 'bitset', functional = {fid = box.func.sum.id}, parts = {{1, 'unsigned'}}})
+s:drop()
+box.schema.func.drop('sum')
diff --git a/test/box/hash.result b/test/box/hash.result
index 9f08c49b8..fabbff850 100644
--- a/test/box/hash.result
+++ b/test/box/hash.result
@@ -847,3 +847,27 @@ _ = s:create_index('hash', {type = 'hash', parts = {{'[2][*]', 'unsigned'}}})
s:drop()
---
...
+-- Hash index cannot be functional.
+s = box.schema.space.create('withdata')
+---
+...
+lua_code = [[function(tuple) return tuple[1] + tuple[2] end]]
+---
+...
+box.schema.func.create('sum', {body = lua_code, is_deterministic = true, is_sandboxed = true})
+---
+...
+_ = s:create_index('pk')
+---
+...
+_ = s:create_index('idx', {type = 'hash', functional = {fid = box.func.sum.id}, parts = {{1, 'unsigned'}}})
+---
+- error: 'Can''t create or modify index ''idx'' in space ''withdata'': HASH index
+ cannot be functional'
+...
+s:drop()
+---
+...
+box.schema.func.drop('sum')
+---
+...
diff --git a/test/box/hash.test.lua b/test/box/hash.test.lua
index 9801873c4..65b2f5071 100644
--- a/test/box/hash.test.lua
+++ b/test/box/hash.test.lua
@@ -353,3 +353,12 @@ s = box.schema.space.create('test')
_ = s:create_index('primary')
_ = s:create_index('hash', {type = 'hash', parts = {{'[2][*]', 'unsigned'}}})
s:drop()
+
+-- Hash index cannot be functional.
+s = box.schema.space.create('withdata')
+lua_code = [[function(tuple) return tuple[1] + tuple[2] end]]
+box.schema.func.create('sum', {body = lua_code, is_deterministic = true, is_sandboxed = true})
+_ = s:create_index('pk')
+_ = s:create_index('idx', {type = 'hash', functional = {fid = box.func.sum.id}, parts = {{1, 'unsigned'}}})
+s:drop()
+box.schema.func.drop('sum')
diff --git a/test/box/misc.result b/test/box/misc.result
index ec2c4fa95..2c5d2d249 100644
--- a/test/box/misc.result
+++ b/test/box/misc.result
@@ -524,6 +524,7 @@ t;
194: box.error.MULTIKEY_INDEX_MISMATCH
195: box.error.CREATE_CK_CONSTRAINT
196: box.error.CK_CONSTRAINT_FAILED
+ 197: box.error.FUNCTIONAL_EXTRACTOR
...
test_run:cmd("setopt delimiter ''");
---
diff --git a/test/box/rtree_misc.result b/test/box/rtree_misc.result
index 6e48bacc7..b6840627f 100644
--- a/test/box/rtree_misc.result
+++ b/test/box/rtree_misc.result
@@ -682,3 +682,27 @@ _ = s:create_index('rtree', {type = 'rtree', parts = {{'[2][*]', 'array'}}})
s:drop()
---
...
+-- Rtree index cannot be functional.
+s = box.schema.space.create('withdata')
+---
+...
+lua_code = [[function(tuple) return {tuple[1] + tuple[2], tuple[1] - tuple[2]} end]]
+---
+...
+box.schema.func.create('fextract', {body = lua_code, is_deterministic = true, is_sandboxed = true})
+---
+...
+_ = s:create_index('pk')
+---
+...
+_ = s:create_index('idx', {type = 'rtree', functional = {fid = box.func.fextract.id}, parts = {{1, 'array'}}})
+---
+- error: 'Can''t create or modify index ''idx'' in space ''withdata'': RTREE index
+ cannot be functional'
+...
+s:drop()
+---
+...
+box.schema.func.drop('fextract')
+---
+...
diff --git a/test/box/rtree_misc.test.lua b/test/box/rtree_misc.test.lua
index 000a928e8..efb03f5bc 100644
--- a/test/box/rtree_misc.test.lua
+++ b/test/box/rtree_misc.test.lua
@@ -243,3 +243,12 @@ s = box.schema.space.create('test')
_ = s:create_index('primary')
_ = s:create_index('rtree', {type = 'rtree', parts = {{'[2][*]', 'array'}}})
s:drop()
+
+-- Rtree index cannot be functional.
+s = box.schema.space.create('withdata')
+lua_code = [[function(tuple) return {tuple[1] + tuple[2], tuple[1] - tuple[2]} end]]
+box.schema.func.create('fextract', {body = lua_code, is_deterministic = true, is_sandboxed = true})
+_ = s:create_index('pk')
+_ = s:create_index('idx', {type = 'rtree', functional = {fid = box.func.fextract.id}, parts = {{1, 'array'}}})
+s:drop()
+box.schema.func.drop('fextract')
diff --git a/test/engine/engine.cfg b/test/engine/engine.cfg
index 9f07629b4..826e8eb1f 100644
--- a/test/engine/engine.cfg
+++ b/test/engine/engine.cfg
@@ -2,7 +2,10 @@
"*": {
"memtx": {"engine": "memtx"},
"vinyl": {"engine": "vinyl"}
- }
+ },
+ "functional.test.lua": {
+ "memtx": {"engine": "memtx"}
+ }
}
diff --git a/test/engine/functional.result b/test/engine/functional.result
new file mode 100644
index 000000000..b707455fd
--- /dev/null
+++ b/test/engine/functional.result
@@ -0,0 +1,689 @@
+test_run = require('test_run').new()
+---
+...
+engine = test_run:get_cfg('engine')
+---
+...
+--
+-- gh-1260: Funclional indexes.
+--
+s = box.schema.space.create('withdata', {engine = engine})
+---
+...
+lua_code = [[function(tuple) return {tuple[1] + tuple[2]} end]]
+---
+...
+lua_code2 = [[function(tuple) return {tuple[1] + tuple[2], 2 * tuple[1] + tuple[2]} end]]
+---
+...
+box.schema.func.create('sum_nonpersistent')
+---
+...
+box.schema.func.create('sum_ivaliddef1', {body = lua_code, is_deterministic = false, is_sandboxed = true})
+---
+...
+box.schema.func.create('sum_ivaliddef2', {body = lua_code, is_deterministic = true, is_sandboxed = false})
+---
+...
+box.schema.func.create('sum', {body = lua_code, is_deterministic = true, is_sandboxed = true})
+---
+...
+box.schema.func.create('sums', {body = lua_code2, is_deterministic = true, is_sandboxed = true})
+---
+...
+-- Functional index can't be primary.
+_ = s:create_index('idx', {functional = {fid = box.func.sum.id}, parts = {{1, 'unsigned'}}})
+---
+- error: 'Can''t create or modify index ''idx'' in space ''withdata'': primary key
+ cannot be functional'
+...
+pk = s:create_index('pk')
+---
+...
+-- Invalid fid.
+_ = s:create_index('idx', {functional = {fid = 6666}, parts = {{1, 'unsigned'}}})
+---
+- error: 'Wrong index options (field 4): referenced function doesn''t exists'
+...
+-- Can't use non-persistent function in functional index.
+_ = s:create_index('idx', {functional = {fid = box.func.sum_nonpersistent.id}, parts = {{1, 'unsigned'}}})
+---
+- error: 'Wrong index options (field 0): referenced function doesn''t satisfy functional
+ index constraints'
+...
+-- Can't use non-deterministic function in functional index.
+_ = s:create_index('idx', {functional = {fid = box.func.sum_ivaliddef1.id}, parts = {{1, 'unsigned'}}})
+---
+- error: 'Wrong index options (field 0): referenced function doesn''t satisfy functional
+ index constraints'
+...
+-- Can't use non-sandboxed function in functional index.
+_ = s:create_index('idx', {functional = {fid = box.func.sum_ivaliddef2.id}, parts = {{1, 'unsigned'}}})
+---
+- error: 'Wrong index options (field 0): referenced function doesn''t satisfy functional
+ index constraints'
+...
+-- Can't use non-sequential parts in returned key definition.
+_ = s:create_index('idx', {functional = {fid = box.func.sums.id}, parts = {{1, 'unsigned'}, {3, 'unsigned'}}})
+---
+- error: 'Wrong index options (field 0): invalid functional key definition'
+...
+-- Can't use parts started not by 1 field.
+_ = s:create_index('idx', {functional = {fid = box.func.sums.id}, parts = {{2, 'unsigned'}, {3, 'unsigned'}}})
+---
+- error: 'Wrong index options (field 0): invalid functional key definition'
+...
+-- Can't use JSON paths in returned key definiton.
+_ = s:create_index('idx', {functional = {fid = box.func.sums.id}, parts = {{"[1]data", 'unsigned'}}})
+---
+- error: 'Wrong index options (field 0): invalid functional key definition'
+...
+-- Can't drop a function referenced by functional index.
+idx = s:create_index('idx', {unique = true, functional = {fid = box.func.sum.id}, parts = {{1, 'unsigned'}}})
+---
+...
+box.schema.func.drop('sum')
+---
+- error: 'Can''t drop function 1: function has references'
+...
+box.snapshot()
+---
+- ok
+...
+test_run:cmd("restart server default")
+box.schema.func.drop('sum')
+---
+- error: 'Can''t drop function 1: function has references'
+...
+s = box.space.withdata
+---
+...
+idx = s.index.idx
+---
+...
+idx:drop()
+---
+...
+box.schema.func.drop('sum')
+---
+...
+test_run = require('test_run').new()
+---
+...
+engine = test_run:get_cfg('engine')
+---
+...
+-- Invalid functional index extractor routine return: the extractor must return keys.
+lua_code = [[function(tuple) return "hello" end]]
+---
+...
+box.schema.func.create('invalidreturn0', {body = lua_code, is_deterministic = true, is_sandboxed = true})
+---
+...
+idx = s:create_index('idx', {functional = {fid = box.func.invalidreturn0.id}, parts = {{1, 'unsigned'}}})
+---
+...
+s:insert({1})
+---
+- error: 'Functional index extractor ''invalidreturn0'' error: returned key type is
+ invalid'
+...
+idx:drop()
+---
+...
+-- Invalid functional index extractor routine return: a stirng instead of unsigned
+lua_code = [[function(tuple) return {"hello"} end]]
+---
+...
+box.schema.func.create('invalidreturn1', {body = lua_code, is_deterministic = true, is_sandboxed = true})
+---
+...
+idx = s:create_index('idx', {functional = {fid = box.func.invalidreturn1.id}, parts = {{1, 'unsigned'}}})
+---
+...
+s:insert({1})
+---
+- error: 'Functional index extractor ''invalidreturn1'' error: Supplied key type of
+ part 0 does not match index part type: expected unsigned'
+...
+idx:drop()
+---
+...
+-- Invalid functional index extractor routine return: undefined multikey return.
+lua_code = [[function(tuple) return {"hello", "world"}, {"my", "hart"} end]]
+---
+...
+box.schema.func.create('invalidreturn2', {body = lua_code, is_deterministic = true, is_sandboxed = true})
+---
+...
+idx = s:create_index('idx', {functional = {fid = box.func.invalidreturn2.id}, parts = {{1, 'string'}, {2, 'string'}}})
+---
+...
+s:insert({1})
+---
+- error: 'Functional index extractor ''invalidreturn2'' error: to many keys were returned'
+...
+idx:drop()
+---
+...
+-- Invalid functional index extractor routine return: the second returned key invalid.
+lua_code = [[function(tuple) return {"hello", "world"}, {1, 2} end]]
+---
+...
+box.schema.func.create('invalidreturn3', {body = lua_code, is_deterministic = true, is_sandboxed = true})
+---
+...
+idx = s:create_index('idx', {functional = {fid = box.func.invalidreturn3.id, is_multikey = true}, parts = {{1, 'unsigned'}, {2, 'unsigned'}}})
+---
+...
+s:insert({1})
+---
+- error: 'Functional index extractor ''invalidreturn3'' error: Supplied key type of
+ part 0 does not match index part type: expected unsigned'
+...
+idx:drop()
+---
+...
+-- Invalid functional extractor: runtime extractor error
+test_run:cmd("setopt delimiter ';'")
+---
+- true
+...
+lua_code = [[function(tuple)
+ local json = require('json')
+ return json.encode(tuple)
+ end]]
+test_run:cmd("setopt delimiter ''");
+---
+...
+box.schema.func.create('runtimeerror', {body = lua_code, is_deterministic = true, is_sandboxed = true})
+---
+...
+idx = s:create_index('idx', {functional = {fid = box.func.runtimeerror.id}, parts = {{1, 'string'}}})
+---
+...
+s:insert({1})
+---
+- error: 'Functional index extractor ''runtimeerror'' error: [string "return function(tuple) local
+ ..."]:1: attempt to call global ''require'' (a nil value)'
+...
+idx:drop()
+---
+...
+-- Remove old persistent functions
+for _, v in pairs(box.func) do if v.is_persistent then box.schema.func.drop(v.name) end end
+---
+...
+s:drop()
+---
+...
+-- Functional test cases.
+s = box.schema.space.create('withdata', {engine = engine})
+---
+...
+lua_code = [[function(tuple) return {tuple[1] + tuple[2]} end]]
+---
+...
+box.schema.func.create('extract', {body = lua_code, is_deterministic = true, is_sandboxed = true})
+---
+...
+pk = s:create_index('pk')
+---
+...
+idx = s:create_index('idx', {unique = true, functional = {fid = box.func.extract.id}, parts = {{1, 'integer'}}})
+---
+...
+s:insert({1, 2})
+---
+- [1, 2]
+...
+s:insert({2, 1})
+---
+- error: Duplicate key exists in unique index 'idx' in space 'withdata'
+...
+idx:get(3)
+---
+- [1, 2]
+...
+idx:delete(3)
+---
+- [1, 2]
+...
+s:select()
+---
+- []
+...
+s:insert({2, 1})
+---
+- [2, 1]
+...
+idx:get(3)
+---
+- [2, 1]
+...
+s:drop()
+---
+...
+box.schema.func.drop('extract')
+---
+- error: 'Can''t drop function 1: function has references'
+...
+collectgarbage()
+---
+- 0
+...
+box.schema.func.drop('extract')
+---
+...
+-- Multikey functional index.
+s = box.schema.space.create('withdata', {engine = engine})
+---
+...
+lua_code = [[function(tuple) return {tuple[1] + tuple[2]}, {tuple[1] + tuple[2]}, {tuple[1]} end]]
+---
+...
+box.schema.func.create('extract', {body = lua_code, is_deterministic = true, is_sandboxed = true})
+---
+...
+pk = s:create_index('pk')
+---
+...
+idx = s:create_index('idx', {unique = true, functional = {fid = box.func.extract.id, is_multikey = true}, parts = {{1, 'integer'}}})
+---
+...
+s:insert({1, 2})
+---
+- [1, 2]
+...
+s:insert({3, 5})
+---
+- error: Duplicate key exists in unique index 'idx' in space 'withdata'
+...
+s:insert({5, 3})
+---
+- [5, 3]
+...
+idx:select()
+---
+- - [1, 2]
+ - [1, 2]
+ - [5, 3]
+ - [5, 3]
+...
+idx:get(8)
+---
+- [5, 3]
+...
+idx:get(3)
+---
+- [1, 2]
+...
+idx:get(1)
+---
+- [1, 2]
+...
+idx:get(5)
+---
+- [5, 3]
+...
+s:drop()
+---
+...
+collectgarbage()
+---
+- 0
+...
+box.schema.func.drop('extract')
+---
+...
+-- Multikey multipart functional index.
+s = box.schema.space.create('withdata', {engine = engine})
+---
+...
+lua_code = [[function(tuple) return {600 + tuple[1], 600 + tuple[2]}, {500 + tuple[1], 500 + tuple[2]} end]]
+---
+...
+box.schema.func.create('extract', {body = lua_code, is_deterministic = true, is_sandboxed = true})
+---
+...
+pk = s:create_index('pk')
+---
+...
+idx = s:create_index('idx', {unique = true, functional = {fid = box.func.extract.id, is_multikey = true}, parts = {{1, 'integer'}, {2, 'integer'}}})
+---
+...
+s:insert({1, 2})
+---
+- [1, 2]
+...
+s:insert({2, 1})
+---
+- [2, 1]
+...
+s:insert({3, 3})
+---
+- [3, 3]
+...
+idx:select({600}, {iterator = "GE"})
+---
+- - [1, 2]
+ - [2, 1]
+ - [3, 3]
+...
+idx:get({603, 603})
+---
+- [3, 3]
+...
+idx:select({503}, {iterator = "LE"})
+---
+- - [3, 3]
+ - [2, 1]
+ - [1, 2]
+...
+s:drop()
+---
+...
+collectgarbage()
+---
+- 0
+...
+box.schema.func.drop('extract')
+---
+...
+-- Multikey non-unique functional index.
+s = box.schema.space.create('withdata', {engine = engine})
+---
+...
+lua_code = [[function(tuple) return {500 + tuple[1]}, {500 + tuple[2]}, {500 + tuple[2]} end]]
+---
+...
+box.schema.func.create('extract', {body = lua_code, is_deterministic = true, is_sandboxed = true})
+---
+...
+pk = s:create_index('pk')
+---
+...
+idx = s:create_index('idx', {unique = false, functional = {fid = box.func.extract.id, is_multikey = true}, parts = {{1, 'integer'}}})
+---
+...
+s:insert({1, 2})
+---
+- [1, 2]
+...
+s:insert({2, 1})
+---
+- [2, 1]
+...
+idx:select({501})
+---
+- - [1, 2]
+ - [2, 1]
+...
+idx:select({502})
+---
+- - [1, 2]
+ - [2, 1]
+...
+s:replace({1, 3})
+---
+- [1, 3]
+...
+idx:select({501})
+---
+- - [1, 3]
+ - [2, 1]
+...
+idx:select({502})
+---
+- - [2, 1]
+...
+idx:select({503})
+---
+- - [1, 3]
+...
+box.snapshot()
+---
+- ok
+...
+test_run:cmd("restart server default")
+s = box.space.withdata
+---
+...
+idx = s.index.idx
+---
+...
+idx:select({501})
+---
+- - [1, 3]
+ - [2, 1]
+...
+idx:select({502})
+---
+- - [2, 1]
+...
+idx:select({503})
+---
+- - [1, 3]
+...
+s:replace({1, 2})
+---
+- [1, 2]
+...
+idx:select({501})
+---
+- - [1, 2]
+ - [2, 1]
+...
+idx:select({502})
+---
+- - [1, 2]
+ - [2, 1]
+...
+idx:select({503})
+---
+- []
+...
+s:drop()
+---
+...
+collectgarbage()
+---
+- 0
+...
+box.schema.func.drop('extract')
+---
+...
+-- Multikey UTF-8 address extractor
+test_run = require('test_run').new()
+---
+...
+engine = test_run:get_cfg('engine')
+---
+...
+s = box.schema.space.create('withdata', {engine = engine})
+---
+...
+s:format({{name = 'name', type = 'string'}, {name = 'address', type = 'string'}})
+---
+...
+pk = s:create_index('name', {parts = {1, 'string'}})
+---
+...
+s:insert({"James", "SIS Building Lambeth London UK"})
+---
+- ['James', 'SIS Building Lambeth London UK']
+...
+s:insert({"Sherlock", "221B Baker St Marylebone London NW1 6XE UK"})
+---
+- ['Sherlock', '221B Baker St Marylebone London NW1 6XE UK']
+...
+-- Create functional index on space with data
+test_run:cmd("setopt delimiter ';'")
+---
+- true
+...
+lua_code = [[function(tuple)
+ local address = string.split(tuple.address)
+ local ret = {}
+ for _, v in pairs(address) do table.insert(ret, {utf8.upper(v)}) end
+ return unpack(ret)
+ end]]
+test_run:cmd("setopt delimiter ''");
+---
+...
+box.schema.func.create('addr_extractor', {body = lua_code, is_deterministic = true, is_sandboxed = true})
+---
+...
+idx = s:create_index('addr', {unique = false, functional = {fid = box.func.addr_extractor.id, is_multikey = true}, parts = {{1, 'string', collation = 'unicode_ci'}}})
+---
+...
+idx:select('uk')
+---
+- - ['James', 'SIS Building Lambeth London UK']
+ - ['Sherlock', '221B Baker St Marylebone London NW1 6XE UK']
+...
+idx:select('Sis')
+---
+- - ['James', 'SIS Building Lambeth London UK']
+...
+s:drop()
+---
+...
+collectgarbage()
+---
+- 0
+...
+box.schema.func.drop('addr_extractor')
+---
+...
+-- Partial index with functional index extractor
+s = box.schema.space.create('withdata', {engine = engine})
+---
+...
+pk = s:create_index('pk')
+---
+...
+lua_code = [[function(tuple) if tuple[1] % 2 == 1 then return {tuple[1]} end end]]
+---
+...
+box.schema.func.create('extract', {body = lua_code, is_deterministic = true, is_sandboxed = true})
+---
+...
+idx = s:create_index('idx', {unique = true, functional = {fid = box.func.extract.id, is_multikey = true}, parts = {{1, 'integer'}}})
+---
+...
+s:insert({1})
+---
+- [1]
+...
+s:insert({2})
+---
+- [2]
+...
+s:insert({3})
+---
+- [3]
+...
+s:insert({4})
+---
+- [4]
+...
+idx:select()
+---
+- - [1]
+ - [3]
+...
+s:drop()
+---
+...
+collectgarbage()
+---
+- 0
+...
+box.schema.func.drop('extract')
+---
+...
+-- Return nil from functional index extractor.
+s = box.schema.space.create('withdata', {engine = engine})
+---
+...
+pk = s:create_index('pk')
+---
+...
+lua_code = [[function(tuple) return {nil} end]]
+---
+...
+box.schema.func.create('extract', {body = lua_code, is_deterministic = true, is_sandboxed = true})
+---
+...
+idx = s:create_index('idx', {unique = false, functional = {fid = box.func.extract.id}, parts = {{1, 'integer', is_nullable = true}}})
+---
+...
+s:insert({1})
+---
+- error: 'Functional index extractor ''extract'' error: Invalid key part count in
+ an exact match (expected 1, got 0)'
+...
+s:drop()
+---
+...
+collectgarbage()
+---
+- 0
+...
+box.schema.func.drop('extract')
+---
+...
+-- Multiple functional indexes. tuple with format
+s = box.schema.space.create('withdata', {engine = engine})
+---
+...
+s:format({{name = 'a', type = 'integer'}, {name = 'b', type = 'integer'}})
+---
+...
+lua_code = [[function(tuple) return {tuple.a + tuple.b} end]]
+---
+...
+box.schema.func.create('sum', {body = lua_code, is_deterministic = true, is_sandboxed = true})
+---
+...
+lua_code = [[function(tuple) return {tuple.a - tuple.b} end]]
+---
+...
+box.schema.func.create('sub', {body = lua_code, is_deterministic = true, is_sandboxed = true})
+---
+...
+pk = s:create_index('pk')
+---
+...
+idx1 = s:create_index('sum_idx', {unique = true, functional = {fid = box.func.sum.id}, parts = {{1, 'integer'}}})
+---
+...
+idx2 = s:create_index('sub_idx', {unique = true, functional = {fid = box.func.sub.id}, parts = {{1, 'integer'}}})
+---
+...
+s:insert({4, 1})
+---
+- [4, 1]
+...
+idx1:get(5)
+---
+- [4, 1]
+...
+idx2:get(3)
+---
+- [4, 1]
+...
+s:drop()
+---
+...
+collectgarbage()
+---
+- 0
+...
+box.schema.func.drop('sum')
+---
+...
+box.schema.func.drop('sub')
+---
+...
diff --git a/test/engine/functional.test.lua b/test/engine/functional.test.lua
new file mode 100644
index 000000000..438b4f578
--- /dev/null
+++ b/test/engine/functional.test.lua
@@ -0,0 +1,241 @@
+test_run = require('test_run').new()
+engine = test_run:get_cfg('engine')
+
+--
+-- gh-1260: Funclional indexes.
+--
+s = box.schema.space.create('withdata', {engine = engine})
+lua_code = [[function(tuple) return {tuple[1] + tuple[2]} end]]
+lua_code2 = [[function(tuple) return {tuple[1] + tuple[2], 2 * tuple[1] + tuple[2]} end]]
+box.schema.func.create('sum_nonpersistent')
+box.schema.func.create('sum_ivaliddef1', {body = lua_code, is_deterministic = false, is_sandboxed = true})
+box.schema.func.create('sum_ivaliddef2', {body = lua_code, is_deterministic = true, is_sandboxed = false})
+
+box.schema.func.create('sum', {body = lua_code, is_deterministic = true, is_sandboxed = true})
+box.schema.func.create('sums', {body = lua_code2, is_deterministic = true, is_sandboxed = true})
+
+-- Functional index can't be primary.
+_ = s:create_index('idx', {functional = {fid = box.func.sum.id}, parts = {{1, 'unsigned'}}})
+pk = s:create_index('pk')
+-- Invalid fid.
+_ = s:create_index('idx', {functional = {fid = 6666}, parts = {{1, 'unsigned'}}})
+-- Can't use non-persistent function in functional index.
+_ = s:create_index('idx', {functional = {fid = box.func.sum_nonpersistent.id}, parts = {{1, 'unsigned'}}})
+-- Can't use non-deterministic function in functional index.
+_ = s:create_index('idx', {functional = {fid = box.func.sum_ivaliddef1.id}, parts = {{1, 'unsigned'}}})
+-- Can't use non-sandboxed function in functional index.
+_ = s:create_index('idx', {functional = {fid = box.func.sum_ivaliddef2.id}, parts = {{1, 'unsigned'}}})
+-- Can't use non-sequential parts in returned key definition.
+_ = s:create_index('idx', {functional = {fid = box.func.sums.id}, parts = {{1, 'unsigned'}, {3, 'unsigned'}}})
+-- Can't use parts started not by 1 field.
+_ = s:create_index('idx', {functional = {fid = box.func.sums.id}, parts = {{2, 'unsigned'}, {3, 'unsigned'}}})
+-- Can't use JSON paths in returned key definiton.
+_ = s:create_index('idx', {functional = {fid = box.func.sums.id}, parts = {{"[1]data", 'unsigned'}}})
+
+-- Can't drop a function referenced by functional index.
+idx = s:create_index('idx', {unique = true, functional = {fid = box.func.sum.id}, parts = {{1, 'unsigned'}}})
+box.schema.func.drop('sum')
+box.snapshot()
+test_run:cmd("restart server default")
+box.schema.func.drop('sum')
+s = box.space.withdata
+idx = s.index.idx
+idx:drop()
+box.schema.func.drop('sum')
+
+test_run = require('test_run').new()
+engine = test_run:get_cfg('engine')
+
+-- Invalid functional index extractor routine return: the extractor must return keys.
+lua_code = [[function(tuple) return "hello" end]]
+box.schema.func.create('invalidreturn0', {body = lua_code, is_deterministic = true, is_sandboxed = true})
+idx = s:create_index('idx', {functional = {fid = box.func.invalidreturn0.id}, parts = {{1, 'unsigned'}}})
+s:insert({1})
+idx:drop()
+
+-- Invalid functional index extractor routine return: a stirng instead of unsigned
+lua_code = [[function(tuple) return {"hello"} end]]
+box.schema.func.create('invalidreturn1', {body = lua_code, is_deterministic = true, is_sandboxed = true})
+idx = s:create_index('idx', {functional = {fid = box.func.invalidreturn1.id}, parts = {{1, 'unsigned'}}})
+s:insert({1})
+idx:drop()
+
+-- Invalid functional index extractor routine return: undefined multikey return.
+lua_code = [[function(tuple) return {"hello", "world"}, {"my", "hart"} end]]
+box.schema.func.create('invalidreturn2', {body = lua_code, is_deterministic = true, is_sandboxed = true})
+idx = s:create_index('idx', {functional = {fid = box.func.invalidreturn2.id}, parts = {{1, 'string'}, {2, 'string'}}})
+s:insert({1})
+idx:drop()
+
+-- Invalid functional index extractor routine return: the second returned key invalid.
+lua_code = [[function(tuple) return {"hello", "world"}, {1, 2} end]]
+box.schema.func.create('invalidreturn3', {body = lua_code, is_deterministic = true, is_sandboxed = true})
+idx = s:create_index('idx', {functional = {fid = box.func.invalidreturn3.id, is_multikey = true}, parts = {{1, 'unsigned'}, {2, 'unsigned'}}})
+s:insert({1})
+idx:drop()
+
+-- Invalid functional extractor: runtime extractor error
+test_run:cmd("setopt delimiter ';'")
+lua_code = [[function(tuple)
+ local json = require('json')
+ return json.encode(tuple)
+ end]]
+test_run:cmd("setopt delimiter ''");
+box.schema.func.create('runtimeerror', {body = lua_code, is_deterministic = true, is_sandboxed = true})
+idx = s:create_index('idx', {functional = {fid = box.func.runtimeerror.id}, parts = {{1, 'string'}}})
+s:insert({1})
+idx:drop()
+
+-- Remove old persistent functions
+for _, v in pairs(box.func) do if v.is_persistent then box.schema.func.drop(v.name) end end
+s:drop()
+
+-- Functional test cases.
+s = box.schema.space.create('withdata', {engine = engine})
+lua_code = [[function(tuple) return {tuple[1] + tuple[2]} end]]
+box.schema.func.create('extract', {body = lua_code, is_deterministic = true, is_sandboxed = true})
+pk = s:create_index('pk')
+idx = s:create_index('idx', {unique = true, functional = {fid = box.func.extract.id}, parts = {{1, 'integer'}}})
+s:insert({1, 2})
+s:insert({2, 1})
+idx:get(3)
+idx:delete(3)
+s:select()
+s:insert({2, 1})
+idx:get(3)
+s:drop()
+box.schema.func.drop('extract')
+collectgarbage()
+box.schema.func.drop('extract')
+
+-- Multikey functional index.
+s = box.schema.space.create('withdata', {engine = engine})
+lua_code = [[function(tuple) return {tuple[1] + tuple[2]}, {tuple[1] + tuple[2]}, {tuple[1]} end]]
+box.schema.func.create('extract', {body = lua_code, is_deterministic = true, is_sandboxed = true})
+pk = s:create_index('pk')
+idx = s:create_index('idx', {unique = true, functional = {fid = box.func.extract.id, is_multikey = true}, parts = {{1, 'integer'}}})
+s:insert({1, 2})
+s:insert({3, 5})
+s:insert({5, 3})
+idx:select()
+idx:get(8)
+idx:get(3)
+idx:get(1)
+idx:get(5)
+s:drop()
+collectgarbage()
+box.schema.func.drop('extract')
+
+-- Multikey multipart functional index.
+s = box.schema.space.create('withdata', {engine = engine})
+lua_code = [[function(tuple) return {600 + tuple[1], 600 + tuple[2]}, {500 + tuple[1], 500 + tuple[2]} end]]
+box.schema.func.create('extract', {body = lua_code, is_deterministic = true, is_sandboxed = true})
+pk = s:create_index('pk')
+idx = s:create_index('idx', {unique = true, functional = {fid = box.func.extract.id, is_multikey = true}, parts = {{1, 'integer'}, {2, 'integer'}}})
+s:insert({1, 2})
+s:insert({2, 1})
+s:insert({3, 3})
+idx:select({600}, {iterator = "GE"})
+idx:get({603, 603})
+idx:select({503}, {iterator = "LE"})
+s:drop()
+collectgarbage()
+box.schema.func.drop('extract')
+
+-- Multikey non-unique functional index.
+s = box.schema.space.create('withdata', {engine = engine})
+lua_code = [[function(tuple) return {500 + tuple[1]}, {500 + tuple[2]}, {500 + tuple[2]} end]]
+box.schema.func.create('extract', {body = lua_code, is_deterministic = true, is_sandboxed = true})
+pk = s:create_index('pk')
+idx = s:create_index('idx', {unique = false, functional = {fid = box.func.extract.id, is_multikey = true}, parts = {{1, 'integer'}}})
+s:insert({1, 2})
+s:insert({2, 1})
+idx:select({501})
+idx:select({502})
+s:replace({1, 3})
+idx:select({501})
+idx:select({502})
+idx:select({503})
+box.snapshot()
+test_run:cmd("restart server default")
+s = box.space.withdata
+idx = s.index.idx
+idx:select({501})
+idx:select({502})
+idx:select({503})
+s:replace({1, 2})
+idx:select({501})
+idx:select({502})
+idx:select({503})
+s:drop()
+collectgarbage()
+box.schema.func.drop('extract')
+
+-- Multikey UTF-8 address extractor
+test_run = require('test_run').new()
+engine = test_run:get_cfg('engine')
+s = box.schema.space.create('withdata', {engine = engine})
+s:format({{name = 'name', type = 'string'}, {name = 'address', type = 'string'}})
+pk = s:create_index('name', {parts = {1, 'string'}})
+s:insert({"James", "SIS Building Lambeth London UK"})
+s:insert({"Sherlock", "221B Baker St Marylebone London NW1 6XE UK"})
+-- Create functional index on space with data
+test_run:cmd("setopt delimiter ';'")
+lua_code = [[function(tuple)
+ local address = string.split(tuple.address)
+ local ret = {}
+ for _, v in pairs(address) do table.insert(ret, {utf8.upper(v)}) end
+ return unpack(ret)
+ end]]
+test_run:cmd("setopt delimiter ''");
+box.schema.func.create('addr_extractor', {body = lua_code, is_deterministic = true, is_sandboxed = true})
+idx = s:create_index('addr', {unique = false, functional = {fid = box.func.addr_extractor.id, is_multikey = true}, parts = {{1, 'string', collation = 'unicode_ci'}}})
+idx:select('uk')
+idx:select('Sis')
+s:drop()
+collectgarbage()
+box.schema.func.drop('addr_extractor')
+
+-- Partial index with functional index extractor
+s = box.schema.space.create('withdata', {engine = engine})
+pk = s:create_index('pk')
+lua_code = [[function(tuple) if tuple[1] % 2 == 1 then return {tuple[1]} end end]]
+box.schema.func.create('extract', {body = lua_code, is_deterministic = true, is_sandboxed = true})
+idx = s:create_index('idx', {unique = true, functional = {fid = box.func.extract.id, is_multikey = true}, parts = {{1, 'integer'}}})
+s:insert({1})
+s:insert({2})
+s:insert({3})
+s:insert({4})
+idx:select()
+s:drop()
+collectgarbage()
+box.schema.func.drop('extract')
+
+-- Return nil from functional index extractor.
+s = box.schema.space.create('withdata', {engine = engine})
+pk = s:create_index('pk')
+lua_code = [[function(tuple) return {nil} end]]
+box.schema.func.create('extract', {body = lua_code, is_deterministic = true, is_sandboxed = true})
+idx = s:create_index('idx', {unique = false, functional = {fid = box.func.extract.id}, parts = {{1, 'integer', is_nullable = true}}})
+s:insert({1})
+s:drop()
+collectgarbage()
+box.schema.func.drop('extract')
+
+-- Multiple functional indexes. tuple with format
+s = box.schema.space.create('withdata', {engine = engine})
+s:format({{name = 'a', type = 'integer'}, {name = 'b', type = 'integer'}})
+lua_code = [[function(tuple) return {tuple.a + tuple.b} end]]
+box.schema.func.create('sum', {body = lua_code, is_deterministic = true, is_sandboxed = true})
+lua_code = [[function(tuple) return {tuple.a - tuple.b} end]]
+box.schema.func.create('sub', {body = lua_code, is_deterministic = true, is_sandboxed = true})
+pk = s:create_index('pk')
+idx1 = s:create_index('sum_idx', {unique = true, functional = {fid = box.func.sum.id}, parts = {{1, 'integer'}}})
+idx2 = s:create_index('sub_idx', {unique = true, functional = {fid = box.func.sub.id}, parts = {{1, 'integer'}}})
+s:insert({4, 1})
+idx1:get(5)
+idx2:get(3)
+s:drop()
+collectgarbage()
+box.schema.func.drop('sum')
+box.schema.func.drop('sub')
diff --git a/test/unit/luaT_tuple_new.c b/test/unit/luaT_tuple_new.c
index 0a16fa039..aaae9e46b 100644
--- a/test/unit/luaT_tuple_new.c
+++ b/test/unit/luaT_tuple_new.c
@@ -124,7 +124,7 @@ test_basic(struct lua_State *L)
part.nullable_action = ON_CONFLICT_ACTION_DEFAULT;
part.sort_order = SORT_ORDER_ASC;
part.path = NULL;
- struct key_def *key_def = key_def_new(&part, 1);
+ struct key_def *key_def = key_def_new(&part, 1, 0);
box_tuple_format_t *another_format = box_tuple_format_new(&key_def, 1);
key_def_delete(key_def);
diff --git a/test/unit/merger.test.c b/test/unit/merger.test.c
index b4a989a20..fe27d9345 100644
--- a/test/unit/merger.test.c
+++ b/test/unit/merger.test.c
@@ -214,7 +214,7 @@ test_merger(struct tuple_format *format)
merge_source_array_new(true),
};
- struct key_def *key_def = key_def_new(&key_part_unsigned, 1);
+ struct key_def *key_def = key_def_new(&key_part_unsigned, 1, 0);
struct merge_source *merger = merger_new(key_def, sources, source_count,
false);
key_def_delete(key_def);
@@ -252,7 +252,7 @@ test_basic()
plan(4);
header();
- struct key_def *key_def = key_def_new(&key_part_integer, 1);
+ struct key_def *key_def = key_def_new(&key_part_integer, 1, 0);
struct tuple_format *format = box_tuple_format_new(&key_def, 1);
assert(format != NULL);
@@ -273,7 +273,7 @@ main()
{
memory_init();
fiber_init(fiber_c_invoke);
- tuple_init(NULL);
+ tuple_init(NULL, NULL);
int rc = test_basic();
diff --git a/test/unit/tuple_bigref.c b/test/unit/tuple_bigref.c
index 20eab61f6..7d290fe93 100644
--- a/test/unit/tuple_bigref.c
+++ b/test/unit/tuple_bigref.c
@@ -143,7 +143,7 @@ main()
memory_init();
fiber_init(fiber_c_invoke);
- tuple_init(NULL);
+ tuple_init(NULL, NULL);
tuple_end = mp_encode_array(tuple_end, 1);
tuple_end = mp_encode_uint(tuple_end, 2);
diff --git a/test/unit/vy_iterators_helper.c b/test/unit/vy_iterators_helper.c
index 0d20f19ef..bea7f7f0a 100644
--- a/test/unit/vy_iterators_helper.c
+++ b/test/unit/vy_iterators_helper.c
@@ -18,7 +18,7 @@ vy_iterator_C_test_init(size_t cache_size)
memory_init();
fiber_init(fiber_c_invoke);
- tuple_init(NULL);
+ tuple_init(NULL, NULL);
vy_stmt_env_create(&stmt_env);
vy_cache_env_create(&cache_env, cord_slab_cache());
vy_cache_env_set_quota(&cache_env, cache_size);
diff --git a/test/vinyl/misc.result b/test/vinyl/misc.result
index b2aacdc55..00a0dde75 100644
--- a/test/vinyl/misc.result
+++ b/test/vinyl/misc.result
@@ -432,3 +432,26 @@ stat.bytes_compressed < stat.bytes / 10
s:drop()
---
...
+-- Vinyl doesn't support functional indexes.
+s = box.schema.space.create('withdata', {engine = 'vinyl'})
+---
+...
+lua_code = [[function(tuple) return tuple[1] + tuple[2] end]]
+---
+...
+box.schema.func.create('sum', {body = lua_code, is_deterministic = true, is_sandboxed = true})
+---
+...
+_ = s:create_index('pk')
+---
+...
+_ = s:create_index('idx', {functional = {fid = box.func.sum.id}, parts = {{1, 'unsigned'}}})
+---
+- error: Vinyl does not support functional indexes
+...
+s:drop()
+---
+...
+box.schema.func.drop('sum')
+---
+...
diff --git a/test/vinyl/misc.test.lua b/test/vinyl/misc.test.lua
index f8da578d0..45dae1599 100644
--- a/test/vinyl/misc.test.lua
+++ b/test/vinyl/misc.test.lua
@@ -182,3 +182,12 @@ test_run:wait_cond(function() return i:stat().disk.compaction.count > 0 end)
stat = i:stat().disk
stat.bytes_compressed < stat.bytes / 10
s:drop()
+
+-- Vinyl doesn't support functional indexes.
+s = box.schema.space.create('withdata', {engine = 'vinyl'})
+lua_code = [[function(tuple) return tuple[1] + tuple[2] end]]
+box.schema.func.create('sum', {body = lua_code, is_deterministic = true, is_sandboxed = true})
+_ = s:create_index('pk')
+_ = s:create_index('idx', {functional = {fid = box.func.sum.id}, parts = {{1, 'unsigned'}}})
+s:drop()
+box.schema.func.drop('sum')
--
2.21.0
^ permalink raw reply [flat|nested] 5+ messages in thread
end of thread, other threads:[~2019-06-24 14:27 UTC | newest]
Thread overview: 5+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
[not found] <cover.1561384554.git.kshcherbatov@tarantool.org>
2019-06-24 14:27 ` [PATCH v2 1/5] box: introduce tuple_extra infrastructure Kirill Shcherbatov
2019-06-24 14:27 ` [PATCH v2 2/5] box: introduce key_def->is_multikey flag Kirill Shcherbatov
2019-06-24 14:27 ` [PATCH v2 3/5] box: refactor key_validate_parts to return key_end Kirill Shcherbatov
2019-06-24 14:27 ` [PATCH v2 4/5] box: move the function hash to the tuple library Kirill Shcherbatov
2019-06-24 14:27 ` [PATCH v2 5/5] box: introduce functional indexes in memtx Kirill Shcherbatov
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox