From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: From: Kirill Shcherbatov Subject: [PATCH v9 3/6] box: introduce JSON Indexes Date: Sun, 3 Feb 2019 13:20:23 +0300 Message-Id: <899474d430fb62a79a0af1c83b5ad6262aec2923.1549187339.git.kshcherbatov@tarantool.org> In-Reply-To: References: MIME-Version: 1.0 Content-Transfer-Encoding: 8bit To: tarantool-patches@freelists.org, vdavydov.dev@gmail.com Cc: Kirill Shcherbatov List-ID: New JSON indexes allows to index documents content. At first, introduced new key_part fields path and path_len representing JSON path string specified by user. Modified tuple_format_use_key_part routine constructs corresponding tuple_fields chain in tuple_format::fields tree to indexed data. The resulting tree is used for type checking and for alloctating indexed fields offset slots. Then refined tuple_init_field_map routine logic parses tuple msgpack in depth using stack allocated on region and initialize field map with corresponding tuple_format::field if any. Finally, to proceed memory allocation for vinyl's secondary key restored by extracted keys loaded from disc without fields tree traversal, introduced format::min_tuple_size field - the size of tuple_format tuple as if all leaf fields are zero. Example: To create a new JSON index specify path to document data as a part of key_part: parts = {{3, 'str', path = '.FIO.fname', is_nullable = false}} idx = s:create_index('json_idx', {parts = parse}) idx:select("Ivanov") Part of #1012 --- src/box/alter.cc | 2 +- src/box/index_def.c | 10 +- src/box/key_def.c | 157 ++++++++-- src/box/key_def.h | 34 +- src/box/lua/space.cc | 5 + src/box/memtx_engine.c | 4 + src/box/sql.c | 1 + src/box/sql/build.c | 1 + src/box/sql/select.c | 3 +- src/box/sql/where.c | 1 + src/box/tuple_compare.cc | 7 +- src/box/tuple_extract_key.cc | 35 ++- src/box/tuple_format.c | 374 ++++++++++++++++++---- src/box/tuple_format.h | 60 +++- src/box/tuple_hash.cc | 2 +- src/box/vinyl.c | 4 + src/box/vy_log.c | 61 +++- src/box/vy_point_lookup.c | 4 +- src/box/vy_stmt.c | 202 +++++++++--- test/engine/json.result | 591 +++++++++++++++++++++++++++++++++++ test/engine/json.test.lua | 167 ++++++++++ 21 files changed, 1546 insertions(+), 179 deletions(-) create mode 100644 test/engine/json.result create mode 100644 test/engine/json.test.lua diff --git a/src/box/alter.cc b/src/box/alter.cc index 0589c9678..9656a4189 100644 --- a/src/box/alter.cc +++ b/src/box/alter.cc @@ -268,7 +268,7 @@ index_def_new_from_tuple(struct tuple *tuple, struct space *space) }); if (key_def_decode_parts(part_def, part_count, &parts, space->def->fields, - space->def->field_count) != 0) + space->def->field_count, &fiber()->gc) != 0) diag_raise(); key_def = key_def_new(part_def, part_count); if (key_def == NULL) diff --git a/src/box/index_def.c b/src/box/index_def.c index 2ba57ee9d..c52aa38d1 100644 --- a/src/box/index_def.c +++ b/src/box/index_def.c @@ -31,6 +31,8 @@ #include "index_def.h" #include "schema_def.h" #include "identifier.h" +#include "tuple_format.h" +#include "json/json.h" const char *index_type_strs[] = { "HASH", "TREE", "BITSET", "RTREE" }; @@ -278,8 +280,12 @@ index_def_is_valid(struct index_def *index_def, const char *space_name) * Courtesy to a user who could have made * a typo. */ - if (index_def->key_def->parts[i].fieldno == - index_def->key_def->parts[j].fieldno) { + struct key_part *part_a = &index_def->key_def->parts[i]; + struct key_part *part_b = &index_def->key_def->parts[j]; + if (part_a->fieldno == part_b->fieldno && + json_path_cmp(part_a->path, part_a->path_len, + part_b->path, part_b->path_len, + TUPLE_INDEX_BASE) == 0) { diag_set(ClientError, ER_MODIFY_INDEX, index_def->name, space_name, "same key part is indexed twice"); diff --git a/src/box/key_def.c b/src/box/key_def.c index dae3580e2..1b00945ca 100644 --- a/src/box/key_def.c +++ b/src/box/key_def.c @@ -28,6 +28,7 @@ * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ +#include "json/json.h" #include "key_def.h" #include "tuple_compare.h" #include "tuple_extract_key.h" @@ -35,6 +36,7 @@ #include "column_mask.h" #include "schema_def.h" #include "coll_id_cache.h" +#include "small/region.h" const char *sort_order_strs[] = { "asc", "desc", "undef" }; @@ -44,7 +46,8 @@ const struct key_part_def key_part_def_default = { COLL_NONE, false, ON_CONFLICT_ACTION_DEFAULT, - SORT_ORDER_ASC + SORT_ORDER_ASC, + NULL }; static int64_t @@ -59,6 +62,7 @@ part_type_by_name_wrapper(const char *str, uint32_t len) #define PART_OPT_NULLABILITY "is_nullable" #define PART_OPT_NULLABLE_ACTION "nullable_action" #define PART_OPT_SORT_ORDER "sort_order" +#define PART_OPT_PATH "path" const struct opt_def part_def_reg[] = { OPT_DEF_ENUM(PART_OPT_TYPE, field_type, struct key_part_def, type, @@ -71,19 +75,33 @@ const struct opt_def part_def_reg[] = { struct key_part_def, nullable_action, NULL), OPT_DEF_ENUM(PART_OPT_SORT_ORDER, sort_order, struct key_part_def, sort_order, NULL), + OPT_DEF(PART_OPT_PATH, OPT_STRPTR, struct key_part_def, path), OPT_END, }; struct key_def * key_def_dup(const struct key_def *src) { - size_t sz = key_def_sizeof(src->part_count); + size_t sz = 0; + for (uint32_t i = 0; i < src->part_count; i++) + sz += src->parts[i].path_len; + sz = key_def_sizeof(src->part_count, sz); struct key_def *res = (struct key_def *)malloc(sz); if (res == NULL) { diag_set(OutOfMemory, sz, "malloc", "res"); return NULL; } memcpy(res, src, sz); + /* + * Update the paths pointers so that they refer to the + * JSON strings bytes in the new allocation. + */ + for (uint32_t i = 0; i < src->part_count; i++) { + if (src->parts[i].path == NULL) + continue; + size_t path_offset = src->parts[i].path - (char *)src; + res->parts[i].path = (char *)res + path_offset; + } return res; } @@ -91,8 +109,16 @@ void key_def_swap(struct key_def *old_def, struct key_def *new_def) { assert(old_def->part_count == new_def->part_count); - for (uint32_t i = 0; i < new_def->part_count; i++) + for (uint32_t i = 0; i < new_def->part_count; i++) { SWAP(old_def->parts[i], new_def->parts[i]); + /* + * Paths are allocated as a part of key_def so + * we need to swap path pointers back - it's OK + * as paths aren't supposed to change. + */ + assert(old_def->parts[i].path_len == new_def->parts[i].path_len); + SWAP(old_def->parts[i].path, new_def->parts[i].path); + } SWAP(*old_def, *new_def); } @@ -115,24 +141,39 @@ static void key_def_set_part(struct key_def *def, uint32_t part_no, uint32_t fieldno, enum field_type type, enum on_conflict_action nullable_action, struct coll *coll, uint32_t coll_id, - enum sort_order sort_order) + enum sort_order sort_order, const char *path, + uint32_t path_len, char **path_pool) { assert(part_no < def->part_count); assert(type < field_type_MAX); def->is_nullable |= (nullable_action == ON_CONFLICT_ACTION_NONE); + def->has_json_paths |= path != NULL; def->parts[part_no].nullable_action = nullable_action; def->parts[part_no].fieldno = fieldno; def->parts[part_no].type = type; def->parts[part_no].coll = coll; def->parts[part_no].coll_id = coll_id; def->parts[part_no].sort_order = sort_order; + if (path != NULL) { + assert(path_pool != NULL); + def->parts[part_no].path = *path_pool; + *path_pool += path_len; + memcpy(def->parts[part_no].path, path, path_len); + def->parts[part_no].path_len = path_len; + } else { + def->parts[part_no].path = NULL; + def->parts[part_no].path_len = 0; + } column_mask_set_fieldno(&def->column_mask, fieldno); } struct key_def * key_def_new(const struct key_part_def *parts, uint32_t part_count) { - size_t sz = key_def_sizeof(part_count); + size_t sz = 0; + for (uint32_t i = 0; i < part_count; i++) + sz += parts[i].path != NULL ? strlen(parts[i].path) : 0; + sz = key_def_sizeof(part_count, sz); struct key_def *def = calloc(1, sz); if (def == NULL) { diag_set(OutOfMemory, sz, "malloc", "struct key_def"); @@ -142,6 +183,8 @@ key_def_new(const struct key_part_def *parts, uint32_t part_count) def->part_count = part_count; def->unique_part_count = part_count; + /* A pointer to the JSON paths data in the new key_def. */ + char *path_pool = (char *)def + key_def_sizeof(part_count, 0); for (uint32_t i = 0; i < part_count; i++) { const struct key_part_def *part = &parts[i]; struct coll *coll = NULL; @@ -155,16 +198,20 @@ key_def_new(const struct key_part_def *parts, uint32_t part_count) } coll = coll_id->coll; } + uint32_t path_len = part->path != NULL ? strlen(part->path) : 0; key_def_set_part(def, i, part->fieldno, part->type, part->nullable_action, coll, part->coll_id, - part->sort_order); + part->sort_order, part->path, path_len, + &path_pool); } + assert(path_pool == (char *)def + sz); key_def_set_cmp(def); return def; } -void -key_def_dump_parts(const struct key_def *def, struct key_part_def *parts) +int +key_def_dump_parts(const struct key_def *def, struct key_part_def *parts, + struct region *region) { for (uint32_t i = 0; i < def->part_count; i++) { const struct key_part *part = &def->parts[i]; @@ -174,13 +221,27 @@ key_def_dump_parts(const struct key_def *def, struct key_part_def *parts) part_def->is_nullable = key_part_is_nullable(part); part_def->nullable_action = part->nullable_action; part_def->coll_id = part->coll_id; + if (part->path != NULL) { + char *path = region_alloc(region, part->path_len + 1); + if (path == NULL) { + diag_set(OutOfMemory, part->path_len + 1, + "region", "part_def->path"); + return -1; + } + memcpy(path, part->path, part->path_len); + path[part->path_len] = '\0'; + part_def->path = path; + } else { + part_def->path = NULL; + } } + return 0; } box_key_def_t * box_key_def_new(uint32_t *fields, uint32_t *types, uint32_t part_count) { - size_t sz = key_def_sizeof(part_count); + size_t sz = key_def_sizeof(part_count, 0); struct key_def *key_def = calloc(1, sz); if (key_def == NULL) { diag_set(OutOfMemory, sz, "malloc", "struct key_def"); @@ -194,7 +255,8 @@ box_key_def_new(uint32_t *fields, uint32_t *types, uint32_t part_count) key_def_set_part(key_def, item, fields[item], (enum field_type)types[item], ON_CONFLICT_ACTION_DEFAULT, - NULL, COLL_NONE, SORT_ORDER_ASC); + NULL, COLL_NONE, SORT_ORDER_ASC, NULL, 0, + NULL); } key_def_set_cmp(key_def); return key_def; @@ -243,6 +305,11 @@ key_part_cmp(const struct key_part *parts1, uint32_t part_count1, if (key_part_is_nullable(part1) != key_part_is_nullable(part2)) return key_part_is_nullable(part1) < key_part_is_nullable(part2) ? -1 : 1; + int rc = json_path_cmp(part1->path, part1->path_len, + part2->path, part2->path_len, + TUPLE_INDEX_BASE); + if (rc != 0) + return rc; } return part_count1 < part_count2 ? -1 : part_count1 > part_count2; } @@ -253,8 +320,9 @@ key_def_update_optionality(struct key_def *def, uint32_t min_field_count) def->has_optional_parts = false; for (uint32_t i = 0; i < def->part_count; ++i) { struct key_part *part = &def->parts[i]; - def->has_optional_parts |= key_part_is_nullable(part) && - min_field_count < part->fieldno + 1; + def->has_optional_parts |= + (min_field_count < part->fieldno + 1 || + part->path != NULL) && key_part_is_nullable(part); /* * One optional part is enough to switch to new * comparators. @@ -274,8 +342,13 @@ key_def_snprint_parts(char *buf, int size, const struct key_part_def *parts, for (uint32_t i = 0; i < part_count; i++) { const struct key_part_def *part = &parts[i]; assert(part->type < field_type_MAX); - SNPRINT(total, snprintf, buf, size, "%d, '%s'", + SNPRINT(total, snprintf, buf, size, "[%d, '%s'", (int)part->fieldno, field_type_strs[part->type]); + if (part->path != NULL) { + SNPRINT(total, snprintf, buf, size, ", path='%s'", + part->path); + } + SNPRINT(total, snprintf, buf, size, "]"); if (i < part_count - 1) SNPRINT(total, snprintf, buf, size, ", "); } @@ -294,6 +367,8 @@ key_def_sizeof_parts(const struct key_part_def *parts, uint32_t part_count) count++; if (part->is_nullable) count++; + if (part->path != NULL) + count++; size += mp_sizeof_map(count); size += mp_sizeof_str(strlen(PART_OPT_FIELD)); size += mp_sizeof_uint(part->fieldno); @@ -308,6 +383,10 @@ key_def_sizeof_parts(const struct key_part_def *parts, uint32_t part_count) size += mp_sizeof_str(strlen(PART_OPT_NULLABILITY)); size += mp_sizeof_bool(part->is_nullable); } + if (part->path != NULL) { + size += mp_sizeof_str(strlen(PART_OPT_PATH)); + size += mp_sizeof_str(strlen(part->path)); + } } return size; } @@ -323,6 +402,8 @@ key_def_encode_parts(char *data, const struct key_part_def *parts, count++; if (part->is_nullable) count++; + if (part->path != NULL) + count++; data = mp_encode_map(data, count); data = mp_encode_str(data, PART_OPT_FIELD, strlen(PART_OPT_FIELD)); @@ -342,6 +423,12 @@ key_def_encode_parts(char *data, const struct key_part_def *parts, strlen(PART_OPT_NULLABILITY)); data = mp_encode_bool(data, part->is_nullable); } + if (part->path != NULL) { + data = mp_encode_str(data, PART_OPT_PATH, + strlen(PART_OPT_PATH)); + data = mp_encode_str(data, part->path, + strlen(part->path)); + } } return data; } @@ -403,6 +490,7 @@ key_def_decode_parts_166(struct key_part_def *parts, uint32_t part_count, fields[part->fieldno].is_nullable : key_part_def_default.is_nullable); part->coll_id = COLL_NONE; + part->path = NULL; } return 0; } @@ -410,7 +498,7 @@ key_def_decode_parts_166(struct key_part_def *parts, uint32_t part_count, int key_def_decode_parts(struct key_part_def *parts, uint32_t part_count, const char **data, const struct field_def *fields, - uint32_t field_count) + uint32_t field_count, struct region *region) { if (mp_typeof(**data) == MP_ARRAY) { return key_def_decode_parts_166(parts, part_count, data, @@ -439,7 +527,7 @@ key_def_decode_parts(struct key_part_def *parts, uint32_t part_count, const char *key = mp_decode_str(data, &key_len); if (opts_parse_key(part, part_def_reg, key, key_len, data, ER_WRONG_INDEX_OPTIONS, - i + TUPLE_INDEX_BASE, NULL, + i + TUPLE_INDEX_BASE, region, false) != 0) return -1; if (is_action_missing && @@ -485,6 +573,13 @@ key_def_decode_parts(struct key_part_def *parts, uint32_t part_count, "index part: unknown sort order"); return -1; } + if (part->path != NULL && + json_path_validate(part->path, strlen(part->path), + TUPLE_INDEX_BASE) != 0) { + diag_set(ClientError, ER_WRONG_INDEX_OPTIONS, + i + TUPLE_INDEX_BASE, "invalid path"); + return -1; + } } return 0; } @@ -504,7 +599,10 @@ key_def_find(const struct key_def *key_def, const struct key_part *to_find) const struct key_part *part = key_def->parts; const struct key_part *end = part + key_def->part_count; for (; part != end; part++) { - if (part->fieldno == to_find->fieldno) + if (part->fieldno == to_find->fieldno && + json_path_cmp(part->path, part->path_len, + to_find->path, to_find->path_len, + TUPLE_INDEX_BASE) == 0) return part; } return NULL; @@ -530,18 +628,25 @@ key_def_merge(const struct key_def *first, const struct key_def *second) * Find and remove part duplicates, i.e. parts counted * twice since they are present in both key defs. */ - const struct key_part *part = second->parts; - const struct key_part *end = part + second->part_count; + size_t sz = 0; + const struct key_part *part = first->parts; + const struct key_part *end = part + first->part_count; + for (; part != end; part++) + sz += part->path_len; + part = second->parts; + end = part + second->part_count; for (; part != end; part++) { if (key_def_find(first, part) != NULL) --new_part_count; + else + sz += part->path_len; } + sz = key_def_sizeof(new_part_count, sz); struct key_def *new_def; - new_def = (struct key_def *)calloc(1, key_def_sizeof(new_part_count)); + new_def = (struct key_def *)calloc(1, sz); if (new_def == NULL) { - diag_set(OutOfMemory, key_def_sizeof(new_part_count), "malloc", - "new_def"); + diag_set(OutOfMemory, sz, "malloc", "new_def"); return NULL; } new_def->part_count = new_part_count; @@ -549,6 +654,9 @@ key_def_merge(const struct key_def *first, const struct key_def *second) new_def->is_nullable = first->is_nullable || second->is_nullable; new_def->has_optional_parts = first->has_optional_parts || second->has_optional_parts; + + /* JSON paths data in the new key_def. */ + char *path_pool = (char *)new_def + key_def_sizeof(new_part_count, 0); /* Write position in the new key def. */ uint32_t pos = 0; /* Append first key def's parts to the new index_def. */ @@ -557,7 +665,8 @@ key_def_merge(const struct key_def *first, const struct key_def *second) for (; part != end; part++) { key_def_set_part(new_def, pos++, part->fieldno, part->type, part->nullable_action, part->coll, - part->coll_id, part->sort_order); + part->coll_id, part->sort_order, part->path, + part->path_len, &path_pool); } /* Set-append second key def's part to the new key def. */ @@ -568,8 +677,10 @@ key_def_merge(const struct key_def *first, const struct key_def *second) continue; key_def_set_part(new_def, pos++, part->fieldno, part->type, part->nullable_action, part->coll, - part->coll_id, part->sort_order); + part->coll_id, part->sort_order, part->path, + part->path_len, &path_pool); } + assert(path_pool == (char *)new_def + sz); key_def_set_cmp(new_def); return new_def; } diff --git a/src/box/key_def.h b/src/box/key_def.h index d1866303b..678d1f070 100644 --- a/src/box/key_def.h +++ b/src/box/key_def.h @@ -64,6 +64,12 @@ struct key_part_def { enum on_conflict_action nullable_action; /** Part sort order. */ enum sort_order sort_order; + /** + * JSON path to indexed data, relative to the field number, + * or NULL if this key part indexes a top-level field. + * This sting is 0-terminated. + */ + const char *path; }; extern const struct key_part_def key_part_def_default; @@ -82,6 +88,15 @@ struct key_part { enum on_conflict_action nullable_action; /** Part sort order. */ enum sort_order sort_order; + /** + * JSON path to indexed data, relative to the field number, + * or NULL if this key part index a top-level field. + * This string is not 0-terminated. String memory is + * allocated at the end of key_def. + */ + char *path; + /** The length of JSON path. */ + uint32_t path_len; }; struct key_def; @@ -148,6 +163,8 @@ struct key_def { uint32_t unique_part_count; /** True, if at least one part can store NULL. */ bool is_nullable; + /** True if some key part has JSON path. */ + bool has_json_paths; /** * True, if some key parts can be absent in a tuple. These * fields assumed to be MP_NIL. @@ -241,9 +258,10 @@ box_tuple_compare_with_key(const box_tuple_t *tuple_a, const char *key_b, /** \endcond public */ static inline size_t -key_def_sizeof(uint32_t part_count) +key_def_sizeof(uint32_t part_count, uint32_t path_pool_size) { - return sizeof(struct key_def) + sizeof(struct key_part) * part_count; + return sizeof(struct key_def) + sizeof(struct key_part) * part_count + + path_pool_size; } /** @@ -255,9 +273,12 @@ key_def_new(const struct key_part_def *parts, uint32_t part_count); /** * Dump part definitions of the given key def. + * The region is used for allocating JSON paths, if any. + * Return -1 on memory allocation error, 0 on success. */ -void -key_def_dump_parts(const struct key_def *def, struct key_part_def *parts); +int +key_def_dump_parts(const struct key_def *def, struct key_part_def *parts, + struct region *region); /** * Update 'has_optional_parts' of @a key_def with correspondence @@ -299,11 +320,12 @@ key_def_encode_parts(char *data, const struct key_part_def *parts, * [NUM, STR, ..][NUM, STR, ..].., * OR * {field=NUM, type=STR, ..}{field=NUM, type=STR, ..}.., + * The region is used for allocating JSON paths, if any. */ int key_def_decode_parts(struct key_part_def *parts, uint32_t part_count, const char **data, const struct field_def *fields, - uint32_t field_count); + uint32_t field_count, struct region *region); /** * Returns the part in index_def->parts for the specified fieldno. @@ -364,6 +386,8 @@ key_validate_parts(const struct key_def *key_def, const char *key, static inline bool key_def_is_sequential(const struct key_def *key_def) { + if (key_def->has_json_paths) + return false; for (uint32_t part_id = 0; part_id < key_def->part_count; part_id++) { if (key_def->parts[part_id].fieldno != part_id) return false; diff --git a/src/box/lua/space.cc b/src/box/lua/space.cc index 7cae436f1..1f152917e 100644 --- a/src/box/lua/space.cc +++ b/src/box/lua/space.cc @@ -296,6 +296,11 @@ lbox_fillspace(struct lua_State *L, struct space *space, int i) lua_pushnumber(L, part->fieldno + TUPLE_INDEX_BASE); lua_setfield(L, -2, "fieldno"); + if (part->path != NULL) { + lua_pushlstring(L, part->path, part->path_len); + lua_setfield(L, -2, "path"); + } + lua_pushboolean(L, key_part_is_nullable(part)); lua_setfield(L, -2, "is_nullable"); diff --git a/src/box/memtx_engine.c b/src/box/memtx_engine.c index 692e41efb..64f43456e 100644 --- a/src/box/memtx_engine.c +++ b/src/box/memtx_engine.c @@ -1312,6 +1312,10 @@ memtx_index_def_change_requires_rebuild(struct index *index, return true; if (old_part->coll != new_part->coll) return true; + if (json_path_cmp(old_part->path, old_part->path_len, + new_part->path, new_part->path_len, + TUPLE_INDEX_BASE) != 0) + return true; } return false; } diff --git a/src/box/sql.c b/src/box/sql.c index 387da7b3d..94fd7e369 100644 --- a/src/box/sql.c +++ b/src/box/sql.c @@ -386,6 +386,7 @@ sql_ephemeral_space_create(uint32_t field_count, struct sql_key_info *key_info) part->nullable_action = ON_CONFLICT_ACTION_NONE; part->is_nullable = true; part->sort_order = SORT_ORDER_ASC; + part->path = NULL; if (def != NULL && i < def->part_count) part->coll_id = def->parts[i].coll_id; else diff --git a/src/box/sql/build.c b/src/box/sql/build.c index 49b90b5d0..947daf8f6 100644 --- a/src/box/sql/build.c +++ b/src/box/sql/build.c @@ -2185,6 +2185,7 @@ index_fill_def(struct Parse *parse, struct index *index, part->is_nullable = part->nullable_action == ON_CONFLICT_ACTION_NONE; part->sort_order = SORT_ORDER_ASC; part->coll_id = coll_id; + part->path = NULL; } key_def = key_def_new(key_parts, expr_list->nExpr); if (key_def == NULL) diff --git a/src/box/sql/select.c b/src/box/sql/select.c index 02ee225f1..3f136a342 100644 --- a/src/box/sql/select.c +++ b/src/box/sql/select.c @@ -1360,6 +1360,7 @@ sql_key_info_new(sqlite3 *db, uint32_t part_count) part->is_nullable = false; part->nullable_action = ON_CONFLICT_ACTION_ABORT; part->sort_order = SORT_ORDER_ASC; + part->path = NULL; } return key_info; } @@ -1377,7 +1378,7 @@ sql_key_info_new_from_key_def(sqlite3 *db, const struct key_def *key_def) key_info->key_def = NULL; key_info->refs = 1; key_info->part_count = key_def->part_count; - key_def_dump_parts(key_def, key_info->parts); + key_def_dump_parts(key_def, key_info->parts, NULL); return key_info; } diff --git a/src/box/sql/where.c b/src/box/sql/where.c index 571b5af78..814bd3926 100644 --- a/src/box/sql/where.c +++ b/src/box/sql/where.c @@ -2807,6 +2807,7 @@ whereLoopAddBtree(WhereLoopBuilder * pBuilder, /* WHERE clause information */ part.is_nullable = false; part.sort_order = SORT_ORDER_ASC; part.coll_id = COLL_NONE; + part.path = NULL; struct key_def *key_def = key_def_new(&part, 1); if (key_def == NULL) { diff --git a/src/box/tuple_compare.cc b/src/box/tuple_compare.cc index 3fe4cae32..7ab6e3bf6 100644 --- a/src/box/tuple_compare.cc +++ b/src/box/tuple_compare.cc @@ -469,7 +469,8 @@ tuple_compare_slowpath(const struct tuple *tuple_a, const struct tuple *tuple_b, struct key_part *part = key_def->parts; const char *tuple_a_raw = tuple_data(tuple_a); const char *tuple_b_raw = tuple_data(tuple_b); - if (key_def->part_count == 1 && part->fieldno == 0) { + if (key_def->part_count == 1 && part->fieldno == 0 && + part->path == NULL) { /* * First field can not be optional - empty tuples * can not exist. @@ -1027,7 +1028,7 @@ tuple_compare_create(const struct key_def *def) } } assert(! def->has_optional_parts); - if (!key_def_has_collation(def)) { + if (!key_def_has_collation(def) && !def->has_json_paths) { /* Precalculated comparators don't use collation */ for (uint32_t k = 0; k < sizeof(cmp_arr) / sizeof(cmp_arr[0]); k++) { @@ -1247,7 +1248,7 @@ tuple_compare_with_key_create(const struct key_def *def) } } assert(! def->has_optional_parts); - if (!key_def_has_collation(def)) { + if (!key_def_has_collation(def) && !def->has_json_paths) { /* Precalculated comparators don't use collation */ for (uint32_t k = 0; k < sizeof(cmp_wk_arr) / sizeof(cmp_wk_arr[0]); diff --git a/src/box/tuple_extract_key.cc b/src/box/tuple_extract_key.cc index ac8b5a44e..1e8ec7588 100644 --- a/src/box/tuple_extract_key.cc +++ b/src/box/tuple_extract_key.cc @@ -8,9 +8,10 @@ enum { MSGPACK_NULL = 0xc0 }; static inline bool key_def_parts_are_sequential(const struct key_def *def, int i) { - uint32_t fieldno1 = def->parts[i].fieldno + 1; - uint32_t fieldno2 = def->parts[i + 1].fieldno; - return fieldno1 == fieldno2; + const struct key_part *part1 = &def->parts[i]; + const struct key_part *part2 = &def->parts[i + 1]; + return part1->fieldno + 1 == part2->fieldno && + part1->path == NULL && part2->path == NULL; } /** True, if a key con contain two or more parts in sequence. */ @@ -241,7 +242,8 @@ tuple_extract_key_slowpath_raw(const char *data, const char *data_end, if (!key_def_parts_are_sequential(key_def, i)) break; } - uint32_t end_fieldno = key_def->parts[i].fieldno; + const struct key_part *part = &key_def->parts[i]; + uint32_t end_fieldno = part->fieldno; if (fieldno < current_fieldno) { /* Rewind. */ @@ -283,8 +285,29 @@ tuple_extract_key_slowpath_raw(const char *data, const char *data_end, current_fieldno++; } } - memcpy(key_buf, field, field_end - field); - key_buf += field_end - field; + const char *src = field; + const char *src_end = field_end; + if (part->path != NULL) { + if (tuple_field_go_to_path(&src, part->path, + part->path_len) != 0) { + /* + * The path must be correct as + * it has already been validated + * in key_def_decode_parts. + */ + unreachable(); + } + assert(src != NULL || has_optional_parts); + if (has_optional_parts && src == NULL) { + null_count += 1; + src = src_end; + } else { + src_end = src; + mp_next(&src_end); + } + } + memcpy(key_buf, src, src_end - src); + key_buf += src_end - src; if (has_optional_parts && null_count != 0) { memset(key_buf, MSGPACK_NULL, null_count); key_buf += null_count * mp_sizeof_nil(); diff --git a/src/box/tuple_format.c b/src/box/tuple_format.c index 4d10b0918..d9c408495 100644 --- a/src/box/tuple_format.c +++ b/src/box/tuple_format.c @@ -42,6 +42,33 @@ static intptr_t recycled_format_ids = FORMAT_ID_NIL; static uint32_t formats_size = 0, formats_capacity = 0; +/** + * Find in format1::fields the field by format2_field's JSON path. + * Routine uses fiber region for temporal path allocation and + * panics on failure. + */ +static struct tuple_field * +tuple_format1_field_by_format2_field(struct tuple_format *format1, + struct tuple_field *format2_field) +{ + struct region *region = &fiber()->gc; + size_t region_svp = region_used(region); + uint32_t path_len = json_tree_snprint_path(NULL, 0, + &format2_field->token, TUPLE_INDEX_BASE); + char *path = region_alloc(region, path_len + 1); + if (path == NULL) + panic("Can not allocate memory for path"); + json_tree_snprint_path(path, path_len + 1, &format2_field->token, + TUPLE_INDEX_BASE); + struct tuple_field *format1_field = + json_tree_lookup_path_entry(&format1->fields, + &format1->fields.root, path, + path_len, TUPLE_INDEX_BASE, + struct tuple_field, token); + region_truncate(region, region_svp); + return format1_field; +} + static int tuple_format_cmp(const struct tuple_format *format1, const struct tuple_format *format2) @@ -50,12 +77,14 @@ tuple_format_cmp(const struct tuple_format *format1, struct tuple_format *b = (struct tuple_format *)format2; if (a->exact_field_count != b->exact_field_count) return a->exact_field_count - b->exact_field_count; - if (tuple_format_field_count(a) != tuple_format_field_count(b)) - return tuple_format_field_count(a) - tuple_format_field_count(b); + if (a->total_field_count != b->total_field_count) + return a->total_field_count - b->total_field_count; - for (uint32_t i = 0; i < tuple_format_field_count(a); ++i) { - struct tuple_field *field_a = tuple_format_field(a, i); - struct tuple_field *field_b = tuple_format_field(b, i); + struct tuple_field *field_a; + json_tree_foreach_entry_preorder(field_a, &a->fields.root, + struct tuple_field, token) { + struct tuple_field *field_b = + tuple_format1_field_by_format2_field(b, field_a); if (field_a->type != field_b->type) return (int)field_a->type - (int)field_b->type; if (field_a->coll_id != field_b->coll_id) @@ -82,8 +111,9 @@ tuple_format_hash(struct tuple_format *format) uint32_t h = 13; uint32_t carry = 0; uint32_t size = 0; - for (uint32_t i = 0; i < tuple_format_field_count(format); ++i) { - struct tuple_field *f = tuple_format_field(format, i); + struct tuple_field *f; + json_tree_foreach_entry_preorder(f, &format->fields.root, + struct tuple_field, token) { TUPLE_FIELD_MEMBER_HASH(f, type, h, carry, size) TUPLE_FIELD_MEMBER_HASH(f, coll_id, h, carry, size) TUPLE_FIELD_MEMBER_HASH(f, nullable_action, h, carry, size) @@ -135,9 +165,14 @@ static const char * tuple_field_path(const struct tuple_field *field) { assert(field->token.parent != NULL); - assert(field->token.parent->parent == NULL); - assert(field->token.type == JSON_TOKEN_NUM); - return int2str(field->token.num + TUPLE_INDEX_BASE); + if (field->token.parent->parent == NULL) { + /* Top-level field, no need to format the path. */ + return int2str(field->token.num + TUPLE_INDEX_BASE); + } + char *path = tt_static_buf(); + json_tree_snprint_path(path, TT_STATIC_BUF_LEN, &field->token, + TUPLE_INDEX_BASE); + return path; } /** @@ -158,18 +193,109 @@ tuple_format_field_by_id(struct tuple_format *format, uint32_t id) return NULL; } +/** + * Given a field number and a path, add the corresponding field + * to the tuple format, allocating intermediate fields if + * necessary. + * + * Return a pointer to the leaf field on success, NULL on memory + * allocation error or type/nullability mistmatch error, diag + * message is set. + */ +static struct tuple_field * +tuple_format_add_field(struct tuple_format *format, uint32_t fieldno, + const char *path, uint32_t path_len, char **path_pool) +{ + struct tuple_field *field = NULL; + struct tuple_field *parent = tuple_format_field(format, fieldno); + assert(parent != NULL); + if (path == NULL) + return parent; + field = tuple_field_new(); + if (field == NULL) + goto fail; + + /* + * Retrieve path_len memory chunk from the path_pool and + * copy path data there. This is necessary in order to + * ensure that each new format::tuple_field refer format + * memory. + */ + memcpy(*path_pool, path, path_len); + path = *path_pool; + *path_pool += path_len; + + int rc = 0; + uint32_t token_count = 0; + struct json_tree *tree = &format->fields; + struct json_lexer lexer; + json_lexer_create(&lexer, path, path_len, TUPLE_INDEX_BASE); + while ((rc = json_lexer_next_token(&lexer, &field->token)) == 0 && + field->token.type != JSON_TOKEN_END) { + enum field_type expected_type = + field->token.type == JSON_TOKEN_STR ? + FIELD_TYPE_MAP : FIELD_TYPE_ARRAY; + if (field_type1_contains_type2(parent->type, expected_type)) { + parent->type = expected_type; + } else { + diag_set(ClientError, ER_INDEX_PART_TYPE_MISMATCH, + tuple_field_path(parent), + field_type_strs[parent->type], + field_type_strs[expected_type]); + goto fail; + } + struct tuple_field *next = + json_tree_lookup_entry(tree, &parent->token, + &field->token, + struct tuple_field, token); + if (next == NULL) { + field->id = format->total_field_count++; + rc = json_tree_add(tree, &parent->token, &field->token); + if (rc != 0) { + diag_set(OutOfMemory, sizeof(struct json_token), + "json_tree_add", "tree"); + goto fail; + } + next = field; + field = tuple_field_new(); + if (field == NULL) + goto fail; + } + parent->is_key_part = true; + parent = next; + token_count++; + } + /* + * The path has already been verified by the + * key_def_decode_parts function. + */ + assert(rc == 0 && field->token.type == JSON_TOKEN_END); + assert(parent != NULL); + /* Update tree depth information. */ + format->fields_depth = MAX(format->fields_depth, token_count + 1); +end: + tuple_field_delete(field); + return parent; +fail: + parent = NULL; + goto end; +} + static int tuple_format_use_key_part(struct tuple_format *format, uint32_t field_count, const struct key_part *part, bool is_sequential, - int *current_slot) + int *current_slot, char **path_pool) { assert(part->fieldno < tuple_format_field_count(format)); - struct tuple_field *field = tuple_format_field(format, part->fieldno); + struct tuple_field *field = + tuple_format_add_field(format, part->fieldno, part->path, + part->path_len, path_pool); + if (field == NULL) + return -1; /* - * If a field is not present in the space format, - * inherit nullable action of the first key part - * referencing it. - */ + * If a field is not present in the space format, inherit + * nullable action of the first key part referencing it. + */ if (part->fieldno >= field_count && !field->is_key_part) field->nullable_action = part->nullable_action; /* @@ -224,7 +350,8 @@ tuple_format_use_key_part(struct tuple_format *format, uint32_t field_count, * simply accessible, so we don't store an offset for it. */ if (field->offset_slot == TUPLE_OFFSET_SLOT_NIL && - is_sequential == false && part->fieldno > 0) { + is_sequential == false && + (part->fieldno > 0 || part->path != NULL)) { *current_slot = *current_slot - 1; field->offset_slot = *current_slot; } @@ -269,6 +396,11 @@ tuple_format_create(struct tuple_format *format, struct key_def * const *keys, int current_slot = 0; + /* + * Set pointer to reserved area in the format chunk + * allocated with tuple_format_alloc call. + */ + char *path_pool = (char *)format + sizeof(struct tuple_format); /* extract field type info */ for (uint16_t key_no = 0; key_no < key_count; ++key_no) { const struct key_def *key_def = keys[key_no]; @@ -279,7 +411,8 @@ tuple_format_create(struct tuple_format *format, struct key_def * const *keys, for (; part < parts_end; part++) { if (tuple_format_use_key_part(format, field_count, part, is_sequential, - ¤t_slot) != 0) + ¤t_slot, + &path_pool) != 0) return -1; } } @@ -302,6 +435,7 @@ tuple_format_create(struct tuple_format *format, struct key_def * const *keys, "malloc", "required field bitmap"); return -1; } + format->min_tuple_size = mp_sizeof_array(format->index_field_count); struct tuple_field *field; json_tree_foreach_entry_preorder(field, &format->fields.root, struct tuple_field, token) { @@ -313,6 +447,44 @@ tuple_format_create(struct tuple_format *format, struct key_def * const *keys, if (json_token_is_leaf(&field->token) && !tuple_field_is_nullable(field)) bit_set(format->required_fields, field->id); + + /* + * Update format::min_tuple_size by field. + * Skip fields not involved in index. + */ + if (!field->is_key_part) + continue; + if (field->token.type == JSON_TOKEN_NUM) { + /* + * Account a gap between omitted array + * items. + */ + struct json_token **neighbors = + field->token.parent->children; + for (int i = field->token.sibling_idx - 1; i >= 0; i--) { + if (neighbors[i] != NULL && + json_tree_entry(neighbors[i], + struct tuple_field, + token)->is_key_part) + break; + format->min_tuple_size += mp_sizeof_nil(); + } + } else { + /* Account a key string for map member. */ + assert(field->token.type == JSON_TOKEN_STR); + format->min_tuple_size += + mp_sizeof_str(field->token.len); + } + int max_child_idx = field->token.max_child_idx; + if (json_token_is_leaf(&field->token)) { + format->min_tuple_size += mp_sizeof_nil(); + } else if (field->type == FIELD_TYPE_ARRAY) { + format->min_tuple_size += + mp_sizeof_array(max_child_idx + 1); + } else if (field->type == FIELD_TYPE_MAP) { + format->min_tuple_size += + mp_sizeof_map(max_child_idx + 1); + } } format->hash = tuple_format_hash(format); return 0; @@ -389,6 +561,8 @@ static struct tuple_format * tuple_format_alloc(struct key_def * const *keys, uint16_t key_count, uint32_t space_field_count, struct tuple_dictionary *dict) { + /* Size of area to store JSON paths data. */ + uint32_t path_pool_size = 0; uint32_t index_field_count = 0; /* find max max field no */ for (uint16_t key_no = 0; key_no < key_count; ++key_no) { @@ -398,13 +572,15 @@ tuple_format_alloc(struct key_def * const *keys, uint16_t key_count, for (; part < pend; part++) { index_field_count = MAX(index_field_count, part->fieldno + 1); + path_pool_size += part->path_len; } } uint32_t field_count = MAX(space_field_count, index_field_count); - struct tuple_format *format = malloc(sizeof(struct tuple_format)); + uint32_t allocation_size = sizeof(struct tuple_format) + path_pool_size; + struct tuple_format *format = malloc(allocation_size); if (format == NULL) { - diag_set(OutOfMemory, sizeof(struct tuple_format), "malloc", + diag_set(OutOfMemory, allocation_size, "malloc", "tuple format"); return NULL; } @@ -440,6 +616,8 @@ tuple_format_alloc(struct key_def * const *keys, uint16_t key_count, } format->total_field_count = field_count; format->required_fields = NULL; + format->fields_depth = 1; + format->min_tuple_size = 0; format->refs = 0; format->id = FORMAT_ID_NIL; format->index_field_count = index_field_count; @@ -581,15 +759,16 @@ tuple_format1_can_store_format2_tuples(struct tuple_format *format1, { if (format1->exact_field_count != format2->exact_field_count) return false; - uint32_t format1_field_count = tuple_format_field_count(format1); - uint32_t format2_field_count = tuple_format_field_count(format2); - for (uint32_t i = 0; i < format1_field_count; ++i) { - struct tuple_field *field1 = tuple_format_field(format1, i); + struct tuple_field *field1; + json_tree_foreach_entry_preorder(field1, &format1->fields.root, + struct tuple_field, token) { + struct tuple_field *field2 = + tuple_format1_field_by_format2_field(format2, field1); /* * The field has a data type in format1, but has * no data type in format2. */ - if (i >= format2_field_count) { + if (field2 == NULL) { /* * The field can get a name added * for it, and this doesn't require a data @@ -605,7 +784,6 @@ tuple_format1_can_store_format2_tuples(struct tuple_format *format1, else return false; } - struct tuple_field *field2 = tuple_format_field(format2, i); if (! field_type1_contains_type2(field1->type, field2->type)) return false; /* @@ -663,52 +841,122 @@ tuple_init_field_map(struct tuple_format *format, uint32_t *field_map, */ if (field_count == 0) { /* Empty tuple, nothing to do. */ - goto skip; - } - /* first field is simply accessible, so we do not store offset to it */ - struct tuple_field *field = tuple_format_field(format, 0); - if (validate && - !field_mp_type_is_compatible(field->type, mp_typeof(*pos), - tuple_field_is_nullable(field))) { - diag_set(ClientError, ER_FIELD_TYPE, tuple_field_path(field), - field_type_strs[field->type]); - goto error; + goto finish; } - if (required_fields != NULL) - bit_clear(required_fields, field->id); - mp_next(&pos); - /* other fields...*/ - uint32_t i = 1; uint32_t defined_field_count = MIN(field_count, validate ? tuple_format_field_count(format) : format->index_field_count); - if (field_count < format->index_field_count) { + /* + * Nullify field map to be able to detect by 0, + * which key fields are absent in tuple_field(). + */ + memset((char *)field_map - format->field_map_size, 0, + format->field_map_size); + /* + * Prepare mp stack of the size equal to the maximum depth + * of the indexed field in the format::fields tree + * (fields_depth) to carry out a simultaneous parsing of + * the tuple and tree traversal to process type + * validations and field map initialization. + */ + uint32_t frames_sz = format->fields_depth * sizeof(struct mp_frame); + struct mp_frame *frames = region_alloc(region, frames_sz); + if (frames == NULL) { + diag_set(OutOfMemory, frames_sz, "region", "frames"); + goto error; + } + struct mp_stack stack; + mp_stack_create(&stack, format->fields_depth, frames); + mp_stack_push(&stack, MP_ARRAY, defined_field_count); + struct tuple_field *field; + struct json_token *parent = &format->fields.root; + while (true) { + int idx; + while ((idx = mp_stack_advance(&stack)) == -1) { + /* + * If the elements of the current frame + * are over, pop this frame out of stack + * and climb one position in the + * format::fields tree to match the + * changed JSON path to the data in the + * tuple. + */ + mp_stack_pop(&stack); + if (mp_stack_is_empty(&stack)) + goto finish; + parent = parent->parent; + } /* - * Nullify field map to be able to detect by 0, - * which key fields are absent in tuple_field(). + * Use the top frame of the stack and the + * current data offset to prepare the JSON token + * for the subsequent format::fields lookup. */ - memset((char *)field_map - format->field_map_size, 0, - format->field_map_size); - } - for (; i < defined_field_count; ++i) { - field = tuple_format_field(format, i); - if (validate && - !field_mp_type_is_compatible(field->type, mp_typeof(*pos), - tuple_field_is_nullable(field))) { - diag_set(ClientError, ER_FIELD_TYPE, - tuple_field_path(field), - field_type_strs[field->type]); - goto error; + struct json_token token; + switch (mp_stack_type(&stack)) { + case MP_ARRAY: + token.type = JSON_TOKEN_NUM; + token.num = idx; + break; + case MP_MAP: + if (mp_typeof(*pos) != MP_STR) { + /* + * JSON path support only string + * keys for map. Skip this entry. + */ + mp_next(&pos); + mp_next(&pos); + continue; + } + token.type = JSON_TOKEN_STR; + token.str = mp_decode_str(&pos, (uint32_t *)&token.len); + break; + default: + unreachable(); } - if (field->offset_slot != TUPLE_OFFSET_SLOT_NIL) { - field_map[field->offset_slot] = - (uint32_t) (pos - tuple); + /* + * Perform lookup for a field in format::fields, + * that represents the field metadata by JSON path + * corresponding to the current position in the + * tuple. + */ + enum mp_type type = mp_typeof(*pos); + assert(parent != NULL); + field = json_tree_lookup_entry(&format->fields, parent, &token, + struct tuple_field, token); + if (field != NULL) { + bool is_nullable = tuple_field_is_nullable(field); + if (validate && + !field_mp_type_is_compatible(field->type, type, + is_nullable) != 0) { + diag_set(ClientError, ER_FIELD_TYPE, + tuple_field_path(field), + field_type_strs[field->type]); + goto error; + } + if (field->offset_slot != TUPLE_OFFSET_SLOT_NIL) + field_map[field->offset_slot] = pos - tuple; + if (required_fields != NULL) + bit_clear(required_fields, field->id); + } + /* + * If the current position of the data in tuple + * matches the container type (MP_MAP or MP_ARRAY) + * and the format::fields tree has such a record, + * prepare a new stack frame because it needs to + * be analyzed in the next iterations. + */ + if ((type == MP_ARRAY || type == MP_MAP) && + !mp_stack_is_full(&stack) && field != NULL) { + uint32_t size = type == MP_ARRAY ? + mp_decode_array(&pos) : + mp_decode_map(&pos); + mp_stack_push(&stack, type, size); + parent = &field->token; + } else { + mp_next(&pos); } - if (required_fields != NULL) - bit_clear(required_fields, field->id); - mp_next(&pos); } -skip: +finish: /* * Check the required field bitmap for missing fields. */ diff --git a/src/box/tuple_format.h b/src/box/tuple_format.h index 60b019194..d4b53195b 100644 --- a/src/box/tuple_format.h +++ b/src/box/tuple_format.h @@ -196,6 +196,15 @@ struct tuple_format { * Shared names storage used by all formats of a space. */ struct tuple_dictionary *dict; + /** + * The size of a minimal tuple conforming to the format + * and filled with nils. + */ + uint32_t min_tuple_size; + /** + * A maximum depth of format::fields subtree. + */ + uint32_t fields_depth; /** * Fields comprising the format, organized in a tree. * First level nodes correspond to tuple fields. @@ -218,18 +227,36 @@ tuple_format_field_count(struct tuple_format *format) } /** - * Return meta information of a top-level tuple field given - * a format and a field index. + * Return meta information of a tuple field given a format, + * field index and path. */ static inline struct tuple_field * -tuple_format_field(struct tuple_format *format, uint32_t fieldno) +tuple_format_field_by_path(struct tuple_format *format, uint32_t fieldno, + const char *path, uint32_t path_len) { assert(fieldno < tuple_format_field_count(format)); struct json_token token; token.type = JSON_TOKEN_NUM; token.num = fieldno; - return json_tree_lookup_entry(&format->fields, &format->fields.root, - &token, struct tuple_field, token); + struct tuple_field *root = + json_tree_lookup_entry(&format->fields, &format->fields.root, + &token, struct tuple_field, token); + assert(root != NULL); + if (path == NULL) + return root; + return json_tree_lookup_path_entry(&format->fields, &root->token, + path, path_len, TUPLE_INDEX_BASE, + struct tuple_field, token); +} + +/** + * Return meta information of a top-level tuple field given + * a format and a field index. + */ +static inline struct tuple_field * +tuple_format_field(struct tuple_format *format, uint32_t fieldno) +{ + return tuple_format_field_by_path(format, fieldno, NULL, 0); } extern struct tuple_format **tuple_formats; @@ -401,12 +428,16 @@ tuple_field_raw_by_path(struct tuple_format *format, const char *tuple, const char *path, uint32_t path_len) { if (likely(fieldno < format->index_field_count)) { - if (fieldno == 0) { + if (path == NULL && fieldno == 0) { mp_decode_array(&tuple); - goto parse_path; + return tuple; } - struct tuple_field *field = tuple_format_field(format, fieldno); - assert(field != NULL); + struct tuple_field *field = + tuple_format_field_by_path(format, fieldno, path, + path_len); + assert(field != NULL || path != NULL); + if (path != NULL && field == NULL) + goto parse; int32_t offset_slot = field->offset_slot; if (offset_slot == TUPLE_OFFSET_SLOT_NIL) goto parse; @@ -423,11 +454,11 @@ parse: return NULL; for (uint32_t k = 0; k < fieldno; k++) mp_next(&tuple); + if (path != NULL && + unlikely(tuple_field_go_to_path(&tuple, path, + path_len) != 0)) + return NULL; } -parse_path: - if (path != NULL && - unlikely(tuple_field_go_to_path(&tuple, path, path_len) != 0)) - return NULL; return tuple; } @@ -482,7 +513,8 @@ static inline const char * tuple_field_by_part_raw(struct tuple_format *format, const char *data, const uint32_t *field_map, struct key_part *part) { - return tuple_field_raw(format, data, field_map, part->fieldno); + return tuple_field_raw_by_path(format, data, field_map, part->fieldno, + part->path, part->path_len); } /** diff --git a/src/box/tuple_hash.cc b/src/box/tuple_hash.cc index 078cc6fe0..825c3e5b3 100644 --- a/src/box/tuple_hash.cc +++ b/src/box/tuple_hash.cc @@ -223,7 +223,7 @@ key_hash_slowpath(const char *key, struct key_def *key_def); void tuple_hash_func_set(struct key_def *key_def) { - if (key_def->is_nullable) + if (key_def->is_nullable || key_def->has_json_paths) goto slowpath; /* * Check that key_def defines sequential a key without holes diff --git a/src/box/vinyl.c b/src/box/vinyl.c index 1832a29c7..065a309f5 100644 --- a/src/box/vinyl.c +++ b/src/box/vinyl.c @@ -1005,6 +1005,10 @@ vinyl_index_def_change_requires_rebuild(struct index *index, return true; if (!field_type1_contains_type2(new_part->type, old_part->type)) return true; + if (json_path_cmp(old_part->path, old_part->path_len, + new_part->path, new_part->path_len, + TUPLE_INDEX_BASE) != 0) + return true; } return false; } diff --git a/src/box/vy_log.c b/src/box/vy_log.c index 11c763cec..f94b60ff2 100644 --- a/src/box/vy_log.c +++ b/src/box/vy_log.c @@ -581,8 +581,9 @@ vy_log_record_decode(struct vy_log_record *record, record->group_id = mp_decode_uint(&pos); break; case VY_LOG_KEY_DEF: { + struct region *region = &fiber()->gc; uint32_t part_count = mp_decode_array(&pos); - struct key_part_def *parts = region_alloc(&fiber()->gc, + struct key_part_def *parts = region_alloc(region, sizeof(*parts) * part_count); if (parts == NULL) { diag_set(OutOfMemory, @@ -591,7 +592,7 @@ vy_log_record_decode(struct vy_log_record *record, return -1; } if (key_def_decode_parts(parts, part_count, &pos, - NULL, 0) != 0) { + NULL, 0, region) != 0) { diag_log(); diag_set(ClientError, ER_INVALID_VYLOG_FILE, "Bad record: failed to decode " @@ -700,7 +701,8 @@ vy_log_record_dup(struct region *pool, const struct vy_log_record *src) "struct key_part_def"); goto err; } - key_def_dump_parts(src->key_def, dst->key_parts); + if (key_def_dump_parts(src->key_def, dst->key_parts, pool) != 0) + goto err; dst->key_part_count = src->key_def->part_count; dst->key_def = NULL; } @@ -1267,6 +1269,46 @@ vy_recovery_lookup_slice(struct vy_recovery *recovery, int64_t slice_id) return mh_i64ptr_node(h, k)->val; } +/** + * Allocate duplicate of the data of key_part_count + * key_part_def objects. This function is required because the + * original key_part passed as an argument can have non-NULL + * path fields referencing other memory fragments. + * + * Returns the key_part_def on success, NULL on error. + */ +struct key_part_def * +vy_recovery_alloc_key_parts(const struct key_part_def *key_parts, + uint32_t key_part_count) +{ + uint32_t new_parts_sz = sizeof(*key_parts) * key_part_count; + for (uint32_t i = 0; i < key_part_count; i++) { + new_parts_sz += key_parts[i].path != NULL ? + strlen(key_parts[i].path) + 1 : 0; + } + struct key_part_def *new_parts = malloc(new_parts_sz); + if (new_parts == NULL) { + diag_set(OutOfMemory, sizeof(*key_parts) * key_part_count, + "malloc", "struct key_part_def"); + return NULL; + } + memcpy(new_parts, key_parts, sizeof(*key_parts) * key_part_count); + char *path_pool = + (char *)new_parts + sizeof(*key_parts) * key_part_count; + for (uint32_t i = 0; i < key_part_count; i++) { + if (key_parts[i].path == NULL) + continue; + char *path = path_pool; + uint32_t path_len = strlen(key_parts[i].path); + path_pool += path_len + 1; + memcpy(path, key_parts[i].path, path_len); + path[path_len] = '\0'; + new_parts[i].path = path; + } + assert(path_pool == (char *)new_parts + new_parts_sz); + return new_parts; +} + /** * Allocate a new LSM tree with the given ID and add it to * the recovery context. @@ -1292,10 +1334,8 @@ vy_recovery_do_create_lsm(struct vy_recovery *recovery, int64_t id, "malloc", "struct vy_lsm_recovery_info"); return NULL; } - lsm->key_parts = malloc(sizeof(*key_parts) * key_part_count); + lsm->key_parts = vy_recovery_alloc_key_parts(key_parts, key_part_count); if (lsm->key_parts == NULL) { - diag_set(OutOfMemory, sizeof(*key_parts) * key_part_count, - "malloc", "struct key_part_def"); free(lsm); return NULL; } @@ -1313,7 +1353,6 @@ vy_recovery_do_create_lsm(struct vy_recovery *recovery, int64_t id, lsm->space_id = space_id; lsm->index_id = index_id; lsm->group_id = group_id; - memcpy(lsm->key_parts, key_parts, sizeof(*key_parts) * key_part_count); lsm->key_part_count = key_part_count; lsm->create_lsn = -1; lsm->modify_lsn = -1; @@ -1440,13 +1479,9 @@ vy_recovery_modify_lsm(struct vy_recovery *recovery, int64_t id, return -1; } free(lsm->key_parts); - lsm->key_parts = malloc(sizeof(*key_parts) * key_part_count); - if (lsm->key_parts == NULL) { - diag_set(OutOfMemory, sizeof(*key_parts) * key_part_count, - "malloc", "struct key_part_def"); + lsm->key_parts = vy_recovery_alloc_key_parts(key_parts, key_part_count); + if (lsm->key_parts == NULL) return -1; - } - memcpy(lsm->key_parts, key_parts, sizeof(*key_parts) * key_part_count); lsm->key_part_count = key_part_count; lsm->modify_lsn = modify_lsn; return 0; diff --git a/src/box/vy_point_lookup.c b/src/box/vy_point_lookup.c index ddbc2d46f..088177a4b 100644 --- a/src/box/vy_point_lookup.c +++ b/src/box/vy_point_lookup.c @@ -196,7 +196,9 @@ vy_point_lookup(struct vy_lsm *lsm, struct vy_tx *tx, const struct vy_read_view **rv, struct tuple *key, struct tuple **ret) { - assert(tuple_field_count(key) >= lsm->cmp_def->part_count); + /* All key parts must be set for a point lookup. */ + assert(vy_stmt_type(key) != IPROTO_SELECT || + tuple_field_count(key) >= lsm->cmp_def->part_count); *ret = NULL; double start_time = ev_monotonic_now(loop()); diff --git a/src/box/vy_stmt.c b/src/box/vy_stmt.c index 47f135c65..af5c64086 100644 --- a/src/box/vy_stmt.c +++ b/src/box/vy_stmt.c @@ -383,51 +383,103 @@ vy_stmt_new_surrogate_from_key(const char *key, enum iproto_type type, /* UPSERT can't be surrogate. */ assert(type != IPROTO_UPSERT); struct region *region = &fiber()->gc; + size_t region_svp = region_used(region); uint32_t field_count = format->index_field_count; - struct iovec *iov = region_alloc(region, sizeof(*iov) * field_count); + uint32_t iov_sz = sizeof(struct iovec) * format->total_field_count; + struct iovec *iov = region_alloc(region, iov_sz); if (iov == NULL) { - diag_set(OutOfMemory, sizeof(*iov) * field_count, - "region", "iov for surrogate key"); + diag_set(OutOfMemory, iov_sz, "region", + "iov for surrogate key"); return NULL; } - memset(iov, 0, sizeof(*iov) * field_count); + memset(iov, 0, iov_sz); uint32_t part_count = mp_decode_array(&key); assert(part_count == cmp_def->part_count); - assert(part_count <= field_count); - uint32_t nulls_count = field_count - cmp_def->part_count; - uint32_t bsize = mp_sizeof_array(field_count) + - mp_sizeof_nil() * nulls_count; + assert(part_count <= format->total_field_count); + /** + * Calculate bsize using format::min_tuple_size tuple + * where parts_count nulls replaced with extracted keys. + */ + uint32_t bsize = format->min_tuple_size - mp_sizeof_nil() * part_count; for (uint32_t i = 0; i < part_count; ++i) { const struct key_part *part = &cmp_def->parts[i]; assert(part->fieldno < field_count); + struct tuple_field *field = + tuple_format_field_by_path(format, part->fieldno, + part->path, part->path_len); + assert(field != NULL); const char *svp = key; - iov[part->fieldno].iov_base = (char *) key; + iov[field->id].iov_base = (char *) key; mp_next(&key); - iov[part->fieldno].iov_len = key - svp; + iov[field->id].iov_len = key - svp; bsize += key - svp; } struct tuple *stmt = vy_stmt_alloc(format, bsize); if (stmt == NULL) - return NULL; + goto out; char *raw = (char *) tuple_data(stmt); uint32_t *field_map = (uint32_t *) raw; + memset((char *)field_map - format->field_map_size, 0, + format->field_map_size); char *wpos = mp_encode_array(raw, field_count); - for (uint32_t i = 0; i < field_count; ++i) { - struct tuple_field *field = tuple_format_field(format, i); - if (field->offset_slot != TUPLE_OFFSET_SLOT_NIL) - field_map[field->offset_slot] = wpos - raw; - if (iov[i].iov_base == NULL) { - wpos = mp_encode_nil(wpos); + struct tuple_field *field; + json_tree_foreach_entry_preorder(field, &format->fields.root, + struct tuple_field, token) { + /* + * Do not restore fields not involved in index + * (except gaps in the mp array that may be filled + * with nils later). + */ + if (!field->is_key_part) + continue; + if (field->token.type == JSON_TOKEN_NUM) { + /* + * Write nil istead of omitted array + * members. + */ + struct json_token **neighbors = + field->token.parent->children; + for (int i = field->token.sibling_idx - 1; i >= 0; i--) { + if (neighbors[i] != NULL && + json_tree_entry(neighbors[i], + struct tuple_field, + token)->is_key_part) + break; + wpos = mp_encode_nil(wpos); + } } else { - memcpy(wpos, iov[i].iov_base, iov[i].iov_len); - wpos += iov[i].iov_len; + /* Write a key string for map member. */ + assert(field->token.type == JSON_TOKEN_STR); + const char *str = field->token.str; + uint32_t len = field->token.len; + wpos = mp_encode_str(wpos, str, len); + } + int max_child_idx = field->token.max_child_idx; + if (json_token_is_leaf(&field->token)) { + if (iov[field->id].iov_len == 0) { + wpos = mp_encode_nil(wpos); + } else { + memcpy(wpos, iov[field->id].iov_base, + iov[field->id].iov_len); + uint32_t data_offset = wpos - raw; + int32_t slot = field->offset_slot; + if (slot != TUPLE_OFFSET_SLOT_NIL) + field_map[slot] = data_offset; + wpos += iov[field->id].iov_len; + } + } else if (field->type == FIELD_TYPE_ARRAY) { + wpos = mp_encode_array(wpos, max_child_idx + 1); + } else if (field->type == FIELD_TYPE_MAP) { + wpos = mp_encode_map(wpos, max_child_idx + 1); } } assert(wpos == raw + bsize); vy_stmt_set_type(stmt, type); +out: + region_truncate(region, region_svp); return stmt; } @@ -443,10 +495,13 @@ struct tuple * vy_stmt_new_surrogate_delete_raw(struct tuple_format *format, const char *src_data, const char *src_data_end) { + struct tuple *stmt = NULL; uint32_t src_size = src_data_end - src_data; uint32_t total_size = src_size + format->field_map_size; /* Surrogate tuple uses less memory than the original tuple */ - char *data = region_alloc(&fiber()->gc, total_size); + struct region *region = &fiber()->gc; + size_t region_svp = region_used(region); + char *data = region_alloc(region, total_size); if (data == NULL) { diag_set(OutOfMemory, src_size, "region", "tuple"); return NULL; @@ -456,47 +511,102 @@ vy_stmt_new_surrogate_delete_raw(struct tuple_format *format, const char *src_pos = src_data; uint32_t src_count = mp_decode_array(&src_pos); - uint32_t field_count; - if (src_count < format->index_field_count) { - field_count = src_count; - /* - * Nullify field map to be able to detect by 0, - * which key fields are absent in tuple_field(). - */ - memset((char *)field_map - format->field_map_size, 0, - format->field_map_size); - } else { - field_count = format->index_field_count; - } + uint32_t field_count = MIN(src_count, format->index_field_count); + /* + * Nullify field map to be able to detect by 0, which key + * fields are absent in tuple_field(). + */ + memset((char *)field_map - format->field_map_size, 0, + format->field_map_size); char *pos = mp_encode_array(data, field_count); - for (uint32_t i = 0; i < field_count; ++i) { - struct tuple_field *field = tuple_format_field(format, i); - if (! field->is_key_part) { - /* Unindexed field - write NIL. */ - assert(i < src_count); - pos = mp_encode_nil(pos); + /* + * Perform simultaneous parsing of the tuple and + * format::fields tree traversal to copy indexed field + * data and initialize field map. In many details the code + * above works like tuple_init_field_map, read it's + * comments for more details. + */ + uint32_t frames_sz = format->fields_depth * sizeof(struct mp_frame); + struct mp_frame *frames = region_alloc(region, frames_sz); + if (frames == NULL) { + diag_set(OutOfMemory, frames_sz, "region", "frames"); + goto out; + } + struct mp_stack stack; + mp_stack_create(&stack, format->fields_depth, frames); + mp_stack_push(&stack, MP_ARRAY, field_count); + struct tuple_field *field; + struct json_token *parent = &format->fields.root; + while (true) { + int idx; + while ((idx = mp_stack_advance(&stack)) == -1) { + mp_stack_pop(&stack); + if (mp_stack_is_empty(&stack)) + goto finish; + parent = parent->parent; + } + struct json_token token; + switch (mp_stack_type(&stack)) { + case MP_ARRAY: + token.type = JSON_TOKEN_NUM; + token.num = idx; + break; + case MP_MAP: + if (mp_typeof(*src_pos) != MP_STR) { + mp_next(&src_pos); + mp_next(&src_pos); + continue; + } + token.type = JSON_TOKEN_STR; + token.str = mp_decode_str(&src_pos, (uint32_t *)&token.len); + pos = mp_encode_str(pos, token.str, token.len); + break; + default: + unreachable(); + } + assert(parent != NULL); + field = json_tree_lookup_entry(&format->fields, parent, &token, + struct tuple_field, token); + if (field == NULL || !field->is_key_part) { mp_next(&src_pos); + pos = mp_encode_nil(pos); continue; } - /* Indexed field - copy */ - const char *src_field = src_pos; - mp_next(&src_pos); - memcpy(pos, src_field, src_pos - src_field); if (field->offset_slot != TUPLE_OFFSET_SLOT_NIL) field_map[field->offset_slot] = pos - data; - pos += src_pos - src_field; + enum mp_type type = mp_typeof(*src_pos); + if ((type == MP_ARRAY || type == MP_MAP) && + !mp_stack_is_full(&stack)) { + uint32_t size; + if (type == MP_ARRAY) { + size = mp_decode_array(&src_pos); + pos = mp_encode_array(pos, size); + } else { + size = mp_decode_map(&src_pos); + pos = mp_encode_map(pos, size); + } + mp_stack_push(&stack, type, size); + parent = &field->token; + } else { + const char *src_field = src_pos; + mp_next(&src_pos); + memcpy(pos, src_field, src_pos - src_field); + pos += src_pos - src_field; + } } +finish: assert(pos <= data + src_size); uint32_t bsize = pos - data; - struct tuple *stmt = vy_stmt_alloc(format, bsize); + stmt = vy_stmt_alloc(format, bsize); if (stmt == NULL) - return NULL; + goto out; char *stmt_data = (char *) tuple_data(stmt); char *stmt_field_map_begin = stmt_data - format->field_map_size; memcpy(stmt_data, data, bsize); memcpy(stmt_field_map_begin, field_map_begin, format->field_map_size); vy_stmt_set_type(stmt, IPROTO_DELETE); - +out: + region_truncate(region, region_svp); return stmt; } diff --git a/test/engine/json.result b/test/engine/json.result new file mode 100644 index 000000000..3a5f472bc --- /dev/null +++ b/test/engine/json.result @@ -0,0 +1,591 @@ +test_run = require('test_run').new() +--- +... +engine = test_run:get_cfg('engine') +--- +... +-- +-- gh-1012: Indexes for JSON-defined paths. +-- +s = box.schema.space.create('withdata', {engine = engine}) +--- +... +-- Test build field tree conflicts. +s:create_index('test1', {parts = {{2, 'number'}, {3, 'str', path = 'FIO["fname"]'}, {3, 'str', path = '["FIO"].fname'}}}) +--- +- error: 'Can''t create or modify index ''test1'' in space ''withdata'': same key + part is indexed twice' +... +s:create_index('test1', {parts = {{2, 'number'}, {3, 'str', path = 666}, {3, 'str', path = '["FIO"]["fname"]'}}}) +--- +- error: 'Wrong index options (field 2): ''path'' must be string' +... +s:create_index('test1', {parts = {{2, 'number'}, {3, 'map', path = 'FIO'}}}) +--- +- error: 'Can''t create or modify index ''test1'' in space ''withdata'': field type + ''map'' is not supported' +... +s:create_index('test1', {parts = {{2, 'number'}, {3, 'array', path = '[1]'}}}) +--- +- error: 'Can''t create or modify index ''test1'' in space ''withdata'': field type + ''array'' is not supported' +... +s:create_index('test1', {parts = {{2, 'number'}, {3, 'str', path = 'FIO'}, {3, 'str', path = 'FIO.fname'}}}) +--- +- error: Field [3]["FIO"] has type 'string' in one index, but type 'map' in another +... +s:create_index('test1', {parts = {{2, 'number'}, {3, 'str', path = '[1].sname'}, {3, 'str', path = '["FIO"].fname'}}}) +--- +- error: Field 3 has type 'array' in one index, but type 'map' in another +... +s:create_index('test1', {parts = {{2, 'number'}, {3, 'str', path = 'FIO....fname'}}}) +--- +- error: 'Wrong index options (field 2): invalid path' +... +idx = s:create_index('test1', {parts = {{2, 'number'}, {3, 'str', path = 'FIO.fname', is_nullable = false}, {3, 'str', path = '["FIO"]["sname"]'}}}) +--- +... +idx ~= nil +--- +- true +... +idx.parts[2].path == 'FIO.fname' +--- +- true +... +-- Test format mismatch. +format = {{'id', 'unsigned'}, {'meta', 'unsigned'}, {'data', 'array'}, {'age', 'unsigned'}, {'level', 'unsigned'}} +--- +... +s:format(format) +--- +- error: Field 3 has type 'array' in one index, but type 'map' in another +... +format = {{'id', 'unsigned'}, {'meta', 'unsigned'}, {'data', 'map'}, {'age', 'unsigned'}, {'level', 'unsigned'}} +--- +... +s:format(format) +--- +... +s:create_index('test2', {parts = {{2, 'number'}, {3, 'number', path = 'FIO.fname'}, {3, 'str', path = '["FIO"]["sname"]'}}}) +--- +- error: Field [3]["FIO"]["fname"] has type 'string' in one index, but type 'number' + in another +... +-- Test incompatable tuple insertion. +s:insert{7, 7, {town = 'London', FIO = 666}, 4, 5} +--- +- error: 'Tuple field [3]["FIO"] type does not match one required by operation: expected + map' +... +s:insert{7, 7, {town = 'London', FIO = {fname = 666, sname = 'Bond'}}, 4, 5} +--- +- error: 'Tuple field [3]["FIO"]["fname"] type does not match one required by operation: + expected string' +... +s:insert{7, 7, {town = 'London', FIO = {fname = "James"}}, 4, 5} +--- +- error: Tuple field [3]["FIO"]["sname"] required by space format is missing +... +s:insert{7, 7, {town = 'London', FIO = {fname = 'James', sname = 'Bond'}}, 4, 5} +--- +- [7, 7, {'town': 'London', 'FIO': {'fname': 'James', 'sname': 'Bond'}}, 4, 5] +... +s:insert{7, 7, {town = 'London', FIO = {fname = 'James', sname = 'Bond'}}, 4, 5} +--- +- error: Duplicate key exists in unique index 'test1' in space 'withdata' +... +s:insert{7, 7, {town = 'London', FIO = {fname = 'James', sname = 'Bond', data = "extra"}}, 4, 5} +--- +- error: Duplicate key exists in unique index 'test1' in space 'withdata' +... +s:insert{7, 7, {town = 'Moscow', FIO = {fname = 'Max', sname = 'Isaev', data = "extra"}}, 4, 5} +--- +- [7, 7, {'town': 'Moscow', 'FIO': {'fname': 'Max', 'data': 'extra', 'sname': 'Isaev'}}, + 4, 5] +... +idx:select() +--- +- - [7, 7, {'town': 'London', 'FIO': {'fname': 'James', 'sname': 'Bond'}}, 4, 5] + - [7, 7, {'town': 'Moscow', 'FIO': {'fname': 'Max', 'data': 'extra', 'sname': 'Isaev'}}, + 4, 5] +... +idx:min() +--- +- [7, 7, {'town': 'London', 'FIO': {'fname': 'James', 'sname': 'Bond'}}, 4, 5] +... +idx:max() +--- +- [7, 7, {'town': 'Moscow', 'FIO': {'fname': 'Max', 'data': 'extra', 'sname': 'Isaev'}}, + 4, 5] +... +s:drop() +--- +... +-- Test upsert of JSON-indexed data. +s = box.schema.create_space('withdata', {engine = engine}) +--- +... +parts = {} +--- +... +parts[1] = {1, 'unsigned', path='[2]'} +--- +... +pk = s:create_index('pk', {parts = parts}) +--- +... +s:insert{{1, 2}, 3} +--- +- [[1, 2], 3] +... +s:upsert({{box.null, 2}}, {{'+', 2, 5}}) +--- +... +s:get(2) +--- +- [[1, 2], 8] +... +s:drop() +--- +... +-- Test index creation on space with data. +s = box.schema.space.create('withdata', {engine = engine}) +--- +... +pk = s:create_index('primary', { type = 'tree', parts = {{2, 'number'}} }) +--- +... +s:insert{1, 1, 7, {town = 'London', FIO = 1234}, 4, 5} +--- +- [1, 1, 7, {'town': 'London', 'FIO': 1234}, 4, 5] +... +s:insert{2, 2, 7, {town = 'London', FIO = {fname = 'James', sname = 'Bond'}}, 4, 5} +--- +- [2, 2, 7, {'town': 'London', 'FIO': {'fname': 'James', 'sname': 'Bond'}}, 4, 5] +... +s:insert{3, 3, 7, {town = 'London', FIO = {fname = 'James', sname = 'Bond'}}, 4, 5} +--- +- [3, 3, 7, {'town': 'London', 'FIO': {'fname': 'James', 'sname': 'Bond'}}, 4, 5] +... +s:insert{4, 4, 7, {town = 'London', FIO = {1,2,3}}, 4, 5} +--- +- [4, 4, 7, {'town': 'London', 'FIO': [1, 2, 3]}, 4, 5] +... +s:create_index('test1', {parts = {{3, 'number'}, {4, 'str', path = '["FIO"]["fname"]'}, {4, 'str', path = '["FIO"]["sname"]'}}}) +--- +- error: 'Tuple field [4]["FIO"] type does not match one required by operation: expected + map' +... +_ = s:delete(1) +--- +... +s:create_index('test1', {parts = {{3, 'number'}, {4, 'str', path = '["FIO"]["fname"]'}, {4, 'str', path = '["FIO"]["sname"]'}}}) +--- +- error: Duplicate key exists in unique index 'test1' in space 'withdata' +... +_ = s:delete(2) +--- +... +s:create_index('test1', {parts = {{3, 'number'}, {4, 'str', path = '["FIO"]["fname"]'}, {4, 'str', path = '["FIO"]["sname"]'}}}) +--- +- error: 'Tuple field [4]["FIO"] type does not match one required by operation: expected + map' +... +_ = s:delete(4) +--- +... +idx = s:create_index('test1', {parts = {{3, 'number'}, {4, 'str', path = '["FIO"]["fname"]', is_nullable = true}, {4, 'str', path = '["FIO"]["sname"]'}, {4, 'str', path = '["FIO"]["extra"]', is_nullable = true}}}) +--- +... +idx ~= nil +--- +- true +... +s:create_index('test2', {parts = {{3, 'number'}, {4, 'number', path = '["FIO"]["fname"]'}}}) +--- +- error: Field [4]["FIO"]["fname"] has type 'string' in one index, but type 'number' + in another +... +idx2 = s:create_index('test2', {parts = {{3, 'number'}, {4, 'str', path = '["FIO"]["fname"]'}}}) +--- +... +idx2 ~= nil +--- +- true +... +t = s:insert{5, 5, 7, {town = 'Matrix', FIO = {fname = 'Agent', sname = 'Smith'}}, 4, 5} +--- +... +idx:select() +--- +- - [5, 5, 7, {'town': 'Matrix', 'FIO': {'fname': 'Agent', 'sname': 'Smith'}}, 4, + 5] + - [3, 3, 7, {'town': 'London', 'FIO': {'fname': 'James', 'sname': 'Bond'}}, 4, 5] +... +idx:min() +--- +- [5, 5, 7, {'town': 'Matrix', 'FIO': {'fname': 'Agent', 'sname': 'Smith'}}, 4, 5] +... +idx:max() +--- +- [3, 3, 7, {'town': 'London', 'FIO': {'fname': 'James', 'sname': 'Bond'}}, 4, 5] +... +idx:drop() +--- +... +s:drop() +--- +... +-- Test complex JSON indexes with nullable fields. +s = box.schema.space.create('withdata', {engine = engine}) +--- +... +parts = {} +--- +... +parts[1] = {1, 'str', path='[3][2].a'} +--- +... +parts[2] = {1, 'unsigned', path = '[3][1]'} +--- +... +parts[3] = {2, 'str', path = '[2].d[1]'} +--- +... +pk = s:create_index('primary', { type = 'tree', parts = parts}) +--- +... +s:insert{{1, 2, {3, {3, a = 'str', b = 5}}}, {'c', {d = {'e', 'f'}, e = 'g'}}, 6, {1, 2, 3}} +--- +- [[1, 2, [3, {1: 3, 'a': 'str', 'b': 5}]], ['c', {'d': ['e', 'f'], 'e': 'g'}], 6, + [1, 2, 3]] +... +s:insert{{1, 2, {3, {a = 'str', b = 1}}}, {'c', {d = {'e', 'f'}, e = 'g'}}, 6} +--- +- error: Duplicate key exists in unique index 'primary' in space 'withdata' +... +parts = {} +--- +... +parts[1] = {4, 'unsigned', path='[1]', is_nullable = false} +--- +... +parts[2] = {4, 'unsigned', path='[2]', is_nullable = true} +--- +... +parts[3] = {4, 'unsigned', path='[4]', is_nullable = true} +--- +... +trap_idx = s:create_index('trap', { type = 'tree', parts = parts}) +--- +... +s:insert{{1, 2, {3, {3, a = 'str2', b = 5}}}, {'c', {d = {'e', 'f'}, e = 'g'}}, 6, {}} +--- +- error: Tuple field [4][1] required by space format is missing +... +parts = {} +--- +... +parts[1] = {1, 'unsigned', path='[3][2].b' } +--- +... +parts[2] = {3, 'unsigned'} +--- +... +crosspart_idx = s:create_index('crosspart', { parts = parts}) +--- +... +s:insert{{1, 2, {3, {a = 'str2', b = 2}}}, {'c', {d = {'e', 'f'}, e = 'g'}}, 6, {9, 2, 3}} +--- +- [[1, 2, [3, {'a': 'str2', 'b': 2}]], ['c', {'d': ['e', 'f'], 'e': 'g'}], 6, [9, + 2, 3]] +... +parts = {} +--- +... +parts[1] = {1, 'unsigned', path='[3][2].b'} +--- +... +num_idx = s:create_index('numeric', {parts = parts}) +--- +... +s:insert{{1, 2, {3, {a = 'str3', b = 9}}}, {'c', {d = {'e', 'f'}, e = 'g'}}, 6, {0}} +--- +- [[1, 2, [3, {'a': 'str3', 'b': 9}]], ['c', {'d': ['e', 'f'], 'e': 'g'}], 6, [0]] +... +num_idx:get(2) +--- +- [[1, 2, [3, {'a': 'str2', 'b': 2}]], ['c', {'d': ['e', 'f'], 'e': 'g'}], 6, [9, + 2, 3]] +... +num_idx:select() +--- +- - [[1, 2, [3, {'a': 'str2', 'b': 2}]], ['c', {'d': ['e', 'f'], 'e': 'g'}], 6, [ + 9, 2, 3]] + - [[1, 2, [3, {1: 3, 'a': 'str', 'b': 5}]], ['c', {'d': ['e', 'f'], 'e': 'g'}], + 6, [1, 2, 3]] + - [[1, 2, [3, {'a': 'str3', 'b': 9}]], ['c', {'d': ['e', 'f'], 'e': 'g'}], 6, [ + 0]] +... +num_idx:max() +--- +- [[1, 2, [3, {'a': 'str3', 'b': 9}]], ['c', {'d': ['e', 'f'], 'e': 'g'}], 6, [0]] +... +num_idx:min() +--- +- [[1, 2, [3, {'a': 'str2', 'b': 2}]], ['c', {'d': ['e', 'f'], 'e': 'g'}], 6, [9, + 2, 3]] +... +crosspart_idx:max() == num_idx:max() +--- +- true +... +crosspart_idx:min() == num_idx:min() +--- +- true +... +trap_idx:max() +--- +- [[1, 2, [3, {'a': 'str2', 'b': 2}]], ['c', {'d': ['e', 'f'], 'e': 'g'}], 6, [9, + 2, 3]] +... +trap_idx:min() +--- +- [[1, 2, [3, {'a': 'str3', 'b': 9}]], ['c', {'d': ['e', 'f'], 'e': 'g'}], 6, [0]] +... +s:drop() +--- +... +-- Test index alter. +s = box.schema.space.create('withdata', {engine = engine}) +--- +... +pk_simplified = s:create_index('primary', { type = 'tree', parts = {{1, 'unsigned'}}}) +--- +... +pk_simplified.path == box.NULL +--- +- true +... +idx = s:create_index('idx', {parts = {{2, 'integer', path = 'a'}}}) +--- +... +s:insert{31, {a = 1, aa = -1}} +--- +- [31, {'a': 1, 'aa': -1}] +... +s:insert{22, {a = 2, aa = -2}} +--- +- [22, {'a': 2, 'aa': -2}] +... +s:insert{13, {a = 3, aa = -3}} +--- +- [13, {'a': 3, 'aa': -3}] +... +idx:select() +--- +- - [31, {'a': 1, 'aa': -1}] + - [22, {'a': 2, 'aa': -2}] + - [13, {'a': 3, 'aa': -3}] +... +idx:alter({parts = {{2, 'integer', path = 'aa'}}}) +--- +... +idx:select() +--- +- - [13, {'a': 3, 'aa': -3}] + - [22, {'a': 2, 'aa': -2}] + - [31, {'a': 1, 'aa': -1}] +... +s:drop() +--- +... +-- Incompatible format change. +s = box.schema.space.create('withdata') +--- +... +i = s:create_index('pk', {parts = {{1, 'integer', path = '[1]'}}}) +--- +... +s:insert{{-1}} +--- +- [[-1]] +... +i:alter{parts = {{1, 'string', path = '[1]'}}} +--- +- error: 'Tuple field [1][1] type does not match one required by operation: expected + string' +... +s:insert{{'a'}} +--- +- error: 'Tuple field [1][1] type does not match one required by operation: expected + integer' +... +i:drop() +--- +... +i = s:create_index('pk', {parts = {{1, 'integer', path = '[1].FIO'}}}) +--- +... +s:insert{{{FIO=-1}}} +--- +- [[{'FIO': -1}]] +... +i:alter{parts = {{1, 'integer', path = '[1][1]'}}} +--- +- error: 'Tuple field [1][1] type does not match one required by operation: expected + array' +... +i:alter{parts = {{1, 'integer', path = '[1].FIO[1]'}}} +--- +- error: 'Tuple field [1][1]["FIO"] type does not match one required by operation: + expected array' +... +s:drop() +--- +... +-- Test snapshotting and recovery. +s = box.schema.space.create('withdata', {engine = engine}) +--- +... +pk = s:create_index('pk', {parts = {{1, 'integer'}, {3, 'string', path = 'town'}}}) +--- +... +name = s:create_index('name', {parts = {{3, 'string', path = 'FIO.fname'}, {3, 'string', path = 'FIO.sname'}, {3, 'string', path = 'FIO.extra', is_nullable = true}}}) +--- +... +s:insert{1, 1, {town = 'Moscow', FIO = {fname = 'Max', sname = 'Isaev'}}} +--- +- [1, 1, {'town': 'Moscow', 'FIO': {'fname': 'Max', 'sname': 'Isaev'}}] +... +s:insert{1, 777, {town = 'London', FIO = {fname = 'James', sname = 'Bond'}}} +--- +- [1, 777, {'town': 'London', 'FIO': {'fname': 'James', 'sname': 'Bond'}}] +... +s:insert{1, 45, {town = 'Berlin', FIO = {fname = 'Richard', sname = 'Sorge'}}} +--- +- [1, 45, {'town': 'Berlin', 'FIO': {'fname': 'Richard', 'sname': 'Sorge'}}] +... +s:insert{4, 45, {town = 'Berlin', FIO = {fname = 'Max', extra = 'Otto', sname = 'Stierlitz'}}} +--- +- [4, 45, {'town': 'Berlin', 'FIO': {'fname': 'Max', 'extra': 'Otto', 'sname': 'Stierlitz'}}] +... +pk:select({1}) +--- +- - [1, 45, {'town': 'Berlin', 'FIO': {'fname': 'Richard', 'sname': 'Sorge'}}] + - [1, 777, {'town': 'London', 'FIO': {'fname': 'James', 'sname': 'Bond'}}] + - [1, 1, {'town': 'Moscow', 'FIO': {'fname': 'Max', 'sname': 'Isaev'}}] +... +pk:select({1, 'Berlin'}) +--- +- - [1, 45, {'town': 'Berlin', 'FIO': {'fname': 'Richard', 'sname': 'Sorge'}}] +... +name:select({}) +--- +- - [1, 777, {'town': 'London', 'FIO': {'fname': 'James', 'sname': 'Bond'}}] + - [1, 1, {'town': 'Moscow', 'FIO': {'fname': 'Max', 'sname': 'Isaev'}}] + - [4, 45, {'town': 'Berlin', 'FIO': {'fname': 'Max', 'extra': 'Otto', 'sname': 'Stierlitz'}}] + - [1, 45, {'town': 'Berlin', 'FIO': {'fname': 'Richard', 'sname': 'Sorge'}}] +... +name:select({'Max'}) +--- +- - [1, 1, {'town': 'Moscow', 'FIO': {'fname': 'Max', 'sname': 'Isaev'}}] + - [4, 45, {'town': 'Berlin', 'FIO': {'fname': 'Max', 'extra': 'Otto', 'sname': 'Stierlitz'}}] +... +name:get({'Max', 'Stierlitz', 'Otto'}) +--- +- [4, 45, {'town': 'Berlin', 'FIO': {'fname': 'Max', 'extra': 'Otto', 'sname': 'Stierlitz'}}] +... +box.snapshot() +--- +- ok +... +test_run:cmd("restart server default") +s = box.space["withdata"] +--- +... +pk = s.index["pk"] +--- +... +name = s.index["name"] +--- +... +pk:select({1}) +--- +- - [1, 45, {'town': 'Berlin', 'FIO': {'fname': 'Richard', 'sname': 'Sorge'}}] + - [1, 777, {'town': 'London', 'FIO': {'fname': 'James', 'sname': 'Bond'}}] + - [1, 1, {'town': 'Moscow', 'FIO': {'fname': 'Max', 'sname': 'Isaev'}}] +... +pk:select({1, 'Berlin'}) +--- +- - [1, 45, {'town': 'Berlin', 'FIO': {'fname': 'Richard', 'sname': 'Sorge'}}] +... +name:select({}) +--- +- - [1, 777, {'town': 'London', 'FIO': {'fname': 'James', 'sname': 'Bond'}}] + - [1, 1, {'town': 'Moscow', 'FIO': {'fname': 'Max', 'sname': 'Isaev'}}] + - [4, 45, {'town': 'Berlin', 'FIO': {'fname': 'Max', 'extra': 'Otto', 'sname': 'Stierlitz'}}] + - [1, 45, {'town': 'Berlin', 'FIO': {'fname': 'Richard', 'sname': 'Sorge'}}] +... +name:select({'Max'}) +--- +- - [1, 1, {'town': 'Moscow', 'FIO': {'fname': 'Max', 'sname': 'Isaev'}}] + - [4, 45, {'town': 'Berlin', 'FIO': {'fname': 'Max', 'extra': 'Otto', 'sname': 'Stierlitz'}}] +... +name:get({'Max', 'Stierlitz', 'Otto'}) +--- +- [4, 45, {'town': 'Berlin', 'FIO': {'fname': 'Max', 'extra': 'Otto', 'sname': 'Stierlitz'}}] +... +s:replace{4, 45, {town = 'Berlin', FIO = {fname = 'Max', sname = 'Stierlitz'}}} +--- +- [4, 45, {'town': 'Berlin', 'FIO': {'fname': 'Max', 'sname': 'Stierlitz'}}] +... +name:select({'Max', 'Stierlitz'}) +--- +- - [4, 45, {'town': 'Berlin', 'FIO': {'fname': 'Max', 'sname': 'Stierlitz'}}] +... +town = s:create_index('town', {unique = false, parts = {{3, 'string', path = 'town'}}}) +--- +... +town:select({'Berlin'}) +--- +- - [1, 45, {'town': 'Berlin', 'FIO': {'fname': 'Richard', 'sname': 'Sorge'}}] + - [4, 45, {'town': 'Berlin', 'FIO': {'fname': 'Max', 'sname': 'Stierlitz'}}] +... +_ = s:delete({4, 'Berlin'}) +--- +... +town:select({'Berlin'}) +--- +- - [1, 45, {'town': 'Berlin', 'FIO': {'fname': 'Richard', 'sname': 'Sorge'}}] +... +s:update({1, 'Berlin'}, {{"+", 2, 45}}) +--- +- [1, 90, {'town': 'Berlin', 'FIO': {'fname': 'Richard', 'sname': 'Sorge'}}] +... +box.snapshot() +--- +- ok +... +s:upsert({1, 90, {town = 'Berlin', FIO = {fname = 'X', sname = 'Y'}}}, {{'+', 2, 1}}) +--- +... +town:select() +--- +- - [1, 91, {'town': 'Berlin', 'FIO': {'fname': 'Richard', 'sname': 'Sorge'}}] + - [1, 777, {'town': 'London', 'FIO': {'fname': 'James', 'sname': 'Bond'}}] + - [1, 1, {'town': 'Moscow', 'FIO': {'fname': 'Max', 'sname': 'Isaev'}}] +... +name:drop() +--- +... +town:select() +--- +- - [1, 91, {'town': 'Berlin', 'FIO': {'fname': 'Richard', 'sname': 'Sorge'}}] + - [1, 777, {'town': 'London', 'FIO': {'fname': 'James', 'sname': 'Bond'}}] + - [1, 1, {'town': 'Moscow', 'FIO': {'fname': 'Max', 'sname': 'Isaev'}}] +... +s:drop() +--- +... diff --git a/test/engine/json.test.lua b/test/engine/json.test.lua new file mode 100644 index 000000000..181eae02c --- /dev/null +++ b/test/engine/json.test.lua @@ -0,0 +1,167 @@ +test_run = require('test_run').new() +engine = test_run:get_cfg('engine') +-- +-- gh-1012: Indexes for JSON-defined paths. +-- +s = box.schema.space.create('withdata', {engine = engine}) +-- Test build field tree conflicts. +s:create_index('test1', {parts = {{2, 'number'}, {3, 'str', path = 'FIO["fname"]'}, {3, 'str', path = '["FIO"].fname'}}}) +s:create_index('test1', {parts = {{2, 'number'}, {3, 'str', path = 666}, {3, 'str', path = '["FIO"]["fname"]'}}}) +s:create_index('test1', {parts = {{2, 'number'}, {3, 'map', path = 'FIO'}}}) +s:create_index('test1', {parts = {{2, 'number'}, {3, 'array', path = '[1]'}}}) +s:create_index('test1', {parts = {{2, 'number'}, {3, 'str', path = 'FIO'}, {3, 'str', path = 'FIO.fname'}}}) +s:create_index('test1', {parts = {{2, 'number'}, {3, 'str', path = '[1].sname'}, {3, 'str', path = '["FIO"].fname'}}}) +s:create_index('test1', {parts = {{2, 'number'}, {3, 'str', path = 'FIO....fname'}}}) +idx = s:create_index('test1', {parts = {{2, 'number'}, {3, 'str', path = 'FIO.fname', is_nullable = false}, {3, 'str', path = '["FIO"]["sname"]'}}}) +idx ~= nil +idx.parts[2].path == 'FIO.fname' +-- Test format mismatch. +format = {{'id', 'unsigned'}, {'meta', 'unsigned'}, {'data', 'array'}, {'age', 'unsigned'}, {'level', 'unsigned'}} +s:format(format) +format = {{'id', 'unsigned'}, {'meta', 'unsigned'}, {'data', 'map'}, {'age', 'unsigned'}, {'level', 'unsigned'}} +s:format(format) +s:create_index('test2', {parts = {{2, 'number'}, {3, 'number', path = 'FIO.fname'}, {3, 'str', path = '["FIO"]["sname"]'}}}) +-- Test incompatable tuple insertion. +s:insert{7, 7, {town = 'London', FIO = 666}, 4, 5} +s:insert{7, 7, {town = 'London', FIO = {fname = 666, sname = 'Bond'}}, 4, 5} +s:insert{7, 7, {town = 'London', FIO = {fname = "James"}}, 4, 5} +s:insert{7, 7, {town = 'London', FIO = {fname = 'James', sname = 'Bond'}}, 4, 5} +s:insert{7, 7, {town = 'London', FIO = {fname = 'James', sname = 'Bond'}}, 4, 5} +s:insert{7, 7, {town = 'London', FIO = {fname = 'James', sname = 'Bond', data = "extra"}}, 4, 5} +s:insert{7, 7, {town = 'Moscow', FIO = {fname = 'Max', sname = 'Isaev', data = "extra"}}, 4, 5} +idx:select() +idx:min() +idx:max() +s:drop() + +-- Test upsert of JSON-indexed data. +s = box.schema.create_space('withdata', {engine = engine}) +parts = {} +parts[1] = {1, 'unsigned', path='[2]'} +pk = s:create_index('pk', {parts = parts}) +s:insert{{1, 2}, 3} +s:upsert({{box.null, 2}}, {{'+', 2, 5}}) +s:get(2) +s:drop() + +-- Test index creation on space with data. +s = box.schema.space.create('withdata', {engine = engine}) +pk = s:create_index('primary', { type = 'tree', parts = {{2, 'number'}} }) +s:insert{1, 1, 7, {town = 'London', FIO = 1234}, 4, 5} +s:insert{2, 2, 7, {town = 'London', FIO = {fname = 'James', sname = 'Bond'}}, 4, 5} +s:insert{3, 3, 7, {town = 'London', FIO = {fname = 'James', sname = 'Bond'}}, 4, 5} +s:insert{4, 4, 7, {town = 'London', FIO = {1,2,3}}, 4, 5} +s:create_index('test1', {parts = {{3, 'number'}, {4, 'str', path = '["FIO"]["fname"]'}, {4, 'str', path = '["FIO"]["sname"]'}}}) +_ = s:delete(1) +s:create_index('test1', {parts = {{3, 'number'}, {4, 'str', path = '["FIO"]["fname"]'}, {4, 'str', path = '["FIO"]["sname"]'}}}) +_ = s:delete(2) +s:create_index('test1', {parts = {{3, 'number'}, {4, 'str', path = '["FIO"]["fname"]'}, {4, 'str', path = '["FIO"]["sname"]'}}}) +_ = s:delete(4) +idx = s:create_index('test1', {parts = {{3, 'number'}, {4, 'str', path = '["FIO"]["fname"]', is_nullable = true}, {4, 'str', path = '["FIO"]["sname"]'}, {4, 'str', path = '["FIO"]["extra"]', is_nullable = true}}}) +idx ~= nil +s:create_index('test2', {parts = {{3, 'number'}, {4, 'number', path = '["FIO"]["fname"]'}}}) +idx2 = s:create_index('test2', {parts = {{3, 'number'}, {4, 'str', path = '["FIO"]["fname"]'}}}) +idx2 ~= nil +t = s:insert{5, 5, 7, {town = 'Matrix', FIO = {fname = 'Agent', sname = 'Smith'}}, 4, 5} +idx:select() +idx:min() +idx:max() +idx:drop() +s:drop() + +-- Test complex JSON indexes with nullable fields. +s = box.schema.space.create('withdata', {engine = engine}) +parts = {} +parts[1] = {1, 'str', path='[3][2].a'} +parts[2] = {1, 'unsigned', path = '[3][1]'} +parts[3] = {2, 'str', path = '[2].d[1]'} +pk = s:create_index('primary', { type = 'tree', parts = parts}) +s:insert{{1, 2, {3, {3, a = 'str', b = 5}}}, {'c', {d = {'e', 'f'}, e = 'g'}}, 6, {1, 2, 3}} +s:insert{{1, 2, {3, {a = 'str', b = 1}}}, {'c', {d = {'e', 'f'}, e = 'g'}}, 6} +parts = {} +parts[1] = {4, 'unsigned', path='[1]', is_nullable = false} +parts[2] = {4, 'unsigned', path='[2]', is_nullable = true} +parts[3] = {4, 'unsigned', path='[4]', is_nullable = true} +trap_idx = s:create_index('trap', { type = 'tree', parts = parts}) +s:insert{{1, 2, {3, {3, a = 'str2', b = 5}}}, {'c', {d = {'e', 'f'}, e = 'g'}}, 6, {}} +parts = {} +parts[1] = {1, 'unsigned', path='[3][2].b' } +parts[2] = {3, 'unsigned'} +crosspart_idx = s:create_index('crosspart', { parts = parts}) +s:insert{{1, 2, {3, {a = 'str2', b = 2}}}, {'c', {d = {'e', 'f'}, e = 'g'}}, 6, {9, 2, 3}} +parts = {} +parts[1] = {1, 'unsigned', path='[3][2].b'} +num_idx = s:create_index('numeric', {parts = parts}) +s:insert{{1, 2, {3, {a = 'str3', b = 9}}}, {'c', {d = {'e', 'f'}, e = 'g'}}, 6, {0}} +num_idx:get(2) +num_idx:select() +num_idx:max() +num_idx:min() +crosspart_idx:max() == num_idx:max() +crosspart_idx:min() == num_idx:min() +trap_idx:max() +trap_idx:min() +s:drop() + +-- Test index alter. +s = box.schema.space.create('withdata', {engine = engine}) +pk_simplified = s:create_index('primary', { type = 'tree', parts = {{1, 'unsigned'}}}) +pk_simplified.path == box.NULL +idx = s:create_index('idx', {parts = {{2, 'integer', path = 'a'}}}) +s:insert{31, {a = 1, aa = -1}} +s:insert{22, {a = 2, aa = -2}} +s:insert{13, {a = 3, aa = -3}} +idx:select() +idx:alter({parts = {{2, 'integer', path = 'aa'}}}) +idx:select() +s:drop() + +-- Incompatible format change. +s = box.schema.space.create('withdata') +i = s:create_index('pk', {parts = {{1, 'integer', path = '[1]'}}}) +s:insert{{-1}} +i:alter{parts = {{1, 'string', path = '[1]'}}} +s:insert{{'a'}} +i:drop() +i = s:create_index('pk', {parts = {{1, 'integer', path = '[1].FIO'}}}) +s:insert{{{FIO=-1}}} +i:alter{parts = {{1, 'integer', path = '[1][1]'}}} +i:alter{parts = {{1, 'integer', path = '[1].FIO[1]'}}} +s:drop() + +-- Test snapshotting and recovery. +s = box.schema.space.create('withdata', {engine = engine}) +pk = s:create_index('pk', {parts = {{1, 'integer'}, {3, 'string', path = 'town'}}}) +name = s:create_index('name', {parts = {{3, 'string', path = 'FIO.fname'}, {3, 'string', path = 'FIO.sname'}, {3, 'string', path = 'FIO.extra', is_nullable = true}}}) +s:insert{1, 1, {town = 'Moscow', FIO = {fname = 'Max', sname = 'Isaev'}}} +s:insert{1, 777, {town = 'London', FIO = {fname = 'James', sname = 'Bond'}}} +s:insert{1, 45, {town = 'Berlin', FIO = {fname = 'Richard', sname = 'Sorge'}}} +s:insert{4, 45, {town = 'Berlin', FIO = {fname = 'Max', extra = 'Otto', sname = 'Stierlitz'}}} +pk:select({1}) +pk:select({1, 'Berlin'}) +name:select({}) +name:select({'Max'}) +name:get({'Max', 'Stierlitz', 'Otto'}) +box.snapshot() +test_run:cmd("restart server default") +s = box.space["withdata"] +pk = s.index["pk"] +name = s.index["name"] +pk:select({1}) +pk:select({1, 'Berlin'}) +name:select({}) +name:select({'Max'}) +name:get({'Max', 'Stierlitz', 'Otto'}) +s:replace{4, 45, {town = 'Berlin', FIO = {fname = 'Max', sname = 'Stierlitz'}}} +name:select({'Max', 'Stierlitz'}) +town = s:create_index('town', {unique = false, parts = {{3, 'string', path = 'town'}}}) +town:select({'Berlin'}) +_ = s:delete({4, 'Berlin'}) +town:select({'Berlin'}) +s:update({1, 'Berlin'}, {{"+", 2, 45}}) +box.snapshot() +s:upsert({1, 90, {town = 'Berlin', FIO = {fname = 'X', sname = 'Y'}}}, {{'+', 2, 1}}) +town:select() +name:drop() +town:select() +s:drop() -- 2.20.1