From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: From: Kirill Shcherbatov Subject: [PATCH v3.1 3/5] box: introduce JSON indexes Date: Thu, 20 Sep 2018 10:46:51 +0300 Message-Id: <5888720ea2e84d6b148ff65c7a7877ce934f33ce.1537429335.git.kshcherbatov@tarantool.org> In-Reply-To: References: In-Reply-To: References: To: tarantool-patches@freelists.org Cc: vdavydov.dev@gmail.com, Kirill Shcherbatov List-ID: As we need to store user-defined JSON path in key_part and key_part_def, we have introduced path and path_len fields. JSON path is verified and transformed to canonical form on index msgpack unpack. Path string stored as a part of the key_def allocation: +-------+---------+-------+---------+-------+-------+-------+ |key_def|key_part1| ... |key_partN| path1 | pathK | pathN | +-------+---------+-------+---------+-------+-------+-------+ | ^ |-> path _________________| Because of field names specified as format could be changed key_part path persisted in Tarantool should be always started with first-level field access via array index(not by name). To work with JSON-defined indexes we use format JSON path hashtable data_path and a tree of intermediate path fields attached to format root fields. : format->data_path [2].FIO.fname -> field "fname" {type=str, off_slot=-1} [2].FIO.sname -> field "sname" {type=str, off_slot=-2} : format->field[2] {type = map} | FIO {type = map} | "fname" | "sname" {type=str,off_slot=-1} ____|____ {type = str,off_slot=-2} Leaf fields used in Index have initialized offset_slot. On new tuple creation we observe fields tree and use leaf records to init tuple field_map. At the same time we use data_path hashtable on tuple data access by index(when cached offset_slot is invalid). All paths stored at the end of format allocation: JSON-tree fields same as format->path_hash point to them. +------------+------------+-------+------------+-------+ |tuple_format|tuple_field1| ... |tuple_fieldN| pathK | +------------+------------+-------+------------+-------+ New routine tuple_format_add_json_path is used to construct all internal structures for JSON path on new format creation and duplicating. Paths are deduplicated in format allocation. Part of #1012. --- src/box/errcode.h | 2 +- src/box/index_def.c | 10 +- src/box/key_def.c | 300 +++++++++++++-- src/box/key_def.h | 38 +- src/box/lua/space.cc | 5 + src/box/memtx_engine.c | 5 + src/box/schema.cc | 12 +- src/box/tuple.c | 12 +- src/box/tuple_compare.cc | 119 ++++-- src/box/tuple_extract_key.cc | 136 +++++-- src/box/tuple_format.c | 873 ++++++++++++++++++++++++++++++++++++------- src/box/tuple_format.h | 63 +++- src/box/tuple_hash.cc | 47 ++- src/box/vinyl.c | 5 + src/box/vy_log.c | 3 +- src/box/vy_lsm.c | 1 + src/box/vy_point_lookup.c | 2 - src/box/vy_stmt.c | 148 ++++++-- test/box/misc.result | 57 +-- test/engine/tuple.result | 387 +++++++++++++++++++ test/engine/tuple.test.lua | 109 ++++++ 21 files changed, 1996 insertions(+), 338 deletions(-) diff --git a/src/box/errcode.h b/src/box/errcode.h index 4115e6b..464f413 100644 --- a/src/box/errcode.h +++ b/src/box/errcode.h @@ -107,7 +107,7 @@ struct errcode_record { /* 52 */_(ER_FUNCTION_EXISTS, "Function '%s' already exists") \ /* 53 */_(ER_BEFORE_REPLACE_RET, "Invalid return value of space:before_replace trigger: expected tuple or nil, got %s") \ /* 54 */_(ER_FUNCTION_MAX, "A limit on the total number of functions has been reached: %u") \ - /* 55 */_(ER_UNUSED4, "") \ + /* 55 */_(ER_DATA_STRUCTURE_MISMATCH, "Tuple doesn't math document structure: %s") \ /* 56 */_(ER_USER_MAX, "A limit on the total number of users has been reached: %u") \ /* 57 */_(ER_NO_SUCH_ENGINE, "Space engine '%s' does not exist") \ /* 58 */_(ER_RELOAD_CFG, "Can't set option '%s' dynamically") \ diff --git a/src/box/index_def.c b/src/box/index_def.c index 9cda63c..f67b952 100644 --- a/src/box/index_def.c +++ b/src/box/index_def.c @@ -209,8 +209,14 @@ index_def_is_valid(struct index_def *index_def, const char *space_name) * Courtesy to a user who could have made * a typo. */ - if (index_def->key_def->parts[i].fieldno == - index_def->key_def->parts[j].fieldno) { + struct key_part *part_a = &index_def->key_def->parts[i]; + struct key_part *part_b = &index_def->key_def->parts[j]; + if ((part_a->fieldno == part_b->fieldno && + part_a->path == NULL && part_b->path == NULL) || + (part_a->path_len != 0 && + part_a->path_len == part_b->path_len && + memcmp(part_a->path, part_b->path, + part_a->path_len) == 0)) { diag_set(ClientError, ER_MODIFY_INDEX, index_def->name, space_name, "same key part is indexed twice"); diff --git a/src/box/key_def.c b/src/box/key_def.c index e8a6fa4..5366eb7 100644 --- a/src/box/key_def.c +++ b/src/box/key_def.c @@ -35,12 +35,16 @@ #include "column_mask.h" #include "schema_def.h" #include "coll_id_cache.h" +#include "fiber.h" +#include "assoc.h" +#include "json/path.h" static const struct key_part_def key_part_def_default = { 0, field_type_MAX, COLL_NONE, false, + NULL }; static int64_t @@ -53,6 +57,7 @@ part_type_by_name_wrapper(const char *str, uint32_t len) #define PART_OPT_FIELD "field" #define PART_OPT_COLLATION "collation" #define PART_OPT_NULLABILITY "is_nullable" +#define PART_OPT_PATH "path" const struct opt_def part_def_reg[] = { OPT_DEF_ENUM(PART_OPT_TYPE, field_type, struct key_part_def, type, @@ -61,6 +66,7 @@ const struct opt_def part_def_reg[] = { OPT_DEF(PART_OPT_COLLATION, OPT_UINT32, struct key_part_def, coll_id), OPT_DEF(PART_OPT_NULLABILITY, OPT_BOOL, struct key_part_def, is_nullable), + OPT_DEF(PART_OPT_PATH, OPT_STRPTR, struct key_part_def, path), OPT_END, }; @@ -96,13 +102,24 @@ const uint32_t key_mp_type[] = { struct key_def * key_def_dup(const struct key_def *src) { - size_t sz = key_def_sizeof(src->part_count); + const struct key_part *parts = src->parts; + const struct key_part *parts_end = parts + src->part_count; + size_t sz = 0; + for (; parts < parts_end; parts++) + sz += parts->path != NULL ? parts->path_len + 1 : 0; + sz = key_def_sizeof(src->part_count, sz); struct key_def *res = (struct key_def *)malloc(sz); if (res == NULL) { diag_set(OutOfMemory, sz, "malloc", "res"); return NULL; } memcpy(res, src, sz); + for (uint32_t i = 0; i < src->part_count; i++) { + if (src->parts[i].path == NULL) + continue; + size_t path_offset = src->parts[i].path - (char *)src; + res->parts[i].path = (char *)res + path_offset; + } return res; } @@ -110,8 +127,17 @@ void key_def_swap(struct key_def *old_def, struct key_def *new_def) { assert(old_def->part_count == new_def->part_count); - for (uint32_t i = 0; i < new_def->part_count; i++) - SWAP(old_def->parts[i], new_def->parts[i]); + for (uint32_t i = 0; i < new_def->part_count; i++) { + if (old_def->parts[i].path == NULL) { + SWAP(old_def->parts[i], new_def->parts[i]); + } else { + size_t path_offset = + old_def->parts[i].path - (char *)old_def; + SWAP(old_def->parts[i], new_def->parts[i]); + old_def->parts[i].path = (char *)old_def + path_offset; + new_def->parts[i].path = (char *)new_def + path_offset; + } + } SWAP(*old_def, *new_def); } @@ -131,9 +157,9 @@ key_def_set_cmp(struct key_def *def) } struct key_def * -key_def_new(uint32_t part_count) +key_def_new(uint32_t part_count, size_t paths_size) { - size_t sz = key_def_sizeof(part_count); + size_t sz = key_def_sizeof(part_count, paths_size); /** Use calloc() to zero comparator function pointers. */ struct key_def *key_def = (struct key_def *) calloc(1, sz); if (key_def == NULL) { @@ -148,10 +174,13 @@ key_def_new(uint32_t part_count) struct key_def * key_def_new_with_parts(struct key_part_def *parts, uint32_t part_count) { - struct key_def *def = key_def_new(part_count); + size_t sz = 0; + for (uint32_t i = 0; i < part_count; i++) + sz += parts[i].path != NULL ? strlen(parts[i].path) + 1 : 0; + struct key_def *def = key_def_new(part_count, sz); if (def == NULL) return NULL; - + char *data = (char *)def + key_def_sizeof(part_count, 0); for (uint32_t i = 0; i < part_count; i++) { struct key_part_def *part = &parts[i]; struct coll *coll = NULL; @@ -165,14 +194,22 @@ key_def_new_with_parts(struct key_part_def *parts, uint32_t part_count) } coll = coll_id->coll; } + uint32_t path_len = 0; + if (part->path != NULL) { + path_len = strlen(part->path); + def->parts[i].path = data; + data += path_len + 1; + } key_def_set_part(def, i, part->fieldno, part->type, - part->is_nullable, coll, part->coll_id); + part->is_nullable, coll, part->coll_id, + part->path, path_len); } return def; } -void -key_def_dump_parts(const struct key_def *def, struct key_part_def *parts) +int +key_def_dump_parts(struct region *pool, const struct key_def *def, + struct key_part_def *parts) { for (uint32_t i = 0; i < def->part_count; i++) { const struct key_part *part = &def->parts[i]; @@ -181,13 +218,26 @@ key_def_dump_parts(const struct key_def *def, struct key_part_def *parts) part_def->type = part->type; part_def->is_nullable = part->is_nullable; part_def->coll_id = part->coll_id; + if (part->path != NULL) { + part_def->path = region_alloc(pool, part->path_len + 1); + if (part_def->path == NULL) { + diag_set(OutOfMemory, part->path_len + 1, + "region_alloc", "part_def->path"); + return -1; + } + memcpy(part_def->path, part->path, part->path_len); + part_def->path[part->path_len] = '\0'; + } else { + part_def->path = NULL; + } } + return 0; } box_key_def_t * box_key_def_new(uint32_t *fields, uint32_t *types, uint32_t part_count) { - struct key_def *key_def = key_def_new(part_count); + struct key_def *key_def = key_def_new(part_count, 0); if (key_def == NULL) return key_def; @@ -195,7 +245,7 @@ box_key_def_new(uint32_t *fields, uint32_t *types, uint32_t part_count) key_def_set_part(key_def, item, fields[item], (enum field_type)types[item], key_part_def_default.is_nullable, NULL, - COLL_NONE); + COLL_NONE, NULL, 0); } return key_def; } @@ -241,6 +291,13 @@ key_part_cmp(const struct key_part *parts1, uint32_t part_count1, if (part1->is_nullable != part2->is_nullable) return part1->is_nullable < part2->is_nullable ? -1 : 1; + /* Lexicographic strings order. */ + if (part1->path_len != part2->path_len) + return part1->path_len - part2->path_len; + int rc = 0; + if ((rc = memcmp(part1->path, part2->path, + part1->path_len)) != 0) + return rc; } return part_count1 < part_count2 ? -1 : part_count1 > part_count2; } @@ -248,16 +305,28 @@ key_part_cmp(const struct key_part *parts1, uint32_t part_count1, void key_def_set_part(struct key_def *def, uint32_t part_no, uint32_t fieldno, enum field_type type, bool is_nullable, struct coll *coll, - uint32_t coll_id) + uint32_t coll_id, const char *path, uint32_t path_len) { assert(part_no < def->part_count); assert(type < field_type_MAX); def->is_nullable |= is_nullable; + def->has_json_paths |= path != NULL; def->parts[part_no].is_nullable = is_nullable; def->parts[part_no].fieldno = fieldno; def->parts[part_no].type = type; def->parts[part_no].coll = coll; def->parts[part_no].coll_id = coll_id; + if (path != NULL) { + def->parts[part_no].path_len = path_len; + assert(def->parts[part_no].path != NULL); + memcpy(def->parts[part_no].path, path, path_len); + def->parts[part_no].path[path_len] = '\0'; + def->parts[part_no].path_hash = mh_strn_hash(path, path_len); + } else { + def->parts[part_no].path_len = 0; + def->parts[part_no].path = NULL; + def->parts[part_no].path_hash = 0; + } column_mask_set_fieldno(&def->column_mask, fieldno); /** * When all parts are set, initialize the tuple @@ -282,13 +351,13 @@ key_def_update_optionality(struct key_def *def, uint32_t min_field_count) for (uint32_t i = 0; i < def->part_count; ++i) { struct key_part *part = &def->parts[i]; def->has_optional_parts |= part->is_nullable && - min_field_count < part->fieldno + 1; + (min_field_count < + part->fieldno + 1); /* * One optional part is enough to switch to new * comparators. */ - if (def->has_optional_parts) - break; + if (def->has_optional_parts) break; } key_def_set_cmp(def); } @@ -302,8 +371,15 @@ key_def_snprint_parts(char *buf, int size, const struct key_part_def *parts, for (uint32_t i = 0; i < part_count; i++) { const struct key_part_def *part = &parts[i]; assert(part->type < field_type_MAX); - SNPRINT(total, snprintf, buf, size, "%d, '%s'", - (int)part->fieldno, field_type_strs[part->type]); + if (part->path != NULL) { + SNPRINT(total, snprintf, buf, size, "%d, '%s', '%s'", + (int) part->fieldno, part->path, + field_type_strs[part->type]); + } else { + SNPRINT(total, snprintf, buf, size, "%d, '%s'", + (int) part->fieldno, + field_type_strs[part->type]); + } if (i < part_count - 1) SNPRINT(total, snprintf, buf, size, ", "); } @@ -322,6 +398,8 @@ key_def_sizeof_parts(const struct key_part_def *parts, uint32_t part_count) count++; if (part->is_nullable) count++; + if (part->path != NULL) + count++; size += mp_sizeof_map(count); size += mp_sizeof_str(strlen(PART_OPT_FIELD)); size += mp_sizeof_uint(part->fieldno); @@ -336,6 +414,10 @@ key_def_sizeof_parts(const struct key_part_def *parts, uint32_t part_count) size += mp_sizeof_str(strlen(PART_OPT_NULLABILITY)); size += mp_sizeof_bool(part->is_nullable); } + if (part->path != NULL) { + size += mp_sizeof_str(strlen(PART_OPT_PATH)); + size += mp_sizeof_str(strlen(part->path)); + } } return size; } @@ -349,6 +431,8 @@ key_def_encode_parts(char *data, const struct key_part_def *parts, int count = 2; if (part->coll_id != COLL_NONE) count++; + if (part->path != NULL) + count++; if (part->is_nullable) count++; data = mp_encode_map(data, count); @@ -370,6 +454,12 @@ key_def_encode_parts(char *data, const struct key_part_def *parts, strlen(PART_OPT_NULLABILITY)); data = mp_encode_bool(data, part->is_nullable); } + if (part->path != NULL) { + data = mp_encode_str(data, PART_OPT_PATH, + strlen(PART_OPT_PATH)); + data = mp_encode_str(data, part->path, + strlen(part->path)); + } } return data; } @@ -430,8 +520,111 @@ key_def_decode_parts_166(struct key_part_def *parts, uint32_t part_count, fields[part->fieldno].is_nullable : key_part_def_default.is_nullable); part->coll_id = COLL_NONE; + part->path = NULL; + } + return 0; +} + +/** + * Verify key_part JSON path and convert to canonical form. + * + * @param region Region to make allocations. + * @param part Part with path to update. + * @param path_extra Extra allocated space to reuse if possible. + * @param path_extra_size The @path_extra size. + * + * @retval -1 on error. + * @retval 0 on success. + */ +static int +key_def_normalize_json_path(struct region *region, struct key_part_def *part, + char **path_extra, uint32_t *path_extra_size) +{ + uint32_t allocated_size = *path_extra_size; + char *path = *path_extra; + + uint32_t path_len = strlen(part->path); + struct json_path_parser parser; + struct json_path_node node; + json_path_parser_create(&parser, part->path, path_len); + /* + * A worst-case scenario is .a -> ["a"] + * i.e. 2.5 * path_len + 1 is enough. + */ + uint32_t new_path_size = 2.5 * path_len + 1; + if (new_path_size >= allocated_size) { + path = region_alloc(region, new_path_size); + if (path == NULL) { + diag_set(OutOfMemory, new_path_size, + "region_alloc", "path"); + return -1; + } + allocated_size = new_path_size; + } + assert(path != NULL); + part->path = path; + int rc = json_path_next(&parser, &node); + if (rc != 0) + goto error_invalid_json; + if (node.type != JSON_PATH_NUM) { + diag_set(ClientError, ER_WRONG_INDEX_OPTIONS, + part->fieldno, + "invalid JSON path: first part should " + "be defined as array index"); + return -1; + } + if (node.num - TUPLE_INDEX_BASE != part->fieldno) { + diag_set(ClientError, ER_WRONG_INDEX_OPTIONS, + part->fieldno, + "invalid JSON path: first part refers " + "to invalid field"); + return -1; + } + uint32_t lexemes = 0; + do { + if (node.type == JSON_PATH_NUM) { + path += sprintf(path, "[%llu]", + (unsigned long long) node.num); + } else if (node.type == JSON_PATH_STR) { + path += sprintf(path, "[\"%.*s\"]", node.len, node.str); + } else { + unreachable(); + } + lexemes++; + } while ((rc = json_path_next(&parser, &node)) == 0 && + node.type != JSON_PATH_END); + if (rc != 0 || node.type != JSON_PATH_END) + goto error_invalid_json; + if (lexemes == 1) { + /* JSON index is useless. */ + path = part->path; + part->path = NULL; + } else { + /* Skip terminating zero. */ + path++; + /* Account constructed string size. */ + allocated_size -= path - part->path; + } + /* Going to try to reuse extra allocation next time. */ + if (allocated_size > (uint32_t)parser.src_len) { + /* Use rest of new buffer next time. */ + *path_extra = path; + *path_extra_size = allocated_size; + } else { + /* Reuse old path buffer. */ + *path_extra = (char *)parser.src; + *path_extra_size = parser.src_len; } return 0; + +error_invalid_json: ; + const char *err_msg = + tt_sprintf("invalid JSON path '%.*s': path has invalid "\ + "structure (error at position %d)", parser.src_len, + parser.src, rc); + diag_set(ClientError, ER_WRONG_INDEX_OPTIONS, + part->fieldno + TUPLE_INDEX_BASE, err_msg); + return -1; } int @@ -443,8 +636,11 @@ key_def_decode_parts(struct key_part_def *parts, uint32_t part_count, return key_def_decode_parts_166(parts, part_count, data, fields, field_count); } - for (uint32_t i = 0; i < part_count; i++) { - struct key_part_def *part = &parts[i]; + char *path = NULL; + uint32_t allocated_size = 0; + struct key_part_def *part = parts; + struct region *region = &fiber()->gc; + for (uint32_t i = 0; i < part_count; i++, part++) { if (mp_typeof(**data) != MP_MAP) { diag_set(ClientError, ER_WRONG_INDEX_OPTIONS, i + TUPLE_INDEX_BASE, @@ -454,7 +650,7 @@ key_def_decode_parts(struct key_part_def *parts, uint32_t part_count, *part = key_part_def_default; if (opts_decode(part, part_def_reg, data, ER_WRONG_INDEX_OPTIONS, i + TUPLE_INDEX_BASE, - NULL) != 0) + region) != 0) return -1; if (part->type == field_type_MAX) { diag_set(ClientError, ER_WRONG_INDEX_OPTIONS, @@ -471,6 +667,10 @@ key_def_decode_parts(struct key_part_def *parts, uint32_t part_count, "string and scalar parts"); return -1; } + if (part->path != NULL && + key_def_normalize_json_path(region, part, &path, + &allocated_size) != 0) + return -1; } return 0; } @@ -495,20 +695,25 @@ key_def_decode_parts_160(struct key_part_def *parts, uint32_t part_count, fields[part->fieldno].is_nullable : key_part_def_default.is_nullable); part->coll_id = COLL_NONE; + part->path = NULL; } return 0; } -const struct key_part * -key_def_find(const struct key_def *key_def, uint32_t fieldno) +bool +key_def_contains_part(const struct key_def *key_def, + const struct key_part *to_find) { const struct key_part *part = key_def->parts; const struct key_part *end = part + key_def->part_count; for (; part != end; part++) { - if (part->fieldno == fieldno) - return part; + if (part->fieldno == to_find->fieldno && + part->path_len == to_find->path_len && + (part->path == NULL || memcmp(part->path, to_find->path, + to_find->path_len) == 0)) + return true; } - return NULL; + return false; } bool @@ -517,7 +722,7 @@ key_def_contains(const struct key_def *first, const struct key_def *second) const struct key_part *part = second->parts; const struct key_part *end = part + second->part_count; for (; part != end; part++) { - if (key_def_find(first, part->fieldno) == NULL) + if (! key_def_contains_part(first, part)) return false; } return true; @@ -531,18 +736,25 @@ key_def_merge(const struct key_def *first, const struct key_def *second) * Find and remove part duplicates, i.e. parts counted * twice since they are present in both key defs. */ - const struct key_part *part = second->parts; - const struct key_part *end = part + second->part_count; + size_t sz = 0; + const struct key_part *part = first->parts; + const struct key_part *end = part + first->part_count; for (; part != end; part++) { - if (key_def_find(first, part->fieldno)) + if (part->path != NULL) + sz += part->path_len + 1; + } + part = second->parts; + end = part + second->part_count; + for (; part != end; part++) { + if (key_def_contains_part(first, part)) --new_part_count; + else if (part->path != NULL) + sz += part->path_len + 1; } - - struct key_def *new_def; - new_def = (struct key_def *)calloc(1, key_def_sizeof(new_part_count)); + sz = key_def_sizeof(new_part_count, sz); + struct key_def *new_def = (struct key_def *)calloc(1, sz); if (new_def == NULL) { - diag_set(OutOfMemory, key_def_sizeof(new_part_count), "malloc", - "new_def"); + diag_set(OutOfMemory, sz, "calloc", "new_def"); return NULL; } new_def->part_count = new_part_count; @@ -550,24 +762,36 @@ key_def_merge(const struct key_def *first, const struct key_def *second) new_def->is_nullable = first->is_nullable || second->is_nullable; new_def->has_optional_parts = first->has_optional_parts || second->has_optional_parts; + /* Path data write position in the new key_def. */ + char *data = (char *)new_def + key_def_sizeof(new_part_count, 0); /* Write position in the new key def. */ uint32_t pos = 0; /* Append first key def's parts to the new index_def. */ part = first->parts; end = part + first->part_count; for (; part != end; part++) { + if (part->path != NULL) { + new_def->parts[pos].path = data; + data += part->path_len + 1; + } key_def_set_part(new_def, pos++, part->fieldno, part->type, - part->is_nullable, part->coll, part->coll_id); + part->is_nullable, part->coll, part->coll_id, + part->path, part->path_len); } /* Set-append second key def's part to the new key def. */ part = second->parts; end = part + second->part_count; for (; part != end; part++) { - if (key_def_find(first, part->fieldno)) + if (key_def_contains_part(first, part)) continue; + if (part->path != NULL) { + new_def->parts[pos].path = data; + data += part->path_len + 1; + } key_def_set_part(new_def, pos++, part->fieldno, part->type, - part->is_nullable, part->coll, part->coll_id); + part->is_nullable, part->coll, part->coll_id, + part->path, part->path_len); } return new_def; } diff --git a/src/box/key_def.h b/src/box/key_def.h index 6cc705b..d451258 100644 --- a/src/box/key_def.h +++ b/src/box/key_def.h @@ -54,6 +54,8 @@ struct key_part_def { uint32_t coll_id; /** True if a key part can store NULLs. */ bool is_nullable; + /** JSON path to data. */ + char *path; }; /** @@ -74,6 +76,12 @@ struct key_part { struct coll *coll; /** True if a part can store NULLs. */ bool is_nullable; + /** JSON path to data in canonical form. */ + char *path; + /** JSON path length. */ + uint32_t path_len; + /** JSON path hash. */ + uint32_t path_hash; }; struct key_def; @@ -133,6 +141,10 @@ struct key_def { * fields assumed to be MP_NIL. */ bool has_optional_parts; + /** + * True, if some key part contain JSON path. + */ + bool has_json_paths; /** Key fields mask. @sa column_mask.h for details. */ uint64_t column_mask; /** The size of the 'parts' array. */ @@ -221,16 +233,17 @@ box_tuple_compare_with_key(const box_tuple_t *tuple_a, const char *key_b, /** \endcond public */ static inline size_t -key_def_sizeof(uint32_t part_count) +key_def_sizeof(uint32_t part_count, size_t paths_size) { - return sizeof(struct key_def) + sizeof(struct key_part) * part_count; + return sizeof(struct key_def) + sizeof(struct key_part) * part_count + + paths_size; } /** * Allocate a new key_def with the given part count. */ struct key_def * -key_def_new(uint32_t part_count); +key_def_new(uint32_t part_count, size_t paths_size); /** * Allocate a new key_def with the given part count @@ -242,8 +255,9 @@ key_def_new_with_parts(struct key_part_def *parts, uint32_t part_count); /** * Dump part definitions of the given key def. */ -void -key_def_dump_parts(const struct key_def *def, struct key_part_def *parts); +int +key_def_dump_parts(struct region *pool, const struct key_def *def, + struct key_part_def *parts); /** * Set a single key part in a key def. @@ -252,7 +266,7 @@ key_def_dump_parts(const struct key_def *def, struct key_part_def *parts); void key_def_set_part(struct key_def *def, uint32_t part_no, uint32_t fieldno, enum field_type type, bool is_nullable, struct coll *coll, - uint32_t coll_id); + uint32_t coll_id, const char *path, uint32_t path_len); /** * Update 'has_optional_parts' of @a key_def with correspondence @@ -314,12 +328,10 @@ key_def_decode_parts_160(struct key_part_def *parts, uint32_t part_count, const char **data, const struct field_def *fields, uint32_t field_count); -/** - * Returns the part in index_def->parts for the specified fieldno. - * If fieldno is not in index_def->parts returns NULL. - */ -const struct key_part * -key_def_find(const struct key_def *key_def, uint32_t fieldno); +/** Check if @a key_def contains @a to_find part. */ +bool +key_def_contains_part(const struct key_def *key_def, + const struct key_part *to_find); /** * Check if key definition @a first contains all parts of @@ -366,6 +378,8 @@ key_validate_parts(const struct key_def *key_def, const char *key, static inline bool key_def_is_sequential(const struct key_def *key_def) { + if (key_def->has_json_paths) + return false; for (uint32_t part_id = 0; part_id < key_def->part_count; part_id++) { if (key_def->parts[part_id].fieldno != part_id) return false; diff --git a/src/box/lua/space.cc b/src/box/lua/space.cc index 25b7e36..875e51f 100644 --- a/src/box/lua/space.cc +++ b/src/box/lua/space.cc @@ -295,6 +295,11 @@ lbox_fillspace(struct lua_State *L, struct space *space, int i) lua_pushnumber(L, part->fieldno + TUPLE_INDEX_BASE); lua_setfield(L, -2, "fieldno"); + if (part->path != NULL) { + lua_pushstring(L, part->path); + lua_setfield(L, -2, "path"); + } + lua_pushboolean(L, part->is_nullable); lua_setfield(L, -2, "is_nullable"); diff --git a/src/box/memtx_engine.c b/src/box/memtx_engine.c index 1f80ce5..554d6f9 100644 --- a/src/box/memtx_engine.c +++ b/src/box/memtx_engine.c @@ -1310,6 +1310,11 @@ memtx_index_def_change_requires_rebuild(struct index *index, return true; if (old_part->coll != new_part->coll) return true; + if (old_part->path_len != new_part->path_len) + return true; + if (memcmp(old_part->path, new_part->path, + old_part->path_len) != 0) + return true; } return false; } diff --git a/src/box/schema.cc b/src/box/schema.cc index 7f20f36..2411c8c 100644 --- a/src/box/schema.cc +++ b/src/box/schema.cc @@ -286,19 +286,19 @@ schema_init() * (and re-created) first. */ /* _schema - key/value space with schema description */ - struct key_def *key_def = key_def_new(1); /* part count */ + struct key_def *key_def = key_def_new(1, 0); if (key_def == NULL) diag_raise(); auto key_def_guard = make_scoped_guard([&] { key_def_delete(key_def); }); key_def_set_part(key_def, 0 /* part no */, 0 /* field no */, - FIELD_TYPE_STRING, false, NULL, COLL_NONE); + FIELD_TYPE_STRING, false, NULL, COLL_NONE, NULL, 0); sc_space_new(BOX_SCHEMA_ID, "_schema", key_def, &on_replace_schema, NULL); /* _space - home for all spaces. */ key_def_set_part(key_def, 0 /* part no */, 0 /* field no */, - FIELD_TYPE_UNSIGNED, false, NULL, COLL_NONE); + FIELD_TYPE_UNSIGNED, false, NULL, COLL_NONE, NULL, 0); /* _collation - collation description. */ sc_space_new(BOX_COLLATION_ID, "_collation", key_def, @@ -341,15 +341,15 @@ schema_init() NULL); key_def_delete(key_def); - key_def = key_def_new(2); /* part count */ + key_def = key_def_new(2, 0); if (key_def == NULL) diag_raise(); /* space no */ key_def_set_part(key_def, 0 /* part no */, 0 /* field no */, - FIELD_TYPE_UNSIGNED, false, NULL, COLL_NONE); + FIELD_TYPE_UNSIGNED, false, NULL, COLL_NONE, NULL, 0); /* index no */ key_def_set_part(key_def, 1 /* part no */, 1 /* field no */, - FIELD_TYPE_UNSIGNED, false, NULL, COLL_NONE); + FIELD_TYPE_UNSIGNED, false, NULL, COLL_NONE, NULL, 0); sc_space_new(BOX_INDEX_ID, "_index", key_def, &alter_space_on_replace_index, &on_stmt_begin_index); diff --git a/src/box/tuple.c b/src/box/tuple.c index d7dbad3..83bdad1 100644 --- a/src/box/tuple.c +++ b/src/box/tuple.c @@ -159,14 +159,22 @@ tuple_validate_raw(struct tuple_format *format, const char *tuple) /* Check field types */ struct tuple_field *field = &format->fields[0]; + const char *pos = tuple; uint32_t i = 0; uint32_t defined_field_count = MIN(field_count, format->field_count); for (; i < defined_field_count; ++i, ++field) { - if (key_mp_type_validate(field->type, mp_typeof(*tuple), + if (key_mp_type_validate(field->type, mp_typeof(*pos), ER_FIELD_TYPE, i + TUPLE_INDEX_BASE, field->is_nullable)) return -1; - mp_next(&tuple); + /* Check all JSON paths. */ + if (field->childs != NULL) { + if (tuple_field_bypass_and_init(field, i, tuple, &pos, + NULL) != 0) + return -1; + } else { + mp_next(&pos); + } } return 0; } diff --git a/src/box/tuple_compare.cc b/src/box/tuple_compare.cc index e21b009..5a3a968 100644 --- a/src/box/tuple_compare.cc +++ b/src/box/tuple_compare.cc @@ -458,18 +458,20 @@ tuple_common_key_parts(const struct tuple *tuple_a, const struct tuple *tuple_b, return i; } -template +template static inline int tuple_compare_slowpath(const struct tuple *tuple_a, const struct tuple *tuple_b, struct key_def *key_def) { + assert(has_json_path == key_def->has_json_paths); assert(!has_optional_parts || is_nullable); assert(is_nullable == key_def->is_nullable); assert(has_optional_parts == key_def->has_optional_parts); struct key_part *part = key_def->parts; const char *tuple_a_raw = tuple_data(tuple_a); const char *tuple_b_raw = tuple_data(tuple_b); - if (key_def->part_count == 1 && part->fieldno == 0) { + if (key_def->part_count == 1 && part->fieldno == 0 && + part->path == NULL) { /* * First field can not be optional - empty tuples * can not exist. @@ -507,10 +509,19 @@ tuple_compare_slowpath(const struct tuple *tuple_a, const struct tuple *tuple_b, end = part + key_def->part_count; for (; part < end; part++) { - field_a = tuple_field_by_part_raw(format_a, tuple_a_raw, - field_map_a, part); - field_b = tuple_field_by_part_raw(format_b, tuple_b_raw, - field_map_b, part); + if (!has_json_path) { + field_a = tuple_field_raw(format_a, tuple_a_raw, + field_map_a, + part->fieldno); + field_b = tuple_field_raw(format_b, tuple_b_raw, + field_map_b, + part->fieldno); + } else { + field_a = tuple_field_by_part_raw(format_a, tuple_a_raw, + field_map_a, part); + field_b = tuple_field_by_part_raw(format_b, tuple_b_raw, + field_map_b, part); + } assert(has_optional_parts || (field_a != NULL && field_b != NULL)); if (! is_nullable) { @@ -557,10 +568,19 @@ tuple_compare_slowpath(const struct tuple *tuple_a, const struct tuple *tuple_b, */ end = key_def->parts + key_def->part_count; for (; part < end; ++part) { - field_a = tuple_field_by_part_raw(format_a, tuple_a_raw, - field_map_a, part); - field_b = tuple_field_by_part_raw(format_b, tuple_b_raw, - field_map_b, part); + if (!has_json_path) { + field_a = tuple_field_raw(format_a, tuple_a_raw, + field_map_a, + part->fieldno); + field_b = tuple_field_raw(format_b, tuple_b_raw, + field_map_b, + part->fieldno); + } else { + field_a = tuple_field_by_part_raw(format_a, tuple_a_raw, + field_map_a, part); + field_b = tuple_field_by_part_raw(format_b, tuple_b_raw, + field_map_b, part); + } /* * Extended parts are primary, and they can not * be absent or be NULLs. @@ -574,11 +594,12 @@ tuple_compare_slowpath(const struct tuple *tuple_a, const struct tuple *tuple_b, return 0; } -template +template static inline int tuple_compare_with_key_slowpath(const struct tuple *tuple, const char *key, uint32_t part_count, struct key_def *key_def) { + assert(has_json_paths == key_def->has_json_paths); assert(!has_optional_parts || is_nullable); assert(is_nullable == key_def->is_nullable); assert(has_optional_parts == key_def->has_optional_parts); @@ -590,9 +611,14 @@ tuple_compare_with_key_slowpath(const struct tuple *tuple, const char *key, const uint32_t *field_map = tuple_field_map(tuple); enum mp_type a_type, b_type; if (likely(part_count == 1)) { - const char *field = - tuple_field_by_part_raw(format, tuple_raw, field_map, - part); + const char *field; + if (!has_json_paths) { + field = tuple_field_raw(format, tuple_raw, field_map, + part->fieldno); + } else { + field = tuple_field_by_part_raw(format, tuple_raw, + field_map, part); + } if (! is_nullable) { return tuple_compare_field(field, key, part->type, part->coll); @@ -616,9 +642,14 @@ tuple_compare_with_key_slowpath(const struct tuple *tuple, const char *key, struct key_part *end = part + part_count; int rc; for (; part < end; ++part, mp_next(&key)) { - const char *field = - tuple_field_by_part_raw(format, tuple_raw, - field_map, part); + const char *field; + if (!has_json_paths) { + field = tuple_field_raw(format, tuple_raw, field_map, + part->fieldno); + } else { + field = tuple_field_by_part_raw(format, tuple_raw, + field_map, part); + } if (! is_nullable) { rc = tuple_compare_field(field, key, part->type, part->coll); @@ -1011,23 +1042,35 @@ static const comparator_signature cmp_arr[] = { #undef COMPARATOR +static const tuple_compare_t compare_slowpath_funcs[] = { + tuple_compare_slowpath, + tuple_compare_slowpath, + tuple_compare_slowpath, + tuple_compare_slowpath, + tuple_compare_slowpath, + tuple_compare_slowpath, + tuple_compare_slowpath, + tuple_compare_slowpath +}; + tuple_compare_t tuple_compare_create(const struct key_def *def) { + int cmp_func_idx = (def->is_nullable ? 1 : 0) + + 2 * (def->has_optional_parts ? 1 : 0) + + 4 * (def->has_json_paths ? 1 : 0); if (def->is_nullable) { if (key_def_is_sequential(def)) { if (def->has_optional_parts) return tuple_compare_sequential; else return tuple_compare_sequential; - } else if (def->has_optional_parts) { - return tuple_compare_slowpath; } else { - return tuple_compare_slowpath; + return compare_slowpath_funcs[cmp_func_idx]; } } assert(! def->has_optional_parts); - if (!key_def_has_collation(def)) { + if (!key_def_has_collation(def) && !def->has_json_paths) { /* Precalculated comparators don't use collation */ for (uint32_t k = 0; k < sizeof(cmp_arr) / sizeof(cmp_arr[0]); k++) { @@ -1043,10 +1086,9 @@ tuple_compare_create(const struct key_def *def) return cmp_arr[k].f; } } - if (key_def_is_sequential(def)) - return tuple_compare_sequential; - else - return tuple_compare_slowpath; + return key_def_is_sequential(def) ? + tuple_compare_sequential : + compare_slowpath_funcs[cmp_func_idx]; } /* }}} tuple_compare */ @@ -1228,9 +1270,23 @@ static const comparator_with_key_signature cmp_wk_arr[] = { #undef KEY_COMPARATOR +static const tuple_compare_with_key_t compare_with_key_slowpath_funcs[] = { + tuple_compare_with_key_slowpath, + tuple_compare_with_key_slowpath, + tuple_compare_with_key_slowpath, + tuple_compare_with_key_slowpath, + tuple_compare_with_key_slowpath, + tuple_compare_with_key_slowpath, + tuple_compare_with_key_slowpath, + tuple_compare_with_key_slowpath +}; + tuple_compare_with_key_t tuple_compare_with_key_create(const struct key_def *def) { + int cmp_func_idx = (def->is_nullable ? 1 : 0) + + 2 * (def->has_optional_parts ? 1 : 0) + + 4 * (def->has_json_paths ? 1 : 0); if (def->is_nullable) { if (key_def_is_sequential(def)) { if (def->has_optional_parts) { @@ -1240,14 +1296,12 @@ tuple_compare_with_key_create(const struct key_def *def) return tuple_compare_with_key_sequential; } - } else if (def->has_optional_parts) { - return tuple_compare_with_key_slowpath; } else { - return tuple_compare_with_key_slowpath; + return compare_with_key_slowpath_funcs[cmp_func_idx]; } } assert(! def->has_optional_parts); - if (!key_def_has_collation(def)) { + if (!key_def_has_collation(def) && !def->has_json_paths) { /* Precalculated comparators don't use collation */ for (uint32_t k = 0; k < sizeof(cmp_wk_arr) / sizeof(cmp_wk_arr[0]); @@ -1266,10 +1320,9 @@ tuple_compare_with_key_create(const struct key_def *def) return cmp_wk_arr[k].f; } } - if (key_def_is_sequential(def)) - return tuple_compare_with_key_sequential; - else - return tuple_compare_with_key_slowpath; + return key_def_is_sequential(def) ? + tuple_compare_with_key_sequential : + compare_with_key_slowpath_funcs[cmp_func_idx]; } /* }}} tuple_compare_with_key */ diff --git a/src/box/tuple_extract_key.cc b/src/box/tuple_extract_key.cc index e493c3b..2474d98 100644 --- a/src/box/tuple_extract_key.cc +++ b/src/box/tuple_extract_key.cc @@ -1,15 +1,31 @@ #include "tuple_extract_key.h" #include "tuple.h" #include "fiber.h" +#include "json/path.h" enum { MSGPACK_NULL = 0xc0 }; +/** True if key part i and i+1 are sequential. */ +template +static inline bool +key_def_parts_are_sequential(const struct key_def *def, int i) +{ + uint32_t fieldno1 = def->parts[i].fieldno + 1; + uint32_t fieldno2 = def->parts[i + 1].fieldno; + if (!has_json_paths) { + return fieldno1 == fieldno2; + } else { + return fieldno1 == fieldno2 && def->parts[i].path == NULL && + def->parts[i + 1].path == NULL; + } +} + /** True, if a key con contain two or more parts in sequence. */ static bool key_def_contains_sequential_parts(const struct key_def *def) { for (uint32_t i = 0; i < def->part_count - 1; ++i) { - if (def->parts[i].fieldno + 1 == def->parts[i + 1].fieldno) + if (key_def_parts_are_sequential(def, i)) return true; } return false; @@ -89,11 +105,13 @@ tuple_extract_key_sequential(const struct tuple *tuple, struct key_def *key_def, * General-purpose implementation of tuple_extract_key() * @copydoc tuple_extract_key() */ -template +template static char * tuple_extract_key_slowpath(const struct tuple *tuple, struct key_def *key_def, uint32_t *key_size) { + assert(has_json_paths == key_def->has_json_paths); assert(!has_optional_parts || key_def->is_nullable); assert(has_optional_parts == key_def->has_optional_parts); assert(contains_sequential_parts == @@ -108,9 +126,14 @@ tuple_extract_key_slowpath(const struct tuple *tuple, /* Calculate the key size. */ for (uint32_t i = 0; i < part_count; ++i) { - const char *field = - tuple_field_by_part_raw(format, data, field_map, - &key_def->parts[i]); + const char *field; + if (!has_json_paths) { + field = tuple_field_raw(format, data, field_map, + key_def->parts[i].fieldno); + } else { + field = tuple_field_by_part_raw(format, data, field_map, + &key_def->parts[i]); + } if (has_optional_parts && field == NULL) { bsize += mp_sizeof_nil(); continue; @@ -123,8 +146,8 @@ tuple_extract_key_slowpath(const struct tuple *tuple, * minimize tuple_field_raw() calls. */ for (; i < part_count - 1; i++) { - if (key_def->parts[i].fieldno + 1 != - key_def->parts[i + 1].fieldno) { + if (!key_def_parts_are_sequential + (key_def, i)) { /* * End of sequential part. */ @@ -150,9 +173,14 @@ tuple_extract_key_slowpath(const struct tuple *tuple, } char *key_buf = mp_encode_array(key, part_count); for (uint32_t i = 0; i < part_count; ++i) { - const char *field = - tuple_field_by_part_raw(format, data, field_map, - &key_def->parts[i]); + const char *field; + if (!has_json_paths) { + field = tuple_field_raw(format, data, field_map, + key_def->parts[i].fieldno); + } else { + field = tuple_field_by_part_raw(format, data, field_map, + &key_def->parts[i]); + } if (has_optional_parts && field == NULL) { key_buf = mp_encode_nil(key_buf); continue; @@ -165,8 +193,8 @@ tuple_extract_key_slowpath(const struct tuple *tuple, * minimize tuple_field_raw() calls. */ for (; i < part_count - 1; i++) { - if (key_def->parts[i].fieldno + 1 != - key_def->parts[i + 1].fieldno) { + if (!key_def_parts_are_sequential + (key_def, i)) { /* * End of sequential part. */ @@ -199,11 +227,12 @@ tuple_extract_key_slowpath(const struct tuple *tuple, * General-purpose version of tuple_extract_key_raw() * @copydoc tuple_extract_key_raw() */ -template +template static char * tuple_extract_key_slowpath_raw(const char *data, const char *data_end, struct key_def *key_def, uint32_t *key_size) { + assert(has_json_paths == key_def->has_json_paths); assert(!has_optional_parts || key_def->is_nullable); assert(has_optional_parts == key_def->has_optional_parts); assert(mp_sizeof_nil() == 1); @@ -231,11 +260,12 @@ tuple_extract_key_slowpath_raw(const char *data, const char *data_end, uint32_t fieldno = key_def->parts[i].fieldno; uint32_t null_count = 0; for (; i < key_def->part_count - 1; i++) { - if (key_def->parts[i].fieldno + 1 != - key_def->parts[i + 1].fieldno) + if (!key_def_parts_are_sequential + (key_def, i)) break; } - uint32_t end_fieldno = key_def->parts[i].fieldno; + const struct key_part *part = &key_def->parts[i]; + uint32_t end_fieldno = part->fieldno; if (fieldno < current_fieldno) { /* Rewind. */ @@ -277,6 +307,21 @@ tuple_extract_key_slowpath_raw(const char *data, const char *data_end, current_fieldno++; } } + const char *field_last, *field_end_last; + if (has_json_paths && part->path != NULL) { + field_last = field; + field_end_last = field_end; + struct json_path_parser parser; + struct json_path_node node; + json_path_parser_create(&parser, part->path, + part->path_len); + /* Skip fieldno. */ + int rc = json_path_next(&parser, &node); + assert(rc == 0); + rc = tuple_field_dig_with_parser(&parser, &field); + field_end = field; + mp_next(&field_end); + } memcpy(key_buf, field, field_end - field); key_buf += field_end - field; if (has_optional_parts && null_count != 0) { @@ -285,12 +330,27 @@ tuple_extract_key_slowpath_raw(const char *data, const char *data_end, } else { assert(key_buf - key <= data_end - data); } + if (has_json_paths && part->path != NULL) { + field = field_last; + field_end = field_end_last; + } } if (key_size != NULL) *key_size = (uint32_t)(key_buf - key); return key; } +static const tuple_extract_key_t extract_key_slowpath_funcs[] = { + tuple_extract_key_slowpath, + tuple_extract_key_slowpath, + tuple_extract_key_slowpath, + tuple_extract_key_slowpath, + tuple_extract_key_slowpath, + tuple_extract_key_slowpath, + tuple_extract_key_slowpath, + tuple_extract_key_slowpath +}; + /** * Initialize tuple_extract_key() and tuple_extract_key_raw() */ @@ -311,32 +371,30 @@ tuple_extract_key_set(struct key_def *key_def) tuple_extract_key_sequential_raw; } } else { - if (key_def->has_optional_parts) { - assert(key_def->is_nullable); - if (key_def_contains_sequential_parts(key_def)) { - key_def->tuple_extract_key = - tuple_extract_key_slowpath; - } else { - key_def->tuple_extract_key = - tuple_extract_key_slowpath; - } - } else { - if (key_def_contains_sequential_parts(key_def)) { - key_def->tuple_extract_key = - tuple_extract_key_slowpath; - } else { - key_def->tuple_extract_key = - tuple_extract_key_slowpath; - } - } + int func_idx = + (key_def_contains_sequential_parts(key_def) ? 1 : 0) + + 2 * (key_def->has_optional_parts ? 1 : 0) + + 4 * (key_def->has_json_paths ? 1 : 0); + key_def->tuple_extract_key = + extract_key_slowpath_funcs[func_idx]; + assert(!key_def->has_optional_parts || key_def->is_nullable); } if (key_def->has_optional_parts) { assert(key_def->is_nullable); - key_def->tuple_extract_key_raw = - tuple_extract_key_slowpath_raw; + if (key_def->has_json_paths) { + key_def->tuple_extract_key_raw = + tuple_extract_key_slowpath_raw; + } else { + key_def->tuple_extract_key_raw = + tuple_extract_key_slowpath_raw; + } } else { - key_def->tuple_extract_key_raw = - tuple_extract_key_slowpath_raw; + if (key_def->has_json_paths) { + key_def->tuple_extract_key_raw = + tuple_extract_key_slowpath_raw; + } else { + key_def->tuple_extract_key_raw = + tuple_extract_key_slowpath_raw; + } } } diff --git a/src/box/tuple_format.c b/src/box/tuple_format.c index 91677d4..fa8b938 100644 --- a/src/box/tuple_format.c +++ b/src/box/tuple_format.c @@ -30,6 +30,7 @@ */ #include "json/path.h" #include "tuple_format.h" +#include "assoc.h" /** Global table of tuple formats */ struct tuple_format **tuple_formats; @@ -38,10 +39,551 @@ static intptr_t recycled_format_ids = FORMAT_ID_NIL; static uint32_t formats_size = 0, formats_capacity = 0; static const struct tuple_field tuple_field_default = { - FIELD_TYPE_ANY, TUPLE_OFFSET_SLOT_NIL, false, false, + FIELD_TYPE_ANY, TUPLE_OFFSET_SLOT_NIL, false, false, {{NULL, 0}} }; /** + * Propagate @a field to MessagePack(field)[key]. + * @param[in][out] field Field to propagate. + * @param key Key to propagate to. + * @param len Length of @a key. + * @param field_idx Field index in map. + * + * @retval 0 Success, the index was found. + * @retval -1 Not found. + */ +static inline int +tuple_field_go_to_key(const char **field, const char *key, int len, + uint32_t *field_idx) +{ + enum mp_type type = mp_typeof(**field); + if (type != MP_MAP) + return -1; + uint32_t count = mp_decode_map(field); + for (uint32_t idx = 0; idx < count; idx++) { + type = mp_typeof(**field); + if (type == MP_STR) { + uint32_t value_len; + const char *value = mp_decode_str(field, &value_len); + if (value_len == (uint)len && + memcmp(value, key, len) == 0) { + *field_idx = idx; + return 0; + } + } else { + /* Skip key. */ + mp_next(field); + } + /* Skip value. */ + mp_next(field); + } + return -1; +} + +struct mh_strnptr_node_t * +json_path_hash_get(struct mh_strnptr_t *hashtable, const char *path, + uint32_t path_len, uint32_t path_hash) +{ + assert(hashtable != NULL); + struct mh_strnptr_key_t key = {path, path_len, path_hash}; + mh_int_t rc = mh_strnptr_find(hashtable, &key, NULL); + if (rc == mh_end(hashtable)) + return NULL; + return mh_strnptr_node(hashtable, rc); +} + +/** + * Create a new hashtable object. + * @param[out] hashtable Pointer to object to create. + * @param records Count of records to reserve. + * @retval -1 On error. + * @retval 0 On success. + */ +static struct mh_strnptr_t * +json_path_hash_create(uint32_t records) +{ + struct mh_strnptr_t *ret = mh_strnptr_new(); + if (ret == NULL) { + diag_set(OutOfMemory, sizeof(struct mh_strnptr_t), + "mh_strnptr_new", "hashtable"); + return NULL; + } + if (records > 0 && + mh_strnptr_reserve(ret, records, NULL) != 0) { + mh_strnptr_delete(ret); + diag_set(OutOfMemory, records, "mh_strnptr_reserve", + "hashtable"); + return NULL; + } + return ret; +} + +/** + * Delete @hashtable object. + * @param hashtable Pointer to object to delete. + */ +static void +json_path_hash_delete(struct mh_strnptr_t *hashtable) +{ + assert(hashtable != NULL); + while (mh_size(hashtable) != 0) { + mh_int_t n = mh_first(hashtable); + mh_strnptr_del(hashtable, n, NULL); + } + mh_strnptr_delete(hashtable); +} + +/** + * Insert a new record to hashtable. + * @param hashtable Storage to insert new record. + * @param path String with path. + * @param path_len Length of @path. + * @param field Value to store in @hashtable. + * @retval -1 On error. + * @retval 0 On success. + */ +static int +json_path_hash_insert(struct mh_strnptr_t *hashtable, const char *path, + uint32_t path_len, struct tuple_field *field) +{ + assert(hashtable != NULL); + uint32_t path_hash = mh_strn_hash(path, path_len); + struct mh_strnptr_node_t name_node = {path, path_len, path_hash, field}; + mh_int_t rc = mh_strnptr_put(hashtable, &name_node, NULL, NULL); + if (rc == mh_end(hashtable)) { + diag_set(OutOfMemory, sizeof(*hashtable), "mh_strnptr_put", + "hashtable"); + return -1; + } + return 0; +} + +/** + * Construct field tree level for JSON path part. + * + * @param[in, out] tuple_field Pointer to record to start with + * would be changed to record that math + * @part lexeme. + * @param fieldno Number of root space field. + * @param part JSON path lexeme to represent in field tree. + * @retval -1 On error. + * @retval 0 On success. + */ +static int +json_field_tree_append(struct tuple_field **field_subtree, uint32_t fieldno, + struct json_path_node *part) +{ + enum field_type type; + struct tuple_field *field = *field_subtree; + switch (part->type) { + case JSON_PATH_NUM: { + type = FIELD_TYPE_ARRAY; + if (field->type != FIELD_TYPE_ANY && field->type != type) + goto error_type_mistmatch; + /* Create or resize field array if required. */ + if (field->array == NULL || part->num > field->array_size) { + struct tuple_field **array = + realloc(field->array, + part->num * sizeof(array[0])); + if (array == NULL) { + diag_set(OutOfMemory, + part->num * sizeof(array[0]), + "realloc","array"); + return -1; + } + memset(&array[field->array_size], 0, + (part->num - field->array_size) * + sizeof(array[0])); + field->array = array; + field->array_size = part->num; + field->type = type; + } else if (field->array[part->num - TUPLE_INDEX_BASE] != NULL) { + /* Record already exists. No actions required */ + *field_subtree = + field->array[part->num - TUPLE_INDEX_BASE]; + return 0; + } + break; + } + case JSON_PATH_STR: { + type = FIELD_TYPE_MAP; + if (field->type != FIELD_TYPE_ANY && field->type != type) + goto error_type_mistmatch; + if (field->map == NULL) { + field->map = json_path_hash_create(1); + if (field->map == NULL) + return -1; + field->type = type; + } else { + uint32_t str_hash = mh_strn_hash(part->str, part->len); + struct mh_strnptr_node_t *ht_record = + json_path_hash_get(field->map, part->str, + part->len, str_hash); + if (ht_record != NULL) { + assert(ht_record->val != NULL); + *field_subtree = ht_record->val; + return 0; + } + } + break; + } + default: + unreachable(); + } + + /* Construct and insert a new record. */ + struct tuple_field *new_field = malloc(sizeof(struct tuple_field)); + if (new_field == NULL) { + diag_set(OutOfMemory, sizeof(struct tuple_field), "malloc", + "new_field"); + return -1; + } + *new_field = tuple_field_default; + if (field->type == FIELD_TYPE_MAP) { + if (json_path_hash_insert(field->map, part->str, part->len, + new_field) != 0) { + free(new_field); + return -1; + } + } else if (field->type == FIELD_TYPE_ARRAY) { + field->array[part->num - TUPLE_INDEX_BASE] = new_field; + } + *field_subtree = new_field; + return 0; + +error_type_mistmatch: + diag_set(ClientError, ER_INDEX_PART_TYPE_MISMATCH, + tt_sprintf("%d", fieldno + TUPLE_INDEX_BASE), + field_type_strs[type], field_type_strs[field->type]); + return -1; +} + +/** + * Delete @field_subtree object. + * @param field_subtree To delete. + */ +static void +json_field_tree_delete(struct tuple_field *field_subtree) +{ + if (field_subtree->type == FIELD_TYPE_MAP && + field_subtree->map != NULL) { + mh_int_t i; + mh_foreach(field_subtree->map, i) { + struct tuple_field *field = + mh_strnptr_node(field_subtree->map, i)->val; + assert(field != NULL); + json_field_tree_delete(field); + free(field); + } + json_path_hash_delete(field_subtree->map); + } else if (field_subtree->type == FIELD_TYPE_ARRAY && + field_subtree->array != NULL) { + for (uint32_t i = 0; i < field_subtree->array_size; i++) { + struct tuple_field *field = field_subtree->array[i]; + if (field == NULL) + continue; + json_field_tree_delete(field_subtree->array[i]); + free(field_subtree->array[i]); + } + free(field_subtree->array); + } +} + +int +tuple_field_bypass_and_init(const struct tuple_field *field, uint32_t idx, + const char *tuple, const char **offset, + uint32_t *field_map) +{ + assert(offset != NULL); + const char *mp_data = *offset; + const char *valid_type_str = NULL; + const char *err = NULL; + enum mp_type type = mp_typeof(**offset); + if (field->type == FIELD_TYPE_MAP) { + if (type != MP_MAP) { + valid_type_str = mp_type_strs[MP_MAP]; + goto error_type_mistmatch; + } + const char *max_offset = *offset; + uint32_t max_idx = 0; + uint32_t count = mp_decode_map(&max_offset); + mh_int_t i; + mh_foreach(field->map, i) { + struct mh_strnptr_node_t *ht_record = + mh_strnptr_node(field->map, i); + struct tuple_field *leaf = ht_record->val; + assert(leaf != NULL); + + const char *raw = *offset; + uint32_t map_idx = 0; + int rc = tuple_field_go_to_key(&raw, ht_record->str, + (int)ht_record->len, + &map_idx); + if (rc != 0 && !leaf->is_nullable) { + err = tt_sprintf("map doesn't contain key " + "'%.*s' defined in index", + ht_record->len,ht_record->str); + goto error_invalid_document; + } + if (rc != 0) { + if (field_map != NULL && + leaf->offset_slot != TUPLE_OFFSET_SLOT_NIL) + field_map[leaf->offset_slot] = 0; + continue; + } + if (tuple_field_bypass_and_init(leaf, idx, tuple, &raw, + field_map) != 0) + return -1; + max_idx = MAX(max_idx, map_idx + 1); + max_offset = MAX(max_offset, raw); + } + *offset = max_offset; + while (count-- > max_idx) { + mp_next(offset); + mp_next(offset); + } + return 0; + } else if (field->type == FIELD_TYPE_ARRAY) { + if (type != MP_ARRAY) { + valid_type_str = mp_type_strs[MP_ARRAY]; + goto error_type_mistmatch; + } + uint32_t count = mp_decode_array(offset); + for (uint32_t i = count; i < field->array_size; i++) { + /* + * Index fields out of document array + * must be nullable. + */ + struct tuple_field *leaf = field->array[i]; + if (leaf == NULL) + continue; + if (leaf->is_nullable) { + if (field_map != NULL && + leaf->offset_slot != TUPLE_OFFSET_SLOT_NIL) + field_map[leaf->offset_slot] = 0; + continue; + } + err = tt_sprintf("array size %d is less than size of %d " + "defined in index", i, i + 1); + goto error_invalid_document; + } + uint32_t fields = MIN(field->array_size, count); + for (uint32_t i = 0; i < fields; i++) { + if (field->array[i] == NULL) { + mp_next(offset); + continue; + } + if (tuple_field_bypass_and_init(field->array[i], idx, + tuple, + offset, field_map) != 0) + return -1; + } + while (count-- > fields) + mp_next(offset); + return 0; + } + /* Tree leaf field */ + if (key_mp_type_validate(field->type, type, ER_KEY_PART_TYPE, idx, + field->is_nullable) != 0) { + valid_type_str = field_type_strs[field->type]; + goto error_type_mistmatch; + } + assert(offset != NULL); + if (field_map != NULL && + field->offset_slot != TUPLE_OFFSET_SLOT_NIL) + field_map[field->offset_slot] = (uint32_t) (*offset - tuple); + mp_next(offset); + return 0; + +error_type_mistmatch: + err = tt_sprintf("type mismatch: have %s, expected %s", + mp_type_strs[type], valid_type_str); +error_invalid_document: + assert(err != NULL); + char *data_buff = tt_static_buf(); + mp_snprint(data_buff, TT_STATIC_BUF_LEN, mp_data); + const char *err_msg = + tt_sprintf("invalid field %d document content '%s': %s", + idx + TUPLE_INDEX_BASE, data_buff, err); + diag_set(ClientError, ER_DATA_STRUCTURE_MISMATCH, err_msg); + return -1; +} + +/** + * Add new JSON @path to @format. + * @param format Tuple format to modify. + * @param path String to add. + * @param path_len Length of @path. + * @param path_hash Hash of @path. + * @param type Type of field by @path. + * @param is_nullable Nullability of field by @path. + * @param strings Area to store unique JSON paths (optional). + * @param[out] leaf Pointer to leaf field. + * @retval -1 On error. + * @retval 0 On success. + */ +static int +tuple_format_add_json_path(struct tuple_format *format, const char *path, + uint32_t path_len, uint32_t path_hash, + enum field_type type, bool is_nullable, + char **strings, struct tuple_field **leaf) +{ + assert(format->path_hash != NULL); + /* + * Get root field by index. + * Path is specified in canonical form: [i]... + */ + struct json_path_parser parser; + struct json_path_node node; + json_path_parser_create(&parser, path, path_len); + int rc = json_path_next(&parser, &node); + assert(rc == 0 && node.type == JSON_PATH_NUM); + assert(node.num - TUPLE_INDEX_BASE < format->field_count); + + /* Test if path is already registered. */ + struct mh_strnptr_node_t *ht_record = + json_path_hash_get(format->path_hash, path, path_len, path_hash); + assert(ht_record != NULL); + struct tuple_field *field = ht_record->val; + if (unlikely(field != NULL)) { + /* Path has been already registered. */ + if (field->is_nullable != is_nullable) + field->is_nullable = false; + if (field_type1_contains_type2(field->type, type)) { + field->type = type; + } else if (!field_type1_contains_type2(type, field->type)) { + const char *err = + tt_sprintf("JSON path '%.*s' has been already " + "constructed for '%s' leaf record", + path_len, path, + field_type_strs[field->type]); + diag_set(ClientError, ER_WRONG_INDEX_OPTIONS, + node.num, err); + return -1; + } + *leaf = field; + return 0; + } else if (strings != NULL) { + /* + * Hashtable should hold memory related to format + * chunk allocation. + */ + memcpy(*strings, path, path_len); + (*strings)[path_len] = '\0'; + ht_record->str = *strings; + *strings += path_len + 1; + } + + /* + * We have to re-init parser with path string located in + * format chunk. + */ + json_path_parser_create(&parser, ht_record->str + parser.offset, + path_len - parser.offset); + /* Build data path tree. */ + uint32_t root_fieldno = node.num - TUPLE_INDEX_BASE; + field = &format->fields[root_fieldno]; + while ((rc = json_path_next(&parser, &node)) == 0 && + node.type != JSON_PATH_END) { + if (json_field_tree_append(&field, root_fieldno, &node) != 0) + return -1; + } + assert(rc == 0 && node.type == JSON_PATH_END); + + /* Leaf record is a new object as JSON path unique. */ + field->type = type; + field->is_nullable = is_nullable; + *leaf = field; + ht_record->val = field; + return 0; +} + +/** + * Add a new key_part to format and initialize format tuple_field + * representation. + * @param format Format to initialize. + * @param fields Fields definition if any. + * @param fields_count Count of @fields. + * @param part An index part to append. + * @param is_sequential Does this part sequential. + * @param data Memory to store path strings. + * @param current_slot Pointer to last offset slot. + * @retval -1 On error. + * @retval 0 On success. + */ +static int +tuple_format_add_key_part(struct tuple_format *format, + const struct field_def *fields, uint32_t field_count, + const struct key_part *part, bool is_sequential, + char **data, int *current_slot) +{ + assert(part->fieldno < format->field_count); + struct tuple_field *field = &format->fields[part->fieldno]; + if (part->path != NULL) { + field->is_key_part = true; + assert(!is_sequential); + struct tuple_field *leaf = NULL; + if (tuple_format_add_json_path(format, part->path, + part->path_len, part->path_hash, + part->type, part->is_nullable, + data, &leaf) != 0) + return -1; + assert(leaf != NULL); + if (leaf->offset_slot == TUPLE_OFFSET_SLOT_NIL) { + *current_slot = *current_slot - 1; + leaf->offset_slot = *current_slot; + } + return 0; + } + if (part->fieldno >= field_count) { + field->is_nullable = part->is_nullable; + } else if (field->is_nullable != part->is_nullable) { + /* + * In case of mismatch set the most + * strict option for is_nullable. + */ + field->is_nullable = false; + } + /* + * Check that there are no conflicts + * between index part types and space + * fields. If a part type is compatible + * with field's one, then the part type is + * more strict and the part type must be + * used in tuple_format. + */ + if (field_type1_contains_type2(field->type, part->type)) { + field->type = part->type; + } else if (!field_type1_contains_type2(part->type, field->type)) { + int fieldno = part->fieldno + TUPLE_INDEX_BASE; + const char *name = part->fieldno >= field_count ? + tt_sprintf("%d", fieldno) : + tt_sprintf("'%s'", + fields[part->fieldno].name); + int errcode = !field->is_key_part ? + ER_FORMAT_MISMATCH_INDEX_PART : + ER_INDEX_PART_TYPE_MISMATCH; + diag_set(ClientError, errcode, name, + field_type_strs[field->type], + field_type_strs[part->type]); + return -1; + } + field->is_key_part = true; + /* + * In the tuple, store only offsets necessary + * to access fields of non-sequential keys. + * First field is always simply accessible, + * so we don't store an offset for it. + */ + if (field->offset_slot == TUPLE_OFFSET_SLOT_NIL && !is_sequential && + part->fieldno > 0) { + *current_slot = *current_slot - 1; + field->offset_slot = *current_slot; + } + return 0; +} + +/** * Extract all available type info from keys and field * definitions. */ @@ -63,12 +605,18 @@ tuple_format_create(struct tuple_format *format, struct key_def * const *keys, format->fields[i].type = fields[i].type; format->fields[i].offset_slot = TUPLE_OFFSET_SLOT_NIL; format->fields[i].is_nullable = fields[i].is_nullable; + /* Don't need to init format->fields[i].map. */ + format->fields[i].childs = NULL; + format->fields[i].array_size = 0; } /* Initialize remaining fields */ for (uint32_t i = field_count; i < format->field_count; i++) format->fields[i] = tuple_field_default; int current_slot = 0; + /* Memory allocated for JSON paths if any. */ + char *data = (char *)format + sizeof(struct tuple_format) + + format->field_count * sizeof(struct tuple_field); /* extract field type info */ for (uint16_t key_no = 0; key_no < key_count; ++key_no) { @@ -76,65 +624,12 @@ tuple_format_create(struct tuple_format *format, struct key_def * const *keys, bool is_sequential = key_def_is_sequential(key_def); const struct key_part *part = key_def->parts; const struct key_part *parts_end = part + key_def->part_count; - for (; part < parts_end; part++) { - assert(part->fieldno < format->field_count); - struct tuple_field *field = - &format->fields[part->fieldno]; - if (part->fieldno >= field_count) { - field->is_nullable = part->is_nullable; - } else if (field->is_nullable != part->is_nullable) { - /* - * In case of mismatch set the most - * strict option for is_nullable. - */ - field->is_nullable = false; - } - - /* - * Check that there are no conflicts - * between index part types and space - * fields. If a part type is compatible - * with field's one, then the part type is - * more strict and the part type must be - * used in tuple_format. - */ - if (field_type1_contains_type2(field->type, - part->type)) { - field->type = part->type; - } else if (! field_type1_contains_type2(part->type, - field->type)) { - const char *name; - int fieldno = part->fieldno + TUPLE_INDEX_BASE; - if (part->fieldno >= field_count) { - name = tt_sprintf("%d", fieldno); - } else { - const struct field_def *def = - &fields[part->fieldno]; - name = tt_sprintf("'%s'", def->name); - } - int errcode; - if (! field->is_key_part) - errcode = ER_FORMAT_MISMATCH_INDEX_PART; - else - errcode = ER_INDEX_PART_TYPE_MISMATCH; - diag_set(ClientError, errcode, name, - field_type_strs[field->type], - field_type_strs[part->type]); + if (tuple_format_add_key_part(format, fields, + field_count, part, + is_sequential, &data, + ¤t_slot) != 0) return -1; - } - field->is_key_part = true; - /* - * In the tuple, store only offsets necessary - * to access fields of non-sequential keys. - * First field is always simply accessible, - * so we don't store an offset for it. - */ - if (field->offset_slot == TUPLE_OFFSET_SLOT_NIL && - is_sequential == false && part->fieldno > 0) { - - field->offset_slot = --current_slot; - } } } @@ -201,32 +696,58 @@ tuple_format_alloc(struct key_def * const *keys, uint16_t key_count, uint32_t space_field_count, struct tuple_dictionary *dict) { uint32_t index_field_count = 0; + /* JSON path hashtable. */ + struct mh_strnptr_t *path_hash = json_path_hash_create(0); + if (path_hash == NULL) + return NULL; /* find max max field no */ for (uint16_t key_no = 0; key_no < key_count; ++key_no) { const struct key_def *key_def = keys[key_no]; const struct key_part *part = key_def->parts; const struct key_part *pend = part + key_def->part_count; for (; part < pend; part++) { + if (part->path != NULL && + json_path_hash_insert(path_hash, part->path, + part->path_len, NULL) != 0) + goto error; index_field_count = MAX(index_field_count, part->fieldno + 1); } } + size_t extra_size = 0; + if (mh_size(path_hash) == 0) { + /* Hashtable is useless. */ + json_path_hash_delete(path_hash); + path_hash = NULL; + } else { + /* + * Calculate unique JSON paths count. + * Path data would be copied later on + * tuple_format_create routine. + */ + mh_int_t i; + mh_foreach(path_hash, i) { + struct mh_strnptr_node_t *node = + mh_strnptr_node(path_hash, i); + extra_size += node->len + 1; + } + } uint32_t field_count = MAX(space_field_count, index_field_count); uint32_t total = sizeof(struct tuple_format) + - field_count * sizeof(struct tuple_field); + field_count * sizeof(struct tuple_field) + extra_size; struct tuple_format *format = (struct tuple_format *) malloc(total); if (format == NULL) { diag_set(OutOfMemory, sizeof(struct tuple_format), "malloc", "tuple format"); - return NULL; + goto error; } if (dict == NULL) { assert(space_field_count == 0); format->dict = tuple_dictionary_new(NULL, 0); if (format->dict == NULL) { free(format); - return NULL; + goto error; } } else { format->dict = dict; @@ -238,13 +759,21 @@ tuple_format_alloc(struct key_def * const *keys, uint16_t key_count, format->index_field_count = index_field_count; format->exact_field_count = 0; format->min_field_count = 0; + format->path_hash = path_hash; return format; +error: + json_path_hash_delete(path_hash); + return NULL; } /** Free tuple format resources, doesn't unregister. */ static inline void tuple_format_destroy(struct tuple_format *format) { + for (uint32_t i = 0; i < format->field_count; i++) + json_field_tree_delete(&format->fields[i]); + if (format->path_hash != NULL) + json_path_hash_delete(format->path_hash); tuple_dictionary_unref(format->dict); } @@ -329,21 +858,61 @@ tuple_format_dup(struct tuple_format *src) { uint32_t total = sizeof(struct tuple_format) + src->field_count * sizeof(struct tuple_field); + if (src->path_hash != NULL) { + mh_int_t i; + mh_foreach(src->path_hash, i) + total += mh_strnptr_node(src->path_hash, i)->len + 1; + } struct tuple_format *format = (struct tuple_format *) malloc(total); if (format == NULL) { diag_set(OutOfMemory, total, "malloc", "tuple format"); return NULL; } memcpy(format, src, total); + + /* Fill with NULLs for normal destruction on error. */ + format->path_hash = NULL; + for (uint32_t i = 0; i < format->field_count; i++) { + format->fields[i].childs = NULL; + format->fields[i].array_size = 0; + } + if (src->path_hash != NULL) { + mh_int_t i; + format->path_hash = + json_path_hash_create(mh_size(src->path_hash)); + if (format->path_hash == NULL) + goto error; + mh_foreach(src->path_hash, i) { + struct mh_strnptr_node_t *node = + mh_strnptr_node(src->path_hash, i); + /* Path data has been already copied. */ + char *path = (char *)format + (node->str - (char *)src); + if (json_path_hash_insert(format->path_hash, path, + node->len, NULL) != 0) + goto error; + /* Store source leaf field offset_slot. */ + struct tuple_field *leaf = node->val; + int32_t offset_slot = leaf->offset_slot; + uint32_t path_hash = mh_strn_hash(path, node->len); + if (tuple_format_add_json_path(format, path, node->len, + path_hash, leaf->type, + leaf->is_nullable, NULL, + &leaf) != 0) + goto error; + /* Store offset_slot in a new leaf record. */ + assert(leaf != NULL); + leaf->offset_slot = offset_slot; + } + } tuple_dictionary_ref(format->dict); format->id = FORMAT_ID_NIL; format->refs = 0; - if (tuple_format_register(format) != 0) { - tuple_format_destroy(format); - free(format); - return NULL; - } - return format; + if (tuple_format_register(format) == 0) + return format; +error: + tuple_format_destroy(format); + free(format); + return NULL; } /** @sa declaration for details. */ @@ -372,18 +941,10 @@ tuple_init_field_map(const struct tuple_format *format, uint32_t *field_map, return -1; } - /* first field is simply accessible, so we do not store offset to it */ - enum mp_type mp_type = mp_typeof(*pos); + uint32_t i = 0; + enum mp_type mp_type; const struct tuple_field *field = &format->fields[0]; - if (key_mp_type_validate(field->type, mp_type, ER_FIELD_TYPE, - TUPLE_INDEX_BASE, field->is_nullable)) - return -1; - mp_next(&pos); - /* other fields...*/ - ++field; - uint32_t i = 1; - uint32_t defined_field_count = MIN(field_count, format->field_count); - if (field_count < format->index_field_count) { + if (field_count < format->index_field_count || field->childs != NULL) { /* * Nullify field map to be able to detect by 0, * which key fields are absent in tuple_field(). @@ -391,6 +952,20 @@ tuple_init_field_map(const struct tuple_format *format, uint32_t *field_map, memset((char *)field_map - format->field_map_size, 0, format->field_map_size); } + if (field->childs == NULL) { + /* + * First field is simply accessible, do not store + * offset to it. + */ + mp_type = mp_typeof(*pos); + if (key_mp_type_validate(field->type, mp_type, ER_FIELD_TYPE, + TUPLE_INDEX_BASE, field->is_nullable)) + return -1; + mp_next(&pos); + ++field; + ++i; + } + uint32_t defined_field_count = MIN(field_count, format->field_count); for (; i < defined_field_count; ++i, ++field) { mp_type = mp_typeof(*pos); if (key_mp_type_validate(field->type, mp_type, ER_FIELD_TYPE, @@ -400,8 +975,12 @@ tuple_init_field_map(const struct tuple_format *format, uint32_t *field_map, if (field->offset_slot != TUPLE_OFFSET_SLOT_NIL) { field_map[field->offset_slot] = (uint32_t) (pos - tuple); - } - mp_next(&pos); + } else if (field->childs != NULL && + tuple_field_bypass_and_init(field, i, tuple, &pos, + field_map) != 0) + return -1; + if (field->childs == NULL) + mp_next(&pos); } return 0; } @@ -507,55 +1086,99 @@ tuple_field_go_to_index(const char **field, uint64_t index) return -1; } -/** - * Propagate @a field to MessagePack(field)[key]. - * @param[in][out] field Field to propagate. - * @param key Key to propagate to. - * @param len Length of @a key. - * - * @retval 0 Success, the index was found. - * @retval -1 Not found. - */ -static inline int -tuple_field_go_to_key(const char **field, const char *key, int len) +const char * +tuple_field_by_part_raw(const struct tuple_format *format, const char *data, + const uint32_t *field_map, struct key_part *part) { - enum mp_type type = mp_typeof(**field); - if (type != MP_MAP) - return -1; - uint64_t count = mp_decode_map(field); - for (; count > 0; --count) { - type = mp_typeof(**field); - if (type == MP_STR) { - uint32_t value_len; - const char *value = mp_decode_str(field, &value_len); - if (value_len == (uint)len && - memcmp(value, key, len) == 0) - return 0; - } else { - /* Skip key. */ - mp_next(field); - } - /* Skip value. */ - mp_next(field); + if (likely(part->path == NULL)) + return tuple_field_raw(format, data, field_map, part->fieldno); + + struct mh_strnptr_node_t *ht_record = NULL; + int32_t offset_slot; + if (format->path_hash != NULL && + (ht_record = json_path_hash_get(format->path_hash, part->path, + part->path_len, + part->path_hash)) != NULL) { + struct tuple_field *field = ht_record->val; + assert(field != NULL); + offset_slot = field->offset_slot; + } else { + /* + * Legacy tuple having no field map for + * JSON index. + */ + uint32_t path_hash = + field_name_hash(part->path, part->path_len); + const char *raw = NULL; + if (tuple_field_raw_by_path(format, data, field_map, + part->path, part->path_len, + path_hash, &raw) != 0) + raw = NULL; + return raw; } - return -1; + assert(offset_slot < 0); + assert(-offset_slot * sizeof(uint32_t) <= format->field_map_size); + if (unlikely(field_map[offset_slot] == 0)) + return NULL; + return data + field_map[offset_slot]; } -const char * -tuple_field_by_part_raw(const struct tuple_format *format, const char *data, - const uint32_t *field_map, struct key_part *part) +int +tuple_field_dig_with_parser(struct json_path_parser *parser, const char **field) { - return tuple_field_raw(format, data, field_map, part->fieldno); + int rc; + struct json_path_node node; + while ((rc = json_path_next(parser, &node)) == 0) { + uint32_t dummy; + switch(node.type) { + case JSON_PATH_NUM: + rc = tuple_field_go_to_index(field, node.num); + break; + case JSON_PATH_STR: + rc = tuple_field_go_to_key(field, node.str, + node.len, &dummy); + break; + default: + assert(node.type == JSON_PATH_END); + return 0; + } + if (rc != 0) { + *field = NULL; + return 0; + } + } + return rc; } int -tuple_field_raw_by_path(struct tuple_format *format, const char *tuple, +tuple_field_raw_by_path(const struct tuple_format *format, const char *tuple, const uint32_t *field_map, const char *path, uint32_t path_len, uint32_t path_hash, const char **field) { assert(path_len > 0); uint32_t fieldno; + if (format->path_hash != NULL) { + /* + * The path hash for format->path_hash hashtable + * may may be different from path_hash specified + * as function argument. + */ + struct mh_strnptr_node_t *ht_record = + json_path_hash_get(format->path_hash, path, path_len, + mh_strn_hash(path, path_len)); + if (ht_record != NULL) { + struct tuple_field *leaf = ht_record->val; + assert(leaf != NULL); + int32_t offset_slot = leaf->offset_slot; + assert(offset_slot != TUPLE_OFFSET_SLOT_NIL); + if (likely(field_map[offset_slot] != 0)) + *field = tuple + field_map[offset_slot]; + else + *field = NULL; + return 0; + } + } /* * It is possible, that a field has a name as * well-formatted JSON. For example 'a.b.c.d' or '[1]' can @@ -611,23 +1234,9 @@ tuple_field_raw_by_path(struct tuple_format *format, const char *tuple, *field = NULL; return 0; } - while ((rc = json_path_next(&parser, &node)) == 0) { - switch(node.type) { - case JSON_PATH_NUM: - rc = tuple_field_go_to_index(field, node.num); - break; - case JSON_PATH_STR: - rc = tuple_field_go_to_key(field, node.str, node.len); - break; - default: - assert(node.type == JSON_PATH_END); - return 0; - } - if (rc != 0) { - *field = NULL; - return 0; - } - } + rc = tuple_field_dig_with_parser(&parser, field); + if (rc == 0) + return 0; error: assert(rc > 0); diag_set(ClientError, ER_ILLEGAL_PARAMS, diff --git a/src/box/tuple_format.h b/src/box/tuple_format.h index 0ffdf7d..df500db 100644 --- a/src/box/tuple_format.h +++ b/src/box/tuple_format.h @@ -63,6 +63,8 @@ enum { TUPLE_OFFSET_SLOT_NIL = INT32_MAX }; struct tuple; struct tuple_format; +struct json_path_parser; +struct mh_strnptr_t; /** Engine-specific tuple format methods. */ struct tuple_format_vtab { @@ -108,6 +110,21 @@ struct tuple_field { bool is_key_part; /** True, if a field can store NULL. */ bool is_nullable; + /** Tree child records. Must at the end of struct */ + union { + /** Array of fields. */ + struct { + struct tuple_field **array; + uint32_t array_size; + }; + /** Hashtable: path -> tuple_field. */ + struct mh_strnptr_t *map; + /** + * Auxiliary pointer to test if field has + * JSON path subtree. + */ + void *childs; + }; }; /** @@ -161,6 +178,8 @@ struct tuple_format { * Shared names storage used by all formats of a space. */ struct tuple_dictionary *dict; + /** JSON path hash table. */ + struct mh_strnptr_t *path_hash; /* Formats of the fields */ struct tuple_field fields[0]; }; @@ -388,7 +407,7 @@ tuple_field_raw(const struct tuple_format *format, const char *tuple, * @retval NULL No field with @a name. */ static inline const char * -tuple_field_raw_by_name(struct tuple_format *format, const char *tuple, +tuple_field_raw_by_name(const struct tuple_format *format, const char *tuple, const uint32_t *field_map, const char *name, uint32_t name_len, uint32_t name_hash) { @@ -413,11 +432,51 @@ tuple_field_raw_by_name(struct tuple_format *format, const char *tuple, * @retval -1 Error in JSON path. */ int -tuple_field_raw_by_path(struct tuple_format *format, const char *tuple, +tuple_field_raw_by_path(const struct tuple_format *format, const char *tuple, const uint32_t *field_map, const char *path, uint32_t path_len, uint32_t path_hash, const char **field); +/** + * Retrieve document data @field with initialized @parser. + * @param parser JSON parser. + * @param[in, out] field Tuple field to lookup. + * @retval 0 On success. + * @retval > 0 On error in path been used to initialize @parser. + */ +int +tuple_field_dig_with_parser(struct json_path_parser *parser, + const char **field); + +/** + * Get @hashtable record by key @path, @path_len. + * @param hashtable Storage to lookup. + * @param path Path string. + * @param path_len Length of @path. + * @param path_hash Hash of @path. + * @retval NULL On nothing found. + * @retval not NULL Leaf field pointer for registered path. + */ +struct mh_strnptr_node_t * +json_path_hash_get(struct mh_strnptr_t *hashtable, const char *path, + uint32_t path_len, uint32_t path_hash); + +/** + * Observe JSON path tree in @field comparing with @tuple + * structure. Initialize field map if specified. + * @param field Field to use on initialization. + * @param idx Root field index to emmit correct error. + * @param tuple Source raw data. + * @param offset Document field offset to process. + * @param field_map Field map to initialize (optional). + * @retval 0 On success. + * @retval -1 On error. + */ +int +tuple_field_bypass_and_init(const struct tuple_field *field, uint32_t idx, + const char *tuple, const char **offset, + uint32_t *field_map); + #if defined(__cplusplus) } /* extern "C" */ #endif /* defined(__cplusplus) */ diff --git a/src/box/tuple_hash.cc b/src/box/tuple_hash.cc index b394804..8ede290 100644 --- a/src/box/tuple_hash.cc +++ b/src/box/tuple_hash.cc @@ -213,7 +213,7 @@ static const hasher_signature hash_arr[] = { #undef HASHER -template +template uint32_t tuple_hash_slowpath(const struct tuple *tuple, struct key_def *key_def); @@ -222,7 +222,7 @@ key_hash_slowpath(const char *key, struct key_def *key_def); void tuple_hash_func_set(struct key_def *key_def) { - if (key_def->is_nullable) + if (key_def->is_nullable || key_def->has_json_paths) goto slowpath; /* * Check that key_def defines sequential a key without holes @@ -256,10 +256,17 @@ tuple_hash_func_set(struct key_def *key_def) { } slowpath: - if (key_def->has_optional_parts) - key_def->tuple_hash = tuple_hash_slowpath; - else - key_def->tuple_hash = tuple_hash_slowpath; + if (key_def->has_optional_parts) { + if (key_def->has_json_paths) + key_def->tuple_hash = tuple_hash_slowpath; + else + key_def->tuple_hash = tuple_hash_slowpath; + } else { + if (key_def->has_json_paths) + key_def->tuple_hash = tuple_hash_slowpath; + else + key_def->tuple_hash = tuple_hash_slowpath; + } key_def->key_hash = key_hash_slowpath; } @@ -319,10 +326,11 @@ tuple_hash_key_part(uint32_t *ph1, uint32_t *pcarry, const struct tuple *tuple, return tuple_hash_field(ph1, pcarry, &field, part->coll); } -template +template uint32_t tuple_hash_slowpath(const struct tuple *tuple, struct key_def *key_def) { + assert(has_json_paths == key_def->has_json_paths); assert(has_optional_parts == key_def->has_optional_parts); uint32_t h = HASH_SEED; uint32_t carry = 0; @@ -331,9 +339,13 @@ tuple_hash_slowpath(const struct tuple *tuple, struct key_def *key_def) struct tuple_format *format = tuple_format(tuple); const char *tuple_raw = tuple_data(tuple); const uint32_t *field_map = tuple_field_map(tuple); - const char *field = - tuple_field_by_part_raw(format, tuple_raw, field_map, - key_def->parts); + const char *field; + if (!has_json_paths) { + field = tuple_field(tuple, prev_fieldno); + } else { + field = tuple_field_by_part_raw(format, tuple_raw, field_map, + key_def->parts); + } const char *end = (char *)tuple + tuple_size(tuple); if (has_optional_parts && field == NULL) { total_size += tuple_hash_null(&h, &carry); @@ -347,9 +359,18 @@ tuple_hash_slowpath(const struct tuple *tuple, struct key_def *key_def) * need of tuple_field */ if (prev_fieldno + 1 != key_def->parts[part_id].fieldno) { - struct key_part *part = &key_def->parts[part_id]; - field = tuple_field_by_part_raw(format, tuple_raw, - field_map, part); + if (!has_json_paths) { + field = tuple_field(tuple, + key_def->parts[part_id]. + fieldno); + } else { + struct key_part *part = + &key_def->parts[part_id]; + field = tuple_field_by_part_raw(format, + tuple_raw, + field_map, + part); + } } if (has_optional_parts && (field == NULL || field >= end)) { total_size += tuple_hash_null(&h, &carry); diff --git a/src/box/vinyl.c b/src/box/vinyl.c index b590487..073466f 100644 --- a/src/box/vinyl.c +++ b/src/box/vinyl.c @@ -982,6 +982,11 @@ vinyl_index_def_change_requires_rebuild(struct index *index, return true; if (!field_type1_contains_type2(new_part->type, old_part->type)) return true; + if (old_part->path_len != new_part->path_len) + return true; + if (memcmp(old_part->path, new_part->path, + old_part->path_len) != 0) + return true; } return false; } diff --git a/src/box/vy_log.c b/src/box/vy_log.c index fc8ede5..f396705 100644 --- a/src/box/vy_log.c +++ b/src/box/vy_log.c @@ -711,7 +711,8 @@ vy_log_record_dup(struct region *pool, const struct vy_log_record *src) "struct key_part_def"); goto err; } - key_def_dump_parts(src->key_def, dst->key_parts); + if (key_def_dump_parts(pool, src->key_def, dst->key_parts) != 0) + goto err; dst->key_part_count = src->key_def->part_count; dst->key_def = NULL; } diff --git a/src/box/vy_lsm.c b/src/box/vy_lsm.c index de5ad31..b55933f 100644 --- a/src/box/vy_lsm.c +++ b/src/box/vy_lsm.c @@ -36,6 +36,7 @@ #include #include #include +#include #include "diag.h" #include "errcode.h" diff --git a/src/box/vy_point_lookup.c b/src/box/vy_point_lookup.c index 7b704b8..9d5e220 100644 --- a/src/box/vy_point_lookup.c +++ b/src/box/vy_point_lookup.c @@ -196,8 +196,6 @@ vy_point_lookup(struct vy_lsm *lsm, struct vy_tx *tx, const struct vy_read_view **rv, struct tuple *key, struct tuple **ret) { - assert(tuple_field_count(key) >= lsm->cmp_def->part_count); - *ret = NULL; double start_time = ev_monotonic_now(loop()); int rc = 0; diff --git a/src/box/vy_stmt.c b/src/box/vy_stmt.c index 8018dee..e06305e 100644 --- a/src/box/vy_stmt.c +++ b/src/box/vy_stmt.c @@ -44,6 +44,7 @@ #include "tuple_format.h" #include "xrow.h" #include "fiber.h" +#include "assoc.h" /** * Statement metadata keys. @@ -330,6 +331,71 @@ vy_stmt_replace_from_upsert(const struct tuple *upsert) return replace; } +static void +vy_stmt_msgpack_build(struct tuple_field *field, char *tuple, + uint32_t *field_map, char **offset, bool write_data, + struct mh_i64ptr_t *fields_iov_ht) +{ + if (field->type == FIELD_TYPE_ARRAY) { + if (write_data) + *offset = mp_encode_array(*offset, field->array_size); + else + *offset += mp_sizeof_array(field->array_size); + for (uint32_t i = 0; i < field->array_size; i++) { + if (field->array[i] == NULL) { + if (write_data) + *offset = mp_encode_nil(*offset); + else + *offset += mp_sizeof_nil(); + continue; + } + vy_stmt_msgpack_build(field->array[i], tuple, field_map, + offset, write_data, + fields_iov_ht); + } + return; + } else if (field->type == FIELD_TYPE_MAP) { + if (write_data) + *offset = mp_encode_map(*offset, mh_size(field->map)); + else + *offset += mp_sizeof_map(mh_size(field->map)); + mh_int_t i; + mh_foreach(field->map, i) { + struct mh_strnptr_node_t *node = + mh_strnptr_node(field->map, i); + assert(node); + if (write_data) { + *offset = mp_encode_str(*offset, node->str, + node->len); + } else { + *offset += mp_sizeof_str(node->len); + } + vy_stmt_msgpack_build(node->val, tuple, field_map, + offset, write_data, + fields_iov_ht); + } + return; + } + + mh_int_t k = mh_i64ptr_find(fields_iov_ht, (uint64_t)field, NULL); + struct iovec *iov = k != mh_end(fields_iov_ht) ? + mh_i64ptr_node(fields_iov_ht, k)->val : NULL; + if (iov == NULL) { + if (write_data) + *offset = mp_encode_nil(*offset); + else + *offset += mp_sizeof_nil(); + } else { + if (write_data) { + uint32_t data_offset = *offset - tuple; + memcpy(*offset, iov->iov_base, iov->iov_len); + if (field->offset_slot != TUPLE_OFFSET_SLOT_NIL) + field_map[field->offset_slot] = data_offset; + } + *offset += iov->iov_len; + } +} + static struct tuple * vy_stmt_new_surrogate_from_key(const char *key, enum iproto_type type, const struct key_def *cmp_def, @@ -338,51 +404,79 @@ vy_stmt_new_surrogate_from_key(const char *key, enum iproto_type type, /* UPSERT can't be surrogate. */ assert(type != IPROTO_UPSERT); struct region *region = &fiber()->gc; + struct tuple *stmt = NULL; uint32_t field_count = format->index_field_count; - struct iovec *iov = region_alloc(region, sizeof(*iov) * field_count); + uint32_t part_count = mp_decode_array(&key); + assert(part_count == cmp_def->part_count); + struct iovec *iov = region_alloc(region, sizeof(*iov) * part_count); if (iov == NULL) { - diag_set(OutOfMemory, sizeof(*iov) * field_count, - "region", "iov for surrogate key"); + diag_set(OutOfMemory, sizeof(*iov) * part_count, "region", + "iov for surrogate key"); return NULL; } - memset(iov, 0, sizeof(*iov) * field_count); - uint32_t part_count = mp_decode_array(&key); - assert(part_count == cmp_def->part_count); - assert(part_count <= field_count); - uint32_t nulls_count = field_count - cmp_def->part_count; - uint32_t bsize = mp_sizeof_array(field_count) + - mp_sizeof_nil() * nulls_count; - for (uint32_t i = 0; i < part_count; ++i) { - const struct key_part *part = &cmp_def->parts[i]; + struct mh_i64ptr_t *fields_iov_ht = mh_i64ptr_new(); + if (fields_iov_ht == NULL) { + diag_set(OutOfMemory, sizeof(struct mh_i64ptr_t), + "mh_i64ptr_new", "fields_iov_ht"); + return NULL; + } + if (mh_i64ptr_reserve(fields_iov_ht, part_count, NULL) != 0) { + diag_set(OutOfMemory, part_count, "mh_i64ptr_reserve", + "fields_iov_ht"); + goto end; + } + uint32_t bsize = mp_sizeof_array(field_count); + uint32_t nulls_count = field_count; + memset(iov, 0, sizeof(*iov) * part_count); + const struct key_part *part = cmp_def->parts; + for (uint32_t i = 0; i < part_count; ++i, ++part) { assert(part->fieldno < field_count); const char *svp = key; - iov[part->fieldno].iov_base = (char *) key; + iov[i].iov_base = (char *) key; mp_next(&key); - iov[part->fieldno].iov_len = key - svp; - bsize += key - svp; + iov[i].iov_len = key - svp; + struct tuple_field *field; + if (part->path == NULL) { + field = &format->fields[part->fieldno]; + --nulls_count; + } else { + struct mh_strnptr_node_t *ht_record = + json_path_hash_get(format->path_hash, + part->path, part->path_len, + part->path_hash); + assert(ht_record != NULL); + field = ht_record->val; + assert(field != NULL); + } + struct mh_i64ptr_node_t node = {(uint64_t)field, &iov[i]}; + mh_int_t k = mh_i64ptr_put(fields_iov_ht, &node, NULL, NULL); + if (k == mh_end(fields_iov_ht)) + goto end; + } + for (uint32_t i = 0; i < field_count; ++i) { + char *data = NULL; + vy_stmt_msgpack_build(&format->fields[i], NULL, NULL, &data, + false, fields_iov_ht); + bsize += data - (char *)NULL; } - struct tuple *stmt = vy_stmt_alloc(format, bsize); + stmt = vy_stmt_alloc(format, bsize); if (stmt == NULL) - return NULL; + goto end; char *raw = (char *) tuple_data(stmt); uint32_t *field_map = (uint32_t *) raw; char *wpos = mp_encode_array(raw, field_count); for (uint32_t i = 0; i < field_count; ++i) { - const struct tuple_field *field = &format->fields[i]; - if (field->offset_slot != TUPLE_OFFSET_SLOT_NIL) - field_map[field->offset_slot] = wpos - raw; - if (iov[i].iov_base == NULL) { - wpos = mp_encode_nil(wpos); - } else { - memcpy(wpos, iov[i].iov_base, iov[i].iov_len); - wpos += iov[i].iov_len; - } + vy_stmt_msgpack_build(&format->fields[i], raw, field_map, &wpos, + true, fields_iov_ht); } - assert(wpos == raw + bsize); + assert(wpos <= raw + bsize); vy_stmt_set_type(stmt, type); + +end: + mh_i64ptr_delete(fields_iov_ht); return stmt; } diff --git a/test/box/misc.result b/test/box/misc.result index 4ee4797..b1fda78 100644 --- a/test/box/misc.result +++ b/test/box/misc.result @@ -350,7 +350,7 @@ t; - 'box.error.CANT_CREATE_COLLATION : 150' - 'box.error.USER_EXISTS : 46' - 'box.error.WAL_IO : 40' - - 'box.error.PROC_RET : 21' + - 'box.error.RTREE_RECT : 101' - 'box.error.PRIV_GRANTED : 89' - 'box.error.CREATE_SPACE : 9' - 'box.error.GRANT : 88' @@ -361,7 +361,7 @@ t; - 'box.error.VINYL_MAX_TUPLE_SIZE : 139' - 'box.error.LOAD_FUNCTION : 99' - 'box.error.INVALID_XLOG : 74' - - 'box.error.READ_VIEW_ABORTED : 130' + - 'box.error.PRIV_NOT_GRANTED : 91' - 'box.error.TRANSACTION_CONFLICT : 97' - 'box.error.GUEST_USER_PASSWORD : 96' - 'box.error.PROC_C : 102' @@ -371,8 +371,8 @@ t; - 'box.error.DROP_FUNCTION : 71' - 'box.error.CFG : 59' - 'box.error.NO_SUCH_FIELD : 37' - - 'box.error.CONNECTION_TO_SELF : 117' - - 'box.error.FUNCTION_MAX : 54' + - 'box.error.MORE_THAN_ONE_TUPLE : 41' + - 'box.error.PROC_LUA : 32' - 'box.error.ILLEGAL_PARAMS : 1' - 'box.error.PARTIAL_KEY : 136' - 'box.error.SAVEPOINT_NO_TRANSACTION : 114' @@ -400,34 +400,35 @@ t; - 'box.error.UPDATE_ARG_TYPE : 26' - 'box.error.CROSS_ENGINE_TRANSACTION : 81' - 'box.error.FORMAT_MISMATCH_INDEX_PART : 27' - - 'box.error.FUNCTION_TX_ACTIVE : 30' - 'box.error.injection : table:
- - 'box.error.ITERATOR_TYPE : 72' + - 'box.error.FUNCTION_TX_ACTIVE : 30' + - 'box.error.IDENTIFIER : 70' + - 'box.error.TRANSACTION_YIELD : 154' - 'box.error.NO_SUCH_ENGINE : 57' - 'box.error.COMMIT_IN_SUB_STMT : 122' - - 'box.error.TRANSACTION_YIELD : 154' - - 'box.error.UNSUPPORTED : 5' - - 'box.error.LAST_DROP : 15' + - 'box.error.RELOAD_CFG : 58' - 'box.error.SPACE_FIELD_IS_DUPLICATE : 149' + - 'box.error.LAST_DROP : 15' + - 'box.error.SEQUENCE_OVERFLOW : 147' - 'box.error.DECOMPRESSION : 124' - 'box.error.CREATE_SEQUENCE : 142' - 'box.error.CREATE_USER : 43' - - 'box.error.SEQUENCE_OVERFLOW : 147' + - 'box.error.FUNCTION_MAX : 54' - 'box.error.INSTANCE_UUID_MISMATCH : 66' - - 'box.error.RELOAD_CFG : 58' + - 'box.error.TUPLE_FORMAT_LIMIT : 16' - 'box.error.SYSTEM : 115' - 'box.error.KEY_PART_IS_TOO_LONG : 118' - - 'box.error.MORE_THAN_ONE_TUPLE : 41' - 'box.error.TRUNCATE_SYSTEM_SPACE : 137' - - 'box.error.NO_SUCH_SAVEPOINT : 61' - 'box.error.VY_QUOTA_TIMEOUT : 135' - - 'box.error.PRIV_NOT_GRANTED : 91' + - 'box.error.NO_SUCH_SAVEPOINT : 61' + - 'box.error.PROTOCOL : 104' + - 'box.error.READ_VIEW_ABORTED : 130' - 'box.error.WRONG_INDEX_OPTIONS : 108' - 'box.error.INVALID_VYLOG_FILE : 133' - 'box.error.INDEX_FIELD_COUNT_LIMIT : 127' - - 'box.error.BEFORE_REPLACE_RET : 53' + - 'box.error.DATA_STRUCTURE_MISMATCH : 55' - 'box.error.USER_MAX : 56' - - 'box.error.INVALID_MSGPACK : 20' + - 'box.error.BEFORE_REPLACE_RET : 53' - 'box.error.TUPLE_NOT_ARRAY : 22' - 'box.error.KEY_PART_COUNT : 31' - 'box.error.ALTER_SPACE : 12' @@ -436,47 +437,47 @@ t; - 'box.error.DROP_SEQUENCE : 144' - 'box.error.INVALID_XLOG_ORDER : 76' - 'box.error.UNKNOWN_REQUEST_TYPE : 48' - - 'box.error.PROC_LUA : 32' + - 'box.error.PROC_RET : 21' - 'box.error.SUB_STMT_MAX : 121' - 'box.error.ROLE_NOT_GRANTED : 92' - 'box.error.SPACE_EXISTS : 10' - - 'box.error.UPDATE_INTEGER_OVERFLOW : 95' + - 'box.error.UNSUPPORTED : 5' - 'box.error.MIN_FIELD_COUNT : 39' - 'box.error.NO_SUCH_SPACE : 36' - 'box.error.WRONG_INDEX_PARTS : 107' - 'box.error.REPLICASET_UUID_MISMATCH : 63' - 'box.error.UPDATE_FIELD : 29' - 'box.error.INDEX_EXISTS : 85' - - 'box.error.SPLICE : 25' + - 'box.error.DROP_SPACE : 11' - 'box.error.COMPRESSION : 119' - 'box.error.INVALID_ORDER : 68' - - 'box.error.UNKNOWN : 0' + - 'box.error.SPLICE : 25' - 'box.error.NO_SUCH_GROUP : 155' - - 'box.error.TUPLE_FORMAT_LIMIT : 16' + - 'box.error.INVALID_MSGPACK : 20' - 'box.error.DROP_PRIMARY_KEY : 17' - 'box.error.NULLABLE_PRIMARY : 152' - 'box.error.NO_SUCH_SEQUENCE : 145' - 'box.error.INJECTION : 8' - 'box.error.INVALID_UUID : 64' - - 'box.error.IDENTIFIER : 70' + - 'box.error.NO_SUCH_ROLE : 82' - 'box.error.TIMEOUT : 78' + - 'box.error.ITERATOR_TYPE : 72' - 'box.error.REPLICA_MAX : 73' - - 'box.error.NO_SUCH_ROLE : 82' - - 'box.error.DROP_SPACE : 11' + - 'box.error.UNKNOWN : 0' - 'box.error.MISSING_REQUEST_FIELD : 69' - 'box.error.MISSING_SNAPSHOT : 93' - 'box.error.WRONG_SPACE_OPTIONS : 111' - 'box.error.READONLY : 7' - - 'box.error.RTREE_RECT : 101' + - 'box.error.UPDATE_INTEGER_OVERFLOW : 95' - 'box.error.UPSERT_UNIQUE_SECONDARY_KEY : 105' - 'box.error.NO_CONNECTION : 77' - 'box.error.UNSUPPORTED_PRIV : 98' - 'box.error.WRONG_SCHEMA_VERSION : 109' - 'box.error.ROLLBACK_IN_SUB_STMT : 123' - - 'box.error.PROTOCOL : 104' - - 'box.error.INVALID_XLOG_TYPE : 125' - - 'box.error.INDEX_PART_TYPE_MISMATCH : 24' - 'box.error.UNSUPPORTED_INDEX_FEATURE : 112' + - 'box.error.CONNECTION_TO_SELF : 117' + - 'box.error.INDEX_PART_TYPE_MISMATCH : 24' + - 'box.error.INVALID_XLOG_TYPE : 125' ... test_run:cmd("setopt delimiter ''"); --- diff --git a/test/engine/tuple.result b/test/engine/tuple.result index 35c700e..b74bb23 100644 --- a/test/engine/tuple.result +++ b/test/engine/tuple.result @@ -954,6 +954,393 @@ type(tuple:tomap().fourth) s:drop() --- ... +-- +-- gh-1012: Indexes for JSON-defined paths. +-- +s = box.schema.space.create('withdata', {engine = engine}) +--- +... +s:create_index('test1', {parts = {{2, 'number'}, {3, 'str', path = '[3].FIO["fname"]'}, {3, 'str', path = '[3]["FIO"].fname'}}}) +--- +- error: 'Can''t create or modify index ''test1'' in space ''withdata'': same key + part is indexed twice' +... +s:create_index('test1', {parts = {{2, 'number'}, {3, 'str', path = 666}, {3, 'str', path = '[3]["FIO"]["fname"]'}}}) +--- +- error: 'Wrong index options (field 2): ''path'' must be string' +... +s:create_index('test1', {parts = {{2, 'number'}, {3, 'str', path = 'field.FIO.fname'}}}) +--- +- error: 'Wrong index options (field 2): invalid JSON path: first part should be defined + as array index' +... +s:create_index('test1', {parts = {{2, 'number'}, {3, 'map', path = '[3].FIO'}}}) +--- +- error: 'Can''t create or modify index ''test1'' in space ''withdata'': field type + ''map'' is not supported' +... +s:create_index('test1', {parts = {{2, 'number'}, {3, 'array', path = '[3][1]'}}}) +--- +- error: 'Can''t create or modify index ''test1'' in space ''withdata'': field type + ''array'' is not supported' +... +s:create_index('test1', {parts = {{2, 'number'}, {3, 'str', path = '[3].FIO'}, {3, 'str', path = '[3]["FIO"].fname'}}}) +--- +- error: Field 3 has type 'map' in one index, but type 'string' in another +... +s:create_index('test1', {parts = {{2, 'number'}, {3, 'str', path = '[3][1].sname'}, {3, 'str', path = '[3]["FIO"].fname'}}}) +--- +- error: Field 3 has type 'map' in one index, but type 'array' in another +... +s:create_index('test1', {parts = {{2, 'number'}, {3, 'str', path = '[2].FIO.fname'}}}) +--- +- error: 'Wrong index options (field 2): invalid JSON path: first part refers to invalid + field' +... +s:create_index('test1', {parts = {{2, 'number'}, {3, 'str', path = '[3].FIO....fname'}}}) +--- +- error: 'Wrong index options (field 3): invalid JSON path ''[3].FIO....fname'': path + has invalid structure (error at position 9)' +... +idx = s:create_index('test1', {parts = {{2, 'number'}, {3, 'str', path = '[3]["FIO"]["fname"]'}, {3, 'str', path = '[3]["FIO"]["sname"]'}}}) +--- +... +assert(idx ~= nil) +--- +- true +... +s:insert{7, 7, {town = 'London', FIO = 666}, 4, 5} +--- +- error: 'Tuple doesn''t math document structure: invalid field 3 document content + ''666'': type mismatch: have unsigned, expected map' +... +s:insert{7, 7, {town = 'London', FIO = {fname = 666, sname = 'Bond'}}, 4, 5} +--- +- error: 'Tuple doesn''t math document structure: invalid field 3 document content + ''666'': type mismatch: have unsigned, expected string' +... +s:insert{7, 7, {town = 'London', FIO = {fname = "James"}}, 4, 5} +--- +- error: 'Tuple doesn''t math document structure: invalid field 3 document content + ''{"fname": "James"}'': map doesn''t contain key ''sname'' defined in index' +... +s:insert{7, 7, {town = 'London', FIO = {fname = 'James', sname = 'Bond'}}, 4, 5} +--- +- [7, 7, {'town': 'London', 'FIO': {'fname': 'James', 'sname': 'Bond'}}, 4, 5] +... +s:insert{7, 7, {town = 'London', FIO = {fname = 'James', sname = 'Bond'}}, 4, 5} +--- +- error: Duplicate key exists in unique index 'test1' in space 'withdata' +... +s:insert{7, 7, {town = 'London', FIO = {fname = 'James', sname = 'Bond', data = "extra"}}, 4, 5} +--- +- error: Duplicate key exists in unique index 'test1' in space 'withdata' +... +s:insert{7, 7, {town = 'Moscow', FIO = {fname = 'Max', sname = 'Isaev', data = "extra"}}, 4, 5} +--- +- [7, 7, {'town': 'Moscow', 'FIO': {'fname': 'Max', 'data': 'extra', 'sname': 'Isaev'}}, + 4, 5] +... +idx:select() +--- +- - [7, 7, {'town': 'London', 'FIO': {'fname': 'James', 'sname': 'Bond'}}, 4, 5] + - [7, 7, {'town': 'Moscow', 'FIO': {'fname': 'Max', 'data': 'extra', 'sname': 'Isaev'}}, + 4, 5] +... +idx:min() +--- +- [7, 7, {'town': 'London', 'FIO': {'fname': 'James', 'sname': 'Bond'}}, 4, 5] +... +idx:max() +--- +- [7, 7, {'town': 'Moscow', 'FIO': {'fname': 'Max', 'data': 'extra', 'sname': 'Isaev'}}, + 4, 5] +... +s:drop() +--- +... +s = box.schema.create_space('withdata', {engine = engine}) +--- +... +parts = {} +--- +... +parts[1] = {1, 'unsigned', path='[1][2]'} +--- +... +pk = s:create_index('pk', {parts = parts}) +--- +... +s:insert{{1, 2}, 3} +--- +- [[1, 2], 3] +... +s:upsert({{box.null, 2}}, {{'+', 2, 5}}) +--- +... +s:get(2) +--- +- [[1, 2], 8] +... +s:drop() +--- +... +-- Create index on space with data +s = box.schema.space.create('withdata', {engine = engine}) +--- +... +pk = s:create_index('primary', { type = 'tree' }) +--- +... +s:insert{1, 7, {town = 'London', FIO = 1234}, 4, 5} +--- +- [1, 7, {'town': 'London', 'FIO': 1234}, 4, 5] +... +s:insert{2, 7, {town = 'London', FIO = {fname = 'James', sname = 'Bond'}}, 4, 5} +--- +- [2, 7, {'town': 'London', 'FIO': {'fname': 'James', 'sname': 'Bond'}}, 4, 5] +... +s:insert{3, 7, {town = 'London', FIO = {fname = 'James', sname = 'Bond'}}, 4, 5} +--- +- [3, 7, {'town': 'London', 'FIO': {'fname': 'James', 'sname': 'Bond'}}, 4, 5] +... +s:insert{4, 7, {town = 'London', FIO = {1,2,3}}, 4, 5} +--- +- [4, 7, {'town': 'London', 'FIO': [1, 2, 3]}, 4, 5] +... +s:create_index('test1', {parts = {{2, 'number'}, {3, 'str', path = '[3]["FIO"]["fname"]'}, {3, 'str', path = '[3]["FIO"]["sname"]'}}}) +--- +- error: 'Tuple doesn''t math document structure: invalid field 3 document content + ''1234'': type mismatch: have unsigned, expected map' +... +_ = s:delete(1) +--- +... +s:create_index('test1', {parts = {{2, 'number'}, {3, 'str', path = '[3]["FIO"]["fname"]'}, {3, 'str', path = '[3]["FIO"]["sname"]'}}}) +--- +- error: Duplicate key exists in unique index 'test1' in space 'withdata' +... +_ = s:delete(2) +--- +... +s:create_index('test1', {parts = {{2, 'number'}, {3, 'str', path = '[3]["FIO"]["fname"]'}, {3, 'str', path = '[3]["FIO"]["sname"]'}}}) +--- +- error: 'Tuple doesn''t math document structure: invalid field 3 document content + ''[1, 2, 3]'': type mismatch: have array, expected map' +... +_ = s:delete(4) +--- +... +idx = s:create_index('test1', {parts = {{2, 'number'}, {3, 'str', path = '[3]["FIO"]["fname"]', is_nullable = true}, {3, 'str', path = '[3]["FIO"]["sname"]'}, {3, 'str', path = '[3]["FIO"]["extra"]', is_nullable = true}}}) +--- +... +assert(idx ~= nil) +--- +- true +... +s:create_index('test2', {parts = {{2, 'number'}, {3, 'number', path = '[3]["FIO"]["fname"]'}}}) +--- +- error: 'Wrong index options (field 3): JSON path ''[3]["FIO"]["fname"]'' has been + already constructed for ''string'' leaf record' +... +idx2 = s:create_index('test2', {parts = {{2, 'number'}, {3, 'str', path = '[3]["FIO"]["fname"]'}}}) +--- +... +assert(idx2 ~= nil) +--- +- true +... +t = s:insert{5, 7, {town = 'Matrix', FIO = {fname = 'Agent', sname = 'Smith'}}, 4, 5} +--- +... +-- Test field_map in tuple speed-up access by indexed path. +t["[3][\"FIO\"][\"fname\"]"] +--- +- Agent +... +idx:select() +--- +- - [5, 7, {'town': 'Matrix', 'FIO': {'fname': 'Agent', 'sname': 'Smith'}}, 4, 5] + - [3, 7, {'town': 'London', 'FIO': {'fname': 'James', 'sname': 'Bond'}}, 4, 5] +... +idx:min() +--- +- [5, 7, {'town': 'Matrix', 'FIO': {'fname': 'Agent', 'sname': 'Smith'}}, 4, 5] +... +idx:max() +--- +- [3, 7, {'town': 'London', 'FIO': {'fname': 'James', 'sname': 'Bond'}}, 4, 5] +... +idx:drop() +--- +... +s:drop() +--- +... +-- Test complex JSON indexes +s = box.schema.space.create('withdata', {engine = engine}) +--- +... +parts = {} +--- +... +parts[1] = {1, 'str', path='[1][3][2].a'} +--- +... +parts[2] = {1, 'unsigned', path = '[1][3][1]'} +--- +... +parts[3] = {2, 'str', path = '[2][2].d[1]'} +--- +... +pk = s:create_index('primary', { type = 'tree', parts = parts}) +--- +... +s:insert{{1, 2, {3, {3, a = 'str', b = 5}}}, {'c', {d = {'e', 'f'}, e = 'g'}}, 6, {1, 2, 3}} +--- +- [[1, 2, [3, {1: 3, 'a': 'str', 'b': 5}]], ['c', {'d': ['e', 'f'], 'e': 'g'}], 6, + [1, 2, 3]] +... +s:insert{{1, 2, {3, {a = 'str', b = 1}}}, {'c', {d = {'e', 'f'}, e = 'g'}}, 6} +--- +- error: Duplicate key exists in unique index 'primary' in space 'withdata' +... +parts = {} +--- +... +parts[1] = {4, 'unsigned', path='[4][1]', is_nullable = false} +--- +... +parts[2] = {4, 'unsigned', path='[4][2]', is_nullable = true} +--- +... +parts[3] = {4, 'unsigned', path='[4][4]', is_nullable = true} +--- +... +trap_idx = s:create_index('trap', { type = 'tree', parts = parts}) +--- +... +s:insert{{1, 2, {3, {3, a = 'str2', b = 5}}}, {'c', {d = {'e', 'f'}, e = 'g'}}, 6, {}} +--- +- error: 'Tuple doesn''t math document structure: invalid field 4 document content + ''[]'': array size 0 is less than size of 1 defined in index' +... +parts = {} +--- +... +parts[1] = {1, 'unsigned', path='[1][3][2].b' } +--- +... +parts[2] = {3, 'unsigned'} +--- +... +crosspart_idx = s:create_index('crosspart', { parts = parts}) +--- +... +s:insert{{1, 2, {3, {a = 'str2', b = 2}}}, {'c', {d = {'e', 'f'}, e = 'g'}}, 6, {9, 2, 3}} +--- +- [[1, 2, [3, {'a': 'str2', 'b': 2}]], ['c', {'d': ['e', 'f'], 'e': 'g'}], 6, [9, + 2, 3]] +... +parts = {} +--- +... +parts[1] = {1, 'unsigned', path='[1][3][2].b'} +--- +... +num_idx = s:create_index('numeric', {parts = parts}) +--- +... +s:insert{{1, 2, {3, {a = 'str3', b = 9}}}, {'c', {d = {'e', 'f'}, e = 'g'}}, 6, {0}} +--- +- [[1, 2, [3, {'a': 'str3', 'b': 9}]], ['c', {'d': ['e', 'f'], 'e': 'g'}], 6, [0]] +... +num_idx:get(2) +--- +- [[1, 2, [3, {'a': 'str2', 'b': 2}]], ['c', {'d': ['e', 'f'], 'e': 'g'}], 6, [9, + 2, 3]] +... +num_idx:select() +--- +- - [[1, 2, [3, {'a': 'str2', 'b': 2}]], ['c', {'d': ['e', 'f'], 'e': 'g'}], 6, [ + 9, 2, 3]] + - [[1, 2, [3, {1: 3, 'a': 'str', 'b': 5}]], ['c', {'d': ['e', 'f'], 'e': 'g'}], + 6, [1, 2, 3]] + - [[1, 2, [3, {'a': 'str3', 'b': 9}]], ['c', {'d': ['e', 'f'], 'e': 'g'}], 6, [ + 0]] +... +num_idx:max() +--- +- [[1, 2, [3, {'a': 'str3', 'b': 9}]], ['c', {'d': ['e', 'f'], 'e': 'g'}], 6, [0]] +... +num_idx:min() +--- +- [[1, 2, [3, {'a': 'str2', 'b': 2}]], ['c', {'d': ['e', 'f'], 'e': 'g'}], 6, [9, + 2, 3]] +... +assert(crosspart_idx:max() == num_idx:max()) +--- +- true +... +assert(crosspart_idx:min() == num_idx:min()) +--- +- true +... +trap_idx:max() +--- +- [[1, 2, [3, {'a': 'str2', 'b': 2}]], ['c', {'d': ['e', 'f'], 'e': 'g'}], 6, [9, + 2, 3]] +... +trap_idx:min() +--- +- [[1, 2, [3, {'a': 'str3', 'b': 9}]], ['c', {'d': ['e', 'f'], 'e': 'g'}], 6, [0]] +... +s:drop() +--- +... +s = box.schema.space.create('withdata', {engine = engine}) +--- +... +pk_simplified = s:create_index('primary', { type = 'tree', parts = {{1, 'unsigned', path = '[1]'}}}) +--- +... +assert(pk_simplified.path == box.NULL) +--- +- true +... +idx = s:create_index('idx', {parts = {{2, 'integer', path = '[2].a'}}}) +--- +... +s:insert{31, {a = 1, aa = -1}} +--- +- [31, {'a': 1, 'aa': -1}] +... +s:insert{22, {a = 2, aa = -2}} +--- +- [22, {'a': 2, 'aa': -2}] +... +s:insert{13, {a = 3, aa = -3}} +--- +- [13, {'a': 3, 'aa': -3}] +... +idx:select() +--- +- - [31, {'a': 1, 'aa': -1}] + - [22, {'a': 2, 'aa': -2}] + - [13, {'a': 3, 'aa': -3}] +... +idx:alter({parts = {{2, 'integer', path = '[2].aa'}}}) +--- +... +idx:select() +--- +- - [13, {'a': 3, 'aa': -3}] + - [22, {'a': 2, 'aa': -2}] + - [31, {'a': 1, 'aa': -1}] +... +s:drop() +--- +... engine = nil --- ... diff --git a/test/engine/tuple.test.lua b/test/engine/tuple.test.lua index edc3dab..d563c66 100644 --- a/test/engine/tuple.test.lua +++ b/test/engine/tuple.test.lua @@ -312,5 +312,114 @@ tuple:tomap().fourth type(tuple:tomap().fourth) s:drop() +-- +-- gh-1012: Indexes for JSON-defined paths. +-- +s = box.schema.space.create('withdata', {engine = engine}) +s:create_index('test1', {parts = {{2, 'number'}, {3, 'str', path = '[3].FIO["fname"]'}, {3, 'str', path = '[3]["FIO"].fname'}}}) +s:create_index('test1', {parts = {{2, 'number'}, {3, 'str', path = 666}, {3, 'str', path = '[3]["FIO"]["fname"]'}}}) +s:create_index('test1', {parts = {{2, 'number'}, {3, 'str', path = 'field.FIO.fname'}}}) +s:create_index('test1', {parts = {{2, 'number'}, {3, 'map', path = '[3].FIO'}}}) +s:create_index('test1', {parts = {{2, 'number'}, {3, 'array', path = '[3][1]'}}}) +s:create_index('test1', {parts = {{2, 'number'}, {3, 'str', path = '[3].FIO'}, {3, 'str', path = '[3]["FIO"].fname'}}}) +s:create_index('test1', {parts = {{2, 'number'}, {3, 'str', path = '[3][1].sname'}, {3, 'str', path = '[3]["FIO"].fname'}}}) +s:create_index('test1', {parts = {{2, 'number'}, {3, 'str', path = '[2].FIO.fname'}}}) +s:create_index('test1', {parts = {{2, 'number'}, {3, 'str', path = '[3].FIO....fname'}}}) +idx = s:create_index('test1', {parts = {{2, 'number'}, {3, 'str', path = '[3]["FIO"]["fname"]'}, {3, 'str', path = '[3]["FIO"]["sname"]'}}}) +assert(idx ~= nil) +s:insert{7, 7, {town = 'London', FIO = 666}, 4, 5} +s:insert{7, 7, {town = 'London', FIO = {fname = 666, sname = 'Bond'}}, 4, 5} +s:insert{7, 7, {town = 'London', FIO = {fname = "James"}}, 4, 5} +s:insert{7, 7, {town = 'London', FIO = {fname = 'James', sname = 'Bond'}}, 4, 5} +s:insert{7, 7, {town = 'London', FIO = {fname = 'James', sname = 'Bond'}}, 4, 5} +s:insert{7, 7, {town = 'London', FIO = {fname = 'James', sname = 'Bond', data = "extra"}}, 4, 5} +s:insert{7, 7, {town = 'Moscow', FIO = {fname = 'Max', sname = 'Isaev', data = "extra"}}, 4, 5} +idx:select() +idx:min() +idx:max() +s:drop() + +s = box.schema.create_space('withdata', {engine = engine}) +parts = {} +parts[1] = {1, 'unsigned', path='[1][2]'} +pk = s:create_index('pk', {parts = parts}) +s:insert{{1, 2}, 3} +s:upsert({{box.null, 2}}, {{'+', 2, 5}}) +s:get(2) +s:drop() + +-- Create index on space with data +s = box.schema.space.create('withdata', {engine = engine}) +pk = s:create_index('primary', { type = 'tree' }) +s:insert{1, 7, {town = 'London', FIO = 1234}, 4, 5} +s:insert{2, 7, {town = 'London', FIO = {fname = 'James', sname = 'Bond'}}, 4, 5} +s:insert{3, 7, {town = 'London', FIO = {fname = 'James', sname = 'Bond'}}, 4, 5} +s:insert{4, 7, {town = 'London', FIO = {1,2,3}}, 4, 5} +s:create_index('test1', {parts = {{2, 'number'}, {3, 'str', path = '[3]["FIO"]["fname"]'}, {3, 'str', path = '[3]["FIO"]["sname"]'}}}) +_ = s:delete(1) +s:create_index('test1', {parts = {{2, 'number'}, {3, 'str', path = '[3]["FIO"]["fname"]'}, {3, 'str', path = '[3]["FIO"]["sname"]'}}}) +_ = s:delete(2) +s:create_index('test1', {parts = {{2, 'number'}, {3, 'str', path = '[3]["FIO"]["fname"]'}, {3, 'str', path = '[3]["FIO"]["sname"]'}}}) +_ = s:delete(4) +idx = s:create_index('test1', {parts = {{2, 'number'}, {3, 'str', path = '[3]["FIO"]["fname"]', is_nullable = true}, {3, 'str', path = '[3]["FIO"]["sname"]'}, {3, 'str', path = '[3]["FIO"]["extra"]', is_nullable = true}}}) +assert(idx ~= nil) +s:create_index('test2', {parts = {{2, 'number'}, {3, 'number', path = '[3]["FIO"]["fname"]'}}}) +idx2 = s:create_index('test2', {parts = {{2, 'number'}, {3, 'str', path = '[3]["FIO"]["fname"]'}}}) +assert(idx2 ~= nil) +t = s:insert{5, 7, {town = 'Matrix', FIO = {fname = 'Agent', sname = 'Smith'}}, 4, 5} +-- Test field_map in tuple speed-up access by indexed path. +t["[3][\"FIO\"][\"fname\"]"] +idx:select() +idx:min() +idx:max() +idx:drop() +s:drop() + +-- Test complex JSON indexes +s = box.schema.space.create('withdata', {engine = engine}) +parts = {} +parts[1] = {1, 'str', path='[1][3][2].a'} +parts[2] = {1, 'unsigned', path = '[1][3][1]'} +parts[3] = {2, 'str', path = '[2][2].d[1]'} +pk = s:create_index('primary', { type = 'tree', parts = parts}) +s:insert{{1, 2, {3, {3, a = 'str', b = 5}}}, {'c', {d = {'e', 'f'}, e = 'g'}}, 6, {1, 2, 3}} +s:insert{{1, 2, {3, {a = 'str', b = 1}}}, {'c', {d = {'e', 'f'}, e = 'g'}}, 6} +parts = {} +parts[1] = {4, 'unsigned', path='[4][1]', is_nullable = false} +parts[2] = {4, 'unsigned', path='[4][2]', is_nullable = true} +parts[3] = {4, 'unsigned', path='[4][4]', is_nullable = true} +trap_idx = s:create_index('trap', { type = 'tree', parts = parts}) +s:insert{{1, 2, {3, {3, a = 'str2', b = 5}}}, {'c', {d = {'e', 'f'}, e = 'g'}}, 6, {}} +parts = {} +parts[1] = {1, 'unsigned', path='[1][3][2].b' } +parts[2] = {3, 'unsigned'} +crosspart_idx = s:create_index('crosspart', { parts = parts}) +s:insert{{1, 2, {3, {a = 'str2', b = 2}}}, {'c', {d = {'e', 'f'}, e = 'g'}}, 6, {9, 2, 3}} +parts = {} +parts[1] = {1, 'unsigned', path='[1][3][2].b'} +num_idx = s:create_index('numeric', {parts = parts}) +s:insert{{1, 2, {3, {a = 'str3', b = 9}}}, {'c', {d = {'e', 'f'}, e = 'g'}}, 6, {0}} +num_idx:get(2) +num_idx:select() +num_idx:max() +num_idx:min() +assert(crosspart_idx:max() == num_idx:max()) +assert(crosspart_idx:min() == num_idx:min()) +trap_idx:max() +trap_idx:min() +s:drop() + +s = box.schema.space.create('withdata', {engine = engine}) +pk_simplified = s:create_index('primary', { type = 'tree', parts = {{1, 'unsigned', path = '[1]'}}}) +assert(pk_simplified.path == box.NULL) +idx = s:create_index('idx', {parts = {{2, 'integer', path = '[2].a'}}}) +s:insert{31, {a = 1, aa = -1}} +s:insert{22, {a = 2, aa = -2}} +s:insert{13, {a = 3, aa = -3}} +idx:select() +idx:alter({parts = {{2, 'integer', path = '[2].aa'}}}) +idx:select() +s:drop() + engine = nil test_run = nil -- 2.7.4