From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from localhost (localhost [127.0.0.1]) by turing.freelists.org (Avenir Technologies Mail Multiplex) with ESMTP id A4B14294F9 for ; Mon, 27 Aug 2018 03:37:35 -0400 (EDT) Received: from turing.freelists.org ([127.0.0.1]) by localhost (turing.freelists.org [127.0.0.1]) (amavisd-new, port 10024) with ESMTP id PGpZ6QGxuHS0 for ; Mon, 27 Aug 2018 03:37:35 -0400 (EDT) Received: from smtpng2.m.smailru.net (smtpng2.m.smailru.net [94.100.179.3]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by turing.freelists.org (Avenir Technologies Mail Multiplex) with ESMTPS id 2F1E4294AA for ; Mon, 27 Aug 2018 03:37:35 -0400 (EDT) From: Kirill Shcherbatov Subject: [tarantool-patches] [PATCH v3 4/4] box: specify indexes in user-friendly form Date: Mon, 27 Aug 2018 10:37:30 +0300 Message-Id: In-Reply-To: References: In-Reply-To: References: Sender: tarantool-patches-bounce@freelists.org Errors-to: tarantool-patches-bounce@freelists.org Reply-To: tarantool-patches@freelists.org List-help: List-unsubscribe: List-software: Ecartis version 1.0.0 List-Id: tarantool-patches List-subscribe: List-owner: List-post: List-archive: To: tarantool-patches@freelists.org Cc: v.shpilevoy@tarantool.org, Kirill Shcherbatov Since now it is possible to create indexes by JSON-path using field names specified in format. Closes #1012. @TarantoolBot document Title: Indexes by JSON path Sometimes field data could have complex document structure. When this structure is consistent across whole document, you are able to create an index by JSON path. Example: s:create_index('json_index', {parts = {{'data.FIO["fname"]', 'str'}}}) --- src/box/lua/index.c | 74 +++++++++++++++++++++++++++++++++++++++++++++ src/box/lua/schema.lua | 20 ++++++------ test/engine/iterator.result | 2 +- test/engine/tuple.result | 37 +++++++++++++++++++++++ test/engine/tuple.test.lua | 10 ++++++ 5 files changed, 131 insertions(+), 12 deletions(-) diff --git a/src/box/lua/index.c b/src/box/lua/index.c index ef89c39..b2b8ca2 100644 --- a/src/box/lua/index.c +++ b/src/box/lua/index.c @@ -28,6 +28,9 @@ * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ +#include "box/schema.h" +#include "box/tuple_format.h" +#include "json/path.h" #include "box/lua/index.h" #include "lua/utils.h" #include "box/box.h" @@ -328,6 +331,76 @@ lbox_index_compact(lua_State *L) return 0; } +static int +lbox_index_resolve_path(struct lua_State *L) +{ + if (lua_gettop(L) != 3 || + !lua_isnumber(L, 1) || !lua_isnumber(L, 2) || !lua_isstring(L, 3)) { + return luaL_error(L, "Usage box.internal." + "path_resolve(part_id, space_id, path)"); + } + uint32_t part_id = lua_tonumber(L, 1); + uint32_t space_id = lua_tonumber(L, 2); + const char *path = lua_tostring(L, 3); + size_t path_len = strlen(path); + struct space *space = space_cache_find(space_id); + if (space == NULL) + return luaT_error(L); + struct json_path_parser parser; + struct json_path_node node; + json_path_parser_create(&parser, path, path_len); + int rc = json_path_next(&parser, &node); + if (rc != 0) { + const char *err_msg = + tt_sprintf("Illegal parameters, options.parts[%d]: " + "error in path on position %d", part_id, rc); + diag_set(IllegalParams, err_msg); + return luaT_error(L); + } + assert(space->format != NULL && space->format->dict != NULL); + uint32_t fieldno = node.num - TUPLE_INDEX_BASE;; + if (node.type == JSON_PATH_NUM && + fieldno >= space->format->field_count) { + const char *err_msg = + tt_sprintf("Illegal parameters, options.parts[%d]: " + "field '%d' referenced in path is greater " + "than format field count %d", part_id, + fieldno + TUPLE_INDEX_BASE, + space->format->field_count); + diag_set(IllegalParams, err_msg); + return luaT_error(L); + } else if (node.type == JSON_PATH_STR && + tuple_fieldno_by_name(space->format->dict, node.str, node.len, + field_name_hash(node.str, node.len), + &fieldno) != 0) { + const char *err_msg = + tt_sprintf("Illegal parameters, options.parts[%d]: " + "field was not found by name '%.*s'", + part_id, node.len, node.str); + diag_set(IllegalParams, err_msg); + return luaT_error(L); + } + fieldno += TUPLE_INDEX_BASE; + + char *path_resolved = region_alloc(&fiber()->gc, path_len + 1); + if (path_resolved == NULL) { + diag_set(OutOfMemory, path_len + 1, "region_alloc", + "path_resolved"); + return luaT_error(L); + } + + path = path_resolved; + path_resolved += + sprintf(path_resolved, "[%d]", fieldno); + memcpy(path_resolved, parser.src + parser.offset, + parser.src_len - parser.offset); + path_resolved[parser.src_len - parser.offset] = '\0'; + + lua_pushnumber(L, fieldno); + lua_pushstring(L, path); + return 2; +} + /* }}} */ void @@ -365,6 +438,7 @@ box_lua_index_init(struct lua_State *L) {"truncate", lbox_truncate}, {"stat", lbox_index_stat}, {"compact", lbox_index_compact}, + {"path_resolve", lbox_index_resolve_path}, {NULL, NULL} }; diff --git a/src/box/lua/schema.lua b/src/box/lua/schema.lua index 540a2a5..bc11375 100644 --- a/src/box/lua/schema.lua +++ b/src/box/lua/schema.lua @@ -556,7 +556,7 @@ local function update_index_parts_1_6_0(parts) return result end -local function update_index_parts(format, parts) +local function update_index_parts(format, parts, space_id) if type(parts) ~= "table" then box.error(box.error.ILLEGAL_PARAMS, "options.parts parameter should be a table") @@ -607,16 +607,14 @@ local function update_index_parts(format, parts) box.error(box.error.ILLEGAL_PARAMS, "options.parts[" .. i .. "]: field (name or number) is expected") elseif type(part.field) == 'string' then - for k,v in pairs(format) do - if v.name == part.field then - part.field = k - break - end - end - if type(part.field) == 'string' then + local idx, path = box.internal.path_resolve(i, space_id, part.field) + if part.path ~= nil and part.path ~= path then box.error(box.error.ILLEGAL_PARAMS, - "options.parts[" .. i .. "]: field was not found by name '" .. part.field .. "'") + "options.parts[" .. i .. "]: field path '"..part.path.." doesn't math path resolved by name '" .. part.field .. "'") end + parts_can_be_simplified = parts_can_be_simplified and path == nil + part.field = idx + part.path = path or part.path elseif part.field == 0 then box.error(box.error.ILLEGAL_PARAMS, "options.parts[" .. i .. "]: field (number) must be one-based") @@ -767,7 +765,7 @@ box.schema.index.create = function(space_id, name, options) end end local parts, parts_can_be_simplified = - update_index_parts(format, options.parts) + update_index_parts(format, options.parts, space_id) -- create_index() options contains type, parts, etc, -- stored separately. Remove these members from index_opts local index_opts = { @@ -934,7 +932,7 @@ box.schema.index.alter = function(space_id, index_id, options) if options.parts then local parts_can_be_simplified parts, parts_can_be_simplified = - update_index_parts(format, options.parts) + update_index_parts(format, options.parts, space_id) -- save parts in old format if possible if parts_can_be_simplified then parts = simplify_index_parts(parts) diff --git a/test/engine/iterator.result b/test/engine/iterator.result index 10097ed..05d892d 100644 --- a/test/engine/iterator.result +++ b/test/engine/iterator.result @@ -4213,7 +4213,7 @@ s:replace{35} ... state, value = gen(param,state) --- -- error: 'builtin/box/schema.lua:1034: usage: next(param, state)' +- error: 'builtin/box/schema.lua:1032: usage: next(param, state)' ... value --- diff --git a/test/engine/tuple.result b/test/engine/tuple.result index 8a0c0bc..e778050 100644 --- a/test/engine/tuple.result +++ b/test/engine/tuple.result @@ -946,6 +946,43 @@ assert(idx ~= nil) --- - true ... +format = {{'int1', 'unsigned'}, {'int2', 'unsigned'}, {'data', 'array'}, {'int3', 'unsigned'}, {'int4', 'unsigned'}} +--- +... +s:format(format) +--- +- error: Field 3 has type 'map' in one index, but type 'array' in another +... +format = {{'int1', 'unsigned'}, {'int2', 'unsigned'}, {'data', 'map'}, {'int3', 'unsigned'}, {'int4', 'unsigned'}} +--- +... +s:format(format) +--- +... +s:create_index('test3', {parts = {{2, 'number'}, {']sad.FIO["fname"]', 'str'}}}) +--- +- error: 'Illegal parameters, options.parts[2]: error in path on position 1' +... +s:create_index('test3', {parts = {{2, 'number'}, {'[666].FIO["fname"]', 'str'}}}) +--- +- error: 'Illegal parameters, options.parts[2]: field ''666'' referenced in path is + greater than format field count 5' +... +s:create_index('test3', {parts = {{2, 'number'}, {'invalid.FIO["fname"]', 'str'}}}) +--- +- error: 'Illegal parameters, options.parts[2]: field was not found by name ''invalid''' +... +idx3 = s:create_index('test3', {parts = {{2, 'number'}, {'data.FIO["fname"]', 'str'}}}) +--- +... +assert(idx3 ~= nil) +--- +- true +... +assert(idx3.parts[2].path == "[3][\"FIO\"][\"fname\"]") +--- +- true +... s:insert{7, 7, {town = 'London', FIO = 666}, 4, 5} --- - error: 'Tuple doesn''t math document structure: invalid field 3 document content diff --git a/test/engine/tuple.test.lua b/test/engine/tuple.test.lua index 3b4ce26..56d24e5 100644 --- a/test/engine/tuple.test.lua +++ b/test/engine/tuple.test.lua @@ -304,6 +304,16 @@ s:create_index('test1', {parts = {{2, 'number'}, {3, 'str', path = '[2].FIO.fnam s:create_index('test1', {parts = {{2, 'number'}, {3, 'str', path = '[3].FIO....fname'}}}) idx = s:create_index('test1', {parts = {{2, 'number'}, {3, 'str', path = '[3]["FIO"]["fname"]'}, {3, 'str', path = '[3]["FIO"]["sname"]'}}}) assert(idx ~= nil) +format = {{'int1', 'unsigned'}, {'int2', 'unsigned'}, {'data', 'array'}, {'int3', 'unsigned'}, {'int4', 'unsigned'}} +s:format(format) +format = {{'int1', 'unsigned'}, {'int2', 'unsigned'}, {'data', 'map'}, {'int3', 'unsigned'}, {'int4', 'unsigned'}} +s:format(format) +s:create_index('test3', {parts = {{2, 'number'}, {']sad.FIO["fname"]', 'str'}}}) +s:create_index('test3', {parts = {{2, 'number'}, {'[666].FIO["fname"]', 'str'}}}) +s:create_index('test3', {parts = {{2, 'number'}, {'invalid.FIO["fname"]', 'str'}}}) +idx3 = s:create_index('test3', {parts = {{2, 'number'}, {'data.FIO["fname"]', 'str'}}}) +assert(idx3 ~= nil) +assert(idx3.parts[2].path == "[3][\"FIO\"][\"fname\"]") s:insert{7, 7, {town = 'London', FIO = 666}, 4, 5} s:insert{7, 7, {town = 'London', FIO = {fname = 666, sname = 'Bond'}}, 4, 5} s:insert{7, 7, {town = 'London', FIO = {fname = "James"}}, 4, 5} -- 2.7.4