From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from localhost (localhost [127.0.0.1]) by turing.freelists.org (Avenir Technologies Mail Multiplex) with ESMTP id 1DDAA299D6 for ; Thu, 29 Mar 2018 10:22:21 -0400 (EDT) Received: from turing.freelists.org ([127.0.0.1]) by localhost (turing.freelists.org [127.0.0.1]) (amavisd-new, port 10024) with ESMTP id htXjTYccKpcT for ; Thu, 29 Mar 2018 10:22:21 -0400 (EDT) Received: from smtpng3.m.smailru.net (smtpng3.m.smailru.net [94.100.177.149]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by turing.freelists.org (Avenir Technologies Mail Multiplex) with ESMTPS id A7C30299CD for ; Thu, 29 Mar 2018 10:22:20 -0400 (EDT) From: Kirill Shcherbatov Subject: [tarantool-patches] [PATCH v2 2/3] lua: implement json path access to tuple fields Date: Thu, 29 Mar 2018 17:22:03 +0300 Message-Id: <92b7caf27491cef803bb6004f1616176a15d96c7.1522333265.git.kshcherbatov@tarantool.org> In-Reply-To: References: In-Reply-To: References: Sender: tarantool-patches-bounce@freelists.org Errors-to: tarantool-patches-bounce@freelists.org Reply-To: tarantool-patches@freelists.org List-help: List-unsubscribe: List-software: Ecartis version 1.0.0 List-Id: tarantool-patches List-subscribe: List-owner: List-post: List-archive: To: tarantool-patches@freelists.org Cc: Vladislav Shpilevoy From: Vladislav Shpilevoy In progress ... Closes #1285 --- src/box/CMakeLists.txt | 2 +- src/box/lua/tuple.c | 176 +++++++++++++++++++++++++++++++++++----- src/box/lua/tuple.lua | 45 +++-------- test/engine/tuple.result | 198 +++++++++++++++++++++++++++++++++++++++++++++ test/engine/tuple.test.lua | 59 ++++++++++++++ 5 files changed, 428 insertions(+), 52 deletions(-) diff --git a/src/box/CMakeLists.txt b/src/box/CMakeLists.txt index e420fe3..add0ff9 100644 --- a/src/box/CMakeLists.txt +++ b/src/box/CMakeLists.txt @@ -130,5 +130,5 @@ add_library(box STATIC ${bin_sources}) target_link_libraries(box box_error tuple stat xrow xlog vclock crc32 scramble - ${common_libraries}) + json_path ${common_libraries}) add_dependencies(box build_bundled_libs) diff --git a/src/box/lua/tuple.c b/src/box/lua/tuple.c index 7ca4299..99b9ff2 100644 --- a/src/box/lua/tuple.c +++ b/src/box/lua/tuple.c @@ -41,6 +41,7 @@ #include "box/tuple_convert.h" #include "box/errcode.h" #include "box/memtx_tuple.h" +#include "json/path.h" /** {{{ box.tuple Lua library * @@ -402,36 +403,175 @@ lbox_tuple_transform(struct lua_State *L) } /** - * Find a tuple field using its name. + * Propagate @a field to MessagePack(field)[index]. + * @param[in][out] field Field to propagate. + * @param index 1-based index to propagate to. + * + * @retval 0 Success, the index was found. + * @retval -1 Not found. + */ +static inline int +tuple_field_go_to_index(const char **field, uint64_t index) +{ + assert(index >= 0); + enum mp_type type = mp_typeof(**field); + if (type == MP_ARRAY) { + if (index == 0) + return -1; + /* Make index 0-based. */ + index -= TUPLE_INDEX_BASE; + uint32_t count = mp_decode_array(field); + if (index >= count) + return -1; + for (; index > 0; --index) + mp_next(field); + return 0; + } else if (type == MP_MAP) { + uint64_t count = mp_decode_map(field); + for (; count > 0; --count) { + type = mp_typeof(**field); + if (type == MP_UINT) { + uint64_t value = mp_decode_uint(field); + if (value == index) + return 0; + } else if (type == MP_INT) { + int64_t value = mp_decode_int(field); + if (value >= 0 && (uint64_t)value == index) + return 0; + } else { + /* Skip key. */ + mp_next(field); + } + /* Skip value. */ + mp_next(field); + } + } + return -1; +} + +/** + * Propagate @a field to MessagePack(field)[key]. + * @param[in][out] field Field to propagate. + * @param key Key to propagate to. + * @param len Length of @a key. + * + * @retval 0 Success, the index was found. + * @retval -1 Not found. + */ +static inline int +tuple_field_go_to_key(const char **field, const char *key, int len) +{ + enum mp_type type = mp_typeof(**field); + if (type != MP_MAP) + return -1; + uint64_t count = mp_decode_map(field); + for (; count > 0; --count) { + type = mp_typeof(**field); + if (type == MP_STR) { + uint32_t value_len; + const char *value = mp_decode_str(field, &value_len); + if (value_len == (uint)len && + memcmp(value, key, len) == 0) + return 0; + } else { + /* Skip key. */ + mp_next(field); + } + /* Skip value. */ + mp_next(field); + } + return -1; +} + +/** + * Find a tuple field by JSON path. * @param L Lua state. - * @param tuple 1-th argument on lua stack, tuple to get field + * @param tuple 1-th argument on a lua stack, tuple to get field * from. - * @param field_name 2-th argument on lua stack, field name to - * get. + * @param path 2-th argument on lua stack. Can be field name, + * JSON path to a field or a field number. * * @retval If a field was not found, return -1 and nil to lua else * return 0 and decoded field. */ static int -lbox_tuple_field_by_name(struct lua_State *L) +lbox_tuple_field_by_path(struct lua_State *L) { + const char *field; struct tuple *tuple = luaT_istuple(L, 1); /* Is checked in Lua wrapper. */ assert(tuple != NULL); - assert(lua_isstring(L, 2)); - size_t name_len; - const char *name = lua_tolstring(L, 2, &name_len); - uint32_t name_hash = lua_hashstring(L, 2); - const char *field = - tuple_field_by_name(tuple, name, name_len, name_hash); - if (field == NULL) { - lua_pushinteger(L, -1); - lua_pushnil(L); + if (lua_isnumber(L, 2)) { + int index = lua_tointeger(L, 2); + index -= TUPLE_INDEX_BASE; + if (index < 0) { +not_found: + lua_pushinteger(L, -1); + lua_pushnil(L); + return 2; + } + field = tuple_field(tuple, index); + if (field == NULL) + goto not_found; +push_value: + lua_pushinteger(L, 0); + luamp_decode(L, luaL_msgpack_default, &field); return 2; } - lua_pushinteger(L, 0); - luamp_decode(L, luaL_msgpack_default, &field); - return 2; + assert(lua_isstring(L, 2)); + size_t path_len; + const char *path = lua_tolstring(L, 2, &path_len); + struct json_path_parser parser; + struct json_path_node node; + json_path_parser_create(&parser, path, path_len); + int rc = json_path_next(&parser, &node); + if (rc != 0 || node.type == JSON_PATH_END) + luaL_error(L, "Error in path on position %d", rc); + if (node.type == JSON_PATH_NUM) { + int index = node.num; + if (index == 0) + goto not_found; + index -= TUPLE_INDEX_BASE; + field = tuple_field(tuple, index); + if (field == NULL) + goto not_found; + } else { + assert(node.type == JSON_PATH_STR); + /* First part of a path is a field name. */ + const char *name = node.str; + uint32_t name_len = node.len; + uint32_t name_hash; + if (path_len == name_len) { + name_hash = lua_hashstring(L, 2); + } else { + /* + * If a string is "field....", then its + * precalculated juajit hash can not be + * used. A tuple dictionary hashes only + * name, not path. + */ + name_hash = lua_hash(name, name_len); + } + field = tuple_field_by_name(tuple, name, name_len, name_hash); + if (field == NULL) + goto not_found; + } + while ((rc = json_path_next(&parser, &node)) == 0 && + node.type != JSON_PATH_END) { + if (node.type == JSON_PATH_NUM) { + rc = tuple_field_go_to_index(&field, node.num); + } else { + assert(node.type == JSON_PATH_STR); + rc = tuple_field_go_to_key(&field, node.str, node.len); + } + if (rc != 0) + goto not_found; + } + if (rc == 0) + goto push_value; + luaL_error(L, "Error in path on position %d", rc); + unreachable(); + goto not_found; } static int @@ -470,8 +610,8 @@ static const struct luaL_Reg lbox_tuple_meta[] = { {"tostring", lbox_tuple_to_string}, {"slice", lbox_tuple_slice}, {"transform", lbox_tuple_transform}, - {"tuple_field_by_name", lbox_tuple_field_by_name}, {"tuple_to_map", lbox_tuple_to_map}, + {"tuple_field_by_path", lbox_tuple_field_by_path}, {NULL, NULL} }; diff --git a/src/box/lua/tuple.lua b/src/box/lua/tuple.lua index 001971a..b51b4df 100644 --- a/src/box/lua/tuple.lua +++ b/src/box/lua/tuple.lua @@ -9,16 +9,9 @@ local internal = require('box.internal') ffi.cdef[[ /** \cond public */ -typedef struct tuple_format box_tuple_format_t; - -box_tuple_format_t * -box_tuple_format_default(void); typedef struct tuple box_tuple_t; -box_tuple_t * -box_tuple_new(box_tuple_format_t *format, const char *data, const char *end); - int box_tuple_ref(box_tuple_t *tuple); @@ -34,9 +27,6 @@ box_tuple_bsize(const box_tuple_t *tuple); ssize_t box_tuple_to_buf(const box_tuple_t *tuple, char *buf, size_t size); -box_tuple_format_t * -box_tuple_format(const box_tuple_t *tuple); - const char * box_tuple_field(const box_tuple_t *tuple, uint32_t i); @@ -278,9 +268,9 @@ end msgpackffi.on_encode(const_tuple_ref_t, tuple_to_msgpack) -local function tuple_field_by_name(tuple, name) +local function tuple_field_by_path(tuple, path) tuple_check(tuple, "tuple['field_name']"); - return internal.tuple.tuple_field_by_name(tuple, name) + return internal.tuple.tuple_field_by_path(tuple, path) end local methods = { @@ -306,33 +296,22 @@ end methods["__serialize"] = tuple_totable -- encode hook for msgpack/yaml/json -local tuple_field = function(tuple, field_n) - local field = builtin.box_tuple_field(tuple, field_n - 1) - if field == nil then - return nil - end - -- Use () to shrink stack to the first return value - return (msgpackffi.decode_unchecked(field)) -end - - ffi.metatype(tuple_t, { __len = function(tuple) return builtin.box_tuple_field_count(tuple) end; __tostring = internal.tuple.tostring; __index = function(tuple, key) - if type(key) == "number" then - return tuple_field(tuple, key) - elseif type(key) == "string" then - -- Try to get a field with a name = key. If it was not - -- found (rc ~= 0) then return a method from the - -- vtable. If a collision occurred, then fields have - -- higher priority. For example, if a tuple T has a - -- field with name 'bsize', then T.bsize returns field - -- value, not tuple_bsize function. To access hidden - -- methods use 'box.tuple.(T, [args...])'. - local rc, field = tuple_field_by_name(tuple, key) + if type(key) == "string" or type(key) == "number" then + -- Try to get a field by json path or by [index]. If + -- it was not found (rc ~= 0) then return a method + -- from the vtable. If a collision occurred, then + -- fields have higher priority. For example, if a + -- tuple T has a field with name 'bsize', then T.bsize + -- returns field value, not tuple_bsize function. To + -- access hidden methods use + -- 'box.tuple.(T, [args...])'. + local rc, field = tuple_field_by_path(tuple, key) if rc == 0 then return field end diff --git a/test/engine/tuple.result b/test/engine/tuple.result index b3b23b2..2d7367a 100644 --- a/test/engine/tuple.result +++ b/test/engine/tuple.result @@ -590,6 +590,204 @@ maplen(t1map), t1map[1], t1map[2], t1map[3] s:drop() --- ... +format = {} +--- +... +format[1] = {name = 'field1', type = 'unsigned'} +--- +... +format[2] = {name = 'field2', type = 'array'} +--- +... +format[3] = {name = 'field3', type = 'map'} +--- +... +format[4] = {name = 'field4', type = 'string'} +--- +... +s = box.schema.space.create('test', {format = format}) +--- +... +pk = s:create_index('pk') +--- +... +field2 = {1, 2, 3, "4", {5,6,7}, {key="key1", value="value1"}} +--- +... +field3 = {[10] = 100, k1 = 100, k2 = {1,2,3}, k3 = { {a=1, b=2}, {c=3, d=4} }, [-1] = 200} +--- +... +t = s:replace{1, field2, field3, "123456"} +--- +... +t[1] +--- +- 1 +... +t[2] +--- +- [1, 2, 3, '4', [5, 6, 7], {'key': 'key1', 'value': 'value1'}] +... +t[3] +--- +- {'k1': 100, 'k3': [{'a': 1, 'b': 2}, {'c': 3, 'd': 4}], -1: 200, 10: 100, 'k2': [ + 1, 2, 3]} +... +t[4] +--- +- '123456' +... +t[2][1] +--- +- 1 +... +t["[2][1]"] +--- +- 1 +... +t[2][5] +--- +- [5, 6, 7] +... +t["[2][5]"] +--- +- [5, 6, 7] +... +t["[2][5][1]"] +--- +- 5 +... +t["[2][5][2]"] +--- +- 6 +... +t["[2][5][3]"] +--- +- 7 +... +t["[2][6].key"] +--- +- key1 +... +t["[2][6].value"] +--- +- value1 +... +t["[2][6]['key']"] +--- +- key1 +... +t["[2][6]['value']"] +--- +- value1 +... +t["[3].k3[2].c"] +--- +- 3 +... +t["[4]"] +--- +- '123456' +... +t.field1 +--- +- 1 +... +t.field2[5] +--- +- [5, 6, 7] +... +t[".field1"] +--- +- 1 +... +t["field1"] +--- +- 1 +... +t["[3][10]"] +--- +- 100 +... +-- Not found. +t[0] +--- +- null +... +t["[0]"] +--- +- null +... +t["[1000]"] +--- +- null +... +t.field1000 +--- +- null +... +t["not_found"] +--- +- null +... +t["[2][5][10]"] +--- +- null +... +t["[2][6].key100"] +--- +- null +... +t["[2][0]"] -- 0-based index in array. +--- +- null +... +t["[4][3]"] -- Can not index string. +--- +- null +... +t["[4]['key']"] +--- +- null +... +-- Not found 'a'. Return 'null' despite of syntax error on a +-- next position. +t["a.b.c d.e.f"] +--- +- null +... +-- Sytax errors. +t[""] +--- +- error: 'builtin/box/tuple.lua:314: Error in path on position 0' +... +t["[2].[5]"] +--- +- error: 'builtin/box/tuple.lua:314: Error in path on position 5' +... +t["[-1]"] +--- +- error: 'builtin/box/tuple.lua:314: Error in path on position 2' +... +t[".."] +--- +- error: 'builtin/box/tuple.lua:314: Error in path on position 2' +... +t["[["] +--- +- error: 'builtin/box/tuple.lua:314: Error in path on position 2' +... +t["]]"] +--- +- error: 'builtin/box/tuple.lua:314: Error in path on position 1' +... +t["{"] +--- +- error: 'builtin/box/tuple.lua:314: Error in path on position 1' +... +s:drop() +--- +... engine = nil --- ... diff --git a/test/engine/tuple.test.lua b/test/engine/tuple.test.lua index 6d7d254..ba3482d 100644 --- a/test/engine/tuple.test.lua +++ b/test/engine/tuple.test.lua @@ -200,5 +200,64 @@ t1map = t1:tomap() maplen(t1map), t1map[1], t1map[2], t1map[3] s:drop() +format = {} +format[1] = {name = 'field1', type = 'unsigned'} +format[2] = {name = 'field2', type = 'array'} +format[3] = {name = 'field3', type = 'map'} +format[4] = {name = 'field4', type = 'string'} +s = box.schema.space.create('test', {format = format}) +pk = s:create_index('pk') +field2 = {1, 2, 3, "4", {5,6,7}, {key="key1", value="value1"}} +field3 = {[10] = 100, k1 = 100, k2 = {1,2,3}, k3 = { {a=1, b=2}, {c=3, d=4} }, [-1] = 200} +t = s:replace{1, field2, field3, "123456"} +t[1] +t[2] +t[3] +t[4] +t[2][1] +t["[2][1]"] +t[2][5] +t["[2][5]"] +t["[2][5][1]"] +t["[2][5][2]"] +t["[2][5][3]"] +t["[2][6].key"] +t["[2][6].value"] +t["[2][6]['key']"] +t["[2][6]['value']"] +t["[3].k3[2].c"] +t["[4]"] +t.field1 +t.field2[5] +t[".field1"] +t["field1"] +t["[3][10]"] + +-- Not found. +t[0] +t["[0]"] +t["[1000]"] +t.field1000 +t["not_found"] +t["[2][5][10]"] +t["[2][6].key100"] +t["[2][0]"] -- 0-based index in array. +t["[4][3]"] -- Can not index string. +t["[4]['key']"] +-- Not found 'a'. Return 'null' despite of syntax error on a +-- next position. +t["a.b.c d.e.f"] + +-- Sytax errors. +t[""] +t["[2].[5]"] +t["[-1]"] +t[".."] +t["[["] +t["]]"] +t["{"] + +s:drop() + engine = nil test_run = nil -- 2.7.4