* [PATCH v2 0/6] Merger
@ 2019-01-09 20:20 Alexander Turenko
2019-01-09 20:20 ` [PATCH v2 1/6] Add luaL_iscallable with support of cdata metatype Alexander Turenko
` (5 more replies)
0 siblings, 6 replies; 28+ messages in thread
From: Alexander Turenko @ 2019-01-09 20:20 UTC (permalink / raw)
To: Vladimir Davydov; +Cc: Alexander Turenko, tarantool-patches
There was made many changes since the previous iteration. Consider the
email in the mailing thread re the first version, where I answer to the
review comments. In short (non-exhaustive list of changes):
* move to box/;
* renames structures;
* change API: merger.pairs(ctx, ...);
* remove decode/encode, added net.box/msgpack helpers;
* removed say_debug() debug prints;
* replaced switched around different sources with virtual functions;
* deduplicated the code was copied from lbox_tuple_new();
* moved key_def creation to box/lua/key_def.[ch];
https://github.com/tarantool/tarantool/issues/3276
https://github.com/tarantool/tarantool/tree/Totktonada/gh-3276-on-board-merger
Alexander Turenko (6):
Add luaL_iscallable with support of cdata metatype
Add functions to ease using Lua iterators from C
lua: add luaT_newtuple()
lua: add luaT_new_key_def()
net.box: add helpers to decode msgpack headers
Add merger for tuple streams
extra/exports | 2 +
src/CMakeLists.txt | 1 +
src/box/CMakeLists.txt | 2 +
src/box/lua/init.c | 3 +
src/box/lua/key_def.c | 217 +++++
src/box/lua/key_def.h | 61 ++
src/box/lua/merger.c | 1402 ++++++++++++++++++++++++++++++
src/box/lua/merger.h | 47 +
src/box/lua/net_box.c | 49 ++
src/box/lua/net_box.lua | 1 +
src/box/lua/tuple.c | 91 +-
src/box/lua/tuple.h | 15 +
src/lua/msgpack.c | 66 ++
src/lua/utils.c | 109 +++
src/lua/utils.h | 38 +
test/app-tap/module_api.c | 23 +
test/app-tap/module_api.test.lua | 172 +++-
test/app-tap/msgpack.test.lua | 157 +++-
test/box-tap/merger.test.lua | 558 ++++++++++++
test/box-tap/suite.ini | 1 +
test/box/net.box.result | 58 ++
test/box/net.box.test.lua | 26 +
22 files changed, 3072 insertions(+), 27 deletions(-)
create mode 100644 src/box/lua/key_def.c
create mode 100644 src/box/lua/key_def.h
create mode 100644 src/box/lua/merger.c
create mode 100644 src/box/lua/merger.h
create mode 100755 test/box-tap/merger.test.lua
--
2.20.1
^ permalink raw reply [flat|nested] 28+ messages in thread* [PATCH v2 1/6] Add luaL_iscallable with support of cdata metatype 2019-01-09 20:20 [PATCH v2 0/6] Merger Alexander Turenko @ 2019-01-09 20:20 ` Alexander Turenko 2019-01-10 12:21 ` Vladimir Davydov 2019-01-09 20:20 ` [PATCH v2 2/6] Add functions to ease using Lua iterators from C Alexander Turenko ` (4 subsequent siblings) 5 siblings, 1 reply; 28+ messages in thread From: Alexander Turenko @ 2019-01-09 20:20 UTC (permalink / raw) To: Vladimir Davydov; +Cc: Alexander Turenko, tarantool-patches Needed for #3276. --- extra/exports | 1 + src/lua/utils.c | 43 ++++++++++++++++ src/lua/utils.h | 10 ++++ test/app-tap/module_api.c | 10 ++++ test/app-tap/module_api.test.lua | 85 +++++++++++++++++++++++++++++++- 5 files changed, 147 insertions(+), 2 deletions(-) diff --git a/extra/exports b/extra/exports index 5f69e0730..af6863963 100644 --- a/extra/exports +++ b/extra/exports @@ -134,6 +134,7 @@ luaT_call luaT_cpcall luaT_state luaT_tolstring +luaL_iscallable box_txn box_txn_begin box_txn_commit diff --git a/src/lua/utils.c b/src/lua/utils.c index 978fe61f1..eefb860ee 100644 --- a/src/lua/utils.c +++ b/src/lua/utils.c @@ -920,6 +920,49 @@ luaT_tolstring(lua_State *L, int idx, size_t *len) return lua_tolstring(L, -1, len); } +/* Based on ffi_meta___call() from luajit/src/lib_ffi.c. */ +static int +luaL_cdata_iscallable(lua_State *L, int idx) +{ + /* Calculate absolute value in the stack. */ + if (idx < 0) + idx = lua_gettop(L) + idx + 1; + + /* Get cdata from the stack. */ + assert(lua_type(L, idx) == LUA_TCDATA); + GCcdata *cd = cdataV(L->base + idx - 1); + + CTState *cts = ctype_cts(L); + CTypeID id = cd->ctypeid; + CType *ct = ctype_raw(cts, id); + if (ctype_isptr(ct->info)) + id = ctype_cid(ct->info); + + /* Get ctype metamethod. */ + cTValue *tv = lj_ctype_meta(cts, id, MM_call); + + return tv != NULL; +} + +int +luaL_iscallable(lua_State *L, int idx) +{ + /* Whether it is function. */ + int res = lua_isfunction(L, idx); + if (res == 1) + return 1; + + /* Whether it is cdata with metatype with __call field. */ + if (lua_type(L, idx) == LUA_TCDATA) + return luaL_cdata_iscallable(L, idx); + + /* Whether it has metatable with __call field. */ + res = luaL_getmetafield(L, idx, "__call"); + if (res == 1) + lua_pop(L, 1); /* Pop __call value. */ + return res; +} + lua_State * luaT_state(void) { diff --git a/src/lua/utils.h b/src/lua/utils.h index a47e3d2b4..bd302d8e9 100644 --- a/src/lua/utils.h +++ b/src/lua/utils.h @@ -438,6 +438,16 @@ luaT_state(void); LUA_API const char * luaT_tolstring(lua_State *L, int idx, size_t *ssize); +/** + * Check whether a Lua object is a function or has + * metatable/metatype with a __call field. + * + * Note: It does not check type of __call metatable/metatype + * field. + */ +LUA_API int +luaL_iscallable(lua_State *L, int idx); + /** \endcond public */ /** diff --git a/test/app-tap/module_api.c b/test/app-tap/module_api.c index 4abe1af48..b81a98056 100644 --- a/test/app-tap/module_api.c +++ b/test/app-tap/module_api.c @@ -440,6 +440,15 @@ test_tostring(lua_State *L) return 1; } +static int +test_iscallable(lua_State *L) +{ + int exp = lua_toboolean(L, 2); + int res = luaL_iscallable(L, 1); + lua_pushboolean(L, res == exp); + return 1; +} + LUA_API int luaopen_module_api(lua_State *L) { @@ -467,6 +476,7 @@ luaopen_module_api(lua_State *L) {"test_cpcall", test_cpcall}, {"test_state", test_state}, {"test_tostring", test_tostring}, + {"iscallable", test_iscallable}, {NULL, NULL} }; luaL_register(L, "module_api", lib); diff --git a/test/app-tap/module_api.test.lua b/test/app-tap/module_api.test.lua index f93257236..a6658cc61 100755 --- a/test/app-tap/module_api.test.lua +++ b/test/app-tap/module_api.test.lua @@ -3,7 +3,9 @@ local fio = require('fio') box.cfg{log = "tarantool.log"} -build_path = os.getenv("BUILDDIR") +-- Use BUILDDIR passed from test-run or cwd when run w/o +-- test-run to find test/app-tap/module_api.{so,dylib}. +build_path = os.getenv("BUILDDIR") or '.' package.cpath = fio.pathjoin(build_path, 'test/app-tap/?.so' ) .. ';' .. fio.pathjoin(build_path, 'test/app-tap/?.dylib') .. ';' .. package.cpath @@ -36,8 +38,86 @@ local function test_pushcdata(test, module) test:is(gc_counter, 1, 'pushcdata gc') end +local function test_iscallable(test, module) + local ffi = require('ffi') + + ffi.cdef([[ + struct cdata_1 { int foo; }; + struct cdata_2 { int foo; }; + ]]) + + local cdata_1 = ffi.new('struct cdata_1') + local cdata_1_ref = ffi.new('struct cdata_1 &') + local cdata_2 = ffi.new('struct cdata_2') + local cdata_2_ref = ffi.new('struct cdata_2 &') + + local nop = function() end + + ffi.metatype('struct cdata_2', { + __call = nop, + }) + + local cases = { + { + obj = nop, + exp = true, + description = 'function', + }, + { + obj = nil, + exp = false, + description = 'nil', + }, + { + obj = 1, + exp = false, + description = 'number', + }, + { + obj = {}, + exp = false, + description = 'table without metatable', + }, + { + obj = setmetatable({}, {}), + exp = false, + description = 'table without __call metatable field', + }, + { + obj = setmetatable({}, {__call = nop}), + exp = true, + description = 'table with __call metatable field' + }, + { + obj = cdata_1, + exp = false, + description = 'cdata without __call metatable field', + }, + { + obj = cdata_1_ref, + exp = false, + description = 'cdata reference without __call metatable field', + }, + { + obj = cdata_2, + exp = true, + description = 'cdata with __call metatable field', + }, + { + obj = cdata_2_ref, + exp = true, + description = 'cdata reference with __call metatable field', + }, + } + + test:plan(#cases) + for _, case in ipairs(cases) do + test:ok(module.iscallable(case.obj, case.exp), case.description) + end +end + local test = require('tap').test("module_api", function(test) - test:plan(23) + test:plan(24) local status, module = pcall(require, 'module_api') test:is(status, true, "module") test:ok(status, "module is loaded") @@ -62,6 +142,7 @@ local test = require('tap').test("module_api", function(test) test:like(msg, 'luaT_error', 'luaT_error') test:test("pushcdata", test_pushcdata, module) + test:test("iscallable", test_iscallable, module) space:drop() end) -- 2.20.1 ^ permalink raw reply [flat|nested] 28+ messages in thread
* Re: [PATCH v2 1/6] Add luaL_iscallable with support of cdata metatype 2019-01-09 20:20 ` [PATCH v2 1/6] Add luaL_iscallable with support of cdata metatype Alexander Turenko @ 2019-01-10 12:21 ` Vladimir Davydov 0 siblings, 0 replies; 28+ messages in thread From: Vladimir Davydov @ 2019-01-10 12:21 UTC (permalink / raw) To: Alexander Turenko; +Cc: tarantool-patches On Wed, Jan 09, 2019 at 11:20:09PM +0300, Alexander Turenko wrote: > Needed for #3276. > --- > extra/exports | 1 + > src/lua/utils.c | 43 ++++++++++++++++ > src/lua/utils.h | 10 ++++ > test/app-tap/module_api.c | 10 ++++ > test/app-tap/module_api.test.lua | 85 +++++++++++++++++++++++++++++++- > 5 files changed, 147 insertions(+), 2 deletions(-) This patch looks OK to me, but I'm not sure if you'll need it after you rework the merger API. So I'm not applying it now. ^ permalink raw reply [flat|nested] 28+ messages in thread
* [PATCH v2 2/6] Add functions to ease using Lua iterators from C 2019-01-09 20:20 [PATCH v2 0/6] Merger Alexander Turenko 2019-01-09 20:20 ` [PATCH v2 1/6] Add luaL_iscallable with support of cdata metatype Alexander Turenko @ 2019-01-09 20:20 ` Alexander Turenko 2019-01-10 12:29 ` Vladimir Davydov 2019-01-09 20:20 ` [PATCH v2 3/6] lua: add luaT_newtuple() Alexander Turenko ` (3 subsequent siblings) 5 siblings, 1 reply; 28+ messages in thread From: Alexander Turenko @ 2019-01-09 20:20 UTC (permalink / raw) To: Vladimir Davydov; +Cc: Alexander Turenko, tarantool-patches Needed for #3276. --- src/lua/utils.c | 66 +++++++++++++++++++++++++++++++++++++++++++++++++ src/lua/utils.h | 28 +++++++++++++++++++++ 2 files changed, 94 insertions(+) diff --git a/src/lua/utils.c b/src/lua/utils.c index eefb860ee..4d1eee6ab 100644 --- a/src/lua/utils.c +++ b/src/lua/utils.c @@ -969,6 +969,72 @@ luaT_state(void) return tarantool_L; } +/* {{{ Helper functions to interact with a Lua iterator from C */ + +struct luaL_iterator { + int gen; + int param; + int state; +}; + +struct luaL_iterator * +luaL_iterator_new_fromtable(lua_State *L, int idx) +{ + struct luaL_iterator *it = (struct luaL_iterator *) malloc( + sizeof(struct luaL_iterator)); + + lua_rawgeti(L, idx, 1); /* Popped by luaL_ref(). */ + it->gen = luaL_ref(L, LUA_REGISTRYINDEX); + lua_rawgeti(L, idx, 2); /* Popped by luaL_ref(). */ + it->param = luaL_ref(L, LUA_REGISTRYINDEX); + lua_rawgeti(L, idx, 3); /* Popped by luaL_ref(). */ + it->state = luaL_ref(L, LUA_REGISTRYINDEX); + + return it; +} + +int +luaL_iterator_next(lua_State *L, struct luaL_iterator *it) +{ + int frame_start = lua_gettop(L); + + /* Call gen(param, state). */ + lua_rawgeti(L, LUA_REGISTRYINDEX, it->gen); + lua_rawgeti(L, LUA_REGISTRYINDEX, it->param); + lua_rawgeti(L, LUA_REGISTRYINDEX, it->state); + lua_call(L, 2, LUA_MULTRET); + int nresults = lua_gettop(L) - frame_start; + if (nresults == 0) { + luaL_error(L, "luaL_iterator_next: gen(param, state) must " + "return at least one result"); + unreachable(); + return 0; + } + + /* The call above returns nil as the first result. */ + if (lua_isnil(L, frame_start + 1)) { + lua_settop(L, frame_start); + return 0; + } + + /* Save the first result to it->state. */ + luaL_unref(L, LUA_REGISTRYINDEX, it->state); + lua_pushvalue(L, frame_start + 1); /* Popped by luaL_ref(). */ + it->state = luaL_ref(L, LUA_REGISTRYINDEX); + + return nresults; +} + +void luaL_iterator_free(lua_State *L, struct luaL_iterator *it) +{ + luaL_unref(L, LUA_REGISTRYINDEX, it->gen); + luaL_unref(L, LUA_REGISTRYINDEX, it->param); + luaL_unref(L, LUA_REGISTRYINDEX, it->state); + free(it); +} + +/* }}} */ + int tarantool_lua_utils_init(struct lua_State *L) { diff --git a/src/lua/utils.h b/src/lua/utils.h index bd302d8e9..6ba2e4767 100644 --- a/src/lua/utils.h +++ b/src/lua/utils.h @@ -525,6 +525,34 @@ luaL_checkfinite(struct lua_State *L, struct luaL_serializer *cfg, luaL_error(L, "number must not be NaN or Inf"); } +/* {{{ Helper functions to interact with a Lua iterator from C */ + +/** + * Holds iterator state (references to Lua objects). + */ +struct luaL_iterator; + +/** + * Create a Lua iterator from {gen, param, state}. + */ +struct luaL_iterator * +luaL_iterator_new_fromtable(lua_State *L, int idx); + +/** + * Move iterator to the next value. Push values returned by + * gen(param, state) and return its count. Zero means no more + * results available. + */ +int +luaL_iterator_next(lua_State *L, struct luaL_iterator *it); + +/** + * Free all resources held by the iterator. + */ +void luaL_iterator_free(lua_State *L, struct luaL_iterator *it); + +/* }}} */ + int tarantool_lua_utils_init(struct lua_State *L); -- 2.20.1 ^ permalink raw reply [flat|nested] 28+ messages in thread
* Re: [PATCH v2 2/6] Add functions to ease using Lua iterators from C 2019-01-09 20:20 ` [PATCH v2 2/6] Add functions to ease using Lua iterators from C Alexander Turenko @ 2019-01-10 12:29 ` Vladimir Davydov 2019-01-15 23:26 ` Alexander Turenko 0 siblings, 1 reply; 28+ messages in thread From: Vladimir Davydov @ 2019-01-10 12:29 UTC (permalink / raw) To: Alexander Turenko; +Cc: tarantool-patches On Wed, Jan 09, 2019 at 11:20:10PM +0300, Alexander Turenko wrote: > Needed for #3276. Again, I'm not quite sure that you'll need this patch after you rework the merger API so I'm not applying it until you send the new API proposal. > --- > src/lua/utils.c | 66 +++++++++++++++++++++++++++++++++++++++++++++++++ > src/lua/utils.h | 28 +++++++++++++++++++++ Some tests would be nice to have. > 2 files changed, 94 insertions(+) > > diff --git a/src/lua/utils.c b/src/lua/utils.c > index eefb860ee..4d1eee6ab 100644 > --- a/src/lua/utils.c > +++ b/src/lua/utils.c > @@ -969,6 +969,72 @@ luaT_state(void) > return tarantool_L; > } > > +/* {{{ Helper functions to interact with a Lua iterator from C */ > + > +struct luaL_iterator { > + int gen; > + int param; > + int state; > +}; > + > +struct luaL_iterator * > +luaL_iterator_new_fromtable(lua_State *L, int idx) > +{ > + struct luaL_iterator *it = (struct luaL_iterator *) malloc( Nit: no need to convert void * to struct luaL_iterator *. > + sizeof(struct luaL_iterator)); > + > + lua_rawgeti(L, idx, 1); /* Popped by luaL_ref(). */ > + it->gen = luaL_ref(L, LUA_REGISTRYINDEX); > + lua_rawgeti(L, idx, 2); /* Popped by luaL_ref(). */ > + it->param = luaL_ref(L, LUA_REGISTRYINDEX); > + lua_rawgeti(L, idx, 3); /* Popped by luaL_ref(). */ > + it->state = luaL_ref(L, LUA_REGISTRYINDEX); > + > + return it; > +} > + > +int > +luaL_iterator_next(lua_State *L, struct luaL_iterator *it) > +{ > + int frame_start = lua_gettop(L); > + > + /* Call gen(param, state). */ > + lua_rawgeti(L, LUA_REGISTRYINDEX, it->gen); > + lua_rawgeti(L, LUA_REGISTRYINDEX, it->param); > + lua_rawgeti(L, LUA_REGISTRYINDEX, it->state); > + lua_call(L, 2, LUA_MULTRET); > + int nresults = lua_gettop(L) - frame_start; > + if (nresults == 0) { > + luaL_error(L, "luaL_iterator_next: gen(param, state) must " > + "return at least one result"); > + unreachable(); > + return 0; > + } > + > + /* The call above returns nil as the first result. */ > + if (lua_isnil(L, frame_start + 1)) { > + lua_settop(L, frame_start); > + return 0; > + } > + > + /* Save the first result to it->state. */ > + luaL_unref(L, LUA_REGISTRYINDEX, it->state); > + lua_pushvalue(L, frame_start + 1); /* Popped by luaL_ref(). */ > + it->state = luaL_ref(L, LUA_REGISTRYINDEX); > + > + return nresults; > +} > + > +void luaL_iterator_free(lua_State *L, struct luaL_iterator *it) > +{ > + luaL_unref(L, LUA_REGISTRYINDEX, it->gen); > + luaL_unref(L, LUA_REGISTRYINDEX, it->param); > + luaL_unref(L, LUA_REGISTRYINDEX, it->state); > + free(it); > +} > + > +/* }}} */ > + > int > tarantool_lua_utils_init(struct lua_State *L) > { > diff --git a/src/lua/utils.h b/src/lua/utils.h > index bd302d8e9..6ba2e4767 100644 > --- a/src/lua/utils.h > +++ b/src/lua/utils.h > @@ -525,6 +525,34 @@ luaL_checkfinite(struct lua_State *L, struct luaL_serializer *cfg, > luaL_error(L, "number must not be NaN or Inf"); > } > > +/* {{{ Helper functions to interact with a Lua iterator from C */ > + > +/** > + * Holds iterator state (references to Lua objects). > + */ > +struct luaL_iterator; I'd make luaL_iterator struct transparent so that one could define it on stack. > + > +/** > + * Create a Lua iterator from {gen, param, state}. May be, we could pass idx == 0 to create an iterator from gen, param, state (without a table)? Would it be worthwhile? > + */ > +struct luaL_iterator * > +luaL_iterator_new_fromtable(lua_State *L, int idx); I don't think that _fromtable suffix is really necessary. > + > +/** > + * Move iterator to the next value. Push values returned by > + * gen(param, state) and return its count. Zero means no more > + * results available. > + */ > +int > +luaL_iterator_next(lua_State *L, struct luaL_iterator *it); > + > +/** > + * Free all resources held by the iterator. > + */ > +void luaL_iterator_free(lua_State *L, struct luaL_iterator *it); We usually match _new with _delete. > + > +/* }}} */ > + > int > tarantool_lua_utils_init(struct lua_State *L); ^ permalink raw reply [flat|nested] 28+ messages in thread
* Re: [PATCH v2 2/6] Add functions to ease using Lua iterators from C 2019-01-10 12:29 ` Vladimir Davydov @ 2019-01-15 23:26 ` Alexander Turenko 2019-01-16 8:18 ` Vladimir Davydov 0 siblings, 1 reply; 28+ messages in thread From: Alexander Turenko @ 2019-01-15 23:26 UTC (permalink / raw) To: Vladimir Davydov; +Cc: tarantool-patches Thanks for the review! I commented inline and fixed all comments except one, where I doubt. The patch from the previous version at end of the email. List of changes: * Fixed zero gen() retvals case. * Added a unit test. * Removed unneded cast from (void *). * Allow to pass values w/o a table. - Also removed _fromtable suffix from luaL_iterator_new(). * luaL_iterator_free() -> luaL_iterator_delete(). WBR, Alexander Turenko. On Thu, Jan 10, 2019 at 03:29:09PM +0300, Vladimir Davydov wrote: > On Wed, Jan 09, 2019 at 11:20:10PM +0300, Alexander Turenko wrote: > > Needed for #3276. > > Again, I'm not quite sure that you'll need this patch after you > rework the merger API so I'm not applying it until you send the > new API proposal. If we'll support iterator sources those helpers are convenient. All APIs we discussing now have them. IMHO, it is possible, but unlikely we'll decide to get rid of them. > > > --- > > src/lua/utils.c | 66 +++++++++++++++++++++++++++++++++++++++++++++++++ > > src/lua/utils.h | 28 +++++++++++++++++++++ > > Some tests would be nice to have. I have added test/unit/luaL_iterator.c. I have to link many parts (*.a libraries) of tarantool to it and system dynamic libraries (dependencies of *.a), but I think size does not matter much here. This test found one error! An iterator can return zero count of values instead of returning nil and ipairs() behaves in that way. I fixed the implementation. > > > 2 files changed, 94 insertions(+) > > > > diff --git a/src/lua/utils.c b/src/lua/utils.c > > index eefb860ee..4d1eee6ab 100644 > > --- a/src/lua/utils.c > > +++ b/src/lua/utils.c > > @@ -969,6 +969,72 @@ luaT_state(void) > > return tarantool_L; > > } > > > > +/* {{{ Helper functions to interact with a Lua iterator from C */ > > + > > +struct luaL_iterator { > > + int gen; > > + int param; > > + int state; > > +}; > > + > > +struct luaL_iterator * > > +luaL_iterator_new_fromtable(lua_State *L, int idx) > > +{ > > + struct luaL_iterator *it = (struct luaL_iterator *) malloc( > > Nit: no need to convert void * to struct luaL_iterator *. Yep. Fixed. > > > + sizeof(struct luaL_iterator)); > > + > > + lua_rawgeti(L, idx, 1); /* Popped by luaL_ref(). */ > > + it->gen = luaL_ref(L, LUA_REGISTRYINDEX); > > + lua_rawgeti(L, idx, 2); /* Popped by luaL_ref(). */ > > + it->param = luaL_ref(L, LUA_REGISTRYINDEX); > > + lua_rawgeti(L, idx, 3); /* Popped by luaL_ref(). */ > > + it->state = luaL_ref(L, LUA_REGISTRYINDEX); > > + > > + return it; > > +} > > + > > +int > > +luaL_iterator_next(lua_State *L, struct luaL_iterator *it) > > +{ > > + int frame_start = lua_gettop(L); > > + > > + /* Call gen(param, state). */ > > + lua_rawgeti(L, LUA_REGISTRYINDEX, it->gen); > > + lua_rawgeti(L, LUA_REGISTRYINDEX, it->param); > > + lua_rawgeti(L, LUA_REGISTRYINDEX, it->state); > > + lua_call(L, 2, LUA_MULTRET); > > + int nresults = lua_gettop(L) - frame_start; > > + if (nresults == 0) { > > + luaL_error(L, "luaL_iterator_next: gen(param, state) must " > > + "return at least one result"); > > + unreachable(); > > + return 0; > > + } > > + > > + /* The call above returns nil as the first result. */ > > + if (lua_isnil(L, frame_start + 1)) { > > + lua_settop(L, frame_start); > > + return 0; > > + } > > + > > + /* Save the first result to it->state. */ > > + luaL_unref(L, LUA_REGISTRYINDEX, it->state); > > + lua_pushvalue(L, frame_start + 1); /* Popped by luaL_ref(). */ > > + it->state = luaL_ref(L, LUA_REGISTRYINDEX); > > + > > + return nresults; > > +} > > + > > +void luaL_iterator_free(lua_State *L, struct luaL_iterator *it) > > +{ > > + luaL_unref(L, LUA_REGISTRYINDEX, it->gen); > > + luaL_unref(L, LUA_REGISTRYINDEX, it->param); > > + luaL_unref(L, LUA_REGISTRYINDEX, it->state); > > + free(it); > > +} > > + > > +/* }}} */ > > + > > int > > tarantool_lua_utils_init(struct lua_State *L) > > { > > diff --git a/src/lua/utils.h b/src/lua/utils.h > > index bd302d8e9..6ba2e4767 100644 > > --- a/src/lua/utils.h > > +++ b/src/lua/utils.h > > @@ -525,6 +525,34 @@ luaL_checkfinite(struct lua_State *L, struct luaL_serializer *cfg, > > luaL_error(L, "number must not be NaN or Inf"); > > } > > > > +/* {{{ Helper functions to interact with a Lua iterator from C */ > > + > > +/** > > + * Holds iterator state (references to Lua objects). > > + */ > > +struct luaL_iterator; > > I'd make luaL_iterator struct transparent so that one could define it > on stack. > But luaL_iterator_new() do malloc and return a pointer. So we need to change the function or add another one to initialize a structure allocated outsize of the module. Can you please suggest me how the API should look if we'll make the structure transparent? I left it unchanged until we'll discuss this aspect. > > + > > +/** > > + * Create a Lua iterator from {gen, param, state}. > > May be, we could pass idx == 0 to create an iterator from > gen, param, state (without a table)? Would it be worthwhile? > I think it is good idea, because cases could be different. My thought was that we'll add another function for this case (if we'll need), but using idx == 0 is better. I created the similar API before for luaT_newtuple(). And I think the new merger API will require it. Added. > > + */ > > +struct luaL_iterator * > > +luaL_iterator_new_fromtable(lua_State *L, int idx); > > I don't think that _fromtable suffix is really necessary. > In light of the idx == 0 support, yep. Fixed. > > + > > +/** > > + * Move iterator to the next value. Push values returned by > > + * gen(param, state) and return its count. Zero means no more > > + * results available. > > + */ > > +int > > +luaL_iterator_next(lua_State *L, struct luaL_iterator *it); > > + > > +/** > > + * Free all resources held by the iterator. > > + */ > > +void luaL_iterator_free(lua_State *L, struct luaL_iterator *it); > > We usually match _new with _delete. Fixed. > > > + > > +/* }}} */ > > + > > int > > tarantool_lua_utils_init(struct lua_State *L); ---- diff --git a/src/lua/utils.c b/src/lua/utils.c index 4d1eee6ab..173d59a59 100644 --- a/src/lua/utils.c +++ b/src/lua/utils.c @@ -978,17 +978,30 @@ struct luaL_iterator { }; struct luaL_iterator * -luaL_iterator_new_fromtable(lua_State *L, int idx) +luaL_iterator_new(lua_State *L, int idx) { - struct luaL_iterator *it = (struct luaL_iterator *) malloc( - sizeof(struct luaL_iterator)); - - lua_rawgeti(L, idx, 1); /* Popped by luaL_ref(). */ - it->gen = luaL_ref(L, LUA_REGISTRYINDEX); - lua_rawgeti(L, idx, 2); /* Popped by luaL_ref(). */ - it->param = luaL_ref(L, LUA_REGISTRYINDEX); - lua_rawgeti(L, idx, 3); /* Popped by luaL_ref(). */ - it->state = luaL_ref(L, LUA_REGISTRYINDEX); + struct luaL_iterator *it = malloc(sizeof(struct luaL_iterator)); + + if (idx == 0) { + /* gen, param, state are on top of a Lua stack. */ + lua_pushvalue(L, -3); /* Popped by luaL_ref(). */ + it->gen = luaL_ref(L, LUA_REGISTRYINDEX); + lua_pushvalue(L, -2); /* Popped by luaL_ref(). */ + it->param = luaL_ref(L, LUA_REGISTRYINDEX); + lua_pushvalue(L, -1); /* Popped by luaL_ref(). */ + it->state = luaL_ref(L, LUA_REGISTRYINDEX); + } else { + /* + * {gen, param, state} table is at idx in a Lua + * stack. + */ + lua_rawgeti(L, idx, 1); /* Popped by luaL_ref(). */ + it->gen = luaL_ref(L, LUA_REGISTRYINDEX); + lua_rawgeti(L, idx, 2); /* Popped by luaL_ref(). */ + it->param = luaL_ref(L, LUA_REGISTRYINDEX); + lua_rawgeti(L, idx, 3); /* Popped by luaL_ref(). */ + it->state = luaL_ref(L, LUA_REGISTRYINDEX); + } return it; } @@ -1004,15 +1017,15 @@ luaL_iterator_next(lua_State *L, struct luaL_iterator *it) lua_rawgeti(L, LUA_REGISTRYINDEX, it->state); lua_call(L, 2, LUA_MULTRET); int nresults = lua_gettop(L) - frame_start; - if (nresults == 0) { - luaL_error(L, "luaL_iterator_next: gen(param, state) must " - "return at least one result"); - unreachable(); - return 0; - } - /* The call above returns nil as the first result. */ - if (lua_isnil(L, frame_start + 1)) { + /* + * gen() function can either return nil when the iterator + * ends or return zero count of values. + * + * In LuaJIT pairs() returns nil, but ipairs() returns + * nothing when ends. + */ + if (nresults == 0 || lua_isnil(L, frame_start + 1)) { lua_settop(L, frame_start); return 0; } @@ -1025,7 +1038,7 @@ luaL_iterator_next(lua_State *L, struct luaL_iterator *it) return nresults; } -void luaL_iterator_free(lua_State *L, struct luaL_iterator *it) +void luaL_iterator_delete(lua_State *L, struct luaL_iterator *it) { luaL_unref(L, LUA_REGISTRYINDEX, it->gen); luaL_unref(L, LUA_REGISTRYINDEX, it->param); diff --git a/src/lua/utils.h b/src/lua/utils.h index 6ba2e4767..2df2f5015 100644 --- a/src/lua/utils.h +++ b/src/lua/utils.h @@ -533,10 +533,16 @@ luaL_checkfinite(struct lua_State *L, struct luaL_serializer *cfg, struct luaL_iterator; /** - * Create a Lua iterator from {gen, param, state}. + * Create a Lua iterator from a gen, param, state triplet. + * + * If idx == 0, then three top stack values are used as the + * triplet. + * + * Otherwise idx is index on Lua stack points to a + * {gen, param, state} table. */ struct luaL_iterator * -luaL_iterator_new_fromtable(lua_State *L, int idx); +luaL_iterator_new(lua_State *L, int idx); /** * Move iterator to the next value. Push values returned by @@ -549,7 +555,7 @@ luaL_iterator_next(lua_State *L, struct luaL_iterator *it); /** * Free all resources held by the iterator. */ -void luaL_iterator_free(lua_State *L, struct luaL_iterator *it); +void luaL_iterator_delete(lua_State *L, struct luaL_iterator *it); /* }}} */ diff --git a/test/unit/CMakeLists.txt b/test/unit/CMakeLists.txt index 0025d3611..c2c45a4b8 100644 --- a/test/unit/CMakeLists.txt +++ b/test/unit/CMakeLists.txt @@ -138,6 +138,10 @@ add_executable(histogram.test histogram.c) target_link_libraries(histogram.test stat unit) add_executable(ratelimit.test ratelimit.c) target_link_libraries(ratelimit.test unit) +add_executable(luaL_iterator.test luaL_iterator.c) +target_link_libraries(luaL_iterator.test unit server core misc + ${CURL_LIBRARIES} ${LIBYAML_LIBRARIES} ${READLINE_LIBRARIES} + ${ICU_LIBRARIES} ${LUAJIT_LIBRARIES}) add_executable(say.test say.c) target_link_libraries(say.test core unit) diff --git a/test/unit/luaL_iterator.c b/test/unit/luaL_iterator.c new file mode 100644 index 000000000..5a254f27d --- /dev/null +++ b/test/unit/luaL_iterator.c @@ -0,0 +1,159 @@ +#include <lua.h> /* lua_*() */ +#include <lauxlib.h> /* luaL_*() */ +#include <lualib.h> /* luaL_openlibs() */ +#include "unit.h" /* plan, header, footer, is */ +#include "lua/utils.h" /* luaL_iterator_*() */ + +extern char fun_lua[]; + +int +main() +{ + struct { + /* A string to output with a test case. */ + const char *description; + /* A string with Lua code to push an iterator. */ + const char *init; + /* + * How much values are pushed by the Lua code + * above. + */ + int init_retvals; + /* + * Start values from this number to distinguish + * them from its ordinal. + */ + int first_value; + /* + * Lua stack index where {gen, param, state} is + * placed or zero. + */ + int idx; + /* How much values are in the iterator. */ + int iterations; + } cases[] = { + { + .description = "pairs, zero idx", + .init = "return pairs({42})", + .init_retvals = 3, + .first_value = 42, + .idx = 0, + .iterations = 1, + }, + { + .description = "ipairs, zero idx", + .init = "return ipairs({42, 43, 44})", + .init_retvals = 3, + .first_value = 42, + .idx = 0, + .iterations = 3, + }, + { + .description = "luafun iterator, zero idx", + .init = "return fun.wrap(ipairs({42, 43, 44}))", + .init_retvals = 3, + .first_value = 42, + .idx = 0, + .iterations = 3, + }, + { + .description = "pairs, from table", + .init = "return {pairs({42})}", + .init_retvals = 1, + .first_value = 42, + .idx = -1, + .iterations = 1, + }, + { + .description = "ipairs, from table", + .init = "return {ipairs({42, 43, 44})}", + .init_retvals = 1, + .first_value = 42, + .idx = -1, + .iterations = 3, + }, + { + .description = "luafun iterator, from table", + .init = "return {fun.wrap(ipairs({42, 43, 44}))}", + .init_retvals = 1, + .first_value = 42, + .idx = -1, + .iterations = 3, + }, + }; + + int cases_cnt = (int) (sizeof(cases) / sizeof(cases[0])); + /* + * * Check stack size after creating luaL_iterator (triple + * times). + * * 4 checks per iteration. + * * Check that values ends. + */ + int planned = 0; + for (int i = 0; i < cases_cnt; ++i) + planned += cases[i].iterations * 4 + 4; + + plan(planned); + header(); + + struct lua_State *L = luaL_newstate(); + luaL_openlibs(L); + + /* + * Expose luafun. + * + * Don't register it in package.loaded for simplicity. + */ + luaL_loadstring(L, fun_lua); + lua_call(L, 0, 1); + lua_setglobal(L, "fun"); + + for (int i = 0; i < cases_cnt; ++i) { + const char *description = cases[i].description; + int top = lua_gettop(L); + + /* Push an iterator to the Lua stack. */ + luaL_loadstring(L, cases[i].init); + lua_call(L, 0, cases[i].init_retvals); + + /* Create the luaL_iterator structure. */ + struct luaL_iterator *it = luaL_iterator_new(L, cases[i].idx); + lua_pop(L, cases[i].init_retvals); + + /* Check stack size. */ + is(lua_gettop(L) - top, 0, "%s: stack size", description); + + /* Iterate over values and check them. */ + for (int j = 0; j < cases[i].iterations; ++j) { + int top = lua_gettop(L); + int rc = luaL_iterator_next(L, it); + is(rc, 2, "%s: iter %d: gen() retval count", + description, j); + is(luaL_checkinteger(L, -2), j + 1, + "%s: iter %d: gen() 1st retval", + description, j); + is(luaL_checkinteger(L, -1), j + cases[i].first_value, + "%s: iter %d: gen() 2nd retval", + description, j); + lua_pop(L, 2); + is(lua_gettop(L) - top, 0, "%s: iter: %d: stack size", + description, j); + } + + /* Check the iterator ends when expected. */ + int rc = luaL_iterator_next(L, it); + is(rc, 0, "%s: iterator ends", description); + + /* Check stack size. */ + is(lua_gettop(L) - top, 0, "%s: stack size", description); + + /* Free the luaL_iterator structure. */ + luaL_iterator_delete(L, it); + + /* Check stack size. */ + is(lua_gettop(L) - top, 0, "%s: stack size", description); + } + + footer(); + return check_plan(); +} diff --git a/test/unit/luaL_iterator.result b/test/unit/luaL_iterator.result new file mode 100644 index 000000000..f4eda5695 --- /dev/null +++ b/test/unit/luaL_iterator.result @@ -0,0 +1,83 @@ +1..80 + *** main *** +ok 1 - pairs, zero idx: stack size +ok 2 - pairs, zero idx: iter 0: gen() retval count +ok 3 - pairs, zero idx: iter 0: gen() 1st retval +ok 4 - pairs, zero idx: iter 0: gen() 2nd retval +ok 5 - pairs, zero idx: iter: 0: stack size +ok 6 - pairs, zero idx: iterator ends +ok 7 - pairs, zero idx: stack size +ok 8 - pairs, zero idx: stack size +ok 9 - ipairs, zero idx: stack size +ok 10 - ipairs, zero idx: iter 0: gen() retval count +ok 11 - ipairs, zero idx: iter 0: gen() 1st retval +ok 12 - ipairs, zero idx: iter 0: gen() 2nd retval +ok 13 - ipairs, zero idx: iter: 0: stack size +ok 14 - ipairs, zero idx: iter 1: gen() retval count +ok 15 - ipairs, zero idx: iter 1: gen() 1st retval +ok 16 - ipairs, zero idx: iter 1: gen() 2nd retval +ok 17 - ipairs, zero idx: iter: 1: stack size +ok 18 - ipairs, zero idx: iter 2: gen() retval count +ok 19 - ipairs, zero idx: iter 2: gen() 1st retval +ok 20 - ipairs, zero idx: iter 2: gen() 2nd retval +ok 21 - ipairs, zero idx: iter: 2: stack size +ok 22 - ipairs, zero idx: iterator ends +ok 23 - ipairs, zero idx: stack size +ok 24 - ipairs, zero idx: stack size +ok 25 - luafun iterator, zero idx: stack size +ok 26 - luafun iterator, zero idx: iter 0: gen() retval count +ok 27 - luafun iterator, zero idx: iter 0: gen() 1st retval +ok 28 - luafun iterator, zero idx: iter 0: gen() 2nd retval +ok 29 - luafun iterator, zero idx: iter: 0: stack size +ok 30 - luafun iterator, zero idx: iter 1: gen() retval count +ok 31 - luafun iterator, zero idx: iter 1: gen() 1st retval +ok 32 - luafun iterator, zero idx: iter 1: gen() 2nd retval +ok 33 - luafun iterator, zero idx: iter: 1: stack size +ok 34 - luafun iterator, zero idx: iter 2: gen() retval count +ok 35 - luafun iterator, zero idx: iter 2: gen() 1st retval +ok 36 - luafun iterator, zero idx: iter 2: gen() 2nd retval +ok 37 - luafun iterator, zero idx: iter: 2: stack size +ok 38 - luafun iterator, zero idx: iterator ends +ok 39 - luafun iterator, zero idx: stack size +ok 40 - luafun iterator, zero idx: stack size +ok 41 - pairs, from table: stack size +ok 42 - pairs, from table: iter 0: gen() retval count +ok 43 - pairs, from table: iter 0: gen() 1st retval +ok 44 - pairs, from table: iter 0: gen() 2nd retval +ok 45 - pairs, from table: iter: 0: stack size +ok 46 - pairs, from table: iterator ends +ok 47 - pairs, from table: stack size +ok 48 - pairs, from table: stack size +ok 49 - ipairs, from table: stack size +ok 50 - ipairs, from table: iter 0: gen() retval count +ok 51 - ipairs, from table: iter 0: gen() 1st retval +ok 52 - ipairs, from table: iter 0: gen() 2nd retval +ok 53 - ipairs, from table: iter: 0: stack size +ok 54 - ipairs, from table: iter 1: gen() retval count +ok 55 - ipairs, from table: iter 1: gen() 1st retval +ok 56 - ipairs, from table: iter 1: gen() 2nd retval +ok 57 - ipairs, from table: iter: 1: stack size +ok 58 - ipairs, from table: iter 2: gen() retval count +ok 59 - ipairs, from table: iter 2: gen() 1st retval +ok 60 - ipairs, from table: iter 2: gen() 2nd retval +ok 61 - ipairs, from table: iter: 2: stack size +ok 62 - ipairs, from table: iterator ends +ok 63 - ipairs, from table: stack size +ok 64 - ipairs, from table: stack size +ok 65 - luafun iterator, from table: stack size +ok 66 - luafun iterator, from table: iter 0: gen() retval count +ok 67 - luafun iterator, from table: iter 0: gen() 1st retval +ok 68 - luafun iterator, from table: iter 0: gen() 2nd retval +ok 69 - luafun iterator, from table: iter: 0: stack size +ok 70 - luafun iterator, from table: iter 1: gen() retval count +ok 71 - luafun iterator, from table: iter 1: gen() 1st retval +ok 72 - luafun iterator, from table: iter 1: gen() 2nd retval +ok 73 - luafun iterator, from table: iter: 1: stack size +ok 74 - luafun iterator, from table: iter 2: gen() retval count +ok 75 - luafun iterator, from table: iter 2: gen() 1st retval +ok 76 - luafun iterator, from table: iter 2: gen() 2nd retval +ok 77 - luafun iterator, from table: iter: 2: stack size +ok 78 - luafun iterator, from table: iterator ends +ok 79 - luafun iterator, from table: stack size +ok 80 - luafun iterator, from table: stack size + *** main: done *** ^ permalink raw reply [flat|nested] 28+ messages in thread
* Re: [PATCH v2 2/6] Add functions to ease using Lua iterators from C 2019-01-15 23:26 ` Alexander Turenko @ 2019-01-16 8:18 ` Vladimir Davydov 2019-01-16 11:40 ` Alexander Turenko 2019-01-28 18:17 ` Alexander Turenko 0 siblings, 2 replies; 28+ messages in thread From: Vladimir Davydov @ 2019-01-16 8:18 UTC (permalink / raw) To: Alexander Turenko; +Cc: tarantool-patches On Wed, Jan 16, 2019 at 02:26:23AM +0300, Alexander Turenko wrote: > > > diff --git a/src/lua/utils.h b/src/lua/utils.h > > > index bd302d8e9..6ba2e4767 100644 > > > --- a/src/lua/utils.h > > > +++ b/src/lua/utils.h > > > @@ -525,6 +525,34 @@ luaL_checkfinite(struct lua_State *L, struct luaL_serializer *cfg, > > > luaL_error(L, "number must not be NaN or Inf"); > > > } > > > > > > +/* {{{ Helper functions to interact with a Lua iterator from C */ > > > + > > > +/** > > > + * Holds iterator state (references to Lua objects). > > > + */ > > > +struct luaL_iterator; > > > > I'd make luaL_iterator struct transparent so that one could define it > > on stack. > > > > But luaL_iterator_new() do malloc and return a pointer. So we need to > change the function or add another one to initialize a structure > allocated outsize of the module. OK, I see it now. Didn't think about it when first looked at the patch. > > Can you please suggest me how the API should look if we'll make the > structure transparent? > > I left it unchanged until we'll discuss this aspect. > > > > + > > > +/** > > > + * Create a Lua iterator from {gen, param, state}. > > > > May be, we could pass idx == 0 to create an iterator from > > gen, param, state (without a table)? Would it be worthwhile? > > > > I think it is good idea, because cases could be different. My thought > was that we'll add another function for this case (if we'll need), but > using idx == 0 is better. I created the similar API before for > luaT_newtuple(). > > And I think the new merger API will require it. The patch looks good to me now, but I'm still not sure the new merger implementation will need to deal with Lua iterators in C at all, as one can easily write a wrapper function in Lua turning an iterator to a 'fetch' closure, which then can be passed to a source constructor. ^ permalink raw reply [flat|nested] 28+ messages in thread
* Re: [PATCH v2 2/6] Add functions to ease using Lua iterators from C 2019-01-16 8:18 ` Vladimir Davydov @ 2019-01-16 11:40 ` Alexander Turenko 2019-01-16 12:20 ` Vladimir Davydov 2019-01-28 18:17 ` Alexander Turenko 1 sibling, 1 reply; 28+ messages in thread From: Alexander Turenko @ 2019-01-16 11:40 UTC (permalink / raw) To: Vladimir Davydov; +Cc: tarantool-patches > > > > + > > > > +/** > > > > + * Create a Lua iterator from {gen, param, state}. > > > > > > May be, we could pass idx == 0 to create an iterator from > > > gen, param, state (without a table)? Would it be worthwhile? > > > > > > > I think it is good idea, because cases could be different. My thought > > was that we'll add another function for this case (if we'll need), but > > using idx == 0 is better. I created the similar API before for > > luaT_newtuple(). > > > > And I think the new merger API will require it. > > The patch looks good to me now, but I'm still not sure the new merger > implementation will need to deal with Lua iterators in C at all, as one > can easily write a wrapper function in Lua turning an iterator to a > 'fetch' closure, which then can be passed to a source constructor. This is not the thread about merger, but the idea looks weird for me. 'next' has one meaning (get one tuple), 'fetch' has another meaning (get next tuples batch). 'next' is written in C and predefined, 'fetch' is user-defined Lua function. When you'll try to express one over another you'll find rough edges, e. g. need of extra wrapping table. But my main objection is that it is hard to think what is going on when 'next' and 'fetch' are mixed. WBR, Alexander Turenko. ^ permalink raw reply [flat|nested] 28+ messages in thread
* Re: [PATCH v2 2/6] Add functions to ease using Lua iterators from C 2019-01-16 11:40 ` Alexander Turenko @ 2019-01-16 12:20 ` Vladimir Davydov 2019-01-17 1:20 ` Alexander Turenko 0 siblings, 1 reply; 28+ messages in thread From: Vladimir Davydov @ 2019-01-16 12:20 UTC (permalink / raw) To: Alexander Turenko; +Cc: tarantool-patches On Wed, Jan 16, 2019 at 02:40:12PM +0300, Alexander Turenko wrote: > > > > > + > > > > > +/** > > > > > + * Create a Lua iterator from {gen, param, state}. > > > > > > > > May be, we could pass idx == 0 to create an iterator from > > > > gen, param, state (without a table)? Would it be worthwhile? > > > > > > > > > > I think it is good idea, because cases could be different. My thought > > > was that we'll add another function for this case (if we'll need), but > > > using idx == 0 is better. I created the similar API before for > > > luaT_newtuple(). > > > > > > And I think the new merger API will require it. > > > > The patch looks good to me now, but I'm still not sure the new merger > > implementation will need to deal with Lua iterators in C at all, as one > > can easily write a wrapper function in Lua turning an iterator to a > > 'fetch' closure, which then can be passed to a source constructor. > > This is not the thread about merger, but the idea looks weird for me. > 'next' has one meaning (get one tuple), 'fetch' has another meaning (get > next tuples batch). 'next' is written in C and predefined, 'fetch' is > user-defined Lua function. When you'll try to express one over another > you'll find rough edges, e. g. need of extra wrapping table. Yep, why not wrap an iterator/table/whatever so that it works as fetch() closure taken by one of available source constructors. > > But my main objection is that it is hard to think what is going on > when 'next' and 'fetch' are mixed. I never mixed those. 'next' would be a C function returning C tuples. This would be a method of a source. It would be used by a merger to get the next tuple to merge. It would also be used by source:pairs() Lua function. 'fetch' would be a Lua function returning either Lua tuples or tuple batches (represented by Lua tables or raw msgpack). What exactly it returns depends on the source type. ^ permalink raw reply [flat|nested] 28+ messages in thread
* Re: [PATCH v2 2/6] Add functions to ease using Lua iterators from C 2019-01-16 12:20 ` Vladimir Davydov @ 2019-01-17 1:20 ` Alexander Turenko 0 siblings, 0 replies; 28+ messages in thread From: Alexander Turenko @ 2019-01-17 1:20 UTC (permalink / raw) To: Vladimir Davydov; +Cc: tarantool-patches On Wed, Jan 16, 2019 at 03:20:38PM +0300, Vladimir Davydov wrote: > On Wed, Jan 16, 2019 at 02:40:12PM +0300, Alexander Turenko wrote: > > > > > > + > > > > > > +/** > > > > > > + * Create a Lua iterator from {gen, param, state}. > > > > > > > > > > May be, we could pass idx == 0 to create an iterator from > > > > > gen, param, state (without a table)? Would it be worthwhile? > > > > > > > > > > > > > I think it is good idea, because cases could be different. My thought > > > > was that we'll add another function for this case (if we'll need), but > > > > using idx == 0 is better. I created the similar API before for > > > > luaT_newtuple(). > > > > > > > > And I think the new merger API will require it. > > > > > > The patch looks good to me now, but I'm still not sure the new merger > > > implementation will need to deal with Lua iterators in C at all, as one > > > can easily write a wrapper function in Lua turning an iterator to a > > > 'fetch' closure, which then can be passed to a source constructor. > > > > This is not the thread about merger, but the idea looks weird for me. > > 'next' has one meaning (get one tuple), 'fetch' has another meaning (get > > next tuples batch). 'next' is written in C and predefined, 'fetch' is > > user-defined Lua function. When you'll try to express one over another > > you'll find rough edges, e. g. need of extra wrapping table. > > Yep, why not wrap an iterator/table/whatever so that it works as fetch() > closure taken by one of available source constructors. > > > > > But my main objection is that it is hard to think what is going on > > when 'next' and 'fetch' are mixed. > > I never mixed those. > > 'next' would be a C function returning C tuples. This would be a method > of a source. It would be used by a merger to get the next tuple to > merge. It would also be used by source:pairs() Lua function. > > 'fetch' would be a Lua function returning either Lua tuples or tuple > batches (represented by Lua tables or raw msgpack). What exactly it > returns depends on the source type. Answered in the '[#3276] Merger API RFC' thread. WBR, Alexander Turenko. ^ permalink raw reply [flat|nested] 28+ messages in thread
* Re: [PATCH v2 2/6] Add functions to ease using Lua iterators from C 2019-01-16 8:18 ` Vladimir Davydov 2019-01-16 11:40 ` Alexander Turenko @ 2019-01-28 18:17 ` Alexander Turenko 2019-03-01 4:04 ` Alexander Turenko 1 sibling, 1 reply; 28+ messages in thread From: Alexander Turenko @ 2019-01-28 18:17 UTC (permalink / raw) To: Vladimir Davydov; +Cc: tarantool-patches Updated a bit: diff --git a/src/lua/utils.c b/src/lua/utils.c index 173d59a59..c8cfa70b3 100644 --- a/src/lua/utils.c +++ b/src/lua/utils.c @@ -981,6 +981,11 @@ struct luaL_iterator * luaL_iterator_new(lua_State *L, int idx) { struct luaL_iterator *it = malloc(sizeof(struct luaL_iterator)); + if (it == NULL) { + diag_set(OutOfMemory, sizeof(struct luaL_iterator), + "malloc", "luaL_iterator"); + return NULL; + } if (idx == 0) { /* gen, param, state are on top of a Lua stack. */ ^ permalink raw reply [flat|nested] 28+ messages in thread
* Re: [PATCH v2 2/6] Add functions to ease using Lua iterators from C 2019-01-28 18:17 ` Alexander Turenko @ 2019-03-01 4:04 ` Alexander Turenko 0 siblings, 0 replies; 28+ messages in thread From: Alexander Turenko @ 2019-03-01 4:04 UTC (permalink / raw) To: Vladimir Davydov; +Cc: tarantool-patches Made several minor updates. I'll squash them into the original commit, but show here separatelly to ease reading. WBR, Alexander Turenko. On Mon, Jan 28, 2019 at 09:17:16PM +0300, Alexander Turenko wrote: > Updated a bit: > > diff --git a/src/lua/utils.c b/src/lua/utils.c > index 173d59a59..c8cfa70b3 100644 > --- a/src/lua/utils.c > +++ b/src/lua/utils.c > @@ -981,6 +981,11 @@ struct luaL_iterator * > luaL_iterator_new(lua_State *L, int idx) > { > struct luaL_iterator *it = malloc(sizeof(struct luaL_iterator)); > + if (it == NULL) { > + diag_set(OutOfMemory, sizeof(struct luaL_iterator), > + "malloc", "luaL_iterator"); > + return NULL; > + } > > if (idx == 0) { > /* gen, param, state are on top of a Lua stack. */ commit c7cf235b5ba68e911a1fd47e950fe205e963f9e7 Author: Alexander Turenko <alexander.turenko@tarantool.org> Date: Thu Feb 21 14:06:39 2019 +0300 FIXUP: luaL_iterator: use global Lua state for unreferencing diff --git a/src/lua/utils.c b/src/lua/utils.c index 002c5e4b9..2e5096626 100644 --- a/src/lua/utils.c +++ b/src/lua/utils.c @@ -1052,11 +1052,11 @@ luaL_iterator_next(lua_State *L, struct luaL_iterator *it) return nresults; } -void luaL_iterator_delete(lua_State *L, struct luaL_iterator *it) +void luaL_iterator_delete(struct luaL_iterator *it) { - luaL_unref(L, LUA_REGISTRYINDEX, it->gen); - luaL_unref(L, LUA_REGISTRYINDEX, it->param); - luaL_unref(L, LUA_REGISTRYINDEX, it->state); + luaL_unref(tarantool_L, LUA_REGISTRYINDEX, it->gen); + luaL_unref(tarantool_L, LUA_REGISTRYINDEX, it->param); + luaL_unref(tarantool_L, LUA_REGISTRYINDEX, it->state); free(it); } diff --git a/src/lua/utils.h b/src/lua/utils.h index 772b5d877..c66a8a4b9 100644 --- a/src/lua/utils.h +++ b/src/lua/utils.h @@ -562,7 +562,7 @@ luaL_iterator_next(lua_State *L, struct luaL_iterator *it); /** * Free all resources held by the iterator. */ -void luaL_iterator_delete(lua_State *L, struct luaL_iterator *it); +void luaL_iterator_delete(struct luaL_iterator *it); /* }}} */ diff --git a/test/unit/luaL_iterator.c b/test/unit/luaL_iterator.c index 5a254f27d..038d34c5c 100644 --- a/test/unit/luaL_iterator.c +++ b/test/unit/luaL_iterator.c @@ -98,6 +98,13 @@ main() struct lua_State *L = luaL_newstate(); luaL_openlibs(L); + tarantool_L = L; + + /* + * Check that everything works fine in a thread (a fiber) + * other then the main one. + */ + L = lua_newthread(L); /* * Expose luafun. @@ -148,7 +155,7 @@ main() is(lua_gettop(L) - top, 0, "%s: stack size", description); /* Free the luaL_iterator structure. */ - luaL_iterator_delete(L, it); + luaL_iterator_delete(it); /* Check stack size. */ is(lua_gettop(L) - top, 0, "%s: stack size", description); commit 1318e30c89efbb58de46fb5ed167cd61e3ff661a Author: Alexander Turenko <alexander.turenko@tarantool.org> Date: Mon Feb 25 01:05:01 2019 +0300 FIXUP: luaL_iterator: tweak a comment diff --git a/src/lua/utils.h b/src/lua/utils.h index c66a8a4b9..c4a16704f 100644 --- a/src/lua/utils.h +++ b/src/lua/utils.h @@ -543,7 +543,7 @@ struct luaL_iterator; * Create a Lua iterator from a gen, param, state triplet. * * If idx == 0, then three top stack values are used as the - * triplet. + * triplet. Note: they are not popped. * * Otherwise idx is index on Lua stack points to a * {gen, param, state} table. commit f95e07b340f4b698c2d6aa0ff6db437cc388ab85 Author: Alexander Turenko <alexander.turenko@tarantool.org> Date: Mon Feb 25 09:50:34 2019 +0300 FIXUP: luaL_iterator: use pcall diff --git a/src/lua/utils.c b/src/lua/utils.c index 2e5096626..94f8c65d6 100644 --- a/src/lua/utils.c +++ b/src/lua/utils.c @@ -1029,7 +1029,15 @@ luaL_iterator_next(lua_State *L, struct luaL_iterator *it) lua_rawgeti(L, LUA_REGISTRYINDEX, it->gen); lua_rawgeti(L, LUA_REGISTRYINDEX, it->param); lua_rawgeti(L, LUA_REGISTRYINDEX, it->state); - lua_call(L, 2, LUA_MULTRET); + if (luaT_call(L, 2, LUA_MULTRET) != 0) { + /* + * Pop garbage from the call (a gen function + * likely will not leave the stack even when raise + * an error), pop a returned error. + */ + lua_settop(L, frame_start); + return -1; + } int nresults = lua_gettop(L) - frame_start; /* diff --git a/src/lua/utils.h b/src/lua/utils.h index c4a16704f..4a87dac45 100644 --- a/src/lua/utils.h +++ b/src/lua/utils.h @@ -553,8 +553,11 @@ luaL_iterator_new(lua_State *L, int idx); /** * Move iterator to the next value. Push values returned by - * gen(param, state) and return its count. Zero means no more - * results available. + * gen(param, state). + * + * Return count of pushed values. Zero means no more results + * available. In case of a Lua error in a gen function return -1 + * and set a diag. */ int luaL_iterator_next(lua_State *L, struct luaL_iterator *it); diff --git a/test/unit/luaL_iterator.c b/test/unit/luaL_iterator.c index 038d34c5c..8d25a0062 100644 --- a/test/unit/luaL_iterator.c +++ b/test/unit/luaL_iterator.c @@ -2,7 +2,12 @@ #include <lauxlib.h> /* luaL_*() */ #include <lualib.h> /* luaL_openlibs() */ #include "unit.h" /* plan, header, footer, is */ +#include "memory.h" /* memory_init() */ +#include "fiber.h" /* fiber_init() */ +#include "diag.h" /* struct error, diag_*() */ +#include "exception.h" /* type_LuajitError */ #include "lua/utils.h" /* luaL_iterator_*() */ +#include "lua/error.h" /* tarantool_lua_error_init() */ extern char fun_lua[]; @@ -31,6 +36,8 @@ main() int idx; /* How much values are in the iterator. */ int iterations; + /* Expected error (if any). */ + const char *exp_err; } cases[] = { { .description = "pairs, zero idx", @@ -39,6 +46,7 @@ main() .first_value = 42, .idx = 0, .iterations = 1, + .exp_err = NULL, }, { .description = "ipairs, zero idx", @@ -47,6 +55,7 @@ main() .first_value = 42, .idx = 0, .iterations = 3, + .exp_err = NULL, }, { .description = "luafun iterator, zero idx", @@ -55,6 +64,7 @@ main() .first_value = 42, .idx = 0, .iterations = 3, + .exp_err = NULL, }, { .description = "pairs, from table", @@ -63,6 +73,7 @@ main() .first_value = 42, .idx = -1, .iterations = 1, + .exp_err = NULL, }, { .description = "ipairs, from table", @@ -71,6 +82,7 @@ main() .first_value = 42, .idx = -1, .iterations = 3, + .exp_err = NULL, }, { .description = "luafun iterator, from table", @@ -79,19 +91,34 @@ main() .first_value = 42, .idx = -1, .iterations = 3, + .exp_err = NULL, + }, + { + .description = "lua error", + .init = "return error, 'I am the error', 0", + .init_retvals = 3, + .first_value = 0, + .idx = 0, + .iterations = 0, + .exp_err = "I am the error", }, }; int cases_cnt = (int) (sizeof(cases) / sizeof(cases[0])); /* - * * Check stack size after creating luaL_iterator (triple - * times). * * 4 checks per iteration. - * * Check that values ends. + * * 3 checks of a stack size. + * * 1 check that values ends (for success cases). + * * 1 check for an iterator error (for error cases). + * * 1 check for an error type (for error cases). + * * 1 check for an error message (for error cases). */ int planned = 0; - for (int i = 0; i < cases_cnt; ++i) + for (int i = 0; i < cases_cnt; ++i) { planned += cases[i].iterations * 4 + 4; + if (cases[i].exp_err != NULL) + planned += 2; + } plan(planned); header(); @@ -100,6 +127,10 @@ main() luaL_openlibs(L); tarantool_L = L; + memory_init(); + fiber_init(fiber_c_invoke); + tarantool_lua_error_init(L); + /* * Check that everything works fine in a thread (a fiber) * other then the main one. @@ -147,9 +178,20 @@ main() description, j); } - /* Check the iterator ends when expected. */ - int rc = luaL_iterator_next(L, it); - is(rc, 0, "%s: iterator ends", description); + if (cases[i].exp_err == NULL) { + /* Check the iterator ends when expected. */ + int rc = luaL_iterator_next(L, it); + is(rc, 0, "%s: iterator ends", description); + } else { + /* Check expected error. */ + int rc = luaL_iterator_next(L, it); + is(rc, -1, "%s: iterator error", description); + struct error *e = diag_last_error(diag_get()); + is(e->type, &type_LuajitError, "%s: check error type", + description); + ok(!strcmp(e->errmsg, cases[i].exp_err), + "%s: check error message", description); + } /* Check stack size. */ is(lua_gettop(L) - top, 0, "%s: stack size", description); diff --git a/test/unit/luaL_iterator.result b/test/unit/luaL_iterator.result index f4eda5695..2472eedcf 100644 --- a/test/unit/luaL_iterator.result +++ b/test/unit/luaL_iterator.result @@ -1,4 +1,4 @@ -1..80 +1..86 *** main *** ok 1 - pairs, zero idx: stack size ok 2 - pairs, zero idx: iter 0: gen() retval count @@ -80,4 +80,10 @@ ok 77 - luafun iterator, from table: iter: 2: stack size ok 78 - luafun iterator, from table: iterator ends ok 79 - luafun iterator, from table: stack size ok 80 - luafun iterator, from table: stack size +ok 81 - lua error: stack size +ok 82 - lua error: iterator error +ok 83 - lua error: check error type +ok 84 - lua error: check error message +ok 85 - lua error: stack size +ok 86 - lua error: stack size *** main: done *** commit 6edef55a8f689f595a760a9c06180de7da37976e Author: Alexander Turenko <alexander.turenko@tarantool.org> Date: Fri Mar 1 02:27:25 2019 +0300 FIXUP: luaL_iterator: after rebase fix diff --git a/test/unit/CMakeLists.txt b/test/unit/CMakeLists.txt index 263856a7a..663ed2ec2 100644 --- a/test/unit/CMakeLists.txt +++ b/test/unit/CMakeLists.txt @@ -139,9 +139,9 @@ target_link_libraries(histogram.test stat unit) add_executable(ratelimit.test ratelimit.c) target_link_libraries(ratelimit.test unit) add_executable(luaL_iterator.test luaL_iterator.c) -target_link_libraries(luaL_iterator.test unit server core misc +target_link_libraries(luaL_iterator.test unit server coll core misc ${CURL_LIBRARIES} ${LIBYAML_LIBRARIES} ${READLINE_LIBRARIES} - ${ICU_LIBRARIES} ${LUAJIT_LIBRARIES}) + ${ICU_LIBRARIES} ${LUAJIT_LIBRARIES} dl) add_executable(say.test say.c) target_link_libraries(say.test core unit) ^ permalink raw reply [flat|nested] 28+ messages in thread
* [PATCH v2 3/6] lua: add luaT_newtuple() 2019-01-09 20:20 [PATCH v2 0/6] Merger Alexander Turenko 2019-01-09 20:20 ` [PATCH v2 1/6] Add luaL_iscallable with support of cdata metatype Alexander Turenko 2019-01-09 20:20 ` [PATCH v2 2/6] Add functions to ease using Lua iterators from C Alexander Turenko @ 2019-01-09 20:20 ` Alexander Turenko 2019-01-10 12:44 ` Vladimir Davydov 2019-01-09 20:20 ` [PATCH v2 4/6] lua: add luaT_new_key_def() Alexander Turenko ` (2 subsequent siblings) 5 siblings, 1 reply; 28+ messages in thread From: Alexander Turenko @ 2019-01-09 20:20 UTC (permalink / raw) To: Vladimir Davydov; +Cc: Alexander Turenko, tarantool-patches The function allows to create a tuple with specific tuple format in C code using a Lua table, an another tuple or objects on a Lua stack. Needed for #3276. --- src/box/lua/tuple.c | 91 +++++++++++++++++++++++++++++++++------------ src/box/lua/tuple.h | 15 ++++++++ 2 files changed, 83 insertions(+), 23 deletions(-) diff --git a/src/box/lua/tuple.c b/src/box/lua/tuple.c index 1867f810f..7e9ad89fe 100644 --- a/src/box/lua/tuple.c +++ b/src/box/lua/tuple.c @@ -92,6 +92,65 @@ luaT_istuple(struct lua_State *L, int narg) return *(struct tuple **) data; } +struct tuple * +luaT_newtuple(struct lua_State *L, int idx, box_tuple_format_t *format) +{ + struct tuple *tuple; + + if (idx == 0 || lua_istable(L, idx)) { + struct ibuf *buf = tarantool_lua_ibuf; + ibuf_reset(buf); + struct mpstream stream; + mpstream_init(&stream, buf, ibuf_reserve_cb, ibuf_alloc_cb, + luamp_error, L); + if (idx == 0) { + /* + * Create the tuple from lua stack + * objects. + */ + int argc = lua_gettop(L); + mpstream_encode_array(&stream, argc); + for (int k = 1; k <= argc; ++k) { + luamp_encode(L, luaL_msgpack_default, &stream, + k); + } + } else { + /* Create the tuple from a Lua table. */ + luamp_encode_tuple(L, luaL_msgpack_default, &stream, + idx); + } + mpstream_flush(&stream); + tuple = box_tuple_new(format, buf->buf, + buf->buf + ibuf_used(buf)); + if (tuple == NULL) { + luaT_pusherror(L, diag_last_error(diag_get())); + return NULL; + } + ibuf_reinit(tarantool_lua_ibuf); + return tuple; + } + + tuple = luaT_istuple(L, idx); + if (tuple == NULL) { + lua_pushfstring(L, "A tuple or a table expected, got %s", + lua_typename(L, lua_type(L, -1))); + return NULL; + } + + /* + * Create the new tuple with the necessary format from + * the another tuple. + */ + const char *tuple_beg = tuple_data(tuple); + const char *tuple_end = tuple_beg + tuple->bsize; + tuple = box_tuple_new(format, tuple_beg, tuple_end); + if (tuple == NULL) { + luaT_pusherror(L, diag_last_error(diag_get())); + return NULL; + } + return tuple; +} + int lbox_tuple_new(lua_State *L) { @@ -100,33 +159,19 @@ lbox_tuple_new(lua_State *L) lua_newtable(L); /* create an empty tuple */ ++argc; } - struct ibuf *buf = tarantool_lua_ibuf; - - ibuf_reset(buf); - struct mpstream stream; - mpstream_init(&stream, buf, ibuf_reserve_cb, ibuf_alloc_cb, - luamp_error, L); - - if (argc == 1 && (lua_istable(L, 1) || luaT_istuple(L, 1))) { - /* New format: box.tuple.new({1, 2, 3}) */ - luamp_encode_tuple(L, luaL_msgpack_default, &stream, 1); - } else { - /* Backward-compatible format: box.tuple.new(1, 2, 3). */ - mpstream_encode_array(&stream, argc); - for (int k = 1; k <= argc; ++k) { - luamp_encode(L, luaL_msgpack_default, &stream, k); - } - } - mpstream_flush(&stream); - + /* + * Use backward-compatible parameters format: + * box.tuple.new(1, 2, 3) (idx == 0), or the new one: + * box.tuple.new({1, 2, 3}) (idx == 1). + */ + int idx = argc == 1 && (lua_istable(L, 1) || + luaT_istuple(L, 1)); box_tuple_format_t *fmt = box_tuple_format_default(); - struct tuple *tuple = box_tuple_new(fmt, buf->buf, - buf->buf + ibuf_used(buf)); + struct tuple *tuple = luaT_newtuple(L, idx, fmt); if (tuple == NULL) - return luaT_error(L); + return lua_error(L); /* box_tuple_new() doesn't leak on exception, see public API doc */ luaT_pushtuple(L, tuple); - ibuf_reinit(tarantool_lua_ibuf); return 1; } diff --git a/src/box/lua/tuple.h b/src/box/lua/tuple.h index 5d7062eb8..3319b951e 100644 --- a/src/box/lua/tuple.h +++ b/src/box/lua/tuple.h @@ -41,6 +41,8 @@ typedef struct tuple box_tuple_t; struct lua_State; struct mpstream; struct luaL_serializer; +struct tuple_format; +typedef struct tuple_format box_tuple_format_t; /** \cond public */ @@ -66,6 +68,19 @@ luaT_istuple(struct lua_State *L, int idx); /** \endcond public */ +/** + * Create the new tuple with specific format from a Lua table, a + * tuple or objects on the lua stack. + * + * Set idx to zero to create the new tuple from objects on the lua + * stack. + * + * In case of an error push the error message to the Lua stack and + * return NULL. + */ +struct tuple * +luaT_newtuple(struct lua_State *L, int idx, box_tuple_format_t *format); + int lbox_tuple_new(struct lua_State *L); -- 2.20.1 ^ permalink raw reply [flat|nested] 28+ messages in thread
* Re: [PATCH v2 3/6] lua: add luaT_newtuple() 2019-01-09 20:20 ` [PATCH v2 3/6] lua: add luaT_newtuple() Alexander Turenko @ 2019-01-10 12:44 ` Vladimir Davydov 2019-01-18 21:58 ` Alexander Turenko 0 siblings, 1 reply; 28+ messages in thread From: Vladimir Davydov @ 2019-01-10 12:44 UTC (permalink / raw) To: Alexander Turenko; +Cc: tarantool-patches On Wed, Jan 09, 2019 at 11:20:11PM +0300, Alexander Turenko wrote: > The function allows to create a tuple with specific tuple format in C > code using a Lua table, an another tuple or objects on a Lua stack. > > Needed for #3276. > --- > src/box/lua/tuple.c | 91 +++++++++++++++++++++++++++++++++------------ > src/box/lua/tuple.h | 15 ++++++++ > 2 files changed, 83 insertions(+), 23 deletions(-) Although a test would be nice to have, I guess we can live without it, because the new function is tested indirectly via lbox_tuple_new(). > > diff --git a/src/box/lua/tuple.c b/src/box/lua/tuple.c > index 1867f810f..7e9ad89fe 100644 > --- a/src/box/lua/tuple.c > +++ b/src/box/lua/tuple.c > @@ -92,6 +92,65 @@ luaT_istuple(struct lua_State *L, int narg) > return *(struct tuple **) data; > } > > +struct tuple * > +luaT_newtuple(struct lua_State *L, int idx, box_tuple_format_t *format) I looked at the Lua reference manual and realized that they usually call a function lua_newsomething if it creates an object on Lua stack. So I guess we'd better rename it to luaT_tuple_new() to avoid confusion. > +{ > + struct tuple *tuple; > + > + if (idx == 0 || lua_istable(L, idx)) { > + struct ibuf *buf = tarantool_lua_ibuf; > + ibuf_reset(buf); > + struct mpstream stream; > + mpstream_init(&stream, buf, ibuf_reserve_cb, ibuf_alloc_cb, > + luamp_error, L); Nit: bad indentation. > + if (idx == 0) { > + /* > + * Create the tuple from lua stack > + * objects. > + */ > + int argc = lua_gettop(L); > + mpstream_encode_array(&stream, argc); > + for (int k = 1; k <= argc; ++k) { > + luamp_encode(L, luaL_msgpack_default, &stream, > + k); > + } > + } else { > + /* Create the tuple from a Lua table. */ > + luamp_encode_tuple(L, luaL_msgpack_default, &stream, > + idx); > + } > + mpstream_flush(&stream); > + tuple = box_tuple_new(format, buf->buf, > + buf->buf + ibuf_used(buf)); > + if (tuple == NULL) { > + luaT_pusherror(L, diag_last_error(diag_get())); Why not simply throw the error with luaT_error()? Other similar functions throw an error, not just push it to the stack. > + return NULL; > + } > + ibuf_reinit(tarantool_lua_ibuf); > + return tuple; > + } > + > + tuple = luaT_istuple(L, idx); > + if (tuple == NULL) { > + lua_pushfstring(L, "A tuple or a table expected, got %s", > + lua_typename(L, lua_type(L, -1))); > + return NULL; > + } > + > + /* > + * Create the new tuple with the necessary format from Nit: a new tuple > + * the another tuple. Nit: 'the' is redundant. > + */ > + const char *tuple_beg = tuple_data(tuple); > + const char *tuple_end = tuple_beg + tuple->bsize; > + tuple = box_tuple_new(format, tuple_beg, tuple_end); > + if (tuple == NULL) { > + luaT_pusherror(L, diag_last_error(diag_get())); > + return NULL; > + } > + return tuple; I see that you reworked the original code so as to avoid tuple data copying in case a new tuple is created from another tuple. That's OK, but I think that it should've been done in a separate patch. > +} > + > int > lbox_tuple_new(lua_State *L) > { > @@ -100,33 +159,19 @@ lbox_tuple_new(lua_State *L) > lua_newtable(L); /* create an empty tuple */ > ++argc; > } > - struct ibuf *buf = tarantool_lua_ibuf; > - > - ibuf_reset(buf); > - struct mpstream stream; > - mpstream_init(&stream, buf, ibuf_reserve_cb, ibuf_alloc_cb, > - luamp_error, L); > - > - if (argc == 1 && (lua_istable(L, 1) || luaT_istuple(L, 1))) { > - /* New format: box.tuple.new({1, 2, 3}) */ > - luamp_encode_tuple(L, luaL_msgpack_default, &stream, 1); > - } else { > - /* Backward-compatible format: box.tuple.new(1, 2, 3). */ > - mpstream_encode_array(&stream, argc); > - for (int k = 1; k <= argc; ++k) { > - luamp_encode(L, luaL_msgpack_default, &stream, k); > - } > - } > - mpstream_flush(&stream); > - > + /* > + * Use backward-compatible parameters format: > + * box.tuple.new(1, 2, 3) (idx == 0), or the new one: > + * box.tuple.new({1, 2, 3}) (idx == 1). > + */ > + int idx = argc == 1 && (lua_istable(L, 1) || > + luaT_istuple(L, 1)); > box_tuple_format_t *fmt = box_tuple_format_default(); > - struct tuple *tuple = box_tuple_new(fmt, buf->buf, > - buf->buf + ibuf_used(buf)); > + struct tuple *tuple = luaT_newtuple(L, idx, fmt); > if (tuple == NULL) > - return luaT_error(L); > + return lua_error(L); > /* box_tuple_new() doesn't leak on exception, see public API doc */ > luaT_pushtuple(L, tuple); > - ibuf_reinit(tarantool_lua_ibuf); > return 1; > } > > diff --git a/src/box/lua/tuple.h b/src/box/lua/tuple.h > index 5d7062eb8..3319b951e 100644 > --- a/src/box/lua/tuple.h > +++ b/src/box/lua/tuple.h > @@ -41,6 +41,8 @@ typedef struct tuple box_tuple_t; > struct lua_State; > struct mpstream; > struct luaL_serializer; > +struct tuple_format; > +typedef struct tuple_format box_tuple_format_t; > > /** \cond public */ > > @@ -66,6 +68,19 @@ luaT_istuple(struct lua_State *L, int idx); > > /** \endcond public */ > > +/** > + * Create the new tuple with specific format from a Lua table, a Nit: a new tuple > + * tuple or objects on the lua stack. Nit: comma before 'or' is missing ;-) > + * > + * Set idx to zero to create the new tuple from objects on the lua > + * stack. > + * > + * In case of an error push the error message to the Lua stack and > + * return NULL. > + */ > +struct tuple * > +luaT_newtuple(struct lua_State *L, int idx, box_tuple_format_t *format); > + > int > lbox_tuple_new(struct lua_State *L); ^ permalink raw reply [flat|nested] 28+ messages in thread
* Re: [PATCH v2 3/6] lua: add luaT_newtuple() 2019-01-10 12:44 ` Vladimir Davydov @ 2019-01-18 21:58 ` Alexander Turenko 2019-01-23 16:12 ` Vladimir Davydov 0 siblings, 1 reply; 28+ messages in thread From: Alexander Turenko @ 2019-01-18 21:58 UTC (permalink / raw) To: Vladimir Davydov; +Cc: tarantool-patches List of changes: * Changed the name: luaT_newtuple() -> luaT_tuple_new(). * Fix an indentation and grammar typos. * Extracted an optimization to a separate commit. * Added a unit test. Answered inline. Still have unresolved questions (naming and error handling). The new version (two commits) is at bottom of the email. WBR, Alexander Turenko. On Thu, Jan 10, 2019 at 03:44:46PM +0300, Vladimir Davydov wrote: > On Wed, Jan 09, 2019 at 11:20:11PM +0300, Alexander Turenko wrote: > > The function allows to create a tuple with specific tuple format in C > > code using a Lua table, an another tuple or objects on a Lua stack. > > > > Needed for #3276. > > --- > > src/box/lua/tuple.c | 91 +++++++++++++++++++++++++++++++++------------ > > src/box/lua/tuple.h | 15 ++++++++ > > 2 files changed, 83 insertions(+), 23 deletions(-) > > Although a test would be nice to have, I guess we can live without it, > because the new function is tested indirectly via lbox_tuple_new(). There are untested cases: using a tuple as an input, using a non-default format, an error in case of an unexpected type. 1st is possible with box.tuple.new(), but is not tested. 2nd and 3rd are not possible with box.tuple.new(). I have added a unit test for all usage cases of luaT_tuple_new() w/o varying a tuple, which is done in box/tuple.test.lua. So now all cases are covered. > > > > > diff --git a/src/box/lua/tuple.c b/src/box/lua/tuple.c > > index 1867f810f..7e9ad89fe 100644 > > --- a/src/box/lua/tuple.c > > +++ b/src/box/lua/tuple.c > > @@ -92,6 +92,65 @@ luaT_istuple(struct lua_State *L, int narg) > > return *(struct tuple **) data; > > } > > > > +struct tuple * > > +luaT_newtuple(struct lua_State *L, int idx, box_tuple_format_t *format) > > I looked at the Lua reference manual and realized that they usually call > a function lua_newsomething if it creates an object on Lua stack. So I > guess we'd better rename it to luaT_tuple_new() to avoid confusion. Renamed, but I'm tentative about this change. We'll have the following convention: * lua{,L,T}_foo_new(): read from a Lua stack, return a pointer, push an error to a Lua stack (errors is the open question, see below); * lua{,L,T}_newfoo(): read from a Lua stack, push on a Lua stack, raise an error; * lbox_foo_new(): the same as previous, but exposed into Lua via a module / instance fields. Don't sure I'll able to remember this convention: a name does not much say about a behaviour. I would stick with luaT_new* or introduce some more explicit naming for a function to grab a definition from lua and create an object in C. Check / to does not fit good: * lua{L,T}_checkfoo() -> struct foo * - Like luaL_checkstring(). - But it looks as if it can raise an exception. - But it looks as if we'll unwrap a pointer from a cdata. * lua{L,T}_tofoo() -> struct foo * - Like lua_tolstring(). - But it looks as if we'll unwrap a pointer from a cdata. Another possible variants (in order of increasing weirdness for me): * lua{L,T}_makefoo(); * lua{L,T}_createfoo(); * lua{L,T}_parsefoo(); * lua{L,T}_newfoo_c(); * lua{L,T}_constructfoo(); * lua{L,T}_foo_new_fromlua(); BTW, '_c' postfix can suggest that a function cannot raise an error. There are lua{,L}_new* functions that returns a pointer: * luaL_newstate(void) -> struct lua_State * * lua_newthread(struct lua_State *) -> struct lua_State * * lua_newuserdata(struct lua_State *, size_t) -> void * Maybe it is not criminal to do the same under luaT_newtuple()? We also can use luaT_create*, I found only one function with such naming: lua_createtable(). I think the general approach could be the following: use lua{L,T}_newfoo() or lua{L,T}_createfoo() if the former one is busy. luaT_maketuple() and lua{L,T}_newtuple_c() looks appropriate for me too. Latter a bit ugly, but '_c' is informative (no exceptions). > > > +{ > > + struct tuple *tuple; > > + > > + if (idx == 0 || lua_istable(L, idx)) { > > + struct ibuf *buf = tarantool_lua_ibuf; > > + ibuf_reset(buf); > > + struct mpstream stream; > > + mpstream_init(&stream, buf, ibuf_reserve_cb, ibuf_alloc_cb, > > + luamp_error, L); > > Nit: bad indentation. Fixed. > > > + if (idx == 0) { > > + /* > > + * Create the tuple from lua stack > > + * objects. > > + */ > > + int argc = lua_gettop(L); > > + mpstream_encode_array(&stream, argc); > > + for (int k = 1; k <= argc; ++k) { > > + luamp_encode(L, luaL_msgpack_default, &stream, > > + k); > > + } > > + } else { > > + /* Create the tuple from a Lua table. */ > > + luamp_encode_tuple(L, luaL_msgpack_default, &stream, > > + idx); > > + } > > + mpstream_flush(&stream); > > + tuple = box_tuple_new(format, buf->buf, > > + buf->buf + ibuf_used(buf)); > > + if (tuple == NULL) { > > + luaT_pusherror(L, diag_last_error(diag_get())); > > Why not simply throw the error with luaT_error()? Other similar > functions throw an error, not just push it to the stack. Because in a caller I need to perform clean up before reraise it. lua_pcall() would be expensive. luaT_tuple_new() is used in a table and an iterator source in next(). next() is used in two places: merger_next() where I just pop and raise the error and in merger_source_new() to acquire a first tuple. If an error occurs here I need to free the newly created source, then in merger_state_new() free newly created merger_state with all successfully created sources (it also perform needed unref's). I cannot allow any function called on this path to raise an error. I can implement reference counting of all that objects and free them in the next call to merger (some kind of simple gc), but this way looks as overengineered. Mine errors are strings and it is convenient to create them with lua_pushfstring() or push memory errors with luaT_pusherror(). There are two variants how to avoid raising an error: * lua_pushfstring(); * diag_set(). Latter seems to be more native for tarantool. I would use something like XlogError: printf-style format string + vararg. But I doubt how should I name such class? ContractError (most of them are about bad args)? There is also unexpected buffer end, it is more RuntimeError. I dislike the idea to copy XlogError code under another name. Maybe we can implement a general class for such errors and inherit it in XlogError, ContractError and RuntimeError? I choose pushing to stack, because it is the most simple solution, and forget to discuss it with you. My bad. Please, give me some pointer here. > > > + return NULL; > > + } > > + ibuf_reinit(tarantool_lua_ibuf); > > + return tuple; > > + } > > + > > + tuple = luaT_istuple(L, idx); > > + if (tuple == NULL) { > > + lua_pushfstring(L, "A tuple or a table expected, got %s", > > + lua_typename(L, lua_type(L, -1))); > > + return NULL; > > + } > > + > > + /* > > + * Create the new tuple with the necessary format from > > Nit: a new tuple Fixed. > > > + * the another tuple. > > Nit: 'the' is redundant. Fixed. > > > + */ > > + const char *tuple_beg = tuple_data(tuple); > > + const char *tuple_end = tuple_beg + tuple->bsize; > > + tuple = box_tuple_new(format, tuple_beg, tuple_end); > > + if (tuple == NULL) { > > + luaT_pusherror(L, diag_last_error(diag_get())); > > + return NULL; > > + } > > + return tuple; > > I see that you reworked the original code so as to avoid tuple data > copying in case a new tuple is created from another tuple. That's OK, > but I think that it should've been done in a separate patch. Extracted. Copying is still performed, but it is one mempcy() in runtime_tuple_new() and does not involve parsing. > > > +} > > + > > int > > lbox_tuple_new(lua_State *L) > > { > > @@ -100,33 +159,19 @@ lbox_tuple_new(lua_State *L) > > lua_newtable(L); /* create an empty tuple */ > > ++argc; > > } > > - struct ibuf *buf = tarantool_lua_ibuf; > > - > > - ibuf_reset(buf); > > - struct mpstream stream; > > - mpstream_init(&stream, buf, ibuf_reserve_cb, ibuf_alloc_cb, > > - luamp_error, L); > > - > > - if (argc == 1 && (lua_istable(L, 1) || luaT_istuple(L, 1))) { > > - /* New format: box.tuple.new({1, 2, 3}) */ > > - luamp_encode_tuple(L, luaL_msgpack_default, &stream, 1); > > - } else { > > - /* Backward-compatible format: box.tuple.new(1, 2, 3). */ > > - mpstream_encode_array(&stream, argc); > > - for (int k = 1; k <= argc; ++k) { > > - luamp_encode(L, luaL_msgpack_default, &stream, k); > > - } > > - } > > - mpstream_flush(&stream); > > - > > + /* > > + * Use backward-compatible parameters format: > > + * box.tuple.new(1, 2, 3) (idx == 0), or the new one: > > + * box.tuple.new({1, 2, 3}) (idx == 1). > > + */ > > + int idx = argc == 1 && (lua_istable(L, 1) || > > + luaT_istuple(L, 1)); > > box_tuple_format_t *fmt = box_tuple_format_default(); > > - struct tuple *tuple = box_tuple_new(fmt, buf->buf, > > - buf->buf + ibuf_used(buf)); > > + struct tuple *tuple = luaT_newtuple(L, idx, fmt); > > if (tuple == NULL) > > - return luaT_error(L); > > + return lua_error(L); > > /* box_tuple_new() doesn't leak on exception, see public API doc */ > > luaT_pushtuple(L, tuple); > > - ibuf_reinit(tarantool_lua_ibuf); > > return 1; > > } > > > > diff --git a/src/box/lua/tuple.h b/src/box/lua/tuple.h > > index 5d7062eb8..3319b951e 100644 > > --- a/src/box/lua/tuple.h > > +++ b/src/box/lua/tuple.h > > @@ -41,6 +41,8 @@ typedef struct tuple box_tuple_t; > > struct lua_State; > > struct mpstream; > > struct luaL_serializer; > > +struct tuple_format; > > +typedef struct tuple_format box_tuple_format_t; > > > > /** \cond public */ > > > > @@ -66,6 +68,19 @@ luaT_istuple(struct lua_State *L, int idx); > > > > /** \endcond public */ > > > > +/** > > + * Create the new tuple with specific format from a Lua table, a > > Nit: a new tuple Fixed. > > > + * tuple or objects on the lua stack. > > Nit: comma before 'or' is missing ;-) An oxford comma is not mandatory :) Added. > > > + * > > + * Set idx to zero to create the new tuple from objects on the lua > > + * stack. > > + * > > + * In case of an error push the error message to the Lua stack and > > + * return NULL. > > + */ > > +struct tuple * > > +luaT_newtuple(struct lua_State *L, int idx, box_tuple_format_t *format); > > + > > int > > lbox_tuple_new(struct lua_State *L); ---- commit 1c49e0cc9dd96da7f4ac45797b427b19168c47a4 Author: Alexander Turenko <alexander.turenko@tarantool.org> Date: Mon Jan 7 16:52:33 2019 +0300 lua: add luaT_tuple_new() The function allows to create a tuple with specific tuple format in C code using a Lua table, another tuple, or objects on a Lua stack. Needed for #3276. diff --git a/src/box/lua/tuple.c b/src/box/lua/tuple.c index 1867f810f..a2a06a214 100644 --- a/src/box/lua/tuple.c +++ b/src/box/lua/tuple.c @@ -92,41 +92,66 @@ luaT_istuple(struct lua_State *L, int narg) return *(struct tuple **) data; } -int -lbox_tuple_new(lua_State *L) +struct tuple * +luaT_tuple_new(struct lua_State *L, int idx, box_tuple_format_t *format) { - int argc = lua_gettop(L); - if (argc < 1) { - lua_newtable(L); /* create an empty tuple */ - ++argc; + if (idx != 0 && !lua_istable(L, idx) && !luaT_istuple(L, idx)) { + lua_pushfstring(L, "A tuple or a table expected, got %s", + lua_typename(L, lua_type(L, idx))); + return NULL; } - struct ibuf *buf = tarantool_lua_ibuf; + struct ibuf *buf = tarantool_lua_ibuf; ibuf_reset(buf); struct mpstream stream; mpstream_init(&stream, buf, ibuf_reserve_cb, ibuf_alloc_cb, luamp_error, L); - - if (argc == 1 && (lua_istable(L, 1) || luaT_istuple(L, 1))) { - /* New format: box.tuple.new({1, 2, 3}) */ - luamp_encode_tuple(L, luaL_msgpack_default, &stream, 1); - } else { - /* Backward-compatible format: box.tuple.new(1, 2, 3). */ + if (idx == 0) { + /* + * Create the tuple from lua stack + * objects. + */ + int argc = lua_gettop(L); mpstream_encode_array(&stream, argc); for (int k = 1; k <= argc; ++k) { luamp_encode(L, luaL_msgpack_default, &stream, k); } + } else { + /* Create the tuple from a Lua table. */ + luamp_encode_tuple(L, luaL_msgpack_default, &stream, idx); } mpstream_flush(&stream); + struct tuple *tuple = box_tuple_new(format, buf->buf, + buf->buf + ibuf_used(buf)); + if (tuple == NULL) { + luaT_pusherror(L, diag_last_error(diag_get())); + return NULL; + } + ibuf_reinit(tarantool_lua_ibuf); + return tuple; +} +int +lbox_tuple_new(lua_State *L) +{ + int argc = lua_gettop(L); + if (argc < 1) { + lua_newtable(L); /* create an empty tuple */ + ++argc; + } + /* + * Use backward-compatible parameters format: + * box.tuple.new(1, 2, 3) (idx == 0), or the new one: + * box.tuple.new({1, 2, 3}) (idx == 1). + */ + int idx = argc == 1 && (lua_istable(L, 1) || + luaT_istuple(L, 1)); box_tuple_format_t *fmt = box_tuple_format_default(); - struct tuple *tuple = box_tuple_new(fmt, buf->buf, - buf->buf + ibuf_used(buf)); + struct tuple *tuple = luaT_tuple_new(L, idx, fmt); if (tuple == NULL) - return luaT_error(L); + return lua_error(L); /* box_tuple_new() doesn't leak on exception, see public API doc */ luaT_pushtuple(L, tuple); - ibuf_reinit(tarantool_lua_ibuf); return 1; } diff --git a/src/box/lua/tuple.h b/src/box/lua/tuple.h index 5d7062eb8..f8c8ccf1c 100644 --- a/src/box/lua/tuple.h +++ b/src/box/lua/tuple.h @@ -41,6 +41,8 @@ typedef struct tuple box_tuple_t; struct lua_State; struct mpstream; struct luaL_serializer; +struct tuple_format; +typedef struct tuple_format box_tuple_format_t; /** \cond public */ @@ -66,6 +68,19 @@ luaT_istuple(struct lua_State *L, int idx); /** \endcond public */ +/** + * Create a new tuple with specific format from a Lua table, a + * tuple, or objects on the lua stack. + * + * Set idx to zero to create the new tuple from objects on the lua + * stack. + * + * In case of an error push the error message to the Lua stack and + * return NULL. + */ +struct tuple * +luaT_tuple_new(struct lua_State *L, int idx, box_tuple_format_t *format); + int lbox_tuple_new(struct lua_State *L); diff --git a/test/unit/CMakeLists.txt b/test/unit/CMakeLists.txt index c2c45a4b8..4cedc09a4 100644 --- a/test/unit/CMakeLists.txt +++ b/test/unit/CMakeLists.txt @@ -138,10 +138,15 @@ add_executable(histogram.test histogram.c) target_link_libraries(histogram.test stat unit) add_executable(ratelimit.test ratelimit.c) target_link_libraries(ratelimit.test unit) + add_executable(luaL_iterator.test luaL_iterator.c) target_link_libraries(luaL_iterator.test unit server core misc ${CURL_LIBRARIES} ${LIBYAML_LIBRARIES} ${READLINE_LIBRARIES} ${ICU_LIBRARIES} ${LUAJIT_LIBRARIES}) +add_executable(luaT_tuple_new.test luaT_tuple_new.c) +target_link_libraries(luaT_tuple_new.test unit box server core misc + ${CURL_LIBRARIES} ${LIBYAML_LIBRARIES} ${READLINE_LIBRARIES} + ${ICU_LIBRARIES} ${LUAJIT_LIBRARIES}) add_executable(say.test say.c) target_link_libraries(say.test core unit) diff --git a/test/unit/luaT_tuple_new.c b/test/unit/luaT_tuple_new.c new file mode 100644 index 000000000..07fa1a792 --- /dev/null +++ b/test/unit/luaT_tuple_new.c @@ -0,0 +1,174 @@ +#include <string.h> /* strncmp() */ +#include <lua.h> /* lua_*() */ +#include <lauxlib.h> /* luaL_*() */ +#include <lualib.h> /* luaL_openlibs() */ +#include "unit.h" /* plan, header, footer, is, ok */ +#include "memory.h" /* memory_init() */ +#include "fiber.h" /* fiber_init() */ +#include "small/ibuf.h" /* struct ibuf */ +#include "box/box.h" /* box_init() */ +#include "box/tuple.h" /* box_tuple_format_default() */ +#include "lua/msgpack.h" /* luaopen_msgpack() */ +#include "box/lua/tuple.h" /* luaL_iterator_*() */ + +/* + * This test checks all usage cases of luaT_tuple_new(): + * + * * Use with idx == 0 and idx != 0. + * * Use with default and non-default formats. + * * Use a table and a tuple as an input. + * * Use with an unexpected lua type as an input. + * + * The test does not vary an input table/tuple. This is done in + * box/tuple.test.lua. + */ + +extern struct ibuf *tarantool_lua_ibuf; + +uint32_t +min_u32(uint32_t a, uint32_t b) +{ + return a < b ? a : b; +} + +void +check_tuple(const struct tuple *tuple, box_tuple_format_t *format, + int retvals, const char *case_name) +{ + uint32_t size; + const char *data = tuple_data_range(tuple, &size); + + ok(tuple != NULL, "%s: tuple != NULL", case_name); + is(tuple->format_id, tuple_format_id(format), + "%s: check tuple format id", case_name); + is(size, 4, "%s: check tuple size", case_name); + ok(!strncmp(data, "\x93\x01\x02\x03", min_u32(size, 4)), + "%s: check tuple data", case_name); + is(retvals, 0, "%s: check retvals count", case_name); +} + +void check_error(struct lua_State *L, const struct tuple *tuple, int retvals, + const char *case_name) +{ + const char *exp_err = "A tuple or a table expected, got number"; + is(tuple, NULL, "%s: tuple == NULL", case_name); + is(retvals, 1, "%s: check retvals count", case_name); + is(lua_type(L, -1), LUA_TSTRING, "%s: check error type", case_name); + ok(!strcmp(lua_tostring(L, -1), exp_err), "%s: check error message", + case_name); +} + +int +test_basic(struct lua_State *L) +{ + plan(19); + header(); + + int top; + struct tuple *tuple; + box_tuple_format_t *default_format = box_tuple_format_default(); + + /* + * Case: a Lua table on idx == -2 as an input. + */ + + /* Prepare the Lua stack. */ + luaL_loadstring(L, "return {1, 2, 3}"); + lua_call(L, 0, 1); + lua_pushnil(L); + + /* Create and check a tuple. */ + top = lua_gettop(L); + tuple = luaT_tuple_new(L, -2, default_format); + check_tuple(tuple, default_format, lua_gettop(L) - top, "table"); + + /* Clean up. */ + lua_pop(L, 2); + assert(lua_gettop(L) == 0); + + /* + * Case: a tuple on idx == -1 as an input. + */ + + /* Prepare the Lua stack. */ + luaT_pushtuple(L, tuple); + + /* Create and check a tuple. */ + top = lua_gettop(L); + tuple = luaT_tuple_new(L, -1, default_format); + check_tuple(tuple, default_format, lua_gettop(L) - top, "tuple"); + + /* Clean up. */ + lua_pop(L, 1); + assert(lua_gettop(L) == 0); + + /* + * Case: elements on the stack (idx == 0) as an input and + * a non-default format. + */ + + /* Prepare the Lua stack. */ + lua_pushinteger(L, 1); + lua_pushinteger(L, 2); + lua_pushinteger(L, 3); + + /* Create a new format. */ + struct key_part_def part; + part.fieldno = 0; + part.type = FIELD_TYPE_INTEGER; + part.coll_id = COLL_NONE; + part.is_nullable = false; + part.nullable_action = ON_CONFLICT_ACTION_DEFAULT; + part.sort_order = SORT_ORDER_ASC; + struct key_def *key_def = key_def_new(&part, 1); + box_tuple_format_t *another_format = box_tuple_format_new(&key_def, 1); + key_def_delete(key_def); + + /* Create and check a tuple. */ + top = lua_gettop(L); + tuple = luaT_tuple_new(L, 0, another_format); + check_tuple(tuple, another_format, lua_gettop(L) - top, "objects"); + + /* Clean up. */ + tuple_format_delete(another_format); + lua_pop(L, 3); + assert(lua_gettop(L) == 0); + + /* + * Case: a lua object of an unexpected type. + */ + + /* Prepare the Lua stack. */ + lua_pushinteger(L, 42); + + /* Try to create and check for the error. */ + top = lua_gettop(L); + tuple = luaT_tuple_new(L, -1, default_format); + check_error(L, tuple, lua_gettop(L) - top, "unexpected type"); + + /* Clean up. */ + lua_pop(L, 2); + assert(lua_gettop(L) == 0); + + footer(); + return check_plan(); +} + +int +main() +{ + memory_init(); + fiber_init(fiber_c_invoke); + + ibuf_create(tarantool_lua_ibuf, &cord()->slabc, 16000); + + struct lua_State *L = luaL_newstate(); + luaL_openlibs(L); + + box_init(); + box_lua_tuple_init(L); + luaopen_msgpack(L); + lua_pop(L, 1); + + return test_basic(L); +} diff --git a/test/unit/luaT_tuple_new.result b/test/unit/luaT_tuple_new.result new file mode 100644 index 000000000..110aa68c2 --- /dev/null +++ b/test/unit/luaT_tuple_new.result @@ -0,0 +1,22 @@ +1..19 + *** test_basic *** +ok 1 - table: tuple != NULL +ok 2 - table: check tuple format id +ok 3 - table: check tuple size +ok 4 - table: check tuple data +ok 5 - table: check retvals count +ok 6 - tuple: tuple != NULL +ok 7 - tuple: check tuple format id +ok 8 - tuple: check tuple size +ok 9 - tuple: check tuple data +ok 10 - tuple: check retvals count +ok 11 - objects: tuple != NULL +ok 12 - objects: check tuple format id +ok 13 - objects: check tuple size +ok 14 - objects: check tuple data +ok 15 - objects: check retvals count +ok 16 - unexpected type: tuple == NULL +ok 17 - unexpected type: check retvals count +ok 18 - unexpected type: check error type +ok 19 - unexpected type: check error message + *** test_basic: done *** ---- commit d66cb0e3769b6f21b06f08ffb38ba3e0c6718346 Author: Alexander Turenko <alexander.turenko@tarantool.org> Date: Thu Jan 17 23:42:34 2019 +0300 lua: optimize creation of a tuple from a tuple Don't parse tuple data, just copy it. diff --git a/src/box/lua/tuple.c b/src/box/lua/tuple.c index a2a06a214..ab861c6d2 100644 --- a/src/box/lua/tuple.c +++ b/src/box/lua/tuple.c @@ -95,39 +95,59 @@ luaT_istuple(struct lua_State *L, int narg) struct tuple * luaT_tuple_new(struct lua_State *L, int idx, box_tuple_format_t *format) { - if (idx != 0 && !lua_istable(L, idx) && !luaT_istuple(L, idx)) { + struct tuple *tuple; + + if (idx == 0 || lua_istable(L, idx)) { + struct ibuf *buf = tarantool_lua_ibuf; + ibuf_reset(buf); + struct mpstream stream; + mpstream_init(&stream, buf, ibuf_reserve_cb, ibuf_alloc_cb, + luamp_error, L); + if (idx == 0) { + /* + * Create the tuple from lua stack + * objects. + */ + int argc = lua_gettop(L); + mpstream_encode_array(&stream, argc); + for (int k = 1; k <= argc; ++k) { + luamp_encode(L, luaL_msgpack_default, &stream, + k); + } + } else { + /* Create the tuple from a Lua table. */ + luamp_encode_tuple(L, luaL_msgpack_default, &stream, + idx); + } + mpstream_flush(&stream); + tuple = box_tuple_new(format, buf->buf, + buf->buf + ibuf_used(buf)); + if (tuple == NULL) { + luaT_pusherror(L, diag_last_error(diag_get())); + return NULL; + } + ibuf_reinit(tarantool_lua_ibuf); + return tuple; + } + + tuple = luaT_istuple(L, idx); + if (tuple == NULL) { lua_pushfstring(L, "A tuple or a table expected, got %s", lua_typename(L, lua_type(L, idx))); return NULL; } - struct ibuf *buf = tarantool_lua_ibuf; - ibuf_reset(buf); - struct mpstream stream; - mpstream_init(&stream, buf, ibuf_reserve_cb, ibuf_alloc_cb, - luamp_error, L); - if (idx == 0) { - /* - * Create the tuple from lua stack - * objects. - */ - int argc = lua_gettop(L); - mpstream_encode_array(&stream, argc); - for (int k = 1; k <= argc; ++k) { - luamp_encode(L, luaL_msgpack_default, &stream, k); - } - } else { - /* Create the tuple from a Lua table. */ - luamp_encode_tuple(L, luaL_msgpack_default, &stream, idx); - } - mpstream_flush(&stream); - struct tuple *tuple = box_tuple_new(format, buf->buf, - buf->buf + ibuf_used(buf)); + /* + * Create a new tuple with the necessary format from + * another tuple. + */ + const char *tuple_beg = tuple_data(tuple); + const char *tuple_end = tuple_beg + tuple->bsize; + tuple = box_tuple_new(format, tuple_beg, tuple_end); if (tuple == NULL) { luaT_pusherror(L, diag_last_error(diag_get())); return NULL; } - ibuf_reinit(tarantool_lua_ibuf); return tuple; } ^ permalink raw reply [flat|nested] 28+ messages in thread
* Re: [PATCH v2 3/6] lua: add luaT_newtuple() 2019-01-18 21:58 ` Alexander Turenko @ 2019-01-23 16:12 ` Vladimir Davydov 2019-01-28 16:51 ` Alexander Turenko 0 siblings, 1 reply; 28+ messages in thread From: Vladimir Davydov @ 2019-01-23 16:12 UTC (permalink / raw) To: Alexander Turenko; +Cc: tarantool-patches On Sat, Jan 19, 2019 at 12:58:20AM +0300, Alexander Turenko wrote: > > > diff --git a/src/box/lua/tuple.c b/src/box/lua/tuple.c > > > index 1867f810f..7e9ad89fe 100644 > > > --- a/src/box/lua/tuple.c > > > +++ b/src/box/lua/tuple.c > > > @@ -92,6 +92,65 @@ luaT_istuple(struct lua_State *L, int narg) > > > return *(struct tuple **) data; > > > } > > > > > > +struct tuple * > > > +luaT_newtuple(struct lua_State *L, int idx, box_tuple_format_t *format) > > > > I looked at the Lua reference manual and realized that they usually call > > a function lua_newsomething if it creates an object on Lua stack. So I > > guess we'd better rename it to luaT_tuple_new() to avoid confusion. > > Renamed, but I'm tentative about this change. > > We'll have the following convention: > > * lua{,L,T}_foo_new(): read from a Lua stack, return a pointer, push an > error to a Lua stack (errors is the open question, see below); > * lua{,L,T}_newfoo(): read from a Lua stack, push on a Lua stack, raise > an error; > * lbox_foo_new(): the same as previous, but exposed into Lua via a > module / instance fields. > > Don't sure I'll able to remember this convention: a name does not much > say about a behaviour. I would stick with luaT_new* or introduce some luaT_newkey_def would look awful, wouldn't it? > more explicit naming for a function to grab a definition from lua and > create an object in C. > > Check / to does not fit good: > > * lua{L,T}_checkfoo() -> struct foo * > - Like luaL_checkstring(). > - But it looks as if it can raise an exception. > - But it looks as if we'll unwrap a pointer from a cdata. > * lua{L,T}_tofoo() -> struct foo * > - Like lua_tolstring(). > - But it looks as if we'll unwrap a pointer from a cdata. > > Another possible variants (in order of increasing weirdness for me): > > * lua{L,T}_makefoo(); > * lua{L,T}_createfoo(); > * lua{L,T}_parsefoo(); > * lua{L,T}_newfoo_c(); > * lua{L,T}_constructfoo(); > * lua{L,T}_foo_new_fromlua(); > > BTW, '_c' postfix can suggest that a function cannot raise an error. Better not (why _c?) > > There are lua{,L}_new* functions that returns a pointer: > > * luaL_newstate(void) -> struct lua_State * > * lua_newthread(struct lua_State *) -> struct lua_State * > * lua_newuserdata(struct lua_State *, size_t) -> void * These function don't use Lua stack as input AFAIK so they are different. > > Maybe it is not criminal to do the same under luaT_newtuple()? > > We also can use luaT_create*, I found only one function with such > naming: lua_createtable(). > > I think the general approach could be the following: use > lua{L,T}_newfoo() or lua{L,T}_createfoo() if the former one is busy. > > luaT_maketuple() and lua{L,T}_newtuple_c() looks appropriate for me too. > Latter a bit ugly, but '_c' is informative (no exceptions). I don't know what to say, really. It's overwhelming. Let's just call this function luaT_tuple_new ;-) > > > > > > +{ > > > + struct tuple *tuple; > > > + > > > + if (idx == 0 || lua_istable(L, idx)) { > > > + struct ibuf *buf = tarantool_lua_ibuf; > > > + ibuf_reset(buf); > > > + struct mpstream stream; > > > + mpstream_init(&stream, buf, ibuf_reserve_cb, ibuf_alloc_cb, > > > + luamp_error, L); > > > + if (idx == 0) { > > > + /* > > > + * Create the tuple from lua stack > > > + * objects. > > > + */ > > > + int argc = lua_gettop(L); > > > + mpstream_encode_array(&stream, argc); > > > + for (int k = 1; k <= argc; ++k) { > > > + luamp_encode(L, luaL_msgpack_default, &stream, > > > + k); > > > + } > > > + } else { > > > + /* Create the tuple from a Lua table. */ > > > + luamp_encode_tuple(L, luaL_msgpack_default, &stream, > > > + idx); > > > + } > > > + mpstream_flush(&stream); > > > + tuple = box_tuple_new(format, buf->buf, > > > + buf->buf + ibuf_used(buf)); > > > + if (tuple == NULL) { > > > + luaT_pusherror(L, diag_last_error(diag_get())); > > > > Why not simply throw the error with luaT_error()? Other similar > > functions throw an error, not just push it to the stack. > > Because in a caller I need to perform clean up before reraise it. > lua_pcall() would be expensive. > > luaT_tuple_new() is used in a table and an iterator source in next(). > next() is used in two places: merger_next() where I just pop and raise > the error and in merger_source_new() to acquire a first tuple. If an Why do you need to acquire the first tuple in merger_source_new()? Can't you do that in next()? > error occurs here I need to free the newly created source, then in > merger_state_new() free newly created merger_state with all successfully > created sources (it also perform needed unref's). I cannot allow any > function called on this path to raise an error. > > I can implement reference counting of all that objects and free them in > the next call to merger (some kind of simple gc), but this way looks as > overengineered. > > Mine errors are strings and it is convenient to create them with > lua_pushfstring() or push memory errors with luaT_pusherror(). > > There are two variants how to avoid raising an error: > > * lua_pushfstring(); > * diag_set(). > > Latter seems to be more native for tarantool. I would use something like > XlogError: printf-style format string + vararg. But I doubt how should I > name such class? ContractError (most of them are about bad args)? There > is also unexpected buffer end, it is more RuntimeError. Use IllegalParams (already exists)? > > I dislike the idea to copy XlogError code under another name. Maybe we > can implement a general class for such errors and inherit it in > XlogError, ContractError and RuntimeError? > > I choose pushing to stack, because it is the most simple solution, and > forget to discuss it with you. My bad. > > Please, give me some pointer here. I'd prefer to either throw an error (if the merger can handle it) or set diag to IllegalParams. ^ permalink raw reply [flat|nested] 28+ messages in thread
* Re: [PATCH v2 3/6] lua: add luaT_newtuple() 2019-01-23 16:12 ` Vladimir Davydov @ 2019-01-28 16:51 ` Alexander Turenko 2019-03-01 4:08 ` Alexander Turenko 0 siblings, 1 reply; 28+ messages in thread From: Alexander Turenko @ 2019-01-28 16:51 UTC (permalink / raw) To: Vladimir Davydov; +Cc: tarantool-patches Comments are below, diff is at the bottom of the email. > Let's just call this function luaT_tuple_new ;-) Ok. > > > > > > > > > > +{ > > > > + struct tuple *tuple; > > > > + > > > > + if (idx == 0 || lua_istable(L, idx)) { > > > > + struct ibuf *buf = tarantool_lua_ibuf; > > > > + ibuf_reset(buf); > > > > + struct mpstream stream; > > > > + mpstream_init(&stream, buf, ibuf_reserve_cb, ibuf_alloc_cb, > > > > + luamp_error, L); > > > > + if (idx == 0) { > > > > + /* > > > > + * Create the tuple from lua stack > > > > + * objects. > > > > + */ > > > > + int argc = lua_gettop(L); > > > > + mpstream_encode_array(&stream, argc); > > > > + for (int k = 1; k <= argc; ++k) { > > > > + luamp_encode(L, luaL_msgpack_default, &stream, > > > > + k); > > > > + } > > > > + } else { > > > > + /* Create the tuple from a Lua table. */ > > > > + luamp_encode_tuple(L, luaL_msgpack_default, &stream, > > > > + idx); > > > > + } > > > > + mpstream_flush(&stream); > > > > + tuple = box_tuple_new(format, buf->buf, > > > > + buf->buf + ibuf_used(buf)); > > > > + if (tuple == NULL) { > > > > + luaT_pusherror(L, diag_last_error(diag_get())); > > > > > > Why not simply throw the error with luaT_error()? Other similar > > > functions throw an error, not just push it to the stack. > > > > Because in a caller I need to perform clean up before reraise it. > > lua_pcall() would be expensive. > > > > luaT_tuple_new() is used in a table and an iterator source in next(). > > next() is used in two places: merger_next() where I just pop and raise > > the error and in merger_source_new() to acquire a first tuple. If an > > Why do you need to acquire the first tuple in merger_source_new()? > Can't you do that in next()? It is possible, but we need to call next() (and, then, luaT_tuple_new()) from C part of merger, so API of those functions should not use a lua stack for errors. So I'll leave diag_set() for errors in luaT_tuple_new(). BTW, we'll need to introduce a flag in merger_state (states that sources was not initialized yet) to roll a loop with inserts into the heap. > > > error occurs here I need to free the newly created source, then in > > merger_state_new() free newly created merger_state with all successfully > > created sources (it also perform needed unref's). I cannot allow any > > function called on this path to raise an error. > > > > I can implement reference counting of all that objects and free them in > > the next call to merger (some kind of simple gc), but this way looks as > > overengineered. > > > > Mine errors are strings and it is convenient to create them with > > lua_pushfstring() or push memory errors with luaT_pusherror(). > > > > There are two variants how to avoid raising an error: > > > > * lua_pushfstring(); > > * diag_set(). > > > > Latter seems to be more native for tarantool. I would use something like > > XlogError: printf-style format string + vararg. But I doubt how should I > > name such class? ContractError (most of them are about bad args)? There > > is also unexpected buffer end, it is more RuntimeError. > > Use IllegalParams (already exists)? Looks okay. Thanks! > > > > > I dislike the idea to copy XlogError code under another name. Maybe we > > can implement a general class for such errors and inherit it in > > XlogError, ContractError and RuntimeError? > > > > I choose pushing to stack, because it is the most simple solution, and > > forget to discuss it with you. My bad. > > > > Please, give me some pointer here. > > I'd prefer to either throw an error (if the merger can handle it) or set > diag to IllegalParams. I'll use diag_set(). ---- (Below are resulting changes, but I amended them into 'lua: add luaT_tuple_new()' and 'lua: optimize creation of a tuple from a tuple' commits as approariate.) diff --git a/src/box/lua/tuple.c b/src/box/lua/tuple.c index ab861c6d2..c4f323717 100644 --- a/src/box/lua/tuple.c +++ b/src/box/lua/tuple.c @@ -122,18 +122,16 @@ luaT_tuple_new(struct lua_State *L, int idx, box_tuple_format_t *format) mpstream_flush(&stream); tuple = box_tuple_new(format, buf->buf, buf->buf + ibuf_used(buf)); - if (tuple == NULL) { - luaT_pusherror(L, diag_last_error(diag_get())); + if (tuple == NULL) return NULL; - } ibuf_reinit(tarantool_lua_ibuf); return tuple; } tuple = luaT_istuple(L, idx); if (tuple == NULL) { - lua_pushfstring(L, "A tuple or a table expected, got %s", - lua_typename(L, lua_type(L, idx))); + diag_set(IllegalParams, "A tuple or a table expected, got %s", + lua_typename(L, lua_type(L, idx))); return NULL; } @@ -144,10 +142,8 @@ luaT_tuple_new(struct lua_State *L, int idx, box_tuple_format_t *format) const char *tuple_beg = tuple_data(tuple); const char *tuple_end = tuple_beg + tuple->bsize; tuple = box_tuple_new(format, tuple_beg, tuple_end); - if (tuple == NULL) { - luaT_pusherror(L, diag_last_error(diag_get())); + if (tuple == NULL) return NULL; - } return tuple; } @@ -169,7 +165,7 @@ lbox_tuple_new(lua_State *L) box_tuple_format_t *fmt = box_tuple_format_default(); struct tuple *tuple = luaT_tuple_new(L, idx, fmt); if (tuple == NULL) - return lua_error(L); + return luaT_error(L); /* box_tuple_new() doesn't leak on exception, see public API doc */ luaT_pushtuple(L, tuple); return 1; diff --git a/src/box/lua/tuple.h b/src/box/lua/tuple.h index f8c8ccf1c..06efa277a 100644 --- a/src/box/lua/tuple.h +++ b/src/box/lua/tuple.h @@ -75,8 +75,7 @@ luaT_istuple(struct lua_State *L, int idx); * Set idx to zero to create the new tuple from objects on the lua * stack. * - * In case of an error push the error message to the Lua stack and - * return NULL. + * In case of an error set diag and return NULL. */ struct tuple * luaT_tuple_new(struct lua_State *L, int idx, box_tuple_format_t *format); diff --git a/test/unit/luaT_tuple_new.c b/test/unit/luaT_tuple_new.c index 07fa1a792..582d6c616 100644 --- a/test/unit/luaT_tuple_new.c +++ b/test/unit/luaT_tuple_new.c @@ -10,6 +10,8 @@ #include "box/tuple.h" /* box_tuple_format_default() */ #include "lua/msgpack.h" /* luaopen_msgpack() */ #include "box/lua/tuple.h" /* luaL_iterator_*() */ +#include "diag.h" /* struct error, diag_*() */ +#include "exception.h" /* type_IllegalParams */ /* * This test checks all usage cases of luaT_tuple_new(): @@ -52,10 +54,10 @@ void check_error(struct lua_State *L, const struct tuple *tuple, int retvals, { const char *exp_err = "A tuple or a table expected, got number"; is(tuple, NULL, "%s: tuple == NULL", case_name); - is(retvals, 1, "%s: check retvals count", case_name); - is(lua_type(L, -1), LUA_TSTRING, "%s: check error type", case_name); - ok(!strcmp(lua_tostring(L, -1), exp_err), "%s: check error message", - case_name); + is(retvals, 0, "%s: check retvals count", case_name); + struct error *e = diag_last_error(diag_get()); + is(e->type, &type_IllegalParams, "%s: check error type", case_name); + ok(!strcmp(e->errmsg, exp_err), "%s: check error message", case_name); } int @@ -147,7 +149,7 @@ test_basic(struct lua_State *L) check_error(L, tuple, lua_gettop(L) - top, "unexpected type"); /* Clean up. */ - lua_pop(L, 2); + lua_pop(L, 1); assert(lua_gettop(L) == 0); footer(); ^ permalink raw reply [flat|nested] 28+ messages in thread
* Re: [PATCH v2 3/6] lua: add luaT_newtuple() 2019-01-28 16:51 ` Alexander Turenko @ 2019-03-01 4:08 ` Alexander Turenko 0 siblings, 0 replies; 28+ messages in thread From: Alexander Turenko @ 2019-03-01 4:08 UTC (permalink / raw) To: Vladimir Davydov; +Cc: tarantool-patches I made two minor updates to the commit. I pasted them below separatelly for ease reading, but I'll squash them into the original commit. WBR, Alexander Turenko. commit 819687fc2a87f96c2c9445b2476dea2a5d140e01 Author: Alexander Turenko <alexander.turenko@tarantool.org> Date: Thu Feb 21 18:37:58 2019 +0300 FIXUP: luaT_tuple_new(): fix test comment diff --git a/test/unit/luaT_tuple_new.c b/test/unit/luaT_tuple_new.c index 582d6c616..e424d50bd 100644 --- a/test/unit/luaT_tuple_new.c +++ b/test/unit/luaT_tuple_new.c @@ -9,7 +9,7 @@ #include "box/box.h" /* box_init() */ #include "box/tuple.h" /* box_tuple_format_default() */ #include "lua/msgpack.h" /* luaopen_msgpack() */ -#include "box/lua/tuple.h" /* luaL_iterator_*() */ +#include "box/lua/tuple.h" /* luaT_tuple_new() */ #include "diag.h" /* struct error, diag_*() */ #include "exception.h" /* type_IllegalParams */ commit 5f3cefa306c43308ebc06f9b50a2e26684841746 Author: Alexander Turenko <alexander.turenko@tarantool.org> Date: Fri Mar 1 02:28:05 2019 +0300 FIXUP: luaT_tuple_new: after rebase fix diff --git a/test/unit/luaT_tuple_new.c b/test/unit/luaT_tuple_new.c index e424d50bd..bcb6347be 100644 --- a/test/unit/luaT_tuple_new.c +++ b/test/unit/luaT_tuple_new.c @@ -122,6 +122,7 @@ test_basic(struct lua_State *L) part.is_nullable = false; part.nullable_action = ON_CONFLICT_ACTION_DEFAULT; part.sort_order = SORT_ORDER_ASC; + part.path = NULL; struct key_def *key_def = key_def_new(&part, 1); box_tuple_format_t *another_format = box_tuple_format_new(&key_def, 1); key_def_delete(key_def); On Mon, Jan 28, 2019 at 07:51:29PM +0300, Alexander Turenko wrote: > Comments are below, diff is at the bottom of the email. > > > Let's just call this function luaT_tuple_new ;-) > > Ok. > > > > > > > > > > > > > > > +{ > > > > > + struct tuple *tuple; > > > > > + > > > > > + if (idx == 0 || lua_istable(L, idx)) { > > > > > + struct ibuf *buf = tarantool_lua_ibuf; > > > > > + ibuf_reset(buf); > > > > > + struct mpstream stream; > > > > > + mpstream_init(&stream, buf, ibuf_reserve_cb, ibuf_alloc_cb, > > > > > + luamp_error, L); > > > > > + if (idx == 0) { > > > > > + /* > > > > > + * Create the tuple from lua stack > > > > > + * objects. > > > > > + */ > > > > > + int argc = lua_gettop(L); > > > > > + mpstream_encode_array(&stream, argc); > > > > > + for (int k = 1; k <= argc; ++k) { > > > > > + luamp_encode(L, luaL_msgpack_default, &stream, > > > > > + k); > > > > > + } > > > > > + } else { > > > > > + /* Create the tuple from a Lua table. */ > > > > > + luamp_encode_tuple(L, luaL_msgpack_default, &stream, > > > > > + idx); > > > > > + } > > > > > + mpstream_flush(&stream); > > > > > + tuple = box_tuple_new(format, buf->buf, > > > > > + buf->buf + ibuf_used(buf)); > > > > > + if (tuple == NULL) { > > > > > + luaT_pusherror(L, diag_last_error(diag_get())); > > > > > > > > Why not simply throw the error with luaT_error()? Other similar > > > > functions throw an error, not just push it to the stack. > > > > > > Because in a caller I need to perform clean up before reraise it. > > > lua_pcall() would be expensive. > > > > > > luaT_tuple_new() is used in a table and an iterator source in next(). > > > next() is used in two places: merger_next() where I just pop and raise > > > the error and in merger_source_new() to acquire a first tuple. If an > > > > Why do you need to acquire the first tuple in merger_source_new()? > > Can't you do that in next()? > > It is possible, but we need to call next() (and, then, luaT_tuple_new()) > from C part of merger, so API of those functions should not use a lua > stack for errors. So I'll leave diag_set() for errors in > luaT_tuple_new(). > > BTW, we'll need to introduce a flag in merger_state (states that sources > was not initialized yet) to roll a loop with inserts into the heap. > > > > > > error occurs here I need to free the newly created source, then in > > > merger_state_new() free newly created merger_state with all successfully > > > created sources (it also perform needed unref's). I cannot allow any > > > function called on this path to raise an error. > > > > > > I can implement reference counting of all that objects and free them in > > > the next call to merger (some kind of simple gc), but this way looks as > > > overengineered. > > > > > > Mine errors are strings and it is convenient to create them with > > > lua_pushfstring() or push memory errors with luaT_pusherror(). > > > > > > There are two variants how to avoid raising an error: > > > > > > * lua_pushfstring(); > > > * diag_set(). > > > > > > Latter seems to be more native for tarantool. I would use something like > > > XlogError: printf-style format string + vararg. But I doubt how should I > > > name such class? ContractError (most of them are about bad args)? There > > > is also unexpected buffer end, it is more RuntimeError. > > > > Use IllegalParams (already exists)? > > Looks okay. Thanks! > > > > > > > > > I dislike the idea to copy XlogError code under another name. Maybe we > > > can implement a general class for such errors and inherit it in > > > XlogError, ContractError and RuntimeError? > > > > > > I choose pushing to stack, because it is the most simple solution, and > > > forget to discuss it with you. My bad. > > > > > > Please, give me some pointer here. > > > > I'd prefer to either throw an error (if the merger can handle it) or set > > diag to IllegalParams. > > I'll use diag_set(). > > ---- > > (Below are resulting changes, but I amended them into 'lua: add > luaT_tuple_new()' and 'lua: optimize creation of a tuple from a tuple' > commits as approariate.) > > diff --git a/src/box/lua/tuple.c b/src/box/lua/tuple.c > index ab861c6d2..c4f323717 100644 > --- a/src/box/lua/tuple.c > +++ b/src/box/lua/tuple.c > @@ -122,18 +122,16 @@ luaT_tuple_new(struct lua_State *L, int idx, box_tuple_format_t *format) > mpstream_flush(&stream); > tuple = box_tuple_new(format, buf->buf, > buf->buf + ibuf_used(buf)); > - if (tuple == NULL) { > - luaT_pusherror(L, diag_last_error(diag_get())); > + if (tuple == NULL) > return NULL; > - } > ibuf_reinit(tarantool_lua_ibuf); > return tuple; > } > > tuple = luaT_istuple(L, idx); > if (tuple == NULL) { > - lua_pushfstring(L, "A tuple or a table expected, got %s", > - lua_typename(L, lua_type(L, idx))); > + diag_set(IllegalParams, "A tuple or a table expected, got %s", > + lua_typename(L, lua_type(L, idx))); > return NULL; > } > > @@ -144,10 +142,8 @@ luaT_tuple_new(struct lua_State *L, int idx, box_tuple_format_t *format) > const char *tuple_beg = tuple_data(tuple); > const char *tuple_end = tuple_beg + tuple->bsize; > tuple = box_tuple_new(format, tuple_beg, tuple_end); > - if (tuple == NULL) { > - luaT_pusherror(L, diag_last_error(diag_get())); > + if (tuple == NULL) > return NULL; > - } > return tuple; > } > > @@ -169,7 +165,7 @@ lbox_tuple_new(lua_State *L) > box_tuple_format_t *fmt = box_tuple_format_default(); > struct tuple *tuple = luaT_tuple_new(L, idx, fmt); > if (tuple == NULL) > - return lua_error(L); > + return luaT_error(L); > /* box_tuple_new() doesn't leak on exception, see public API doc */ > luaT_pushtuple(L, tuple); > return 1; > diff --git a/src/box/lua/tuple.h b/src/box/lua/tuple.h > index f8c8ccf1c..06efa277a 100644 > --- a/src/box/lua/tuple.h > +++ b/src/box/lua/tuple.h > @@ -75,8 +75,7 @@ luaT_istuple(struct lua_State *L, int idx); > * Set idx to zero to create the new tuple from objects on the lua > * stack. > * > - * In case of an error push the error message to the Lua stack and > - * return NULL. > + * In case of an error set diag and return NULL. > */ > struct tuple * > luaT_tuple_new(struct lua_State *L, int idx, box_tuple_format_t *format); > diff --git a/test/unit/luaT_tuple_new.c b/test/unit/luaT_tuple_new.c > index 07fa1a792..582d6c616 100644 > --- a/test/unit/luaT_tuple_new.c > +++ b/test/unit/luaT_tuple_new.c > @@ -10,6 +10,8 @@ > #include "box/tuple.h" /* box_tuple_format_default() */ > #include "lua/msgpack.h" /* luaopen_msgpack() */ > #include "box/lua/tuple.h" /* luaL_iterator_*() */ > +#include "diag.h" /* struct error, diag_*() */ > +#include "exception.h" /* type_IllegalParams */ > > /* > * This test checks all usage cases of luaT_tuple_new(): > @@ -52,10 +54,10 @@ void check_error(struct lua_State *L, const struct tuple *tuple, int retvals, > { > const char *exp_err = "A tuple or a table expected, got number"; > is(tuple, NULL, "%s: tuple == NULL", case_name); > - is(retvals, 1, "%s: check retvals count", case_name); > - is(lua_type(L, -1), LUA_TSTRING, "%s: check error type", case_name); > - ok(!strcmp(lua_tostring(L, -1), exp_err), "%s: check error message", > - case_name); > + is(retvals, 0, "%s: check retvals count", case_name); > + struct error *e = diag_last_error(diag_get()); > + is(e->type, &type_IllegalParams, "%s: check error type", case_name); > + ok(!strcmp(e->errmsg, exp_err), "%s: check error message", case_name); > } > > int > @@ -147,7 +149,7 @@ test_basic(struct lua_State *L) > check_error(L, tuple, lua_gettop(L) - top, "unexpected type"); > > /* Clean up. */ > - lua_pop(L, 2); > + lua_pop(L, 1); > assert(lua_gettop(L) == 0); > > footer(); ^ permalink raw reply [flat|nested] 28+ messages in thread
* [PATCH v2 4/6] lua: add luaT_new_key_def() 2019-01-09 20:20 [PATCH v2 0/6] Merger Alexander Turenko ` (2 preceding siblings ...) 2019-01-09 20:20 ` [PATCH v2 3/6] lua: add luaT_newtuple() Alexander Turenko @ 2019-01-09 20:20 ` Alexander Turenko 2019-01-10 13:07 ` Vladimir Davydov 2019-01-09 20:20 ` [PATCH v2 5/6] net.box: add helpers to decode msgpack headers Alexander Turenko 2019-01-09 20:20 ` [PATCH v2 6/6] Add merger for tuple streams Alexander Turenko 5 siblings, 1 reply; 28+ messages in thread From: Alexander Turenko @ 2019-01-09 20:20 UTC (permalink / raw) To: Vladimir Davydov; +Cc: Alexander Turenko, tarantool-patches The function is needed to create the new struct key_def from C code using a Lua table in the format compatible with box.space[...].index[...].parts and net_box_conn.space[...].index[...].parts. Needed for #3276. --- extra/exports | 1 + src/CMakeLists.txt | 1 + src/box/CMakeLists.txt | 1 + src/box/lua/key_def.c | 217 +++++++++++++++++++++++++++++++ src/box/lua/key_def.h | 61 +++++++++ test/app-tap/module_api.c | 13 ++ test/app-tap/module_api.test.lua | 89 ++++++++++++- 7 files changed, 381 insertions(+), 2 deletions(-) create mode 100644 src/box/lua/key_def.c create mode 100644 src/box/lua/key_def.h diff --git a/extra/exports b/extra/exports index af6863963..497719ed8 100644 --- a/extra/exports +++ b/extra/exports @@ -210,6 +210,7 @@ clock_realtime64 clock_monotonic64 clock_process64 clock_thread64 +luaT_new_key_def # Lua / LuaJIT diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 04de5ad04..494c8d391 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -202,6 +202,7 @@ set(api_headers ${CMAKE_SOURCE_DIR}/src/lua/error.h ${CMAKE_SOURCE_DIR}/src/box/txn.h ${CMAKE_SOURCE_DIR}/src/box/key_def.h + ${CMAKE_SOURCE_DIR}/src/box/lua/key_def.h ${CMAKE_SOURCE_DIR}/src/box/field_def.h ${CMAKE_SOURCE_DIR}/src/box/tuple.h ${CMAKE_SOURCE_DIR}/src/box/tuple_format.h diff --git a/src/box/CMakeLists.txt b/src/box/CMakeLists.txt index 5521e489e..0db093768 100644 --- a/src/box/CMakeLists.txt +++ b/src/box/CMakeLists.txt @@ -139,6 +139,7 @@ add_library(box STATIC lua/net_box.c lua/xlog.c lua/sql.c + lua/key_def.c ${bin_sources}) target_link_libraries(box box_error tuple stat xrow xlog vclock crc32 scramble diff --git a/src/box/lua/key_def.c b/src/box/lua/key_def.c new file mode 100644 index 000000000..60247f427 --- /dev/null +++ b/src/box/lua/key_def.c @@ -0,0 +1,217 @@ +/* + * Copyright 2010-2018, Tarantool AUTHORS, please see AUTHORS file. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * 1. Redistributions of source code must retain the above + * copyright notice, this list of conditions and the + * following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY <COPYRIGHT HOLDER> ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL + * <COPYRIGHT HOLDER> OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF + * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include "box/lua/key_def.h" + +#include <lua.h> +#include <lauxlib.h> +#include "diag.h" +#include "box/key_def.h" +#include "box/box.h" +#include "box/coll_id_cache.h" +#include "lua/utils.h" + +struct key_def * +luaT_new_key_def(struct lua_State *L, int idx) +{ + if (lua_istable(L, idx) != 1) { + luaL_error(L, "Bad params, use: luaT_new_key_def({" + "{fieldno = fieldno, type = type" + "[, is_nullable = is_nullable" + "[, collation_id = collation_id" + "[, collation = collation]]]}, ...}"); + unreachable(); + return NULL; + } + uint32_t key_parts_count = 0; + uint32_t capacity = 8; + + const ssize_t parts_size = sizeof(struct key_part_def) * capacity; + struct key_part_def *parts = NULL; + parts = (struct key_part_def *) malloc(parts_size); + if (parts == NULL) { + diag_set(OutOfMemory, parts_size, "malloc", "parts"); + luaT_error(L); + unreachable(); + return NULL; + } + + while (true) { + lua_pushinteger(L, key_parts_count + 1); + lua_gettable(L, idx); + if (lua_isnil(L, -1)) + break; + + /* Extend parts if necessary. */ + if (key_parts_count == capacity) { + capacity *= 2; + struct key_part_def *old_parts = parts; + const ssize_t parts_size = + sizeof(struct key_part_def) * capacity; + parts = (struct key_part_def *) realloc(parts, + parts_size); + if (parts == NULL) { + free(old_parts); + diag_set(OutOfMemory, parts_size / 2, "malloc", + "parts"); + luaT_error(L); + unreachable(); + return NULL; + } + } + + /* Set parts[key_parts_count].fieldno. */ + lua_pushstring(L, "fieldno"); + lua_gettable(L, -2); + if (lua_isnil(L, -1)) { + free(parts); + luaL_error(L, "fieldno must not be nil"); + unreachable(); + return NULL; + } + /* + * Transform one-based Lua fieldno to zero-based + * fieldno to use in key_def_new(). + */ + parts[key_parts_count].fieldno = lua_tointeger(L, -1) - 1; + lua_pop(L, 1); + + /* Set parts[key_parts_count].type. */ + lua_pushstring(L, "type"); + lua_gettable(L, -2); + if (lua_isnil(L, -1)) { + free(parts); + luaL_error(L, "type must not be nil"); + unreachable(); + return NULL; + } + size_t type_len; + const char *type_name = lua_tolstring(L, -1, &type_len); + lua_pop(L, 1); + parts[key_parts_count].type = field_type_by_name(type_name, + type_len); + if (parts[key_parts_count].type == field_type_MAX) { + free(parts); + luaL_error(L, "Unknown field type: %s", type_name); + unreachable(); + return NULL; + } + + /* + * Set parts[key_parts_count].is_nullable and + * parts[key_parts_count].nullable_action. + */ + lua_pushstring(L, "is_nullable"); + lua_gettable(L, -2); + if (lua_isnil(L, -1)) { + parts[key_parts_count].is_nullable = false; + parts[key_parts_count].nullable_action = + ON_CONFLICT_ACTION_DEFAULT; + } else { + parts[key_parts_count].is_nullable = + lua_toboolean(L, -1); + parts[key_parts_count].nullable_action = + ON_CONFLICT_ACTION_NONE; + } + lua_pop(L, 1); + + /* Set parts[key_parts_count].coll_id using collation_id. */ + lua_pushstring(L, "collation_id"); + lua_gettable(L, -2); + if (lua_isnil(L, -1)) + parts[key_parts_count].coll_id = COLL_NONE; + else + parts[key_parts_count].coll_id = lua_tointeger(L, -1); + lua_pop(L, 1); + + /* Set parts[key_parts_count].coll_id using collation. */ + lua_pushstring(L, "collation"); + lua_gettable(L, -2); + /* Check whether box.cfg{} was called. */ + if ((parts[key_parts_count].coll_id != COLL_NONE || + !lua_isnil(L, -1)) && !box_is_configured()) { + free(parts); + luaL_error(L, "Cannot use collations: " + "please call box.cfg{}"); + unreachable(); + return NULL; + } + if (!lua_isnil(L, -1)) { + if (parts[key_parts_count].coll_id != COLL_NONE) { + free(parts); + luaL_error(L, "Conflicting options: " + "collation_id and collation"); + unreachable(); + return NULL; + } + size_t coll_name_len; + const char *coll_name = lua_tolstring(L, -1, + &coll_name_len); + struct coll_id *coll_id = coll_by_name(coll_name, + coll_name_len); + if (coll_id == NULL) { + free(parts); + luaL_error(L, "Unknown collation: \"%s\"", + coll_name); + unreachable(); + return NULL; + } + parts[key_parts_count].coll_id = coll_id->id; + } + lua_pop(L, 1); + + /* Check coll_id. */ + struct coll_id *coll_id = + coll_by_id(parts[key_parts_count].coll_id); + if (parts[key_parts_count].coll_id != COLL_NONE && + coll_id == NULL) { + uint32_t collation_id = parts[key_parts_count].coll_id; + free(parts); + luaL_error(L, "Unknown collation_id: %d", collation_id); + unreachable(); + return NULL; + } + + /* Set parts[key_parts_count].sort_order. */ + parts[key_parts_count].sort_order = SORT_ORDER_ASC; + + ++key_parts_count; + } + + struct key_def *key_def = key_def_new(parts, key_parts_count); + free(parts); + if (key_def == NULL) { + luaL_error(L, "Cannot create key_def"); + unreachable(); + return NULL; + } + return key_def; +} diff --git a/src/box/lua/key_def.h b/src/box/lua/key_def.h new file mode 100644 index 000000000..55292fb7e --- /dev/null +++ b/src/box/lua/key_def.h @@ -0,0 +1,61 @@ +#ifndef TARANTOOL_BOX_LUA_KEY_DEF_H_INCLUDED +#define TARANTOOL_BOX_LUA_KEY_DEF_H_INCLUDED +/* + * Copyright 2010-2018, Tarantool AUTHORS, please see AUTHORS file. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * 1. Redistributions of source code must retain the above + * copyright notice, this list of conditions and the + * following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY AUTHORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL + * AUTHORS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF + * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#if defined(__cplusplus) +extern "C" { +#endif /* defined(__cplusplus) */ + +struct key_def; +struct lua_State; + +/** \cond public */ + +/** + * Create the new key_def from a Lua table. + * + * Expected a table of key parts on the Lua stack. The format is + * the same as box.space.<...>.index.<...>.parts or corresponding + * net.box's one. + * + * Returns the new key_def. + */ +struct key_def * +luaT_new_key_def(struct lua_State *L, int idx); + +/** \endcond public */ + +#if defined(__cplusplus) +} /* extern "C" */ +#endif /* defined(__cplusplus) */ + +#endif /* TARANTOOL_BOX_LUA_KEY_DEF_H_INCLUDED */ diff --git a/test/app-tap/module_api.c b/test/app-tap/module_api.c index b81a98056..34ab54bc0 100644 --- a/test/app-tap/module_api.c +++ b/test/app-tap/module_api.c @@ -449,6 +449,18 @@ test_iscallable(lua_State *L) return 1; } +static int +test_luaT_new_key_def(lua_State *L) +{ + /* + * Ignore the return value. Here we test whether the + * function raises an error. + */ + luaT_new_key_def(L, 1); + lua_pop(L, 1); + return 0; +} + LUA_API int luaopen_module_api(lua_State *L) { @@ -477,6 +489,7 @@ luaopen_module_api(lua_State *L) {"test_state", test_state}, {"test_tostring", test_tostring}, {"iscallable", test_iscallable}, + {"luaT_new_key_def", test_luaT_new_key_def}, {NULL, NULL} }; luaL_register(L, "module_api", lib); diff --git a/test/app-tap/module_api.test.lua b/test/app-tap/module_api.test.lua index a6658cc61..994275425 100755 --- a/test/app-tap/module_api.test.lua +++ b/test/app-tap/module_api.test.lua @@ -2,7 +2,6 @@ local fio = require('fio') -box.cfg{log = "tarantool.log"} -- Use BUILDDIR passed from test-run or cwd when run w/o -- test-run to find test/app-tap/module_api.{so,dylib}. build_path = os.getenv("BUILDDIR") or '.' @@ -116,8 +115,91 @@ local function test_iscallable(test, module) end end +local function test_luaT_new_key_def(test, module) + local cases = { + -- Cases to call before box.cfg{}. + { + 'Pass a field on an unknown type', + parts = {{ + fieldno = 2, + type = 'unknown', + }}, + exp_err = 'Unknown field type: unknown', + }, + { + 'Try to use collation_id before box.cfg{}', + parts = {{ + fieldno = 1, + type = 'string', + collation_id = 2, + }}, + exp_err = 'Cannot use collations: please call box.cfg{}', + }, + { + 'Try to use collation before box.cfg{}', + parts = {{ + fieldno = 1, + type = 'string', + collation = 'unicode_ci', + }}, + exp_err = 'Cannot use collations: please call box.cfg{}', + }, + function() + -- For collations. + box.cfg{} + end, + -- Cases to call after box.cfg{}. + { + 'Try to use both collation_id and collation', + parts = {{ + fieldno = 1, + type = 'string', + collation_id = 2, + collation = 'unicode_ci', + }}, + exp_err = 'Conflicting options: collation_id and collation', + }, + { + 'Unknown collation_id', + parts = {{ + fieldno = 1, + type = 'string', + collation_id = 42, + }}, + exp_err = 'Unknown collation_id: 42', + }, + { + 'Unknown collation name', + parts = {{ + fieldno = 1, + type = 'string', + collation = 'unknown', + }}, + exp_err = 'Unknown collation: "unknown"', + }, + { + parts = 1, + exp_err = 'Bad params, use: luaT_new_key_def({' .. + '{fieldno = fieldno, type = type' .. + '[, is_nullable = is_nullable' .. + '[, collation_id = collation_id' .. + '[, collation = collation]]]}, ...}', + }, + } + + test:plan(#cases - 1) + for _, case in ipairs(cases) do + if type(case) == 'function' then + case() + else + local ok, err = pcall(module.luaT_new_key_def, case.parts) + test:is_deeply({ok, err}, {false, case.exp_err}, case[1]) + end + end +end + local test = require('tap').test("module_api", function(test) - test:plan(24) + test:plan(25) local status, module = pcall(require, 'module_api') test:is(status, true, "module") test:ok(status, "module is loaded") @@ -129,6 +211,9 @@ local test = require('tap').test("module_api", function(test) return end + -- Should be called before box.cfg{}. Calls box.cfg{} itself. + test:test("luaT_new_key_def", test_luaT_new_key_def, module) + local space = box.schema.space.create("test") space:create_index('primary') -- 2.20.1 ^ permalink raw reply [flat|nested] 28+ messages in thread
* Re: [PATCH v2 4/6] lua: add luaT_new_key_def() 2019-01-09 20:20 ` [PATCH v2 4/6] lua: add luaT_new_key_def() Alexander Turenko @ 2019-01-10 13:07 ` Vladimir Davydov 2019-01-29 18:52 ` Alexander Turenko 0 siblings, 1 reply; 28+ messages in thread From: Vladimir Davydov @ 2019-01-10 13:07 UTC (permalink / raw) To: Alexander Turenko; +Cc: tarantool-patches On Wed, Jan 09, 2019 at 11:20:12PM +0300, Alexander Turenko wrote: > The function is needed to create the new struct key_def from C code > using a Lua table in the format compatible with > box.space[...].index[...].parts and > net_box_conn.space[...].index[...].parts. > > Needed for #3276. > --- > extra/exports | 1 + > src/CMakeLists.txt | 1 + > src/box/CMakeLists.txt | 1 + > src/box/lua/key_def.c | 217 +++++++++++++++++++++++++++++++ > src/box/lua/key_def.h | 61 +++++++++ > test/app-tap/module_api.c | 13 ++ > test/app-tap/module_api.test.lua | 89 ++++++++++++- > 7 files changed, 381 insertions(+), 2 deletions(-) > create mode 100644 src/box/lua/key_def.c > create mode 100644 src/box/lua/key_def.h > > diff --git a/extra/exports b/extra/exports > index af6863963..497719ed8 100644 > --- a/extra/exports > +++ b/extra/exports > @@ -210,6 +210,7 @@ clock_realtime64 > clock_monotonic64 > clock_process64 > clock_thread64 > +luaT_new_key_def > > # Lua / LuaJIT > > diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt > index 04de5ad04..494c8d391 100644 > --- a/src/CMakeLists.txt > +++ b/src/CMakeLists.txt > @@ -202,6 +202,7 @@ set(api_headers > ${CMAKE_SOURCE_DIR}/src/lua/error.h > ${CMAKE_SOURCE_DIR}/src/box/txn.h > ${CMAKE_SOURCE_DIR}/src/box/key_def.h > + ${CMAKE_SOURCE_DIR}/src/box/lua/key_def.h > ${CMAKE_SOURCE_DIR}/src/box/field_def.h > ${CMAKE_SOURCE_DIR}/src/box/tuple.h > ${CMAKE_SOURCE_DIR}/src/box/tuple_format.h > diff --git a/src/box/CMakeLists.txt b/src/box/CMakeLists.txt > index 5521e489e..0db093768 100644 > --- a/src/box/CMakeLists.txt > +++ b/src/box/CMakeLists.txt > @@ -139,6 +139,7 @@ add_library(box STATIC > lua/net_box.c > lua/xlog.c > lua/sql.c > + lua/key_def.c > ${bin_sources}) > > target_link_libraries(box box_error tuple stat xrow xlog vclock crc32 scramble > diff --git a/src/box/lua/key_def.c b/src/box/lua/key_def.c > new file mode 100644 > index 000000000..60247f427 > --- /dev/null > +++ b/src/box/lua/key_def.c > @@ -0,0 +1,217 @@ > +/* > + * Copyright 2010-2018, Tarantool AUTHORS, please see AUTHORS file. > + * > + * Redistribution and use in source and binary forms, with or > + * without modification, are permitted provided that the following > + * conditions are met: > + * > + * 1. Redistributions of source code must retain the above > + * copyright notice, this list of conditions and the > + * following disclaimer. > + * > + * 2. Redistributions in binary form must reproduce the above > + * copyright notice, this list of conditions and the following > + * disclaimer in the documentation and/or other materials > + * provided with the distribution. > + * > + * THIS SOFTWARE IS PROVIDED BY <COPYRIGHT HOLDER> ``AS IS'' AND > + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED > + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL > + * <COPYRIGHT HOLDER> OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, > + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL > + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF > + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR > + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF > + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF > + * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF > + * SUCH DAMAGE. > + */ > + > +#include "box/lua/key_def.h" > + > +#include <lua.h> > +#include <lauxlib.h> > +#include "diag.h" > +#include "box/key_def.h" > +#include "box/box.h" > +#include "box/coll_id_cache.h" > +#include "lua/utils.h" > + > +struct key_def * > +luaT_new_key_def(struct lua_State *L, int idx) If you agree with luaT_tuple_new, then rename this function to luaT_key_def_new pls. > +{ > + if (lua_istable(L, idx) != 1) { > + luaL_error(L, "Bad params, use: luaT_new_key_def({" > + "{fieldno = fieldno, type = type" > + "[, is_nullable = is_nullable" > + "[, collation_id = collation_id" Hm, what's collation_id for? > + "[, collation = collation]]]}, ...}"); This looks like you can't specify collation without is_nullable. Should be luaT_new_key_def({{fieldno = FIELDNO, type = TYPE[, is_nullable = true | false][, collation = COLLATION]}}) > + unreachable(); > + return NULL; > + } > + uint32_t key_parts_count = 0; > + uint32_t capacity = 8; > + > + const ssize_t parts_size = sizeof(struct key_part_def) * capacity; Can't we figure out the table length right away instead of reallocaing key_part_def array? > + struct key_part_def *parts = NULL; > + parts = (struct key_part_def *) malloc(parts_size); > + if (parts == NULL) { > + diag_set(OutOfMemory, parts_size, "malloc", "parts"); > + luaT_error(L); > + unreachable(); > + return NULL; > + } > + > + while (true) { Would be nice to factor out part creation to a separate function. > + lua_pushinteger(L, key_parts_count + 1); We would call this variable key_part_count (without 's') or even just part_count, as you called the array of key parts simply 'parts'. > + lua_gettable(L, idx); > + if (lua_isnil(L, -1)) > + break; > + > + /* Extend parts if necessary. */ > + if (key_parts_count == capacity) { > + capacity *= 2; > + struct key_part_def *old_parts = parts; > + const ssize_t parts_size = > + sizeof(struct key_part_def) * capacity; > + parts = (struct key_part_def *) realloc(parts, > + parts_size); > + if (parts == NULL) { > + free(old_parts); > + diag_set(OutOfMemory, parts_size / 2, "malloc", > + "parts"); > + luaT_error(L); > + unreachable(); > + return NULL; > + } > + } > + > + /* Set parts[key_parts_count].fieldno. */ > + lua_pushstring(L, "fieldno"); > + lua_gettable(L, -2); > + if (lua_isnil(L, -1)) { > + free(parts); > + luaL_error(L, "fieldno must not be nil"); > + unreachable(); > + return NULL; > + } > + /* > + * Transform one-based Lua fieldno to zero-based > + * fieldno to use in key_def_new(). > + */ > + parts[key_parts_count].fieldno = lua_tointeger(L, -1) - 1; Use TUPLE_INDEX_BASE instead of 1 pls. > + lua_pop(L, 1); > + > + /* Set parts[key_parts_count].type. */ > + lua_pushstring(L, "type"); > + lua_gettable(L, -2); > + if (lua_isnil(L, -1)) { > + free(parts); > + luaL_error(L, "type must not be nil"); > + unreachable(); > + return NULL; > + } > + size_t type_len; > + const char *type_name = lua_tolstring(L, -1, &type_len); > + lua_pop(L, 1); > + parts[key_parts_count].type = field_type_by_name(type_name, > + type_len); > + if (parts[key_parts_count].type == field_type_MAX) { > + free(parts); > + luaL_error(L, "Unknown field type: %s", type_name); > + unreachable(); > + return NULL; > + } > + > + /* > + * Set parts[key_parts_count].is_nullable and > + * parts[key_parts_count].nullable_action. > + */ > + lua_pushstring(L, "is_nullable"); > + lua_gettable(L, -2); > + if (lua_isnil(L, -1)) { > + parts[key_parts_count].is_nullable = false; > + parts[key_parts_count].nullable_action = > + ON_CONFLICT_ACTION_DEFAULT; > + } else { > + parts[key_parts_count].is_nullable = > + lua_toboolean(L, -1); > + parts[key_parts_count].nullable_action = > + ON_CONFLICT_ACTION_NONE; > + } > + lua_pop(L, 1); > + > + /* Set parts[key_parts_count].coll_id using collation_id. */ > + lua_pushstring(L, "collation_id"); > + lua_gettable(L, -2); > + if (lua_isnil(L, -1)) > + parts[key_parts_count].coll_id = COLL_NONE; > + else > + parts[key_parts_count].coll_id = lua_tointeger(L, -1); > + lua_pop(L, 1); > + > + /* Set parts[key_parts_count].coll_id using collation. */ > + lua_pushstring(L, "collation"); > + lua_gettable(L, -2); > + /* Check whether box.cfg{} was called. */ Collations should be usable even without box.cfg IIRC. Well, not all of them I think, but still you don't need to check box.cfg() here AFAIU. > + if ((parts[key_parts_count].coll_id != COLL_NONE || > + !lua_isnil(L, -1)) && !box_is_configured()) { > + free(parts); > + luaL_error(L, "Cannot use collations: " > + "please call box.cfg{}"); > + unreachable(); > + return NULL; > + } > + if (!lua_isnil(L, -1)) { > + if (parts[key_parts_count].coll_id != COLL_NONE) { > + free(parts); > + luaL_error(L, "Conflicting options: " > + "collation_id and collation"); > + unreachable(); > + return NULL; > + } > + size_t coll_name_len; > + const char *coll_name = lua_tolstring(L, -1, > + &coll_name_len); > + struct coll_id *coll_id = coll_by_name(coll_name, > + coll_name_len); Ouch, this doesn't seem to belong here. Ideally, it should be done by key_def_new(). Can we rework key_part_def so that it stores collation string instead of collation id? > + if (coll_id == NULL) { > + free(parts); > + luaL_error(L, "Unknown collation: \"%s\"", > + coll_name); > + unreachable(); > + return NULL; > + } > + parts[key_parts_count].coll_id = coll_id->id; > + } > + lua_pop(L, 1); > + > + /* Check coll_id. */ > + struct coll_id *coll_id = > + coll_by_id(parts[key_parts_count].coll_id); > + if (parts[key_parts_count].coll_id != COLL_NONE && > + coll_id == NULL) { > + uint32_t collation_id = parts[key_parts_count].coll_id; > + free(parts); > + luaL_error(L, "Unknown collation_id: %d", collation_id); > + unreachable(); > + return NULL; > + } > + > + /* Set parts[key_parts_count].sort_order. */ > + parts[key_parts_count].sort_order = SORT_ORDER_ASC; > + > + ++key_parts_count; > + } > + > + struct key_def *key_def = key_def_new(parts, key_parts_count); > + free(parts); > + if (key_def == NULL) { > + luaL_error(L, "Cannot create key_def"); > + unreachable(); > + return NULL; > + } > + return key_def; > +} > diff --git a/src/box/lua/key_def.h b/src/box/lua/key_def.h > new file mode 100644 > index 000000000..55292fb7e > --- /dev/null > +++ b/src/box/lua/key_def.h > @@ -0,0 +1,61 @@ > +#ifndef TARANTOOL_BOX_LUA_KEY_DEF_H_INCLUDED > +#define TARANTOOL_BOX_LUA_KEY_DEF_H_INCLUDED > +/* > + * Copyright 2010-2018, Tarantool AUTHORS, please see AUTHORS file. > + * > + * Redistribution and use in source and binary forms, with or > + * without modification, are permitted provided that the following > + * conditions are met: > + * > + * 1. Redistributions of source code must retain the above > + * copyright notice, this list of conditions and the > + * following disclaimer. > + * > + * 2. Redistributions in binary form must reproduce the above > + * copyright notice, this list of conditions and the following > + * disclaimer in the documentation and/or other materials > + * provided with the distribution. > + * > + * THIS SOFTWARE IS PROVIDED BY AUTHORS ``AS IS'' AND > + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED > + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL > + * AUTHORS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, > + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL > + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF > + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR > + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF > + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF > + * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF > + * SUCH DAMAGE. > + */ > + > +#if defined(__cplusplus) > +extern "C" { > +#endif /* defined(__cplusplus) */ > + > +struct key_def; > +struct lua_State; > + > +/** \cond public */ > + > +/** > + * Create the new key_def from a Lua table. a key_def > + * > + * Expected a table of key parts on the Lua stack. The format is > + * the same as box.space.<...>.index.<...>.parts or corresponding > + * net.box's one. > + * > + * Returns the new key_def. > + */ > +struct key_def * > +luaT_new_key_def(struct lua_State *L, int idx); > + > +/** \endcond public */ > + > +#if defined(__cplusplus) > +} /* extern "C" */ > +#endif /* defined(__cplusplus) */ > + > +#endif /* TARANTOOL_BOX_LUA_KEY_DEF_H_INCLUDED */ > diff --git a/test/app-tap/module_api.c b/test/app-tap/module_api.c > index b81a98056..34ab54bc0 100644 > --- a/test/app-tap/module_api.c > +++ b/test/app-tap/module_api.c > @@ -449,6 +449,18 @@ test_iscallable(lua_State *L) > return 1; > } > > +static int > +test_luaT_new_key_def(lua_State *L) > +{ > + /* > + * Ignore the return value. Here we test whether the > + * function raises an error. > + */ > + luaT_new_key_def(L, 1); It would be nice to test that it actually creates a valid key_def. Testing error conditions is less important. > + lua_pop(L, 1); > + return 0; > +} > + > LUA_API int > luaopen_module_api(lua_State *L) > { > @@ -477,6 +489,7 @@ luaopen_module_api(lua_State *L) > {"test_state", test_state}, > {"test_tostring", test_tostring}, > {"iscallable", test_iscallable}, > + {"luaT_new_key_def", test_luaT_new_key_def}, > {NULL, NULL} > }; > luaL_register(L, "module_api", lib); ^ permalink raw reply [flat|nested] 28+ messages in thread
* Re: [PATCH v2 4/6] lua: add luaT_new_key_def() 2019-01-10 13:07 ` Vladimir Davydov @ 2019-01-29 18:52 ` Alexander Turenko 2019-01-30 10:58 ` Alexander Turenko 0 siblings, 1 reply; 28+ messages in thread From: Alexander Turenko @ 2019-01-29 18:52 UTC (permalink / raw) To: Vladimir Davydov; +Cc: tarantool-patches I considered https://github.com/tarantool/tarantool/issues/3398 and decided to implement full-featured key_def lua module. Now I only created the module stub with key_def.new() function. It is enough for merger and I hope we can leave #3398 unimplemented for now (to fix it a bit later). Removed exports, moved the test cases from module-api test to a separate file. I replaced merger.context.new(key_parts) with merger.context.new(key_def.new(key_parts)). I added docbot comment, because the module becomes user visible and is employed in the merger's docbot examples. That is why I think it is better to have it documented despite the fact it is just stub for now. Other comments are below. The new patch at the end of the email. NB: branch: Totktonada/gh-3276-on-board-merger WBR, Alexander Turenko. > > +#include "box/lua/key_def.h" > > + > > +#include <lua.h> > > +#include <lauxlib.h> > > +#include "diag.h" > > +#include "box/key_def.h" > > +#include "box/box.h" > > +#include "box/coll_id_cache.h" > > +#include "lua/utils.h" > > + > > +struct key_def * > > +luaT_new_key_def(struct lua_State *L, int idx) > > If you agree with luaT_tuple_new, then rename this function to > luaT_key_def_new pls. The code was moved to lbox_key_def_new() and luaT_key_def_set_part(). > > > +{ > > + if (lua_istable(L, idx) != 1) { > > + luaL_error(L, "Bad params, use: luaT_new_key_def({" > > + "{fieldno = fieldno, type = type" > > + "[, is_nullable = is_nullable" > > + "[, collation_id = collation_id" > > Hm, what's collation_id for? net.box exposes index parts in that way: https://github.com/tarantool/tarantool/issues/3941 I'll leave collation_id here for now if you don't mind and will remove it in the scope of #3941. > > > + "[, collation = collation]]]}, ...}"); > > This looks like you can't specify collation without is_nullable. > Should be > > luaT_new_key_def({{fieldno = FIELDNO, type = TYPE[, is_nullable = true | false][, collation = COLLATION]}}) Changed to: luaL_error(L, "Bad params, use: key_def.new({" "{fieldno = fieldno, type = type" "[, is_nullable = <boolean>]" "[, collation_id = <number>]" "[, collation = <string>]}, ...}"); > > > + unreachable(); > > + return NULL; > > + } > > + uint32_t key_parts_count = 0; > > + uint32_t capacity = 8; > > + > > + const ssize_t parts_size = sizeof(struct key_part_def) * capacity; > > Can't we figure out the table length right away instead of reallocaing > key_part_def array? Sure. Fixed. > > > + struct key_part_def *parts = NULL; > > + parts = (struct key_part_def *) malloc(parts_size); > > + if (parts == NULL) { > > + diag_set(OutOfMemory, parts_size, "malloc", "parts"); > > + luaT_error(L); > > + unreachable(); > > + return NULL; > > + } > > + > > + while (true) { > > Would be nice to factor out part creation to a separate function. Done. > > > + lua_pushinteger(L, key_parts_count + 1); > > We would call this variable key_part_count (without 's') or even just > part_count, as you called the array of key parts simply 'parts'. Ok. Fixed. > > > + lua_gettable(L, idx); > > + if (lua_isnil(L, -1)) > > + break; > > + > > + /* Extend parts if necessary. */ > > + if (key_parts_count == capacity) { > > + capacity *= 2; > > + struct key_part_def *old_parts = parts; > > + const ssize_t parts_size = > > + sizeof(struct key_part_def) * capacity; > > + parts = (struct key_part_def *) realloc(parts, > > + parts_size); > > + if (parts == NULL) { > > + free(old_parts); > > + diag_set(OutOfMemory, parts_size / 2, "malloc", > > + "parts"); > > + luaT_error(L); > > + unreachable(); > > + return NULL; > > + } > > + } > > + > > + /* Set parts[key_parts_count].fieldno. */ > > + lua_pushstring(L, "fieldno"); > > + lua_gettable(L, -2); > > + if (lua_isnil(L, -1)) { > > + free(parts); > > + luaL_error(L, "fieldno must not be nil"); > > + unreachable(); > > + return NULL; > > + } > > + /* > > + * Transform one-based Lua fieldno to zero-based > > + * fieldno to use in key_def_new(). > > + */ > > + parts[key_parts_count].fieldno = lua_tointeger(L, -1) - 1; > > Use TUPLE_INDEX_BASE instead of 1 pls. Fixed. > > > + lua_pop(L, 1); > > + > > + /* Set parts[key_parts_count].type. */ > > + lua_pushstring(L, "type"); > > + lua_gettable(L, -2); > > + if (lua_isnil(L, -1)) { > > + free(parts); > > + luaL_error(L, "type must not be nil"); > > + unreachable(); > > + return NULL; > > + } > > + size_t type_len; > > + const char *type_name = lua_tolstring(L, -1, &type_len); > > + lua_pop(L, 1); > > + parts[key_parts_count].type = field_type_by_name(type_name, > > + type_len); > > + if (parts[key_parts_count].type == field_type_MAX) { > > + free(parts); > > + luaL_error(L, "Unknown field type: %s", type_name); > > + unreachable(); > > + return NULL; > > + } > > + > > + /* > > + * Set parts[key_parts_count].is_nullable and > > + * parts[key_parts_count].nullable_action. > > + */ > > + lua_pushstring(L, "is_nullable"); > > + lua_gettable(L, -2); > > + if (lua_isnil(L, -1)) { > > + parts[key_parts_count].is_nullable = false; > > + parts[key_parts_count].nullable_action = > > + ON_CONFLICT_ACTION_DEFAULT; > > + } else { > > + parts[key_parts_count].is_nullable = > > + lua_toboolean(L, -1); > > + parts[key_parts_count].nullable_action = > > + ON_CONFLICT_ACTION_NONE; > > + } > > + lua_pop(L, 1); > > + > > + /* Set parts[key_parts_count].coll_id using collation_id. */ > > + lua_pushstring(L, "collation_id"); > > + lua_gettable(L, -2); > > + if (lua_isnil(L, -1)) > > + parts[key_parts_count].coll_id = COLL_NONE; > > + else > > + parts[key_parts_count].coll_id = lua_tointeger(L, -1); > > + lua_pop(L, 1); > > + > > + /* Set parts[key_parts_count].coll_id using collation. */ > > + lua_pushstring(L, "collation"); > > + lua_gettable(L, -2); > > + /* Check whether box.cfg{} was called. */ > > Collations should be usable even without box.cfg IIRC. Well, not all of > them I think, but still you don't need to check box.cfg() here AFAIU. Removed box.cfg{} check. No collations are available before box.cfg{} and we'll get an error for any collation ('Unknown collation "foo"' when it is pointed by name). Removed coll_id correctness check: it is performed if key_def_new() anyway. > > > + if ((parts[key_parts_count].coll_id != COLL_NONE || > > + !lua_isnil(L, -1)) && !box_is_configured()) { > > + free(parts); > > + luaL_error(L, "Cannot use collations: " > > + "please call box.cfg{}"); > > + unreachable(); > > + return NULL; > > + } > > + if (!lua_isnil(L, -1)) { > > + if (parts[key_parts_count].coll_id != COLL_NONE) { > > + free(parts); > > + luaL_error(L, "Conflicting options: " > > + "collation_id and collation"); > > + unreachable(); > > + return NULL; > > + } > > + size_t coll_name_len; > > + const char *coll_name = lua_tolstring(L, -1, > > + &coll_name_len); > > + struct coll_id *coll_id = coll_by_name(coll_name, > > + coll_name_len); > > Ouch, this doesn't seem to belong here. Ideally, it should be done by > key_def_new(). Can we rework key_part_def so that it stores collation > string instead of collation id? Can it have negative performance impact? It seems we don't compare key_defs, but I don't sure. The format of vy_log_record_encode() will change and we'll need to create upgrade script for old format. We use numeric collation IDs in many places. It seems the change is possible, but will heavily increase scope of work. I would skip it for now if you don't mind. I didn't filed an issue, because I don't sure how refactored collation support should look at whole. Maybe later. > > > + if (coll_id == NULL) { > > + free(parts); > > + luaL_error(L, "Unknown collation: \"%s\"", > > + coll_name); > > + unreachable(); > > + return NULL; > > + } > > + parts[key_parts_count].coll_id = coll_id->id; > > + } > > + lua_pop(L, 1); > > + > > + /* Check coll_id. */ > > + struct coll_id *coll_id = > > + coll_by_id(parts[key_parts_count].coll_id); > > + if (parts[key_parts_count].coll_id != COLL_NONE && > > + coll_id == NULL) { > > + uint32_t collation_id = parts[key_parts_count].coll_id; > > + free(parts); > > + luaL_error(L, "Unknown collation_id: %d", collation_id); > > + unreachable(); > > + return NULL; > > + } > > + > > + /* Set parts[key_parts_count].sort_order. */ > > + parts[key_parts_count].sort_order = SORT_ORDER_ASC; > > + > > + ++key_parts_count; > > + } > > + > > + struct key_def *key_def = key_def_new(parts, key_parts_count); > > + free(parts); > > + if (key_def == NULL) { > > + luaL_error(L, "Cannot create key_def"); > > + unreachable(); > > + return NULL; > > + } > > + return key_def; > > +} > > +#if defined(__cplusplus) > > +extern "C" { > > +#endif /* defined(__cplusplus) */ > > + > > +struct key_def; > > +struct lua_State; > > + > > +/** \cond public */ > > + > > +/** > > + * Create the new key_def from a Lua table. > > a key_def Fixed. > > diff --git a/test/app-tap/module_api.c b/test/app-tap/module_api.c > > index b81a98056..34ab54bc0 100644 > > --- a/test/app-tap/module_api.c > > +++ b/test/app-tap/module_api.c > > @@ -449,6 +449,18 @@ test_iscallable(lua_State *L) > > return 1; > > } > > > > +static int > > +test_luaT_new_key_def(lua_State *L) > > +{ > > + /* > > + * Ignore the return value. Here we test whether the > > + * function raises an error. > > + */ > > + luaT_new_key_def(L, 1); > > It would be nice to test that it actually creates a valid key_def. > Testing error conditions is less important. Now I can check a type of returned lua object in a successful case and that is all. When we'll add compare functions (in the scope #3398) we can test they are works as expected. I have added type checks of results of some successful key_def.new() invocations. ---- commit 7eae92c26421b99159892638c366a90f0d6af877 Author: Alexander Turenko <alexander.turenko@tarantool.org> Date: Mon Jan 7 19:12:50 2019 +0300 lua: add key_def lua module There are two reasons to add this module: * Incapsulate key_def creation from a Lua table (factor it out from merger's code). * Support comparing tuple with tuple and/or tuple with key from Lua in the future. The format of `parts` parameter in the `key_def.new(parts)` call is compatible with the following structures: * box.space[...].index[...].parts; * net_box_conn.space[...].index[...].parts. Needed for #3276. Needed for #3398. @TarantoolBot document Title: Document built-in key_def lua module Now there is only stub with the `key_def.new(parts)` function that returns cdata<struct key_def &>. The only way to use it for now is pass it to the merger. This module will be improved in the scope of https://github.com/tarantool/tarantool/issues/3398 See the commit message for more info. diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 04de5ad04..494c8d391 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -202,6 +202,7 @@ set(api_headers ${CMAKE_SOURCE_DIR}/src/lua/error.h ${CMAKE_SOURCE_DIR}/src/box/txn.h ${CMAKE_SOURCE_DIR}/src/box/key_def.h + ${CMAKE_SOURCE_DIR}/src/box/lua/key_def.h ${CMAKE_SOURCE_DIR}/src/box/field_def.h ${CMAKE_SOURCE_DIR}/src/box/tuple.h ${CMAKE_SOURCE_DIR}/src/box/tuple_format.h diff --git a/src/box/CMakeLists.txt b/src/box/CMakeLists.txt index 5521e489e..0db093768 100644 --- a/src/box/CMakeLists.txt +++ b/src/box/CMakeLists.txt @@ -139,6 +139,7 @@ add_library(box STATIC lua/net_box.c lua/xlog.c lua/sql.c + lua/key_def.c ${bin_sources}) target_link_libraries(box box_error tuple stat xrow xlog vclock crc32 scramble diff --git a/src/box/lua/init.c b/src/box/lua/init.c index 0e90f6be5..885354ace 100644 --- a/src/box/lua/init.c +++ b/src/box/lua/init.c @@ -59,6 +59,7 @@ #include "box/lua/console.h" #include "box/lua/tuple.h" #include "box/lua/sql.h" +#include "box/lua/key_def.h" extern char session_lua[], tuple_lua[], @@ -312,6 +313,8 @@ box_lua_init(struct lua_State *L) lua_pop(L, 1); tarantool_lua_console_init(L); lua_pop(L, 1); + luaopen_key_def(L); + lua_pop(L, 1); /* Load Lua extension */ for (const char **s = lua_sources; *s; s += 2) { diff --git a/src/box/lua/key_def.c b/src/box/lua/key_def.c new file mode 100644 index 000000000..f372048a6 --- /dev/null +++ b/src/box/lua/key_def.c @@ -0,0 +1,226 @@ +/* + * Copyright 2010-2018, Tarantool AUTHORS, please see AUTHORS file. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * 1. Redistributions of source code must retain the above + * copyright notice, this list of conditions and the + * following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY <COPYRIGHT HOLDER> ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL + * <COPYRIGHT HOLDER> OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF + * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include "box/lua/key_def.h" + +#include <lua.h> +#include <lauxlib.h> +#include "diag.h" +#include "box/key_def.h" +#include "box/box.h" +#include "box/coll_id_cache.h" +#include "lua/utils.h" +#include "box/tuple_format.h" /* TUPLE_INDEX_BASE */ + +static uint32_t key_def_type_id = 0; + +/** + * Set key_part_def from a table on top of a Lua stack. + * + * When successful return 0, otherwise return -1 and set a diag. + */ +static int +luaT_key_def_set_part(struct lua_State *L, struct key_part_def *part) +{ + /* Set part->fieldno. */ + lua_pushstring(L, "fieldno"); + lua_gettable(L, -2); + if (lua_isnil(L, -1)) { + diag_set(IllegalParams, "fieldno must not be nil"); + return -1; + } + /* + * Transform one-based Lua fieldno to zero-based + * fieldno to use in key_def_new(). + */ + part->fieldno = lua_tointeger(L, -1) - TUPLE_INDEX_BASE; + lua_pop(L, 1); + + /* Set part->type. */ + lua_pushstring(L, "type"); + lua_gettable(L, -2); + if (lua_isnil(L, -1)) { + diag_set(IllegalParams, "type must not be nil"); + return -1; + } + size_t type_len; + const char *type_name = lua_tolstring(L, -1, &type_len); + lua_pop(L, 1); + part->type = field_type_by_name(type_name, type_len); + if (part->type == field_type_MAX) { + diag_set(IllegalParams, "Unknown field type: %s", type_name); + return -1; + } + + /* Set part->is_nullable and part->nullable_action. */ + lua_pushstring(L, "is_nullable"); + lua_gettable(L, -2); + if (lua_isnil(L, -1)) { + part->is_nullable = false; + part->nullable_action = ON_CONFLICT_ACTION_DEFAULT; + } else { + part->is_nullable = lua_toboolean(L, -1); + part->nullable_action = ON_CONFLICT_ACTION_NONE; + } + lua_pop(L, 1); + + /* + * Set part->coll_id using collation_id. + * + * The value will be checked in key_def_new(). + */ + lua_pushstring(L, "collation_id"); + lua_gettable(L, -2); + if (lua_isnil(L, -1)) + part->coll_id = COLL_NONE; + else + part->coll_id = lua_tointeger(L, -1); + lua_pop(L, 1); + + /* Set part->coll_id using collation. */ + lua_pushstring(L, "collation"); + lua_gettable(L, -2); + if (!lua_isnil(L, -1)) { + /* Check for conflicting options. */ + if (part->coll_id != COLL_NONE) { + diag_set(IllegalParams, "Conflicting options: " + "collation_id and collation"); + return -1; + } + + size_t coll_name_len; + const char *coll_name = lua_tolstring(L, -1, &coll_name_len); + struct coll_id *coll_id = coll_by_name(coll_name, + coll_name_len); + if (coll_id == NULL) { + diag_set(IllegalParams, "Unknown collation: \"%s\"", + coll_name); + return -1; + } + part->coll_id = coll_id->id; + } + lua_pop(L, 1); + + /* Set part->sort_order. */ + part->sort_order = SORT_ORDER_ASC; + + return 0; +} + +struct key_def * +check_key_def(struct lua_State *L, int idx) +{ + if (lua_type(L, idx) != LUA_TCDATA) + return NULL; + + uint32_t cdata_type; + struct key_def **key_def_ptr = luaL_checkcdata(L, idx, &cdata_type); + if (key_def_ptr == NULL || cdata_type != key_def_type_id) + return NULL; + return *key_def_ptr; +} + +/** + * Free a key_def from a Lua code. + */ +static int +lbox_key_def_gc(struct lua_State *L) +{ + struct key_def *key_def = check_key_def(L, 1); + if (key_def == NULL) + return 0; + box_key_def_delete(key_def); + return 0; +} + +/** + * Create a new key_def from a Lua table. + * + * Expected a table of key parts on the Lua stack. The format is + * the same as box.space.<...>.index.<...>.parts or corresponding + * net.box's one. + * + * Return the new key_def as cdata. + */ +static int +lbox_key_def_new(struct lua_State *L) +{ + if (lua_gettop(L) != 1 || lua_istable(L, 1) != 1) + return luaL_error(L, "Bad params, use: key_def.new({" + "{fieldno = fieldno, type = type" + "[, is_nullable = <boolean>]" + "[, collation_id = <number>]" + "[, collation = <string>]}, ...}"); + + uint32_t part_count = lua_objlen(L, 1); + const ssize_t parts_size = sizeof(struct key_part_def) * part_count; + struct key_part_def *parts = malloc(parts_size); + if (parts == NULL) { + diag_set(OutOfMemory, parts_size, "malloc", "parts"); + return luaT_error(L); + } + + for (uint32_t i = 0; i < part_count; ++i) { + lua_pushinteger(L, i + 1); + lua_gettable(L, 1); + if (luaT_key_def_set_part(L, &parts[i]) != 0) { + free(parts); + return luaT_error(L); + } + } + + struct key_def *key_def = key_def_new(parts, part_count); + free(parts); + if (key_def == NULL) + return luaT_error(L); + + *(struct key_def **) luaL_pushcdata(L, key_def_type_id) = key_def; + lua_pushcfunction(L, lbox_key_def_gc); + luaL_setcdatagc(L, -2); + + return 1; +} + +LUA_API int +luaopen_key_def(struct lua_State *L) +{ + luaL_cdef(L, "struct key_def;"); + key_def_type_id = luaL_ctypeid(L, "struct key_def&"); + + /* Export C functions to Lua. */ + static const struct luaL_Reg meta[] = { + {"new", lbox_key_def_new}, + {NULL, NULL} + }; + luaL_register_module(L, "key_def", meta); + + return 1; +} diff --git a/src/box/lua/key_def.h b/src/box/lua/key_def.h new file mode 100644 index 000000000..11cc0bfd4 --- /dev/null +++ b/src/box/lua/key_def.h @@ -0,0 +1,56 @@ +#ifndef TARANTOOL_BOX_LUA_KEY_DEF_H_INCLUDED +#define TARANTOOL_BOX_LUA_KEY_DEF_H_INCLUDED +/* + * Copyright 2010-2018, Tarantool AUTHORS, please see AUTHORS file. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * 1. Redistributions of source code must retain the above + * copyright notice, this list of conditions and the + * following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY AUTHORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL + * AUTHORS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF + * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#if defined(__cplusplus) +extern "C" { +#endif /* defined(__cplusplus) */ + +struct lua_State; + +/** + * Extract a key_def object from a Lua stack. + */ +struct key_def * +check_key_def(struct lua_State *L, int idx); + +/** + * Register the module. + */ +int +luaopen_key_def(struct lua_State *L); + +#if defined(__cplusplus) +} /* extern "C" */ +#endif /* defined(__cplusplus) */ + +#endif /* TARANTOOL_BOX_LUA_KEY_DEF_H_INCLUDED */ diff --git a/test/box-tap/key_def.test.lua b/test/box-tap/key_def.test.lua new file mode 100755 index 000000000..7e6e0e330 --- /dev/null +++ b/test/box-tap/key_def.test.lua @@ -0,0 +1,137 @@ +#!/usr/bin/env tarantool + +local tap = require('tap') +local ffi = require('ffi') +local key_def = require('key_def') + +local usage_error = 'Bad params, use: key_def.new({' .. + '{fieldno = fieldno, type = type' .. + '[, is_nullable = <boolean>]' .. + '[, collation_id = <number>]' .. + '[, collation = <string>]}, ...}' + +local function coll_not_found(fieldno, collation) + if type(collation) == 'number' then + return ('Wrong index options (field %d): ' .. + 'collation was not found by ID'):format(fieldno) + end + + return ('Unknown collation: "%s"'):format(collation) +end + +local cases = { + -- Cases to call before box.cfg{}. + { + 'Pass a field on an unknown type', + parts = {{ + fieldno = 2, + type = 'unknown', + }}, + exp_err = 'Unknown field type: unknown', + }, + { + 'Try to use collation_id before box.cfg{}', + parts = {{ + fieldno = 1, + type = 'string', + collation_id = 2, + }}, + exp_err = coll_not_found(1, 2), + }, + { + 'Try to use collation before box.cfg{}', + parts = {{ + fieldno = 1, + type = 'string', + collation = 'unicode_ci', + }}, + exp_err = coll_not_found(1, 'unicode_ci'), + }, + function() + -- For collations. + box.cfg{} + end, + -- Cases to call after box.cfg{}. + { + 'Try to use both collation_id and collation', + parts = {{ + fieldno = 1, + type = 'string', + collation_id = 2, + collation = 'unicode_ci', + }}, + exp_err = 'Conflicting options: collation_id and collation', + }, + { + 'Unknown collation_id', + parts = {{ + fieldno = 1, + type = 'string', + collation_id = 42, + }}, + exp_err = coll_not_found(1, 42), + }, + { + 'Unknown collation name', + parts = {{ + fieldno = 1, + type = 'string', + collation = 'unknown', + }}, + exp_err = 'Unknown collation: "unknown"', + }, + { + 'Bad parts parameter type', + parts = 1, + exp_err = usage_error, + }, + { + 'No parameters', + params = {}, + exp_err = usage_error, + }, + { + 'Two parameters', + params = {{}, {}}, + exp_err = usage_error, + }, + { + 'Success case; zero parts', + parts = {}, + exp_err = nil, + }, + { + 'Success case; one part', + parts = { + fieldno = 1, + type = 'string', + }, + exp_err = nil, + }, +} + +local test = tap.test('key_def') + +test:plan(#cases - 1) +for _, case in ipairs(cases) do + if type(case) == 'function' then + case() + else + local ok, res + if case.params then + ok, res = pcall(key_def.new, unpack(case.params)) + else + ok, res = pcall(key_def.new, case.parts) + end + if case.exp_err == nil then + ok = ok and type(res) == 'cdata' and + ffi.istype('struct key_def', res) + test:ok(ok, case[1]) + else + local err = tostring(res) -- cdata -> string + test:is_deeply({ok, err}, {false, case.exp_err}, case[1]) + end + end +end + +os.exit(test:check() and 0 or 1) ^ permalink raw reply [flat|nested] 28+ messages in thread
* Re: [PATCH v2 4/6] lua: add luaT_new_key_def() 2019-01-29 18:52 ` Alexander Turenko @ 2019-01-30 10:58 ` Alexander Turenko 2019-03-01 4:10 ` Alexander Turenko 0 siblings, 1 reply; 28+ messages in thread From: Alexander Turenko @ 2019-01-30 10:58 UTC (permalink / raw) To: Vladimir Davydov; +Cc: tarantool-patches Updated a bit: diff --git a/src/box/lua/key_def.c b/src/box/lua/key_def.c index f372048a6..a77497384 100644 --- a/src/box/lua/key_def.c +++ b/src/box/lua/key_def.c @@ -75,9 +75,20 @@ luaT_key_def_set_part(struct lua_State *L, struct key_part_def *part) const char *type_name = lua_tolstring(L, -1, &type_len); lua_pop(L, 1); part->type = field_type_by_name(type_name, type_len); - if (part->type == field_type_MAX) { + switch (part->type) { + case FIELD_TYPE_ANY: + case FIELD_TYPE_ARRAY: + case FIELD_TYPE_MAP: + /* Tuple comparators don't support these types. */ + diag_set(IllegalParams, "Unsupported field type: %s", + type_name); + return -1; + case field_type_MAX: diag_set(IllegalParams, "Unknown field type: %s", type_name); return -1; + default: + /* Pass though. */ + break; } /* Set part->is_nullable and part->nullable_action. */ WBR, Alexander Turenko. On Tue, Jan 29, 2019 at 09:52:38PM +0300, Alexander Turenko wrote: > I considered https://github.com/tarantool/tarantool/issues/3398 and > decided to implement full-featured key_def lua module. Now I only > created the module stub with key_def.new() function. It is enough for > merger and I hope we can leave #3398 unimplemented for now (to fix it a > bit later). > > Removed exports, moved the test cases from module-api test to a separate > file. > > I replaced merger.context.new(key_parts) with > merger.context.new(key_def.new(key_parts)). > > I added docbot comment, because the module becomes user visible and is > employed in the merger's docbot examples. That is why I think it is > better to have it documented despite the fact it is just stub for now. > > Other comments are below. The new patch at the end of the email. > > NB: branch: Totktonada/gh-3276-on-board-merger > > WBR, Alexander Turenko. > > > > +#include "box/lua/key_def.h" > > > + > > > +#include <lua.h> > > > +#include <lauxlib.h> > > > +#include "diag.h" > > > +#include "box/key_def.h" > > > +#include "box/box.h" > > > +#include "box/coll_id_cache.h" > > > +#include "lua/utils.h" > > > + > > > +struct key_def * > > > +luaT_new_key_def(struct lua_State *L, int idx) > > > > If you agree with luaT_tuple_new, then rename this function to > > luaT_key_def_new pls. > > The code was moved to lbox_key_def_new() and luaT_key_def_set_part(). > > > > > > +{ > > > + if (lua_istable(L, idx) != 1) { > > > + luaL_error(L, "Bad params, use: luaT_new_key_def({" > > > + "{fieldno = fieldno, type = type" > > > + "[, is_nullable = is_nullable" > > > + "[, collation_id = collation_id" > > > > Hm, what's collation_id for? > > net.box exposes index parts in that way: > https://github.com/tarantool/tarantool/issues/3941 > > I'll leave collation_id here for now if you don't mind and will remove > it in the scope of #3941. > > > > > > + "[, collation = collation]]]}, ...}"); > > > > This looks like you can't specify collation without is_nullable. > > Should be > > > > luaT_new_key_def({{fieldno = FIELDNO, type = TYPE[, is_nullable = true | false][, collation = COLLATION]}}) > > Changed to: > > luaL_error(L, "Bad params, use: key_def.new({" > "{fieldno = fieldno, type = type" > "[, is_nullable = <boolean>]" > "[, collation_id = <number>]" > "[, collation = <string>]}, ...}"); > > > > > > + unreachable(); > > > + return NULL; > > > + } > > > + uint32_t key_parts_count = 0; > > > + uint32_t capacity = 8; > > > + > > > + const ssize_t parts_size = sizeof(struct key_part_def) * capacity; > > > > Can't we figure out the table length right away instead of reallocaing > > key_part_def array? > > Sure. Fixed. > > > > > > + struct key_part_def *parts = NULL; > > > + parts = (struct key_part_def *) malloc(parts_size); > > > + if (parts == NULL) { > > > + diag_set(OutOfMemory, parts_size, "malloc", "parts"); > > > + luaT_error(L); > > > + unreachable(); > > > + return NULL; > > > + } > > > + > > > + while (true) { > > > > Would be nice to factor out part creation to a separate function. > > Done. > > > > > > + lua_pushinteger(L, key_parts_count + 1); > > > > We would call this variable key_part_count (without 's') or even just > > part_count, as you called the array of key parts simply 'parts'. > > Ok. Fixed. > > > > > > + lua_gettable(L, idx); > > > + if (lua_isnil(L, -1)) > > > + break; > > > + > > > + /* Extend parts if necessary. */ > > > + if (key_parts_count == capacity) { > > > + capacity *= 2; > > > + struct key_part_def *old_parts = parts; > > > + const ssize_t parts_size = > > > + sizeof(struct key_part_def) * capacity; > > > + parts = (struct key_part_def *) realloc(parts, > > > + parts_size); > > > + if (parts == NULL) { > > > + free(old_parts); > > > + diag_set(OutOfMemory, parts_size / 2, "malloc", > > > + "parts"); > > > + luaT_error(L); > > > + unreachable(); > > > + return NULL; > > > + } > > > + } > > > + > > > + /* Set parts[key_parts_count].fieldno. */ > > > + lua_pushstring(L, "fieldno"); > > > + lua_gettable(L, -2); > > > + if (lua_isnil(L, -1)) { > > > + free(parts); > > > + luaL_error(L, "fieldno must not be nil"); > > > + unreachable(); > > > + return NULL; > > > + } > > > + /* > > > + * Transform one-based Lua fieldno to zero-based > > > + * fieldno to use in key_def_new(). > > > + */ > > > + parts[key_parts_count].fieldno = lua_tointeger(L, -1) - 1; > > > > Use TUPLE_INDEX_BASE instead of 1 pls. > > Fixed. > > > > > > + lua_pop(L, 1); > > > + > > > + /* Set parts[key_parts_count].type. */ > > > + lua_pushstring(L, "type"); > > > + lua_gettable(L, -2); > > > + if (lua_isnil(L, -1)) { > > > + free(parts); > > > + luaL_error(L, "type must not be nil"); > > > + unreachable(); > > > + return NULL; > > > + } > > > + size_t type_len; > > > + const char *type_name = lua_tolstring(L, -1, &type_len); > > > + lua_pop(L, 1); > > > + parts[key_parts_count].type = field_type_by_name(type_name, > > > + type_len); > > > + if (parts[key_parts_count].type == field_type_MAX) { > > > + free(parts); > > > + luaL_error(L, "Unknown field type: %s", type_name); > > > + unreachable(); > > > + return NULL; > > > + } > > > + > > > + /* > > > + * Set parts[key_parts_count].is_nullable and > > > + * parts[key_parts_count].nullable_action. > > > + */ > > > + lua_pushstring(L, "is_nullable"); > > > + lua_gettable(L, -2); > > > + if (lua_isnil(L, -1)) { > > > + parts[key_parts_count].is_nullable = false; > > > + parts[key_parts_count].nullable_action = > > > + ON_CONFLICT_ACTION_DEFAULT; > > > + } else { > > > + parts[key_parts_count].is_nullable = > > > + lua_toboolean(L, -1); > > > + parts[key_parts_count].nullable_action = > > > + ON_CONFLICT_ACTION_NONE; > > > + } > > > + lua_pop(L, 1); > > > + > > > + /* Set parts[key_parts_count].coll_id using collation_id. */ > > > + lua_pushstring(L, "collation_id"); > > > + lua_gettable(L, -2); > > > + if (lua_isnil(L, -1)) > > > + parts[key_parts_count].coll_id = COLL_NONE; > > > + else > > > + parts[key_parts_count].coll_id = lua_tointeger(L, -1); > > > + lua_pop(L, 1); > > > + > > > + /* Set parts[key_parts_count].coll_id using collation. */ > > > + lua_pushstring(L, "collation"); > > > + lua_gettable(L, -2); > > > + /* Check whether box.cfg{} was called. */ > > > > Collations should be usable even without box.cfg IIRC. Well, not all of > > them I think, but still you don't need to check box.cfg() here AFAIU. > > Removed box.cfg{} check. No collations are available before box.cfg{} > and we'll get an error for any collation ('Unknown collation "foo"' when > it is pointed by name). > > Removed coll_id correctness check: it is performed if key_def_new() > anyway. > > > > > > + if ((parts[key_parts_count].coll_id != COLL_NONE || > > > + !lua_isnil(L, -1)) && !box_is_configured()) { > > > + free(parts); > > > + luaL_error(L, "Cannot use collations: " > > > + "please call box.cfg{}"); > > > + unreachable(); > > > + return NULL; > > > + } > > > + if (!lua_isnil(L, -1)) { > > > + if (parts[key_parts_count].coll_id != COLL_NONE) { > > > + free(parts); > > > + luaL_error(L, "Conflicting options: " > > > + "collation_id and collation"); > > > + unreachable(); > > > + return NULL; > > > + } > > > + size_t coll_name_len; > > > + const char *coll_name = lua_tolstring(L, -1, > > > + &coll_name_len); > > > + struct coll_id *coll_id = coll_by_name(coll_name, > > > + coll_name_len); > > > > Ouch, this doesn't seem to belong here. Ideally, it should be done by > > key_def_new(). Can we rework key_part_def so that it stores collation > > string instead of collation id? > > Can it have negative performance impact? It seems we don't compare > key_defs, but I don't sure. > > The format of vy_log_record_encode() will change and we'll need to > create upgrade script for old format. > > We use numeric collation IDs in many places. It seems the change is > possible, but will heavily increase scope of work. I would skip it for > now if you don't mind. I didn't filed an issue, because I don't sure how > refactored collation support should look at whole. Maybe later. > > > > > > + if (coll_id == NULL) { > > > + free(parts); > > > + luaL_error(L, "Unknown collation: \"%s\"", > > > + coll_name); > > > + unreachable(); > > > + return NULL; > > > + } > > > + parts[key_parts_count].coll_id = coll_id->id; > > > + } > > > + lua_pop(L, 1); > > > + > > > + /* Check coll_id. */ > > > + struct coll_id *coll_id = > > > + coll_by_id(parts[key_parts_count].coll_id); > > > + if (parts[key_parts_count].coll_id != COLL_NONE && > > > + coll_id == NULL) { > > > + uint32_t collation_id = parts[key_parts_count].coll_id; > > > + free(parts); > > > + luaL_error(L, "Unknown collation_id: %d", collation_id); > > > + unreachable(); > > > + return NULL; > > > + } > > > + > > > + /* Set parts[key_parts_count].sort_order. */ > > > + parts[key_parts_count].sort_order = SORT_ORDER_ASC; > > > + > > > + ++key_parts_count; > > > + } > > > + > > > + struct key_def *key_def = key_def_new(parts, key_parts_count); > > > + free(parts); > > > + if (key_def == NULL) { > > > + luaL_error(L, "Cannot create key_def"); > > > + unreachable(); > > > + return NULL; > > > + } > > > + return key_def; > > > +} > > > > +#if defined(__cplusplus) > > > +extern "C" { > > > +#endif /* defined(__cplusplus) */ > > > + > > > +struct key_def; > > > +struct lua_State; > > > + > > > +/** \cond public */ > > > + > > > +/** > > > + * Create the new key_def from a Lua table. > > > > a key_def > > Fixed. > > > > diff --git a/test/app-tap/module_api.c b/test/app-tap/module_api.c > > > index b81a98056..34ab54bc0 100644 > > > --- a/test/app-tap/module_api.c > > > +++ b/test/app-tap/module_api.c > > > @@ -449,6 +449,18 @@ test_iscallable(lua_State *L) > > > return 1; > > > } > > > > > > +static int > > > +test_luaT_new_key_def(lua_State *L) > > > +{ > > > + /* > > > + * Ignore the return value. Here we test whether the > > > + * function raises an error. > > > + */ > > > + luaT_new_key_def(L, 1); > > > > It would be nice to test that it actually creates a valid key_def. > > Testing error conditions is less important. > > Now I can check a type of returned lua object in a successful case and > that is all. When we'll add compare functions (in the scope #3398) we > can test they are works as expected. > > I have added type checks of results of some successful key_def.new() > invocations. > > ---- > > commit 7eae92c26421b99159892638c366a90f0d6af877 > Author: Alexander Turenko <alexander.turenko@tarantool.org> > Date: Mon Jan 7 19:12:50 2019 +0300 > > lua: add key_def lua module > > There are two reasons to add this module: > > * Incapsulate key_def creation from a Lua table (factor it out from > merger's code). > * Support comparing tuple with tuple and/or tuple with key from Lua in > the future. > > The format of `parts` parameter in the `key_def.new(parts)` call is > compatible with the following structures: > > * box.space[...].index[...].parts; > * net_box_conn.space[...].index[...].parts. > > Needed for #3276. > Needed for #3398. > > @TarantoolBot document > Title: Document built-in key_def lua module > > Now there is only stub with the `key_def.new(parts)` function that > returns cdata<struct key_def &>. The only way to use it for now is pass > it to the merger. > > This module will be improved in the scope of > https://github.com/tarantool/tarantool/issues/3398 > > See the commit message for more info. > > diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt > index 04de5ad04..494c8d391 100644 > --- a/src/CMakeLists.txt > +++ b/src/CMakeLists.txt > @@ -202,6 +202,7 @@ set(api_headers > ${CMAKE_SOURCE_DIR}/src/lua/error.h > ${CMAKE_SOURCE_DIR}/src/box/txn.h > ${CMAKE_SOURCE_DIR}/src/box/key_def.h > + ${CMAKE_SOURCE_DIR}/src/box/lua/key_def.h > ${CMAKE_SOURCE_DIR}/src/box/field_def.h > ${CMAKE_SOURCE_DIR}/src/box/tuple.h > ${CMAKE_SOURCE_DIR}/src/box/tuple_format.h > diff --git a/src/box/CMakeLists.txt b/src/box/CMakeLists.txt > index 5521e489e..0db093768 100644 > --- a/src/box/CMakeLists.txt > +++ b/src/box/CMakeLists.txt > @@ -139,6 +139,7 @@ add_library(box STATIC > lua/net_box.c > lua/xlog.c > lua/sql.c > + lua/key_def.c > ${bin_sources}) > > target_link_libraries(box box_error tuple stat xrow xlog vclock crc32 scramble > diff --git a/src/box/lua/init.c b/src/box/lua/init.c > index 0e90f6be5..885354ace 100644 > --- a/src/box/lua/init.c > +++ b/src/box/lua/init.c > @@ -59,6 +59,7 @@ > #include "box/lua/console.h" > #include "box/lua/tuple.h" > #include "box/lua/sql.h" > +#include "box/lua/key_def.h" > > extern char session_lua[], > tuple_lua[], > @@ -312,6 +313,8 @@ box_lua_init(struct lua_State *L) > lua_pop(L, 1); > tarantool_lua_console_init(L); > lua_pop(L, 1); > + luaopen_key_def(L); > + lua_pop(L, 1); > > /* Load Lua extension */ > for (const char **s = lua_sources; *s; s += 2) { > diff --git a/src/box/lua/key_def.c b/src/box/lua/key_def.c > new file mode 100644 > index 000000000..f372048a6 > --- /dev/null > +++ b/src/box/lua/key_def.c > @@ -0,0 +1,226 @@ > +/* > + * Copyright 2010-2018, Tarantool AUTHORS, please see AUTHORS file. > + * > + * Redistribution and use in source and binary forms, with or > + * without modification, are permitted provided that the following > + * conditions are met: > + * > + * 1. Redistributions of source code must retain the above > + * copyright notice, this list of conditions and the > + * following disclaimer. > + * > + * 2. Redistributions in binary form must reproduce the above > + * copyright notice, this list of conditions and the following > + * disclaimer in the documentation and/or other materials > + * provided with the distribution. > + * > + * THIS SOFTWARE IS PROVIDED BY <COPYRIGHT HOLDER> ``AS IS'' AND > + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED > + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL > + * <COPYRIGHT HOLDER> OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, > + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL > + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF > + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR > + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF > + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF > + * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF > + * SUCH DAMAGE. > + */ > + > +#include "box/lua/key_def.h" > + > +#include <lua.h> > +#include <lauxlib.h> > +#include "diag.h" > +#include "box/key_def.h" > +#include "box/box.h" > +#include "box/coll_id_cache.h" > +#include "lua/utils.h" > +#include "box/tuple_format.h" /* TUPLE_INDEX_BASE */ > + > +static uint32_t key_def_type_id = 0; > + > +/** > + * Set key_part_def from a table on top of a Lua stack. > + * > + * When successful return 0, otherwise return -1 and set a diag. > + */ > +static int > +luaT_key_def_set_part(struct lua_State *L, struct key_part_def *part) > +{ > + /* Set part->fieldno. */ > + lua_pushstring(L, "fieldno"); > + lua_gettable(L, -2); > + if (lua_isnil(L, -1)) { > + diag_set(IllegalParams, "fieldno must not be nil"); > + return -1; > + } > + /* > + * Transform one-based Lua fieldno to zero-based > + * fieldno to use in key_def_new(). > + */ > + part->fieldno = lua_tointeger(L, -1) - TUPLE_INDEX_BASE; > + lua_pop(L, 1); > + > + /* Set part->type. */ > + lua_pushstring(L, "type"); > + lua_gettable(L, -2); > + if (lua_isnil(L, -1)) { > + diag_set(IllegalParams, "type must not be nil"); > + return -1; > + } > + size_t type_len; > + const char *type_name = lua_tolstring(L, -1, &type_len); > + lua_pop(L, 1); > + part->type = field_type_by_name(type_name, type_len); > + if (part->type == field_type_MAX) { > + diag_set(IllegalParams, "Unknown field type: %s", type_name); > + return -1; > + } > + > + /* Set part->is_nullable and part->nullable_action. */ > + lua_pushstring(L, "is_nullable"); > + lua_gettable(L, -2); > + if (lua_isnil(L, -1)) { > + part->is_nullable = false; > + part->nullable_action = ON_CONFLICT_ACTION_DEFAULT; > + } else { > + part->is_nullable = lua_toboolean(L, -1); > + part->nullable_action = ON_CONFLICT_ACTION_NONE; > + } > + lua_pop(L, 1); > + > + /* > + * Set part->coll_id using collation_id. > + * > + * The value will be checked in key_def_new(). > + */ > + lua_pushstring(L, "collation_id"); > + lua_gettable(L, -2); > + if (lua_isnil(L, -1)) > + part->coll_id = COLL_NONE; > + else > + part->coll_id = lua_tointeger(L, -1); > + lua_pop(L, 1); > + > + /* Set part->coll_id using collation. */ > + lua_pushstring(L, "collation"); > + lua_gettable(L, -2); > + if (!lua_isnil(L, -1)) { > + /* Check for conflicting options. */ > + if (part->coll_id != COLL_NONE) { > + diag_set(IllegalParams, "Conflicting options: " > + "collation_id and collation"); > + return -1; > + } > + > + size_t coll_name_len; > + const char *coll_name = lua_tolstring(L, -1, &coll_name_len); > + struct coll_id *coll_id = coll_by_name(coll_name, > + coll_name_len); > + if (coll_id == NULL) { > + diag_set(IllegalParams, "Unknown collation: \"%s\"", > + coll_name); > + return -1; > + } > + part->coll_id = coll_id->id; > + } > + lua_pop(L, 1); > + > + /* Set part->sort_order. */ > + part->sort_order = SORT_ORDER_ASC; > + > + return 0; > +} > + > +struct key_def * > +check_key_def(struct lua_State *L, int idx) > +{ > + if (lua_type(L, idx) != LUA_TCDATA) > + return NULL; > + > + uint32_t cdata_type; > + struct key_def **key_def_ptr = luaL_checkcdata(L, idx, &cdata_type); > + if (key_def_ptr == NULL || cdata_type != key_def_type_id) > + return NULL; > + return *key_def_ptr; > +} > + > +/** > + * Free a key_def from a Lua code. > + */ > +static int > +lbox_key_def_gc(struct lua_State *L) > +{ > + struct key_def *key_def = check_key_def(L, 1); > + if (key_def == NULL) > + return 0; > + box_key_def_delete(key_def); > + return 0; > +} > + > +/** > + * Create a new key_def from a Lua table. > + * > + * Expected a table of key parts on the Lua stack. The format is > + * the same as box.space.<...>.index.<...>.parts or corresponding > + * net.box's one. > + * > + * Return the new key_def as cdata. > + */ > +static int > +lbox_key_def_new(struct lua_State *L) > +{ > + if (lua_gettop(L) != 1 || lua_istable(L, 1) != 1) > + return luaL_error(L, "Bad params, use: key_def.new({" > + "{fieldno = fieldno, type = type" > + "[, is_nullable = <boolean>]" > + "[, collation_id = <number>]" > + "[, collation = <string>]}, ...}"); > + > + uint32_t part_count = lua_objlen(L, 1); > + const ssize_t parts_size = sizeof(struct key_part_def) * part_count; > + struct key_part_def *parts = malloc(parts_size); > + if (parts == NULL) { > + diag_set(OutOfMemory, parts_size, "malloc", "parts"); > + return luaT_error(L); > + } > + > + for (uint32_t i = 0; i < part_count; ++i) { > + lua_pushinteger(L, i + 1); > + lua_gettable(L, 1); > + if (luaT_key_def_set_part(L, &parts[i]) != 0) { > + free(parts); > + return luaT_error(L); > + } > + } > + > + struct key_def *key_def = key_def_new(parts, part_count); > + free(parts); > + if (key_def == NULL) > + return luaT_error(L); > + > + *(struct key_def **) luaL_pushcdata(L, key_def_type_id) = key_def; > + lua_pushcfunction(L, lbox_key_def_gc); > + luaL_setcdatagc(L, -2); > + > + return 1; > +} > + > +LUA_API int > +luaopen_key_def(struct lua_State *L) > +{ > + luaL_cdef(L, "struct key_def;"); > + key_def_type_id = luaL_ctypeid(L, "struct key_def&"); > + > + /* Export C functions to Lua. */ > + static const struct luaL_Reg meta[] = { > + {"new", lbox_key_def_new}, > + {NULL, NULL} > + }; > + luaL_register_module(L, "key_def", meta); > + > + return 1; > +} > diff --git a/src/box/lua/key_def.h b/src/box/lua/key_def.h > new file mode 100644 > index 000000000..11cc0bfd4 > --- /dev/null > +++ b/src/box/lua/key_def.h > @@ -0,0 +1,56 @@ > +#ifndef TARANTOOL_BOX_LUA_KEY_DEF_H_INCLUDED > +#define TARANTOOL_BOX_LUA_KEY_DEF_H_INCLUDED > +/* > + * Copyright 2010-2018, Tarantool AUTHORS, please see AUTHORS file. > + * > + * Redistribution and use in source and binary forms, with or > + * without modification, are permitted provided that the following > + * conditions are met: > + * > + * 1. Redistributions of source code must retain the above > + * copyright notice, this list of conditions and the > + * following disclaimer. > + * > + * 2. Redistributions in binary form must reproduce the above > + * copyright notice, this list of conditions and the following > + * disclaimer in the documentation and/or other materials > + * provided with the distribution. > + * > + * THIS SOFTWARE IS PROVIDED BY AUTHORS ``AS IS'' AND > + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED > + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL > + * AUTHORS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, > + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL > + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF > + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR > + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF > + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF > + * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF > + * SUCH DAMAGE. > + */ > + > +#if defined(__cplusplus) > +extern "C" { > +#endif /* defined(__cplusplus) */ > + > +struct lua_State; > + > +/** > + * Extract a key_def object from a Lua stack. > + */ > +struct key_def * > +check_key_def(struct lua_State *L, int idx); > + > +/** > + * Register the module. > + */ > +int > +luaopen_key_def(struct lua_State *L); > + > +#if defined(__cplusplus) > +} /* extern "C" */ > +#endif /* defined(__cplusplus) */ > + > +#endif /* TARANTOOL_BOX_LUA_KEY_DEF_H_INCLUDED */ > diff --git a/test/box-tap/key_def.test.lua b/test/box-tap/key_def.test.lua > new file mode 100755 > index 000000000..7e6e0e330 > --- /dev/null > +++ b/test/box-tap/key_def.test.lua > @@ -0,0 +1,137 @@ > +#!/usr/bin/env tarantool > + > +local tap = require('tap') > +local ffi = require('ffi') > +local key_def = require('key_def') > + > +local usage_error = 'Bad params, use: key_def.new({' .. > + '{fieldno = fieldno, type = type' .. > + '[, is_nullable = <boolean>]' .. > + '[, collation_id = <number>]' .. > + '[, collation = <string>]}, ...}' > + > +local function coll_not_found(fieldno, collation) > + if type(collation) == 'number' then > + return ('Wrong index options (field %d): ' .. > + 'collation was not found by ID'):format(fieldno) > + end > + > + return ('Unknown collation: "%s"'):format(collation) > +end > + > +local cases = { > + -- Cases to call before box.cfg{}. > + { > + 'Pass a field on an unknown type', > + parts = {{ > + fieldno = 2, > + type = 'unknown', > + }}, > + exp_err = 'Unknown field type: unknown', > + }, > + { > + 'Try to use collation_id before box.cfg{}', > + parts = {{ > + fieldno = 1, > + type = 'string', > + collation_id = 2, > + }}, > + exp_err = coll_not_found(1, 2), > + }, > + { > + 'Try to use collation before box.cfg{}', > + parts = {{ > + fieldno = 1, > + type = 'string', > + collation = 'unicode_ci', > + }}, > + exp_err = coll_not_found(1, 'unicode_ci'), > + }, > + function() > + -- For collations. > + box.cfg{} > + end, > + -- Cases to call after box.cfg{}. > + { > + 'Try to use both collation_id and collation', > + parts = {{ > + fieldno = 1, > + type = 'string', > + collation_id = 2, > + collation = 'unicode_ci', > + }}, > + exp_err = 'Conflicting options: collation_id and collation', > + }, > + { > + 'Unknown collation_id', > + parts = {{ > + fieldno = 1, > + type = 'string', > + collation_id = 42, > + }}, > + exp_err = coll_not_found(1, 42), > + }, > + { > + 'Unknown collation name', > + parts = {{ > + fieldno = 1, > + type = 'string', > + collation = 'unknown', > + }}, > + exp_err = 'Unknown collation: "unknown"', > + }, > + { > + 'Bad parts parameter type', > + parts = 1, > + exp_err = usage_error, > + }, > + { > + 'No parameters', > + params = {}, > + exp_err = usage_error, > + }, > + { > + 'Two parameters', > + params = {{}, {}}, > + exp_err = usage_error, > + }, > + { > + 'Success case; zero parts', > + parts = {}, > + exp_err = nil, > + }, > + { > + 'Success case; one part', > + parts = { > + fieldno = 1, > + type = 'string', > + }, > + exp_err = nil, > + }, > +} > + > +local test = tap.test('key_def') > + > +test:plan(#cases - 1) > +for _, case in ipairs(cases) do > + if type(case) == 'function' then > + case() > + else > + local ok, res > + if case.params then > + ok, res = pcall(key_def.new, unpack(case.params)) > + else > + ok, res = pcall(key_def.new, case.parts) > + end > + if case.exp_err == nil then > + ok = ok and type(res) == 'cdata' and > + ffi.istype('struct key_def', res) > + test:ok(ok, case[1]) > + else > + local err = tostring(res) -- cdata -> string > + test:is_deeply({ok, err}, {false, case.exp_err}, case[1]) > + end > + end > +end > + > +os.exit(test:check() and 0 or 1) ^ permalink raw reply [flat|nested] 28+ messages in thread
* Re: [PATCH v2 4/6] lua: add luaT_new_key_def() 2019-01-30 10:58 ` Alexander Turenko @ 2019-03-01 4:10 ` Alexander Turenko 0 siblings, 0 replies; 28+ messages in thread From: Alexander Turenko @ 2019-03-01 4:10 UTC (permalink / raw) To: Vladimir Davydov; +Cc: tarantool-patches Updated after recent rebase: diff --git a/src/box/lua/key_def.c b/src/box/lua/key_def.c index a77497384..48d111b03 100644 --- a/src/box/lua/key_def.c +++ b/src/box/lua/key_def.c @@ -143,6 +143,9 @@ luaT_key_def_set_part(struct lua_State *L, struct key_part_def *part) /* Set part->sort_order. */ part->sort_order = SORT_ORDER_ASC; + /* Set part->path. */ + part->path = NULL; + return 0; } WBR, Alexander Turenko. On Wed, Jan 30, 2019 at 01:58:33PM +0300, Alexander Turenko wrote: > Updated a bit: > > diff --git a/src/box/lua/key_def.c b/src/box/lua/key_def.c > index f372048a6..a77497384 100644 > --- a/src/box/lua/key_def.c > +++ b/src/box/lua/key_def.c > @@ -75,9 +75,20 @@ luaT_key_def_set_part(struct lua_State *L, struct key_part_def *part) > const char *type_name = lua_tolstring(L, -1, &type_len); > lua_pop(L, 1); > part->type = field_type_by_name(type_name, type_len); > - if (part->type == field_type_MAX) { > + switch (part->type) { > + case FIELD_TYPE_ANY: > + case FIELD_TYPE_ARRAY: > + case FIELD_TYPE_MAP: > + /* Tuple comparators don't support these types. */ > + diag_set(IllegalParams, "Unsupported field type: %s", > + type_name); > + return -1; > + case field_type_MAX: > diag_set(IllegalParams, "Unknown field type: %s", type_name); > return -1; > + default: > + /* Pass though. */ > + break; > } > > /* Set part->is_nullable and part->nullable_action. */ > > WBR, Alexander Turenko. > > On Tue, Jan 29, 2019 at 09:52:38PM +0300, Alexander Turenko wrote: > > I considered https://github.com/tarantool/tarantool/issues/3398 and > > decided to implement full-featured key_def lua module. Now I only > > created the module stub with key_def.new() function. It is enough for > > merger and I hope we can leave #3398 unimplemented for now (to fix it a > > bit later). > > > > Removed exports, moved the test cases from module-api test to a separate > > file. > > > > I replaced merger.context.new(key_parts) with > > merger.context.new(key_def.new(key_parts)). > > > > I added docbot comment, because the module becomes user visible and is > > employed in the merger's docbot examples. That is why I think it is > > better to have it documented despite the fact it is just stub for now. > > > > Other comments are below. The new patch at the end of the email. > > > > NB: branch: Totktonada/gh-3276-on-board-merger > > > > WBR, Alexander Turenko. > > > > > > +#include "box/lua/key_def.h" > > > > + > > > > +#include <lua.h> > > > > +#include <lauxlib.h> > > > > +#include "diag.h" > > > > +#include "box/key_def.h" > > > > +#include "box/box.h" > > > > +#include "box/coll_id_cache.h" > > > > +#include "lua/utils.h" > > > > + > > > > +struct key_def * > > > > +luaT_new_key_def(struct lua_State *L, int idx) > > > > > > If you agree with luaT_tuple_new, then rename this function to > > > luaT_key_def_new pls. > > > > The code was moved to lbox_key_def_new() and luaT_key_def_set_part(). > > > > > > > > > +{ > > > > + if (lua_istable(L, idx) != 1) { > > > > + luaL_error(L, "Bad params, use: luaT_new_key_def({" > > > > + "{fieldno = fieldno, type = type" > > > > + "[, is_nullable = is_nullable" > > > > + "[, collation_id = collation_id" > > > > > > Hm, what's collation_id for? > > > > net.box exposes index parts in that way: > > https://github.com/tarantool/tarantool/issues/3941 > > > > I'll leave collation_id here for now if you don't mind and will remove > > it in the scope of #3941. > > > > > > > > > + "[, collation = collation]]]}, ...}"); > > > > > > This looks like you can't specify collation without is_nullable. > > > Should be > > > > > > luaT_new_key_def({{fieldno = FIELDNO, type = TYPE[, is_nullable = true | false][, collation = COLLATION]}}) > > > > Changed to: > > > > luaL_error(L, "Bad params, use: key_def.new({" > > "{fieldno = fieldno, type = type" > > "[, is_nullable = <boolean>]" > > "[, collation_id = <number>]" > > "[, collation = <string>]}, ...}"); > > > > > > > > > + unreachable(); > > > > + return NULL; > > > > + } > > > > + uint32_t key_parts_count = 0; > > > > + uint32_t capacity = 8; > > > > + > > > > + const ssize_t parts_size = sizeof(struct key_part_def) * capacity; > > > > > > Can't we figure out the table length right away instead of reallocaing > > > key_part_def array? > > > > Sure. Fixed. > > > > > > > > > + struct key_part_def *parts = NULL; > > > > + parts = (struct key_part_def *) malloc(parts_size); > > > > + if (parts == NULL) { > > > > + diag_set(OutOfMemory, parts_size, "malloc", "parts"); > > > > + luaT_error(L); > > > > + unreachable(); > > > > + return NULL; > > > > + } > > > > + > > > > + while (true) { > > > > > > Would be nice to factor out part creation to a separate function. > > > > Done. > > > > > > > > > + lua_pushinteger(L, key_parts_count + 1); > > > > > > We would call this variable key_part_count (without 's') or even just > > > part_count, as you called the array of key parts simply 'parts'. > > > > Ok. Fixed. > > > > > > > > > + lua_gettable(L, idx); > > > > + if (lua_isnil(L, -1)) > > > > + break; > > > > + > > > > + /* Extend parts if necessary. */ > > > > + if (key_parts_count == capacity) { > > > > + capacity *= 2; > > > > + struct key_part_def *old_parts = parts; > > > > + const ssize_t parts_size = > > > > + sizeof(struct key_part_def) * capacity; > > > > + parts = (struct key_part_def *) realloc(parts, > > > > + parts_size); > > > > + if (parts == NULL) { > > > > + free(old_parts); > > > > + diag_set(OutOfMemory, parts_size / 2, "malloc", > > > > + "parts"); > > > > + luaT_error(L); > > > > + unreachable(); > > > > + return NULL; > > > > + } > > > > + } > > > > + > > > > + /* Set parts[key_parts_count].fieldno. */ > > > > + lua_pushstring(L, "fieldno"); > > > > + lua_gettable(L, -2); > > > > + if (lua_isnil(L, -1)) { > > > > + free(parts); > > > > + luaL_error(L, "fieldno must not be nil"); > > > > + unreachable(); > > > > + return NULL; > > > > + } > > > > + /* > > > > + * Transform one-based Lua fieldno to zero-based > > > > + * fieldno to use in key_def_new(). > > > > + */ > > > > + parts[key_parts_count].fieldno = lua_tointeger(L, -1) - 1; > > > > > > Use TUPLE_INDEX_BASE instead of 1 pls. > > > > Fixed. > > > > > > > > > + lua_pop(L, 1); > > > > + > > > > + /* Set parts[key_parts_count].type. */ > > > > + lua_pushstring(L, "type"); > > > > + lua_gettable(L, -2); > > > > + if (lua_isnil(L, -1)) { > > > > + free(parts); > > > > + luaL_error(L, "type must not be nil"); > > > > + unreachable(); > > > > + return NULL; > > > > + } > > > > + size_t type_len; > > > > + const char *type_name = lua_tolstring(L, -1, &type_len); > > > > + lua_pop(L, 1); > > > > + parts[key_parts_count].type = field_type_by_name(type_name, > > > > + type_len); > > > > + if (parts[key_parts_count].type == field_type_MAX) { > > > > + free(parts); > > > > + luaL_error(L, "Unknown field type: %s", type_name); > > > > + unreachable(); > > > > + return NULL; > > > > + } > > > > + > > > > + /* > > > > + * Set parts[key_parts_count].is_nullable and > > > > + * parts[key_parts_count].nullable_action. > > > > + */ > > > > + lua_pushstring(L, "is_nullable"); > > > > + lua_gettable(L, -2); > > > > + if (lua_isnil(L, -1)) { > > > > + parts[key_parts_count].is_nullable = false; > > > > + parts[key_parts_count].nullable_action = > > > > + ON_CONFLICT_ACTION_DEFAULT; > > > > + } else { > > > > + parts[key_parts_count].is_nullable = > > > > + lua_toboolean(L, -1); > > > > + parts[key_parts_count].nullable_action = > > > > + ON_CONFLICT_ACTION_NONE; > > > > + } > > > > + lua_pop(L, 1); > > > > + > > > > + /* Set parts[key_parts_count].coll_id using collation_id. */ > > > > + lua_pushstring(L, "collation_id"); > > > > + lua_gettable(L, -2); > > > > + if (lua_isnil(L, -1)) > > > > + parts[key_parts_count].coll_id = COLL_NONE; > > > > + else > > > > + parts[key_parts_count].coll_id = lua_tointeger(L, -1); > > > > + lua_pop(L, 1); > > > > + > > > > + /* Set parts[key_parts_count].coll_id using collation. */ > > > > + lua_pushstring(L, "collation"); > > > > + lua_gettable(L, -2); > > > > + /* Check whether box.cfg{} was called. */ > > > > > > Collations should be usable even without box.cfg IIRC. Well, not all of > > > them I think, but still you don't need to check box.cfg() here AFAIU. > > > > Removed box.cfg{} check. No collations are available before box.cfg{} > > and we'll get an error for any collation ('Unknown collation "foo"' when > > it is pointed by name). > > > > Removed coll_id correctness check: it is performed if key_def_new() > > anyway. > > > > > > > > > + if ((parts[key_parts_count].coll_id != COLL_NONE || > > > > + !lua_isnil(L, -1)) && !box_is_configured()) { > > > > + free(parts); > > > > + luaL_error(L, "Cannot use collations: " > > > > + "please call box.cfg{}"); > > > > + unreachable(); > > > > + return NULL; > > > > + } > > > > + if (!lua_isnil(L, -1)) { > > > > + if (parts[key_parts_count].coll_id != COLL_NONE) { > > > > + free(parts); > > > > + luaL_error(L, "Conflicting options: " > > > > + "collation_id and collation"); > > > > + unreachable(); > > > > + return NULL; > > > > + } > > > > + size_t coll_name_len; > > > > + const char *coll_name = lua_tolstring(L, -1, > > > > + &coll_name_len); > > > > + struct coll_id *coll_id = coll_by_name(coll_name, > > > > + coll_name_len); > > > > > > Ouch, this doesn't seem to belong here. Ideally, it should be done by > > > key_def_new(). Can we rework key_part_def so that it stores collation > > > string instead of collation id? > > > > Can it have negative performance impact? It seems we don't compare > > key_defs, but I don't sure. > > > > The format of vy_log_record_encode() will change and we'll need to > > create upgrade script for old format. > > > > We use numeric collation IDs in many places. It seems the change is > > possible, but will heavily increase scope of work. I would skip it for > > now if you don't mind. I didn't filed an issue, because I don't sure how > > refactored collation support should look at whole. Maybe later. > > > > > > > > > + if (coll_id == NULL) { > > > > + free(parts); > > > > + luaL_error(L, "Unknown collation: \"%s\"", > > > > + coll_name); > > > > + unreachable(); > > > > + return NULL; > > > > + } > > > > + parts[key_parts_count].coll_id = coll_id->id; > > > > + } > > > > + lua_pop(L, 1); > > > > + > > > > + /* Check coll_id. */ > > > > + struct coll_id *coll_id = > > > > + coll_by_id(parts[key_parts_count].coll_id); > > > > + if (parts[key_parts_count].coll_id != COLL_NONE && > > > > + coll_id == NULL) { > > > > + uint32_t collation_id = parts[key_parts_count].coll_id; > > > > + free(parts); > > > > + luaL_error(L, "Unknown collation_id: %d", collation_id); > > > > + unreachable(); > > > > + return NULL; > > > > + } > > > > + > > > > + /* Set parts[key_parts_count].sort_order. */ > > > > + parts[key_parts_count].sort_order = SORT_ORDER_ASC; > > > > + > > > > + ++key_parts_count; > > > > + } > > > > + > > > > + struct key_def *key_def = key_def_new(parts, key_parts_count); > > > > + free(parts); > > > > + if (key_def == NULL) { > > > > + luaL_error(L, "Cannot create key_def"); > > > > + unreachable(); > > > > + return NULL; > > > > + } > > > > + return key_def; > > > > +} > > > > > > +#if defined(__cplusplus) > > > > +extern "C" { > > > > +#endif /* defined(__cplusplus) */ > > > > + > > > > +struct key_def; > > > > +struct lua_State; > > > > + > > > > +/** \cond public */ > > > > + > > > > +/** > > > > + * Create the new key_def from a Lua table. > > > > > > a key_def > > > > Fixed. > > > > > > diff --git a/test/app-tap/module_api.c b/test/app-tap/module_api.c > > > > index b81a98056..34ab54bc0 100644 > > > > --- a/test/app-tap/module_api.c > > > > +++ b/test/app-tap/module_api.c > > > > @@ -449,6 +449,18 @@ test_iscallable(lua_State *L) > > > > return 1; > > > > } > > > > > > > > +static int > > > > +test_luaT_new_key_def(lua_State *L) > > > > +{ > > > > + /* > > > > + * Ignore the return value. Here we test whether the > > > > + * function raises an error. > > > > + */ > > > > + luaT_new_key_def(L, 1); > > > > > > It would be nice to test that it actually creates a valid key_def. > > > Testing error conditions is less important. > > > > Now I can check a type of returned lua object in a successful case and > > that is all. When we'll add compare functions (in the scope #3398) we > > can test they are works as expected. > > > > I have added type checks of results of some successful key_def.new() > > invocations. > > > > ---- > > > > commit 7eae92c26421b99159892638c366a90f0d6af877 > > Author: Alexander Turenko <alexander.turenko@tarantool.org> > > Date: Mon Jan 7 19:12:50 2019 +0300 > > > > lua: add key_def lua module > > > > There are two reasons to add this module: > > > > * Incapsulate key_def creation from a Lua table (factor it out from > > merger's code). > > * Support comparing tuple with tuple and/or tuple with key from Lua in > > the future. > > > > The format of `parts` parameter in the `key_def.new(parts)` call is > > compatible with the following structures: > > > > * box.space[...].index[...].parts; > > * net_box_conn.space[...].index[...].parts. > > > > Needed for #3276. > > Needed for #3398. > > > > @TarantoolBot document > > Title: Document built-in key_def lua module > > > > Now there is only stub with the `key_def.new(parts)` function that > > returns cdata<struct key_def &>. The only way to use it for now is pass > > it to the merger. > > > > This module will be improved in the scope of > > https://github.com/tarantool/tarantool/issues/3398 > > > > See the commit message for more info. > > > > diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt > > index 04de5ad04..494c8d391 100644 > > --- a/src/CMakeLists.txt > > +++ b/src/CMakeLists.txt > > @@ -202,6 +202,7 @@ set(api_headers > > ${CMAKE_SOURCE_DIR}/src/lua/error.h > > ${CMAKE_SOURCE_DIR}/src/box/txn.h > > ${CMAKE_SOURCE_DIR}/src/box/key_def.h > > + ${CMAKE_SOURCE_DIR}/src/box/lua/key_def.h > > ${CMAKE_SOURCE_DIR}/src/box/field_def.h > > ${CMAKE_SOURCE_DIR}/src/box/tuple.h > > ${CMAKE_SOURCE_DIR}/src/box/tuple_format.h > > diff --git a/src/box/CMakeLists.txt b/src/box/CMakeLists.txt > > index 5521e489e..0db093768 100644 > > --- a/src/box/CMakeLists.txt > > +++ b/src/box/CMakeLists.txt > > @@ -139,6 +139,7 @@ add_library(box STATIC > > lua/net_box.c > > lua/xlog.c > > lua/sql.c > > + lua/key_def.c > > ${bin_sources}) > > > > target_link_libraries(box box_error tuple stat xrow xlog vclock crc32 scramble > > diff --git a/src/box/lua/init.c b/src/box/lua/init.c > > index 0e90f6be5..885354ace 100644 > > --- a/src/box/lua/init.c > > +++ b/src/box/lua/init.c > > @@ -59,6 +59,7 @@ > > #include "box/lua/console.h" > > #include "box/lua/tuple.h" > > #include "box/lua/sql.h" > > +#include "box/lua/key_def.h" > > > > extern char session_lua[], > > tuple_lua[], > > @@ -312,6 +313,8 @@ box_lua_init(struct lua_State *L) > > lua_pop(L, 1); > > tarantool_lua_console_init(L); > > lua_pop(L, 1); > > + luaopen_key_def(L); > > + lua_pop(L, 1); > > > > /* Load Lua extension */ > > for (const char **s = lua_sources; *s; s += 2) { > > diff --git a/src/box/lua/key_def.c b/src/box/lua/key_def.c > > new file mode 100644 > > index 000000000..f372048a6 > > --- /dev/null > > +++ b/src/box/lua/key_def.c > > @@ -0,0 +1,226 @@ > > +/* > > + * Copyright 2010-2018, Tarantool AUTHORS, please see AUTHORS file. > > + * > > + * Redistribution and use in source and binary forms, with or > > + * without modification, are permitted provided that the following > > + * conditions are met: > > + * > > + * 1. Redistributions of source code must retain the above > > + * copyright notice, this list of conditions and the > > + * following disclaimer. > > + * > > + * 2. Redistributions in binary form must reproduce the above > > + * copyright notice, this list of conditions and the following > > + * disclaimer in the documentation and/or other materials > > + * provided with the distribution. > > + * > > + * THIS SOFTWARE IS PROVIDED BY <COPYRIGHT HOLDER> ``AS IS'' AND > > + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED > > + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > > + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL > > + * <COPYRIGHT HOLDER> OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, > > + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL > > + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF > > + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR > > + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF > > + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > > + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF > > + * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF > > + * SUCH DAMAGE. > > + */ > > + > > +#include "box/lua/key_def.h" > > + > > +#include <lua.h> > > +#include <lauxlib.h> > > +#include "diag.h" > > +#include "box/key_def.h" > > +#include "box/box.h" > > +#include "box/coll_id_cache.h" > > +#include "lua/utils.h" > > +#include "box/tuple_format.h" /* TUPLE_INDEX_BASE */ > > + > > +static uint32_t key_def_type_id = 0; > > + > > +/** > > + * Set key_part_def from a table on top of a Lua stack. > > + * > > + * When successful return 0, otherwise return -1 and set a diag. > > + */ > > +static int > > +luaT_key_def_set_part(struct lua_State *L, struct key_part_def *part) > > +{ > > + /* Set part->fieldno. */ > > + lua_pushstring(L, "fieldno"); > > + lua_gettable(L, -2); > > + if (lua_isnil(L, -1)) { > > + diag_set(IllegalParams, "fieldno must not be nil"); > > + return -1; > > + } > > + /* > > + * Transform one-based Lua fieldno to zero-based > > + * fieldno to use in key_def_new(). > > + */ > > + part->fieldno = lua_tointeger(L, -1) - TUPLE_INDEX_BASE; > > + lua_pop(L, 1); > > + > > + /* Set part->type. */ > > + lua_pushstring(L, "type"); > > + lua_gettable(L, -2); > > + if (lua_isnil(L, -1)) { > > + diag_set(IllegalParams, "type must not be nil"); > > + return -1; > > + } > > + size_t type_len; > > + const char *type_name = lua_tolstring(L, -1, &type_len); > > + lua_pop(L, 1); > > + part->type = field_type_by_name(type_name, type_len); > > + if (part->type == field_type_MAX) { > > + diag_set(IllegalParams, "Unknown field type: %s", type_name); > > + return -1; > > + } > > + > > + /* Set part->is_nullable and part->nullable_action. */ > > + lua_pushstring(L, "is_nullable"); > > + lua_gettable(L, -2); > > + if (lua_isnil(L, -1)) { > > + part->is_nullable = false; > > + part->nullable_action = ON_CONFLICT_ACTION_DEFAULT; > > + } else { > > + part->is_nullable = lua_toboolean(L, -1); > > + part->nullable_action = ON_CONFLICT_ACTION_NONE; > > + } > > + lua_pop(L, 1); > > + > > + /* > > + * Set part->coll_id using collation_id. > > + * > > + * The value will be checked in key_def_new(). > > + */ > > + lua_pushstring(L, "collation_id"); > > + lua_gettable(L, -2); > > + if (lua_isnil(L, -1)) > > + part->coll_id = COLL_NONE; > > + else > > + part->coll_id = lua_tointeger(L, -1); > > + lua_pop(L, 1); > > + > > + /* Set part->coll_id using collation. */ > > + lua_pushstring(L, "collation"); > > + lua_gettable(L, -2); > > + if (!lua_isnil(L, -1)) { > > + /* Check for conflicting options. */ > > + if (part->coll_id != COLL_NONE) { > > + diag_set(IllegalParams, "Conflicting options: " > > + "collation_id and collation"); > > + return -1; > > + } > > + > > + size_t coll_name_len; > > + const char *coll_name = lua_tolstring(L, -1, &coll_name_len); > > + struct coll_id *coll_id = coll_by_name(coll_name, > > + coll_name_len); > > + if (coll_id == NULL) { > > + diag_set(IllegalParams, "Unknown collation: \"%s\"", > > + coll_name); > > + return -1; > > + } > > + part->coll_id = coll_id->id; > > + } > > + lua_pop(L, 1); > > + > > + /* Set part->sort_order. */ > > + part->sort_order = SORT_ORDER_ASC; > > + > > + return 0; > > +} > > + > > +struct key_def * > > +check_key_def(struct lua_State *L, int idx) > > +{ > > + if (lua_type(L, idx) != LUA_TCDATA) > > + return NULL; > > + > > + uint32_t cdata_type; > > + struct key_def **key_def_ptr = luaL_checkcdata(L, idx, &cdata_type); > > + if (key_def_ptr == NULL || cdata_type != key_def_type_id) > > + return NULL; > > + return *key_def_ptr; > > +} > > + > > +/** > > + * Free a key_def from a Lua code. > > + */ > > +static int > > +lbox_key_def_gc(struct lua_State *L) > > +{ > > + struct key_def *key_def = check_key_def(L, 1); > > + if (key_def == NULL) > > + return 0; > > + box_key_def_delete(key_def); > > + return 0; > > +} > > + > > +/** > > + * Create a new key_def from a Lua table. > > + * > > + * Expected a table of key parts on the Lua stack. The format is > > + * the same as box.space.<...>.index.<...>.parts or corresponding > > + * net.box's one. > > + * > > + * Return the new key_def as cdata. > > + */ > > +static int > > +lbox_key_def_new(struct lua_State *L) > > +{ > > + if (lua_gettop(L) != 1 || lua_istable(L, 1) != 1) > > + return luaL_error(L, "Bad params, use: key_def.new({" > > + "{fieldno = fieldno, type = type" > > + "[, is_nullable = <boolean>]" > > + "[, collation_id = <number>]" > > + "[, collation = <string>]}, ...}"); > > + > > + uint32_t part_count = lua_objlen(L, 1); > > + const ssize_t parts_size = sizeof(struct key_part_def) * part_count; > > + struct key_part_def *parts = malloc(parts_size); > > + if (parts == NULL) { > > + diag_set(OutOfMemory, parts_size, "malloc", "parts"); > > + return luaT_error(L); > > + } > > + > > + for (uint32_t i = 0; i < part_count; ++i) { > > + lua_pushinteger(L, i + 1); > > + lua_gettable(L, 1); > > + if (luaT_key_def_set_part(L, &parts[i]) != 0) { > > + free(parts); > > + return luaT_error(L); > > + } > > + } > > + > > + struct key_def *key_def = key_def_new(parts, part_count); > > + free(parts); > > + if (key_def == NULL) > > + return luaT_error(L); > > + > > + *(struct key_def **) luaL_pushcdata(L, key_def_type_id) = key_def; > > + lua_pushcfunction(L, lbox_key_def_gc); > > + luaL_setcdatagc(L, -2); > > + > > + return 1; > > +} > > + > > +LUA_API int > > +luaopen_key_def(struct lua_State *L) > > +{ > > + luaL_cdef(L, "struct key_def;"); > > + key_def_type_id = luaL_ctypeid(L, "struct key_def&"); > > + > > + /* Export C functions to Lua. */ > > + static const struct luaL_Reg meta[] = { > > + {"new", lbox_key_def_new}, > > + {NULL, NULL} > > + }; > > + luaL_register_module(L, "key_def", meta); > > + > > + return 1; > > +} > > diff --git a/src/box/lua/key_def.h b/src/box/lua/key_def.h > > new file mode 100644 > > index 000000000..11cc0bfd4 > > --- /dev/null > > +++ b/src/box/lua/key_def.h > > @@ -0,0 +1,56 @@ > > +#ifndef TARANTOOL_BOX_LUA_KEY_DEF_H_INCLUDED > > +#define TARANTOOL_BOX_LUA_KEY_DEF_H_INCLUDED > > +/* > > + * Copyright 2010-2018, Tarantool AUTHORS, please see AUTHORS file. > > + * > > + * Redistribution and use in source and binary forms, with or > > + * without modification, are permitted provided that the following > > + * conditions are met: > > + * > > + * 1. Redistributions of source code must retain the above > > + * copyright notice, this list of conditions and the > > + * following disclaimer. > > + * > > + * 2. Redistributions in binary form must reproduce the above > > + * copyright notice, this list of conditions and the following > > + * disclaimer in the documentation and/or other materials > > + * provided with the distribution. > > + * > > + * THIS SOFTWARE IS PROVIDED BY AUTHORS ``AS IS'' AND > > + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED > > + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > > + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL > > + * AUTHORS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, > > + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL > > + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF > > + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR > > + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF > > + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > > + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF > > + * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF > > + * SUCH DAMAGE. > > + */ > > + > > +#if defined(__cplusplus) > > +extern "C" { > > +#endif /* defined(__cplusplus) */ > > + > > +struct lua_State; > > + > > +/** > > + * Extract a key_def object from a Lua stack. > > + */ > > +struct key_def * > > +check_key_def(struct lua_State *L, int idx); > > + > > +/** > > + * Register the module. > > + */ > > +int > > +luaopen_key_def(struct lua_State *L); > > + > > +#if defined(__cplusplus) > > +} /* extern "C" */ > > +#endif /* defined(__cplusplus) */ > > + > > +#endif /* TARANTOOL_BOX_LUA_KEY_DEF_H_INCLUDED */ > > diff --git a/test/box-tap/key_def.test.lua b/test/box-tap/key_def.test.lua > > new file mode 100755 > > index 000000000..7e6e0e330 > > --- /dev/null > > +++ b/test/box-tap/key_def.test.lua > > @@ -0,0 +1,137 @@ > > +#!/usr/bin/env tarantool > > + > > +local tap = require('tap') > > +local ffi = require('ffi') > > +local key_def = require('key_def') > > + > > +local usage_error = 'Bad params, use: key_def.new({' .. > > + '{fieldno = fieldno, type = type' .. > > + '[, is_nullable = <boolean>]' .. > > + '[, collation_id = <number>]' .. > > + '[, collation = <string>]}, ...}' > > + > > +local function coll_not_found(fieldno, collation) > > + if type(collation) == 'number' then > > + return ('Wrong index options (field %d): ' .. > > + 'collation was not found by ID'):format(fieldno) > > + end > > + > > + return ('Unknown collation: "%s"'):format(collation) > > +end > > + > > +local cases = { > > + -- Cases to call before box.cfg{}. > > + { > > + 'Pass a field on an unknown type', > > + parts = {{ > > + fieldno = 2, > > + type = 'unknown', > > + }}, > > + exp_err = 'Unknown field type: unknown', > > + }, > > + { > > + 'Try to use collation_id before box.cfg{}', > > + parts = {{ > > + fieldno = 1, > > + type = 'string', > > + collation_id = 2, > > + }}, > > + exp_err = coll_not_found(1, 2), > > + }, > > + { > > + 'Try to use collation before box.cfg{}', > > + parts = {{ > > + fieldno = 1, > > + type = 'string', > > + collation = 'unicode_ci', > > + }}, > > + exp_err = coll_not_found(1, 'unicode_ci'), > > + }, > > + function() > > + -- For collations. > > + box.cfg{} > > + end, > > + -- Cases to call after box.cfg{}. > > + { > > + 'Try to use both collation_id and collation', > > + parts = {{ > > + fieldno = 1, > > + type = 'string', > > + collation_id = 2, > > + collation = 'unicode_ci', > > + }}, > > + exp_err = 'Conflicting options: collation_id and collation', > > + }, > > + { > > + 'Unknown collation_id', > > + parts = {{ > > + fieldno = 1, > > + type = 'string', > > + collation_id = 42, > > + }}, > > + exp_err = coll_not_found(1, 42), > > + }, > > + { > > + 'Unknown collation name', > > + parts = {{ > > + fieldno = 1, > > + type = 'string', > > + collation = 'unknown', > > + }}, > > + exp_err = 'Unknown collation: "unknown"', > > + }, > > + { > > + 'Bad parts parameter type', > > + parts = 1, > > + exp_err = usage_error, > > + }, > > + { > > + 'No parameters', > > + params = {}, > > + exp_err = usage_error, > > + }, > > + { > > + 'Two parameters', > > + params = {{}, {}}, > > + exp_err = usage_error, > > + }, > > + { > > + 'Success case; zero parts', > > + parts = {}, > > + exp_err = nil, > > + }, > > + { > > + 'Success case; one part', > > + parts = { > > + fieldno = 1, > > + type = 'string', > > + }, > > + exp_err = nil, > > + }, > > +} > > + > > +local test = tap.test('key_def') > > + > > +test:plan(#cases - 1) > > +for _, case in ipairs(cases) do > > + if type(case) == 'function' then > > + case() > > + else > > + local ok, res > > + if case.params then > > + ok, res = pcall(key_def.new, unpack(case.params)) > > + else > > + ok, res = pcall(key_def.new, case.parts) > > + end > > + if case.exp_err == nil then > > + ok = ok and type(res) == 'cdata' and > > + ffi.istype('struct key_def', res) > > + test:ok(ok, case[1]) > > + else > > + local err = tostring(res) -- cdata -> string > > + test:is_deeply({ok, err}, {false, case.exp_err}, case[1]) > > + end > > + end > > +end > > + > > +os.exit(test:check() and 0 or 1) ^ permalink raw reply [flat|nested] 28+ messages in thread
* [PATCH v2 5/6] net.box: add helpers to decode msgpack headers 2019-01-09 20:20 [PATCH v2 0/6] Merger Alexander Turenko ` (3 preceding siblings ...) 2019-01-09 20:20 ` [PATCH v2 4/6] lua: add luaT_new_key_def() Alexander Turenko @ 2019-01-09 20:20 ` Alexander Turenko 2019-01-10 17:29 ` Vladimir Davydov 2019-01-09 20:20 ` [PATCH v2 6/6] Add merger for tuple streams Alexander Turenko 5 siblings, 1 reply; 28+ messages in thread From: Alexander Turenko @ 2019-01-09 20:20 UTC (permalink / raw) To: Vladimir Davydov; +Cc: Alexander Turenko, tarantool-patches Needed for #3276. @TarantoolBot document Title: net.box: helpers to decode msgpack headers They allow to skip iproto packet and msgpack array headers and pass raw msgpack data to some other function, say, merger. Contracts: ``` net_box.check_iproto_data(buf.rpos, buf.wpos - buf.rpos) -> new_rpos -> nil, err_msg msgpack.check_array(buf.rpos, buf.wpos - buf.rpos, [, arr_len]) -> new_rpos, arr_len -> nil, err_msg ``` Below the example with msgpack.decode() as the function that need raw msgpack data. It is just to illustrate the approach, there is no sense to skip iproto/array headers manually in Lua and then decode the rest in Lua. But it worth when the raw msgpack data is subject to process in a C module. ```lua local function single_select(space, ...) return box.space[space]:select(...) end local function batch_select(spaces, ...) local res = {} for _, space in ipairs(spaces) do table.insert(res, box.space[space]:select(...)) end return res end _G.single_select = single_select _G.batch_select = batch_select local res local buf = buffer.ibuf() conn.space.s:select(nil, {buffer = buf}) -- check and skip iproto_data header buf.rpos = assert(net_box.check_iproto_data(buf.rpos, buf.wpos - buf.rpos)) -- check that we really got data from :select() as result res, buf.rpos = msgpack.decode(buf.rpos, buf.wpos - buf.rpos) -- check that the buffer ends assert(buf.rpos == buf.wpos) buf:recycle() conn:call('single_select', {'s'}, {buffer = buf}) -- check and skip the iproto_data header buf.rpos = assert(net_box.check_iproto_data(buf.rpos, buf.wpos - buf.rpos)) -- check and skip the array around return values buf.rpos = assert(msgpack.check_array(buf.rpos, buf.wpos - buf.rpos, 1)) -- check that we really got data from :select() as result res, buf.rpos = msgpack.decode(buf.rpos, buf.wpos - buf.rpos) -- check that the buffer ends assert(buf.rpos == buf.wpos) buf:recycle() local spaces = {'s', 't'} conn:call('batch_select', {spaces}, {buffer = buf}) -- check and skip the iproto_data header buf.rpos = assert(net_box.check_iproto_data(buf.rpos, buf.wpos - buf.rpos)) -- check and skip the array around return values buf.rpos = assert(msgpack.check_array(buf.rpos, buf.wpos - buf.rpos, 1)) -- check and skip the array header before the first select result buf.rpos = assert(msgpack.check_array(buf.rpos, buf.wpos - buf.rpos, #spaces)) -- check that we really got data from s:select() as result res, buf.rpos = msgpack.decode(buf.rpos, buf.wpos - buf.rpos) -- t:select() data res, buf.rpos = msgpack.decode(buf.rpos, buf.wpos - buf.rpos) -- check that the buffer ends assert(buf.rpos == buf.wpos) ``` --- src/box/lua/net_box.c | 49 +++++++++++ src/box/lua/net_box.lua | 1 + src/lua/msgpack.c | 66 ++++++++++++++ test/app-tap/msgpack.test.lua | 157 +++++++++++++++++++++++++++++++++- test/box/net.box.result | 58 +++++++++++++ test/box/net.box.test.lua | 26 ++++++ 6 files changed, 356 insertions(+), 1 deletion(-) diff --git a/src/box/lua/net_box.c b/src/box/lua/net_box.c index c7063d9c8..d71f33768 100644 --- a/src/box/lua/net_box.c +++ b/src/box/lua/net_box.c @@ -51,6 +51,9 @@ #define cfg luaL_msgpack_default +static uint32_t CTID_CHAR_PTR; +static uint32_t CTID_CONST_CHAR_PTR; + static inline size_t netbox_prepare_request(lua_State *L, struct mpstream *stream, uint32_t r_type) { @@ -745,9 +748,54 @@ netbox_decode_execute(struct lua_State *L) return 2; } +/** + * net_box.check_iproto_data(buf.rpos, buf.wpos - buf.rpos) + * -> new_rpos + * -> nil, err_msg + */ +int +netbox_check_iproto_data(struct lua_State *L) +{ + uint32_t ctypeid; + const char *data = *(const char **) luaL_checkcdata(L, 1, &ctypeid); + if (ctypeid != CTID_CHAR_PTR && ctypeid != CTID_CONST_CHAR_PTR) + return luaL_error(L, + "net_box.check_iproto_data: 'char *' or " + "'const char *' expected"); + + if (!lua_isnumber(L, 2)) + return luaL_error(L, "net_box.check_iproto_data: number " + "expected as 2nd argument"); + const char *end = data + lua_tointeger(L, 2); + + int ok = data < end && + mp_typeof(*data) == MP_MAP && + mp_check_map(data, end) <= 0 && + mp_decode_map(&data) == 1 && + data < end && + mp_typeof(*data) == MP_UINT && + mp_check_uint(data, end) <= 0 && + mp_decode_uint(&data) == IPROTO_DATA; + + if (!ok) { + lua_pushnil(L); + lua_pushstring(L, + "net_box.check_iproto_data: wrong iproto data packet"); + return 2; + } + + *(const char **) luaL_pushcdata(L, ctypeid) = data; + return 1; +} + int luaopen_net_box(struct lua_State *L) { + CTID_CHAR_PTR = luaL_ctypeid(L, "char *"); + assert(CTID_CHAR_PTR != 0); + CTID_CONST_CHAR_PTR = luaL_ctypeid(L, "const char *"); + assert(CTID_CONST_CHAR_PTR != 0); + static const luaL_Reg net_box_lib[] = { { "encode_ping", netbox_encode_ping }, { "encode_call_16", netbox_encode_call_16 }, @@ -765,6 +813,7 @@ luaopen_net_box(struct lua_State *L) { "communicate", netbox_communicate }, { "decode_select", netbox_decode_select }, { "decode_execute", netbox_decode_execute }, + { "check_iproto_data", netbox_check_iproto_data }, { NULL, NULL} }; /* luaL_register_module polutes _G */ diff --git a/src/box/lua/net_box.lua b/src/box/lua/net_box.lua index 2bf772aa8..0a38efa5a 100644 --- a/src/box/lua/net_box.lua +++ b/src/box/lua/net_box.lua @@ -1424,6 +1424,7 @@ local this_module = { new = connect, -- Tarantool < 1.7.1 compatibility, wrap = wrap, establish_connection = establish_connection, + check_iproto_data = internal.check_iproto_data, } function this_module.timeout(timeout, ...) diff --git a/src/lua/msgpack.c b/src/lua/msgpack.c index b47006038..fca440660 100644 --- a/src/lua/msgpack.c +++ b/src/lua/msgpack.c @@ -51,6 +51,7 @@ luamp_error(void *error_ctx) } static uint32_t CTID_CHAR_PTR; +static uint32_t CTID_CONST_CHAR_PTR; static uint32_t CTID_STRUCT_IBUF; struct luaL_serializer *luaL_msgpack_default = NULL; @@ -418,6 +419,68 @@ lua_ibuf_msgpack_decode(lua_State *L) return 2; } +/** + * msgpack.check_array(buf.rpos, buf.wpos - buf.rpos, [, arr_len]) + * -> new_rpos, arr_len + * -> nil, err_msg + */ +static int +lua_check_array(lua_State *L) +{ + uint32_t ctypeid; + const char *data = *(const char **) luaL_checkcdata(L, 1, &ctypeid); + if (ctypeid != CTID_CHAR_PTR && ctypeid != CTID_CONST_CHAR_PTR) + return luaL_error(L, "msgpack.check_array: 'char *' or " + "'const char *' expected"); + + if (!lua_isnumber(L, 2)) + return luaL_error(L, "msgpack.check_array: number expected as " + "2nd argument"); + const char *end = data + lua_tointeger(L, 2); + + if (!lua_isnoneornil(L, 3) && !lua_isnumber(L, 3)) + return luaL_error(L, "msgpack.check_array: number or nil " + "expected as 3rd argument"); + + static const char *end_of_buffer_msg = "msgpack.check_array: " + "unexpected end of buffer"; + + if (data >= end) { + lua_pushnil(L); + lua_pushstring(L, end_of_buffer_msg); + return 2; + } + + if (mp_typeof(*data) != MP_ARRAY) { + lua_pushnil(L); + lua_pushstring(L, "msgpack.check_array: wrong array header"); + return 2; + } + + if (mp_check_array(data, end) > 0) { + lua_pushnil(L); + lua_pushstring(L, end_of_buffer_msg); + return 2; + } + + uint32_t len = mp_decode_array(&data); + + if (!lua_isnoneornil(L, 3)) { + uint32_t exp_len = (uint32_t) lua_tointeger(L, 3); + if (len != exp_len) { + lua_pushnil(L); + lua_pushfstring(L, "msgpack.check_array: expected " + "array of length %d, got length %d", + len, exp_len); + return 2; + } + } + + *(const char **) luaL_pushcdata(L, ctypeid) = data; + lua_pushinteger(L, len); + return 2; +} + static int lua_msgpack_new(lua_State *L); @@ -426,6 +489,7 @@ static const luaL_Reg msgpacklib[] = { { "decode", lua_msgpack_decode }, { "decode_unchecked", lua_msgpack_decode_unchecked }, { "ibuf_decode", lua_ibuf_msgpack_decode }, + { "check_array", lua_check_array }, { "new", lua_msgpack_new }, { NULL, NULL } }; @@ -447,6 +511,8 @@ luaopen_msgpack(lua_State *L) assert(CTID_STRUCT_IBUF != 0); CTID_CHAR_PTR = luaL_ctypeid(L, "char *"); assert(CTID_CHAR_PTR != 0); + CTID_CONST_CHAR_PTR = luaL_ctypeid(L, "const char *"); + assert(CTID_CONST_CHAR_PTR != 0); luaL_msgpack_default = luaL_newserializer(L, "msgpack", msgpacklib); return 1; } diff --git a/test/app-tap/msgpack.test.lua b/test/app-tap/msgpack.test.lua index 0e1692ad9..d481d2da9 100755 --- a/test/app-tap/msgpack.test.lua +++ b/test/app-tap/msgpack.test.lua @@ -49,9 +49,163 @@ local function test_misc(test, s) test:ok(not st and e:match("null"), "null ibuf") end +local function test_check_array(test, s) + local ffi = require('ffi') + + local good_cases = { + { + 'fixarray', + data = '\x94\x01\x02\x03\x04', + exp_len = 4, + exp_rewind = 1, + }, + { + 'array 16', + data = '\xdc\x00\x04\x01\x02\x03\x04', + exp_len = 4, + exp_rewind = 3, + }, + { + 'array 32', + data = '\xdd\x00\x00\x00\x04\x01\x02\x03\x04', + exp_len = 4, + exp_rewind = 5, + }, + } + + local bad_cases = { + { + 'fixmap', + data = '\x80', + exp_err = 'msgpack.check_array: wrong array header', + }, + { + 'truncated array 16', + data = '\xdc\x00', + exp_err = 'msgpack.check_array: unexpected end of buffer', + }, + { + 'truncated array 32', + data = '\xdd\x00\x00\x00', + exp_err = 'msgpack.check_array: unexpected end of buffer', + }, + { + 'zero size buffer', + data = '\x90', + size = 0, + exp_err = 'msgpack.check_array: unexpected end of buffer', + }, + { + 'negative size buffer', + data = '\x90', + size = -1, + exp_err = 'msgpack.check_array: unexpected end of buffer', + }, + } + + local wrong_1_arg_not_cdata_err = 'expected cdata as 1 argument' + local wrong_1_arg_err = "msgpack.check_array: 'char *' or " .. + "'const char *' expected" + local wrong_2_arg_err = 'msgpack.check_array: number expected as 2nd ' .. + 'argument' + local wrong_3_arg_err = 'msgpack.check_array: number or nil expected as ' .. + '3rd argument' + + local bad_api_cases = { + { + '1st argument: nil', + args = {nil, 1}, + exp_err = wrong_1_arg_not_cdata_err, + }, + { + '1st argument: not cdata', + args = {1, 1}, + exp_err = wrong_1_arg_not_cdata_err, + }, + { + '1st argument: wrong cdata type', + args = {box.tuple.new(), 1}, + exp_err = wrong_1_arg_err, + }, + { + '2nd argument: nil', + args = {ffi.cast('char *', '\x90'), nil}, + exp_err = wrong_2_arg_err, + }, + { + '2nd argument: wrong type', + args = {ffi.cast('char *', '\x90'), 'eee'}, + exp_err = wrong_2_arg_err, + }, + { + '3rd argument: wrong type', + args = {ffi.cast('char *', '\x90'), 1, 'eee'}, + exp_err = wrong_3_arg_err, + }, + } + + -- Add good cases with wrong expected length to the bad cases. + for _, case in ipairs(good_cases) do + table.insert(bad_cases, { + case[1], + data = case.data, + exp_len = case.exp_len + 1, + exp_err = 'msgpack.check_array: expected array of length' + }) + end + + test:plan(4 * #good_cases + 2 * #bad_cases + #bad_api_cases) + + -- Good cases: don't pass 2nd argument. + for _, ctype in ipairs({'char *', 'const char *'}) do + for _, case in ipairs(good_cases) do + local buf = ffi.cast(ctype, case.data) + local size = case.size or #case.data + local new_buf, len = s.check_array(buf, size) + local rewind = new_buf - buf + local description = ('good; %s; %s; %s'):format(case[1], ctype, + 'do not pass 2nd argument') + test:is_deeply({len, rewind}, {case.exp_len, case.exp_rewind}, + description) + end + end + + -- Good cases: pass right 2nd argument. + for _, ctype in ipairs({'char *', 'const char *'}) do + for _, case in ipairs(good_cases) do + local buf = ffi.cast(ctype, case.data) + local size = case.size or #case.data + local new_buf, len = s.check_array(buf, size, case.exp_len) + local rewind = new_buf - buf + local description = ('good; %s; %s; %s'):format(case[1], ctype, + 'pass right 2nd argument') + test:is_deeply({len, rewind}, {case.exp_len, case.exp_rewind}, + description) + end + end + + -- Bad cases. + for _, ctype in ipairs({'char *', 'const char *'}) do + for _, case in ipairs(bad_cases) do + local buf = ffi.cast(ctype, case.data) + local size = case.size or #case.data + local n, err = s.check_array(buf, size, case.exp_len) + local description = ('bad; %s; %s'):format(case[1], ctype) + test:ok(n == nil and err:startswith(case.exp_err), description) + end + end + + -- Bad API usage cases. + for _, case in ipairs(bad_api_cases) do + local ok, err = pcall(s.check_array, unpack(case.args)) + local description = 'bad API usage; ' .. case[1] + test:is_deeply({ok, err}, {false, case.exp_err}, description) + end +end + tap.test("msgpack", function(test) local serializer = require('msgpack') - test:plan(10) + test:plan(11) test:test("unsigned", common.test_unsigned, serializer) test:test("signed", common.test_signed, serializer) test:test("double", common.test_double, serializer) @@ -62,4 +216,5 @@ tap.test("msgpack", function(test) test:test("ucdata", common.test_ucdata, serializer) test:test("offsets", test_offsets, serializer) test:test("misc", test_misc, serializer) + test:test("check_array", test_check_array, serializer) end) diff --git a/test/box/net.box.result b/test/box/net.box.result index 2b5a84646..98ba9598e 100644 --- a/test/box/net.box.result +++ b/test/box/net.box.result @@ -3433,6 +3433,64 @@ c c:close() --- ... +ffi = require('ffi') +--- +... +-- Case: valid iproto_data packet; char *. +data = '\x81\x30\x90' +--- +... +rpos = ffi.cast('char *', data) +--- +... +net.check_iproto_data(rpos, #data) - rpos -- 2 +--- +- 2 +... +-- Case: valid iproto_data packet; const char *. +rpos = ffi.cast('const char *', data) +--- +... +net.check_iproto_data(rpos, #data) - rpos -- 2 +--- +- 2 +... +-- Case: invalid iproto_data packet. +data = '\x91\x01' +--- +... +rpos = ffi.cast('char *', data) +--- +... +net.check_iproto_data(rpos, #data) -- error +--- +- null +- 'net_box.check_iproto_data: wrong iproto data packet' +... +-- Case: truncated msgpack. +data = '\x81' +--- +... +rpos = ffi.cast('char *', data) +--- +... +net.check_iproto_data(rpos, #data) -- error +--- +- null +- 'net_box.check_iproto_data: wrong iproto data packet' +... +-- Case: zero size buffer. +data = '' +--- +... +rpos = ffi.cast('char *', data) +--- +... +net.check_iproto_data(rpos, #data) -- error +--- +- null +- 'net_box.check_iproto_data: wrong iproto data packet' +... box.schema.func.drop('do_long') --- ... diff --git a/test/box/net.box.test.lua b/test/box/net.box.test.lua index 96d822820..f89cf7f4d 100644 --- a/test/box/net.box.test.lua +++ b/test/box/net.box.test.lua @@ -1388,6 +1388,32 @@ c = net.connect('8.8.8.8:123456', {wait_connected = false}) c c:close() +ffi = require('ffi') + +-- Case: valid iproto_data packet; char *. +data = '\x81\x30\x90' +rpos = ffi.cast('char *', data) +net.check_iproto_data(rpos, #data) - rpos -- 2 + +-- Case: valid iproto_data packet; const char *. +rpos = ffi.cast('const char *', data) +net.check_iproto_data(rpos, #data) - rpos -- 2 + +-- Case: invalid iproto_data packet. +data = '\x91\x01' +rpos = ffi.cast('char *', data) +net.check_iproto_data(rpos, #data) -- error + +-- Case: truncated msgpack. +data = '\x81' +rpos = ffi.cast('char *', data) +net.check_iproto_data(rpos, #data) -- error + +-- Case: zero size buffer. +data = '' +rpos = ffi.cast('char *', data) +net.check_iproto_data(rpos, #data) -- error + box.schema.func.drop('do_long') box.schema.user.revoke('guest', 'write', 'space', '_schema') box.schema.user.revoke('guest', 'read,write', 'space', '_space') -- 2.20.1 ^ permalink raw reply [flat|nested] 28+ messages in thread
* Re: [PATCH v2 5/6] net.box: add helpers to decode msgpack headers 2019-01-09 20:20 ` [PATCH v2 5/6] net.box: add helpers to decode msgpack headers Alexander Turenko @ 2019-01-10 17:29 ` Vladimir Davydov 2019-02-01 15:11 ` Alexander Turenko 0 siblings, 1 reply; 28+ messages in thread From: Vladimir Davydov @ 2019-01-10 17:29 UTC (permalink / raw) To: Alexander Turenko; +Cc: tarantool-patches On Wed, Jan 09, 2019 at 11:20:13PM +0300, Alexander Turenko wrote: > Needed for #3276. > > @TarantoolBot document > Title: net.box: helpers to decode msgpack headers > > They allow to skip iproto packet and msgpack array headers and pass raw > msgpack data to some other function, say, merger. > > Contracts: > > ``` > net_box.check_iproto_data(buf.rpos, buf.wpos - buf.rpos) > -> new_rpos > -> nil, err_msg I'd prefer if this was done right in net.box.select or whatever function writing the response to ibuf. Yes, this is going to break backward compatibility, but IMO it's OK for 2.1 - I doubt anybody have used this weird high perf API anyway. > msgpack.check_array(buf.rpos, buf.wpos - buf.rpos, [, arr_len]) > -> new_rpos, arr_len > -> nil, err_msg This seems to be OK, although I'm not sure if we really need to check the length in this function. Looks like we will definitely need it because of net.box.call, which wraps function return value in an array. Not sure about the name either, because it doesn't just checks the msgpack - it decodes it, but can't come up with anything substantially better. May be, msgpack.decode_array? > ``` > > Below the example with msgpack.decode() as the function that need raw > msgpack data. It is just to illustrate the approach, there is no sense > to skip iproto/array headers manually in Lua and then decode the rest in > Lua. But it worth when the raw msgpack data is subject to process in a C > module. > > ```lua > local function single_select(space, ...) > return box.space[space]:select(...) > end > > local function batch_select(spaces, ...) > local res = {} > for _, space in ipairs(spaces) do > table.insert(res, box.space[space]:select(...)) > end > return res > end > > _G.single_select = single_select > _G.batch_select = batch_select > > local res > > local buf = buffer.ibuf() > conn.space.s:select(nil, {buffer = buf}) > -- check and skip iproto_data header > buf.rpos = assert(net_box.check_iproto_data(buf.rpos, buf.wpos - buf.rpos)) > -- check that we really got data from :select() as result > res, buf.rpos = msgpack.decode(buf.rpos, buf.wpos - buf.rpos) > -- check that the buffer ends > assert(buf.rpos == buf.wpos) > > buf:recycle() > conn:call('single_select', {'s'}, {buffer = buf}) > -- check and skip the iproto_data header > buf.rpos = assert(net_box.check_iproto_data(buf.rpos, buf.wpos - buf.rpos)) > -- check and skip the array around return values > buf.rpos = assert(msgpack.check_array(buf.rpos, buf.wpos - buf.rpos, 1)) > -- check that we really got data from :select() as result > res, buf.rpos = msgpack.decode(buf.rpos, buf.wpos - buf.rpos) > -- check that the buffer ends > assert(buf.rpos == buf.wpos) > > buf:recycle() > local spaces = {'s', 't'} > conn:call('batch_select', {spaces}, {buffer = buf}) > -- check and skip the iproto_data header > buf.rpos = assert(net_box.check_iproto_data(buf.rpos, buf.wpos - buf.rpos)) > -- check and skip the array around return values > buf.rpos = assert(msgpack.check_array(buf.rpos, buf.wpos - buf.rpos, 1)) > -- check and skip the array header before the first select result > buf.rpos = assert(msgpack.check_array(buf.rpos, buf.wpos - buf.rpos, #spaces)) > -- check that we really got data from s:select() as result > res, buf.rpos = msgpack.decode(buf.rpos, buf.wpos - buf.rpos) > -- t:select() data > res, buf.rpos = msgpack.decode(buf.rpos, buf.wpos - buf.rpos) > -- check that the buffer ends > assert(buf.rpos == buf.wpos) > ``` > --- > src/box/lua/net_box.c | 49 +++++++++++ > src/box/lua/net_box.lua | 1 + > src/lua/msgpack.c | 66 ++++++++++++++ > test/app-tap/msgpack.test.lua | 157 +++++++++++++++++++++++++++++++++- > test/box/net.box.result | 58 +++++++++++++ > test/box/net.box.test.lua | 26 ++++++ > 6 files changed, 356 insertions(+), 1 deletion(-) > > diff --git a/src/box/lua/net_box.c b/src/box/lua/net_box.c > index c7063d9c8..d71f33768 100644 > --- a/src/box/lua/net_box.c > +++ b/src/box/lua/net_box.c > @@ -51,6 +51,9 @@ > > #define cfg luaL_msgpack_default > > +static uint32_t CTID_CHAR_PTR; > +static uint32_t CTID_CONST_CHAR_PTR; > + > static inline size_t > netbox_prepare_request(lua_State *L, struct mpstream *stream, uint32_t r_type) > { > @@ -745,9 +748,54 @@ netbox_decode_execute(struct lua_State *L) > return 2; > } > > +/** > + * net_box.check_iproto_data(buf.rpos, buf.wpos - buf.rpos) > + * -> new_rpos > + * -> nil, err_msg > + */ > +int > +netbox_check_iproto_data(struct lua_State *L) Instead of adding this function to net_box.c, I'd rather try to add msgpack helpers for decoding a map, similar to msgpack.check_array added by your patch, and use them in net_box.lua. > +{ > + uint32_t ctypeid; > + const char *data = *(const char **) luaL_checkcdata(L, 1, &ctypeid); > + if (ctypeid != CTID_CHAR_PTR && ctypeid != CTID_CONST_CHAR_PTR) > + return luaL_error(L, > + "net_box.check_iproto_data: 'char *' or " > + "'const char *' expected"); > + > + if (!lua_isnumber(L, 2)) > + return luaL_error(L, "net_box.check_iproto_data: number " > + "expected as 2nd argument"); > + const char *end = data + lua_tointeger(L, 2); > + > + int ok = data < end && > + mp_typeof(*data) == MP_MAP && > + mp_check_map(data, end) <= 0 && > + mp_decode_map(&data) == 1 && > + data < end && > + mp_typeof(*data) == MP_UINT && > + mp_check_uint(data, end) <= 0 && > + mp_decode_uint(&data) == IPROTO_DATA; > + > + if (!ok) { > + lua_pushnil(L); > + lua_pushstring(L, > + "net_box.check_iproto_data: wrong iproto data packet"); > + return 2; > + } > + > + *(const char **) luaL_pushcdata(L, ctypeid) = data; > + return 1; > +} > + > int > luaopen_net_box(struct lua_State *L) > { > + CTID_CHAR_PTR = luaL_ctypeid(L, "char *"); > + assert(CTID_CHAR_PTR != 0); > + CTID_CONST_CHAR_PTR = luaL_ctypeid(L, "const char *"); > + assert(CTID_CONST_CHAR_PTR != 0); > + > static const luaL_Reg net_box_lib[] = { > { "encode_ping", netbox_encode_ping }, > { "encode_call_16", netbox_encode_call_16 }, > @@ -765,6 +813,7 @@ luaopen_net_box(struct lua_State *L) > { "communicate", netbox_communicate }, > { "decode_select", netbox_decode_select }, > { "decode_execute", netbox_decode_execute }, > + { "check_iproto_data", netbox_check_iproto_data }, > { NULL, NULL} > }; > /* luaL_register_module polutes _G */ > diff --git a/src/box/lua/net_box.lua b/src/box/lua/net_box.lua > index 2bf772aa8..0a38efa5a 100644 > --- a/src/box/lua/net_box.lua > +++ b/src/box/lua/net_box.lua > @@ -1424,6 +1424,7 @@ local this_module = { > new = connect, -- Tarantool < 1.7.1 compatibility, > wrap = wrap, > establish_connection = establish_connection, > + check_iproto_data = internal.check_iproto_data, > } > > function this_module.timeout(timeout, ...) > diff --git a/src/lua/msgpack.c b/src/lua/msgpack.c > index b47006038..fca440660 100644 > --- a/src/lua/msgpack.c > +++ b/src/lua/msgpack.c > @@ -51,6 +51,7 @@ luamp_error(void *error_ctx) > } > > static uint32_t CTID_CHAR_PTR; > +static uint32_t CTID_CONST_CHAR_PTR; > static uint32_t CTID_STRUCT_IBUF; > > struct luaL_serializer *luaL_msgpack_default = NULL; > @@ -418,6 +419,68 @@ lua_ibuf_msgpack_decode(lua_State *L) > return 2; > } > > +/** > + * msgpack.check_array(buf.rpos, buf.wpos - buf.rpos, [, arr_len]) > + * -> new_rpos, arr_len > + * -> nil, err_msg > + */ > +static int > +lua_check_array(lua_State *L) > +{ > + uint32_t ctypeid; > + const char *data = *(const char **) luaL_checkcdata(L, 1, &ctypeid); > + if (ctypeid != CTID_CHAR_PTR && ctypeid != CTID_CONST_CHAR_PTR) Hm, msgpack.decode doesn't care about CTID_CONST_CHAR_PTR. Why should we? > + return luaL_error(L, "msgpack.check_array: 'char *' or " > + "'const char *' expected"); > + > + if (!lua_isnumber(L, 2)) > + return luaL_error(L, "msgpack.check_array: number expected as " > + "2nd argument"); > + const char *end = data + lua_tointeger(L, 2); > + > + if (!lua_isnoneornil(L, 3) && !lua_isnumber(L, 3)) > + return luaL_error(L, "msgpack.check_array: number or nil " > + "expected as 3rd argument"); Why not simply luaL_checkinteger? > + > + static const char *end_of_buffer_msg = "msgpack.check_array: " > + "unexpected end of buffer"; No point to make this variable static. > + > + if (data >= end) { > + lua_pushnil(L); > + lua_pushstring(L, end_of_buffer_msg); msgpack.decode throws an error when it fails to decode msgpack data, so I think this function should throw too. > + return 2; > + } > + > + if (mp_typeof(*data) != MP_ARRAY) { > + lua_pushnil(L); > + lua_pushstring(L, "msgpack.check_array: wrong array header"); > + return 2; > + } > + > + if (mp_check_array(data, end) > 0) { > + lua_pushnil(L); > + lua_pushstring(L, end_of_buffer_msg); > + return 2; > + } > + > + uint32_t len = mp_decode_array(&data); > + > + if (!lua_isnoneornil(L, 3)) { > + uint32_t exp_len = (uint32_t) lua_tointeger(L, 3); IMO it would be better if you set exp_len when you checked the arguments (using luaL_checkinteger). > + if (len != exp_len) { > + lua_pushnil(L); > + lua_pushfstring(L, "msgpack.check_array: expected " > + "array of length %d, got length %d", > + len, exp_len); > + return 2; > + } > + } > + > + *(const char **) luaL_pushcdata(L, ctypeid) = data; > + lua_pushinteger(L, len); > + return 2; > +} > + > static int > lua_msgpack_new(lua_State *L); > > @@ -426,6 +489,7 @@ static const luaL_Reg msgpacklib[] = { > { "decode", lua_msgpack_decode }, > { "decode_unchecked", lua_msgpack_decode_unchecked }, > { "ibuf_decode", lua_ibuf_msgpack_decode }, > + { "check_array", lua_check_array }, > { "new", lua_msgpack_new }, > { NULL, NULL } > }; > @@ -447,6 +511,8 @@ luaopen_msgpack(lua_State *L) > assert(CTID_STRUCT_IBUF != 0); > CTID_CHAR_PTR = luaL_ctypeid(L, "char *"); > assert(CTID_CHAR_PTR != 0); > + CTID_CONST_CHAR_PTR = luaL_ctypeid(L, "const char *"); > + assert(CTID_CONST_CHAR_PTR != 0); > luaL_msgpack_default = luaL_newserializer(L, "msgpack", msgpacklib); > return 1; > } ^ permalink raw reply [flat|nested] 28+ messages in thread
* Re: [PATCH v2 5/6] net.box: add helpers to decode msgpack headers 2019-01-10 17:29 ` Vladimir Davydov @ 2019-02-01 15:11 ` Alexander Turenko 2019-03-05 12:00 ` Alexander Turenko 0 siblings, 1 reply; 28+ messages in thread From: Alexander Turenko @ 2019-02-01 15:11 UTC (permalink / raw) To: Vladimir Davydov; +Cc: tarantool-patches Splitted this patch to two ones: * lua: add non-recursive msgpack decoding functions * net.box: add skip_header option to use with buffer I attached them at end of the email. WBR, Alexander Turenko. On Thu, Jan 10, 2019 at 08:29:33PM +0300, Vladimir Davydov wrote: > On Wed, Jan 09, 2019 at 11:20:13PM +0300, Alexander Turenko wrote: > > Needed for #3276. > > > > @TarantoolBot document > > Title: net.box: helpers to decode msgpack headers > > > > They allow to skip iproto packet and msgpack array headers and pass raw > > msgpack data to some other function, say, merger. > > > > Contracts: > > > > ``` > > net_box.check_iproto_data(buf.rpos, buf.wpos - buf.rpos) > > -> new_rpos > > -> nil, err_msg > > I'd prefer if this was done right in net.box.select or whatever function > writing the response to ibuf. Yes, this is going to break backward > compatibility, but IMO it's OK for 2.1 - I doubt anybody have used this > weird high perf API anyway. 1. This will break tarantool/shard. 2. Hey, Guido thinks it is okay to break compatibility btw Python 2 and Python 3 and it seems that Python 2 is in use ten years or like so. I can do it under a separate option: skip_iproto_header or skip_header. It is not about a packet header, but part of body, however I have no better variants. > > msgpack.check_array(buf.rpos, buf.wpos - buf.rpos, [, arr_len]) > > -> new_rpos, arr_len > > -> nil, err_msg > > This seems to be OK, although I'm not sure if we really need to check > the length in this function. Looks like we will definitely need it > because of net.box.call, which wraps function return value in an array. > Not sure about the name either, because it doesn't just checks the > msgpack - it decodes it, but can't come up with anything substantially > better. May be, msgpack.decode_array? Re check length: the reason was to simplify user's code, but ok, it will not much more complex if we'll factor this check out. Like so (except from the merger's commit message): ``` conn:call('batch_select', <...>, {buffer = buf, skip_header = true}) local len, _ len, buf.rpos = msgpack.decode_array(buf.rpos, buf:size()) assert(len == 1) _, buf.rpos = msgpack.decode_array(buf.rpos, buf:size()) ``` Re name: now I understood: decode_unchecked() is like mp_decode(), decode() is like mp_check() + mp_decode(). So it worth to rename it to decode_array(). Done. Also I changed order of return values to match msgpack.decode() (before it matches msgpack.ibuf_decode()). > > ``` > > > > Below the example with msgpack.decode() as the function that need raw > > msgpack data. It is just to illustrate the approach, there is no sense > > to skip iproto/array headers manually in Lua and then decode the rest in > > Lua. But it worth when the raw msgpack data is subject to process in a C > > module. > > > > ```lua > > local function single_select(space, ...) > > return box.space[space]:select(...) > > end > > > > local function batch_select(spaces, ...) > > local res = {} > > for _, space in ipairs(spaces) do > > table.insert(res, box.space[space]:select(...)) > > end > > return res > > end > > > > _G.single_select = single_select > > _G.batch_select = batch_select > > > > local res > > > > local buf = buffer.ibuf() > > conn.space.s:select(nil, {buffer = buf}) > > -- check and skip iproto_data header > > buf.rpos = assert(net_box.check_iproto_data(buf.rpos, buf.wpos - buf.rpos)) > > -- check that we really got data from :select() as result > > res, buf.rpos = msgpack.decode(buf.rpos, buf.wpos - buf.rpos) > > -- check that the buffer ends > > assert(buf.rpos == buf.wpos) > > > > buf:recycle() > > conn:call('single_select', {'s'}, {buffer = buf}) > > -- check and skip the iproto_data header > > buf.rpos = assert(net_box.check_iproto_data(buf.rpos, buf.wpos - buf.rpos)) > > -- check and skip the array around return values > > buf.rpos = assert(msgpack.check_array(buf.rpos, buf.wpos - buf.rpos, 1)) > > -- check that we really got data from :select() as result > > res, buf.rpos = msgpack.decode(buf.rpos, buf.wpos - buf.rpos) > > -- check that the buffer ends > > assert(buf.rpos == buf.wpos) > > > > buf:recycle() > > local spaces = {'s', 't'} > > conn:call('batch_select', {spaces}, {buffer = buf}) > > -- check and skip the iproto_data header > > buf.rpos = assert(net_box.check_iproto_data(buf.rpos, buf.wpos - buf.rpos)) > > -- check and skip the array around return values > > buf.rpos = assert(msgpack.check_array(buf.rpos, buf.wpos - buf.rpos, 1)) > > -- check and skip the array header before the first select result > > buf.rpos = assert(msgpack.check_array(buf.rpos, buf.wpos - buf.rpos, #spaces)) > > -- check that we really got data from s:select() as result > > res, buf.rpos = msgpack.decode(buf.rpos, buf.wpos - buf.rpos) > > -- t:select() data > > res, buf.rpos = msgpack.decode(buf.rpos, buf.wpos - buf.rpos) > > -- check that the buffer ends > > assert(buf.rpos == buf.wpos) > > ``` > > --- > > src/box/lua/net_box.c | 49 +++++++++++ > > src/box/lua/net_box.lua | 1 + > > src/lua/msgpack.c | 66 ++++++++++++++ > > test/app-tap/msgpack.test.lua | 157 +++++++++++++++++++++++++++++++++- > > test/box/net.box.result | 58 +++++++++++++ > > test/box/net.box.test.lua | 26 ++++++ > > 6 files changed, 356 insertions(+), 1 deletion(-) > > > > diff --git a/src/box/lua/net_box.c b/src/box/lua/net_box.c > > index c7063d9c8..d71f33768 100644 > > --- a/src/box/lua/net_box.c > > +++ b/src/box/lua/net_box.c > > @@ -51,6 +51,9 @@ > > > > #define cfg luaL_msgpack_default > > > > +static uint32_t CTID_CHAR_PTR; > > +static uint32_t CTID_CONST_CHAR_PTR; > > + > > static inline size_t > > netbox_prepare_request(lua_State *L, struct mpstream *stream, uint32_t r_type) > > { > > @@ -745,9 +748,54 @@ netbox_decode_execute(struct lua_State *L) > > return 2; > > } > > > > +/** > > + * net_box.check_iproto_data(buf.rpos, buf.wpos - buf.rpos) > > + * -> new_rpos > > + * -> nil, err_msg > > + */ > > +int > > +netbox_check_iproto_data(struct lua_State *L) > > Instead of adding this function to net_box.c, I'd rather try to add > msgpack helpers for decoding a map, similar to msgpack.check_array added > by your patch, and use them in net_box.lua. Done. We discussed that we should add such helpers for all types like nil, bool, number, string, maybe bin. I think we can reuse recursive msgpack.decode() if we expect a scalar value. > > +{ > > + uint32_t ctypeid; > > + const char *data = *(const char **) luaL_checkcdata(L, 1, &ctypeid); > > + if (ctypeid != CTID_CHAR_PTR && ctypeid != CTID_CONST_CHAR_PTR) > > + return luaL_error(L, > > + "net_box.check_iproto_data: 'char *' or " > > + "'const char *' expected"); > > + > > + if (!lua_isnumber(L, 2)) > > + return luaL_error(L, "net_box.check_iproto_data: number " > > + "expected as 2nd argument"); > > + const char *end = data + lua_tointeger(L, 2); > > + > > + int ok = data < end && > > + mp_typeof(*data) == MP_MAP && > > + mp_check_map(data, end) <= 0 && > > + mp_decode_map(&data) == 1 && > > + data < end && > > + mp_typeof(*data) == MP_UINT && > > + mp_check_uint(data, end) <= 0 && > > + mp_decode_uint(&data) == IPROTO_DATA; > > + > > + if (!ok) { > > + lua_pushnil(L); > > + lua_pushstring(L, > > + "net_box.check_iproto_data: wrong iproto data packet"); > > + return 2; > > + } > > + > > + *(const char **) luaL_pushcdata(L, ctypeid) = data; > > + return 1; > > +} > > + > > int > > luaopen_net_box(struct lua_State *L) > > { > > + CTID_CHAR_PTR = luaL_ctypeid(L, "char *"); > > + assert(CTID_CHAR_PTR != 0); > > + CTID_CONST_CHAR_PTR = luaL_ctypeid(L, "const char *"); > > + assert(CTID_CONST_CHAR_PTR != 0); > > + > > static const luaL_Reg net_box_lib[] = { > > { "encode_ping", netbox_encode_ping }, > > { "encode_call_16", netbox_encode_call_16 }, > > @@ -765,6 +813,7 @@ luaopen_net_box(struct lua_State *L) > > { "communicate", netbox_communicate }, > > { "decode_select", netbox_decode_select }, > > { "decode_execute", netbox_decode_execute }, > > + { "check_iproto_data", netbox_check_iproto_data }, > > { NULL, NULL} > > }; > > /* luaL_register_module polutes _G */ > > diff --git a/src/box/lua/net_box.lua b/src/box/lua/net_box.lua > > index 2bf772aa8..0a38efa5a 100644 > > --- a/src/box/lua/net_box.lua > > +++ b/src/box/lua/net_box.lua > > @@ -1424,6 +1424,7 @@ local this_module = { > > new = connect, -- Tarantool < 1.7.1 compatibility, > > wrap = wrap, > > establish_connection = establish_connection, > > + check_iproto_data = internal.check_iproto_data, > > } > > > > function this_module.timeout(timeout, ...) > > diff --git a/src/lua/msgpack.c b/src/lua/msgpack.c > > index b47006038..fca440660 100644 > > --- a/src/lua/msgpack.c > > +++ b/src/lua/msgpack.c > > @@ -51,6 +51,7 @@ luamp_error(void *error_ctx) > > } > > > > static uint32_t CTID_CHAR_PTR; > > +static uint32_t CTID_CONST_CHAR_PTR; > > static uint32_t CTID_STRUCT_IBUF; > > > > struct luaL_serializer *luaL_msgpack_default = NULL; > > @@ -418,6 +419,68 @@ lua_ibuf_msgpack_decode(lua_State *L) > > return 2; > > } > > > > +/** > > + * msgpack.check_array(buf.rpos, buf.wpos - buf.rpos, [, arr_len]) > > + * -> new_rpos, arr_len > > + * -> nil, err_msg > > + */ > > +static int > > +lua_check_array(lua_State *L) > > +{ > > + uint32_t ctypeid; > > + const char *data = *(const char **) luaL_checkcdata(L, 1, &ctypeid); > > + if (ctypeid != CTID_CHAR_PTR && ctypeid != CTID_CONST_CHAR_PTR) > > Hm, msgpack.decode doesn't care about CTID_CONST_CHAR_PTR. Why should we? It looks natural to support a const pointer where we allow non-const one. But I don't have an example where we can obtain 'const char *' buffer with msgpack in Lua (w/o ffi.cast()). Msgpackffi returns 'const unsigned char *', but it is the bug and should be fixed in https://github.com/tarantool/tarantool/issues/3926 > > + return luaL_error(L, "msgpack.check_array: 'char *' or " > > + "'const char *' expected"); > > + > > + if (!lua_isnumber(L, 2)) > > + return luaL_error(L, "msgpack.check_array: number expected as " > > + "2nd argument"); > > + const char *end = data + lua_tointeger(L, 2); > > + > > + if (!lua_isnoneornil(L, 3) && !lua_isnumber(L, 3)) > > + return luaL_error(L, "msgpack.check_array: number or nil " > > + "expected as 3rd argument"); > > Why not simply luaL_checkinteger? We can separatelly check lua_gettop() and use luaL_checkinteger(). It looks shorter, now I see. Fixed. > > + > > + static const char *end_of_buffer_msg = "msgpack.check_array: " > > + "unexpected end of buffer"; > > No point to make this variable static. Ok. But now I removed it. > > + > > + if (data >= end) { > > + lua_pushnil(L); > > + lua_pushstring(L, end_of_buffer_msg); > > msgpack.decode throws an error when it fails to decode msgpack data, so > I think this function should throw too. Or Lua code style states we should report errors with `nil, err`. But this aspect is more about external modules as I see. It is quite unclear what is the best option for built-in modules. If one likely want to handle an error in Lua the `nil, err` approach looks better. As far as I know at least some of our commercial projects primarily use this approach and have to wrap many functions with pcall. Don't sure how much the overhead is. But anyway other msgpack functions just raise an error and it seems the new functions should have similar contract. Changed. > > + return 2; > > + } > > + > > + if (mp_typeof(*data) != MP_ARRAY) { > > + lua_pushnil(L); > > + lua_pushstring(L, "msgpack.check_array: wrong array header"); > > + return 2; > > + } > > + > > + if (mp_check_array(data, end) > 0) { > > + lua_pushnil(L); > > + lua_pushstring(L, end_of_buffer_msg); > > + return 2; > > + } > > + > > + uint32_t len = mp_decode_array(&data); > > + > > + if (!lua_isnoneornil(L, 3)) { > > + uint32_t exp_len = (uint32_t) lua_tointeger(L, 3); > > IMO it would be better if you set exp_len when you checked the arguments > (using luaL_checkinteger). Expected length was removed from the function as you suggested. > > + if (len != exp_len) { > > + lua_pushnil(L); > > + lua_pushfstring(L, "msgpack.check_array: expected " > > + "array of length %d, got length %d", > > + len, exp_len); > > + return 2; > > + } > > + } > > + > > + *(const char **) luaL_pushcdata(L, ctypeid) = data; > > + lua_pushinteger(L, len); > > + return 2; > > +} > > + > > static int > > lua_msgpack_new(lua_State *L); > > > > @@ -426,6 +489,7 @@ static const luaL_Reg msgpacklib[] = { > > { "decode", lua_msgpack_decode }, > > { "decode_unchecked", lua_msgpack_decode_unchecked }, > > { "ibuf_decode", lua_ibuf_msgpack_decode }, > > + { "check_array", lua_check_array }, > > { "new", lua_msgpack_new }, > > { NULL, NULL } > > }; > > @@ -447,6 +511,8 @@ luaopen_msgpack(lua_State *L) > > assert(CTID_STRUCT_IBUF != 0); > > CTID_CHAR_PTR = luaL_ctypeid(L, "char *"); > > assert(CTID_CHAR_PTR != 0); > > + CTID_CONST_CHAR_PTR = luaL_ctypeid(L, "const char *"); > > + assert(CTID_CONST_CHAR_PTR != 0); > > luaL_msgpack_default = luaL_newserializer(L, "msgpack", msgpacklib); > > return 1; > > } ---- commit 8c820dff279734d79e26591dcb771f7c6ab13639 Author: Alexander Turenko <alexander.turenko@tarantool.org> Date: Thu Jan 31 01:45:22 2019 +0300 lua: add non-recursive msgpack decoding functions Needed for #3276. @TarantoolBot document Title: Non-recursive msgpack decoding functions Contracts: ``` msgpack.decode_array(buf.rpos, buf:size()) -> arr_len, new_rpos msgpack.decode_map(buf.rpos, buf:size()) -> map_len, new_rpos ``` These functions are intended to be used with a msgpack buffer received from net.box. A user may want to skip {[IPROTO_DATA_KEY] = ...} wrapper and an array header before pass the buffer to decode in some C function. See https://github.com/tarantool/tarantool/issues/2195 for more information re this net.box's API. Consider merger's docbot comment for usage examples. diff --git a/src/lua/msgpack.c b/src/lua/msgpack.c index b47006038..92a9efd25 100644 --- a/src/lua/msgpack.c +++ b/src/lua/msgpack.c @@ -418,6 +418,84 @@ lua_ibuf_msgpack_decode(lua_State *L) return 2; } +/** + * Verify and set arguments: data and size. + * + * Always return 0. In case of any fail raise a Lua error. + */ +static int +verify_decode_args(lua_State *L, const char *func_name, const char **data_p, + ptrdiff_t *size_p) +{ + /* Verify arguments count. */ + if (lua_gettop(L) != 2) + return luaL_error(L, "Usage: %s(ptr, size)", func_name); + + /* Verify ptr type. */ + uint32_t ctypeid; + const char *data = *(char **) luaL_checkcdata(L, 1, &ctypeid); + if (ctypeid != CTID_CHAR_PTR) + return luaL_error(L, "%s: 'char *' expected", func_name); + + /* Verify size type and value. */ + ptrdiff_t size = (ptrdiff_t) luaL_checkinteger(L, 2); + if (size <= 0) + return luaL_error(L, "%s: non-positive size", func_name); + + *data_p = data; + *size_p = size; + + return 0; +} + +/** + * msgpack.decode_array(buf.rpos, buf:size()) -> arr_len, new_rpos + */ +static int +lua_decode_array(lua_State *L) +{ + const char *func_name = "msgpack.decode_array"; + const char *data; + ptrdiff_t size; + verify_decode_args(L, func_name, &data, &size); + + if (mp_typeof(*data) != MP_ARRAY) + return luaL_error(L, "%s: unexpected msgpack type", func_name); + + if (mp_check_array(data, data + size) > 0) + return luaL_error(L, "%s: unexpected end of buffer", func_name); + + uint32_t len = mp_decode_array(&data); + + lua_pushinteger(L, len); + *(const char **) luaL_pushcdata(L, CTID_CHAR_PTR) = data; + return 2; +} + +/** + * msgpack.decode_map(buf.rpos, buf:size()) -> map_len, new_rpos + */ +static int +lua_decode_map(lua_State *L) +{ + const char *func_name = "msgpack.decode_map"; + const char *data; + ptrdiff_t size; + verify_decode_args(L, func_name, &data, &size); + + if (mp_typeof(*data) != MP_MAP) + return luaL_error(L, "%s: unexpected msgpack type", func_name); + + if (mp_check_map(data, data + size) > 0) + return luaL_error(L, "%s: unexpected end of buffer", func_name); + + uint32_t len = mp_decode_map(&data); + + lua_pushinteger(L, len); + *(const char **) luaL_pushcdata(L, CTID_CHAR_PTR) = data; + return 2; +} + static int lua_msgpack_new(lua_State *L); @@ -426,6 +504,8 @@ static const luaL_Reg msgpacklib[] = { { "decode", lua_msgpack_decode }, { "decode_unchecked", lua_msgpack_decode_unchecked }, { "ibuf_decode", lua_ibuf_msgpack_decode }, + { "decode_array", lua_decode_array }, + { "decode_map", lua_decode_map }, { "new", lua_msgpack_new }, { NULL, NULL } }; diff --git a/test/app-tap/msgpack.test.lua b/test/app-tap/msgpack.test.lua index 0e1692ad9..ee215dfb1 100755 --- a/test/app-tap/msgpack.test.lua +++ b/test/app-tap/msgpack.test.lua @@ -49,9 +49,186 @@ local function test_misc(test, s) test:ok(not st and e:match("null"), "null ibuf") end +local function test_decode_array_map(test, s) + local ffi = require('ffi') + + local usage_err = 'Usage: msgpack%.decode_[^_(]+%(ptr, size%)' + local end_of_buffer_err = 'msgpack%.decode_[^_]+: unexpected end of buffer' + local non_positive_size_err = 'msgpack.decode_[^_]+: non%-positive size' + + local decode_cases = { + { + 'fixarray', + func = s.decode_array, + data = ffi.cast('char *', '\x94'), + size = 1, + exp_len = 4, + exp_rewind = 1, + }, + { + 'array 16', + func = s.decode_array, + data = ffi.cast('char *', '\xdc\x00\x04'), + size = 3, + exp_len = 4, + exp_rewind = 3, + }, + { + 'array 32', + func = s.decode_array, + data = ffi.cast('char *', '\xdd\x00\x00\x00\x04'), + size = 5, + exp_len = 4, + exp_rewind = 5, + }, + { + 'truncated array 16', + func = s.decode_array, + data = ffi.cast('char *', '\xdc\x00'), + size = 2, + exp_err = end_of_buffer_err, + }, + { + 'truncated array 32', + func = s.decode_array, + data = ffi.cast('char *', '\xdd\x00\x00\x00'), + size = 4, + exp_err = end_of_buffer_err, + }, + { + 'fixmap', + func = s.decode_map, + data = ffi.cast('char *', '\x84'), + size = 1, + exp_len = 4, + exp_rewind = 1, + }, + { + 'map 16', + func = s.decode_map, + data = ffi.cast('char *', '\xde\x00\x04'), + size = 3, + exp_len = 4, + exp_rewind = 3, + }, + { + 'array 32', + func = s.decode_map, + data = ffi.cast('char *', '\xdf\x00\x00\x00\x04'), + size = 5, + exp_len = 4, + exp_rewind = 5, + }, + { + 'truncated map 16', + func = s.decode_map, + data = ffi.cast('char *', '\xde\x00'), + size = 2, + exp_err = end_of_buffer_err, + }, + { + 'truncated map 32', + func = s.decode_map, + data = ffi.cast('char *', '\xdf\x00\x00\x00'), + size = 4, + exp_err = end_of_buffer_err, + }, + } + + local bad_api_cases = { + { + 'wrong msgpack type', + data = ffi.cast('char *', '\xc0'), + size = 1, + exp_err = 'msgpack.decode_[^_]+: unexpected msgpack type', + }, + { + 'zero size buffer', + data = ffi.cast('char *', ''), + size = 0, + exp_err = non_positive_size_err, + }, + { + 'negative size buffer', + data = ffi.cast('char *', ''), + size = -1, + exp_err = non_positive_size_err, + }, + { + 'size is nil', + data = ffi.cast('char *', ''), + size = nil, + exp_err = 'bad argument', + }, + { + 'no arguments', + args = {}, + exp_err = usage_err, + }, + { + 'one argument', + args = {ffi.cast('char *', '')}, + exp_err = usage_err, + }, + { + 'data is nil', + args = {nil, 1}, + exp_err = 'expected cdata as 1 argument', + }, + { + 'data is not cdata', + args = {1, 1}, + exp_err = 'expected cdata as 1 argument', + }, + { + 'data with wrong cdata type', + args = {box.tuple.new(), 1}, + exp_err = "msgpack.decode_[^_]+: 'char %*' expected", + }, + { + 'size has wrong type', + args = {ffi.cast('char *', ''), 'eee'}, + exp_err = 'bad argument', + }, + } + + test:plan(#decode_cases + 2 * #bad_api_cases) + + -- Decode cases. + for _, case in ipairs(decode_cases) do + if case.exp_err ~= nil then + local ok, err = pcall(case.func, case.data, case.size) + local description = ('bad; %s'):format(case[1]) + test:ok(ok == false and err:match(case.exp_err), description) + else + local len, new_buf = case.func(case.data, case.size) + local rewind = new_buf - case.data + local description = ('good; %s'):format(case[1]) + test:is_deeply({len, rewind}, {case.exp_len, case.exp_rewind}, + description) + end + end + + -- Bad api usage cases. + for _, func_name in ipairs({'decode_array', 'decode_map'}) do + for _, case in ipairs(bad_api_cases) do + local ok, err + if case.args ~= nil then + local args_len = table.maxn(case.args) + ok, err = pcall(s[func_name], unpack(case.args, 1, args_len)) + else + ok, err = pcall(s[func_name], case.data, case.size) + end + local description = ('%s bad api usage; %s'):format(func_name, + case[1]) + test:ok(ok == false and err:match(case.exp_err), description) + end + end +end + tap.test("msgpack", function(test) local serializer = require('msgpack') - test:plan(10) + test:plan(11) test:test("unsigned", common.test_unsigned, serializer) test:test("signed", common.test_signed, serializer) test:test("double", common.test_double, serializer) @@ -62,4 +239,5 @@ tap.test("msgpack", function(test) test:test("ucdata", common.test_ucdata, serializer) test:test("offsets", test_offsets, serializer) test:test("misc", test_misc, serializer) + test:test("decode_array_map", test_decode_array_map, serializer) end) ---- commit 3868d5c2551c893f16bd05c79d4d52a564c6a833 Author: Alexander Turenko <alexander.turenko@tarantool.org> Date: Thu Jan 31 01:59:18 2019 +0300 net.box: add skip_header option to use with buffer Needed for #3276. @TarantoolBot document Title: net.box: skip_header option This option instructs net.box to skip {[IPROTO_DATA_KEY] = ...} wrapper from a buffer. This may be needed to pass this buffer to some C function when it expects some specific msgpack input. See src/box/lua/net_box.lua for examples. Also consider merger's docbot comment for more examples. diff --git a/src/box/lua/net_box.lua b/src/box/lua/net_box.lua index 2bf772aa8..53c93cafb 100644 --- a/src/box/lua/net_box.lua +++ b/src/box/lua/net_box.lua @@ -15,6 +15,7 @@ local max = math.max local fiber_clock = fiber.clock local fiber_self = fiber.self local decode = msgpack.decode_unchecked +local decode_map = msgpack.decode_map local table_new = require('table.new') local check_iterator_type = box.internal.check_iterator_type @@ -483,8 +484,8 @@ local function create_transport(host, port, user, password, callback, -- @retval nil, error Error occured. -- @retval not nil Future object. -- - local function perform_async_request(buffer, method, on_push, on_push_ctx, - ...) + local function perform_async_request(buffer, skip_header, method, on_push, + on_push_ctx, ...) if state ~= 'active' and state ~= 'fetch_schema' then return nil, box.error.new({code = last_errno or E_NO_CONNECTION, reason = last_error}) @@ -497,12 +498,13 @@ local function create_transport(host, port, user, password, callback, local id = next_request_id method_encoder[method](send_buf, id, ...) next_request_id = next_id(id) - -- Request in most cases has maximum 8 members: - -- method, buffer, id, cond, errno, response, on_push, - -- on_push_ctx. - local request = setmetatable(table_new(0, 8), request_mt) + -- Request in most cases has maximum 9 members: + -- method, buffer, skip_header, id, cond, errno, response, + -- on_push, on_push_ctx. + local request = setmetatable(table_new(0, 9), request_mt) request.method = method request.buffer = buffer + request.skip_header = skip_header request.id = id request.cond = fiber.cond() requests[id] = request @@ -516,10 +518,11 @@ local function create_transport(host, port, user, password, callback, -- @retval nil, error Error occured. -- @retval not nil Response object. -- - local function perform_request(timeout, buffer, method, on_push, - on_push_ctx, ...) + local function perform_request(timeout, buffer, skip_header, method, + on_push, on_push_ctx, ...) local request, err = - perform_async_request(buffer, method, on_push, on_push_ctx, ...) + perform_async_request(buffer, skip_header, method, on_push, + on_push_ctx, ...) if not request then return nil, err end @@ -554,6 +557,15 @@ local function create_transport(host, port, user, password, callback, local wpos = buffer:alloc(body_len) ffi.copy(wpos, body_rpos, body_len) body_len = tonumber(body_len) + if request.skip_header then + -- Skip {[IPROTO_DATA_KEY] = ...} wrapper. + local map_len, key + map_len, buffer.rpos = decode_map(buffer.rpos, buffer:size()) + assert(map_len == 1) + key, buffer.rpos = decode(buffer.rpos) + assert(key == IPROTO_DATA_KEY) + body_len = buffer:size() + end if status == IPROTO_OK_KEY then request.response = body_len requests[id] = nil @@ -1047,17 +1059,18 @@ end function remote_methods:_request(method, opts, ...) local transport = self._transport - local on_push, on_push_ctx, buffer, deadline + local on_push, on_push_ctx, buffer, skip_header, deadline -- Extract options, set defaults, check if the request is -- async. if opts then buffer = opts.buffer + skip_header = opts.skip_header if opts.is_async then if opts.on_push or opts.on_push_ctx then error('To handle pushes in an async request use future:pairs()') end - return transport.perform_async_request(buffer, method, table.insert, - {}, ...) + return transport.perform_async_request(buffer, skip_header, method, + table.insert, {}, ...) end if opts.timeout then -- conn.space:request(, { timeout = timeout }) @@ -1079,8 +1092,9 @@ function remote_methods:_request(method, opts, ...) transport.wait_state('active', timeout) timeout = deadline and max(0, deadline - fiber_clock()) end - local res, err = transport.perform_request(timeout, buffer, method, - on_push, on_push_ctx, ...) + local res, err = transport.perform_request(timeout, buffer, skip_header, + method, on_push, on_push_ctx, + ...) if err then box.error(err) end @@ -1283,10 +1297,10 @@ function console_methods:eval(line, timeout) end if self.protocol == 'Binary' then local loader = 'return require("console").eval(...)' - res, err = pr(timeout, nil, 'eval', nil, nil, loader, {line}) + res, err = pr(timeout, nil, false, 'eval', nil, nil, loader, {line}) else assert(self.protocol == 'Lua console') - res, err = pr(timeout, nil, 'inject', nil, nil, line..'$EOF$\n') + res, err = pr(timeout, nil, false, 'inject', nil, nil, line..'$EOF$\n') end if err then box.error(err) diff --git a/test/box/net.box.result b/test/box/net.box.result index 2b5a84646..71d0e0a50 100644 --- a/test/box/net.box.result +++ b/test/box/net.box.result @@ -29,7 +29,7 @@ function x_select(cn, space_id, index_id, iterator, offset, limit, key, opts) offset, limit, key) return ret end -function x_fatal(cn) cn._transport.perform_request(nil, nil, 'inject', nil, nil, '\x80') end +function x_fatal(cn) cn._transport.perform_request(nil, nil, false, 'inject', nil, nil, '\x80') end test_run:cmd("setopt delimiter ''"); --- ... @@ -1573,6 +1573,18 @@ result --- - {48: [[2]]} ... +-- replace + skip_header +c.space.test:replace({2}, {buffer = ibuf, skip_header = true}) +--- +- 7 +... +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) +--- +... +result +--- +- [[2]] +... -- insert c.space.test:insert({3}, {buffer = ibuf}) --- @@ -1585,6 +1597,21 @@ result --- - {48: [[3]]} ... +-- insert + skip_header +_ = space:delete({3}) +--- +... +c.space.test:insert({3}, {buffer = ibuf, skip_header = true}) +--- +- 7 +... +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) +--- +... +result +--- +- [[3]] +... -- update c.space.test:update({3}, {}, {buffer = ibuf}) --- @@ -1608,6 +1635,29 @@ result --- - {48: [[3]]} ... +-- update + skip_header +c.space.test:update({3}, {}, {buffer = ibuf, skip_header = true}) +--- +- 7 +... +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) +--- +... +result +--- +- [[3]] +... +c.space.test.index.primary:update({3}, {}, {buffer = ibuf, skip_header = true}) +--- +- 7 +... +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) +--- +... +result +--- +- [[3]] +... -- upsert c.space.test:upsert({4}, {}, {buffer = ibuf}) --- @@ -1620,6 +1670,18 @@ result --- - {48: []} ... +-- upsert + skip_header +c.space.test:upsert({4}, {}, {buffer = ibuf, skip_header = true}) +--- +- 5 +... +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) +--- +... +result +--- +- [] +... -- delete c.space.test:upsert({4}, {}, {buffer = ibuf}) --- @@ -1632,6 +1694,18 @@ result --- - {48: []} ... +-- delete + skip_header +c.space.test:upsert({4}, {}, {buffer = ibuf, skip_header = true}) +--- +- 5 +... +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) +--- +... +result +--- +- [] +... -- select c.space.test.index.primary:select({3}, {iterator = 'LE', buffer = ibuf}) --- @@ -1644,6 +1718,18 @@ result --- - {48: [[3], [2], [1, 'hello']]} ... +-- select + skip_header +c.space.test.index.primary:select({3}, {iterator = 'LE', buffer = ibuf, skip_header = true}) +--- +- 17 +... +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) +--- +... +result +--- +- [[3], [2], [1, 'hello']] +... -- select len = c.space.test:select({}, {buffer = ibuf}) --- @@ -1667,6 +1753,29 @@ result --- - {48: [[1, 'hello'], [2], [3], [4]]} ... +-- select + skip_header +len = c.space.test:select({}, {buffer = ibuf, skip_header = true}) +--- +... +ibuf.rpos + len == ibuf.wpos +--- +- true +... +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) +--- +... +ibuf.rpos == ibuf.wpos +--- +- true +... +len +--- +- 19 +... +result +--- +- [[1, 'hello'], [2], [3], [4]] +... -- call c:call("echo", {1, 2, 3}, {buffer = ibuf}) --- @@ -1701,6 +1810,40 @@ result --- - {48: []} ... +-- call + skip_header +c:call("echo", {1, 2, 3}, {buffer = ibuf, skip_header = true}) +--- +- 8 +... +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) +--- +... +result +--- +- [1, 2, 3] +... +c:call("echo", {}, {buffer = ibuf, skip_header = true}) +--- +- 5 +... +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) +--- +... +result +--- +- [] +... +c:call("echo", nil, {buffer = ibuf, skip_header = true}) +--- +- 5 +... +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) +--- +... +result +--- +- [] +... -- eval c:eval("echo(...)", {1, 2, 3}, {buffer = ibuf}) --- @@ -1735,6 +1878,40 @@ result --- - {48: []} ... +-- eval + skip_header +c:eval("echo(...)", {1, 2, 3}, {buffer = ibuf, skip_header = true}) +--- +- 5 +... +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) +--- +... +result +--- +- [] +... +c:eval("echo(...)", {}, {buffer = ibuf, skip_header = true}) +--- +- 5 +... +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) +--- +... +result +--- +- [] +... +c:eval("echo(...)", nil, {buffer = ibuf, skip_header = true}) +--- +- 5 +... +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) +--- +... +result +--- +- [] +... -- unsupported methods c.space.test:get({1}, { buffer = ibuf}) --- @@ -2571,7 +2748,7 @@ c.space.test:delete{1} -- -- Break a connection to test reconnect_after. -- -_ = c._transport.perform_request(nil, nil, 'inject', nil, nil, '\x80') +_ = c._transport.perform_request(nil, nil, false, 'inject', nil, nil, '\x80') --- ... c.state @@ -3205,7 +3382,7 @@ c = net:connect(box.cfg.listen, {reconnect_after = 0.01}) future = c:call('long_function', {1, 2, 3}, {is_async = true}) --- ... -_ = c._transport.perform_request(nil, nil, 'inject', nil, nil, '\x80') +_ = c._transport.perform_request(nil, nil, false, 'inject', nil, nil, '\x80') --- ... while not c:is_connected() do fiber.sleep(0.01) end @@ -3340,7 +3517,7 @@ c:ping() -- new attempts to read any data - the connection is closed -- already. -- -f = fiber.create(c._transport.perform_request, nil, nil, 'call_17', nil, nil, 'long', {}) c._transport.perform_request(nil, nil, 'inject', nil, nil, '\x80') +f = fiber.create(c._transport.perform_request, nil, nil, false, 'call_17', nil, nil, 'long', {}) c._transport.perform_request(nil, nil, false, 'inject', nil, nil, '\x80') --- ... while f:status() ~= 'dead' do fiber.sleep(0.01) end @@ -3359,7 +3536,7 @@ c = net:connect(box.cfg.listen) data = msgpack.encode(18400000000000000000)..'aaaaaaa' --- ... -c._transport.perform_request(nil, nil, 'inject', nil, nil, data) +c._transport.perform_request(nil, nil, false, 'inject', nil, nil, data) --- - null - Peer closed diff --git a/test/box/net.box.test.lua b/test/box/net.box.test.lua index 96d822820..48cc7147d 100644 --- a/test/box/net.box.test.lua +++ b/test/box/net.box.test.lua @@ -12,7 +12,7 @@ function x_select(cn, space_id, index_id, iterator, offset, limit, key, opts) offset, limit, key) return ret end -function x_fatal(cn) cn._transport.perform_request(nil, nil, 'inject', nil, nil, '\x80') end +function x_fatal(cn) cn._transport.perform_request(nil, nil, false, 'inject', nil, nil, '\x80') end test_run:cmd("setopt delimiter ''"); LISTEN = require('uri').parse(box.cfg.listen) @@ -615,11 +615,22 @@ c.space.test:replace({2}, {buffer = ibuf}) result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) result +-- replace + skip_header +c.space.test:replace({2}, {buffer = ibuf, skip_header = true}) +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) +result + -- insert c.space.test:insert({3}, {buffer = ibuf}) result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) result +-- insert + skip_header +_ = space:delete({3}) +c.space.test:insert({3}, {buffer = ibuf, skip_header = true}) +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) +result + -- update c.space.test:update({3}, {}, {buffer = ibuf}) result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) @@ -628,21 +639,44 @@ c.space.test.index.primary:update({3}, {}, {buffer = ibuf}) result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) result +-- update + skip_header +c.space.test:update({3}, {}, {buffer = ibuf, skip_header = true}) +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) +result +c.space.test.index.primary:update({3}, {}, {buffer = ibuf, skip_header = true}) +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) +result + -- upsert c.space.test:upsert({4}, {}, {buffer = ibuf}) result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) result +-- upsert + skip_header +c.space.test:upsert({4}, {}, {buffer = ibuf, skip_header = true}) +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) +result + -- delete c.space.test:upsert({4}, {}, {buffer = ibuf}) result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) result +-- delete + skip_header +c.space.test:upsert({4}, {}, {buffer = ibuf, skip_header = true}) +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) +result + -- select c.space.test.index.primary:select({3}, {iterator = 'LE', buffer = ibuf}) result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) result +-- select + skip_header +c.space.test.index.primary:select({3}, {iterator = 'LE', buffer = ibuf, skip_header = true}) +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) +result + -- select len = c.space.test:select({}, {buffer = ibuf}) ibuf.rpos + len == ibuf.wpos @@ -651,6 +685,14 @@ ibuf.rpos == ibuf.wpos len result +-- select + skip_header +len = c.space.test:select({}, {buffer = ibuf, skip_header = true}) +ibuf.rpos + len == ibuf.wpos +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) +ibuf.rpos == ibuf.wpos +len +result + -- call c:call("echo", {1, 2, 3}, {buffer = ibuf}) result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) @@ -662,6 +704,17 @@ c:call("echo", nil, {buffer = ibuf}) result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) result +-- call + skip_header +c:call("echo", {1, 2, 3}, {buffer = ibuf, skip_header = true}) +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) +result +c:call("echo", {}, {buffer = ibuf, skip_header = true}) +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) +result +c:call("echo", nil, {buffer = ibuf, skip_header = true}) +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) +result + -- eval c:eval("echo(...)", {1, 2, 3}, {buffer = ibuf}) result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) @@ -673,6 +726,17 @@ c:eval("echo(...)", nil, {buffer = ibuf}) result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) result +-- eval + skip_header +c:eval("echo(...)", {1, 2, 3}, {buffer = ibuf, skip_header = true}) +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) +result +c:eval("echo(...)", {}, {buffer = ibuf, skip_header = true}) +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) +result +c:eval("echo(...)", nil, {buffer = ibuf, skip_header = true}) +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) +result + -- unsupported methods c.space.test:get({1}, { buffer = ibuf}) c.space.test.index.primary:min({}, { buffer = ibuf}) @@ -1063,7 +1127,7 @@ c.space.test:delete{1} -- -- Break a connection to test reconnect_after. -- -_ = c._transport.perform_request(nil, nil, 'inject', nil, nil, '\x80') +_ = c._transport.perform_request(nil, nil, false, 'inject', nil, nil, '\x80') c.state while not c:is_connected() do fiber.sleep(0.01) end c:ping() @@ -1291,7 +1355,7 @@ finalize_long() -- c = net:connect(box.cfg.listen, {reconnect_after = 0.01}) future = c:call('long_function', {1, 2, 3}, {is_async = true}) -_ = c._transport.perform_request(nil, nil, 'inject', nil, nil, '\x80') +_ = c._transport.perform_request(nil, nil, false, 'inject', nil, nil, '\x80') while not c:is_connected() do fiber.sleep(0.01) end finalize_long() future:wait_result(100) @@ -1348,7 +1412,7 @@ c:ping() -- new attempts to read any data - the connection is closed -- already. -- -f = fiber.create(c._transport.perform_request, nil, nil, 'call_17', nil, nil, 'long', {}) c._transport.perform_request(nil, nil, 'inject', nil, nil, '\x80') +f = fiber.create(c._transport.perform_request, nil, nil, false, 'call_17', nil, nil, 'long', {}) c._transport.perform_request(nil, nil, false, 'inject', nil, nil, '\x80') while f:status() ~= 'dead' do fiber.sleep(0.01) end c:close() @@ -1358,7 +1422,7 @@ c:close() -- c = net:connect(box.cfg.listen) data = msgpack.encode(18400000000000000000)..'aaaaaaa' -c._transport.perform_request(nil, nil, 'inject', nil, nil, data) +c._transport.perform_request(nil, nil, false, 'inject', nil, nil, data) c:close() test_run:grep_log('default', 'too big packet size in the header') ~= nil ^ permalink raw reply [flat|nested] 28+ messages in thread
* Re: [PATCH v2 5/6] net.box: add helpers to decode msgpack headers 2019-02-01 15:11 ` Alexander Turenko @ 2019-03-05 12:00 ` Alexander Turenko 0 siblings, 0 replies; 28+ messages in thread From: Alexander Turenko @ 2019-03-05 12:00 UTC (permalink / raw) To: Vladimir Davydov; +Cc: tarantool-patches Fixed for the case when we're read into a non-empty buffer. The patch is below. WBR, Alexander Turenko. diff --git a/src/box/lua/net_box.lua b/src/box/lua/net_box.lua index 08bfb3444..c6ed3e138 100644 --- a/src/box/lua/net_box.lua +++ b/src/box/lua/net_box.lua @@ -554,18 +554,18 @@ local function create_transport(host, port, user, password, callback, if buffer ~= nil then -- Copy xrow.body to user-provided buffer local body_len = body_end - body_rpos - local wpos = buffer:alloc(body_len) - ffi.copy(wpos, body_rpos, body_len) - body_len = tonumber(body_len) if request.skip_header then -- Skip {[IPROTO_DATA_KEY] = ...} wrapper. local map_len, key - map_len, buffer.rpos = decode_map(buffer.rpos, buffer:size()) + map_len, body_rpos = decode_map(body_rpos, body_len) assert(map_len == 1) - key, buffer.rpos = decode(buffer.rpos) + key, body_rpos = decode(body_rpos) assert(key == IPROTO_DATA_KEY) - body_len = buffer:size() + body_len = body_end - body_rpos end + local wpos = buffer:alloc(body_len) + ffi.copy(wpos, body_rpos, body_len) + body_len = tonumber(body_len) if status == IPROTO_OK_KEY then request.response = body_len requests[id] = nil diff --git a/test/box/net.box.result b/test/box/net.box.result index 1cba78a5f..37f615323 100644 --- a/test/box/net.box.result +++ b/test/box/net.box.result @@ -1912,6 +1912,41 @@ result --- - [] ... +-- make several request into a buffer with skip_header, then read +-- results +c:call("echo", {1, 2, 3}, {buffer = ibuf, skip_header = true}) +--- +- 8 +... +c:call("echo", {1, 2, 3}, {buffer = ibuf, skip_header = true}) +--- +- 8 +... +c:call("echo", {1, 2, 3}, {buffer = ibuf, skip_header = true}) +--- +- 8 +... +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) +--- +... +result +--- +- [1, 2, 3] +... +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) +--- +... +result +--- +- [1, 2, 3] +... +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) +--- +... +result +--- +- [1, 2, 3] +... -- unsupported methods c.space.test:get({1}, { buffer = ibuf}) --- diff --git a/test/box/net.box.test.lua b/test/box/net.box.test.lua index 0fe948c29..9fda23088 100644 --- a/test/box/net.box.test.lua +++ b/test/box/net.box.test.lua @@ -737,6 +737,18 @@ c:eval("echo(...)", nil, {buffer = ibuf, skip_header = true}) result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) result +-- make several request into a buffer with skip_header, then read +-- results +c:call("echo", {1, 2, 3}, {buffer = ibuf, skip_header = true}) +c:call("echo", {1, 2, 3}, {buffer = ibuf, skip_header = true}) +c:call("echo", {1, 2, 3}, {buffer = ibuf, skip_header = true}) +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) +result +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) +result +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) +result + -- unsupported methods c.space.test:get({1}, { buffer = ibuf}) c.space.test.index.primary:min({}, { buffer = ibuf}) On Fri, Feb 01, 2019 at 06:11:41PM +0300, Alexander Turenko wrote: > Splitted this patch to two ones: > > * lua: add non-recursive msgpack decoding functions > * net.box: add skip_header option to use with buffer > > I attached them at end of the email. > > WBR, Alexander Turenko. > > On Thu, Jan 10, 2019 at 08:29:33PM +0300, Vladimir Davydov wrote: > > On Wed, Jan 09, 2019 at 11:20:13PM +0300, Alexander Turenko wrote: > > > Needed for #3276. > > > > > > @TarantoolBot document > > > Title: net.box: helpers to decode msgpack headers > > > > > > They allow to skip iproto packet and msgpack array headers and pass raw > > > msgpack data to some other function, say, merger. > > > > > > Contracts: > > > > > > ``` > > > net_box.check_iproto_data(buf.rpos, buf.wpos - buf.rpos) > > > -> new_rpos > > > -> nil, err_msg > > > > I'd prefer if this was done right in net.box.select or whatever function > > writing the response to ibuf. Yes, this is going to break backward > > compatibility, but IMO it's OK for 2.1 - I doubt anybody have used this > > weird high perf API anyway. > > 1. This will break tarantool/shard. > 2. Hey, Guido thinks it is okay to break compatibility btw Python 2 and > Python 3 and it seems that Python 2 is in use ten years or like so. > > I can do it under a separate option: skip_iproto_header or skip_header. > It is not about a packet header, but part of body, however I have no > better variants. > > > > msgpack.check_array(buf.rpos, buf.wpos - buf.rpos, [, arr_len]) > > > -> new_rpos, arr_len > > > -> nil, err_msg > > > > This seems to be OK, although I'm not sure if we really need to check > > the length in this function. Looks like we will definitely need it > > because of net.box.call, which wraps function return value in an array. > > Not sure about the name either, because it doesn't just checks the > > msgpack - it decodes it, but can't come up with anything substantially > > better. May be, msgpack.decode_array? > > Re check length: the reason was to simplify user's code, but ok, it will > not much more complex if we'll factor this check out. Like so (except > from the merger's commit message): > > ``` > conn:call('batch_select', <...>, {buffer = buf, skip_header = true}) > local len, _ > len, buf.rpos = msgpack.decode_array(buf.rpos, buf:size()) > assert(len == 1) > _, buf.rpos = msgpack.decode_array(buf.rpos, buf:size()) > ``` > > Re name: now I understood: decode_unchecked() is like mp_decode(), > decode() is like mp_check() + mp_decode(). So it worth to rename it to > decode_array(). Done. > > Also I changed order of return values to match msgpack.decode() (before > it matches msgpack.ibuf_decode()). > > > > ``` > > > > > > Below the example with msgpack.decode() as the function that need raw > > > msgpack data. It is just to illustrate the approach, there is no sense > > > to skip iproto/array headers manually in Lua and then decode the rest in > > > Lua. But it worth when the raw msgpack data is subject to process in a C > > > module. > > > > > > ```lua > > > local function single_select(space, ...) > > > return box.space[space]:select(...) > > > end > > > > > > local function batch_select(spaces, ...) > > > local res = {} > > > for _, space in ipairs(spaces) do > > > table.insert(res, box.space[space]:select(...)) > > > end > > > return res > > > end > > > > > > _G.single_select = single_select > > > _G.batch_select = batch_select > > > > > > local res > > > > > > local buf = buffer.ibuf() > > > conn.space.s:select(nil, {buffer = buf}) > > > -- check and skip iproto_data header > > > buf.rpos = assert(net_box.check_iproto_data(buf.rpos, buf.wpos - buf.rpos)) > > > -- check that we really got data from :select() as result > > > res, buf.rpos = msgpack.decode(buf.rpos, buf.wpos - buf.rpos) > > > -- check that the buffer ends > > > assert(buf.rpos == buf.wpos) > > > > > > buf:recycle() > > > conn:call('single_select', {'s'}, {buffer = buf}) > > > -- check and skip the iproto_data header > > > buf.rpos = assert(net_box.check_iproto_data(buf.rpos, buf.wpos - buf.rpos)) > > > -- check and skip the array around return values > > > buf.rpos = assert(msgpack.check_array(buf.rpos, buf.wpos - buf.rpos, 1)) > > > -- check that we really got data from :select() as result > > > res, buf.rpos = msgpack.decode(buf.rpos, buf.wpos - buf.rpos) > > > -- check that the buffer ends > > > assert(buf.rpos == buf.wpos) > > > > > > buf:recycle() > > > local spaces = {'s', 't'} > > > conn:call('batch_select', {spaces}, {buffer = buf}) > > > -- check and skip the iproto_data header > > > buf.rpos = assert(net_box.check_iproto_data(buf.rpos, buf.wpos - buf.rpos)) > > > -- check and skip the array around return values > > > buf.rpos = assert(msgpack.check_array(buf.rpos, buf.wpos - buf.rpos, 1)) > > > -- check and skip the array header before the first select result > > > buf.rpos = assert(msgpack.check_array(buf.rpos, buf.wpos - buf.rpos, #spaces)) > > > -- check that we really got data from s:select() as result > > > res, buf.rpos = msgpack.decode(buf.rpos, buf.wpos - buf.rpos) > > > -- t:select() data > > > res, buf.rpos = msgpack.decode(buf.rpos, buf.wpos - buf.rpos) > > > -- check that the buffer ends > > > assert(buf.rpos == buf.wpos) > > > ``` > > > --- > > > src/box/lua/net_box.c | 49 +++++++++++ > > > src/box/lua/net_box.lua | 1 + > > > src/lua/msgpack.c | 66 ++++++++++++++ > > > test/app-tap/msgpack.test.lua | 157 +++++++++++++++++++++++++++++++++- > > > test/box/net.box.result | 58 +++++++++++++ > > > test/box/net.box.test.lua | 26 ++++++ > > > 6 files changed, 356 insertions(+), 1 deletion(-) > > > > > > diff --git a/src/box/lua/net_box.c b/src/box/lua/net_box.c > > > index c7063d9c8..d71f33768 100644 > > > --- a/src/box/lua/net_box.c > > > +++ b/src/box/lua/net_box.c > > > @@ -51,6 +51,9 @@ > > > > > > #define cfg luaL_msgpack_default > > > > > > +static uint32_t CTID_CHAR_PTR; > > > +static uint32_t CTID_CONST_CHAR_PTR; > > > + > > > static inline size_t > > > netbox_prepare_request(lua_State *L, struct mpstream *stream, uint32_t r_type) > > > { > > > @@ -745,9 +748,54 @@ netbox_decode_execute(struct lua_State *L) > > > return 2; > > > } > > > > > > +/** > > > + * net_box.check_iproto_data(buf.rpos, buf.wpos - buf.rpos) > > > + * -> new_rpos > > > + * -> nil, err_msg > > > + */ > > > +int > > > +netbox_check_iproto_data(struct lua_State *L) > > > > Instead of adding this function to net_box.c, I'd rather try to add > > msgpack helpers for decoding a map, similar to msgpack.check_array added > > by your patch, and use them in net_box.lua. > > Done. > > We discussed that we should add such helpers for all types like nil, > bool, number, string, maybe bin. I think we can reuse recursive > msgpack.decode() if we expect a scalar value. > > > > +{ > > > + uint32_t ctypeid; > > > + const char *data = *(const char **) luaL_checkcdata(L, 1, &ctypeid); > > > + if (ctypeid != CTID_CHAR_PTR && ctypeid != CTID_CONST_CHAR_PTR) > > > + return luaL_error(L, > > > + "net_box.check_iproto_data: 'char *' or " > > > + "'const char *' expected"); > > > + > > > + if (!lua_isnumber(L, 2)) > > > + return luaL_error(L, "net_box.check_iproto_data: number " > > > + "expected as 2nd argument"); > > > + const char *end = data + lua_tointeger(L, 2); > > > + > > > + int ok = data < end && > > > + mp_typeof(*data) == MP_MAP && > > > + mp_check_map(data, end) <= 0 && > > > + mp_decode_map(&data) == 1 && > > > + data < end && > > > + mp_typeof(*data) == MP_UINT && > > > + mp_check_uint(data, end) <= 0 && > > > + mp_decode_uint(&data) == IPROTO_DATA; > > > + > > > + if (!ok) { > > > + lua_pushnil(L); > > > + lua_pushstring(L, > > > + "net_box.check_iproto_data: wrong iproto data packet"); > > > + return 2; > > > + } > > > + > > > + *(const char **) luaL_pushcdata(L, ctypeid) = data; > > > + return 1; > > > +} > > > + > > > int > > > luaopen_net_box(struct lua_State *L) > > > { > > > + CTID_CHAR_PTR = luaL_ctypeid(L, "char *"); > > > + assert(CTID_CHAR_PTR != 0); > > > + CTID_CONST_CHAR_PTR = luaL_ctypeid(L, "const char *"); > > > + assert(CTID_CONST_CHAR_PTR != 0); > > > + > > > static const luaL_Reg net_box_lib[] = { > > > { "encode_ping", netbox_encode_ping }, > > > { "encode_call_16", netbox_encode_call_16 }, > > > @@ -765,6 +813,7 @@ luaopen_net_box(struct lua_State *L) > > > { "communicate", netbox_communicate }, > > > { "decode_select", netbox_decode_select }, > > > { "decode_execute", netbox_decode_execute }, > > > + { "check_iproto_data", netbox_check_iproto_data }, > > > { NULL, NULL} > > > }; > > > /* luaL_register_module polutes _G */ > > > diff --git a/src/box/lua/net_box.lua b/src/box/lua/net_box.lua > > > index 2bf772aa8..0a38efa5a 100644 > > > --- a/src/box/lua/net_box.lua > > > +++ b/src/box/lua/net_box.lua > > > @@ -1424,6 +1424,7 @@ local this_module = { > > > new = connect, -- Tarantool < 1.7.1 compatibility, > > > wrap = wrap, > > > establish_connection = establish_connection, > > > + check_iproto_data = internal.check_iproto_data, > > > } > > > > > > function this_module.timeout(timeout, ...) > > > diff --git a/src/lua/msgpack.c b/src/lua/msgpack.c > > > index b47006038..fca440660 100644 > > > --- a/src/lua/msgpack.c > > > +++ b/src/lua/msgpack.c > > > @@ -51,6 +51,7 @@ luamp_error(void *error_ctx) > > > } > > > > > > static uint32_t CTID_CHAR_PTR; > > > +static uint32_t CTID_CONST_CHAR_PTR; > > > static uint32_t CTID_STRUCT_IBUF; > > > > > > struct luaL_serializer *luaL_msgpack_default = NULL; > > > @@ -418,6 +419,68 @@ lua_ibuf_msgpack_decode(lua_State *L) > > > return 2; > > > } > > > > > > +/** > > > + * msgpack.check_array(buf.rpos, buf.wpos - buf.rpos, [, arr_len]) > > > + * -> new_rpos, arr_len > > > + * -> nil, err_msg > > > + */ > > > +static int > > > +lua_check_array(lua_State *L) > > > +{ > > > + uint32_t ctypeid; > > > + const char *data = *(const char **) luaL_checkcdata(L, 1, &ctypeid); > > > + if (ctypeid != CTID_CHAR_PTR && ctypeid != CTID_CONST_CHAR_PTR) > > > > Hm, msgpack.decode doesn't care about CTID_CONST_CHAR_PTR. Why should we? > > It looks natural to support a const pointer where we allow non-const > one. But I don't have an example where we can obtain 'const char *' > buffer with msgpack in Lua (w/o ffi.cast()). Msgpackffi returns 'const > unsigned char *', but it is the bug and should be fixed in > https://github.com/tarantool/tarantool/issues/3926 > > > > + return luaL_error(L, "msgpack.check_array: 'char *' or " > > > + "'const char *' expected"); > > > + > > > + if (!lua_isnumber(L, 2)) > > > + return luaL_error(L, "msgpack.check_array: number expected as " > > > + "2nd argument"); > > > + const char *end = data + lua_tointeger(L, 2); > > > + > > > + if (!lua_isnoneornil(L, 3) && !lua_isnumber(L, 3)) > > > + return luaL_error(L, "msgpack.check_array: number or nil " > > > + "expected as 3rd argument"); > > > > Why not simply luaL_checkinteger? > > We can separatelly check lua_gettop() and use luaL_checkinteger(). It > looks shorter, now I see. Fixed. > > > > + > > > + static const char *end_of_buffer_msg = "msgpack.check_array: " > > > + "unexpected end of buffer"; > > > > No point to make this variable static. > > Ok. But now I removed it. > > > > + > > > + if (data >= end) { > > > + lua_pushnil(L); > > > + lua_pushstring(L, end_of_buffer_msg); > > > > msgpack.decode throws an error when it fails to decode msgpack data, so > > I think this function should throw too. > > Or Lua code style states we should report errors with `nil, err`. But > this aspect is more about external modules as I see. It is quite unclear > what is the best option for built-in modules. > > If one likely want to handle an error in Lua the `nil, err` approach > looks better. As far as I know at least some of our commercial projects > primarily use this approach and have to wrap many functions with pcall. > Don't sure how much the overhead is. > > But anyway other msgpack functions just raise an error and it seems the > new functions should have similar contract. > > Changed. > > > > + return 2; > > > + } > > > + > > > + if (mp_typeof(*data) != MP_ARRAY) { > > > + lua_pushnil(L); > > > + lua_pushstring(L, "msgpack.check_array: wrong array header"); > > > + return 2; > > > + } > > > + > > > + if (mp_check_array(data, end) > 0) { > > > + lua_pushnil(L); > > > + lua_pushstring(L, end_of_buffer_msg); > > > + return 2; > > > + } > > > + > > > + uint32_t len = mp_decode_array(&data); > > > + > > > + if (!lua_isnoneornil(L, 3)) { > > > + uint32_t exp_len = (uint32_t) lua_tointeger(L, 3); > > > > IMO it would be better if you set exp_len when you checked the arguments > > (using luaL_checkinteger). > > Expected length was removed from the function as you suggested. > > > > + if (len != exp_len) { > > > + lua_pushnil(L); > > > + lua_pushfstring(L, "msgpack.check_array: expected " > > > + "array of length %d, got length %d", > > > + len, exp_len); > > > + return 2; > > > + } > > > + } > > > + > > > + *(const char **) luaL_pushcdata(L, ctypeid) = data; > > > + lua_pushinteger(L, len); > > > + return 2; > > > +} > > > + > > > static int > > > lua_msgpack_new(lua_State *L); > > > > > > @@ -426,6 +489,7 @@ static const luaL_Reg msgpacklib[] = { > > > { "decode", lua_msgpack_decode }, > > > { "decode_unchecked", lua_msgpack_decode_unchecked }, > > > { "ibuf_decode", lua_ibuf_msgpack_decode }, > > > + { "check_array", lua_check_array }, > > > { "new", lua_msgpack_new }, > > > { NULL, NULL } > > > }; > > > @@ -447,6 +511,8 @@ luaopen_msgpack(lua_State *L) > > > assert(CTID_STRUCT_IBUF != 0); > > > CTID_CHAR_PTR = luaL_ctypeid(L, "char *"); > > > assert(CTID_CHAR_PTR != 0); > > > + CTID_CONST_CHAR_PTR = luaL_ctypeid(L, "const char *"); > > > + assert(CTID_CONST_CHAR_PTR != 0); > > > luaL_msgpack_default = luaL_newserializer(L, "msgpack", msgpacklib); > > > return 1; > > > } > > ---- > > commit 8c820dff279734d79e26591dcb771f7c6ab13639 > Author: Alexander Turenko <alexander.turenko@tarantool.org> > Date: Thu Jan 31 01:45:22 2019 +0300 > > lua: add non-recursive msgpack decoding functions > > Needed for #3276. > > @TarantoolBot document > Title: Non-recursive msgpack decoding functions > > Contracts: > > ``` > msgpack.decode_array(buf.rpos, buf:size()) -> arr_len, new_rpos > msgpack.decode_map(buf.rpos, buf:size()) -> map_len, new_rpos > ``` > > These functions are intended to be used with a msgpack buffer received > from net.box. A user may want to skip {[IPROTO_DATA_KEY] = ...} wrapper > and an array header before pass the buffer to decode in some C function. > > See https://github.com/tarantool/tarantool/issues/2195 for more > information re this net.box's API. > > Consider merger's docbot comment for usage examples. > > diff --git a/src/lua/msgpack.c b/src/lua/msgpack.c > index b47006038..92a9efd25 100644 > --- a/src/lua/msgpack.c > +++ b/src/lua/msgpack.c > @@ -418,6 +418,84 @@ lua_ibuf_msgpack_decode(lua_State *L) > return 2; > } > > +/** > + * Verify and set arguments: data and size. > + * > + * Always return 0. In case of any fail raise a Lua error. > + */ > +static int > +verify_decode_args(lua_State *L, const char *func_name, const char **data_p, > + ptrdiff_t *size_p) > +{ > + /* Verify arguments count. */ > + if (lua_gettop(L) != 2) > + return luaL_error(L, "Usage: %s(ptr, size)", func_name); > + > + /* Verify ptr type. */ > + uint32_t ctypeid; > + const char *data = *(char **) luaL_checkcdata(L, 1, &ctypeid); > + if (ctypeid != CTID_CHAR_PTR) > + return luaL_error(L, "%s: 'char *' expected", func_name); > + > + /* Verify size type and value. */ > + ptrdiff_t size = (ptrdiff_t) luaL_checkinteger(L, 2); > + if (size <= 0) > + return luaL_error(L, "%s: non-positive size", func_name); > + > + *data_p = data; > + *size_p = size; > + > + return 0; > +} > + > +/** > + * msgpack.decode_array(buf.rpos, buf:size()) -> arr_len, new_rpos > + */ > +static int > +lua_decode_array(lua_State *L) > +{ > + const char *func_name = "msgpack.decode_array"; > + const char *data; > + ptrdiff_t size; > + verify_decode_args(L, func_name, &data, &size); > + > + if (mp_typeof(*data) != MP_ARRAY) > + return luaL_error(L, "%s: unexpected msgpack type", func_name); > + > + if (mp_check_array(data, data + size) > 0) > + return luaL_error(L, "%s: unexpected end of buffer", func_name); > + > + uint32_t len = mp_decode_array(&data); > + > + lua_pushinteger(L, len); > + *(const char **) luaL_pushcdata(L, CTID_CHAR_PTR) = data; > + return 2; > +} > + > +/** > + * msgpack.decode_map(buf.rpos, buf:size()) -> map_len, new_rpos > + */ > +static int > +lua_decode_map(lua_State *L) > +{ > + const char *func_name = "msgpack.decode_map"; > + const char *data; > + ptrdiff_t size; > + verify_decode_args(L, func_name, &data, &size); > + > + if (mp_typeof(*data) != MP_MAP) > + return luaL_error(L, "%s: unexpected msgpack type", func_name); > + > + if (mp_check_map(data, data + size) > 0) > + return luaL_error(L, "%s: unexpected end of buffer", func_name); > + > + uint32_t len = mp_decode_map(&data); > + > + lua_pushinteger(L, len); > + *(const char **) luaL_pushcdata(L, CTID_CHAR_PTR) = data; > + return 2; > +} > + > static int > lua_msgpack_new(lua_State *L); > > @@ -426,6 +504,8 @@ static const luaL_Reg msgpacklib[] = { > { "decode", lua_msgpack_decode }, > { "decode_unchecked", lua_msgpack_decode_unchecked }, > { "ibuf_decode", lua_ibuf_msgpack_decode }, > + { "decode_array", lua_decode_array }, > + { "decode_map", lua_decode_map }, > { "new", lua_msgpack_new }, > { NULL, NULL } > }; > diff --git a/test/app-tap/msgpack.test.lua b/test/app-tap/msgpack.test.lua > index 0e1692ad9..ee215dfb1 100755 > --- a/test/app-tap/msgpack.test.lua > +++ b/test/app-tap/msgpack.test.lua > @@ -49,9 +49,186 @@ local function test_misc(test, s) > test:ok(not st and e:match("null"), "null ibuf") > end > > +local function test_decode_array_map(test, s) > + local ffi = require('ffi') > + > + local usage_err = 'Usage: msgpack%.decode_[^_(]+%(ptr, size%)' > + local end_of_buffer_err = 'msgpack%.decode_[^_]+: unexpected end of buffer' > + local non_positive_size_err = 'msgpack.decode_[^_]+: non%-positive size' > + > + local decode_cases = { > + { > + 'fixarray', > + func = s.decode_array, > + data = ffi.cast('char *', '\x94'), > + size = 1, > + exp_len = 4, > + exp_rewind = 1, > + }, > + { > + 'array 16', > + func = s.decode_array, > + data = ffi.cast('char *', '\xdc\x00\x04'), > + size = 3, > + exp_len = 4, > + exp_rewind = 3, > + }, > + { > + 'array 32', > + func = s.decode_array, > + data = ffi.cast('char *', '\xdd\x00\x00\x00\x04'), > + size = 5, > + exp_len = 4, > + exp_rewind = 5, > + }, > + { > + 'truncated array 16', > + func = s.decode_array, > + data = ffi.cast('char *', '\xdc\x00'), > + size = 2, > + exp_err = end_of_buffer_err, > + }, > + { > + 'truncated array 32', > + func = s.decode_array, > + data = ffi.cast('char *', '\xdd\x00\x00\x00'), > + size = 4, > + exp_err = end_of_buffer_err, > + }, > + { > + 'fixmap', > + func = s.decode_map, > + data = ffi.cast('char *', '\x84'), > + size = 1, > + exp_len = 4, > + exp_rewind = 1, > + }, > + { > + 'map 16', > + func = s.decode_map, > + data = ffi.cast('char *', '\xde\x00\x04'), > + size = 3, > + exp_len = 4, > + exp_rewind = 3, > + }, > + { > + 'array 32', > + func = s.decode_map, > + data = ffi.cast('char *', '\xdf\x00\x00\x00\x04'), > + size = 5, > + exp_len = 4, > + exp_rewind = 5, > + }, > + { > + 'truncated map 16', > + func = s.decode_map, > + data = ffi.cast('char *', '\xde\x00'), > + size = 2, > + exp_err = end_of_buffer_err, > + }, > + { > + 'truncated map 32', > + func = s.decode_map, > + data = ffi.cast('char *', '\xdf\x00\x00\x00'), > + size = 4, > + exp_err = end_of_buffer_err, > + }, > + } > + > + local bad_api_cases = { > + { > + 'wrong msgpack type', > + data = ffi.cast('char *', '\xc0'), > + size = 1, > + exp_err = 'msgpack.decode_[^_]+: unexpected msgpack type', > + }, > + { > + 'zero size buffer', > + data = ffi.cast('char *', ''), > + size = 0, > + exp_err = non_positive_size_err, > + }, > + { > + 'negative size buffer', > + data = ffi.cast('char *', ''), > + size = -1, > + exp_err = non_positive_size_err, > + }, > + { > + 'size is nil', > + data = ffi.cast('char *', ''), > + size = nil, > + exp_err = 'bad argument', > + }, > + { > + 'no arguments', > + args = {}, > + exp_err = usage_err, > + }, > + { > + 'one argument', > + args = {ffi.cast('char *', '')}, > + exp_err = usage_err, > + }, > + { > + 'data is nil', > + args = {nil, 1}, > + exp_err = 'expected cdata as 1 argument', > + }, > + { > + 'data is not cdata', > + args = {1, 1}, > + exp_err = 'expected cdata as 1 argument', > + }, > + { > + 'data with wrong cdata type', > + args = {box.tuple.new(), 1}, > + exp_err = "msgpack.decode_[^_]+: 'char %*' expected", > + }, > + { > + 'size has wrong type', > + args = {ffi.cast('char *', ''), 'eee'}, > + exp_err = 'bad argument', > + }, > + } > + > + test:plan(#decode_cases + 2 * #bad_api_cases) > + > + -- Decode cases. > + for _, case in ipairs(decode_cases) do > + if case.exp_err ~= nil then > + local ok, err = pcall(case.func, case.data, case.size) > + local description = ('bad; %s'):format(case[1]) > + test:ok(ok == false and err:match(case.exp_err), description) > + else > + local len, new_buf = case.func(case.data, case.size) > + local rewind = new_buf - case.data > + local description = ('good; %s'):format(case[1]) > + test:is_deeply({len, rewind}, {case.exp_len, case.exp_rewind}, > + description) > + end > + end > + > + -- Bad api usage cases. > + for _, func_name in ipairs({'decode_array', 'decode_map'}) do > + for _, case in ipairs(bad_api_cases) do > + local ok, err > + if case.args ~= nil then > + local args_len = table.maxn(case.args) > + ok, err = pcall(s[func_name], unpack(case.args, 1, args_len)) > + else > + ok, err = pcall(s[func_name], case.data, case.size) > + end > + local description = ('%s bad api usage; %s'):format(func_name, > + case[1]) > + test:ok(ok == false and err:match(case.exp_err), description) > + end > + end > +end > + > tap.test("msgpack", function(test) > local serializer = require('msgpack') > - test:plan(10) > + test:plan(11) > test:test("unsigned", common.test_unsigned, serializer) > test:test("signed", common.test_signed, serializer) > test:test("double", common.test_double, serializer) > @@ -62,4 +239,5 @@ tap.test("msgpack", function(test) > test:test("ucdata", common.test_ucdata, serializer) > test:test("offsets", test_offsets, serializer) > test:test("misc", test_misc, serializer) > + test:test("decode_array_map", test_decode_array_map, serializer) > end) > > ---- > > commit 3868d5c2551c893f16bd05c79d4d52a564c6a833 > Author: Alexander Turenko <alexander.turenko@tarantool.org> > Date: Thu Jan 31 01:59:18 2019 +0300 > > net.box: add skip_header option to use with buffer > > Needed for #3276. > > @TarantoolBot document > Title: net.box: skip_header option > > This option instructs net.box to skip {[IPROTO_DATA_KEY] = ...} wrapper > from a buffer. This may be needed to pass this buffer to some C function > when it expects some specific msgpack input. > > See src/box/lua/net_box.lua for examples. Also consider merger's docbot > comment for more examples. > > diff --git a/src/box/lua/net_box.lua b/src/box/lua/net_box.lua > index 2bf772aa8..53c93cafb 100644 > --- a/src/box/lua/net_box.lua > +++ b/src/box/lua/net_box.lua > @@ -15,6 +15,7 @@ local max = math.max > local fiber_clock = fiber.clock > local fiber_self = fiber.self > local decode = msgpack.decode_unchecked > +local decode_map = msgpack.decode_map > > local table_new = require('table.new') > local check_iterator_type = box.internal.check_iterator_type > @@ -483,8 +484,8 @@ local function create_transport(host, port, user, password, callback, > -- @retval nil, error Error occured. > -- @retval not nil Future object. > -- > - local function perform_async_request(buffer, method, on_push, on_push_ctx, > - ...) > + local function perform_async_request(buffer, skip_header, method, on_push, > + on_push_ctx, ...) > if state ~= 'active' and state ~= 'fetch_schema' then > return nil, box.error.new({code = last_errno or E_NO_CONNECTION, > reason = last_error}) > @@ -497,12 +498,13 @@ local function create_transport(host, port, user, password, callback, > local id = next_request_id > method_encoder[method](send_buf, id, ...) > next_request_id = next_id(id) > - -- Request in most cases has maximum 8 members: > - -- method, buffer, id, cond, errno, response, on_push, > - -- on_push_ctx. > - local request = setmetatable(table_new(0, 8), request_mt) > + -- Request in most cases has maximum 9 members: > + -- method, buffer, skip_header, id, cond, errno, response, > + -- on_push, on_push_ctx. > + local request = setmetatable(table_new(0, 9), request_mt) > request.method = method > request.buffer = buffer > + request.skip_header = skip_header > request.id = id > request.cond = fiber.cond() > requests[id] = request > @@ -516,10 +518,11 @@ local function create_transport(host, port, user, password, callback, > -- @retval nil, error Error occured. > -- @retval not nil Response object. > -- > - local function perform_request(timeout, buffer, method, on_push, > - on_push_ctx, ...) > + local function perform_request(timeout, buffer, skip_header, method, > + on_push, on_push_ctx, ...) > local request, err = > - perform_async_request(buffer, method, on_push, on_push_ctx, ...) > + perform_async_request(buffer, skip_header, method, on_push, > + on_push_ctx, ...) > if not request then > return nil, err > end > @@ -554,6 +557,15 @@ local function create_transport(host, port, user, password, callback, > local wpos = buffer:alloc(body_len) > ffi.copy(wpos, body_rpos, body_len) > body_len = tonumber(body_len) > + if request.skip_header then > + -- Skip {[IPROTO_DATA_KEY] = ...} wrapper. > + local map_len, key > + map_len, buffer.rpos = decode_map(buffer.rpos, buffer:size()) > + assert(map_len == 1) > + key, buffer.rpos = decode(buffer.rpos) > + assert(key == IPROTO_DATA_KEY) > + body_len = buffer:size() > + end > if status == IPROTO_OK_KEY then > request.response = body_len > requests[id] = nil > @@ -1047,17 +1059,18 @@ end > > function remote_methods:_request(method, opts, ...) > local transport = self._transport > - local on_push, on_push_ctx, buffer, deadline > + local on_push, on_push_ctx, buffer, skip_header, deadline > -- Extract options, set defaults, check if the request is > -- async. > if opts then > buffer = opts.buffer > + skip_header = opts.skip_header > if opts.is_async then > if opts.on_push or opts.on_push_ctx then > error('To handle pushes in an async request use future:pairs()') > end > - return transport.perform_async_request(buffer, method, table.insert, > - {}, ...) > + return transport.perform_async_request(buffer, skip_header, method, > + table.insert, {}, ...) > end > if opts.timeout then > -- conn.space:request(, { timeout = timeout }) > @@ -1079,8 +1092,9 @@ function remote_methods:_request(method, opts, ...) > transport.wait_state('active', timeout) > timeout = deadline and max(0, deadline - fiber_clock()) > end > - local res, err = transport.perform_request(timeout, buffer, method, > - on_push, on_push_ctx, ...) > + local res, err = transport.perform_request(timeout, buffer, skip_header, > + method, on_push, on_push_ctx, > + ...) > if err then > box.error(err) > end > @@ -1283,10 +1297,10 @@ function console_methods:eval(line, timeout) > end > if self.protocol == 'Binary' then > local loader = 'return require("console").eval(...)' > - res, err = pr(timeout, nil, 'eval', nil, nil, loader, {line}) > + res, err = pr(timeout, nil, false, 'eval', nil, nil, loader, {line}) > else > assert(self.protocol == 'Lua console') > - res, err = pr(timeout, nil, 'inject', nil, nil, line..'$EOF$\n') > + res, err = pr(timeout, nil, false, 'inject', nil, nil, line..'$EOF$\n') > end > if err then > box.error(err) > diff --git a/test/box/net.box.result b/test/box/net.box.result > index 2b5a84646..71d0e0a50 100644 > --- a/test/box/net.box.result > +++ b/test/box/net.box.result > @@ -29,7 +29,7 @@ function x_select(cn, space_id, index_id, iterator, offset, limit, key, opts) > offset, limit, key) > return ret > end > -function x_fatal(cn) cn._transport.perform_request(nil, nil, 'inject', nil, nil, '\x80') end > +function x_fatal(cn) cn._transport.perform_request(nil, nil, false, 'inject', nil, nil, '\x80') end > test_run:cmd("setopt delimiter ''"); > --- > ... > @@ -1573,6 +1573,18 @@ result > --- > - {48: [[2]]} > ... > +-- replace + skip_header > +c.space.test:replace({2}, {buffer = ibuf, skip_header = true}) > +--- > +- 7 > +... > +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) > +--- > +... > +result > +--- > +- [[2]] > +... > -- insert > c.space.test:insert({3}, {buffer = ibuf}) > --- > @@ -1585,6 +1597,21 @@ result > --- > - {48: [[3]]} > ... > +-- insert + skip_header > +_ = space:delete({3}) > +--- > +... > +c.space.test:insert({3}, {buffer = ibuf, skip_header = true}) > +--- > +- 7 > +... > +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) > +--- > +... > +result > +--- > +- [[3]] > +... > -- update > c.space.test:update({3}, {}, {buffer = ibuf}) > --- > @@ -1608,6 +1635,29 @@ result > --- > - {48: [[3]]} > ... > +-- update + skip_header > +c.space.test:update({3}, {}, {buffer = ibuf, skip_header = true}) > +--- > +- 7 > +... > +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) > +--- > +... > +result > +--- > +- [[3]] > +... > +c.space.test.index.primary:update({3}, {}, {buffer = ibuf, skip_header = true}) > +--- > +- 7 > +... > +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) > +--- > +... > +result > +--- > +- [[3]] > +... > -- upsert > c.space.test:upsert({4}, {}, {buffer = ibuf}) > --- > @@ -1620,6 +1670,18 @@ result > --- > - {48: []} > ... > +-- upsert + skip_header > +c.space.test:upsert({4}, {}, {buffer = ibuf, skip_header = true}) > +--- > +- 5 > +... > +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) > +--- > +... > +result > +--- > +- [] > +... > -- delete > c.space.test:upsert({4}, {}, {buffer = ibuf}) > --- > @@ -1632,6 +1694,18 @@ result > --- > - {48: []} > ... > +-- delete + skip_header > +c.space.test:upsert({4}, {}, {buffer = ibuf, skip_header = true}) > +--- > +- 5 > +... > +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) > +--- > +... > +result > +--- > +- [] > +... > -- select > c.space.test.index.primary:select({3}, {iterator = 'LE', buffer = ibuf}) > --- > @@ -1644,6 +1718,18 @@ result > --- > - {48: [[3], [2], [1, 'hello']]} > ... > +-- select + skip_header > +c.space.test.index.primary:select({3}, {iterator = 'LE', buffer = ibuf, skip_header = true}) > +--- > +- 17 > +... > +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) > +--- > +... > +result > +--- > +- [[3], [2], [1, 'hello']] > +... > -- select > len = c.space.test:select({}, {buffer = ibuf}) > --- > @@ -1667,6 +1753,29 @@ result > --- > - {48: [[1, 'hello'], [2], [3], [4]]} > ... > +-- select + skip_header > +len = c.space.test:select({}, {buffer = ibuf, skip_header = true}) > +--- > +... > +ibuf.rpos + len == ibuf.wpos > +--- > +- true > +... > +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) > +--- > +... > +ibuf.rpos == ibuf.wpos > +--- > +- true > +... > +len > +--- > +- 19 > +... > +result > +--- > +- [[1, 'hello'], [2], [3], [4]] > +... > -- call > c:call("echo", {1, 2, 3}, {buffer = ibuf}) > --- > @@ -1701,6 +1810,40 @@ result > --- > - {48: []} > ... > +-- call + skip_header > +c:call("echo", {1, 2, 3}, {buffer = ibuf, skip_header = true}) > +--- > +- 8 > +... > +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) > +--- > +... > +result > +--- > +- [1, 2, 3] > +... > +c:call("echo", {}, {buffer = ibuf, skip_header = true}) > +--- > +- 5 > +... > +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) > +--- > +... > +result > +--- > +- [] > +... > +c:call("echo", nil, {buffer = ibuf, skip_header = true}) > +--- > +- 5 > +... > +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) > +--- > +... > +result > +--- > +- [] > +... > -- eval > c:eval("echo(...)", {1, 2, 3}, {buffer = ibuf}) > --- > @@ -1735,6 +1878,40 @@ result > --- > - {48: []} > ... > +-- eval + skip_header > +c:eval("echo(...)", {1, 2, 3}, {buffer = ibuf, skip_header = true}) > +--- > +- 5 > +... > +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) > +--- > +... > +result > +--- > +- [] > +... > +c:eval("echo(...)", {}, {buffer = ibuf, skip_header = true}) > +--- > +- 5 > +... > +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) > +--- > +... > +result > +--- > +- [] > +... > +c:eval("echo(...)", nil, {buffer = ibuf, skip_header = true}) > +--- > +- 5 > +... > +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) > +--- > +... > +result > +--- > +- [] > +... > -- unsupported methods > c.space.test:get({1}, { buffer = ibuf}) > --- > @@ -2571,7 +2748,7 @@ c.space.test:delete{1} > -- > -- Break a connection to test reconnect_after. > -- > -_ = c._transport.perform_request(nil, nil, 'inject', nil, nil, '\x80') > +_ = c._transport.perform_request(nil, nil, false, 'inject', nil, nil, '\x80') > --- > ... > c.state > @@ -3205,7 +3382,7 @@ c = net:connect(box.cfg.listen, {reconnect_after = 0.01}) > future = c:call('long_function', {1, 2, 3}, {is_async = true}) > --- > ... > -_ = c._transport.perform_request(nil, nil, 'inject', nil, nil, '\x80') > +_ = c._transport.perform_request(nil, nil, false, 'inject', nil, nil, '\x80') > --- > ... > while not c:is_connected() do fiber.sleep(0.01) end > @@ -3340,7 +3517,7 @@ c:ping() > -- new attempts to read any data - the connection is closed > -- already. > -- > -f = fiber.create(c._transport.perform_request, nil, nil, 'call_17', nil, nil, 'long', {}) c._transport.perform_request(nil, nil, 'inject', nil, nil, '\x80') > +f = fiber.create(c._transport.perform_request, nil, nil, false, 'call_17', nil, nil, 'long', {}) c._transport.perform_request(nil, nil, false, 'inject', nil, nil, '\x80') > --- > ... > while f:status() ~= 'dead' do fiber.sleep(0.01) end > @@ -3359,7 +3536,7 @@ c = net:connect(box.cfg.listen) > data = msgpack.encode(18400000000000000000)..'aaaaaaa' > --- > ... > -c._transport.perform_request(nil, nil, 'inject', nil, nil, data) > +c._transport.perform_request(nil, nil, false, 'inject', nil, nil, data) > --- > - null > - Peer closed > diff --git a/test/box/net.box.test.lua b/test/box/net.box.test.lua > index 96d822820..48cc7147d 100644 > --- a/test/box/net.box.test.lua > +++ b/test/box/net.box.test.lua > @@ -12,7 +12,7 @@ function x_select(cn, space_id, index_id, iterator, offset, limit, key, opts) > offset, limit, key) > return ret > end > -function x_fatal(cn) cn._transport.perform_request(nil, nil, 'inject', nil, nil, '\x80') end > +function x_fatal(cn) cn._transport.perform_request(nil, nil, false, 'inject', nil, nil, '\x80') end > test_run:cmd("setopt delimiter ''"); > > LISTEN = require('uri').parse(box.cfg.listen) > @@ -615,11 +615,22 @@ c.space.test:replace({2}, {buffer = ibuf}) > result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) > result > > +-- replace + skip_header > +c.space.test:replace({2}, {buffer = ibuf, skip_header = true}) > +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) > +result > + > -- insert > c.space.test:insert({3}, {buffer = ibuf}) > result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) > result > > +-- insert + skip_header > +_ = space:delete({3}) > +c.space.test:insert({3}, {buffer = ibuf, skip_header = true}) > +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) > +result > + > -- update > c.space.test:update({3}, {}, {buffer = ibuf}) > result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) > @@ -628,21 +639,44 @@ c.space.test.index.primary:update({3}, {}, {buffer = ibuf}) > result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) > result > > +-- update + skip_header > +c.space.test:update({3}, {}, {buffer = ibuf, skip_header = true}) > +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) > +result > +c.space.test.index.primary:update({3}, {}, {buffer = ibuf, skip_header = true}) > +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) > +result > + > -- upsert > c.space.test:upsert({4}, {}, {buffer = ibuf}) > result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) > result > > +-- upsert + skip_header > +c.space.test:upsert({4}, {}, {buffer = ibuf, skip_header = true}) > +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) > +result > + > -- delete > c.space.test:upsert({4}, {}, {buffer = ibuf}) > result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) > result > > +-- delete + skip_header > +c.space.test:upsert({4}, {}, {buffer = ibuf, skip_header = true}) > +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) > +result > + > -- select > c.space.test.index.primary:select({3}, {iterator = 'LE', buffer = ibuf}) > result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) > result > > +-- select + skip_header > +c.space.test.index.primary:select({3}, {iterator = 'LE', buffer = ibuf, skip_header = true}) > +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) > +result > + > -- select > len = c.space.test:select({}, {buffer = ibuf}) > ibuf.rpos + len == ibuf.wpos > @@ -651,6 +685,14 @@ ibuf.rpos == ibuf.wpos > len > result > > +-- select + skip_header > +len = c.space.test:select({}, {buffer = ibuf, skip_header = true}) > +ibuf.rpos + len == ibuf.wpos > +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) > +ibuf.rpos == ibuf.wpos > +len > +result > + > -- call > c:call("echo", {1, 2, 3}, {buffer = ibuf}) > result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) > @@ -662,6 +704,17 @@ c:call("echo", nil, {buffer = ibuf}) > result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) > result > > +-- call + skip_header > +c:call("echo", {1, 2, 3}, {buffer = ibuf, skip_header = true}) > +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) > +result > +c:call("echo", {}, {buffer = ibuf, skip_header = true}) > +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) > +result > +c:call("echo", nil, {buffer = ibuf, skip_header = true}) > +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) > +result > + > -- eval > c:eval("echo(...)", {1, 2, 3}, {buffer = ibuf}) > result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) > @@ -673,6 +726,17 @@ c:eval("echo(...)", nil, {buffer = ibuf}) > result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) > result > > +-- eval + skip_header > +c:eval("echo(...)", {1, 2, 3}, {buffer = ibuf, skip_header = true}) > +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) > +result > +c:eval("echo(...)", {}, {buffer = ibuf, skip_header = true}) > +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) > +result > +c:eval("echo(...)", nil, {buffer = ibuf, skip_header = true}) > +result, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) > +result > + > -- unsupported methods > c.space.test:get({1}, { buffer = ibuf}) > c.space.test.index.primary:min({}, { buffer = ibuf}) > @@ -1063,7 +1127,7 @@ c.space.test:delete{1} > -- > -- Break a connection to test reconnect_after. > -- > -_ = c._transport.perform_request(nil, nil, 'inject', nil, nil, '\x80') > +_ = c._transport.perform_request(nil, nil, false, 'inject', nil, nil, '\x80') > c.state > while not c:is_connected() do fiber.sleep(0.01) end > c:ping() > @@ -1291,7 +1355,7 @@ finalize_long() > -- > c = net:connect(box.cfg.listen, {reconnect_after = 0.01}) > future = c:call('long_function', {1, 2, 3}, {is_async = true}) > -_ = c._transport.perform_request(nil, nil, 'inject', nil, nil, '\x80') > +_ = c._transport.perform_request(nil, nil, false, 'inject', nil, nil, '\x80') > while not c:is_connected() do fiber.sleep(0.01) end > finalize_long() > future:wait_result(100) > @@ -1348,7 +1412,7 @@ c:ping() > -- new attempts to read any data - the connection is closed > -- already. > -- > -f = fiber.create(c._transport.perform_request, nil, nil, 'call_17', nil, nil, 'long', {}) c._transport.perform_request(nil, nil, 'inject', nil, nil, '\x80') > +f = fiber.create(c._transport.perform_request, nil, nil, false, 'call_17', nil, nil, 'long', {}) c._transport.perform_request(nil, nil, false, 'inject', nil, nil, '\x80') > while f:status() ~= 'dead' do fiber.sleep(0.01) end > c:close() > > @@ -1358,7 +1422,7 @@ c:close() > -- > c = net:connect(box.cfg.listen) > data = msgpack.encode(18400000000000000000)..'aaaaaaa' > -c._transport.perform_request(nil, nil, 'inject', nil, nil, data) > +c._transport.perform_request(nil, nil, false, 'inject', nil, nil, data) > c:close() > test_run:grep_log('default', 'too big packet size in the header') ~= nil ^ permalink raw reply [flat|nested] 28+ messages in thread
* [PATCH v2 6/6] Add merger for tuple streams 2019-01-09 20:20 [PATCH v2 0/6] Merger Alexander Turenko ` (4 preceding siblings ...) 2019-01-09 20:20 ` [PATCH v2 5/6] net.box: add helpers to decode msgpack headers Alexander Turenko @ 2019-01-09 20:20 ` Alexander Turenko 5 siblings, 0 replies; 28+ messages in thread From: Alexander Turenko @ 2019-01-09 20:20 UTC (permalink / raw) To: Vladimir Davydov; +Cc: Alexander Turenko, tarantool-patches Fixes #3276. @TarantoolBot document Title: Merger for tuple streams ## API and basic usage The following example demonstrates API of the module: ```lua local msgpack = require('msgpack') local net_box = require('net.box') local buffer = require('buffer') local merger = require('merger') -- The format of key_parts parameter is the same as -- `{box,conn}.space.<...>.index.<...>.parts` (where conn is -- net.box connection). local key_parts = { { fieldno = <number>, type = <string>, [ is_nullable = <boolean>, ] [ collation_id = <number>, ] [ collation = <string>, ] }, ... } -- Create the merger context. local ctx = merger.context.new(key_parts) -- Optional parameters. local opts = { -- Output buffer, only for merger.select(ctx, <...>). [ buffer = <buffer>, ] -- Ascending (default) or descending result order. [ descending = <boolean>, ] -- Optional callback to fetch more data. [ fetch_source = <function>, ] } -- Prepare buffer source. local conn = net_box.connect('localhost:3301') local buf = buffer.ibuf() conn.space.s:select(nil, {buffer = buf}) -- read to buffer buf.rpos = assert(net_box.check_iproto_data(buf.rpos, buf.wpos - buf.rpos)) -- We have three sources here. local sources = { buf, -- buffer source box.space.s:select(), -- table source {box.space.s:pairs()}, -- iterator source } -- Read the whole result at once. local res = merger.select(ctx, sources, opts) -- Read the result tuple per tuple. local res = {} for _, tuple in merger.pairs(ctx, sources, opts) do -- Some stop merge condition. if tuple[1] > MAX_VALUE then break end table.insert(res, tuple) end -- The same in the functional style. local function cond(tuple) return tuple[1] <= MAX_VALUE end local res = merger.pairs(ctx, sources, opts):take(cond):totable() ``` The basic case of using merger is when there are M storages and data are partitioned (sharded) across them. A client want to fetch the data (tuples stream) from each storage and merge them into one tuple stream: ```lua local msgpack = require('msgpack') local net_box = require('net.box') local buffer = require('buffer') local merger = require('merger') -- Prepare M sources. local net_box_opts = {reconnect_after = 0.1} local connects = { net_box.connect('localhost:3301', net_box_opts), net_box.connect('localhost:3302', net_box_opts), ... net_box.connect('localhost:<...>', net_box_opts), } local sources = {} for _, conn in ipairs(connects) do local buf = buffer.ibuf() conn.space.<...>.index.<...>:select(<...>, {buffer = buf}) buf.rpos = assert(net_box.check_iproto_data(buf.rpos, buf.wpos - buf.rpos)) table.insert(sources, buf) end -- See the 'Notes...' section below. local key_parts = {} local space = connects[1].space.<...> local index = space.index.<...> for _, part in ipairs(index.parts) do table.insert(key_parts, part) end if not index.unique then for _, part in ipairs(space.index[0]) do table.insert(key_parts, part) end end -- Create the merger context. local ctx = merger.context.new(key_parts) -- Merge. local res = merger.select(ctx, sources) ``` ## Notes re source sorting and key parts The merger expects that each source tuples stream is sorted according to provided key parts and perform a kind of merge sort (choose minimal / maximal tuple across sources on each step). Tuples from select() from Tarantool's space are sorted according to key parts from index that was used. When secondary non-unique index is used tuples are sorted according to the key parts of the secondary index and, then, key parts of the primary index. ## Preparing buffers We'll use the symbol T below to represent an msgpack array that corresponds to a tuple. A select response has the following structure: `{[48] = {T, T, ...}}`, while a call response is `{[48] = {{T, T, ...}}}` (because it should support multiple return values). A user should skip extra headers and pass a buffer with the read position on `{T, T, ...}` to merger. Typical headers are the following: Cases | Buffer structure ---------------- | ---------------- raw data | {T, T, ...} net.box select | {[48] = {T, T, ...}} net.box call | {[48] = {{T, T, ...}}} The example how to skip iproto_data (`{[48] = ...}`) and array headers and obtain raw data in a buffer is [here](XXX). XXX: link the example from 'net.box: helpers to decode msgpack headers' docbot request. How to check buffer data structure myself: ```lua #!usr/bin/env tarantool local net_box = require('net.box') local buffer = require('buffer') local ffi = require('ffi') local msgpack = require('msgpack') local yaml = require('yaml') box.cfg{listen = 3301} box.once('load_data', function() box.schema.user.grant('guest', 'read,write,execute', 'universe') box.schema.space.create('s') box.space.s:create_index('pk') box.space.s:insert({1}) box.space.s:insert({2}) box.space.s:insert({3}) box.space.s:insert({4}) end) local function foo() return box.space.s:select() end _G.foo = foo local conn = net_box.connect('localhost:3301') local buf = buffer.ibuf() conn.space.s:select(nil, {buffer = buf}) local buf_str = ffi.string(buf.rpos, buf.wpos - buf.rpos) local buf_lua = msgpack.decode(buf_str) print('select:\n' .. yaml.encode(buf_lua)) local buf = buffer.ibuf() conn:call('foo', nil, {buffer = buf}) local buf_str = ffi.string(buf.rpos, buf.wpos - buf.rpos) local buf_lua = msgpack.decode(buf_str) print('call:\n' .. yaml.encode(buf_lua)) os.exit() ``` A user can use its own shape of the result under net.box call wrapper, so additional actions could be necessary before pass a buffer to merger as the source. See the 'merge multiplexed requests' section for the example. ## Chunked data transfer The merger can ask for further data for a drained source using `fetch_source` callback with the following signature: ```lua fetch_source = function(source, last_tuple, processed) <...> end ``` If this callback is provided the merger will invoke it when a buffer or a table source reaches the end (but it doesn't called for an iterator source). If the new data become available after the call, the merger will use the new data or will consider the source entirely drained otherwise. `fetch_source` should update provided buffer in case of a buffer source or return a new table in case of a table source. An empty buffer, a buffer with zero tuples count, an empty/nil table are considered as stoppers: the callback will not called anymore. `source` is the table with the following fields: - `source.idx` is one-based index of the source; - `source.type` is a string: 'buffer' or 'table'; - `source.buffer` is a cdata<struct ibuf> or nil; - `source.table` is a previous table or nil. `last_tuple` is a last tuple was fetched from the source (can be nil), `processed` is a count of tuples were extracted from this source (over all previous iterations). If no data are available in a source when the merge starts it will call the callback. `last_tuple` will be `nil` in the case, `processed` will be 0. This allows to just define the `fetch_source` callback and don't fill buffers / tables before start. When using `is_async = true` net.box option one can lean on the fact that net.box writes an answer w/o yield: partial result cannot be observed. The following example fetches a data from two storages in chunks, the requests are performed from the `fetch_source` callback. The first request uses ALL iterator and BLOCK_SIZE limit, the following ones use GT iterator (with a key extracted from the last fetched tuple) and the same limit. Note: such way to implement a cursor / a pagination will work smoothly only with unique indexes. See also #3898. More complex scenarious are possible: using futures (`is_async = true` parameters of net.box methods) to fetch a next chunk while merge a current one or, say, call a function with several return values (some of them need to be skipped manually in the callback to let merger read tuples). ```lua -- Storage script -- -------------- box.cfg({<...>}) box.schema.space.create('s') box.space.s:create_index('pk') if instance_name == 'storage_1' then box.space.s:insert({1, 'one'}) box.space.s:insert({3, 'three'}) box.space.s:insert({5, 'five'}) box.space.s:insert({7, 'seven'}) box.space.s:insert({9, 'nine'}) else box.space.s:insert({2, 'two'}) box.space.s:insert({4, 'four'}) box.space.s:insert({6, 'six'}) box.space.s:insert({8, 'eight'}) box.space.s:insert({10, 'ten'}) end box.schema.user.grant('guest', 'read', 'space', 's') box.cfg({listen = <...>}) -- Client script -- ------------- <...requires...> local BLOCK_SIZE = 2 local function key_from_tuple(tuple, key_parts) local key = {} for _, part in ipairs(key_parts) do table.insert(key, tuple[part.fieldno] or box.NULL) end return key end local function gen_fetch_source(conns, key_parts) return function(source, last_tuple, _) local conn = conns[source.idx] local buf = source.buffer local opts = { limit = BLOCK_SIZE, buffer = buf, } -- the first request: ALL iterator + limit if last_tuple == nil then conn.space.s:select(nil, opts) buf.rpos = assert(net_box.check_iproto_data(buf.rpos, buf.wpos - buf.rpos)) return end -- subsequent requests: GT iterator + limit local key = key_from_tuple(last_tuple, key_parts) opts.iterator = box.index.GT conn.space.s:select(key, opts) buf.rpos = assert(net_box.check_iproto_data(buf.rpos, buf.wpos - buf.rpos)) end end local conns = <...> local buffers = <...> local key_parts = conns[1].space.s.index.pk.parts local ctx = merger.context.new(key_parts) local fetch_source = gen_fetch_source(conns, key_parts) local res = merger.select(ctx, buffers, {fetch_source = fetch_source}) print(yaml.encode(res)) os.exit() ``` ## Merge multiplexed requests Consider the case when a network latency between storage machines and frontend machine(s) is much larger then time to process a request on the frontend. This situation is typical when a workload consists of many amount of light requests. So it could be worth to 'multiplex' different requests to storage machines within one network request. We'll consider approach when a storage function returns many box.space.<...>:select(<...>) results instead of one. One need to skip iproto_data packet header, two array headers and then run merger N times on the same buffers (with the same or different contexts). No extra data copies, no tuples decoding into the Lua memory. ```lua -- Storage script -- -------------- -- Return N results in a table. -- Each result is table of tuples. local function batch_select(<...>) local res = {} for i = 1, N do local tuples = box.space.<...>:select(<...>) table.insert(res, tuples) end return res end -- Expose to call it using net.box. _G.batch_select = batch_select -- Client script -- ------------- local net_box = require('net.box') local buffer = require('buffer') local merger = require('merger') -- Prepare M sources. local connects = <...> local sources = {} for _, conn in ipairs(connects) do local buf = buffer.ibuf() conn:call('batch_select', <...>, {buffer = buf}) buf.rpos = assert(net_box.check_iproto_data(buf.rpos, buf.wpos - buf.rpos)) buf.rpos = assert(msgpack.check_array(buf.rpos, buf.wpos - buf.rpos, 1)) buf.rpos = assert(msgpack.check_array(buf.rpos, buf.wpos - buf.rpos)) table.insert(sources, buf) end -- Now we have M sources and each have N results. We want to -- merge all 1st results, all 2nd results, ..., all Nth -- results. local ctx = merger.context.new(<...>) local res = {} for i = 1, N do -- We use the same merger instance for each merge, but it -- is possible to use different ones. local tuples = merger.select(ctx, sources) table.insert(res, tuples) end ``` The result of these N merges can be written into a buffer (using the `buffer` option) and this buffer can be used as the source in N merges of the next level (see cascading mergers below for the idea). ## Cascading mergers The idea is simple: the merger output formats are the same as source formats, so it is possible to merge results of previous merges. The example below is synthetic to be simple. Real cases when cascading can be profitable likely involve additional layers of Tarantool instances between storages and clients or separate threads to merge blocks of each level. To be honest no one use this ability for now. It exists, because the same input and output formats looks as good property of the API. ```lua <...requires...> local sources = <...100 buffers...> local ctx = merger.context.new(<...>) -- We use buffer sources at 1st and 2nd merge layers, but read -- the final result as the table. local sources_level_2 = {} for i = 1, 10 do -- Take next 10 first level sources. local sources_level_1 = {} for j = 1, 10 do sources_level_1[j] = sources[(i - 1) * 10 + j] end -- Merge 10 sources into a second level source. local result_level_1 = buffer.ibuf() merger.select(ctx, sources_level_1, {buffer = result_level_1}) sources_level_2[i] = result_level_1 end local res = merger.select(ctx, sources_level_2) ``` --- src/box/CMakeLists.txt | 1 + src/box/lua/init.c | 3 + src/box/lua/merger.c | 1402 ++++++++++++++++++++++++++++++++++ src/box/lua/merger.h | 47 ++ test/box-tap/merger.test.lua | 558 ++++++++++++++ test/box-tap/suite.ini | 1 + 6 files changed, 2012 insertions(+) create mode 100644 src/box/lua/merger.c create mode 100644 src/box/lua/merger.h create mode 100755 test/box-tap/merger.test.lua diff --git a/src/box/CMakeLists.txt b/src/box/CMakeLists.txt index 0db093768..b6a60a618 100644 --- a/src/box/CMakeLists.txt +++ b/src/box/CMakeLists.txt @@ -140,6 +140,7 @@ add_library(box STATIC lua/xlog.c lua/sql.c lua/key_def.c + lua/merger.c ${bin_sources}) target_link_libraries(box box_error tuple stat xrow xlog vclock crc32 scramble diff --git a/src/box/lua/init.c b/src/box/lua/init.c index 0e90f6be5..c08bcc288 100644 --- a/src/box/lua/init.c +++ b/src/box/lua/init.c @@ -59,6 +59,7 @@ #include "box/lua/console.h" #include "box/lua/tuple.h" #include "box/lua/sql.h" +#include "box/lua/merger.h" extern char session_lua[], tuple_lua[], @@ -312,6 +313,8 @@ box_lua_init(struct lua_State *L) lua_pop(L, 1); tarantool_lua_console_init(L); lua_pop(L, 1); + luaopen_merger(L); + lua_pop(L, 1); /* Load Lua extension */ for (const char **s = lua_sources; *s; s += 2) { diff --git a/src/box/lua/merger.c b/src/box/lua/merger.c new file mode 100644 index 000000000..cac6918d4 --- /dev/null +++ b/src/box/lua/merger.c @@ -0,0 +1,1402 @@ +/* + * Copyright 2010-2018, Tarantool AUTHORS, please see AUTHORS file. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * 1. Redistributions of source code must retain the above + * copyright notice, this list of conditions and the + * following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY <COPYRIGHT HOLDER> ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL + * <COPYRIGHT HOLDER> OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF + * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include "box/lua/merger.h" + +#include <string.h> +#include <stdio.h> +#include <stdlib.h> +#include <stdint.h> + +#include <lua.h> +#include <lauxlib.h> + +#include "lua/error.h" +#include "lua/utils.h" +#include "small/ibuf.h" +#include "msgpuck.h" +#include "mpstream.h" +#include "lua/msgpack.h" + +#define HEAP_FORWARD_DECLARATION +#include "salad/heap.h" + +#include "box/field_def.h" +#include "box/key_def.h" +#include "box/lua/key_def.h" +#include "box/schema_def.h" +#include "box/tuple.h" +#include "box/lua/tuple.h" +#include "box/box.h" +#include "box/index.h" +#include "diag.h" + +static bool +source_less(const heap_t *heap, const struct heap_node *a, + const struct heap_node *b); +#define HEAP_NAME merger_heap +#define HEAP_LESS source_less +#include "salad/heap.h" + +static uint32_t merger_context_type_id = 0; +static uint32_t merger_state_type_id = 0; +static uint32_t ibuf_type_id = 0; + +/* {{{ Merger structures */ + +struct merger_source; +struct merger_context; +struct merger_state; + +struct merger_source_vtab { + /** + * Free the merger source. + * + * We need to know Lua state here, because sources of + * table and iterator types are saved as references within + * the Lua state. + */ + void (*delete)(struct merger_source *base, struct lua_State *L); + /** + * Update source->tuple of specific source. + * + * Increases the reference counter of the tuple. + * + * Return 0 when successfully fetched a tuple or NULL. In + * case of an error push an error message to the Lua stack + * and return 1. + */ + int (*next)(struct merger_source *base, box_tuple_format_t *format, + const struct merger_state *state, struct lua_State *L); +}; + +/** + * Base (abstract) structure to represent a merge source state. + * Concrete implementations are in box/lua/merger.c. + */ +struct merger_source { + /* Source-specific methods. */ + struct merger_source_vtab *vtab; + /* Ordinal number of the source. */ + int idx; + /* How huch tuples were used from this source. */ + uint32_t processed; + /* Next tuple. */ + struct tuple *tuple; + /* + * A source is the heap node. Compared by the next tuple. + */ + struct heap_node hnode; +}; + +/** + * Holds immutable parameters of a merger. + */ +struct merger_context { + struct key_def *key_def; + box_tuple_format_t *format; +}; + +/** + * Holds parameters of merge process, sources, result storage + * (if any), heap of sources and utility flags / counters. + */ +struct merger_state { + /* Heap of sources. */ + heap_t heap; + /* + * Copy of key_def from merger_context. + * + * A merger_context can be collected by LuaJIT GC + * independently from a merger_state, so we need either + * copy key_def or implement reference counting for + * merger_context and save the pointer. + * + * key_def is needed in source_less(), where merger_state + * is known, but merger_context is not. + */ + struct key_def *key_def; + /* Parsed sources. */ + uint32_t sources_count; + struct merger_source **sources; + /* Ascending / descending order. */ + int order; + /* Optional output buffer. */ + struct ibuf *obuf; + /* Optional fetch_source() callback. */ + int fetch_source_ref; +}; + +/* }}} */ + +/* {{{ Helpers for source methods and merger functions */ + +/** + * How much more memory the heap will reserve at the next grow. + * + * See HEAP(reserve)() function in lib/salad/heap.h. + */ +size_t heap_next_grow_size(const heap_t *heap) +{ + heap_off_t heap_capacity_diff = heap->capacity == 0 ? + HEAP_INITIAL_CAPACITY : heap->capacity; + return heap_capacity_diff * sizeof(struct heap_node *); +} + +/** + * Extract an ibuf object from the Lua stack. + */ +static struct ibuf * +check_ibuf(struct lua_State *L, int idx) +{ + if (lua_type(L, idx) != LUA_TCDATA) + return NULL; + + uint32_t cdata_type; + struct ibuf *ibuf_ptr = luaL_checkcdata(L, idx, &cdata_type); + if (ibuf_ptr == NULL || cdata_type != ibuf_type_id) + return NULL; + return ibuf_ptr; +} + +/** + * Extract a merger context from the Lua stack. + */ +static struct merger_context * +check_merger_context(struct lua_State *L, int idx) +{ + uint32_t cdata_type; + struct merger_context **ctx_ptr = luaL_checkcdata(L, idx, &cdata_type); + if (ctx_ptr == NULL || cdata_type != merger_context_type_id) + return NULL; + return *ctx_ptr; +} + +/** + * Extract a merger state from the Lua stack. + */ +static struct merger_state * +check_merger_state(struct lua_State *L, int idx) +{ + uint32_t cdata_type; + struct merger_state **state_ptr = luaL_checkcdata(L, idx, &cdata_type); + if (state_ptr == NULL || cdata_type != merger_state_type_id) + return NULL; + return *state_ptr; +} + +/** + * Skip the array around tuples and save its length. + */ +static int +decode_header(struct ibuf *buf, size_t *len_p) +{ + /* Check the buffer is correct. */ + if (buf->rpos > buf->wpos) + return 0; + + /* Skip decoding if the buffer is empty. */ + if (ibuf_used(buf) == 0) { + *len_p = 0; + return 1; + } + + /* Check and skip the array around tuples. */ + int ok = mp_typeof(*buf->rpos) == MP_ARRAY; + if (ok) + ok = mp_check_array(buf->rpos, buf->wpos) <= 0; + if (ok) + *len_p = mp_decode_array((const char **) &buf->rpos); + return ok; +} + +/** + * Encode the array around tuples. + */ +static void +encode_header(struct ibuf *obuf, uint32_t result_len) +{ + ibuf_reserve(obuf, mp_sizeof_array(result_len)); + obuf->wpos = mp_encode_array(obuf->wpos, result_len); +} + +/* }}} */ + +/* {{{ Buffer merger source */ + +struct merger_source_buffer { + struct merger_source base; + /* + * The reference is needed to push the + * buffer to Lua as a part of the source + * table to the fetch_source callback. + * + * See luaL_merger_source_buffer_push(). + */ + int ref; + struct ibuf *buf; + /* + * A merger stops before end of a buffer + * when it is not the last merger in the + * chain. + */ + size_t remaining_tuples_cnt; +}; + +/* Virtual methods declarations */ + +static void +luaL_merger_source_buffer_delete(struct merger_source *base, + struct lua_State *L); +static int +luaL_merger_source_buffer_next(struct merger_source *base, + box_tuple_format_t *format, + const struct merger_state *state, + struct lua_State *L); + +/* Non-virtual methods */ + +/** + * Create the new merger source of buffer type using content of a + * Lua stack. + * + * In case of an error it returns NULL and pushes the error to the + * Lua stack. + */ +static struct merger_source * +luaL_merger_source_buffer_new(struct lua_State *L, int idx, int ordinal, + struct merger_state *state) +{ + static struct merger_source_vtab merger_source_buffer_vtab = { + .delete = luaL_merger_source_buffer_delete, + .next = luaL_merger_source_buffer_next, + }; + + struct merger_source_buffer *source = (struct merger_source_buffer *) + malloc(sizeof(struct merger_source_buffer)); + + if (source == NULL) { + diag_set(OutOfMemory, sizeof(struct merger_source_buffer), + "malloc", "merger_source_buffer"); + luaT_pusherror(L, diag_last_error(diag_get())); + return NULL; + } + + source->base.idx = ordinal; + source->base.processed = 0; + source->base.tuple = NULL; + /* source->base.hnode does not need to be initialized. */ + + lua_pushvalue(L, idx); /* Popped by luaL_ref(). */ + source->ref = luaL_ref(L, LUA_REGISTRYINDEX); + source->buf = check_ibuf(L, idx); + assert(source->buf != NULL); + source->remaining_tuples_cnt = 0; + + /* + * We decode a buffer header once at start when no fetch + * callback is provided. In case when there is the + * callback we should call it first: it is performed in + * the source->base.vtab->next() function. + * + * The reason is that a user can want to skip some data + * (say, a request metainformation) before proceed with + * merge. + */ + if (state->fetch_source_ref <= 0) { + if (!decode_header(source->buf, + &source->remaining_tuples_cnt)) { + luaL_unref(L, LUA_REGISTRYINDEX, source->ref); + free(source); + lua_pushfstring(L, "Invalid merge source %d", + ordinal + 1); + return NULL; + } + } + + source->base.vtab = &merger_source_buffer_vtab; + return &source->base; +} + +/** + * Push certain fields of a source to Lua. + */ +static int +luaL_merger_source_buffer_push(const struct merger_source_buffer *source, + struct lua_State *L) +{ + lua_createtable(L, 0, 3); + + lua_pushinteger(L, source->base.idx + 1); + lua_setfield(L, -2, "idx"); + + lua_pushstring(L, "buffer"); + lua_setfield(L, -2, "type"); + + lua_rawgeti(L, LUA_REGISTRYINDEX, source->ref); + lua_setfield(L, -2, "buffer"); + + return 1; +} + +/** + * Call a user provided function to fill the source and, maybe, + * to skip data before tuples array. + * + * Return 0 at success and 1 at error (push the error object). + */ +static int +luaL_merger_source_buffer_fetch(struct merger_source_buffer *source, + const struct merger_state *state, + struct tuple *last_tuple, struct lua_State *L) +{ + /* No fetch callback: do nothing. */ + if (state->fetch_source_ref <= 0) + return 0; + /* Push fetch callback. */ + lua_rawgeti(L, LUA_REGISTRYINDEX, state->fetch_source_ref); + /* Push source, last_tuple, processed. */ + luaL_merger_source_buffer_push(source, L); + if (last_tuple == NULL) + lua_pushnil(L); + else + luaT_pushtuple(L, last_tuple); + lua_pushinteger(L, source->base.processed); + /* Invoke the callback and process data. */ + if (lua_pcall(L, 3, 0, 0)) + return 1; + /* Update remaining_tuples_cnt and skip the header. */ + if (!decode_header(source->buf, &source->remaining_tuples_cnt)) { + lua_pushfstring(L, "Invalid merge source %d", + source->base.idx + 1); + return 1; + } + return 0; +} + +/* Virtual methods */ + +static void +luaL_merger_source_buffer_delete(struct merger_source *base, + struct lua_State *L) +{ + struct merger_source_buffer *source = container_of(base, + struct merger_source_buffer, base); + + luaL_unref(L, LUA_REGISTRYINDEX, source->ref); + + if (base->tuple != NULL) + box_tuple_unref(base->tuple); + + free(source); +} + +static int +luaL_merger_source_buffer_next(struct merger_source *base, + box_tuple_format_t *format, + const struct merger_state *state, + struct lua_State *L) +{ + struct merger_source_buffer *source = container_of(base, + struct merger_source_buffer, base); + + struct tuple *last_tuple = base->tuple; + base->tuple = NULL; + + /* + * Handle the case when all data were processed: + * ask more and stop if no data arrived. + */ + if (source->remaining_tuples_cnt == 0) { + int rc = luaL_merger_source_buffer_fetch(source, state, + last_tuple, L); + if (rc != 0) + return 1; + if (source->remaining_tuples_cnt == 0) + return 0; + } + if (ibuf_used(source->buf) == 0) { + lua_pushstring(L, "Unexpected msgpack buffer end"); + return 1; + } + const char *tuple_beg = source->buf->rpos; + const char *tuple_end = tuple_beg; + /* + * mp_next() is faster then mp_check(), but can + * read bytes outside of the buffer and so can + * cause segmentation faults or incorrect result. + * + * We check buffer boundaries after the mp_next() + * call and throw an error when the boundaries are + * violated, but it does not save us from possible + * segmentation faults. + * + * It is in a user responsibility to provide valid + * msgpack. + */ + mp_next(&tuple_end); + --source->remaining_tuples_cnt; + if (tuple_end > source->buf->wpos) { + lua_pushstring(L, "Unexpected msgpack buffer end"); + return 1; + } + ++base->processed; + source->buf->rpos = (char *) tuple_end; + base->tuple = box_tuple_new(format, tuple_beg, tuple_end); + if (base->tuple == NULL) { + luaT_pusherror(L, diag_last_error(diag_get())); + return 1; + } + + box_tuple_ref(base->tuple); + return 0; +} + +/* }}} */ + +/* {{{ Table merger source */ + +struct merger_source_table { + struct merger_source base; + int ref; + int next_idx; +}; + +/* Virtual methods declarations */ + +static void +luaL_merger_source_table_delete(struct merger_source *base, + struct lua_State *L); +static int +luaL_merger_source_table_next(struct merger_source *base, + box_tuple_format_t *format, + const struct merger_state *state, + struct lua_State *L); + +/* Non-virtual methods */ + +/** + * Create the new merger source of table type using content of a + * Lua stack. + * + * In case of an error it returns NULL and pushes the error to the + * Lua stack. + */ +static struct merger_source * +luaL_merger_source_table_new(struct lua_State *L, int idx, int ordinal, + struct merger_state *state) +{ + (void) state; + + static struct merger_source_vtab merger_source_table_vtab = { + .delete = luaL_merger_source_table_delete, + .next = luaL_merger_source_table_next, + }; + + struct merger_source_table *source = (struct merger_source_table *) + malloc(sizeof(struct merger_source_table)); + + if (source == NULL) { + diag_set(OutOfMemory, sizeof(struct merger_source_table), + "malloc", "merger_source_table"); + luaT_pusherror(L, diag_last_error(diag_get())); + return NULL; + } + + source->base.idx = ordinal; + source->base.processed = 0; + source->base.tuple = NULL; + /* source->base.hnode does not need to be initialized. */ + + lua_pushvalue(L, idx); /* Popped by luaL_ref(). */ + source->ref = luaL_ref(L, LUA_REGISTRYINDEX); + source->next_idx = 1; + + source->base.vtab = &merger_source_table_vtab; + return &source->base; +} + +/** + * Push certain fields of a source to Lua. + */ +static int +luaL_merger_source_table_push(const struct merger_source_table *source, + struct lua_State *L) +{ + lua_createtable(L, 0, 3); + + lua_pushinteger(L, source->base.idx + 1); + lua_setfield(L, -2, "idx"); + + lua_pushstring(L, "table"); + lua_setfield(L, -2, "type"); + + lua_rawgeti(L, LUA_REGISTRYINDEX, source->ref); + lua_setfield(L, -2, "table"); + + return 1; +} + +/** + * Call a user provided function to fill the source. + * + * Return 0 at success and 1 at error (push the error object). + */ +static int +luaL_merger_source_table_fetch(struct merger_source_table *source, + const struct merger_state *state, + struct tuple *last_tuple, struct lua_State *L) +{ + /* No fetch callback: do nothing. */ + if (state->fetch_source_ref <= 0) + return 0; + /* Push fetch callback. */ + lua_rawgeti(L, LUA_REGISTRYINDEX, state->fetch_source_ref); + /* Push source, last_tuple, processed. */ + luaL_merger_source_table_push(source, L); + if (last_tuple == NULL) + lua_pushnil(L); + else + luaT_pushtuple(L, last_tuple); + lua_pushinteger(L, source->base.processed); + /* Invoke the callback and process data. */ + if (lua_pcall(L, 3, 1, 0)) + return 1; + /* No more data: do nothing. */ + if (lua_isnil(L, -1)) { + lua_pop(L, 1); + return 0; + } + /* Set the new table as the source. */ + luaL_unref(L, LUA_REGISTRYINDEX, source->ref); + source->ref = luaL_ref(L, LUA_REGISTRYINDEX); + source->next_idx = 1; + return 0; + +} + +/* Virtual methods */ + +static void +luaL_merger_source_table_delete(struct merger_source *base, + struct lua_State *L) +{ + struct merger_source_buffer *source = container_of(base, + struct merger_source_buffer, base); + + luaL_unref(L, LUA_REGISTRYINDEX, source->ref); + + if (base->tuple != NULL) + box_tuple_unref(base->tuple); + + free(source); +} + +static int +luaL_merger_source_table_next(struct merger_source *base, + box_tuple_format_t *format, + const struct merger_state *state, + struct lua_State *L) +{ + struct merger_source_table *source = container_of(base, + struct merger_source_table, base); + + struct tuple *last_tuple = base->tuple; + base->tuple = NULL; + + lua_rawgeti(L, LUA_REGISTRYINDEX, source->ref); + lua_pushinteger(L, source->next_idx); + lua_gettable(L, -2); + /* + * If all data were processed, try to fetch more. + */ + if (lua_isnil(L, -1)) { + lua_pop(L, 2); + int rc = luaL_merger_source_table_fetch(source, state, + last_tuple, L); + if (rc != 0) + return 1; + /* + * Retry tuple extracting after fetching + * of the source. + */ + lua_rawgeti(L, LUA_REGISTRYINDEX, source->ref); + lua_pushinteger(L, source->next_idx); + lua_gettable(L, -2); + if (lua_isnil(L, -1)) { + lua_pop(L, 2); + return 0; + } + } + base->tuple = luaT_newtuple(L, -1, format); + if (base->tuple == NULL) + return 1; + ++source->next_idx; + ++base->processed; + lua_pop(L, 2); + + box_tuple_ref(base->tuple); + return 0; +} + +/* }}} */ + +/* {{{ Iterator merger source */ + +struct merger_source_iterator { + struct merger_source base; + struct luaL_iterator *it; +}; + +/* Virtual methods declarations */ + +static void +luaL_merger_source_iterator_delete(struct merger_source *base, + struct lua_State *L); +static int +luaL_merger_source_iterator_next(struct merger_source *base, + box_tuple_format_t *format, + const struct merger_state *state, + struct lua_State *L); + +/* Non-virtual methods */ + +/** + * Create the new merger source of iterator type using content of + * a Lua stack. + * + * In case of an error it returns NULL and pushes the error to the + * Lua stack. + */ +static struct merger_source * +luaL_merger_source_iterator_new(struct lua_State *L, int idx, int ordinal, + struct merger_state *state) +{ + (void) state; + + static struct merger_source_vtab merger_source_iterator_vtab = { + .delete = luaL_merger_source_iterator_delete, + .next = luaL_merger_source_iterator_next, + }; + + struct merger_source_iterator *source = + (struct merger_source_iterator *) malloc( + sizeof(struct merger_source_iterator)); + + if (source == NULL) { + diag_set(OutOfMemory, sizeof(struct merger_source_iterator), + "malloc", "merger_source_iterator"); + luaT_pusherror(L, diag_last_error(diag_get())); + return NULL; + } + + source->base.idx = ordinal; + source->base.processed = 0; + source->base.tuple = NULL; + /* source->base.hnode does not need to be initialized. */ + + source->it = luaL_iterator_new_fromtable(L, idx); + + source->base.vtab = &merger_source_iterator_vtab; + return &source->base; +} + +/* Virtual methods */ + +static void +luaL_merger_source_iterator_delete(struct merger_source *base, + struct lua_State *L) +{ + struct merger_source_iterator *source = container_of(base, + struct merger_source_iterator, base); + + luaL_iterator_free(L, source->it); + + if (base->tuple != NULL) + box_tuple_unref(base->tuple); + + free(source); +} + +static int +luaL_merger_source_iterator_next(struct merger_source *base, + box_tuple_format_t *format, + const struct merger_state *state, + struct lua_State *L) +{ + (void) state; + + struct merger_source_iterator *source = container_of(base, + struct merger_source_iterator, base); + + base->tuple = NULL; + + int nresult = luaL_iterator_next(L, source->it); + if (nresult == 0) + return 0; + base->tuple = luaT_newtuple(L, -nresult + 1, format); + if (base->tuple == NULL) + return 1; + ++base->processed; + lua_pop(L, nresult); + + box_tuple_ref(base->tuple); + return 0; +} + +/* }}} */ + +/* {{{ Create a source using Lua stack */ + +/** + * Create the new merger source using content of a Lua stack. + * + * In case of an error it returns NULL and pushes the error to the + * Lua stack. + */ +struct merger_source * +merger_source_new(struct lua_State *L, int idx, int ordinal, + struct merger_context *ctx, struct merger_state *state) +{ + struct merger_source *base = NULL; + + /* Determine type of a merger source on the Lua stack. */ + if (lua_type(L, idx) == LUA_TCDATA) { + struct ibuf *buf = check_ibuf(L, idx); + if (buf == NULL) + goto err; + /* Create the new buffer source. */ + base = luaL_merger_source_buffer_new(L, idx, ordinal, state); + } else if (lua_istable(L, idx)) { + lua_rawgeti(L, idx, 1); + int iscallable = luaL_iscallable(L, idx); + lua_pop(L, 1); + if (iscallable) { + /* Create the new iterator source. */ + base = luaL_merger_source_iterator_new(L, idx, ordinal, + state); + } else { + /* Create the new table source. */ + base = luaL_merger_source_table_new(L, idx, ordinal, + state); + } + } else { + goto err; + } + + if (base == NULL) + return NULL; + + /* Acquire the next tuple. */ + int rc = base->vtab->next(base, ctx->format, state, L); + if (rc) { + base->vtab->delete(base, L); + return NULL; + } + + /* Update the heap. */ + if (base->tuple != NULL) { + rc = merger_heap_insert(&state->heap, &base->hnode); + if (rc) { + base->vtab->delete(base, L); + diag_set(OutOfMemory, heap_next_grow_size(&state->heap), + "malloc", "merger heap"); + luaT_pusherror(L, diag_last_error(diag_get())); + return NULL; + } + } + + return base; + +err: + lua_pushfstring(L, "Unknown source type at index %d", ordinal + 1); + return NULL; +} + +/* }}} */ + +/* {{{ merger_context functions */ + +/** + * Free the merger context from a Lua code. + */ +static int +lbox_merger_context_gc(struct lua_State *L) +{ + struct merger_context *ctx; + if ((ctx = check_merger_context(L, 1)) == NULL) + return 0; + box_key_def_delete(ctx->key_def); + box_tuple_format_unref(ctx->format); + free(ctx); + return 0; +} + +/** + * Create the new merger context. + * + * Expected a table of key parts on the Lua stack. + * + * Returns the new instance. + */ +static int +lbox_merger_context_new(struct lua_State *L) +{ + if (lua_gettop(L) != 1) + return luaL_error(L, "Usage: merger.context.new(key_parts)"); + + struct merger_context *ctx = (struct merger_context *) malloc( + sizeof(struct merger_context)); + if (ctx == NULL) { + diag_set(OutOfMemory, sizeof(struct merger_context), "malloc", + "merger_context"); + return luaT_error(L); + } + ctx->key_def = luaT_new_key_def(L, 1); + if (ctx->key_def == NULL) { + free(ctx); + return luaL_error(L, "Cannot create key_def"); + } + + ctx->format = box_tuple_format_new(&ctx->key_def, 1); + if (ctx->format == NULL) { + box_key_def_delete(ctx->key_def); + free(ctx); + return luaL_error(L, "Cannot create format"); + } + + *(struct merger_context **) luaL_pushcdata(L, merger_context_type_id) = + ctx; + + lua_pushcfunction(L, lbox_merger_context_gc); + luaL_setcdatagc(L, -2); + + return 1; +} + +/* }}} */ + +/* {{{ merger_state functions */ + +/** + * Free the merger state. + * + * We need to know Lua state here, because sources of table and + * iterator types are saved as references within the Lua state. + */ +static void +merger_state_delete(struct lua_State *L, struct merger_state *state) +{ + merger_heap_destroy(&state->heap); + box_key_def_delete(state->key_def); + + for (uint32_t i = 0; i < state->sources_count; ++i) { + assert(state->sources != NULL); + assert(state->sources[i] != NULL); + state->sources[i]->vtab->delete(state->sources[i], L); + } + + if (state->sources != NULL) + free(state->sources); + + if (state->fetch_source_ref > 0) + luaL_unref(L, LUA_REGISTRYINDEX, state->fetch_source_ref); + + free(state); +} + +/** + * Free the merger state from a Lua code. + */ +static int +lbox_merger_state_gc(struct lua_State *L) +{ + struct merger_state *state; + if ((state = check_merger_state(L, 1)) == NULL) + return 0; + merger_state_delete(L, state); + return 0; +} + +/** + * Push 'bad params' / 'bad param X' and the usage info to the Lua + * stack. + */ +static int +merger_usage(struct lua_State *L, const char *param_name) +{ + static const char *usage = "merger.{ipairs,pairs,select}(" + "merger_context, " + "{source, source, ...}[, {" + "descending = <boolean> or <nil>, " + "buffer = <cdata<struct ibuf>> or <nil>, " + "fetch_source = <function> or <nil>}])"; + if (param_name == NULL) + lua_pushfstring(L, "Bad params, use: %s", usage); + else + lua_pushfstring(L, "Bad param \"%s\", use: %s", param_name, + usage); + return 1; +} + +/** + * Parse optional third parameter of merger.pairs() and + * merger.select() into the merger_state structure. + * + * Returns 0 on success. In case of an error it pushes an error + * message to the Lua stack and returns 1. + * + * It is the helper for merger_state_new(). + */ +static int +parse_opts(struct lua_State *L, int idx, struct merger_state *state) +{ + /* No opts: use defaults. */ + if (lua_isnoneornil(L, idx)) + return 0; + + /* Not a table: error. */ + if (!lua_istable(L, idx)) + return merger_usage(L, NULL); + + /* Parse descending to state->order. */ + lua_pushstring(L, "descending"); + lua_gettable(L, idx); + if (!lua_isnil(L, -1)) { + if (lua_isboolean(L, -1)) + state->order = lua_toboolean(L, -1) ? -1 : 1; + else + return merger_usage(L, "descending"); + } + lua_pop(L, 1); + + /* Parse buffer. */ + lua_pushstring(L, "buffer"); + lua_gettable(L, idx); + if (!lua_isnil(L, -1)) { + if ((state->obuf = check_ibuf(L, -1)) == NULL) + return merger_usage(L, "buffer"); + } + lua_pop(L, 1); + + /* Parse fetch_source. */ + lua_pushstring(L, "fetch_source"); + lua_gettable(L, idx); + if (!lua_isnil(L, -1)) { + if (!luaL_iscallable(L, -1)) + return merger_usage(L, "fetch_source"); + lua_pushvalue(L, -1); /* Popped by luaL_ref(). */ + state->fetch_source_ref = luaL_ref(L, LUA_REGISTRYINDEX); + } + lua_pop(L, 1); + + return 0; +} + +/** + * Parse sources table: second parameter of merger.pairs() + * and merger.select() into the merger_state structure. + * + * Note: This function should be called when options are already + * parsed (using parse_opts()). + * + * Returns 0 on success. In case of an error it pushes an error + * message to the Lua stack and returns 1. + * + * It is the helper for merger_state_new(). + */ +static int +parse_sources(struct lua_State *L, int idx, struct merger_context *ctx, + struct merger_state *state) +{ + /* Allocate sources array. */ + uint32_t capacity = 8; + const ssize_t sources_size = capacity * sizeof(struct merger_source *); + state->sources = (struct merger_source **) malloc(sources_size); + if (state->sources == NULL) { + diag_set(OutOfMemory, sources_size, "malloc", "state->sources"); + luaT_pusherror(L, diag_last_error(diag_get())); + return 1; + } + + /* Fetch all sources. */ + while (true) { + lua_pushinteger(L, state->sources_count + 1); + lua_gettable(L, idx); + if (lua_isnil(L, -1)) + break; + + /* Grow sources array if needed. */ + if (state->sources_count == capacity) { + capacity *= 2; + struct merger_source **new_sources; + const ssize_t new_sources_size = + capacity * sizeof(struct merger_source *); + new_sources = (struct merger_source **) realloc( + state->sources, new_sources_size); + if (new_sources == NULL) { + diag_set(OutOfMemory, new_sources_size / 2, + "malloc", "new_sources"); + luaT_pusherror(L, diag_last_error(diag_get())); + return 1; + } + state->sources = new_sources; + } + + /* Create the new source. */ + struct merger_source *source = merger_source_new(L, -1, + state->sources_count, ctx, state); + if (source == NULL) + return 1; + state->sources[state->sources_count] = source; + ++state->sources_count; + } + lua_pop(L, state->sources_count + 1); + + return 0; +} + +/** + * Parse sources and options on Lua stack and create the new + * merger_state instance. + * + * It is common code for parsing parameters for + * lbox_merger_ipairs() and lbox_merger_select(). + */ +static struct merger_state * +merger_state_new(struct lua_State *L) +{ + struct merger_context *ctx; + int ok = (lua_gettop(L) == 2 || lua_gettop(L) == 3) && + /* Merger context. */ + (ctx = check_merger_context(L, 1)) != NULL && + /* Sources. */ + lua_istable(L, 2) == 1 && + /* Opts. */ + (lua_isnoneornil(L, 3) == 1 || lua_istable(L, 3) == 1); + if (!ok) { + merger_usage(L, NULL); + lua_error(L); + unreachable(); + return NULL; + } + + struct merger_state *state = (struct merger_state *) + malloc(sizeof(struct merger_state)); + merger_heap_create(&state->heap); + state->key_def = key_def_dup(ctx->key_def); + state->sources_count = 0; + state->sources = NULL; + state->order = 1; + state->obuf = NULL; + state->fetch_source_ref = 0; + + if (parse_opts(L, 3, state) != 0 || + parse_sources(L, 2, ctx, state) != 0) { + merger_state_delete(L, state); + lua_error(L); + unreachable(); + return NULL; + } + + return state; +} + +/* }}} */ + +/* {{{ merger module logic */ + +/** + * Data comparing function to construct heap of sources. + */ +static bool +source_less(const heap_t *heap, const struct heap_node *a, + const struct heap_node *b) +{ + struct merger_source *left = container_of(a, struct merger_source, + hnode); + struct merger_source *right = container_of(b, struct merger_source, + hnode); + if (left->tuple == NULL && right->tuple == NULL) + return false; + if (left->tuple == NULL) + return false; + if (right->tuple == NULL) + return true; + struct merger_state *state = container_of(heap, struct merger_state, + heap); + return state->order * box_tuple_compare(left->tuple, right->tuple, + state->key_def) < 0; +} + +/** + * Get a tuple from a top source, update the source, update the + * heap. + * + * The reference counter of the tuple is increased (in + * source->vtab->next()). + * + * Return NULL when all sources are drained. + */ +static struct tuple * +merger_next(struct lua_State *L, struct merger_context *ctx, + struct merger_state *state) +{ + struct heap_node *hnode = merger_heap_top(&state->heap); + if (hnode == NULL) + return NULL; + + struct merger_source *source = container_of(hnode, struct merger_source, + hnode); + struct tuple *tuple = source->tuple; + assert(tuple != NULL); + int rc = source->vtab->next(source, ctx->format, state, L); + if (rc != 0) { + lua_error(L); + unreachable(); + return NULL; + } + if (source->tuple == NULL) + merger_heap_delete(&state->heap, hnode); + else + merger_heap_update(&state->heap, hnode); + + return tuple; +} + + +/** + * Iterator gen function to traverse merger results. + * + * Expected a merger context as the first parameter (state) and a + * merger_state as the second parameter (param) on the Lua + * stack. + * + * Push the merger_state (the new param) and the next tuple. + */ +static int +lbox_merger_gen(struct lua_State *L) +{ + struct merger_context *ctx; + struct merger_state *state; + bool ok = (ctx = check_merger_context(L, -2)) != NULL && + (state = check_merger_state(L, -1)) != NULL; + if (!ok) + return luaL_error(L, "Bad params, use: " + "lbox_merger_gen(merger_context, " + "merger_state)"); + + struct tuple *tuple = merger_next(L, ctx, state); + if (tuple == NULL) { + lua_pushnil(L); + lua_pushnil(L); + return 2; + } + + /* Push merger_state, tuple. */ + *(struct merger_state **) + luaL_pushcdata(L, merger_state_type_id) = state; + luaT_pushtuple(L, tuple); + + box_tuple_unref(tuple); + return 2; +} + +/** + * Iterate over merge results from Lua. + * + * Push three values to the Lua stack: + * + * 1. gen (lbox_merger_gen wrapped by fun.wrap()); + * 2. param (merger_context); + * 3. state (merger_state). + */ +static int +lbox_merger_ipairs(struct lua_State *L) +{ + /* Create merger_state. */ + struct merger_state *state = merger_state_new(L); + lua_settop(L, 1); /* Pop sources, [opts]. */ + /* Stack: merger_context. */ + + if (state->obuf != NULL) + return luaL_error(L, "\"buffer\" option is forbidden with " + "merger.pairs(<...>)"); + + luaL_loadstring(L, "return require('fun').wrap"); + lua_call(L, 0, 1); + lua_insert(L, -2); /* Swap merger_context and wrap. */ + /* Stack: wrap, merger_context. */ + + lua_pushcfunction(L, lbox_merger_gen); + lua_insert(L, -2); /* Swap merger_context and gen. */ + /* Stack: wrap, gen, merger_context. */ + + *(struct merger_state **) + luaL_pushcdata(L, merger_state_type_id) = state; + lua_pushcfunction(L, lbox_merger_state_gc); + luaL_setcdatagc(L, -2); + /* Stack: wrap, gen, merger_context, merger_state. */ + + /* Call fun.wrap(gen, merger_context, merger_state). */ + lua_call(L, 3, 3); + return 3; +} + +/** + * Write merge results into ibuf. + * + * It is the helper for lbox_merger_select(). + */ +static void +encode_result_buffer(struct lua_State *L, struct merger_context *ctx, + struct merger_state *state) +{ + struct ibuf *obuf = state->obuf; + uint32_t result_len = 0; + uint32_t result_len_offset = 4; + + /* + * Reserve maximum size for the array around resulting + * tuples to set it later. + */ + encode_header(state->obuf, UINT32_MAX); + + /* Fetch, merge and copy tuples to the buffer. */ + struct tuple *tuple; + while ((tuple = merger_next(L, ctx, state)) != NULL) { + uint32_t bsize = tuple->bsize; + ibuf_reserve(obuf, bsize); + memcpy(obuf->wpos, tuple_data(tuple), bsize); + obuf->wpos += bsize; + result_len_offset += bsize; + box_tuple_unref(tuple); + ++result_len; + } + + /* Write the real array size. */ + mp_store_u32(obuf->wpos - result_len_offset, result_len); +} + +/** + * Write merge results into the new Lua table. + * + * It is the helper for lbox_merger_select(). + */ +static int +create_result_table(struct lua_State *L, struct merger_context *ctx, + struct merger_state *state) +{ + /* Create result table. */ + lua_newtable(L); + + uint32_t cur = 1; + + /* Fetch, merge and save tuples to the table. */ + struct tuple *tuple; + while ((tuple = merger_next(L, ctx, state)) != NULL) { + luaT_pushtuple(L, tuple); + lua_rawseti(L, -2, cur); + box_tuple_unref(tuple); + ++cur; + } + + return 1; +} + +/** + * Perform the merge. + * + * Write results into a buffer or a Lua table depending on + * options. + * + * Expected a merger context, sources table and options (optional) + * on the Lua stack. + * + * Return the Lua table or nothing when the 'buffer' option is + * provided. + */ +static int +lbox_merger_select(struct lua_State *L) +{ + struct merger_context *ctx = check_merger_context(L, 1); + if (ctx == NULL) { + merger_usage(L, NULL); + lua_error(L); + } + + struct merger_state *state = merger_state_new(L); + lua_settop(L, 0); /* Pop merger_context, sources, [opts]. */ + + if (state->obuf == NULL) { + create_result_table(L, ctx, state); + merger_state_delete(L, state); + return 1; + } else { + encode_result_buffer(L, ctx, state); + merger_state_delete(L, state); + return 0; + } +} + +/** + * Register the module. + */ +LUA_API int +luaopen_merger(lua_State *L) +{ + luaL_cdef(L, "struct merger_context;"); + luaL_cdef(L, "struct merger_state;"); + luaL_cdef(L, "struct ibuf;"); + + merger_context_type_id = luaL_ctypeid(L, "struct merger_context&"); + merger_state_type_id = luaL_ctypeid(L, "struct merger_state&"); + ibuf_type_id = luaL_ctypeid(L, "struct ibuf"); + + /* Export C functions to Lua. */ + static const struct luaL_Reg meta[] = { + {"select", lbox_merger_select}, + {"ipairs", lbox_merger_ipairs}, + {"pairs", lbox_merger_ipairs}, + {NULL, NULL} + }; + luaL_register_module(L, "merger", meta); + + /* Add context.new(). */ + lua_newtable(L); /* merger.context */ + lua_pushcfunction(L, lbox_merger_context_new); + lua_setfield(L, -2, "new"); + lua_setfield(L, -2, "context"); + + return 1; +} + +/* }}} */ diff --git a/src/box/lua/merger.h b/src/box/lua/merger.h new file mode 100644 index 000000000..e444e99e4 --- /dev/null +++ b/src/box/lua/merger.h @@ -0,0 +1,47 @@ +#ifndef TARANTOOL_BOX_LUA_MERGER_H_INCLUDED +#define TARANTOOL_BOX_LUA_MERGER_H_INCLUDED +/* + * Copyright 2010-2018, Tarantool AUTHORS, please see AUTHORS file. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * 1. Redistributions of source code must retain the above + * copyright notice, this list of conditions and the + * following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY AUTHORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL + * AUTHORS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF + * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#if defined(__cplusplus) +extern "C" { +#endif /* defined(__cplusplus) */ + +struct lua_State; + +int +luaopen_merger(struct lua_State *L); + +#if defined(__cplusplus) +} /* extern "C" */ +#endif /* defined(__cplusplus) */ + +#endif /* TARANTOOL_BOX_LUA_MERGER_H_INCLUDED */ diff --git a/test/box-tap/merger.test.lua b/test/box-tap/merger.test.lua new file mode 100755 index 000000000..c6f890edf --- /dev/null +++ b/test/box-tap/merger.test.lua @@ -0,0 +1,558 @@ +#!/usr/bin/env tarantool + +local tap = require('tap') +local buffer = require('buffer') +local msgpackffi = require('msgpackffi') +local digest = require('digest') +local merger = require('merger') +local fiber = require('fiber') +local utf8 = require('utf8') +local ffi = require('ffi') +local fun = require('fun') + +local FETCH_BLOCK_SIZE = 10 + +local function merger_usage(param) + local msg = 'merger.{ipairs,pairs,select}(' .. + 'merger_context, ' .. + '{source, source, ...}[, {' .. + 'descending = <boolean> or <nil>, ' .. + 'buffer = <cdata<struct ibuf>> or <nil>, ' .. + 'fetch_source = <function> or <nil>}])' + if not param then + return ('Bad params, use: %s'):format(msg) + else + return ('Bad param "%s", use: %s'):format(param, msg) + end +end + +-- Get buffer with data encoded without last 'trunc' bytes. +local function truncated_msgpack_buffer(data, trunc) + local data = msgpackffi.encode(data) + data = data:sub(1, data:len() - trunc) + local len = data:len() + local buf = buffer.ibuf() + -- Ensure we have enough buffer to write len + trunc bytes. + buf:reserve(len + trunc) + local p = buf:alloc(len) + -- Ensure len bytes follows with trunc zero bytes. + ffi.copy(p, data .. string.rep('\0', trunc), len + trunc) + return buf +end + +local bad_merger_methods_calls = { + { + 'Bad opts', + sources = {}, + opts = 1, + exp_err = merger_usage(nil), + }, + { + 'Bad opts.descending', + sources = {}, + opts = {descending = 1}, + exp_err = merger_usage('descending'), + }, + { + 'Bad source', + sources = {1}, + opts = nil, + exp_err = 'Unknown source type at index 1', + }, + { + 'Bad cdata source', + sources = {ffi.new('char *')}, + opts = nil, + exp_err = 'Unknown source type at index 1', + }, + { + 'Wrong source of table type', + sources = {{1}}, + opts = nil, + exp_err = 'A tuple or a table expected, got number', + }, + { + 'Use buffer with an iterator result', + sources = {}, + opts = {buffer = buffer.ibuf()}, + funcs = {'pairs', 'ipairs'}, + exp_err = '"buffer" option is forbidden with merger.pairs(<...>)', + }, + { + 'Bad msgpack source: wrong length of the tuples array', + -- Remove the last tuple from msgpack data, but keep old + -- tuples array size. + sources = { + truncated_msgpack_buffer({{''}, {''}, {''}}, 2), + }, + opts = {}, + funcs = {'select'}, + exp_err = 'Unexpected msgpack buffer end', + }, + { + 'Bad msgpack source: wrong length of a tuple', + -- Remove half of the last tuple, but keep old tuple size. + sources = { + truncated_msgpack_buffer({{''}, {''}, {''}}, 1), + }, + opts = {}, + funcs = {'select'}, + exp_err = 'Unexpected msgpack buffer end', + }, + { + 'Bad fetch_source type', + sources = {}, + opts = {fetch_source = 1}, + exp_err = merger_usage('fetch_source'), + }, +} + +local schemas = { + { + name = 'small_unsigned', + parts = { + { + fieldno = 2, + type = 'unsigned', + } + }, + gen_tuple = function(tupleno) + return {'id_' .. tostring(tupleno), tupleno} + end, + }, + -- Merger allocates a memory for 8 parts by default. + -- Test that reallocation works properly. + -- Test with N-1 equal parts and Nth different. + { + name = 'many_parts', + parts = (function() + local parts = {} + for i = 1, 128 do + parts[i] = { + fieldno = i, + type = 'unsigned', + } + end + return parts + end)(), + gen_tuple = function(tupleno) + local tuple = {} + -- 127 constant parts + for i = 1, 127 do + tuple[i] = i + end + -- 128th part is varying + tuple[128] = tupleno + return tuple + end, + -- reduce tuples count to decrease test run time + tuples_cnt = 16, + }, + -- Test null value in nullable field of an index. + { + name = 'nullable', + parts = { + { + fieldno = 1, + type = 'unsigned', + }, + { + fieldno = 2, + type = 'string', + is_nullable = true, + }, + }, + gen_tuple = function(i) + if i % 1 == 1 then + return {0, tostring(i)} + else + return {0, box.NULL} + end + end, + }, + -- Test index part with 'collation_id' option (as in net.box's + -- response). + { + name = 'collation_id', + parts = { + { + fieldno = 1, + type = 'string', + collation_id = 2, -- unicode_ci + }, + }, + gen_tuple = function(i) + local letters = {'a', 'b', 'c', 'A', 'B', 'C'} + if i <= #letters then + return {letters[i]} + else + return {''} + end + end, + }, + -- Test index part with 'collation' option (as in local index + -- parts). + { + name = 'collation', + parts = { + { + fieldno = 1, + type = 'string', + collation = 'unicode_ci', + }, + }, + gen_tuple = function(i) + local letters = {'a', 'b', 'c', 'A', 'B', 'C'} + if i <= #letters then + return {letters[i]} + else + return {''} + end + end, + }, +} + +local function is_unicode_ci_part(part) + return part.collation_id == 2 or part.collation == 'unicode_ci' +end + +local function tuple_comparator(a, b, parts) + for _, part in ipairs(parts) do + local fieldno = part.fieldno + if a[fieldno] ~= b[fieldno] then + if a[fieldno] == nil then + return -1 + end + if b[fieldno] == nil then + return 1 + end + if is_unicode_ci_part(part) then + return utf8.casecmp(a[fieldno], b[fieldno]) + end + return a[fieldno] < b[fieldno] and -1 or 1 + end + end + + return 0 +end + +local function sort_tuples(tuples, parts, opts) + local function tuple_comparator_wrapper(a, b) + local cmp = tuple_comparator(a, b, parts) + if cmp < 0 then + return not opts.descending + elseif cmp > 0 then + return opts.descending + else + return false + end + end + + table.sort(tuples, tuple_comparator_wrapper) +end + +local function lowercase_unicode_ci_fields(tuples, parts) + for i = 1, #tuples do + local tuple = tuples[i] + for _, part in ipairs(parts) do + if is_unicode_ci_part(part) then + -- Workaround #3709. + if tuple[part.fieldno]:len() > 0 then + tuple[part.fieldno] = utf8.lower(tuple[part.fieldno]) + end + end + end + end +end + +local function gen_fetch_source(schema, tuples, opts) + local opts = opts or {} + local input_type = opts.input_type + local sources_cnt = #tuples + + local sources = {} + local last_positions = {} + for i = 1, sources_cnt do + sources[i] = input_type == 'table' and {} or buffer.ibuf() + last_positions[i] = 0 + end + + local fetch_source = function(source, last_tuple, processed) + assert(source.type == input_type) + if source.type == 'buffer' then + assert(type(source.buffer) == 'cdata') + assert(ffi.istype('struct ibuf', source.buffer)) + assert(source.table == nil) + else + assert(source.type == 'table') + assert(type(source.table) == 'table') + assert(source.buffer == nil) + end + local idx = source.idx + local last_pos = last_positions[idx] + local exp_last_tuple = tuples[idx][last_pos] + assert((last_tuple == nil and exp_last_tuple == nil) or + tuple_comparator(last_tuple, exp_last_tuple, + schema.parts) == 0) + assert(last_pos == processed) + local data = fun.iter(tuples[idx]):drop(last_pos):take( + FETCH_BLOCK_SIZE):totable() + assert(#data > 0 or processed == #tuples[idx]) + last_positions[idx] = last_pos + #data + if source.type == 'table' then + return data + elseif source.type == 'buffer' then + msgpackffi.internal.encode_r(source.buffer, data, 0) + else + assert(false) + end + end + + return sources, fetch_source +end + +local function prepare_data(schema, tuples_cnt, sources_cnt, opts) + local opts = opts or {} + local input_type = opts.input_type + local use_table_as_tuple = opts.use_table_as_tuple + local use_fetch_source = opts.use_fetch_source + + local tuples = {} + local exp_result = {} + local fetch_source + + -- Ensure empty sources are empty table and not nil. + for i = 1, sources_cnt do + if tuples[i] == nil then + tuples[i] = {} + end + end + + -- Prepare N tables with tuples as input for merger. + for i = 1, tuples_cnt do + -- [1, sources_cnt] + local guava = digest.guava(i, sources_cnt) + 1 + local tuple = schema.gen_tuple(i) + table.insert(exp_result, tuple) + if not use_table_as_tuple then + assert(input_type ~= 'buffer') + tuple = box.tuple.new(tuple) + end + table.insert(tuples[guava], tuple) + end + + -- Sort tuples within each source. + for _, source_tuples in pairs(tuples) do + sort_tuples(source_tuples, schema.parts, opts) + end + + -- Sort expected result. + sort_tuples(exp_result, schema.parts, opts) + + -- Fill sources. + local sources + if input_type == 'table' then + -- Imitate netbox's select w/o {buffer = ...}. + if use_fetch_source then + sources, fetch_source = gen_fetch_source(schema, tuples, opts) + else + sources = tuples + end + elseif input_type == 'buffer' then + -- Imitate netbox's select with {buffer = ...}. + if use_fetch_source then + sources, fetch_source = gen_fetch_source(schema, tuples, opts) + else + sources = {} + for i = 1, sources_cnt do + sources[i] = buffer.ibuf() + msgpackffi.internal.encode_r(sources[i], tuples[i], 0) + end + end + elseif input_type == 'iterator' then + -- Lua iterator. + assert(not use_fetch_source) + sources = {} + for i = 1, sources_cnt do + sources[i] = { + -- gen (next) + next, + -- param (tuples) + tuples[i], + -- state (idx) + nil + } + end + end + + return sources, exp_result, fetch_source +end + +local function test_case_opts_str(opts) + local params = {} + + if opts.input_type then + table.insert(params, 'input_type: ' .. opts.input_type) + end + + if opts.output_type then + table.insert(params, 'output_type: ' .. opts.output_type) + end + + if opts.descending then + table.insert(params, 'descending') + end + + if opts.use_table_as_tuple then + table.insert(params, 'use_table_as_tuple') + end + + if opts.use_fetch_source then + table.insert(params, 'use_fetch_source') + end + + if next(params) == nil then + return '' + end + + return (' (%s)'):format(table.concat(params, ', ')) +end + +local function run_merger(test, schema, tuples_cnt, sources_cnt, opts) + fiber.yield() + + local opts = opts or {} + + -- Prepare data. + local sources, exp_result, fetch_source = + prepare_data(schema, tuples_cnt, sources_cnt, opts) + + -- Create a merger instance and fill options. + local ctx = merger.context.new(schema.parts) + local merger_opts = { + descending = opts.descending, + fetch_source = fetch_source, + } + if opts.output_type == 'buffer' then + merger_opts.buffer = buffer.ibuf() + end + + local res + + -- Run merger and prepare output for compare. + if opts.output_type == 'table' then + -- Table output. + res = merger.select(ctx, sources, merger_opts) + elseif opts.output_type == 'buffer' then + -- Buffer output. + merger.select(ctx, sources, merger_opts) + local obuf = merger_opts.buffer + res = msgpackffi.decode(obuf.rpos) + else + -- Iterator output. + assert(opts.output_type == 'iterator') + res = merger.pairs(ctx, sources, merger_opts):totable() + end + + -- A bit more postprocessing to compare. + for i = 1, #res do + if type(res[i]) ~= 'table' then + res[i] = res[i]:totable() + end + end + + -- unicode_ci does not differentiate btw 'A' and 'a', so the + -- order is arbitrary. We transform fields with unicode_ci + -- collation in parts to lower case before comparing. + lowercase_unicode_ci_fields(res, schema.parts) + lowercase_unicode_ci_fields(exp_result, schema.parts) + + test:is_deeply(res, exp_result, + ('check order on %3d tuples in %4d sources%s') + :format(tuples_cnt, sources_cnt, test_case_opts_str(opts))) +end + +local function run_case(test, schema, opts) + local opts = opts or {} + + local case_name = ('testing on schema %s%s'):format( + schema.name, test_case_opts_str(opts)) + local tuples_cnt = schema.tuples_cnt or 100 + + local input_type = opts.input_type + local use_table_as_tuple = opts.use_table_as_tuple + local use_fetch_source = opts.use_fetch_source + + -- Skip meaningless flags combinations. + if input_type == 'buffer' and not use_table_as_tuple then + return + end + if input_type == 'iterator' and use_fetch_source then + return + end + + test:test(case_name, function(test) + test:plan(6) + + -- Check with small buffers count. + run_merger(test, schema, tuples_cnt, 1, opts) + run_merger(test, schema, tuples_cnt, 2, opts) + run_merger(test, schema, tuples_cnt, 3, opts) + run_merger(test, schema, tuples_cnt, 4, opts) + run_merger(test, schema, tuples_cnt, 5, opts) + + -- Check more buffers then tuples count. + run_merger(test, schema, tuples_cnt, 1000, opts) + end) +end + +local test = tap.test('merger') +test:plan(#bad_merger_methods_calls + #schemas * 48) + +-- For collations. +box.cfg{} + +-- Create the instance to use in testing merger's methods below. +local ctx = merger.context.new({{ + fieldno = 1, + type = 'string', +}}) + +-- Bad source or/and opts parameters for merger's methods. +for _, case in ipairs(bad_merger_methods_calls) do + test:test(case[1], function(test) + local funcs = case.funcs or {'pairs', 'ipairs', 'select'} + test:plan(#funcs) + for _, func in ipairs(funcs) do + local exp_ok = case.exp_err == nil + local ok, err = pcall(merger[func], ctx, case.sources, case.opts) + if ok then + err = nil + end + test:is_deeply({ok, err}, {exp_ok, case.exp_err}, func) + end + end) +end + +-- Merging cases. +for _, input_type in ipairs({'buffer', 'table', 'iterator'}) do + for _, output_type in ipairs({'buffer', 'table', 'iterator'}) do + for _, descending in ipairs({false, true}) do + for _, use_table_as_tuple in ipairs({false, true}) do + for _, use_fetch_source in ipairs({false, true}) do + for _, schema in ipairs(schemas) do + run_case(test, schema, { + input_type = input_type, + output_type = output_type, + descending = descending, + use_table_as_tuple = use_table_as_tuple, + use_fetch_source = use_fetch_source, + }) + end + end + end + end + end +end + +os.exit(test:check() and 0 or 1) diff --git a/test/box-tap/suite.ini b/test/box-tap/suite.ini index 50dc1f435..3361102be 100644 --- a/test/box-tap/suite.ini +++ b/test/box-tap/suite.ini @@ -1,4 +1,5 @@ [default] core = app description = Database tests with #! using TAP +long_run = merger.test.lua is_parallel = True -- 2.20.1 ^ permalink raw reply [flat|nested] 28+ messages in thread
end of thread, other threads:[~2019-03-05 12:00 UTC | newest] Thread overview: 28+ messages (download: mbox.gz / follow: Atom feed) -- links below jump to the message on this page -- 2019-01-09 20:20 [PATCH v2 0/6] Merger Alexander Turenko 2019-01-09 20:20 ` [PATCH v2 1/6] Add luaL_iscallable with support of cdata metatype Alexander Turenko 2019-01-10 12:21 ` Vladimir Davydov 2019-01-09 20:20 ` [PATCH v2 2/6] Add functions to ease using Lua iterators from C Alexander Turenko 2019-01-10 12:29 ` Vladimir Davydov 2019-01-15 23:26 ` Alexander Turenko 2019-01-16 8:18 ` Vladimir Davydov 2019-01-16 11:40 ` Alexander Turenko 2019-01-16 12:20 ` Vladimir Davydov 2019-01-17 1:20 ` Alexander Turenko 2019-01-28 18:17 ` Alexander Turenko 2019-03-01 4:04 ` Alexander Turenko 2019-01-09 20:20 ` [PATCH v2 3/6] lua: add luaT_newtuple() Alexander Turenko 2019-01-10 12:44 ` Vladimir Davydov 2019-01-18 21:58 ` Alexander Turenko 2019-01-23 16:12 ` Vladimir Davydov 2019-01-28 16:51 ` Alexander Turenko 2019-03-01 4:08 ` Alexander Turenko 2019-01-09 20:20 ` [PATCH v2 4/6] lua: add luaT_new_key_def() Alexander Turenko 2019-01-10 13:07 ` Vladimir Davydov 2019-01-29 18:52 ` Alexander Turenko 2019-01-30 10:58 ` Alexander Turenko 2019-03-01 4:10 ` Alexander Turenko 2019-01-09 20:20 ` [PATCH v2 5/6] net.box: add helpers to decode msgpack headers Alexander Turenko 2019-01-10 17:29 ` Vladimir Davydov 2019-02-01 15:11 ` Alexander Turenko 2019-03-05 12:00 ` Alexander Turenko 2019-01-09 20:20 ` [PATCH v2 6/6] Add merger for tuple streams Alexander Turenko
This is a public inbox, see mirroring instructions for how to clone and mirror all data and code used for this inbox