* [PATCH v2 01/10] yaml: don't throw OOM on any error in yaml encoding
2018-04-20 13:24 [PATCH v2 00/10] session: introduce box.session.push Vladislav Shpilevoy
@ 2018-04-20 13:24 ` Vladislav Shpilevoy
2018-05-10 18:10 ` [tarantool-patches] " Konstantin Osipov
2018-04-20 13:24 ` [tarantool-patches] [PATCH v2 10/10] session: introduce binary box.session.push Vladislav Shpilevoy
` (9 subsequent siblings)
10 siblings, 1 reply; 34+ messages in thread
From: Vladislav Shpilevoy @ 2018-04-20 13:24 UTC (permalink / raw)
To: tarantool-patches; +Cc: vdavydov.dev
OOM is not single possible error. Yaml library during dump can
raise such errors as YAML_MEMORY_ERROR, YAML_WRITER_ERROR,
YAML_EMITTER_ERROR. And each of them can contain any error
message that is skipped now, because Tarantool YAML does not
provide such API, that can lead to a non-OOM error.
But it is changed in next commits.
---
third_party/lua-yaml/lyaml.cc | 49 ++++++++++++++++++++++++-------------------
1 file changed, 27 insertions(+), 22 deletions(-)
diff --git a/third_party/lua-yaml/lyaml.cc b/third_party/lua-yaml/lyaml.cc
index 4d875fab4..430f4be2c 100644
--- a/third_party/lua-yaml/lyaml.cc
+++ b/third_party/lua-yaml/lyaml.cc
@@ -401,7 +401,7 @@ static int dump_table(struct lua_yaml_dumper *dumper, struct luaL_field *field){
? (YAML_FLOW_MAPPING_STYLE) : YAML_BLOCK_MAPPING_STYLE;
if (!yaml_mapping_start_event_initialize(&ev, anchor, NULL, 0, yaml_style) ||
!yaml_emitter_emit(&dumper->emitter, &ev))
- luaL_error(dumper->L, OOM_ERRMSG);
+ return 0;
lua_pushnil(dumper->L);
while (lua_next(dumper->L, -2)) {
@@ -414,11 +414,8 @@ static int dump_table(struct lua_yaml_dumper *dumper, struct luaL_field *field){
lua_pop(dumper->L, 1);
}
- if (!yaml_mapping_end_event_initialize(&ev) ||
- !yaml_emitter_emit(&dumper->emitter, &ev))
- luaL_error(dumper->L, OOM_ERRMSG);
-
- return 1;
+ return yaml_mapping_end_event_initialize(&ev) != 0 &&
+ yaml_emitter_emit(&dumper->emitter, &ev) != 0 ? 1 : 0;
}
static int dump_array(struct lua_yaml_dumper *dumper, struct luaL_field *field){
@@ -433,7 +430,7 @@ static int dump_array(struct lua_yaml_dumper *dumper, struct luaL_field *field){
? (YAML_FLOW_SEQUENCE_STYLE) : YAML_BLOCK_SEQUENCE_STYLE;
if (!yaml_sequence_start_event_initialize(&ev, anchor, NULL, 0, yaml_style) ||
!yaml_emitter_emit(&dumper->emitter, &ev))
- luaL_error(dumper->L, OOM_ERRMSG);
+ return 0;
for (i = 0; i < field->size; i++) {
lua_rawgeti(dumper->L, -1, i + 1);
@@ -442,11 +439,8 @@ static int dump_array(struct lua_yaml_dumper *dumper, struct luaL_field *field){
lua_pop(dumper->L, 1);
}
- if (!yaml_sequence_end_event_initialize(&ev) ||
- !yaml_emitter_emit(&dumper->emitter, &ev))
- luaL_error(dumper->L, OOM_ERRMSG);
-
- return 1;
+ return yaml_sequence_end_event_initialize(&ev) != 0 &&
+ yaml_emitter_emit(&dumper->emitter, &ev) != 0 ? 1 : 0;
}
static int yaml_is_flow_mode(struct lua_yaml_dumper *dumper) {
@@ -566,7 +560,7 @@ static int dump_node(struct lua_yaml_dumper *dumper)
if (!yaml_scalar_event_initialize(&ev, NULL, tag, (unsigned char *)str, len,
!is_binary, !is_binary, style) ||
!yaml_emitter_emit(&dumper->emitter, &ev))
- luaL_error(dumper->L, OOM_ERRMSG);
+ return 0;
if (is_binary)
lua_pop(dumper->L, 1);
@@ -579,14 +573,14 @@ static void dump_document(struct lua_yaml_dumper *dumper) {
if (!yaml_document_start_event_initialize(&ev, NULL, NULL, NULL, 0) ||
!yaml_emitter_emit(&dumper->emitter, &ev))
- luaL_error(dumper->L, OOM_ERRMSG);
+ return;
if (!dump_node(dumper) || dumper->error)
return;
if (!yaml_document_end_event_initialize(&ev, 0) ||
!yaml_emitter_emit(&dumper->emitter, &ev))
- luaL_error(dumper->L, OOM_ERRMSG);
+ return;
}
static int append_output(void *arg, unsigned char *buf, size_t len) {
@@ -637,7 +631,7 @@ static int l_dump(lua_State *L) {
luaL_buffinit(dumper.outputL, &dumper.yamlbuf);
if (!yaml_emitter_initialize(&dumper.emitter))
- luaL_error(L, OOM_ERRMSG);
+ goto error;
yaml_emitter_set_unicode(&dumper.emitter, 1);
yaml_emitter_set_indent(&dumper.emitter, 2);
@@ -647,7 +641,7 @@ static int l_dump(lua_State *L) {
if (!yaml_stream_start_event_initialize(&ev, YAML_UTF8_ENCODING) ||
!yaml_emitter_emit(&dumper.emitter, &ev))
- luaL_error(L, OOM_ERRMSG);
+ goto error;
for (i = 0; i < argcount; i++) {
lua_newtable(L);
@@ -657,26 +651,37 @@ static int l_dump(lua_State *L) {
find_references(&dumper);
dump_document(&dumper);
if (dumper.error)
- break;
+ goto error;
lua_pop(L, 2); /* pop copied arg and anchor table */
}
if (!yaml_stream_end_event_initialize(&ev) ||
!yaml_emitter_emit(&dumper.emitter, &ev) ||
!yaml_emitter_flush(&dumper.emitter))
- luaL_error(L, OOM_ERRMSG);
-
- yaml_emitter_delete(&dumper.emitter);
+ goto error;
/* finalize and push YAML buffer */
luaL_pushresult(&dumper.yamlbuf);
if (dumper.error)
- lua_error(L);
+ goto error;
+ yaml_emitter_delete(&dumper.emitter);
/* move buffer to original thread */
lua_xmove(dumper.outputL, L, 1);
return 1;
+
+error:
+ if (dumper.emitter.error == YAML_NO_ERROR ||
+ dumper.emitter.error == YAML_MEMORY_ERROR) {
+ yaml_emitter_delete(&dumper.emitter);
+ return luaL_error(L, OOM_ERRMSG);
+ } else {
+ lua_pushnil(L);
+ lua_pushstring(L, dumper.emitter.problem);
+ yaml_emitter_delete(&dumper.emitter);
+ return 2;
+ }
}
static int
--
2.15.1 (Apple Git-101)
^ permalink raw reply [flat|nested] 34+ messages in thread* [tarantool-patches] [PATCH v2 10/10] session: introduce binary box.session.push
2018-04-20 13:24 [PATCH v2 00/10] session: introduce box.session.push Vladislav Shpilevoy
2018-04-20 13:24 ` [PATCH v2 01/10] yaml: don't throw OOM on any error in yaml encoding Vladislav Shpilevoy
@ 2018-04-20 13:24 ` Vladislav Shpilevoy
2018-05-10 19:50 ` Konstantin Osipov
2018-04-20 13:24 ` [PATCH v2 02/10] yaml: introduce yaml.encode_tagged Vladislav Shpilevoy
` (8 subsequent siblings)
10 siblings, 1 reply; 34+ messages in thread
From: Vladislav Shpilevoy @ 2018-04-20 13:24 UTC (permalink / raw)
To: tarantool-patches; +Cc: vdavydov.dev
Box.session.push() allows to send a message to a client with no
finishing a main request.
Tarantool after this patch supports pushes over binary protocol.
IProto message is encoded using a new header code - IPROTO_CHUNK.
TX thread to notify IProto thread about new data in obuf sends
a message 'push_msg'. IProto thread, got this message, notifies
libev about new data, and then sends 'push_msg' back with
updated write position. TX thread, received the message back,
updates its version of a write position. If IProto would not send
a write position, then TX would write to the same obuf again and
again, because it can not know that IProto already flushed
another obuf.
To avoid multiple 'push_msg' in fly between IProto and TX, the
only one 'push_msg' per connection is used. To deliver pushes,
appeared when 'push_msg' was in fly, TX thread sets a flag every
time when sees, that 'push_msg' is sent, and there is a new push.
When 'push_msg' returns, it checks this flag, and if it is set,
the IProto is notified again.
Closes #2677
---
src/box/iproto.cc | 221 +++++++++++++++++++++++++++++++++++--------
src/box/iproto_constants.h | 3 +
src/box/lua/net_box.lua | 50 ++++++----
src/box/lua/session.c | 33 ++++++-
src/box/xrow.c | 13 +++
src/box/xrow.h | 12 +++
src/fiber.h | 14 ++-
test/box/net.box.result | 4 +-
test/box/net.box.test.lua | 4 +-
test/box/push.result | 231 ++++++++++++++++++++++++++++++++++++++++++---
test/box/push.test.lua | 134 +++++++++++++++++++++++---
11 files changed, 635 insertions(+), 84 deletions(-)
diff --git a/src/box/iproto.cc b/src/box/iproto.cc
index 38baf1b8d..78a551039 100644
--- a/src/box/iproto.cc
+++ b/src/box/iproto.cc
@@ -67,6 +67,41 @@ enum { IPROTO_MSG_MAX = 768 };
enum { IPROTO_SALT_SIZE = 32 };
+/**
+ * This structure represents a position in the output.
+ * Since we use rotating buffers to recycle memory,
+ * it includes not only a position in obuf, but also
+ * a pointer to obuf the position is for.
+ */
+struct iproto_wpos {
+ struct obuf *obuf;
+ struct obuf_svp svp;
+};
+
+static void
+iproto_wpos_create(struct iproto_wpos *wpos, struct obuf *out)
+{
+ wpos->obuf = out;
+ wpos->svp = obuf_create_svp(out);
+}
+
+/**
+ * Message to notify IProto thread about new data in an output
+ * buffer. Struct iproto_msg is not used here, because push
+ * notification can be much more compact: it does not have
+ * request, ibuf, length, flags ...
+ */
+struct iproto_push_msg {
+ struct cmsg base;
+ /**
+ * Before sending to IProto thread, the wpos is set to a
+ * current position in an output buffer. Before IProto
+ * returns the message to TX, it sets wpos to the last
+ * flushed position (works like iproto_msg.wpos).
+ */
+ struct iproto_wpos wpos;
+};
+
/**
* Network readahead. A signed integer to avoid
* automatic type coercion to an unsigned type.
@@ -111,24 +146,6 @@ iproto_reset_input(struct ibuf *ibuf)
}
}
-/**
- * This structure represents a position in the output.
- * Since we use rotating buffers to recycle memory,
- * it includes not only a position in obuf, but also
- * a pointer to obuf the position is for.
- */
-struct iproto_wpos {
- struct obuf *obuf;
- struct obuf_svp svp;
-};
-
-static void
-iproto_wpos_create(struct iproto_wpos *wpos, struct obuf *out)
-{
- wpos->obuf = out;
- wpos->svp = obuf_create_svp(out);
-}
-
/* {{{ iproto_msg - declaration */
/**
@@ -367,6 +384,37 @@ struct iproto_connection
/** Pointer to the current output buffer. */
struct obuf *p_obuf;
} tx;
+ /**
+ * Is_push_in_progress is set, when a push_msg is sent to
+ * IProto thread, and reset, when the message is returned
+ * to TX. If a new push sees, that a push_msg is already
+ * sent to IProto, then has_new_pushes is set. After push
+ * notification is returned to TX, it checks
+ * has_new_pushes. If it is set, then the notification is
+ * sent again. This ping-pong continues, until TX stopped
+ * pushing. It allows to
+ * 1) avoid multiple push_msg from one session in fly,
+ * 2) do not block push() until a previous push() is
+ * finished.
+ *
+ * IProto TX
+ * -------------------------------------------------------
+ * + [push message]
+ * start socket <--- notification ----
+ * write
+ * + [push message]
+ * + [push message]
+ * ...
+ * end socket
+ * write ----------------> check for new
+ * pushes - found
+ * <--- notification ---
+ * ....
+ */
+ bool has_new_pushes;
+ bool is_push_in_progress;
+ /** Push notification for IProto thread. */
+ struct iproto_push_msg push_msg;
/** Authentication salt. */
char salt[IPROTO_SALT_SIZE];
};
@@ -806,6 +854,7 @@ iproto_connection_new(int fd)
{
struct iproto_connection *con = (struct iproto_connection *)
mempool_alloc_xc(&iproto_connection_pool);
+ memset(con, 0, sizeof(*con));
con->input.data = con->output.data = con;
con->loop = loop();
ev_io_init(&con->input, iproto_connection_on_input, fd, EV_READ);
@@ -818,9 +867,6 @@ iproto_connection_new(int fd)
con->tx.p_obuf = &con->obuf[0];
iproto_wpos_create(&con->wpos, con->tx.p_obuf);
iproto_wpos_create(&con->wend, con->tx.p_obuf);
- con->parse_size = 0;
- con->long_poll_requests = 0;
- con->session = NULL;
rlist_create(&con->in_stop_list);
/* It may be very awkward to allocate at close. */
con->disconnect = iproto_msg_new(con);
@@ -1010,9 +1056,9 @@ error:
}
static void
-tx_fiber_init(struct session *session, uint64_t sync)
+tx_fiber_init(struct session *session, uint64_t *sync)
{
- session->meta.sync = sync;
+ session->meta.sync = sync != NULL ? *sync : 0;
/*
* We do not cleanup fiber keys at the end of each request.
* This does not lead to privilege escalation as long as
@@ -1025,6 +1071,7 @@ tx_fiber_init(struct session *session, uint64_t sync)
*/
fiber_set_session(fiber(), session);
fiber_set_user(fiber(), &session->credentials);
+ fiber_set_key(fiber(), FIBER_KEY_SYNC, (void *) sync);
}
/**
@@ -1038,7 +1085,7 @@ tx_process_disconnect(struct cmsg *m)
struct iproto_msg *msg = (struct iproto_msg *) m;
struct iproto_connection *con = msg->connection;
if (con->session) {
- tx_fiber_init(con->session, 0);
+ tx_fiber_init(con->session, NULL);
if (! rlist_empty(&session_on_disconnect))
session_run_on_disconnect_triggers(con->session);
session_destroy(con->session);
@@ -1108,15 +1155,16 @@ tx_discard_input(struct iproto_msg *msg)
* not, the empty buffer is selected.
* - if neither of the buffers are empty, the function
* does not rotate the buffer.
+ *
+ * @param con IProto connection.
+ * @param wpos Last flushed write position, received from IProto
+ * thread.
*/
-static struct iproto_msg *
-tx_accept_msg(struct cmsg *m)
+static void
+tx_accept_wpos(struct iproto_connection *con, const struct iproto_wpos *wpos)
{
- struct iproto_msg *msg = (struct iproto_msg *) m;
- struct iproto_connection *con = msg->connection;
-
struct obuf *prev = &con->obuf[con->tx.p_obuf == con->obuf];
- if (msg->wpos.obuf == con->tx.p_obuf) {
+ if (wpos->obuf == con->tx.p_obuf) {
/*
* We got a message advancing the buffer which
* is being appended to. The previous buffer is
@@ -1134,6 +1182,13 @@ tx_accept_msg(struct cmsg *m)
*/
con->tx.p_obuf = prev;
}
+}
+
+static inline struct iproto_msg *
+tx_accept_msg(struct cmsg *m)
+{
+ struct iproto_msg *msg = (struct iproto_msg *) m;
+ tx_accept_wpos(msg->connection, &msg->wpos);
return msg;
}
@@ -1179,7 +1234,7 @@ tx_process1(struct cmsg *m)
{
struct iproto_msg *msg = tx_accept_msg(m);
- tx_fiber_init(msg->connection->session, msg->header.sync);
+ tx_fiber_init(msg->connection->session, &msg->header.sync);
if (tx_check_schema(msg->header.schema_version))
goto error;
@@ -1213,7 +1268,7 @@ tx_process_select(struct cmsg *m)
int rc;
struct request *req = &msg->dml;
- tx_fiber_init(msg->connection->session, msg->header.sync);
+ tx_fiber_init(msg->connection->session, &msg->header.sync);
if (tx_check_schema(msg->header.schema_version))
goto error;
@@ -1263,7 +1318,7 @@ tx_process_call(struct cmsg *m)
{
struct iproto_msg *msg = tx_accept_msg(m);
- tx_fiber_init(msg->connection->session, msg->header.sync);
+ tx_fiber_init(msg->connection->session, &msg->header.sync);
if (tx_check_schema(msg->header.schema_version))
goto error;
@@ -1347,7 +1402,7 @@ tx_process_misc(struct cmsg *m)
struct iproto_connection *con = msg->connection;
struct obuf *out = con->tx.p_obuf;
- tx_fiber_init(con->session, msg->header.sync);
+ tx_fiber_init(con->session, &msg->header.sync);
if (tx_check_schema(msg->header.schema_version))
goto error;
@@ -1386,7 +1441,7 @@ tx_process_join_subscribe(struct cmsg *m)
struct iproto_msg *msg = tx_accept_msg(m);
struct iproto_connection *con = msg->connection;
- tx_fiber_init(con->session, msg->header.sync);
+ tx_fiber_init(con->session, &msg->header.sync);
try {
switch (msg->header.type) {
@@ -1503,7 +1558,7 @@ tx_process_connect(struct cmsg *m)
if (con->session == NULL)
diag_raise();
con->session->meta.conn = con;
- tx_fiber_init(con->session, 0);
+ tx_fiber_init(con->session, NULL);
static __thread char greeting[IPROTO_GREETING_SIZE];
/* TODO: dirty read from tx thread */
struct tt_uuid uuid = INSTANCE_UUID;
@@ -1653,6 +1708,98 @@ iproto_session_sync(struct session *session)
return session->meta.sync;
}
+/** {{{ IPROTO_PUSH implementation. */
+
+/**
+ * Send to IProto thread a notification about new pushes.
+ * @param conn IProto connection.
+ */
+static void
+tx_begin_push(struct iproto_connection *conn);
+
+/**
+ * Create an event to send push.
+ * @param m IProto push message.
+ */
+static void
+net_push_msg(struct cmsg *m)
+{
+ struct iproto_push_msg *msg = (struct iproto_push_msg *) m;
+ struct iproto_connection *conn =
+ container_of(msg, struct iproto_connection, push_msg);
+ conn->wend = msg->wpos;
+ msg->wpos = conn->wpos;
+ if (evio_has_fd(&conn->output) && !ev_is_active(&conn->output))
+ ev_feed_event(conn->loop, &conn->output, EV_WRITE);
+}
+
+/**
+ * After a message notifies IProto thread about pushed data, TX
+ * thread can already have a new push in one of obufs. This
+ * function checks for new pushes and possibly re-sends push
+ * notification to IProto thread.
+ */
+static void
+tx_check_for_new_push(struct cmsg *m)
+{
+ struct iproto_push_msg *msg = (struct iproto_push_msg *) m;
+ struct iproto_connection *conn =
+ container_of(msg, struct iproto_connection, push_msg);
+ tx_accept_wpos(conn, &msg->wpos);
+ conn->is_push_in_progress = false;
+ if (conn->has_new_pushes)
+ tx_begin_push(conn);
+}
+
+static const struct cmsg_hop push_route[] = {
+ { net_push_msg, &tx_pipe },
+ { tx_check_for_new_push, NULL }
+};
+
+static void
+tx_begin_push(struct iproto_connection *conn)
+{
+ assert(! conn->is_push_in_progress);
+ cmsg_init((struct cmsg *) &conn->push_msg, push_route);
+ iproto_wpos_create(&conn->push_msg.wpos, conn->tx.p_obuf);
+ conn->has_new_pushes = false;
+ conn->is_push_in_progress = true;
+ cpipe_push(&net_pipe, (struct cmsg *) &conn->push_msg);
+}
+
+/**
+ * Push a message from @a port to a remote client.
+ * @param session IProto session.
+ * @param port Port with data to send.
+ *
+ * @retval -1 Memory error.
+ * @retval 0 Success, a message is wrote to an output buffer. But
+ * it is not guaranteed, that it will be sent
+ * successfully.
+ */
+static int
+iproto_session_push(struct session *session, struct port *port)
+{
+ struct iproto_connection *conn =
+ (struct iproto_connection *) session->meta.conn;
+ struct obuf_svp svp;
+ if (iproto_prepare_select(conn->tx.p_obuf, &svp) != 0)
+ return -1;
+ if (port_dump_msgpack(port, conn->tx.p_obuf) != 0) {
+ obuf_rollback_to_svp(conn->tx.p_obuf, &svp);
+ return -1;
+ }
+ iproto_reply_chunk(conn->tx.p_obuf, &svp, fiber_sync(fiber()),
+ ::schema_version);
+ if (! conn->is_push_in_progress)
+ tx_begin_push(conn);
+ else
+ conn->has_new_pushes = true;
+ return 0;
+}
+
+/** }}} */
+
/** Initialize the iproto subsystem and start network io thread */
void
iproto_init()
@@ -1666,7 +1813,7 @@ iproto_init()
cpipe_create(&net_pipe, "net");
cpipe_set_max_input(&net_pipe, IPROTO_MSG_MAX/2);
struct session_vtab iproto_session_vtab = {
- /* .push = */ generic_session_push,
+ /* .push = */ iproto_session_push,
/* .fd = */ iproto_session_fd,
/* .sync = */ iproto_session_sync,
};
diff --git a/src/box/iproto_constants.h b/src/box/iproto_constants.h
index 92534ef6e..67514ca03 100644
--- a/src/box/iproto_constants.h
+++ b/src/box/iproto_constants.h
@@ -163,6 +163,9 @@ enum iproto_type {
/** Vinyl row index stored in .run file */
VY_RUN_ROW_INDEX = 102,
+ /** Non-final response type. */
+ IPROTO_CHUNK = 128,
+
/**
* Error codes = (IPROTO_TYPE_ERROR | ER_XXX from errcode.h)
*/
diff --git a/src/box/lua/net_box.lua b/src/box/lua/net_box.lua
index 4ed2b375d..814358a2e 100644
--- a/src/box/lua/net_box.lua
+++ b/src/box/lua/net_box.lua
@@ -40,6 +40,8 @@ local IPROTO_SCHEMA_VERSION_KEY = 0x05
local IPROTO_DATA_KEY = 0x30
local IPROTO_ERROR_KEY = 0x31
local IPROTO_GREETING_SIZE = 128
+local IPROTO_CHUNK_KEY = 128
+local IPROTO_OK_KEY = 0
-- select errors from box.error
local E_UNKNOWN = box.error.UNKNOWN
@@ -267,7 +269,8 @@ local function create_transport(host, port, user, password, callback,
end
-- REQUEST/RESPONSE --
- local function perform_request(timeout, buffer, method, schema_version, ...)
+ local function perform_request(timeout, buffer, method, on_push,
+ schema_version, ...)
if state ~= 'active' then
return last_errno or E_NO_CONNECTION, last_error
end
@@ -280,11 +283,12 @@ local function create_transport(host, port, user, password, callback,
local id = next_request_id
method_codec[method](send_buf, id, schema_version, ...)
next_request_id = next_id(id)
- local request = table_new(0, 6) -- reserve space for 6 keys
+ local request = table_new(0, 7) -- reserve space for 7 keys
request.client = fiber_self()
request.method = method
request.schema_version = schema_version
request.buffer = buffer
+ request.on_push = on_push
requests[id] = request
repeat
local timeout = max(0, deadline - fiber_clock())
@@ -308,12 +312,12 @@ local function create_transport(host, port, user, password, callback,
if request == nil then -- nobody is waiting for the response
return
end
- requests[id] = nil
local status = hdr[IPROTO_STATUS_KEY]
local body, body_end_check
- if status ~= 0 then
+ if status > IPROTO_CHUNK_KEY then
-- Handle errors
+ requests[id] = nil
body, body_end_check = decode(body_rpos)
assert(body_end == body_end_check, "invalid xrow length")
request.errno = band(status, IPROTO_ERRNO_MASK)
@@ -328,16 +332,27 @@ local function create_transport(host, port, user, password, callback,
local body_len = body_end - body_rpos
local wpos = buffer:alloc(body_len)
ffi.copy(wpos, body_rpos, body_len)
- request.response = tonumber(body_len)
- wakeup_client(request.client)
- return
+ if status == IPROTO_OK_KEY then
+ request.response = tonumber(body_len)
+ requests[id] = nil
+ wakeup_client(request.client)
+ elseif request.on_push then
+ assert(status == IPROTO_CHUNK_KEY)
+ request.on_push(tonumber(body_len))
+ end
+ else
+ -- Decode xrow.body[DATA] to Lua objects
+ body, body_end_check = decode(body_rpos)
+ assert(body_end == body_end_check, "invalid xrow length")
+ if status == IPROTO_OK_KEY then
+ request.response = body[IPROTO_DATA_KEY]
+ requests[id] = nil
+ wakeup_client(request.client)
+ elseif request.on_push then
+ assert(status == IPROTO_CHUNK_KEY)
+ request.on_push(body[IPROTO_DATA_KEY][1])
+ end
end
-
- -- Decode xrow.body[DATA] to Lua objects
- body, body_end_check = decode(body_rpos)
- assert(body_end == body_end_check, "invalid xrow length")
- request.response = body[IPROTO_DATA_KEY]
- wakeup_client(request.client)
end
local function new_request_id()
@@ -834,7 +849,8 @@ function remote_methods:_request(method, opts, ...)
timeout = deadline and max(0, deadline - fiber_clock())
end
err, res = perform_request(timeout, buffer, method,
- self.schema_version, ...)
+ opts and opts.on_push, self.schema_version,
+ ...)
if not err and buffer ~= nil then
return res -- the length of xrow.body
elseif not err then
@@ -864,7 +880,7 @@ function remote_methods:ping(opts)
timeout = deadline and max(0, deadline - fiber_clock())
or (opts and opts.timeout)
end
- local err = self._transport.perform_request(timeout, nil, 'ping',
+ local err = self._transport.perform_request(timeout, nil, 'ping', nil,
self.schema_version)
return not err or err == E_WRONG_SCHEMA_VERSION
end
@@ -1045,10 +1061,10 @@ function console_methods:eval(line, timeout)
end
if self.protocol == 'Binary' then
local loader = 'return require("console").eval(...)'
- err, res = pr(timeout, nil, 'eval', nil, loader, {line})
+ err, res = pr(timeout, nil, 'eval', nil, nil, loader, {line})
else
assert(self.protocol == 'Lua console')
- err, res = pr(timeout, nil, 'inject', nil, line..'$EOF$\n')
+ err, res = pr(timeout, nil, 'inject', nil, nil, line..'$EOF$\n')
end
if err then
box.error({code = err, reason = res})
diff --git a/src/box/lua/session.c b/src/box/lua/session.c
index 306271809..c3db93627 100644
--- a/src/box/lua/session.c
+++ b/src/box/lua/session.c
@@ -31,6 +31,7 @@
#include "session.h"
#include "lua/utils.h"
#include "lua/trigger.h"
+#include "lua/msgpack.h"
#include <lua.h>
#include <lauxlib.h>
@@ -43,6 +44,7 @@
#include "box/schema.h"
#include "box/port.h"
#include "box/lua/console.h"
+#include "small/obuf.h"
static const char *sessionlib_name = "box.session";
@@ -371,8 +373,11 @@ struct lua_push_port {
static const char *
lua_push_port_dump_plain(struct port *port, uint32_t *size);
+static int
+lua_push_port_dump_msgpack(struct port *port, struct obuf *obuf);
+
static const struct port_vtab lua_push_port_vtab = {
- .dump_msgpack = NULL,
+ .dump_msgpack = lua_push_port_dump_msgpack,
/*
* Dump_16 has no sense, since push appears since 1.10
* protocol.
@@ -403,6 +408,32 @@ lua_push_port_dump_plain(struct port *port, uint32_t *size)
return result;
}
+static void
+obuf_error_cb(void *ctx)
+{
+ *((int *)ctx) = -1;
+}
+
+static int
+lua_push_port_dump_msgpack(struct port *port, struct obuf *obuf)
+{
+ struct lua_push_port *lua_port = (struct lua_push_port *) port;
+ assert(lua_port->vtab == &lua_push_port_vtab);
+ struct mpstream stream;
+ int rc = 0;
+ /*
+ * Do not use luamp_error to allow a caller to clear the
+ * obuf, if it already has allocated something (for
+ * example, iproto push reserves memory for a header).
+ */
+ mpstream_init(&stream, obuf, obuf_reserve_cb, obuf_alloc_cb,
+ obuf_error_cb, &rc);
+ luamp_encode(lua_port->L, luaL_msgpack_default, &stream, 1);
+ if (rc == 0)
+ mpstream_flush(&stream);
+ return rc;
+}
+
/**
* Push a message using a protocol, depending on a session type.
* @param data Data to push, first argument on a stack.
diff --git a/src/box/xrow.c b/src/box/xrow.c
index f48525645..adb52deeb 100644
--- a/src/box/xrow.c
+++ b/src/box/xrow.c
@@ -373,6 +373,19 @@ iproto_reply_select(struct obuf *buf, struct obuf_svp *svp, uint64_t sync,
memcpy(pos + IPROTO_HEADER_LEN, &body, sizeof(body));
}
+void
+iproto_reply_chunk(struct obuf *buf, struct obuf_svp *svp, uint64_t sync,
+ uint32_t schema_version)
+{
+ char *pos = (char *) obuf_svp_to_ptr(buf, svp);
+ iproto_header_encode(pos, IPROTO_CHUNK, sync, schema_version,
+ obuf_size(buf) - svp->used - IPROTO_HEADER_LEN);
+ struct iproto_body_bin body = iproto_body_bin;
+ body.v_data_len = mp_bswap_u32(1);
+
+ memcpy(pos + IPROTO_HEADER_LEN, &body, sizeof(body));
+}
+
int
xrow_decode_dml(struct xrow_header *row, struct request *request,
uint64_t key_map)
diff --git a/src/box/xrow.h b/src/box/xrow.h
index d407d151b..7fe1debbf 100644
--- a/src/box/xrow.h
+++ b/src/box/xrow.h
@@ -390,6 +390,18 @@ int
iproto_reply_error(struct obuf *out, const struct error *e, uint64_t sync,
uint32_t schema_version);
+/**
+ * Write an IPROTO_CHUNK header from a specified position in a
+ * buffer.
+ * @param buf Buffer to write to.
+ * @param svp Position to write from.
+ * @param sync Request sync.
+ * @param schema_version Actual schema version.
+ */
+void
+iproto_reply_chunk(struct obuf *buf, struct obuf_svp *svp, uint64_t sync,
+ uint32_t schema_version);
+
/** Write error directly to a socket. */
void
iproto_write_error(int fd, const struct error *e, uint32_t schema_version,
diff --git a/src/fiber.h b/src/fiber.h
index 8231bba24..eb89c48cd 100644
--- a/src/fiber.h
+++ b/src/fiber.h
@@ -105,8 +105,13 @@ enum fiber_key {
/** User global privilege and authentication token */
FIBER_KEY_USER = 3,
FIBER_KEY_MSG = 4,
- /** Storage for lua stack */
+ /**
+ * The storage cell number 5 is shared between lua stack
+ * for fibers created from Lua, and IProto sync for fibers
+ * created to execute a binary request.
+ */
FIBER_KEY_LUA_STACK = 5,
+ FIBER_KEY_SYNC = 5,
FIBER_KEY_MAX = 6
};
@@ -610,6 +615,13 @@ fiber_get_key(struct fiber *fiber, enum fiber_key key)
return fiber->fls[key];
}
+static inline uint64_t
+fiber_sync(struct fiber *fiber)
+{
+ uint64_t *sync = (uint64_t *) fiber_get_key(fiber, FIBER_KEY_SYNC);
+ return sync != NULL ? *sync : 0;
+}
+
/**
* Finalizer callback
* \sa fiber_key_on_gc()
diff --git a/test/box/net.box.result b/test/box/net.box.result
index cf7b27f0b..93561dc64 100644
--- a/test/box/net.box.result
+++ b/test/box/net.box.result
@@ -28,7 +28,7 @@ function x_select(cn, space_id, index_id, iterator, offset, limit, key, opts)
return cn:_request('select', opts, space_id, index_id, iterator,
offset, limit, key)
end
-function x_fatal(cn) cn._transport.perform_request(nil, nil, 'inject', nil, '\x80') end
+function x_fatal(cn) cn._transport.perform_request(nil, nil, 'inject', nil, nil, '\x80') end
test_run:cmd("setopt delimiter ''");
---
...
@@ -2363,7 +2363,7 @@ c.space.test:delete{1}
--
-- Break a connection to test reconnect_after.
--
-_ = c._transport.perform_request(nil, nil, 'inject', nil, '\x80')
+_ = c._transport.perform_request(nil, nil, 'inject', nil, nil, '\x80')
---
...
c.state
diff --git a/test/box/net.box.test.lua b/test/box/net.box.test.lua
index 576b5cfea..b05e1f0be 100644
--- a/test/box/net.box.test.lua
+++ b/test/box/net.box.test.lua
@@ -11,7 +11,7 @@ function x_select(cn, space_id, index_id, iterator, offset, limit, key, opts)
return cn:_request('select', opts, space_id, index_id, iterator,
offset, limit, key)
end
-function x_fatal(cn) cn._transport.perform_request(nil, nil, 'inject', nil, '\x80') end
+function x_fatal(cn) cn._transport.perform_request(nil, nil, 'inject', nil, nil, '\x80') end
test_run:cmd("setopt delimiter ''");
LISTEN = require('uri').parse(box.cfg.listen)
@@ -965,7 +965,7 @@ c.space.test:delete{1}
--
-- Break a connection to test reconnect_after.
--
-_ = c._transport.perform_request(nil, nil, 'inject', nil, '\x80')
+_ = c._transport.perform_request(nil, nil, 'inject', nil, nil, '\x80')
c.state
while not c:is_connected() do fiber.sleep(0.01) end
c:ping()
diff --git a/test/box/push.result b/test/box/push.result
index 816f06e00..91e2981ed 100644
--- a/test/box/push.result
+++ b/test/box/push.result
@@ -1,5 +1,8 @@
+test_run = require('test_run').new()
+---
+...
--
--- gh-2677: box.session.push.
+-- gh-2677: box.session.push binary protocol tests.
--
--
-- Usage.
@@ -12,18 +15,34 @@ box.session.push(1, 2)
---
- error: 'Usage: box.session.push(data)'
...
-ok = nil
+fiber = require('fiber')
---
...
-err = nil
+messages = {}
+---
+...
+test_run:cmd("setopt delimiter ';'")
---
+- true
...
-function do_push() ok, err = box.session.push(1) end
+function on_push(message)
+ table.insert(messages, message)
+end;
---
...
---
--- Test binary protocol.
---
+function do_pushes()
+ for i = 1, 5 do
+ box.session.push(i)
+ fiber.sleep(0.01)
+ end
+ return 300
+end;
+---
+...
+test_run:cmd("setopt delimiter ''");
+---
+- true
+...
netbox = require('net.box')
---
...
@@ -37,27 +56,213 @@ c:ping()
---
- true
...
-c:call('do_push')
+c:call('do_pushes', {}, {on_push = on_push})
---
+- 300
...
-ok, err
+messages
+---
+- - 1
+ - 2
+ - 3
+ - 4
+ - 5
+...
+-- Add a little stress: many pushes with different syncs, from
+-- different fibers and DML/DQL requests.
+catchers = {}
+---
+...
+started = 0
+---
+...
+finished = 0
+---
+...
+s = box.schema.create_space('test', {format = {{'field1', 'integer'}}})
+---
+...
+pk = s:create_index('pk')
+---
+...
+c:reload_schema()
+---
+...
+test_run:cmd("setopt delimiter ';'")
+---
+- true
+...
+function dml_push_and_dml(key)
+ box.session.push('started dml')
+ s:replace{key}
+ box.session.push('continued dml')
+ s:replace{-key}
+ box.session.push('finished dml')
+ return key
+end;
+---
+...
+function do_pushes(val)
+ for i = 1, 5 do
+ box.session.push(i)
+ fiber.yield()
+ end
+ return val
+end;
+---
+...
+function push_catcher_f()
+ fiber.yield()
+ started = started + 1
+ local catcher = {messages = {}, retval = nil, is_dml = false}
+ catcher.retval = c:call('do_pushes', {started}, {on_push = function(message)
+ table.insert(catcher.messages, message)
+ end})
+ table.insert(catchers, catcher)
+ finished = finished + 1
+end;
+---
+...
+function dml_push_and_dml_f()
+ fiber.yield()
+ started = started + 1
+ local catcher = {messages = {}, retval = nil, is_dml = true}
+ catcher.retval = c:call('dml_push_and_dml', {started}, {on_push = function(message)
+ table.insert(catcher.messages, message)
+ end})
+ table.insert(catchers, catcher)
+ finished = finished + 1
+end;
+---
+...
+-- At first check that a pushed message can be ignored in a binary
+-- protocol too.
+c:call('do_pushes', {300});
+---
+- 300
+...
+-- Then do stress.
+for i = 1, 200 do
+ fiber.create(dml_push_and_dml_f)
+ fiber.create(push_catcher_f)
+end;
+---
+...
+while finished ~= 400 do fiber.sleep(0.1) end;
+---
+...
+for _, c in pairs(catchers) do
+ if c.is_dml then
+ assert(#c.messages == 3, 'dml sends 3 messages')
+ assert(c.messages[1] == 'started dml', 'started')
+ assert(c.messages[2] == 'continued dml', 'continued')
+ assert(c.messages[3] == 'finished dml', 'finished')
+ assert(s:get{c.retval}, 'inserted +')
+ assert(s:get{-c.retval}, 'inserted -')
+ else
+ assert(c.retval, 'something is returned')
+ assert(#c.messages == 5, 'non-dml sends 5 messages')
+ for k, v in pairs(c.messages) do
+ assert(k == v, 'with equal keys and values')
+ end
+ end
+end;
---
-- null
-- Session 'binary' does not support push()
+...
+test_run:cmd("setopt delimiter ''");
+---
+- true
+...
+#s:select{}
+---
+- 400
+...
+--
+-- Ok to push NULL.
+--
+function push_null() box.session.push(box.NULL) end
+---
+...
+messages = {}
+---
+...
+c:call('push_null', {}, {on_push = on_push})
+---
+...
+messages
+---
+- - null
+...
+--
+-- Test binary pushes.
+--
+ibuf = require('buffer').ibuf()
+---
+...
+msgpack = require('msgpack')
+---
+...
+messages = {}
+---
+...
+resp_len = c:call('do_pushes', {300}, {on_push = on_push, buffer = ibuf})
+---
+...
+resp_len
+---
+- 10
+...
+messages
+---
+- - 8
+ - 8
+ - 8
+ - 8
+ - 8
+...
+decoded = {}
+---
+...
+r = nil
+---
+...
+for i = 1, #messages do r, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) table.insert(decoded, r) end
+---
+...
+decoded
+---
+- - {48: [1]}
+ - {48: [2]}
+ - {48: [3]}
+ - {48: [4]}
+ - {48: [5]}
+...
+r, _ = msgpack.decode_unchecked(ibuf.rpos)
+---
+...
+r
+---
+- {48: [300]}
...
c:close()
---
...
+s:drop()
+---
+...
box.schema.user.revoke('guest', 'read,write,execute', 'universe')
---
...
--
-- Ensure can not push in background.
--
-fiber = require('fiber')
+ok = nil
+---
+...
+err = nil
---
...
-f = fiber.create(do_push)
+f = fiber.create(function() ok, err = box.session.push(100) end)
---
...
while f:status() ~= 'dead' do fiber.sleep(0.01) end
diff --git a/test/box/push.test.lua b/test/box/push.test.lua
index a59fe0a4c..e5bd3287f 100644
--- a/test/box/push.test.lua
+++ b/test/box/push.test.lua
@@ -1,5 +1,6 @@
+test_run = require('test_run').new()
--
--- gh-2677: box.session.push.
+-- gh-2677: box.session.push binary protocol tests.
--
--
@@ -8,28 +9,139 @@
box.session.push()
box.session.push(1, 2)
-ok = nil
-err = nil
-function do_push() ok, err = box.session.push(1) end
+fiber = require('fiber')
+messages = {}
+test_run:cmd("setopt delimiter ';'")
+function on_push(message)
+ table.insert(messages, message)
+end;
+
+function do_pushes()
+ for i = 1, 5 do
+ box.session.push(i)
+ fiber.sleep(0.01)
+ end
+ return 300
+end;
+test_run:cmd("setopt delimiter ''");
---
--- Test binary protocol.
---
netbox = require('net.box')
box.schema.user.grant('guest', 'read,write,execute', 'universe')
c = netbox.connect(box.cfg.listen)
c:ping()
-c:call('do_push')
-ok, err
+c:call('do_pushes', {}, {on_push = on_push})
+messages
+
+-- Add a little stress: many pushes with different syncs, from
+-- different fibers and DML/DQL requests.
+
+catchers = {}
+started = 0
+finished = 0
+s = box.schema.create_space('test', {format = {{'field1', 'integer'}}})
+pk = s:create_index('pk')
+c:reload_schema()
+test_run:cmd("setopt delimiter ';'")
+function dml_push_and_dml(key)
+ box.session.push('started dml')
+ s:replace{key}
+ box.session.push('continued dml')
+ s:replace{-key}
+ box.session.push('finished dml')
+ return key
+end;
+function do_pushes(val)
+ for i = 1, 5 do
+ box.session.push(i)
+ fiber.yield()
+ end
+ return val
+end;
+function push_catcher_f()
+ fiber.yield()
+ started = started + 1
+ local catcher = {messages = {}, retval = nil, is_dml = false}
+ catcher.retval = c:call('do_pushes', {started}, {on_push = function(message)
+ table.insert(catcher.messages, message)
+ end})
+ table.insert(catchers, catcher)
+ finished = finished + 1
+end;
+function dml_push_and_dml_f()
+ fiber.yield()
+ started = started + 1
+ local catcher = {messages = {}, retval = nil, is_dml = true}
+ catcher.retval = c:call('dml_push_and_dml', {started}, {on_push = function(message)
+ table.insert(catcher.messages, message)
+ end})
+ table.insert(catchers, catcher)
+ finished = finished + 1
+end;
+-- At first check that a pushed message can be ignored in a binary
+-- protocol too.
+c:call('do_pushes', {300});
+-- Then do stress.
+for i = 1, 200 do
+ fiber.create(dml_push_and_dml_f)
+ fiber.create(push_catcher_f)
+end;
+while finished ~= 400 do fiber.sleep(0.1) end;
+
+for _, c in pairs(catchers) do
+ if c.is_dml then
+ assert(#c.messages == 3, 'dml sends 3 messages')
+ assert(c.messages[1] == 'started dml', 'started')
+ assert(c.messages[2] == 'continued dml', 'continued')
+ assert(c.messages[3] == 'finished dml', 'finished')
+ assert(s:get{c.retval}, 'inserted +')
+ assert(s:get{-c.retval}, 'inserted -')
+ else
+ assert(c.retval, 'something is returned')
+ assert(#c.messages == 5, 'non-dml sends 5 messages')
+ for k, v in pairs(c.messages) do
+ assert(k == v, 'with equal keys and values')
+ end
+ end
+end;
+test_run:cmd("setopt delimiter ''");
+
+#s:select{}
+
+--
+-- Ok to push NULL.
+--
+function push_null() box.session.push(box.NULL) end
+messages = {}
+c:call('push_null', {}, {on_push = on_push})
+messages
+
+--
+-- Test binary pushes.
+--
+ibuf = require('buffer').ibuf()
+msgpack = require('msgpack')
+messages = {}
+resp_len = c:call('do_pushes', {300}, {on_push = on_push, buffer = ibuf})
+resp_len
+messages
+decoded = {}
+r = nil
+for i = 1, #messages do r, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) table.insert(decoded, r) end
+decoded
+r, _ = msgpack.decode_unchecked(ibuf.rpos)
+r
+
c:close()
+s:drop()
box.schema.user.revoke('guest', 'read,write,execute', 'universe')
--
-- Ensure can not push in background.
--
-fiber = require('fiber')
-f = fiber.create(do_push)
+ok = nil
+err = nil
+f = fiber.create(function() ok, err = box.session.push(100) end)
while f:status() ~= 'dead' do fiber.sleep(0.01) end
ok, err
--
2.15.1 (Apple Git-101)
^ permalink raw reply [flat|nested] 34+ messages in thread* Re: [tarantool-patches] [PATCH v2 10/10] session: introduce binary box.session.push
2018-04-20 13:24 ` [tarantool-patches] [PATCH v2 10/10] session: introduce binary box.session.push Vladislav Shpilevoy
@ 2018-05-10 19:50 ` Konstantin Osipov
2018-05-24 20:50 ` [tarantool-patches] " Vladislav Shpilevoy
0 siblings, 1 reply; 34+ messages in thread
From: Konstantin Osipov @ 2018-05-10 19:50 UTC (permalink / raw)
To: tarantool-patches; +Cc: vdavydov.dev
* Vladislav Shpilevoy <v.shpilevoy@tarantool.org> [18/04/20 16:25]:
> Box.session.push() allows to send a message to a client with no
> finishing a main request.
>
> Tarantool after this patch supports pushes over binary protocol.
>
> IProto message is encoded using a new header code - IPROTO_CHUNK.
> TX thread to notify IProto thread about new data in obuf sends
> a message 'push_msg'. IProto thread, got this message, notifies
> libev about new data, and then sends 'push_msg' back with
> updated write position. TX thread, received the message back,
> updates its version of a write position. If IProto would not send
> a write position, then TX would write to the same obuf again and
> again, because it can not know that IProto already flushed
> another obuf.
>
> To avoid multiple 'push_msg' in fly between IProto and TX, the
> only one 'push_msg' per connection is used. To deliver pushes,
> appeared when 'push_msg' was in fly, TX thread sets a flag every
> time when sees, that 'push_msg' is sent, and there is a new push.
> When 'push_msg' returns, it checks this flag, and if it is set,
> the IProto is notified again.
I don't see any reason for this restriction.
Any connection has two independent rotating output buffers
of infinite size. If you ever want to block a push message, you
should block it because both buffers are busy.
> +/**
> + * Message to notify IProto thread about new data in an output
> + * buffer. Struct iproto_msg is not used here, because push
> + * notification can be much more compact: it does not have
> + * request, ibuf, length, flags ...
> + */
> +struct iproto_push_msg {
> + struct cmsg base;
> + /**
> + * Before sending to IProto thread, the wpos is set to a
> + * current position in an output buffer. Before IProto
> + * returns the message to TX, it sets wpos to the last
> + * flushed position (works like iproto_msg.wpos).
> + */
> + struct iproto_wpos wpos;
> +};
> +
Looks good to me.
> + * Is_push_in_progress is set, when a push_msg is sent to
> + * IProto thread, and reset, when the message is returned
> + * to TX. If a new push sees, that a push_msg is already
> + * sent to IProto, then has_new_pushes is set. After push
> + * notification is returned to TX, it checks
> + * has_new_pushes. If it is set, then the notification is
> + * sent again. This ping-pong continues, until TX stopped
> + * pushing. It allows to
> + * 1) avoid multiple push_msg from one session in fly,
> + * 2) do not block push() until a previous push() is
> + * finished.
Please make it radically simpler, every push can create a new
message which has an independent life cycle. Messages can never
run one over each other, so you have nothing to worry about.
> @@ -1038,7 +1085,7 @@ tx_process_disconnect(struct cmsg *m)
> struct iproto_msg *msg = (struct iproto_msg *) m;
> struct iproto_connection *con = msg->connection;
> if (con->session) {
> - tx_fiber_init(con->session, 0);
> + tx_fiber_init(con->session, NULL);
Why do you need to make it more complex than it is now?
Every Lua procedure which makes a push takes a long-polling
reference to the connection already. Until this procedure
ends, you can't disconnect a connection.
> +static void
> +tx_accept_wpos(struct iproto_connection *con, const struct iproto_wpos *wpos)
> {
> - struct iproto_msg *msg = (struct iproto_msg *) m;
> - struct iproto_connection *con = msg->connection;
> -
> struct obuf *prev = &con->obuf[con->tx.p_obuf == con->obuf];
> - if (msg->wpos.obuf == con->tx.p_obuf) {
> + if (wpos->obuf == con->tx.p_obuf) {
> /*
> * We got a message advancing the buffer which
> * is being appended to. The previous buffer is
> @@ -1134,6 +1182,13 @@ tx_accept_msg(struct cmsg *m)
> */
> con->tx.p_obuf = prev;
> }
> +}
> +
> +static inline struct iproto_msg *
> +tx_accept_msg(struct cmsg *m)
> +{
> + struct iproto_msg *msg = (struct iproto_msg *) m;
> + tx_accept_wpos(msg->connection, &msg->wpos);
> return msg;
> }
This somehow looks half-baked, I don't know how yet.
> +c:call('push_null', {}, {on_push = on_push})
What happens if on_push handler is not set? Can I get the entire
data set in a result when all pushes are over?
Can I get a data set as an iterable and yield in the iterator
instead?
--
Konstantin Osipov, Moscow, Russia, +7 903 626 22 32
http://tarantool.io - www.twitter.com/kostja_osipov
^ permalink raw reply [flat|nested] 34+ messages in thread* Re: [tarantool-patches] Re: [PATCH v2 10/10] session: introduce binary box.session.push
2018-05-10 19:50 ` Konstantin Osipov
@ 2018-05-24 20:50 ` Vladislav Shpilevoy
0 siblings, 0 replies; 34+ messages in thread
From: Vladislav Shpilevoy @ 2018-05-24 20:50 UTC (permalink / raw)
To: tarantool-patches, Konstantin Osipov; +Cc: vdavydov.dev
On 10/05/2018 22:50, Konstantin Osipov wrote:
> * Vladislav Shpilevoy <v.shpilevoy@tarantool.org> [18/04/20 16:25]:
>> Box.session.push() allows to send a message to a client with no
>> finishing a main request.
>>
>> Tarantool after this patch supports pushes over binary protocol.
>>
>> IProto message is encoded using a new header code - IPROTO_CHUNK.
>> TX thread to notify IProto thread about new data in obuf sends
>> a message 'push_msg'. IProto thread, got this message, notifies
>> libev about new data, and then sends 'push_msg' back with
>> updated write position. TX thread, received the message back,
>> updates its version of a write position. If IProto would not send
>> a write position, then TX would write to the same obuf again and
>> again, because it can not know that IProto already flushed
>> another obuf.
>>
>> To avoid multiple 'push_msg' in fly between IProto and TX, the
>> only one 'push_msg' per connection is used. To deliver pushes,
>> appeared when 'push_msg' was in fly, TX thread sets a flag every
>> time when sees, that 'push_msg' is sent, and there is a new push.
>> When 'push_msg' returns, it checks this flag, and if it is set,
>> the IProto is notified again.
>
> I don't see any reason for this restriction.
> Any connection has two independent rotating output buffers
> of infinite size. If you ever want to block a push message, you
> should block it because both buffers are busy.
I do not block a user. We have discussed it verbally, but I
duplicate the explanation here for the record.
When a message is pushed, it is written to obuf, and either cmsg
is created or the flag is set, and the control is immediately
returned to the caller.
If the flag was not set on push, then at the end of event loop
iteration the cmsg with push notification is delivered to IProto thead,
where it trigger on_output event. Then the message is returned to TX.
If the flag was set on push, then push notification will be delivered to
IProto on the next event loop iteration.
>
>> + * Is_push_in_progress is set, when a push_msg is sent to
>> + * IProto thread, and reset, when the message is returned
>> + * to TX. If a new push sees, that a push_msg is already
>> + * sent to IProto, then has_new_pushes is set. After push
>> + * notification is returned to TX, it checks
>> + * has_new_pushes. If it is set, then the notification is
>> + * sent again. This ping-pong continues, until TX stopped
>> + * pushing. It allows to
>> + * 1) avoid multiple push_msg from one session in fly,
>> + * 2) do not block push() until a previous push() is
>> + * finished.
>
> Please make it radically simpler, every push can create a new
> message which has an independent life cycle. Messages can never
> run one over each other, so you have nothing to worry about.
Same. Discussed verbally and you agreed, that you misunderstood me.
This way is ok, and moreover, it can be reused to get rid of two-end
routes that now deliver iproto_msg to TX thread and are kept there
until the request is finished.
>
>> @@ -1038,7 +1085,7 @@ tx_process_disconnect(struct cmsg *m)
>> struct iproto_msg *msg = (struct iproto_msg *) m;
>> struct iproto_connection *con = msg->connection;
>> if (con->session) {
>> - tx_fiber_init(con->session, 0);
>> + tx_fiber_init(con->session, NULL);
>
> Why do you need to make it more complex than it is now?
>
> Every Lua procedure which makes a push takes a long-polling
> reference to the connection already. Until this procedure
> ends, you can't disconnect a connection.
I know, but I need sync of this request. Not the whole connection.
Session->sync is valid only until the next yield. After it can be
changed to sync of a new request. Here I reuse one of fiber local
storage fields, that it is never used in TX fiber pool fibers.
>
>> +static void
>> +tx_accept_wpos(struct iproto_connection *con, const struct iproto_wpos *wpos)
>> {
>> - struct iproto_msg *msg = (struct iproto_msg *) m;
>> - struct iproto_connection *con = msg->connection;
>> -
>> struct obuf *prev = &con->obuf[con->tx.p_obuf == con->obuf];
>> - if (msg->wpos.obuf == con->tx.p_obuf) {
>> + if (wpos->obuf == con->tx.p_obuf) {
>> /*
>> * We got a message advancing the buffer which
>> * is being appended to. The previous buffer is
>> @@ -1134,6 +1182,13 @@ tx_accept_msg(struct cmsg *m)
>> */
>> con->tx.p_obuf = prev;
>> }
>> +}
>> +
>> +static inline struct iproto_msg *
>> +tx_accept_msg(struct cmsg *m)
>> +{
>> + struct iproto_msg *msg = (struct iproto_msg *) m;
>> + tx_accept_wpos(msg->connection, &msg->wpos);
>> return msg;
>> }
>
> This somehow looks half-baked, I don't know how yet.
>
>> +c:call('push_null', {}, {on_push = on_push})
>
> What happens if on_push handler is not set? Can I get the entire
> data set in a result when all pushes are over?
>
> Can I get a data set as an iterable and yield in the iterator
> instead?
>
I have discussed it with community (if it can be called 'discussion' -
almost every one does not care about API). The accepted decision - make
on_push and is_async be mutually exclusive, and make future object be
iterable.
If no one is set, then pushed messages are ignored.
The iterable API I have pushed as a separate commit, that I sends to the
same thread.
Also verbally you said that we should name Kharon iproto push message, that
notifies IProto thread about updated obuf. I have done it. Look the complete
diff below:
==========================================================================
commit 7525c7db82d01b5ad77c50f0fb70a6cede8f16c2
Author: Vladislav Shpilevoy <v.shpilevoy@tarantool.org>
Date: Fri Apr 20 00:45:32 2018 +0300
session: introduce binary box.session.push
Box.session.push() allows to send a message to a client with no
finishing a main request. Tarantool after this patch supports
pushes over binary protocol.
IProto message is encoded using a new header code - IPROTO_CHUNK.
Push works as follows: a user calls box.session.push(message).
The message is encoded into currently active obuf in TX thread,
and then Kharon notifies IProto thread about new data.
Originally Kharon is the ferryman of Hades who carries souls of
the newly deceased across the rivers Styx and Acheron that
divided the world of the living from the world of the dead. In
Tarantool Kharon is a message and does the similar work. It
notifies IProto thread about new data in an output buffer
carrying pushed messages to IProto. Styx here is cpipe, and the
boat is cbus message.
One connection has single Kharon for all pushes. But Kharon can
not be in two places at the time. So once he got away from TX to
IProto, new messages can not send Kharon. They just set a special
flag. When Kharon is back to TX and sees the flag is set, he
immediately takes the road back to IProto.
Herewith a user is not blocked to write to obuf when Kharon is
busy. The user just updates obuf and set the flag if not set.
There is no waiting for Kharon arrival back.
Closes #2677
diff --git a/src/box/iproto.cc b/src/box/iproto.cc
index 736a699a2..f9508ed7d 100644
--- a/src/box/iproto.cc
+++ b/src/box/iproto.cc
@@ -66,6 +66,44 @@
enum { IPROTO_SALT_SIZE = 32 };
+/**
+ * This structure represents a position in the output.
+ * Since we use rotating buffers to recycle memory,
+ * it includes not only a position in obuf, but also
+ * a pointer to obuf the position is for.
+ */
+struct iproto_wpos {
+ struct obuf *obuf;
+ struct obuf_svp svp;
+};
+
+static void
+iproto_wpos_create(struct iproto_wpos *wpos, struct obuf *out)
+{
+ wpos->obuf = out;
+ wpos->svp = obuf_create_svp(out);
+}
+
+/**
+ * Kharon is the ferryman of Hades who carries souls of the newly
+ * deceased across the rivers Styx and Acheron that divided the
+ * world of the living from the world of the dead.
+ * Here Kharon is a message and does the similar work. It notifies
+ * IProto thread about new data in an output buffer carrying
+ * pushed messages to IProto. Styx here is cpipe, and the boat is
+ * cbus message.
+ */
+struct kharon {
+ struct cmsg base;
+ /**
+ * Before sending to IProto thread, the wpos is set to a
+ * current position in an output buffer. Before IProto
+ * returns the message to TX, it sets wpos to the last
+ * flushed position (works like iproto_msg.wpos).
+ */
+ struct iproto_wpos wpos;
+};
+
/**
* Network readahead. A signed integer to avoid
* automatic type coercion to an unsigned type.
@@ -113,24 +151,6 @@ iproto_reset_input(struct ibuf *ibuf)
}
}
-/**
- * This structure represents a position in the output.
- * Since we use rotating buffers to recycle memory,
- * it includes not only a position in obuf, but also
- * a pointer to obuf the position is for.
- */
-struct iproto_wpos {
- struct obuf *obuf;
- struct obuf_svp svp;
-};
-
-static void
-iproto_wpos_create(struct iproto_wpos *wpos, struct obuf *out)
-{
- wpos->obuf = out;
- wpos->svp = obuf_create_svp(out);
-}
-
/* {{{ iproto_msg - declaration */
/**
@@ -365,6 +385,48 @@ struct iproto_connection
/** Pointer to the current output buffer. */
struct obuf *p_obuf;
} tx;
+ /**
+ * Kharon serves to notify IProto about new data in obuf
+ * when TX thread is initiator of the transfer. It is
+ * possible for example on box.session.push.
+ *
+ * But Kharon can not be in two places at the time. So
+ * once he got away from TX to IProto, new messages can
+ * not send Kharon. They just set a special flag:
+ * has_new_pushes. When Kharon is back to TX and sees the
+ * flag is set, he immediately takes the road back to
+ * IProto.
+ *
+ * Herewith a user is not blocked to write to obuf when
+ * Kharon is busy. The user just updates obuf and set the
+ * flag if not set. There is no waiting for Kharon arrival
+ * back.
+ *
+ * Kharon allows to
+ * 1) avoid multiple cbus messages from one session in
+ * fly,
+ * 2) do not block a new push() until a previous push() is
+ * finished.
+ *
+ * IProto TX
+ * -------------------------------------------------------
+ * + [push message]
+ * start socket <--- notification ----
+ * write
+ * + [push message]
+ * + [push message]
+ * ...
+ * end socket
+ * write ----------------> check for new
+ * pushes - found
+ * <--- notification ---
+ * ....
+ */
+ bool has_new_pushes;
+ /** True if Kharon is on the way. */
+ bool is_kharon_on_road;
+ /** Push notification for IProto thread. */
+ struct kharon kharon;
/** Authentication salt. */
char salt[IPROTO_SALT_SIZE];
};
@@ -882,6 +944,7 @@ iproto_connection_new(int fd)
diag_set(OutOfMemory, sizeof(*con), "mempool_alloc", "con");
return NULL;
}
+ memset(con, 0, sizeof(*con));
con->input.data = con->output.data = con;
con->loop = loop();
ev_io_init(&con->input, iproto_connection_on_input, fd, EV_READ);
@@ -894,9 +957,6 @@ iproto_connection_new(int fd)
con->tx.p_obuf = &con->obuf[0];
iproto_wpos_create(&con->wpos, con->tx.p_obuf);
iproto_wpos_create(&con->wend, con->tx.p_obuf);
- con->parse_size = 0;
- con->long_poll_requests = 0;
- con->session = NULL;
rlist_create(&con->in_stop_list);
/* It may be very awkward to allocate at close. */
cmsg_init(&con->disconnect, disconnect_route);
@@ -1084,9 +1144,9 @@ error:
}
static void
-tx_fiber_init(struct session *session, uint64_t sync)
+tx_fiber_init(struct session *session, uint64_t *sync)
{
- session->meta.sync = sync;
+ session->meta.sync = sync != NULL ? *sync : 0;
/*
* We do not cleanup fiber keys at the end of each request.
* This does not lead to privilege escalation as long as
@@ -1099,6 +1159,7 @@ tx_fiber_init(struct session *session, uint64_t sync)
*/
fiber_set_session(fiber(), session);
fiber_set_user(fiber(), &session->credentials);
+ fiber_set_key(fiber(), FIBER_KEY_SYNC, (void *) sync);
}
/**
@@ -1112,7 +1173,7 @@ tx_process_disconnect(struct cmsg *m)
struct iproto_connection *con =
container_of(m, struct iproto_connection, disconnect);
if (con->session) {
- tx_fiber_init(con->session, 0);
+ tx_fiber_init(con->session, NULL);
if (! rlist_empty(&session_on_disconnect))
session_run_on_disconnect_triggers(con->session);
session_destroy(con->session);
@@ -1185,15 +1246,16 @@ tx_discard_input(struct iproto_msg *msg)
* not, the empty buffer is selected.
* - if neither of the buffers are empty, the function
* does not rotate the buffer.
+ *
+ * @param con IProto connection.
+ * @param wpos Last flushed write position, received from IProto
+ * thread.
*/
-static struct iproto_msg *
-tx_accept_msg(struct cmsg *m)
+static void
+tx_accept_wpos(struct iproto_connection *con, const struct iproto_wpos *wpos)
{
- struct iproto_msg *msg = (struct iproto_msg *) m;
- struct iproto_connection *con = msg->connection;
-
struct obuf *prev = &con->obuf[con->tx.p_obuf == con->obuf];
- if (msg->wpos.obuf == con->tx.p_obuf) {
+ if (wpos->obuf == con->tx.p_obuf) {
/*
* We got a message advancing the buffer which
* is being appended to. The previous buffer is
@@ -1211,6 +1273,14 @@ tx_accept_msg(struct cmsg *m)
*/
con->tx.p_obuf = prev;
}
+}
+
+static inline struct iproto_msg *
+tx_accept_msg(struct cmsg *m)
+{
+ struct iproto_msg *msg = (struct iproto_msg *) m;
+ tx_accept_wpos(msg->connection, &msg->wpos);
+ tx_fiber_init(msg->connection->session, &msg->header.sync);
return msg;
}
@@ -1255,8 +1325,6 @@ static void
tx_process1(struct cmsg *m)
{
struct iproto_msg *msg = tx_accept_msg(m);
-
- tx_fiber_init(msg->connection->session, msg->header.sync);
if (tx_check_schema(msg->header.schema_version))
goto error;
@@ -1289,9 +1357,6 @@ tx_process_select(struct cmsg *m)
int count;
int rc;
struct request *req = &msg->dml;
-
- tx_fiber_init(msg->connection->session, msg->header.sync);
-
if (tx_check_schema(msg->header.schema_version))
goto error;
@@ -1339,9 +1404,6 @@ static void
tx_process_call(struct cmsg *m)
{
struct iproto_msg *msg = tx_accept_msg(m);
-
- tx_fiber_init(msg->connection->session, msg->header.sync);
-
if (tx_check_schema(msg->header.schema_version))
goto error;
@@ -1423,9 +1485,6 @@ tx_process_misc(struct cmsg *m)
struct iproto_msg *msg = tx_accept_msg(m);
struct iproto_connection *con = msg->connection;
struct obuf *out = con->tx.p_obuf;
-
- tx_fiber_init(con->session, msg->header.sync);
-
if (tx_check_schema(msg->header.schema_version))
goto error;
@@ -1463,9 +1522,6 @@ tx_process_join_subscribe(struct cmsg *m)
{
struct iproto_msg *msg = tx_accept_msg(m);
struct iproto_connection *con = msg->connection;
-
- tx_fiber_init(con->session, msg->header.sync);
-
try {
switch (msg->header.type) {
case IPROTO_JOIN:
@@ -1582,7 +1638,7 @@ tx_process_connect(struct cmsg *m)
if (con->session == NULL)
diag_raise();
con->session->meta.connection = con;
- tx_fiber_init(con->session, 0);
+ tx_fiber_init(con->session, NULL);
static __thread char greeting[IPROTO_GREETING_SIZE];
/* TODO: dirty read from tx thread */
struct tt_uuid uuid = INSTANCE_UUID;
@@ -1741,6 +1797,99 @@ iproto_session_sync(struct session *session)
return session->meta.sync;
}
+/** {{{ IPROTO_PUSH implementation. */
+
+/**
+ * Send to IProto thread a notification about new pushes.
+ * @param conn IProto connection.
+ */
+static void
+tx_begin_push(struct iproto_connection *conn);
+
+/**
+ * Kharon reached the dead world (IProto). He schedules an event
+ * to flush new obuf data up to brought wpos.
+ * @param m Kharon.
+ */
+static void
+kharon_go_dead(struct cmsg *m)
+{
+ struct kharon *kharon = (struct kharon *) m;
+ struct iproto_connection *conn =
+ container_of(kharon, struct iproto_connection, kharon);
+ conn->wend = kharon->wpos;
+ kharon->wpos = conn->wpos;
+ if (evio_has_fd(&conn->output) && !ev_is_active(&conn->output))
+ ev_feed_event(conn->loop, &conn->output, EV_WRITE);
+}
+
+/**
+ * Kharon is in living world (TX) back from dead one (IProto). He
+ * checks if new push messages appeared during its trip and goes
+ * to IProto again if needed.
+ * @param m Kharon.
+ */
+static void
+kharon_go_alive(struct cmsg *m)
+{
+ struct kharon *kharon = (struct kharon *) m;
+ struct iproto_connection *conn =
+ container_of(kharon, struct iproto_connection, kharon);
+ tx_accept_wpos(conn, &kharon->wpos);
+ conn->is_kharon_on_road = false;
+ if (conn->has_new_pushes)
+ tx_begin_push(conn);
+}
+
+static const struct cmsg_hop push_route[] = {
+ { kharon_go_dead, &tx_pipe },
+ { kharon_go_alive, NULL }
+};
+
+static void
+tx_begin_push(struct iproto_connection *conn)
+{
+ assert(! conn->is_kharon_on_road);
+ cmsg_init((struct cmsg *) &conn->kharon, push_route);
+ iproto_wpos_create(&conn->kharon.wpos, conn->tx.p_obuf);
+ conn->has_new_pushes = false;
+ conn->is_kharon_on_road = true;
+ cpipe_push(&net_pipe, (struct cmsg *) &conn->kharon);
+}
+
+/**
+ * Push a message from @a port to a remote client.
+ * @param session IProto session.
+ * @param port Port with data to send.
+ *
+ * @retval -1 Memory error.
+ * @retval 0 Success, a message is wrote to an output buffer. But
+ * it is not guaranteed, that it will be sent
+ * successfully.
+ */
+static int
+iproto_session_push(struct session *session, struct port *port)
+{
+ struct iproto_connection *con =
+ (struct iproto_connection *) session->meta.connection;
+ struct obuf_svp svp;
+ if (iproto_prepare_select(con->tx.p_obuf, &svp) != 0)
+ return -1;
+ if (port_dump_msgpack(port, con->tx.p_obuf) != 0) {
+ obuf_rollback_to_svp(con->tx.p_obuf, &svp);
+ return -1;
+ }
+ iproto_reply_chunk(con->tx.p_obuf, &svp, fiber_sync(fiber()),
+ ::schema_version);
+ if (! con->is_kharon_on_road)
+ tx_begin_push(con);
+ else
+ con->has_new_pushes = true;
+ return 0;
+}
+
+/** }}} */
+
/** Initialize the iproto subsystem and start network io thread */
void
iproto_init()
@@ -1754,7 +1903,7 @@ iproto_init()
cpipe_create(&net_pipe, "net");
cpipe_set_max_input(&net_pipe, iproto_msg_max / 2);
struct session_vtab iproto_session_vtab = {
- /* .push = */ generic_session_push,
+ /* .push = */ iproto_session_push,
/* .fd = */ iproto_session_fd,
/* .sync = */ iproto_session_sync,
};
diff --git a/src/box/iproto_constants.h b/src/box/iproto_constants.h
index 46d47719b..dd50bf874 100644
--- a/src/box/iproto_constants.h
+++ b/src/box/iproto_constants.h
@@ -164,6 +164,9 @@ enum iproto_type {
/** Vinyl row index stored in .run file */
VY_RUN_ROW_INDEX = 102,
+ /** Non-final response type. */
+ IPROTO_CHUNK = 128,
+
/**
* Error codes = (IPROTO_TYPE_ERROR | ER_XXX from errcode.h)
*/
diff --git a/src/box/lua/net_box.lua b/src/box/lua/net_box.lua
index 8fcaf89e8..fe113e740 100644
--- a/src/box/lua/net_box.lua
+++ b/src/box/lua/net_box.lua
@@ -39,6 +39,8 @@ local IPROTO_SCHEMA_VERSION_KEY = 0x05
local IPROTO_DATA_KEY = 0x30
local IPROTO_ERROR_KEY = 0x31
local IPROTO_GREETING_SIZE = 128
+local IPROTO_CHUNK_KEY = 128
+local IPROTO_OK_KEY = 0
-- select errors from box.error
local E_UNKNOWN = box.error.UNKNOWN
@@ -74,6 +76,10 @@ local function decode_count(raw_data)
local response, raw_end = decode(raw_data)
return response[IPROTO_DATA_KEY][1], raw_end
end
+local function decode_push(raw_data)
+ local response, raw_end = decode(raw_data)
+ return response[IPROTO_DATA_KEY][1], raw_end
+end
local method_encoder = {
ping = internal.encode_ping,
@@ -114,6 +120,7 @@ local method_decoder = {
max = decode_get,
count = decode_count,
inject = decode_data,
+ push = decode_push,
}
local function next_id(id) return band(id + 1, 0x7FFFFFFF) end
@@ -149,6 +156,12 @@ local function establish_connection(host, port, timeout)
return s, greeting
end
+--
+-- Default action on push during a synchronous request -
+-- ignore.
+--
+local function on_push_sync_default(...) end
+
--
-- Basically, *transport* is a TCP connection speaking one of
-- Tarantool network protocols. This is a low-level interface.
@@ -397,7 +410,8 @@ local function create_transport(host, port, user, password, callback,
-- @retval nil, error Error occured.
-- @retval not nil Future object.
--
- local function perform_async_request(buffer, method, ...)
+ local function perform_async_request(buffer, method, on_push, on_push_ctx,
+ ...)
if state ~= 'active' and state ~= 'fetch_schema' then
return nil, box.error.new({code = last_errno or E_NO_CONNECTION,
reason = last_error})
@@ -410,14 +424,17 @@ local function create_transport(host, port, user, password, callback,
local id = next_request_id
method_encoder[method](send_buf, id, ...)
next_request_id = next_id(id)
- -- Request has maximum 6 members:
- -- method, buffer, id, cond, errno, response.
- local request = setmetatable(table_new(0, 6), request_mt)
+ -- Request in most cases has maximum 8 members:
+ -- method, buffer, id, cond, errno, response, on_push,
+ -- on_push_ctx.
+ local request = setmetatable(table_new(0, 8), request_mt)
request.method = method
request.buffer = buffer
request.id = id
request.cond = fiber.cond()
requests[id] = request
+ request.on_push = on_push
+ request.on_push_ctx = on_push_ctx
return request
end
@@ -426,9 +443,10 @@ local function create_transport(host, port, user, password, callback,
-- @retval nil, error Error occured.
-- @retval not nil Response object.
--
- local function perform_request(timeout, buffer, method, ...)
+ local function perform_request(timeout, buffer, method, on_push,
+ on_push_ctx, ...)
local request, err =
- perform_async_request(buffer, method, ...)
+ perform_async_request(buffer, method, on_push, on_push_ctx, ...)
if not request then
return nil, err
end
@@ -441,13 +459,13 @@ local function create_transport(host, port, user, password, callback,
if request == nil then -- nobody is waiting for the response
return
end
- requests[id] = nil
- request.id = nil
local status = hdr[IPROTO_STATUS_KEY]
local body, body_end_check
- if status ~= 0 then
+ if status > IPROTO_CHUNK_KEY then
-- Handle errors
+ requests[id] = nil
+ request.id = nil
body, body_end_check = decode(body_rpos)
assert(body_end == body_end_check, "invalid xrow length")
request.errno = band(status, IPROTO_ERRNO_MASK)
@@ -462,16 +480,33 @@ local function create_transport(host, port, user, password, callback,
local body_len = body_end - body_rpos
local wpos = buffer:alloc(body_len)
ffi.copy(wpos, body_rpos, body_len)
- request.response = tonumber(body_len)
+ body_len = tonumber(body_len)
+ if status == IPROTO_OK_KEY then
+ request.response = body_len
+ requests[id] = nil
+ request.id = nil
+ else
+ request.on_push(request.on_push_ctx, body_len)
+ end
request.cond:broadcast()
return
end
- -- Decode xrow.body[DATA] to Lua objects
local real_end
- request.response, real_end, request.errno =
- method_decoder[request.method](body_rpos, body_end)
- assert(real_end == body_end, "invalid body length")
+ -- Decode xrow.body[DATA] to Lua objects
+ if status == IPROTO_OK_KEY then
+ request.response, real_end, request.errno =
+ method_decoder[request.method](body_rpos, body_end)
+ assert(real_end == body_end, "invalid body length")
+ requests[id] = nil
+ request.id = nil
+ else
+ local msg
+ msg, real_end, request.errno =
+ method_decoder.push(body_rpos, body_end)
+ assert(real_end == body_end, "invalid body length")
+ request.on_push(request.on_push_ctx, msg)
+ end
request.cond:broadcast()
end
@@ -947,25 +982,40 @@ end
function remote_methods:_request(method, opts, ...)
local transport = self._transport
- local buffer = opts and opts.buffer
- if opts and opts.is_async then
- return transport.perform_async_request(buffer, method, ...)
- end
- local deadline
- if opts and opts.timeout then
- -- conn.space:request(, { timeout = timeout })
- deadline = fiber_clock() + opts.timeout
+ local on_push, on_push_ctx, buffer, deadline
+ -- Extract options, set defaults, check if the request is
+ -- async.
+ if opts then
+ buffer = opts.buffer
+ if opts.is_async then
+ if opts.on_push or opts.on_push_ctx then
+ error('To handle pushes in an async request use future:pairs()')
+ end
+ return transport.perform_async_request(buffer, method, table.insert,
+ {}, ...)
+ end
+ if opts.timeout then
+ -- conn.space:request(, { timeout = timeout })
+ deadline = fiber_clock() + opts.timeout
+ else
+ -- conn:timeout(timeout).space:request()
+ -- @deprecated since 1.7.4
+ deadline = self._deadlines[fiber_self()]
+ end
+ on_push = opts.on_push or on_push_sync_default
+ on_push_ctx = opts.on_push_ctx
else
- -- conn:timeout(timeout).space:request()
- -- @deprecated since 1.7.4
deadline = self._deadlines[fiber_self()]
+ on_push = on_push_sync_default
end
+ -- Execute synchronous request.
local timeout = deadline and max(0, deadline - fiber_clock())
if self.state ~= 'active' then
transport.wait_state('active', timeout)
timeout = deadline and max(0, deadline - fiber_clock())
end
- local res, err = transport.perform_request(timeout, buffer, method, ...)
+ local res, err = transport.perform_request(timeout, buffer, method,
+ on_push, on_push_ctx, ...)
if err then
box.error(err)
end
@@ -1159,10 +1209,10 @@ function console_methods:eval(line, timeout)
end
if self.protocol == 'Binary' then
local loader = 'return require("console").eval(...)'
- res, err = pr(timeout, nil, 'eval', loader, {line})
+ res, err = pr(timeout, nil, 'eval', nil, nil, loader, {line})
else
assert(self.protocol == 'Lua console')
- res, err = pr(timeout, nil, 'inject', line..'$EOF$\n')
+ res, err = pr(timeout, nil, 'inject', nil, nil, line..'$EOF$\n')
end
if err then
box.error(err)
diff --git a/src/box/lua/session.c b/src/box/lua/session.c
index 306271809..c3db93627 100644
--- a/src/box/lua/session.c
+++ b/src/box/lua/session.c
@@ -31,6 +31,7 @@
#include "session.h"
#include "lua/utils.h"
#include "lua/trigger.h"
+#include "lua/msgpack.h"
#include <lua.h>
#include <lauxlib.h>
@@ -43,6 +44,7 @@
#include "box/schema.h"
#include "box/port.h"
#include "box/lua/console.h"
+#include "small/obuf.h"
static const char *sessionlib_name = "box.session";
@@ -371,8 +373,11 @@ struct lua_push_port {
static const char *
lua_push_port_dump_plain(struct port *port, uint32_t *size);
+static int
+lua_push_port_dump_msgpack(struct port *port, struct obuf *obuf);
+
static const struct port_vtab lua_push_port_vtab = {
- .dump_msgpack = NULL,
+ .dump_msgpack = lua_push_port_dump_msgpack,
/*
* Dump_16 has no sense, since push appears since 1.10
* protocol.
@@ -403,6 +408,32 @@ lua_push_port_dump_plain(struct port *port, uint32_t *size)
return result;
}
+static void
+obuf_error_cb(void *ctx)
+{
+ *((int *)ctx) = -1;
+}
+
+static int
+lua_push_port_dump_msgpack(struct port *port, struct obuf *obuf)
+{
+ struct lua_push_port *lua_port = (struct lua_push_port *) port;
+ assert(lua_port->vtab == &lua_push_port_vtab);
+ struct mpstream stream;
+ int rc = 0;
+ /*
+ * Do not use luamp_error to allow a caller to clear the
+ * obuf, if it already has allocated something (for
+ * example, iproto push reserves memory for a header).
+ */
+ mpstream_init(&stream, obuf, obuf_reserve_cb, obuf_alloc_cb,
+ obuf_error_cb, &rc);
+ luamp_encode(lua_port->L, luaL_msgpack_default, &stream, 1);
+ if (rc == 0)
+ mpstream_flush(&stream);
+ return rc;
+}
+
/**
* Push a message using a protocol, depending on a session type.
* @param data Data to push, first argument on a stack.
diff --git a/src/box/xrow.c b/src/box/xrow.c
index b3f81a86f..64f490995 100644
--- a/src/box/xrow.c
+++ b/src/box/xrow.c
@@ -377,6 +377,19 @@ iproto_reply_select(struct obuf *buf, struct obuf_svp *svp, uint64_t sync,
memcpy(pos + IPROTO_HEADER_LEN, &body, sizeof(body));
}
+void
+iproto_reply_chunk(struct obuf *buf, struct obuf_svp *svp, uint64_t sync,
+ uint32_t schema_version)
+{
+ char *pos = (char *) obuf_svp_to_ptr(buf, svp);
+ iproto_header_encode(pos, IPROTO_CHUNK, sync, schema_version,
+ obuf_size(buf) - svp->used - IPROTO_HEADER_LEN);
+ struct iproto_body_bin body = iproto_body_bin;
+ body.v_data_len = mp_bswap_u32(1);
+
+ memcpy(pos + IPROTO_HEADER_LEN, &body, sizeof(body));
+}
+
int
xrow_decode_dml(struct xrow_header *row, struct request *request,
uint64_t key_map)
diff --git a/src/box/xrow.h b/src/box/xrow.h
index b10bf26d5..1bb5f103b 100644
--- a/src/box/xrow.h
+++ b/src/box/xrow.h
@@ -411,6 +411,18 @@ int
iproto_reply_error(struct obuf *out, const struct error *e, uint64_t sync,
uint32_t schema_version);
+/**
+ * Write an IPROTO_CHUNK header from a specified position in a
+ * buffer.
+ * @param buf Buffer to write to.
+ * @param svp Position to write from.
+ * @param sync Request sync.
+ * @param schema_version Actual schema version.
+ */
+void
+iproto_reply_chunk(struct obuf *buf, struct obuf_svp *svp, uint64_t sync,
+ uint32_t schema_version);
+
/** Write error directly to a socket. */
void
iproto_write_error(int fd, const struct error *e, uint32_t schema_version,
diff --git a/src/fiber.h b/src/fiber.h
index 8231bba24..eb89c48cd 100644
--- a/src/fiber.h
+++ b/src/fiber.h
@@ -105,8 +105,13 @@ enum fiber_key {
/** User global privilege and authentication token */
FIBER_KEY_USER = 3,
FIBER_KEY_MSG = 4,
- /** Storage for lua stack */
+ /**
+ * The storage cell number 5 is shared between lua stack
+ * for fibers created from Lua, and IProto sync for fibers
+ * created to execute a binary request.
+ */
FIBER_KEY_LUA_STACK = 5,
+ FIBER_KEY_SYNC = 5,
FIBER_KEY_MAX = 6
};
@@ -610,6 +615,13 @@ fiber_get_key(struct fiber *fiber, enum fiber_key key)
return fiber->fls[key];
}
+static inline uint64_t
+fiber_sync(struct fiber *fiber)
+{
+ uint64_t *sync = (uint64_t *) fiber_get_key(fiber, FIBER_KEY_SYNC);
+ return sync != NULL ? *sync : 0;
+}
+
/**
* Finalizer callback
* \sa fiber_key_on_gc()
diff --git a/test/box/net.box.result b/test/box/net.box.result
index 1674c27e1..1a9758cd2 100644
--- a/test/box/net.box.result
+++ b/test/box/net.box.result
@@ -28,7 +28,7 @@ function x_select(cn, space_id, index_id, iterator, offset, limit, key, opts)
return cn:_request('select', opts, space_id, index_id, iterator,
offset, limit, key)
end
-function x_fatal(cn) cn._transport.perform_request(nil, nil, 'inject', '\x80') end
+function x_fatal(cn) cn._transport.perform_request(nil, nil, 'inject', nil, nil, '\x80') end
test_run:cmd("setopt delimiter ''");
---
...
@@ -2365,7 +2365,7 @@ c.space.test:delete{1}
--
-- Break a connection to test reconnect_after.
--
-_ = c._transport.perform_request(nil, nil, 'inject', '\x80')
+_ = c._transport.perform_request(nil, nil, 'inject', nil, nil, '\x80')
---
...
c.state
@@ -2939,7 +2939,7 @@ c = net:connect(box.cfg.listen, {reconnect_after = 0.01})
future = c:call('long_function', {1, 2, 3}, {is_async = true})
---
...
-_ = c._transport.perform_request(nil, nil, 'inject', '\x80')
+_ = c._transport.perform_request(nil, nil, 'inject', nil, nil, '\x80')
---
...
while not c:is_connected() do fiber.sleep(0.01) end
@@ -3053,7 +3053,7 @@ c:ping()
-- new attempts to read any data - the connection is closed
-- already.
--
-f = fiber.create(c._transport.perform_request, nil, nil, 'call_17', 'long', {}) c._transport.perform_request(nil, nil, 'inject', '\x80')
+f = fiber.create(c._transport.perform_request, nil, nil, 'call_17', nil, nil, 'long', {}) c._transport.perform_request(nil, nil, 'inject', nil, nil, '\x80')
---
...
while f:status() ~= 'dead' do fiber.sleep(0.01) end
diff --git a/test/box/net.box.test.lua b/test/box/net.box.test.lua
index c34616aec..6a36c812d 100644
--- a/test/box/net.box.test.lua
+++ b/test/box/net.box.test.lua
@@ -11,7 +11,7 @@ function x_select(cn, space_id, index_id, iterator, offset, limit, key, opts)
return cn:_request('select', opts, space_id, index_id, iterator,
offset, limit, key)
end
-function x_fatal(cn) cn._transport.perform_request(nil, nil, 'inject', '\x80') end
+function x_fatal(cn) cn._transport.perform_request(nil, nil, 'inject', nil, nil, '\x80') end
test_run:cmd("setopt delimiter ''");
LISTEN = require('uri').parse(box.cfg.listen)
@@ -965,7 +965,7 @@ c.space.test:delete{1}
--
-- Break a connection to test reconnect_after.
--
-_ = c._transport.perform_request(nil, nil, 'inject', '\x80')
+_ = c._transport.perform_request(nil, nil, 'inject', nil, nil, '\x80')
c.state
while not c:is_connected() do fiber.sleep(0.01) end
c:ping()
@@ -1173,7 +1173,7 @@ finalize_long()
--
c = net:connect(box.cfg.listen, {reconnect_after = 0.01})
future = c:call('long_function', {1, 2, 3}, {is_async = true})
-_ = c._transport.perform_request(nil, nil, 'inject', '\x80')
+_ = c._transport.perform_request(nil, nil, 'inject', nil, nil, '\x80')
while not c:is_connected() do fiber.sleep(0.01) end
finalize_long()
future:wait_result(100)
@@ -1222,7 +1222,7 @@ c:ping()
-- new attempts to read any data - the connection is closed
-- already.
--
-f = fiber.create(c._transport.perform_request, nil, nil, 'call_17', 'long', {}) c._transport.perform_request(nil, nil, 'inject', '\x80')
+f = fiber.create(c._transport.perform_request, nil, nil, 'call_17', nil, nil, 'long', {}) c._transport.perform_request(nil, nil, 'inject', nil, nil, '\x80')
while f:status() ~= 'dead' do fiber.sleep(0.01) end
c:close()
diff --git a/test/box/push.result b/test/box/push.result
index 816f06e00..04cdc474b 100644
--- a/test/box/push.result
+++ b/test/box/push.result
@@ -1,5 +1,8 @@
+test_run = require('test_run').new()
+---
+...
--
--- gh-2677: box.session.push.
+-- gh-2677: box.session.push binary protocol tests.
--
--
-- Usage.
@@ -12,18 +15,29 @@ box.session.push(1, 2)
---
- error: 'Usage: box.session.push(data)'
...
-ok = nil
+fiber = require('fiber')
---
...
-err = nil
+messages = {}
+---
+...
+test_run:cmd("setopt delimiter ';'")
---
+- true
...
-function do_push() ok, err = box.session.push(1) end
+function do_pushes()
+ for i = 1, 5 do
+ box.session.push(i)
+ fiber.sleep(0.01)
+ end
+ return 300
+end;
---
...
---
--- Test binary protocol.
---
+test_run:cmd("setopt delimiter ''");
+---
+- true
+...
netbox = require('net.box')
---
...
@@ -37,27 +51,213 @@ c:ping()
---
- true
...
-c:call('do_push')
+c:call('do_pushes', {}, {on_push = table.insert, on_push_ctx = messages})
---
+- 300
...
-ok, err
+messages
+---
+- - 1
+ - 2
+ - 3
+ - 4
+ - 5
+...
+-- Add a little stress: many pushes with different syncs, from
+-- different fibers and DML/DQL requests.
+catchers = {}
+---
+...
+started = 0
+---
+...
+finished = 0
+---
+...
+s = box.schema.create_space('test', {format = {{'field1', 'integer'}}})
+---
+...
+pk = s:create_index('pk')
+---
+...
+c:reload_schema()
+---
+...
+test_run:cmd("setopt delimiter ';'")
+---
+- true
+...
+function dml_push_and_dml(key)
+ box.session.push('started dml')
+ s:replace{key}
+ box.session.push('continued dml')
+ s:replace{-key}
+ box.session.push('finished dml')
+ return key
+end;
+---
+...
+function do_pushes(val)
+ for i = 1, 5 do
+ box.session.push(i)
+ fiber.yield()
+ end
+ return val
+end;
+---
+...
+function push_catcher_f()
+ fiber.yield()
+ started = started + 1
+ local catcher = {messages = {}, retval = nil, is_dml = false}
+ catcher.retval = c:call('do_pushes', {started},
+ {on_push = table.insert,
+ on_push_ctx = catcher.messages})
+ table.insert(catchers, catcher)
+ finished = finished + 1
+end;
+---
+...
+function dml_push_and_dml_f()
+ fiber.yield()
+ started = started + 1
+ local catcher = {messages = {}, retval = nil, is_dml = true}
+ catcher.retval = c:call('dml_push_and_dml', {started},
+ {on_push = table.insert,
+ on_push_ctx = catcher.messages})
+ table.insert(catchers, catcher)
+ finished = finished + 1
+end;
+---
+...
+-- At first check that a pushed message can be ignored in a binary
+-- protocol too.
+c:call('do_pushes', {300});
+---
+- 300
+...
+-- Then do stress.
+for i = 1, 200 do
+ fiber.create(dml_push_and_dml_f)
+ fiber.create(push_catcher_f)
+end;
+---
+...
+while finished ~= 400 do fiber.sleep(0.1) end;
+---
+...
+for _, c in pairs(catchers) do
+ if c.is_dml then
+ assert(#c.messages == 3, 'dml sends 3 messages')
+ assert(c.messages[1] == 'started dml', 'started')
+ assert(c.messages[2] == 'continued dml', 'continued')
+ assert(c.messages[3] == 'finished dml', 'finished')
+ assert(s:get{c.retval}, 'inserted +')
+ assert(s:get{-c.retval}, 'inserted -')
+ else
+ assert(c.retval, 'something is returned')
+ assert(#c.messages == 5, 'non-dml sends 5 messages')
+ for k, v in pairs(c.messages) do
+ assert(k == v, 'with equal keys and values')
+ end
+ end
+end;
+---
+...
+test_run:cmd("setopt delimiter ''");
+---
+- true
+...
+#s:select{}
+---
+- 400
+...
+--
+-- Ok to push NULL.
+--
+function push_null() box.session.push(box.NULL) end
+---
+...
+messages = {}
+---
+...
+c:call('push_null', {}, {on_push = table.insert, on_push_ctx = messages})
+---
+...
+messages
+---
+- - null
+...
+--
+-- Test binary pushes.
+--
+ibuf = require('buffer').ibuf()
+---
+...
+msgpack = require('msgpack')
+---
+...
+messages = {}
+---
+...
+resp_len = c:call('do_pushes', {300}, {on_push = table.insert, on_push_ctx = messages, buffer = ibuf})
---
-- null
-- Session 'binary' does not support push()
+...
+resp_len
+---
+- 10
+...
+messages
+---
+- - 8
+ - 8
+ - 8
+ - 8
+ - 8
+...
+decoded = {}
+---
+...
+r = nil
+---
+...
+for i = 1, #messages do r, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) table.insert(decoded, r) end
+---
+...
+decoded
+---
+- - {48: [1]}
+ - {48: [2]}
+ - {48: [3]}
+ - {48: [4]}
+ - {48: [5]}
+...
+r, _ = msgpack.decode_unchecked(ibuf.rpos)
+---
+...
+r
+---
+- {48: [300]}
...
c:close()
---
...
+s:drop()
+---
+...
box.schema.user.revoke('guest', 'read,write,execute', 'universe')
---
...
--
-- Ensure can not push in background.
--
-fiber = require('fiber')
+ok = nil
+---
+...
+err = nil
---
...
-f = fiber.create(do_push)
+f = fiber.create(function() ok, err = box.session.push(100) end)
---
...
while f:status() ~= 'dead' do fiber.sleep(0.01) end
diff --git a/test/box/push.test.lua b/test/box/push.test.lua
index a59fe0a4c..b0ff218bb 100644
--- a/test/box/push.test.lua
+++ b/test/box/push.test.lua
@@ -1,5 +1,6 @@
+test_run = require('test_run').new()
--
--- gh-2677: box.session.push.
+-- gh-2677: box.session.push binary protocol tests.
--
--
@@ -8,28 +9,135 @@
box.session.push()
box.session.push(1, 2)
-ok = nil
-err = nil
-function do_push() ok, err = box.session.push(1) end
+fiber = require('fiber')
+messages = {}
+test_run:cmd("setopt delimiter ';'")
+function do_pushes()
+ for i = 1, 5 do
+ box.session.push(i)
+ fiber.sleep(0.01)
+ end
+ return 300
+end;
+test_run:cmd("setopt delimiter ''");
---
--- Test binary protocol.
---
netbox = require('net.box')
box.schema.user.grant('guest', 'read,write,execute', 'universe')
c = netbox.connect(box.cfg.listen)
c:ping()
-c:call('do_push')
-ok, err
+c:call('do_pushes', {}, {on_push = table.insert, on_push_ctx = messages})
+messages
+
+-- Add a little stress: many pushes with different syncs, from
+-- different fibers and DML/DQL requests.
+
+catchers = {}
+started = 0
+finished = 0
+s = box.schema.create_space('test', {format = {{'field1', 'integer'}}})
+pk = s:create_index('pk')
+c:reload_schema()
+test_run:cmd("setopt delimiter ';'")
+function dml_push_and_dml(key)
+ box.session.push('started dml')
+ s:replace{key}
+ box.session.push('continued dml')
+ s:replace{-key}
+ box.session.push('finished dml')
+ return key
+end;
+function do_pushes(val)
+ for i = 1, 5 do
+ box.session.push(i)
+ fiber.yield()
+ end
+ return val
+end;
+function push_catcher_f()
+ fiber.yield()
+ started = started + 1
+ local catcher = {messages = {}, retval = nil, is_dml = false}
+ catcher.retval = c:call('do_pushes', {started},
+ {on_push = table.insert,
+ on_push_ctx = catcher.messages})
+ table.insert(catchers, catcher)
+ finished = finished + 1
+end;
+function dml_push_and_dml_f()
+ fiber.yield()
+ started = started + 1
+ local catcher = {messages = {}, retval = nil, is_dml = true}
+ catcher.retval = c:call('dml_push_and_dml', {started},
+ {on_push = table.insert,
+ on_push_ctx = catcher.messages})
+ table.insert(catchers, catcher)
+ finished = finished + 1
+end;
+-- At first check that a pushed message can be ignored in a binary
+-- protocol too.
+c:call('do_pushes', {300});
+-- Then do stress.
+for i = 1, 200 do
+ fiber.create(dml_push_and_dml_f)
+ fiber.create(push_catcher_f)
+end;
+while finished ~= 400 do fiber.sleep(0.1) end;
+
+for _, c in pairs(catchers) do
+ if c.is_dml then
+ assert(#c.messages == 3, 'dml sends 3 messages')
+ assert(c.messages[1] == 'started dml', 'started')
+ assert(c.messages[2] == 'continued dml', 'continued')
+ assert(c.messages[3] == 'finished dml', 'finished')
+ assert(s:get{c.retval}, 'inserted +')
+ assert(s:get{-c.retval}, 'inserted -')
+ else
+ assert(c.retval, 'something is returned')
+ assert(#c.messages == 5, 'non-dml sends 5 messages')
+ for k, v in pairs(c.messages) do
+ assert(k == v, 'with equal keys and values')
+ end
+ end
+end;
+test_run:cmd("setopt delimiter ''");
+
+#s:select{}
+
+--
+-- Ok to push NULL.
+--
+function push_null() box.session.push(box.NULL) end
+messages = {}
+c:call('push_null', {}, {on_push = table.insert, on_push_ctx = messages})
+messages
+
+--
+-- Test binary pushes.
+--
+ibuf = require('buffer').ibuf()
+msgpack = require('msgpack')
+messages = {}
+resp_len = c:call('do_pushes', {300}, {on_push = table.insert, on_push_ctx = messages, buffer = ibuf})
+resp_len
+messages
+decoded = {}
+r = nil
+for i = 1, #messages do r, ibuf.rpos = msgpack.decode_unchecked(ibuf.rpos) table.insert(decoded, r) end
+decoded
+r, _ = msgpack.decode_unchecked(ibuf.rpos)
+r
+
c:close()
+s:drop()
box.schema.user.revoke('guest', 'read,write,execute', 'universe')
--
-- Ensure can not push in background.
--
-fiber = require('fiber')
-f = fiber.create(do_push)
+ok = nil
+err = nil
+f = fiber.create(function() ok, err = box.session.push(100) end)
while f:status() ~= 'dead' do fiber.sleep(0.01) end
ok, err
^ permalink raw reply [flat|nested] 34+ messages in thread
* [PATCH v2 02/10] yaml: introduce yaml.encode_tagged
2018-04-20 13:24 [PATCH v2 00/10] session: introduce box.session.push Vladislav Shpilevoy
2018-04-20 13:24 ` [PATCH v2 01/10] yaml: don't throw OOM on any error in yaml encoding Vladislav Shpilevoy
2018-04-20 13:24 ` [tarantool-patches] [PATCH v2 10/10] session: introduce binary box.session.push Vladislav Shpilevoy
@ 2018-04-20 13:24 ` Vladislav Shpilevoy
2018-05-10 18:22 ` [tarantool-patches] " Konstantin Osipov
2018-04-20 13:24 ` [PATCH v2 03/10] yaml: introduce yaml.decode_tag Vladislav Shpilevoy
` (7 subsequent siblings)
10 siblings, 1 reply; 34+ messages in thread
From: Vladislav Shpilevoy @ 2018-04-20 13:24 UTC (permalink / raw)
To: tarantool-patches; +Cc: vdavydov.dev
Encode_tagged allows to define one global YAML tag for a
document. Tagged YAML documents are going to be used for
console text pushes to distinguish actual box.session.push() from
console.print(). The first will have tag !push, and the
second - !print.
---
test/app-tap/yaml.test.lua | 23 ++++++++++++++-
third_party/lua-yaml/lyaml.cc | 69 +++++++++++++++++++++++++++++++++++++++++--
2 files changed, 88 insertions(+), 4 deletions(-)
diff --git a/test/app-tap/yaml.test.lua b/test/app-tap/yaml.test.lua
index def278570..b81402cc0 100755
--- a/test/app-tap/yaml.test.lua
+++ b/test/app-tap/yaml.test.lua
@@ -69,9 +69,29 @@ local function test_output(test, s)
"--- |-\n Tutorial -- Header\n ====\n\n Text\n...\n", "tutorial string");
end
+local function test_tagged(test, s)
+ test:plan(7)
+ local must_be_err = 'Usage: encode_tagged({prefix = <string>, handle = <string>}, object)'
+ local prefix = 'tag:tarantool.io/push,2018'
+ local ok, err = pcall(s.encode_tagged)
+ test:is(err, must_be_err, "encode_tagged usage")
+ ok, err = pcall(s.encode_tagged, {}, 100)
+ test:is(err, must_be_err, "encode_tagged usage")
+ ok, err = pcall(s.encode_tagged, {handle = true, prefix = 100}, 200)
+ test:is(err, must_be_err, "encode_tagged usage")
+ local ret
+ ret, err = s.encode_tagged({handle = '!push', prefix = prefix}, 300)
+ test:is(ret, nil, 'non-usage and non-oom errors do not raise')
+ test:is(err, "tag handle must end with '!'", "encode_tagged usage")
+ ret = s.encode_tagged({handle = '!push!', prefix = prefix}, 300)
+ test:is(ret, "%TAG !push! "..prefix.."\n--- 300\n...\n", "encode_tagged usage")
+ ret = s.encode_tagged({handle = '!print!', prefix = prefix}, {a = 100, b = 200})
+ test:is(ret, "%TAG !print! tag:tarantool.io/push,2018\n---\na: 100\nb: 200\n...\n", 'encode_tagged usage')
+end
+
tap.test("yaml", function(test)
local serializer = require('yaml')
- test:plan(10)
+ test:plan(11)
test:test("unsigned", common.test_unsigned, serializer)
test:test("signed", common.test_signed, serializer)
test:test("double", common.test_double, serializer)
@@ -82,4 +102,5 @@ tap.test("yaml", function(test)
test:test("ucdata", common.test_ucdata, serializer)
test:test("compact", test_compact, serializer)
test:test("output", test_output, serializer)
+ test:test("tagged", test_tagged, serializer)
end)
diff --git a/third_party/lua-yaml/lyaml.cc b/third_party/lua-yaml/lyaml.cc
index 430f4be2c..8329f84e9 100644
--- a/third_party/lua-yaml/lyaml.cc
+++ b/third_party/lua-yaml/lyaml.cc
@@ -78,6 +78,8 @@ struct lua_yaml_dumper {
unsigned int anchor_number;
yaml_emitter_t emitter;
char error;
+ yaml_tag_directive_t begin_tag;
+ yaml_tag_directive_t *end_tag;
lua_State *outputL;
luaL_Buffer yamlbuf;
@@ -571,7 +573,8 @@ static int dump_node(struct lua_yaml_dumper *dumper)
static void dump_document(struct lua_yaml_dumper *dumper) {
yaml_event_t ev;
- if (!yaml_document_start_event_initialize(&ev, NULL, NULL, NULL, 0) ||
+ if (!yaml_document_start_event_initialize(&ev, NULL, &dumper->begin_tag,
+ dumper->end_tag, 0) ||
!yaml_emitter_emit(&dumper->emitter, &ev))
return;
@@ -618,13 +621,38 @@ static void find_references(struct lua_yaml_dumper *dumper) {
}
}
-static int l_dump(lua_State *L) {
+/**
+ * Encode a Lua object or objects into YAML documents onto Lua
+ * stack.
+ * @param L Lua stack to get arguments and push result.
+ * @param serializer Lua YAML serializer.
+ * @param tag_handle NULL, or a global tag handle. Handle becomes
+ * a synonym for prefix.
+ * @param tag_prefix NULL, or a global tag prefix, to which @a
+ * handle is expanded.
+ * @retval nil, error Error.
+ * @retval not nil Lua string with dumped object.
+ */
+static int
+lua_yaml_encode_impl(lua_State *L, struct luaL_serializer *serializer,
+ const char *tag_handle, const char *tag_prefix)
+{
struct lua_yaml_dumper dumper;
int i, argcount = lua_gettop(L);
yaml_event_t ev;
dumper.L = L;
- dumper.cfg = luaL_checkserializer(L);
+ dumper.begin_tag.handle = (yaml_char_t *) tag_handle;
+ dumper.begin_tag.prefix = (yaml_char_t *) tag_prefix;
+ /*
+ * If a tag is specified, then tag list is not empty and
+ * consists of a single tag.
+ */
+ if (tag_handle != NULL && tag_prefix != NULL)
+ dumper.end_tag = &dumper.begin_tag + 1;
+ else
+ dumper.end_tag = &dumper.begin_tag;
+ dumper.cfg = serializer;
dumper.error = 0;
/* create thread to use for YAML buffer */
dumper.outputL = lua_newthread(L);
@@ -684,11 +712,46 @@ error:
}
}
+static int l_dump(lua_State *L) {
+ return lua_yaml_encode_impl(L, luaL_checkserializer(L), NULL, NULL);
+}
+
+/**
+ * Dump a Lua object into YAML string with a global TAG specified.
+ * @param options First argument on a stack, consists of two
+ * options: tag prefix and tag handle.
+ * @param object Lua object to dump under the global tag.
+ * @retval Lua string with dumped object.
+ */
+static int l_dump_tagged(lua_State *L)
+{
+ struct luaL_serializer *serializer = luaL_checkserializer(L);
+ int argcount = lua_gettop(L);
+ if (argcount != 2 || !lua_istable(L, 1)) {
+usage_error:
+ return luaL_error(L, "Usage: encode_tagged({prefix = <string>, "\
+ "handle = <string>}, object)");
+ }
+ lua_getfield(L, 1, "prefix");
+ if (! lua_isstring(L, -1))
+ goto usage_error;
+ const char *prefix = lua_tostring(L, -1);
+ lua_getfield(L, 1, "handle");
+ if (! lua_isstring(L, -1))
+ goto usage_error;
+ const char *handle = lua_tostring(L, -1);
+ lua_pop(L, 2);
+ lua_replace(L, 1);
+ lua_settop(L, 1);
+ return lua_yaml_encode_impl(L, serializer, handle, prefix);
+}
+
static int
l_new(lua_State *L);
static const luaL_Reg yamllib[] = {
{ "encode", l_dump },
+ { "encode_tagged", l_dump_tagged },
{ "decode", l_load },
{ "new", l_new },
{ NULL, NULL}
--
2.15.1 (Apple Git-101)
^ permalink raw reply [flat|nested] 34+ messages in thread* Re: [tarantool-patches] [PATCH v2 02/10] yaml: introduce yaml.encode_tagged
2018-04-20 13:24 ` [PATCH v2 02/10] yaml: introduce yaml.encode_tagged Vladislav Shpilevoy
@ 2018-05-10 18:22 ` Konstantin Osipov
2018-05-24 20:50 ` [tarantool-patches] " Vladislav Shpilevoy
0 siblings, 1 reply; 34+ messages in thread
From: Konstantin Osipov @ 2018-05-10 18:22 UTC (permalink / raw)
To: tarantool-patches; +Cc: vdavydov.dev
* Vladislav Shpilevoy <v.shpilevoy@tarantool.org> [18/04/20 16:25]:
> index 430f4be2c..8329f84e9 100644
> --- a/third_party/lua-yaml/lyaml.cc
> +++ b/third_party/lua-yaml/lyaml.cc
> @@ -78,6 +78,8 @@ struct lua_yaml_dumper {
> unsigned int anchor_number;
> yaml_emitter_t emitter;
> char error;
> + yaml_tag_directive_t begin_tag;
> + yaml_tag_directive_t *end_tag;
Please add comments for the new members.
>
> +/**
> + * Encode a Lua object or objects into YAML documents onto Lua
> + * stack.
Encode an object or objects on Lua stack into YAML stream.
> + * @param L Lua stack to get arguments and push result.
> + * @param serializer Lua YAML serializer.
> + * @param tag_handle NULL, or a global tag handle. Handle becomes
> + * a synonym for prefix.
The handle becomes a synonym for prefix.
I don't understand what this means.
> + * @param tag_prefix NULL, or a global tag prefix, to which @a
> + * handle is expanded.
Perhaps you could say a few words here about handles, prefixes and
expansions, or, better yet, quote the relevant parts of YAML
standard.
> + * @retval nil, error Error.
> + * @retval not nil Lua string with dumped object.
The return value is integer. What did you mean to say?
For Lua functions it's better to write something like -2, +1
(pops two, pushes 1).
> + */
> +static int
> +lua_yaml_encode_impl(lua_State *L, struct luaL_serializer *serializer,
> + const char *tag_handle, const char *tag_prefix)
> +{
> struct lua_yaml_dumper dumper;
> int i, argcount = lua_gettop(L);
> yaml_event_t ev;
>
> dumper.L = L;
> - dumper.cfg = luaL_checkserializer(L);
> + dumper.begin_tag.handle = (yaml_char_t *) tag_handle;
> + dumper.begin_tag.prefix = (yaml_char_t *) tag_prefix;
> + /*
> + * If a tag is specified, then tag list is not empty and
> + * consists of a single tag.
> + */
> + if (tag_handle != NULL && tag_prefix != NULL)
Why do you need to check both?
dumper.end_tag = &dumper.begin_tag + (tag_handle != NULL && tag_prefix != NULL);
> + dumper.end_tag = &dumper.begin_tag + 1;
> + else
> + dumper.end_tag = &dumper.begin_tag;
> + dumper.cfg = serializer;
> dumper.error = 0;
> /* create thread to use for YAML buffer */
> dumper.outputL = lua_newthread(L);
> @@ -684,11 +712,46 @@ error:
> }
> }
>
> +static int l_dump(lua_State *L) {
Missing newline.
The function needs a formal comment, even though it's trivial.
> + return lua_yaml_encode_impl(L, luaL_checkserializer(L), NULL, NULL);
> +}
> +
> +/**
> + * Dump a Lua object into YAML string with a global TAG specified.
Serialize a Lua object as YAML string, taking into account a
global tag, if it's supplied in the arguments.
> + * @param options First argument on a stack, consists of two
> + * options: tag prefix and tag handle.
> + * @param object Lua object to dump under the global tag.
> + * @retval Lua string with dumped object.
Why do you take options first, object second? AFAICS we usually
take object first, options second. Let's be consistent?
Which begs the question, why do you need a new function rather
than extend an existing one with options?
> + */
> +static int l_dump_tagged(lua_State *L)
> +{
> + struct luaL_serializer *serializer = luaL_checkserializer(L);
> + int argcount = lua_gettop(L);
> + if (argcount != 2 || !lua_istable(L, 1)) {
> +usage_error:
> + return luaL_error(L, "Usage: encode_tagged({prefix = <string>, "\
> + "handle = <string>}, object)");
> + }
> + lua_getfield(L, 1, "prefix");
> + if (! lua_isstring(L, -1))
> + goto usage_error;
> + const char *prefix = lua_tostring(L, -1);
> + lua_getfield(L, 1, "handle");
> + if (! lua_isstring(L, -1))
> + goto usage_error;
> + const char *handle = lua_tostring(L, -1);
> + lua_pop(L, 2);
> + lua_replace(L, 1);
> + lua_settop(L, 1);
AFAIR you invalidate handle and prefix pointers as soon as you pop
and replace things on the stack.
> + return lua_yaml_encode_impl(L, serializer, handle, prefix);
> +}
> +
> static int
> l_new(lua_State *L);
--
Konstantin Osipov, Moscow, Russia, +7 903 626 22 32
http://tarantool.io - www.twitter.com/kostja_osipov
^ permalink raw reply [flat|nested] 34+ messages in thread* Re: [tarantool-patches] Re: [PATCH v2 02/10] yaml: introduce yaml.encode_tagged
2018-05-10 18:22 ` [tarantool-patches] " Konstantin Osipov
@ 2018-05-24 20:50 ` Vladislav Shpilevoy
2018-05-30 19:15 ` Konstantin Osipov
0 siblings, 1 reply; 34+ messages in thread
From: Vladislav Shpilevoy @ 2018-05-24 20:50 UTC (permalink / raw)
To: tarantool-patches, Konstantin Osipov; +Cc: vdavydov.dev
Hello. Thanks for the review.
On 10/05/2018 21:22, Konstantin Osipov wrote:
> * Vladislav Shpilevoy <v.shpilevoy@tarantool.org> [18/04/20 16:25]:
>> index 430f4be2c..8329f84e9 100644
>> --- a/third_party/lua-yaml/lyaml.cc
>> +++ b/third_party/lua-yaml/lyaml.cc
>> @@ -78,6 +78,8 @@ struct lua_yaml_dumper {
>> unsigned int anchor_number;
>> yaml_emitter_t emitter;
>> char error;
>> + yaml_tag_directive_t begin_tag;
>> + yaml_tag_directive_t *end_tag;
>
> Please add comments for the new members.
@@ -78,7 +78,16 @@ struct lua_yaml_dumper {
unsigned int anchor_number;
yaml_emitter_t emitter;
char error;
+ /** Global tag to label the result document by. */
yaml_tag_directive_t begin_tag;
+ /**
+ * - end_tag == &begin_tag - a document is not labeld with a
+ * global tag.
+ * - end_tag == &begin_tag + 1 - a document is labeled with a
+ * global tag specified in begin_tag attribute. End_tag pointer
+ * is used instead of tag count because of libyaml API - it
+ * takes begin and end pointers of tags array.
+ */
yaml_tag_directive_t *end_tag;
>
>>
>> +/**
>> + * Encode a Lua object or objects into YAML documents onto Lua
>> + * stack.
>
> Encode an object or objects on Lua stack into YAML stream.
@@ -622,9 +631,9 @@ static void find_references(struct lua_yaml_dumper *dumper) {
}
/**
- * Encode a Lua object or objects into YAML documents onto Lua
- * stack.
+ * Encode an object or objects on Lua stack into YAML stream.
>
>> + * @param L Lua stack to get arguments and push result.
>> + * @param serializer Lua YAML serializer.
>> + * @param tag_handle NULL, or a global tag handle. Handle becomes
>> + * a synonym for prefix.
>
> The handle becomes a synonym for prefix.
>
>
> I don't understand what this means.
Handle and prefix are YAML standard terms. Consider this tag:
%TAG !push! tag:tarantool.io/push,2018
Here !push! is a handle, tag:tarantool.io/push,2018 - prefix.
When in a document !push! is written, it is exposed to
!tag:tarantool.io/push,2018.
Example:
-- !push!str string_value
Same as:
-- !tag:tarantool.io/push,2018!str string_value
So here handle '!push!' is exposed to 'tag:tarantool.io/push,2018'.
Look:
tarantool> a = yaml.decode('!tag:tarantool.io/push,2018!str string_value')
---
...
tarantool> a
---
- string_value
...
So handle is synonym for prefix.
>
>> + * @param tag_prefix NULL, or a global tag prefix, to which @a
>> + * handle is expanded.
>
> Perhaps you could say a few words here about handles, prefixes and
> expansions, or, better yet, quote the relevant parts of YAML
> standard.
- * @param tag_handle NULL, or a global tag handle. Handle becomes
- * a synonym for prefix.
+ * @param tag_handle NULL, or a global tag handle. For global tags
+ * details see the standard:
+ * http://yaml.org/spec/1.2/spec.html#tag/shorthand/.
* @param tag_prefix NULL, or a global tag prefix, to which @a
- * handle is expanded.
+ * handle is expanded. Example of a tagged document:
+ * handle prefix
+ * ____ ________________________
+ * / \ / \
+ * '%TAG !push! tag:tarantool.io/push,2018
+ * --- value
+ * ...
+ *
>
>> + * @retval nil, error Error.
>> + * @retval not nil Lua string with dumped object.
>
> The return value is integer. What did you mean to say?
> For Lua functions it's better to write something like -2, +1
> (pops two, pushes 1).
- * @retval nil, error Error.
- * @retval not nil Lua string with dumped object.
+ * @retval 2 Pushes two values on error: nil, error description.
+ * @retval 1 Pushes one value on success: string with dumped
+ * object.
*/
>
>> + */
>> +static int
>> +lua_yaml_encode_impl(lua_State *L, struct luaL_serializer *serializer,
>> + const char *tag_handle, const char *tag_prefix)
>> +{
>> struct lua_yaml_dumper dumper;
>> int i, argcount = lua_gettop(L);
>> yaml_event_t ev;
>>
>> dumper.L = L;
>> - dumper.cfg = luaL_checkserializer(L);
>> + dumper.begin_tag.handle = (yaml_char_t *) tag_handle;
>> + dumper.begin_tag.prefix = (yaml_char_t *) tag_prefix;
>> + /*
>> + * If a tag is specified, then tag list is not empty and
>> + * consists of a single tag.
>> + */
>> + if (tag_handle != NULL && tag_prefix != NULL)
>
> Why do you need to check both?
@@ -661,11 +661,12 @@ lua_yaml_encode_impl(lua_State *L, int argcount,
dumper.L = L;
dumper.begin_tag.handle = (yaml_char_t *) tag_handle;
dumper.begin_tag.prefix = (yaml_char_t *) tag_prefix;
+ assert((tag_handle == NULL) == (tag_prefix == NULL));
/*
* If a tag is specified, then tag list is not empty and
* consists of a single tag.
*/
- if (tag_handle != NULL && tag_prefix != NULL)
+ if (tag_prefix != NULL)
dumper.end_tag = &dumper.begin_tag + 1;
else
dumper.end_tag = &dumper.begin_tag;
>
> dumper.end_tag = &dumper.begin_tag + (tag_handle != NULL && tag_prefix != NULL);
I very do not like arithmetic operations on boolean and integers, so no.
>> @@ -684,11 +712,46 @@ error:
>> }
>> }
>>
>> +static int l_dump(lua_State *L) {
>
> Missing newline.
Not missing. I did it advisedly. The lyaml.cc file has its own code
style, that slightly differs from Tarantool's. We must either convert
the entire file to Tarantool, or obey lyaml style.
>
> The function needs a formal comment, even though it's trivial.
+/**
+ * Dump Lua objects into YAML string. It takes any argument count,
+ * and dumps each in a separate document.
+ * @retval 1 Pushes one value: string with dumped documents.
+ */
>
>> + return lua_yaml_encode_impl(L, luaL_checkserializer(L), NULL, NULL);
>> +}
>> +
>> +/**
>> + * Dump a Lua object into YAML string with a global TAG specified.
>
> Serialize a Lua object as YAML string, taking into account a
> global tag, if it's supplied in the arguments.
- * Dump a Lua object into YAML string with a global TAG specified.
+ * Serialize a Lua object as YAML string, taking into account a
+ * global tag.
I did not add "if it's supplied in the arguments", because a tag is
required.
>
>> + * @param options First argument on a stack, consists of two
>> + * options: tag prefix and tag handle.
>> + * @param object Lua object to dump under the global tag.
>> + * @retval Lua string with dumped object.
>
> Why do you take options first, object second? AFAICS we usually
> take object first, options second. Let's be consistent?
+ * @param object Lua object to dump under the global tag.
+ * @param options Table with two options: tag prefix and tag
+ * handle.
>
> Which begs the question, why do you need a new function rather
> than extend an existing one with options?
Original yaml.encode has this signature: encode(...). It takes
any argument count, and dumps each. So I can not add an option to
this. Example:
yaml.encode(object, {prefix = ..., handle = ...})
What to do, dump both arguments or use the second as options? I think
that we lost a chance to change encode() signature, it is already public
and documented, and will always take (...) arguments.
But here it is documented
https://tarantool.io/en/doc/1.9/reference/reference_lua/yaml.html?highlight=yaml#lua-function.yaml.encode
that yaml.encode has one argument. Maybe no one noticed that it takes multiple
values. If you want, I could change its signature from (...) to (value, [{tag = <tag>}]).
>
>> + */
>> +static int l_dump_tagged(lua_State *L)
>> +{
>> + struct luaL_serializer *serializer = luaL_checkserializer(L);
>> + int argcount = lua_gettop(L);
>> + if (argcount != 2 || !lua_istable(L, 1)) {
>> +usage_error:
>> + return luaL_error(L, "Usage: encode_tagged({prefix = <string>, "\
>> + "handle = <string>}, object)");
>> + }
>> + lua_getfield(L, 1, "prefix");
>> + if (! lua_isstring(L, -1))
>> + goto usage_error;
>> + const char *prefix = lua_tostring(L, -1);
>> + lua_getfield(L, 1, "handle");
>> + if (! lua_isstring(L, -1))
>> + goto usage_error;
>> + const char *handle = lua_tostring(L, -1);
>> + lua_pop(L, 2);
>> + lua_replace(L, 1);
>> + lua_settop(L, 1);
>
> AFAIR you invalidate handle and prefix pointers as soon as you pop
> and replace things on the stack.
Fixed on the branch.
From the next review I fixed usage tests:
@@ -71,14 +71,13 @@ end
local function test_tagged(test, s)
test:plan(7)
- local must_be_err = 'Usage: encode_tagged(object, {prefix = <string>, handle = <string>})'
local prefix = 'tag:tarantool.io/push,2018'
local ok, err = pcall(s.encode_tagged)
- test:is(err, must_be_err, "encode_tagged usage")
+ test:isnt(err:find('Usage'), nil, "encode_tagged usage")
ok, err = pcall(s.encode_tagged, 100, {})
- test:is(err, must_be_err, "encode_tagged usage")
+ test:isnt(err:find('Usage'), nil, "encode_tagged usage")
ok, err = pcall(s.encode_tagged, 200, {handle = true, prefix = 100})
- test:is(err, must_be_err, "encode_tagged usage")
+ test:isnt(err:find('Usage'), nil, "encode_tagged usage")
local ret
ret, err = s.encode_tagged(300, {handle = '!push', prefix = prefix})
test:is(ret, nil, 'non-usage and non-oom errors do not raise')
See below the entire patch.
=======================================================================
commit 0117eac46095ac7fbf7d36c7ebfeb7fe16f96cc2
Author: Vladislav Shpilevoy <v.shpilevoy@tarantool.org>
Date: Mon Apr 2 23:40:30 2018 +0300
yaml: introduce yaml.encode_tagged
Encode_tagged allows to define one global YAML tag for a
document. Tagged YAML documents are going to be used for
console text pushes to distinguish actual box.session.push() from
console.print(). The first will have tag !push, and the
second - !print.
diff --git a/test/app-tap/yaml.test.lua b/test/app-tap/yaml.test.lua
index def278570..bb75ce4ea 100755
--- a/test/app-tap/yaml.test.lua
+++ b/test/app-tap/yaml.test.lua
@@ -69,9 +69,28 @@ local function test_output(test, s)
"--- |-\n Tutorial -- Header\n ====\n\n Text\n...\n", "tutorial string");
end
+local function test_tagged(test, s)
+ test:plan(7)
+ local prefix = 'tag:tarantool.io/push,2018'
+ local ok, err = pcall(s.encode_tagged)
+ test:isnt(err:find('Usage'), nil, "encode_tagged usage")
+ ok, err = pcall(s.encode_tagged, 100, {})
+ test:isnt(err:find('Usage'), nil, "encode_tagged usage")
+ ok, err = pcall(s.encode_tagged, 200, {handle = true, prefix = 100})
+ test:isnt(err:find('Usage'), nil, "encode_tagged usage")
+ local ret
+ ret, err = s.encode_tagged(300, {handle = '!push', prefix = prefix})
+ test:is(ret, nil, 'non-usage and non-oom errors do not raise')
+ test:is(err, "tag handle must end with '!'", "encode_tagged usage")
+ ret = s.encode_tagged(300, {handle = '!push!', prefix = prefix})
+ test:is(ret, "%TAG !push! "..prefix.."\n--- 300\n...\n", "encode_tagged usage")
+ ret = s.encode_tagged({a = 100, b = 200}, {handle = '!print!', prefix = prefix})
+ test:is(ret, "%TAG !print! tag:tarantool.io/push,2018\n---\na: 100\nb: 200\n...\n", 'encode_tagged usage')
+end
+
tap.test("yaml", function(test)
local serializer = require('yaml')
- test:plan(10)
+ test:plan(11)
test:test("unsigned", common.test_unsigned, serializer)
test:test("signed", common.test_signed, serializer)
test:test("double", common.test_double, serializer)
@@ -82,4 +101,5 @@ tap.test("yaml", function(test)
test:test("ucdata", common.test_ucdata, serializer)
test:test("compact", test_compact, serializer)
test:test("output", test_output, serializer)
+ test:test("tagged", test_tagged, serializer)
end)
diff --git a/third_party/lua-yaml/lyaml.cc b/third_party/lua-yaml/lyaml.cc
index 430f4be2c..ca7f4d224 100644
--- a/third_party/lua-yaml/lyaml.cc
+++ b/third_party/lua-yaml/lyaml.cc
@@ -78,6 +78,17 @@ struct lua_yaml_dumper {
unsigned int anchor_number;
yaml_emitter_t emitter;
char error;
+ /** Global tag to label the result document by. */
+ yaml_tag_directive_t begin_tag;
+ /**
+ * - end_tag == &begin_tag - a document is not labeld with a
+ * global tag.
+ * - end_tag == &begin_tag + 1 - a document is labeled with a
+ * global tag specified in begin_tag attribute. End_tag pointer
+ * is used instead of tag count because of libyaml API - it
+ * takes begin and end pointers of tags array.
+ */
+ yaml_tag_directive_t *end_tag;
lua_State *outputL;
luaL_Buffer yamlbuf;
@@ -571,7 +582,8 @@ static int dump_node(struct lua_yaml_dumper *dumper)
static void dump_document(struct lua_yaml_dumper *dumper) {
yaml_event_t ev;
- if (!yaml_document_start_event_initialize(&ev, NULL, NULL, NULL, 0) ||
+ if (!yaml_document_start_event_initialize(&ev, NULL, &dumper->begin_tag,
+ dumper->end_tag, 0) ||
!yaml_emitter_emit(&dumper->emitter, &ev))
return;
@@ -618,13 +630,52 @@ static void find_references(struct lua_yaml_dumper *dumper) {
}
}
-static int l_dump(lua_State *L) {
+/**
+ * Encode an object or objects on Lua stack into YAML stream.
+ * @param L Lua stack to get arguments and push result.
+ * @param serializer Lua YAML serializer.
+ * @param tag_handle NULL, or a global tag handle. For global tags
+ * details see the standard:
+ * http://yaml.org/spec/1.2/spec.html#tag/shorthand/.
+ * @param tag_prefix NULL, or a global tag prefix, to which @a
+ * handle is expanded. Example of a tagged document:
+ * handle prefix
+ * ____ ________________________
+ * / \ / \
+ * '%TAG !push! tag:tarantool.io/push,2018
+ * --- value
+ * ...
+ *
+ * @retval 2 Pushes two values on error: nil, error description.
+ * @retval 1 Pushes one value on success: string with dumped
+ * object.
+ */
+static int
+lua_yaml_encode_impl(lua_State *L, struct luaL_serializer *serializer,
+ const char *tag_handle, const char *tag_prefix)
+{
struct lua_yaml_dumper dumper;
- int i, argcount = lua_gettop(L);
yaml_event_t ev;
+ int argcount;
dumper.L = L;
- dumper.cfg = luaL_checkserializer(L);
+ dumper.begin_tag.handle = (yaml_char_t *) tag_handle;
+ dumper.begin_tag.prefix = (yaml_char_t *) tag_prefix;
+ assert((tag_handle == NULL) == (tag_prefix == NULL));
+ /*
+ * If a tag is specified, then tag list is not empty and
+ * consists of a single tag.
+ */
+ if (tag_prefix != NULL) {
+ dumper.end_tag = &dumper.begin_tag + 1;
+ /* Only one object can be encoded with a tag. */
+ argcount = 1;
+ } else {
+ dumper.end_tag = &dumper.begin_tag;
+ /* When no tags - encode all the stack. */
+ argcount = lua_gettop(L);
+ }
+ dumper.cfg = serializer;
dumper.error = 0;
/* create thread to use for YAML buffer */
dumper.outputL = lua_newthread(L);
@@ -643,7 +694,7 @@ static int l_dump(lua_State *L) {
!yaml_emitter_emit(&dumper.emitter, &ev))
goto error;
- for (i = 0; i < argcount; i++) {
+ for (int i = 0; i < argcount; i++) {
lua_newtable(L);
dumper.anchortable_index = lua_gettop(L);
dumper.anchor_number = 0;
@@ -684,11 +735,49 @@ error:
}
}
+/**
+ * Dump Lua objects into YAML string. It takes any argument count,
+ * and dumps each in a separate document.
+ * @retval 1 Pushes one value: string with dumped documents.
+ */
+static int l_dump(lua_State *L) {
+ return lua_yaml_encode_impl(L, luaL_checkserializer(L), NULL, NULL);
+}
+
+/**
+ * Serialize a Lua object as YAML string, taking into account a
+ * global tag.
+ * @param object Lua object to dump under the global tag.
+ * @param options Table with two options: tag prefix and tag
+ * handle.
+ * @retval 1 Pushes Lua string with dumped object.
+ * @retval 2 Pushes nil and error message.
+ */
+static int l_dump_tagged(lua_State *L)
+{
+ struct luaL_serializer *serializer = luaL_checkserializer(L);
+ if (lua_gettop(L) != 2 || !lua_istable(L, 2)) {
+usage_error:
+ return luaL_error(L, "Usage: encode_tagged(object, {prefix = <string>, "\
+ "handle = <string>})");
+ }
+ lua_getfield(L, 2, "prefix");
+ if (! lua_isstring(L, -1))
+ goto usage_error;
+ const char *prefix = lua_tostring(L, -1);
+ lua_getfield(L, 2, "handle");
+ if (! lua_isstring(L, -1))
+ goto usage_error;
+ const char *handle = lua_tostring(L, -1);
+ return lua_yaml_encode_impl(L, serializer, handle, prefix);
+}
+
static int
l_new(lua_State *L);
static const luaL_Reg yamllib[] = {
{ "encode", l_dump },
+ { "encode_tagged", l_dump_tagged },
{ "decode", l_load },
{ "new", l_new },
{ NULL, NULL}
^ permalink raw reply [flat|nested] 34+ messages in thread* Re: [tarantool-patches] Re: [PATCH v2 02/10] yaml: introduce yaml.encode_tagged
2018-05-24 20:50 ` [tarantool-patches] " Vladislav Shpilevoy
@ 2018-05-30 19:15 ` Konstantin Osipov
2018-05-30 20:49 ` Vladislav Shpilevoy
0 siblings, 1 reply; 34+ messages in thread
From: Konstantin Osipov @ 2018-05-30 19:15 UTC (permalink / raw)
To: Vladislav Shpilevoy; +Cc: tarantool-patches, vdavydov.dev
* Vladislav Shpilevoy <v.shpilevoy@tarantool.org> [18/05/24 23:53]:
> Hello. Thanks for the review.
>
> On 10/05/2018 21:22, Konstantin Osipov wrote:
> >* Vladislav Shpilevoy <v.shpilevoy@tarantool.org> [18/04/20 16:25]:
> >>index 430f4be2c..8329f84e9 100644
> >>--- a/third_party/lua-yaml/lyaml.cc
> >>+++ b/third_party/lua-yaml/lyaml.cc
> >>@@ -78,6 +78,8 @@ struct lua_yaml_dumper {
> >> unsigned int anchor_number;
> >> yaml_emitter_t emitter;
> >> char error;
> >>+ yaml_tag_directive_t begin_tag;
> >>+ yaml_tag_directive_t *end_tag;
> >
> >Please add comments for the new members.
>
> @@ -78,7 +78,16 @@ struct lua_yaml_dumper {
> unsigned int anchor_number;
> yaml_emitter_t emitter;
> char error;
> + /** Global tag to label the result document by. */
> yaml_tag_directive_t begin_tag;
> + /**
> + * - end_tag == &begin_tag - a document is not labeld with a
> + * global tag.
> + * - end_tag == &begin_tag + 1 - a document is labeled with a
> + * global tag specified in begin_tag attribute. End_tag pointer
> + * is used instead of tag count because of libyaml API - it
> + * takes begin and end pointers of tags array.
> + */
> yaml_tag_directive_t *end_tag;
>
> >
> >>+/**
> >>+ * Encode a Lua object or objects into YAML documents onto Lua
> >>+ * stack.
> >
> >Encode an object or objects on Lua stack into YAML stream.
>
> @@ -622,9 +631,9 @@ static void find_references(struct lua_yaml_dumper *dumper) {
> }
> /**
> - * Encode a Lua object or objects into YAML documents onto Lua
> - * stack.
> + * Encode an object or objects on Lua stack into YAML stream.
>
> >
> >>+ * @param L Lua stack to get arguments and push result.
> >>+ * @param serializer Lua YAML serializer.
> >>+ * @param tag_handle NULL, or a global tag handle. Handle becomes
> >>+ * a synonym for prefix.
> >
> >The handle becomes a synonym for prefix.
> >
> >
> >I don't understand what this means.
>
> Handle and prefix are YAML standard terms. Consider this tag:
> %TAG !push! tag:tarantool.io/push,2018
>
> Here !push! is a handle, tag:tarantool.io/push,2018 - prefix.
> When in a document !push! is written, it is exposed to
> !tag:tarantool.io/push,2018.
>
> Example:
> -- !push!str string_value
>
> Same as:
> -- !tag:tarantool.io/push,2018!str string_value
>
> So here handle '!push!' is exposed to 'tag:tarantool.io/push,2018'.
>
> Look:
> tarantool> a = yaml.decode('!tag:tarantool.io/push,2018!str string_value')
> ---
> ...
>
> tarantool> a
> ---
> - string_value
> ...
>
> So handle is synonym for prefix.
>
> >
> >>+ * @param tag_prefix NULL, or a global tag prefix, to which @a
> >>+ * handle is expanded.
> >
> >Perhaps you could say a few words here about handles, prefixes and
> >expansions, or, better yet, quote the relevant parts of YAML
> >standard.
>
> - * @param tag_handle NULL, or a global tag handle. Handle becomes
> - * a synonym for prefix.
> + * @param tag_handle NULL, or a global tag handle. For global tags
> + * details see the standard:
> + * http://yaml.org/spec/1.2/spec.html#tag/shorthand/.
> * @param tag_prefix NULL, or a global tag prefix, to which @a
> - * handle is expanded.
> + * handle is expanded. Example of a tagged document:
> + * handle prefix
> + * ____ ________________________
> + * / \ / \
> + * '%TAG !push! tag:tarantool.io/push,2018
> + * --- value
> + * ...
> + *
>
> >
> >>+ * @retval nil, error Error.
> >>+ * @retval not nil Lua string with dumped object.
> >
> >The return value is integer. What did you mean to say?
> >For Lua functions it's better to write something like -2, +1
> >(pops two, pushes 1).
>
> - * @retval nil, error Error.
> - * @retval not nil Lua string with dumped object.
> + * @retval 2 Pushes two values on error: nil, error description.
> + * @retval 1 Pushes one value on success: string with dumped
> + * object.
> */
>
> >
> >>+ */
> >>+static int
> >>+lua_yaml_encode_impl(lua_State *L, struct luaL_serializer *serializer,
> >>+ const char *tag_handle, const char *tag_prefix)
> >>+{
> >> struct lua_yaml_dumper dumper;
> >> int i, argcount = lua_gettop(L);
> >> yaml_event_t ev;
> >> dumper.L = L;
> >>- dumper.cfg = luaL_checkserializer(L);
> >>+ dumper.begin_tag.handle = (yaml_char_t *) tag_handle;
> >>+ dumper.begin_tag.prefix = (yaml_char_t *) tag_prefix;
> >>+ /*
> >>+ * If a tag is specified, then tag list is not empty and
> >>+ * consists of a single tag.
> >>+ */
> >>+ if (tag_handle != NULL && tag_prefix != NULL)
> >
> >Why do you need to check both?
>
> @@ -661,11 +661,12 @@ lua_yaml_encode_impl(lua_State *L, int argcount,
> dumper.L = L;
> dumper.begin_tag.handle = (yaml_char_t *) tag_handle;
> dumper.begin_tag.prefix = (yaml_char_t *) tag_prefix;
> + assert((tag_handle == NULL) == (tag_prefix == NULL));
> /*
> * If a tag is specified, then tag list is not empty and
> * consists of a single tag.
> */
> - if (tag_handle != NULL && tag_prefix != NULL)
> + if (tag_prefix != NULL)
> dumper.end_tag = &dumper.begin_tag + 1;
> else
> dumper.end_tag = &dumper.begin_tag;
>
> >
> >dumper.end_tag = &dumper.begin_tag + (tag_handle != NULL && tag_prefix != NULL);
>
> I very do not like arithmetic operations on boolean and integers, so no.
>
> >>@@ -684,11 +712,46 @@ error:
> >> }
> >> }
> >>+static int l_dump(lua_State *L) {
> >
> >Missing newline.
>
> Not missing. I did it advisedly. The lyaml.cc file has its own code
> style, that slightly differs from Tarantool's. We must either convert
> the entire file to Tarantool, or obey lyaml style.
>
> >
> >The function needs a formal comment, even though it's trivial.
>
> +/**
> + * Dump Lua objects into YAML string. It takes any argument count,
> + * and dumps each in a separate document.
> + * @retval 1 Pushes one value: string with dumped documents.
> + */
>
> >
> >>+ return lua_yaml_encode_impl(L, luaL_checkserializer(L), NULL, NULL);
> >>+}
> >>+
> >>+/**
> >>+ * Dump a Lua object into YAML string with a global TAG specified.
> >
> >Serialize a Lua object as YAML string, taking into account a
> >global tag, if it's supplied in the arguments.
>
> - * Dump a Lua object into YAML string with a global TAG specified.
> + * Serialize a Lua object as YAML string, taking into account a
> + * global tag.
>
> I did not add "if it's supplied in the arguments", because a tag is
> required.
>
> >
> >>+ * @param options First argument on a stack, consists of two
> >>+ * options: tag prefix and tag handle.
> >>+ * @param object Lua object to dump under the global tag.
> >>+ * @retval Lua string with dumped object.
> >
> >Why do you take options first, object second? AFAICS we usually
> >take object first, options second. Let's be consistent?
>
> + * @param object Lua object to dump under the global tag.
> + * @param options Table with two options: tag prefix and tag
> + * handle.
>
> >
> >Which begs the question, why do you need a new function rather
> >than extend an existing one with options?
>
> Original yaml.encode has this signature: encode(...). It takes
> any argument count, and dumps each. So I can not add an option to
> this. Example:
> yaml.encode(object, {prefix = ..., handle = ...})
>
> What to do, dump both arguments or use the second as options? I think
> that we lost a chance to change encode() signature, it is already public
> and documented, and will always take (...) arguments.
>
> But here it is documented
> https://tarantool.io/en/doc/1.9/reference/reference_lua/yaml.html?highlight=yaml#lua-function.yaml.encode
> that yaml.encode has one argument. Maybe no one noticed that it takes multiple
> values. If you want, I could change its signature from (...) to (value, [{tag = <tag>}]).
Please do.
We can see if anyone complains and revert the change.
I am pushing the patch. Please make this last remaining change on
a separate branch, maybe independently of the rest of the patch
stack.
--
Konstantin Osipov, Moscow, Russia, +7 903 626 22 32
http://tarantool.io - www.twitter.com/kostja_osipov
^ permalink raw reply [flat|nested] 34+ messages in thread* Re: [tarantool-patches] Re: [PATCH v2 02/10] yaml: introduce yaml.encode_tagged
2018-05-30 19:15 ` Konstantin Osipov
@ 2018-05-30 20:49 ` Vladislav Shpilevoy
2018-05-31 10:46 ` Konstantin Osipov
0 siblings, 1 reply; 34+ messages in thread
From: Vladislav Shpilevoy @ 2018-05-30 20:49 UTC (permalink / raw)
To: Konstantin Osipov; +Cc: tarantool-patches, vdavydov.dev
Hello. Thanks for the review!
>>>> +/**
>>>> + * Dump a Lua object into YAML string with a global TAG specified.
>>>
>>> Serialize a Lua object as YAML string, taking into account a
>>> global tag, if it's supplied in the arguments.
>>
>> - * Dump a Lua object into YAML string with a global TAG specified.
>> + * Serialize a Lua object as YAML string, taking into account a
>> + * global tag.
>>
>> I did not add "if it's supplied in the arguments", because a tag is
>> required.
>>
>>>
>>>> + * @param options First argument on a stack, consists of two
>>>> + * options: tag prefix and tag handle.
>>>> + * @param object Lua object to dump under the global tag.
>>>> + * @retval Lua string with dumped object.
>>>
>>> Why do you take options first, object second? AFAICS we usually
>>> take object first, options second. Let's be consistent?
>>
>> + * @param object Lua object to dump under the global tag.
>> + * @param options Table with two options: tag prefix and tag
>> + * handle.
>>
>>>
>>> Which begs the question, why do you need a new function rather
>>> than extend an existing one with options?
>>
>> Original yaml.encode has this signature: encode(...). It takes
>> any argument count, and dumps each. So I can not add an option to
>> this. Example:
>> yaml.encode(object, {prefix = ..., handle = ...})
>>
>> What to do, dump both arguments or use the second as options? I think
>> that we lost a chance to change encode() signature, it is already public
>> and documented, and will always take (...) arguments.
>>
>> But here it is documented
>> https://tarantool.io/en/doc/1.9/reference/reference_lua/yaml.html?highlight=yaml#lua-function.yaml.encode
>> that yaml.encode has one argument. Maybe no one noticed that it takes multiple
>> values. If you want, I could change its signature from (...) to (value, [{tag = <tag>}]).
>
> Please do.
> We can see if anyone complains and revert the change.
>
> I am pushing the patch. Please make this last remaining change on
> a separate branch, maybe independently of the rest of the patch
> stack.
>
Done. See the patch below. It is the first commit on the same branch.
======================================================================
commit d94518c0f084a1e1f35140c8b1dd972037ffddf2
Author: Vladislav Shpilevoy <v.shpilevoy@tarantool.org>
Date: Wed May 30 23:14:44 2018 +0300
lua: merge encode_tagged into encode options
Encode_tagged is a workaround to be able to pass options to
yaml.encode().
Before the patch yaml.encode() in fact has this signature:
yaml.encode(...). So it was impossible to add any options to this
function - all of them would be treated as the parameters. But
documentation says: https://tarantool.io/en/doc/1.9/reference/reference_lua/yaml.html?highlight=yaml#lua-function.yaml.encode
that the function has this signature: yaml.encode(value).
I hope if anyone uses yaml.encode(), he does it according to the
documentation. And I can add the {tag_prefix, tag_handle} options
to yaml.encode() and remove yaml.encode_tagged() workaround.
diff --git a/test/app-tap/yaml.test.lua b/test/app-tap/yaml.test.lua
index bb75ce4ea..30e94ddd4 100755
--- a/test/app-tap/yaml.test.lua
+++ b/test/app-tap/yaml.test.lua
@@ -72,20 +72,20 @@ end
local function test_tagged(test, s)
test:plan(7)
local prefix = 'tag:tarantool.io/push,2018'
- local ok, err = pcall(s.encode_tagged)
- test:isnt(err:find('Usage'), nil, "encode_tagged usage")
- ok, err = pcall(s.encode_tagged, 100, {})
- test:isnt(err:find('Usage'), nil, "encode_tagged usage")
- ok, err = pcall(s.encode_tagged, 200, {handle = true, prefix = 100})
- test:isnt(err:find('Usage'), nil, "encode_tagged usage")
+ local ok, err = pcall(s.encode, 200, {tag_handle = true, tag_prefix = 100})
+ test:isnt(err:find('Usage'), nil, "encode usage")
+ ok, err = pcall(s.encode, 100, {tag_handle = 'handle'})
+ test:isnt(err:find('Usage'), nil, "encode usage, no prefix")
+ ok, err = pcall(s.encode, 100, {tag_prefix = 'prefix'})
+ test:isnt(err:find('Usage'), nil, "encode usage, no handle")
local ret
- ret, err = s.encode_tagged(300, {handle = '!push', prefix = prefix})
+ ret, err = s.encode(300, {tag_handle = '!push', tag_prefix = prefix})
test:is(ret, nil, 'non-usage and non-oom errors do not raise')
- test:is(err, "tag handle must end with '!'", "encode_tagged usage")
- ret = s.encode_tagged(300, {handle = '!push!', prefix = prefix})
- test:is(ret, "%TAG !push! "..prefix.."\n--- 300\n...\n", "encode_tagged usage")
- ret = s.encode_tagged({a = 100, b = 200}, {handle = '!print!', prefix = prefix})
- test:is(ret, "%TAG !print! tag:tarantool.io/push,2018\n---\na: 100\nb: 200\n...\n", 'encode_tagged usage')
+ test:is(err, "tag handle must end with '!'", "encode usage")
+ ret = s.encode(300, {tag_handle = '!push!', tag_prefix = prefix})
+ test:is(ret, "%TAG !push! "..prefix.."\n--- 300\n...\n", "encode usage")
+ ret = s.encode({a = 100, b = 200}, {tag_handle = '!print!', tag_prefix = prefix})
+ test:is(ret, "%TAG !print! tag:tarantool.io/push,2018\n---\na: 100\nb: 200\n...\n", 'encode usage')
end
tap.test("yaml", function(test)
diff --git a/third_party/lua-yaml/lyaml.cc b/third_party/lua-yaml/lyaml.cc
index ca7f4d224..a07804a09 100644
--- a/third_party/lua-yaml/lyaml.cc
+++ b/third_party/lua-yaml/lyaml.cc
@@ -631,8 +631,8 @@ static void find_references(struct lua_yaml_dumper *dumper) {
}
/**
- * Encode an object or objects on Lua stack into YAML stream.
- * @param L Lua stack to get arguments and push result.
+ * Encode an object on Lua stack into YAML stream.
+ * @param L Lua stack to get an argument and push the result.
* @param serializer Lua YAML serializer.
* @param tag_handle NULL, or a global tag handle. For global tags
* details see the standard:
@@ -651,12 +651,11 @@ static void find_references(struct lua_yaml_dumper *dumper) {
* object.
*/
static int
-lua_yaml_encode_impl(lua_State *L, struct luaL_serializer *serializer,
- const char *tag_handle, const char *tag_prefix)
+lua_yaml_encode(lua_State *L, struct luaL_serializer *serializer,
+ const char *tag_handle, const char *tag_prefix)
{
struct lua_yaml_dumper dumper;
yaml_event_t ev;
- int argcount;
dumper.L = L;
dumper.begin_tag.handle = (yaml_char_t *) tag_handle;
@@ -666,15 +665,10 @@ lua_yaml_encode_impl(lua_State *L, struct luaL_serializer *serializer,
* If a tag is specified, then tag list is not empty and
* consists of a single tag.
*/
- if (tag_prefix != NULL) {
+ if (tag_prefix != NULL)
dumper.end_tag = &dumper.begin_tag + 1;
- /* Only one object can be encoded with a tag. */
- argcount = 1;
- } else {
+ else
dumper.end_tag = &dumper.begin_tag;
- /* When no tags - encode all the stack. */
- argcount = lua_gettop(L);
- }
dumper.cfg = serializer;
dumper.error = 0;
/* create thread to use for YAML buffer */
@@ -694,17 +688,15 @@ lua_yaml_encode_impl(lua_State *L, struct luaL_serializer *serializer,
!yaml_emitter_emit(&dumper.emitter, &ev))
goto error;
- for (int i = 0; i < argcount; i++) {
- lua_newtable(L);
- dumper.anchortable_index = lua_gettop(L);
- dumper.anchor_number = 0;
- lua_pushvalue(L, i + 1); /* push copy of arg we're processing */
- find_references(&dumper);
- dump_document(&dumper);
- if (dumper.error)
- goto error;
- lua_pop(L, 2); /* pop copied arg and anchor table */
- }
+ lua_newtable(L);
+ dumper.anchortable_index = lua_gettop(L);
+ dumper.anchor_number = 0;
+ lua_pushvalue(L, 1); /* push copy of arg we're processing */
+ find_references(&dumper);
+ dump_document(&dumper);
+ if (dumper.error)
+ goto error;
+ lua_pop(L, 2); /* pop copied arg and anchor table */
if (!yaml_stream_end_event_initialize(&ev) ||
!yaml_emitter_emit(&dumper.emitter, &ev) ||
@@ -735,41 +727,43 @@ error:
}
}
-/**
- * Dump Lua objects into YAML string. It takes any argument count,
- * and dumps each in a separate document.
- * @retval 1 Pushes one value: string with dumped documents.
- */
-static int l_dump(lua_State *L) {
- return lua_yaml_encode_impl(L, luaL_checkserializer(L), NULL, NULL);
-}
-
/**
* Serialize a Lua object as YAML string, taking into account a
- * global tag.
+ * global tag if specified.
* @param object Lua object to dump under the global tag.
* @param options Table with two options: tag prefix and tag
* handle.
* @retval 1 Pushes Lua string with dumped object.
* @retval 2 Pushes nil and error message.
*/
-static int l_dump_tagged(lua_State *L)
-{
+static int l_dump(lua_State *L) {
struct luaL_serializer *serializer = luaL_checkserializer(L);
- if (lua_gettop(L) != 2 || !lua_istable(L, 2)) {
+ int top = lua_gettop(L);
+ if (top > 2) {
usage_error:
- return luaL_error(L, "Usage: encode_tagged(object, {prefix = <string>, "\
- "handle = <string>})");
+ return luaL_error(L, "Usage: encode(object, {tag_prefix = <string>, "\
+ "tag_handle = <string>})");
+ }
+ const char *prefix = NULL, *handle = NULL;
+ if (top == 2 && !lua_isnil(L, 2)) {
+ if (! lua_istable(L, 2))
+ goto usage_error;
+ lua_getfield(L, 2, "tag_prefix");
+ if (lua_isstring(L, -1))
+ prefix = lua_tostring(L, -1);
+ else if (! lua_isnil(L, -1))
+ goto usage_error;
+
+ lua_getfield(L, 2, "tag_handle");
+ if (lua_isstring(L, -1))
+ handle = lua_tostring(L, -1);
+ else if (! lua_isnil(L, -1))
+ goto usage_error;
+
+ if ((prefix == NULL) != (handle == NULL))
+ goto usage_error;
}
- lua_getfield(L, 2, "prefix");
- if (! lua_isstring(L, -1))
- goto usage_error;
- const char *prefix = lua_tostring(L, -1);
- lua_getfield(L, 2, "handle");
- if (! lua_isstring(L, -1))
- goto usage_error;
- const char *handle = lua_tostring(L, -1);
- return lua_yaml_encode_impl(L, serializer, handle, prefix);
+ return lua_yaml_encode(L, serializer, handle, prefix);
}
static int
@@ -777,7 +771,6 @@ l_new(lua_State *L);
static const luaL_Reg yamllib[] = {
{ "encode", l_dump },
- { "encode_tagged", l_dump_tagged },
{ "decode", l_load },
{ "new", l_new },
{ NULL, NULL}
^ permalink raw reply [flat|nested] 34+ messages in thread
* [PATCH v2 03/10] yaml: introduce yaml.decode_tag
2018-04-20 13:24 [PATCH v2 00/10] session: introduce box.session.push Vladislav Shpilevoy
` (2 preceding siblings ...)
2018-04-20 13:24 ` [PATCH v2 02/10] yaml: introduce yaml.encode_tagged Vladislav Shpilevoy
@ 2018-04-20 13:24 ` Vladislav Shpilevoy
2018-05-10 18:41 ` [tarantool-patches] " Konstantin Osipov
2018-04-20 13:24 ` [PATCH v2 04/10] console: use Lua C API to do formatting for console Vladislav Shpilevoy
` (6 subsequent siblings)
10 siblings, 1 reply; 34+ messages in thread
From: Vladislav Shpilevoy @ 2018-04-20 13:24 UTC (permalink / raw)
To: tarantool-patches; +Cc: vdavydov.dev
Yaml.decode_tag allows to decode a single tag of a YAML document.
For #2677 it is needed to detect different push types in text
console: print pushes via console.print, and actual pushes via
box.session.push.
To distinguish them YAML tags will be used. A client console for
each message will try to find a tag. If a tag is absent, then the
message is a simple response on a request.
If a tag is !print!, then the document consists of a single
string, that must be printed. Such a document must be decoded to
get the printed string. So the calls sequence is yaml.decode_tag
+ yaml.decode. The reason why a print message must be decoded
is that a print() result on a server side can be not
well-formatted YAML, and must be encoded into it to be correctly
send. For example, when I do on a server side something like
this:
console.print('very bad YAML string')
The result of a print() is not a YAML document, and to be sent it
must be encoded into YAML on a server side.
If a tag is !push!, then the document is sent via
box.session.push, and must not be decoded. It can be just printed
or ignored or something.
Needed for #2677
---
test/app-tap/yaml.test.lua | 30 ++++++++++++++++++-
third_party/lua-yaml/lyaml.cc | 67 +++++++++++++++++++++++++++++++++++++++++++
2 files changed, 96 insertions(+), 1 deletion(-)
diff --git a/test/app-tap/yaml.test.lua b/test/app-tap/yaml.test.lua
index b81402cc0..ef64509fe 100755
--- a/test/app-tap/yaml.test.lua
+++ b/test/app-tap/yaml.test.lua
@@ -70,7 +70,10 @@ local function test_output(test, s)
end
local function test_tagged(test, s)
- test:plan(7)
+ test:plan(15)
+ --
+ -- Test encoding tags.
+ --
local must_be_err = 'Usage: encode_tagged({prefix = <string>, handle = <string>}, object)'
local prefix = 'tag:tarantool.io/push,2018'
local ok, err = pcall(s.encode_tagged)
@@ -87,6 +90,31 @@ local function test_tagged(test, s)
test:is(ret, "%TAG !push! "..prefix.."\n--- 300\n...\n", "encode_tagged usage")
ret = s.encode_tagged({handle = '!print!', prefix = prefix}, {a = 100, b = 200})
test:is(ret, "%TAG !print! tag:tarantool.io/push,2018\n---\na: 100\nb: 200\n...\n", 'encode_tagged usage')
+ --
+ -- Test decoding tags.
+ --
+ must_be_err = "Usage: decode_tag(<string>)"
+ ok, err = pcall(s.decode_tag)
+ test:is(err, must_be_err, "decode_tag usage")
+ ok, err = pcall(s.decode_tag, false)
+ test:is(err, must_be_err, "decode_tag usage")
+ local handle, prefix = s.decode_tag(ret)
+ test:is(handle, "!print!", "handle is decoded ok")
+ test:is(prefix, "tag:tarantool.io/push,2018", "prefix is decoded ok")
+ local several_tags =
+[[%TAG !tag1! tag:tarantool.io/tag1,2018
+%TAG !tag2! tag:tarantool.io/tag2,2018
+---
+- 100
+...
+]]
+ ok, err = s.decode_tag(several_tags)
+ test:is(ok, nil, "can not decode multiple tags")
+ test:is(err, "can not decode multiple tags", "same")
+ local no_tags = s.encode(100)
+ handle, prefix = s.decode_tag(no_tags)
+ test:is(handle, nil, "no tag - no handle")
+ test:is(prefix, nil, "no tag - no prefix")
end
tap.test("yaml", function(test)
diff --git a/third_party/lua-yaml/lyaml.cc b/third_party/lua-yaml/lyaml.cc
index 8329f84e9..d24715edd 100644
--- a/third_party/lua-yaml/lyaml.cc
+++ b/third_party/lua-yaml/lyaml.cc
@@ -361,6 +361,72 @@ static int l_load(lua_State *L) {
return loader.document_count;
}
+/**
+ * Decode a global tag of document. Multiple tags can not be
+ * decoded. In a case of multiple documents only first is
+ * processed.
+ * @param YAML document in string.
+ * @retval nil, err Error occured during decoding. In the second
+ * value is error description.
+ * @retval nil, nil A document does not contain tags.
+ * @retval handle, prefix Handle and prefix of a tag.
+ */
+static int
+l_load_tag(struct lua_State *L)
+{
+ if (lua_gettop(L) != 1 || !lua_isstring(L, 1))
+ return luaL_error(L, "Usage: decode_tag(<string>)");
+ size_t len;
+ const char *str = lua_tolstring(L, 1, &len);
+ struct lua_yaml_loader loader;
+ memset(&loader, 0, sizeof(loader));
+ loader.L = L;
+ loader.cfg = luaL_checkserializer(L);
+ if (yaml_parser_initialize(&loader.parser) == 0)
+ luaL_error(L, OOM_ERRMSG);
+ yaml_tag_directive_t *start, *end;
+ yaml_parser_set_input_string(&loader.parser, (const unsigned char *) str,
+ len);
+ /* Initial parser step. Detect the documents start position. */
+ if (do_parse(&loader) == 0)
+ goto parse_error;
+ if (loader.event.type != YAML_STREAM_START_EVENT) {
+ lua_pushnil(L);
+ lua_pushstring(L, "expected STREAM_START_EVENT");
+ return 2;
+ }
+ /* Parse a document start. */
+ if (do_parse(&loader) == 0)
+ goto parse_error;
+ if (loader.event.type == YAML_STREAM_END_EVENT)
+ goto no_tags;
+ assert(loader.event.type == YAML_DOCUMENT_START_EVENT);
+ start = loader.event.data.document_start.tag_directives.start;
+ end = loader.event.data.document_start.tag_directives.end;
+ if (start == end)
+ goto no_tags;
+ if (end - start > 1) {
+ lua_pushnil(L);
+ lua_pushstring(L, "can not decode multiple tags");
+ return 2;
+ }
+ lua_pushstring(L, (const char *) start->handle);
+ lua_pushstring(L, (const char *) start->prefix);
+ delete_event(&loader);
+ yaml_parser_delete(&loader.parser);
+ return 2;
+
+parse_error:
+ lua_pushnil(L);
+ /* Make nil be before an error message. */
+ lua_insert(L, -2);
+ return 2;
+
+no_tags:
+ lua_pushnil(L);
+ return 1;
+}
+
static int dump_node(struct lua_yaml_dumper *dumper);
static yaml_char_t *get_yaml_anchor(struct lua_yaml_dumper *dumper) {
@@ -753,6 +819,7 @@ static const luaL_Reg yamllib[] = {
{ "encode", l_dump },
{ "encode_tagged", l_dump_tagged },
{ "decode", l_load },
+ { "decode_tag", l_load_tag },
{ "new", l_new },
{ NULL, NULL}
};
--
2.15.1 (Apple Git-101)
^ permalink raw reply [flat|nested] 34+ messages in thread* Re: [tarantool-patches] [PATCH v2 03/10] yaml: introduce yaml.decode_tag
2018-04-20 13:24 ` [PATCH v2 03/10] yaml: introduce yaml.decode_tag Vladislav Shpilevoy
@ 2018-05-10 18:41 ` Konstantin Osipov
2018-05-24 20:50 ` [tarantool-patches] " Vladislav Shpilevoy
0 siblings, 1 reply; 34+ messages in thread
From: Konstantin Osipov @ 2018-05-10 18:41 UTC (permalink / raw)
To: tarantool-patches; +Cc: vdavydov.dev
* Vladislav Shpilevoy <v.shpilevoy@tarantool.org> [18/04/20 16:25]:
> Yaml.decode_tag allows to decode a single tag of a YAML document.
Why do you need a function to decode just the tag, not an entire
document with tags?
>
> To distinguish them YAML tags will be used. A client console for
> each message will try to find a tag. If a tag is absent, then the
> message is a simple response on a request.
response to
> If a tag is !print!, then the document consists of a single
> string, that must be printed. Such a document must be decoded to
> get the printed string. So the calls sequence is yaml.decode_tag
> + yaml.decode. The reason why a print message must be decoded
> is that a print() result on a server side can be not
> well-formatted YAML, and must be encoded into it to be correctly
> send. For example, when I do on a server side something like
> console.print('very bad YAML string')
>
> The result of a print() is not a YAML document, and to be sent it
> must be encoded into YAML on a server side.
>
> If a tag is !push!, then the document is sent via
> box.session.push, and must not be decoded. It can be just printed
> or ignored or something.
It is nice you explain this convention in a changeset comment, but
I'd suggest to move the explanation to the relevant commit, i.e.
the one which uses the api you're adding in this commit.
>
> Needed for #2677
> ---
> test/app-tap/yaml.test.lua | 30 ++++++++++++++++++-
> third_party/lua-yaml/lyaml.cc | 67 +++++++++++++++++++++++++++++++++++++++++++
> 2 files changed, 96 insertions(+), 1 deletion(-)
>
> diff --git a/test/app-tap/yaml.test.lua b/test/app-tap/yaml.test.lua
> index b81402cc0..ef64509fe 100755
> --- a/test/app-tap/yaml.test.lua
> +++ b/test/app-tap/yaml.test.lua
> @@ -70,7 +70,10 @@ local function test_output(test, s)
> end
>
> local function test_tagged(test, s)
> - test:plan(7)
> + test:plan(15)
> + --
> + -- Test encoding tags.
> + --
> local must_be_err = 'Usage: encode_tagged({prefix = <string>, handle = <string>}, object)'
> local prefix = 'tag:tarantool.io/push,2018'
> local ok, err = pcall(s.encode_tagged)
> @@ -87,6 +90,31 @@ local function test_tagged(test, s)
> test:is(ret, "%TAG !push! "..prefix.."\n--- 300\n...\n", "encode_tagged usage")
> ret = s.encode_tagged({handle = '!print!', prefix = prefix}, {a = 100, b = 200})
> test:is(ret, "%TAG !print! tag:tarantool.io/push,2018\n---\na: 100\nb: 200\n...\n", 'encode_tagged usage')
> + --
> + -- Test decoding tags.
> + --
> + must_be_err = "Usage: decode_tag(<string>)"
This is a bad test - any change to the error message will require
changes to the test. If you're testing that there is a relevant
usage message, you could search for substring.
As a person reading your code and test for it, I am at a loss what
kind of string the API expects. Is it a fragment of YAML document
containing the tag? Is it a single YAML document? Is it a stream
with multiple documents? I can see you're passing the return value
from the encoder into decode_tag(). Why is the API a-symmetric?
It should not be asymmetric in the first place, but if you decided
to make it one please this deserves an explanation in the
changeset comment.
> + ok, err = pcall(s.decode_tag)
I suggest you simply have encode_tagged and decode_tagged.
Or even simpler, extend dump/load, with tag support.
> + test:is(err, must_be_err, "decode_tag usage")
> + ok, err = pcall(s.decode_tag, false)
> + test:is(err, must_be_err, "decode_tag usage")
> + local handle, prefix = s.decode_tag(ret)
> + test:is(handle, "!print!", "handle is decoded ok")
> + test:is(prefix, "tag:tarantool.io/push,2018", "prefix is decoded ok")
> + local several_tags =
> +[[%TAG !tag1! tag:tarantool.io/tag1,2018
> +%TAG !tag2! tag:tarantool.io/tag2,2018
Please add a test case for multiple documents.
> +---
> +- 100
> +...
> +]]
> + ok, err = s.decode_tag(several_tags)
> + test:is(ok, nil, "can not decode multiple tags")
> + test:is(err, "can not decode multiple tags", "same")
> + local no_tags = s.encode(100)
> + handle, prefix = s.decode_tag(no_tags)
> + test:is(handle, nil, "no tag - no handle")
> + test:is(prefix, nil, "no tag - no prefix")
> end
>
> tap.test("yaml", function(test)
> diff --git a/third_party/lua-yaml/lyaml.cc b/third_party/lua-yaml/lyaml.cc
> index 8329f84e9..d24715edd 100644
> --- a/third_party/lua-yaml/lyaml.cc
> +++ b/third_party/lua-yaml/lyaml.cc
> @@ -361,6 +361,72 @@ static int l_load(lua_State *L) {
> return loader.document_count;
> }
>
> +/**
> + * Decode a global tag of document. Multiple tags can not be
> + * decoded. In a case of multiple documents only first is
> + * processed.
Decode a document taking into account document tags.
In case of success, pops the input from the stack and pushed
the document and a table with options, containing tag prefix and
tag handle.
> + * @param YAML document in string.
> + * @retval nil, err Error occured during decoding. In the second
> + * value is error description.
> + * @retval nil, nil A document does not contain tags.
> + * @retval handle, prefix Handle and prefix of a tag.
> + */
> +static int
> +l_load_tag(struct lua_State *L)
Function name in C does not match the Lua name. Please make sure
the names match.
I understand you sometimes may avoid extra work because you don't
believe I am ever going to look at the patch, but this is not
extra work, this is just sloppy code.
> +{
> + if (lua_gettop(L) != 1 || !lua_isstring(L, 1))
> + return luaL_error(L, "Usage: decode_tag(<string>)");
> + size_t len;
> + const char *str = lua_tolstring(L, 1, &len);
> + struct lua_yaml_loader loader;
> + memset(&loader, 0, sizeof(loader));
> + loader.L = L;
> + loader.cfg = luaL_checkserializer(L);
> + if (yaml_parser_initialize(&loader.parser) == 0)
> + luaL_error(L, OOM_ERRMSG);
> + yaml_tag_directive_t *start, *end;
> + yaml_parser_set_input_string(&loader.parser, (const unsigned char *) str,
> + len);
Shouldn't you use yaml typedef here rather than const unsigned
char *?
> + /* Initial parser step. Detect the documents start position. */
> + if (do_parse(&loader) == 0)
> + goto parse_error;
> + if (loader.event.type != YAML_STREAM_START_EVENT) {
> + lua_pushnil(L);
> + lua_pushstring(L, "expected STREAM_START_EVENT");
> + return 2;
> + }
What is the current convention for dump/load API? Does it use nil,
err or lua_error() for errors?
Why did you decide to depart from the current convention?
> + /* Parse a document start. */
> + if (do_parse(&loader) == 0)
> + goto parse_error;
> + if (loader.event.type == YAML_STREAM_END_EVENT)
> + goto no_tags;
> + assert(loader.event.type == YAML_DOCUMENT_START_EVENT);
> + start = loader.event.data.document_start.tag_directives.start;
> + end = loader.event.data.document_start.tag_directives.end;
> + if (start == end)
> + goto no_tags;
> + if (end - start > 1) {
> + lua_pushnil(L);
> + lua_pushstring(L, "can not decode multiple tags");
> + return 2;
> + }
> + lua_pushstring(L, (const char *) start->handle);
> + lua_pushstring(L, (const char *) start->prefix);
> + delete_event(&loader);
> + yaml_parser_delete(&loader.parser);
> + return 2;
Why not make the API symmetric in what it expects and returns?
dump(object, options) -> stream
load(stream) -> object, options or error
> +
> +parse_error:
> + lua_pushnil(L);
> + /* Make nil be before an error message. */
> + lua_insert(L, -2);
> + return 2;
> +
> +no_tags:
> + lua_pushnil(L);
> + return 1;
> +}
> +
> static int dump_node(struct lua_yaml_dumper *dumper);
>
> static yaml_char_t *get_yaml_anchor(struct lua_yaml_dumper *dumper) {
> @@ -753,6 +819,7 @@ static const luaL_Reg yamllib[] = {
> { "encode", l_dump },
> { "encode_tagged", l_dump_tagged },
> { "decode", l_load },
> + { "decode_tag", l_load_tag },
> { "new", l_new },
> { NULL, NULL}
> };
--
Konstantin Osipov, Moscow, Russia, +7 903 626 22 32
http://tarantool.io - www.twitter.com/kostja_osipov
^ permalink raw reply [flat|nested] 34+ messages in thread* Re: [tarantool-patches] Re: [PATCH v2 03/10] yaml: introduce yaml.decode_tag
2018-05-10 18:41 ` [tarantool-patches] " Konstantin Osipov
@ 2018-05-24 20:50 ` Vladislav Shpilevoy
2018-05-31 10:54 ` Konstantin Osipov
2018-05-31 11:36 ` Konstantin Osipov
0 siblings, 2 replies; 34+ messages in thread
From: Vladislav Shpilevoy @ 2018-05-24 20:50 UTC (permalink / raw)
To: tarantool-patches, Konstantin Osipov; +Cc: vdavydov.dev
On 10/05/2018 21:41, Konstantin Osipov wrote:
> * Vladislav Shpilevoy <v.shpilevoy@tarantool.org> [18/04/20 16:25]:
>
>> Yaml.decode_tag allows to decode a single tag of a YAML document.
>
> Why do you need a function to decode just the tag, not an entire
> document with tags?
Fixed on the branch. See the complete patch at the end of the letter. But
IMO now decode() function looks ugly. Option 'tag_only' requires to
create a Lua table on each call with the single option when it is needed.
And 'tag_only' itself is very strange. It is the same as we would make
message pack types as options like this:
- msgpack.decode_array/unsigned/string ...
+ msgpack.decode(..., {array/unsigned/string... = true})
But as you wish.
>
>>
>> To distinguish them YAML tags will be used. A client console for
>> each message will try to find a tag. If a tag is absent, then the
>> message is a simple response on a request.
>
> response to
Ditto.
>> @@ -87,6 +90,31 @@ local function test_tagged(test, s)
>> test:is(ret, "%TAG !push! "..prefix.."\n--- 300\n...\n", "encode_tagged usage")
>> ret = s.encode_tagged({handle = '!print!', prefix = prefix}, {a = 100, b = 200})
>> test:is(ret, "%TAG !print! tag:tarantool.io/push,2018\n---\na: 100\nb: 200\n...\n", 'encode_tagged usage')
>> + --
>> + -- Test decoding tags.
>> + --
>> + must_be_err = "Usage: decode_tag(<string>)"
>
> This is a bad test - any change to the error message will require
> changes to the test. If you're testing that there is a relevant
> usage message, you could search for substring.
@@ -93,11 +93,10 @@ local function test_tagged(test, s)
--
-- Test decoding tags.
--
- must_be_err = "Usage: yaml.decode(document, [{tag_only = boolean}])"
ok, err = pcall(s.decode)
- test:is(err, must_be_err, "decode usage")
+ test:isnt(err:find('Usage'), nil, "decode usage")
ok, err = pcall(s.decode, false)
- test:is(err, must_be_err, "decode usage")
+ test:isnt(err:find('Usage'), nil, "decode usage")
local handle, prefix = s.decode(ret, {tag_only = true})
test:is(handle, "!print!", "handle is decoded ok")
test:is(prefix, "tag:tarantool.io/push,2018", "prefix is decoded ok")
Answers on your questions below are in the commit comment already. I will
copy-paste them here for you.
>
> As a person reading your code and test for it, I am at a loss what
> kind of string the API expects. Is it a fragment of YAML document
> containing the tag? Is it a single YAML document? Is it a stream
> with multiple documents?
Cite: "Yaml.decode tag_only option allows to decode a single tag of a YAML document.".
> I can see you're passing the return value
> from the encoder into decode_tag(). Why is the API a-symmetric?
>
> It should not be asymmetric in the first place, but if you decided
> to make it one please this deserves an explanation in the
> changeset comment.
Cite: "If a tag is !print!, then the document consists of a single
string, that must be printed. Such a document must be decoded to
get the printed string."
Cite: "If a tag is !push!, then the document is sent via
box.session.push, and must not be decoded. It can be just printed
or ignored or something."
Summary: it makes no sense to decode a push message in console, if it will be
in any case encoded back and printed. So the message can be just printed without
decode+encode.
>
>> + ok, err = pcall(s.decode_tag)
>
> I suggest you simply have encode_tagged and decode_tagged.
> Or even simpler, extend dump/load, with tag support.
I do not need full tag support. I need to be able to encode with
a single tag - 'push' or 'print', and decode a *single* tag, not
the entire document. I explained why in the commit comment, and above.
>
>> + test:is(err, must_be_err, "decode_tag usage")
>> + ok, err = pcall(s.decode_tag, false)
>> + test:is(err, must_be_err, "decode_tag usage")
>> + local handle, prefix = s.decode_tag(ret)
>> + test:is(handle, "!print!", "handle is decoded ok")
>> + test:is(prefix, "tag:tarantool.io/push,2018", "prefix is decoded ok")
>> + local several_tags =
>> +[[%TAG !tag1! tag:tarantool.io/tag1,2018
>> +%TAG !tag2! tag:tarantool.io/tag2,2018
>
> Please add a test case for multiple documents.
+ local several_documents =
+[[
+%TAG !tag1! tag:tarantool.io/tag1,2018
+--- 1
+...
+
+%TAG !tag2! tag:tarantool.io/tag2,2018
+--- 2
+...
+
+]]
+ handle, prefix = s.decode(several_documents, {tag_only = true})
+ test:is(handle, "!tag1!", "tag handle on multiple documents")
+ test:is(prefix, "tag:tarantool.io/tag1,2018",
+ "tag prefix on multiple documents")
>> diff --git a/third_party/lua-yaml/lyaml.cc b/third_party/lua-yaml/lyaml.cc
>> index 8329f84e9..d24715edd 100644
>> --- a/third_party/lua-yaml/lyaml.cc
>> +++ b/third_party/lua-yaml/lyaml.cc
>> @@ -361,6 +361,72 @@ static int l_load(lua_State *L) {
>> return loader.document_count;
>> }
>>
>> +/**
>> + * Decode a global tag of document. Multiple tags can not be
>> + * decoded. In a case of multiple documents only first is
>> + * processed.
>
> Decode a document taking into account document tags.
> In case of success, pops the input from the stack and pushed
> the document and a table with options, containing tag prefix and
> tag handle.
Is not relevant after the previous comment.
>
>> + * @param YAML document in string.
>> + * @retval nil, err Error occured during decoding. In the second
>> + * value is error description.
>
>
>> + * @retval nil, nil A document does not contain tags.
>> + * @retval handle, prefix Handle and prefix of a tag.
>> + */
>> +static int
>> +l_load_tag(struct lua_State *L)
>
> Function name in C does not match the Lua name. Please make sure
> the names match.
I did it advisedly. Lyaml library has such naming convention. We must
either convert the entire file, or obey this style.
>
> I understand you sometimes may avoid extra work
Ok, thanks for the 'compliment'. I am 'happy' you think so about me. I
am working every day more then 12 hours for you, so 'obviously' I very very
'do not like' to work trying to avoid any. You are right as always.
> because you don't believe I am ever going to look at the patch, but this is not
> extra work, this is just sloppy code.
>
>> +{
>> + if (lua_gettop(L) != 1 || !lua_isstring(L, 1))
>> + return luaL_error(L, "Usage: decode_tag(<string>)");
>> + size_t len;
>> + const char *str = lua_tolstring(L, 1, &len);
>> + struct lua_yaml_loader loader;
>> + memset(&loader, 0, sizeof(loader));
>> + loader.L = L;
>> + loader.cfg = luaL_checkserializer(L);
>> + if (yaml_parser_initialize(&loader.parser) == 0)
>> + luaL_error(L, OOM_ERRMSG);
>> + yaml_tag_directive_t *start, *end;
>> + yaml_parser_set_input_string(&loader.parser, (const unsigned char *) str,
>> + len);
>
> Shouldn't you use yaml typedef here rather than const unsigned
> char *?
It is not used in yaml_parser_set_input_string declaration, but as you wish.
- yaml_parser_set_input_string(&loader.parser,
- (const unsigned char *) document, len);
+ yaml_parser_set_input_string(&loader.parser, (yaml_char_t *) document, len);
if (lua_gettop(L) > 1) {
>
>> + /* Initial parser step. Detect the documents start position. */
>> + if (do_parse(&loader) == 0)
>> + goto parse_error;
>> + if (loader.event.type != YAML_STREAM_START_EVENT) {
>> + lua_pushnil(L);
>> + lua_pushstring(L, "expected STREAM_START_EVENT");
>> + return 2;
>> + }
>
> What is the current convention for dump/load API? Does it use nil,
> err or lua_error() for errors?
Throw on OOM, return nil and error object on other error types.
>
> Why did you decide to depart from the current convention?
I respect the convention. Our convention is throw on OOM, and nil + error
object else.
>
>> + /* Parse a document start. */
>> + if (do_parse(&loader) == 0)
>> + goto parse_error;
>> + if (loader.event.type == YAML_STREAM_END_EVENT)
>> + goto no_tags;
>> + assert(loader.event.type == YAML_DOCUMENT_START_EVENT);
>> + start = loader.event.data.document_start.tag_directives.start;
>> + end = loader.event.data.document_start.tag_directives.end;
>> + if (start == end)
>> + goto no_tags;
>> + if (end - start > 1) {
>> + lua_pushnil(L);
>> + lua_pushstring(L, "can not decode multiple tags");
>> + return 2;
>> + }
>> + lua_pushstring(L, (const char *) start->handle);
>> + lua_pushstring(L, (const char *) start->prefix);
>> + delete_event(&loader);
>> + yaml_parser_delete(&loader.parser);
>> + return 2;
>
> Why not make the API symmetric in what it expects and returns?
>
> dump(object, options) -> stream
Dump does not return a stream. It returns a single string with YAML documents.
Maybe you meant load? Load returns multiple values, and throws on any error, when
no options are specified. This is its old behavior that I can not break. But for
new functionality (tag_only option) I use the correct convention: objects or
nil + error.
See below the entire patch.
===============================================================================
commit 2d30cb1f3c28faf25fc39e33d7215d09d0814e3f
Author: Vladislav Shpilevoy <v.shpilevoy@tarantool.org>
Date: Tue Apr 3 23:12:50 2018 +0300
yaml: introduce yaml.decode tag_only option
Yaml.decode tag_only option allows to decode a single tag of a
YAML document. For #2677 it is needed to detect different push
types in text console: print pushes via console.print, and actual
pushes via box.session.push.
To distinguish them YAML tags will be used. A client console for
each message will try to find a tag. If a tag is absent, then the
message is a simple response to a request.
If a tag is !print!, then the document consists of a single
string, that must be printed. Such a document must be decoded to
get the printed string. So the calls sequence is
yaml.decode(tag_only) + yaml.decode. The reason why a print
message must be decoded is that a print() result on a server side
can be not well-formatted YAML, and must be encoded into it to be
correctly sent. For example, when I do on a server side something
like this:
console.print('very bad YAML string')
The result of a print() is not a YAML document, and to be sent it
must be encoded into YAML on a server side.
If a tag is !push!, then the document is sent via
box.session.push, and must not be decoded. It can be just printed
or ignored or something.
Needed for #2677
diff --git a/test/app-tap/yaml.test.lua b/test/app-tap/yaml.test.lua
index bb75ce4ea..c6eca100d 100755
--- a/test/app-tap/yaml.test.lua
+++ b/test/app-tap/yaml.test.lua
@@ -70,7 +70,10 @@ local function test_output(test, s)
end
local function test_tagged(test, s)
- test:plan(7)
+ test:plan(17)
+ --
+ -- Test encoding tags.
+ --
local prefix = 'tag:tarantool.io/push,2018'
local ok, err = pcall(s.encode_tagged)
test:isnt(err:find('Usage'), nil, "encode_tagged usage")
@@ -86,6 +89,45 @@ local function test_tagged(test, s)
test:is(ret, "%TAG !push! "..prefix.."\n--- 300\n...\n", "encode_tagged usage")
ret = s.encode_tagged({a = 100, b = 200}, {handle = '!print!', prefix = prefix})
test:is(ret, "%TAG !print! tag:tarantool.io/push,2018\n---\na: 100\nb: 200\n...\n", 'encode_tagged usage')
+ --
+ -- Test decoding tags.
+ --
+ ok, err = pcall(s.decode)
+ test:isnt(err:find('Usage'), nil, "decode usage")
+ ok, err = pcall(s.decode, false)
+ test:isnt(err:find('Usage'), nil, "decode usage")
+ local handle, prefix = s.decode(ret, {tag_only = true})
+ test:is(handle, "!print!", "handle is decoded ok")
+ test:is(prefix, "tag:tarantool.io/push,2018", "prefix is decoded ok")
+ local several_tags =
+[[%TAG !tag1! tag:tarantool.io/tag1,2018
+%TAG !tag2! tag:tarantool.io/tag2,2018
+---
+- 100
+...
+]]
+ ok, err = s.decode(several_tags, {tag_only = true})
+ test:is(ok, nil, "can not decode multiple tags")
+ test:is(err, "can not decode multiple tags", "same")
+ local no_tags = s.encode(100)
+ handle, prefix = s.decode(no_tags, {tag_only = true})
+ test:is(handle, nil, "no tag - no handle")
+ test:is(prefix, nil, "no tag - no prefix")
+ local several_documents =
+[[
+%TAG !tag1! tag:tarantool.io/tag1,2018
+--- 1
+...
+
+%TAG !tag2! tag:tarantool.io/tag2,2018
+--- 2
+...
+
+]]
+ handle, prefix = s.decode(several_documents, {tag_only = true})
+ test:is(handle, "!tag1!", "tag handle on multiple documents")
+ test:is(prefix, "tag:tarantool.io/tag1,2018",
+ "tag prefix on multiple documents")
end
tap.test("yaml", function(test)
diff --git a/third_party/lua-yaml/lyaml.cc b/third_party/lua-yaml/lyaml.cc
index 33057da5d..03573dd39 100644
--- a/third_party/lua-yaml/lyaml.cc
+++ b/third_party/lua-yaml/lyaml.cc
@@ -309,6 +309,53 @@ static int load_node(struct lua_yaml_loader *loader) {
}
}
+/**
+ * Decode YAML document global tag onto Lua stack.
+ * @param loader Initialized loader to load tag from.
+ * @retval 2 Tag handle and prefix are pushed. Both are not nil.
+ * @retval 2 If an error occurred during decoding. Nil and error
+ * string are pushed.
+ */
+static int load_tag(struct lua_yaml_loader *loader) {
+ yaml_tag_directive_t *start, *end;
+ /* Initial parser step. Detect the documents start position. */
+ if (do_parse(loader) == 0)
+ goto parse_error;
+ if (loader->event.type != YAML_STREAM_START_EVENT) {
+ lua_pushnil(loader->L);
+ lua_pushstring(loader->L, "expected STREAM_START_EVENT");
+ return 2;
+ }
+ /* Parse a document start. */
+ if (do_parse(loader) == 0)
+ goto parse_error;
+ if (loader->event.type == YAML_STREAM_END_EVENT)
+ goto no_tags;
+ assert(loader->event.type == YAML_DOCUMENT_START_EVENT);
+ start = loader->event.data.document_start.tag_directives.start;
+ end = loader->event.data.document_start.tag_directives.end;
+ if (start == end)
+ goto no_tags;
+ if (end - start > 1) {
+ lua_pushnil(loader->L);
+ lua_pushstring(loader->L, "can not decode multiple tags");
+ return 2;
+ }
+ lua_pushstring(loader->L, (const char *) start->handle);
+ lua_pushstring(loader->L, (const char *) start->prefix);
+ return 2;
+
+parse_error:
+ lua_pushnil(loader->L);
+ /* Make nil be before an error message. */
+ lua_insert(loader->L, -2);
+ return 2;
+
+no_tags:
+ lua_pushnil(loader->L);
+ return 1;
+}
+
static void load(struct lua_yaml_loader *loader) {
if (!do_parse(loader))
return;
@@ -340,34 +387,59 @@ static void load(struct lua_yaml_loader *loader) {
}
}
+/**
+ * Decode YAML documents onto Lua stack. First value on stack
+ * is string with YAML document. Second value is options:
+ * {tag_only = boolean}. Options are not required.
+ * @retval N Pushed document count, if tag_only option is not
+ * specified, or equals to false.
+ * @retval 2 If tag_only option is true. Tag handle and prefix
+ * are pushed.
+ * @retval 2 If an error occurred during decoding. Nil and error
+ * string are pushed.
+ */
static int l_load(lua_State *L) {
struct lua_yaml_loader loader;
-
- luaL_argcheck(L, lua_isstring(L, 1), 1, "must provide a string argument");
-
+ if (! lua_isstring(L, 1)) {
+usage_error:
+ return luaL_error(L, "Usage: yaml.decode(document, "\
+ "[{tag_only = boolean}])");
+ }
+ size_t len;
+ const char *document = lua_tolstring(L, 1, &len);
loader.L = L;
loader.cfg = luaL_checkserializer(L);
loader.validevent = 0;
loader.error = 0;
loader.document_count = 0;
-
- /* create table used to track anchors */
- lua_newtable(L);
- loader.anchortable_index = lua_gettop(L);
-
if (!yaml_parser_initialize(&loader.parser))
- luaL_error(L, OOM_ERRMSG);
- yaml_parser_set_input_string(&loader.parser,
- (const unsigned char *)lua_tostring(L, 1), lua_strlen(L, 1));
- load(&loader);
+ return luaL_error(L, OOM_ERRMSG);
+ yaml_parser_set_input_string(&loader.parser, (yaml_char_t *) document, len);
+ bool tag_only;
+ if (lua_gettop(L) > 1) {
+ if (! lua_istable(L, 2))
+ goto usage_error;
+ lua_getfield(L, 2, "tag_only");
+ tag_only = lua_isboolean(L, -1) && lua_toboolean(L, -1);
+ } else {
+ tag_only = false;
+ }
+ int rc;
+ if (! tag_only) {
+ /* create table used to track anchors */
+ lua_newtable(L);
+ loader.anchortable_index = lua_gettop(L);
+ load(&loader);
+ if (loader.error)
+ lua_error(L);
+ rc = loader.document_count;
+ } else {
+ rc = load_tag(&loader);
+ }
delete_event(&loader);
yaml_parser_delete(&loader.parser);
-
- if (loader.error)
- lua_error(L);
-
- return loader.document_count;
+ return rc;
}
static int dump_node(struct lua_yaml_dumper *dumper);
^ permalink raw reply [flat|nested] 34+ messages in thread* Re: [tarantool-patches] Re: [PATCH v2 03/10] yaml: introduce yaml.decode_tag
2018-05-24 20:50 ` [tarantool-patches] " Vladislav Shpilevoy
@ 2018-05-31 10:54 ` Konstantin Osipov
2018-05-31 11:36 ` Konstantin Osipov
1 sibling, 0 replies; 34+ messages in thread
From: Konstantin Osipov @ 2018-05-31 10:54 UTC (permalink / raw)
To: Vladislav Shpilevoy; +Cc: tarantool-patches, vdavydov.dev
* Vladislav Shpilevoy <v.shpilevoy@tarantool.org> [18/05/24 23:53]:
> Fixed on the branch. See the complete patch at the end of the letter. But
> IMO now decode() function looks ugly. Option 'tag_only' requires to
> create a Lua table on each call with the single option when it is needed.
> And 'tag_only' itself is very strange. It is the same as we would make
> message pack types as options like this:
>
> - msgpack.decode_array/unsigned/string ...
> + msgpack.decode(..., {array/unsigned/string... = true})
>
> But as you wish.
No, there is a difference. With tags, you still potentially decode
entire stream, the option specifies that you're only interested in
tags. With msgpack.decode_array() you explicitly request to decode
a single element on the stream, and it must be in front of the
stream.
Re extra overhead to build a Lua table, passing options as named
arguments in a single table is our standard convention. If I were
worried about performance, I'd be looking at
https://github.com/tarantool/tarantool/issues/30
Notice how old it is. No likes, no upvotes. I hope now that you're
no longer a student we can finally put the issue of premature
optimization at rest :)
But I have no strong opinion about this actually. It was only a
suggestion, not something I would fight for. If you want to put
encode_tagged/decode_tagged back, I'm OK with it - we can add
options to encode()/decode() any time we want.
--
Konstantin Osipov, Moscow, Russia, +7 903 626 22 32
http://tarantool.io - www.twitter.com/kostja_osipov
^ permalink raw reply [flat|nested] 34+ messages in thread* Re: [tarantool-patches] Re: [PATCH v2 03/10] yaml: introduce yaml.decode_tag
2018-05-24 20:50 ` [tarantool-patches] " Vladislav Shpilevoy
2018-05-31 10:54 ` Konstantin Osipov
@ 2018-05-31 11:36 ` Konstantin Osipov
1 sibling, 0 replies; 34+ messages in thread
From: Konstantin Osipov @ 2018-05-31 11:36 UTC (permalink / raw)
To: Vladislav Shpilevoy; +Cc: tarantool-patches, vdavydov.dev
* Vladislav Shpilevoy <v.shpilevoy@tarantool.org> [18/05/24 23:53]:
I pushed this patch.
It's very unfortunate decode can only decode tags when
tag_only is true.
--
Konstantin Osipov, Moscow, Russia, +7 903 626 22 32
http://tarantool.io - www.twitter.com/kostja_osipov
^ permalink raw reply [flat|nested] 34+ messages in thread
* [PATCH v2 04/10] console: use Lua C API to do formatting for console
2018-04-20 13:24 [PATCH v2 00/10] session: introduce box.session.push Vladislav Shpilevoy
` (3 preceding siblings ...)
2018-04-20 13:24 ` [PATCH v2 03/10] yaml: introduce yaml.decode_tag Vladislav Shpilevoy
@ 2018-04-20 13:24 ` Vladislav Shpilevoy
2018-05-10 18:46 ` [tarantool-patches] " Konstantin Osipov
2018-04-20 13:24 ` [PATCH v2 05/10] session: move salt into iproto connection Vladislav Shpilevoy
` (5 subsequent siblings)
10 siblings, 1 reply; 34+ messages in thread
From: Vladislav Shpilevoy @ 2018-04-20 13:24 UTC (permalink / raw)
To: tarantool-patches; +Cc: vdavydov.dev
YAML formatting C API is needed for #2677, where it will be used
to send text pushes and prints as tagged YAML documents.
Needed for #2677
---
src/box/lua/console.c | 48 +++++++++++++++++++++++++++++++++++++++++++
src/box/lua/console.lua | 32 ++++++-----------------------
third_party/lua-yaml/lyaml.cc | 34 ++++++++++++------------------
third_party/lua-yaml/lyaml.h | 29 ++++++++++++++++++++++++++
4 files changed, 96 insertions(+), 47 deletions(-)
diff --git a/src/box/lua/console.c b/src/box/lua/console.c
index d27d7ecac..f6009d387 100644
--- a/src/box/lua/console.c
+++ b/src/box/lua/console.c
@@ -34,6 +34,7 @@
#include "lua/fiber.h"
#include "fiber.h"
#include "coio.h"
+#include "lua-yaml/lyaml.h"
#include <lua.h>
#include <lauxlib.h>
#include <lualib.h>
@@ -42,6 +43,8 @@
#include <stdlib.h>
#include <ctype.h>
+static struct luaL_serializer *luaL_yaml_default = NULL;
+
/*
* Completion engine (Mike Paul's).
* Used internally when collecting completions locally. Also a Lua
@@ -328,6 +331,32 @@ lbox_console_add_history(struct lua_State *L)
return 0;
}
+/**
+ * Encode Lua object into YAML documents. Gets variable count of
+ * parameters.
+ * @retval String with YAML documents - one per parameter.
+ */
+static int
+lbox_console_format(struct lua_State *L)
+{
+ int arg_count = lua_gettop(L);
+ if (arg_count == 0) {
+ lua_pushstring(L, "---\n...\n");
+ return 1;
+ }
+ lua_createtable(L, arg_count, 0);
+ for (int i = 0; i < arg_count; ++i) {
+ if (lua_isnil(L, i + 1))
+ luaL_pushnull(L);
+ else
+ lua_pushvalue(L, i + 1);
+ lua_rawseti(L, -2, i + 1);
+ }
+ lua_replace(L, 1);
+ lua_settop(L, 1);
+ return lua_yaml_encode(L, luaL_yaml_default);
+}
+
void
tarantool_lua_console_init(struct lua_State *L)
{
@@ -336,6 +365,7 @@ tarantool_lua_console_init(struct lua_State *L)
{"save_history", lbox_console_save_history},
{"add_history", lbox_console_add_history},
{"completion_handler", lbox_console_completion_handler},
+ {"format", lbox_console_format},
{NULL, NULL}
};
luaL_register_module(L, "console", consolelib);
@@ -344,6 +374,24 @@ tarantool_lua_console_init(struct lua_State *L)
lua_getfield(L, -1, "completion_handler");
lua_pushcclosure(L, lbox_console_readline, 1);
lua_setfield(L, -2, "readline");
+
+ luaL_yaml_default = lua_yaml_new_serializer(L);
+ luaL_yaml_default->encode_invalid_numbers = 1;
+ luaL_yaml_default->encode_load_metatables = 1;
+ luaL_yaml_default->encode_use_tostring = 1;
+ luaL_yaml_default->encode_invalid_as_nil = 1;
+ /*
+ * Hold reference to a formatter (the Lua representation
+ * of luaL_yaml_default). It is not visible to a user
+ * here, because require('console') returns modified
+ * package with no formatter. This problem is absent in
+ * similar places like lua/msgpack.c, because they are
+ * serializers and nothing more - they hold
+ * luaL_serializer in LUA_REGISTRYINDEX. Console does the
+ * same, but here a YAML serializer is just a part of
+ * console.
+ */
+ lua_setfield(L, -2, "formatter");
}
/*
diff --git a/src/box/lua/console.lua b/src/box/lua/console.lua
index f1a8f4915..bc4e02bfc 100644
--- a/src/box/lua/console.lua
+++ b/src/box/lua/console.lua
@@ -12,34 +12,11 @@ local net_box = require('net.box')
local YAML_TERM = '\n...\n'
--- admin formatter must be able to encode any Lua variable
-local formatter = yaml.new()
-formatter.cfg{
- encode_invalid_numbers = true;
- encode_load_metatables = true;
- encode_use_tostring = true;
- encode_invalid_as_nil = true;
-}
-
local function format(status, ...)
- -- When storing a nil in a Lua table, there is no way to
- -- distinguish nil value from no value. This is a trick to
- -- make sure yaml converter correctly
- local function wrapnull(v)
- return v == nil and formatter.NULL or v
- end
local err
if status then
- local count = select('#', ...)
- if count == 0 then
- return "---\n...\n"
- end
- local res = {}
- for i=1,count,1 do
- table.insert(res, wrapnull(select(i, ...)))
- end
-- serializer can raise an exception
- status, err = pcall(formatter.encode, res)
+ status, err = pcall(internal.format, ...)
if status then
return err
else
@@ -47,9 +24,12 @@ local function format(status, ...)
tostring(err)
end
else
- err = wrapnull(...)
+ err = ...
+ if err == nil then
+ err = box.NULL
+ end
end
- return formatter.encode({{error = err }})
+ return internal.format({ error = err })
end
--
diff --git a/third_party/lua-yaml/lyaml.cc b/third_party/lua-yaml/lyaml.cc
index d24715edd..2b1860c82 100644
--- a/third_party/lua-yaml/lyaml.cc
+++ b/third_party/lua-yaml/lyaml.cc
@@ -687,21 +687,9 @@ static void find_references(struct lua_yaml_dumper *dumper) {
}
}
-/**
- * Encode a Lua object or objects into YAML documents onto Lua
- * stack.
- * @param L Lua stack to get arguments and push result.
- * @param serializer Lua YAML serializer.
- * @param tag_handle NULL, or a global tag handle. Handle becomes
- * a synonym for prefix.
- * @param tag_prefix NULL, or a global tag prefix, to which @a
- * handle is expanded.
- * @retval nil, error Error.
- * @retval not nil Lua string with dumped object.
- */
-static int
-lua_yaml_encode_impl(lua_State *L, struct luaL_serializer *serializer,
- const char *tag_handle, const char *tag_prefix)
+int
+lua_yaml_encode_tagged(lua_State *L, struct luaL_serializer *serializer,
+ const char *tag_handle, const char *tag_prefix)
{
struct lua_yaml_dumper dumper;
int i, argcount = lua_gettop(L);
@@ -779,7 +767,7 @@ error:
}
static int l_dump(lua_State *L) {
- return lua_yaml_encode_impl(L, luaL_checkserializer(L), NULL, NULL);
+ return lua_yaml_encode_tagged(L, luaL_checkserializer(L), NULL, NULL);
}
/**
@@ -809,11 +797,15 @@ usage_error:
lua_pop(L, 2);
lua_replace(L, 1);
lua_settop(L, 1);
- return lua_yaml_encode_impl(L, serializer, handle, prefix);
+ return lua_yaml_encode_tagged(L, serializer, handle, prefix);
}
static int
-l_new(lua_State *L);
+l_new(lua_State *L)
+{
+ lua_yaml_new_serializer(L);
+ return 1;
+}
static const luaL_Reg yamllib[] = {
{ "encode", l_dump },
@@ -824,12 +816,12 @@ static const luaL_Reg yamllib[] = {
{ NULL, NULL}
};
-static int
-l_new(lua_State *L)
+struct luaL_serializer *
+lua_yaml_new_serializer(lua_State *L)
{
struct luaL_serializer *s = luaL_newserializer(L, NULL, yamllib);
s->has_compact = 1;
- return 1;
+ return s;
}
int
diff --git a/third_party/lua-yaml/lyaml.h b/third_party/lua-yaml/lyaml.h
index 650a2d747..fccb7d18a 100644
--- a/third_party/lua-yaml/lyaml.h
+++ b/third_party/lua-yaml/lyaml.h
@@ -7,9 +7,38 @@ extern "C" {
#include <lua.h>
+struct luaL_serializer;
+
LUALIB_API int
luaopen_yaml(lua_State *L);
+/** @Sa luaL_newserializer(). */
+struct luaL_serializer *
+lua_yaml_new_serializer(lua_State *L);
+
+/**
+ * Encode a Lua object or objects into YAML documents onto Lua
+ * stack.
+ * @param L Lua stack to get arguments and push result.
+ * @param serializer Lua YAML serializer.
+ * @param tag_handle NULL, or a global tag handle. Handle becomes
+ * a synonym for prefix.
+ * @param tag_prefix NULL, or a global tag prefix, to which @a
+ * handle is expanded.
+ * @retval nil, error Error.
+ * @retval not nil Lua string with dumped object.
+ */
+int
+lua_yaml_encode_tagged(lua_State *L, struct luaL_serializer *serializer,
+ const char *tag_handle, const char *tag_prefix);
+
+/** Same as lua_yaml_encode_tagged, but encode with no tags. */
+static inline int
+lua_yaml_encode(lua_State *L, struct luaL_serializer *serializer)
+{
+ return lua_yaml_encode_tagged(L, serializer, NULL, NULL);
+}
+
#ifdef __cplusplus
}
#endif
--
2.15.1 (Apple Git-101)
^ permalink raw reply [flat|nested] 34+ messages in thread* Re: [tarantool-patches] [PATCH v2 04/10] console: use Lua C API to do formatting for console
2018-04-20 13:24 ` [PATCH v2 04/10] console: use Lua C API to do formatting for console Vladislav Shpilevoy
@ 2018-05-10 18:46 ` Konstantin Osipov
2018-05-24 20:50 ` [tarantool-patches] " Vladislav Shpilevoy
0 siblings, 1 reply; 34+ messages in thread
From: Konstantin Osipov @ 2018-05-10 18:46 UTC (permalink / raw)
To: tarantool-patches; +Cc: vdavydov.dev
* Vladislav Shpilevoy <v.shpilevoy@tarantool.org> [18/04/20 16:25]:
> YAML formatting C API is needed for #2677, where it will be used
> to send text pushes and prints as tagged YAML documents.
I don't understand why you need to move dump/format into C
from this comment.
What was wrong with keeping it in Lua implementation?
>
> Needed for #2677
> ---
> src/box/lua/console.c | 48 +++++++++++++++++++++++++++++++++++++++++++
> src/box/lua/console.lua | 32 ++++++-----------------------
> third_party/lua-yaml/lyaml.cc | 34 ++++++++++++------------------
> third_party/lua-yaml/lyaml.h | 29 ++++++++++++++++++++++++++
> 4 files changed, 96 insertions(+), 47 deletions(-)
>
> diff --git a/src/box/lua/console.c b/src/box/lua/console.c
> index d27d7ecac..f6009d387 100644
> --- a/src/box/lua/console.c
> +++ b/src/box/lua/console.c
> @@ -34,6 +34,7 @@
> #include "lua/fiber.h"
> #include "fiber.h"
> #include "coio.h"
> +#include "lua-yaml/lyaml.h"
> #include <lua.h>
> #include <lauxlib.h>
> #include <lualib.h>
> @@ -42,6 +43,8 @@
> #include <stdlib.h>
> #include <ctype.h>
>
> +static struct luaL_serializer *luaL_yaml_default = NULL;
> +
> /*
> * Completion engine (Mike Paul's).
> * Used internally when collecting completions locally. Also a Lua
> @@ -328,6 +331,32 @@ lbox_console_add_history(struct lua_State *L)
> return 0;
> }
>
> +/**
> + * Encode Lua object into YAML documents. Gets variable count of
> + * parameters.
> + * @retval String with YAML documents - one per parameter.
> + */
> +static int
> +lbox_console_format(struct lua_State *L)
> +{
> + int arg_count = lua_gettop(L);
> + if (arg_count == 0) {
> + lua_pushstring(L, "---\n...\n");
> + return 1;
> + }
> + lua_createtable(L, arg_count, 0);
> + for (int i = 0; i < arg_count; ++i) {
> + if (lua_isnil(L, i + 1))
> + luaL_pushnull(L);
> + else
> + lua_pushvalue(L, i + 1);
> + lua_rawseti(L, -2, i + 1);
> + }
> + lua_replace(L, 1);
> + lua_settop(L, 1);
> + return lua_yaml_encode(L, luaL_yaml_default);
> +}
> +
> void
> tarantool_lua_console_init(struct lua_State *L)
> {
> @@ -336,6 +365,7 @@ tarantool_lua_console_init(struct lua_State *L)
> {"save_history", lbox_console_save_history},
> {"add_history", lbox_console_add_history},
> {"completion_handler", lbox_console_completion_handler},
> + {"format", lbox_console_format},
> {NULL, NULL}
> };
> luaL_register_module(L, "console", consolelib);
> @@ -344,6 +374,24 @@ tarantool_lua_console_init(struct lua_State *L)
> lua_getfield(L, -1, "completion_handler");
> lua_pushcclosure(L, lbox_console_readline, 1);
> lua_setfield(L, -2, "readline");
> +
> + luaL_yaml_default = lua_yaml_new_serializer(L);
> + luaL_yaml_default->encode_invalid_numbers = 1;
> + luaL_yaml_default->encode_load_metatables = 1;
> + luaL_yaml_default->encode_use_tostring = 1;
> + luaL_yaml_default->encode_invalid_as_nil = 1;
> + /*
> + * Hold reference to a formatter (the Lua representation
> + * of luaL_yaml_default).
> It is not visible to a user
> + * here, because require('console') returns modified
> + * package with no formatter. This problem is absent in
> + * similar places like lua/msgpack.c, because they are
> + * serializers and nothing more - they hold
> + * luaL_serializer in LUA_REGISTRYINDEX. Console does the
> + * same, but here a YAML serializer is just a part of
> + * console.
> + */
I don't understand the comment beyond the first sentence.
> + lua_setfield(L, -2, "formatter");
> }
>
> /*
> +/**
> + * Encode a Lua object or objects into YAML documents onto Lua
> + * stack.
> + * @param L Lua stack to get arguments and push result.
> + * @param serializer Lua YAML serializer.
> + * @param tag_handle NULL, or a global tag handle. Handle becomes
> + * a synonym for prefix.
> + * @param tag_prefix NULL, or a global tag prefix, to which @a
> + * handle is expanded.
> + * @retval nil, error Error.
> + * @retval not nil Lua string with dumped object.
> + */
> +int
> +lua_yaml_encode_tagged(lua_State *L, struct luaL_serializer *serializer,
> + const char *tag_handle, const char *tag_prefix);
No need to keep the comments in two places. We have decided to
keep the comments around declarations, not definitions, a while
ago (please make sure to update the server engineering guidelines
if it is not reflected there or in the coding style).
> +
> +/** Same as lua_yaml_encode_tagged, but encode with no tags. */
> +static inline int
> +lua_yaml_encode(lua_State *L, struct luaL_serializer *serializer)
> +{
> + return lua_yaml_encode_tagged(L, serializer, NULL, NULL);
> +}
--
Konstantin Osipov, Moscow, Russia, +7 903 626 22 32
http://tarantool.io - www.twitter.com/kostja_osipov
^ permalink raw reply [flat|nested] 34+ messages in thread* Re: [tarantool-patches] Re: [PATCH v2 04/10] console: use Lua C API to do formatting for console
2018-05-10 18:46 ` [tarantool-patches] " Konstantin Osipov
@ 2018-05-24 20:50 ` Vladislav Shpilevoy
0 siblings, 0 replies; 34+ messages in thread
From: Vladislav Shpilevoy @ 2018-05-24 20:50 UTC (permalink / raw)
To: tarantool-patches, Konstantin Osipov; +Cc: vdavydov.dev
On 10/05/2018 21:46, Konstantin Osipov wrote:
> * Vladislav Shpilevoy <v.shpilevoy@tarantool.org> [18/04/20 16:25]:
>> YAML formatting C API is needed for #2677, where it will be used
>> to send text pushes and prints as tagged YAML documents.
>
> I don't understand why you need to move dump/format into C
> from this comment.
>
> What was wrong with keeping it in Lua implementation?
New commit message:
console: use Lua C API to do formatting for console
YAML formatting C API is needed for #2677, where it will be used
to send text pushes and prints as tagged YAML documents.
Push in the next patches is implemented as a virtual C method of
struct session, so it is necessary to be able format YAML from C.
Needed for #2677
>
>>
>> Needed for #2677
>> ---
>> src/box/lua/console.c | 48 +++++++++++++++++++++++++++++++++++++++++++
>> src/box/lua/console.lua | 32 ++++++-----------------------
>> third_party/lua-yaml/lyaml.cc | 34 ++++++++++++------------------
>> third_party/lua-yaml/lyaml.h | 29 ++++++++++++++++++++++++++
>> 4 files changed, 96 insertions(+), 47 deletions(-)
>>
>> diff --git a/src/box/lua/console.c b/src/box/lua/console.c
>> index d27d7ecac..f6009d387 100644
>> --- a/src/box/lua/console.c
>> +++ b/src/box/lua/console.c
>> @@ -34,6 +34,7 @@
>> #include "lua/fiber.h"
>> #include "fiber.h"
>> #include "coio.h"
>> +#include "lua-yaml/lyaml.h"
>> #include <lua.h>
>> #include <lauxlib.h>
>> #include <lualib.h>
>> @@ -42,6 +43,8 @@
>> #include <stdlib.h>
>> #include <ctype.h>
>>
>> +static struct luaL_serializer *luaL_yaml_default = NULL;
>> +
>> /*
>> * Completion engine (Mike Paul's).
>> * Used internally when collecting completions locally. Also a Lua
>> @@ -328,6 +331,32 @@ lbox_console_add_history(struct lua_State *L)
>> return 0;
>> }
>>
>> +/**
>> + * Encode Lua object into YAML documents. Gets variable count of
>> + * parameters.
>> + * @retval String with YAML documents - one per parameter.
>> + */
>> +static int
>> +lbox_console_format(struct lua_State *L)
>> +{
>> + int arg_count = lua_gettop(L);
>> + if (arg_count == 0) {
>> + lua_pushstring(L, "---\n...\n");
>> + return 1;
>> + }
>> + lua_createtable(L, arg_count, 0);
>> + for (int i = 0; i < arg_count; ++i) {
>> + if (lua_isnil(L, i + 1))
>> + luaL_pushnull(L);
>> + else
>> + lua_pushvalue(L, i + 1);
>> + lua_rawseti(L, -2, i + 1);
>> + }
>> + lua_replace(L, 1);
>> + lua_settop(L, 1);
>> + return lua_yaml_encode(L, luaL_yaml_default);
>> +}
>> +
>> void
>> tarantool_lua_console_init(struct lua_State *L)
>> {
>> @@ -336,6 +365,7 @@ tarantool_lua_console_init(struct lua_State *L)
>> {"save_history", lbox_console_save_history},
>> {"add_history", lbox_console_add_history},
>> {"completion_handler", lbox_console_completion_handler},
>> + {"format", lbox_console_format},
>> {NULL, NULL}
>> };
>> luaL_register_module(L, "console", consolelib);
>> @@ -344,6 +374,24 @@ tarantool_lua_console_init(struct lua_State *L)
>> lua_getfield(L, -1, "completion_handler");
>> lua_pushcclosure(L, lbox_console_readline, 1);
>> lua_setfield(L, -2, "readline");
>> +
>> + luaL_yaml_default = lua_yaml_new_serializer(L);
>> + luaL_yaml_default->encode_invalid_numbers = 1;
>> + luaL_yaml_default->encode_load_metatables = 1;
>> + luaL_yaml_default->encode_use_tostring = 1;
>> + luaL_yaml_default->encode_invalid_as_nil = 1;
>> + /*
>> + * Hold reference to a formatter (the Lua representation
>> + * of luaL_yaml_default).
>> It is not visible to a user
>> + * here, because require('console') returns modified
>> + * package with no formatter. This problem is absent in
>> + * similar places like lua/msgpack.c, because they are
>> + * serializers and nothing more - they hold
>> + * luaL_serializer in LUA_REGISTRYINDEX. Console does the
>> + * same, but here a YAML serializer is just a part of
>> + * console.
>> + */
>
> I don't understand the comment beyond the first sentence.
I commiserate. Maybe the new comment is more understandable.
diff --git a/src/box/lua/console.c b/src/box/lua/console.c
index a85a43dbd..ffd1347de 100644
--- a/src/box/lua/console.c
+++ b/src/box/lua/console.c
@@ -383,15 +383,12 @@ tarantool_lua_console_init(struct lua_State *L)
luaL_yaml_default->encode_use_tostring = 1;
luaL_yaml_default->encode_invalid_as_nil = 1;
/*
- * Hold reference to a formatter (the Lua representation
- * of luaL_yaml_default). It is not visible to a user
- * here, because require('console') returns modified
- * package with no formatter. This problem is absent in
- * similar places like lua/msgpack.c, because they are
- * serializers and nothing more - they hold
- * luaL_serializer in LUA_REGISTRYINDEX. Console does the
- * same, but here a YAML serializer is just a part of
- * console.
+ * Hold reference to the formatter in module local
+ * variable.
+ *
+ * This member is not visible to a user, because
+ * console.lua modifies itself, holding the formatter in
+ * module local variable. Add_history, save_history,
+ * load_history work the same way.
*/
lua_setfield(L, -2, "formatter");
}
>
>> + lua_setfield(L, -2, "formatter");
>> }
>>
>> /*
>> +/**
>> + * Encode a Lua object or objects into YAML documents onto Lua
>> + * stack.
>> + * @param L Lua stack to get arguments and push result.
>> + * @param serializer Lua YAML serializer.
>> + * @param tag_handle NULL, or a global tag handle. Handle becomes
>> + * a synonym for prefix.
>> + * @param tag_prefix NULL, or a global tag prefix, to which @a
>> + * handle is expanded.
>> + * @retval nil, error Error.
>> + * @retval not nil Lua string with dumped object.
>> + */
>> +int
>> +lua_yaml_encode_tagged(lua_State *L, struct luaL_serializer *serializer,
>> + const char *tag_handle, const char *tag_prefix);
>
> No need to keep the comments in two places. We have decided to
> keep the comments around declarations, not definitions, a while
> ago (please make sure to update the server engineering guidelines
> if it is not reflected there or in the coding style).
I do not keep comments in two places. This comment is removed from lyaml.cc.
And you could see it in the diff above. Anyway in the new patch based on previous
review fixes this function is new, and is not just exposed into the header file. So
the comment is not moved or duplicated.
See the entire diff below:
=======================================================================
commit 52dbf38e8657b21d3870297611e439125dd12235
Author: Vladislav Shpilevoy <v.shpilevoy@tarantool.org>
Date: Thu Apr 5 20:20:24 2018 +0300
console: use Lua C API to do formatting for console
YAML formatting C API is needed for #2677, where it will be used
to send text pushes and prints as tagged YAML documents.
Push in the next patches is implemented as a virtual C method of
struct session, so it is necessary to be able format YAML from C.
Needed for #2677
diff --git a/src/box/lua/console.c b/src/box/lua/console.c
index 7a5ac5550..31614b4a8 100644
--- a/src/box/lua/console.c
+++ b/src/box/lua/console.c
@@ -34,6 +34,7 @@
#include "lua/fiber.h"
#include "fiber.h"
#include "coio.h"
+#include "lua-yaml/lyaml.h"
#include <lua.h>
#include <lauxlib.h>
#include <lualib.h>
@@ -42,6 +43,8 @@
#include <stdlib.h>
#include <ctype.h>
+static struct luaL_serializer *luaL_yaml_default = NULL;
+
/*
* Completion engine (Mike Paul's).
* Used internally when collecting completions locally. Also a Lua
@@ -329,6 +332,33 @@ lbox_console_add_history(struct lua_State *L)
return 0;
}
+/**
+ * Encode Lua object into YAML documents. Gets variable count of
+ * parameters.
+ * @retval 1 Pushes string with YAML documents - one per
+ * parameter.
+ */
+static int
+lbox_console_format(struct lua_State *L)
+{
+ int arg_count = lua_gettop(L);
+ if (arg_count == 0) {
+ lua_pushstring(L, "---\n...\n");
+ return 1;
+ }
+ lua_createtable(L, arg_count, 0);
+ for (int i = 0; i < arg_count; ++i) {
+ if (lua_isnil(L, i + 1))
+ luaL_pushnull(L);
+ else
+ lua_pushvalue(L, i + 1);
+ lua_rawseti(L, -2, i + 1);
+ }
+ lua_replace(L, 1);
+ lua_settop(L, 1);
+ return lua_yaml_encode(L, luaL_yaml_default);
+}
+
void
tarantool_lua_console_init(struct lua_State *L)
{
@@ -337,6 +367,7 @@ tarantool_lua_console_init(struct lua_State *L)
{"save_history", lbox_console_save_history},
{"add_history", lbox_console_add_history},
{"completion_handler", lbox_console_completion_handler},
+ {"format", lbox_console_format},
{NULL, NULL}
};
luaL_register_module(L, "console", consolelib);
@@ -345,6 +376,22 @@ tarantool_lua_console_init(struct lua_State *L)
lua_getfield(L, -1, "completion_handler");
lua_pushcclosure(L, lbox_console_readline, 1);
lua_setfield(L, -2, "readline");
+
+ luaL_yaml_default = lua_yaml_new_serializer(L);
+ luaL_yaml_default->encode_invalid_numbers = 1;
+ luaL_yaml_default->encode_load_metatables = 1;
+ luaL_yaml_default->encode_use_tostring = 1;
+ luaL_yaml_default->encode_invalid_as_nil = 1;
+ /*
+ * Hold reference to the formatter in module local
+ * variable.
+ *
+ * This member is not visible to a user, because
+ * console.lua modifies itself, holding the formatter in
+ * module local variable. Add_history, save_history,
+ * load_history work the same way.
+ */
+ lua_setfield(L, -2, "formatter");
}
/*
diff --git a/src/box/lua/console.lua b/src/box/lua/console.lua
index f1a8f4915..bc4e02bfc 100644
--- a/src/box/lua/console.lua
+++ b/src/box/lua/console.lua
@@ -12,34 +12,11 @@ local net_box = require('net.box')
local YAML_TERM = '\n...\n'
--- admin formatter must be able to encode any Lua variable
-local formatter = yaml.new()
-formatter.cfg{
- encode_invalid_numbers = true;
- encode_load_metatables = true;
- encode_use_tostring = true;
- encode_invalid_as_nil = true;
-}
-
local function format(status, ...)
- -- When storing a nil in a Lua table, there is no way to
- -- distinguish nil value from no value. This is a trick to
- -- make sure yaml converter correctly
- local function wrapnull(v)
- return v == nil and formatter.NULL or v
- end
local err
if status then
- local count = select('#', ...)
- if count == 0 then
- return "---\n...\n"
- end
- local res = {}
- for i=1,count,1 do
- table.insert(res, wrapnull(select(i, ...)))
- end
-- serializer can raise an exception
- status, err = pcall(formatter.encode, res)
+ status, err = pcall(internal.format, ...)
if status then
return err
else
@@ -47,9 +24,12 @@ local function format(status, ...)
tostring(err)
end
else
- err = wrapnull(...)
+ err = ...
+ if err == nil then
+ err = box.NULL
+ end
end
- return formatter.encode({{error = err }})
+ return internal.format({ error = err })
end
--
diff --git a/third_party/lua-yaml/lyaml.cc b/third_party/lua-yaml/lyaml.cc
index 1544e08b4..c66afd817 100644
--- a/third_party/lua-yaml/lyaml.cc
+++ b/third_party/lua-yaml/lyaml.cc
@@ -807,6 +807,19 @@ error:
}
}
+int lua_yaml_encode_tagged(lua_State *L, struct luaL_serializer *serializer,
+ const char *tag_handle, const char *tag_prefix)
+{
+ assert(tag_handle != NULL && tag_prefix != NULL);
+ assert(lua_gettop(L) == 1);
+ return lua_yaml_encode_impl(L, serializer, tag_handle, tag_prefix);
+}
+
+int lua_yaml_encode(lua_State *L, struct luaL_serializer *serializer)
+{
+ return lua_yaml_encode_impl(L, serializer, NULL, NULL);
+}
+
/**
* Dump Lua objects into YAML string. It takes any argument count,
* and dumps each in a separate document.
@@ -845,7 +858,11 @@ usage_error:
}
static int
-l_new(lua_State *L);
+l_new(lua_State *L)
+{
+ lua_yaml_new_serializer(L);
+ return 1;
+}
static const luaL_Reg yamllib[] = {
{ "encode", l_dump },
@@ -855,12 +872,12 @@ static const luaL_Reg yamllib[] = {
{ NULL, NULL}
};
-static int
-l_new(lua_State *L)
+struct luaL_serializer *
+lua_yaml_new_serializer(lua_State *L)
{
struct luaL_serializer *s = luaL_newserializer(L, NULL, yamllib);
s->has_compact = 1;
- return 1;
+ return s;
}
int
diff --git a/third_party/lua-yaml/lyaml.h b/third_party/lua-yaml/lyaml.h
index 650a2d747..ac3cb8ca7 100644
--- a/third_party/lua-yaml/lyaml.h
+++ b/third_party/lua-yaml/lyaml.h
@@ -7,9 +7,37 @@ extern "C" {
#include <lua.h>
+struct luaL_serializer;
+
LUALIB_API int
luaopen_yaml(lua_State *L);
+/** @Sa luaL_newserializer(). */
+struct luaL_serializer *
+lua_yaml_new_serializer(lua_State *L);
+
+/**
+ * Encode a Lua object into YAML document onto Lua stack.
+ * @param L Lua stack to get an argument and push result.
+ * @param serializer Lua YAML serializer.
+ * @param tag_handle NULL, or a global tag handle. Handle becomes
+ * a synonym for prefix.
+ * @param tag_prefix NULL, or a global tag prefix, to which @a
+ * handle is expanded.
+ * @retval nil, error Error.
+ * @retval not nil Lua string with dumped object.
+ */
+int
+lua_yaml_encode_tagged(lua_State *L, struct luaL_serializer *serializer,
+ const char *tag_handle, const char *tag_prefix);
+
+/**
+ * Same as lua_yaml_encode_tagged, but encode with no tags and
+ * with multiple arguments.
+ */
+int
+lua_yaml_encode(lua_State *L, struct luaL_serializer *serializer);
+
#ifdef __cplusplus
}
#endif
^ permalink raw reply [flat|nested] 34+ messages in thread
* [PATCH v2 05/10] session: move salt into iproto connection
2018-04-20 13:24 [PATCH v2 00/10] session: introduce box.session.push Vladislav Shpilevoy
` (4 preceding siblings ...)
2018-04-20 13:24 ` [PATCH v2 04/10] console: use Lua C API to do formatting for console Vladislav Shpilevoy
@ 2018-04-20 13:24 ` Vladislav Shpilevoy
2018-05-10 18:47 ` [tarantool-patches] " Konstantin Osipov
2018-04-20 13:24 ` [PATCH v2 06/10] session: introduce session vtab and meta Vladislav Shpilevoy
` (4 subsequent siblings)
10 siblings, 1 reply; 34+ messages in thread
From: Vladislav Shpilevoy @ 2018-04-20 13:24 UTC (permalink / raw)
To: tarantool-patches; +Cc: vdavydov.dev
Session salt is 32 random bytes, that are used to encode password
when a user is authorized. The salt is not used in non-binary
sessions, and can be moved to iproto connection.
---
src/box/authentication.cc | 4 ++--
src/box/authentication.h | 3 ++-
src/box/box.cc | 4 ++--
src/box/box.h | 2 +-
src/box/iproto.cc | 17 ++++++++++++-----
src/box/session.cc | 3 ---
src/box/session.h | 4 ----
7 files changed, 19 insertions(+), 18 deletions(-)
diff --git a/src/box/authentication.cc b/src/box/authentication.cc
index fef549c55..811974cb9 100644
--- a/src/box/authentication.cc
+++ b/src/box/authentication.cc
@@ -37,7 +37,7 @@
static char zero_hash[SCRAMBLE_SIZE];
void
-authenticate(const char *user_name, uint32_t len,
+authenticate(const char *user_name, uint32_t len, const char *salt,
const char *tuple)
{
struct user *user = user_find_by_name_xc(user_name, len);
@@ -84,7 +84,7 @@ authenticate(const char *user_name, uint32_t len,
"invalid scramble size");
}
- if (scramble_check(scramble, session->salt, user->def->hash2)) {
+ if (scramble_check(scramble, salt, user->def->hash2)) {
auth_res.is_authenticated = false;
if (session_run_on_auth_triggers(&auth_res) != 0)
diag_raise();
diff --git a/src/box/authentication.h b/src/box/authentication.h
index e91fe0a0e..9935e3548 100644
--- a/src/box/authentication.h
+++ b/src/box/authentication.h
@@ -45,6 +45,7 @@ struct on_auth_trigger_ctx {
void
-authenticate(const char *user_name, uint32_t len, const char *tuple);
+authenticate(const char *user_name, uint32_t len, const char *salt,
+ const char *tuple);
#endif /* INCLUDES_TARANTOOL_BOX_AUTHENTICATION_H */
diff --git a/src/box/box.cc b/src/box/box.cc
index d2dfc5b5f..ad21f051d 100644
--- a/src/box/box.cc
+++ b/src/box/box.cc
@@ -1233,7 +1233,7 @@ box_on_join(const tt_uuid *instance_uuid)
}
void
-box_process_auth(struct auth_request *request)
+box_process_auth(struct auth_request *request, const char *salt)
{
rmean_collect(rmean_box, IPROTO_AUTH, 1);
@@ -1243,7 +1243,7 @@ box_process_auth(struct auth_request *request)
const char *user = request->user_name;
uint32_t len = mp_decode_strl(&user);
- authenticate(user, len, request->scramble);
+ authenticate(user, len, salt, request->scramble);
}
void
diff --git a/src/box/box.h b/src/box/box.h
index c9b5aad01..84899cc13 100644
--- a/src/box/box.h
+++ b/src/box/box.h
@@ -150,7 +150,7 @@ box_reset_stat(void);
} /* extern "C" */
void
-box_process_auth(struct auth_request *request);
+box_process_auth(struct auth_request *request, const char *salt);
void
box_process_join(struct ev_io *io, struct xrow_header *header);
diff --git a/src/box/iproto.cc b/src/box/iproto.cc
index 96a8b708e..9e809b2e5 100644
--- a/src/box/iproto.cc
+++ b/src/box/iproto.cc
@@ -48,6 +48,7 @@
#include "coio.h"
#include "scoped_guard.h"
#include "memory.h"
+#include "random.h"
#include "port.h"
#include "box.h"
@@ -64,6 +65,8 @@
/* The number of iproto messages in flight */
enum { IPROTO_MSG_MAX = 768 };
+enum { IPROTO_SALT_SIZE = 32 };
+
/**
* Network readahead. A signed integer to avoid
* automatic type coercion to an unsigned type.
@@ -364,6 +367,8 @@ struct iproto_connection
/** Pointer to the current output buffer. */
struct obuf *p_obuf;
} tx;
+ /** Authentication salt. */
+ char salt[IPROTO_SALT_SIZE];
};
static struct mempool iproto_connection_pool;
@@ -1344,9 +1349,10 @@ static void
tx_process_misc(struct cmsg *m)
{
struct iproto_msg *msg = tx_accept_msg(m);
- struct obuf *out = msg->connection->tx.p_obuf;
+ struct iproto_connection *con = msg->connection;
+ struct obuf *out = con->tx.p_obuf;
- tx_fiber_init(msg->connection->session, msg->header.sync);
+ tx_fiber_init(con->session, msg->header.sync);
if (tx_check_schema(msg->header.schema_version))
goto error;
@@ -1354,7 +1360,7 @@ tx_process_misc(struct cmsg *m)
try {
switch (msg->header.type) {
case IPROTO_AUTH:
- box_process_auth(&msg->auth);
+ box_process_auth(&msg->auth, con->salt);
iproto_reply_ok_xc(out, msg->header.sync,
::schema_version);
break;
@@ -1505,8 +1511,9 @@ tx_process_connect(struct cmsg *m)
static __thread char greeting[IPROTO_GREETING_SIZE];
/* TODO: dirty read from tx thread */
struct tt_uuid uuid = INSTANCE_UUID;
- greeting_encode(greeting, tarantool_version_id(),
- &uuid, con->session->salt, SESSION_SEED_SIZE);
+ random_bytes(con->salt, IPROTO_SALT_SIZE);
+ greeting_encode(greeting, tarantool_version_id(), &uuid,
+ con->salt, IPROTO_SALT_SIZE);
obuf_dup_xc(out, greeting, IPROTO_GREETING_SIZE);
if (! rlist_empty(&session_on_connect)) {
if (session_run_on_connect_triggers(con->session) != 0)
diff --git a/src/box/session.cc b/src/box/session.cc
index ea6d76bb4..3d787bd51 100644
--- a/src/box/session.cc
+++ b/src/box/session.cc
@@ -33,7 +33,6 @@
#include "memory.h"
#include "assoc.h"
#include "trigger.h"
-#include "random.h"
#include "user.h"
#include "error.h"
@@ -96,8 +95,6 @@ session_create(int fd, enum session_type type)
/* For on_connect triggers. */
credentials_init(&session->credentials, guest_user->auth_token,
guest_user->def->uid);
- if (fd >= 0)
- random_bytes(session->salt, SESSION_SEED_SIZE);
struct mh_i64ptr_node_t node;
node.key = session->id;
node.val = session;
diff --git a/src/box/session.h b/src/box/session.h
index 4f9235ea8..c387e6f95 100644
--- a/src/box/session.h
+++ b/src/box/session.h
@@ -47,8 +47,6 @@ session_init();
void
session_free();
-enum { SESSION_SEED_SIZE = 32, SESSION_DELIM_SIZE = 16 };
-
enum session_type {
SESSION_TYPE_BACKGROUND = 0,
SESSION_TYPE_BINARY,
@@ -86,8 +84,6 @@ struct session {
*/
uint64_t sync;
enum session_type type;
- /** Authentication salt. */
- char salt[SESSION_SEED_SIZE];
/** Session user id and global grants */
struct credentials credentials;
/** Trigger for fiber on_stop to cleanup created on-demand session */
--
2.15.1 (Apple Git-101)
^ permalink raw reply [flat|nested] 34+ messages in thread* Re: [tarantool-patches] [PATCH v2 05/10] session: move salt into iproto connection
2018-04-20 13:24 ` [PATCH v2 05/10] session: move salt into iproto connection Vladislav Shpilevoy
@ 2018-05-10 18:47 ` Konstantin Osipov
0 siblings, 0 replies; 34+ messages in thread
From: Konstantin Osipov @ 2018-05-10 18:47 UTC (permalink / raw)
To: tarantool-patches; +Cc: vdavydov.dev
* Vladislav Shpilevoy <v.shpilevoy@tarantool.org> [18/04/20 16:25]:
> Session salt is 32 random bytes, that are used to encode password
> when a user is authorized. The salt is not used in non-binary
> sessions, and can be moved to iproto connection.
> ---
> src/box/authentication.cc | 4 ++--
> src/box/authentication.h | 3 ++-
> src/box/box.cc | 4 ++--
> src/box/box.h | 2 +-
> src/box/iproto.cc | 17 ++++++++++++-----
> src/box/session.cc | 3 ---
> src/box/session.h | 4 ----
OK to push.
--
Konstantin Osipov, Moscow, Russia, +7 903 626 22 32
http://tarantool.io - www.twitter.com/kostja_osipov
^ permalink raw reply [flat|nested] 34+ messages in thread
* [PATCH v2 06/10] session: introduce session vtab and meta
2018-04-20 13:24 [PATCH v2 00/10] session: introduce box.session.push Vladislav Shpilevoy
` (5 preceding siblings ...)
2018-04-20 13:24 ` [PATCH v2 05/10] session: move salt into iproto connection Vladislav Shpilevoy
@ 2018-04-20 13:24 ` Vladislav Shpilevoy
2018-05-10 19:20 ` [tarantool-patches] " Konstantin Osipov
2018-04-20 13:24 ` [PATCH v2 07/10] port: rename dump() into dump_msgpack() Vladislav Shpilevoy
` (3 subsequent siblings)
10 siblings, 1 reply; 34+ messages in thread
From: Vladislav Shpilevoy @ 2018-04-20 13:24 UTC (permalink / raw)
To: tarantool-patches; +Cc: vdavydov.dev
box.session.push implementation depends on session type -
console session must send YAML tagged text, binary session must
send MessagePack via another thread, other sessions must return
error.
Add virtual table to a session with a single 'push' function.
The same virtual table together with struct session meta can be
used to use memory of struct session more effectively. Before the
patch session stored sync and fd as attributes, but:
* fd was duplicated for iproto, which already has fd in
connection;
* sync is used only by iproto, and just occupies 8 byte in other
sessions;
* after the #2677 session additionaly must be able to store
iproto connection pointer.
Struct session meta uses C union to store either iproto, or
console, or another meta, but not all of them together.
Part of #2677
---
src/box/iproto.cc | 30 +++++++--
src/box/lua/console.c | 13 ++++
src/box/lua/session.c | 33 ++++++++--
src/box/session.cc | 47 ++++++++++++--
src/box/session.h | 107 ++++++++++++++++++++++++++-----
test/app-tap/console.test.lua | 8 ++-
test/box/push.result | 70 ++++++++++++++++++++
test/box/push.test.lua | 35 ++++++++++
test/replication/before_replace.result | 14 ++++
test/replication/before_replace.test.lua | 11 ++++
10 files changed, 333 insertions(+), 35 deletions(-)
create mode 100644 test/box/push.result
create mode 100644 test/box/push.test.lua
diff --git a/src/box/iproto.cc b/src/box/iproto.cc
index 9e809b2e5..a284368d3 100644
--- a/src/box/iproto.cc
+++ b/src/box/iproto.cc
@@ -1012,7 +1012,7 @@ error:
static void
tx_fiber_init(struct session *session, uint64_t sync)
{
- session->sync = sync;
+ session->meta.sync = sync;
/*
* We do not cleanup fiber keys at the end of each request.
* This does not lead to privilege escalation as long as
@@ -1039,11 +1039,6 @@ tx_process_disconnect(struct cmsg *m)
struct iproto_connection *con = msg->connection;
if (con->session) {
tx_fiber_init(con->session, 0);
- /*
- * The socket is already closed in iproto thread,
- * prevent box.session.peer() from using it.
- */
- con->session->fd = -1;
if (! rlist_empty(&session_on_disconnect))
session_run_on_disconnect_triggers(con->session);
session_destroy(con->session);
@@ -1504,9 +1499,10 @@ tx_process_connect(struct cmsg *m)
struct iproto_connection *con = msg->connection;
struct obuf *out = msg->connection->tx.p_obuf;
try { /* connect. */
- con->session = session_create(con->input.fd, SESSION_TYPE_BINARY);
+ con->session = session_create(SESSION_TYPE_BINARY);
if (con->session == NULL)
diag_raise();
+ con->session->meta.conn = con;
tx_fiber_init(con->session, 0);
static __thread char greeting[IPROTO_GREETING_SIZE];
/* TODO: dirty read from tx thread */
@@ -1643,6 +1639,20 @@ net_cord_f(va_list /* ap */)
return 0;
}
+int
+iproto_session_fd(struct session *session)
+{
+ struct iproto_connection *conn =
+ (struct iproto_connection *) session->meta.conn;
+ return conn->output.fd;
+}
+
+int64_t
+iproto_session_sync(struct session *session)
+{
+ return session->meta.sync;
+}
+
/** Initialize the iproto subsystem and start network io thread */
void
iproto_init()
@@ -1655,6 +1665,12 @@ iproto_init()
/* Create a pipe to "net" thread. */
cpipe_create(&net_pipe, "net");
cpipe_set_max_input(&net_pipe, IPROTO_MSG_MAX/2);
+ struct session_vtab iproto_session_vtab = {
+ /* .push = */ generic_session_push,
+ /* .fd = */ iproto_session_fd,
+ /* .sync = */ iproto_session_sync,
+ };
+ session_vtab_registry[SESSION_TYPE_BINARY] = iproto_session_vtab;
}
/**
diff --git a/src/box/lua/console.c b/src/box/lua/console.c
index f6009d387..a3bf83cb1 100644
--- a/src/box/lua/console.c
+++ b/src/box/lua/console.c
@@ -30,6 +30,7 @@
*/
#include "box/lua/console.h"
+#include "box/session.h"
#include "lua/utils.h"
#include "lua/fiber.h"
#include "fiber.h"
@@ -357,6 +358,12 @@ lbox_console_format(struct lua_State *L)
return lua_yaml_encode(L, luaL_yaml_default);
}
+int
+console_session_fd(struct session *session)
+{
+ return session->meta.fd;
+}
+
void
tarantool_lua_console_init(struct lua_State *L)
{
@@ -392,6 +399,12 @@ tarantool_lua_console_init(struct lua_State *L)
* console.
*/
lua_setfield(L, -2, "formatter");
+ struct session_vtab console_session_vtab = {
+ /* .push = */ generic_session_push,
+ /* .fd = */ console_session_fd,
+ /* .sync = */ generic_session_sync,
+ };
+ session_vtab_registry[SESSION_TYPE_CONSOLE] = console_session_vtab;
}
/*
diff --git a/src/box/lua/session.c b/src/box/lua/session.c
index 51caf199f..5fe5f08d4 100644
--- a/src/box/lua/session.c
+++ b/src/box/lua/session.c
@@ -50,10 +50,10 @@ lbox_session_create(struct lua_State *L)
{
struct session *session = fiber_get_session(fiber());
if (session == NULL) {
- int fd = luaL_optinteger(L, 1, -1);
- session = session_create_on_demand(fd);
+ session = session_create_on_demand();
if (session == NULL)
return luaT_error(L);
+ session->meta.fd = luaL_optinteger(L, 1, -1);
}
/* If a session already exists, simply reset its type */
session->type = STR2ENUM(session_type, luaL_optstring(L, 2, "console"));
@@ -96,7 +96,7 @@ lbox_session_type(struct lua_State *L)
static int
lbox_session_sync(struct lua_State *L)
{
- lua_pushnumber(L, current_session()->sync);
+ lua_pushnumber(L, session_sync(current_session()));
return 1;
}
@@ -231,7 +231,7 @@ lbox_session_fd(struct lua_State *L)
struct session *session = session_find(sid);
if (session == NULL)
luaL_error(L, "session.fd(): session does not exist");
- lua_pushinteger(L, session->fd);
+ lua_pushinteger(L, session_fd(session));
return 1;
}
@@ -253,7 +253,7 @@ lbox_session_peer(struct lua_State *L)
session = current_session();
if (session == NULL)
luaL_error(L, "session.peer(): session does not exist");
- fd = session->fd;
+ fd = session_fd(session);
if (fd < 0) {
lua_pushnil(L); /* no associated peer */
return 1;
@@ -355,6 +355,28 @@ lbox_push_on_access_denied_event(struct lua_State *L, void *event)
return 3;
}
+/**
+ * Push a message using a protocol, depending on a session type.
+ * @param data Data to push, first argument on a stack.
+ * @retval true Success.
+ * @retval nil, error Error occured.
+ */
+static int
+lbox_session_push(struct lua_State *L)
+{
+ if (lua_gettop(L) != 1)
+ return luaL_error(L, "Usage: box.session.push(data)");
+
+ if (session_push(current_session(), NULL) != 0) {
+ lua_pushnil(L);
+ luaT_pusherror(L, box_error_last());
+ return 2;
+ } else {
+ lua_pushboolean(L, true);
+ return 1;
+ }
+}
+
/**
* Sets trigger on_access_denied.
* For test purposes only.
@@ -429,6 +451,7 @@ box_lua_session_init(struct lua_State *L)
{"on_disconnect", lbox_session_on_disconnect},
{"on_auth", lbox_session_on_auth},
{"on_access_denied", lbox_session_on_access_denied},
+ {"push", lbox_session_push},
{NULL, NULL}
};
luaL_register_module(L, sessionlib_name, sessionlib);
diff --git a/src/box/session.cc b/src/box/session.cc
index 3d787bd51..4a1397c24 100644
--- a/src/box/session.cc
+++ b/src/box/session.cc
@@ -45,6 +45,20 @@ const char *session_type_strs[] = {
"unknown",
};
+static struct session_vtab generic_session_vtab = {
+ /* .push = */ generic_session_push,
+ /* .fd = */ generic_session_fd,
+ /* .sync = */ generic_session_sync,
+};
+
+struct session_vtab session_vtab_registry[] = {
+ /* BACKGROUND */ generic_session_vtab,
+ /* BINARY */ generic_session_vtab,
+ /* CONSOLE */ generic_session_vtab,
+ /* REPL */ generic_session_vtab,
+ /* APPLIER */ generic_session_vtab,
+};
+
static struct mh_i64ptr_t *session_registry;
struct mempool session_pool;
@@ -79,7 +93,7 @@ session_on_stop(struct trigger *trigger, void * /* event */)
}
struct session *
-session_create(int fd, enum session_type type)
+session_create(enum session_type type)
{
struct session *session =
(struct session *) mempool_alloc(&session_pool);
@@ -89,8 +103,7 @@ session_create(int fd, enum session_type type)
return NULL;
}
session->id = sid_max();
- session->fd = fd;
- session->sync = 0;
+ memset(&session->meta, 0, sizeof(session->meta));
session->type = type;
/* For on_connect triggers. */
credentials_init(&session->credentials, guest_user->auth_token,
@@ -110,12 +123,12 @@ session_create(int fd, enum session_type type)
}
struct session *
-session_create_on_demand(int fd)
+session_create_on_demand()
{
assert(fiber_get_session(fiber()) == NULL);
/* Create session on demand */
- struct session *s = session_create(fd, SESSION_TYPE_BACKGROUND);
+ struct session *s = session_create(SESSION_TYPE_BACKGROUND);
if (s == NULL)
return NULL;
s->fiber_on_stop = {
@@ -278,3 +291,27 @@ access_check_universe(user_access_t access)
}
return 0;
}
+
+int
+generic_session_push(struct session *session, struct port *port)
+{
+ (void) port;
+ const char *name =
+ tt_sprintf("Session '%s'", session_type_strs[session->type]);
+ diag_set(ClientError, ER_UNSUPPORTED, name, "push()");
+ return -1;
+}
+
+int
+generic_session_fd(struct session *session)
+{
+ (void) session;
+ return -1;
+}
+
+int64_t
+generic_session_sync(struct session *session)
+{
+ (void) session;
+ return 0;
+}
diff --git a/src/box/session.h b/src/box/session.h
index c387e6f95..e583c8c6b 100644
--- a/src/box/session.h
+++ b/src/box/session.h
@@ -41,6 +41,9 @@
extern "C" {
#endif /* defined(__cplusplus) */
+struct port;
+struct session_vtab;
+
void
session_init();
@@ -58,6 +61,23 @@ enum session_type {
extern const char *session_type_strs[];
+/**
+ * Session meta is used in different ways by sessions of different
+ * types, and allows to do not store attributes in struct session,
+ * that are used only by a session of particular type.
+ */
+struct session_meta {
+ union {
+ /** IProto connection meta. */
+ struct {
+ uint64_t sync;
+ void *conn;
+ };
+ /** Only by console is used. */
+ int fd;
+ };
+};
+
/**
* Abstraction of a single user session:
* for now, only provides accounting of established
@@ -70,26 +90,48 @@ extern const char *session_type_strs[];
struct session {
/** Session id. */
uint64_t id;
- /** File descriptor - socket of the connected peer.
- * Only if the session has a peer.
- */
- int fd;
- /**
- * For iproto requests, we set this field
- * to the value of packet sync. Since the
- * session may be reused between many requests,
- * the value is true only at the beginning
- * of the request, and gets distorted after
- * the first yield.
- */
- uint64_t sync;
enum session_type type;
+ /** Session metadata. */
+ struct session_meta meta;
/** Session user id and global grants */
struct credentials credentials;
/** Trigger for fiber on_stop to cleanup created on-demand session */
struct trigger fiber_on_stop;
};
+struct session_vtab {
+ /**
+ * Push a port data into a session data channel - socket,
+ * console or something.
+ * @param session Session to push into.
+ * @param port Port with data to push.
+ *
+ * @retval 0 Success.
+ * @retval -1 Error.
+ */
+ int
+ (*push)(struct session *session, struct port *port);
+ /**
+ * Get session file descriptor if exists.
+ * @param session Session to get descriptor from.
+ * @retval -1 No fd.
+ * @retval >=0 Found fd.
+ */
+ int
+ (*fd)(struct session *session);
+ /**
+ * For iproto requests, we set sync to the value of packet
+ * sync. Since the session may be reused between many
+ * requests, the value is true only at the beginning
+ * of the request, and gets distorted after the first
+ * yield. For other sessions it is 0.
+ */
+ int64_t
+ (*sync)(struct session *session);
+};
+
+extern struct session_vtab session_vtab_registry[];
+
/**
* Find a session by id.
*/
@@ -150,7 +192,7 @@ extern struct credentials admin_credentials;
* trigger to destroy it when this fiber ends.
*/
struct session *
-session_create_on_demand(int fd);
+session_create_on_demand();
/*
* When creating a new fiber, the database (box)
@@ -167,7 +209,7 @@ current_session()
{
struct session *session = fiber_get_session(fiber());
if (session == NULL) {
- session = session_create_on_demand(-1);
+ session = session_create_on_demand();
if (session == NULL)
diag_raise();
}
@@ -187,7 +229,7 @@ effective_user()
(struct credentials *) fiber_get_key(fiber(),
FIBER_KEY_USER);
if (u == NULL) {
- session_create_on_demand(-1);
+ session_create_on_demand();
u = (struct credentials *) fiber_get_key(fiber(),
FIBER_KEY_USER);
}
@@ -212,7 +254,7 @@ session_storage_cleanup(int sid);
* trigger fails or runs out of resources.
*/
struct session *
-session_create(int fd, enum session_type type);
+session_create(enum session_type type);
/**
* Destroy a session.
@@ -251,6 +293,37 @@ access_check_session(struct user *user);
int
access_check_universe(user_access_t access);
+static inline int
+session_push(struct session *session, struct port *port)
+{
+ return session_vtab_registry[session->type].push(session, port);
+}
+
+static inline int
+session_fd(struct session *session)
+{
+ return session_vtab_registry[session->type].fd(session);
+}
+
+static inline int
+session_sync(struct session *session)
+{
+ return session_vtab_registry[session->type].sync(session);
+}
+
+/**
+ * In a common case, a session does not support push. This
+ * function always returns -1 and sets ER_UNSUPPORTED error.
+ */
+int
+generic_session_push(struct session *session, struct port *port);
+
+int
+generic_session_fd(struct session *session);
+
+int64_t
+generic_session_sync(struct session *session);
+
#if defined(__cplusplus)
} /* extern "C" */
diff --git a/test/app-tap/console.test.lua b/test/app-tap/console.test.lua
index 48d28bd6d..d2e88b55b 100755
--- a/test/app-tap/console.test.lua
+++ b/test/app-tap/console.test.lua
@@ -21,7 +21,7 @@ local EOL = "\n...\n"
test = tap.test("console")
-test:plan(59)
+test:plan(60)
-- Start console and connect to it
local server = console.listen(CONSOLE_SOCKET)
@@ -31,6 +31,12 @@ local handshake = client:read{chunk = 128}
test:ok(string.match(handshake, '^Tarantool .*console') ~= nil, 'Handshake')
test:ok(client ~= nil, "connect to console")
+--
+-- gh-2677: box.session.push, text protocol support.
+--
+client:write('box.session.push(200)\n')
+test:is(client:read(EOL), "---\n- null\n- Session 'console' does not support push()\n...\n", "push does not work")
+
-- Execute some command
client:write("1\n")
test:is(yaml.decode(client:read(EOL))[1], 1, "eval")
diff --git a/test/box/push.result b/test/box/push.result
new file mode 100644
index 000000000..816f06e00
--- /dev/null
+++ b/test/box/push.result
@@ -0,0 +1,70 @@
+--
+-- gh-2677: box.session.push.
+--
+--
+-- Usage.
+--
+box.session.push()
+---
+- error: 'Usage: box.session.push(data)'
+...
+box.session.push(1, 2)
+---
+- error: 'Usage: box.session.push(data)'
+...
+ok = nil
+---
+...
+err = nil
+---
+...
+function do_push() ok, err = box.session.push(1) end
+---
+...
+--
+-- Test binary protocol.
+--
+netbox = require('net.box')
+---
+...
+box.schema.user.grant('guest', 'read,write,execute', 'universe')
+---
+...
+c = netbox.connect(box.cfg.listen)
+---
+...
+c:ping()
+---
+- true
+...
+c:call('do_push')
+---
+...
+ok, err
+---
+- null
+- Session 'binary' does not support push()
+...
+c:close()
+---
+...
+box.schema.user.revoke('guest', 'read,write,execute', 'universe')
+---
+...
+--
+-- Ensure can not push in background.
+--
+fiber = require('fiber')
+---
+...
+f = fiber.create(do_push)
+---
+...
+while f:status() ~= 'dead' do fiber.sleep(0.01) end
+---
+...
+ok, err
+---
+- null
+- Session 'background' does not support push()
+...
diff --git a/test/box/push.test.lua b/test/box/push.test.lua
new file mode 100644
index 000000000..a59fe0a4c
--- /dev/null
+++ b/test/box/push.test.lua
@@ -0,0 +1,35 @@
+--
+-- gh-2677: box.session.push.
+--
+
+--
+-- Usage.
+--
+box.session.push()
+box.session.push(1, 2)
+
+ok = nil
+err = nil
+function do_push() ok, err = box.session.push(1) end
+
+--
+-- Test binary protocol.
+--
+netbox = require('net.box')
+box.schema.user.grant('guest', 'read,write,execute', 'universe')
+
+c = netbox.connect(box.cfg.listen)
+c:ping()
+c:call('do_push')
+ok, err
+c:close()
+
+box.schema.user.revoke('guest', 'read,write,execute', 'universe')
+
+--
+-- Ensure can not push in background.
+--
+fiber = require('fiber')
+f = fiber.create(do_push)
+while f:status() ~= 'dead' do fiber.sleep(0.01) end
+ok, err
diff --git a/test/replication/before_replace.result b/test/replication/before_replace.result
index d561b4813..9937c5769 100644
--- a/test/replication/before_replace.result
+++ b/test/replication/before_replace.result
@@ -49,7 +49,17 @@ test_run:cmd("switch autobootstrap3");
---
- true
...
+--
+-- gh-2677 - test that an applier can not push() messages. Applier
+-- session is available in Lua, so the test is here instead of
+-- box/push.test.lua.
+--
+push_ok = nil
+push_err = nil
_ = box.space.test:before_replace(function(old, new)
+ if box.session.type() == 'applier' and not push_err then
+ push_ok, push_err = box.session.push(100)
+ end
if old ~= nil and new ~= nil then
return new[2] > old[2] and new or old
end
@@ -187,6 +197,10 @@ box.space.test:select()
- [9, 90]
- [10, 100]
...
+push_err
+---
+- Session 'applier' does not support push()
+...
test_run:cmd('restart server autobootstrap3')
box.space.test:select()
---
diff --git a/test/replication/before_replace.test.lua b/test/replication/before_replace.test.lua
index 2c6912d06..52ace490a 100644
--- a/test/replication/before_replace.test.lua
+++ b/test/replication/before_replace.test.lua
@@ -26,7 +26,17 @@ _ = box.space.test:before_replace(function(old, new)
end
end);
test_run:cmd("switch autobootstrap3");
+--
+-- gh-2677 - test that an applier can not push() messages. Applier
+-- session is available in Lua, so the test is here instead of
+-- box/push.test.lua.
+--
+push_ok = nil
+push_err = nil
_ = box.space.test:before_replace(function(old, new)
+ if box.session.type() == 'applier' and not push_err then
+ push_ok, push_err = box.session.push(100)
+ end
if old ~= nil and new ~= nil then
return new[2] > old[2] and new or old
end
@@ -62,6 +72,7 @@ test_run:cmd('restart server autobootstrap2')
box.space.test:select()
test_run:cmd("switch autobootstrap3")
box.space.test:select()
+push_err
test_run:cmd('restart server autobootstrap3')
box.space.test:select()
--
2.15.1 (Apple Git-101)
^ permalink raw reply [flat|nested] 34+ messages in thread* Re: [tarantool-patches] [PATCH v2 06/10] session: introduce session vtab and meta
2018-04-20 13:24 ` [PATCH v2 06/10] session: introduce session vtab and meta Vladislav Shpilevoy
@ 2018-05-10 19:20 ` Konstantin Osipov
2018-05-24 20:50 ` [tarantool-patches] " Vladislav Shpilevoy
0 siblings, 1 reply; 34+ messages in thread
From: Konstantin Osipov @ 2018-05-10 19:20 UTC (permalink / raw)
To: tarantool-patches; +Cc: vdavydov.dev
* Vladislav Shpilevoy <v.shpilevoy@tarantool.org> [18/04/20 16:25]:
> +int
> +iproto_session_fd(struct session *session)
> +{
> + struct iproto_connection *conn =
> + (struct iproto_connection *) session->meta.conn;
> + return conn->output.fd;
> +}
Nitpick: I would not abbreviate connection when in a member, and
use 'con' as in other places for name of the variable declared on
the stack.
> +/**
> + * Push a message using a protocol, depending on a session type.
> + * @param data Data to push, first argument on a stack.
> + * @retval true Success.
> + * @retval nil, error Error occured.
> + */
> +static int
> +lbox_session_push(struct lua_State *L)
> +{
> + if (lua_gettop(L) != 1)
> + return luaL_error(L, "Usage: box.session.push(data)");
> +
> + if (session_push(current_session(), NULL) != 0) {
> + lua_pushnil(L);
> + luaT_pusherror(L, box_error_last());
> + return 2;
> + } else {
> + lua_pushboolean(L, true);
> + return 1;
> + }
> +}
I'm not sure we the calling convention should differ from the rest
of box API, i.e. we should return nil, error rather than
exceptions. Could you run a poll in the community chat?
> +
> /**
> * Sets trigger on_access_denied.
> * For test purposes only.
> @@ -429,6 +451,7 @@ box_lua_session_init(struct lua_State *L)
> {"on_disconnect", lbox_session_on_disconnect},
> {"on_auth", lbox_session_on_auth},
> {"on_access_denied", lbox_session_on_access_denied},
> + {"push", lbox_session_push},
> {NULL, NULL}
> };
> luaL_register_module(L, sessionlib_name, sessionlib);
> diff --git a/src/box/session.cc b/src/box/session.cc
> index 3d787bd51..4a1397c24 100644
> --- a/src/box/session.cc
> +++ b/src/box/session.cc
> @@ -45,6 +45,20 @@ const char *session_type_strs[] = {
> "unknown",
> };
>
> +static struct session_vtab generic_session_vtab = {
> + /* .push = */ generic_session_push,
> + /* .fd = */ generic_session_fd,
> + /* .sync = */ generic_session_sync,
> +};
> +
> +struct session_vtab session_vtab_registry[] = {
> + /* BACKGROUND */ generic_session_vtab,
> + /* BINARY */ generic_session_vtab,
> + /* CONSOLE */ generic_session_vtab,
> + /* REPL */ generic_session_vtab,
> + /* APPLIER */ generic_session_vtab,
> +};
> +
> static struct mh_i64ptr_t *session_registry;
>
> struct mempool session_pool;
> @@ -79,7 +93,7 @@ session_on_stop(struct trigger *trigger, void * /* event */)
> }
>
> struct session *
> -session_create(int fd, enum session_type type)
> +session_create(enum session_type type)
> {
> struct session *session =
> (struct session *) mempool_alloc(&session_pool);
> @@ -89,8 +103,7 @@ session_create(int fd, enum session_type type)
> return NULL;
> }
> session->id = sid_max();
> - session->fd = fd;
> - session->sync = 0;
> + memset(&session->meta, 0, sizeof(session->meta));
> session->type = type;
> /* For on_connect triggers. */
> credentials_init(&session->credentials, guest_user->auth_token,
> @@ -110,12 +123,12 @@ session_create(int fd, enum session_type type)
> }
>
> struct session *
> -session_create_on_demand(int fd)
> +session_create_on_demand()
> {
> assert(fiber_get_session(fiber()) == NULL);
>
> /* Create session on demand */
> - struct session *s = session_create(fd, SESSION_TYPE_BACKGROUND);
> + struct session *s = session_create(SESSION_TYPE_BACKGROUND);
> if (s == NULL)
> return NULL;
> s->fiber_on_stop = {
> @@ -278,3 +291,27 @@ access_check_universe(user_access_t access)
> }
> return 0;
> }
> +
> +int
> +generic_session_push(struct session *session, struct port *port)
> +{
> + (void) port;
> + const char *name =
> + tt_sprintf("Session '%s'", session_type_strs[session->type]);
> + diag_set(ClientError, ER_UNSUPPORTED, name, "push()");
> + return -1;
> +}
> +
> +int
> +generic_session_fd(struct session *session)
> +{
> + (void) session;
> + return -1;
> +}
> +
> +int64_t
> +generic_session_sync(struct session *session)
> +{
> + (void) session;
> + return 0;
> +}
> diff --git a/src/box/session.h b/src/box/session.h
> index c387e6f95..e583c8c6b 100644
> --- a/src/box/session.h
> +++ b/src/box/session.h
> @@ -41,6 +41,9 @@
> extern "C" {
> #endif /* defined(__cplusplus) */
>
> +struct port;
> +struct session_vtab;
> +
> void
> session_init();
>
> @@ -58,6 +61,23 @@ enum session_type {
>
> extern const char *session_type_strs[];
>
> +/**
> + * Session meta is used in different ways by sessions of different
> + * types, and allows to do not store attributes in struct session,
> + * that are used only by a session of particular type.
> + */
> +struct session_meta {
> + union {
> + /** IProto connection meta. */
> + struct {
> + uint64_t sync;
> + void *conn;
> + };
> + /** Only by console is used. */
> + int fd;
> + };
> +};
> +
> /**
Otherwise the patch LGTM.
--
Konstantin Osipov, Moscow, Russia, +7 903 626 22 32
http://tarantool.io - www.twitter.com/kostja_osipov
^ permalink raw reply [flat|nested] 34+ messages in thread* Re: [tarantool-patches] Re: [PATCH v2 06/10] session: introduce session vtab and meta
2018-05-10 19:20 ` [tarantool-patches] " Konstantin Osipov
@ 2018-05-24 20:50 ` Vladislav Shpilevoy
0 siblings, 0 replies; 34+ messages in thread
From: Vladislav Shpilevoy @ 2018-05-24 20:50 UTC (permalink / raw)
To: tarantool-patches, Konstantin Osipov; +Cc: vdavydov.dev
On 10/05/2018 22:20, Konstantin Osipov wrote:
> * Vladislav Shpilevoy <v.shpilevoy@tarantool.org> [18/04/20 16:25]:
>> +int
>> +iproto_session_fd(struct session *session)
>> +{
>> + struct iproto_connection *conn =
>> + (struct iproto_connection *) session->meta.conn;
>> + return conn->output.fd;
>> +}
>
> Nitpick: I would not abbreviate connection when in a member, and
> use 'con' as in other places for name of the variable declared on
> the stack.
diff --git a/src/box/iproto.cc b/src/box/iproto.cc
index e612ba173..cd76fc4c5 100644
--- a/src/box/iproto.cc
+++ b/src/box/iproto.cc
@@ -1581,7 +1581,7 @@ tx_process_connect(struct cmsg *m)
con->session = session_create(SESSION_TYPE_BINARY);
if (con->session == NULL)
diag_raise();
- con->session->meta.conn = con;
+ con->session->meta.connection = con;
tx_fiber_init(con->session, 0);
static __thread char greeting[IPROTO_GREETING_SIZE];
/* TODO: dirty read from tx thread */
@@ -1730,9 +1730,9 @@ net_cord_f(va_list /* ap */)
int
iproto_session_fd(struct session *session)
{
- struct iproto_connection *conn =
- (struct iproto_connection *) session->meta.conn;
- return conn->output.fd;
+ struct iproto_connection *con =
+ (struct iproto_connection *) session->meta.connection;
+ return con->output.fd;
}
int64_t
diff --git a/src/box/session.h b/src/box/session.h
index e583c8c6b..ff3c7b2fa 100644
--- a/src/box/session.h
+++ b/src/box/session.h
@@ -71,7 +71,7 @@ struct session_meta {
/** IProto connection meta. */
struct {
uint64_t sync;
- void *conn;
+ void *connection;
};
/** Only by console is used. */
int fd;
>
>> +/**
>> + * Push a message using a protocol, depending on a session type.
>> + * @param data Data to push, first argument on a stack.
>> + * @retval true Success.
>> + * @retval nil, error Error occured.
>> + */
>> +static int
>> +lbox_session_push(struct lua_State *L)
>> +{
>> + if (lua_gettop(L) != 1)
>> + return luaL_error(L, "Usage: box.session.push(data)");
>> +
>> + if (session_push(current_session(), NULL) != 0) {
>> + lua_pushnil(L);
>> + luaT_pusherror(L, box_error_last());
>> + return 2;
>> + } else {
>> + lua_pushboolean(L, true);
>> + return 1;
>> + }
>> +}
>
> I'm not sure we the calling convention should differ from the rest
> of box API, i.e. we should return nil, error rather than
> exceptions. Could you run a poll in the community chat?
It is not a subject to run a poll for. Out convention is already established,
and it is throw on OOM, and return nil + error object on other errors.
See the new diff below:
========================================================================
commit 24eaadece9219a3eacc46798d7c5662540a645f6
Author: Vladislav Shpilevoy <v.shpilevoy@tarantool.org>
Date: Thu Apr 19 13:43:10 2018 +0300
session: introduce session vtab and meta
box.session.push implementation depends on session type -
console session must send YAML tagged text, binary session must
send MessagePack via another thread, other sessions must return
error.
Add virtual table to a session with 'push', 'fd' and 'sync'
functions.
The same virtual table together with struct session meta can be
used to use memory of struct session more effectively. Before the
patch session stored sync and fd as attributes, but:
* fd was duplicated for iproto, which already has fd in
connection;
* sync is used only by iproto, and just occupies 8 byte in other
sessions;
* after the #2677 session additionaly must be able to store
iproto connection pointer.
Struct session meta uses C union to store either iproto, or
console, or another meta, but not all of them together.
Part of #2677
diff --git a/src/box/iproto.cc b/src/box/iproto.cc
index fd6022305..cd76fc4c5 100644
--- a/src/box/iproto.cc
+++ b/src/box/iproto.cc
@@ -1086,7 +1086,7 @@ error:
static void
tx_fiber_init(struct session *session, uint64_t sync)
{
- session->sync = sync;
+ session->meta.sync = sync;
/*
* We do not cleanup fiber keys at the end of each request.
* This does not lead to privilege escalation as long as
@@ -1113,11 +1113,6 @@ tx_process_disconnect(struct cmsg *m)
container_of(m, struct iproto_connection, disconnect);
if (con->session) {
tx_fiber_init(con->session, 0);
- /*
- * The socket is already closed in iproto thread,
- * prevent box.session.peer() from using it.
- */
- con->session->fd = -1;
if (! rlist_empty(&session_on_disconnect))
session_run_on_disconnect_triggers(con->session);
session_destroy(con->session);
@@ -1583,9 +1578,10 @@ tx_process_connect(struct cmsg *m)
struct iproto_connection *con = msg->connection;
struct obuf *out = msg->connection->tx.p_obuf;
try { /* connect. */
- con->session = session_create(con->input.fd, SESSION_TYPE_BINARY);
+ con->session = session_create(SESSION_TYPE_BINARY);
if (con->session == NULL)
diag_raise();
+ con->session->meta.connection = con;
tx_fiber_init(con->session, 0);
static __thread char greeting[IPROTO_GREETING_SIZE];
/* TODO: dirty read from tx thread */
@@ -1731,6 +1727,20 @@ net_cord_f(va_list /* ap */)
return 0;
}
+int
+iproto_session_fd(struct session *session)
+{
+ struct iproto_connection *con =
+ (struct iproto_connection *) session->meta.connection;
+ return con->output.fd;
+}
+
+int64_t
+iproto_session_sync(struct session *session)
+{
+ return session->meta.sync;
+}
+
/** Initialize the iproto subsystem and start network io thread */
void
iproto_init()
@@ -1743,6 +1753,12 @@ iproto_init()
/* Create a pipe to "net" thread. */
cpipe_create(&net_pipe, "net");
cpipe_set_max_input(&net_pipe, iproto_msg_max / 2);
+ struct session_vtab iproto_session_vtab = {
+ /* .push = */ generic_session_push,
+ /* .fd = */ iproto_session_fd,
+ /* .sync = */ iproto_session_sync,
+ };
+ session_vtab_registry[SESSION_TYPE_BINARY] = iproto_session_vtab;
}
/** Available IProto configuration changes. */
diff --git a/src/box/lua/console.c b/src/box/lua/console.c
index 31614b4a8..7e17fa30a 100644
--- a/src/box/lua/console.c
+++ b/src/box/lua/console.c
@@ -30,6 +30,7 @@
*/
#include "box/lua/console.h"
+#include "box/session.h"
#include "lua/utils.h"
#include "lua/fiber.h"
#include "fiber.h"
@@ -359,6 +360,12 @@ lbox_console_format(struct lua_State *L)
return lua_yaml_encode(L, luaL_yaml_default);
}
+int
+console_session_fd(struct session *session)
+{
+ return session->meta.fd;
+}
+
void
tarantool_lua_console_init(struct lua_State *L)
{
@@ -392,6 +399,12 @@ tarantool_lua_console_init(struct lua_State *L)
* load_history work the same way.
*/
lua_setfield(L, -2, "formatter");
+ struct session_vtab console_session_vtab = {
+ /* .push = */ generic_session_push,
+ /* .fd = */ console_session_fd,
+ /* .sync = */ generic_session_sync,
+ };
+ session_vtab_registry[SESSION_TYPE_CONSOLE] = console_session_vtab;
}
/*
diff --git a/src/box/lua/session.c b/src/box/lua/session.c
index 51caf199f..05010c4c3 100644
--- a/src/box/lua/session.c
+++ b/src/box/lua/session.c
@@ -50,10 +50,10 @@ lbox_session_create(struct lua_State *L)
{
struct session *session = fiber_get_session(fiber());
if (session == NULL) {
- int fd = luaL_optinteger(L, 1, -1);
- session = session_create_on_demand(fd);
+ session = session_create_on_demand();
if (session == NULL)
return luaT_error(L);
+ session->meta.fd = luaL_optinteger(L, 1, -1);
}
/* If a session already exists, simply reset its type */
session->type = STR2ENUM(session_type, luaL_optstring(L, 2, "console"));
@@ -96,7 +96,7 @@ lbox_session_type(struct lua_State *L)
static int
lbox_session_sync(struct lua_State *L)
{
- lua_pushnumber(L, current_session()->sync);
+ lua_pushnumber(L, session_sync(current_session()));
return 1;
}
@@ -231,7 +231,7 @@ lbox_session_fd(struct lua_State *L)
struct session *session = session_find(sid);
if (session == NULL)
luaL_error(L, "session.fd(): session does not exist");
- lua_pushinteger(L, session->fd);
+ lua_pushinteger(L, session_fd(session));
return 1;
}
@@ -253,7 +253,7 @@ lbox_session_peer(struct lua_State *L)
session = current_session();
if (session == NULL)
luaL_error(L, "session.peer(): session does not exist");
- fd = session->fd;
+ fd = session_fd(session);
if (fd < 0) {
lua_pushnil(L); /* no associated peer */
return 1;
@@ -355,6 +355,29 @@ lbox_push_on_access_denied_event(struct lua_State *L, void *event)
return 3;
}
+/**
+ * Push a message using a protocol, depending on a session type.
+ * @param L Lua state. First argument on the stack is data to
+ * push.
+ * @retval 1 Success, true is pushed.
+ * @retval 2 Error. Nil and error object are pushed.
+ */
+static int
+lbox_session_push(struct lua_State *L)
+{
+ if (lua_gettop(L) != 1)
+ return luaL_error(L, "Usage: box.session.push(data)");
+
+ if (session_push(current_session(), NULL) != 0) {
+ lua_pushnil(L);
+ luaT_pusherror(L, box_error_last());
+ return 2;
+ } else {
+ lua_pushboolean(L, true);
+ return 1;
+ }
+}
+
/**
* Sets trigger on_access_denied.
* For test purposes only.
@@ -429,6 +452,7 @@ box_lua_session_init(struct lua_State *L)
{"on_disconnect", lbox_session_on_disconnect},
{"on_auth", lbox_session_on_auth},
{"on_access_denied", lbox_session_on_access_denied},
+ {"push", lbox_session_push},
{NULL, NULL}
};
luaL_register_module(L, sessionlib_name, sessionlib);
diff --git a/src/box/session.cc b/src/box/session.cc
index 3d787bd51..4a1397c24 100644
--- a/src/box/session.cc
+++ b/src/box/session.cc
@@ -45,6 +45,20 @@ const char *session_type_strs[] = {
"unknown",
};
+static struct session_vtab generic_session_vtab = {
+ /* .push = */ generic_session_push,
+ /* .fd = */ generic_session_fd,
+ /* .sync = */ generic_session_sync,
+};
+
+struct session_vtab session_vtab_registry[] = {
+ /* BACKGROUND */ generic_session_vtab,
+ /* BINARY */ generic_session_vtab,
+ /* CONSOLE */ generic_session_vtab,
+ /* REPL */ generic_session_vtab,
+ /* APPLIER */ generic_session_vtab,
+};
+
static struct mh_i64ptr_t *session_registry;
struct mempool session_pool;
@@ -79,7 +93,7 @@ session_on_stop(struct trigger *trigger, void * /* event */)
}
struct session *
-session_create(int fd, enum session_type type)
+session_create(enum session_type type)
{
struct session *session =
(struct session *) mempool_alloc(&session_pool);
@@ -89,8 +103,7 @@ session_create(int fd, enum session_type type)
return NULL;
}
session->id = sid_max();
- session->fd = fd;
- session->sync = 0;
+ memset(&session->meta, 0, sizeof(session->meta));
session->type = type;
/* For on_connect triggers. */
credentials_init(&session->credentials, guest_user->auth_token,
@@ -110,12 +123,12 @@ session_create(int fd, enum session_type type)
}
struct session *
-session_create_on_demand(int fd)
+session_create_on_demand()
{
assert(fiber_get_session(fiber()) == NULL);
/* Create session on demand */
- struct session *s = session_create(fd, SESSION_TYPE_BACKGROUND);
+ struct session *s = session_create(SESSION_TYPE_BACKGROUND);
if (s == NULL)
return NULL;
s->fiber_on_stop = {
@@ -278,3 +291,27 @@ access_check_universe(user_access_t access)
}
return 0;
}
+
+int
+generic_session_push(struct session *session, struct port *port)
+{
+ (void) port;
+ const char *name =
+ tt_sprintf("Session '%s'", session_type_strs[session->type]);
+ diag_set(ClientError, ER_UNSUPPORTED, name, "push()");
+ return -1;
+}
+
+int
+generic_session_fd(struct session *session)
+{
+ (void) session;
+ return -1;
+}
+
+int64_t
+generic_session_sync(struct session *session)
+{
+ (void) session;
+ return 0;
+}
diff --git a/src/box/session.h b/src/box/session.h
index c387e6f95..a37fbdad0 100644
--- a/src/box/session.h
+++ b/src/box/session.h
@@ -41,6 +41,9 @@
extern "C" {
#endif /* defined(__cplusplus) */
+struct port;
+struct session_vtab;
+
void
session_init();
@@ -58,6 +61,23 @@ enum session_type {
extern const char *session_type_strs[];
+/**
+ * Session meta is used in different ways by sessions of different
+ * types, and allows to do not store attributes in struct session,
+ * that are used only by a session of particular type.
+ */
+struct session_meta {
+ union {
+ /** IProto connection meta. */
+ struct {
+ uint64_t sync;
+ void *connection;
+ };
+ /** Only by console is used. */
+ int fd;
+ };
+};
+
/**
* Abstraction of a single user session:
* for now, only provides accounting of established
@@ -70,26 +90,48 @@ extern const char *session_type_strs[];
struct session {
/** Session id. */
uint64_t id;
- /** File descriptor - socket of the connected peer.
- * Only if the session has a peer.
- */
- int fd;
- /**
- * For iproto requests, we set this field
- * to the value of packet sync. Since the
- * session may be reused between many requests,
- * the value is true only at the beginning
- * of the request, and gets distorted after
- * the first yield.
- */
- uint64_t sync;
enum session_type type;
+ /** Session metadata. */
+ struct session_meta meta;
/** Session user id and global grants */
struct credentials credentials;
/** Trigger for fiber on_stop to cleanup created on-demand session */
struct trigger fiber_on_stop;
};
+struct session_vtab {
+ /**
+ * Push a port data into a session data channel - socket,
+ * console or something.
+ * @param session Session to push into.
+ * @param port Port with data to push.
+ *
+ * @retval 0 Success.
+ * @retval -1 Error.
+ */
+ int
+ (*push)(struct session *session, struct port *port);
+ /**
+ * Get session file descriptor if exists.
+ * @param session Session to get descriptor from.
+ * @retval -1 No fd.
+ * @retval >=0 Found fd.
+ */
+ int
+ (*fd)(struct session *session);
+ /**
+ * For iproto requests, we set sync to the value of packet
+ * sync. Since the session may be reused between many
+ * requests, the value is true only at the beginning
+ * of the request, and gets distorted after the first
+ * yield. For other sessions it is 0.
+ */
+ int64_t
+ (*sync)(struct session *session);
+};
+
+extern struct session_vtab session_vtab_registry[];
+
/**
* Find a session by id.
*/
@@ -150,7 +192,7 @@ extern struct credentials admin_credentials;
* trigger to destroy it when this fiber ends.
*/
struct session *
-session_create_on_demand(int fd);
+session_create_on_demand();
/*
* When creating a new fiber, the database (box)
@@ -167,7 +209,7 @@ current_session()
{
struct session *session = fiber_get_session(fiber());
if (session == NULL) {
- session = session_create_on_demand(-1);
+ session = session_create_on_demand();
if (session == NULL)
diag_raise();
}
@@ -187,7 +229,7 @@ effective_user()
(struct credentials *) fiber_get_key(fiber(),
FIBER_KEY_USER);
if (u == NULL) {
- session_create_on_demand(-1);
+ session_create_on_demand();
u = (struct credentials *) fiber_get_key(fiber(),
FIBER_KEY_USER);
}
@@ -212,7 +254,7 @@ session_storage_cleanup(int sid);
* trigger fails or runs out of resources.
*/
struct session *
-session_create(int fd, enum session_type type);
+session_create(enum session_type type);
/**
* Destroy a session.
@@ -251,6 +293,39 @@ access_check_session(struct user *user);
int
access_check_universe(user_access_t access);
+static inline int
+session_push(struct session *session, struct port *port)
+{
+ return session_vtab_registry[session->type].push(session, port);
+}
+
+static inline int
+session_fd(struct session *session)
+{
+ return session_vtab_registry[session->type].fd(session);
+}
+
+static inline int
+session_sync(struct session *session)
+{
+ return session_vtab_registry[session->type].sync(session);
+}
+
+/**
+ * In a common case, a session does not support push. This
+ * function always returns -1 and sets ER_UNSUPPORTED error.
+ */
+int
+generic_session_push(struct session *session, struct port *port);
+
+/** Return -1 from any session. */
+int
+generic_session_fd(struct session *session);
+
+/** Return 0 from any session. */
+int64_t
+generic_session_sync(struct session *session);
+
#if defined(__cplusplus)
} /* extern "C" */
diff --git a/test/app-tap/console.test.lua b/test/app-tap/console.test.lua
index fecfa52c8..a5b3061a9 100755
--- a/test/app-tap/console.test.lua
+++ b/test/app-tap/console.test.lua
@@ -21,7 +21,7 @@ local EOL = "\n...\n"
test = tap.test("console")
-test:plan(59)
+test:plan(60)
-- Start console and connect to it
local server = console.listen(CONSOLE_SOCKET)
@@ -31,6 +31,12 @@ local handshake = client:read{chunk = 128}
test:ok(string.match(handshake, '^Tarantool .*console') ~= nil, 'Handshake')
test:ok(client ~= nil, "connect to console")
+--
+-- gh-2677: box.session.push, text protocol support.
+--
+client:write('box.session.push(200)\n')
+test:is(client:read(EOL), "---\n- null\n- Session 'console' does not support push()\n...\n", "push does not work")
+
-- Execute some command
client:write("1\n")
test:is(yaml.decode(client:read(EOL))[1], 1, "eval")
diff --git a/test/box/push.result b/test/box/push.result
new file mode 100644
index 000000000..816f06e00
--- /dev/null
+++ b/test/box/push.result
@@ -0,0 +1,70 @@
+--
+-- gh-2677: box.session.push.
+--
+--
+-- Usage.
+--
+box.session.push()
+---
+- error: 'Usage: box.session.push(data)'
+...
+box.session.push(1, 2)
+---
+- error: 'Usage: box.session.push(data)'
+...
+ok = nil
+---
+...
+err = nil
+---
+...
+function do_push() ok, err = box.session.push(1) end
+---
+...
+--
+-- Test binary protocol.
+--
+netbox = require('net.box')
+---
+...
+box.schema.user.grant('guest', 'read,write,execute', 'universe')
+---
+...
+c = netbox.connect(box.cfg.listen)
+---
+...
+c:ping()
+---
+- true
+...
+c:call('do_push')
+---
+...
+ok, err
+---
+- null
+- Session 'binary' does not support push()
+...
+c:close()
+---
+...
+box.schema.user.revoke('guest', 'read,write,execute', 'universe')
+---
+...
+--
+-- Ensure can not push in background.
+--
+fiber = require('fiber')
+---
+...
+f = fiber.create(do_push)
+---
+...
+while f:status() ~= 'dead' do fiber.sleep(0.01) end
+---
+...
+ok, err
+---
+- null
+- Session 'background' does not support push()
+...
diff --git a/test/box/push.test.lua b/test/box/push.test.lua
new file mode 100644
index 000000000..a59fe0a4c
--- /dev/null
+++ b/test/box/push.test.lua
@@ -0,0 +1,35 @@
+--
+-- gh-2677: box.session.push.
+--
+
+--
+-- Usage.
+--
+box.session.push()
+box.session.push(1, 2)
+
+ok = nil
+err = nil
+function do_push() ok, err = box.session.push(1) end
+
+--
+-- Test binary protocol.
+--
+netbox = require('net.box')
+box.schema.user.grant('guest', 'read,write,execute', 'universe')
+
+c = netbox.connect(box.cfg.listen)
+c:ping()
+c:call('do_push')
+ok, err
+c:close()
+
+box.schema.user.revoke('guest', 'read,write,execute', 'universe')
+
+--
+-- Ensure can not push in background.
+--
+fiber = require('fiber')
+f = fiber.create(do_push)
+while f:status() ~= 'dead' do fiber.sleep(0.01) end
+ok, err
diff --git a/test/replication/before_replace.result b/test/replication/before_replace.result
index d561b4813..9937c5769 100644
--- a/test/replication/before_replace.result
+++ b/test/replication/before_replace.result
@@ -49,7 +49,17 @@ test_run:cmd("switch autobootstrap3");
---
- true
...
+--
+-- gh-2677 - test that an applier can not push() messages. Applier
+-- session is available in Lua, so the test is here instead of
+-- box/push.test.lua.
+--
+push_ok = nil
+push_err = nil
_ = box.space.test:before_replace(function(old, new)
+ if box.session.type() == 'applier' and not push_err then
+ push_ok, push_err = box.session.push(100)
+ end
if old ~= nil and new ~= nil then
return new[2] > old[2] and new or old
end
@@ -187,6 +197,10 @@ box.space.test:select()
- [9, 90]
- [10, 100]
...
+push_err
+---
+- Session 'applier' does not support push()
+...
test_run:cmd('restart server autobootstrap3')
box.space.test:select()
---
diff --git a/test/replication/before_replace.test.lua b/test/replication/before_replace.test.lua
index 2c6912d06..52ace490a 100644
--- a/test/replication/before_replace.test.lua
+++ b/test/replication/before_replace.test.lua
@@ -26,7 +26,17 @@ _ = box.space.test:before_replace(function(old, new)
end
end);
test_run:cmd("switch autobootstrap3");
+--
+-- gh-2677 - test that an applier can not push() messages. Applier
+-- session is available in Lua, so the test is here instead of
+-- box/push.test.lua.
+--
+push_ok = nil
+push_err = nil
_ = box.space.test:before_replace(function(old, new)
+ if box.session.type() == 'applier' and not push_err then
+ push_ok, push_err = box.session.push(100)
+ end
if old ~= nil and new ~= nil then
return new[2] > old[2] and new or old
end
@@ -62,6 +72,7 @@ test_run:cmd('restart server autobootstrap2')
box.space.test:select()
test_run:cmd("switch autobootstrap3")
box.space.test:select()
+push_err
test_run:cmd('restart server autobootstrap3')
box.space.test:select()
^ permalink raw reply [flat|nested] 34+ messages in thread
* [PATCH v2 07/10] port: rename dump() into dump_msgpack()
2018-04-20 13:24 [PATCH v2 00/10] session: introduce box.session.push Vladislav Shpilevoy
` (6 preceding siblings ...)
2018-04-20 13:24 ` [PATCH v2 06/10] session: introduce session vtab and meta Vladislav Shpilevoy
@ 2018-04-20 13:24 ` Vladislav Shpilevoy
2018-05-10 19:21 ` [tarantool-patches] " Konstantin Osipov
2018-04-20 13:24 ` [PATCH v2 08/10] session: introduce text box.session.push Vladislav Shpilevoy
` (2 subsequent siblings)
10 siblings, 1 reply; 34+ messages in thread
From: Vladislav Shpilevoy @ 2018-04-20 13:24 UTC (permalink / raw)
To: tarantool-patches; +Cc: vdavydov.dev
In #2677 a port must be able to dump both msgpack into obuf for
binary box.session.push(), and text with no obuf for text
box.session.push.
---
src/box/iproto.cc | 6 +++---
src/box/lua/call.c | 4 ++--
src/box/port.c | 18 +++++++++---------
src/box/port.h | 12 ++++++------
4 files changed, 20 insertions(+), 20 deletions(-)
diff --git a/src/box/iproto.cc b/src/box/iproto.cc
index a284368d3..38baf1b8d 100644
--- a/src/box/iproto.cc
+++ b/src/box/iproto.cc
@@ -1233,7 +1233,7 @@ tx_process_select(struct cmsg *m)
/*
* SELECT output format has not changed since Tarantool 1.6
*/
- count = port_dump_16(&port, out);
+ count = port_dump_msgpack_16(&port, out);
port_destroy(&port);
if (count < 0) {
/* Discard the prepared select. */
@@ -1323,9 +1323,9 @@ tx_process_call(struct cmsg *m)
}
if (msg->header.type == IPROTO_CALL_16)
- count = port_dump_16(&port, out);
+ count = port_dump_msgpack_16(&port, out);
else
- count = port_dump(&port, out);
+ count = port_dump_msgpack(&port, out);
port_destroy(&port);
if (count < 0) {
obuf_rollback_to_svp(out, &svp);
diff --git a/src/box/lua/call.c b/src/box/lua/call.c
index be13812aa..2f5c9deec 100644
--- a/src/box/lua/call.c
+++ b/src/box/lua/call.c
@@ -416,8 +416,8 @@ port_lua_destroy(struct port *base)
}
static const struct port_vtab port_lua_vtab = {
- .dump = port_lua_dump,
- .dump_16 = port_lua_dump_16,
+ .dump_msgpack = port_lua_dump,
+ .dump_msgpack_16 = port_lua_dump_16,
.destroy = port_lua_destroy,
};
diff --git a/src/box/port.c b/src/box/port.c
index 03f6be79d..255eb732c 100644
--- a/src/box/port.c
+++ b/src/box/port.c
@@ -96,7 +96,7 @@ port_tuple_destroy(struct port *base)
}
static int
-port_tuple_dump_16(struct port *base, struct obuf *out)
+port_tuple_dump_msgpack_16(struct port *base, struct obuf *out)
{
struct port_tuple *port = port_tuple(base);
struct port_tuple_entry *pe;
@@ -113,14 +113,14 @@ port_tuple_dump_16(struct port *base, struct obuf *out)
}
static int
-port_tuple_dump(struct port *base, struct obuf *out)
+port_tuple_dump_msgpack(struct port *base, struct obuf *out)
{
struct port_tuple *port = port_tuple(base);
char *size_buf = obuf_alloc(out, mp_sizeof_array(port->size));
if (size_buf == NULL)
return -1;
mp_encode_array(size_buf, port->size);
- if (port_tuple_dump_16(base, out) < 0)
+ if (port_tuple_dump_msgpack_16(base, out) < 0)
return -1;
return 1;
}
@@ -132,15 +132,15 @@ port_destroy(struct port *port)
}
int
-port_dump(struct port *port, struct obuf *out)
+port_dump_msgpack(struct port *port, struct obuf *out)
{
- return port->vtab->dump(port, out);
+ return port->vtab->dump_msgpack(port, out);
}
int
-port_dump_16(struct port *port, struct obuf *out)
+port_dump_msgpack_16(struct port *port, struct obuf *out)
{
- return port->vtab->dump_16(port, out);
+ return port->vtab->dump_msgpack_16(port, out);
}
void
@@ -157,7 +157,7 @@ port_free(void)
}
const struct port_vtab port_tuple_vtab = {
- .dump = port_tuple_dump,
- .dump_16 = port_tuple_dump_16,
+ .dump_msgpack = port_tuple_dump_msgpack,
+ .dump_msgpack_16 = port_tuple_dump_msgpack_16,
.destroy = port_tuple_destroy,
};
diff --git a/src/box/port.h b/src/box/port.h
index 7cf3339b5..1c44b9b00 100644
--- a/src/box/port.h
+++ b/src/box/port.h
@@ -70,12 +70,12 @@ struct port_vtab {
* On success returns number of entries dumped.
* On failure sets diag and returns -1.
*/
- int (*dump)(struct port *port, struct obuf *out);
+ int (*dump_msgpack)(struct port *port, struct obuf *out);
/**
- * Same as dump(), but use the legacy Tarantool 1.6
- * format.
+ * Same as dump_msgpack(), but use the legacy Tarantool
+ * 1.6 format.
*/
- int (*dump_16)(struct port *port, struct obuf *out);
+ int (*dump_msgpack_16)(struct port *port, struct obuf *out);
/**
* Destroy a port and release associated resources.
*/
@@ -149,14 +149,14 @@ port_destroy(struct port *port);
* Return number of entries dumped on success, -1 on error.
*/
int
-port_dump(struct port *port, struct obuf *out);
+port_dump_msgpack(struct port *port, struct obuf *out);
/**
* Same as port_dump(), but use the legacy Tarantool 1.6
* format.
*/
int
-port_dump_16(struct port *port, struct obuf *out);
+port_dump_msgpack_16(struct port *port, struct obuf *out);
void
port_init(void);
--
2.15.1 (Apple Git-101)
^ permalink raw reply [flat|nested] 34+ messages in thread* [PATCH v2 08/10] session: introduce text box.session.push
2018-04-20 13:24 [PATCH v2 00/10] session: introduce box.session.push Vladislav Shpilevoy
` (7 preceding siblings ...)
2018-04-20 13:24 ` [PATCH v2 07/10] port: rename dump() into dump_msgpack() Vladislav Shpilevoy
@ 2018-04-20 13:24 ` Vladislav Shpilevoy
2018-05-10 19:27 ` [tarantool-patches] " Konstantin Osipov
2018-04-20 13:24 ` [PATCH v2 09/10] session: enable box.session.push in local console Vladislav Shpilevoy
2018-05-24 20:50 ` [tarantool-patches] [PATCH 1/1] netbox: introduce iterable future objects Vladislav Shpilevoy
10 siblings, 1 reply; 34+ messages in thread
From: Vladislav Shpilevoy @ 2018-04-20 13:24 UTC (permalink / raw)
To: tarantool-patches; +Cc: vdavydov.dev
box.session.push allows to send some intermediate results in the
scope of main request with no finalizing it. Messages can be
sent over text and binary protocol. This patch allows to send
text pushes.
Text push is a YAML document tagged with '!push!' handle and
'tag:tarantool.io/push,2018' prefix. YAML tags is a standard way
to define a type of the document.
Console received push message just prints it to the stdout (or
sends to a next console, if it is remote console too).
Part of #2677
---
src/box/lua/console.c | 45 +++++++++++++++++++++++++++++++++++-
src/box/lua/console.h | 10 ++++++++
src/box/lua/console.lua | 29 +++++++++++++++++++----
src/box/lua/session.c | 54 ++++++++++++++++++++++++++++++++++++++++++-
src/box/port.c | 6 +++++
src/box/port.h | 17 ++++++++++++++
src/fio.c | 2 +-
src/fio.h | 2 +-
test/app-tap/console.test.lua | 6 +++--
9 files changed, 160 insertions(+), 11 deletions(-)
diff --git a/src/box/lua/console.c b/src/box/lua/console.c
index a3bf83cb1..c4886db8c 100644
--- a/src/box/lua/console.c
+++ b/src/box/lua/console.c
@@ -31,10 +31,12 @@
#include "box/lua/console.h"
#include "box/session.h"
+#include "box/port.h"
#include "lua/utils.h"
#include "lua/fiber.h"
#include "fiber.h"
#include "coio.h"
+#include "fio.h"
#include "lua-yaml/lyaml.h"
#include <lua.h>
#include <lauxlib.h>
@@ -364,6 +366,47 @@ console_session_fd(struct session *session)
return session->meta.fd;
}
+int
+console_encode_push(struct lua_State *L)
+{
+ return lua_yaml_encode_tagged(L, luaL_yaml_default, "!push!",
+ "tag:tarantool.io/push,2018");
+}
+
+/**
+ * Push a tagged YAML document into a console socket.
+ * @param session Console session.
+ * @param port Port with YAML to push.
+ *
+ * @retval 0 Success.
+ * @retval -1 Error.
+ */
+static int
+console_session_push(struct session *session, struct port *port)
+{
+ assert(session_vtab_registry[session->type].push ==
+ console_session_push);
+ uint32_t text_len;
+ const char *text = port_dump_plain(port, &text_len);
+ if (text == NULL)
+ return -1;
+ int fd = session_fd(session);
+ while (text_len > 0) {
+ while (coio_wait(fd, COIO_WRITE,
+ TIMEOUT_INFINITY) != COIO_WRITE);
+ const struct iovec iov = {
+ .iov_base = (void *) text,
+ .iov_len = text_len
+ };
+ ssize_t rc = fio_writev(fd, &iov, 1);
+ if (rc < 0)
+ return -1;
+ text_len -= rc;
+ text += rc;
+ }
+ return 0;
+}
+
void
tarantool_lua_console_init(struct lua_State *L)
{
@@ -400,7 +443,7 @@ tarantool_lua_console_init(struct lua_State *L)
*/
lua_setfield(L, -2, "formatter");
struct session_vtab console_session_vtab = {
- /* .push = */ generic_session_push,
+ /* .push = */ console_session_push,
/* .fd = */ console_session_fd,
/* .sync = */ generic_session_sync,
};
diff --git a/src/box/lua/console.h b/src/box/lua/console.h
index 208b31490..6d1449810 100644
--- a/src/box/lua/console.h
+++ b/src/box/lua/console.h
@@ -36,6 +36,16 @@ extern "C" {
struct lua_State;
+/**
+ * Encode a single value on top of the stack into YAML document
+ * tagged as push message.
+ * @param object Any lua object on top of the stack.
+ * @retval nil, error Error occured.
+ * @retval not nil Tagged YAML document.
+ */
+int
+console_encode_push(struct lua_State *L);
+
void
tarantool_lua_console_init(struct lua_State *L);
diff --git a/src/box/lua/console.lua b/src/box/lua/console.lua
index bc4e02bfc..b8ae5ba59 100644
--- a/src/box/lua/console.lua
+++ b/src/box/lua/console.lua
@@ -11,6 +11,7 @@ local yaml = require('yaml')
local net_box = require('net.box')
local YAML_TERM = '\n...\n'
+local PUSH_TAG_HANDLE = '!push!'
local function format(status, ...)
local err
@@ -92,13 +93,27 @@ local text_connection_mt = {
--
eval = function(self, text)
text = text..'$EOF$\n'
- if self:write(text) then
+ if not self:write(text) then
+ error(self:set_error())
+ end
+ while true do
local rc = self:read()
- if rc then
+ if not rc then
+ break
+ end
+ local handle, prefix = yaml.decode_tag(rc)
+ assert(handle or not prefix)
+ if not handle then
+ -- Can not fail - tags are encoded with no
+ -- user participation and are correct always.
+ assert(not prefix)
return rc
end
+ if handle == PUSH_TAG_HANDLE and self.print_f then
+ self.print_f(rc)
+ end
end
- error(self:set_error())
+ return rc
end,
--
-- Make the connection be in error state, set error
@@ -121,15 +136,18 @@ local text_connection_mt = {
-- netbox-like object.
-- @param connection Socket to wrap.
-- @param url Parsed destination URL.
+-- @param print_f Function to print push messages.
+--
-- @retval nil, err Error, and err contains an error message.
-- @retval not nil Netbox-like object.
--
-local function wrap_text_socket(connection, url)
+local function wrap_text_socket(connection, url, print_f)
local conn = setmetatable({
_socket = connection,
state = 'active',
host = url.host or 'localhost',
port = url.service,
+ print_f = print_f,
}, text_connection_mt)
if not conn:write('require("console").delimiter("$EOF$")\n') or
not conn:read() then
@@ -369,7 +387,8 @@ local function connect(uri, opts)
end
local remote
if greeting.protocol == 'Lua console' then
- remote = wrap_text_socket(connection, u)
+ remote = wrap_text_socket(connection, u,
+ function(msg) self:print(msg) end)
else
opts = {
connect_timeout = opts.timeout,
diff --git a/src/box/lua/session.c b/src/box/lua/session.c
index 5fe5f08d4..306271809 100644
--- a/src/box/lua/session.c
+++ b/src/box/lua/session.c
@@ -41,6 +41,8 @@
#include "box/session.h"
#include "box/user.h"
#include "box/schema.h"
+#include "box/port.h"
+#include "box/lua/console.h"
static const char *sessionlib_name = "box.session";
@@ -355,6 +357,52 @@ lbox_push_on_access_denied_event(struct lua_State *L, void *event)
return 3;
}
+/**
+ * Port to push a message from Lua.
+ */
+struct lua_push_port {
+ const struct port_vtab *vtab;
+ /**
+ * Lua state, containing data to dump on top of the stack.
+ */
+ struct lua_State *L;
+};
+
+static const char *
+lua_push_port_dump_plain(struct port *port, uint32_t *size);
+
+static const struct port_vtab lua_push_port_vtab = {
+ .dump_msgpack = NULL,
+ /*
+ * Dump_16 has no sense, since push appears since 1.10
+ * protocol.
+ */
+ .dump_msgpack_16 = NULL,
+ .dump_plain = lua_push_port_dump_plain,
+ .destroy = NULL,
+};
+
+static const char *
+lua_push_port_dump_plain(struct port *port, uint32_t *size)
+{
+ struct lua_push_port *lua_port = (struct lua_push_port *) port;
+ assert(lua_port->vtab == &lua_push_port_vtab);
+ struct lua_State *L = lua_port->L;
+ int rc = console_encode_push(L);
+ if (rc == 2) {
+ assert(lua_isnil(L, -2));
+ assert(lua_isstring(L, -1));
+ diag_set(ClientError, ER_PROC_LUA, lua_tostring(L, -1));
+ return NULL;
+ }
+ assert(rc == 1);
+ assert(lua_isstring(L, -1));
+ size_t len;
+ const char *result = lua_tolstring(L, -1, &len);
+ *size = (uint32_t) len;
+ return result;
+}
+
/**
* Push a message using a protocol, depending on a session type.
* @param data Data to push, first argument on a stack.
@@ -367,7 +415,11 @@ lbox_session_push(struct lua_State *L)
if (lua_gettop(L) != 1)
return luaL_error(L, "Usage: box.session.push(data)");
- if (session_push(current_session(), NULL) != 0) {
+ struct lua_push_port port;
+ port.vtab = &lua_push_port_vtab;
+ port.L = L;
+
+ if (session_push(current_session(), (struct port *) &port) != 0) {
lua_pushnil(L);
luaT_pusherror(L, box_error_last());
return 2;
diff --git a/src/box/port.c b/src/box/port.c
index 255eb732c..f9b655840 100644
--- a/src/box/port.c
+++ b/src/box/port.c
@@ -143,6 +143,12 @@ port_dump_msgpack_16(struct port *port, struct obuf *out)
return port->vtab->dump_msgpack_16(port, out);
}
+const char *
+port_dump_plain(struct port *port, uint32_t *size)
+{
+ return port->vtab->dump_plain(port, size);
+}
+
void
port_init(void)
{
diff --git a/src/box/port.h b/src/box/port.h
index 1c44b9b00..7fc1b8972 100644
--- a/src/box/port.h
+++ b/src/box/port.h
@@ -76,6 +76,11 @@ struct port_vtab {
* 1.6 format.
*/
int (*dump_msgpack_16)(struct port *port, struct obuf *out);
+ /**
+ * Dump a port content as a plain text into a buffer,
+ * allocated inside.
+ */
+ const char *(*dump_plain)(struct port *port, uint32_t *size);
/**
* Destroy a port and release associated resources.
*/
@@ -158,6 +163,18 @@ port_dump_msgpack(struct port *port, struct obuf *out);
int
port_dump_msgpack_16(struct port *port, struct obuf *out);
+/**
+ * Dump a port content as a plain text into a buffer,
+ * allocated inside.
+ * @param port Port with data to dump.
+ * @param[out] size Length of a result plain text.
+ *
+ * @retval nil Error.
+ * @retval not nil Plain text.
+ */
+const char *
+port_dump_plain(struct port *port, uint32_t *size);
+
void
port_init(void);
diff --git a/src/fio.c b/src/fio.c
index b79d3d058..b1d9ecf44 100644
--- a/src/fio.c
+++ b/src/fio.c
@@ -135,7 +135,7 @@ fio_writen(int fd, const void *buf, size_t count)
}
ssize_t
-fio_writev(int fd, struct iovec *iov, int iovcnt)
+fio_writev(int fd, const struct iovec *iov, int iovcnt)
{
assert(iov && iovcnt >= 0);
ssize_t nwr;
diff --git a/src/fio.h b/src/fio.h
index 12749afcb..fb6383c81 100644
--- a/src/fio.h
+++ b/src/fio.h
@@ -133,7 +133,7 @@ fio_writen(int fd, const void *buf, size_t count);
* returns the total number of bytes written, or -1 if error.
*/
ssize_t
-fio_writev(int fd, struct iovec *iov, int iovcnt);
+fio_writev(int fd, const struct iovec *iov, int iovcnt);
/**
* A wrapper around writev, but retries for partial writes
diff --git a/test/app-tap/console.test.lua b/test/app-tap/console.test.lua
index d2e88b55b..b1d7166d4 100755
--- a/test/app-tap/console.test.lua
+++ b/test/app-tap/console.test.lua
@@ -21,7 +21,7 @@ local EOL = "\n...\n"
test = tap.test("console")
-test:plan(60)
+test:plan(61)
-- Start console and connect to it
local server = console.listen(CONSOLE_SOCKET)
@@ -35,7 +35,9 @@ test:ok(client ~= nil, "connect to console")
-- gh-2677: box.session.push, text protocol support.
--
client:write('box.session.push(200)\n')
-test:is(client:read(EOL), "---\n- null\n- Session 'console' does not support push()\n...\n", "push does not work")
+test:is(client:read(EOL), '%TAG !push! tag:tarantool.io/push,2018\n--- 200\n...\n',
+ "pushed message")
+test:is(client:read(EOL), '---\n- true\n...\n', "pushed message")
-- Execute some command
client:write("1\n")
--
2.15.1 (Apple Git-101)
^ permalink raw reply [flat|nested] 34+ messages in thread* Re: [tarantool-patches] [PATCH v2 08/10] session: introduce text box.session.push
2018-04-20 13:24 ` [PATCH v2 08/10] session: introduce text box.session.push Vladislav Shpilevoy
@ 2018-05-10 19:27 ` Konstantin Osipov
2018-05-24 20:50 ` [tarantool-patches] " Vladislav Shpilevoy
0 siblings, 1 reply; 34+ messages in thread
From: Konstantin Osipov @ 2018-05-10 19:27 UTC (permalink / raw)
To: tarantool-patches; +Cc: vdavydov.dev
* Vladislav Shpilevoy <v.shpilevoy@tarantool.org> [18/04/20 16:31]:
> +/**
> + * Push a tagged YAML document into a console socket.
> + * @param session Console session.
> + * @param port Port with YAML to push.
> + *
> + * @retval 0 Success.
> + * @retval -1 Error.
> + */
> +static int
> +console_session_push(struct session *session, struct port *port)
> +{
> + assert(session_vtab_registry[session->type].push ==
> + console_session_push);
> + uint32_t text_len;
> + const char *text = port_dump_plain(port, &text_len);
> + if (text == NULL)
> + return -1;
> + int fd = session_fd(session);
> + while (text_len > 0) {
> + while (coio_wait(fd, COIO_WRITE,
> + TIMEOUT_INFINITY) != COIO_WRITE);
Nitpick: the socket is ready in 99% of cases, there is no reason
to call coio_wait() unless you get EINTR.
Second, why the choice of fio_writev() rather than write() or
send?
What is wrong with coio_writev or similar?
> + const struct iovec iov = {
> + .iov_base = (void *) text,
> + .iov_len = text_len
> + };
> + ssize_t rc = fio_writev(fd, &iov, 1);
> + if (rc < 0)
> + return -1;
> + text_len -= rc;
> + text += rc;
> + }
> + return 0;
> +}
> @@ -92,13 +93,27 @@ local text_connection_mt = {
> --
> eval = function(self, text)
> text = text..'$EOF$\n'
> - if self:write(text) then
> + if not self:write(text) then
> + error(self:set_error())
> + end
> + while true do
> local rc = self:read()
> - if rc then
> + if not rc then
> + break
> + end
> + local handle, prefix = yaml.decode_tag(rc)
> + assert(handle or not prefix)
> + if not handle then
> + -- Can not fail - tags are encoded with no
> + -- user participation and are correct always.
> + assert(not prefix)
In Lua, asserts take CPU time. Please don't use them unless in a
test.
> return rc
> end
> + if handle == PUSH_TAG_HANDLE and self.print_f then
> + self.print_f(rc)
> + end
> end
> - error(self:set_error())
> + return rc
> end,
> --
> -- Make the connection be in error state, set error
> @@ -121,15 +136,18 @@ local text_connection_mt = {
> -- netbox-like object.
> -- @param connection Socket to wrap.
> -- @param url Parsed destination URL.
> +-- @param print_f Function to print push messages.
> +--
> -- @retval nil, err Error, and err contains an error message.
> -- @retval not nil Netbox-like object.
> --
> -local function wrap_text_socket(connection, url)
> +local function wrap_text_socket(connection, url, print_f)
> local conn = setmetatable({
> _socket = connection,
> state = 'active',
> host = url.host or 'localhost',
> port = url.service,
> + print_f = print_f,
> }, text_connection_mt)
> if not conn:write('require("console").delimiter("$EOF$")\n') or
> not conn:read() then
> @@ -369,7 +387,8 @@ local function connect(uri, opts)
> end
> local remote
> if greeting.protocol == 'Lua console' then
> - remote = wrap_text_socket(connection, u)
> + remote = wrap_text_socket(connection, u,
> + function(msg) self:print(msg) end)
> else
> opts = {
> connect_timeout = opts.timeout,
> diff --git a/src/box/lua/session.c b/src/box/lua/session.c
> index 5fe5f08d4..306271809 100644
> --- a/src/box/lua/session.c
> +++ b/src/box/lua/session.c
> @@ -41,6 +41,8 @@
> #include "box/session.h"
> #include "box/user.h"
> #include "box/schema.h"
> +#include "box/port.h"
> +#include "box/lua/console.h"
>
> static const char *sessionlib_name = "box.session";
>
> @@ -355,6 +357,52 @@ lbox_push_on_access_denied_event(struct lua_State *L, void *event)
> return 3;
> }
>
> +/**
> + * Port to push a message from Lua.
> + */
> +struct lua_push_port {
> + const struct port_vtab *vtab;
> + /**
> + * Lua state, containing data to dump on top of the stack.
> + */
> + struct lua_State *L;
> +};
> +
> +static const char *
> +lua_push_port_dump_plain(struct port *port, uint32_t *size);
> +
> +static const struct port_vtab lua_push_port_vtab = {
> + .dump_msgpack = NULL,
> + /*
> + * Dump_16 has no sense, since push appears since 1.10
> + * protocol.
> + */
> + .dump_msgpack_16 = NULL,
> + .dump_plain = lua_push_port_dump_plain,
> + .destroy = NULL,
> +};
> +
> +static const char *
> +lua_push_port_dump_plain(struct port *port, uint32_t *size)
> +{
> + struct lua_push_port *lua_port = (struct lua_push_port *) port;
> + assert(lua_port->vtab == &lua_push_port_vtab);
> + struct lua_State *L = lua_port->L;
> + int rc = console_encode_push(L);
> + if (rc == 2) {
> + assert(lua_isnil(L, -2));
> + assert(lua_isstring(L, -1));
> + diag_set(ClientError, ER_PROC_LUA, lua_tostring(L, -1));
> + return NULL;
> + }
> + assert(rc == 1);
> + assert(lua_isstring(L, -1));
> + size_t len;
> + const char *result = lua_tolstring(L, -1, &len);
> + *size = (uint32_t) len;
> + return result;
> +}
> +
> /**
> * Push a message using a protocol, depending on a session type.
> * @param data Data to push, first argument on a stack.
> @@ -367,7 +415,11 @@ lbox_session_push(struct lua_State *L)
> if (lua_gettop(L) != 1)
> return luaL_error(L, "Usage: box.session.push(data)");
>
> - if (session_push(current_session(), NULL) != 0) {
> + struct lua_push_port port;
> + port.vtab = &lua_push_port_vtab;
Why do you need a separate port?
And why do you need to create a port for every push? Can't you
reuse the same port as is used for the Lua call itself?
> + port.L = L;
> +
> + if (session_push(current_session(), (struct port *) &port) != 0) {
> lua_pushnil(L);
> luaT_pusherror(L, box_error_last());
> return 2;
> diff --git a/src/box/port.c b/src/box/port.c
> index 255eb732c..f9b655840 100644
--
Konstantin Osipov, Moscow, Russia, +7 903 626 22 32
http://tarantool.io - www.twitter.com/kostja_osipov
^ permalink raw reply [flat|nested] 34+ messages in thread* Re: [tarantool-patches] Re: [PATCH v2 08/10] session: introduce text box.session.push
2018-05-10 19:27 ` [tarantool-patches] " Konstantin Osipov
@ 2018-05-24 20:50 ` Vladislav Shpilevoy
0 siblings, 0 replies; 34+ messages in thread
From: Vladislav Shpilevoy @ 2018-05-24 20:50 UTC (permalink / raw)
To: tarantool-patches, Konstantin Osipov; +Cc: vdavydov.dev
On 10/05/2018 22:27, Konstantin Osipov wrote:
> * Vladislav Shpilevoy <v.shpilevoy@tarantool.org> [18/04/20 16:31]:
>> +/**
>> + * Push a tagged YAML document into a console socket.
>> + * @param session Console session.
>> + * @param port Port with YAML to push.
>> + *
>> + * @retval 0 Success.
>> + * @retval -1 Error.
>> + */
>> +static int
>> +console_session_push(struct session *session, struct port *port)
>> +{
>> + assert(session_vtab_registry[session->type].push ==
>> + console_session_push);
>> + uint32_t text_len;
>> + const char *text = port_dump_plain(port, &text_len);
>> + if (text == NULL)
>> + return -1;
>> + int fd = session_fd(session);
>> + while (text_len > 0) {
>> + while (coio_wait(fd, COIO_WRITE,
>> + TIMEOUT_INFINITY) != COIO_WRITE);
>
> Nitpick: the socket is ready in 99% of cases, there is no reason
> to call coio_wait() unless you get EINTR.
> Second, why the choice of fio_writev() rather than write() or
> send?
We have discussed that verbally, but I duplicate it here for the
record. I used fio_writev since it does not return -1 on EINTR, but
it appeared to return -1 on EWOULDBLOCK/EAGAIN, so I was wrong -
I can not use it here, you are right.
>
> What is wrong with coio_writev or similar?
Because console socket has no ev_io object. You asked me investigate if
lua socket has encapsulated ev_io - it does not. Lua socket always
uses file descriptor only, and does coio_wait(int fd). So I can not pass it
to session meta directly from socket.lua with no big refactoring of Lua
socket. If you really want it, I can open a separate issue for that,
label 'good first issue' and assign a student. It is a simple but big
task, that does not block push.
I refactored console push to write into a socket, and only then do
coio_wait if necessary.
diff --git a/src/box/lua/console.c b/src/box/lua/console.c
index 65a2192da..5ded99c98 100644
--- a/src/box/lua/console.c
+++ b/src/box/lua/console.c
@@ -394,17 +394,20 @@ console_session_push(struct session *session, struct port *port)
return -1;
int fd = session_fd(session);
while (text_len > 0) {
- while (coio_wait(fd, COIO_WRITE,
- TIMEOUT_INFINITY) != COIO_WRITE);
- const struct iovec iov = {
- .iov_base = (void *) text,
- .iov_len = text_len
- };
- ssize_t rc = fio_writev(fd, &iov, 1);
- if (rc < 0)
- return -1;
- text_len -= rc;
- text += rc;
+ ssize_t rc = write(fd, text, text_len);
+ if (rc < 0) {
+ if (errno == EAGAIN || errno == EWOULDBLOCK) {
+ while (coio_wait(fd, COIO_WRITE,
+ TIMEOUT_INFINITY) !=
+ COIO_WRITE);
+ } else if (errno != EINTR) {
+ diag_set(SocketError, fd, strerror(errno));
+ return -1;
+ }
+ } else {
+ text_len -= (uint32_t) rc;
+ text += rc;
+ }
}
return 0;
}
>
>> + const struct iovec iov = {
>> + .iov_base = (void *) text,
>> + .iov_len = text_len
>> + };
>> + ssize_t rc = fio_writev(fd, &iov, 1);
>> + if (rc < 0)
>> + return -1;
>> + text_len -= rc;
>> + text += rc;
>> + }
>> + return 0;
>> +}
>> @@ -92,13 +93,27 @@ local text_connection_mt = {
>> --
>> eval = function(self, text)
>> text = text..'$EOF$\n'
>> - if self:write(text) then
>> + if not self:write(text) then
>> + error(self:set_error())
>> + end
>> + while true do
>> local rc = self:read()
>> - if rc then
>> + if not rc then
>> + break
>> + end
>> + local handle, prefix = yaml.decode_tag(rc)
>> + assert(handle or not prefix)
>> + if not handle then
>> + -- Can not fail - tags are encoded with no
>> + -- user participation and are correct always.
>> + assert(not prefix)
>
> In Lua, asserts take CPU time. Please don't use them unless in a
> test.
diff --git a/src/box/lua/console.lua b/src/box/lua/console.lua
index b8ae5ba59..574882188 100644
--- a/src/box/lua/console.lua
+++ b/src/box/lua/console.lua
@@ -102,11 +102,9 @@ local text_connection_mt = {
break
end
local handle, prefix = yaml.decode_tag(rc)
- assert(handle or not prefix)
if not handle then
-- Can not fail - tags are encoded with no
-- user participation and are correct always.
- assert(not prefix)
return rc
end
if handle == PUSH_TAG_HANDLE and self.print_f then
>
>> return rc
>> end
>> + if handle == PUSH_TAG_HANDLE and self.print_f then
>> + self.print_f(rc)
>> + end
>> end
>> - error(self:set_error())
>> + return rc
>> end,
>> --
>> -- Make the connection be in error state, set error
>> @@ -121,15 +136,18 @@ local text_connection_mt = {
>> -- netbox-like object.
>> -- @param connection Socket to wrap.
>> -- @param url Parsed destination URL.
>> +-- @param print_f Function to print push messages.
>> +--
>> -- @retval nil, err Error, and err contains an error message.
>> -- @retval not nil Netbox-like object.
>> --
>> -local function wrap_text_socket(connection, url)
>> +local function wrap_text_socket(connection, url, print_f)
>> local conn = setmetatable({
>> _socket = connection,
>> state = 'active',
>> host = url.host or 'localhost',
>> port = url.service,
>> + print_f = print_f,
>> }, text_connection_mt)
>> if not conn:write('require("console").delimiter("$EOF$")\n') or
>> not conn:read() then
>> @@ -369,7 +387,8 @@ local function connect(uri, opts)
>> end
>> local remote
>> if greeting.protocol == 'Lua console' then
>> - remote = wrap_text_socket(connection, u)
>> + remote = wrap_text_socket(connection, u,
>> + function(msg) self:print(msg) end)
>> else
>> opts = {
>> connect_timeout = opts.timeout,
>> diff --git a/src/box/lua/session.c b/src/box/lua/session.c
>> index 5fe5f08d4..306271809 100644
>> --- a/src/box/lua/session.c
>> +++ b/src/box/lua/session.c
>> @@ -367,7 +415,11 @@ lbox_session_push(struct lua_State *L)
>> if (lua_gettop(L) != 1)
>> return luaL_error(L, "Usage: box.session.push(data)");
>>
>> - if (session_push(current_session(), NULL) != 0) {
>> + struct lua_push_port port;
>> + port.vtab = &lua_push_port_vtab;
>
> Why do you need a separate port?
I need a separate port, because origin struct port does not have
struct lua_State *L. I need the lua stack in console push to get
the object, encode it into YAML and send to a socket.
>
> And why do you need to create a port for every push? Can'st you
> reuse the same port as is used for the Lua call itself?
Lua call does not reuse port. The port is created on each call in
iproto.cc on stack, exactly like here. I can not understand why do
you think that creating an object on the stack and setting a pair
of its fields makes noticeable performance input. Especially taking
into account that Lua does most of work here.
>
>> + port.L = L;
>> +
>> + if (session_push(current_session(), (struct port *) &port) != 0) {
>> lua_pushnil(L);
>> luaT_pusherror(L, box_error_last());
>> return 2;
>> diff --git a/src/box/port.c b/src/box/port.c
>> index 255eb732c..f9b655840 100644
>
The patch is below:
=============================================================================
commit 9c626685ce752c03029f49fc8b8298fe6a2e2f9f
Author: Vladislav Shpilevoy <v.shpilevoy@tarantool.org>
Date: Thu Apr 19 19:50:25 2018 +0300
session: introduce text box.session.push
box.session.push allows to send some intermediate results in the
scope of main request with no finalizing it. Messages can be
sent over text and binary protocol. This patch allows to send
text pushes.
Text push is a YAML document tagged with '!push!' handle and
'tag:tarantool.io/push,2018' prefix. YAML tags is a standard way
to define a type of the document.
Console received push message just prints it to the stdout (or
sends to a next console, if it is remote console too).
Part of #2677
diff --git a/src/box/lua/console.c b/src/box/lua/console.c
index 7e17fa30a..5ded99c98 100644
--- a/src/box/lua/console.c
+++ b/src/box/lua/console.c
@@ -31,10 +31,12 @@
#include "box/lua/console.h"
#include "box/session.h"
+#include "box/port.h"
#include "lua/utils.h"
#include "lua/fiber.h"
#include "fiber.h"
#include "coio.h"
#include "lua-yaml/lyaml.h"
#include <lua.h>
#include <lauxlib.h>
@@ -366,6 +368,50 @@ console_session_fd(struct session *session)
return session->meta.fd;
}
+int
+console_encode_push(struct lua_State *L)
+{
+ return lua_yaml_encode_tagged(L, luaL_yaml_default, "!push!",
+ "tag:tarantool.io/push,2018");
+}
+
+/**
+ * Push a tagged YAML document into a console socket.
+ * @param session Console session.
+ * @param port Port with YAML to push.
+ *
+ * @retval 0 Success.
+ * @retval -1 Error.
+ */
+static int
+console_session_push(struct session *session, struct port *port)
+{
+ assert(session_vtab_registry[session->type].push ==
+ console_session_push);
+ uint32_t text_len;
+ const char *text = port_dump_plain(port, &text_len);
+ if (text == NULL)
+ return -1;
+ int fd = session_fd(session);
+ while (text_len > 0) {
+ ssize_t rc = write(fd, text, text_len);
+ if (rc < 0) {
+ if (errno == EAGAIN || errno == EWOULDBLOCK) {
+ while (coio_wait(fd, COIO_WRITE,
+ TIMEOUT_INFINITY) !=
+ COIO_WRITE);
+ } else if (errno != EINTR) {
+ diag_set(SocketError, fd, strerror(errno));
+ return -1;
+ }
+ } else {
+ text_len -= (uint32_t) rc;
+ text += rc;
+ }
+ }
+ return 0;
+}
+
void
tarantool_lua_console_init(struct lua_State *L)
{
@@ -400,7 +446,7 @@ tarantool_lua_console_init(struct lua_State *L)
*/
lua_setfield(L, -2, "formatter");
struct session_vtab console_session_vtab = {
- /* .push = */ generic_session_push,
+ /* .push = */ console_session_push,
/* .fd = */ console_session_fd,
/* .sync = */ generic_session_sync,
};
diff --git a/src/box/lua/console.h b/src/box/lua/console.h
index 208b31490..6d1449810 100644
--- a/src/box/lua/console.h
+++ b/src/box/lua/console.h
@@ -36,6 +36,16 @@ extern "C" {
struct lua_State;
+/**
+ * Encode a single value on top of the stack into YAML document
+ * tagged as push message.
+ * @param object Any lua object on top of the stack.
+ * @retval nil, error Error occured.
+ * @retval not nil Tagged YAML document.
+ */
+int
+console_encode_push(struct lua_State *L);
+
void
tarantool_lua_console_init(struct lua_State *L);
diff --git a/src/box/lua/console.lua b/src/box/lua/console.lua
index bc4e02bfc..574882188 100644
--- a/src/box/lua/console.lua
+++ b/src/box/lua/console.lua
@@ -11,6 +11,7 @@ local yaml = require('yaml')
local net_box = require('net.box')
local YAML_TERM = '\n...\n'
+local PUSH_TAG_HANDLE = '!push!'
local function format(status, ...)
local err
@@ -92,13 +93,25 @@ local text_connection_mt = {
--
eval = function(self, text)
text = text..'$EOF$\n'
- if self:write(text) then
+ if not self:write(text) then
+ error(self:set_error())
+ end
+ while true do
local rc = self:read()
- if rc then
+ if not rc then
+ break
+ end
+ local handle, prefix = yaml.decode_tag(rc)
+ if not handle then
+ -- Can not fail - tags are encoded with no
+ -- user participation and are correct always.
return rc
end
+ if handle == PUSH_TAG_HANDLE and self.print_f then
+ self.print_f(rc)
+ end
end
- error(self:set_error())
+ return rc
end,
--
-- Make the connection be in error state, set error
@@ -121,15 +134,18 @@ local text_connection_mt = {
-- netbox-like object.
-- @param connection Socket to wrap.
-- @param url Parsed destination URL.
+-- @param print_f Function to print push messages.
+--
-- @retval nil, err Error, and err contains an error message.
-- @retval not nil Netbox-like object.
--
-local function wrap_text_socket(connection, url)
+local function wrap_text_socket(connection, url, print_f)
local conn = setmetatable({
_socket = connection,
state = 'active',
host = url.host or 'localhost',
port = url.service,
+ print_f = print_f,
}, text_connection_mt)
if not conn:write('require("console").delimiter("$EOF$")\n') or
not conn:read() then
@@ -369,7 +385,8 @@ local function connect(uri, opts)
end
local remote
if greeting.protocol == 'Lua console' then
- remote = wrap_text_socket(connection, u)
+ remote = wrap_text_socket(connection, u,
+ function(msg) self:print(msg) end)
else
opts = {
connect_timeout = opts.timeout,
diff --git a/src/box/lua/session.c b/src/box/lua/session.c
index 05010c4c3..ade85491f 100644
--- a/src/box/lua/session.c
+++ b/src/box/lua/session.c
@@ -41,6 +41,8 @@
#include "box/session.h"
#include "box/user.h"
#include "box/schema.h"
+#include "box/port.h"
+#include "box/lua/console.h"
static const char *sessionlib_name = "box.session";
@@ -355,6 +357,52 @@ lbox_push_on_access_denied_event(struct lua_State *L, void *event)
return 3;
}
+/**
+ * Port to push a message from Lua.
+ */
+struct lua_push_port {
+ const struct port_vtab *vtab;
+ /**
+ * Lua state, containing data to dump on top of the stack.
+ */
+ struct lua_State *L;
+};
+
+static const char *
+lua_push_port_dump_plain(struct port *port, uint32_t *size);
+
+static const struct port_vtab lua_push_port_vtab = {
+ .dump_msgpack = NULL,
+ /*
+ * Dump_16 has no sense, since push appears since 1.10
+ * protocol.
+ */
+ .dump_msgpack_16 = NULL,
+ .dump_plain = lua_push_port_dump_plain,
+ .destroy = NULL,
+};
+
+static const char *
+lua_push_port_dump_plain(struct port *port, uint32_t *size)
+{
+ struct lua_push_port *lua_port = (struct lua_push_port *) port;
+ assert(lua_port->vtab == &lua_push_port_vtab);
+ struct lua_State *L = lua_port->L;
+ int rc = console_encode_push(L);
+ if (rc == 2) {
+ assert(lua_isnil(L, -2));
+ assert(lua_isstring(L, -1));
+ diag_set(ClientError, ER_PROC_LUA, lua_tostring(L, -1));
+ return NULL;
+ }
+ assert(rc == 1);
+ assert(lua_isstring(L, -1));
+ size_t len;
+ const char *result = lua_tolstring(L, -1, &len);
+ *size = (uint32_t) len;
+ return result;
+}
+
/**
* Push a message using a protocol, depending on a session type.
* @param L Lua state. First argument on the stack is data to
@@ -368,7 +416,11 @@ lbox_session_push(struct lua_State *L)
if (lua_gettop(L) != 1)
return luaL_error(L, "Usage: box.session.push(data)");
- if (session_push(current_session(), NULL) != 0) {
+ struct lua_push_port port;
+ port.vtab = &lua_push_port_vtab;
+ port.L = L;
+
+ if (session_push(current_session(), (struct port *) &port) != 0) {
lua_pushnil(L);
luaT_pusherror(L, box_error_last());
return 2;
diff --git a/src/box/port.c b/src/box/port.c
index 255eb732c..f9b655840 100644
--- a/src/box/port.c
+++ b/src/box/port.c
@@ -143,6 +143,12 @@ port_dump_msgpack_16(struct port *port, struct obuf *out)
return port->vtab->dump_msgpack_16(port, out);
}
+const char *
+port_dump_plain(struct port *port, uint32_t *size)
+{
+ return port->vtab->dump_plain(port, size);
+}
+
void
port_init(void)
{
diff --git a/src/box/port.h b/src/box/port.h
index 1c44b9b00..7fc1b8972 100644
--- a/src/box/port.h
+++ b/src/box/port.h
@@ -76,6 +76,11 @@ struct port_vtab {
* 1.6 format.
*/
int (*dump_msgpack_16)(struct port *port, struct obuf *out);
+ /**
+ * Dump a port content as a plain text into a buffer,
+ * allocated inside.
+ */
+ const char *(*dump_plain)(struct port *port, uint32_t *size);
/**
* Destroy a port and release associated resources.
*/
@@ -158,6 +163,18 @@ port_dump_msgpack(struct port *port, struct obuf *out);
int
port_dump_msgpack_16(struct port *port, struct obuf *out);
+/**
+ * Dump a port content as a plain text into a buffer,
+ * allocated inside.
+ * @param port Port with data to dump.
+ * @param[out] size Length of a result plain text.
+ *
+ * @retval nil Error.
+ * @retval not nil Plain text.
+ */
+const char *
+port_dump_plain(struct port *port, uint32_t *size);
+
void
port_init(void);
diff --git a/src/diag.h b/src/diag.h
index dc6c132d5..4fcbab5c3 100644
--- a/src/diag.h
+++ b/src/diag.h
@@ -249,6 +249,9 @@ struct error *
BuildSystemError(const char *file, unsigned line, const char *format, ...);
struct error *
BuildXlogError(const char *file, unsigned line, const char *format, ...);
+struct error *
+BuildSocketError(const char *file, unsigned line, int fd, const char *format,
+ ...);
struct index_def;
diff --git a/src/sio.cc b/src/sio.cc
index c906a97a8..8d71f0382 100644
--- a/src/sio.cc
+++ b/src/sio.cc
@@ -67,6 +67,22 @@ SocketError::SocketError(const char *file, unsigned line, int fd,
errno = save_errno;
}
+struct error *
+BuildSocketError(const char *file, unsigned line, int fd, const char *format,
+ ...)
+{
+ try {
+ SocketError *e = new SocketError(file, line, fd, "");
+ va_list ap;
+ va_start(ap, format);
+ error_vformat_msg(e, format, ap);
+ va_end(ap);
+ return e;
+ } catch (OutOfMemory *e) {
+ return e;
+ }
+}
+
/** Pretty print socket name and peer (for exceptions) */
const char *
sio_socketname(int fd)
diff --git a/test/app-tap/console.test.lua b/test/app-tap/console.test.lua
index a5b3061a9..237b3d002 100755
--- a/test/app-tap/console.test.lua
+++ b/test/app-tap/console.test.lua
@@ -21,7 +21,7 @@ local EOL = "\n...\n"
test = tap.test("console")
-test:plan(60)
+test:plan(61)
-- Start console and connect to it
local server = console.listen(CONSOLE_SOCKET)
@@ -35,7 +35,9 @@ test:ok(client ~= nil, "connect to console")
-- gh-2677: box.session.push, text protocol support.
--
client:write('box.session.push(200)\n')
-test:is(client:read(EOL), "---\n- null\n- Session 'console' does not support push()\n...\n", "push does not work")
+test:is(client:read(EOL), '%TAG !push! tag:tarantool.io/push,2018\n--- 200\n...\n',
+ "pushed message")
+test:is(client:read(EOL), '---\n- true\n...\n', "pushed message")
-- Execute some command
client:write("1\n")
^ permalink raw reply [flat|nested] 34+ messages in thread
* [PATCH v2 09/10] session: enable box.session.push in local console
2018-04-20 13:24 [PATCH v2 00/10] session: introduce box.session.push Vladislav Shpilevoy
` (8 preceding siblings ...)
2018-04-20 13:24 ` [PATCH v2 08/10] session: introduce text box.session.push Vladislav Shpilevoy
@ 2018-04-20 13:24 ` Vladislav Shpilevoy
2018-05-10 19:28 ` [tarantool-patches] " Konstantin Osipov
2018-05-24 20:50 ` [tarantool-patches] [PATCH 1/1] netbox: introduce iterable future objects Vladislav Shpilevoy
10 siblings, 1 reply; 34+ messages in thread
From: Vladislav Shpilevoy @ 2018-04-20 13:24 UTC (permalink / raw)
To: tarantool-patches; +Cc: vdavydov.dev
It is quite simple - just use stdout file descriptor as the
destination for push messages. It is needed to make remote and
local console be similar.
---
src/box/lua/console.c | 1 +
src/box/lua/console.lua | 2 +-
2 files changed, 2 insertions(+), 1 deletion(-)
diff --git a/src/box/lua/console.c b/src/box/lua/console.c
index c4886db8c..fcbaf299d 100644
--- a/src/box/lua/console.c
+++ b/src/box/lua/console.c
@@ -448,6 +448,7 @@ tarantool_lua_console_init(struct lua_State *L)
/* .sync = */ generic_session_sync,
};
session_vtab_registry[SESSION_TYPE_CONSOLE] = console_session_vtab;
+ session_vtab_registry[SESSION_TYPE_REPL] = console_session_vtab;
}
/*
diff --git a/src/box/lua/console.lua b/src/box/lua/console.lua
index b8ae5ba59..b15ca145a 100644
--- a/src/box/lua/console.lua
+++ b/src/box/lua/console.lua
@@ -355,7 +355,7 @@ local function start()
self.history_file = home_dir .. '/.tarantool_history'
internal.load_history(self.history_file)
end
- session_internal.create(-1, "repl")
+ session_internal.create(1, "repl")
repl(self)
started = false
end
--
2.15.1 (Apple Git-101)
^ permalink raw reply [flat|nested] 34+ messages in thread* [tarantool-patches] [PATCH 1/1] netbox: introduce iterable future objects
2018-04-20 13:24 [PATCH v2 00/10] session: introduce box.session.push Vladislav Shpilevoy
` (9 preceding siblings ...)
2018-04-20 13:24 ` [PATCH v2 09/10] session: enable box.session.push in local console Vladislav Shpilevoy
@ 2018-05-24 20:50 ` Vladislav Shpilevoy
2018-06-04 22:17 ` [tarantool-patches] " Vladislav Shpilevoy
10 siblings, 1 reply; 34+ messages in thread
From: Vladislav Shpilevoy @ 2018-05-24 20:50 UTC (permalink / raw)
To: tarantool-patches; +Cc: kostja
Netbox has two major ways to execute a request: sync and async.
During execution of any a server can send multiplie responses via
IPROTO_CHUNK. And the execution ways differ in how to handle the
chunks (called messages or pushes).
For a sync request a one can specify on_push callback and its
on_push_ctx argument called on each message.
When a request is async a user has a future object only, and can
not specify any callbacks. To get the pushed messages a one must
iterate over future object like this:
for i, message in future:pairs(one_iteration_timeout) do
...
end
Or ignore messages just calling future:wait_result(). Anyway
messages are not deleted, so a one can iterate over future object
again and again.
Follow up #2677
---
src/box/lua/net_box.lua | 84 +++++++++++++++++++++
test/box/push.result | 191 +++++++++++++++++++++++++++++++++++++++++++++++-
test/box/push.test.lua | 79 +++++++++++++++++++-
3 files changed, 349 insertions(+), 5 deletions(-)
diff --git a/src/box/lua/net_box.lua b/src/box/lua/net_box.lua
index fe113e740..5d896f7e3 100644
--- a/src/box/lua/net_box.lua
+++ b/src/box/lua/net_box.lua
@@ -272,6 +272,90 @@ local function create_transport(host, port, user, password, callback,
end
end
--
+ -- Get the next message or the final result.
+ -- @param iterator Iterator object.
+ -- @param i Index to get a next message from.
+ --
+ -- @retval nil, nil The request is finished.
+ -- @retval i + 1, object A message/response and its index.
+ -- @retval box.NULL, error An error occured. When this
+ -- function is called in 'for k, v in future:pairs()',
+ -- `k` becomes box.NULL, and `v` becomes error object.
+ -- If a one want to stop the cycle, he can do break.
+ -- With no break the cycle will be continued until
+ -- the request is finished. The iteration continuation
+ -- is useful for example when time is out during a
+ -- next message waiting, but a one does not consider
+ -- this error be critical.
+ -- On error the key becomes exactly box.NULL instead
+ -- of nil, because nil is treated by Lua as iteration
+ -- end marker. Nil does not participate in iteration,
+ -- and does not allow to continue it.
+ --
+ local function request_iterator_next(iterator, i)
+ if i == box.NULL then
+ -- If a user continues iteration after an error -
+ -- restore position.
+ if not iterator.next_i then
+ return nil, nil
+ end
+ i = iterator.next_i
+ iterator.next_i = nil
+ else
+ i = i + 1
+ end
+ local request = iterator.request
+ local messages = request.on_push_ctx
+ ::retry::
+ if i <= #messages then
+ return i, messages[i]
+ end
+ if request:is_ready() then
+ -- After all the messages are iterated, `i` is equal
+ -- to #messages + 1. After response reading `i`
+ -- becomes #messages + 2. It is the trigger to finish
+ -- the iteration.
+ if i > #messages + 1 then
+ return nil, nil
+ end
+ local response, err = request:result()
+ if err then
+ return box.NULL, err
+ end
+ return i, response
+ end
+ local old_message_count = #messages
+ local timeout = iterator.timeout
+ repeat
+ local ts = fiber_clock()
+ request.cond:wait(timeout)
+ timeout = timeout - (fiber_clock() - ts)
+ if request:is_ready() or old_message_count ~= #messages then
+ goto retry
+ end
+ until timeout <= 0
+ iterator.next_i = i
+ return box.NULL, box.error.new(E_TIMEOUT)
+ end
+ --
+ -- Iterate over all messages, received by a request. @Sa
+ -- request_iterator_next for details what to expect in `for`
+ -- key/value pairs.
+ -- @param timeout One iteration timeout.
+ -- @retval next() callback, iterator, zero key.
+ --
+ function request_index:pairs(timeout)
+ if timeout then
+ if type(timeout) ~= 'number' or timeout < 0 then
+ error('Usage: future:pairs(timeout)')
+ end
+ else
+ timeout = TIMEOUT_INFINITY
+ end
+ local iterator = {request = self, timeout = timeout}
+ return request_iterator_next, iterator, 0
+ end
+ --
-- Wait for a response or error max timeout seconds.
-- @param timeout Max seconds to wait.
-- @retval result, nil Success, the response is returned.
diff --git a/test/box/push.result b/test/box/push.result
index 04cdc474b..d90e4d190 100644
--- a/test/box/push.result
+++ b/test/box/push.result
@@ -245,9 +245,6 @@ c:close()
s:drop()
---
...
-box.schema.user.revoke('guest', 'read,write,execute', 'universe')
----
-...
--
-- Ensure can not push in background.
--
@@ -268,3 +265,191 @@ ok, err
- null
- Session 'background' does not support push()
...
+--
+-- Async iterable pushes.
+--
+c = netbox.connect(box.cfg.listen)
+---
+...
+cond = fiber.cond()
+---
+...
+test_run:cmd("setopt delimiter ';'")
+---
+- true
+...
+function do_pushes()
+ for i = 1, 5 do
+ box.session.push(i + 100)
+ cond:wait()
+ end
+ return true
+end;
+---
+...
+test_run:cmd("setopt delimiter ''");
+---
+- true
+...
+-- Can not combine callback and async mode.
+ok, err = pcall(c.call, c, 'do_pushes', {}, {is_async = true, on_push = function() end})
+---
+...
+assert(not ok)
+---
+- true
+...
+assert(err:find('use future:pairs()') ~= nil)
+---
+- true
+...
+future = c:call('do_pushes', {}, {is_async = true})
+---
+...
+-- Try to ignore pushes.
+while not future:wait_result(0.01) do cond:signal() end
+---
+...
+future:result()
+---
+- [true]
+...
+-- Even if pushes are ignored, they still are available via pairs.
+messages = {}
+---
+...
+keys = {}
+---
+...
+for i, message in future:pairs() do table.insert(messages, message) table.insert(keys, i) end
+---
+...
+messages
+---
+- - 101
+ - 102
+ - 103
+ - 104
+ - 105
+ - [true]
+...
+keys
+---
+- - 1
+ - 2
+ - 3
+ - 4
+ - 5
+ - 6
+...
+-- Test timeouts inside `for`. Even if a timeout is got, a user
+-- can continue iteration making as many attempts to get a message
+-- as he wants.
+future = c:call('do_pushes', {}, {is_async = true})
+---
+...
+messages = {}
+---
+...
+keys = {}
+---
+...
+err_count = 0
+---
+...
+test_run:cmd("setopt delimiter ';'")
+---
+- true
+...
+for i, message in future:pairs(0.01) do
+ if i == nil then
+ err_count = err_count + 1
+ assert(message.code == box.error.TIMEOUT)
+ if err_count % 2 == 0 then
+ cond:signal()
+ end
+ else
+ table.insert(messages, message)
+ table.insert(keys, i)
+ end
+end;
+---
+...
+test_run:cmd("setopt delimiter ''");
+---
+- true
+...
+-- Messages and keys are got in the correct order and with no
+-- duplicates regardless of big timeout count.
+messages
+---
+- - 101
+ - 102
+ - 103
+ - 104
+ - 105
+ - [true]
+...
+keys
+---
+- - 1
+ - 2
+ - 3
+ - 4
+ - 5
+ - 6
+...
+err_count
+---
+- 10
+...
+-- Test non-timeout error.
+s = box.schema.create_space('test')
+---
+...
+pk = s:create_index('pk')
+---
+...
+s:replace{1}
+---
+- [1]
+...
+function do_push_and_duplicate() box.session.push(100) s:insert{1} end
+---
+...
+future = c:call('do_push_and_duplicate', {}, {is_async = true})
+---
+...
+future:wait_result(1000)
+---
+- null
+- Duplicate key exists in unique index 'pk' in space 'test'
+...
+messages = {}
+---
+...
+keys = {}
+---
+...
+for i, message in future:pairs() do table.insert(messages, message) table.insert(keys, i) end
+---
+...
+messages
+---
+- - 100
+ - Duplicate key exists in unique index 'pk' in space 'test'
+...
+keys
+---
+- - 1
+ - null
+...
+s:drop()
+---
+...
+c:close()
+---
+...
+box.schema.user.revoke('guest', 'read,write,execute', 'universe')
+---
+...
diff --git a/test/box/push.test.lua b/test/box/push.test.lua
index b0ff218bb..829883835 100644
--- a/test/box/push.test.lua
+++ b/test/box/push.test.lua
@@ -131,8 +131,6 @@ r
c:close()
s:drop()
-box.schema.user.revoke('guest', 'read,write,execute', 'universe')
-
--
-- Ensure can not push in background.
--
@@ -141,3 +139,80 @@ err = nil
f = fiber.create(function() ok, err = box.session.push(100) end)
while f:status() ~= 'dead' do fiber.sleep(0.01) end
ok, err
+
+--
+-- Async iterable pushes.
+--
+c = netbox.connect(box.cfg.listen)
+cond = fiber.cond()
+test_run:cmd("setopt delimiter ';'")
+function do_pushes()
+ for i = 1, 5 do
+ box.session.push(i + 100)
+ cond:wait()
+ end
+ return true
+end;
+test_run:cmd("setopt delimiter ''");
+
+-- Can not combine callback and async mode.
+ok, err = pcall(c.call, c, 'do_pushes', {}, {is_async = true, on_push = function() end})
+assert(not ok)
+assert(err:find('use future:pairs()') ~= nil)
+future = c:call('do_pushes', {}, {is_async = true})
+-- Try to ignore pushes.
+while not future:wait_result(0.01) do cond:signal() end
+future:result()
+
+-- Even if pushes are ignored, they still are available via pairs.
+messages = {}
+keys = {}
+for i, message in future:pairs() do table.insert(messages, message) table.insert(keys, i) end
+messages
+keys
+
+-- Test timeouts inside `for`. Even if a timeout is got, a user
+-- can continue iteration making as many attempts to get a message
+-- as he wants.
+future = c:call('do_pushes', {}, {is_async = true})
+messages = {}
+keys = {}
+err_count = 0
+test_run:cmd("setopt delimiter ';'")
+for i, message in future:pairs(0.01) do
+ if i == nil then
+ err_count = err_count + 1
+ assert(message.code == box.error.TIMEOUT)
+ if err_count % 2 == 0 then
+ cond:signal()
+ end
+ else
+ table.insert(messages, message)
+ table.insert(keys, i)
+ end
+end;
+test_run:cmd("setopt delimiter ''");
+-- Messages and keys are got in the correct order and with no
+-- duplicates regardless of big timeout count.
+messages
+keys
+err_count
+
+-- Test non-timeout error.
+s = box.schema.create_space('test')
+pk = s:create_index('pk')
+s:replace{1}
+
+function do_push_and_duplicate() box.session.push(100) s:insert{1} end
+future = c:call('do_push_and_duplicate', {}, {is_async = true})
+future:wait_result(1000)
+messages = {}
+keys = {}
+for i, message in future:pairs() do table.insert(messages, message) table.insert(keys, i) end
+messages
+keys
+
+s:drop()
+c:close()
+
+box.schema.user.revoke('guest', 'read,write,execute', 'universe')
--
2.15.1 (Apple Git-101)
^ permalink raw reply [flat|nested] 34+ messages in thread* [tarantool-patches] Re: [PATCH 1/1] netbox: introduce iterable future objects
2018-05-24 20:50 ` [tarantool-patches] [PATCH 1/1] netbox: introduce iterable future objects Vladislav Shpilevoy
@ 2018-06-04 22:17 ` Vladislav Shpilevoy
0 siblings, 0 replies; 34+ messages in thread
From: Vladislav Shpilevoy @ 2018-06-04 22:17 UTC (permalink / raw)
To: tarantool-patches; +Cc: kostja
Hi. I have removed ability to continue iteration after a
timeout error. I have decided it can be added later if
anybody needs.
On 24/05/2018 23:50, Vladislav Shpilevoy wrote:
> Netbox has two major ways to execute a request: sync and async.
> During execution of any a server can send multiplie responses via
> IPROTO_CHUNK. And the execution ways differ in how to handle the
> chunks (called messages or pushes).
>
> For a sync request a one can specify on_push callback and its
> on_push_ctx argument called on each message.
>
> When a request is async a user has a future object only, and can
> not specify any callbacks. To get the pushed messages a one must
> iterate over future object like this:
> for i, message in future:pairs(one_iteration_timeout) do
> ...
> end
> Or ignore messages just calling future:wait_result(). Anyway
> messages are not deleted, so a one can iterate over future object
> again and again.
>
> Follow up #2677
> ---
Below the diff of removal:
================================================
diff --git a/src/box/lua/net_box.lua b/src/box/lua/net_box.lua
index 5d896f7e3..851f245b1 100644
--- a/src/box/lua/net_box.lua
+++ b/src/box/lua/net_box.lua
@@ -281,12 +281,6 @@ local function create_transport(host, port, user, password, callback,
-- @retval box.NULL, error An error occured. When this
-- function is called in 'for k, v in future:pairs()',
-- `k` becomes box.NULL, and `v` becomes error object.
- -- If a one want to stop the cycle, he can do break.
- -- With no break the cycle will be continued until
- -- the request is finished. The iteration continuation
- -- is useful for example when time is out during a
- -- next message waiting, but a one does not consider
- -- this error be critical.
-- On error the key becomes exactly box.NULL instead
-- of nil, because nil is treated by Lua as iteration
-- end marker. Nil does not participate in iteration,
@@ -294,13 +288,7 @@ local function create_transport(host, port, user, password, callback,
--
local function request_iterator_next(iterator, i)
if i == box.NULL then
- -- If a user continues iteration after an error -
- -- restore position.
- if not iterator.next_i then
- return nil, nil
- end
- i = iterator.next_i
- iterator.next_i = nil
+ return nil, nil
else
i = i + 1
end
@@ -334,7 +322,6 @@ local function create_transport(host, port, user, password, callback,
goto retry
end
until timeout <= 0
- iterator.next_i = i
return box.NULL, box.error.new(E_TIMEOUT)
end
--
diff --git a/test/box/push.result b/test/box/push.result
index 0816c6754..340a6c04f 100644
--- a/test/box/push.result
+++ b/test/box/push.result
@@ -390,68 +390,7 @@ keys
- 5
- 6
...
--- Test timeouts inside `for`. Even if a timeout is got, a user
--- can continue iteration making as many attempts to get a message
--- as he wants.
-future = c:call('do_pushes', {}, {is_async = true})
----
-...
-messages = {}
----
-...
-keys = {}
----
-...
-err_count = 0
----
-...
-test_run:cmd("setopt delimiter ';'")
----
-- true
-...
-for i, message in future:pairs(0.01) do
- if i == nil then
- err_count = err_count + 1
- assert(message.code == box.error.TIMEOUT)
- if err_count % 2 == 0 then
- cond:signal()
- end
- else
- table.insert(messages, message)
- table.insert(keys, i)
- end
-end;
----
-...
-test_run:cmd("setopt delimiter ''");
----
-- true
-...
--- Messages and keys are got in the correct order and with no
--- duplicates regardless of big timeout count.
-messages
----
-- - 101
- - 102
- - 103
- - 104
- - 105
- - [true]
-...
-keys
----
-- - 1
- - 2
- - 3
- - 4
- - 5
- - 6
-...
-err_count
----
-- 10
-...
--- Test non-timeout error.
+-- Test error.
s = box.schema.create_space('test')
---
...
diff --git a/test/box/push.test.lua b/test/box/push.test.lua
index 10bc201df..480c58ca3 100644
--- a/test/box/push.test.lua
+++ b/test/box/push.test.lua
@@ -191,34 +191,7 @@ for i, message in future:pairs() do table.insert(messages, message) table.insert
messages
keys
--- Test timeouts inside `for`. Even if a timeout is got, a user
--- can continue iteration making as many attempts to get a message
--- as he wants.
-future = c:call('do_pushes', {}, {is_async = true})
-messages = {}
-keys = {}
-err_count = 0
-test_run:cmd("setopt delimiter ';'")
-for i, message in future:pairs(0.01) do
- if i == nil then
- err_count = err_count + 1
- assert(message.code == box.error.TIMEOUT)
- if err_count % 2 == 0 then
- cond:signal()
- end
- else
- table.insert(messages, message)
- table.insert(keys, i)
- end
-end;
-test_run:cmd("setopt delimiter ''");
--- Messages and keys are got in the correct order and with no
--- duplicates regardless of big timeout count.
-messages
-keys
-err_count
-
--- Test non-timeout error.
+-- Test error.
s = box.schema.create_space('test')
pk = s:create_index('pk')
s:replace{1}
^ permalink raw reply [flat|nested] 34+ messages in thread