* [tarantool-patches] [PATCH 01/13] tuple: remove alloc and alloc_ctx args from update()
2019-08-12 23:05 [tarantool-patches] [PATCH 00/13] JSON updates Vladislav Shpilevoy
@ 2019-08-12 23:05 ` Vladislav Shpilevoy
2019-08-12 23:05 ` [tarantool-patches] [PATCH 10/13] tuple: enable JSON bar updates Vladislav Shpilevoy
` (12 subsequent siblings)
13 siblings, 0 replies; 15+ messages in thread
From: Vladislav Shpilevoy @ 2019-08-12 23:05 UTC (permalink / raw)
To: tarantool-patches
They are always region_aligned_alloc and region pointer. Lets use
them always inside tuple_update.c, with no necessity to pass
explicitly.
The patch is motivated by forthcoming updates by JSON path, which
will strongly complicate and perhaps slow down the tuple_update.c
code. The present patch as well as some next ones should smooth
these problems.
---
src/box/lua/tuple.c | 9 +++--
src/box/memtx_space.c | 18 ++++------
src/box/space.c | 23 ++++++-------
src/box/tuple.c | 6 ++--
src/box/tuple_update.c | 74 +++++++++++++++++++++--------------------
src/box/tuple_update.h | 14 +++-----
src/box/vinyl.c | 10 ++----
src/box/vy_upsert.c | 29 ++++------------
test/unit/column_mask.c | 30 +++++++----------
9 files changed, 88 insertions(+), 125 deletions(-)
diff --git a/src/box/lua/tuple.c b/src/box/lua/tuple.c
index 183c3901d..2fbfb1473 100644
--- a/src/box/lua/tuple.c
+++ b/src/box/lua/tuple.c
@@ -447,11 +447,10 @@ lbox_tuple_transform(struct lua_State *L)
* to use the default one with no restrictions on field
* count or types.
*/
- const char *new_data = tuple_update_execute(region_aligned_alloc_cb,
- region, buf->buf,
- buf->buf + ibuf_used(buf),
- old_data, old_data + bsize,
- &new_size, 1, NULL);
+ const char *new_data =
+ tuple_update_execute(buf->buf, buf->buf + ibuf_used(buf),
+ old_data, old_data + bsize, &new_size, 1,
+ NULL);
if (new_data != NULL)
new_tuple = tuple_new(box_tuple_format_default(),
new_data, new_data + new_size);
diff --git a/src/box/memtx_space.c b/src/box/memtx_space.c
index cf29cf328..d63e8a5ca 100644
--- a/src/box/memtx_space.c
+++ b/src/box/memtx_space.c
@@ -419,8 +419,7 @@ memtx_space_execute_update(struct space *space, struct txn *txn,
uint32_t new_size = 0, bsize;
const char *old_data = tuple_data_range(old_tuple, &bsize);
const char *new_data =
- tuple_update_execute(region_aligned_alloc_cb, &fiber()->gc,
- request->tuple, request->tuple_end,
+ tuple_update_execute(request->tuple, request->tuple_end,
old_data, old_data + bsize,
&new_size, request->index_base, NULL);
if (new_data == NULL)
@@ -489,9 +488,8 @@ memtx_space_execute_upsert(struct space *space, struct txn *txn,
* we get it here, it's also OK to throw it
* @sa https://github.com/tarantool/tarantool/issues/1156
*/
- if (tuple_update_check_ops(region_aligned_alloc_cb, &fiber()->gc,
- request->ops, request->ops_end,
- request->index_base)) {
+ if (tuple_update_check_ops(request->ops, request->ops_end,
+ request->index_base) != 0) {
return -1;
}
stmt->new_tuple = memtx_tuple_new(space->format,
@@ -511,12 +509,10 @@ memtx_space_execute_upsert(struct space *space, struct txn *txn,
*/
uint64_t column_mask = COLUMN_MASK_FULL;
const char *new_data =
- tuple_upsert_execute(region_aligned_alloc_cb,
- &fiber()->gc, request->ops,
- request->ops_end, old_data,
- old_data + bsize, &new_size,
- request->index_base, false,
- &column_mask);
+ tuple_upsert_execute(request->ops, request->ops_end,
+ old_data, old_data + bsize,
+ &new_size, request->index_base,
+ false, &column_mask);
if (new_data == NULL)
return -1;
diff --git a/src/box/space.c b/src/box/space.c
index b4d48fcc3..fffe03c41 100644
--- a/src/box/space.c
+++ b/src/box/space.c
@@ -330,7 +330,6 @@ static int
space_before_replace(struct space *space, struct txn *txn,
struct request *request)
{
- struct region *gc = &fiber()->gc;
enum iproto_type type = request->type;
struct index *pk = space_index(space, 0);
@@ -396,10 +395,10 @@ space_before_replace(struct space *space, struct txn *txn,
}
old_data = tuple_data_range(old_tuple, &old_size);
old_data_end = old_data + old_size;
- new_data = tuple_update_execute(region_aligned_alloc_cb, gc,
- request->tuple, request->tuple_end,
- old_data, old_data_end, &new_size,
- request->index_base, NULL);
+ new_data = tuple_update_execute(request->tuple,
+ request->tuple_end, old_data,
+ old_data_end, &new_size,
+ request->index_base, NULL);
if (new_data == NULL)
return -1;
new_data_end = new_data + new_size;
@@ -419,18 +418,18 @@ space_before_replace(struct space *space, struct txn *txn,
*/
new_data = request->tuple;
new_data_end = request->tuple_end;
- if (tuple_update_check_ops(region_aligned_alloc_cb, gc,
- request->ops, request->ops_end,
- request->index_base) != 0)
+ if (tuple_update_check_ops(request->ops,
+ request->ops_end,
+ request->index_base) != 0)
return -1;
break;
}
old_data = tuple_data_range(old_tuple, &old_size);
old_data_end = old_data + old_size;
- new_data = tuple_upsert_execute(region_aligned_alloc_cb, gc,
- request->ops, request->ops_end,
- old_data, old_data_end, &new_size,
- request->index_base, false, NULL);
+ new_data = tuple_upsert_execute(request->ops, request->ops_end,
+ old_data, old_data_end,
+ &new_size, request->index_base,
+ false, NULL);
new_data_end = new_data + new_size;
break;
default:
diff --git a/src/box/tuple.c b/src/box/tuple.c
index a01646c68..beb290a9f 100644
--- a/src/box/tuple.c
+++ b/src/box/tuple.c
@@ -708,8 +708,7 @@ box_tuple_update(box_tuple_t *tuple, const char *expr, const char *expr_end)
struct region *region = &fiber()->gc;
size_t used = region_used(region);
const char *new_data =
- tuple_update_execute(region_aligned_alloc_cb, region, expr,
- expr_end, old_data, old_data + bsize,
+ tuple_update_execute(expr, expr_end, old_data, old_data + bsize,
&new_size, 1, NULL);
if (new_data == NULL) {
region_truncate(region, used);
@@ -731,8 +730,7 @@ box_tuple_upsert(box_tuple_t *tuple, const char *expr, const char *expr_end)
struct region *region = &fiber()->gc;
size_t used = region_used(region);
const char *new_data =
- tuple_upsert_execute(region_aligned_alloc_cb, region, expr,
- expr_end, old_data, old_data + bsize,
+ tuple_upsert_execute(expr, expr_end, old_data, old_data + bsize,
&new_size, 1, false, NULL);
if (new_data == NULL) {
region_truncate(region, used);
diff --git a/src/box/tuple_update.c b/src/box/tuple_update.c
index 7a203ced8..599952189 100644
--- a/src/box/tuple_update.c
+++ b/src/box/tuple_update.c
@@ -43,7 +43,7 @@
#include <bit/int96.h>
#include <salad/rope.h>
#include "column_mask.h"
-
+#include "fiber.h"
/** UPDATE request implementation.
* UPDATE request is represented by a sequence of operations, each
@@ -96,10 +96,27 @@
* deleted one.
*/
+static inline void *
+update_alloc(void *ctx, size_t size)
+{
+ void *ptr = region_aligned_alloc((struct region *) ctx, size,
+ alignof(uint64_t));
+ if (ptr == NULL)
+ diag_set(OutOfMemory, size, "region_aligned_alloc",
+ "update internals");
+ return ptr;
+}
+
+static void
+update_free(void *ctx, void *mem)
+{
+ (void) ctx;
+ (void) mem;
+}
+
/** Update internal state */
struct tuple_update
{
- tuple_update_alloc_func alloc;
void *alloc_ctx;
struct rope *rope;
struct update_op *ops;
@@ -508,7 +525,7 @@ do_op_insert(struct tuple_update *update, struct update_op *op)
if (op_adjust_field_no(update, op, rope_size(update->rope) + 1))
return -1;
struct update_field *field = (struct update_field *)
- update->alloc(update->alloc_ctx, sizeof(*field));
+ update_alloc(update->alloc_ctx, sizeof(*field));
if (field == NULL)
return -1;
update_field_init(field, op->arg.set.value, op->arg.set.length, 0);
@@ -778,7 +795,7 @@ update_field_split(void *split_ctx, void *data, size_t size, size_t offset)
struct update_field *prev = (struct update_field *) data;
struct update_field *next = (struct update_field *)
- update->alloc(update->alloc_ctx, sizeof(*next));
+ update_alloc(update->alloc_ctx, sizeof(*next));
if (next == NULL)
return NULL;
assert(offset > 0 && prev->tail_len > 0);
@@ -799,14 +816,6 @@ update_field_split(void *split_ctx, void *data, size_t size, size_t offset)
return next;
}
-/** Free rope node - do nothing, since we use a pool allocator. */
-static void
-region_alloc_free_stub(void *ctx, void *mem)
-{
- (void) ctx;
- (void) mem;
-}
-
/**
* We found a tuple to do the update on. Prepare a rope
* to perform operations on.
@@ -822,14 +831,14 @@ int
update_create_rope(struct tuple_update *update, const char *tuple_data,
const char *tuple_data_end, uint32_t field_count)
{
- update->rope = rope_new(update_field_split, update, update->alloc,
- region_alloc_free_stub, update->alloc_ctx);
+ update->rope = rope_new(update_field_split, update, update_alloc,
+ update_free, update->alloc_ctx);
if (update->rope == NULL)
return -1;
/* Initialize the rope with the old tuple. */
struct update_field *first = (struct update_field *)
- update->alloc(update->alloc_ctx, sizeof(*first));
+ update_alloc(update->alloc_ctx, sizeof(*first));
if (first == NULL)
return -1;
const char *field = tuple_data;
@@ -970,7 +979,7 @@ update_read_ops(struct tuple_update *update, const char *expr,
}
/* Read update operations. */
- update->ops = (struct update_op *) update->alloc(update->alloc_ctx,
+ update->ops = (struct update_op *) update_alloc(update->alloc_ctx,
update->op_count * sizeof(struct update_op));
if (update->ops == NULL)
return -1;
@@ -1160,13 +1169,10 @@ upsert_do_ops(struct tuple_update *update, const char *old_data,
}
static void
-update_init(struct tuple_update *update,
- tuple_update_alloc_func alloc, void *alloc_ctx,
- int index_base)
+update_init(struct tuple_update *update, int index_base)
{
memset(update, 0, sizeof(*update));
- update->alloc = alloc;
- update->alloc_ctx = alloc_ctx;
+ update->alloc_ctx = &fiber()->gc;
/*
* Base field offset, e.g. 0 for C and 1 for Lua. Used only for
* error messages. All fields numbers must be zero-based!
@@ -1178,7 +1184,7 @@ const char *
update_finish(struct tuple_update *update, uint32_t *p_tuple_len)
{
uint32_t tuple_len = update_calc_tuple_length(update);
- char *buffer = (char *) update->alloc(update->alloc_ctx, tuple_len);
+ char *buffer = (char *) update_alloc(update->alloc_ctx, tuple_len);
if (buffer == NULL)
return NULL;
*p_tuple_len = update_write_tuple(update, buffer, buffer + tuple_len);
@@ -1186,23 +1192,21 @@ update_finish(struct tuple_update *update, uint32_t *p_tuple_len)
}
int
-tuple_update_check_ops(tuple_update_alloc_func alloc, void *alloc_ctx,
- const char *expr, const char *expr_end, int index_base)
+tuple_update_check_ops(const char *expr, const char *expr_end, int index_base)
{
struct tuple_update update;
- update_init(&update, alloc, alloc_ctx, index_base);
+ update_init(&update, index_base);
return update_read_ops(&update, expr, expr_end, 0);
}
const char *
-tuple_update_execute(tuple_update_alloc_func alloc, void *alloc_ctx,
- const char *expr,const char *expr_end,
+tuple_update_execute(const char *expr,const char *expr_end,
const char *old_data, const char *old_data_end,
uint32_t *p_tuple_len, int index_base,
uint64_t *column_mask)
{
struct tuple_update update;
- update_init(&update, alloc, alloc_ctx, index_base);
+ update_init(&update, index_base);
uint32_t field_count = mp_decode_array(&old_data);
if (update_read_ops(&update, expr, expr_end, field_count) != 0)
@@ -1216,14 +1220,13 @@ tuple_update_execute(tuple_update_alloc_func alloc, void *alloc_ctx,
}
const char *
-tuple_upsert_execute(tuple_update_alloc_func alloc, void *alloc_ctx,
- const char *expr,const char *expr_end,
+tuple_upsert_execute(const char *expr,const char *expr_end,
const char *old_data, const char *old_data_end,
uint32_t *p_tuple_len, int index_base, bool suppress_error,
uint64_t *column_mask)
{
struct tuple_update update;
- update_init(&update, alloc, alloc_ctx, index_base);
+ update_init(&update, index_base);
uint32_t field_count = mp_decode_array(&old_data);
if (update_read_ops(&update, expr, expr_end, field_count) != 0)
@@ -1238,8 +1241,7 @@ tuple_upsert_execute(tuple_update_alloc_func alloc, void *alloc_ctx,
}
const char *
-tuple_upsert_squash(tuple_update_alloc_func alloc, void *alloc_ctx,
- const char *expr1, const char *expr1_end,
+tuple_upsert_squash(const char *expr1, const char *expr1_end,
const char *expr2, const char *expr2_end,
size_t *result_size, int index_base)
{
@@ -1247,7 +1249,7 @@ tuple_upsert_squash(tuple_update_alloc_func alloc, void *alloc_ctx,
const char *expr_end[2] = {expr1_end, expr2_end};
struct tuple_update update[2];
for (int j = 0; j < 2; j++) {
- update_init(&update[j], alloc, alloc_ctx, index_base);
+ update_init(&update[j], index_base);
if (update_read_ops(&update[j], expr[j], expr_end[j], 0))
return NULL;
mp_decode_array(&expr[j]);
@@ -1264,8 +1266,8 @@ tuple_upsert_squash(tuple_update_alloc_func alloc, void *alloc_ctx,
}
size_t possible_size = expr1_end - expr1 + expr2_end - expr2;
const uint32_t space_for_arr_tag = 5;
- char *buf = (char *)alloc(alloc_ctx,
- possible_size + space_for_arr_tag);
+ char *buf = (char *) update_alloc(&fiber()->gc,
+ possible_size + space_for_arr_tag);
if (buf == NULL)
return NULL;
/* reserve some space for mp array header */
diff --git a/src/box/tuple_update.h b/src/box/tuple_update.h
index 706edd55a..37faa1918 100644
--- a/src/box/tuple_update.h
+++ b/src/box/tuple_update.h
@@ -44,22 +44,17 @@ enum {
BOX_UPDATE_OP_CNT_MAX = 4000,
};
-typedef void *(*tuple_update_alloc_func)(void *, size_t);
-
int
-tuple_update_check_ops(tuple_update_alloc_func alloc, void *alloc_ctx,
- const char *expr, const char *expr_end, int index_base);
+tuple_update_check_ops(const char *expr, const char *expr_end, int index_base);
const char *
-tuple_update_execute(tuple_update_alloc_func alloc, void *alloc_ctx,
- const char *expr,const char *expr_end,
+tuple_update_execute(const char *expr,const char *expr_end,
const char *old_data, const char *old_data_end,
uint32_t *p_new_size, int index_base,
uint64_t *column_mask);
const char *
-tuple_upsert_execute(tuple_update_alloc_func alloc, void *alloc_ctx,
- const char *expr, const char *expr_end,
+tuple_upsert_execute(const char *expr, const char *expr_end,
const char *old_data, const char *old_data_end,
uint32_t *p_new_size, int index_base, bool suppress_error,
uint64_t *column_mask);
@@ -75,8 +70,7 @@ tuple_upsert_execute(tuple_update_alloc_func alloc, void *alloc_ctx,
* If it isn't possible to merge expressions NULL is returned.
*/
const char *
-tuple_upsert_squash(tuple_update_alloc_func alloc, void *alloc_ctx,
- const char *expr1, const char *expr1_end,
+tuple_upsert_squash(const char *expr1, const char *expr1_end,
const char *expr2, const char *expr2_end,
size_t *result_size, int index_base);
diff --git a/src/box/vinyl.c b/src/box/vinyl.c
index cd009c1c2..4c5c08852 100644
--- a/src/box/vinyl.c
+++ b/src/box/vinyl.c
@@ -1942,8 +1942,7 @@ vy_update(struct vy_env *env, struct vy_tx *tx, struct txn_stmt *stmt,
uint32_t new_size, old_size;
const char *old_tuple = tuple_data_range(stmt->old_tuple, &old_size);
const char *old_tuple_end = old_tuple + old_size;
- new_tuple = tuple_update_execute(region_aligned_alloc_cb, &fiber()->gc,
- request->tuple, request->tuple_end,
+ new_tuple = tuple_update_execute(request->tuple, request->tuple_end,
old_tuple, old_tuple_end, &new_size,
request->index_base, &column_mask);
if (new_tuple == NULL)
@@ -2122,8 +2121,7 @@ vy_upsert(struct vy_env *env, struct vy_tx *tx, struct txn_stmt *stmt,
if (vy_is_committed(env, pk))
return 0;
/* Check update operations. */
- if (tuple_update_check_ops(region_aligned_alloc_cb, &fiber()->gc,
- request->ops, request->ops_end,
+ if (tuple_update_check_ops(request->ops, request->ops_end,
request->index_base)) {
return -1;
}
@@ -2181,9 +2179,7 @@ vy_upsert(struct vy_env *env, struct vy_tx *tx, struct txn_stmt *stmt,
old_tuple_end = old_tuple + old_size;
/* Apply upsert operations to the old tuple. */
- new_tuple = tuple_upsert_execute(region_aligned_alloc_cb,
- &fiber()->gc, ops, ops_end,
- old_tuple, old_tuple_end,
+ new_tuple = tuple_upsert_execute(ops, ops_end, old_tuple, old_tuple_end,
&new_size, 0, false, &column_mask);
if (new_tuple == NULL)
return -1;
diff --git a/src/box/vy_upsert.c b/src/box/vy_upsert.c
index e5e53be79..1c55f86c2 100644
--- a/src/box/vy_upsert.c
+++ b/src/box/vy_upsert.c
@@ -38,17 +38,6 @@
#include "fiber.h"
#include "column_mask.h"
-static void *
-vy_update_alloc(void *arg, size_t size)
-{
- /* TODO: rewrite tuple_upsert_execute() without exceptions */
- struct region *region = (struct region *) arg;
- void *data = region_aligned_alloc(region, size, sizeof(uint64_t));
- if (data == NULL)
- diag_set(OutOfMemory, size, "region", "upsert");
- return data;
-}
-
/**
* Try to squash two upsert series (msgspacked index_base + ops)
* Try to create a tuple with squahed operations
@@ -58,7 +47,7 @@ vy_update_alloc(void *arg, size_t size)
* @retval -1 - memory error
*/
static int
-vy_upsert_try_to_squash(struct tuple_format *format, struct region *region,
+vy_upsert_try_to_squash(struct tuple_format *format,
const char *key_mp, const char *key_mp_end,
const char *old_serie, const char *old_serie_end,
const char *new_serie, const char *new_serie_end,
@@ -68,8 +57,7 @@ vy_upsert_try_to_squash(struct tuple_format *format, struct region *region,
size_t squashed_size;
const char *squashed =
- tuple_upsert_squash(vy_update_alloc, region,
- old_serie, old_serie_end,
+ tuple_upsert_squash(old_serie, old_serie_end,
new_serie, new_serie_end,
&squashed_size, 0);
if (squashed == NULL)
@@ -130,10 +118,9 @@ vy_apply_upsert(struct tuple *new_stmt, struct tuple *old_stmt,
size_t region_svp = region_used(region);
uint8_t old_type = vy_stmt_type(old_stmt);
uint64_t column_mask = COLUMN_MASK_FULL;
- result_mp = tuple_upsert_execute(vy_update_alloc, region, new_ops,
- new_ops_end, result_mp, result_mp_end,
- &mp_size, 0, suppress_error,
- &column_mask);
+ result_mp = tuple_upsert_execute(new_ops, new_ops_end, result_mp,
+ result_mp_end, &mp_size, 0,
+ suppress_error, &column_mask);
result_mp_end = result_mp + mp_size;
if (old_type != IPROTO_UPSERT) {
assert(old_type == IPROTO_INSERT ||
@@ -163,10 +150,8 @@ vy_apply_upsert(struct tuple *new_stmt, struct tuple *old_stmt,
* UPSERT + UPSERT case: combine operations
*/
assert(old_ops_end - old_ops > 0);
- if (vy_upsert_try_to_squash(format, region,
- result_mp, result_mp_end,
- old_ops, old_ops_end,
- new_ops, new_ops_end,
+ if (vy_upsert_try_to_squash(format, result_mp, result_mp_end,
+ old_ops, old_ops_end, new_ops, new_ops_end,
&result_stmt) != 0) {
region_truncate(region, region_svp);
return NULL;
diff --git a/test/unit/column_mask.c b/test/unit/column_mask.c
index 7859626f8..3ffd351ac 100644
--- a/test/unit/column_mask.c
+++ b/test/unit/column_mask.c
@@ -3,6 +3,9 @@
#include "unit.h"
#include "msgpuck.h"
#include "trivia/util.h"
+#include "fiber.h"
+#include "memory.h"
+#include "tuple.h"
#define MAX_OPS 20
#define MAX_FIELDS 100
@@ -102,20 +105,6 @@ tuple_new_update(const struct tuple_update_template *update, char **end)
return ret;
}
-static char buffer[2048];
-static size_t pos = 0;
-
-/** Allocator for the tuple_update function. */
-static void *
-tuple_update_alloc_f(void *unused, size_t size)
-{
- (void) unused;
- fail_if(pos + size > sizeof(buffer));
- char *ret = &buffer[pos];
- pos += size;
- return ret;
-}
-
/**
* Execute an update operation from the template and compare it
* with the expected tuple and expected column_mask.
@@ -138,20 +127,19 @@ check_update_result(const struct tuple_template *original,
uint32_t actual_len;
uint64_t column_mask;
+ struct region *region = &fiber()->gc;
const char *actual =
- tuple_update_execute(tuple_update_alloc_f, NULL, ops, ops_end,
- old, old_end, &actual_len, 1,
+ tuple_update_execute(ops, ops_end, old, old_end, &actual_len, 1,
&column_mask);
fail_if(actual == NULL);
is((int32_t)actual_len, new_end - new, "check result length");
is(memcmp(actual, new, actual_len), 0, "tuple update is correct");
is(column_mask, expected_mask, "column_mask is correct");
+ fiber_gc();
free(old);
free(new);
free(ops);
- /* reset the global buffer. */
- pos = 0;
}
static inline void
@@ -239,6 +227,9 @@ basic_test()
int
main()
{
+ memory_init();
+ fiber_init(fiber_c_invoke);
+ tuple_init(NULL);
header();
plan(27);
@@ -246,4 +237,7 @@ main()
footer();
check_plan();
+ tuple_free();
+ fiber_free();
+ memory_free();
}
--
2.20.1 (Apple Git-117)
^ permalink raw reply [flat|nested] 15+ messages in thread* [tarantool-patches] [PATCH 10/13] tuple: enable JSON bar updates
2019-08-12 23:05 [tarantool-patches] [PATCH 00/13] JSON updates Vladislav Shpilevoy
2019-08-12 23:05 ` [tarantool-patches] [PATCH 01/13] tuple: remove alloc and alloc_ctx args from update() Vladislav Shpilevoy
@ 2019-08-12 23:05 ` Vladislav Shpilevoy
2019-08-12 23:05 ` [tarantool-patches] [PATCH 11/13] tuple: make update operation tokens consumable Vladislav Shpilevoy
` (11 subsequent siblings)
13 siblings, 0 replies; 15+ messages in thread
From: Vladislav Shpilevoy @ 2019-08-12 23:05 UTC (permalink / raw)
To: tarantool-patches
Bar update is an update by JSON path of a tuple's internal object
with no other updates along the path. It is an optimization to do
not store each part of the path and link to it in memory. Bar
stores just string with JSON and pointer to the MessagePack
object to update.
So far bar can not have another updates along the path, this
patch introduces only non-intersected JSON path updates (that is
actually one of the most common cases, so bar is really useful
optimization).
An example. There is an update {'=', 'a.b.c[1][2][3]', 100}.
For this whole update only one object will be created:
{
op = '=',
path = 'a.b.c[1][2][3]',
new_value = 100,
ptr_to_tuple_msgpack = 0x....,
...
}
The elements 'a', 'b', 'c', '[1]', '[2]', '[3]' are not a list
of objects each occupying memory. The whole bar is stored as one
object, and the path as a pointer to a string in MessagePack
obtained from a user.
This makes JSON updates memory complexity not depend on path
legths.
Part of #1261
---
src/box/CMakeLists.txt | 1 +
src/box/tuple_update.c | 4 +-
src/box/update/update_array.c | 24 +-
src/box/update/update_bar.c | 408 +++++++++++++++++++++++++++++++++
src/box/update/update_field.c | 44 +++-
src/box/update/update_field.h | 102 +++++++++
src/box/vinyl.c | 17 +-
test/box/update.result | 410 +++++++++++++++++++++++++++++++++-
test/box/update.test.lua | 145 ++++++++++++
test/engine/update.result | 5 -
test/engine/update.test.lua | 2 -
11 files changed, 1141 insertions(+), 21 deletions(-)
create mode 100644 src/box/update/update_bar.c
diff --git a/src/box/CMakeLists.txt b/src/box/CMakeLists.txt
index 555e0f47a..94775daba 100644
--- a/src/box/CMakeLists.txt
+++ b/src/box/CMakeLists.txt
@@ -43,6 +43,7 @@ add_library(tuple STATIC
tuple_update.c
update/update_field.c
update/update_array.c
+ update/update_bar.c
tuple_compare.cc
tuple_extract_key.cc
tuple_hash.cc
diff --git a/src/box/tuple_update.c b/src/box/tuple_update.c
index 81e1f7e97..8998f4dd9 100644
--- a/src/box/tuple_update.c
+++ b/src/box/tuple_update.c
@@ -331,8 +331,8 @@ tuple_upsert_squash(const char *expr1, const char *expr1_end,
int32_t prev_field_no = index_base - 1;
for (uint32_t i = 0; i < update[j].op_count; i++) {
struct update_op *op = &update[j].ops[i];
- if (op->opcode != '+' && op->opcode != '-' &&
- op->opcode != '=')
+ if ((op->opcode != '+' && op->opcode != '-' &&
+ op->opcode != '=') || op->lexer.src != NULL)
return NULL;
if (op->field_no <= prev_field_no)
return NULL;
diff --git a/src/box/update/update_array.c b/src/box/update/update_array.c
index 5b834b644..fe50a605a 100644
--- a/src/box/update/update_array.c
+++ b/src/box/update/update_array.c
@@ -216,12 +216,19 @@ do_op_array_insert(struct update_op *op, struct update_field *field)
{
assert(field->type == UPDATE_ARRAY);
struct rope *rope = field->array.rope;
+ struct update_array_item *item;
+ if (! update_op_is_term(op)) {
+ item = update_array_extract_item(field, op);
+ if (item == NULL)
+ return -1;
+ return do_op_insert(op, &item->field);
+ }
+
if (update_op_adjust_field_no(op, rope_size(rope) + 1) != 0)
return -1;
- struct update_array_item *item =
- (struct update_array_item *) rope_alloc(rope->ctx,
- sizeof(*item));
+ item = (struct update_array_item *) rope_alloc(rope->ctx,
+ sizeof(*item));
if (item == NULL)
return -1;
update_array_item_create(item, UPDATE_NOP, op->arg.set.value,
@@ -242,6 +249,8 @@ do_op_array_set(struct update_op *op, struct update_field *field)
update_array_extract_item(field, op);
if (item == NULL)
return -1;
+ if (! update_op_is_term(op))
+ return do_op_set(op, &item->field);
op->new_field_len = op->arg.set.length;
/* Ignore the previous op, if any. */
item->field.type = UPDATE_SCALAR;
@@ -253,6 +262,13 @@ int
do_op_array_delete(struct update_op *op, struct update_field *field)
{
assert(field->type == UPDATE_ARRAY);
+ if (! update_op_is_term(op)) {
+ struct update_array_item *item =
+ update_array_extract_item(field, op);
+ if (item == NULL)
+ return -1;
+ return do_op_delete(op, &item->field);
+ }
struct rope *rope = field->array.rope;
uint32_t size = rope_size(rope);
if (update_op_adjust_field_no(op, size) != 0)
@@ -273,6 +289,8 @@ do_op_array_##op_type(struct update_op *op, struct update_field *field) \
update_array_extract_item(field, op); \
if (item == NULL) \
return -1; \
+ if (! update_op_is_term(op)) \
+ return do_op_##op_type(op, &item->field); \
if (item->field.type != UPDATE_NOP) \
return update_err_double(op); \
if (update_op_do_##op_type(op, item->field.data) != 0) \
diff --git a/src/box/update/update_bar.c b/src/box/update/update_bar.c
new file mode 100644
index 000000000..f4fc00716
--- /dev/null
+++ b/src/box/update/update_bar.c
@@ -0,0 +1,408 @@
+/*
+ * Copyright 2010-2019, Tarantool AUTHORS, please see AUTHORS file.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the
+ * following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY <COPYRIGHT HOLDER> ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+ * <COPYRIGHT HOLDER> OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+ * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+#include "update_field.h"
+#include "box/tuple.h"
+
+/**
+ * Locate the field to update by JSON path in @a op->path. If
+ * found, initialize @a field as a bar update.
+ * @param op Update operation.
+ * @param field Field to locate in.
+ *
+ * @retval 0 Success.
+ * @retval -1 Not found or invalid JSON.
+ */
+static inline int
+update_bar_locate(struct update_op *op, struct update_field *field)
+{
+ assert(! update_op_is_term(op));
+ const char *parent = NULL, *pos = field->data;
+ field->bar.path = op->lexer.src + op->lexer.offset;
+ field->bar.path_len = op->lexer.src_len - op->lexer.offset;
+ int rc;
+ struct json_token token;
+ while ((rc = json_lexer_next_token(&op->lexer, &token)) == 0 &&
+ token.type != JSON_TOKEN_END) {
+
+ parent = pos;
+ switch (token.type) {
+ case JSON_TOKEN_NUM:
+ rc = tuple_field_go_to_index(&pos, token.num);
+ break;
+ case JSON_TOKEN_STR:
+ rc = tuple_field_go_to_key(&pos, token.str, token.len);
+ break;
+ default:
+ assert(token.type == JSON_TOKEN_ANY);
+ return update_err_bad_json(op,
+ op->lexer.symbol_count - 1);
+ }
+ if (rc != 0)
+ return update_err_no_such_field(op);
+ }
+ if (rc > 0)
+ return update_err_bad_json(op, rc);
+
+ field->type = UPDATE_BAR;
+ field->bar.point = pos;
+ mp_next(&pos);
+ field->bar.point_size = pos - field->bar.point;
+ field->bar.op = op;
+ field->bar.parent = parent;
+ return 0;
+}
+
+/**
+ * Locate the optional field to set by JSON path in @a op->path.
+ * If found or only a last path part is not found, initialize @a
+ * field.
+ * @param op Update operation.
+ * @param field Field to locate in.
+ * @param[out] is_found Set if the field was found.
+ * @param[out] key_len_or_index One parameter for two values,
+ * depending on where the target point is located: in an
+ * array or a map. In case of map it is size of a key
+ * before the found point. It is used to find range of the
+ * both key and value in '#' operation to drop the pair.
+ * In case of array it is index of the point to be able to
+ * check how many fields are left for deletion.
+ *
+ * @retval 0 Success.
+ * @retval -1 Not found non-last path part or invalid JSON.
+ */
+static inline int
+update_bar_locate_opt(struct update_op *op, struct update_field *field,
+ bool *is_found, int *key_len_or_index)
+{
+ assert(! update_op_is_term(op));
+ int rc;
+ field->type = UPDATE_BAR;
+ field->bar.op = op;
+ field->bar.path = op->lexer.src + op->lexer.offset;
+ field->bar.path_len = op->lexer.src_len - op->lexer.offset;
+ const char *pos = field->data;
+ struct json_token token;
+ do {
+ rc = json_lexer_next_token(&op->lexer, &token);
+ if (rc != 0)
+ return update_err_bad_json(op, rc);
+
+ switch (token.type) {
+ case JSON_TOKEN_END:
+ *is_found = true;
+ field->bar.point = pos;
+ mp_next(&pos);
+ field->bar.point_size = pos - field->bar.point;
+ return 0;
+ case JSON_TOKEN_NUM:
+ field->bar.parent = pos;
+ *key_len_or_index = token.num;
+ rc = tuple_field_go_to_index(&pos, token.num);
+ break;
+ case JSON_TOKEN_STR:
+ field->bar.parent = pos;
+ *key_len_or_index = token.len;
+ rc = tuple_field_go_to_key(&pos, token.str, token.len);
+ break;
+ default:
+ assert(token.type == JSON_TOKEN_ANY);
+ return update_err_bad_json(op,
+ op->lexer.symbol_count - 1);
+ }
+ } while (rc == 0);
+ assert(rc == -1);
+ struct json_token tmp_token;
+ rc = json_lexer_next_token(&op->lexer, &tmp_token);
+ if (rc != 0)
+ return update_err_bad_json(op, rc);
+ if (tmp_token.type != JSON_TOKEN_END)
+ return update_err_no_such_field(op);
+
+ *is_found = false;
+ if (token.type == JSON_TOKEN_NUM) {
+ if (mp_typeof(*field->bar.parent) != MP_ARRAY) {
+ return update_err(op, "can not access by index a "\
+ "non-array field");
+ }
+ const char *tmp = field->bar.parent;
+ uint32_t size = mp_decode_array(&tmp);
+ if ((uint32_t) token.num > size)
+ return update_err_no_such_field(op);
+ /*
+ * The only way not to find an element in an array
+ * by an index is to use array size as the index.
+ */
+ assert((uint32_t) token.num == size);
+ if (field->bar.parent == field->data) {
+ field->bar.point = field->data + field->size;
+ } else {
+ field->bar.point = field->bar.parent;
+ mp_next(&field->bar.point);
+ }
+ } else {
+ assert(token.type == JSON_TOKEN_STR);
+ field->bar.new_key = token.str;
+ field->bar.new_key_len = token.len;
+ if (mp_typeof(*field->bar.parent) != MP_MAP) {
+ return update_err(op, "can not access by key a "\
+ "non-map field");
+ }
+ }
+ return 0;
+}
+
+int
+do_op_nop_insert(struct update_op *op, struct update_field *field)
+{
+ assert(op->opcode == '!');
+ assert(field->type == UPDATE_NOP);
+ bool is_found = false;
+ int key_len = 0;
+ if (update_bar_locate_opt(op, field, &is_found, &key_len) != 0)
+ return -1;
+ op->new_field_len = op->arg.set.length;
+ if (mp_typeof(*field->bar.parent) == MP_MAP) {
+ if (is_found)
+ return update_err_duplicate(op);
+ op->new_field_len += mp_sizeof_str(key_len);
+ }
+ return 0;
+}
+
+int
+do_op_nop_set(struct update_op *op, struct update_field *field)
+{
+ assert(op->opcode == '=');
+ assert(field->type == UPDATE_NOP);
+ bool is_found = false;
+ int key_len = 0;
+ if (update_bar_locate_opt(op, field, &is_found, &key_len) != 0)
+ return -1;
+ op->new_field_len = op->arg.set.length;
+ if (! is_found) {
+ op->opcode = '!';
+ if (mp_typeof(*field->bar.parent) == MP_MAP)
+ op->new_field_len += mp_sizeof_str(key_len);
+ }
+ return 0;
+}
+
+int
+do_op_nop_delete(struct update_op *op, struct update_field *field)
+{
+ assert(op->opcode == '#');
+ assert(field->type == UPDATE_NOP);
+ bool is_found = false;
+ int key_len_or_index = 0;
+ if (update_bar_locate_opt(op, field, &is_found, &key_len_or_index) != 0)
+ return -1;
+ if (! is_found)
+ return update_err_no_such_field(op);
+ if (mp_typeof(*field->bar.parent) == MP_ARRAY) {
+ const char *tmp = field->bar.parent;
+ uint32_t size = mp_decode_array(&tmp);
+ if (key_len_or_index + op->arg.del.count > size)
+ op->arg.del.count = size - key_len_or_index;
+ const char *end = field->bar.point + field->bar.point_size;
+ for (uint32_t i = 1; i < op->arg.del.count; ++i)
+ mp_next(&end);
+ field->bar.point_size = end - field->bar.point;
+ } else {
+ if (op->arg.del.count != 1)
+ return update_err_delete1(op);
+ /* Take key size into account to delete it too. */
+ uint32_t key_size = mp_sizeof_str(key_len_or_index);
+ field->bar.point -= key_size;
+ field->bar.point_size += key_size;
+ }
+ return 0;
+}
+
+#define DO_SCALAR_OP_GENERIC(op_type) \
+int \
+do_op_bar_##op_type(struct update_op *op, struct update_field *field) \
+{ \
+ (void) op; \
+ (void) field; \
+ assert(field->type == UPDATE_BAR); \
+ diag_set(ClientError, ER_UNSUPPORTED, "update", \
+ "intersected JSON paths"); \
+ return -1; \
+}
+
+DO_SCALAR_OP_GENERIC(insert)
+
+DO_SCALAR_OP_GENERIC(set)
+
+DO_SCALAR_OP_GENERIC(delete)
+
+DO_SCALAR_OP_GENERIC(arith)
+
+DO_SCALAR_OP_GENERIC(bit)
+
+DO_SCALAR_OP_GENERIC(splice)
+
+#undef DO_SCALAR_OP_GENERIC
+
+#define DO_SCALAR_OP_GENERIC(op_type) \
+int \
+do_op_nop_##op_type(struct update_op *op, struct update_field *field) \
+{ \
+ assert(field->type == UPDATE_NOP); \
+ if (update_bar_locate(op, field) != 0) \
+ return -1; \
+ return update_op_do_##op_type(op, field->bar.point); \
+}
+
+DO_SCALAR_OP_GENERIC(arith)
+
+DO_SCALAR_OP_GENERIC(bit)
+
+DO_SCALAR_OP_GENERIC(splice)
+
+uint32_t
+update_bar_sizeof(struct update_field *field)
+{
+ assert(field->type == UPDATE_BAR);
+ switch(field->bar.op->opcode) {
+ case '!': {
+ const char *parent = field->bar.parent;
+ uint32_t size = field->size + field->bar.op->new_field_len;
+ if (mp_typeof(*parent) == MP_ARRAY) {
+ uint32_t array_size = mp_decode_array(&parent);
+ return size + mp_sizeof_array(array_size + 1) -
+ mp_sizeof_array(array_size);
+ } else {
+ uint32_t map_size = mp_decode_map(&parent);
+ return size + mp_sizeof_map(map_size + 1) -
+ mp_sizeof_map(map_size);
+ }
+ }
+ case '#': {
+ const char *parent = field->bar.parent;
+ uint32_t delete_count = field->bar.op->arg.del.count;
+ uint32_t size = field->size - field->bar.point_size;
+ if (mp_typeof(*parent) == MP_ARRAY) {
+ uint32_t array_size = mp_decode_array(&parent);
+ assert(array_size >= delete_count);
+ return size - mp_sizeof_array(array_size) +
+ mp_sizeof_array(array_size - delete_count);
+ } else {
+ uint32_t map_size = mp_decode_map(&parent);
+ assert(delete_count == 1);
+ return size - mp_sizeof_map(map_size) +
+ mp_sizeof_map(map_size - 1);
+ }
+ }
+ default: {
+ return field->size - field->bar.point_size +
+ field->bar.op->new_field_len;
+ }
+ }
+}
+
+uint32_t
+update_bar_store(struct update_field *field, char *out, char *out_end)
+{
+ assert(field->type == UPDATE_BAR);
+ (void) out_end;
+ struct update_op *op = field->bar.op;
+ char *out_saved = out;
+ switch(op->opcode) {
+ case '!': {
+ const char *pos = field->bar.parent;
+ uint32_t before_parent = pos - field->data;
+ /* Before parent. */
+ memcpy(out, field->data, before_parent);
+ out += before_parent;
+ if (mp_typeof(*pos) == MP_ARRAY) {
+ /* New array header. */
+ uint32_t size = mp_decode_array(&pos);
+ out = mp_encode_array(out, size + 1);
+ /* Before insertion point. */
+ size = field->bar.point - pos;
+ memcpy(out, pos, size);
+ out += size;
+ pos += size;
+ } else {
+ /* New map header. */
+ uint32_t size = mp_decode_map(&pos);
+ out = mp_encode_map(out, size + 1);
+ /* New key. */
+ out = mp_encode_str(out, field->bar.new_key,
+ field->bar.new_key_len);
+ }
+ /* New value. */
+ memcpy(out, op->arg.set.value, op->arg.set.length);
+ out += op->arg.set.length;
+ /* Old values and field tail. */
+ uint32_t after_point = field->data + field->size - pos;
+ memcpy(out, pos, after_point);
+ out += after_point;
+ return out - out_saved;
+ }
+ case '#': {
+ const char *pos = field->bar.parent;
+ uint32_t size, before_parent = pos - field->data;
+ memcpy(out, field->data, before_parent);
+ out += before_parent;
+ if (mp_typeof(*pos) == MP_ARRAY) {
+ size = mp_decode_array(&pos);
+ out = mp_encode_array(out, size - op->arg.del.count);
+ } else {
+ size = mp_decode_map(&pos);
+ out = mp_encode_map(out, size - 1);
+ }
+ size = field->bar.point - pos;
+ memcpy(out, pos, size);
+ out += size;
+ pos = field->bar.point + field->bar.point_size;
+
+ size = field->data + field->size - pos;
+ memcpy(out, pos, size);
+ return out + size - out_saved;
+ }
+ default: {
+ uint32_t before_point = field->bar.point - field->data;
+ const char *field_end = field->data + field->size;
+ const char *point_end =
+ field->bar.point + field->bar.point_size;
+ uint32_t after_point = field_end - point_end;
+
+ memcpy(out, field->data, before_point);
+ out += before_point;
+ op->meta->store_cb(op, field->bar.point, out);
+ out += op->new_field_len;
+ memcpy(out, point_end, after_point);
+ return out + after_point - out_saved;
+ }
+ }
+}
diff --git a/src/box/update/update_field.c b/src/box/update/update_field.c
index a3200a266..820dc427b 100644
--- a/src/box/update/update_field.c
+++ b/src/box/update/update_field.c
@@ -41,7 +41,9 @@
static inline const char *
update_op_field_str(const struct update_op *op)
{
- if (op->field_no >= 0)
+ if (op->lexer.src != NULL)
+ return tt_sprintf("'%.*s'", op->lexer.src_len, op->lexer.src);
+ else if (op->field_no >= 0)
return tt_sprintf("%d", op->field_no + TUPLE_INDEX_BASE);
else
return tt_sprintf("%d", op->field_no);
@@ -74,8 +76,12 @@ update_err_splice_bound(const struct update_op *op)
int
update_err_no_such_field(const struct update_op *op)
{
- diag_set(ClientError, ER_NO_SUCH_FIELD_NO, op->field_no >= 0 ?
- TUPLE_INDEX_BASE + op->field_no : op->field_no);
+ if (op->lexer.src == NULL) {
+ diag_set(ClientError, ER_NO_SUCH_FIELD_NO, op->field_no +
+ (op->field_no >= 0 ? TUPLE_INDEX_BASE : 0));
+ return -1;
+ }
+ diag_set(ClientError, ER_NO_SUCH_FIELD_NAME, update_op_field_str(op));
return -1;
}
@@ -99,6 +105,8 @@ update_field_sizeof(struct update_field *field)
return field->scalar.op->new_field_len;
case UPDATE_ARRAY:
return update_array_sizeof(field);
+ case UPDATE_BAR:
+ return update_bar_sizeof(field);
default:
unreachable();
}
@@ -123,6 +131,8 @@ update_field_store(struct update_field *field, char *out, char *out_end)
return size;
case UPDATE_ARRAY:
return update_array_store(field, out, out_end);
+ case UPDATE_BAR:
+ return update_bar_store(field, out, out_end);
default:
unreachable();
}
@@ -540,6 +550,7 @@ update_op_decode(struct update_op *op, int index_base,
switch(mp_typeof(**expr)) {
case MP_INT:
case MP_UINT: {
+ json_lexer_create(&op->lexer, NULL, 0, 0);
if (mp_read_i32(op, expr, &field_no) != 0)
return -1;
if (field_no - index_base >= 0) {
@@ -555,14 +566,35 @@ update_op_decode(struct update_op *op, int index_base,
case MP_STR: {
const char *path = mp_decode_str(expr, &len);
uint32_t field_no, hash = field_name_hash(path, len);
+ json_lexer_create(&op->lexer, path, len, TUPLE_INDEX_BASE);
if (tuple_fieldno_by_name(dict, path, len, hash,
&field_no) == 0) {
op->field_no = (int32_t) field_no;
+ op->lexer.offset = len;
break;
}
- diag_set(ClientError, ER_NO_SUCH_FIELD_NAME,
- tt_cstr(path, len));
- return -1;
+ struct json_token token;
+ int rc = json_lexer_next_token(&op->lexer, &token);
+ if (rc != 0)
+ return update_err_bad_json(op, rc);
+ switch (token.type) {
+ case JSON_TOKEN_NUM:
+ op->field_no = token.num;
+ break;
+ case JSON_TOKEN_STR:
+ hash = field_name_hash(token.str, token.len);
+ if (tuple_fieldno_by_name(dict, token.str, token.len,
+ hash, &field_no) == 0) {
+ op->field_no = (int32_t) field_no;
+ break;
+ }
+ FALLTHROUGH;
+ default:
+ diag_set(ClientError, ER_NO_SUCH_FIELD_NAME,
+ tt_cstr(path, len));
+ return -1;
+ }
+ break;
}
default:
diag_set(ClientError, ER_ILLEGAL_PARAMS,
diff --git a/src/box/update/update_field.h b/src/box/update/update_field.h
index 35ec85238..fc4b1053f 100644
--- a/src/box/update/update_field.h
+++ b/src/box/update/update_field.h
@@ -33,6 +33,7 @@
#include "trivia/util.h"
#include "tt_static.h"
#include <stdint.h>
+#include "json/json.h"
#include "bit/int96.h"
/**
@@ -177,6 +178,12 @@ struct update_op {
uint32_t new_field_len;
/** Opcode symbol: = + - / ... */
char opcode;
+ /**
+ * Operation target path and its lexer in one. This lexer
+ * is used when the operation is applied down through the
+ * update tree.
+ */
+ struct json_lexer lexer;
};
/**
@@ -193,6 +200,16 @@ int
update_op_decode(struct update_op *op, int index_base,
struct tuple_dictionary *dict, const char **expr);
+/**
+ * Check if the operation should be applied on the current path
+ * node.
+ */
+static inline bool
+update_op_is_term(const struct update_op *op)
+{
+ return json_lexer_is_eof(&op->lexer);
+}
+
/* }}} update_op */
/* {{{ update_field */
@@ -217,6 +234,14 @@ enum update_type {
* of individual fields.
*/
UPDATE_ARRAY,
+ /**
+ * Field of this type stores such update, that has
+ * non-empty JSON path non-intersected with any another
+ * update. In such optimized case it is possible to do not
+ * allocate neither fields nor ops nor anything for path
+ * nodes. And this is the most common case.
+ */
+ UPDATE_BAR,
};
/**
@@ -251,6 +276,49 @@ struct update_field {
struct {
struct rope *rope;
} array;
+ /**
+ * Bar update - by JSON path non-intersected with
+ * any another update.
+ */
+ struct {
+ /** Bar update is a single operation. */
+ struct update_op *op;
+ /**
+ * Always has a non-empty head path
+ * leading inside this field's data.
+ */
+ const char *path;
+ int path_len;
+ /**
+ * For insertion/deletion to change parent
+ * header.
+ */
+ const char *parent;
+ union {
+ /**
+ * For scalar op; insertion into
+ * array; deletion. This is the
+ * point to delete, change or
+ * insert after.
+ */
+ struct {
+ const char *point;
+ uint32_t point_size;
+ };
+ /*
+ * For insertion into map. New
+ * key. On insertion into a map
+ * there is no strict order as in
+ * array and no point. The field
+ * is inserted just right after
+ * the parent header.
+ */
+ struct {
+ const char *new_key;
+ uint32_t new_key_len;
+ };
+ };
+ } bar;
};
};
@@ -320,6 +388,18 @@ OP_DECL_GENERIC(array)
/* }}} update_field.array */
+/* {{{ update_field.bar */
+
+OP_DECL_GENERIC(bar)
+
+/* }}} update_field.bar */
+
+/* {{{ update_field.nop */
+
+OP_DECL_GENERIC(nop)
+
+/* }}} update_field.nop */
+
#undef OP_DECL_GENERIC
/* {{{ Common helpers. */
@@ -339,6 +419,10 @@ do_op_##op_type(struct update_op *op, struct update_field *field) \
switch (field->type) { \
case UPDATE_ARRAY: \
return do_op_array_##op_type(op, field); \
+ case UPDATE_NOP: \
+ return do_op_nop_##op_type(op, field); \
+ case UPDATE_BAR: \
+ return do_op_bar_##op_type(op, field); \
default: \
unreachable(); \
} \
@@ -404,6 +488,24 @@ update_err_double(const struct update_op *op)
return update_err(op, "double update of the same field");
}
+static inline int
+update_err_bad_json(const struct update_op *op, int pos)
+{
+ return update_err(op, tt_sprintf("invalid JSON in position %d", pos));
+}
+
+static inline int
+update_err_delete1(const struct update_op *op)
+{
+ return update_err(op, "can delete only 1 field from a map in a row");
+}
+
+static inline int
+update_err_duplicate(const struct update_op *op)
+{
+ return update_err(op, "the key exists already");
+}
+
/** }}} Error helpers. */
#endif /* TARANTOOL_BOX_TUPLE_UPDATE_FIELD_H */
diff --git a/src/box/vinyl.c b/src/box/vinyl.c
index 5fdfe0aa7..f6cb03550 100644
--- a/src/box/vinyl.c
+++ b/src/box/vinyl.c
@@ -2055,15 +2055,28 @@ request_normalize_ops(struct request *request)
ops_end = mp_encode_str(ops_end, op_name, op_name_len);
int field_no;
- if (mp_typeof(*pos) == MP_INT) {
+ const char *field_name;
+ switch (mp_typeof(*pos)) {
+ case MP_INT:
field_no = mp_decode_int(&pos);
ops_end = mp_encode_int(ops_end, field_no);
- } else {
+ break;
+ case MP_UINT:
field_no = mp_decode_uint(&pos);
field_no -= request->index_base;
ops_end = mp_encode_uint(ops_end, field_no);
+ break;
+ case MP_STR:
+ field_name = pos;
+ mp_next(&pos);
+ memcpy(ops_end, field_name, pos - field_name);
+ ops_end += pos - field_name;
+ break;
+ default:
+ unreachable();
}
+
if (*op_name == ':') {
/**
* splice op adjust string pos and copy
diff --git a/test/box/update.result b/test/box/update.result
index f5de3bc09..ec882d20d 100644
--- a/test/box/update.result
+++ b/test/box/update.result
@@ -794,7 +794,7 @@ s:update({0}, {{'+', 0}})
...
s:update({0}, {{'+', '+', '+'}})
---
-- error: Field '+' was not found in the tuple
+- error: 'Field ''+'' UPDATE error: invalid JSON in position 1'
...
s:update({0}, {{0, 0, 0}})
---
@@ -849,3 +849,411 @@ s:update(1, {{'=', 3, map}})
s:drop()
---
...
+--
+-- gh-1261: update by JSON path.
+--
+format = {}
+---
+...
+format[1] = {'field1', 'unsigned'}
+---
+...
+format[2] = {'f', 'map'}
+---
+...
+format[3] = {'g', 'array'}
+---
+...
+s = box.schema.create_space('test', {format = format})
+---
+...
+pk = s:create_index('pk')
+---
+...
+t = {}
+---
+...
+t[1] = 1
+---
+...
+t[2] = { \
+ a = 100, \
+ b = 200, \
+ c = { \
+ d = 400, \
+ e = 500, \
+ f = {4, 5, 6, 7, 8}, \
+ g = {k = 600, l = 700} \
+ }, \
+ m = true, \
+ g = {800, 900} \
+}; \
+t[3] = { \
+ 100, \
+ 200, \
+ { \
+ {300, 350}, \
+ {400, 450} \
+ }, \
+ {a = 500, b = 600}, \
+ {c = 700, d = 800} \
+}
+---
+...
+t = s:insert(t)
+---
+...
+t4_array = t:update({{'!', 4, setmetatable({}, {__serialize = 'array'})}})
+---
+...
+t4_map = t:update({{'!', 4, setmetatable({}, {__serialize = 'map'})}})
+---
+...
+t
+---
+- [1, {'b': 200, 'm': true, 'a': 100, 'c': {'d': 400, 'f': [4, 5, 6, 7, 8], 'e': 500,
+ 'g': {'k': 600, 'l': 700}}, 'g': [800, 900]}, [100, 200, [[300, 350], [400,
+ 450]], {'a': 500, 'b': 600}, {'c': 700, 'd': 800}]]
+...
+--
+-- At first, test simple non-intersected paths.
+--
+--
+-- !
+--
+t:update({{'!', 'f.c.f[1]', 3}, {'!', '[3][1]', {100, 200, 300}}})
+---
+- [1, {'b': 200, 'm': true, 'a': 100, 'c': {'d': 400, 'f': [3, 4, 5, 6, 7, 8], 'e': 500,
+ 'g': {'k': 600, 'l': 700}}, 'g': [800, 900]}, [[100, 200, 300], 100, 200, [
+ [300, 350], [400, 450]], {'a': 500, 'b': 600}, {'c': 700, 'd': 800}]]
+...
+t:update({{'!', 'f.g[3]', 1000}})
+---
+- [1, {'b': 200, 'm': true, 'a': 100, 'c': {'d': 400, 'f': [4, 5, 6, 7, 8], 'e': 500,
+ 'g': {'k': 600, 'l': 700}}, 'g': [800, 900, 1000]}, [100, 200, [[300, 350],
+ [400, 450]], {'a': 500, 'b': 600}, {'c': 700, 'd': 800}]]
+...
+t:update({{'!', 'g[6]', 'new element'}})
+---
+- [1, {'b': 200, 'm': true, 'a': 100, 'c': {'d': 400, 'f': [4, 5, 6, 7, 8], 'e': 500,
+ 'g': {'k': 600, 'l': 700}}, 'g': [800, 900]}, [100, 200, [[300, 350], [400,
+ 450]], {'a': 500, 'b': 600}, {'c': 700, 'd': 800}, 'new element']]
+...
+t:update({{'!', 'f.e', 300}, {'!', 'g[4].c', 700}})
+---
+- [1, {'b': 200, 'm': true, 'g': [800, 900], 'a': 100, 'c': {'d': 400, 'f': [4, 5,
+ 6, 7, 8], 'e': 500, 'g': {'k': 600, 'l': 700}}, 'e': 300}, [100, 200, [[300,
+ 350], [400, 450]], {'b': 600, 'c': 700, 'a': 500}, {'c': 700, 'd': 800}]]
+...
+t:update({{'!', 'f.c.f[2]', 4.5}, {'!', 'g[3][2][2]', 425}})
+---
+- [1, {'b': 200, 'm': true, 'a': 100, 'c': {'d': 400, 'f': [4, 4.5, 5, 6, 7, 8], 'e': 500,
+ 'g': {'k': 600, 'l': 700}}, 'g': [800, 900]}, [100, 200, [[300, 350], [400,
+ 425, 450]], {'a': 500, 'b': 600}, {'c': 700, 'd': 800}]]
+...
+t2 = t:update({{'!', 'g[6]', {100}}})
+---
+...
+-- Test single element array update.
+t2:update({{'!', 'g[6][2]', 200}})
+---
+- [1, {'b': 200, 'm': true, 'a': 100, 'c': {'d': 400, 'f': [4, 5, 6, 7, 8], 'e': 500,
+ 'g': {'k': 600, 'l': 700}}, 'g': [800, 900]}, [100, 200, [[300, 350], [400,
+ 450]], {'a': 500, 'b': 600}, {'c': 700, 'd': 800}, [100, 200]]]
+...
+t2:update({{'!', 'g[6][1]', 50}})
+---
+- [1, {'b': 200, 'm': true, 'a': 100, 'c': {'d': 400, 'f': [4, 5, 6, 7, 8], 'e': 500,
+ 'g': {'k': 600, 'l': 700}}, 'g': [800, 900]}, [100, 200, [[300, 350], [400,
+ 450]], {'a': 500, 'b': 600}, {'c': 700, 'd': 800}, [50, 100]]]
+...
+-- Test empty array/map.
+t4_array:update({{'!', '[4][1]', 100}})
+---
+- [1, {'b': 200, 'm': true, 'a': 100, 'c': {'d': 400, 'f': [4, 5, 6, 7, 8], 'e': 500,
+ 'g': {'k': 600, 'l': 700}}, 'g': [800, 900]}, [100, 200, [[300, 350], [400,
+ 450]], {'a': 500, 'b': 600}, {'c': 700, 'd': 800}], [100]]
+...
+t4_map:update({{'!', '[4].a', 100}})
+---
+- [1, {'b': 200, 'm': true, 'a': 100, 'c': {'d': 400, 'f': [4, 5, 6, 7, 8], 'e': 500,
+ 'g': {'k': 600, 'l': 700}}, 'g': [800, 900]}, [100, 200, [[300, 350], [400,
+ 450]], {'a': 500, 'b': 600}, {'c': 700, 'd': 800}], {'a': 100}]
+...
+-- Test errors.
+t:update({{'!', 'a', 100}}) -- No such field.
+---
+- error: Field 'a' was not found in the tuple
+...
+t:update({{'!', 'f.a', 300}}) -- Key already exists.
+---
+- error: 'Field ''f.a'' UPDATE error: the key exists already'
+...
+t:update({{'!', 'f.c.f[0]', 3.5}}) -- No such index, too small.
+---
+- error: 'Field ''f.c.f[0]'' UPDATE error: invalid JSON in position 7'
+...
+t:update({{'!', 'f.c.f[100]', 100}}) -- No such index, too big.
+---
+- error: Field ''f.c.f[100]'' was not found in the tuple
+...
+t:update({{'!', 'g[4][100]', 700}}) -- Insert index into map.
+---
+- error: 'Field ''g[4][100]'' UPDATE error: can not access by index a non-array field'
+...
+t:update({{'!', 'g[1][1]', 300}})
+---
+- error: 'Field ''g[1][1]'' UPDATE error: can not access by index a non-array field'
+...
+t:update({{'!', 'f.g.a', 700}}) -- Insert key into array.
+---
+- error: 'Field ''f.g.a'' UPDATE error: can not access by key a non-map field'
+...
+t:update({{'!', 'f.g[1].a', 700}})
+---
+- error: 'Field ''f.g[1].a'' UPDATE error: can not access by key a non-map field'
+...
+t:update({{'!', 'f[*].k', 20}}) -- 'Any' is not considered valid JSON.
+---
+- error: 'Field ''f[*].k'' UPDATE error: invalid JSON in position 3'
+...
+-- JSON error after the not existing field to insert.
+t:update({{'!', '[2].e.100000', 100}})
+---
+- error: 'Field ''[2].e.100000'' UPDATE error: invalid JSON in position 7'
+...
+-- Correct JSON, but next to last field does not exist. '!' can't
+-- create the whole path.
+t:update({{'!', '[2].e.f', 100}})
+---
+- error: Field ''[2].e.f'' was not found in the tuple
+...
+--
+-- =
+--
+-- Set existing fields.
+t:update({{'=', 'f.a', 150}, {'=', 'g[3][1][2]', 400}})
+---
+- [1, {'b': 200, 'm': true, 'a': 150, 'c': {'d': 400, 'f': [4, 5, 6, 7, 8], 'e': 500,
+ 'g': {'k': 600, 'l': 700}}, 'g': [800, 900]}, [100, 200, [[300, 400], [400,
+ 450]], {'a': 500, 'b': 600}, {'c': 700, 'd': 800}]]
+...
+t:update({{'=', 'f', {a = 100, b = 200}}})
+---
+- [1, {'a': 100, 'b': 200}, [100, 200, [[300, 350], [400, 450]], {'a': 500, 'b': 600},
+ {'c': 700, 'd': 800}]]
+...
+t:update({{'=', 'g[4].b', 700}})
+---
+- [1, {'b': 200, 'm': true, 'a': 100, 'c': {'d': 400, 'f': [4, 5, 6, 7, 8], 'e': 500,
+ 'g': {'k': 600, 'l': 700}}, 'g': [800, 900]}, [100, 200, [[300, 350], [400,
+ 450]], {'a': 500, 'b': 700}, {'c': 700, 'd': 800}]]
+...
+-- Insert via set.
+t:update({{'=', 'f.e', 300}})
+---
+- [1, {'b': 200, 'm': true, 'g': [800, 900], 'a': 100, 'c': {'d': 400, 'f': [4, 5,
+ 6, 7, 8], 'e': 500, 'g': {'k': 600, 'l': 700}}, 'e': 300}, [100, 200, [[300,
+ 350], [400, 450]], {'a': 500, 'b': 600}, {'c': 700, 'd': 800}]]
+...
+t:update({{'=', 'f.g[3]', 1000}})
+---
+- [1, {'b': 200, 'm': true, 'a': 100, 'c': {'d': 400, 'f': [4, 5, 6, 7, 8], 'e': 500,
+ 'g': {'k': 600, 'l': 700}}, 'g': [800, 900, 1000]}, [100, 200, [[300, 350],
+ [400, 450]], {'a': 500, 'b': 600}, {'c': 700, 'd': 800}]]
+...
+t:update({{'=', 'f.g[1]', 0}})
+---
+- [1, {'b': 200, 'm': true, 'a': 100, 'c': {'d': 400, 'f': [4, 5, 6, 7, 8], 'e': 500,
+ 'g': {'k': 600, 'l': 700}}, 'g': [0, 900]}, [100, 200, [[300, 350], [400, 450]],
+ {'a': 500, 'b': 600}, {'c': 700, 'd': 800}]]
+...
+-- Test empty array/map.
+t4_array:update({{'=', '[4][1]', 100}})
+---
+- [1, {'b': 200, 'm': true, 'a': 100, 'c': {'d': 400, 'f': [4, 5, 6, 7, 8], 'e': 500,
+ 'g': {'k': 600, 'l': 700}}, 'g': [800, 900]}, [100, 200, [[300, 350], [400,
+ 450]], {'a': 500, 'b': 600}, {'c': 700, 'd': 800}], [100]]
+...
+t4_map:update({{'=', '[4]["a"]', 100}})
+---
+- [1, {'b': 200, 'm': true, 'a': 100, 'c': {'d': 400, 'f': [4, 5, 6, 7, 8], 'e': 500,
+ 'g': {'k': 600, 'l': 700}}, 'g': [800, 900]}, [100, 200, [[300, 350], [400,
+ 450]], {'a': 500, 'b': 600}, {'c': 700, 'd': 800}], {'a': 100}]
+...
+-- Test errors.
+t:update({{'=', 'f.a[1]', 100}})
+---
+- error: 'Field ''f.a[1]'' UPDATE error: can not access by index a non-array field'
+...
+t:update({{'=', 'f.a.k', 100}})
+---
+- error: 'Field ''f.a.k'' UPDATE error: can not access by key a non-map field'
+...
+t:update({{'=', 'f.c.f[1]', 100}})
+---
+- [1, {'b': 200, 'm': true, 'a': 100, 'c': {'d': 400, 'f': [100, 5, 6, 7, 8], 'e': 500,
+ 'g': {'k': 600, 'l': 700}}, 'g': [800, 900]}, [100, 200, [[300, 350], [400,
+ 450]], {'a': 500, 'b': 600}, {'c': 700, 'd': 800}]]
+...
+t:update({{'=', 'f.c.f[100]', 100}})
+---
+- error: Field ''f.c.f[100]'' was not found in the tuple
+...
+t:update({{'=', '[2].c.f 1 1 1 1', 100}})
+---
+- error: 'Field ''[2].c.f 1 1 1 1'' UPDATE error: invalid JSON in position 8'
+...
+--
+-- #
+--
+t:update({{'#', '[2].b', 1}})
+---
+- [1, {'a': 100, 'm': true, 'c': {'d': 400, 'f': [4, 5, 6, 7, 8], 'e': 500, 'g': {
+ 'k': 600, 'l': 700}}, 'g': [800, 900]}, [100, 200, [[300, 350], [400, 450]],
+ {'a': 500, 'b': 600}, {'c': 700, 'd': 800}]]
+...
+t:update({{'#', 'f.c.f[1]', 1}})
+---
+- [1, {'b': 200, 'm': true, 'a': 100, 'c': {'d': 400, 'f': [5, 6, 7, 8], 'e': 500,
+ 'g': {'k': 600, 'l': 700}}, 'g': [800, 900]}, [100, 200, [[300, 350], [400,
+ 450]], {'a': 500, 'b': 600}, {'c': 700, 'd': 800}]]
+...
+t:update({{'#', 'f.c.f[1]', 2}})
+---
+- [1, {'b': 200, 'm': true, 'a': 100, 'c': {'d': 400, 'f': [6, 7, 8], 'e': 500, 'g': {
+ 'k': 600, 'l': 700}}, 'g': [800, 900]}, [100, 200, [[300, 350], [400, 450]],
+ {'a': 500, 'b': 600}, {'c': 700, 'd': 800}]]
+...
+t:update({{'#', 'f.c.f[1]', 100}})
+---
+- [1, {'b': 200, 'm': true, 'a': 100, 'c': {'d': 400, 'f': [], 'e': 500, 'g': {'k': 600,
+ 'l': 700}}, 'g': [800, 900]}, [100, 200, [[300, 350], [400, 450]], {'a': 500,
+ 'b': 600}, {'c': 700, 'd': 800}]]
+...
+t:update({{'#', 'f.c.f[5]', 1}})
+---
+- [1, {'b': 200, 'm': true, 'a': 100, 'c': {'d': 400, 'f': [4, 5, 6, 7], 'e': 500,
+ 'g': {'k': 600, 'l': 700}}, 'g': [800, 900]}, [100, 200, [[300, 350], [400,
+ 450]], {'a': 500, 'b': 600}, {'c': 700, 'd': 800}]]
+...
+t:update({{'#', 'f.c.f[5]', 2}})
+---
+- [1, {'b': 200, 'm': true, 'a': 100, 'c': {'d': 400, 'f': [4, 5, 6, 7], 'e': 500,
+ 'g': {'k': 600, 'l': 700}}, 'g': [800, 900]}, [100, 200, [[300, 350], [400,
+ 450]], {'a': 500, 'b': 600}, {'c': 700, 'd': 800}]]
+...
+-- Test errors.
+t:update({{'#', 'f.h', 1}})
+---
+- error: Field ''f.h'' was not found in the tuple
+...
+t:update({{'#', 'f.c.f[100]', 1}})
+---
+- error: Field ''f.c.f[100]'' was not found in the tuple
+...
+t:update({{'#', 'f.b', 2}})
+---
+- error: 'Field ''f.b'' UPDATE error: can delete only 1 field from a map in a row'
+...
+t:update({{'#', 'f.b', 0}})
+---
+- error: 'Field ''f.b'' UPDATE error: cannot delete 0 fields'
+...
+t:update({{'#', 'f', 0}})
+---
+- error: 'Field ''f'' UPDATE error: cannot delete 0 fields'
+...
+--
+-- Scalar operations.
+--
+t:update({{'+', 'f.a', 50}})
+---
+- [1, {'b': 200, 'm': true, 'a': 150, 'c': {'d': 400, 'f': [4, 5, 6, 7, 8], 'e': 500,
+ 'g': {'k': 600, 'l': 700}}, 'g': [800, 900]}, [100, 200, [[300, 350], [400,
+ 450]], {'a': 500, 'b': 600}, {'c': 700, 'd': 800}]]
+...
+t:update({{'-', 'f.c.f[1]', 0.5}})
+---
+- [1, {'b': 200, 'm': true, 'a': 100, 'c': {'d': 400, 'f': [3.5, 5, 6, 7, 8], 'e': 500,
+ 'g': {'k': 600, 'l': 700}}, 'g': [800, 900]}, [100, 200, [[300, 350], [400,
+ 450]], {'a': 500, 'b': 600}, {'c': 700, 'd': 800}]]
+...
+t:update({{'&', 'f.c.f[2]', 4}})
+---
+- [1, {'b': 200, 'm': true, 'a': 100, 'c': {'d': 400, 'f': [4, 4, 6, 7, 8], 'e': 500,
+ 'g': {'k': 600, 'l': 700}}, 'g': [800, 900]}, [100, 200, [[300, 350], [400,
+ 450]], {'a': 500, 'b': 600}, {'c': 700, 'd': 800}]]
+...
+t2 = t:update({{'=', 4, {str = 'abcd'}}})
+---
+...
+t2:update({{':', '[4].str', 2, 2, 'e'}})
+---
+- [1, {'b': 200, 'm': true, 'a': 100, 'c': {'d': 400, 'f': [4, 5, 6, 7, 8], 'e': 500,
+ 'g': {'k': 600, 'l': 700}}, 'g': [800, 900]}, [100, 200, [[300, 350], [400,
+ 450]], {'a': 500, 'b': 600}, {'c': 700, 'd': 800}], {'str': 'aed'}]
+...
+-- Test errors.
+t:update({{'+', 'g[3]', 50}})
+---
+- error: 'Argument type in operation ''+'' on field ''g[3]'' does not match field
+ type: expected a number'
+...
+t:update({{'+', '[2].b.......', 100}})
+---
+- error: 'Field ''[2].b.......'' UPDATE error: invalid JSON in position 7'
+...
+t:update({{'+', '[2].b.c.d.e', 100}})
+---
+- error: Field ''[2].b.c.d.e'' was not found in the tuple
+...
+t:update({{'-', '[2][*]', 20}})
+---
+- error: 'Field ''[2][*]'' UPDATE error: invalid JSON in position 5'
+...
+-- Vinyl normalizes field numbers. It should not touch paths,
+-- and they should not affect squashing.
+format = {}
+---
+...
+format[1] = {'field1', 'unsigned'}
+---
+...
+format[2] = {'field2', 'any'}
+---
+...
+vy_s = box.schema.create_space('test2', {engine = 'vinyl', format = format})
+---
+...
+pk = vy_s:create_index('pk')
+---
+...
+_ = vy_s:replace(t)
+---
+...
+box.begin()
+---
+...
+-- Use a scalar operation, only they can be squashed.
+vy_s:upsert({1, 1}, {{'+', 'field2.c.f[1]', 1}})
+---
+...
+vy_s:upsert({1, 1}, {{'+', '[3][3][1][1]', 1}})
+---
+...
+box.commit()
+---
+...
+vy_s:select()
+---
+- - [1, {'b': 200, 'm': true, 'a': 100, 'c': {'d': 400, 'f': [5, 5, 6, 7, 8], 'e': 500,
+ 'g': {'k': 600, 'l': 700}}, 'g': [800, 900]}, [100, 200, [[301, 350], [400,
+ 450]], {'a': 500, 'b': 600}, {'c': 700, 'd': 800}]]
+...
+vy_s:drop()
+---
+...
+s:drop()
+---
+...
diff --git a/test/box/update.test.lua b/test/box/update.test.lua
index c455bc1e1..702c0dbd7 100644
--- a/test/box/update.test.lua
+++ b/test/box/update.test.lua
@@ -269,3 +269,148 @@ t:update({{'=', 3, map}})
s:update(1, {{'=', 3, map}})
s:drop()
+
+--
+-- gh-1261: update by JSON path.
+--
+format = {}
+format[1] = {'field1', 'unsigned'}
+format[2] = {'f', 'map'}
+format[3] = {'g', 'array'}
+s = box.schema.create_space('test', {format = format})
+pk = s:create_index('pk')
+t = {}
+t[1] = 1
+t[2] = { \
+ a = 100, \
+ b = 200, \
+ c = { \
+ d = 400, \
+ e = 500, \
+ f = {4, 5, 6, 7, 8}, \
+ g = {k = 600, l = 700} \
+ }, \
+ m = true, \
+ g = {800, 900} \
+}; \
+t[3] = { \
+ 100, \
+ 200, \
+ { \
+ {300, 350}, \
+ {400, 450} \
+ }, \
+ {a = 500, b = 600}, \
+ {c = 700, d = 800} \
+}
+t = s:insert(t)
+
+t4_array = t:update({{'!', 4, setmetatable({}, {__serialize = 'array'})}})
+t4_map = t:update({{'!', 4, setmetatable({}, {__serialize = 'map'})}})
+
+t
+--
+-- At first, test simple non-intersected paths.
+--
+
+--
+-- !
+--
+t:update({{'!', 'f.c.f[1]', 3}, {'!', '[3][1]', {100, 200, 300}}})
+t:update({{'!', 'f.g[3]', 1000}})
+t:update({{'!', 'g[6]', 'new element'}})
+t:update({{'!', 'f.e', 300}, {'!', 'g[4].c', 700}})
+t:update({{'!', 'f.c.f[2]', 4.5}, {'!', 'g[3][2][2]', 425}})
+t2 = t:update({{'!', 'g[6]', {100}}})
+-- Test single element array update.
+t2:update({{'!', 'g[6][2]', 200}})
+t2:update({{'!', 'g[6][1]', 50}})
+-- Test empty array/map.
+t4_array:update({{'!', '[4][1]', 100}})
+t4_map:update({{'!', '[4].a', 100}})
+-- Test errors.
+t:update({{'!', 'a', 100}}) -- No such field.
+t:update({{'!', 'f.a', 300}}) -- Key already exists.
+t:update({{'!', 'f.c.f[0]', 3.5}}) -- No such index, too small.
+t:update({{'!', 'f.c.f[100]', 100}}) -- No such index, too big.
+t:update({{'!', 'g[4][100]', 700}}) -- Insert index into map.
+t:update({{'!', 'g[1][1]', 300}})
+t:update({{'!', 'f.g.a', 700}}) -- Insert key into array.
+t:update({{'!', 'f.g[1].a', 700}})
+t:update({{'!', 'f[*].k', 20}}) -- 'Any' is not considered valid JSON.
+-- JSON error after the not existing field to insert.
+t:update({{'!', '[2].e.100000', 100}})
+-- Correct JSON, but next to last field does not exist. '!' can't
+-- create the whole path.
+t:update({{'!', '[2].e.f', 100}})
+
+--
+-- =
+--
+-- Set existing fields.
+t:update({{'=', 'f.a', 150}, {'=', 'g[3][1][2]', 400}})
+t:update({{'=', 'f', {a = 100, b = 200}}})
+t:update({{'=', 'g[4].b', 700}})
+-- Insert via set.
+t:update({{'=', 'f.e', 300}})
+t:update({{'=', 'f.g[3]', 1000}})
+t:update({{'=', 'f.g[1]', 0}})
+-- Test empty array/map.
+t4_array:update({{'=', '[4][1]', 100}})
+t4_map:update({{'=', '[4]["a"]', 100}})
+-- Test errors.
+t:update({{'=', 'f.a[1]', 100}})
+t:update({{'=', 'f.a.k', 100}})
+t:update({{'=', 'f.c.f[1]', 100}})
+t:update({{'=', 'f.c.f[100]', 100}})
+t:update({{'=', '[2].c.f 1 1 1 1', 100}})
+
+--
+-- #
+--
+t:update({{'#', '[2].b', 1}})
+t:update({{'#', 'f.c.f[1]', 1}})
+t:update({{'#', 'f.c.f[1]', 2}})
+t:update({{'#', 'f.c.f[1]', 100}})
+t:update({{'#', 'f.c.f[5]', 1}})
+t:update({{'#', 'f.c.f[5]', 2}})
+-- Test errors.
+t:update({{'#', 'f.h', 1}})
+t:update({{'#', 'f.c.f[100]', 1}})
+t:update({{'#', 'f.b', 2}})
+t:update({{'#', 'f.b', 0}})
+t:update({{'#', 'f', 0}})
+
+--
+-- Scalar operations.
+--
+t:update({{'+', 'f.a', 50}})
+t:update({{'-', 'f.c.f[1]', 0.5}})
+t:update({{'&', 'f.c.f[2]', 4}})
+t2 = t:update({{'=', 4, {str = 'abcd'}}})
+t2:update({{':', '[4].str', 2, 2, 'e'}})
+-- Test errors.
+t:update({{'+', 'g[3]', 50}})
+t:update({{'+', '[2].b.......', 100}})
+t:update({{'+', '[2].b.c.d.e', 100}})
+t:update({{'-', '[2][*]', 20}})
+
+-- Vinyl normalizes field numbers. It should not touch paths,
+-- and they should not affect squashing.
+format = {}
+format[1] = {'field1', 'unsigned'}
+format[2] = {'field2', 'any'}
+vy_s = box.schema.create_space('test2', {engine = 'vinyl', format = format})
+pk = vy_s:create_index('pk')
+_ = vy_s:replace(t)
+
+box.begin()
+-- Use a scalar operation, only they can be squashed.
+vy_s:upsert({1, 1}, {{'+', 'field2.c.f[1]', 1}})
+vy_s:upsert({1, 1}, {{'+', '[3][3][1][1]', 1}})
+box.commit()
+
+vy_s:select()
+vy_s:drop()
+
+s:drop()
diff --git a/test/engine/update.result b/test/engine/update.result
index f181924f3..ddb13bd5b 100644
--- a/test/engine/update.result
+++ b/test/engine/update.result
@@ -843,11 +843,6 @@ t:update({{'+', '[1]', 50}})
---
- [1, [10, 11, 12], {'b': 21, 'a': 20, 'c': 22}, 'abcdefgh', true, -100, 250]
...
--- JSON paths are not allowed yet.
-t:update({{'=', 'field2[1]', 13}})
----
-- error: Field 'field2[1]' was not found in the tuple
-...
s:update({1}, {{'=', 'field3', {d = 30, e = 31, f = 32}}})
---
- [1, [10, 11, 12], {'d': 30, 'f': 32, 'e': 31}, 'abcdefgh', true, -100, 200]
diff --git a/test/engine/update.test.lua b/test/engine/update.test.lua
index 4ca2589e4..31fca2b7b 100644
--- a/test/engine/update.test.lua
+++ b/test/engine/update.test.lua
@@ -156,8 +156,6 @@ t:update({{':', 'field4', 3, 3, 'bbccdd'}, {'+', 'field6', 50}, {'!', 7, 300}})
-- Any path is interpreted as a field name first. And only then
-- as JSON.
t:update({{'+', '[1]', 50}})
--- JSON paths are not allowed yet.
-t:update({{'=', 'field2[1]', 13}})
s:update({1}, {{'=', 'field3', {d = 30, e = 31, f = 32}}})
--
2.20.1 (Apple Git-117)
^ permalink raw reply [flat|nested] 15+ messages in thread* [tarantool-patches] [PATCH 11/13] tuple: make update operation tokens consumable
2019-08-12 23:05 [tarantool-patches] [PATCH 00/13] JSON updates Vladislav Shpilevoy
2019-08-12 23:05 ` [tarantool-patches] [PATCH 01/13] tuple: remove alloc and alloc_ctx args from update() Vladislav Shpilevoy
2019-08-12 23:05 ` [tarantool-patches] [PATCH 10/13] tuple: enable JSON bar updates Vladislav Shpilevoy
@ 2019-08-12 23:05 ` Vladislav Shpilevoy
2019-08-12 23:05 ` [tarantool-patches] [PATCH 12/13] tuple: JSON updates support intersection by arrays Vladislav Shpilevoy
` (10 subsequent siblings)
13 siblings, 0 replies; 15+ messages in thread
From: Vladislav Shpilevoy @ 2019-08-12 23:05 UTC (permalink / raw)
To: tarantool-patches
There is a case: [1][2][3][4] = 100. It is not a problem when it
is a single operation, not intersecting with anything. It is a
bar then, and works ok. But the next patch allows intersections,
and the path [1][2][3][4] can become a tree of arrays.
The root array has its child [1] array. The child has its own
child [2] array, and so on. They each would call
do_op_set(child). But all array update routines expect, that
field_no is already extracted from the patch, and will look at
update_op.field_no to find a target field.
It means, that perhaps each next level of the update [1][2][3][4]
should prepare field_no for the next child? In such a case they
would need to check type of the child if it is an array, which
would complicate code even more: each array update operation
would check child types; each do_op_##optype() would check field
type. Even if it is not enough, there are maps - they double the
complexity.
So this patch goes another way - lets each next level of update
will check if its field_no/map_key is already prepared by the
caller. And if not, extract a next token from the operation path.
Part of #1261
---
src/box/update/update_array.c | 42 +++++++++++++++++++++++++++++++++--
src/box/update/update_field.c | 17 ++++++++++++++
src/box/update/update_field.h | 25 +++++++++++++++++++--
3 files changed, 80 insertions(+), 4 deletions(-)
diff --git a/src/box/update/update_array.c b/src/box/update/update_array.c
index fe50a605a..262eceafb 100644
--- a/src/box/update/update_array.c
+++ b/src/box/update/update_array.c
@@ -32,6 +32,26 @@
#include "msgpuck.h"
#include "fiber.h"
+/**
+ * Make sure @a op contains a valid field number. It can be not
+ * so, if the array's parent didn't propagate operation's lexer.
+ * In fact, the parent fills fieldno only in some rare cases like
+ * branching. Generally, an array should care about fieldno by
+ * itself.
+ */
+static inline int
+update_op_prepare_num_token(struct update_op *op)
+{
+ if (op->token_type == JSON_TOKEN_END &&
+ update_op_consume_token(op) != 0)
+ return -1;
+ if (op->token_type != JSON_TOKEN_NUM) {
+ return update_err(op, "can't update an array by not a "\
+ "number index");
+ }
+ return 0;
+}
+
/**
* Make field number positive and check for the field existence.
* @param op Update operation.
@@ -43,6 +63,7 @@
static inline int
update_op_adjust_field_no(struct update_op *op, int32_t field_count)
{
+ assert(op->token_type == JSON_TOKEN_NUM);
if (op->field_no >= 0) {
if (op->field_no < field_count)
return 0;
@@ -217,10 +238,14 @@ do_op_array_insert(struct update_op *op, struct update_field *field)
assert(field->type == UPDATE_ARRAY);
struct rope *rope = field->array.rope;
struct update_array_item *item;
+ if (update_op_prepare_num_token(op) != 0)
+ return -1;
+
if (! update_op_is_term(op)) {
item = update_array_extract_item(field, op);
if (item == NULL)
return -1;
+ op->token_type = JSON_TOKEN_END;
return do_op_insert(op, &item->field);
}
@@ -241,6 +266,9 @@ do_op_array_set(struct update_op *op, struct update_field *field)
{
assert(field->type == UPDATE_ARRAY);
struct rope *rope = field->array.rope;
+ if (update_op_prepare_num_token(op) != 0)
+ return -1;
+
/* Interpret '=' for n + 1 field as insert. */
if (op->field_no == (int32_t) rope_size(rope))
return do_op_array_insert(op, field);
@@ -249,8 +277,10 @@ do_op_array_set(struct update_op *op, struct update_field *field)
update_array_extract_item(field, op);
if (item == NULL)
return -1;
- if (! update_op_is_term(op))
+ if (! update_op_is_term(op)) {
+ op->token_type = JSON_TOKEN_END;
return do_op_set(op, &item->field);
+ }
op->new_field_len = op->arg.set.length;
/* Ignore the previous op, if any. */
item->field.type = UPDATE_SCALAR;
@@ -262,11 +292,15 @@ int
do_op_array_delete(struct update_op *op, struct update_field *field)
{
assert(field->type == UPDATE_ARRAY);
+ if (update_op_prepare_num_token(op) != 0)
+ return -1;
+
if (! update_op_is_term(op)) {
struct update_array_item *item =
update_array_extract_item(field, op);
if (item == NULL)
return -1;
+ op->token_type = JSON_TOKEN_END;
return do_op_delete(op, &item->field);
}
struct rope *rope = field->array.rope;
@@ -285,12 +319,16 @@ do_op_array_delete(struct update_op *op, struct update_field *field)
int \
do_op_array_##op_type(struct update_op *op, struct update_field *field) \
{ \
+ if (update_op_prepare_num_token(op) != 0) \
+ return -1; \
struct update_array_item *item = \
update_array_extract_item(field, op); \
if (item == NULL) \
return -1; \
- if (! update_op_is_term(op)) \
+ if (! update_op_is_term(op)) { \
+ op->token_type = JSON_TOKEN_END; \
return do_op_##op_type(op, &item->field); \
+ } \
if (item->field.type != UPDATE_NOP) \
return update_err_double(op); \
if (update_op_do_##op_type(op, item->field.data) != 0) \
diff --git a/src/box/update/update_field.c b/src/box/update/update_field.c
index 820dc427b..c3270b825 100644
--- a/src/box/update/update_field.c
+++ b/src/box/update/update_field.c
@@ -518,6 +518,22 @@ update_op_by(char opcode)
}
}
+int
+update_op_consume_token(struct update_op *op)
+{
+ struct json_token token;
+ int rc = json_lexer_next_token(&op->lexer, &token);
+ if (rc != 0)
+ return update_err_bad_json(op, rc);
+ if (token.type == JSON_TOKEN_END)
+ return update_err_no_such_field(op);
+ op->token_type = token.type;
+ op->key = token.str;
+ op->key_len = token.len;
+ op->field_no = token.num;
+ return 0;
+}
+
int
update_op_decode(struct update_op *op, int index_base,
struct tuple_dictionary *dict, const char **expr)
@@ -546,6 +562,7 @@ update_op_decode(struct update_op *op, int index_base,
diag_set(ClientError, ER_UNKNOWN_UPDATE_OP);
return -1;
}
+ op->token_type = JSON_TOKEN_NUM;
int32_t field_no;
switch(mp_typeof(**expr)) {
case MP_INT:
diff --git a/src/box/update/update_field.h b/src/box/update/update_field.h
index fc4b1053f..784cb3261 100644
--- a/src/box/update/update_field.h
+++ b/src/box/update/update_field.h
@@ -172,8 +172,18 @@ struct update_op {
const struct update_op_meta *meta;
/** Operation arguments. */
union update_op_arg arg;
- /** First level field no. */
- int32_t field_no;
+ /**
+ * Current level token. END means that it is invalid and
+ * a next token should be extracted from the lexer.
+ */
+ enum json_token_type token_type;
+ union {
+ struct {
+ const char *key;
+ uint32_t key_len;
+ };
+ int32_t field_no;
+ };
/** Size of a new field after it is updated. */
uint32_t new_field_len;
/** Opcode symbol: = + - / ... */
@@ -186,6 +196,17 @@ struct update_op {
struct json_lexer lexer;
};
+/**
+ * Extract a next token from the operation path lexer. The result
+ * is used to decide to which child of a current map/array the
+ * operation should be forwarded. It is not just a synonym to
+ * json_lexer_next_token, because fills some fields of @a op,
+ * and should be used only to chose a next child inside a current
+ * map/array.
+ */
+int
+update_op_consume_token(struct update_op *op);
+
/**
* Decode an update operation from MessagePack.
* @param[out] op Update operation.
--
2.20.1 (Apple Git-117)
^ permalink raw reply [flat|nested] 15+ messages in thread* [tarantool-patches] [PATCH 12/13] tuple: JSON updates support intersection by arrays
2019-08-12 23:05 [tarantool-patches] [PATCH 00/13] JSON updates Vladislav Shpilevoy
` (2 preceding siblings ...)
2019-08-12 23:05 ` [tarantool-patches] [PATCH 11/13] tuple: make update operation tokens consumable Vladislav Shpilevoy
@ 2019-08-12 23:05 ` Vladislav Shpilevoy
2019-08-12 23:05 ` [tarantool-patches] [PATCH 13/13] tuple: JSON updates support intersection by maps Vladislav Shpilevoy
` (9 subsequent siblings)
13 siblings, 0 replies; 15+ messages in thread
From: Vladislav Shpilevoy @ 2019-08-12 23:05 UTC (permalink / raw)
To: tarantool-patches
Before the patch only not intersected JSON updates were supported
as the simplest and fastest to implement. This patch allows paths
to intersect. They can have a common JSON prefix, but difference
should start from an array index.
For example, this is allowed:
[1][2][3].a.b.c = 20
[1][2][4].e.f.g = 30
This is not allowed yet:
[1][2][3].a.b.c = 20
[1][2][3].a.e.f = 30
Here first difference is 'b' vs 'e' - intersection by map
keys.
Architecture of the solution is similar to bar updates: a new
update tree node type is added: UPDATE_ROUTE. When several
update operations have the same prefix, this prefix becomes an
UPDATE_ROUTE tree field. It stores the prefix and a subtree with
these operations.
Bar and route update nodes can branch and produce more bars and
routes, when new operations come.
Part of #1261
---
src/box/CMakeLists.txt | 1 +
src/box/update/update_array.c | 49 +++++
src/box/update/update_bar.c | 9 +-
src/box/update/update_field.c | 4 +
src/box/update/update_field.h | 93 ++++++++++
src/box/update/update_route.c | 334 ++++++++++++++++++++++++++++++++++
test/box/update.result | 177 ++++++++++++++++++
test/box/update.test.lua | 91 +++++++++
8 files changed, 753 insertions(+), 5 deletions(-)
create mode 100644 src/box/update/update_route.c
diff --git a/src/box/CMakeLists.txt b/src/box/CMakeLists.txt
index 94775daba..06f0629cb 100644
--- a/src/box/CMakeLists.txt
+++ b/src/box/CMakeLists.txt
@@ -44,6 +44,7 @@ add_library(tuple STATIC
update/update_field.c
update/update_array.c
update/update_bar.c
+ update/update_route.c
tuple_compare.cc
tuple_extract_key.cc
tuple_hash.cc
diff --git a/src/box/update/update_array.c b/src/box/update/update_array.c
index 262eceafb..2e354ed42 100644
--- a/src/box/update/update_array.c
+++ b/src/box/update/update_array.c
@@ -190,6 +190,55 @@ update_array_create(struct update_field *field, const char *header,
return rope_append(field->array.rope, item, field_count);
}
+int
+update_array_create_with_child(struct update_field *field,
+ const struct update_field *child,
+ int32_t field_no, const char *header)
+{
+ const char *data = header;
+ uint32_t field_count = mp_decode_array(&data);
+ const char *first_field = data;
+ const char *first_field_end = first_field;
+ mp_next(&first_field_end);
+ struct region *region = &fiber()->gc;
+ struct rope *rope = rope_new(region);
+ if (rope == NULL)
+ return -1;
+ struct update_array_item *item =
+ (struct update_array_item *) rope_alloc(region, sizeof(*item));
+ if (item == NULL)
+ return -1;
+ const char *end = first_field_end;
+ if (field_no > 0) {
+ for (int32_t i = 1; i < field_no; ++i)
+ mp_next(&end);
+ update_array_item_create(item, UPDATE_NOP, first_field,
+ first_field_end - first_field,
+ end - first_field_end);
+ if (rope_append(rope, item, field_no) != 0)
+ return -1;
+ item = (struct update_array_item *) rope_alloc(region,
+ sizeof(*item));
+ if (item == NULL)
+ return -1;
+ first_field = end;
+ first_field_end = first_field;
+ mp_next(&first_field_end);
+ end = first_field_end;
+ }
+ for (uint32_t i = field_no + 1; i < field_count; ++i)
+ mp_next(&end);
+ item->field = *child;
+ update_array_item_create(item, child->type, first_field,
+ first_field_end - first_field,
+ end - first_field_end);
+ field->type = UPDATE_ARRAY;
+ field->data = header;
+ field->size = end - header;
+ field->array.rope = rope;
+ return rope_append(rope, item, field_count - field_no);
+}
+
uint32_t
update_array_sizeof(struct update_field *field)
{
diff --git a/src/box/update/update_bar.c b/src/box/update/update_bar.c
index f4fc00716..c8e28347e 100644
--- a/src/box/update/update_bar.c
+++ b/src/box/update/update_bar.c
@@ -250,12 +250,11 @@ do_op_nop_delete(struct update_op *op, struct update_field *field)
int \
do_op_bar_##op_type(struct update_op *op, struct update_field *field) \
{ \
- (void) op; \
- (void) field; \
assert(field->type == UPDATE_BAR); \
- diag_set(ClientError, ER_UNSUPPORTED, "update", \
- "intersected JSON paths"); \
- return -1; \
+ field = update_route_branch(field, op); \
+ if (field == NULL) \
+ return -1; \
+ return do_op_##op_type(op, field); \
}
DO_SCALAR_OP_GENERIC(insert)
diff --git a/src/box/update/update_field.c b/src/box/update/update_field.c
index c3270b825..2c249a59d 100644
--- a/src/box/update/update_field.c
+++ b/src/box/update/update_field.c
@@ -107,6 +107,8 @@ update_field_sizeof(struct update_field *field)
return update_array_sizeof(field);
case UPDATE_BAR:
return update_bar_sizeof(field);
+ case UPDATE_ROUTE:
+ return update_route_sizeof(field);
default:
unreachable();
}
@@ -133,6 +135,8 @@ update_field_store(struct update_field *field, char *out, char *out_end)
return update_array_store(field, out, out_end);
case UPDATE_BAR:
return update_bar_store(field, out, out_end);
+ case UPDATE_ROUTE:
+ return update_route_store(field, out, out_end);
default:
unreachable();
}
diff --git a/src/box/update/update_field.h b/src/box/update/update_field.h
index 784cb3261..fc748f741 100644
--- a/src/box/update/update_field.h
+++ b/src/box/update/update_field.h
@@ -263,6 +263,22 @@ enum update_type {
* nodes. And this is the most common case.
*/
UPDATE_BAR,
+ /**
+ * Field with a subtree of updates having the same prefix,
+ * stored here explicitly. New updates with the same
+ * prefix just follow it without decoding of JSON nor
+ * MessagePack. It can be quite helpful when an update
+ * works with the same internal object via several
+ * operations like this:
+ *
+ * [1][2].a.b.c[3].key1 = 20
+ * [1][2].a.b.c[3].key2 = 30
+ * [1][2].a.b.c[3].key3 = true
+ *
+ * Here [1][2].a.b.c[3] is stored only once as a route
+ * with a subtree 'key1 = 20', 'key2 = 30', 'key3 = true'.
+ */
+ UPDATE_ROUTE,
};
/**
@@ -340,6 +356,17 @@ struct update_field {
};
};
} bar;
+ /** Route update - path to an update subtree. */
+ struct {
+ /**
+ * Common prefix of all updates in the
+ * subtree.
+ */
+ const char *path;
+ int path_len;
+ /** Update subtree. */
+ struct update_field *next_hop;
+ } route;
};
};
@@ -405,6 +432,31 @@ update_array_create(struct update_field *field, const char *header,
const char *data, const char *data_end,
uint32_t field_count);
+/**
+ * The same as mere create, but the array is created with a first
+ * child already. It allows to make it more effective, because
+ * under the hood a rope is created not as a one huge range which
+ * is then spit in parts, but as two rope nodes from the
+ * beginning. On the summary: -1 rope node split, -1 decoding of
+ * fields to @a field_no.
+ *
+ * The function is used during branching, where there was an
+ * existing update, but another one came with the same prefix, and
+ * a different suffix.
+ *
+ * @param[out] field Field to initialize.
+ * @param child A child subtree.
+ * @param field_no Field number of @a child.
+ * @param header Beginning of the array, MessagePack header.
+ *
+ * @retval 0 Success.
+ * @retval -1 Error.
+ */
+int
+update_array_create_with_child(struct update_field *field,
+ const struct update_field *child,
+ int32_t field_no, const char *header);
+
OP_DECL_GENERIC(array)
/* }}} update_field.array */
@@ -421,6 +473,34 @@ OP_DECL_GENERIC(nop)
/* }}} update_field.nop */
+/* {{{ update_field.route */
+
+/**
+ * Take a bar or a route @a field and split its path in the place
+ * where @a new_op should be applied. Prefix becomes a new route,
+ * suffix becomes a child of the result field. In the result @a
+ * field stays root of its subtree, and a node of that subtree
+ * is returned, to which @a new_op should be applied.
+ *
+ * Note, this function does not apply @a new_op. It just finds to
+ * where it should be applied and does all preparations. It is
+ * done deliberately, because otherwise do_cb() virtual function
+ * of @a new_op would have been called here, since there is no
+ * context. But a caller always knows exactly if it was insert,
+ * set, arith, etc. And a caller can and does use more specific
+ * function like do_op_set, do_op_insert ...
+ *
+ * @param field Field to find where to apply @a new_op.
+ * @param new_op New operation to apply.
+ * @return A field to which @a new_op should be applied.
+ */
+struct update_field *
+update_route_branch(struct update_field *field, struct update_op *new_op);
+
+OP_DECL_GENERIC(route)
+
+/* }}} update_field.route */
+
#undef OP_DECL_GENERIC
/* {{{ Common helpers. */
@@ -432,6 +512,17 @@ OP_DECL_GENERIC(nop)
* Each child can be another array, a bar, a route, a map -
* anything. The functions below help to make such places shorter
* and simpler.
+ *
+ * Note, that they are recursive, although it is not clearly
+ * visible. For example, if an update tree contains several array
+ * nodes on one tree branch, then update of the deepest array goes
+ * through each of these nodes and calls do_op_array_<opname> on
+ * needed children. But it is ok, because operation count is
+ * usually small (<<50 in the most cases, usually <= 5), and the
+ * update tree hight is not bigger than operation count. Also,
+ * fiber stack is big enough to fit ~10k update tree depth -
+ * incredible number, even though the real limit is 4k due to
+ * limited number of operations.
*/
#define OP_DECL_GENERIC(op_type) \
static inline int \
@@ -444,6 +535,8 @@ do_op_##op_type(struct update_op *op, struct update_field *field) \
return do_op_nop_##op_type(op, field); \
case UPDATE_BAR: \
return do_op_bar_##op_type(op, field); \
+ case UPDATE_ROUTE: \
+ return do_op_route_##op_type(op, field); \
default: \
unreachable(); \
} \
diff --git a/src/box/update/update_route.c b/src/box/update/update_route.c
new file mode 100644
index 000000000..f1d2ced65
--- /dev/null
+++ b/src/box/update/update_route.c
@@ -0,0 +1,334 @@
+/*
+ * Copyright 2010-2019, Tarantool AUTHORS, please see AUTHORS file.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the
+ * following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY <COPYRIGHT HOLDER> ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+ * <COPYRIGHT HOLDER> OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+ * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+#include "update_field.h"
+#include "fiber.h"
+#include "box/tuple.h"
+
+/**
+ * Do the actual branch. This is the case when an existing
+ * bar/route path is different from a new operation's path in an
+ * array.
+ * @param[out] next_hop A field which will be initialized as an
+ * array, and which will be a point to apply a new
+ * operation.
+ * @param child Current field from which the branch happens. It
+ * already contains an old update subtree.
+ * @param parent MessagePack array to be taken by @a next_hop.
+ */
+static int
+update_route_branch_array(struct update_field *next_hop,
+ const struct update_field *child, int32_t field_no,
+ const char *parent)
+{
+ struct update_op *op = child->bar.op;
+ /*
+ * There are rules when a subtree can be just copied as is
+ * from one parent to another.
+ * 1) It should not be a leaf. If it is not leaf, then it
+ * does not change header and other fields of this
+ * particular array, and can be safely copied into one
+ * of its fields. Otherwise check next rules.
+ * 2) It should not be a bar. Because if it is not a bar,
+ * and is a leaf, then it is scalar. Again, changes
+ * only its own field, does not affect anything else,
+ * can be copied. Otherwise go down.
+ * 3) Ok, it is a bar, a leaf. It is a bar with zero path
+ * length. In this case it should be scalar bar, it is
+ * checked below. The only non-scalar operations are
+ * '!' and '#'.
+ *
+ * Why '#' and '!' can't be copied? '!', for example,
+ * being applied to a field [1], affects all fields [2-*],
+ * and the array header. The same but in an even worse
+ * form about '#'. Such operations should be redone. They
+ * affect many fields and the parent.
+ *
+ * There is a tricky thing though - why not to just redo
+ * all operations here, for the sake of code simplicity?
+ * It would allow to remove 'create_with_child' crutch.
+ * The answer is - it is not possible. If a field is
+ * copyable, it is not re-applicable. And vice-versa. For
+ * example, if it is not a leaf, then there may be many
+ * operations, not one. A subtree just can't be
+ * 're-applied'.
+ *
+ * If the operation is scalar and a leaf, then its result
+ * has already overridden its arguments. This is because
+ * scalar operations save result into the arguments, to
+ * save memory. A second operation appliance would lead
+ * to very surprising results.
+ *
+ * Another reason - performance. This path should be
+ * quite hot, and to copy a struct is for sure much faster
+ * than to reapply an operation using a virtual function.
+ * Operations '!' and '#' are quite rare, so their
+ * optimization is not the goal.
+ */
+ if (child->type != UPDATE_BAR || child->bar.path_len > 0 ||
+ (op->opcode != '!' && op->opcode != '#')) {
+ return update_array_create_with_child(next_hop, child,
+ field_no, parent);
+ }
+ op->token_type = JSON_TOKEN_NUM;
+ op->field_no = field_no;
+ const char *data = parent;
+ uint32_t field_count = mp_decode_array(&data);
+ const char *end = data;
+ for (uint32_t i = 0; i < field_count; ++i)
+ mp_next(&end);
+ if (update_array_create(next_hop, parent, data, end, field_count) != 0)
+ return -1;
+ return op->meta->do_cb(op, next_hop);
+}
+
+struct update_field *
+update_route_branch(struct update_field *field, struct update_op *new_op)
+{
+ assert(new_op->lexer.src != NULL);
+ const char *old_path;
+ int old_path_len;
+ if (field->type == UPDATE_BAR) {
+ old_path = field->bar.path;
+ old_path_len = field->bar.path_len;
+ } else {
+ assert(field->type == UPDATE_ROUTE);
+ old_path = field->route.path;
+ old_path_len = field->route.path_len;
+ }
+ assert(old_path != NULL);
+ struct json_lexer old_path_lexer;
+ struct json_token old_token, new_token;
+ /*
+ * Offset is going to be used as a length of the route
+ * node created as a parent of the old subtree and the
+ * new operation. As it is described in the route
+ * definition in struct update_field - route is the common
+ * prefix of all operations of the subtree. Here its
+ * length is calculated.
+ * In addition, it is used here to determine if the new
+ * operation is different from the current subtree from
+ * the very beginning. Basically, it means the offset is
+ * 0, and no route is generated. Root becomes a regular
+ * update field (array, map), not a route. Example:
+ * @a field is a bar '[1].a.b = 20', @a new_op is
+ * '[2].c.d = 30'. In this case the paths are already
+ * different from the beginning, no common prefix = no
+ * route. Array with children [1].a.b and [2].c.d becomes
+ * a root.
+ */
+ int saved_old_offset;
+ json_lexer_create(&old_path_lexer, old_path, old_path_len,
+ TUPLE_INDEX_BASE);
+ const char *parent = field->data;
+ int rc;
+ do {
+ saved_old_offset = old_path_lexer.offset;
+ rc = json_lexer_next_token(&old_path_lexer, &old_token);
+ /* Old path is already validated. */
+ assert(rc == 0);
+ rc = json_lexer_next_token(&new_op->lexer, &new_token);
+ if (rc != 0) {
+ update_err_bad_json(new_op, rc);
+ return NULL;
+ }
+ if (json_token_cmp(&old_token, &new_token) != 0)
+ break;
+ switch(new_token.type) {
+ case JSON_TOKEN_NUM:
+ rc = tuple_field_go_to_index(&parent, new_token.num);
+ break;
+ case JSON_TOKEN_STR:
+ rc = tuple_field_go_to_key(&parent, new_token.str,
+ new_token.len);
+ break;
+ default:
+ /*
+ * Can't be ANY, because old and new
+ * tokens are equal, but '*' is invalid in
+ * paths and old was already checked for
+ * that.
+ */
+ assert(new_token.type == JSON_TOKEN_END);
+ update_err_double(new_op);
+ return NULL;
+ }
+ /*
+ * Must always find a field, because old
+ * token already went that path.
+ */
+ assert(rc == 0);
+ } while (true);
+ enum mp_type type = mp_typeof(*parent);
+ /*
+ * The paths are different from the very beginning. It
+ * means, that the old field should be transformed instead
+ * of creating a new route node.
+ */
+ bool transform_root = saved_old_offset == 0;
+ struct update_field *next_hop;
+ if (! transform_root) {
+ next_hop = (struct update_field *)
+ region_alloc(&fiber()->gc, sizeof(*next_hop));
+ if (next_hop == NULL) {
+ diag_set(OutOfMemory, sizeof(*next_hop), "region_alloc",
+ "next_hop");
+ return NULL;
+ }
+ } else {
+ next_hop = field;
+ }
+ if (type == MP_MAP) {
+ diag_set(ClientError, ER_UNSUPPORTED, "update",
+ "path intersection on map");
+ return NULL;
+ }
+ if (new_token.type != JSON_TOKEN_NUM) {
+ update_err(new_op, "can not update array by non-integer index");
+ return NULL;
+ }
+ if (type != MP_ARRAY) {
+ update_err_no_such_field(new_op);
+ return NULL;
+ }
+
+ int path_offset = old_path_lexer.offset;
+ struct update_field child = *field;
+ if (child.type == UPDATE_ROUTE) {
+ child.route.path += path_offset;
+ child.route.path_len -= path_offset;
+ if (child.route.path_len == 0)
+ child = *child.route.next_hop;
+ } else {
+ assert(child.type == UPDATE_BAR);
+ child.bar.path += path_offset;
+ child.bar.path_len -= path_offset;
+ /*
+ * Yeah, bar length can become 0 here, but it is
+ * ok as long as it is not a non-scalar operation
+ * like '!' or '#'. In these cases it changes the
+ * parent too. When a bar is scalar ('=', arith,
+ * splice, ...), it operates on one concrete field
+ * and works even if its path len is 0. Of course,
+ * it is not a big deal to fix such bar here
+ * making it scalar, but it would be additional
+ * branching on an actually hot path.
+ * Talking of '#' and '!' - they are handled by
+ * array and map 'branchers' internally, below.
+ */
+ }
+
+ new_op->token_type = JSON_TOKEN_NUM;
+ new_op->field_no = new_token.num;
+ if (update_route_branch_array(next_hop, &child, old_token.num,
+ parent) != 0)
+ return NULL;
+ if (! transform_root) {
+ field->type = UPDATE_ROUTE;
+ field->route.path = old_path;
+ field->route.path_len = saved_old_offset;
+ field->route.next_hop = next_hop;
+ }
+ return next_hop;
+}
+
+/**
+ * Obtain a next node of the update tree to which @a op should be
+ * propagated. It is the same as branch, but has a fast path in
+ * case of @a field is route, and operation prefix matches this
+ * route - then no need to parse JSON and dive into MessagePack,
+ * the route is just followed, via a lexer offset increase.
+ *
+ * @param field Field to find where to apply @a op.
+ * @param op New operation to apply.
+ * @return A field to which @a op should be applied.
+ */
+static struct update_field *
+update_route_next(struct update_field *field, struct update_op *op)
+{
+ assert(field->type == UPDATE_ROUTE);
+ assert(! update_op_is_term(op));
+ const char *new_path = op->lexer.src + op->lexer.offset;
+ int new_path_len = op->lexer.src_len - op->lexer.offset;
+ if (field->route.path_len <= new_path_len &&
+ memcmp(field->route.path, new_path, field->route.path_len) == 0) {
+ /*
+ * Fast path: jump to the next hop with no
+ * decoding. Is used, when several JSON updates
+ * have the same prefix.
+ */
+ op->lexer.offset += field->route.path_len;
+ return field->route.next_hop;
+ }
+ return update_route_branch(field, op);
+}
+
+#define DO_SCALAR_OP_GENERIC(op_type) \
+int \
+do_op_route_##op_type(struct update_op *op, struct update_field *field) \
+{ \
+ assert(field->type == UPDATE_ROUTE); \
+ struct update_field *next_hop = update_route_next(field, op); \
+ if (next_hop == NULL) \
+ return -1; \
+ return do_op_##op_type(op, next_hop); \
+}
+
+DO_SCALAR_OP_GENERIC(set)
+
+DO_SCALAR_OP_GENERIC(insert)
+
+DO_SCALAR_OP_GENERIC(delete)
+
+DO_SCALAR_OP_GENERIC(arith)
+
+DO_SCALAR_OP_GENERIC(bit)
+
+DO_SCALAR_OP_GENERIC(splice)
+
+uint32_t
+update_route_sizeof(struct update_field *field)
+{
+ return field->size - field->route.next_hop->size +
+ update_field_sizeof(field->route.next_hop);
+}
+
+uint32_t
+update_route_store(struct update_field *field, char *out, char *out_end)
+{
+ char *saved_out = out;
+ int before_hop = field->route.next_hop->data - field->data;
+ memcpy(out, field->data, before_hop);
+ out += before_hop;
+ out += update_field_store(field->route.next_hop, out, out_end);
+ int after_hop = before_hop + field->route.next_hop->size;
+ memcpy(out, field->data + after_hop, field->size - after_hop);
+ return out + field->size - after_hop - saved_out;
+}
diff --git a/test/box/update.result b/test/box/update.result
index ec882d20d..748bcf14a 100644
--- a/test/box/update.result
+++ b/test/box/update.result
@@ -1254,6 +1254,183 @@ vy_s:select()
vy_s:drop()
---
...
+--
+-- Complex updates. Intersected paths, different types.
+--
+--
+-- =
+--
+-- Difference in the last part.
+t:update({{'=', '[3][3][1][1]', 325}, {'=', '[3][3][1][2]', 375}})
+---
+- [1, {'b': 200, 'm': true, 'a': 100, 'c': {'d': 400, 'f': [4, 5, 6, 7, 8], 'e': 500,
+ 'g': {'k': 600, 'l': 700}}, 'g': [800, 900]}, [100, 200, [[325, 375], [400,
+ 450]], {'a': 500, 'b': 600}, {'c': 700, 'd': 800}]]
+...
+-- Different starting from the non-last part.
+t:update({{'=', '[3][3][1][1]', 325}, {'=', '[3][3][2][2]', 475}})
+---
+- [1, {'b': 200, 'm': true, 'a': 100, 'c': {'d': 400, 'f': [4, 5, 6, 7, 8], 'e': 500,
+ 'g': {'k': 600, 'l': 700}}, 'g': [800, 900]}, [100, 200, [[325, 350], [400,
+ 475]], {'a': 500, 'b': 600}, {'c': 700, 'd': 800}]]
+...
+-- Contains string keys in the middle.
+t:update({{'=', '[2].c.f[1]', 4000}, {'=', '[2].c.f[2]', 5000}})
+---
+- [1, {'b': 200, 'm': true, 'a': 100, 'c': {'d': 400, 'f': [4000, 5000, 6, 7, 8],
+ 'e': 500, 'g': {'k': 600, 'l': 700}}, 'g': [800, 900]}, [100, 200, [[300, 350],
+ [400, 450]], {'a': 500, 'b': 600}, {'c': 700, 'd': 800}]]
+...
+t = {1}
+---
+...
+t[2] = setmetatable({}, {__serialize = 'map'})
+---
+...
+t[3] = {}
+---
+...
+t[4] = { \
+ 1, \
+ 2, \
+ 3, \
+ { \
+ 4, \
+ 5, \
+ 6, \
+ 7, \
+ { \
+ 8, \
+ 9, \
+ {10, 11, 12}, \
+ 13, \
+ 14, \
+ }, \
+ 15, \
+ 16, \
+ {17, 18, 19} \
+ }, \
+ 20, \
+ { \
+ a = 21, \
+ b = 22, \
+ c = { \
+ d = 23, \
+ e = 24 \
+ } \
+ }, \
+ 25 \
+}
+---
+...
+t = s:replace(t)
+---
+...
+-- Non-first and non-last field updates.
+t:update({{'=', '[4][4][5][3][2]', 11000}, {'=', '[4][4][8][3]', 19000}})
+---
+- [1, {}, [], [1, 2, 3, [4, 5, 6, 7, [8, 9, [10, 11000, 12], 13, 14], 15, 16, [17,
+ 18, 19000]], 20, {'b': 22, 'a': 21, 'c': {'d': 23, 'e': 24}}, 25]]
+...
+-- Triple intersection with enabled fast jump by route.
+t:update({{'=', '[4][4][3]', 6000}, {'=', '[4][4][5][2]', 9000}, {'=', '[4][4][8][2]', 18000}})
+---
+- [1, {}, [], [1, 2, 3, [4, 5, 6000, 7, [8, 9000, [10, 11, 12], 13, 14], 15, 16, [
+ 17, 18000, 19]], 20, {'b': 22, 'a': 21, 'c': {'d': 23, 'e': 24}}, 25]]
+...
+-- Triple intersection with no optimizations.
+t:update({{'=', '[4][3]', 3000}, {'=', '[4][4][2]', 5000}, {'=', '[4][4][5][1]', 8000}})
+---
+- [1, {}, [], [1, 2, 3000, [4, 5000, 6, 7, [8000, 9, [10, 11, 12], 13, 14], 15, 16,
+ [17, 18, 19]], 20, {'b': 22, 'a': 21, 'c': {'d': 23, 'e': 24}}, 25]]
+...
+-- Last operation splits previous ones.
+t:update({{'=', '[4][4][5][3][2]', 11000}, {'=', '[4][4][5][3][1]', 10000}, {'=', '[4][4][3]', 6000}})
+---
+- [1, {}, [], [1, 2, 3, [4, 5, 6000, 7, [8, 9, [10000, 11000, 12], 13, 14], 15, 16,
+ [17, 18, 19]], 20, {'b': 22, 'a': 21, 'c': {'d': 23, 'e': 24}}, 25]]
+...
+-- First two operations generate a route '[4]' in the 4th field of
+-- the tuple. With childs [1] and [2]. The last operation should
+-- delete the route and replace it with a full featured array.
+t:update({{'=', '[4][4][1]', 4.5}, {'=', '[4][4][2]', 5.5}, {'=', '[4][3]', 3.5}})
+---
+- [1, {}, [], [1, 2, 3.5, [4.5, 5.5, 6, 7, [8, 9, [10, 11, 12], 13, 14], 15, 16, [
+ 17, 18, 19]], 20, {'b': 22, 'a': 21, 'c': {'d': 23, 'e': 24}}, 25]]
+...
+-- Test errors.
+t:update({{'=', '[4][4][5][1]', 8000}, {'=', '[4][4][5].a', -1}})
+---
+- error: 'Field ''[4][4][5].a'' UPDATE error: can not update array by non-integer
+ index'
+...
+t:update({{'=', '[4][4][5][1]', 8000}, {'=', '[4][4][*].a', -1}})
+---
+- error: 'Field ''[4][4][*].a'' UPDATE error: can not update array by non-integer
+ index'
+...
+t:update({{'=', '[4][4][*].b', 8000}, {'=', '[4][4][*].a', -1}})
+---
+- error: 'Field ''[4][4][*].b'' UPDATE error: invalid JSON in position 8'
+...
+t:update({{'=', '[4][4][1]', 4000}, {'=', '[4][4]-badjson', -1}})
+---
+- error: 'Field ''[4][4]-badjson'' UPDATE error: invalid JSON in position 7'
+...
+t:update({{'=', '[4][4][1]', 1}, {'=', '[4][4][1]', 2}})
+---
+- error: 'Field ''[4][4][1]'' UPDATE error: double update of the same field'
+...
+-- First two operations produce zero length bar update in field
+-- [4][4][5][4]. The third operation should raise an error.
+t:update({{'+', '[4][4][5][4]', 13000}, {'=', '[4][4][5][1]', 8000}, {'+', '[4][4][5][4]', 1}})
+---
+- error: 'Field ''[4][4][5][4]'' UPDATE error: double update of the same field'
+...
+--
+-- !
+--
+t:update({{'!', '[4][5]', 19.5}, {'!', '[4][6]', 19.75}, {'!', '[4][4][3]', 5.5}, {'=', '[4][4][2]', 5.25}})
+---
+- [1, {}, [], [1, 2, 3, [4, 5.25, 5.5, 6, 7, [8, 9, [10, 11, 12], 13, 14], 15, 16,
+ [17, 18, 19]], 19.5, 19.75, 20, {'b': 22, 'a': 21, 'c': {'d': 23, 'e': 24}},
+ 25]]
+...
+--
+-- #
+--
+t:update({{'#', '[4][4][5]', 1}, {'#', '[4][4][4]', 1}, {'#', '[4][4][5]', 1}})
+---
+- [1, {}, [], [1, 2, 3, [4, 5, 6, 15, [17, 18, 19]], 20, {'b': 22, 'a': 21, 'c': {
+ 'd': 23, 'e': 24}}, 25]]
+...
+--
+-- Scalar operations.
+--
+t:update({{'+', '[4][3]', 0.5}, {'+', '[4][4][5][5]', 0.75}})
+---
+- [1, {}, [], [1, 2, 3.5, [4, 5, 6, 7, [8, 9, [10, 11, 12], 13, 14.75], 15, 16, [
+ 17, 18, 19]], 20, {'b': 22, 'a': 21, 'c': {'d': 23, 'e': 24}}, 25]]
+...
+t:update({{'+', '[4][3]', 0.5}, {'+', '[4][4][5][5]', 0.75}, {'+', '[4][4][3]', 0.25}})
+---
+- [1, {}, [], [1, 2, 3.5, [4, 5, 6.25, 7, [8, 9, [10, 11, 12], 13, 14.75], 15, 16,
+ [17, 18, 19]], 20, {'b': 22, 'a': 21, 'c': {'d': 23, 'e': 24}}, 25]]
+...
+-- Combined - check that operations following '!' take into
+-- account that there is a new element. Indexes should be relative
+-- to the new array size.
+t:update({{'!', '[4][3]', 2.5}, {'+', '[4][5][5][5]', 0.75}, {'+', '[4][5][3]', 0.25}})
+---
+- [1, {}, [], [1, 2, 2.5, 3, [4, 5, 6.25, 7, [8, 9, [10, 11, 12], 13, 14.75], 15,
+ 16, [17, 18, 19]], 20, {'b': 22, 'a': 21, 'c': {'d': 23, 'e': 24}}, 25]]
+...
+-- Error - an attempt to follow JSON path in a scalar field
+-- during branching.
+t:update({{'+', '[4][3]', 0.5}, {'+', '[4][3][2]', 0.5}})
+---
+- error: Field ''[4][3][2]'' was not found in the tuple
+...
s:drop()
---
...
diff --git a/test/box/update.test.lua b/test/box/update.test.lua
index 702c0dbd7..27347d66d 100644
--- a/test/box/update.test.lua
+++ b/test/box/update.test.lua
@@ -413,4 +413,95 @@ box.commit()
vy_s:select()
vy_s:drop()
+--
+-- Complex updates. Intersected paths, different types.
+--
+
+--
+-- =
+--
+-- Difference in the last part.
+t:update({{'=', '[3][3][1][1]', 325}, {'=', '[3][3][1][2]', 375}})
+-- Different starting from the non-last part.
+t:update({{'=', '[3][3][1][1]', 325}, {'=', '[3][3][2][2]', 475}})
+-- Contains string keys in the middle.
+t:update({{'=', '[2].c.f[1]', 4000}, {'=', '[2].c.f[2]', 5000}})
+t = {1}
+t[2] = setmetatable({}, {__serialize = 'map'})
+t[3] = {}
+t[4] = { \
+ 1, \
+ 2, \
+ 3, \
+ { \
+ 4, \
+ 5, \
+ 6, \
+ 7, \
+ { \
+ 8, \
+ 9, \
+ {10, 11, 12}, \
+ 13, \
+ 14, \
+ }, \
+ 15, \
+ 16, \
+ {17, 18, 19} \
+ }, \
+ 20, \
+ { \
+ a = 21, \
+ b = 22, \
+ c = { \
+ d = 23, \
+ e = 24 \
+ } \
+ }, \
+ 25 \
+}
+t = s:replace(t)
+-- Non-first and non-last field updates.
+t:update({{'=', '[4][4][5][3][2]', 11000}, {'=', '[4][4][8][3]', 19000}})
+-- Triple intersection with enabled fast jump by route.
+t:update({{'=', '[4][4][3]', 6000}, {'=', '[4][4][5][2]', 9000}, {'=', '[4][4][8][2]', 18000}})
+-- Triple intersection with no optimizations.
+t:update({{'=', '[4][3]', 3000}, {'=', '[4][4][2]', 5000}, {'=', '[4][4][5][1]', 8000}})
+-- Last operation splits previous ones.
+t:update({{'=', '[4][4][5][3][2]', 11000}, {'=', '[4][4][5][3][1]', 10000}, {'=', '[4][4][3]', 6000}})
+-- First two operations generate a route '[4]' in the 4th field of
+-- the tuple. With childs [1] and [2]. The last operation should
+-- delete the route and replace it with a full featured array.
+t:update({{'=', '[4][4][1]', 4.5}, {'=', '[4][4][2]', 5.5}, {'=', '[4][3]', 3.5}})
+-- Test errors.
+t:update({{'=', '[4][4][5][1]', 8000}, {'=', '[4][4][5].a', -1}})
+t:update({{'=', '[4][4][5][1]', 8000}, {'=', '[4][4][*].a', -1}})
+t:update({{'=', '[4][4][*].b', 8000}, {'=', '[4][4][*].a', -1}})
+t:update({{'=', '[4][4][1]', 4000}, {'=', '[4][4]-badjson', -1}})
+t:update({{'=', '[4][4][1]', 1}, {'=', '[4][4][1]', 2}})
+-- First two operations produce zero length bar update in field
+-- [4][4][5][4]. The third operation should raise an error.
+t:update({{'+', '[4][4][5][4]', 13000}, {'=', '[4][4][5][1]', 8000}, {'+', '[4][4][5][4]', 1}})
+
+--
+-- !
+--
+t:update({{'!', '[4][5]', 19.5}, {'!', '[4][6]', 19.75}, {'!', '[4][4][3]', 5.5}, {'=', '[4][4][2]', 5.25}})
+--
+-- #
+--
+t:update({{'#', '[4][4][5]', 1}, {'#', '[4][4][4]', 1}, {'#', '[4][4][5]', 1}})
+--
+-- Scalar operations.
+--
+t:update({{'+', '[4][3]', 0.5}, {'+', '[4][4][5][5]', 0.75}})
+t:update({{'+', '[4][3]', 0.5}, {'+', '[4][4][5][5]', 0.75}, {'+', '[4][4][3]', 0.25}})
+-- Combined - check that operations following '!' take into
+-- account that there is a new element. Indexes should be relative
+-- to the new array size.
+t:update({{'!', '[4][3]', 2.5}, {'+', '[4][5][5][5]', 0.75}, {'+', '[4][5][3]', 0.25}})
+-- Error - an attempt to follow JSON path in a scalar field
+-- during branching.
+t:update({{'+', '[4][3]', 0.5}, {'+', '[4][3][2]', 0.5}})
+
s:drop()
--
2.20.1 (Apple Git-117)
^ permalink raw reply [flat|nested] 15+ messages in thread* [tarantool-patches] [PATCH 13/13] tuple: JSON updates support intersection by maps
2019-08-12 23:05 [tarantool-patches] [PATCH 00/13] JSON updates Vladislav Shpilevoy
` (3 preceding siblings ...)
2019-08-12 23:05 ` [tarantool-patches] [PATCH 12/13] tuple: JSON updates support intersection by arrays Vladislav Shpilevoy
@ 2019-08-12 23:05 ` Vladislav Shpilevoy
2019-08-12 23:05 ` [tarantool-patches] [PATCH 02/13] rope: make rope library macro template Vladislav Shpilevoy
` (8 subsequent siblings)
13 siblings, 0 replies; 15+ messages in thread
From: Vladislav Shpilevoy @ 2019-08-12 23:05 UTC (permalink / raw)
To: tarantool-patches
Previous commits introduced JSON updates with any non-intersecting
JSON paths, and allowed intersection by arrays. This one completes
the puzzle, now any JSON updates are available.
Closes #1261
@TarantoolBot document
Title: JSON updates
Tuple/space/index:update/upsert now support JSON paths. All the
same paths as allowed in tuple["..."]. Including field names in
the first JSON path token.
Example:
box.cfg{}
format = {}
format[1] = {'field1', 'unsigned'}
format[2] = {'field2', 'map'}
format[3] = {'field3', 'array'}
format[4] = {'field4', 'string', is_nullable = true}
s = box.schema.create_space('test', {format = format})
_ = s:create_index('pk')
t = {
1,
{
key1 = 'value',
key2 = 10
},
{
2,
3,
{key3 = 20}
}
}
t = s:replace(t)
tarantool> t:update({{'=', 'field2.key1', 'new_value'}})
---
- [1, {'key1': 'new_value', 'key2': 10}, [2, 3, {'key3': 20}]]
...
tarantool> t:update({{'+', 'field3[2]', 1}})
---
- [1, {'key1': 'value', 'key2': 10}, [2, 4, {'key3': 20}]]
...
tarantool> s:update({1}, {{'!', 'field4', 'inserted value'}})
---
- [1, {'key1': 'value', 'key2': 10}, [2, 3, {'key3': 20}], 'inserted value']
...
tarantool> s:update({1}, {{'#', '[2].key2', 1}, {'=', '[3][3].key4', 'value4'}})
---
- [1, {'key1': 'value'}, [2, 3, {'key3': 20, 'key4': 'value4'}], 'inserted value']
...
tarantool> s:upsert({1, {k = 'v'}, {}}, {{'#', '[2].key1', 1}})
---
...
tarantool> s:select{}
---
- - [1, {}, [2, 3, {'key3': 20, 'key4': 'value4'}], 'inserted value']
...
Note, that there is the same rule, as in tuple field access by
JSON, for field names looking like JSON paths. The rule is that
firstly the whole path is interpreted as a field name. If such a
name does not exist, then it is treated as a path. For example,
if there is a field name 'field.name.like.json', then this update
<obj>:update({..., 'field.name.like.json', ...})
will update this field, instead of keys 'field' -> 'name' ->
'like' -> 'json'. If such a name is needed as a part of a bigger
path, then it should be wrapped in quotes and []:
<obj>:update({..., '["field.name.like.json"].next.fields', ...})
There are some new rules for JSON updates:
- Operation '!' can't be used to create all intermediate nodes of
a path. For example, {'!', 'field1[1].field3', ...} can't
create fields 'field1' and '[1]', they should exist.
- Operation '#', when applied to maps, can't delete more than one
key at once. That is, its argument should be always 1 for maps.
{'#', 'field1.field2', 1} - this is allowed;
{'#', 'field1.field2', 10} - this is not.
That limitation originates from a problem, that keys in a map
are not ordered anyhow, and '#' with more than 1 key would lead
to undefined behaviour.
- Operation '!' on maps can't create a key, if it exists already.
- If a map contains non-string keys (booleans, numbers, maps,
arrays - anything), then these keys can't be updated via JSON
paths. But it is still allowed to update string keys in such a
map.
Why JSON updates are good, and should be preferred when only a
part of a tuple needs to be updated?
- They consume less space in WAL, because for an update only its
keys, operations, and arguments are stored. It is cheaper to
store update of one deep field, than the whole tuple subtree.
- They are faster. Firstly, this is because they are implemented
in C, and have no problems with Lua GC and dynamic typing.
Secondly, some cases of JSON paths are highly optimized. For
example, an update with a single JSON path costs O(1) memory
regardless of how deep that path goes (not counting update
arguments).
- They are available from remote clients, as well as any other
DML. Before JSON updates to update one deep part of a tuple it
would be necessary to download that tuple, update it in memory,
send it back - 2 network hops. With JSON paths it can be 1 when
the update can be described in paths.
---
src/box/CMakeLists.txt | 1 +
src/box/update/update_field.c | 4 +
src/box/update/update_field.h | 84 ++++++-
src/box/update/update_map.c | 441 ++++++++++++++++++++++++++++++++++
src/box/update/update_route.c | 74 ++++--
test/box/update.result | 152 ++++++++++++
test/box/update.test.lua | 75 ++++++
7 files changed, 813 insertions(+), 18 deletions(-)
create mode 100644 src/box/update/update_map.c
diff --git a/src/box/CMakeLists.txt b/src/box/CMakeLists.txt
index 06f0629cb..505b19061 100644
--- a/src/box/CMakeLists.txt
+++ b/src/box/CMakeLists.txt
@@ -45,6 +45,7 @@ add_library(tuple STATIC
update/update_array.c
update/update_bar.c
update/update_route.c
+ update/update_map.c
tuple_compare.cc
tuple_extract_key.cc
tuple_hash.cc
diff --git a/src/box/update/update_field.c b/src/box/update/update_field.c
index 2c249a59d..592d02f9d 100644
--- a/src/box/update/update_field.c
+++ b/src/box/update/update_field.c
@@ -109,6 +109,8 @@ update_field_sizeof(struct update_field *field)
return update_bar_sizeof(field);
case UPDATE_ROUTE:
return update_route_sizeof(field);
+ case UPDATE_MAP:
+ return update_map_sizeof(field);
default:
unreachable();
}
@@ -137,6 +139,8 @@ update_field_store(struct update_field *field, char *out, char *out_end)
return update_bar_store(field, out, out_end);
case UPDATE_ROUTE:
return update_route_store(field, out, out_end);
+ case UPDATE_MAP:
+ return update_map_store(field, out, out_end);
default:
unreachable();
}
diff --git a/src/box/update/update_field.h b/src/box/update/update_field.h
index fc748f741..9f91fdccf 100644
--- a/src/box/update/update_field.h
+++ b/src/box/update/update_field.h
@@ -33,6 +33,7 @@
#include "trivia/util.h"
#include "tt_static.h"
#include <stdint.h>
+#include "salad/stailq.h"
#include "json/json.h"
#include "bit/int96.h"
@@ -242,7 +243,8 @@ enum update_type {
* for example, when a rope is split in two parts: not
* changed left range of fields, and a right range with
* its first field changed. In this case the left range is
- * NOP.
+ * NOP. And when a map is updated and split into rages,
+ * when only certain points are not NOP.
*/
UPDATE_NOP,
/**
@@ -279,6 +281,11 @@ enum update_type {
* with a subtree 'key1 = 20', 'key2 = 30', 'key3 = true'.
*/
UPDATE_ROUTE,
+ /**
+ * Field is an updated map. Check item list for updates of
+ * individual fields.
+ */
+ UPDATE_MAP,
};
/**
@@ -367,6 +374,48 @@ struct update_field {
/** Update subtree. */
struct update_field *next_hop;
} route;
+ /**
+ * The field is an updated map. Individual fields
+ * are stored very similar to array update and its
+ * rope nodes. Each item is a key, a value, and a
+ * tail of unchanged key-value pairs. The items
+ * are stored in a list. But the list is not
+ * sorted anyhow by keys, because it does not
+ * really make any sense:
+ *
+ * 1) Keys in MessagePack are not sorted anyway,
+ * and any kind of search would not be possible
+ * even if they were sorted. Sort of a map
+ * would require Nlog(N) time and N memory even
+ * if only a few fields were updated.
+ *
+ * 2) Double scalar update of the same key is not
+ * possible.
+ *
+ * Although there is something which can be and is
+ * optimized. When a key is updated first time,
+ * it is moved to the beginning of the list, and
+ * after all operations are done, it is stored
+ * in the result tuple before unchanged fields. On
+ * a second update of the same tuple it is found
+ * immediately.
+ */
+ struct {
+ /**
+ * List of map update items. Each item is
+ * a key, a value, and an unchanged tail.
+ */
+ struct stailq items;
+ /**
+ * Number of key-value pairs in the list.
+ * Note, it is not a number of items, but
+ * exactly number of key-value pairs. It
+ * is used to store MessagePack map header
+ * without decoding each item again just
+ * to learn the size.
+ */
+ int size;
+ } map;
};
};
@@ -461,6 +510,37 @@ OP_DECL_GENERIC(array)
/* }}} update_field.array */
+/* {{{ update_field.map */
+
+/**
+ * Initialize @a field as a map to update.
+ * @param[out] field Field to initialize.
+ * @param header Header of the MessagePack map @a data.
+ * @param data MessagePack data of the map to update.
+ * @param data_end End of @a data.
+ * @param field_count Key-value pair count in @data.
+ *
+ * @retval 0 Success.
+ * @retval -1 Error.
+ */
+int
+update_map_create(struct update_field *field, const char *header,
+ const char *data, const char *data_end, int field_count);
+
+/**
+ * Create a map update with an existing child. Motivation is
+ * exactly the same as with a similar array constructor. It allows
+ * to avoid unnecessary MessagePack decoding.
+ */
+int
+update_map_create_with_child(struct update_field *field,
+ const struct update_field *child, const char *key,
+ uint32_t key_len, const char *header);
+
+OP_DECL_GENERIC(map)
+
+/* }}} update_field.map */
+
/* {{{ update_field.bar */
OP_DECL_GENERIC(bar)
@@ -537,6 +617,8 @@ do_op_##op_type(struct update_op *op, struct update_field *field) \
return do_op_bar_##op_type(op, field); \
case UPDATE_ROUTE: \
return do_op_route_##op_type(op, field); \
+ case UPDATE_MAP: \
+ return do_op_map_##op_type(op, field); \
default: \
unreachable(); \
} \
diff --git a/src/box/update/update_map.c b/src/box/update/update_map.c
new file mode 100644
index 000000000..14838404e
--- /dev/null
+++ b/src/box/update/update_map.c
@@ -0,0 +1,441 @@
+/*
+ * Copyright 2010-2019, Tarantool AUTHORS, please see AUTHORS file.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the
+ * following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY <COPYRIGHT HOLDER> ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+ * <COPYRIGHT HOLDER> OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+ * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+#include "update_field.h"
+#include "fiber.h"
+#include "small/region.h"
+#include "box/tuple_format.h"
+
+/**
+ * Descriptor of one updated key-value pair. Besides updated data
+ * it contains a tail with unchanged pairs, just so as not to
+ * create a separate object for them, and to be similar with array
+ * update items.
+ */
+struct update_map_item {
+ /**
+ * Updated key. Can be NULL. In such a case this item
+ * contains only an unchanged tail. A key can be nullified
+ * if it was removed from the map, or when a map is just
+ * created and has no any update yet.
+ */
+ const char *key;
+ /** Length of @a key. */
+ uint32_t key_len;
+ /** Updated value. */
+ struct update_field field;
+ /** Link in the list of updated map keys. */
+ struct stailq_entry in_items;
+ /**
+ * Size in bytes of unchanged tail data. It goes right
+ * after @a field.data.
+ */
+ uint32_t tail_size;
+};
+
+static inline struct update_map_item *
+update_map_item_alloc(void)
+{
+ struct update_map_item *item = (struct update_map_item *)
+ region_alloc(&fiber()->gc, sizeof(*item));
+ if (item == NULL)
+ diag_set(OutOfMemory, sizeof(*item), "region_alloc", "item");
+ return item;
+}
+
+static void
+update_map_create_item(struct update_field *field, struct update_map_item *item,
+ enum update_type type, const char *key, uint32_t key_len,
+ const char *data, uint32_t data_size, uint32_t tail_size)
+{
+ assert(field->type == UPDATE_MAP);
+ item->key = key;
+ item->key_len = key_len;
+ item->field.type = type;
+ item->field.data = data;
+ item->field.size = data_size,
+ item->tail_size = tail_size;
+ /*
+ * Each time a new item it created it is stored in the
+ * head of update map item list. It helps in case the
+ * tuple is regularly updated, because on all next updates
+ * this key will be found from the very beginning of the
+ * map.
+ */
+ stailq_add_entry(&field->map.items, item, in_items);
+}
+
+static inline struct update_map_item *
+update_map_new_item(struct update_field *field, enum update_type type,
+ const char *key, uint32_t key_len, const char *data,
+ uint32_t data_size, uint32_t tail_size)
+{
+ struct update_map_item *item = update_map_item_alloc();
+ if (item != NULL) {
+ update_map_create_item(field, item, type, key, key_len, data,
+ data_size, tail_size);
+ }
+ return item;
+}
+
+/**
+ * Find an update item to which @a op should be applied. The
+ * target field may do not exist, but at least its parent should.
+ */
+static int
+update_map_extract_opt_item(struct update_field *field, struct update_op *op,
+ struct update_map_item **res)
+{
+ assert(field->type == UPDATE_MAP);
+ if (op->token_type == JSON_TOKEN_END) {
+ if (update_op_consume_token(op) != 0)
+ return -1;
+ if (op->token_type != JSON_TOKEN_STR) {
+ return update_err(op, "can't update a map by not a "\
+ "string key");
+ }
+ }
+ struct stailq *items = &field->map.items;
+ struct update_map_item *i, *new_item;
+ /*
+ * Firstly, try to find the key among already updated
+ * ones. Perhaps, the needed node is just an intermediate
+ * key of a bigger JSON path, and there are many updates
+ * passing this key, so it should be here for all except
+ * first updates.
+ */
+ stailq_foreach_entry(i, items, in_items) {
+ if (i->key != NULL && i->key_len == op->key_len &&
+ memcmp(i->key, op->key, i->key_len) == 0) {
+ *res = i;
+ return 0;
+ }
+ }
+ /*
+ * Slow path - the key is updated first time, need to
+ * decode tails.
+ */
+ uint32_t key_len, i_tail_size;
+ const char *pos, *key, *end, *tmp, *begin;
+ stailq_foreach_entry(i, items, in_items) {
+ begin = i->field.data + i->field.size;
+ pos = begin;
+ end = begin + i->tail_size;
+ i_tail_size = 0;
+ while(pos < end) {
+ if (mp_typeof(*pos) != MP_STR) {
+ mp_next(&pos);
+ mp_next(&pos);
+ continue;
+ }
+ i_tail_size = pos - begin;
+ key = mp_decode_str(&pos, &key_len);
+ if (key_len == op->key_len &&
+ memcmp(key, op->key, key_len) == 0)
+ goto key_is_found;
+ mp_next(&pos);
+ }
+ }
+ *res = NULL;
+ return 0;
+
+key_is_found:
+ tmp = pos;
+ mp_next(&tmp);
+ if (i_tail_size == 0 && i->key == NULL) {
+ /*
+ * Looks like the needed key was found from the
+ * beginning of tail of an item without a key.
+ * This is actually good, because this item can
+ * be transformed instead of a new item creation.
+ */
+ i->key = op->key;
+ i->key_len = op->key_len;
+ i->field.data = pos;
+ i->field.size = tmp - pos;
+ i->tail_size = end - tmp;
+ new_item = i;
+ } else {
+ new_item = update_map_new_item(field, UPDATE_NOP, op->key,
+ op->key_len, pos, tmp - pos,
+ end - tmp);
+ if (new_item == NULL)
+ return -1;
+ i->tail_size = i_tail_size;
+ }
+ *res = new_item;
+ return 0;
+}
+
+/**
+ * The same as optional extractor, but the field to update should
+ * exist. It is the case of all scalar operations (except '=' - it
+ * can work as insert).
+ */
+static inline struct update_map_item *
+update_map_extract_item(struct update_field *field, struct update_op *op)
+{
+ assert(field->type == UPDATE_MAP);
+ struct update_map_item *res;
+ if (update_map_extract_opt_item(field, op, &res) != 0)
+ return NULL;
+ if (res == NULL)
+ update_err_no_such_field(op);
+ return res;
+}
+
+int
+do_op_map_insert(struct update_op *op, struct update_field *field)
+{
+ assert(field->type == UPDATE_MAP);
+ struct update_map_item *item;
+ if (update_map_extract_opt_item(field, op, &item) != 0)
+ return -1;
+ if (! update_op_is_term(op)) {
+ if (item == NULL)
+ return update_err_no_such_field(op);
+ op->token_type = JSON_TOKEN_END;
+ return do_op_insert(op, &item->field);
+ }
+ if (item != NULL)
+ return update_err_duplicate(op);
+ ++field->map.size;
+ item = update_map_new_item(field, UPDATE_NOP, op->key, op->key_len,
+ op->arg.set.value, op->arg.set.length, 0);
+ return item != NULL ? 0 : -1;
+}
+
+int
+do_op_map_set(struct update_op *op, struct update_field *field)
+{
+ assert(field->type == UPDATE_MAP);
+ struct update_map_item *item;
+ if (update_map_extract_opt_item(field, op, &item) != 0)
+ return -1;
+ if (! update_op_is_term(op)) {
+ if (item == NULL)
+ return update_err_no_such_field(op);
+ op->token_type = JSON_TOKEN_END;
+ return do_op_set(op, &item->field);
+ }
+ if (item != NULL) {
+ op->new_field_len = op->arg.set.length;
+ /* Ignore the previous op, if any. */
+ item->field.type = UPDATE_SCALAR;
+ item->field.scalar.op = op;
+ return 0;
+ }
+ ++field->map.size;
+ item = update_map_new_item(field, UPDATE_NOP, op->key, op->key_len,
+ op->arg.set.value, op->arg.set.length, 0);
+ return item != NULL ? 0 : -1;
+}
+
+int
+do_op_map_delete(struct update_op *op, struct update_field *field)
+{
+ assert(field->type == UPDATE_MAP);
+ struct update_map_item *item;
+ if (update_map_extract_opt_item(field, op, &item) != 0)
+ return -1;
+ if (! update_op_is_term(op)) {
+ if (item == NULL)
+ return update_err_no_such_field(op);
+ op->token_type = JSON_TOKEN_END;
+ return do_op_delete(op, &item->field);
+ }
+ if (op->arg.del.count != 1)
+ return update_err_delete1(op);
+ if (item == NULL)
+ return 0;
+ /*
+ * Note, even if tail size becomes 0, this item is not
+ * deleted. This is because items are linked via stailq,
+ * elements of which can't be deleted as simple as that.
+ * But it is not a big deal, because '#' is a really rare
+ * operation.
+ * Why just a next key from the tail can't be decoded?
+ * Why key should be NULL here? This is because the JSON
+ * updates allow to update a map containing non-string
+ * keys. If the next key is not a string, it can't be
+ * saved as key of the item. Therefore, it is better not
+ * to touch unchanged tails unless a new update operation
+ * needs it.
+ */
+ item->key = NULL;
+ item->key_len = 0;
+ item->field.data += item->field.size;
+ item->field.size = 0;
+ item->field.type = UPDATE_NOP;
+ --field->map.size;
+ return 0;
+}
+
+#define DO_SCALAR_OP_GENERIC(op_type) \
+int \
+do_op_map_##op_type(struct update_op *op, struct update_field *field) \
+{ \
+ assert(field->type == UPDATE_MAP); \
+ struct update_map_item *item = \
+ update_map_extract_item(field, op); \
+ if (item == NULL) \
+ return -1; \
+ if (! update_op_is_term(op)) { \
+ op->token_type = JSON_TOKEN_END; \
+ return do_op_##op_type(op, &item->field); \
+ } \
+ if (item->field.type != UPDATE_NOP) \
+ return update_err_double(op); \
+ if (update_op_do_##op_type(op, item->field.data) != 0) \
+ return -1; \
+ item->field.type = UPDATE_SCALAR; \
+ item->field.scalar.op = op; \
+ return 0; \
+}
+
+DO_SCALAR_OP_GENERIC(arith)
+
+DO_SCALAR_OP_GENERIC(bit)
+
+DO_SCALAR_OP_GENERIC(splice)
+
+int
+update_map_create(struct update_field *field, const char *header,
+ const char *data, const char *data_end, int field_count)
+{
+ field->type = UPDATE_MAP;
+ field->data = header;
+ field->size = data_end - header;
+ field->map.size = field_count;
+ stailq_create(&field->map.items);
+ if (field_count == 0)
+ return 0;
+ struct update_map_item *first =
+ update_map_new_item(field, UPDATE_NOP, NULL, 0, data, 0,
+ data_end - data);
+ return first != NULL ? 0 : -1;
+}
+
+int
+update_map_create_with_child(struct update_field *field,
+ const struct update_field *child, const char *key,
+ uint32_t key_len, const char *header)
+{
+ field->type = UPDATE_MAP;
+ field->data = header;
+ stailq_create(&field->map.items);
+
+ const char *pos = header;
+ uint32_t i, ikey_len, field_count = mp_decode_map(&pos);
+ const char *begin = pos;
+ struct update_map_item *item = NULL;
+ for (i = 0; i < field_count; ++i) {
+ if (mp_typeof(*pos) != MP_STR) {
+ mp_next(&pos);
+ mp_next(&pos);
+ continue;
+ }
+ const char *before_key = pos;
+ const char *ikey = mp_decode_str(&pos, &ikey_len);
+ if (ikey_len == key_len && memcmp(ikey, key, key_len) == 0) {
+ item = update_map_new_item(field, UPDATE_NOP, NULL, 0,
+ begin, 0,
+ before_key - begin);
+ if (item == NULL)
+ return -1;
+ ++i;
+ break;
+ }
+ mp_next(&pos);
+ }
+ /*
+ * When a map is initialized with an existing child, it
+ * means, that it was already found earlier. I.e. it is
+ * impossible to miss it.
+ */
+ assert(item != NULL);
+ const char *data = pos;
+ mp_next(&pos);
+ uint32_t data_size = pos - data;
+ for (; i < field_count; ++i) {
+ mp_next(&pos);
+ mp_next(&pos);
+ }
+ item = update_map_item_alloc();
+ if (item == NULL)
+ return -1;
+ item->field = *child;
+ update_map_create_item(field, item, child->type, key, key_len, data,
+ data_size, pos - data - data_size);
+ field->map.size = field_count;
+ field->size = pos - header;
+ return 0;
+}
+
+uint32_t
+update_map_sizeof(struct update_field *field)
+{
+ assert(field->type == UPDATE_MAP);
+ uint32_t res = mp_sizeof_map(field->map.size);
+ struct update_map_item *i;
+ stailq_foreach_entry(i, &field->map.items, in_items) {
+ res += i->tail_size;
+ if (i->key != NULL) {
+ res += mp_sizeof_str(i->key_len) +
+ update_field_sizeof(&i->field);
+ }
+ }
+ return res;
+}
+
+uint32_t
+update_map_store(struct update_field *field, char *out, char *out_end)
+{
+ assert(field->type == UPDATE_MAP);
+ char *out_begin = out;
+ out = mp_encode_map(out, field->map.size);
+ struct update_map_item *i;
+ /*
+ * This is the trick about saving updated keys before
+ * others. The first cycle doesn't save unchanged tails.
+ */
+ stailq_foreach_entry(i, &field->map.items, in_items) {
+ if (i->key != NULL) {
+ out = mp_encode_str(out, i->key, i->key_len);
+ out += update_field_store(&i->field, out, out_end);
+ }
+ }
+ stailq_foreach_entry(i, &field->map.items, in_items) {
+ memcpy(out, i->field.data + i->field.size, i->tail_size);
+ out += i->tail_size;
+ }
+ assert(out <= out_end);
+ return out - out_begin;
+}
diff --git a/src/box/update/update_route.c b/src/box/update/update_route.c
index f1d2ced65..41fcf8b3b 100644
--- a/src/box/update/update_route.c
+++ b/src/box/update/update_route.c
@@ -109,6 +109,36 @@ update_route_branch_array(struct update_field *next_hop,
return op->meta->do_cb(op, next_hop);
}
+/**
+ * Do the actual branch, but by a map and a key in that map. Works
+ * exactly the same as the array-counterpart.
+ */
+static int
+update_route_branch_map(struct update_field *next_hop,
+ const struct update_field *child, const char *key,
+ int key_len, const char *parent)
+{
+ struct update_op *op = child->bar.op;
+ if (child->type != UPDATE_BAR || child->bar.path_len > 0 ||
+ (op->opcode != '!' && op->opcode != '#')) {
+ return update_map_create_with_child(next_hop, child, key,
+ key_len, parent);
+ }
+ op->token_type = JSON_TOKEN_STR;
+ op->key = key;
+ op->key_len = key_len;
+ const char *data = parent;
+ uint32_t field_count = mp_decode_map(&data);
+ const char *end = data;
+ for (uint32_t i = 0; i < field_count; ++i) {
+ mp_next(&end);
+ mp_next(&end);
+ }
+ if (update_map_create(next_hop, parent, data, end, field_count) != 0)
+ return -1;
+ return op->meta->do_cb(op, next_hop);
+}
+
struct update_field *
update_route_branch(struct update_field *field, struct update_op *new_op)
{
@@ -205,19 +235,6 @@ update_route_branch(struct update_field *field, struct update_op *new_op)
} else {
next_hop = field;
}
- if (type == MP_MAP) {
- diag_set(ClientError, ER_UNSUPPORTED, "update",
- "path intersection on map");
- return NULL;
- }
- if (new_token.type != JSON_TOKEN_NUM) {
- update_err(new_op, "can not update array by non-integer index");
- return NULL;
- }
- if (type != MP_ARRAY) {
- update_err_no_such_field(new_op);
- return NULL;
- }
int path_offset = old_path_lexer.offset;
struct update_field child = *field;
@@ -245,11 +262,34 @@ update_route_branch(struct update_field *field, struct update_op *new_op)
*/
}
- new_op->token_type = JSON_TOKEN_NUM;
- new_op->field_no = new_token.num;
- if (update_route_branch_array(next_hop, &child, old_token.num,
- parent) != 0)
+ if (type == MP_ARRAY) {
+ if (new_token.type != JSON_TOKEN_NUM) {
+ update_err(new_op, "can not update array by "\
+ "non-integer index");
+ return NULL;
+ }
+ new_op->token_type = JSON_TOKEN_NUM;
+ new_op->field_no = new_token.num;
+ if (update_route_branch_array(next_hop, &child, old_token.num,
+ parent) != 0)
+ return NULL;
+ } else if (type == MP_MAP) {
+ if (new_token.type != JSON_TOKEN_STR) {
+ update_err(new_op, "can not update map by "\
+ "non-string key");
+ return NULL;
+ }
+ new_op->token_type = JSON_TOKEN_STR;
+ new_op->key = new_token.str;
+ new_op->key_len = new_token.len;
+ if (update_route_branch_map(next_hop, &child, old_token.str,
+ old_token.len, parent) != 0)
+ return NULL;
+ } else {
+ update_err_no_such_field(new_op);
return NULL;
+ }
+
if (! transform_root) {
field->type = UPDATE_ROUTE;
field->route.path = old_path;
diff --git a/test/box/update.result b/test/box/update.result
index 748bcf14a..7062d874b 100644
--- a/test/box/update.result
+++ b/test/box/update.result
@@ -1431,6 +1431,158 @@ t:update({{'+', '[4][3]', 0.5}, {'+', '[4][3][2]', 0.5}})
---
- error: Field ''[4][3][2]'' was not found in the tuple
...
+--
+-- Intersecting map updates.
+--
+t:update({{'=', '[4][6].c.d', 2300}, {'=', '[4][6].c.e', 2400}})
+---
+- [1, {}, [], [1, 2, 3, [4, 5, 6, 7, [8, 9, [10, 11, 12], 13, 14], 15, 16, [17, 18,
+ 19]], 20, {'b': 22, 'a': 21, 'c': {'d': 2300, 'e': 2400}}, 25]]
+...
+-- Fill an empty map.
+t:update({{'=', '[2].k', 100}, {'=', '[2].v', 200}})
+---
+- [1, {'k': 100, 'v': 200}, [], [1, 2, 3, [4, 5, 6, 7, [8, 9, [10, 11, 12], 13, 14],
+ 15, 16, [17, 18, 19]], 20, {'b': 22, 'a': 21, 'c': {'d': 23, 'e': 24}}, 25]]
+...
+t = {} \
+t[1] = 1 \
+t[2] = { \
+ a = 1, \
+ b = 2, \
+ c = {3, 4, 5}, \
+ d = { \
+ [0] = 0, \
+ e = 6, \
+ [true] = false, \
+ f = 7, \
+ [-1] = -1, \
+ }, \
+ g = { \
+ {k = 8, v = 9}, \
+ [box.NULL] = box.NULL, \
+ {k = 10, v = 11}, \
+ {k = '12str', v = '13str'}, \
+ }, \
+ h = { \
+ [-1] = {{{{-1}, -1}, -1}, -1}, \
+ i = { \
+ j = 14, \
+ k = 15, \
+ }, \
+ m = 16, \
+ } \
+}
+---
+...
+t[3] = {}
+---
+...
+t = box.tuple.new(t)
+---
+...
+-- Insert and delete from the same map at once.
+t:update({{'!', '[2].n', 17}, {'#', '[2].b', 1}})
+---
+- [1, {'a': 1, 'n': 17, 'd': {0: 0, -1: -1, true: false, 'f': 7, 'e': 6}, 'h': {'i': {
+ 'j': 14, 'k': 15}, -1: [[[[-1], -1], -1], -1], 'm': 16}, 'c': [3, 4, 5], 'g': {
+ 1: {'k': 8, 'v': 9}, 2: {'k': 10, 'v': 11}, 3: {'k': '12str', 'v': '13str'},
+ null: null}}, []]
+...
+-- Delete a not existing key.
+t:update({{'=', '[2].d.g', 6000}, {'#', '[2].d.old', 1}})
+---
+- [1, {'b': 2, 'a': 1, 'd': {0: 0, 'f': 7, 'e': 6, -1: -1, true: false, 'g': 6000},
+ 'h': {'i': {'j': 14, 'k': 15}, -1: [[[[-1], -1], -1], -1], 'm': 16}, 'c': [3,
+ 4, 5], 'g': {1: {'k': 8, 'v': 9}, 2: {'k': 10, 'v': 11}, 3: {'k': '12str', 'v': '13str'},
+ null: null}}, []]
+...
+-- Scalar operations.
+t:update({{'+', '[2].d.e', 1}, {'+', '[2].b', 1}, {'+', '[2].d.f', 1}})
+---
+- [1, {'b': 3, 'c': [3, 4, 5], 'd': {0: 0, true: false, 'f': 8, -1: -1, 'e': 7}, 'h': {
+ 'i': {'j': 14, 'k': 15}, -1: [[[[-1], -1], -1], -1], 'm': 16}, 'a': 1, 'g': {
+ 1: {'k': 8, 'v': 9}, 2: {'k': 10, 'v': 11}, 3: {'k': '12str', 'v': '13str'},
+ null: null}}, []]
+...
+-- Errors.
+-- This operation array creates an update tree with several full
+-- featured maps, not bars. It is required to cover some complex
+-- cases about fields extraction.
+ops = {{'=', '[2].h.i.j', 14000}, {'=', '[2].h.i.k', 15000}, {'=', '[2].h.m', 16000}, {'=', '[2].b', 2000}, {}}
+---
+...
+-- When a map update extracts a next token on demand, it should
+-- reject bad json, obviously. Scalar and '!'/'#' operations are
+-- implemented separately and tested both.
+ops[#ops] = {'+', '[2].h.i.<badjson>', -1}
+---
+...
+t:update(ops)
+---
+- error: 'Field ''[2].h.i.<badjson>'' UPDATE error: invalid JSON in position 9'
+...
+ops[#ops][1] = '!'
+---
+...
+t:update(ops)
+---
+- error: 'Field ''[2].h.i.<badjson>'' UPDATE error: invalid JSON in position 9'
+...
+ops[#ops][1] = '#'
+---
+...
+ops[#ops][3] = 1
+---
+...
+t:update(ops)
+---
+- error: 'Field ''[2].h.i.<badjson>'' UPDATE error: invalid JSON in position 9'
+...
+-- Key extractor should reject any attempt to use non-string keys.
+ops[#ops] = {'-', '[2].h.i[20]', -1}
+---
+...
+t:update(ops)
+---
+- error: 'Field ''[2].h.i[20]'' UPDATE error: can''t update a map by not a string
+ key'
+...
+ops[#ops][1] = '='
+---
+...
+t:update(ops)
+---
+- error: 'Field ''[2].h.i[20]'' UPDATE error: can''t update a map by not a string
+ key'
+...
+t:update({{'+', '[2].d.e', 1}, {'+', '[2].d.f', 1}, {'+', '[2].d.k', 1}})
+---
+- error: Field ''[2].d.k'' was not found in the tuple
+...
+t:update({{'=', '[2].d.e', 6000}, {'!', '[2].d.g.h', -1}})
+---
+- error: Field ''[2].d.g.h'' was not found in the tuple
+...
+t:update({{'!', '[2].d.g', 6000}, {'!', '[2].d.e', -1}})
+---
+- error: 'Field ''[2].d.e'' UPDATE error: the key exists already'
+...
+t:update({{'=', '[2].d.g', 6000}, {'#', '[2].d.old', 10}})
+---
+- error: 'Field ''[2].d.old'' UPDATE error: can delete only 1 field from a map in
+ a row'
+...
+-- '!'/'=' can be used to create a field, but only if it is a
+-- tail. These operations can't create the whole path.
+t:update({{'=', '[2].d.g', 6000}, {'=', '[2].d.new.new', -1}})
+---
+- error: Field ''[2].d.new.new'' was not found in the tuple
+...
+t:update({{'=', '[2].d.g', 6000}, {'#', '[2].d.old.old', 1}})
+---
+- error: Field ''[2].d.old.old'' was not found in the tuple
+...
s:drop()
---
...
diff --git a/test/box/update.test.lua b/test/box/update.test.lua
index 27347d66d..1e001bf67 100644
--- a/test/box/update.test.lua
+++ b/test/box/update.test.lua
@@ -504,4 +504,79 @@ t:update({{'!', '[4][3]', 2.5}, {'+', '[4][5][5][5]', 0.75}, {'+', '[4][5][3]',
-- during branching.
t:update({{'+', '[4][3]', 0.5}, {'+', '[4][3][2]', 0.5}})
+--
+-- Intersecting map updates.
+--
+t:update({{'=', '[4][6].c.d', 2300}, {'=', '[4][6].c.e', 2400}})
+-- Fill an empty map.
+t:update({{'=', '[2].k', 100}, {'=', '[2].v', 200}})
+
+t = {} \
+t[1] = 1 \
+t[2] = { \
+ a = 1, \
+ b = 2, \
+ c = {3, 4, 5}, \
+ d = { \
+ [0] = 0, \
+ e = 6, \
+ [true] = false, \
+ f = 7, \
+ [-1] = -1, \
+ }, \
+ g = { \
+ {k = 8, v = 9}, \
+ [box.NULL] = box.NULL, \
+ {k = 10, v = 11}, \
+ {k = '12str', v = '13str'}, \
+ }, \
+ h = { \
+ [-1] = {{{{-1}, -1}, -1}, -1}, \
+ i = { \
+ j = 14, \
+ k = 15, \
+ }, \
+ m = 16, \
+ } \
+}
+t[3] = {}
+t = box.tuple.new(t)
+-- Insert and delete from the same map at once.
+t:update({{'!', '[2].n', 17}, {'#', '[2].b', 1}})
+-- Delete a not existing key.
+t:update({{'=', '[2].d.g', 6000}, {'#', '[2].d.old', 1}})
+
+-- Scalar operations.
+t:update({{'+', '[2].d.e', 1}, {'+', '[2].b', 1}, {'+', '[2].d.f', 1}})
+
+-- Errors.
+-- This operation array creates an update tree with several full
+-- featured maps, not bars. It is required to cover some complex
+-- cases about fields extraction.
+ops = {{'=', '[2].h.i.j', 14000}, {'=', '[2].h.i.k', 15000}, {'=', '[2].h.m', 16000}, {'=', '[2].b', 2000}, {}}
+-- When a map update extracts a next token on demand, it should
+-- reject bad json, obviously. Scalar and '!'/'#' operations are
+-- implemented separately and tested both.
+ops[#ops] = {'+', '[2].h.i.<badjson>', -1}
+t:update(ops)
+ops[#ops][1] = '!'
+t:update(ops)
+ops[#ops][1] = '#'
+ops[#ops][3] = 1
+t:update(ops)
+-- Key extractor should reject any attempt to use non-string keys.
+ops[#ops] = {'-', '[2].h.i[20]', -1}
+t:update(ops)
+ops[#ops][1] = '='
+t:update(ops)
+
+t:update({{'+', '[2].d.e', 1}, {'+', '[2].d.f', 1}, {'+', '[2].d.k', 1}})
+t:update({{'=', '[2].d.e', 6000}, {'!', '[2].d.g.h', -1}})
+t:update({{'!', '[2].d.g', 6000}, {'!', '[2].d.e', -1}})
+t:update({{'=', '[2].d.g', 6000}, {'#', '[2].d.old', 10}})
+-- '!'/'=' can be used to create a field, but only if it is a
+-- tail. These operations can't create the whole path.
+t:update({{'=', '[2].d.g', 6000}, {'=', '[2].d.new.new', -1}})
+t:update({{'=', '[2].d.g', 6000}, {'#', '[2].d.old.old', 1}})
+
s:drop()
--
2.20.1 (Apple Git-117)
^ permalink raw reply [flat|nested] 15+ messages in thread* [tarantool-patches] [PATCH 02/13] rope: make rope library macro template
2019-08-12 23:05 [tarantool-patches] [PATCH 00/13] JSON updates Vladislav Shpilevoy
` (4 preceding siblings ...)
2019-08-12 23:05 ` [tarantool-patches] [PATCH 13/13] tuple: JSON updates support intersection by maps Vladislav Shpilevoy
@ 2019-08-12 23:05 ` Vladislav Shpilevoy
2019-08-12 23:05 ` [tarantool-patches] [PATCH 03/13] tuple: relax struct tuple_update dependency on rope Vladislav Shpilevoy
` (7 subsequent siblings)
13 siblings, 0 replies; 15+ messages in thread
From: Vladislav Shpilevoy @ 2019-08-12 23:05 UTC (permalink / raw)
To: tarantool-patches
Rope library stores alloc, split and free functions by pointer,
that is a huge slog at performance, as any other virtual
function call. There is no reason, why the rope library may not
become a template, except historical ones.
The patch not only removes virtual functions, but also makes rope
deletion almost no-op in case if free() function is not defined.
A second motivation point of that patch is that original rope
structure was too big. Again, because it stored several pointers
to functions. In forthcoming patches on #1261 multiple ropes can
be created per each update, so it makes sense to reduce size of
this structure.
---
debian/copyright | 2 +-
src/box/tuple_update.c | 74 +++--
src/lib/salad/rope.c | 479 +++++--------------------------
src/lib/salad/rope.h | 617 ++++++++++++++++++++++++++++++++++------
test/unit/rope.c | 1 -
test/unit/rope_avl.c | 1 -
test/unit/rope_basic.c | 1 -
test/unit/rope_common.h | 41 ++-
test/unit/rope_stress.c | 5 +-
9 files changed, 674 insertions(+), 547 deletions(-)
diff --git a/debian/copyright b/debian/copyright
index a48171371..3d68dc390 100644
--- a/debian/copyright
+++ b/debian/copyright
@@ -218,7 +218,7 @@ Files: third_party/lua-cjson/*
Copyright: 2010-2012 Mark Pulford <mark@kyne.com.au>
License: Expat
-Files: src/lib/salad/rope.c
+Files: src/lib/salad/rope.*
Copyright: 1993-1994 by Xerox Corporation.
License: BSD-2-Clause
diff --git a/src/box/tuple_update.c b/src/box/tuple_update.c
index 599952189..7498d558a 100644
--- a/src/box/tuple_update.c
+++ b/src/box/tuple_update.c
@@ -30,8 +30,6 @@
*/
#include "tuple_update.h"
-
-#include <stdio.h>
#include <stdbool.h>
#include "say.h"
@@ -41,7 +39,6 @@
#include "third_party/queue.h"
#include <msgpuck/msgpuck.h>
#include <bit/int96.h>
-#include <salad/rope.h>
#include "column_mask.h"
#include "fiber.h"
@@ -97,27 +94,19 @@
*/
static inline void *
-update_alloc(void *ctx, size_t size)
+update_alloc(struct region *region, size_t size)
{
- void *ptr = region_aligned_alloc((struct region *) ctx, size,
- alignof(uint64_t));
+ void *ptr = region_aligned_alloc(region, size, alignof(uint64_t));
if (ptr == NULL)
diag_set(OutOfMemory, size, "region_aligned_alloc",
"update internals");
return ptr;
}
-static void
-update_free(void *ctx, void *mem)
-{
- (void) ctx;
- (void) mem;
-}
-
/** Update internal state */
struct tuple_update
{
- void *alloc_ctx;
+ struct region *region;
struct rope *rope;
struct update_op *ops;
uint32_t op_count;
@@ -206,6 +195,17 @@ union update_op_arg {
struct update_field;
struct update_op;
+static struct update_field *
+update_field_split(struct region *region, struct update_field *data,
+ size_t size, size_t offset);
+
+#define ROPE_SPLIT_F update_field_split
+#define ROPE_ALLOC_F update_alloc
+#define rope_data_t struct update_field *
+#define rope_ctx_t struct region *
+
+#include "salad/rope.h"
+
typedef int (*do_op_func)(struct tuple_update *update, struct update_op *op);
typedef int (*read_arg_func)(int index_base, struct update_op *op,
const char **expr);
@@ -525,7 +525,7 @@ do_op_insert(struct tuple_update *update, struct update_op *op)
if (op_adjust_field_no(update, op, rope_size(update->rope) + 1))
return -1;
struct update_field *field = (struct update_field *)
- update_alloc(update->alloc_ctx, sizeof(*field));
+ update_alloc(update->region, sizeof(*field));
if (field == NULL)
return -1;
update_field_init(field, op->arg.set.value, op->arg.set.length, 0);
@@ -540,8 +540,7 @@ do_op_set(struct tuple_update *update, struct update_op *op)
return do_op_insert(update, op);
if (op_adjust_field_no(update, op, rope_size(update->rope)))
return -1;
- struct update_field *field = (struct update_field *)
- rope_extract(update->rope, op->field_no);
+ struct update_field *field = rope_extract(update->rope, op->field_no);
if (field == NULL)
return -1;
/* Ignore the previous op, if any. */
@@ -578,8 +577,7 @@ do_op_arith(struct tuple_update *update, struct update_op *op)
if (op_adjust_field_no(update, op, rope_size(update->rope)))
return -1;
- struct update_field *field = (struct update_field *)
- rope_extract(update->rope, op->field_no);
+ struct update_field *field = rope_extract(update->rope, op->field_no);
if (field == NULL)
return -1;
if (field->op) {
@@ -608,8 +606,7 @@ do_op_bit(struct tuple_update *update, struct update_op *op)
{
if (op_adjust_field_no(update, op, rope_size(update->rope)))
return -1;
- struct update_field *field = (struct update_field *)
- rope_extract(update->rope, op->field_no);
+ struct update_field *field = rope_extract(update->rope, op->field_no);
if (field == NULL)
return -1;
struct op_bit_arg *arg = &op->arg.bit;
@@ -646,8 +643,7 @@ do_op_splice(struct tuple_update *update, struct update_op *op)
{
if (op_adjust_field_no(update, op, rope_size(update->rope)))
return -1;
- struct update_field *field = (struct update_field *)
- rope_extract(update->rope, op->field_no);
+ struct update_field *field = rope_extract(update->rope, op->field_no);
if (field == NULL)
return -1;
if (field->op) {
@@ -786,16 +782,13 @@ static const struct update_op_meta op_delete =
/** Split a range of fields in two, allocating update_field
* context for the new range.
*/
-static void *
-update_field_split(void *split_ctx, void *data, size_t size, size_t offset)
+static struct update_field *
+update_field_split(struct region *region, struct update_field *prev,
+ size_t size, size_t offset)
{
- (void)size;
- struct tuple_update *update = (struct tuple_update *) split_ctx;
-
- struct update_field *prev = (struct update_field *) data;
-
- struct update_field *next = (struct update_field *)
- update_alloc(update->alloc_ctx, sizeof(*next));
+ (void) size;
+ struct update_field *next =
+ (struct update_field *) update_alloc(region, sizeof(*next));
if (next == NULL)
return NULL;
assert(offset > 0 && prev->tail_len > 0);
@@ -831,14 +824,13 @@ int
update_create_rope(struct tuple_update *update, const char *tuple_data,
const char *tuple_data_end, uint32_t field_count)
{
- update->rope = rope_new(update_field_split, update, update_alloc,
- update_free, update->alloc_ctx);
+ update->rope = rope_new(update->region);
if (update->rope == NULL)
return -1;
/* Initialize the rope with the old tuple. */
struct update_field *first = (struct update_field *)
- update_alloc(update->alloc_ctx, sizeof(*first));
+ update_alloc(update->region, sizeof(*first));
if (first == NULL)
return -1;
const char *field = tuple_data;
@@ -864,8 +856,7 @@ update_calc_tuple_length(struct tuple_update *update)
rope_iter_create(&it, update->rope);
for (node = rope_iter_start(&it); node; node = rope_iter_next(&it)) {
- struct update_field *field =
- (struct update_field *) rope_leaf_data(node);
+ struct update_field *field = rope_leaf_data(node);
uint32_t field_len = (field->op ? field->op->new_field_len :
(uint32_t)(field->tail - field->old));
res += field_len + field->tail_len;
@@ -889,8 +880,7 @@ update_write_tuple(struct tuple_update *update, char *buffer, char *buffer_end)
rope_iter_create(&it, update->rope);
for (node = rope_iter_start(&it); node; node = rope_iter_next(&it)) {
- struct update_field *field = (struct update_field *)
- rope_leaf_data(node);
+ struct update_field *field = rope_leaf_data(node);
uint32_t field_count = rope_leaf_size(node);
const char *old_field = field->old;
struct update_op *op = field->op;
@@ -979,7 +969,7 @@ update_read_ops(struct tuple_update *update, const char *expr,
}
/* Read update operations. */
- update->ops = (struct update_op *) update_alloc(update->alloc_ctx,
+ update->ops = (struct update_op *) update_alloc(update->region,
update->op_count * sizeof(struct update_op));
if (update->ops == NULL)
return -1;
@@ -1172,7 +1162,7 @@ static void
update_init(struct tuple_update *update, int index_base)
{
memset(update, 0, sizeof(*update));
- update->alloc_ctx = &fiber()->gc;
+ update->region = &fiber()->gc;
/*
* Base field offset, e.g. 0 for C and 1 for Lua. Used only for
* error messages. All fields numbers must be zero-based!
@@ -1184,7 +1174,7 @@ const char *
update_finish(struct tuple_update *update, uint32_t *p_tuple_len)
{
uint32_t tuple_len = update_calc_tuple_length(update);
- char *buffer = (char *) update_alloc(update->alloc_ctx, tuple_len);
+ char *buffer = (char *) update_alloc(update->region, tuple_len);
if (buffer == NULL)
return NULL;
*p_tuple_len = update_write_tuple(update, buffer, buffer + tuple_len);
diff --git a/src/lib/salad/rope.c b/src/lib/salad/rope.c
index 06324ab65..5de951c78 100644
--- a/src/lib/salad/rope.c
+++ b/src/lib/salad/rope.c
@@ -1,5 +1,5 @@
/*
- * Copyright 2010-2016, Tarantool AUTHORS, please see AUTHORS file.
+ * Copyright 2010-2019, Tarantool AUTHORS, please see AUTHORS file.
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
@@ -42,127 +42,73 @@
*
* Author: Hans-J. Boehm (boehm@parc.xerox.com)
*/
-/*
- * This is a rope implementation which uses AVL tree
- * balancing algorithm for rope tree balance.
+#include <stdlib.h>
+
+/**
+ * This macro helps to implement some common rope functions, not
+ * depending on a specific rope, only once, in a single object
+ * file.
*/
+#define ROPE_SRC
+#define rope_data_t void *
+#define rope_ctx_t void *
#include "rope.h"
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
-#include <assert.h>
static inline int
-rope_node_height(struct rope_node *node)
+avl_node_height(struct avl_node *node)
{
return node ? node->height : 0;
}
-#if !defined(MAX)
-#define MAX(a, b) ((a) > (b) ? (a) : (b))
-#endif /* MAX */
-
static inline void
-rope_relink(struct rope_node *node)
+avl_node_relink(struct avl_node *node)
{
- node->tree_size = (rope_node_size(node->link[0]) +
- rope_node_size(node->link[1]) +
+ node->tree_size = (avl_node_size(node->link[0]) +
+ avl_node_size(node->link[1]) +
node->leaf_size);
- node->height = MAX(rope_node_height(node->link[0]),
- rope_node_height(node->link[1])) + 1;
-}
-
-static inline struct rope_node *
-rope_node_new(struct rope *rope, void *data, rope_size_t size)
-{
- struct rope_node *node =
- (struct rope_node *) rope->alloc(rope->alloc_ctx,
- sizeof(struct rope_node));
-
- if (node == NULL)
- return NULL;
-
- node->height = 1;
- node->tree_size = node->leaf_size = size;
- node->data = data;
- node->link[0] = node->link[1] = NULL;
-
- return node;
+ node->height = MAX(avl_node_height(node->link[0]),
+ avl_node_height(node->link[1])) + 1;
}
-void
-rope_clear(struct rope *rope)
+static inline struct avl_node *
+avl_rotate_single(struct avl_node *parent, int direction)
{
- struct rope_node *it = rope->root;
- struct rope_node *save;
-
- /* Destruction by rotation */
- while (it != NULL) {
- if (it->link[0] == NULL) {
- /* Remove node */
- save = it->link[1];
- rope->free(rope->alloc_ctx, it);
- } else {
- /* Rotate right */
- save = it->link[0];
- it->link[0] = save->link[1];
- save->link[1] = it;
- }
- it = save;
- }
- rope->root = NULL;
-}
-
-static struct rope_node *
-rope_node_split(struct rope *rope, struct rope_node *node, rope_size_t offset)
-{
- rope_size_t old_size = node->leaf_size;
- node->leaf_size = offset;
-
- void *data = rope->split(rope->split_ctx, node->data, old_size, offset);
- return rope_node_new(rope, data, old_size - offset);
-}
-
-static inline struct rope_node *
-avl_rotate_single(struct rope_node *parent, int direction)
-{
- struct rope_node *save = parent->link[!direction];
+ struct avl_node *save = parent->link[!direction];
parent->link[!direction] = save->link[direction];
save->link[direction] = parent;
/* First relink the parent, since it's now a child. */
- rope_relink(parent);
- rope_relink(save);
+ avl_node_relink(parent);
+ avl_node_relink(save);
return save;
}
-static inline struct rope_node *
-avl_rotate_double(struct rope_node *parent, int direction)
+static inline struct avl_node *
+avl_rotate_double(struct avl_node *parent, int direction)
{
parent->link[!direction] =
avl_rotate_single(parent->link[!direction], !direction);
return avl_rotate_single(parent, direction);
}
-/** Rebalance the tree. */
-static inline void
-avl_rebalance_after_insert(struct rope_node ***path,
- struct rope_node ***p_end, int insert_height)
+void
+avl_rebalance_after_insert(struct avl_node ***path, struct avl_node ***p_end,
+ int insert_height)
{
while (p_end > path) {
- struct rope_node *left = **p_end--;
- struct rope_node *parent = **p_end;
+ struct avl_node *left = **p_end--;
+ struct avl_node *parent = **p_end;
/*
* To use the same rotation functions, set mirror
* to 1 if left is right and right is left.
*/
int mirror = left != parent->link[0];
- struct rope_node *right = parent->link[!mirror];
+ struct avl_node *right = parent->link[!mirror];
- int left_height = rope_node_height(left);
- int right_height = rope_node_height(right);
+ int left_height = avl_node_height(left);
+ int right_height = avl_node_height(right);
parent->height = MAX(left_height, right_height) + 1;
/*
* Rotations flattened the tree, so there is no
@@ -179,10 +125,10 @@ avl_rebalance_after_insert(struct rope_node ***path,
* in the range -1..1 + height(new_node).
*/
if (left_height - right_height >= 2) {
- struct rope_node *l_left = left->link[mirror];
- struct rope_node *l_right = left->link[!mirror];
- int l_left_height = rope_node_height(l_left);
- int l_right_height = rope_node_height(l_right);
+ struct avl_node *l_left = left->link[mirror];
+ struct avl_node *l_right = left->link[!mirror];
+ int l_left_height = avl_node_height(l_left);
+ int l_right_height = avl_node_height(l_right);
/*
* Rotate in the direction, opposite to
* the skew. E.g. if we have two left-left
@@ -212,24 +158,20 @@ avl_rebalance_after_insert(struct rope_node ***path,
}
}
-/* This is a copy-cat of the previous loop,
- * with the exception that the heuristic to break
- * the loop is different.
- */
-static inline void
-avl_rebalance_after_delete(struct rope_node ***path,
- struct rope_node ***p_end)
+void
+avl_rebalance_after_delete(struct avl_node ***path,
+ struct avl_node ***p_end)
{
while (p_end > path) {
- struct rope_node *left = **p_end--;
- struct rope_node *parent = **p_end;
+ struct avl_node *left = **p_end--;
+ struct avl_node *parent = **p_end;
int mirror = left != parent->link[0];
- struct rope_node *right = parent->link[!mirror];
+ struct avl_node *right = parent->link[!mirror];
- int left_height = rope_node_height(left);
- int right_height = rope_node_height(right);
+ int left_height = avl_node_height(left);
+ int right_height = avl_node_height(right);
parent->height = MAX(left_height, right_height) + 1;
/*
@@ -241,10 +183,10 @@ avl_rebalance_after_delete(struct rope_node ***path,
break;
if (left_height - right_height <= -2) {
- struct rope_node *r_left = right->link[mirror];
- struct rope_node *r_right = right->link[!mirror];
- int r_left_height = rope_node_height(r_left);
- int r_right_height = rope_node_height(r_right);
+ struct avl_node *r_left = right->link[mirror];
+ struct avl_node *r_right = right->link[!mirror];
+ int r_left_height = avl_node_height(r_left);
+ int r_right_height = avl_node_height(r_right);
if (r_left_height <= r_right_height)
**p_end = avl_rotate_single(parent, mirror);
@@ -254,24 +196,17 @@ avl_rebalance_after_delete(struct rope_node ***path,
}
}
-/**
- * Find a rope node which contains the substring at offset,
- * adjusting tree size with adjust_size and saving the path
- * in path.
- *
- * @return the end of the route.
- */
-static inline struct rope_node ***
-avl_route_to_offset(struct rope_node ***path, rope_size_t *p_offset,
+struct avl_node ***
+avl_route_to_offset(struct avl_node ***path, uint32_t *p_offset,
ssize_t adjust_size)
{
- rope_size_t offset = *p_offset;
+ uint32_t offset = *p_offset;
while (**path) {
- struct rope_node *node = **path;
+ struct avl_node *node = **path;
node->tree_size += adjust_size;
- rope_size_t left_size = rope_node_size(node->link[0]);
+ uint32_t left_size = avl_node_size(node->link[0]);
if (offset < left_size) {
/* The offset lays in the left subtree. */
@@ -297,15 +232,10 @@ avl_route_to_offset(struct rope_node ***path, rope_size_t *p_offset,
return path;
}
-/**
- * Route to successor or predecessor node of the node
- * in **path. It's either the rightmost leaf of the left child
- * (previous node) or leftmost leaf of the right child.
- */
-static inline struct rope_node ***
-avl_route_to_next(struct rope_node ***path, int dir, rope_ssize_t adjust_size)
+struct avl_node ***
+avl_route_to_next(struct avl_node ***path, int dir, int32_t adjust_size)
{
- struct rope_node *node = **path;
+ struct avl_node *node = **path;
*++path = &node->link[dir];
while (**path) {
node = **path;
@@ -315,198 +245,6 @@ avl_route_to_next(struct rope_node ***path, int dir, rope_ssize_t adjust_size)
return path;
}
-/**
- * A new node is always inserted at a leaf position.
- * If insertion unbalances the tree, the rebalancing
- * procedure may put the node into an intermediate position.
- *
- * While traversing the tree, we simultaneously update
- * tree sizes of all intermediate nodes, taking into account
- * the size of the new node.
- *
- * When insertion offset falls at the middle of an existing node,
- * we truncate this node and attach its tail to the left leaf
- * of the new node. This trim operation doesn't decrease the old
- * subtree height, and, while it does change subtree size
- * temporarily, as long as we attach the new node to the right
- * subtree of the truncated node, truncation has no effect on the
- * tree size either.
- *
- * Rebalancing, when it occurs, will correctly update subtree
- * heights and sizes of all modified nodes.
- */
-int
-rope_insert(struct rope *rope, rope_size_t offset, void *data, rope_size_t size)
-{
- if (offset > rope_size(rope))
- offset = rope_size(rope);
-
- assert(size);
-
- struct rope_node *new_node = rope_node_new(rope, data, size);
-
- if (new_node == NULL)
- return -1;
-
- struct rope_node **path[ROPE_HEIGHT_MAX];
- path[0] = &rope->root;
-
- struct rope_node ***p_end = avl_route_to_offset(path, &offset, size);
- if (**p_end != NULL) {
- /*
- * The offset is inside an existing
- * substring in the rope. If offset is 0,
- * then insert the new node at the rightmost leaf
- * of the left child. Otherwise, cut the tail of
- * the substring, make it a prefix of the inserted
- * string, and insert the result at the leftmost
- * leaf of the right child.
- */
- if (offset != 0) {
- struct rope_node *split_node;
- split_node = rope_node_split(rope, **p_end, offset);
- if (split_node == NULL)
- return -1;
- split_node->link[0] = new_node;
- split_node->height++;
- split_node->tree_size += new_node->tree_size;
- new_node = split_node;
- }
- p_end = avl_route_to_next(p_end, offset != 0,
- new_node->tree_size);
- }
- **p_end = new_node;
- avl_rebalance_after_insert(path, p_end, new_node->height);
- return 0;
-}
-
-/** Make sure there is a rope node at the given offset. */
-struct rope_node *
-rope_extract_node(struct rope *rope, rope_size_t offset)
-{
- assert(offset < rope_size(rope));
-
- struct rope_node **path[ROPE_HEIGHT_MAX];
- path[0] = &rope->root;
-
- struct rope_node ***p_end = avl_route_to_offset(path, &offset, 0);
- if (offset == 0)
- return **p_end;
- struct rope_node *new_node = rope_node_split(rope, **p_end, offset);
- if (new_node == NULL)
- return NULL;
- p_end = avl_route_to_next(p_end, 1, new_node->tree_size);
- **p_end = new_node;
- avl_rebalance_after_insert(path, p_end, new_node->height);
- return new_node;
-}
-
-/**
- * Erase a single element from the rope.
- * This is a straightforward implementation for a single-element
- * deletion from a rope. A generic cut from a rope involves
- * 2 tree splits and one merge.
- *
- * When deleting a single element, 3 cases are possible:
- * - offset falls at a node with a single element. In this
- * case we perform a normal AVL tree delete.
- * - offset falls at the end or the beginning of an existing node
- * with leaf_size > 1. In that case we trim the existing node
- * and return.
- * - offset falls inside an existing node. In that case
- * we split the existing node at offset, and insert the tail.
- *
- * The implementation is a copycat of rope_insert(). If you're
- * trying to understand the code, it's recommended to start
- * from rope_insert().
- */
-int
-rope_erase(struct rope *rope, rope_size_t offset)
-{
- assert(offset < rope_size(rope));
-
- struct rope_node **path[ROPE_HEIGHT_MAX];
- path[0] = &rope->root;
-
- struct rope_node ***p_end = avl_route_to_offset(path, &offset, -1);
-
- struct rope_node *node = **p_end;
-
- if (node->leaf_size > 1) {
- /* Check if we can simply trim the node. */
- if (offset == 0) {
- /* Cut the head. */
- node->data = rope->split(rope->split_ctx, node->data,
- node->leaf_size, 1);
- node->leaf_size -= 1;
- return 0;
- }
- rope_size_t size = node->leaf_size;
- /* Cut the tail */
- void *next = rope->split(rope->split_ctx, node->data,
- node->leaf_size, offset);
- node->leaf_size = offset;
- if (offset == size - 1)
- return 0; /* Trimmed the tail, nothing else to do */
- /*
- * Offset falls inside a substring. Erase the
- * first field and insert the tail.
- */
- next = rope->split(rope->split_ctx, next, size - offset, 1);
- struct rope_node *new_node =
- rope_node_new(rope, next, size - offset - 1);
- if (new_node == NULL)
- return -1;
- /* Trim the old node. */
- p_end = avl_route_to_next(p_end, 1, new_node->tree_size);
- **p_end = new_node;
- avl_rebalance_after_insert(path, p_end, new_node->height);
- return 0;
- }
- /* We need to delete the node. */
- assert(offset == 0);
- int direction;
- if (node->link[0] != NULL && node->link[1] != NULL) {
- /*
- * The node has two non-NULL leaves. We can't
- * simply delete the node since in that case we
- * won't know what to do with one of the leaves.
- * Instead of deleting the node, store in it data
- * from the rightmost node in the left subtree, or
- * the leftmost node in the right subtree,
- * (depending on which subtree is taller), and
- * delete this leftmost/rightmost node instead.
- */
- struct rope_node *save = node;
- direction = node->link[1]->height > node->link[0]->height;
- p_end = avl_route_to_next(p_end, direction, 0) - 1;
- node = **p_end;
- /* Move the data pointers. */
- save->data = node->data;
- save->leaf_size = node->leaf_size;
- /*
- * Now follow the path again and update tree_size
- * in the parents of the moved child.
- */
- save = save->link[direction];
- while (save != node) {
- save->tree_size -= node->leaf_size;
- save = save->link[!direction];
- }
- } else {
- /*
- * Left or right subtree are NULL, so we
- * can simply put the non-NULL leaf in place
- * of the parent.
- */
- direction = node->link[0] == NULL;
- }
- **p_end = node->link[direction];
- rope->free(rope, node);
- avl_rebalance_after_delete(path, p_end);
- return 0;
-}
-
/**
* Traverse left until the left subtree is NULL,
* save the path in iter->path.
@@ -515,7 +253,7 @@ rope_erase(struct rope *rope, rope_size_t offset)
* not-NULL node.
*/
static inline void
-rope_iter_down_to_leaf(struct rope_iter *it)
+avl_iter_down_to_leaf(struct avl_iter *it)
{
while (it->top[0]->link[0] != NULL) {
it->top[1] = it->top[0]->link[0];
@@ -523,24 +261,24 @@ rope_iter_down_to_leaf(struct rope_iter *it)
}
}
-struct rope_node *
-rope_iter_start(struct rope_iter *it)
+struct avl_node *
+avl_iter_start(struct avl_iter *it)
{
it->top = it->path;
it->top[0] = it->rope->root;
if (it->top[0] != NULL)
- rope_iter_down_to_leaf(it);
+ avl_iter_down_to_leaf(it);
return it->top[0];
}
-struct rope_node *
-rope_iter_next(struct rope_iter *it)
+struct avl_node *
+avl_iter_next(struct avl_iter *it)
{
if (it->top[0]->link[1] != NULL) {
it->top[1] = it->top[0]->link[1];
it->top++;
- rope_iter_down_to_leaf(it);
+ avl_iter_down_to_leaf(it);
} else {
/*
* Right subtree is NULL. Left subtree is fully
@@ -569,96 +307,25 @@ rope_iter_next(struct rope_iter *it)
return *it->top;
}
-/** Apply visit_leaf function to every rope leaf. */
void
-rope_traverse(struct rope *rope, void (*visit_leaf)(void *, size_t))
+avl_iter_check(struct avl_iter *iter)
{
- struct rope_iter iter;
- rope_iter_create(&iter, rope);
-
- struct rope_node *leaf;
-
- for (leaf = rope_iter_start(&iter);
- leaf != NULL;
- leaf = rope_iter_next(&iter)) {
-
- visit_leaf(leaf->data, leaf->leaf_size);
- }
-}
-
-void
-rope_check(struct rope *rope)
-{
- struct rope_iter iter;
- rope_iter_create(&iter, rope);
-
- struct rope_node *node;
-
- for (node = rope_iter_start(&iter);
- node != NULL;
- node = rope_iter_next(&iter)) {
+ for (struct avl_node *node = avl_iter_start(iter); node != NULL;
+ node = avl_iter_next(iter)) {
assert(node->leaf_size != 0);
- assert(node->tree_size == rope_node_size(node->link[0])
- + rope_node_size(node->link[1]) + node->leaf_size);
+ assert(node->tree_size == avl_node_size(node->link[0])
+ + avl_node_size(node->link[1]) + node->leaf_size);
- assert(node->height == (MAX(rope_node_height(node->link[0]),
- rope_node_height(node->link[1])) + 1));
+ assert(node->height == (MAX(avl_node_height(node->link[0]),
+ avl_node_height(node->link[1])) + 1));
if (node->leaf_size == 0 ||
- node->tree_size != (rope_node_size(node->link[0]) +
- rope_node_size(node->link[1]) +
+ node->tree_size != (avl_node_size(node->link[0]) +
+ avl_node_size(node->link[1]) +
node->leaf_size) ||
- node->height != MAX(rope_node_height(node->link[0]),
- rope_node_height(node->link[1])) + 1)
+ node->height != MAX(avl_node_height(node->link[0]),
+ avl_node_height(node->link[1])) + 1)
abort();
}
}
-
-static void
-rope_node_print(struct rope_node *node,
- void (*print)(void *, size_t),
- const char *prefix, int dir)
-{
- const char *conn[] = { "┌──", "└──" };
-
- const char *padding[] = { "│ ", " " };
-
- rope_size_t child_prefix_len = strlen(prefix) + strlen(padding[0]) + 1;
- char *child_prefix = malloc(child_prefix_len);
-
- if (node && (node->link[0] || node->link[1])) {
- snprintf(child_prefix, child_prefix_len - 1,
- "%s%s", prefix, padding[!dir]);
- rope_node_print(node->link[0], print, child_prefix, 0);
- }
-
- snprintf(child_prefix, child_prefix_len - 1, "%s%s",
- prefix, padding[dir]);
- printf("%s%s", prefix, conn[dir]);
-
- if (node == NULL) {
- printf("nil\n");
- } else {
-
- printf("{ len = %zu, height = %d, data = '",
- (size_t) node->leaf_size, node->height);
- print(node->data, node->leaf_size);
- printf("'}\n");
-
- if (node->link[0] || node->link[1])
- rope_node_print(node->link[1], print, child_prefix, 1);
- }
-
- free(child_prefix);
-}
-
-void
-rope_pretty_print(struct rope *rope, void (*print_leaf)(void *, size_t))
-{
- printf("size = %zu\nstring = '", (size_t) rope_size(rope));
- rope_traverse(rope, print_leaf);
- printf("'\n");
- rope_node_print(rope->root, print_leaf, "", true);
- printf("\n");
-}
diff --git a/src/lib/salad/rope.h b/src/lib/salad/rope.h
index c19f0e319..f38ac0e79 100644
--- a/src/lib/salad/rope.h
+++ b/src/lib/salad/rope.h
@@ -1,7 +1,5 @@
-#ifndef TARANTOOL_LIB_SALAD_ROPE_H_INCLUDED
-#define TARANTOOL_LIB_SALAD_ROPE_H_INCLUDED
/*
- * Copyright 2010-2016, Tarantool AUTHORS, please see AUTHORS file.
+ * Copyright 2010-2019, Tarantool AUTHORS, please see AUTHORS file.
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
@@ -30,49 +28,161 @@
* THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
+/*
+ * Copyright (c) 1993-1994 by Xerox Corporation. All rights reserved.
+ *
+ * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
+ * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
+ *
+ * Permission is hereby granted to use or copy this program
+ * for any purpose, provided the above notices are retained on all copies.
+ * Permission to modify the code and to distribute modified code is granted,
+ * provided the above notices are retained, and a notice that the code was
+ * modified is included with the above copyright notice.
+ *
+ * Author: Hans-J. Boehm (boehm@parc.xerox.com)
+ */
+/*
+ * This is a rope implementation which uses AVL tree
+ * balancing algorithm for rope tree balance.
+ */
#include <stdint.h>
-#include <stddef.h>
#include <stdbool.h>
+#include <stdio.h>
+#include <assert.h>
#if defined(__cplusplus)
extern "C" {
#endif /* defined(__cplusplus) */
+#ifndef MAX
+#define NEED_UNDEF_MAX
+#define MAX(a, b) ((a) > (b) ? (a) : (b))
+#endif /* MAX */
+
+/** {{{ AVL private API */
+
+struct avl_node;
+struct avl_iter;
+
+void
+avl_rebalance_after_insert(struct avl_node ***path,
+ struct avl_node ***p_end, int insert_height);
+
+void
+avl_rebalance_after_delete(struct avl_node ***path,
+ struct avl_node ***p_end);
+
+/**
+ * Find a rope node which contains the substring at @a p_offset,
+ * adjusting tree size with @a adjust_size and saving the path in
+ * @a path.
+ * @return The end of the route.
+ */
+struct avl_node ***
+avl_route_to_offset(struct avl_node ***path, uint32_t *p_offset,
+ ssize_t adjust_size);
+
+/**
+ * Route to successor or predecessor node of the node in @a path.
+ * It's either the rightmost leaf of the left child (previous
+ * node) or leftmost leaf of the right child.
+ */
+struct avl_node ***
+avl_route_to_next(struct avl_node ***path, int dir,
+ int32_t adjust_size);
+
+struct avl_node *
+avl_iter_start(struct avl_iter *it);
+
+struct avl_node *
+avl_iter_next(struct avl_iter *it);
+
+void
+avl_iter_check(struct avl_iter *iter);
+
+/** }}} AVL private API */
+
+/**
+ * Needed definitions:
+ *
+ * rope_name, optional
+ * rope_data_t
+ * rope_ctx_t
+ * ROPE_SPLIT_F
+ * ROPE_ALLOC_F
+ * ROPE_FREE_F, optional
+ */
+
+#if defined(ROPE_SRC)
+ #define rope_api(x) avl_##x
+#elif defined(rope_name)
+ #define rope_api(x) rope_##rope_name##_##x
+ #define rope rope_##rope_name
+#else
+ #define rope_api(x) rope_##x
+#endif
+
+#ifndef ROPE_FREE_F
+ #define ROPE_FREE(a, b) do { (void) a; (void) b; } while(0)
+#else
+ #define ROPE_FREE(a, b) ROPE_FREE_F(a, b)
+#endif
+
+#define rope_node rope_api(node)
+#define rope_node_size rope_api(node_size)
+#define rope_leaf_size rope_api(leaf_size)
+#define rope_leaf_data rope_api(leaf_data)
+#define rope_iter rope_api(iter)
+#define rope_create rope_api(create)
+#define rope_new rope_api(new)
+#define rope_size rope_api(size)
+#define rope_clear rope_api(clear)
+#define rope_delete rope_api(delete)
+#define rope_node_new rope_api(node_new)
+#define rope_node_split rope_api(node_split)
+#define rope_insert rope_api(insert)
+#define rope_append rope_api(append)
+#define rope_extract_node rope_api(extract_node)
+#define rope_extract rope_api(extract)
+#define rope_erase rope_api(erase)
+#define rope_iter_create rope_api(iter_create)
+#define rope_iter_new rope_api(iter_new)
+#define rope_iter_start rope_api(iter_start)
+#define rope_iter_next rope_api(iter_next)
+#define rope_iter_delete rope_api(iter_delete)
+#define rope_traverse rope_api(traverse)
+#define rope_check rope_api(check)
+#define rope_node_print rope_api(node_print)
+#define rope_pretty_print rope_api(pretty_print)
+
typedef uint32_t rope_size_t;
typedef int32_t rope_ssize_t;
-typedef void *(*rope_split_func)(void *, void *, size_t, size_t);
-typedef void *(*rope_alloc_func)(void *, size_t);
-typedef void (*rope_free_func)(void *, void *);
/** Tallest allowable tree, 1.44*log(2^32) */
-enum { ROPE_HEIGHT_MAX = 46 };
+#define ROPE_HEIGHT_MAX 46
struct rope_node {
- /** Node height, see rope_node_height(), used for AVL balance. */
+ /**
+ * Node height, see avl_node_height(), used for AVL
+ * balance.
+ */
int height;
/** Subtree size. */
rope_size_t tree_size;
/* Substring size. */
rope_size_t leaf_size;
- /* Substring. */
- void *data;
/* Left (0) and right (1) links */
struct rope_node *link[2];
+ /* Substring. */
+ rope_data_t data;
};
struct rope {
/** Top of the tree */
struct rope_node *root;
- /** Memory management context. */
- void *alloc_ctx;
- /** Get a sequence tail, given offset. */
- rope_split_func split;
- /** Split function context. */
- void *split_ctx;
- /** Allocate memory (context, size). */
- void *(*alloc)(void *, size_t);
- /** Free memory (context, pointer) */
- void (*free)(void *, void *);
+ /** Rope context. */
+ rope_ctx_t ctx;
};
struct rope_iter {
@@ -96,7 +206,7 @@ rope_leaf_size(struct rope_node *node)
return node->leaf_size;
}
-static inline void *
+static inline rope_data_t
rope_leaf_data(struct rope_node *node)
{
return node->data;
@@ -108,78 +218,178 @@ rope_size(struct rope *rope)
return rope_node_size(rope->root);
}
+/**
+ * Everything below is not needed in the rope's own source file,
+ * so it is macrosed out.
+ */
+#ifndef ROPE_SRC
+
/** Initialize an empty rope. */
static inline void
-rope_create(struct rope *rope, rope_split_func split_func, void *split_ctx,
- rope_alloc_func alloc_func, rope_free_func free_func,
- void *alloc_ctx)
+rope_create(struct rope *rope, rope_ctx_t ctx)
{
rope->root = NULL;
- rope->split = split_func;
- rope->split_ctx = split_ctx;
- rope->alloc = alloc_func;
- rope->free = free_func;
- rope->alloc_ctx = alloc_ctx;
+ rope->ctx = ctx;
}
/** Create a new empty rope.
- * @param split_func a function which returns
- * a pointer to substring
- * given an offset. Used
- * to split substrings when
- * inserting into a rope.
- * @param alloc_func used to allocate memory
- * @param free_func used to free memory
* @param alloc_ctx allocator context
*
* @return an empty rope, or NULL if failed
* to allocate memory
*/
static inline struct rope *
-rope_new(rope_split_func split_func, void *split_ctx,
- rope_alloc_func alloc_func, rope_free_func free_func, void *alloc_ctx)
+rope_new(rope_ctx_t ctx)
{
- struct rope *rope= (struct rope *) alloc_func(alloc_ctx,
- sizeof(struct rope));
- if (rope == NULL)
- return NULL;
- rope_create(rope, split_func, split_ctx, alloc_func, free_func, alloc_ctx);
+ struct rope *rope = (struct rope *) ROPE_ALLOC_F(ctx, sizeof(*rope));
+ if (rope != NULL)
+ rope_create(rope, ctx);
return rope;
}
-
/** Delete rope contents. Can also be used
* to free a rope which is allocated on stack.
* Doesn't delete rope substrings, only
* rope nodes.
*/
-void
-rope_clear(struct rope *rope);
+static void
+rope_clear(struct rope *rope)
+{
+#ifdef ROPE_FREE_F
+ struct rope_node *it = rope->root;
+ struct rope_node *save;
+
+ /* Destruction by rotation */
+ while (it != NULL) {
+ if (it->link[0] == NULL) {
+ /* Remove node */
+ save = it->link[1];
+ ROPE_FREE(rope->ctx, it);
+ } else {
+ /* Rotate right */
+ save = it->link[0];
+ it->link[0] = save->link[1];
+ save->link[1] = it;
+ }
+ it = save;
+ }
+#endif /* ROPE_FREE_F */
+ rope->root = NULL;
+}
/** Delete a rope allocated with rope_new() */
static inline void
rope_delete(struct rope *rope)
{
rope_clear(rope);
- rope->free(rope->alloc_ctx, rope);
+ ROPE_FREE(rope->ctx, rope);
+}
+
+static inline struct rope_node *
+rope_node_new(struct rope *rope, rope_data_t data, rope_size_t size)
+{
+ struct rope_node *node =
+ (struct rope_node *) ROPE_ALLOC_F(rope->ctx, sizeof(*node));
+ if (node == NULL)
+ return NULL;
+ node->height = 1;
+ node->tree_size = node->leaf_size = size;
+ node->data = data;
+ node->link[0] = node->link[1] = NULL;
+
+ return node;
+}
+
+static inline struct rope_node *
+rope_node_split(struct rope *rope, struct rope_node *node, rope_size_t offset)
+{
+ rope_size_t old_size = node->leaf_size;
+ node->leaf_size = offset;
+
+ rope_data_t data = ROPE_SPLIT_F(rope->ctx, node->data, old_size,
+ offset);
+
+ return rope_node_new(rope, data, old_size - offset);
}
-/** Insert a substring into a rope at the given
- * offset.
- * If offset is greater than rope size, insertion
- * happens at the end.
+/**
+ * Insert a substring into a rope at the given offset. If offset
+ * is greater than rope size, insertion happens at the end. A new
+ * node is always inserted at a leaf position. If insertion
+ * unbalances the tree, the rebalancing procedure may put the node
+ * into an intermediate position.
+ *
+ * While traversing the tree, we simultaneously update tree sizes
+ * of all intermediate nodes, taking into account the size of the
+ * new node.
+ *
+ * When insertion offset falls at the middle of an existing node,
+ * we truncate this node and attach its tail to the left leaf of
+ * the new node. This trim operation doesn't decrease the old
+ * subtree height, and, while it does change subtree size
+ * temporarily, as long as we attach the new node to the right
+ * subtree of the truncated node, truncation has no effect on the
+ * tree size either.
*
- * @retval 0 success
- * @retval -1 failed to allocate memory for a new
- * tree node
+ * Rebalancing, when it occurs, will correctly update subtree
+ * heights and sizes of all modified nodes.
+ *
+ * @retval 0 Success.
+ * @retval -1 Memory error.
*/
-int
-rope_insert(struct rope *rope, rope_size_t offset, void *data,
- rope_size_t size);
+static int
+rope_insert(struct rope *rope, rope_size_t offset, rope_data_t data,
+ rope_size_t size)
+{
+ if (offset > rope_size(rope))
+ offset = rope_size(rope);
+
+ assert(size);
+
+ struct rope_node *new_node = rope_node_new(rope, data, size);
+
+ if (new_node == NULL)
+ return -1;
+
+ struct rope_node **path[ROPE_HEIGHT_MAX];
+ path[0] = &rope->root;
+
+ struct rope_node ***p_end = (struct rope_node ***)
+ avl_route_to_offset((struct avl_node ***) path, &offset, size);
+ if (**p_end != NULL) {
+ /*
+ * The offset is inside an existing
+ * substring in the rope. If offset is 0,
+ * then insert the new node at the rightmost leaf
+ * of the left child. Otherwise, cut the tail of
+ * the substring, make it a prefix of the inserted
+ * string, and insert the result at the leftmost
+ * leaf of the right child.
+ */
+ if (offset != 0) {
+ struct rope_node *split_node;
+ split_node = rope_node_split(rope, **p_end, offset);
+ if (split_node == NULL)
+ return -1;
+ split_node->link[0] = new_node;
+ split_node->height++;
+ split_node->tree_size += new_node->tree_size;
+ new_node = split_node;
+ }
+ p_end = (struct rope_node ***)
+ avl_route_to_next((struct avl_node ***) p_end,
+ offset != 0, new_node->tree_size);
+ }
+ **p_end = new_node;
+ avl_rebalance_after_insert((struct avl_node ***) path,
+ (struct avl_node ***) p_end,
+ new_node->height);
+ return 0;
+}
/** Append a substring at rope tail. */
static inline int
-rope_append(struct rope *rope, void *data, size_t size)
+rope_append(struct rope *rope, rope_data_t data, size_t size)
{
return rope_insert(rope, rope_size(rope), data, size);
}
@@ -191,23 +401,149 @@ rope_append(struct rope *rope, void *data, size_t size)
* @retval NULL failed to allocate memory for a new
* tree node
*/
-struct rope_node *
-rope_extract_node(struct rope *rope, rope_size_t offset);
+static struct rope_node *
+rope_extract_node(struct rope *rope, rope_size_t offset)
+{
+ assert(offset < rope_size(rope));
+
+ struct rope_node **path[ROPE_HEIGHT_MAX];
+ path[0] = &rope->root;
-static inline void *
+ struct rope_node ***p_end = (struct rope_node ***)
+ avl_route_to_offset((struct avl_node ***) path, &offset, 0);
+ if (offset == 0)
+ return **p_end;
+ struct rope_node *new_node = rope_node_split(rope, **p_end, offset);
+ if (new_node == NULL)
+ return NULL;
+ p_end = (struct rope_node ***)
+ avl_route_to_next((struct avl_node ***) p_end, 1,
+ new_node->tree_size);
+ **p_end = new_node;
+ avl_rebalance_after_insert((struct avl_node ***) path,
+ (struct avl_node ***) p_end,
+ new_node->height);
+ return new_node;
+}
+
+static inline rope_data_t
rope_extract(struct rope *rope, rope_size_t offset)
{
return rope_leaf_data(rope_extract_node(rope, offset));
}
/**
- * Erase a single element from a rope at the given
- * offset.
+ * Erase a single element from the rope. This is a straightforward
+ * implementation for a single-element deletion from a rope. A
+ * generic cut from a rope involves 2 tree splits and one merge.
+ *
+ * When deleting a single element, 3 cases are possible:
+ * - offset falls at a node with a single element. In this case we
+ * perform a normal AVL tree delete.
+ * - offset falls at the end or the beginning of an existing node
+ * with leaf_size > 1. In that case we trim the existing node
+ * and return.
+ * - offset falls inside an existing node. In that case we split
+ * the existing node at offset, and insert the tail.
*
- * @pre offset < rope_size(rope)
+ * The implementation is a copycat of rope_insert(). If you're
+ * trying to understand the code, it's recommended to start from
+ * rope_insert().
*/
-int
-rope_erase(struct rope *rope, rope_size_t offset);
+static inline int
+rope_erase(struct rope *rope, rope_size_t offset)
+{
+ assert(offset < rope_size(rope));
+
+ struct rope_node **path[ROPE_HEIGHT_MAX];
+ path[0] = &rope->root;
+
+ struct rope_node ***p_end = (struct rope_node ***)
+ avl_route_to_offset((struct avl_node ***) path, &offset, -1);
+
+ struct rope_node *node = **p_end;
+
+ if (node->leaf_size > 1) {
+ /* Check if we can simply trim the node. */
+ if (offset == 0) {
+ /* Cut the head. */
+ node->data = ROPE_SPLIT_F(rope->ctx, node->data,
+ node->leaf_size, 1);
+ node->leaf_size -= 1;
+ return 0;
+ }
+ rope_size_t size = node->leaf_size;
+ /* Cut the tail */
+ rope_data_t next = ROPE_SPLIT_F(rope->ctx, node->data,
+ node->leaf_size, offset);
+ node->leaf_size = offset;
+ if (offset == size - 1)
+ return 0; /* Trimmed the tail, nothing else to do */
+ /*
+ * Offset falls inside a substring. Erase the
+ * first field and insert the tail.
+ */
+ next = ROPE_SPLIT_F(rope->ctx, next, size - offset, 1);
+ struct rope_node *new_node =
+ rope_node_new(rope, next, size - offset - 1);
+ if (new_node == NULL)
+ return -1;
+ /* Trim the old node. */
+ p_end = (struct rope_node ***)
+ avl_route_to_next((struct avl_node ***) p_end, 1,
+ new_node->tree_size);
+ **p_end = new_node;
+ avl_rebalance_after_insert((struct avl_node ***) path,
+ (struct avl_node ***) p_end,
+ new_node->height);
+ return 0;
+ }
+ /* We need to delete the node. */
+ assert(offset == 0);
+ int direction;
+ if (node->link[0] != NULL && node->link[1] != NULL) {
+ /*
+ * The node has two non-NULL leaves. We can't
+ * simply delete the node since in that case we
+ * won't know what to do with one of the leaves.
+ * Instead of deleting the node, store in it data
+ * from the rightmost node in the left subtree, or
+ * the leftmost node in the right subtree,
+ * (depending on which subtree is taller), and
+ * delete this leftmost/rightmost node instead.
+ */
+ struct rope_node *save = node;
+ direction = node->link[1]->height > node->link[0]->height;
+ p_end = (struct rope_node ***)
+ avl_route_to_next((struct avl_node ***) p_end,
+ direction, 0) - 1;
+ node = **p_end;
+ /* Move the data pointers. */
+ save->data = node->data;
+ save->leaf_size = node->leaf_size;
+ /*
+ * Now follow the path again and update tree_size
+ * in the parents of the moved child.
+ */
+ save = save->link[direction];
+ while (save != node) {
+ save->tree_size -= node->leaf_size;
+ save = save->link[!direction];
+ }
+ } else {
+ /*
+ * Left or right subtree are NULL, so we
+ * can simply put the non-NULL leaf in place
+ * of the parent.
+ */
+ direction = node->link[0] == NULL;
+ }
+ **p_end = node->link[direction];
+ ROPE_FREE(rope->ctx, node);
+ avl_rebalance_after_delete((struct avl_node ***) path,
+ (struct avl_node ***) p_end);
+ return 0;
+}
/** Initialize an iterator. */
static inline void
@@ -220,12 +556,10 @@ rope_iter_create(struct rope_iter *it, struct rope *rope)
static inline struct rope_iter *
rope_iter_new(struct rope *rope)
{
- struct rope_iter *it = (struct rope_iter *)
- rope->alloc(rope->alloc_ctx, sizeof(struct rope_iter));
-
- if (it == NULL)
- return NULL;
- rope_iter_create(it, rope);
+ struct rope_iter *it =
+ (struct rope_iter *) ROPE_ALLOC_F(rope->ctx, sizeof(*it));
+ if (it != NULL)
+ rope_iter_create(it, rope);
return it;
}
@@ -233,8 +567,11 @@ rope_iter_new(struct rope *rope)
* Begin iteration.
* @retval NULL the rope is empty
*/
-struct rope_node *
-rope_iter_start(struct rope_iter *it);
+static inline struct rope_node *
+rope_iter_start(struct rope_iter *it)
+{
+ return (struct rope_node *) avl_iter_start((struct avl_iter *) it);
+}
/**
* Advance to the next rope node.
@@ -243,30 +580,138 @@ rope_iter_start(struct rope_iter *it);
* has advanced beyond the last
* node.
*/
-struct rope_node *
-rope_iter_next(struct rope_iter *it);
+static struct rope_node *
+rope_iter_next(struct rope_iter *it)
+{
+ return (struct rope_node *) avl_iter_next((struct avl_iter *) it);
+}
/** Free iterator. */
static inline void
rope_iter_delete(struct rope_iter *it)
{
- it->rope->free(it->rope->alloc_ctx, it);
+ ROPE_FREE(it->rope->ctx, it);
}
/** Apply visit_leaf function to every rope leaf. */
-void
-rope_traverse(struct rope *rope, void (*visit_leaf)(void *, size_t));
+static void
+rope_traverse(struct rope *rope, void (*visit_leaf)(rope_data_t, size_t))
+{
+ struct rope_iter iter;
+ rope_iter_create(&iter, rope);
+
+ struct rope_node *leaf;
+
+ for (leaf = rope_iter_start(&iter);
+ leaf != NULL;
+ leaf = rope_iter_next(&iter)) {
+
+ visit_leaf(leaf->data, leaf->leaf_size);
+ }
+}
/** Check AVL tree consistency. */
-void
-rope_check(struct rope *rope);
+static inline void
+rope_check(struct rope *rope)
+{
+ struct rope_iter iter;
+ rope_iter_create(&iter, rope);
+ avl_iter_check((struct avl_iter *) &iter);
+}
+
+static void
+rope_node_print(struct rope_node *node, void (*print)(rope_data_t, size_t),
+ const char *prefix, int dir)
+{
+ const char *conn[] = { "┌──", "└──" };
+
+ const char *padding[] = { "│ ", " " };
+
+ rope_size_t child_prefix_len = strlen(prefix) + strlen(padding[0]) + 1;
+ char *child_prefix = malloc(child_prefix_len);
+
+ if (node && (node->link[0] || node->link[1])) {
+ snprintf(child_prefix, child_prefix_len - 1,
+ "%s%s", prefix, padding[!dir]);
+ rope_node_print(node->link[0], print, child_prefix, 0);
+ }
+
+ snprintf(child_prefix, child_prefix_len - 1, "%s%s",
+ prefix, padding[dir]);
+ printf("%s%s", prefix, conn[dir]);
+
+ if (node == NULL) {
+ printf("nil\n");
+ } else {
+
+ printf("{ len = %zu, height = %d, data = '",
+ (size_t) node->leaf_size, node->height);
+ print(node->data, node->leaf_size);
+ printf("'}\n");
+
+ if (node->link[0] || node->link[1])
+ rope_node_print(node->link[1], print, child_prefix, 1);
+ }
+
+ free(child_prefix);
+}
/** Pretty print a rope. */
-void
-rope_pretty_print(struct rope *rope, void (*print_leaf)(void *, size_t));
+static inline void
+rope_pretty_print(struct rope *rope, void (*print_leaf)(rope_data_t, size_t))
+{
+ printf("size = %zu\nstring = '", (size_t) rope_size(rope));
+ rope_traverse(rope, print_leaf);
+ printf("'\n");
+ rope_node_print(rope->root, print_leaf, "", true);
+ printf("\n");
+}
+
+#ifdef NEED_UNDEF_MAX
+#undef NEED_UNDEF_MAX
+#undef MAX
+#endif
+
+#endif /* ROPE_SRC */
+
+#undef rope_node
+#undef rope_node_size
+#undef rope_leaf_size
+#undef rope_leaf_data
+#undef rope_iter
+#undef rope_create
+#undef rope_new
+#undef rope_size
+#undef rope_clear
+#undef rope_delete
+#undef rope_node_new
+#undef rope_node_split
+#undef rope_insert
+#undef rope_append
+#undef rope_extract_node
+#undef rope_extract
+#undef rope_erase
+#undef rope_iter_create
+#undef rope_iter_new
+#undef rope_iter_start
+#undef rope_iter_next
+#undef rope_iter_delete
+#undef rope_traverse
+#undef rope_check
+#undef rope_node_print
+#undef rope_pretty_print
+
+#undef rope
+#undef rope_api
+#undef rope_name
+#undef ROPE_HEIGHT_MAX
+#undef rope_ctx_t
+#undef rope_data_t
+#undef ROPE_SPLIT_F
+#undef ROPE_ALLOC_F
+#undef ROPE_FREE_F
+#undef ROPE_FREE
#if defined(__cplusplus)
}
#endif /* defined(__cplusplus) */
-
-#endif /* TARANTOOL_LIB_SALAD_ROPE_H_INCLUDED */
diff --git a/test/unit/rope.c b/test/unit/rope.c
index e19dcaa10..5ab4c51f7 100644
--- a/test/unit/rope.c
+++ b/test/unit/rope.c
@@ -1,4 +1,3 @@
-#include "salad/rope.h"
#include "unit.h"
#include "rope_common.h"
diff --git a/test/unit/rope_avl.c b/test/unit/rope_avl.c
index 9e978f46c..b095fb1a6 100644
--- a/test/unit/rope_avl.c
+++ b/test/unit/rope_avl.c
@@ -1,4 +1,3 @@
-#include "salad/rope.h"
#include "unit.h"
#include "rope_common.h"
diff --git a/test/unit/rope_basic.c b/test/unit/rope_basic.c
index b4a0e7876..d9e116292 100644
--- a/test/unit/rope_basic.c
+++ b/test/unit/rope_basic.c
@@ -1,4 +1,3 @@
-#include "salad/rope.h"
#include "unit.h"
#include "rope_common.h"
diff --git a/test/unit/rope_common.h b/test/unit/rope_common.h
index 6c52153b2..3392e87e3 100644
--- a/test/unit/rope_common.h
+++ b/test/unit/rope_common.h
@@ -3,17 +3,17 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
-static inline void *
-str_getn(void *ctx, void *data, size_t size, size_t offset)
+static inline char *
+str_getn(void *ctx, char *data, size_t size, size_t offset)
{
(void) ctx;
- return (char *) data + offset;
+ return data + offset;
}
static inline void
-str_print(void *data, size_t n)
+str_print(char *data, size_t n)
{
- printf("%.*s", (int) n, (char *) data);
+ printf("%.*s", (int) n, data);
}
static inline void *
@@ -30,10 +30,39 @@ mem_free(void *data, void *ptr)
free(ptr);
}
+#define ROPE_ALLOC_F mem_alloc
+#define ROPE_FREE_F mem_free
+#define ROPE_SPLIT_F str_getn
+#define rope_data_t char *
+#define rope_ctx_t void *
+
+#include "salad/rope.h"
+
+/**
+ * Define a second rope just to check if compilation of two
+ * ropes in one object file works.
+ */
+
+static inline int *
+str_getn2(int *ctx, int *data, size_t size, size_t offset)
+{
+ (void) ctx;
+ return data + offset;
+}
+
+#define rope_name second
+#define ROPE_ALLOC_F mem_alloc
+#define ROPE_FREE_F mem_free
+#define ROPE_SPLIT_F str_getn2
+#define rope_data_t int *
+#define rope_ctx_t int *
+
+#include "salad/rope.h"
+
static inline struct rope *
test_rope_new()
{
- return rope_new(str_getn, NULL, mem_alloc, mem_free, NULL);
+ return rope_new(NULL);
}
static inline void
diff --git a/test/unit/rope_stress.c b/test/unit/rope_stress.c
index dc3f2817e..bc312d7c0 100644
--- a/test/unit/rope_stress.c
+++ b/test/unit/rope_stress.c
@@ -1,4 +1,3 @@
-#include "salad/rope.h"
#include <time.h>
#include "unit.h"
#include "rope_common.h"
@@ -10,7 +9,7 @@ test_rope_stress_small()
{
header();
- struct rope *rope = rope_new(str_getn, NULL, mem_alloc, mem_free, NULL);
+ struct rope *rope = test_rope_new();
const int iterations = 500;
int i = 0;
for (i = 0; i < iterations; ++i) {
@@ -39,7 +38,7 @@ test_rope_stress_large()
{
header();
- struct rope *rope = rope_new(str_getn, NULL, mem_alloc, mem_free, NULL);
+ struct rope *rope = test_rope_new();
const int iterations = 50000;
int i = 0;
for (i = 0; i < iterations; ++i) {
--
2.20.1 (Apple Git-117)
^ permalink raw reply [flat|nested] 15+ messages in thread* [tarantool-patches] [PATCH 03/13] tuple: relax struct tuple_update dependency on rope
2019-08-12 23:05 [tarantool-patches] [PATCH 00/13] JSON updates Vladislav Shpilevoy
` (5 preceding siblings ...)
2019-08-12 23:05 ` [tarantool-patches] [PATCH 02/13] rope: make rope library macro template Vladislav Shpilevoy
@ 2019-08-12 23:05 ` Vladislav Shpilevoy
2019-08-12 23:05 ` [tarantool-patches] [PATCH 04/13] int96: add a missing header Vladislav Shpilevoy
` (6 subsequent siblings)
13 siblings, 0 replies; 15+ messages in thread
From: Vladislav Shpilevoy @ 2019-08-12 23:05 UTC (permalink / raw)
To: tarantool-patches
Rope will not be used by tuple_update directly in
next patches.
---
src/box/tuple_update.c | 34 ++++++++++++++++++----------------
1 file changed, 18 insertions(+), 16 deletions(-)
diff --git a/src/box/tuple_update.c b/src/box/tuple_update.c
index 7498d558a..21ea876c9 100644
--- a/src/box/tuple_update.c
+++ b/src/box/tuple_update.c
@@ -820,31 +820,29 @@ update_field_split(struct region *region, struct update_field *prev,
* @retval 0 Success.
* @retval -1 Error.
*/
-int
-update_create_rope(struct tuple_update *update, const char *tuple_data,
- const char *tuple_data_end, uint32_t field_count)
+struct rope *
+tuple_rope_new(struct region *region, const char *tuple_data,
+ const char *tuple_data_end, uint32_t field_count)
{
- update->rope = rope_new(update->region);
- if (update->rope == NULL)
- return -1;
+ struct rope *rope = rope_new(region);
+ if (rope == NULL)
+ return NULL;
/* Initialize the rope with the old tuple. */
-
- struct update_field *first = (struct update_field *)
- update_alloc(update->region, sizeof(*first));
+ struct update_field *first =
+ (struct update_field *) update_alloc(region, sizeof(*first));
if (first == NULL)
- return -1;
+ return NULL;
const char *field = tuple_data;
const char *end = tuple_data_end;
if (field == end)
- return 0;
+ return rope;
/* Add first field to rope */
mp_next(&tuple_data);
uint32_t field_len = tuple_data - field;
- update_field_init(first, field, field_len,
- end - field - field_len);
+ update_field_init(first, field, field_len, end - field - field_len);
- return rope_append(update->rope, first, field_count);
+ return rope_append(rope, first, field_count) != 0 ? NULL : rope;
}
static uint32_t
@@ -1119,7 +1117,9 @@ static int
update_do_ops(struct tuple_update *update, const char *old_data,
const char *old_data_end, uint32_t part_count)
{
- if (update_create_rope(update, old_data, old_data_end, part_count) != 0)
+ update->rope = tuple_rope_new(update->region, old_data, old_data_end,
+ part_count);
+ if (update->rope == NULL)
return -1;
struct update_op *op = update->ops;
struct update_op *ops_end = op + update->op_count;
@@ -1140,7 +1140,9 @@ upsert_do_ops(struct tuple_update *update, const char *old_data,
const char *old_data_end, uint32_t part_count,
bool suppress_error)
{
- if (update_create_rope(update, old_data, old_data_end, part_count) != 0)
+ update->rope = tuple_rope_new(update->region, old_data, old_data_end,
+ part_count);
+ if (update->rope == NULL)
return -1;
struct update_op *op = update->ops;
struct update_op *ops_end = op + update->op_count;
--
2.20.1 (Apple Git-117)
^ permalink raw reply [flat|nested] 15+ messages in thread* [tarantool-patches] [PATCH 04/13] int96: add a missing header
2019-08-12 23:05 [tarantool-patches] [PATCH 00/13] JSON updates Vladislav Shpilevoy
` (6 preceding siblings ...)
2019-08-12 23:05 ` [tarantool-patches] [PATCH 03/13] tuple: relax struct tuple_update dependency on rope Vladislav Shpilevoy
@ 2019-08-12 23:05 ` Vladislav Shpilevoy
2019-08-12 23:05 ` [tarantool-patches] [PATCH 05/13] tuple: implement update by field name Vladislav Shpilevoy
` (5 subsequent siblings)
13 siblings, 0 replies; 15+ messages in thread
From: Vladislav Shpilevoy @ 2019-08-12 23:05 UTC (permalink / raw)
To: tarantool-patches
---
src/lib/bit/int96.h | 1 +
1 file changed, 1 insertion(+)
diff --git a/src/lib/bit/int96.h b/src/lib/bit/int96.h
index f615e2fe9..cdde0e995 100644
--- a/src/lib/bit/int96.h
+++ b/src/lib/bit/int96.h
@@ -33,6 +33,7 @@
#include <inttypes.h>
#include <assert.h>
+#include <stdbool.h>
/**
* 96-bit signed integer.
--
2.20.1 (Apple Git-117)
^ permalink raw reply [flat|nested] 15+ messages in thread* [tarantool-patches] [PATCH 05/13] tuple: implement update by field name
2019-08-12 23:05 [tarantool-patches] [PATCH 00/13] JSON updates Vladislav Shpilevoy
` (7 preceding siblings ...)
2019-08-12 23:05 ` [tarantool-patches] [PATCH 04/13] int96: add a missing header Vladislav Shpilevoy
@ 2019-08-12 23:05 ` Vladislav Shpilevoy
2019-08-12 23:05 ` [tarantool-patches] [PATCH 06/13] tuple: expose JSON go_to_key and go_to_index functions Vladislav Shpilevoy
` (4 subsequent siblings)
13 siblings, 0 replies; 15+ messages in thread
From: Vladislav Shpilevoy @ 2019-08-12 23:05 UTC (permalink / raw)
To: tarantool-patches
Tuple fields can be named, accessed by name, indexed by name, but
till this commit field names couldn't be used in update
operations. Now it is possible.
This patch is a teaser of updates by JSON path.
Part of #1261
---
src/box/errcode.h | 4 +-
src/box/lua/tuple.c | 5 +-
src/box/memtx_space.c | 17 ++--
src/box/space.c | 9 ++-
src/box/sql/insert.c | 3 +-
src/box/sql/resolve.c | 4 +-
src/box/sql/update.c | 2 +-
src/box/tuple.c | 12 +--
src/box/tuple_update.c | 152 ++++++++++++++++++++++--------------
src/box/tuple_update.h | 15 ++--
src/box/vinyl.c | 9 ++-
src/box/vy_upsert.c | 6 +-
test/box/misc.result | 3 +-
test/box/update.result | 2 +-
test/engine/update.result | 67 ++++++++++++++++
test/engine/update.test.lua | 28 +++++++
test/engine/upsert.result | 2 +-
test/unit/column_mask.c | 5 +-
18 files changed, 247 insertions(+), 98 deletions(-)
diff --git a/src/box/errcode.h b/src/box/errcode.h
index 817275b97..dd55bd42a 100644
--- a/src/box/errcode.h
+++ b/src/box/errcode.h
@@ -205,14 +205,14 @@ struct errcode_record {
/*150 */_(ER_CANT_CREATE_COLLATION, "Failed to initialize collation: %s.") \
/*151 */_(ER_WRONG_COLLATION_OPTIONS, "Wrong collation options (field %u): %s") \
/*152 */_(ER_NULLABLE_PRIMARY, "Primary index of space '%s' can not contain nullable parts") \
- /*153 */_(ER_NO_SUCH_FIELD_NAME, "Field '%s' was not found in space '%s' format") \
+ /*153 */_(ER_NO_SUCH_FIELD_NAME_IN_SPACE, "Field '%s' was not found in space '%s' format") \
/*154 */_(ER_TRANSACTION_YIELD, "Transaction has been aborted by a fiber yield") \
/*155 */_(ER_NO_SUCH_GROUP, "Replication group '%s' does not exist") \
/*156 */_(ER_SQL_BIND_VALUE, "Bind value for parameter %s is out of range for type %s") \
/*157 */_(ER_SQL_BIND_TYPE, "Bind value type %s for parameter %s is not supported") \
/*158 */_(ER_SQL_BIND_PARAMETER_MAX, "SQL bind parameter limit reached: %d") \
/*159 */_(ER_SQL_EXECUTE, "Failed to execute SQL statement: %s") \
- /*160 */_(ER_UNUSED, "") \
+ /*160 */_(ER_NO_SUCH_FIELD_NAME, "Field '%s' was not found in the tuple") \
/*161 */_(ER_SQL_BIND_NOT_FOUND, "Parameter %s was not found in the statement") \
/*162 */_(ER_ACTION_MISMATCH, "Field %s contains %s on conflict action, but %s in index parts") \
/*163 */_(ER_VIEW_MISSING_SQL, "Space declared as a view must have SQL statement") \
diff --git a/src/box/lua/tuple.c b/src/box/lua/tuple.c
index 2fbfb1473..3902288bf 100644
--- a/src/box/lua/tuple.c
+++ b/src/box/lua/tuple.c
@@ -439,6 +439,7 @@ lbox_tuple_transform(struct lua_State *L)
const char *old_data = tuple_data_range(tuple, &bsize);
struct region *region = &fiber()->gc;
size_t used = region_used(region);
+ struct tuple_format *format = tuple_format(tuple);
struct tuple *new_tuple = NULL;
/*
* Can't use box_tuple_update() since transform must reset
@@ -449,8 +450,8 @@ lbox_tuple_transform(struct lua_State *L)
*/
const char *new_data =
tuple_update_execute(buf->buf, buf->buf + ibuf_used(buf),
- old_data, old_data + bsize, &new_size, 1,
- NULL);
+ old_data, old_data + bsize, format->dict,
+ &new_size, 1, NULL);
if (new_data != NULL)
new_tuple = tuple_new(box_tuple_format_default(),
new_data, new_data + new_size);
diff --git a/src/box/memtx_space.c b/src/box/memtx_space.c
index d63e8a5ca..29956c059 100644
--- a/src/box/memtx_space.c
+++ b/src/box/memtx_space.c
@@ -417,15 +417,16 @@ memtx_space_execute_update(struct space *space, struct txn *txn,
/* Update the tuple; legacy, request ops are in request->tuple */
uint32_t new_size = 0, bsize;
+ struct tuple_format *format = space->format;
const char *old_data = tuple_data_range(old_tuple, &bsize);
const char *new_data =
tuple_update_execute(request->tuple, request->tuple_end,
- old_data, old_data + bsize,
+ old_data, old_data + bsize, format->dict,
&new_size, request->index_base, NULL);
if (new_data == NULL)
return -1;
- stmt->new_tuple = memtx_tuple_new(space->format, new_data,
+ stmt->new_tuple = memtx_tuple_new(format, new_data,
new_data + new_size);
if (stmt->new_tuple == NULL)
return -1;
@@ -471,6 +472,7 @@ memtx_space_execute_upsert(struct space *space, struct txn *txn,
if (index_get(index, key, part_count, &old_tuple) != 0)
return -1;
+ struct tuple_format *format = space->format;
if (old_tuple == NULL) {
/**
* Old tuple was not found. A write optimized
@@ -489,11 +491,11 @@ memtx_space_execute_upsert(struct space *space, struct txn *txn,
* @sa https://github.com/tarantool/tarantool/issues/1156
*/
if (tuple_update_check_ops(request->ops, request->ops_end,
+ format->dict,
request->index_base) != 0) {
return -1;
}
- stmt->new_tuple = memtx_tuple_new(space->format,
- request->tuple,
+ stmt->new_tuple = memtx_tuple_new(format, request->tuple,
request->tuple_end);
if (stmt->new_tuple == NULL)
return -1;
@@ -511,12 +513,13 @@ memtx_space_execute_upsert(struct space *space, struct txn *txn,
const char *new_data =
tuple_upsert_execute(request->ops, request->ops_end,
old_data, old_data + bsize,
- &new_size, request->index_base,
- false, &column_mask);
+ format->dict, &new_size,
+ request->index_base, false,
+ &column_mask);
if (new_data == NULL)
return -1;
- stmt->new_tuple = memtx_tuple_new(space->format, new_data,
+ stmt->new_tuple = memtx_tuple_new(format, new_data,
new_data + new_size);
if (stmt->new_tuple == NULL)
return -1;
diff --git a/src/box/space.c b/src/box/space.c
index fffe03c41..b9a830e78 100644
--- a/src/box/space.c
+++ b/src/box/space.c
@@ -397,7 +397,8 @@ space_before_replace(struct space *space, struct txn *txn,
old_data_end = old_data + old_size;
new_data = tuple_update_execute(request->tuple,
request->tuple_end, old_data,
- old_data_end, &new_size,
+ old_data_end,
+ space->format->dict, &new_size,
request->index_base, NULL);
if (new_data == NULL)
return -1;
@@ -420,6 +421,7 @@ space_before_replace(struct space *space, struct txn *txn,
new_data_end = request->tuple_end;
if (tuple_update_check_ops(request->ops,
request->ops_end,
+ space->format->dict,
request->index_base) != 0)
return -1;
break;
@@ -428,8 +430,9 @@ space_before_replace(struct space *space, struct txn *txn,
old_data_end = old_data + old_size;
new_data = tuple_upsert_execute(request->ops, request->ops_end,
old_data, old_data_end,
- &new_size, request->index_base,
- false, NULL);
+ space->format->dict, &new_size,
+ request->index_base, false,
+ NULL);
new_data_end = new_data + new_size;
break;
default:
diff --git a/src/box/sql/insert.c b/src/box/sql/insert.c
index 1c956567f..42b839166 100644
--- a/src/box/sql/insert.c
+++ b/src/box/sql/insert.c
@@ -373,7 +373,8 @@ sqlInsert(Parse * pParse, /* Parser context */
}
}
if (j >= (int) space_def->field_count) {
- diag_set(ClientError, ER_NO_SUCH_FIELD_NAME,
+ diag_set(ClientError,
+ ER_NO_SUCH_FIELD_NAME_IN_SPACE,
pColumn->a[i].zName,
pTabList->a[0].zName);
pParse->is_aborted = true;
diff --git a/src/box/sql/resolve.c b/src/box/sql/resolve.c
index 0b90edd06..bb2e94e9e 100644
--- a/src/box/sql/resolve.c
+++ b/src/box/sql/resolve.c
@@ -431,8 +431,8 @@ lookupName(Parse * pParse, /* The parsing context */
if (zTab == NULL) {
diag_set(ClientError, ER_SQL_CANT_RESOLVE_FIELD, zCol);
} else {
- diag_set(ClientError, ER_NO_SUCH_FIELD_NAME, zCol,
- zTab);
+ diag_set(ClientError, ER_NO_SUCH_FIELD_NAME_IN_SPACE,
+ zCol, zTab);
}
pParse->is_aborted = true;
pTopNC->nErr++;
diff --git a/src/box/sql/update.c b/src/box/sql/update.c
index 7f9358c05..2d7ebf8cd 100644
--- a/src/box/sql/update.c
+++ b/src/box/sql/update.c
@@ -167,7 +167,7 @@ sqlUpdate(Parse * pParse, /* The parser context */
}
}
if (j >= (int)def->field_count) {
- diag_set(ClientError, ER_NO_SUCH_FIELD_NAME,
+ diag_set(ClientError, ER_NO_SUCH_FIELD_NAME_IN_SPACE,
pChanges->a[i].zName, def->name);
pParse->is_aborted = true;
goto update_cleanup;
diff --git a/src/box/tuple.c b/src/box/tuple.c
index beb290a9f..bf4ea711d 100644
--- a/src/box/tuple.c
+++ b/src/box/tuple.c
@@ -707,15 +707,15 @@ box_tuple_update(box_tuple_t *tuple, const char *expr, const char *expr_end)
const char *old_data = tuple_data_range(tuple, &bsize);
struct region *region = &fiber()->gc;
size_t used = region_used(region);
+ struct tuple_format *format = tuple_format(tuple);
const char *new_data =
tuple_update_execute(expr, expr_end, old_data, old_data + bsize,
- &new_size, 1, NULL);
+ format->dict, &new_size, 1, NULL);
if (new_data == NULL) {
region_truncate(region, used);
return NULL;
}
- struct tuple *ret = tuple_new(tuple_format(tuple), new_data,
- new_data + new_size);
+ struct tuple *ret = tuple_new(format, new_data, new_data + new_size);
region_truncate(region, used);
if (ret != NULL)
return tuple_bless(ret);
@@ -729,16 +729,16 @@ box_tuple_upsert(box_tuple_t *tuple, const char *expr, const char *expr_end)
const char *old_data = tuple_data_range(tuple, &bsize);
struct region *region = &fiber()->gc;
size_t used = region_used(region);
+ struct tuple_format *format = tuple_format(tuple);
const char *new_data =
tuple_upsert_execute(expr, expr_end, old_data, old_data + bsize,
- &new_size, 1, false, NULL);
+ format->dict, &new_size, 1, false, NULL);
if (new_data == NULL) {
region_truncate(region, used);
return NULL;
}
- struct tuple *ret = tuple_new(tuple_format(tuple),
- new_data, new_data + new_size);
+ struct tuple *ret = tuple_new(format, new_data, new_data + new_size);
region_truncate(region, used);
if (ret != NULL)
return tuple_bless(ret);
diff --git a/src/box/tuple_update.c b/src/box/tuple_update.c
index 21ea876c9..34e0e767f 100644
--- a/src/box/tuple_update.c
+++ b/src/box/tuple_update.c
@@ -41,6 +41,8 @@
#include <bit/int96.h>
#include "column_mask.h"
#include "fiber.h"
+#include "tuple_dictionary.h"
+#include "tt_static.h"
/** UPDATE request implementation.
* UPDATE request is represented by a sequence of operations, each
@@ -929,12 +931,87 @@ update_op_by(char opcode)
}
}
+/**
+ * Decode an update operation from MessagePack.
+ * @param[out] op Update operation.
+ * @param index_base Field numbers base: 0 or 1.
+ * @param dict Dictionary to lookup field number by a name.
+ * @param expr MessagePack.
+ *
+ * @retval 0 Success.
+ * @retval -1 Client error.
+ */
+static inline int
+update_op_decode(struct update_op *op, int index_base,
+ struct tuple_dictionary *dict, const char **expr)
+{
+ if (mp_typeof(**expr) != MP_ARRAY) {
+ diag_set(ClientError, ER_ILLEGAL_PARAMS, "update operation "
+ "must be an array {op,..}");
+ return -1;
+ }
+ uint32_t len, args = mp_decode_array(expr);
+ if (args < 1) {
+ diag_set(ClientError, ER_ILLEGAL_PARAMS, "update operation "\
+ "must be an array {op,..}, got empty array");
+ return -1;
+ }
+ if (mp_typeof(**expr) != MP_STR) {
+ diag_set(ClientError, ER_ILLEGAL_PARAMS,
+ "update operation name must be a string");
+ return -1;
+ }
+ op->opcode = *mp_decode_str(expr, &len);
+ op->meta = update_op_by(op->opcode);
+ if (op->meta == NULL)
+ return -1;
+ if (args != op->meta->args) {
+ diag_set(ClientError, ER_UNKNOWN_UPDATE_OP);
+ return -1;
+ }
+ int32_t field_no;
+ switch(mp_typeof(**expr)) {
+ case MP_INT:
+ case MP_UINT: {
+ if (mp_read_i32(index_base, op, expr, &field_no) != 0)
+ return -1;
+ if (field_no - index_base >= 0) {
+ op->field_no = field_no - index_base;
+ } else if (field_no < 0) {
+ op->field_no = field_no;
+ } else {
+ diag_set(ClientError, ER_NO_SUCH_FIELD_NO, field_no);
+ return -1;
+ }
+ break;
+ }
+ case MP_STR: {
+ const char *path = mp_decode_str(expr, &len);
+ uint32_t field_no, hash = field_name_hash(path, len);
+ if (tuple_fieldno_by_name(dict, path, len, hash,
+ &field_no) != 0) {
+ diag_set(ClientError, ER_NO_SUCH_FIELD_NAME,
+ tt_cstr(path, len));
+ return -1;
+ }
+ op->field_no = (int32_t) field_no;
+ break;
+ }
+ default:
+ diag_set(ClientError, ER_ILLEGAL_PARAMS,
+ "field id must be a number or a string");
+ return -1;
+ }
+ return op->meta->read_arg(index_base, op, expr);
+}
+
/**
* Read and check update operations and fill column mask.
*
* @param[out] update Update meta.
* @param expr MessagePack array of operations.
* @param expr_end End of the @expr.
+ * @param dict Dictionary to lookup field number by a name.
* @param field_count_hint Field count in the updated tuple. If
* there is no tuple at hand (for example, when we are
* reading UPSERT operations), then 0 for field count will
@@ -948,7 +1025,8 @@ update_op_by(char opcode)
*/
static int
update_read_ops(struct tuple_update *update, const char *expr,
- const char *expr_end, int32_t field_count_hint)
+ const char *expr_end, struct tuple_dictionary *dict,
+ int32_t field_count_hint)
{
if (mp_typeof(*expr) != MP_ARRAY) {
diag_set(ClientError, ER_ILLEGAL_PARAMS,
@@ -974,59 +1052,14 @@ update_read_ops(struct tuple_update *update, const char *expr,
struct update_op *op = update->ops;
struct update_op *ops_end = op + update->op_count;
for (; op < ops_end; op++) {
- if (mp_typeof(*expr) != MP_ARRAY) {
- diag_set(ClientError, ER_ILLEGAL_PARAMS,
- "update operation"
- " must be an array {op,..}");
- return -1;
- }
- /* Read operation */
- uint32_t args, len;
- args = mp_decode_array(&expr);
- if (args < 1) {
- diag_set(ClientError, ER_ILLEGAL_PARAMS,
- "update operation must be an "
- "array {op,..}, got empty array");
- return -1;
- }
- if (mp_typeof(*expr) != MP_STR) {
- diag_set(ClientError, ER_ILLEGAL_PARAMS,
- "update operation name must be a string");
- return -1;
- }
-
- op->opcode = *mp_decode_str(&expr, &len);
- op->meta = update_op_by(op->opcode);
- if (op->meta == NULL)
- return -1;
- if (args != op->meta->args) {
- diag_set(ClientError, ER_UNKNOWN_UPDATE_OP);
- return -1;
- }
- if (mp_typeof(*expr) != MP_INT && mp_typeof(*expr) != MP_UINT) {
- diag_set(ClientError, ER_ILLEGAL_PARAMS,
- "field id must be a number");
- return -1;
- }
- int32_t field_no = 0;
- if (mp_read_i32(update->index_base, op, &expr, &field_no))
- return -1;
- if (field_no - update->index_base >= 0) {
- op->field_no = field_no - update->index_base;
- } else if (field_no < 0) {
- op->field_no = field_no;
- } else {
- diag_set(ClientError, ER_NO_SUCH_FIELD_NO, field_no);
+ if (update_op_decode(op, update->index_base, dict, &expr) != 0)
return -1;
- }
- if (op->meta->read_arg(update->index_base, op, &expr))
- return -1;
-
/*
* Continue collecting the changed columns
* only if there are unset bits in the mask.
*/
if (column_mask != COLUMN_MASK_FULL) {
+ int32_t field_no;
if (op->field_no >= 0)
field_no = op->field_no;
else if (op->opcode != '!')
@@ -1184,24 +1217,25 @@ update_finish(struct tuple_update *update, uint32_t *p_tuple_len)
}
int
-tuple_update_check_ops(const char *expr, const char *expr_end, int index_base)
+tuple_update_check_ops(const char *expr, const char *expr_end,
+ struct tuple_dictionary *dict, int index_base)
{
struct tuple_update update;
update_init(&update, index_base);
- return update_read_ops(&update, expr, expr_end, 0);
+ return update_read_ops(&update, expr, expr_end, dict, 0);
}
const char *
tuple_update_execute(const char *expr,const char *expr_end,
const char *old_data, const char *old_data_end,
- uint32_t *p_tuple_len, int index_base,
- uint64_t *column_mask)
+ struct tuple_dictionary *dict, uint32_t *p_tuple_len,
+ int index_base, uint64_t *column_mask)
{
struct tuple_update update;
update_init(&update, index_base);
uint32_t field_count = mp_decode_array(&old_data);
- if (update_read_ops(&update, expr, expr_end, field_count) != 0)
+ if (update_read_ops(&update, expr, expr_end, dict, field_count) != 0)
return NULL;
if (update_do_ops(&update, old_data, old_data_end, field_count))
return NULL;
@@ -1214,14 +1248,14 @@ tuple_update_execute(const char *expr,const char *expr_end,
const char *
tuple_upsert_execute(const char *expr,const char *expr_end,
const char *old_data, const char *old_data_end,
- uint32_t *p_tuple_len, int index_base, bool suppress_error,
- uint64_t *column_mask)
+ struct tuple_dictionary *dict, uint32_t *p_tuple_len,
+ int index_base, bool suppress_error, uint64_t *column_mask)
{
struct tuple_update update;
update_init(&update, index_base);
uint32_t field_count = mp_decode_array(&old_data);
- if (update_read_ops(&update, expr, expr_end, field_count) != 0)
+ if (update_read_ops(&update, expr, expr_end, dict, field_count) != 0)
return NULL;
if (upsert_do_ops(&update, old_data, old_data_end, field_count,
suppress_error))
@@ -1235,14 +1269,16 @@ tuple_upsert_execute(const char *expr,const char *expr_end,
const char *
tuple_upsert_squash(const char *expr1, const char *expr1_end,
const char *expr2, const char *expr2_end,
- size_t *result_size, int index_base)
+ struct tuple_dictionary *dict, size_t *result_size,
+ int index_base)
{
const char *expr[2] = {expr1, expr2};
const char *expr_end[2] = {expr1_end, expr2_end};
struct tuple_update update[2];
for (int j = 0; j < 2; j++) {
update_init(&update[j], index_base);
- if (update_read_ops(&update[j], expr[j], expr_end[j], 0))
+ if (update_read_ops(&update[j], expr[j], expr_end[j],
+ dict, 0) != 0)
return NULL;
mp_decode_array(&expr[j]);
int32_t prev_field_no = index_base - 1;
diff --git a/src/box/tuple_update.h b/src/box/tuple_update.h
index 37faa1918..b6210dd38 100644
--- a/src/box/tuple_update.h
+++ b/src/box/tuple_update.h
@@ -44,19 +44,23 @@ enum {
BOX_UPDATE_OP_CNT_MAX = 4000,
};
+struct tuple_dictionary;
+
int
-tuple_update_check_ops(const char *expr, const char *expr_end, int index_base);
+tuple_update_check_ops(const char *expr, const char *expr_end,
+ struct tuple_dictionary *dict, int index_base);
const char *
tuple_update_execute(const char *expr,const char *expr_end,
const char *old_data, const char *old_data_end,
- uint32_t *p_new_size, int index_base,
- uint64_t *column_mask);
+ struct tuple_dictionary *dict, uint32_t *p_new_size,
+ int index_base, uint64_t *column_mask);
const char *
tuple_upsert_execute(const char *expr, const char *expr_end,
const char *old_data, const char *old_data_end,
- uint32_t *p_new_size, int index_base, bool suppress_error,
+ struct tuple_dictionary *dict, uint32_t *p_new_size,
+ int index_base, bool suppress_error,
uint64_t *column_mask);
/**
@@ -72,7 +76,8 @@ tuple_upsert_execute(const char *expr, const char *expr_end,
const char *
tuple_upsert_squash(const char *expr1, const char *expr1_end,
const char *expr2, const char *expr2_end,
- size_t *result_size, int index_base);
+ struct tuple_dictionary *dict, size_t *result_size,
+ int index_base);
#if defined(__cplusplus)
} /* extern "C" */
diff --git a/src/box/vinyl.c b/src/box/vinyl.c
index 4c5c08852..5fdfe0aa7 100644
--- a/src/box/vinyl.c
+++ b/src/box/vinyl.c
@@ -1943,7 +1943,8 @@ vy_update(struct vy_env *env, struct vy_tx *tx, struct txn_stmt *stmt,
const char *old_tuple = tuple_data_range(stmt->old_tuple, &old_size);
const char *old_tuple_end = old_tuple + old_size;
new_tuple = tuple_update_execute(request->tuple, request->tuple_end,
- old_tuple, old_tuple_end, &new_size,
+ old_tuple, old_tuple_end,
+ pk->mem_format->dict, &new_size,
request->index_base, &column_mask);
if (new_tuple == NULL)
return -1;
@@ -2122,7 +2123,8 @@ vy_upsert(struct vy_env *env, struct vy_tx *tx, struct txn_stmt *stmt,
return 0;
/* Check update operations. */
if (tuple_update_check_ops(request->ops, request->ops_end,
- request->index_base)) {
+ pk->mem_format->dict,
+ request->index_base) != 0) {
return -1;
}
if (request->index_base != 0) {
@@ -2180,7 +2182,8 @@ vy_upsert(struct vy_env *env, struct vy_tx *tx, struct txn_stmt *stmt,
/* Apply upsert operations to the old tuple. */
new_tuple = tuple_upsert_execute(ops, ops_end, old_tuple, old_tuple_end,
- &new_size, 0, false, &column_mask);
+ pk->mem_format->dict, &new_size, 0,
+ false, &column_mask);
if (new_tuple == NULL)
return -1;
/*
diff --git a/src/box/vy_upsert.c b/src/box/vy_upsert.c
index 1c55f86c2..817d29cfd 100644
--- a/src/box/vy_upsert.c
+++ b/src/box/vy_upsert.c
@@ -58,7 +58,7 @@ vy_upsert_try_to_squash(struct tuple_format *format,
size_t squashed_size;
const char *squashed =
tuple_upsert_squash(old_serie, old_serie_end,
- new_serie, new_serie_end,
+ new_serie, new_serie_end, format->dict,
&squashed_size, 0);
if (squashed == NULL)
return 0;
@@ -119,8 +119,8 @@ vy_apply_upsert(struct tuple *new_stmt, struct tuple *old_stmt,
uint8_t old_type = vy_stmt_type(old_stmt);
uint64_t column_mask = COLUMN_MASK_FULL;
result_mp = tuple_upsert_execute(new_ops, new_ops_end, result_mp,
- result_mp_end, &mp_size, 0,
- suppress_error, &column_mask);
+ result_mp_end, format->dict, &mp_size,
+ 0, suppress_error, &column_mask);
result_mp_end = result_mp + mp_size;
if (old_type != IPROTO_UPSERT) {
assert(old_type == IPROTO_INSERT ||
diff --git a/test/box/misc.result b/test/box/misc.result
index 7a15dabf0..f45e0514c 100644
--- a/test/box/misc.result
+++ b/test/box/misc.result
@@ -483,13 +483,14 @@ t;
150: box.error.CANT_CREATE_COLLATION
151: box.error.WRONG_COLLATION_OPTIONS
152: box.error.NULLABLE_PRIMARY
- 153: box.error.NO_SUCH_FIELD_NAME
+ 153: box.error.NO_SUCH_FIELD_NAME_IN_SPACE
154: box.error.TRANSACTION_YIELD
155: box.error.NO_SUCH_GROUP
156: box.error.SQL_BIND_VALUE
157: box.error.SQL_BIND_TYPE
158: box.error.SQL_BIND_PARAMETER_MAX
159: box.error.SQL_EXECUTE
+ 160: box.error.NO_SUCH_FIELD_NAME
161: box.error.SQL_BIND_NOT_FOUND
162: box.error.ACTION_MISMATCH
163: box.error.VIEW_MISSING_SQL
diff --git a/test/box/update.result b/test/box/update.result
index a3f731b55..f5de3bc09 100644
--- a/test/box/update.result
+++ b/test/box/update.result
@@ -794,7 +794,7 @@ s:update({0}, {{'+', 0}})
...
s:update({0}, {{'+', '+', '+'}})
---
-- error: Illegal parameters, field id must be a number
+- error: Field '+' was not found in the tuple
...
s:update({0}, {{0, 0, 0}})
---
diff --git a/test/engine/update.result b/test/engine/update.result
index 69293e468..f181924f3 100644
--- a/test/engine/update.result
+++ b/test/engine/update.result
@@ -788,3 +788,70 @@ sk:select()
s:drop()
---
...
+--
+-- gh-1261: tuple update by JSON.
+-- At first, test tuple update by field names.
+--
+format = {}
+---
+...
+format[1] = {'field1', 'unsigned'}
+---
+...
+format[2] = {'field2', 'array'}
+---
+...
+format[3] = {'field3', 'map'}
+---
+...
+format[4] = {'field4', 'string'}
+---
+...
+format[5] = {'field5', 'any'}
+---
+...
+format[6] = {'field6', 'integer'}
+---
+...
+format[7] = {'[1]', 'unsigned'}
+---
+...
+s = box.schema.create_space('test', {format = format})
+---
+...
+pk = s:create_index('pk')
+---
+...
+t = s:replace{1, {10, 11, 12}, {a = 20, b = 21, c = 22}, 'abcdefgh', true, -100, 200}
+---
+...
+t:update({{'+', 'field1', 1}})
+---
+- [2, [10, 11, 12], {'b': 21, 'a': 20, 'c': 22}, 'abcdefgh', true, -100, 200]
+...
+t:update({{'=', 'field2', {13, 14, 15}}})
+---
+- [1, [13, 14, 15], {'b': 21, 'a': 20, 'c': 22}, 'abcdefgh', true, -100, 200]
+...
+t:update({{':', 'field4', 3, 3, 'bbccdd'}, {'+', 'field6', 50}, {'!', 7, 300}})
+---
+- [1, [10, 11, 12], {'b': 21, 'a': 20, 'c': 22}, 'abbbccddfgh', true, -50, 300, 200]
+...
+-- Any path is interpreted as a field name first. And only then
+-- as JSON.
+t:update({{'+', '[1]', 50}})
+---
+- [1, [10, 11, 12], {'b': 21, 'a': 20, 'c': 22}, 'abcdefgh', true, -100, 250]
+...
+-- JSON paths are not allowed yet.
+t:update({{'=', 'field2[1]', 13}})
+---
+- error: Field 'field2[1]' was not found in the tuple
+...
+s:update({1}, {{'=', 'field3', {d = 30, e = 31, f = 32}}})
+---
+- [1, [10, 11, 12], {'d': 30, 'f': 32, 'e': 31}, 'abcdefgh', true, -100, 200]
+...
+s:drop()
+---
+...
diff --git a/test/engine/update.test.lua b/test/engine/update.test.lua
index 51263f577..4ca2589e4 100644
--- a/test/engine/update.test.lua
+++ b/test/engine/update.test.lua
@@ -134,3 +134,31 @@ box.begin() s:update(1, {{'=', 2, 2}}) s:update(1, {{'=', 3, 2}}) box.commit()
pk:select()
sk:select()
s:drop()
+
+--
+-- gh-1261: tuple update by JSON.
+-- At first, test tuple update by field names.
+--
+format = {}
+format[1] = {'field1', 'unsigned'}
+format[2] = {'field2', 'array'}
+format[3] = {'field3', 'map'}
+format[4] = {'field4', 'string'}
+format[5] = {'field5', 'any'}
+format[6] = {'field6', 'integer'}
+format[7] = {'[1]', 'unsigned'}
+s = box.schema.create_space('test', {format = format})
+pk = s:create_index('pk')
+t = s:replace{1, {10, 11, 12}, {a = 20, b = 21, c = 22}, 'abcdefgh', true, -100, 200}
+t:update({{'+', 'field1', 1}})
+t:update({{'=', 'field2', {13, 14, 15}}})
+t:update({{':', 'field4', 3, 3, 'bbccdd'}, {'+', 'field6', 50}, {'!', 7, 300}})
+-- Any path is interpreted as a field name first. And only then
+-- as JSON.
+t:update({{'+', '[1]', 50}})
+-- JSON paths are not allowed yet.
+t:update({{'=', 'field2[1]', 13}})
+
+s:update({1}, {{'=', 'field3', {d = 30, e = 31, f = 32}}})
+
+s:drop()
diff --git a/test/engine/upsert.result b/test/engine/upsert.result
index b35545588..47da307fa 100644
--- a/test/engine/upsert.result
+++ b/test/engine/upsert.result
@@ -2108,7 +2108,7 @@ test(t, {{'&', 3, 1}, {'&', 2, 1}, {'&', 4, 1}}, {{1, '1', 1, '1'}})
...
test(t, {{'&', 'a', 3}, {'+', 3, 3}}, {{1, '1', 1, '1'}})
---
-- error: Illegal parameters, field id must be a number
+- error: Field 'a' was not found in the tuple
...
test(t, {{'+', 3, 3}, {'&', 3, 'a'}}, {{1, '1', 1, '1'}})
---
diff --git a/test/unit/column_mask.c b/test/unit/column_mask.c
index 3ffd351ac..5ee8b7332 100644
--- a/test/unit/column_mask.c
+++ b/test/unit/column_mask.c
@@ -129,8 +129,9 @@ check_update_result(const struct tuple_template *original,
uint64_t column_mask;
struct region *region = &fiber()->gc;
const char *actual =
- tuple_update_execute(ops, ops_end, old, old_end, &actual_len, 1,
- &column_mask);
+ tuple_update_execute(ops, ops_end, old, old_end,
+ box_tuple_format_default()->dict,
+ &actual_len, 1, &column_mask);
fail_if(actual == NULL);
is((int32_t)actual_len, new_end - new, "check result length");
is(memcmp(actual, new, actual_len), 0, "tuple update is correct");
--
2.20.1 (Apple Git-117)
^ permalink raw reply [flat|nested] 15+ messages in thread* [tarantool-patches] [PATCH 06/13] tuple: expose JSON go_to_key and go_to_index functions
2019-08-12 23:05 [tarantool-patches] [PATCH 00/13] JSON updates Vladislav Shpilevoy
` (8 preceding siblings ...)
2019-08-12 23:05 ` [tarantool-patches] [PATCH 05/13] tuple: implement update by field name Vladislav Shpilevoy
@ 2019-08-12 23:05 ` Vladislav Shpilevoy
2019-08-12 23:05 ` [tarantool-patches] [PATCH 07/13] tuple: rework updates to improve code extendibility Vladislav Shpilevoy
` (3 subsequent siblings)
13 siblings, 0 replies; 15+ messages in thread
From: Vladislav Shpilevoy @ 2019-08-12 23:05 UTC (permalink / raw)
To: tarantool-patches
They are needed in JSON path update for a case, when a final part
of the path may not exist, and go_to_path returns an error.
That case is '=' and '!' on not existing fields. For example,
in {'=', '[1][2][3]', 20} field [3] can be not existing.
For these cases JSON update will have its own implementation of
go_to_path allowing optional last field.
Needed for #1261
---
src/box/tuple.c | 21 ++-------------------
src/box/tuple.h | 23 +++++++++++++++++++++++
2 files changed, 25 insertions(+), 19 deletions(-)
diff --git a/src/box/tuple.c b/src/box/tuple.c
index bf4ea711d..261505f9a 100644
--- a/src/box/tuple.c
+++ b/src/box/tuple.c
@@ -385,15 +385,7 @@ tuple_free(void)
/* {{{ tuple_field_* getters */
-/**
- * Propagate @a field to MessagePack(field)[index].
- * @param[in][out] field Field to propagate.
- * @param index 0-based index to propagate to.
- *
- * @retval 0 Success, the index was found.
- * @retval -1 Not found.
- */
-static inline int
+int
tuple_field_go_to_index(const char **field, uint64_t index)
{
enum mp_type type = mp_typeof(**field);
@@ -428,16 +420,7 @@ tuple_field_go_to_index(const char **field, uint64_t index)
return -1;
}
-/**
- * Propagate @a field to MessagePack(field)[key].
- * @param[in][out] field Field to propagate.
- * @param key Key to propagate to.
- * @param len Length of @a key.
- *
- * @retval 0 Success, the index was found.
- * @retval -1 Not found.
- */
-static inline int
+int
tuple_field_go_to_key(const char **field, const char *key, int len)
{
enum mp_type type = mp_typeof(**field);
diff --git a/src/box/tuple.h b/src/box/tuple.h
index 4c4050ca8..ab0c7a99b 100644
--- a/src/box/tuple.h
+++ b/src/box/tuple.h
@@ -551,6 +551,29 @@ int
tuple_go_to_path(const char **data, const char *path, uint32_t path_len,
int multikey_idx);
+/**
+ * Propagate @a field to MessagePack(field)[index].
+ * @param[in][out] field Field to propagate.
+ * @param index 0-based index to propagate to.
+ *
+ * @retval 0 Success, the index was found.
+ * @retval -1 Not found.
+ */
+int
+tuple_field_go_to_index(const char **field, uint64_t index);
+
+/**
+ * Propagate @a field to MessagePack(field)[key].
+ * @param[in][out] field Field to propagate.
+ * @param key Key to propagate to.
+ * @param len Length of @a key.
+ *
+ * @retval 0 Success, the index was found.
+ * @retval -1 Not found.
+ */
+int
+tuple_field_go_to_key(const char **field, const char *key, int len);
+
/**
* Get tuple field by field index, relative JSON path and
* multikey_idx.
--
2.20.1 (Apple Git-117)
^ permalink raw reply [flat|nested] 15+ messages in thread* [tarantool-patches] [PATCH 07/13] tuple: rework updates to improve code extendibility
2019-08-12 23:05 [tarantool-patches] [PATCH 00/13] JSON updates Vladislav Shpilevoy
` (9 preceding siblings ...)
2019-08-12 23:05 ` [tarantool-patches] [PATCH 06/13] tuple: expose JSON go_to_key and go_to_index functions Vladislav Shpilevoy
@ 2019-08-12 23:05 ` Vladislav Shpilevoy
2019-08-12 23:05 ` [tarantool-patches] [PATCH 08/13] json: lexer_eof and token_cmp helper functions Vladislav Shpilevoy
` (2 subsequent siblings)
13 siblings, 0 replies; 15+ messages in thread
From: Vladislav Shpilevoy @ 2019-08-12 23:05 UTC (permalink / raw)
To: tarantool-patches
Before the patch update was implemented as a set of operations
applicable for arrays only. It was ok until field names and JSON
paths appearance, because tuple is array on the top level.
But now there are four reasons to allow more complex updates of
tuple field internals via JSON paths:
* tuple field access by JSON path is allowed so for consistency
JSON paths should be allowed in updates as well;
* JSON indexes are supported. JSON update should be able to
change an indexed field without rewriting half of a tuple;
* Tarantool is going to support documents in storage so JSON
path updates is one more step forward;
* JSON updates are going to be faster than get + in-memory
Lua/connector update + replace (or update of a whole tuple
field).
The patch prepares current update code to be extended by updates
of map, so named 'bar' updates, and multilevel updates.
The concept is to build a tree of update objects. Each updates a
scalar terminal value (leaf of that tree), or a complex object
like array or map. In the latter case these objects contain >= 2
children updates.
This organization allows to split update implementation into
several independent files-modules for each update type. Next
commits will introduce them one by one.
Needed for #1261
---
src/box/CMakeLists.txt | 2 +
src/box/errcode.h | 8 +-
src/box/tuple_update.c | 1048 ++-------------------------------
src/box/update/update_array.c | 288 +++++++++
src/box/update/update_field.c | 573 ++++++++++++++++++
src/box/update/update_field.h | 407 +++++++++++++
test/box/misc.result | 2 +-
7 files changed, 1320 insertions(+), 1008 deletions(-)
create mode 100644 src/box/update/update_array.c
create mode 100644 src/box/update/update_field.c
create mode 100644 src/box/update/update_field.h
diff --git a/src/box/CMakeLists.txt b/src/box/CMakeLists.txt
index 9bba37bcb..555e0f47a 100644
--- a/src/box/CMakeLists.txt
+++ b/src/box/CMakeLists.txt
@@ -41,6 +41,8 @@ add_library(tuple STATIC
field_map.c
tuple_format.c
tuple_update.c
+ update/update_field.c
+ update/update_array.c
tuple_compare.cc
tuple_extract_key.cc
tuple_hash.cc
diff --git a/src/box/errcode.h b/src/box/errcode.h
index dd55bd42a..5e02f2db4 100644
--- a/src/box/errcode.h
+++ b/src/box/errcode.h
@@ -77,11 +77,11 @@ struct errcode_record {
/* 22 */_(ER_TUPLE_NOT_ARRAY, "Tuple/Key must be MsgPack array") \
/* 23 */_(ER_FIELD_TYPE, "Tuple field %s type does not match one required by operation: expected %s") \
/* 24 */_(ER_INDEX_PART_TYPE_MISMATCH, "Field %s has type '%s' in one index, but type '%s' in another") \
- /* 25 */_(ER_SPLICE, "SPLICE error on field %u: %s") \
- /* 26 */_(ER_UPDATE_ARG_TYPE, "Argument type in operation '%c' on field %u does not match field type: expected %s") \
+ /* 25 */_(ER_UPDATE_SPLICE, "SPLICE error on field %s: %s") \
+ /* 26 */_(ER_UPDATE_ARG_TYPE, "Argument type in operation '%c' on field %s does not match field type: expected %s") \
/* 27 */_(ER_FORMAT_MISMATCH_INDEX_PART, "Field %s has type '%s' in space format, but type '%s' in index definition") \
/* 28 */_(ER_UNKNOWN_UPDATE_OP, "Unknown UPDATE operation") \
- /* 29 */_(ER_UPDATE_FIELD, "Field %u UPDATE error: %s") \
+ /* 29 */_(ER_UPDATE_FIELD, "Field %s UPDATE error: %s") \
/* 30 */_(ER_FUNCTION_TX_ACTIVE, "Transaction is active at return from function") \
/* 31 */_(ER_KEY_PART_COUNT, "Invalid key part count (expected [0..%u], got %u)") \
/* 32 */_(ER_PROC_LUA, "%s") \
@@ -147,7 +147,7 @@ struct errcode_record {
/* 92 */_(ER_ROLE_NOT_GRANTED, "User '%s' does not have role '%s'") \
/* 93 */_(ER_MISSING_SNAPSHOT, "Can't find snapshot") \
/* 94 */_(ER_CANT_UPDATE_PRIMARY_KEY, "Attempt to modify a tuple field which is part of index '%s' in space '%s'") \
- /* 95 */_(ER_UPDATE_INTEGER_OVERFLOW, "Integer overflow when performing '%c' operation on field %u") \
+ /* 95 */_(ER_UPDATE_INTEGER_OVERFLOW, "Integer overflow when performing '%c' operation on field %s") \
/* 96 */_(ER_GUEST_USER_PASSWORD, "Setting password for guest user has no effect") \
/* 97 */_(ER_TRANSACTION_CONFLICT, "Transaction has been aborted by conflict") \
/* 98 */_(ER_UNSUPPORTED_PRIV, "Unsupported %s privilege '%s'") \
diff --git a/src/box/tuple_update.c b/src/box/tuple_update.c
index 34e0e767f..87946919d 100644
--- a/src/box/tuple_update.c
+++ b/src/box/tuple_update.c
@@ -28,983 +28,31 @@
* THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
-
#include "tuple_update.h"
-#include <stdbool.h>
-
-#include "say.h"
#include "error.h"
#include "diag.h"
-#include "trivia/util.h"
-#include "third_party/queue.h"
#include <msgpuck/msgpuck.h>
-#include <bit/int96.h>
#include "column_mask.h"
#include "fiber.h"
-#include "tuple_dictionary.h"
-#include "tt_static.h"
+#include "update/update_field.h"
-/** UPDATE request implementation.
- * UPDATE request is represented by a sequence of operations, each
- * working with a single field. There also are operations which
- * add or remove fields. Only one operation on the same field
- * is allowed.
- *
- * Supported field change operations are: SET, ADD, SUBTRACT;
- * bitwise AND, XOR and OR; SPLICE.
- *
- * Supported tuple change operations are: SET, DELETE, INSERT,
- * PUSH and POP.
- * If the number of fields in a tuple is altered by an operation,
- * field index of all following operations is evaluated against the
- * new tuple.
- *
- * Despite the allowed complexity, a typical use case for UPDATE
- * is when the operation count is much less than field count in
- * a tuple.
- *
- * With the common case in mind, UPDATE tries to minimize
- * the amount of unnecessary temporary tuple copies.
- *
- * First, operations are parsed and initialized. Then, the
- * resulting tuple length is calculated. A new tuple is allocated.
- * Finally, operations are applied sequentially, each copying data
- * from the old tuple to the new tuple.
- *
- * With this approach, cost of UPDATE is proportional to O(tuple
- * length) + O(C * log C), where C is the number of operations in
- * the request, and data is copied from the old tuple to the new
- * one only once.
- *
- * As long as INSERT, DELETE, PUSH and POP change the relative
- * field order, an auxiliary data structure is necessary to look
- * up fields in the "old" tuple by field number. Such field
- * index is built on demand, using "rope" data structure.
- *
- * A rope is a binary tree designed to store long strings built
- * from pieces. Each tree node points to a substring of a large
- * string. In our case, each rope node points at a range of
- * fields, initially in the old tuple, and then, as fields are
- * added and deleted by UPDATE, in the "current" tuple.
- * Note, that the tuple itself is not materialized: when
- * operations which affect field count are initialized, the rope
- * is updated to reflect the new field order.
- * In particular, if a field is deleted by an operation,
- * it disappears from the rope and all subsequent operations
- * on this field number instead affect the field following the
- * deleted one.
- */
-
-static inline void *
-update_alloc(struct region *region, size_t size)
-{
- void *ptr = region_aligned_alloc(region, size, alignof(uint64_t));
- if (ptr == NULL)
- diag_set(OutOfMemory, size, "region_aligned_alloc",
- "update internals");
- return ptr;
-}
-
-/** Update internal state */
-struct tuple_update
-{
- struct region *region;
- struct rope *rope;
+/** Tuple update operations and arguments. */
+struct tuple_update {
+ /** Operations array. */
struct update_op *ops;
+ /** Length of ops. */
uint32_t op_count;
- int index_base; /* 0 for C and 1 for Lua */
- /** A bitmask of all columns modified by this update */
- uint64_t column_mask;
-};
-
-/** Argument of SET (and INSERT) operation. */
-struct op_set_arg {
- uint32_t length;
- const char *value;
-};
-
-/** Argument of DELETE operation. */
-struct op_del_arg {
- uint32_t count;
-};
-
-/**
- * MsgPack format code of an arithmetic argument or result.
- * MsgPack codes are not used to simplify type calculation.
- */
-enum arith_type {
- AT_DOUBLE = 0, /* MP_DOUBLE */
- AT_FLOAT = 1, /* MP_FLOAT */
- AT_INT = 2 /* MP_INT/MP_UINT */
-};
-
-/**
- * Argument (left and right) and result of ADD, SUBTRACT.
- *
- * To perform an arithmetic operation, update first loads
- * left and right arguments into corresponding value objects,
- * then performs arithmetics on types of arguments, thus
- * calculating the type of the result, and then
- * performs the requested operation according to the calculated
- * type rules.
- *
- * The rules are as follows:
- * - when one of the argument types is double, the result is
- * double
- * - when one of the argument types is float, the result is
- * float
- * - for integer arguments, the result type code depends on
- * the range in which falls the result of the operation.
- * If the result is in negative range, it's MP_INT, otherwise
- * it's MP_UINT. If the result is out of bounds of (-2^63,
- * 2^64), and exception is raised for overflow.
- */
-struct op_arith_arg {
- enum arith_type type;
- union {
- double dbl;
- float flt;
- struct int96_num int96;
- };
-};
-
-/** Argument of AND, XOR, OR operations. */
-struct op_bit_arg {
- uint64_t val;
-};
-
-/** Argument of SPLICE. */
-struct op_splice_arg {
- int32_t offset; /** splice position */
- int32_t cut_length; /** cut this many bytes. */
- const char *paste; /** paste what? */
- uint32_t paste_length; /** paste this many bytes. */
-
- /** Offset of the tail in the old field */
- int32_t tail_offset;
- /** Size of the tail. */
- int32_t tail_length;
-};
-
-union update_op_arg {
- struct op_set_arg set;
- struct op_del_arg del;
- struct op_arith_arg arith;
- struct op_bit_arg bit;
- struct op_splice_arg splice;
-};
-
-struct update_field;
-struct update_op;
-
-static struct update_field *
-update_field_split(struct region *region, struct update_field *data,
- size_t size, size_t offset);
-
-#define ROPE_SPLIT_F update_field_split
-#define ROPE_ALLOC_F update_alloc
-#define rope_data_t struct update_field *
-#define rope_ctx_t struct region *
-
-#include "salad/rope.h"
-
-typedef int (*do_op_func)(struct tuple_update *update, struct update_op *op);
-typedef int (*read_arg_func)(int index_base, struct update_op *op,
- const char **expr);
-typedef void (*store_op_func)(union update_op_arg *arg, const char *in,
- char *out);
-
-/** A set of functions and properties to initialize and do an op. */
-struct update_op_meta {
- read_arg_func read_arg;
- do_op_func do_op;
- store_op_func store;
- /* Argument count */
- uint32_t args;
-};
-
-/** A single UPDATE operation. */
-struct update_op {
- const struct update_op_meta *meta;
- union update_op_arg arg;
- /* Subject field no. */
- int32_t field_no;
- uint32_t new_field_len;
- uint8_t opcode;
-};
-
-/**
- * We can have more than one operation on the same field.
- * A descriptor of one changed field.
- */
-struct update_field {
- /** UPDATE operation against the first field in the range. */
- struct update_op *op;
- /** Points at start of field *data* in the old tuple. */
- const char *old;
- /** End of the old field. */
- const char *tail;
/**
- * Length of the "tail" in the old tuple from end
- * of old data to the beginning of the field in the
- * next update_field structure.
+ * Index base for MessagePack update operations. If update
+ * is from Lua, then the base is 1. Otherwise 0.
*/
- uint32_t tail_len;
+ int index_base;
+ /** A bitmask of all columns modified by this update. */
+ uint64_t column_mask;
+ /** First level of update tree. It is always array. */
+ struct update_field root_array;
};
-static void
-update_field_init(struct update_field *field,
- const char *old, uint32_t old_len, uint32_t tail_len)
-{
- field->op = NULL;
- field->old = old;
- field->tail = old + old_len;
- field->tail_len = tail_len;
-}
-
-/* {{{ read_arg helpers */
-
-/** Read a field index or any other integer field. */
-static inline int
-mp_read_i32(int index_base, struct update_op *op,
- const char **expr, int32_t *ret)
-{
- if (mp_read_int32(expr, ret) == 0)
- return 0;
- diag_set(ClientError, ER_UPDATE_ARG_TYPE, (char)op->opcode,
- index_base + op->field_no, "an integer");
- return -1;
-}
-
-static inline int
-mp_read_uint(int index_base, struct update_op *op,
- const char **expr, uint64_t *ret)
-{
- if (mp_typeof(**expr) == MP_UINT) {
- *ret = mp_decode_uint(expr);
- return 0;
- } else {
- diag_set(ClientError, ER_UPDATE_ARG_TYPE, (char)op->opcode,
- index_base + op->field_no, "a positive integer");
- return -1;
- }
-}
-
-/**
- * Load an argument of an arithmetic operation either from tuple
- * or from the UPDATE command.
- */
-static inline int
-mp_read_arith_arg(int index_base, struct update_op *op,
- const char **expr, struct op_arith_arg *ret)
-{
- if (mp_typeof(**expr) == MP_UINT) {
- ret->type = AT_INT;
- int96_set_unsigned(&ret->int96, mp_decode_uint(expr));
- } else if (mp_typeof(**expr) == MP_INT) {
- ret->type = AT_INT;
- int96_set_signed(&ret->int96, mp_decode_int(expr));
- } else if (mp_typeof(**expr) == MP_DOUBLE) {
- ret->type = AT_DOUBLE;
- ret->dbl = mp_decode_double(expr);
- } else if (mp_typeof(**expr) == MP_FLOAT) {
- ret->type = AT_FLOAT;
- ret->flt = mp_decode_float(expr);
- } else {
- diag_set(ClientError, ER_UPDATE_ARG_TYPE, (char)op->opcode,
- index_base + op->field_no, "a number");
- return -1;
- }
- return 0;
-}
-
-static inline int
-mp_read_str(int index_base, struct update_op *op,
- const char **expr, uint32_t *len, const char **ret)
-{
- if (mp_typeof(**expr) != MP_STR) {
- diag_set(ClientError, ER_UPDATE_ARG_TYPE, (char) op->opcode,
- index_base + op->field_no, "a string");
- return -1;
- }
- *ret = mp_decode_str(expr, len); /* value */
- return 0;
-}
-
-/* }}} read_arg helpers */
-
-/* {{{ read_arg */
-
-static int
-read_arg_set(int index_base, struct update_op *op,
- const char **expr)
-{
- (void)index_base;
- op->arg.set.value = *expr;
- mp_next(expr);
- op->arg.set.length = (uint32_t) (*expr - op->arg.set.value);
- return 0;
-}
-
-static int
-read_arg_insert(int index_base, struct update_op *op,
- const char **expr)
-{
- return read_arg_set(index_base, op, expr);
-}
-
-static int
-read_arg_delete(int index_base, struct update_op *op,
- const char **expr)
-{
- if (mp_typeof(**expr) == MP_UINT) {
- op->arg.del.count = (uint32_t) mp_decode_uint(expr);
- return 0;
- } else {
- diag_set(ClientError, ER_UPDATE_ARG_TYPE, (char)op->opcode,
- index_base + op->field_no,
- "a number of fields to delete");
- return -1;
- }
-}
-
-static int
-read_arg_arith(int index_base, struct update_op *op,
- const char **expr)
-{
- return mp_read_arith_arg(index_base, op, expr, &op->arg.arith);
-}
-
-static int
-read_arg_bit(int index_base, struct update_op *op,
- const char **expr)
-{
- struct op_bit_arg *arg = &op->arg.bit;
- return mp_read_uint(index_base, op, expr, &arg->val);
-}
-
-static int
-read_arg_splice(int index_base, struct update_op *op,
- const char **expr)
-{
- struct op_splice_arg *arg = &op->arg.splice;
- if (mp_read_i32(index_base, op, expr, &arg->offset))
- return -1;
- /* cut length */
- if (mp_read_i32(index_base, op, expr, &arg->cut_length))
- return -1;
- /* value */
- return mp_read_str(index_base, op, expr, &arg->paste_length,
- &arg->paste);
-}
-
-/* }}} read_arg */
-
-/* {{{ do_op helpers */
-
-static inline int
-op_adjust_field_no(struct tuple_update *update, struct update_op *op,
- int32_t field_max)
-{
- if (op->field_no >= 0) {
- if (op->field_no < field_max)
- return 0;
- diag_set(ClientError, ER_NO_SUCH_FIELD_NO, update->index_base +
- op->field_no);
- return -1;
- } else {
- if (op->field_no + field_max >= 0) {
- op->field_no += field_max;
- return 0;
- }
- diag_set(ClientError, ER_NO_SUCH_FIELD_NO, op->field_no);
- return -1;
- }
-}
-
-static inline double
-cast_arith_arg_to_double(struct op_arith_arg arg)
-{
- if (arg.type == AT_DOUBLE) {
- return arg.dbl;
- } else if (arg.type == AT_FLOAT) {
- return arg.flt;
- } else {
- assert(arg.type == AT_INT);
- if (int96_is_uint64(&arg.int96)) {
- return int96_extract_uint64(&arg.int96);
- } else {
- assert(int96_is_neg_int64(&arg.int96));
- return int96_extract_neg_int64(&arg.int96);
- }
- }
-}
-
-/** Return the MsgPack size of an arithmetic operation result. */
-static inline uint32_t
-mp_sizeof_op_arith_arg(struct op_arith_arg arg)
-{
- if (arg.type == AT_INT) {
- if (int96_is_uint64(&arg.int96)) {
- uint64_t val = int96_extract_uint64(&arg.int96);
- return mp_sizeof_uint(val);
- } else {
- int64_t val = int96_extract_neg_int64(&arg.int96);
- return mp_sizeof_int(val);
- }
- } else if (arg.type == AT_DOUBLE) {
- return mp_sizeof_double(arg.dbl);
- } else {
- assert(arg.type == AT_FLOAT);
- return mp_sizeof_float(arg.flt);
- }
-}
-
-static inline int
-make_arith_operation(struct op_arith_arg arg1, struct op_arith_arg arg2,
- char opcode, uint32_t err_fieldno, struct op_arith_arg *ret)
-{
- enum arith_type lowest_type = arg1.type;
- if (arg1.type > arg2.type)
- lowest_type = arg2.type;
-
- if (lowest_type == AT_INT) {
- switch(opcode) {
- case '+':
- int96_add(&arg1.int96, &arg2.int96);
- break;
- case '-':
- int96_invert(&arg2.int96);
- int96_add(&arg1.int96, &arg2.int96);
- break;
- default:
- unreachable(); /* checked by update_read_ops */
- break;
- }
- if (!int96_is_uint64(&arg1.int96) &&
- !int96_is_neg_int64(&arg1.int96)) {
- diag_set(ClientError,
- ER_UPDATE_INTEGER_OVERFLOW,
- opcode, err_fieldno);
- return -1;
- }
- *ret = arg1;
- return 0;
- } else {
- /* At least one of operands is double or float */
- double a = cast_arith_arg_to_double(arg1);
- double b = cast_arith_arg_to_double(arg2);
- double c;
- switch(opcode) {
- case '+': c = a + b; break;
- case '-': c = a - b; break;
- default:
- diag_set(ClientError, ER_UPDATE_ARG_TYPE, (char)opcode,
- err_fieldno, "a positive integer");
- return -1;
- }
- if (lowest_type == AT_DOUBLE) {
- /* result is DOUBLE */
- ret->type = AT_DOUBLE;
- ret->dbl = c;
- } else {
- /* result is FLOAT */
- assert(lowest_type == AT_FLOAT);
- ret->type = AT_FLOAT;
- ret->flt = (float)c;
- }
- }
- return 0;
-}
-
-/* }}} do_op helpers */
-
-/* {{{ do_op */
-
-static int
-do_op_insert(struct tuple_update *update, struct update_op *op)
-{
- if (op_adjust_field_no(update, op, rope_size(update->rope) + 1))
- return -1;
- struct update_field *field = (struct update_field *)
- update_alloc(update->region, sizeof(*field));
- if (field == NULL)
- return -1;
- update_field_init(field, op->arg.set.value, op->arg.set.length, 0);
- return rope_insert(update->rope, op->field_no, field, 1);
-}
-
-static int
-do_op_set(struct tuple_update *update, struct update_op *op)
-{
- /* intepret '=' for n +1 field as insert */
- if (op->field_no == (int32_t) rope_size(update->rope))
- return do_op_insert(update, op);
- if (op_adjust_field_no(update, op, rope_size(update->rope)))
- return -1;
- struct update_field *field = rope_extract(update->rope, op->field_no);
- if (field == NULL)
- return -1;
- /* Ignore the previous op, if any. */
- field->op = op;
- op->new_field_len = op->arg.set.length;
- return 0;
-}
-
-static int
-do_op_delete(struct tuple_update *update, struct update_op *op)
-{
- if (op_adjust_field_no(update, op, rope_size(update->rope)))
- return -1;
- uint32_t delete_count = op->arg.del.count;
-
- if ((uint64_t) op->field_no + delete_count > rope_size(update->rope))
- delete_count = rope_size(update->rope) - op->field_no;
-
- if (delete_count == 0) {
- diag_set(ClientError, ER_UPDATE_FIELD,
- update->index_base + op->field_no,
- "cannot delete 0 fields");
- return -1;
- }
-
- for (uint32_t u = 0; u < delete_count; u++)
- rope_erase(update->rope, op->field_no);
- return 0;
-}
-
-static int
-do_op_arith(struct tuple_update *update, struct update_op *op)
-{
- if (op_adjust_field_no(update, op, rope_size(update->rope)))
- return -1;
-
- struct update_field *field = rope_extract(update->rope, op->field_no);
- if (field == NULL)
- return -1;
- if (field->op) {
- diag_set(ClientError, ER_UPDATE_FIELD,
- update->index_base + op->field_no,
- "double update of the same field");
- return -1;
- }
- const char *old = field->old;
- struct op_arith_arg left_arg;
- if (mp_read_arith_arg(update->index_base, op, &old, &left_arg))
- return -1;
-
- struct op_arith_arg right_arg = op->arg.arith;
- if (make_arith_operation(left_arg, right_arg, op->opcode,
- update->index_base + op->field_no,
- &op->arg.arith))
- return -1;
- field->op = op;
- op->new_field_len = mp_sizeof_op_arith_arg(op->arg.arith);
- return 0;
-}
-
-static int
-do_op_bit(struct tuple_update *update, struct update_op *op)
-{
- if (op_adjust_field_no(update, op, rope_size(update->rope)))
- return -1;
- struct update_field *field = rope_extract(update->rope, op->field_no);
- if (field == NULL)
- return -1;
- struct op_bit_arg *arg = &op->arg.bit;
- if (field->op) {
- diag_set(ClientError, ER_UPDATE_FIELD,
- update->index_base + op->field_no,
- "double update of the same field");
- return -1;
- }
- const char *old = field->old;
- uint64_t val;
- if (mp_read_uint(update->index_base, op, &old, &val))
- return -1;
- switch (op->opcode) {
- case '&':
- arg->val &= val;
- break;
- case '^':
- arg->val ^= val;
- break;
- case '|':
- arg->val |= val;
- break;
- default:
- unreachable(); /* checked by update_read_ops */
- }
- field->op = op;
- op->new_field_len = mp_sizeof_uint(arg->val);
- return 0;
-}
-
-static int
-do_op_splice(struct tuple_update *update, struct update_op *op)
-{
- if (op_adjust_field_no(update, op, rope_size(update->rope)))
- return -1;
- struct update_field *field = rope_extract(update->rope, op->field_no);
- if (field == NULL)
- return -1;
- if (field->op) {
- diag_set(ClientError, ER_UPDATE_FIELD,
- update->index_base + op->field_no,
- "double update of the same field");
- return -1;
- }
-
- struct op_splice_arg *arg = &op->arg.splice;
-
- const char *in = field->old;
- int32_t str_len;
- if (mp_read_str(update->index_base, op, &in, (uint32_t *) &str_len, &in))
- return -1;
-
- if (arg->offset < 0) {
- if (-arg->offset > str_len + 1) {
- diag_set(ClientError, ER_SPLICE,
- update->index_base + op->field_no,
- "offset is out of bound");
- return -1;
- }
- arg->offset = arg->offset + str_len + 1;
- } else if (arg->offset - update->index_base >= 0) {
- arg->offset -= update->index_base;
- if (arg->offset > str_len)
- arg->offset = str_len;
- } else /* (offset <= 0) */ {
- diag_set(ClientError, ER_SPLICE,
- update->index_base + op->field_no,
- "offset is out of bound");
- return -1;
- }
-
- assert(arg->offset >= 0 && arg->offset <= str_len);
-
- if (arg->cut_length < 0) {
- if (-arg->cut_length > (str_len - arg->offset))
- arg->cut_length = 0;
- else
- arg->cut_length += str_len - arg->offset;
- } else if (arg->cut_length > str_len - arg->offset) {
- arg->cut_length = str_len - arg->offset;
- }
-
- assert(arg->offset <= str_len);
-
- /* Fill tail part */
- arg->tail_offset = arg->offset + arg->cut_length;
- arg->tail_length = str_len - arg->tail_offset;
-
-
- field->op = op;
- /* Record the new field length (maximal). */
- op->new_field_len = mp_sizeof_str(arg->offset + arg->paste_length +
- arg->tail_length);
- return 0;
-}
-
-/* }}} do_op */
-
-/* {{{ store_op */
-
-static void
-store_op_set(struct op_set_arg *arg, const char *in, char *out)
-{
- (void)in;
- memcpy(out, arg->value, arg->length);
-}
-
-static void
-store_op_insert(struct op_set_arg *arg, const char *in, char *out)
-{
- (void)in;
- memcpy(out, arg->value, arg->length);
-}
-
-static void
-store_op_arith(struct op_arith_arg *arg, const char *in, char *out)
-{
- (void)in;
- if (arg->type == AT_INT) {
- if (int96_is_uint64(&arg->int96)) {
- mp_encode_uint(out, int96_extract_uint64(&arg->int96));
- } else {
- assert(int96_is_neg_int64(&arg->int96));
- mp_encode_int(out, int96_extract_neg_int64(&arg->int96));
- }
- } else if (arg->type == AT_DOUBLE) {
- mp_encode_double(out, arg->dbl);
- } else {
- assert(arg->type == AT_FLOAT);
- mp_encode_float(out, arg->flt);
- }
-}
-
-static void
-store_op_bit(struct op_bit_arg *arg, const char *in, char *out)
-{
- (void)in;
- mp_encode_uint(out, arg->val);
-}
-
-static void
-store_op_splice(struct op_splice_arg *arg, const char *in, char *out)
-{
- uint32_t new_str_len = arg->offset + arg->paste_length
- + arg->tail_length;
-
- (void) mp_decode_strl(&in);
-
- out = mp_encode_strl(out, new_str_len);
- memcpy(out, in, arg->offset); /* copy field head. */
- out = out + arg->offset;
- memcpy(out, arg->paste, arg->paste_length); /* copy the paste */
- out = out + arg->paste_length;
- memcpy(out, in + arg->tail_offset, arg->tail_length); /* copy tail */
-}
-
-/* }}} store_op */
-
-static const struct update_op_meta op_set =
- { read_arg_set, do_op_set, (store_op_func) store_op_set, 3 };
-static const struct update_op_meta op_insert =
- { read_arg_insert, do_op_insert, (store_op_func) store_op_insert, 3 };
-static const struct update_op_meta op_arith =
- { read_arg_arith, do_op_arith, (store_op_func) store_op_arith, 3 };
-static const struct update_op_meta op_bit =
- { read_arg_bit, do_op_bit, (store_op_func) store_op_bit, 3 };
-static const struct update_op_meta op_splice =
- { read_arg_splice, do_op_splice, (store_op_func) store_op_splice, 5 };
-static const struct update_op_meta op_delete =
- { read_arg_delete, do_op_delete, (store_op_func) NULL, 3 };
-
-/** Split a range of fields in two, allocating update_field
- * context for the new range.
- */
-static struct update_field *
-update_field_split(struct region *region, struct update_field *prev,
- size_t size, size_t offset)
-{
- (void) size;
- struct update_field *next =
- (struct update_field *) update_alloc(region, sizeof(*next));
- if (next == NULL)
- return NULL;
- assert(offset > 0 && prev->tail_len > 0);
-
- const char *field = prev->tail;
- const char *end = field + prev->tail_len;
-
- for (uint32_t i = 1; i < offset; i++) {
- mp_next(&field);
- }
-
- prev->tail_len = field - prev->tail;
- const char *f = field;
- mp_next(&f);
- uint32_t field_len = f - field;
-
- update_field_init(next, field, field_len, end - field - field_len);
- return next;
-}
-
-/**
- * We found a tuple to do the update on. Prepare a rope
- * to perform operations on.
- * @param update Update meta.
- * @param tuple_data MessagePack array without the array header.
- * @param tuple_data_end End of the @tuple_data.
- * @param field_count Field count in @tuple_data.
- *
- * @retval 0 Success.
- * @retval -1 Error.
- */
-struct rope *
-tuple_rope_new(struct region *region, const char *tuple_data,
- const char *tuple_data_end, uint32_t field_count)
-{
- struct rope *rope = rope_new(region);
- if (rope == NULL)
- return NULL;
- /* Initialize the rope with the old tuple. */
- struct update_field *first =
- (struct update_field *) update_alloc(region, sizeof(*first));
- if (first == NULL)
- return NULL;
- const char *field = tuple_data;
- const char *end = tuple_data_end;
- if (field == end)
- return rope;
-
- /* Add first field to rope */
- mp_next(&tuple_data);
- uint32_t field_len = tuple_data - field;
- update_field_init(first, field, field_len, end - field - field_len);
-
- return rope_append(rope, first, field_count) != 0 ? NULL : rope;
-}
-
-static uint32_t
-update_calc_tuple_length(struct tuple_update *update)
-{
- uint32_t res = mp_sizeof_array(rope_size(update->rope));
- struct rope_iter it;
- struct rope_node *node;
-
- rope_iter_create(&it, update->rope);
- for (node = rope_iter_start(&it); node; node = rope_iter_next(&it)) {
- struct update_field *field = rope_leaf_data(node);
- uint32_t field_len = (field->op ? field->op->new_field_len :
- (uint32_t)(field->tail - field->old));
- res += field_len + field->tail_len;
- }
-
- return res;
-}
-
-static uint32_t
-update_write_tuple(struct tuple_update *update, char *buffer, char *buffer_end)
-{
- char *new_data = buffer;
- new_data = mp_encode_array(new_data, rope_size(update->rope));
-
- (void) buffer_end;
-
- uint32_t total_field_count = 0;
-
- struct rope_iter it;
- struct rope_node *node;
-
- rope_iter_create(&it, update->rope);
- for (node = rope_iter_start(&it); node; node = rope_iter_next(&it)) {
- struct update_field *field = rope_leaf_data(node);
- uint32_t field_count = rope_leaf_size(node);
- const char *old_field = field->old;
- struct update_op *op = field->op;
- if (op) {
- op->meta->store(&op->arg, old_field, new_data);
- new_data += op->new_field_len;
- } else {
- uint32_t field_len = field->tail - field->old;
- memcpy(new_data, old_field, field_len);
- new_data += field_len;
- }
- /* Copy tail_len from the old tuple. */
- assert(field->tail_len == 0 || field_count > 1);
- if (field_count > 1) {
- memcpy(new_data, field->tail, field->tail_len);
- new_data += field->tail_len;
- }
- total_field_count += field_count;
- }
-
- assert(rope_size(update->rope) == total_field_count);
- assert(new_data <= buffer_end);
- return new_data - buffer; /* real_tuple_size */
-}
-
-static const struct update_op_meta *
-update_op_by(char opcode)
-{
- switch (opcode) {
- case '=':
- return &op_set;
- case '+':
- case '-':
- return &op_arith;
- case '&':
- case '|':
- case '^':
- return &op_bit;
- case ':':
- return &op_splice;
- case '#':
- return &op_delete;
- case '!':
- return &op_insert;
- default:
- diag_set(ClientError, ER_UNKNOWN_UPDATE_OP);
- return NULL;
- }
-}
-
-/**
- * Decode an update operation from MessagePack.
- * @param[out] op Update operation.
- * @param index_base Field numbers base: 0 or 1.
- * @param dict Dictionary to lookup field number by a name.
- * @param expr MessagePack.
- *
- * @retval 0 Success.
- * @retval -1 Client error.
- */
-static inline int
-update_op_decode(struct update_op *op, int index_base,
- struct tuple_dictionary *dict, const char **expr)
-{
- if (mp_typeof(**expr) != MP_ARRAY) {
- diag_set(ClientError, ER_ILLEGAL_PARAMS, "update operation "
- "must be an array {op,..}");
- return -1;
- }
- uint32_t len, args = mp_decode_array(expr);
- if (args < 1) {
- diag_set(ClientError, ER_ILLEGAL_PARAMS, "update operation "\
- "must be an array {op,..}, got empty array");
- return -1;
- }
- if (mp_typeof(**expr) != MP_STR) {
- diag_set(ClientError, ER_ILLEGAL_PARAMS,
- "update operation name must be a string");
- return -1;
- }
- op->opcode = *mp_decode_str(expr, &len);
- op->meta = update_op_by(op->opcode);
- if (op->meta == NULL)
- return -1;
- if (args != op->meta->args) {
- diag_set(ClientError, ER_UNKNOWN_UPDATE_OP);
- return -1;
- }
- int32_t field_no;
- switch(mp_typeof(**expr)) {
- case MP_INT:
- case MP_UINT: {
- if (mp_read_i32(index_base, op, expr, &field_no) != 0)
- return -1;
- if (field_no - index_base >= 0) {
- op->field_no = field_no - index_base;
- } else if (field_no < 0) {
- op->field_no = field_no;
- } else {
- diag_set(ClientError, ER_NO_SUCH_FIELD_NO, field_no);
- return -1;
- }
- break;
- }
- case MP_STR: {
- const char *path = mp_decode_str(expr, &len);
- uint32_t field_no, hash = field_name_hash(path, len);
- if (tuple_fieldno_by_name(dict, path, len, hash,
- &field_no) != 0) {
- diag_set(ClientError, ER_NO_SUCH_FIELD_NAME,
- tt_cstr(path, len));
- return -1;
- }
- op->field_no = (int32_t) field_no;
- break;
- }
- default:
- diag_set(ClientError, ER_ILLEGAL_PARAMS,
- "field id must be a number or a string");
- return -1;
- }
- return op->meta->read_arg(index_base, op, expr);
-}
-
/**
* Read and check update operations and fill column mask.
*
@@ -1035,20 +83,18 @@ update_read_ops(struct tuple_update *update, const char *expr,
return -1;
}
uint64_t column_mask = 0;
- /* number of operations */
update->op_count = mp_decode_array(&expr);
-
if (update->op_count > BOX_UPDATE_OP_CNT_MAX) {
diag_set(ClientError, ER_ILLEGAL_PARAMS,
"too many operations for update");
return -1;
}
-
- /* Read update operations. */
- update->ops = (struct update_op *) update_alloc(update->region,
- update->op_count * sizeof(struct update_op));
- if (update->ops == NULL)
+ int size = update->op_count * sizeof(update->ops[0]);
+ update->ops = (struct update_op *) region_alloc(&fiber()->gc, size);
+ if (update->ops == NULL) {
+ diag_set(OutOfMemory, size, "region_alloc", "update->ops");
return -1;
+ }
struct update_op *op = update->ops;
struct update_op *ops_end = op + update->op_count;
for (; op < ops_end; op++) {
@@ -1150,14 +196,13 @@ static int
update_do_ops(struct tuple_update *update, const char *old_data,
const char *old_data_end, uint32_t part_count)
{
- update->rope = tuple_rope_new(update->region, old_data, old_data_end,
- part_count);
- if (update->rope == NULL)
+ if (update_array_create(&update->root_array, old_data, old_data_end,
+ part_count) != 0)
return -1;
struct update_op *op = update->ops;
struct update_op *ops_end = op + update->op_count;
for (; op < ops_end; op++) {
- if (op->meta->do_op(update, op))
+ if (op->meta->do_cb(op, &update->root_array) != 0)
return -1;
}
return 0;
@@ -1173,14 +218,13 @@ upsert_do_ops(struct tuple_update *update, const char *old_data,
const char *old_data_end, uint32_t part_count,
bool suppress_error)
{
- update->rope = tuple_rope_new(update->region, old_data, old_data_end,
- part_count);
- if (update->rope == NULL)
+ if (update_array_create(&update->root_array, old_data, old_data_end,
+ part_count) != 0)
return -1;
struct update_op *op = update->ops;
struct update_op *ops_end = op + update->op_count;
for (; op < ops_end; op++) {
- if (op->meta->do_op(update, op) == 0)
+ if (op->meta->do_cb(op, &update->root_array) == 0)
continue;
struct error *e = diag_last_error(diag_get());
if (e->type != &type_ClientError)
@@ -1197,23 +241,22 @@ static void
update_init(struct tuple_update *update, int index_base)
{
memset(update, 0, sizeof(*update));
- update->region = &fiber()->gc;
- /*
- * Base field offset, e.g. 0 for C and 1 for Lua. Used only for
- * error messages. All fields numbers must be zero-based!
- */
update->index_base = index_base;
}
-const char *
+static const char *
update_finish(struct tuple_update *update, uint32_t *p_tuple_len)
{
- uint32_t tuple_len = update_calc_tuple_length(update);
- char *buffer = (char *) update_alloc(update->region, tuple_len);
- if (buffer == NULL)
+ uint32_t tuple_size = update_array_sizeof(&update->root_array);
+ char *out = (char *) region_alloc(&fiber()->gc, tuple_size);
+ if (out == NULL) {
+ diag_set(OutOfMemory, tuple_size, "region_alloc", "buffer");
return NULL;
- *p_tuple_len = update_write_tuple(update, buffer, buffer + tuple_len);
- return buffer;
+ }
+ *p_tuple_len = update_array_store(&update->root_array, out,
+ out + tuple_size);
+ assert(*p_tuple_len == tuple_size);
+ return out;
}
int
@@ -1294,10 +337,13 @@ tuple_upsert_squash(const char *expr1, const char *expr1_end,
}
size_t possible_size = expr1_end - expr1 + expr2_end - expr2;
const uint32_t space_for_arr_tag = 5;
- char *buf = (char *) update_alloc(&fiber()->gc,
+ char *buf = (char *) region_alloc(&fiber()->gc,
possible_size + space_for_arr_tag);
- if (buf == NULL)
+ if (buf == NULL) {
+ diag_set(OutOfMemory, possible_size + space_for_arr_tag,
+ "region_alloc", "buf");
return NULL;
+ }
/* reserve some space for mp array header */
char *res_ops = buf + space_for_arr_tag;
uint32_t res_count = 0; /* number of resulting operations */
@@ -1351,20 +397,16 @@ tuple_upsert_squash(const char *expr1, const char *expr1_end,
op[0]->opcode = '+';
int96_invert(&op[0]->arg.arith.int96);
}
- struct op_arith_arg res;
- if (make_arith_operation(op[0]->arg.arith, op[1]->arg.arith,
- op[1]->opcode,
- update[0].index_base +
- op[0]->field_no, &res))
+ struct update_op res;
+ if (make_arith_operation(op[1], op[0]->arg.arith,
+ &res.arg.arith) != 0)
return NULL;
res_ops = mp_encode_array(res_ops, 3);
- res_ops = mp_encode_str(res_ops,
- (const char *)&op[0]->opcode, 1);
- res_ops = mp_encode_uint(res_ops,
- op[0]->field_no +
- update[0].index_base);
+ res_ops = mp_encode_str(res_ops, &op[0]->opcode, 1);
+ res_ops = mp_encode_uint(res_ops, op[0]->field_no +
+ update[0].index_base);
store_op_arith(&res, NULL, res_ops);
- res_ops += mp_sizeof_op_arith_arg(res);
+ res_ops += update_arith_sizeof(&res.arg.arith);
mp_next(&expr[0]);
mp_next(&expr[1]);
op_no[0]++;
diff --git a/src/box/update/update_array.c b/src/box/update/update_array.c
new file mode 100644
index 000000000..985d555af
--- /dev/null
+++ b/src/box/update/update_array.c
@@ -0,0 +1,288 @@
+/*
+ * Copyright 2010-2019, Tarantool AUTHORS, please see AUTHORS file.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the
+ * following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY <COPYRIGHT HOLDER> ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+ * <COPYRIGHT HOLDER> OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+ * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+#include "update_field.h"
+#include "msgpuck.h"
+#include "fiber.h"
+
+/**
+ * Make field number positive and check for the field existence.
+ * @param op Update operation.
+ * @param field_count Field count in the array having this field.
+ *
+ * @retval 0 Success.
+ * @retval -1 Error.
+ */
+static inline int
+update_op_adjust_field_no(struct update_op *op, int32_t field_count)
+{
+ if (op->field_no >= 0) {
+ if (op->field_no < field_count)
+ return 0;
+ return update_err_no_such_field(op);
+ }
+ if (op->field_no + field_count >= 0) {
+ op->field_no += field_count;
+ return 0;
+ }
+ return update_err_no_such_field(op);
+}
+
+/**
+ * Update of an array consists of array items. Each item is a
+ * range of fields. Item updates first field of the range and
+ * stores size of others to save them with no changes into a new
+ * tuple later. It allows on update of a single field in an array
+ * store at most 2 objects - item for the previous fields, and
+ * item for this one + its tail. This is how rope data structure
+ * works.
+ */
+struct update_array_item {
+ /** Update of the first field in the range. */
+ struct update_field field;
+ /** Size of other fields in the range. */
+ uint32_t tail_size;
+};
+
+/** Initialize an array item. */
+static inline void
+update_array_item_create(struct update_array_item *item, enum update_type type,
+ const char *data, uint32_t size, uint32_t tail_size)
+{
+ item->field.type = type;
+ item->field.data = data;
+ item->field.size = size;
+ item->tail_size = tail_size;
+}
+
+/** Rope allocator for nodes, paths, items etc. */
+static inline void *
+rope_alloc(struct region *region, size_t size)
+{
+ void *ptr = region_aligned_alloc(region, size, alignof(uint64_t));
+ if (ptr == NULL) {
+ diag_set(OutOfMemory, size, "region_aligned_alloc",
+ "update internals");
+ }
+ return ptr;
+}
+
+/** Split a range of fields in two. */
+static struct update_array_item *
+rope_field_split(struct region *region, struct update_array_item *prev,
+ size_t size, size_t offset)
+{
+ (void) size;
+ struct update_array_item *next =
+ (struct update_array_item *) rope_alloc(region, sizeof(*next));
+ if (next == NULL)
+ return NULL;
+ assert(offset > 0 && prev->tail_size > 0);
+
+ const char *tail = prev->field.data + prev->field.size;
+ const char *field = tail;
+ const char *range_end = tail + prev->tail_size;
+
+ for (uint32_t i = 1; i < offset; ++i)
+ mp_next(&field);
+
+ prev->tail_size = field - tail;
+ const char *field_end = field;
+ mp_next(&field_end);
+ update_array_item_create(next, UPDATE_NOP, field, field_end - field,
+ range_end - field_end);
+ return next;
+}
+
+#define ROPE_SPLIT_F rope_field_split
+#define ROPE_ALLOC_F rope_alloc
+#define rope_data_t struct update_array_item *
+#define rope_ctx_t struct region *
+
+#include "salad/rope.h"
+
+/**
+ * Extract from the array an item whose range starts from the
+ * field affected by @a op.
+ */
+static inline struct update_array_item *
+update_array_extract_item(struct update_field *field, struct update_op *op)
+{
+ assert(field->type == UPDATE_ARRAY);
+ struct rope *rope = field->array.rope;
+ if (update_op_adjust_field_no(op, rope_size(rope)) != 0)
+ return NULL;
+ return rope_extract(rope, op->field_no);
+}
+
+int
+update_array_create(struct update_field *field, const char *data,
+ const char *data_end, uint32_t field_count)
+{
+ field->type = UPDATE_ARRAY;
+ field->data = data;
+ field->size = data_end - data;
+ field->array.rope = rope_new(&fiber()->gc);
+ if (field->array.rope == NULL)
+ return -1;
+ struct update_array_item *item = (struct update_array_item *)
+ rope_alloc(&fiber()->gc, sizeof(*item));
+ if (item == NULL)
+ return -1;
+ if (data == data_end)
+ return 0;
+ /*
+ * Initial item consists of one range - the whole array.
+ */
+ const char *begin = data;
+ mp_next(&data);
+ update_array_item_create(item, UPDATE_NOP, begin, data - begin,
+ data_end - data);
+ return rope_append(field->array.rope, item, field_count);
+}
+
+uint32_t
+update_array_sizeof(struct update_field *field)
+{
+ assert(field->type == UPDATE_ARRAY);
+ struct rope_iter it;
+ rope_iter_create(&it, field->array.rope);
+
+ uint32_t res = mp_sizeof_array(rope_size(field->array.rope));
+ for (struct rope_node *node = rope_iter_start(&it); node != NULL;
+ node = rope_iter_next(&it)) {
+ struct update_array_item *item = rope_leaf_data(node);
+ res += update_field_sizeof(&item->field) + item->tail_size;
+ }
+ return res;
+}
+
+uint32_t
+update_array_store(struct update_field *field, char *out, char *out_end)
+{
+ assert(field->type == UPDATE_ARRAY);
+ char *out_begin = out;
+ out = mp_encode_array(out, rope_size(field->array.rope));
+ uint32_t total_field_count = 0;
+ struct rope_iter it;
+ rope_iter_create(&it, field->array.rope);
+ for (struct rope_node *node = rope_iter_start(&it); node != NULL;
+ node = rope_iter_next(&it)) {
+ struct update_array_item *item = rope_leaf_data(node);
+ uint32_t field_count = rope_leaf_size(node);
+ out += update_field_store(&item->field, out, out_end);
+ assert(item->tail_size == 0 || field_count > 1);
+ memcpy(out, item->field.data + item->field.size,
+ item->tail_size);
+ out += item->tail_size;
+ total_field_count += field_count;
+ }
+ (void) total_field_count;
+ assert(rope_size(field->array.rope) == total_field_count);
+ assert(out <= out_end);
+ return out - out_begin;
+}
+
+int
+do_op_array_insert(struct update_op *op, struct update_field *field)
+{
+ assert(field->type == UPDATE_ARRAY);
+ struct rope *rope = field->array.rope;
+ if (update_op_adjust_field_no(op, rope_size(rope) + 1) != 0)
+ return -1;
+
+ struct update_array_item *item =
+ (struct update_array_item *) rope_alloc(rope->ctx,
+ sizeof(*item));
+ if (item == NULL)
+ return -1;
+ update_array_item_create(item, UPDATE_NOP, op->arg.set.value,
+ op->arg.set.length, 0);
+ return rope_insert(rope, op->field_no, item, 1);
+}
+
+int
+do_op_array_set(struct update_op *op, struct update_field *field)
+{
+ assert(field->type == UPDATE_ARRAY);
+ struct rope *rope = field->array.rope;
+ /* Interpret '=' for n + 1 field as insert. */
+ if (op->field_no == (int32_t) rope_size(rope))
+ return do_op_array_insert(op, field);
+
+ struct update_array_item *item =
+ update_array_extract_item(field, op);
+ if (item == NULL)
+ return -1;
+ op->new_field_len = op->arg.set.length;
+ /* Ignore the previous op, if any. */
+ item->field.type = UPDATE_SCALAR;
+ item->field.scalar.op = op;
+ return 0;
+}
+
+int
+do_op_array_delete(struct update_op *op, struct update_field *field)
+{
+ assert(field->type == UPDATE_ARRAY);
+ struct rope *rope = field->array.rope;
+ uint32_t size = rope_size(rope);
+ if (update_op_adjust_field_no(op, size) != 0)
+ return -1;
+ uint32_t delete_count = op->arg.del.count;
+ if ((uint64_t) op->field_no + delete_count > size)
+ delete_count = size - op->field_no;
+ for (uint32_t u = delete_count; u != 0; --u)
+ rope_erase(rope, op->field_no);
+ return 0;
+}
+
+#define DO_SCALAR_OP_GENERIC(op_type) \
+int \
+do_op_array_##op_type(struct update_op *op, struct update_field *field) \
+{ \
+ struct update_array_item *item = \
+ update_array_extract_item(field, op); \
+ if (item == NULL) \
+ return -1; \
+ if (item->field.type != UPDATE_NOP) \
+ return update_err_double(op); \
+ if (update_op_do_##op_type(op, item->field.data) != 0) \
+ return -1; \
+ item->field.type = UPDATE_SCALAR; \
+ item->field.scalar.op = op; \
+ return 0; \
+}
+
+DO_SCALAR_OP_GENERIC(arith)
+
+DO_SCALAR_OP_GENERIC(bit)
+
+DO_SCALAR_OP_GENERIC(splice)
diff --git a/src/box/update/update_field.c b/src/box/update/update_field.c
new file mode 100644
index 000000000..a3200a266
--- /dev/null
+++ b/src/box/update/update_field.c
@@ -0,0 +1,573 @@
+/*
+ * Copyright 2010-2019, Tarantool AUTHORS, please see AUTHORS file.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the
+ * following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY <COPYRIGHT HOLDER> ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+ * <COPYRIGHT HOLDER> OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+ * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+#include "update_field.h"
+#include "box/tuple_format.h"
+
+/* {{{ Error helpers. */
+
+/**
+ * Take a string identifier of a field being updated by @a op.
+ * @param op Operation to convert field of.
+ * @return String with field ID.
+ */
+static inline const char *
+update_op_field_str(const struct update_op *op)
+{
+ if (op->field_no >= 0)
+ return tt_sprintf("%d", op->field_no + TUPLE_INDEX_BASE);
+ else
+ return tt_sprintf("%d", op->field_no);
+}
+
+static inline int
+update_err_arg_type(const struct update_op *op, const char *needed_type)
+{
+ diag_set(ClientError, ER_UPDATE_ARG_TYPE, op->opcode,
+ update_op_field_str(op), needed_type);
+ return -1;
+}
+
+static inline int
+update_err_int_overflow(const struct update_op *op)
+{
+ diag_set(ClientError, ER_UPDATE_INTEGER_OVERFLOW, op->opcode,
+ update_op_field_str(op));
+ return -1;
+}
+
+static inline int
+update_err_splice_bound(const struct update_op *op)
+{
+ diag_set(ClientError, ER_UPDATE_SPLICE, update_op_field_str(op),
+ "offset is out of bound");
+ return -1;
+}
+
+int
+update_err_no_such_field(const struct update_op *op)
+{
+ diag_set(ClientError, ER_NO_SUCH_FIELD_NO, op->field_no >= 0 ?
+ TUPLE_INDEX_BASE + op->field_no : op->field_no);
+ return -1;
+}
+
+int
+update_err(const struct update_op *op, const char *reason)
+{
+ diag_set(ClientError, ER_UPDATE_FIELD, update_op_field_str(op),
+ reason);
+ return -1;
+}
+
+/* }}} Error helpers. */
+
+uint32_t
+update_field_sizeof(struct update_field *field)
+{
+ switch (field->type) {
+ case UPDATE_NOP:
+ return field->size;
+ case UPDATE_SCALAR:
+ return field->scalar.op->new_field_len;
+ case UPDATE_ARRAY:
+ return update_array_sizeof(field);
+ default:
+ unreachable();
+ }
+ return 0;
+}
+
+uint32_t
+update_field_store(struct update_field *field, char *out, char *out_end)
+{
+ struct update_op *op;
+ uint32_t size;
+ switch(field->type) {
+ case UPDATE_NOP:
+ assert(out_end - out >= field->size);
+ memcpy(out, field->data, field->size);
+ return field->size;
+ case UPDATE_SCALAR:
+ op = field->scalar.op;
+ size = op->new_field_len;
+ assert(out_end - out >= size);
+ op->meta->store_cb(op, field->data, out);
+ return size;
+ case UPDATE_ARRAY:
+ return update_array_store(field, out, out_end);
+ default:
+ unreachable();
+ }
+ return 0;
+}
+
+/* {{{ read_arg helpers. */
+
+static inline int
+mp_read_i32(struct update_op *op, const char **expr, int32_t *ret)
+{
+ if (mp_read_int32(expr, ret) == 0)
+ return 0;
+ return update_err_arg_type(op, "an integer");
+}
+
+static inline int
+mp_read_uint(struct update_op *op, const char **expr, uint64_t *ret)
+{
+ if (mp_typeof(**expr) == MP_UINT) {
+ *ret = mp_decode_uint(expr);
+ return 0;
+ }
+ return update_err_arg_type(op, "a positive integer");
+}
+
+static inline int
+mp_read_arith_arg(struct update_op *op, const char **expr,
+ struct op_arith_arg *ret)
+{
+ switch(mp_typeof(**expr)) {
+ case MP_UINT:
+ ret->type = AT_INT;
+ int96_set_unsigned(&ret->int96, mp_decode_uint(expr));
+ return 0;
+ case MP_INT:
+ ret->type = AT_INT;
+ int96_set_signed(&ret->int96, mp_decode_int(expr));
+ return 0;
+ case MP_DOUBLE:
+ ret->type = AT_DOUBLE;
+ ret->dbl = mp_decode_double(expr);
+ return 0;
+ case MP_FLOAT:
+ ret->type = AT_FLOAT;
+ ret->flt = mp_decode_float(expr);
+ return 0;
+ default:
+ return update_err_arg_type(op, "a number");
+ }
+}
+
+static inline int
+mp_read_str(struct update_op *op, const char **expr, uint32_t *len,
+ const char **ret)
+{
+ if (mp_typeof(**expr) == MP_STR) {
+ *ret = mp_decode_str(expr, len);
+ return 0;
+ }
+ return update_err_arg_type(op, "a string");
+}
+
+/* }}} read_arg helpers. */
+
+/* {{{ read_arg */
+
+static int
+read_arg_set(struct update_op *op, const char **expr, int index_base)
+{
+ (void) index_base;
+ op->arg.set.value = *expr;
+ mp_next(expr);
+ op->arg.set.length = (uint32_t) (*expr - op->arg.set.value);
+ return 0;
+}
+
+static int
+read_arg_insert(struct update_op *op, const char **expr, int index_base)
+{
+ return read_arg_set(op, expr, index_base);
+}
+
+static int
+read_arg_delete(struct update_op *op, const char **expr, int index_base)
+{
+ (void) index_base;
+ if (mp_typeof(**expr) == MP_UINT) {
+ op->arg.del.count = (uint32_t) mp_decode_uint(expr);
+ if (op->arg.del.count != 0)
+ return 0;
+ return update_err(op, "cannot delete 0 fields");
+ }
+ return update_err_arg_type(op, "a number of fields to delete");
+}
+
+static int
+read_arg_arith(struct update_op *op, const char **expr, int index_base)
+{
+ (void) index_base;
+ return mp_read_arith_arg(op, expr, &op->arg.arith);
+}
+
+static int
+read_arg_bit(struct update_op *op, const char **expr, int index_base)
+{
+ (void) index_base;
+ return mp_read_uint(op, expr, &op->arg.bit.val);
+}
+
+static int
+read_arg_splice(struct update_op *op, const char **expr, int index_base)
+{
+ struct op_splice_arg *arg = &op->arg.splice;
+ if (mp_read_i32(op, expr, &arg->offset))
+ return -1;
+ if (arg->offset >= 0) {
+ if (arg->offset - index_base < 0)
+ return update_err_splice_bound(op);
+ arg->offset -= index_base;
+ }
+ if (mp_read_i32(op, expr, &arg->cut_length) == 0)
+ return mp_read_str(op, expr, &arg->paste_length, &arg->paste);
+ return -1;
+}
+
+/* }}} read_arg */
+
+/* {{{ do_op helpers. */
+
+static inline double
+cast_arith_arg_to_double(struct op_arith_arg arg)
+{
+ if (arg.type == AT_DOUBLE) {
+ return arg.dbl;
+ } else if (arg.type == AT_FLOAT) {
+ return arg.flt;
+ } else {
+ assert(arg.type == AT_INT);
+ if (int96_is_uint64(&arg.int96)) {
+ return int96_extract_uint64(&arg.int96);
+ } else {
+ assert(int96_is_neg_int64(&arg.int96));
+ return int96_extract_neg_int64(&arg.int96);
+ }
+ }
+}
+
+uint32_t
+update_arith_sizeof(struct op_arith_arg *arg)
+{
+ if (arg->type == AT_INT) {
+ if (int96_is_uint64(&arg->int96)) {
+ uint64_t val = int96_extract_uint64(&arg->int96);
+ return mp_sizeof_uint(val);
+ } else {
+ int64_t val = int96_extract_neg_int64(&arg->int96);
+ return mp_sizeof_int(val);
+ }
+ } else if (arg->type == AT_DOUBLE) {
+ return mp_sizeof_double(arg->dbl);
+ } else {
+ assert(arg->type == AT_FLOAT);
+ return mp_sizeof_float(arg->flt);
+ }
+}
+
+int
+make_arith_operation(struct update_op *op, struct op_arith_arg arg,
+ struct op_arith_arg *ret)
+{
+ struct op_arith_arg value = op->arg.arith;
+ enum arith_type lowest_type = arg.type;
+ if (arg.type > value.type)
+ lowest_type = value.type;
+
+ if (lowest_type == AT_INT) {
+ switch(op->opcode) {
+ case '+':
+ int96_add(&arg.int96, &value.int96);
+ break;
+ case '-':
+ int96_invert(&value.int96);
+ int96_add(&arg.int96, &value.int96);
+ break;
+ default:
+ unreachable();
+ break;
+ }
+ if (!int96_is_uint64(&arg.int96) &&
+ !int96_is_neg_int64(&arg.int96))
+ return update_err_int_overflow(op);
+ *ret = arg;
+ } else {
+ /* At least one of operands is double or float. */
+ double a = cast_arith_arg_to_double(arg);
+ double b = cast_arith_arg_to_double(value);
+ double c;
+ switch(op->opcode) {
+ case '+':
+ c = a + b;
+ break;
+ case '-':
+ c = a - b;
+ break;
+ default:
+ return update_err_arg_type(op, "a positive integer");
+ }
+ if (lowest_type == AT_DOUBLE) {
+ ret->type = AT_DOUBLE;
+ ret->dbl = c;
+ } else {
+ assert(lowest_type == AT_FLOAT);
+ ret->type = AT_FLOAT;
+ ret->flt = (float)c;
+ }
+ }
+ return 0;
+}
+
+int
+update_op_do_arith(struct update_op *op, const char *old)
+{
+ struct op_arith_arg left_arg;
+ if (mp_read_arith_arg(op, &old, &left_arg) != 0 ||
+ make_arith_operation(op, left_arg, &op->arg.arith) != 0)
+ return -1;
+ op->new_field_len = update_arith_sizeof(&op->arg.arith);
+ return 0;
+}
+
+int
+update_op_do_bit(struct update_op *op, const char *old)
+{
+ uint64_t val;
+ if (mp_read_uint(op, &old, &val) != 0)
+ return -1;
+ struct op_bit_arg *arg = &op->arg.bit;
+ switch (op->opcode) {
+ case '&':
+ arg->val &= val;
+ break;
+ case '^':
+ arg->val ^= val;
+ break;
+ case '|':
+ arg->val |= val;
+ break;
+ default:
+ /* Checked by update_read_ops. */
+ unreachable();
+ }
+ op->new_field_len = mp_sizeof_uint(arg->val);
+ return 0;
+}
+
+int
+update_op_do_splice(struct update_op *op, const char *old)
+{
+ struct op_splice_arg *arg = &op->arg.splice;
+ int32_t str_len;
+ if (mp_read_str(op, &old, (uint32_t *) &str_len, &old) != 0)
+ return -1;
+
+ if (arg->offset < 0) {
+ if (-arg->offset > str_len + 1)
+ return update_err_splice_bound(op);
+ arg->offset += str_len + 1;
+ } else if (arg->offset > str_len) {
+ arg->offset = str_len;
+ }
+ assert(arg->offset >= 0 && arg->offset <= str_len);
+ if (arg->cut_length < 0) {
+ if (-arg->cut_length > (str_len - arg->offset))
+ arg->cut_length = 0;
+ else
+ arg->cut_length += str_len - arg->offset;
+ } else if (arg->cut_length > str_len - arg->offset) {
+ arg->cut_length = str_len - arg->offset;
+ }
+ assert(arg->offset <= str_len);
+
+ arg->tail_offset = arg->offset + arg->cut_length;
+ arg->tail_length = str_len - arg->tail_offset;
+ op->new_field_len = mp_sizeof_str(arg->offset + arg->paste_length +
+ arg->tail_length);
+ return 0;
+}
+
+/* }}} do_op helpers. */
+
+/* {{{ store_op */
+
+static void
+store_op_set(struct update_op *op, const char *in, char *out)
+{
+ (void) in;
+ memcpy(out, op->arg.set.value, op->arg.set.length);
+}
+
+void
+store_op_arith(struct update_op *op, const char *in, char *out)
+{
+ (void) in;
+ struct op_arith_arg *arg = &op->arg.arith;
+ if (arg->type == AT_INT) {
+ if (int96_is_uint64(&arg->int96)) {
+ mp_encode_uint(out, int96_extract_uint64(&arg->int96));
+ } else {
+ assert(int96_is_neg_int64(&arg->int96));
+ mp_encode_int(out, int96_extract_neg_int64(&arg->int96));
+ }
+ } else if (arg->type == AT_DOUBLE) {
+ mp_encode_double(out, arg->dbl);
+ } else {
+ assert(arg->type == AT_FLOAT);
+ mp_encode_float(out, arg->flt);
+ }
+}
+
+static void
+store_op_bit(struct update_op *op, const char *in, char *out)
+{
+ (void) in;
+ mp_encode_uint(out, op->arg.bit.val);
+}
+
+static void
+store_op_splice(struct update_op *op, const char *in, char *out)
+{
+ struct op_splice_arg *arg = &op->arg.splice;
+ uint32_t new_str_len = arg->offset + arg->paste_length +
+ arg->tail_length;
+ (void) mp_decode_strl(&in);
+ out = mp_encode_strl(out, new_str_len);
+ /* Copy field head. */
+ memcpy(out, in, arg->offset);
+ out = out + arg->offset;
+ /* Copy the paste. */
+ memcpy(out, arg->paste, arg->paste_length);
+ out = out + arg->paste_length;
+ /* Copy tail. */
+ memcpy(out, in + arg->tail_offset, arg->tail_length);
+}
+
+/* }}} store_op */
+
+static const struct update_op_meta op_set =
+ { read_arg_set, do_op_set, store_op_set, 3 };
+static const struct update_op_meta op_insert =
+ { read_arg_insert, do_op_insert, store_op_set, 3 };
+static const struct update_op_meta op_arith =
+ { read_arg_arith, do_op_arith, store_op_arith, 3 };
+static const struct update_op_meta op_bit =
+ { read_arg_bit, do_op_bit, store_op_bit, 3 };
+static const struct update_op_meta op_splice =
+ { read_arg_splice, do_op_splice, store_op_splice, 5 };
+static const struct update_op_meta op_delete =
+ { read_arg_delete, do_op_delete, NULL, 3 };
+
+static inline const struct update_op_meta *
+update_op_by(char opcode)
+{
+ switch (opcode) {
+ case '=':
+ return &op_set;
+ case '+':
+ case '-':
+ return &op_arith;
+ case '&':
+ case '|':
+ case '^':
+ return &op_bit;
+ case ':':
+ return &op_splice;
+ case '#':
+ return &op_delete;
+ case '!':
+ return &op_insert;
+ default:
+ diag_set(ClientError, ER_UNKNOWN_UPDATE_OP);
+ return NULL;
+ }
+}
+
+int
+update_op_decode(struct update_op *op, int index_base,
+ struct tuple_dictionary *dict, const char **expr)
+{
+ if (mp_typeof(**expr) != MP_ARRAY) {
+ diag_set(ClientError, ER_ILLEGAL_PARAMS, "update operation "
+ "must be an array {op,..}");
+ return -1;
+ }
+ uint32_t len, arg_count = mp_decode_array(expr);
+ if (arg_count < 1) {
+ diag_set(ClientError, ER_ILLEGAL_PARAMS, "update operation "\
+ "must be an array {op,..}, got empty array");
+ return -1;
+ }
+ if (mp_typeof(**expr) != MP_STR) {
+ diag_set(ClientError, ER_ILLEGAL_PARAMS,
+ "update operation name must be a string");
+ return -1;
+ }
+ op->opcode = *mp_decode_str(expr, &len);
+ op->meta = update_op_by(op->opcode);
+ if (op->meta == NULL)
+ return -1;
+ if (arg_count != op->meta->arg_count) {
+ diag_set(ClientError, ER_UNKNOWN_UPDATE_OP);
+ return -1;
+ }
+ int32_t field_no;
+ switch(mp_typeof(**expr)) {
+ case MP_INT:
+ case MP_UINT: {
+ if (mp_read_i32(op, expr, &field_no) != 0)
+ return -1;
+ if (field_no - index_base >= 0) {
+ op->field_no = field_no - index_base;
+ } else if (field_no < 0) {
+ op->field_no = field_no;
+ } else {
+ diag_set(ClientError, ER_NO_SUCH_FIELD_NO, field_no);
+ return -1;
+ }
+ break;
+ }
+ case MP_STR: {
+ const char *path = mp_decode_str(expr, &len);
+ uint32_t field_no, hash = field_name_hash(path, len);
+ if (tuple_fieldno_by_name(dict, path, len, hash,
+ &field_no) == 0) {
+ op->field_no = (int32_t) field_no;
+ break;
+ }
+ diag_set(ClientError, ER_NO_SUCH_FIELD_NAME,
+ tt_cstr(path, len));
+ return -1;
+ }
+ default:
+ diag_set(ClientError, ER_ILLEGAL_PARAMS,
+ "field id must be a number or a string");
+ return -1;
+ }
+ return op->meta->read_arg_cb(op, expr, index_base);
+}
diff --git a/src/box/update/update_field.h b/src/box/update/update_field.h
new file mode 100644
index 000000000..b03c7479f
--- /dev/null
+++ b/src/box/update/update_field.h
@@ -0,0 +1,407 @@
+#ifndef TARANTOOL_BOX_TUPLE_UPDATE_FIELD_H
+#define TARANTOOL_BOX_TUPLE_UPDATE_FIELD_H
+/*
+ * Copyright 2010-2019, Tarantool AUTHORS, please see AUTHORS file.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the
+ * following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY <COPYRIGHT HOLDER> ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+ * <COPYRIGHT HOLDER> OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+ * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+#include "trivia/util.h"
+#include "tt_static.h"
+#include <stdint.h>
+#include "bit/int96.h"
+
+/**
+ * This file is a link between all the update operations for all
+ * the field types. It functions like a router, when an update
+ * operation is being parsed step by step, going down the update
+ * tree. For example, when an update operation goes through an
+ * array, a map, another array, and ends with a scalar operation,
+ * at the end of each step the operation goes to the next one via
+ * functions of this file: update_array.c -> update_field.c ->
+ * update_map.c -> update_field.c -> update_array.c -> ... . The
+ * functions, doing the routing, are do_op_<operation_type>(), see
+ * them below.
+ */
+
+struct rope;
+struct update_field;
+struct update_op;
+struct tuple_dictionary;
+
+/* {{{ update_op */
+
+/** Argument of SET (and INSERT) operation. */
+struct op_set_arg {
+ uint32_t length;
+ const char *value;
+};
+
+/** Argument of DELETE operation. */
+struct op_del_arg {
+ uint32_t count;
+};
+
+/**
+ * MessagePack format code of an arithmetic argument or result.
+ * MessagePack codes are not used to simplify type calculation.
+ */
+enum arith_type {
+ AT_DOUBLE = 0, /* MP_DOUBLE */
+ AT_FLOAT = 1, /* MP_FLOAT */
+ AT_INT = 2 /* MP_INT/MP_UINT */
+};
+
+/**
+ * Argument (left and right) and result of ADD, SUBTRACT.
+ *
+ * To perform an arithmetic operation, update first loads left
+ * and right arguments into corresponding value objects, then
+ * performs arithmetics on types of arguments, thus calculating
+ * the type of the result, and then performs the requested
+ * operation according to the calculated type rules.
+ *
+ * The rules are as follows:
+ * - when one of the argument types is double, the result is
+ * double;
+ * - when one of the argument types is float, the result is float;
+ * - for integer arguments, the result type code depends on the
+ * range in which falls the result of the operation. If the
+ * result is in negative range, it's MP_INT, otherwise it's
+ * MP_UINT. If the result is out of bounds of (-2^63, 2^64), an
+ * exception is raised for overflow.
+ */
+struct op_arith_arg {
+ enum arith_type type;
+ union {
+ double dbl;
+ float flt;
+ struct int96_num int96;
+ };
+};
+
+/** Argument of AND, XOR, OR operations. */
+struct op_bit_arg {
+ uint64_t val;
+};
+
+/** Argument of SPLICE. */
+struct op_splice_arg {
+ /** Splice position. */
+ int32_t offset;
+ /** Byte count to delete. */
+ int32_t cut_length;
+ /** New content. */
+ const char *paste;
+ /** New content length. */
+ uint32_t paste_length;
+
+ /** Offset of the tail in the old field. */
+ int32_t tail_offset;
+ /** Size of the tail. */
+ int32_t tail_length;
+};
+
+/** Update operation argument. */
+union update_op_arg {
+ struct op_set_arg set;
+ struct op_del_arg del;
+ struct op_arith_arg arith;
+ struct op_bit_arg bit;
+ struct op_splice_arg splice;
+};
+
+typedef int
+(*update_op_do_f)(struct update_op *op, struct update_field *field);
+
+typedef int
+(*update_op_read_arg_f)(struct update_op *op, const char **expr,
+ int index_base);
+
+typedef void
+(*update_op_store_f)(struct update_op *op, const char *in, char *out);
+
+/**
+ * A set of functions and properties to initialize, do and store
+ * an operation.
+ */
+struct update_op_meta {
+ /**
+ * Virtual function to read the arguments of the
+ * operation.
+ */
+ update_op_read_arg_f read_arg_cb;
+ /** Virtual function to execute the operation. */
+ update_op_do_f do_cb;
+ /**
+ * Virtual function to store a result of the operation.
+ */
+ update_op_store_f store_cb;
+ /** Argument count. */
+ uint32_t arg_count;
+};
+
+/** A single UPDATE operation. */
+struct update_op {
+ /** Operation meta depending on the op type. */
+ const struct update_op_meta *meta;
+ /** Operation arguments. */
+ union update_op_arg arg;
+ /** First level field no. */
+ int32_t field_no;
+ /** Size of a new field after it is updated. */
+ uint32_t new_field_len;
+ /** Opcode symbol: = + - / ... */
+ char opcode;
+};
+
+/**
+ * Decode an update operation from MessagePack.
+ * @param[out] op Update operation.
+ * @param index_base Field numbers base: 0 or 1.
+ * @param dict Dictionary to lookup field number by a name.
+ * @param expr MessagePack.
+ *
+ * @retval 0 Success.
+ * @retval -1 Client error.
+ */
+int
+update_op_decode(struct update_op *op, int index_base,
+ struct tuple_dictionary *dict, const char **expr);
+
+/* }}} update_op */
+
+/* {{{ update_field */
+
+/** Types of field update. */
+enum update_type {
+ /**
+ * Field is not updated. Just save it as is. It is used,
+ * for example, when a rope is split in two parts: not
+ * changed left range of fields, and a right range with
+ * its first field changed. In this case the left range is
+ * NOP.
+ */
+ UPDATE_NOP,
+ /**
+ * Field is a scalar value, updated via set, arith, bit,
+ * splice, or any other scalar-applicable operation.
+ */
+ UPDATE_SCALAR,
+ /**
+ * Field is an updated array. Check the rope for updates
+ * of individual fields.
+ */
+ UPDATE_ARRAY,
+};
+
+/**
+ * Generic structure describing update of a field. It can be
+ * tuple field, field of a tuple field, or any another tuple
+ * internal object: map, array, scalar, or unchanged field of any
+ * type without op. This is a node of the whole update tree.
+ *
+ * If the field is array or map, it contains more children fields
+ * inside corresponding sub-structures.
+ */
+struct update_field {
+ /**
+ * Type of this field's update. Union below depends on it.
+ */
+ enum update_type type;
+ /** Field data to update. */
+ const char *data;
+ uint32_t size;
+ union {
+ /**
+ * This update is terminal. It does a scalar
+ * operation and has no children fields.
+ */
+ struct {
+ struct update_op *op;
+ } scalar;
+ /**
+ * This update changes an array. Children fields
+ * are stored in rope nodes.
+ */
+ struct {
+ struct rope *rope;
+ } array;
+ };
+};
+
+/**
+ * Size of the updated field, including all children recursively.
+ */
+uint32_t
+update_field_sizeof(struct update_field *field);
+
+/** Save the updated field, including all children recursively. */
+uint32_t
+update_field_store(struct update_field *field, char *out, char *out_end);
+
+/**
+ * Generate declarations for a concrete field type: array, bar
+ * etc. Each complex type has basic operations of the same
+ * signature: insert, set, delete, arith, bit, splice.
+ *
+ * These operations can call each other going down a JSON path.
+ */
+#define OP_DECL_GENERIC(type) \
+int \
+do_op_##type##_insert(struct update_op *op, struct update_field *field); \
+ \
+int \
+do_op_##type##_set(struct update_op *op, struct update_field *field); \
+ \
+int \
+do_op_##type##_delete(struct update_op *op, struct update_field *field); \
+ \
+int \
+do_op_##type##_arith(struct update_op *op, struct update_field *field); \
+ \
+int \
+do_op_##type##_bit(struct update_op *op, struct update_field *field); \
+ \
+int \
+do_op_##type##_splice(struct update_op *op, struct update_field *field); \
+ \
+uint32_t \
+update_##type##_sizeof(struct update_field *field); \
+ \
+uint32_t \
+update_##type##_store(struct update_field *field, char *out, char *out_end);
+
+/* }}} update_field */
+
+/* {{{ update_field.array */
+
+/**
+ * Initialize @a field as an array to update.
+ * @param[out] field Field to initialize.
+ * @param data MessagePack data of the array to update.
+ * @param data_end End of @a data.
+ * @param field_count Field count in @data.
+ *
+ * @retval 0 Success.
+ * @retval -1 Error.
+ */
+int
+update_array_create(struct update_field *field, const char *data,
+ const char *data_end, uint32_t field_count);
+
+OP_DECL_GENERIC(array)
+
+/* }}} update_field.array */
+
+#undef OP_DECL_GENERIC
+
+/* {{{ Common helpers. */
+
+/**
+ * These helper functions are used when a caller function doesn't
+ * know type of a child node to propagate an update down the tree.
+ * For example, do_op_array_set calls do_op_set on its childs.
+ * Each child can be another array, a bar, a route, a map -
+ * anything. The functions below help to make such places shorter
+ * and simpler.
+ */
+#define OP_DECL_GENERIC(op_type) \
+static inline int \
+do_op_##op_type(struct update_op *op, struct update_field *field) \
+{ \
+ switch (field->type) { \
+ case UPDATE_ARRAY: \
+ return do_op_array_##op_type(op, field); \
+ default: \
+ unreachable(); \
+ } \
+ return 0; \
+}
+
+OP_DECL_GENERIC(insert)
+
+OP_DECL_GENERIC(set)
+
+OP_DECL_GENERIC(delete)
+
+OP_DECL_GENERIC(arith)
+
+OP_DECL_GENERIC(bit)
+
+OP_DECL_GENERIC(splice)
+
+#undef OP_DECL_GENERIC
+
+/* }}} Common helpers. */
+
+/* {{{ Scalar helpers. */
+
+int
+make_arith_operation(struct update_op *op, struct op_arith_arg arg,
+ struct op_arith_arg *ret);
+
+void
+store_op_arith(struct update_op *op, const char *in, char *out);
+
+uint32_t
+update_arith_sizeof(struct op_arith_arg *arg);
+
+int
+update_op_do_arith(struct update_op *op, const char *old);
+
+int
+update_op_do_bit(struct update_op *op, const char *old);
+
+int
+update_op_do_splice(struct update_op *op, const char *old);
+
+/* }}} Scalar helpers. */
+
+/** {{{ Error helpers. */
+/**
+ * All the error helpers below set diag with appropriate error
+ * code, taking into account field_no < 0, complex paths. They all
+ * return -1 to shorten error returning in a caller function to
+ * single line.
+ */
+
+int
+update_err_no_such_field(const struct update_op *op);
+
+int
+update_err(const struct update_op *op, const char *reason);
+
+static inline int
+update_err_double(const struct update_op *op)
+{
+ return update_err(op, "double update of the same field");
+}
+
+/** }}} Error helpers. */
+
+#endif /* TARANTOOL_BOX_TUPLE_UPDATE_FIELD_H */
diff --git a/test/box/misc.result b/test/box/misc.result
index f45e0514c..4d17162ff 100644
--- a/test/box/misc.result
+++ b/test/box/misc.result
@@ -356,7 +356,7 @@ t;
22: box.error.TUPLE_NOT_ARRAY
23: box.error.FIELD_TYPE
24: box.error.INDEX_PART_TYPE_MISMATCH
- 25: box.error.SPLICE
+ 25: box.error.UPDATE_SPLICE
26: box.error.UPDATE_ARG_TYPE
27: box.error.FORMAT_MISMATCH_INDEX_PART
28: box.error.UNKNOWN_UPDATE_OP
--
2.20.1 (Apple Git-117)
^ permalink raw reply [flat|nested] 15+ messages in thread* [tarantool-patches] [PATCH 08/13] json: lexer_eof and token_cmp helper functions
2019-08-12 23:05 [tarantool-patches] [PATCH 00/13] JSON updates Vladislav Shpilevoy
` (10 preceding siblings ...)
2019-08-12 23:05 ` [tarantool-patches] [PATCH 07/13] tuple: rework updates to improve code extendibility Vladislav Shpilevoy
@ 2019-08-12 23:05 ` Vladislav Shpilevoy
2019-08-12 23:05 ` [tarantool-patches] [PATCH 09/13] tuple: account the whole array in field.data and size Vladislav Shpilevoy
2019-08-20 18:49 ` [tarantool-patches] Re: [PATCH 00/13] JSON updates Vladislav Shpilevoy
13 siblings, 0 replies; 15+ messages in thread
From: Vladislav Shpilevoy @ 2019-08-12 23:05 UTC (permalink / raw)
To: tarantool-patches
They are needed in incoming JSON updates, which are going to
solve a task of comparison of two JSON paths, their simultaneous
parsing, and digging into a tuple.
json_token_cmp() existed before this patch, but it was trying to
compare parent pointers too, which is not needed in the JSON
updates, since they won't use JSON trees.
Needed for #1261
---
src/lib/json/json.c | 37 +++++++++++++------------------------
src/lib/json/json.h | 28 ++++++++++++++++++++++++++++
2 files changed, 41 insertions(+), 24 deletions(-)
diff --git a/src/lib/json/json.c b/src/lib/json/json.c
index 7ef511d4e..370217076 100644
--- a/src/lib/json/json.c
+++ b/src/lib/json/json.c
@@ -55,7 +55,7 @@
static inline int
json_read_symbol(struct json_lexer *lexer, UChar32 *out)
{
- if (lexer->offset == lexer->src_len) {
+ if (json_lexer_is_eof(lexer)) {
*out = U_SENTINEL;
return lexer->symbol_count + 1;
}
@@ -211,7 +211,7 @@ json_parse_identifier(struct json_lexer *lexer, struct json_token *token)
int
json_lexer_next_token(struct json_lexer *lexer, struct json_token *token)
{
- if (lexer->offset == lexer->src_len) {
+ if (json_lexer_is_eof(lexer)) {
token->type = JSON_TOKEN_END;
return 0;
}
@@ -223,7 +223,7 @@ json_lexer_next_token(struct json_lexer *lexer, struct json_token *token)
switch(c) {
case (UChar32)'[':
/* Error for '[\0'. */
- if (lexer->offset == lexer->src_len)
+ if (json_lexer_is_eof(lexer))
return lexer->symbol_count;
c = json_current_char(lexer);
if (c == '"' || c == '\'') {
@@ -240,14 +240,14 @@ json_lexer_next_token(struct json_lexer *lexer, struct json_token *token)
* Expression, started from [ must be finished
* with ] regardless of its type.
*/
- if (lexer->offset == lexer->src_len ||
+ if (json_lexer_is_eof(lexer) ||
json_current_char(lexer) != ']')
return lexer->symbol_count + 1;
/* Skip ] - one byte char. */
json_skip_char(lexer);
return 0;
case (UChar32)'.':
- if (lexer->offset == lexer->src_len)
+ if (json_lexer_is_eof(lexer))
return lexer->symbol_count + 1;
return json_parse_identifier(lexer, token);
default:
@@ -257,26 +257,15 @@ json_lexer_next_token(struct json_lexer *lexer, struct json_token *token)
}
/**
- * Compare JSON token keys.
+ * Compare JSON tokens as nodes of a JSON tree. That is, including
+ * parent references.
*/
static int
-json_token_cmp(const struct json_token *a, const struct json_token *b)
+json_token_cmp_in_tree(const struct json_token *a, const struct json_token *b)
{
if (a->parent != b->parent)
return a->parent - b->parent;
- if (a->type != b->type)
- return a->type - b->type;
- int ret = 0;
- if (a->type == JSON_TOKEN_STR) {
- if (a->len != b->len)
- return a->len - b->len;
- ret = memcmp(a->str, b->str, a->len);
- } else if (a->type == JSON_TOKEN_NUM) {
- ret = a->num - b->num;
- } else {
- assert(a->type == JSON_TOKEN_ANY);
- }
- return ret;
+ return json_token_cmp(a, b);
}
int
@@ -287,7 +276,7 @@ json_path_cmp(const char *a, int a_len, const char *b, int b_len,
json_lexer_create(&lexer_a, a, a_len, index_base);
json_lexer_create(&lexer_b, b, b_len, index_base);
struct json_token token_a, token_b;
- /* For the sake of json_token_cmp(). */
+ /* For the sake of json_token_cmp_in_tree(). */
token_a.parent = NULL;
token_b.parent = NULL;
int rc_a, rc_b;
@@ -295,7 +284,7 @@ json_path_cmp(const char *a, int a_len, const char *b, int b_len,
(rc_b = json_lexer_next_token(&lexer_b, &token_b)) == 0 &&
token_a.type != JSON_TOKEN_END &&
token_b.type != JSON_TOKEN_END) {
- int rc = json_token_cmp(&token_a, &token_b);
+ int rc = json_token_cmp_in_tree(&token_a, &token_b);
if (rc != 0)
return rc;
}
@@ -421,8 +410,8 @@ json_tree_snprint_path(char *buf, int size, const struct json_token *token,
#define mh_arg_t void *
#define mh_hash(a, arg) ((*(a))->hash)
#define mh_hash_key(a, arg) ((a)->hash)
-#define mh_cmp(a, b, arg) (json_token_cmp(*(a), *(b)))
-#define mh_cmp_key(a, b, arg) (json_token_cmp((a), *(b)))
+#define mh_cmp(a, b, arg) (json_token_cmp_in_tree(*(a), *(b)))
+#define mh_cmp_key(a, b, arg) (json_token_cmp_in_tree((a), *(b)))
#include "salad/mhash.h"
static const uint32_t hash_seed = 13U;
diff --git a/src/lib/json/json.h b/src/lib/json/json.h
index d66a9c7a4..08a0ee96c 100644
--- a/src/lib/json/json.h
+++ b/src/lib/json/json.h
@@ -241,6 +241,13 @@ json_lexer_create(struct json_lexer *lexer, const char *src, int src_len,
int
json_lexer_next_token(struct json_lexer *lexer, struct json_token *token);
+/** Check if @a lexer has finished parsing. */
+static inline bool
+json_lexer_is_eof(const struct json_lexer *lexer)
+{
+ return lexer->offset == lexer->src_len;
+}
+
/**
* Compare two JSON paths using Lexer class.
* - in case of paths that have same token-sequence prefix,
@@ -279,6 +286,27 @@ json_token_is_leaf(struct json_token *token)
return token->max_child_idx < 0;
}
+/**
+ * Compare two JSON tokens, not taking into account their tree
+ * attributes.
+ */
+static inline int
+json_token_cmp(const struct json_token *l, const struct json_token *r)
+{
+ if (l->type != r->type)
+ return l->type - r->type;
+ switch(l->type) {
+ case JSON_TOKEN_NUM:
+ return l->num - r->num;
+ case JSON_TOKEN_STR:
+ if (l->len != r->len)
+ return l->len - r->len;
+ return memcmp(l->str, r->str, l->len);
+ default:
+ return 0;
+ }
+}
+
/**
* Test if a given JSON token is multikey.
*/
--
2.20.1 (Apple Git-117)
^ permalink raw reply [flat|nested] 15+ messages in thread* [tarantool-patches] [PATCH 09/13] tuple: account the whole array in field.data and size
2019-08-12 23:05 [tarantool-patches] [PATCH 00/13] JSON updates Vladislav Shpilevoy
` (11 preceding siblings ...)
2019-08-12 23:05 ` [tarantool-patches] [PATCH 08/13] json: lexer_eof and token_cmp helper functions Vladislav Shpilevoy
@ 2019-08-12 23:05 ` Vladislav Shpilevoy
2019-08-20 18:49 ` [tarantool-patches] Re: [PATCH 00/13] JSON updates Vladislav Shpilevoy
13 siblings, 0 replies; 15+ messages in thread
From: Vladislav Shpilevoy @ 2019-08-12 23:05 UTC (permalink / raw)
To: tarantool-patches
Before the patch a struct field object, describing an array,
didn't account array header in its size and data. Indeed, it
was not needed, because anyway updates could be only 'flat',
and the header was regenerated anyway.
But after next patches an array update may be not on the top
level of an update tree. And its parent node with the current way
of accounting would not be able to see exact borders of each
child.
Part of #1261
---
src/box/tuple_update.c | 28 ++++++++++++++++------------
src/box/update/update_array.c | 9 +++++----
src/box/update/update_field.h | 6 ++++--
3 files changed, 25 insertions(+), 18 deletions(-)
diff --git a/src/box/tuple_update.c b/src/box/tuple_update.c
index 87946919d..81e1f7e97 100644
--- a/src/box/tuple_update.c
+++ b/src/box/tuple_update.c
@@ -193,11 +193,12 @@ update_read_ops(struct tuple_update *update, const char *expr,
* @retval -1 Error.
*/
static int
-update_do_ops(struct tuple_update *update, const char *old_data,
- const char *old_data_end, uint32_t part_count)
+update_do_ops(struct tuple_update *update, const char *header,
+ const char *old_data, const char *old_data_end,
+ uint32_t part_count)
{
- if (update_array_create(&update->root_array, old_data, old_data_end,
- part_count) != 0)
+ if (update_array_create(&update->root_array, header, old_data,
+ old_data_end, part_count) != 0)
return -1;
struct update_op *op = update->ops;
struct update_op *ops_end = op + update->op_count;
@@ -214,12 +215,12 @@ update_do_ops(struct tuple_update *update, const char *old_data,
* and it is enough to simply write the error to the log.
*/
static int
-upsert_do_ops(struct tuple_update *update, const char *old_data,
- const char *old_data_end, uint32_t part_count,
- bool suppress_error)
+upsert_do_ops(struct tuple_update *update, const char *header,
+ const char *old_data, const char *old_data_end,
+ uint32_t part_count, bool suppress_error)
{
- if (update_array_create(&update->root_array, old_data, old_data_end,
- part_count) != 0)
+ if (update_array_create(&update->root_array, header, old_data,
+ old_data_end, part_count) != 0)
return -1;
struct update_op *op = update->ops;
struct update_op *ops_end = op + update->op_count;
@@ -276,11 +277,13 @@ tuple_update_execute(const char *expr,const char *expr_end,
{
struct tuple_update update;
update_init(&update, index_base);
+ const char *header = old_data;
uint32_t field_count = mp_decode_array(&old_data);
if (update_read_ops(&update, expr, expr_end, dict, field_count) != 0)
return NULL;
- if (update_do_ops(&update, old_data, old_data_end, field_count))
+ if (update_do_ops(&update, header, old_data, old_data_end,
+ field_count) != 0)
return NULL;
if (column_mask)
*column_mask = update.column_mask;
@@ -296,12 +299,13 @@ tuple_upsert_execute(const char *expr,const char *expr_end,
{
struct tuple_update update;
update_init(&update, index_base);
+ const char *header = old_data;
uint32_t field_count = mp_decode_array(&old_data);
if (update_read_ops(&update, expr, expr_end, dict, field_count) != 0)
return NULL;
- if (upsert_do_ops(&update, old_data, old_data_end, field_count,
- suppress_error))
+ if (upsert_do_ops(&update, header, old_data, old_data_end, field_count,
+ suppress_error) != 0)
return NULL;
if (column_mask)
*column_mask = update.column_mask;
diff --git a/src/box/update/update_array.c b/src/box/update/update_array.c
index 985d555af..5b834b644 100644
--- a/src/box/update/update_array.c
+++ b/src/box/update/update_array.c
@@ -143,12 +143,13 @@ update_array_extract_item(struct update_field *field, struct update_op *op)
}
int
-update_array_create(struct update_field *field, const char *data,
- const char *data_end, uint32_t field_count)
+update_array_create(struct update_field *field, const char *header,
+ const char *data, const char *data_end,
+ uint32_t field_count)
{
field->type = UPDATE_ARRAY;
- field->data = data;
- field->size = data_end - data;
+ field->data = header;
+ field->size = data_end - header;
field->array.rope = rope_new(&fiber()->gc);
if (field->array.rope == NULL)
return -1;
diff --git a/src/box/update/update_field.h b/src/box/update/update_field.h
index b03c7479f..35ec85238 100644
--- a/src/box/update/update_field.h
+++ b/src/box/update/update_field.h
@@ -303,6 +303,7 @@ update_##type##_store(struct update_field *field, char *out, char *out_end);
/**
* Initialize @a field as an array to update.
* @param[out] field Field to initialize.
+ * @param header Header of the MessagePack array @a data.
* @param data MessagePack data of the array to update.
* @param data_end End of @a data.
* @param field_count Field count in @data.
@@ -311,8 +312,9 @@ update_##type##_store(struct update_field *field, char *out, char *out_end);
* @retval -1 Error.
*/
int
-update_array_create(struct update_field *field, const char *data,
- const char *data_end, uint32_t field_count);
+update_array_create(struct update_field *field, const char *header,
+ const char *data, const char *data_end,
+ uint32_t field_count);
OP_DECL_GENERIC(array)
--
2.20.1 (Apple Git-117)
^ permalink raw reply [flat|nested] 15+ messages in thread* [tarantool-patches] Re: [PATCH 00/13] JSON updates
2019-08-12 23:05 [tarantool-patches] [PATCH 00/13] JSON updates Vladislav Shpilevoy
` (12 preceding siblings ...)
2019-08-12 23:05 ` [tarantool-patches] [PATCH 09/13] tuple: account the whole array in field.data and size Vladislav Shpilevoy
@ 2019-08-20 18:49 ` Vladislav Shpilevoy
13 siblings, 0 replies; 15+ messages in thread
From: Vladislav Shpilevoy @ 2019-08-20 18:49 UTC (permalink / raw)
To: tarantool-patches
Hello, darkness, my old friend ...
^ permalink raw reply [flat|nested] 15+ messages in thread