This patch-set introduces a new syntax that can be used to create ARRAY values. https://github.com/tarantool/tarantool/issues/4762 https://github.com/tarantool/tarantool/tree/imeevma/gh-4762-syntax-for-array Mergen Imeev (3): sql: change mpstream_encode_vdbe_mem() signature sql: refactor sql_vdbe_mem_encode_tuple() sql: introduce syntax for ARRAY values .../gh-4762-introduce-array-to-sql.md | 3 +- src/box/sql.c | 2 +- src/box/sql/expr.c | 20 +++++++++++++ src/box/sql/mem.c | 26 ++++++++--------- src/box/sql/mem.h | 23 +++++++-------- src/box/sql/parse.y | 14 ++++++++++ src/box/sql/tokenize.c | 10 ++++++- src/box/sql/vdbe.c | 25 +++++++++++++++-- src/box/sql/vdbeapi.c | 4 +-- test/sql-tap/array.test.lua | 28 ++++++++++++++++++- test/sql-tap/colname.test.lua | 4 +-- 11 files changed, 123 insertions(+), 36 deletions(-) -- 2.25.1
This patch changes the signature of mpstream_encode_vdbe_mem(), so it now follows the general rule that applies to most functions in mem.c. --- src/box/sql/mem.c | 6 +++--- src/box/sql/mem.h | 7 ++++--- src/box/sql/vdbe.c | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/box/sql/mem.c b/src/box/sql/mem.c index aaed522ef..356b2c7be 100644 --- a/src/box/sql/mem.c +++ b/src/box/sql/mem.c @@ -2971,7 +2971,7 @@ mem_from_mp(struct Mem *mem, const char *buf, uint32_t *len) } void -mpstream_encode_vdbe_mem(struct mpstream *stream, struct Mem *var) +mem_to_mpstream(const struct Mem *var, struct mpstream *stream) { assert(memIsValid(var)); switch (var->type) { @@ -3023,7 +3023,7 @@ sql_vdbe_mem_encode_tuple(struct Mem *fields, uint32_t field_count, set_encode_error, &is_error); mpstream_encode_array(&stream, field_count); for (struct Mem *field = fields; field < fields + field_count; field++) - mpstream_encode_vdbe_mem(&stream, field); + mem_to_mpstream(field, &stream); mpstream_flush(&stream); if (is_error) { diag_set(OutOfMemory, stream.pos - stream.buf, @@ -3120,7 +3120,7 @@ port_vdbemem_get_msgpack(struct port *base, uint32_t *size) set_encode_error, &is_error); mpstream_encode_array(&stream, port->mem_count); for (uint32_t i = 0; i < port->mem_count && !is_error; i++) - mpstream_encode_vdbe_mem(&stream, (struct Mem *)port->mem + i); + mem_to_mpstream((struct Mem *)port->mem + i, &stream); mpstream_flush(&stream); *size = region_used(region) - region_svp; if (is_error) diff --git a/src/box/sql/mem.h b/src/box/sql/mem.h index c983534d5..c1a18fd3a 100644 --- a/src/box/sql/mem.h +++ b/src/box/sql/mem.h @@ -836,12 +836,13 @@ int mem_from_mp(struct Mem *mem, const char *buf, uint32_t *len); /** - * Perform encoding memory variable to stream. + * Perform encoding of MEM to stream. + * + * @param var MEM to encode to stream. * @param stream Initialized mpstream encoder object. - * @param var Vdbe memory variable to encode with stream. */ void -mpstream_encode_vdbe_mem(struct mpstream *stream, struct Mem *var); +mem_to_mpstream(const struct Mem *var, struct mpstream *stream); /** * Perform encoding field_count Vdbe memory fields on region as diff --git a/src/box/sql/vdbe.c b/src/box/sql/vdbe.c index 0c4e38557..3892cc102 100644 --- a/src/box/sql/vdbe.c +++ b/src/box/sql/vdbe.c @@ -3497,7 +3497,7 @@ case OP_Update: { mpstream_encode_array(&stream, 3); mpstream_encode_strn(&stream, "=", 1); mpstream_encode_uint(&stream, field_idx); - mpstream_encode_vdbe_mem(&stream, new_tuple + field_idx); + mem_to_mpstream(new_tuple + field_idx, &stream); } mpstream_flush(&stream); if (is_error) { -- 2.25.1
This function is used to create ARRAY value from array of MEMs. Since ARRAY was added to SQL, this function needs to be refactored. Part of #4762 --- src/box/sql.c | 2 +- src/box/sql/mem.c | 22 +++++++++++----------- src/box/sql/mem.h | 16 ++++++++-------- src/box/sql/vdbe.c | 3 +-- src/box/sql/vdbeapi.c | 4 ++-- 5 files changed, 23 insertions(+), 24 deletions(-) diff --git a/src/box/sql.c b/src/box/sql.c index d15159d6e..2a78a96d5 100644 --- a/src/box/sql.c +++ b/src/box/sql.c @@ -211,7 +211,7 @@ sql_cursor_seek(struct BtCursor *cur, struct Mem *mems, uint32_t len, int *res) struct region *region = &fiber()->gc; size_t used = region_used(region); uint32_t size; - const char *tuple = sql_vdbe_mem_encode_tuple(mems, len, &size, region); + const char *tuple = mem_encode_array(mems, len, &size, region); if (tuple == NULL) return -1; if (key_alloc(cur, size) != 0) diff --git a/src/box/sql/mem.c b/src/box/sql/mem.c index 356b2c7be..2ba4135b0 100644 --- a/src/box/sql/mem.c +++ b/src/box/sql/mem.c @@ -3013,31 +3013,31 @@ mem_to_mpstream(const struct Mem *var, struct mpstream *stream) } char * -sql_vdbe_mem_encode_tuple(struct Mem *fields, uint32_t field_count, - uint32_t *tuple_size, struct region *region) +mem_encode_array(const struct Mem *mems, uint32_t count, uint32_t *size, + struct region *region) { size_t used = region_used(region); bool is_error = false; struct mpstream stream; mpstream_init(&stream, region, region_reserve_cb, region_alloc_cb, set_encode_error, &is_error); - mpstream_encode_array(&stream, field_count); - for (struct Mem *field = fields; field < fields + field_count; field++) - mem_to_mpstream(field, &stream); + mpstream_encode_array(&stream, count); + for (const struct Mem *mem = mems; mem < mems + count; mem++) + mem_to_mpstream(mem, &stream); mpstream_flush(&stream); if (is_error) { diag_set(OutOfMemory, stream.pos - stream.buf, "mpstream_flush", "stream"); return NULL; } - *tuple_size = region_used(region) - used; - char *tuple = region_join(region, *tuple_size); - if (tuple == NULL) { - diag_set(OutOfMemory, *tuple_size, "region_join", "tuple"); + *size = region_used(region) - used; + char *array = region_join(region, *size); + if (array == NULL) { + diag_set(OutOfMemory, *size, "region_join", "array"); return NULL; } - mp_tuple_assert(tuple, tuple + *tuple_size); - return tuple; + mp_tuple_assert(array, array + *size); + return array; } /** diff --git a/src/box/sql/mem.h b/src/box/sql/mem.h index c1a18fd3a..1732152b4 100644 --- a/src/box/sql/mem.h +++ b/src/box/sql/mem.h @@ -845,15 +845,15 @@ void mem_to_mpstream(const struct Mem *var, struct mpstream *stream); /** - * Perform encoding field_count Vdbe memory fields on region as - * msgpack array. - * @param fields The first Vdbe memory field to encode. - * @param field_count Count of fields to encode. - * @param[out] tuple_size Size of encoded tuple. + * Encode array of MEMs as msgpack array on region. + * + * @param mems array of MEMs to encode. + * @param count number of elements in the array. + * @param[out] size Size of encoded msgpack array. * @param region Region to use. * @retval NULL on error, diag message is set. - * @retval Pointer to valid tuple on success. + * @retval Pointer to valid msgpack array on success. */ char * -sql_vdbe_mem_encode_tuple(struct Mem *fields, uint32_t field_count, - uint32_t *tuple_size, struct region *region); +mem_encode_array(const struct Mem *mems, uint32_t count, uint32_t *size, + struct region *region); diff --git a/src/box/sql/vdbe.c b/src/box/sql/vdbe.c index 3892cc102..2e6893f1a 100644 --- a/src/box/sql/vdbe.c +++ b/src/box/sql/vdbe.c @@ -2036,8 +2036,7 @@ case OP_MakeRecord: { struct region *region = &fiber()->gc; size_t used = region_used(region); uint32_t tuple_size; - char *tuple = - sql_vdbe_mem_encode_tuple(pData0, nField, &tuple_size, region); + char *tuple = mem_encode_array(pData0, nField, &tuple_size, region); if (tuple == NULL) goto abort_due_to_error; if ((int64_t)tuple_size > db->aLimit[SQL_LIMIT_LENGTH]) diff --git a/src/box/sql/vdbeapi.c b/src/box/sql/vdbeapi.c index 3894bb943..4ce5feeae 100644 --- a/src/box/sql/vdbeapi.c +++ b/src/box/sql/vdbeapi.c @@ -212,8 +212,8 @@ sql_stmt_result_to_msgpack(struct sql_stmt *stmt, uint32_t *tuple_size, struct region *region) { struct Vdbe *vdbe = (struct Vdbe *)stmt; - return sql_vdbe_mem_encode_tuple(vdbe->pResultSet, vdbe->nResColumn, - tuple_size, region); + return mem_encode_array(vdbe->pResultSet, vdbe->nResColumn, tuple_size, + region); } /* -- 2.25.1
This patch introduces a new syntax that allows to create ARRAY values in an SQL query. Part of #4762 @TarantoolBot document Title: Syntax for ARRAY in SQL The syntax for creating ARRAY values is available in SQL. You can use `[` and `]` to create an ARRAY value - all values in those brackets will be part of ARRAY. The position of the values will be translated to the same positions in ARRAY. Examples: ``` tarantool> box.execute("SELECT [1, 'a', 1.5];") --- - metadata: - name: COLUMN_1 type: array rows: - [[1, 'a', 1.5]] ... ``` ``` tarantool> box.execute("SELECT [1, 'a', ['abc', 321], 1.5];") --- - metadata: - name: COLUMN_1 type: array rows: - [[1, 'a', ['abc', 321], 1.5]] ... ``` --- .../gh-4762-introduce-array-to-sql.md | 3 +- src/box/sql/expr.c | 20 +++++++++++++ src/box/sql/parse.y | 14 ++++++++++ src/box/sql/tokenize.c | 10 ++++++- src/box/sql/vdbe.c | 20 +++++++++++++ test/sql-tap/array.test.lua | 28 ++++++++++++++++++- test/sql-tap/colname.test.lua | 4 +-- 7 files changed, 93 insertions(+), 6 deletions(-) diff --git a/changelogs/unreleased/gh-4762-introduce-array-to-sql.md b/changelogs/unreleased/gh-4762-introduce-array-to-sql.md index 1446ab1cb..fcfa53c73 100644 --- a/changelogs/unreleased/gh-4762-introduce-array-to-sql.md +++ b/changelogs/unreleased/gh-4762-introduce-array-to-sql.md @@ -1,3 +1,4 @@ ## feature/core - * Field type ARRAY is now available in SQL (gh-4762). + * Field type ARRAY is now available in SQL. The syntax has also been + implemented to allow the creation of ARRAY values (gh-4762). diff --git a/src/box/sql/expr.c b/src/box/sql/expr.c index 2c8021060..eb169aeb8 100644 --- a/src/box/sql/expr.c +++ b/src/box/sql/expr.c @@ -3370,6 +3370,22 @@ int_overflow: is_neg ? P4_INT64 : P4_UINT64); } +static void +expr_code_array(struct Parse *parser, struct Expr *expr, int reg) +{ + struct Vdbe *vdbe = parser->pVdbe; + struct ExprList *list = expr->x.pList; + if (list == NULL) { + sqlVdbeAddOp3(vdbe, OP_Array, 0, reg, 0); + return; + } + int count = list->nExpr; + int values_reg = parser->nMem + 1; + parser->nMem += count; + sqlExprCodeExprList(parser, list, values_reg, 0, SQL_ECEL_FACTOR); + sqlVdbeAddOp3(vdbe, OP_Array, count, reg, values_reg); +} + /* * Erase column-cache entry number i */ @@ -3821,6 +3837,10 @@ sqlExprCodeTarget(Parse * pParse, Expr * pExpr, int target) return inReg; } + case TK_ARRAY: + expr_code_array(pParse, pExpr, target); + break; + case TK_LT: case TK_LE: case TK_GT: diff --git a/src/box/sql/parse.y b/src/box/sql/parse.y index c93125a8b..499f235be 100644 --- a/src/box/sql/parse.y +++ b/src/box/sql/parse.y @@ -1128,6 +1128,20 @@ expr(A) ::= CAST(X) LP expr(E) AS typedef(T) RP(Y). { sqlExprAttachSubtrees(pParse->db, A.pExpr, E.pExpr, 0); } +expr(A) ::= LB(X) exprlist(Y) RB(E). { + struct Expr *expr = sql_expr_new_dequoted(pParse->db, TK_ARRAY, NULL); + if (expr == NULL) { + sql_expr_list_delete(pParse->db, Y); + pParse->is_aborted = true; + return; + } + expr->x.pList = Y; + expr->type = FIELD_TYPE_ARRAY; + sqlExprSetHeightAndFlags(pParse, expr); + A.pExpr = expr; + spanSet(&A, &X, &E); +} + expr(A) ::= TRIM(X) LP trim_operands(Y) RP(E). { A.pExpr = sqlExprFunction(pParse, Y, &X); spanSet(&A, &X, &E); diff --git a/src/box/sql/tokenize.c b/src/box/sql/tokenize.c index b3cf8f6e6..f2d5a2df5 100644 --- a/src/box/sql/tokenize.c +++ b/src/box/sql/tokenize.c @@ -83,6 +83,8 @@ #define CC_DOT 26 /* '.' */ #define CC_ILLEGAL 27 /* Illegal character */ #define CC_LINEFEED 28 /* '\n' */ +#define CC_LB 29 /* '[' */ +#define CC_RB 30 /* ']' */ static const char sql_ascii_class[] = { /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xa xb xc xd xe xf */ @@ -91,7 +93,7 @@ static const char sql_ascii_class[] = { /* 2x */ 7, 15, 9, 5, 4, 22, 24, 8, 17, 18, 21, 20, 23, 11, 26, 16, /* 3x */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 5, 19, 12, 14, 13, 6, /* 4x */ 5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -/* 5x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 27, 27, 27, 27, 1, +/* 5x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 29, 27, 30, 27, 1, /* 6x */ 27, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 7x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 27, 10, 27, 25, 27, /* 8x */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, @@ -220,6 +222,12 @@ sql_token(const char *z, int *type, bool *is_reserved) case CC_RP: *type = TK_RP; return 1; + case CC_LB: + *type = TK_LB; + return 1; + case CC_RB: + *type = TK_RB; + return 1; case CC_SEMI: *type = TK_SEMI; return 1; diff --git a/src/box/sql/vdbe.c b/src/box/sql/vdbe.c index 2e6893f1a..55e494332 100644 --- a/src/box/sql/vdbe.c +++ b/src/box/sql/vdbe.c @@ -1418,6 +1418,26 @@ case OP_Cast: { /* in1 */ goto abort_due_to_error; } +/* Opcode: Array P1 P2 P3 * * + * Synopsis: r[P2]=array(P3@P1) + * + * Construct an ARRAY value from P1 registers starting at reg(P3). + */ +case OP_Array: { + pOut = &aMem[pOp->p2]; + + uint32_t size; + struct region *region = &fiber()->gc; + size_t svp = region_used(region); + char *val = mem_encode_array(&aMem[pOp->p3], pOp->p1, &size, region); + if (val == NULL || mem_copy_array(pOut, val, size) != 0) { + region_truncate(region, svp); + goto abort_due_to_error; + } + region_truncate(region, svp); + break; +} + /* Opcode: Eq P1 P2 P3 P4 P5 * Synopsis: IF r[P3]==r[P1] * diff --git a/test/sql-tap/array.test.lua b/test/sql-tap/array.test.lua index 752cb24f2..1f2ba6d5d 100755 --- a/test/sql-tap/array.test.lua +++ b/test/sql-tap/array.test.lua @@ -1,6 +1,6 @@ #!/usr/bin/env tarantool local test = require("sqltester") -test:plan(110) +test:plan(113) box.schema.func.create('A1', { language = 'Lua', @@ -979,6 +979,32 @@ test:do_catchsql_test( 1, "Failed to execute SQL statement: wrong arguments for function ZEROBLOB()" }) +-- Make sure syntax for ARRAY values works as intended. +test:do_execsql_test( + "array-13.1", + [[ + SELECT [a, g, t, n, f, i, b, v, s, d, u] FROM t1 WHERE id = 1; + ]], { + {{1}, 1, '1', 1, 1, 1, true, '1', 1, require('decimal').new(1), + require('uuid').fromstr('11111111-1111-1111-1111-111111111111')} + }) + +test:do_execsql_test( + "array-13.2", + [[ + SELECT [1, true, 1.5e0, ['asd', x'32'], 1234.0]; + ]], { + {1, true, 1.5, {'asd', '2'}, require('decimal').new(1234)} + }) + +test:do_execsql_test( + "array-13.3", + [[ + SELECT typeof([1]); + ]], { + "array" + }) + box.execute([[DROP TABLE t1;]]) box.execute([[DROP TABLE t;]]) diff --git a/test/sql-tap/colname.test.lua b/test/sql-tap/colname.test.lua index ff7585c7a..698a446e1 100755 --- a/test/sql-tap/colname.test.lua +++ b/test/sql-tap/colname.test.lua @@ -1,6 +1,6 @@ #!/usr/bin/env tarantool local test = require("sqltester") -test:plan(79) +test:plan(76) --!./tcltestrunner.lua -- 2008 July 15 @@ -546,7 +546,6 @@ test:do_test( local data = { [[`a`]], - "[a]", } for i, val in ipairs(data) do test:do_catchsql_test( @@ -559,7 +558,6 @@ end local data2 = { {[['a']],{1, "/Syntax error/"}}, -- because ' is delimiter for strings {[[`a`]],{1, "/unrecognized token/"}}, -- because ` is undefined symbol - {"[a]",{1, "/unrecognized token/"}} -- because [ is undefined symbol } for i, val in ipairs(data2) do test:do_catchsql_test( -- 2.25.1
Hi! Thanks for the patch! > diff --git a/src/box/sql/mem.c b/src/box/sql/mem.c > index 356b2c7be..2ba4135b0 100644 > --- a/src/box/sql/mem.c > +++ b/src/box/sql/mem.c > @@ -3013,31 +3013,31 @@ mem_to_mpstream(const struct Mem *var, struct mpstream *stream) > } > > char * > -sql_vdbe_mem_encode_tuple(struct Mem *fields, uint32_t field_count, > - uint32_t *tuple_size, struct region *region) > +mem_encode_array(const struct Mem *mems, uint32_t count, uint32_t *size, > + struct region *region) > { > size_t used = region_used(region); > bool is_error = false; > struct mpstream stream; > mpstream_init(&stream, region, region_reserve_cb, region_alloc_cb, > set_encode_error, &is_error); > - mpstream_encode_array(&stream, field_count); > - for (struct Mem *field = fields; field < fields + field_count; field++) > - mem_to_mpstream(field, &stream); > + mpstream_encode_array(&stream, count); > + for (const struct Mem *mem = mems; mem < mems + count; mem++) > + mem_to_mpstream(mem, &stream); > mpstream_flush(&stream); > if (is_error) { If some memory was allocated/reserved before mpstream_flush() failed (inside of mem_to_mpstream()), then the region might leak. I would recommend to add a truncate here. > diag_set(OutOfMemory, stream.pos - stream.buf, > "mpstream_flush", "stream"); > return NULL; > }
Thanks for the patch! Cool feature! Could you also please add a test for an empty array creation? As [].
Hi! Thank you for the review! My answer, diff and new patch below. Also, I fixed one comment about region_truncate() from the letters about MAP. On Sat, Nov 20, 2021 at 12:35:46AM +0100, Vladislav Shpilevoy wrote: > Hi! Thanks for the patch! > > > diff --git a/src/box/sql/mem.c b/src/box/sql/mem.c > > index 356b2c7be..2ba4135b0 100644 > > --- a/src/box/sql/mem.c > > +++ b/src/box/sql/mem.c > > @@ -3013,31 +3013,31 @@ mem_to_mpstream(const struct Mem *var, struct mpstream *stream) > > } > > > > char * > > -sql_vdbe_mem_encode_tuple(struct Mem *fields, uint32_t field_count, > > - uint32_t *tuple_size, struct region *region) > > +mem_encode_array(const struct Mem *mems, uint32_t count, uint32_t *size, > > + struct region *region) > > { > > size_t used = region_used(region); > > bool is_error = false; > > struct mpstream stream; > > mpstream_init(&stream, region, region_reserve_cb, region_alloc_cb, > > set_encode_error, &is_error); > > - mpstream_encode_array(&stream, field_count); > > - for (struct Mem *field = fields; field < fields + field_count; field++) > > - mem_to_mpstream(field, &stream); > > + mpstream_encode_array(&stream, count); > > + for (const struct Mem *mem = mems; mem < mems + count; mem++) > > + mem_to_mpstream(mem, &stream); > > mpstream_flush(&stream); > > if (is_error) { > > If some memory was allocated/reserved before mpstream_flush() failed (inside > of mem_to_mpstream()), then the region might leak. I would recommend to add a > truncate here. > Thank you. Fixed. > > diag_set(OutOfMemory, stream.pos - stream.buf, > > "mpstream_flush", "stream"); > > return NULL; > > } Diff: diff --git a/src/box/sql/mem.c b/src/box/sql/mem.c index 71a10c9b3..32b8825bc 100644 --- a/src/box/sql/mem.c +++ b/src/box/sql/mem.c @@ -3054,6 +3054,7 @@ mem_encode_array(const struct Mem *mems, uint32_t count, uint32_t *size, mem_to_mpstream(mem, &stream); mpstream_flush(&stream); if (is_error) { + region_truncate(region, used); diag_set(OutOfMemory, stream.pos - stream.buf, "mpstream_flush", "stream"); return NULL; @@ -3061,6 +3062,7 @@ mem_encode_array(const struct Mem *mems, uint32_t count, uint32_t *size, *size = region_used(region) - used; char *array = region_join(region, *size); if (array == NULL) { + region_truncate(region, used); diag_set(OutOfMemory, *size, "region_join", "array"); return NULL; } New patch: commit 14ddc911967143075a373a24d318b36120144267 Author: Mergen Imeev <imeevma@gmail.com> Date: Tue Nov 16 09:54:09 2021 +0300 sql: refactor sql_vdbe_mem_encode_tuple() This function is used to create ARRAY value from array of MEMs. Since ARRAY was added to SQL, this function needs to be refactored. Part of #4762 diff --git a/src/box/sql.c b/src/box/sql.c index d15159d6e..2a78a96d5 100644 --- a/src/box/sql.c +++ b/src/box/sql.c @@ -211,7 +211,7 @@ sql_cursor_seek(struct BtCursor *cur, struct Mem *mems, uint32_t len, int *res) struct region *region = &fiber()->gc; size_t used = region_used(region); uint32_t size; - const char *tuple = sql_vdbe_mem_encode_tuple(mems, len, &size, region); + const char *tuple = mem_encode_array(mems, len, &size, region); if (tuple == NULL) return -1; if (key_alloc(cur, size) != 0) diff --git a/src/box/sql/mem.c b/src/box/sql/mem.c index 303c3a1a3..32b8825bc 100644 --- a/src/box/sql/mem.c +++ b/src/box/sql/mem.c @@ -3041,31 +3041,33 @@ mem_to_mpstream(const struct Mem *var, struct mpstream *stream) } char * -sql_vdbe_mem_encode_tuple(struct Mem *fields, uint32_t field_count, - uint32_t *tuple_size, struct region *region) +mem_encode_array(const struct Mem *mems, uint32_t count, uint32_t *size, + struct region *region) { size_t used = region_used(region); bool is_error = false; struct mpstream stream; mpstream_init(&stream, region, region_reserve_cb, region_alloc_cb, set_encode_error, &is_error); - mpstream_encode_array(&stream, field_count); - for (struct Mem *field = fields; field < fields + field_count; field++) - mem_to_mpstream(field, &stream); + mpstream_encode_array(&stream, count); + for (const struct Mem *mem = mems; mem < mems + count; mem++) + mem_to_mpstream(mem, &stream); mpstream_flush(&stream); if (is_error) { + region_truncate(region, used); diag_set(OutOfMemory, stream.pos - stream.buf, "mpstream_flush", "stream"); return NULL; } - *tuple_size = region_used(region) - used; - char *tuple = region_join(region, *tuple_size); - if (tuple == NULL) { - diag_set(OutOfMemory, *tuple_size, "region_join", "tuple"); + *size = region_used(region) - used; + char *array = region_join(region, *size); + if (array == NULL) { + region_truncate(region, used); + diag_set(OutOfMemory, *size, "region_join", "array"); return NULL; } - mp_tuple_assert(tuple, tuple + *tuple_size); - return tuple; + mp_tuple_assert(array, array + *size); + return array; } /** diff --git a/src/box/sql/mem.h b/src/box/sql/mem.h index ea9ef709d..7f5ecf954 100644 --- a/src/box/sql/mem.h +++ b/src/box/sql/mem.h @@ -862,15 +862,15 @@ void mem_to_mpstream(const struct Mem *var, struct mpstream *stream); /** - * Perform encoding field_count Vdbe memory fields on region as - * msgpack array. - * @param fields The first Vdbe memory field to encode. - * @param field_count Count of fields to encode. - * @param[out] tuple_size Size of encoded tuple. + * Encode array of MEMs as msgpack array on region. + * + * @param mems array of MEMs to encode. + * @param count number of elements in the array. + * @param[out] size Size of encoded msgpack array. * @param region Region to use. * @retval NULL on error, diag message is set. - * @retval Pointer to valid tuple on success. + * @retval Pointer to valid msgpack array on success. */ char * -sql_vdbe_mem_encode_tuple(struct Mem *fields, uint32_t field_count, - uint32_t *tuple_size, struct region *region); +mem_encode_array(const struct Mem *mems, uint32_t count, uint32_t *size, + struct region *region); diff --git a/src/box/sql/vdbe.c b/src/box/sql/vdbe.c index 3892cc102..2e6893f1a 100644 --- a/src/box/sql/vdbe.c +++ b/src/box/sql/vdbe.c @@ -2036,8 +2036,7 @@ case OP_MakeRecord: { struct region *region = &fiber()->gc; size_t used = region_used(region); uint32_t tuple_size; - char *tuple = - sql_vdbe_mem_encode_tuple(pData0, nField, &tuple_size, region); + char *tuple = mem_encode_array(pData0, nField, &tuple_size, region); if (tuple == NULL) goto abort_due_to_error; if ((int64_t)tuple_size > db->aLimit[SQL_LIMIT_LENGTH]) diff --git a/src/box/sql/vdbeapi.c b/src/box/sql/vdbeapi.c index 3894bb943..4ce5feeae 100644 --- a/src/box/sql/vdbeapi.c +++ b/src/box/sql/vdbeapi.c @@ -212,8 +212,8 @@ sql_stmt_result_to_msgpack(struct sql_stmt *stmt, uint32_t *tuple_size, struct region *region) { struct Vdbe *vdbe = (struct Vdbe *)stmt; - return sql_vdbe_mem_encode_tuple(vdbe->pResultSet, vdbe->nResColumn, - tuple_size, region); + return mem_encode_array(vdbe->pResultSet, vdbe->nResColumn, tuple_size, + region); } /*
Thank you for the review! My answer, diff and new patch below. Also I replaced sql_expr_new_dequoted() by sql_expr_new_anon(). On Sat, Nov 20, 2021 at 12:36:19AM +0100, Vladislav Shpilevoy wrote: > Thanks for the patch! Cool feature! > > Could you also please add a test for an empty array creation? As []. Added. Also I added a test about creation of ARRAY value with more than 1000 elements. This is possible because creation of ARRAY is an operation and not a built-in function. Diff: diff --git a/src/box/sql/parse.y b/src/box/sql/parse.y index 92fb37dd4..ee319d5ad 100644 --- a/src/box/sql/parse.y +++ b/src/box/sql/parse.y @@ -1129,7 +1129,7 @@ expr(A) ::= CAST(X) LP expr(E) AS typedef(T) RP(Y). { } expr(A) ::= LB(X) exprlist(Y) RB(E). { - struct Expr *expr = sql_expr_new_dequoted(pParse->db, TK_ARRAY, NULL); + struct Expr *expr = sql_expr_new_anon(pParse->db, TK_ARRAY); if (expr == NULL) { sql_expr_list_delete(pParse->db, Y); pParse->is_aborted = true; diff --git a/test/sql-tap/array.test.lua b/test/sql-tap/array.test.lua index 1f2ba6d5d..79a1c831d 100755 --- a/test/sql-tap/array.test.lua +++ b/test/sql-tap/array.test.lua @@ -1,6 +1,6 @@ #!/usr/bin/env tarantool local test = require("sqltester") -test:plan(113) +test:plan(115) box.schema.func.create('A1', { language = 'Lua', @@ -999,6 +999,25 @@ test:do_execsql_test( test:do_execsql_test( "array-13.3", + [[ + SELECT []; + ]], { + {} + }) + +local arr = {0} +local arr_str = '0' +for i = 1, 1000 do table.insert(arr, i) arr_str = arr_str .. ', ' .. i end +test:do_execsql_test( + "array-13.4", + [[ + SELECT []] .. arr_str .. [[]; + ]], { + arr + }) + +test:do_execsql_test( + "array-13.5", [[ SELECT typeof([1]); ]], { New patch: commit 9036493dd4085cc5ce73fc0e8160cfdcd3412228 Author: Mergen Imeev <imeevma@gmail.com> Date: Tue Nov 16 10:16:25 2021 +0300 sql: introduce syntax for ARRAY values This patch introduces a new syntax that allows to create ARRAY values in an SQL query. Part of #4762 @TarantoolBot document Title: Syntax for ARRAY in SQL The syntax for creating ARRAY values is available in SQL. You can use `[` and `]` to create an ARRAY value - all values in those brackets will be part of ARRAY. The position of the values will be translated to the same positions in ARRAY. Examples: ``` tarantool> box.execute("SELECT [1, 'a', 1.5];") --- - metadata: - name: COLUMN_1 type: array rows: - [[1, 'a', 1.5]] ... ``` ``` tarantool> box.execute("SELECT [1, 'a', ['abc', 321], 1.5];") --- - metadata: - name: COLUMN_1 type: array rows: - [[1, 'a', ['abc', 321], 1.5]] ... ``` diff --git a/changelogs/unreleased/gh-4762-introduce-array-to-sql.md b/changelogs/unreleased/gh-4762-introduce-array-to-sql.md index 1446ab1cb..fcfa53c73 100644 --- a/changelogs/unreleased/gh-4762-introduce-array-to-sql.md +++ b/changelogs/unreleased/gh-4762-introduce-array-to-sql.md @@ -1,3 +1,4 @@ ## feature/core - * Field type ARRAY is now available in SQL (gh-4762). + * Field type ARRAY is now available in SQL. The syntax has also been + implemented to allow the creation of ARRAY values (gh-4762). diff --git a/src/box/sql/expr.c b/src/box/sql/expr.c index 2c8021060..eb169aeb8 100644 --- a/src/box/sql/expr.c +++ b/src/box/sql/expr.c @@ -3370,6 +3370,22 @@ int_overflow: is_neg ? P4_INT64 : P4_UINT64); } +static void +expr_code_array(struct Parse *parser, struct Expr *expr, int reg) +{ + struct Vdbe *vdbe = parser->pVdbe; + struct ExprList *list = expr->x.pList; + if (list == NULL) { + sqlVdbeAddOp3(vdbe, OP_Array, 0, reg, 0); + return; + } + int count = list->nExpr; + int values_reg = parser->nMem + 1; + parser->nMem += count; + sqlExprCodeExprList(parser, list, values_reg, 0, SQL_ECEL_FACTOR); + sqlVdbeAddOp3(vdbe, OP_Array, count, reg, values_reg); +} + /* * Erase column-cache entry number i */ @@ -3821,6 +3837,10 @@ sqlExprCodeTarget(Parse * pParse, Expr * pExpr, int target) return inReg; } + case TK_ARRAY: + expr_code_array(pParse, pExpr, target); + break; + case TK_LT: case TK_LE: case TK_GT: diff --git a/src/box/sql/parse.y b/src/box/sql/parse.y index 548004252..ee319d5ad 100644 --- a/src/box/sql/parse.y +++ b/src/box/sql/parse.y @@ -1128,6 +1128,20 @@ expr(A) ::= CAST(X) LP expr(E) AS typedef(T) RP(Y). { sqlExprAttachSubtrees(pParse->db, A.pExpr, E.pExpr, 0); } +expr(A) ::= LB(X) exprlist(Y) RB(E). { + struct Expr *expr = sql_expr_new_anon(pParse->db, TK_ARRAY); + if (expr == NULL) { + sql_expr_list_delete(pParse->db, Y); + pParse->is_aborted = true; + return; + } + expr->x.pList = Y; + expr->type = FIELD_TYPE_ARRAY; + sqlExprSetHeightAndFlags(pParse, expr); + A.pExpr = expr; + spanSet(&A, &X, &E); +} + expr(A) ::= TRIM(X) LP trim_operands(Y) RP(E). { A.pExpr = sqlExprFunction(pParse, Y, &X); spanSet(&A, &X, &E); diff --git a/src/box/sql/tokenize.c b/src/box/sql/tokenize.c index b3cf8f6e6..f2d5a2df5 100644 --- a/src/box/sql/tokenize.c +++ b/src/box/sql/tokenize.c @@ -83,6 +83,8 @@ #define CC_DOT 26 /* '.' */ #define CC_ILLEGAL 27 /* Illegal character */ #define CC_LINEFEED 28 /* '\n' */ +#define CC_LB 29 /* '[' */ +#define CC_RB 30 /* ']' */ static const char sql_ascii_class[] = { /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xa xb xc xd xe xf */ @@ -91,7 +93,7 @@ static const char sql_ascii_class[] = { /* 2x */ 7, 15, 9, 5, 4, 22, 24, 8, 17, 18, 21, 20, 23, 11, 26, 16, /* 3x */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 5, 19, 12, 14, 13, 6, /* 4x */ 5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -/* 5x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 27, 27, 27, 27, 1, +/* 5x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 29, 27, 30, 27, 1, /* 6x */ 27, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 7x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 27, 10, 27, 25, 27, /* 8x */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, @@ -220,6 +222,12 @@ sql_token(const char *z, int *type, bool *is_reserved) case CC_RP: *type = TK_RP; return 1; + case CC_LB: + *type = TK_LB; + return 1; + case CC_RB: + *type = TK_RB; + return 1; case CC_SEMI: *type = TK_SEMI; return 1; diff --git a/src/box/sql/vdbe.c b/src/box/sql/vdbe.c index 2e6893f1a..55e494332 100644 --- a/src/box/sql/vdbe.c +++ b/src/box/sql/vdbe.c @@ -1418,6 +1418,26 @@ case OP_Cast: { /* in1 */ goto abort_due_to_error; } +/* Opcode: Array P1 P2 P3 * * + * Synopsis: r[P2]=array(P3@P1) + * + * Construct an ARRAY value from P1 registers starting at reg(P3). + */ +case OP_Array: { + pOut = &aMem[pOp->p2]; + + uint32_t size; + struct region *region = &fiber()->gc; + size_t svp = region_used(region); + char *val = mem_encode_array(&aMem[pOp->p3], pOp->p1, &size, region); + if (val == NULL || mem_copy_array(pOut, val, size) != 0) { + region_truncate(region, svp); + goto abort_due_to_error; + } + region_truncate(region, svp); + break; +} + /* Opcode: Eq P1 P2 P3 P4 P5 * Synopsis: IF r[P3]==r[P1] * diff --git a/test/sql-tap/array.test.lua b/test/sql-tap/array.test.lua index 752cb24f2..79a1c831d 100755 --- a/test/sql-tap/array.test.lua +++ b/test/sql-tap/array.test.lua @@ -1,6 +1,6 @@ #!/usr/bin/env tarantool local test = require("sqltester") -test:plan(110) +test:plan(115) box.schema.func.create('A1', { language = 'Lua', @@ -979,6 +979,51 @@ test:do_catchsql_test( 1, "Failed to execute SQL statement: wrong arguments for function ZEROBLOB()" }) +-- Make sure syntax for ARRAY values works as intended. +test:do_execsql_test( + "array-13.1", + [[ + SELECT [a, g, t, n, f, i, b, v, s, d, u] FROM t1 WHERE id = 1; + ]], { + {{1}, 1, '1', 1, 1, 1, true, '1', 1, require('decimal').new(1), + require('uuid').fromstr('11111111-1111-1111-1111-111111111111')} + }) + +test:do_execsql_test( + "array-13.2", + [[ + SELECT [1, true, 1.5e0, ['asd', x'32'], 1234.0]; + ]], { + {1, true, 1.5, {'asd', '2'}, require('decimal').new(1234)} + }) + +test:do_execsql_test( + "array-13.3", + [[ + SELECT []; + ]], { + {} + }) + +local arr = {0} +local arr_str = '0' +for i = 1, 1000 do table.insert(arr, i) arr_str = arr_str .. ', ' .. i end +test:do_execsql_test( + "array-13.4", + [[ + SELECT []] .. arr_str .. [[]; + ]], { + arr + }) + +test:do_execsql_test( + "array-13.5", + [[ + SELECT typeof([1]); + ]], { + "array" + }) + box.execute([[DROP TABLE t1;]]) box.execute([[DROP TABLE t;]]) diff --git a/test/sql-tap/colname.test.lua b/test/sql-tap/colname.test.lua index ff7585c7a..698a446e1 100755 --- a/test/sql-tap/colname.test.lua +++ b/test/sql-tap/colname.test.lua @@ -1,6 +1,6 @@ #!/usr/bin/env tarantool local test = require("sqltester") -test:plan(79) +test:plan(76) --!./tcltestrunner.lua -- 2008 July 15 @@ -546,7 +546,6 @@ test:do_test( local data = { [[`a`]], - "[a]", } for i, val in ipairs(data) do test:do_catchsql_test( @@ -559,7 +558,6 @@ end local data2 = { {[['a']],{1, "/Syntax error/"}}, -- because ' is delimiter for strings {[[`a`]],{1, "/unrecognized token/"}}, -- because ` is undefined symbol - {"[a]",{1, "/unrecognized token/"}} -- because [ is undefined symbol } for i, val in ipairs(data2) do test:do_catchsql_test(
Hi! Thanks for the patchset! LGTM.
This patch-set introduces a new syntax that can be used to create ARRAY values in SQL. https://github.com/tarantool/tarantool/issues/4762 https://github.com/tarantool/tarantool/tree/imeevma/gh-4762-syntax-for-array Mergen Imeev (3): sql: change mpstream_encode_vdbe_mem() signature sql: refactor sql_vdbe_mem_encode_tuple() sql: introduce syntax for ARRAY values .../gh-4762-introduce-array-to-sql.md | 3 +- src/box/sql.c | 2 +- src/box/sql/expr.c | 20 ++++++++ src/box/sql/mem.c | 28 ++++++----- src/box/sql/mem.h | 23 ++++----- src/box/sql/parse.y | 14 ++++++ src/box/sql/tokenize.c | 10 +++- src/box/sql/vdbe.c | 25 ++++++++-- src/box/sql/vdbeapi.c | 4 +- test/sql-tap/array.test.lua | 47 ++++++++++++++++++- test/sql-tap/colname.test.lua | 4 +- 11 files changed, 144 insertions(+), 36 deletions(-) -- 2.25.1
Hello,
On 30 ноя 11:43, imeevma@tarantool.org wrote:
> This patch-set introduces a new syntax that can be used to create ARRAY values
> in SQL.
>
> https://github.com/tarantool/tarantool/issues/4762
> https://github.com/tarantool/tarantool/tree/imeevma/gh-4762-syntax-for-array
LGTM.
I've checked your patchset into master.
--
Regards, Kirill Yukhin