From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from localhost (localhost [127.0.0.1]) by turing.freelists.org (Avenir Technologies Mail Multiplex) with ESMTP id 052452BA98 for ; Thu, 29 Mar 2018 14:04:37 -0400 (EDT) Received: from turing.freelists.org ([127.0.0.1]) by localhost (turing.freelists.org [127.0.0.1]) (amavisd-new, port 10024) with ESMTP id 6G2HRShrgRTg for ; Thu, 29 Mar 2018 14:04:36 -0400 (EDT) Received: from smtp62.i.mail.ru (smtp62.i.mail.ru [217.69.128.42]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by turing.freelists.org (Avenir Technologies Mail Multiplex) with ESMTPS id 497D12BA83 for ; Thu, 29 Mar 2018 14:04:36 -0400 (EDT) Subject: [tarantool-patches] Re: [PATCH v2 3/3] Multibyte characters support References: From: Kirill Shcherbatov Message-ID: Date: Thu, 29 Mar 2018 21:04:33 +0300 MIME-Version: 1.0 In-Reply-To: Content-Type: text/plain; charset="utf-8"; format="flowed" Content-Language: en-US Content-Transfer-Encoding: 8bit Sender: tarantool-patches-bounce@freelists.org Errors-to: tarantool-patches-bounce@freelists.org Reply-To: tarantool-patches@freelists.org List-help: List-unsubscribe: List-software: Ecartis version 1.0.0 List-Id: tarantool-patches List-subscribe: List-owner: List-post: List-archive: To: tarantool-patches@freelists.org Cc: "v.shpilevoy@tarantool.org" From eb16ee6ea2d2ca5801b6b09c403c6e2f23984bd9 Mon Sep 17 00:00:00 2001 From: Kirill Shcherbatov Date: Thu, 29 Mar 2018 21:03:21 +0300 Subject: [PATCH] Multibyte characters support --- src/box/lua/tuple.c | 1 - src/lib/json/path.c | 35 ++++++++++++++++++++++++++++++----- src/lib/json/path.h | 2 ++ test/engine/tuple.result | 16 ++++++++++++++-- test/engine/tuple.test.lua | 5 ++++- 5 files changed, 50 insertions(+), 9 deletions(-) diff --git a/src/box/lua/tuple.c b/src/box/lua/tuple.c index 99b9ff2..c3a435b 100644 --- a/src/box/lua/tuple.c +++ b/src/box/lua/tuple.c @@ -413,7 +413,6 @@ lbox_tuple_transform(struct lua_State *L) static inline int tuple_field_go_to_index(const char **field, uint64_t index) { - assert(index >= 0); enum mp_type type = mp_typeof(**field); if (type == MP_ARRAY) { if (index == 0) diff --git a/src/lib/json/path.c b/src/lib/json/path.c index 4a6174e..1234f6b 100644 --- a/src/lib/json/path.c +++ b/src/lib/json/path.c @@ -31,6 +31,8 @@ #include "path.h" #include +#include +#include #include "trivia/util.h" /** Same as strtoull(), but with limited length. */ @@ -44,6 +46,26 @@ strntoull(const char *src, int len) { return value; } +static inline size_t +mbsize(const char *str, size_t str_size) +{ + mbstate_t ps; + memset(&ps, 0, sizeof(ps)); + mbrlen(NULL,0,&ps); + return mbrlen(str, str_size, &ps); +} + +static inline int +mb2wcisalpha(char *mb_char, size_t mb_char_size) +{ + assert(mb_char_size < 5); + wchar_t buff[2]; + mbstate_t ps; + memset(&ps, 0, sizeof(ps)); + mbsrtowcs(buff, (const char **)&mb_char, mb_char_size + 1, &ps); + return iswalpha((wint_t)buff[0]); +} + /** * Parse string identifier in quotes. Parser either stops right * after the closing quote, or returns an error position. @@ -126,17 +148,20 @@ json_parse_identifier(struct json_path_parser *parser, const char *str = pos; char c = *pos; /* First symbol can not be digit. */ - if (!isalpha(c) && c != '_') - return pos - parser->src + 1; + size_t mb_size = 0; + if ((mb_size = mbsize(pos, end - pos)) && !mb2wcisalpha((char *)pos, mb_size) && c != '_') + return pos - parser->src + mb_size; int len = 1; - for (c = *++pos; pos < end && (isalpha(c) || c == '_' || isdigit(c)); - c = *++pos) + for (c = *(pos += mb_size); + pos < end && (((mb_size = mbsize(pos, end - pos)) + && mb2wcisalpha((char *)pos, mb_size)) || c == '_' || isdigit(c)); + c = *(pos += mb_size)) ++len; assert(len > 0); parser->pos = pos; node->type = JSON_PATH_STR; node->str = str; - node->len = len; + node->len = (int)(pos - str); return 0; } diff --git a/src/lib/json/path.h b/src/lib/json/path.h index 6e8db4c..b1028f6 100644 --- a/src/lib/json/path.h +++ b/src/lib/json/path.h @@ -33,6 +33,8 @@ #include #include +#include +#include #ifdef __cplusplus extern "C" { diff --git a/test/engine/tuple.result b/test/engine/tuple.result index 2d7367a..c4b361a 100644 --- a/test/engine/tuple.result +++ b/test/engine/tuple.result @@ -611,7 +611,7 @@ s = box.schema.space.create('test', {format = format}) pk = s:create_index('pk') --- ... -field2 = {1, 2, 3, "4", {5,6,7}, {key="key1", value="value1"}} +field2 = {1, 2, 3, "4", {5,6,7}, {key="key1", value="value1", hello中国world = {中国 = 'test'}}} --- ... field3 = {[10] = 100, k1 = 100, k2 = {1,2,3}, k3 = { {a=1, b=2}, {c=3, d=4} }, [-1] = 200} @@ -626,7 +626,7 @@ t[1] ... t[2] --- -- [1, 2, 3, '4', [5, 6, 7], {'key': 'key1', 'value': 'value1'}] +- [1, 2, 3, '4', [5, 6, 7], {'hello中国world': {'中国': 'test'}, 'key': 'key1', 'value': 'value1'}] ... t[3] --- @@ -673,6 +673,10 @@ t["[2][6].value"] --- - value1 ... +t["[2][6].hello中国world"] +--- +- {'中国': 'test'} +... t["[2][6]['key']"] --- - key1 @@ -681,10 +685,18 @@ t["[2][6]['value']"] --- - value1 ... +t["[2][6]['hello中国world']"] +--- +- {'中国': 'test'} +... t["[3].k3[2].c"] --- - 3 ... +t["[2][6]['hello中国world'].中国"] +--- +- test +... t["[4]"] --- - '123456' diff --git a/test/engine/tuple.test.lua b/test/engine/tuple.test.lua index ba3482d..476be90 100644 --- a/test/engine/tuple.test.lua +++ b/test/engine/tuple.test.lua @@ -207,7 +207,7 @@ format[3] = {name = 'field3', type = 'map'} format[4] = {name = 'field4', type = 'string'} s = box.schema.space.create('test', {format = format}) pk = s:create_index('pk') -field2 = {1, 2, 3, "4", {5,6,7}, {key="key1", value="value1"}} +field2 = {1, 2, 3, "4", {5,6,7}, {key="key1", value="value1", hello中国world = {中国 = 'test'}}} field3 = {[10] = 100, k1 = 100, k2 = {1,2,3}, k3 = { {a=1, b=2}, {c=3, d=4} }, [-1] = 200} t = s:replace{1, field2, field3, "123456"} t[1] @@ -223,9 +223,12 @@ t["[2][5][2]"] t["[2][5][3]"] t["[2][6].key"] t["[2][6].value"] +t["[2][6].hello中国world"] t["[2][6]['key']"] t["[2][6]['value']"] +t["[2][6]['hello中国world']"] t["[3].k3[2].c"] +t["[2][6]['hello中国world'].中国"] t["[4]"] t.field1 t.field2[5] -- 2.7.4 On 29.03.2018 17:22, Kirill Shcherbatov wrote: > --- > src/box/lua/tuple.c | 1 - > src/lib/json/path.c | 19 +++++++++++++++++-- > test/engine/tuple.result | 20 ++++++++++++++++++-- > test/engine/tuple.test.lua | 6 +++++- > 4 files changed, 40 insertions(+), 6 deletions(-) > > diff --git a/src/box/lua/tuple.c b/src/box/lua/tuple.c > index 99b9ff2..c3a435b 100644 > --- a/src/box/lua/tuple.c > +++ b/src/box/lua/tuple.c > @@ -413,7 +413,6 @@ lbox_tuple_transform(struct lua_State *L) > static inline int > tuple_field_go_to_index(const char **field, uint64_t index) > { > - assert(index >= 0); > enum mp_type type = mp_typeof(**field); > if (type == MP_ARRAY) { > if (index == 0) > diff --git a/src/lib/json/path.c b/src/lib/json/path.c > index 4a6174e..3e1bb80 100644 > --- a/src/lib/json/path.c > +++ b/src/lib/json/path.c > @@ -31,6 +31,8 @@ > > #include "path.h" > #include > +#include > +#include > #include "trivia/util.h" > > /** Same as strtoull(), but with limited length. */ > @@ -44,6 +46,19 @@ strntoull(const char *src, int len) { > return value; > } > > +static inline int > +ismbaswcalpha(const char *str, size_t str_len_max) > +{ > + assert(str_len_max < 1024); > + wchar_t buff[1024]; > + mbstate_t ps; > + memset(&ps, 0, sizeof(ps)); > + str_len_max = mbrlen(str, str_len_max, &ps); > + memset(&ps, 0, sizeof(ps)); > + mbsrtowcs(buff, &str, str_len_max, &ps); > + return iswalpha((wint_t)buff[0]); > +} > + > /** > * Parse string identifier in quotes. Parser either stops right > * after the closing quote, or returns an error position. > @@ -126,10 +141,10 @@ json_parse_identifier(struct json_path_parser *parser, > const char *str = pos; > char c = *pos; > /* First symbol can not be digit. */ > - if (!isalpha(c) && c != '_') > + if (!ismbaswcalpha(pos, end - pos) && c != '_') > return pos - parser->src + 1; > int len = 1; > - for (c = *++pos; pos < end && (isalpha(c) || c == '_' || isdigit(c)); > + for (c = *++pos; pos < end && (ismbaswcalpha(pos, end - pos) || c == '_' || isdigit(c)); > c = *++pos) > ++len; > assert(len > 0); > diff --git a/test/engine/tuple.result b/test/engine/tuple.result > index 2d7367a..d6eb4fa 100644 > --- a/test/engine/tuple.result > +++ b/test/engine/tuple.result > @@ -611,7 +611,7 @@ s = box.schema.space.create('test', {format = format}) > pk = s:create_index('pk') > --- > ... > -field2 = {1, 2, 3, "4", {5,6,7}, {key="key1", value="value1"}} > +field2 = {1, 2, 3, "4", {5,6,7}, {key="key1", value="value1", hello中国world = {中国 = 'test'}}} > --- > ... > field3 = {[10] = 100, k1 = 100, k2 = {1,2,3}, k3 = { {a=1, b=2}, {c=3, d=4} }, [-1] = 200} > @@ -626,7 +626,7 @@ t[1] > ... > t[2] > --- > -- [1, 2, 3, '4', [5, 6, 7], {'key': 'key1', 'value': 'value1'}] > +- [1, 2, 3, '4', [5, 6, 7], {'hello中国world': {'中国': 'test'}, 'key': 'key1', 'value': 'value1'}] > ... > t[3] > --- > @@ -665,6 +665,10 @@ t["[2][5][3]"] > --- > - 7 > ... > +t["[2][5][3]['hello中国world'].中"] > +--- > +- null > +... > t["[2][6].key"] > --- > - key1 > @@ -673,6 +677,10 @@ t["[2][6].value"] > --- > - value1 > ... > +t["[2][6].hello中国world"] > +--- > +- {'中国': 'test'} > +... > t["[2][6]['key']"] > --- > - key1 > @@ -681,10 +689,18 @@ t["[2][6]['value']"] > --- > - value1 > ... > +t["[2][6]['hello中国world']"] > +--- > +- {'中国': 'test'} > +... > t["[3].k3[2].c"] > --- > - 3 > ... > +t["[2][6]['hello中国world'].中国"] > +--- > +- test > +... > t["[4]"] > --- > - '123456' > diff --git a/test/engine/tuple.test.lua b/test/engine/tuple.test.lua > index ba3482d..5a3bcfa 100644 > --- a/test/engine/tuple.test.lua > +++ b/test/engine/tuple.test.lua > @@ -207,7 +207,7 @@ format[3] = {name = 'field3', type = 'map'} > format[4] = {name = 'field4', type = 'string'} > s = box.schema.space.create('test', {format = format}) > pk = s:create_index('pk') > -field2 = {1, 2, 3, "4", {5,6,7}, {key="key1", value="value1"}} > +field2 = {1, 2, 3, "4", {5,6,7}, {key="key1", value="value1", hello中国world = {中国 = 'test'}}} > field3 = {[10] = 100, k1 = 100, k2 = {1,2,3}, k3 = { {a=1, b=2}, {c=3, d=4} }, [-1] = 200} > t = s:replace{1, field2, field3, "123456"} > t[1] > @@ -221,11 +221,15 @@ t["[2][5]"] > t["[2][5][1]"] > t["[2][5][2]"] > t["[2][5][3]"] > +t["[2][5][3]['hello中国world'].中"] > t["[2][6].key"] > t["[2][6].value"] > +t["[2][6].hello中国world"] > t["[2][6]['key']"] > t["[2][6]['value']"] > +t["[2][6]['hello中国world']"] > t["[3].k3[2].c"] > +t["[2][6]['hello中国world'].中国"] > t["[4]"] > t.field1 > t.field2[5] >