[Tarantool-patches] [PATCH 0/2] Borrow encoding/decoding functions from cURL

[Tarantool-patches] [PATCH 0/2] Borrow encoding/decoding functions from cURL

[olegrok]

From: Oleg Babin <babinoleg@mail.ru>

This patchest introduces four new functions for
http.client module

First patch properly closes #3682 and introduces
url_encode and url_decode functions - simple
wrappers via libcurl functions

However it could be enough for our users that
will use it for query arguments formatting and parsing

The second one introduces more high-level functions -
format_query and parse_query

Issue: https://github.com/tarantool/tarantool/issues/3682
Branch: https://github.com/tarantool/tarantool/tree/olegrok/3682-url-escape 

Oleg Babin (2):
  httpc: introduce url_escape/url_unescape functions
  httpc: introduce format_query and parse_query functions

 src/httpc.c                       | 19 ++++++++
 src/httpc.h                       | 36 +++++++++++++++
 src/lua/httpc.c                   | 45 +++++++++++++++++++
 src/lua/httpc.lua                 | 75 +++++++++++++++++++++++++++++--
 test/app-tap/http_client.test.lua | 58 +++++++++++++++++++++++-
 5 files changed, 229 insertions(+), 4 deletions(-)

-- 
2.23.0

[Tarantool-patches] [PATCH 1/2] httpc: introduce url_escape/url_unescape functions

[olegrok]

From: Oleg Babin <babinoleg@mail.ru>

cURL easy API contains functions curl_easy_escape and
curl_easy_unescape:

  - https://curl.haxx.se/libcurl/c/curl_easy_escape.html
  - https://curl.haxx.se/libcurl/c/curl_easy_unescape.html

This functions can be sometimes useful for our users
especially when they works with http client and form
query arguments for http request

This patch introduces two functions available from
our http client:
```lua
httpc.url_escape
httpc.url_unescape
```

Need for #3682
---
Issue: https://github.com/tarantool/tarantool/issues/3682
My related github PR: https://github.com/tarantool/tarantool/pull/4153
 src/httpc.c                       | 19 +++++++++++++
 src/httpc.h                       | 36 +++++++++++++++++++++++++
 src/lua/httpc.c                   | 45 +++++++++++++++++++++++++++++++
 src/lua/httpc.lua                 | 17 +++++++++---
 test/app-tap/http_client.test.lua | 21 ++++++++++++++-
 5 files changed, 134 insertions(+), 4 deletions(-)

diff --git a/src/httpc.c b/src/httpc.c
index be73e3684..d152275e3 100644
--- a/src/httpc.c
+++ b/src/httpc.c
@@ -467,3 +467,22 @@ httpc_execute(struct httpc_request *req, double timeout)
 
 	return 0;
 }
+
+char *
+httpc_str_escape(struct httpc_env *env, const char *str, int len)
+{
+	return curl_easy_escape(env->curl_env.multi, str, len);
+}
+
+char *
+httpc_str_unescape(struct httpc_env *env, const char *str, int len,
+		int *outlen)
+{
+	return curl_easy_unescape(env->curl_env.multi, str, len, outlen);
+}
+
+void
+httpc_str_free(char *str)
+{
+	curl_free(str);
+}
diff --git a/src/httpc.h b/src/httpc.h
index 94d543a40..22a83f72c 100644
--- a/src/httpc.h
+++ b/src/httpc.h
@@ -413,4 +413,40 @@ httpc_execute(struct httpc_request *req, double timeout);
 
 /** Request }}} */
 
+/** {{{ Encode/Decode */
+
+/**
+ * This function URL encodes the given string
+ * according RFC3986
+ * @param req - reference to request object with filled fields
+ * @param str - input string to be encoded
+ * @param len - length of input string
+ * @return encoded string or NULL
+ */
+char *
+httpc_str_escape(struct httpc_env *env, const char *str, int len);
+
+/**
+ * This function URL decodes the given string
+ * according RFC3986
+ * @param req - reference to request object with filled fields
+ * @param str - input string to be encoded
+ * @param len - length of input string
+ * @param outlen - - length of output string
+ * @return decoded string or NULL
+ */
+
+char *
+httpc_str_unescape(struct httpc_env *env, const char *str, int len, int *outlen);
+
+/**
+ * This function reclaims memory that has been obtained
+ * through a libcurl call
+ * @param str - pointer to string returned from escape/unescape
+ */
+void
+httpc_str_free(char *str);
+
+/** Encode/Decode }}} */
+
 #endif /* TARANTOOL_HTTPC_H_INCLUDED */
diff --git a/src/lua/httpc.c b/src/lua/httpc.c
index c3dd611fa..e0c55f49e 100644
--- a/src/lua/httpc.c
+++ b/src/lua/httpc.c
@@ -425,6 +425,49 @@ luaT_httpc_cleanup(lua_State *L)
 	return 2;
 }
 
+static int
+luaT_httpc_escape(lua_State *L)
+{
+	struct httpc_env *ctx = luaT_httpc_checkenv(L);
+	if (ctx == NULL)
+		return luaL_error(L, "can't get httpc environment");
+
+	size_t str_len;
+	const char *str = luaL_checklstring(L, 2, &str_len);
+	if (str == NULL)
+		return luaL_error(L, "second argument should be a string");
+
+	char *outstr = httpc_str_escape(ctx, str, str_len);
+	if (outstr == NULL)
+		return luaL_error(L, "string escaping error");
+
+	lua_pushstring(L, outstr);
+	httpc_str_free(outstr);
+	return 1;
+}
+
+static int
+luaT_httpc_unescape(lua_State *L)
+{
+	struct httpc_env *ctx = luaT_httpc_checkenv(L);
+	if (ctx == NULL)
+		return luaL_error(L, "can't get httpc environment");
+
+	size_t str_len;
+	const char *str = luaL_checklstring(L, 2, &str_len);
+	if (str == NULL)
+		return luaL_error(L, "second argument should be a string");
+
+	int outstr_len;
+	char *outstr = httpc_str_unescape(ctx, str, str_len, &outstr_len);
+	if (outstr == NULL)
+		return luaL_error(L, "string unescaping error");
+
+	lua_pushlstring(L, outstr, outstr_len);
+	httpc_str_free(outstr);
+	return 1;
+}
+
 /*
  * }}}
  */
@@ -435,6 +478,8 @@ luaT_httpc_cleanup(lua_State *L)
 
 static const struct luaL_Reg Module[] = {
 	{"new", luaT_httpc_new},
+	{"escape", luaT_httpc_escape},
+	{"unescape", luaT_httpc_unescape},
 	{NULL, NULL}
 };
 
diff --git a/src/lua/httpc.lua b/src/lua/httpc.lua
index 6381c6a1a..33cb78f3c 100644
--- a/src/lua/httpc.lua
+++ b/src/lua/httpc.lua
@@ -29,8 +29,6 @@
 --  SUCH DAMAGE.
 --
 
-local fiber = require('fiber')
-
 local driver = package.loaded.http.client
 package.loaded.http = nil
 
@@ -442,7 +440,20 @@ curl_mt = {
 -- Export
 --
 local http_default = http_new()
-local this_module = { new = http_new, }
+
+local function url_escape(str)
+    return driver.escape(http_default.curl, str)
+end
+
+local function url_unescape(str)
+    return driver.unescape(http_default.curl, str)
+end
+
+local this_module = {
+    new = http_new,
+    url_escape = url_escape,
+    url_unescape = url_unescape
+}
 
 local function http_default_wrap(fname)
     return function(...) return http_default[fname](http_default, ...) end
diff --git a/test/app-tap/http_client.test.lua b/test/app-tap/http_client.test.lua
index b85b605cf..3676f56b4 100755
--- a/test/app-tap/http_client.test.lua
+++ b/test/app-tap/http_client.test.lua
@@ -611,7 +611,7 @@ function run_tests(test, sock_family, sock_addr)
     stop_server(test, server)
 end
 
-test:plan(2)
+test:plan(3)
 
 test:test("http over AF_INET", function(test)
     local s = socketlib('AF_INET', 'SOCK_STREAM', 0)
@@ -634,4 +634,23 @@ test:test("http over AF_UNIX", function(test)
     os.remove(path)
 end)
 
+test:test("url_escape/url_unescape", function(test)
+    test:plan(11)
+    test:is('hello', client.url_escape('hello'), 'correct encoding of eng')
+    test:is('%D0%BF%D1%80%D0%B8%D0%B2%D0%B5%D1%82', client.url_escape('привет'), 'correct encoding of rus')
+
+    test:is('hello', client.url_unescape('hello'), 'correct decoding of eng')
+    test:is('привет', client.url_unescape('%D0%BF%D1%80%D0%B8%D0%B2%D0%B5%D1%82'), 'correct decoding of rus')
+
+    test:is('hello', client.url_escape(client.url_unescape('hello')), 'decoding and encoding of "hello"')
+    test:is('привет', client.url_unescape(client.url_escape('привет')), 'encoding and decoding of "привет"')
+
+    test:is('%00', client.url_escape(client.url_unescape('%00')), 'decoding and encoding of %00')
+    test:is('\0', client.url_unescape(client.url_escape('\0')), 'encoding and decoding of \\0')
+
+    test:is('%20', client.url_escape(' '), 'space is escaped as %20')
+    test:is('%24%26%2B%2C%3A%3B%3D%3F%40', client.url_escape('$&+,:;=?@'), 'special characters escaping')
+    test:is('-._~', client.url_escape('-._~'), '-._~ are not escaped according RFC 3986')
+end)
+
 os.exit(test:check() == true and 0 or -1)
-- 
2.23.0

[Tarantool-patches] [PATCH 2/2] httpc: introduce format_query and parse_query functions

[olegrok]

From: Oleg Babin <babinoleg@mail.ru>

Previous patch introduced url_escape and url_unescape.
However it's quite poorly to give our users too
low-level functions. The main use case is to form and parse
http query parameters

This patch introduces format_query and parse_query functions

Closes #3682

@TarantoolBot document
Title: New http.client functions

Four new functions are available in http.client module:

  - url_escape - simple wrapper over curl_easy_escape
      (https://curl.haxx.se/libcurl/c/curl_easy_escape.html)

  - url_unescape - simple wrapper over curl_easy_unescape
      (https://curl.haxx.se/libcurl/c/curl_easy_unescape.html)

Examples:
```lua
-- According RFC3986

tarantool> httpc.url_escape('hello')
---
- hello
...

tarantool> httpc.url_escape('Привет')
---
- '%D0%9F%D1%80%D0%B8%D0%B2%D0%B5%D1%82'
...

tarantool> httpc.url_escape('$&+,:;=?@-._~')
---
- '%24%26%2B%2C%3A%3B%3D%3F%40-._~'
...
```

  - format_query - format query arguments from key-value string pairs
for HTTP request

  - parse_query - parse query string into table

Example:
```lua
-- The query string is composed of a series of field-value pairs
-- Within each pair, the field name and value are
--   separated by an equals sign, "="
-- The series of pairs is separated by the ampersand, "&"

tarantool> httpc.format_query({['hello'] = 'world', ['привет'] = 'мир'})
---
- '%D0%BF%D1%80%D0%B8%D0%B2%D0%B5%D1%82=%D0%BC%D0%B8%D1%80&hello=world'
...

tarantool> httpc.parse_query('%D0%BF%D1%80%D0%B8%D0%B2%D0%B5%D1%82=' ..
'%D0%BC%D0%B8%D1%80&hello=world')
---
- привет: мир
  hello: world
...

```
---
Issue: https://github.com/tarantool/tarantool/issues/3682
 src/lua/httpc.lua                 | 60 ++++++++++++++++++++++++++++++-
 test/app-tap/http_client.test.lua | 39 +++++++++++++++++++-
 2 files changed, 97 insertions(+), 2 deletions(-)

diff --git a/src/lua/httpc.lua b/src/lua/httpc.lua
index 33cb78f3c..7d40b3bcc 100644
--- a/src/lua/httpc.lua
+++ b/src/lua/httpc.lua
@@ -449,10 +449,68 @@ local function url_unescape(str)
     return driver.unescape(http_default.curl, str)
 end
 
+--
+--  This function format query params from table with arguments
+--
+--  Parameters:
+--
+--  args  -  table with key-value pairs that will be
+--               encoded as string "key1=value1&key2=value2"
+--               each element of pair is encoded with url_escape
+--
+local function format_query(args)
+    if type(args) ~= "table" then
+        error("httpc.format_query expected table with args")
+    end
+    local encoded = {}
+    for k, v in pairs(args) do
+        k = tostring(k)
+        v = tostring(v)
+        k = url_escape(k)
+        v = url_escape(v)
+        table.insert(encoded, k .. '=' .. v)
+    end
+
+    return table.concat(encoded, '&')
+end
+
+--
+--  This function parse query params from string to table
+--
+--  Parameters:
+--
+--  str  -  string in format "key1=value1&key2=value2"
+--              that will be parsed into {key1 = value1, key2 = value2}
+--              each element of pair is decoded with url_unescape
+--
+local function parse_query(str)
+    if type(str) ~= "string" then
+        error("httpc.parse_query expected string with args")
+    end
+    local args = string.split(str, '&')
+
+    local decoded = {}
+    for _, arg in ipairs(args) do
+        local k_v = string.split(arg, '=', 1)
+        if #k_v ~= 2 then
+            error("httpc.parse_query expected arguments in format 'key=value'"
+                .. " separated by '&', got: " .. tostring(arg))
+        end
+        local key = url_unescape(k_v[1])
+        local value = url_unescape(k_v[2])
+
+        decoded[key] = value
+    end
+
+    return decoded
+end
+
 local this_module = {
     new = http_new,
     url_escape = url_escape,
-    url_unescape = url_unescape
+    url_unescape = url_unescape,
+    format_query = format_query,
+    parse_query = parse_query,
 }
 
 local function http_default_wrap(fname)
diff --git a/test/app-tap/http_client.test.lua b/test/app-tap/http_client.test.lua
index 3676f56b4..4b5e6bbc5 100755
--- a/test/app-tap/http_client.test.lua
+++ b/test/app-tap/http_client.test.lua
@@ -611,7 +611,7 @@ function run_tests(test, sock_family, sock_addr)
     stop_server(test, server)
 end
 
-test:plan(3)
+test:plan(5)
 
 test:test("http over AF_INET", function(test)
     local s = socketlib('AF_INET', 'SOCK_STREAM', 0)
@@ -653,4 +653,41 @@ test:test("url_escape/url_unescape", function(test)
     test:is('-._~', client.url_escape('-._~'), '-._~ are not escaped according RFC 3986')
 end)
 
+test:test("format_query", function(test)
+    test:plan(6)
+    test:is(client.format_query({['hello'] = 'world'}), 'hello=world', 'correct formation for one eng arg')
+    test:is(client.format_query({['привет'] = 'мир'}),
+        '%D0%BF%D1%80%D0%B8%D0%B2%D0%B5%D1%82=%D0%BC%D0%B8%D1%80',  'correct formation for one rus arg')
+    test:is(client.format_query({['key=key'] = 'value;value'}), 'key%3Dkey=value%3Bvalue',
+        'correct formation for args with special characters')
+    test:is(client.format_query({['key1'] = 'value1', ['ключ2'] = 'значение'}),
+        'key1=value1&%D0%BA%D0%BB%D1%8E%D1%872=%D0%B7%D0%BD%D0%B0%D1%87%D0%B5%D0%BD%D0%B8%D0%B5',
+        'correct formation for several args')
+
+    local ok, err = pcall(client.format_query, 'str')
+    test:is(ok, false, 'only table could be passed as argument')
+    test:ok(err:endswith('httpc.format_query expected table with args'), 'correct error message')
+end)
+
+test:test("parse_query", function(test)
+    test:plan(8)
+        test:is_deeply(client.parse_query('hello=world'), {['hello'] = 'world'}, 'correct parsing for one eng arg')
+        test:is_deeply(client.parse_query('%D0%BF%D1%80%D0%B8%D0%B2%D0%B5%D1%82=%D0%BC%D0%B8%D1%80'),
+        {['привет'] = 'мир'}, 'correct parsing for one rus arg')
+        test:is_deeply(client.parse_query('key%3Dkey=value%3Bvalue'), {['key=key'] = 'value;value'},
+            'correct formation for args with special characters')
+        test:is_deeply(client.parse_query(
+            'key1=value1&%D0%BA%D0%BB%D1%8E%D1%872=%D0%B7%D0%BD%D0%B0%D1%87%D0%B5%D0%BD%D0%B8%D0%B5'),
+            {['key1'] = 'value1', ['ключ2'] = 'значение'}, 'correct parsing for several args')
+
+        local ok, err = pcall(client.parse_query, {})
+        test:is(ok, false, 'only string could be passed as argument')
+        test:ok(err:endswith('httpc.parse_query expected string with args'), 'correct error message')
+
+        local ok, err = pcall(client.parse_query, '1=2&3&4')
+        test:is(ok, false, 'unexpected input format error')
+        test:ok(err:endswith('httpc.parse_query expected arguments in format \'key=value\'' ..
+            ' separated by \'&\', got: 3'), 'correct error message')
+end)
+
 os.exit(test:check() == true and 0 or -1)
-- 
2.23.0