[tarantool-patches] [PATCH] Pass max_total_connections parameter to httpclient

[Ilya Konyukhov]

There are some usecases when current http client may leak sockets (due
TIME_WAIT timeout). The reason is that the `max_connection` option of
tarantool's http client (actually is a binding for CURLMOPT_MAXCONNECTS
option in libcurl) may not be enough for some situations.

This option sets up a size of connection cache, which curl maintains
during its lifetime. When this cache is full (i.e. all connections are
waiting for a response from a server), newly created request would
create a new connection to handle it. In that case right after old
connection got response, curl has to close that connection to keep up an
appropriate number of connections in cache. That results in a new socket
in TIME_WAIT state (for 60 seconds by default).

This is actually easy to calculate. Imagine we have http client with
default 8 maximum connections in cache. Also lets say we have 16384
sockets available in the system. We need to make requests to external
system which responses in 100ms in average. So, if we start making
request every 11ms (100/9), every finished request will be followed by
curl closing that socket (switches to TIME_WAIT). Result is about 90
wasted sockets per second and additional overhead for socket creation
for each new request.

Also if we do more than 274 requests per second, we will likely to be
out of available sockets in 1 minute.

The solution is to add another binding for max_total_connections option,
which will let curl hold new requests, before old ones have finished.
It also adds more control over sockets resources.

-

New opt is ignored if curl version is lower 7.30.0
Option CURLMOPT_MAX_TOTAL_CONNECTIONS was added from 7.30.0 version
https://curl.haxx.se/changes.html#7_30_0

-

Fixes #3945

https://github.com/RunsFor/tarantool/tree/master
http://github.com/tarantool/tarantool/issues/3945

---

Changes:
- Added new option (max_total_connections) to http client (curl version >= 7.30.0)
- Increased default connection buffer from 5 to 8

 src/curl.c        | 7 ++++++-
 src/curl.h        | 2 +-
 src/httpc.c       | 4 ++--
 src/httpc.h       | 2 +-
 src/lua/httpc.c   | 3 ++-
 src/lua/httpc.lua | 8 +++++---
 6 files changed, 17 insertions(+), 9 deletions(-)

diff --git a/src/curl.c b/src/curl.c
index b85a4553b..354e9615e 100644
--- a/src/curl.c
+++ b/src/curl.c
@@ -229,7 +229,7 @@ curl_multi_sock_cb(CURL *easy, curl_socket_t fd, int what, void *envp,
 
 
 int
-curl_env_create(struct curl_env *env, long max_conns)
+curl_env_create(struct curl_env *env, long max_conns, long max_total_conns)
 {
 	memset(env, 0, sizeof(*env));
 	mempool_create(&env->sock_pool, &cord()->slabc,
@@ -252,6 +252,11 @@ curl_env_create(struct curl_env *env, long max_conns)
 	curl_multi_setopt(env->multi, CURLMOPT_SOCKETDATA, (void *) env);
 
 	curl_multi_setopt(env->multi, CURLMOPT_MAXCONNECTS, max_conns);
+#if LIBCURL_VERSION_NUM >= 0x071e00
+	curl_multi_setopt(env->multi, CURLMOPT_MAX_TOTAL_CONNECTIONS, max_total_conns);
+#else
+	(void) max_total_conns;
+#endif
 
 	return 0;
 
diff --git a/src/curl.h b/src/curl.h
index cf9163664..ea50afe3e 100644
--- a/src/curl.h
+++ b/src/curl.h
@@ -87,7 +87,7 @@ struct curl_request {
  * @retval -1 on error, check diag
  */
 int
-curl_env_create(struct curl_env *env, long max_conns);
+curl_env_create(struct curl_env *env, long max_conns, long max_total_conns);
 
 /**
  * Destroy HTTP client environment
diff --git a/src/httpc.c b/src/httpc.c
index b673ec3e8..7e35f8fc0 100644
--- a/src/httpc.c
+++ b/src/httpc.c
@@ -82,13 +82,13 @@ curl_easy_header_cb(char *buffer, size_t size, size_t nitems, void *ctx)
 }
 
 int
-httpc_env_create(struct httpc_env *env, int max_conns)
+httpc_env_create(struct httpc_env *env, int max_conns, int max_total_conns)
 {
 	memset(env, 0, sizeof(*env));
 	mempool_create(&env->req_pool, &cord()->slabc,
 			sizeof(struct httpc_request));
 
-	return curl_env_create(&env->curl_env, max_conns);
+	return curl_env_create(&env->curl_env, max_conns, max_total_conns);
 }
 
 void
diff --git a/src/httpc.h b/src/httpc.h
index 821c73955..41e75515a 100644
--- a/src/httpc.h
+++ b/src/httpc.h
@@ -76,7 +76,7 @@ struct httpc_env {
  * @retval -1 on error, check diag
  */
 int
-httpc_env_create(struct httpc_env *ctx, int max_conns);
+httpc_env_create(struct httpc_env *ctx, int max_conns, int max_total_conns);
 
 /**
  * Destroy HTTP client environment
diff --git a/src/lua/httpc.c b/src/lua/httpc.c
index 706b9d90b..51133df12 100644
--- a/src/lua/httpc.c
+++ b/src/lua/httpc.c
@@ -368,7 +368,8 @@ luaT_httpc_new(lua_State *L)
 		return luaL_error(L, "lua_newuserdata failed: httpc_env");
 
 	long max_conns = luaL_checklong(L, 1);
-	if (httpc_env_create(ctx, max_conns) != 0)
+	long max_total_conns = luaL_checklong(L, 2);
+	if (httpc_env_create(ctx, max_conns, max_total_conns) != 0)
 		return luaT_error(L);
 
 	luaL_getmetatable(L, DRIVER_LUA_UDATA_NAME);
diff --git a/src/lua/httpc.lua b/src/lua/httpc.lua
index cd44b6054..8566db88d 100644
--- a/src/lua/httpc.lua
+++ b/src/lua/httpc.lua
@@ -41,7 +41,8 @@ local curl_mt
 --
 --  Parameters:
 --
---  max_connectionss -  Maximum number of entries in the connection cache */
+--  max_connections -  Maximum number of entries in the connection cache */
+--  max_total_connections -  Maximum number of active connections */
 --
 --  Returns:
 --  curl object or raise error()
@@ -51,9 +52,10 @@ local http_new = function(opts)
 
     opts = opts or {}
 
-    opts.max_connections = opts.max_connections or 5
+    opts.max_connections = opts.max_connections or 8
+    opts.max_total_connections = opts.max_total_connections or 8
 
-    local curl = driver.new(opts.max_connections)
+    local curl = driver.new(opts.max_connections, opts.max_total_connections)
     return setmetatable({ curl = curl, }, curl_mt )
 end
 
-- 
2.20.1 (Apple Git-117)