[tarantool-patches] [PATCH v2 1/2] buffer: port static allocator to Lua

[Vladislav Shpilevoy <v.shpilevoy@tarantool.org>]

Static allocator gives memory blocks from cyclic BSS memory
block of 3 pages 4096 bytes each. It is much faster than
malloc, when a temporary buffer is needed.

This commit exposes the allocator to Lua, which suffers from
lack of ability to pass values by pointers into FFI functions,
nor has a stack to allocate small buffers like 'char[256]'.
Also these allocations complicate and slow down GC job.

Static allocator solves most of these problems provindg a way to
swiftly allocate temporary memory blocks.

A simple micro benchmark showed, that ffi.new() vs
buffer.static_alloc() is ~100 times slower, even on small
allocations of 1Kb, and ~1.5 times slower on tiny allocations
of 10 bytes. The results do not account GC. It is remarkable,
buffer.static_alloc() speed does not depend on size, while
ffi.new() strongly depends.
---
 extra/exports            |  2 ++
 src/CMakeLists.txt       |  1 +
 src/box/lua/schema.lua   |  3 ++-
 src/lua/buffer.c         | 42 ++++++++++++++++++++++++++++++++++++++++
 src/lua/buffer.lua       | 23 ++++++++++++++++++++++
 src/lua/digest.lua       |  7 ++++---
 src/lua/fio.lua          |  3 +--
 src/lua/init.c           |  2 +-
 src/lua/socket.lua       | 11 ++++++-----
 src/lua/string.lua       |  6 ++++--
 src/lua/uri.lua          |  7 ++++---
 src/lua/uuid.lua         |  9 +++++----
 test/app/buffer.result   | 26 +++++++++++++++++++++++++
 test/app/buffer.test.lua | 12 ++++++++++++
 14 files changed, 133 insertions(+), 21 deletions(-)
 create mode 100644 src/lua/buffer.c
 create mode 100644 test/app/buffer.result
 create mode 100644 test/app/buffer.test.lua

diff --git a/extra/exports b/extra/exports
index 4f41a17b3..5375a01e4 100644
--- a/extra/exports
+++ b/extra/exports
@@ -85,6 +85,8 @@ tnt_EVP_MD_CTX_free
 tnt_HMAC_CTX_new
 tnt_HMAC_CTX_free
 
+lua_static_aligned_alloc
+
 # Module API
 
 _say
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index a6a18142b..492b8712e 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -116,6 +116,7 @@ set (server_sources
      lua/utf8.c
      lua/info.c
      lua/string.c
+     lua/buffer.c
      ${lua_sources}
      ${PROJECT_SOURCE_DIR}/third_party/lua-yaml/lyaml.cc
      ${PROJECT_SOURCE_DIR}/third_party/lua-yaml/b64.c
diff --git a/src/box/lua/schema.lua b/src/box/lua/schema.lua
index f31cf7f2c..fa6c7c9a4 100644
--- a/src/box/lua/schema.lua
+++ b/src/box/lua/schema.lua
@@ -7,6 +7,7 @@ local fun = require('fun')
 local log = require('log')
 local fio = require('fio')
 local json = require('json')
+local static_alloc = require('buffer').static_alloc
 local session = box.session
 local internal = require('box.internal')
 local function setmap(table)
@@ -2103,7 +2104,7 @@ box.schema.user = {}
 
 box.schema.user.password = function(password)
     local BUF_SIZE = 128
-    local buf = ffi.new("char[?]", BUF_SIZE)
+    local buf = static_alloc('char', BUF_SIZE)
     builtin.password_prepare(password, #password, buf, BUF_SIZE)
     return ffi.string(buf)
 end
diff --git a/src/lua/buffer.c b/src/lua/buffer.c
new file mode 100644
index 000000000..5fd349261
--- /dev/null
+++ b/src/lua/buffer.c
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2010-2019, Tarantool AUTHORS, please see AUTHORS file.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above
+ *    copyright notice, this list of conditions and the
+ *    following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above
+ *    copyright notice, this list of conditions and the following
+ *    disclaimer in the documentation and/or other materials
+ *    provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY <COPYRIGHT HOLDER> ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+ * <COPYRIGHT HOLDER> OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+ * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+#include "small/static.h"
+
+/**
+ * Static inline functions like in static.h can't be exported.
+ * Here they are given a symbol to export.
+ */
+
+void *
+lua_static_aligned_alloc(size_t size, size_t alignment)
+{
+	return static_aligned_alloc(size, alignment);
+}
diff --git a/src/lua/buffer.lua b/src/lua/buffer.lua
index a72d8d1f9..5e9209387 100644
--- a/src/lua/buffer.lua
+++ b/src/lua/buffer.lua
@@ -33,6 +33,9 @@ ibuf_reinit(struct ibuf *ibuf);
 
 void *
 ibuf_reserve_slow(struct ibuf *ibuf, size_t size);
+
+void *
+lua_static_aligned_alloc(size_t size, size_t alignment);
 ]]
 
 local builtin = ffi.C
@@ -174,8 +177,28 @@ local function ibuf_new(arg, arg2)
     errorf('Usage: ibuf([size])')
 end
 
+--
+-- Allocate a chunk of static BSS memory, or use ordinal ffi.new,
+-- when too big size.
+-- @param type C type - a struct, a basic type, etc. Should be a
+--        string: 'int', 'char *', 'struct tuple', etc.
+-- @param size Optional argument, number of elements of @a type to
+--        allocate. 1 by default.
+-- @return Cdata pointer to @a type.
+--
+local function static_alloc(type, size)
+    size = size or 1
+    local bsize = size * ffi.sizeof(type)
+    local ptr = builtin.lua_static_aligned_alloc(bsize, ffi.alignof(type))
+    if ptr ~= nil then
+        return ffi.cast(type..' *', ptr)
+    end
+    return ffi.new(type..'[?]', size)
+end
+
 return {
     ibuf = ibuf_new;
     IBUF_SHARED = ffi.C.tarantool_lua_ibuf;
     READAHEAD = READAHEAD;
+    static_alloc = static_alloc,
 }
diff --git a/src/lua/digest.lua b/src/lua/digest.lua
index 8f199c0af..6ed91cfa2 100644
--- a/src/lua/digest.lua
+++ b/src/lua/digest.lua
@@ -3,6 +3,7 @@
 local ffi = require('ffi')
 local crypto = require('crypto')
 local bit = require('bit')
+local static_alloc = require('buffer').static_alloc
 
 ffi.cdef[[
     /* internal implementation */
@@ -179,7 +180,7 @@ local m = {
         end
         local blen = #bin
         local slen = ffi.C.base64_bufsize(blen, mask)
-        local str  = ffi.new('char[?]', slen)
+        local str  = static_alloc('char', slen)
         local len = ffi.C.base64_encode(bin, blen, str, slen, mask)
         return ffi.string(str, len)
     end,
@@ -190,7 +191,7 @@ local m = {
         end
         local slen = #str
         local blen = math.ceil(slen * 3 / 4)
-        local bin  = ffi.new('char[?]', blen)
+        local bin  = static_alloc('char', blen)
         local len = ffi.C.base64_decode(str, slen, bin, blen)
         return ffi.string(bin, len)
     end,
@@ -228,7 +229,7 @@ local m = {
         if n == nil then
             error('Usage: digest.urandom(len)')
         end
-        local buf = ffi.new('char[?]', n)
+        local buf = static_alloc('char', n)
         ffi.C.random_bytes(buf, n)
         return ffi.string(buf, n)
     end,
diff --git a/src/lua/fio.lua b/src/lua/fio.lua
index 38664a556..fce79c277 100644
--- a/src/lua/fio.lua
+++ b/src/lua/fio.lua
@@ -275,8 +275,7 @@ fio.dirname = function(path)
     if type(path) ~= 'string' then
         error("Usage: fio.dirname(path)")
     end
-    path = ffi.new('char[?]', #path + 1, path)
-    return ffi.string(ffi.C.dirname(path))
+    return ffi.string(ffi.C.dirname(ffi.cast('char *', path)))
 end
 
 fio.umask = function(umask)
diff --git a/src/lua/init.c b/src/lua/init.c
index be164bdfb..303369841 100644
--- a/src/lua/init.c
+++ b/src/lua/init.c
@@ -126,9 +126,9 @@ static const char *lua_modules[] = {
 	"errno", errno_lua,
 	"fiber", fiber_lua,
 	"env", env_lua,
+	"buffer", buffer_lua,
 	"string", string_lua,
 	"table", table_lua,
-	"buffer", buffer_lua,
 	"msgpackffi", msgpackffi_lua,
 	"crypto", crypto_lua,
 	"digest", digest_lua,
diff --git a/src/lua/socket.lua b/src/lua/socket.lua
index b2700e0c0..cbf8c0f29 100644
--- a/src/lua/socket.lua
+++ b/src/lua/socket.lua
@@ -10,6 +10,7 @@ local fiber = require('fiber')
 local fio = require('fio')
 local log = require('log')
 local buffer = require('buffer')
+local static_alloc = buffer.static_alloc
 
 local format = string.format
 
@@ -581,9 +582,10 @@ local function socket_linger(self, active, timeout)
         iactive = 0
     end
 
-    local value = ffi.new("linger_t[1]",
-        { { active = iactive, timeout = timeout } })
-    local len = 2 * ffi.sizeof('int')
+    local value = static_alloc('linger_t')
+    value[0].active = iactive
+    value[0].timeout = timeout
+    local len = ffi.sizeof('linger_t')
     local res = ffi.C.setsockopt(fd, level, info.iname, value, len)
     if res < 0 then
         self._errno = boxerrno()
@@ -836,8 +838,7 @@ local function socket_recv(self, size, flags)
     end
 
     self._errno = nil
-    local buf = ffi.new("char[?]", size)
-
+    local buf = static_alloc('char', size)
     local res = ffi.C.recv(fd, buf, size, iflags)
 
     if res == -1 then
diff --git a/src/lua/string.lua b/src/lua/string.lua
index 8216ace6a..bb4adfc78 100644
--- a/src/lua/string.lua
+++ b/src/lua/string.lua
@@ -1,4 +1,6 @@
 local ffi = require('ffi')
+local buffer = require('buffer')
+local static_alloc = buffer.static_alloc
 
 ffi.cdef[[
     const char *
@@ -290,7 +292,7 @@ local function string_hex(inp)
         error(err_string_arg:format(1, 'string.hex', 'string', type(inp)), 2)
     end
     local len = inp:len() * 2
-    local res = ffi.new('char[?]', len + 1)
+    local res = static_alloc('char', len + 1)
 
     local uinp = ffi.cast('const unsigned char *', inp)
     for i = 0, inp:len() - 1 do
@@ -333,7 +335,7 @@ local function string_fromhex(inp)
     end
     local len = inp:len() / 2
     local casted_inp = ffi.cast('const char *', inp)
-    local res = ffi.new('char[?]', len)
+    local res = static_alloc('char', len)
     for i = 0, len - 1 do
         local first = hexadecimals_mapping[casted_inp[i * 2]]
         local second = hexadecimals_mapping[casted_inp[i * 2 + 1]]
diff --git a/src/lua/uri.lua b/src/lua/uri.lua
index d2946cd2d..5967c8bf2 100644
--- a/src/lua/uri.lua
+++ b/src/lua/uri.lua
@@ -1,6 +1,7 @@
 -- uri.lua (internal file)
 
 local ffi = require('ffi')
+local static_alloc = require('buffer').static_alloc
 
 ffi.cdef[[
 struct uri {
@@ -32,12 +33,11 @@ uri_format(char *str, size_t len, struct uri *uri, bool write_password);
 
 local builtin = ffi.C;
 
-local uribuf = ffi.new('struct uri')
-
 local function parse(str)
     if str == nil then
         error("Usage: uri.parse(string)")
     end
+    local uribuf = static_alloc('struct uri')
     if builtin.uri_parse(uribuf, str) ~= 0 then
         return nil
     end
@@ -59,6 +59,7 @@ local function parse(str)
 end
 
 local function format(uri, write_password)
+    local uribuf = static_alloc('struct uri')
     uribuf.scheme = uri.scheme
     uribuf.scheme_len = string.len(uri.scheme or '')
     uribuf.login = uri.login
@@ -75,7 +76,7 @@ local function format(uri, write_password)
     uribuf.query_len = string.len(uri.query or '')
     uribuf.fragment = uri.fragment
     uribuf.fragment_len = string.len(uri.fragment or '')
-    local str = ffi.new('char[1024]')
+    local str = static_alloc('char', 1024)
     builtin.uri_format(str, 1024, uribuf, write_password and 1 or 0)
     return ffi.string(str)
 end
diff --git a/src/lua/uuid.lua b/src/lua/uuid.lua
index 956ad6e36..f8418cf4d 100644
--- a/src/lua/uuid.lua
+++ b/src/lua/uuid.lua
@@ -1,6 +1,7 @@
 -- uuid.lua (internal file)
 
 local ffi = require("ffi")
+local static_alloc = require('buffer').static_alloc
 local builtin = ffi.C
 
 ffi.cdef[[
@@ -33,7 +34,6 @@ extern const struct tt_uuid uuid_nil;
 local uuid_t = ffi.typeof('struct tt_uuid')
 local UUID_STR_LEN = 36
 local UUID_LEN = ffi.sizeof(uuid_t)
-local uuidbuf = ffi.new(uuid_t)
 
 local uuid_tostring = function(uu)
     if not ffi.istype(uuid_t, uu) then
@@ -69,9 +69,8 @@ local uuid_tobin = function(uu, byteorder)
         return error('Usage: uuid:bin([byteorder])')
     end
     if need_bswap(byteorder) then
-        if uu ~= uuidbuf then
-            ffi.copy(uuidbuf, uu, UUID_LEN)
-        end
+        local uuidbuf = static_alloc('struct tt_uuid')
+        ffi.copy(uuidbuf, uu, UUID_LEN)
         builtin.tt_uuid_bswap(uuidbuf)
         return ffi.string(ffi.cast('char *', uuidbuf), UUID_LEN)
     end
@@ -114,10 +113,12 @@ local uuid_new = function()
 end
 
 local uuid_new_bin = function(byteorder)
+    local uuidbuf = static_alloc('struct tt_uuid')
     builtin.tt_uuid_create(uuidbuf)
     return uuid_tobin(uuidbuf, byteorder)
 end
 local uuid_new_str = function()
+    local uuidbuf = static_alloc('struct tt_uuid')
     builtin.tt_uuid_create(uuidbuf)
     return uuid_tostring(uuidbuf)
 end
diff --git a/test/app/buffer.result b/test/app/buffer.result
new file mode 100644
index 000000000..e0aad9bc7
--- /dev/null
+++ b/test/app/buffer.result
@@ -0,0 +1,26 @@
+test_run = require('test_run').new()
+---
+...
+buffer = require('buffer')
+---
+...
+ffi = require('ffi')
+---
+...
+-- Alignment.
+_ = buffer.static_alloc('char') -- This makes buffer pos unaligned.
+---
+...
+p = buffer.static_alloc('int')
+---
+...
+ffi.cast('int', p) % ffi.alignof('int') == 0 -- But next alloc is aligned.
+---
+- true
+...
+-- Able to allocate bigger than static buffer - such allocations
+-- are on the heap.
+type(buffer.static_alloc('char', 13000))
+---
+- cdata
+...
diff --git a/test/app/buffer.test.lua b/test/app/buffer.test.lua
new file mode 100644
index 000000000..ba7299f33
--- /dev/null
+++ b/test/app/buffer.test.lua
@@ -0,0 +1,12 @@
+test_run = require('test_run').new()
+buffer = require('buffer')
+ffi = require('ffi')
+
+-- Alignment.
+_ = buffer.static_alloc('char') -- This makes buffer pos unaligned.
+p = buffer.static_alloc('int')
+ffi.cast('int', p) % ffi.alignof('int') == 0 -- But next alloc is aligned.
+
+-- Able to allocate bigger than static buffer - such allocations
+-- are on the heap.
+type(buffer.static_alloc('char', 13000))
-- 
2.20.1 (Apple Git-117)