* [Tarantool-patches] [PATCH] Add new 'allocator' option in box.cfg (cpp version)
@ 2020-12-18 14:11 mechanik20051988
0 siblings, 0 replies; only message in thread
From: mechanik20051988 @ 2020-12-18 14:11 UTC (permalink / raw)
To: tarantool-patches, Vladislav Shpilevoy; +Cc: mechanik20051988
From: mechanik20051988 <mechanik20.05.1988@gmail.com>
Branch: https://github.com/tarantool/tarantool/tree/mechanik20051988/gh-5419-choose-allocator-for-memtx-cpp
Issue: https://github.com/tarantool/tarantool/issues/5419
Slab allocator, which is used for tuples allocation, has a certan
disadvantage - it tends to unresolvable fragmentation on certain
workloads (size migration). New option allows to select
the appropriate allocator if necessary.
@TarantoolBot document
Title: Add new 'allocator' option
Add new 'allocator' option which allows to select
the appropriate allocator for memtx tuples if necessary.
Use box.cfg{allocator="small"} or no option to use default
small allocator, use box.cfg{allocator="system"} to use
libc malloc.
Closes #5419
---
CMakeLists.txt | 11 +
perf/allocator_perf.test.lua | 34 +++
src/box/CMakeLists.txt | 4 +-
src/box/allocator.h | 162 +++++++++++++++
src/box/box.cc | 1 +
src/box/field_map.h | 8 +
src/box/lua/load_cfg.lua | 2 +
src/box/lua/{slab.c => slab.cc} | 39 +++-
src/box/{memtx_engine.c => memtx_engine.cc} | 193 +++++++++++++----
src/box/memtx_engine.h | 41 ++--
src/box/memtx_hash.c | 4 +-
src/box/memtx_space.c | 12 +-
src/box/memtx_tree.cc | 4 +-
src/box/system_allocator.h | 218 ++++++++++++++++++++
src/trivia/config.h.cmake | 3 +
test/app-tap/init_script.result | 1 +
test/box/admin.result | 4 +-
test/box/cfg.result | 8 +-
test/box/choose_memtx_allocator.lua | 9 +
test/box/choose_memtx_allocator.result | 135 ++++++++++++
test/box/choose_memtx_allocator.test.lua | 43 ++++
21 files changed, 856 insertions(+), 80 deletions(-)
create mode 100755 perf/allocator_perf.test.lua
create mode 100644 src/box/allocator.h
rename src/box/lua/{slab.c => slab.cc} (88%)
rename src/box/{memtx_engine.c => memtx_engine.cc} (87%)
create mode 100644 src/box/system_allocator.h
create mode 100644 test/box/choose_memtx_allocator.lua
create mode 100644 test/box/choose_memtx_allocator.result
create mode 100644 test/box/choose_memtx_allocator.test.lua
diff --git a/CMakeLists.txt b/CMakeLists.txt
index fa6818f8e..290cd535a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -92,6 +92,17 @@ check_symbol_exists(posix_fadvise fcntl.h HAVE_POSIX_FADVISE)
check_symbol_exists(fallocate fcntl.h HAVE_FALLOCATE)
check_symbol_exists(mremap sys/mman.h HAVE_MREMAP)
+check_function_exists(malloc_usable_size HAVE_MALLOC_USABLE_SIZE)
+check_symbol_exists(malloc_size malloc/malloc.h HAVE_MALLOC_SIZE_DARWIN)
+
+if (HAVE_MALLOC_USABLE_SIZE)
+ if (TARGET_OS_LINUX)
+ set(HAVE_MALLOC_USABLE_SIZE_LINUX 1)
+ else ()
+ set(HAVE_MALLOC_USABLE_SIZE_BSD 1)
+ endif ()
+endif ()
+
check_function_exists(sync_file_range HAVE_SYNC_FILE_RANGE)
check_function_exists(memmem HAVE_MEMMEM)
check_function_exists(memrchr HAVE_MEMRCHR)
diff --git a/perf/allocator_perf.test.lua b/perf/allocator_perf.test.lua
new file mode 100755
index 000000000..be270379b
--- /dev/null
+++ b/perf/allocator_perf.test.lua
@@ -0,0 +1,34 @@
+#!/usr/bin/env ../src/tarantool
+os.execute('rm -rf *.snap *.xlog *.vylog ./512 ./513 ./514 ./515 ./516 ./517 ./518 ./519 ./520 ./521')
+local clock = require('clock')
+box.cfg{listen = 3301, wal_mode='none', allocator=arg[1]}
+local space = box.schema.space.create('test')
+space:format({ {name = 'id', type = 'unsigned'}, {name = 'year', type = 'unsigned'} })
+space:create_index('primary', { parts = {'id'} })
+local time_insert = 0
+local time_replace = 0
+local time_delete = 0
+local cnt = 0
+local cnt_max = 20
+local op_max = 2500000
+local nanosec = 1.0e9
+while cnt < cnt_max do
+ cnt = cnt + 1
+ local time_before = clock.monotonic64()
+ for key = 1, op_max do space:insert({key, key + 1000}) end
+ local time_after = clock.monotonic64()
+ time_insert = time_insert + (time_after - time_before)
+ time_before = clock.monotonic64()
+ for key = 1, op_max do space:replace({key, key + 5000}) end
+ time_after = clock.monotonic64()
+ time_replace = time_replace + (time_after - time_before)
+ time_before = clock.monotonic64()
+ for key = 1, op_max do space:delete(key) end
+ time_after = clock.monotonic64()
+ time_delete = time_delete + (time_after - time_before)
+end
+io.write("{\n")
+io.write(string.format(" \"alloc time\": \"%.3f\"\n", tonumber(time_insert) / (nanosec * cnt_max)))
+io.write(string.format(" \"replace time\": \"%.3f\"\n", tonumber(time_replace) / (nanosec * cnt_max)))
+io.write(string.format(" \"delete time\": \"%.3f\"\n}\n", tonumber(time_delete) / (nanosec * cnt_max)))
+os.exit()
diff --git a/src/box/CMakeLists.txt b/src/box/CMakeLists.txt
index 19203f770..ba8c4248a 100644
--- a/src/box/CMakeLists.txt
+++ b/src/box/CMakeLists.txt
@@ -127,7 +127,7 @@ add_library(box STATIC
memtx_bitset.c
memtx_tx.c
engine.c
- memtx_engine.c
+ memtx_engine.cc
memtx_space.c
sysview.c
blackhole.c
@@ -197,7 +197,7 @@ add_library(box STATIC
lua/console.c
lua/serialize_lua.c
lua/tuple.c
- lua/slab.c
+ lua/slab.cc
lua/index.c
lua/space.cc
lua/sequence.c
diff --git a/src/box/allocator.h b/src/box/allocator.h
new file mode 100644
index 000000000..c40afe6d2
--- /dev/null
+++ b/src/box/allocator.h
@@ -0,0 +1,162 @@
+#pragma once
+/*
+ * Copyright 2010-2020, Tarantool AUTHORS, please see AUTHORS file.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the
+ * following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY <COPYRIGHT HOLDER> ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+ * <COPYRIGHT HOLDER> OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+ * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+#include <small/small.h>
+#include <stdarg.h>
+
+#include "system_allocator.h"
+
+enum memtx_allocator_type {
+ SMALL_ALLOCATOR_TYPE = 0,
+ SYSTEM_ALLOCATOR_TYPE,
+};
+
+struct allocator_stats {
+ size_t used;
+ size_t total;
+};
+
+struct SmallAlloc
+{
+ /* Tuple allocator */
+ static struct small_alloc small_alloc;
+ /** Slab cache for allocating tuples. */
+ static struct slab_cache slab_cache;
+ static inline void
+ init(struct slab_arena *arena, uint32_t objsize_min, float alloc_factor)
+ {
+ slab_cache_create(&slab_cache, arena);
+ small_alloc_create(&small_alloc, &slab_cache, objsize_min, alloc_factor);
+ }
+ static inline void
+ destroy()
+ {
+ small_alloc_destroy(&small_alloc);
+ slab_cache_destroy(&slab_cache);
+ }
+ static inline void *
+ alloc(size_t size)
+ {
+ return smalloc(&small_alloc, size);
+ }
+ static inline void
+ free(void *ptr, size_t size)
+ {
+ return smfree(&small_alloc, ptr, size);
+ }
+ static inline void
+ free_delayed(void *ptr, size_t size)
+ {
+ return smfree_delayed(&small_alloc, ptr, size);
+ }
+ static inline void
+ enter_delayed_free_mode()
+ {
+ return small_alloc_setopt(&small_alloc, SMALL_DELAYED_FREE_MODE, true);
+ }
+ static inline void
+ leave_delayed_free_mode()
+ {
+ return small_alloc_setopt(&small_alloc, SMALL_DELAYED_FREE_MODE, false);
+ }
+ static inline void
+ mem_check()
+ {
+ return slab_cache_check(&slab_cache);
+ }
+ static inline void
+ stats(struct allocator_stats *stats, va_list argptr)
+ {
+ mempool_stats_cb stats_cb = va_arg(argptr, mempool_stats_cb);
+ void *cb_ctx = va_arg(argptr, void *);
+
+ struct small_stats data_stats;
+ small_stats(&small_alloc, &data_stats, stats_cb, cb_ctx);
+ stats->used = data_stats.used;
+ stats->total = data_stats.total;
+ }
+};
+extern struct SmallAlloc small_alloc;
+
+struct SystemAlloc
+{
+ /* Tuple allocator */
+ static struct system_alloc system_alloc;
+ static inline void
+ init(struct quota *quota)
+ {
+ system_alloc_create(&system_alloc, quota);
+ }
+ static inline void
+ destroy()
+ {
+ system_alloc_destroy(&system_alloc);
+ }
+ static inline void *
+ alloc(size_t size)
+ {
+ return sysalloc(&system_alloc, size);
+ }
+ static inline void
+ free(void *ptr, size_t size)
+ {
+ return sysfree(&system_alloc, ptr, size);
+ }
+ static inline void
+ free_delayed(void *ptr, size_t size)
+ {
+ return sysfree_delayed(&system_alloc, ptr, size);
+ }
+ static inline void
+ enter_delayed_free_mode()
+ {
+ return system_alloc_setopt(&system_alloc, SYSTEM_DELAYED_FREE_MODE, true);
+ }
+ static inline void
+ leave_delayed_free_mode()
+ {
+ return system_alloc_setopt(&system_alloc, SYSTEM_DELAYED_FREE_MODE, false);
+ }
+ static inline void
+ mem_check()
+ {
+
+ }
+ static inline void
+ stats(struct allocator_stats *stats, MAYBE_UNUSED va_list argptr)
+ {
+ struct system_stats data_stats;
+ system_stats(&system_alloc, &data_stats);
+ stats->used = data_stats.used;
+ stats->total = data_stats.total;
+ }
+};
+extern struct SystemAlloc system_alloc;
\ No newline at end of file
diff --git a/src/box/box.cc b/src/box/box.cc
index a8bc3471d..66f6030df 100644
--- a/src/box/box.cc
+++ b/src/box/box.cc
@@ -2250,6 +2250,7 @@ engine_init()
cfg_getd("memtx_memory"),
cfg_geti("memtx_min_tuple_size"),
cfg_geti("strip_core"),
+ cfg_gets("allocator"),
cfg_getd("slab_alloc_factor"));
engine_register((struct engine *)memtx);
box_set_memtx_max_tuple_size();
diff --git a/src/box/field_map.h b/src/box/field_map.h
index d8ef726a1..5087d25e5 100644
--- a/src/box/field_map.h
+++ b/src/box/field_map.h
@@ -35,6 +35,10 @@
#include <stddef.h>
#include "bit/bit.h"
+#if defined(__cplusplus)
+extern "C" {
+#endif /* defined(__cplusplus) */
+
struct region;
struct field_map_builder_slot;
@@ -257,4 +261,8 @@ field_map_build_size(struct field_map_builder *builder)
void
field_map_build(struct field_map_builder *builder, char *buffer);
+#if defined(__cplusplus)
+} /* extern "C" */
+#endif /* defined(__plusplus) */
+
#endif /* TARANTOOL_BOX_FIELD_MAP_H_INCLUDED */
diff --git a/src/box/lua/load_cfg.lua b/src/box/lua/load_cfg.lua
index 770442052..817b8dbd5 100644
--- a/src/box/lua/load_cfg.lua
+++ b/src/box/lua/load_cfg.lua
@@ -43,6 +43,7 @@ local default_cfg = {
memtx_min_tuple_size = 16,
memtx_max_tuple_size = 1024 * 1024,
slab_alloc_factor = 1.05,
+ allocator = "small",
work_dir = nil,
memtx_dir = ".",
wal_dir = ".",
@@ -123,6 +124,7 @@ local template_cfg = {
memtx_min_tuple_size = 'number',
memtx_max_tuple_size = 'number',
slab_alloc_factor = 'number',
+ allocator = 'string',
work_dir = 'string',
memtx_dir = 'string',
wal_dir = 'string',
diff --git a/src/box/lua/slab.c b/src/box/lua/slab.cc
similarity index 88%
rename from src/box/lua/slab.c
rename to src/box/lua/slab.cc
index 9f5e7e95c..d44e77e94 100644
--- a/src/box/lua/slab.c
+++ b/src/box/lua/slab.cc
@@ -43,6 +43,7 @@
#include "memory.h"
#include "box/engine.h"
#include "box/memtx_engine.h"
+#include "box/allocator.h"
static int
small_stats_noop_cb(const struct mempool_stats *stats, void *cb_ctx)
@@ -108,13 +109,28 @@ lbox_slab_stats(struct lua_State *L)
struct memtx_engine *memtx;
memtx = (struct memtx_engine *)engine_by_name("memtx");
- struct small_stats totals;
+ struct allocator_stats totals;
lua_newtable(L);
/*
* List all slabs used for tuples and slabs used for
* indexes, with their stats.
*/
- small_stats(&memtx->alloc, &totals, small_stats_lua_cb, L);
+ if (memtx->type == SMALL_ALLOCATOR_TYPE) {
+ g_memtx_allocator_stats(&totals, small_stats_lua_cb, L);
+ } else {
+ g_memtx_allocator_stats(&totals);
+ lua_pushnumber(L, lua_objlen(L, -1) + 1);
+ lua_newtable(L);
+ luaL_setmaphint(L, -1);
+ lua_pushstring(L, "mem_used");
+ luaL_pushuint64(L, totals.used);
+ lua_settable(L, -3);
+
+ lua_pushstring(L, "mem_free");
+ luaL_pushuint64(L, totals.total - totals.used);
+ lua_settable(L, -3);
+ lua_settable(L, -3);
+ }
struct mempool_stats index_stats;
mempool_stats(&memtx->index_extent_pool, &index_stats);
small_stats_lua_cb(&index_stats, L);
@@ -128,14 +144,21 @@ lbox_slab_info(struct lua_State *L)
struct memtx_engine *memtx;
memtx = (struct memtx_engine *)engine_by_name("memtx");
- struct small_stats totals;
+ struct allocator_stats totals;
+ bool is_small_alloc;
/*
* List all slabs used for tuples and slabs used for
* indexes, with their stats.
*/
lua_newtable(L);
- small_stats(&memtx->alloc, &totals, small_stats_noop_cb, L);
+ if (memtx->type == SMALL_ALLOCATOR_TYPE) {
+ g_memtx_allocator_stats(&totals, small_stats_noop_cb, L);
+ is_small_alloc = true;
+ } else {
+ g_memtx_allocator_stats(&totals);
+ is_small_alloc = false;
+ }
struct mempool_stats index_stats;
mempool_stats(&memtx->index_extent_pool, &index_stats);
@@ -187,10 +210,10 @@ lbox_slab_info(struct lua_State *L)
* data (tuples and indexes).
*/
lua_pushstring(L, "arena_used");
- luaL_pushuint64(L, totals.used + index_stats.totals.used);
+ luaL_pushuint64(L, (is_small_alloc ? totals.used : 0) + index_stats.totals.used);
lua_settable(L, -3);
- ratio = 100 * ((double) (totals.used + index_stats.totals.used)
+ ratio = 100 * ((double) ((is_small_alloc ? totals.used : 0) + index_stats.totals.used)
/ (double) arena_size);
snprintf(ratio_buf, sizeof(ratio_buf), "%0.1lf%%", ratio);
@@ -257,9 +280,7 @@ lbox_runtime_info(struct lua_State *L)
static int
lbox_slab_check(MAYBE_UNUSED struct lua_State *L)
{
- struct memtx_engine *memtx;
- memtx = (struct memtx_engine *)engine_by_name("memtx");
- slab_cache_check(memtx->alloc.cache);
+ g_memtx_mem_check();
return 0;
}
diff --git a/src/box/memtx_engine.c b/src/box/memtx_engine.cc
similarity index 87%
rename from src/box/memtx_engine.c
rename to src/box/memtx_engine.cc
index db2bb2333..76f314223 100644
--- a/src/box/memtx_engine.c
+++ b/src/box/memtx_engine.cc
@@ -50,10 +50,83 @@
#include "schema.h"
#include "gc.h"
#include "raft.h"
+#include "allocator.h"
/* sync snapshot every 16MB */
#define SNAP_SYNC_INTERVAL (1 << 24)
+struct slab_cache SmallAlloc::slab_cache;
+struct small_alloc SmallAlloc::small_alloc;
+struct SmallAlloc small_alloc;
+
+struct system_alloc SystemAlloc::system_alloc;
+struct SystemAlloc system_alloc;
+
+struct tuple_format_vtab memtx_tuple_format_vtab;
+
+template <class Alloc>
+struct tuple *
+memtx_tuple_new(struct tuple_format *format, const char *data, const char *end);
+
+template <class Alloc>
+void
+memtx_tuple_delete(struct tuple_format *format, struct tuple *tuple);
+
+template <class Alloc>
+void
+metmx_tuple_chunk_delete(struct tuple_format *format, const char *data);
+
+template <class Alloc>
+const char *
+memtx_tuple_chunk_new(struct tuple_format *format, struct tuple *tuple,
+ const char *data, uint32_t data_sz);
+
+template <class Alloc>
+void
+memtx_enter_delayed_free_mode(struct memtx_engine *memtx);
+
+template <class Alloc>
+void
+memtx_leave_delayed_free_mode(struct memtx_engine *memtx);
+
+template <class Alloc>
+void
+memtx_mem_check();
+
+template <class Alloc>
+void
+memtx_allocator_stats(struct allocator_stats *stats, ...);
+
+global_memtx_tuple_new g_memtx_tuple_new;
+global_memtx_tuple_delete g_memtx_tuple_delete;
+global_memtx_enter_delayed_free_mode g_memtx_enter_delayed_free_mode;
+global_memtx_leave_delayed_free_mode g_memtx_leave_delayed_free_mode;
+global_memtx_mem_check g_memtx_mem_check;
+global_memtx_allocator_stats g_memtx_allocator_stats;
+
+#define DECLARE_MEMTX_ALLOCATOR_CHOICE(PREFIX, prefix, AllocType) \
+static inline void \
+prefix##_allocator_choice(struct memtx_engine *memtx) \
+{ \
+ memtx->type = PREFIX##_ALLOCATOR_TYPE; \
+ memtx_tuple_format_vtab.tuple_delete = memtx_tuple_delete<AllocType>; \
+ memtx_tuple_format_vtab.tuple_new = memtx_tuple_new<AllocType>; \
+ memtx_tuple_format_vtab.tuple_chunk_delete = \
+ metmx_tuple_chunk_delete<AllocType>; \
+ memtx_tuple_format_vtab.tuple_chunk_new = \
+ memtx_tuple_chunk_new<AllocType>; \
+ g_memtx_tuple_new = memtx_tuple_new<AllocType>; \
+ g_memtx_tuple_delete = memtx_tuple_delete<AllocType>; \
+ g_memtx_enter_delayed_free_mode = \
+ memtx_enter_delayed_free_mode<AllocType>; \
+ g_memtx_leave_delayed_free_mode = \
+ memtx_leave_delayed_free_mode<AllocType>; \
+ g_memtx_mem_check = memtx_mem_check<AllocType>; \
+ g_memtx_allocator_stats = memtx_allocator_stats<AllocType>; \
+}
+DECLARE_MEMTX_ALLOCATOR_CHOICE(SMALL, small, SmallAlloc)
+DECLARE_MEMTX_ALLOCATOR_CHOICE(SYSTEM, system, SystemAlloc)
+
static void
checkpoint_cancel(struct checkpoint *ckpt);
@@ -141,8 +214,10 @@ memtx_engine_shutdown(struct engine *engine)
mempool_destroy(&memtx->rtree_iterator_pool);
mempool_destroy(&memtx->index_extent_pool);
slab_cache_destroy(&memtx->index_slab_cache);
- small_alloc_destroy(&memtx->alloc);
- slab_cache_destroy(&memtx->slab_cache);
+ if (memtx->type == SMALL_ALLOCATOR_TYPE)
+ small_alloc.destroy();
+ else
+ system_alloc.destroy();
tuple_arena_destroy(&memtx->arena);
xdir_destroy(&memtx->snap_dir);
free(memtx);
@@ -540,7 +615,7 @@ struct checkpoint {
static struct checkpoint *
checkpoint_new(const char *snap_dirname, uint64_t snap_io_rate_limit)
{
- struct checkpoint *ckpt = malloc(sizeof(*ckpt));
+ struct checkpoint *ckpt = (struct checkpoint *)malloc(sizeof(*ckpt));
if (ckpt == NULL) {
diag_set(OutOfMemory, sizeof(*ckpt), "malloc",
"struct checkpoint");
@@ -609,7 +684,7 @@ checkpoint_add_space(struct space *sp, void *data)
if (!pk)
return 0;
struct checkpoint *ckpt = (struct checkpoint *)data;
- struct checkpoint_entry *entry = malloc(sizeof(*entry));
+ struct checkpoint_entry *entry = (struct checkpoint_entry *)malloc(sizeof(*entry));
if (entry == NULL) {
diag_set(OutOfMemory, sizeof(*entry),
"malloc", "struct checkpoint_entry");
@@ -839,7 +914,7 @@ struct memtx_join_ctx {
static int
memtx_join_add_space(struct space *space, void *arg)
{
- struct memtx_join_ctx *ctx = arg;
+ struct memtx_join_ctx *ctx = (struct memtx_join_ctx *)arg;
if (!space_is_memtx(space))
return 0;
if (space_is_temporary(space))
@@ -849,7 +924,7 @@ memtx_join_add_space(struct space *space, void *arg)
struct index *pk = space_index(space, 0);
if (pk == NULL)
return 0;
- struct memtx_join_entry *entry = malloc(sizeof(*entry));
+ struct memtx_join_entry *entry = (struct memtx_join_entry *)malloc(sizeof(*entry));
if (entry == NULL) {
diag_set(OutOfMemory, sizeof(*entry),
"malloc", "struct memtx_join_entry");
@@ -869,7 +944,7 @@ static int
memtx_engine_prepare_join(struct engine *engine, void **arg)
{
(void)engine;
- struct memtx_join_ctx *ctx = malloc(sizeof(*ctx));
+ struct memtx_join_ctx *ctx = (struct memtx_join_ctx *)malloc(sizeof(*ctx));
if (ctx == NULL) {
diag_set(OutOfMemory, sizeof(*ctx),
"malloc", "struct memtx_join_ctx");
@@ -907,7 +982,7 @@ memtx_join_send_tuple(struct xstream *stream, uint32_t space_id,
static int
memtx_join_f(va_list ap)
{
- struct memtx_join_ctx *ctx = va_arg(ap, struct memtx_join_ctx *);
+ struct memtx_join_ctx *ctx = (struct memtx_join_ctx *)va_arg(ap, struct memtx_join_ctx *);
struct memtx_join_entry *entry;
rlist_foreach_entry(entry, &ctx->entries, in_ctx) {
struct snapshot_iterator *it = entry->iterator;
@@ -929,7 +1004,7 @@ static int
memtx_engine_join(struct engine *engine, void *arg, struct xstream *stream)
{
(void)engine;
- struct memtx_join_ctx *ctx = arg;
+ struct memtx_join_ctx *ctx = (struct memtx_join_ctx *)arg;
ctx->stream = stream;
/*
* Memtx snapshot iterators are safe to use from another
@@ -950,7 +1025,7 @@ static void
memtx_engine_complete_join(struct engine *engine, void *arg)
{
(void)engine;
- struct memtx_join_ctx *ctx = arg;
+ struct memtx_join_ctx *ctx = (struct memtx_join_ctx *)arg;
struct memtx_join_entry *entry, *next;
rlist_foreach_entry_safe(entry, &ctx->entries, in_ctx, next) {
entry->iterator->free(entry->iterator);
@@ -971,10 +1046,14 @@ static void
memtx_engine_memory_stat(struct engine *engine, struct engine_memory_stat *stat)
{
struct memtx_engine *memtx = (struct memtx_engine *)engine;
- struct small_stats data_stats;
+ struct allocator_stats data_stats;
struct mempool_stats index_stats;
mempool_stats(&memtx->index_extent_pool, &index_stats);
- small_stats(&memtx->alloc, &data_stats, small_stats_noop_cb, NULL);
+ if (memtx->type == SMALL_ALLOCATOR_TYPE) {
+ g_memtx_allocator_stats(&data_stats, small_stats_noop_cb, NULL);
+ } else {
+ g_memtx_allocator_stats(&data_stats);
+ }
stat->data += data_stats.used;
stat->index += index_stats.totals.used;
}
@@ -1052,15 +1131,27 @@ memtx_engine_gc_f(va_list va)
struct memtx_engine *
memtx_engine_new(const char *snap_dirname, bool force_recovery,
uint64_t tuple_arena_max_size, uint32_t objsize_min,
- bool dontdump, float alloc_factor)
+ bool dontdump, const char* allocator, float alloc_factor)
{
- struct memtx_engine *memtx = calloc(1, sizeof(*memtx));
+ int64_t snap_signature;
+ struct memtx_engine *memtx = (struct memtx_engine *)calloc(1, sizeof(*memtx));
if (memtx == NULL) {
diag_set(OutOfMemory, sizeof(*memtx),
"malloc", "struct memtx_engine");
return NULL;
}
+ assert(allocator != NULL);
+ if (!strcmp(allocator, "small")) {
+ small_allocator_choice(memtx);
+ } else if (!strcmp(allocator, "system")) {
+ system_allocator_choice(memtx);
+ } else {
+ diag_set(IllegalParams, "Bad memory allocator name");
+ free(memtx);
+ return NULL;
+ }
+
xdir_create(&memtx->snap_dir, snap_dirname, SNAP, &INSTANCE_UUID,
&xlog_opts_default);
memtx->snap_dir.force_recovery = force_recovery;
@@ -1078,7 +1169,7 @@ memtx_engine_new(const char *snap_dirname, bool force_recovery,
* So if the local directory isn't empty, read the snapshot
* signature right now to initialize the instance UUID.
*/
- int64_t snap_signature = xdir_last_vclock(&memtx->snap_dir, NULL);
+ snap_signature = xdir_last_vclock(&memtx->snap_dir, NULL);
if (snap_signature >= 0) {
struct xlog_cursor cursor;
if (xdir_open_cursor(&memtx->snap_dir,
@@ -1108,9 +1199,10 @@ memtx_engine_new(const char *snap_dirname, bool force_recovery,
quota_init(&memtx->quota, tuple_arena_max_size);
tuple_arena_create(&memtx->arena, &memtx->quota, tuple_arena_max_size,
SLAB_SIZE, dontdump, "memtx");
- slab_cache_create(&memtx->slab_cache, &memtx->arena);
- small_alloc_create(&memtx->alloc, &memtx->slab_cache,
- objsize_min, alloc_factor);
+ if (memtx->type == SMALL_ALLOCATOR_TYPE)
+ small_alloc.init(&memtx->arena, objsize_min, alloc_factor);
+ else
+ system_alloc.init(&memtx->quota);
/* Initialize index extent allocator. */
slab_cache_create(&memtx->index_slab_cache, &memtx->arena);
@@ -1170,22 +1262,42 @@ memtx_engine_set_max_tuple_size(struct memtx_engine *memtx, size_t max_size)
memtx->max_tuple_size = max_size;
}
+template <class Alloc>
+void
+memtx_allocator_stats(struct allocator_stats *stats, ...)
+{
+ va_list argptr;
+ va_start(argptr, stats);
+ Alloc::stats(stats, argptr);
+ va_end(argptr);
+}
+
+template <class Alloc>
+void
+memtx_mem_check()
+{
+ return Alloc::mem_check();
+}
+
+template <class Alloc>
void
memtx_enter_delayed_free_mode(struct memtx_engine *memtx)
{
memtx->snapshot_version++;
if (memtx->delayed_free_mode++ == 0)
- small_alloc_setopt(&memtx->alloc, SMALL_DELAYED_FREE_MODE, true);
+ Alloc::enter_delayed_free_mode();
}
+template<class Alloc>
void
memtx_leave_delayed_free_mode(struct memtx_engine *memtx)
{
assert(memtx->delayed_free_mode > 0);
if (--memtx->delayed_free_mode == 0)
- small_alloc_setopt(&memtx->alloc, SMALL_DELAYED_FREE_MODE, false);
+ Alloc::leave_delayed_free_mode();
}
+template <class Alloc>
struct tuple *
memtx_tuple_new(struct tuple_format *format, const char *data, const char *end)
{
@@ -1195,15 +1307,19 @@ memtx_tuple_new(struct tuple_format *format, const char *data, const char *end)
struct region *region = &fiber()->gc;
size_t region_svp = region_used(region);
struct field_map_builder builder;
+ uint32_t field_map_size, data_offset;
+ size_t tuple_len, total;
+ char *raw;
+
if (tuple_field_map_create(format, data, true, &builder) != 0)
goto end;
- uint32_t field_map_size = field_map_build_size(&builder);
+ field_map_size = field_map_build_size(&builder);
/*
* Data offset is calculated from the begin of the struct
* tuple base, not from memtx_tuple, because the struct
* tuple is not the first field of the memtx_tuple.
*/
- uint32_t data_offset = sizeof(struct tuple) + field_map_size;
+ data_offset = sizeof(struct tuple) + field_map_size;
if (data_offset > INT16_MAX) {
/** tuple->data_offset is 15 bits */
diag_set(ClientError, ER_TUPLE_METADATA_IS_TOO_BIG,
@@ -1211,8 +1327,8 @@ memtx_tuple_new(struct tuple_format *format, const char *data, const char *end)
goto end;
}
- size_t tuple_len = end - data;
- size_t total = sizeof(struct memtx_tuple) + field_map_size + tuple_len;
+ tuple_len = end - data;
+ total = sizeof(struct memtx_tuple) + field_map_size + tuple_len;
ERROR_INJECT(ERRINJ_TUPLE_ALLOC, {
diag_set(OutOfMemory, total, "slab allocator", "memtx_tuple");
@@ -1225,7 +1341,7 @@ memtx_tuple_new(struct tuple_format *format, const char *data, const char *end)
}
struct memtx_tuple *memtx_tuple;
- while ((memtx_tuple = smalloc(&memtx->alloc, total)) == NULL) {
+ while ((memtx_tuple = (struct memtx_tuple *)Alloc::alloc(total)) == NULL) {
bool stop;
memtx_engine_run_gc(memtx, &stop);
if (stop)
@@ -1244,7 +1360,7 @@ memtx_tuple_new(struct tuple_format *format, const char *data, const char *end)
tuple_format_ref(format);
tuple->data_offset = data_offset;
tuple->is_dirty = false;
- char *raw = (char *) tuple + tuple->data_offset;
+ raw = (char *) tuple + tuple->data_offset;
field_map_build(&builder, raw - field_map_size);
memcpy(raw, data, tuple_len);
say_debug("%s(%zu) = %p", __func__, tuple_len, memtx_tuple);
@@ -1253,6 +1369,7 @@ end:
return tuple;
}
+template <class Alloc>
void
memtx_tuple_delete(struct tuple_format *format, struct tuple *tuple)
{
@@ -1262,34 +1379,35 @@ memtx_tuple_delete(struct tuple_format *format, struct tuple *tuple)
struct memtx_tuple *memtx_tuple =
container_of(tuple, struct memtx_tuple, base);
size_t total = tuple_size(tuple) + offsetof(struct memtx_tuple, base);
- if (memtx->alloc.free_mode != SMALL_DELAYED_FREE ||
- memtx_tuple->version == memtx->snapshot_version ||
+ if (memtx_tuple->version == memtx->snapshot_version ||
format->is_temporary)
- smfree(&memtx->alloc, memtx_tuple, total);
+ Alloc::free(memtx_tuple, total);
else
- smfree_delayed(&memtx->alloc, memtx_tuple, total);
+ Alloc::free_delayed(memtx_tuple, total);
tuple_format_unref(format);
}
+template <class Alloc>
void
metmx_tuple_chunk_delete(struct tuple_format *format, const char *data)
{
- struct memtx_engine *memtx = (struct memtx_engine *)format->engine;
+ (void)format;
struct tuple_chunk *tuple_chunk =
container_of((const char (*)[0])data,
struct tuple_chunk, data);
uint32_t sz = tuple_chunk_sz(tuple_chunk->data_sz);
- smfree(&memtx->alloc, tuple_chunk, sz);
+ Alloc::free(tuple_chunk, sz);
}
+template <class Alloc>
const char *
memtx_tuple_chunk_new(struct tuple_format *format, struct tuple *tuple,
const char *data, uint32_t data_sz)
{
- struct memtx_engine *memtx = (struct memtx_engine *)format->engine;
+ (void)format;
uint32_t sz = tuple_chunk_sz(data_sz);
struct tuple_chunk *tuple_chunk =
- (struct tuple_chunk *) smalloc(&memtx->alloc, sz);
+ (struct tuple_chunk *) Alloc::alloc(sz);
if (tuple == NULL) {
diag_set(OutOfMemory, sz, "smalloc", "tuple");
return NULL;
@@ -1299,13 +1417,6 @@ memtx_tuple_chunk_new(struct tuple_format *format, struct tuple *tuple,
return tuple_chunk->data;
}
-struct tuple_format_vtab memtx_tuple_format_vtab = {
- memtx_tuple_delete,
- memtx_tuple_new,
- metmx_tuple_chunk_delete,
- memtx_tuple_chunk_new,
-};
-
/**
* Allocate a block of size MEMTX_EXTENT_SIZE for memtx index
*/
diff --git a/src/box/memtx_engine.h b/src/box/memtx_engine.h
index 8b380bf3c..652f634fb 100644
--- a/src/box/memtx_engine.h
+++ b/src/box/memtx_engine.h
@@ -49,6 +49,7 @@ struct index;
struct fiber;
struct tuple;
struct tuple_format;
+struct allocator_stats;
/**
* The state of memtx recovery process.
@@ -133,10 +134,6 @@ struct memtx_engine {
* is reflected in box.slab.info(), @sa lua/slab.c.
*/
struct slab_arena arena;
- /** Slab cache for allocating tuples. */
- struct slab_cache slab_cache;
- /** Tuple allocator. */
- struct small_alloc alloc;
/** Slab cache for allocating index extents. */
struct slab_cache index_slab_cache;
/** Index extent allocator. */
@@ -178,6 +175,8 @@ struct memtx_engine {
* memtx_gc_task::link.
*/
struct stailq gc_queue;
+ /** Memtx allocator type */
+ int type;
};
struct memtx_gc_task;
@@ -213,7 +212,7 @@ struct memtx_engine *
memtx_engine_new(const char *snap_dirname, bool force_recovery,
uint64_t tuple_arena_max_size,
uint32_t objsize_min, bool dontdump,
- float alloc_factor);
+ const char *allocator, float alloc_factor);
int
memtx_engine_recover_snapshot(struct memtx_engine *memtx,
@@ -235,23 +234,35 @@ memtx_engine_set_max_tuple_size(struct memtx_engine *memtx, size_t max_size);
* times from the same or different fibers - one just has to leave
* the delayed free mode the same amount of times then.
*/
-void
-memtx_enter_delayed_free_mode(struct memtx_engine *memtx);
+typedef void
+(*global_memtx_enter_delayed_free_mode)(struct memtx_engine *memtx);
+extern global_memtx_enter_delayed_free_mode g_memtx_enter_delayed_free_mode;
/**
* Leave tuple delayed free mode. This function undoes the effect
* of memtx_enter_delayed_free_mode().
*/
-void
-memtx_leave_delayed_free_mode(struct memtx_engine *memtx);
+typedef void
+(*global_memtx_leave_delayed_free_mode)(struct memtx_engine *memtx);
+extern global_memtx_leave_delayed_free_mode g_memtx_leave_delayed_free_mode;
/** Allocate a memtx tuple. @sa tuple_new(). */
-struct tuple *
-memtx_tuple_new(struct tuple_format *format, const char *data, const char *end);
+typedef struct tuple *
+(*global_memtx_tuple_new)(struct tuple_format *format, const char *data, const char *end);
+extern global_memtx_tuple_new g_memtx_tuple_new;
/** Free a memtx tuple. @sa tuple_delete(). */
-void
-memtx_tuple_delete(struct tuple_format *format, struct tuple *tuple);
+typedef void
+(*global_memtx_tuple_delete)(struct tuple_format *format, struct tuple *tuple);
+extern global_memtx_tuple_delete g_memtx_tuple_delete;
+
+typedef void
+(*global_memtx_mem_check)();
+extern global_memtx_mem_check g_memtx_mem_check;
+
+typedef void
+(*global_memtx_allocator_stats)(struct allocator_stats *stats, ...);
+extern global_memtx_allocator_stats g_memtx_allocator_stats;
/** Tuple format vtab for memtx engine. */
extern struct tuple_format_vtab memtx_tuple_format_vtab;
@@ -299,13 +310,13 @@ static inline struct memtx_engine *
memtx_engine_new_xc(const char *snap_dirname, bool force_recovery,
uint64_t tuple_arena_max_size,
uint32_t objsize_min, bool dontdump,
- float alloc_factor)
+ const char *allocator, float alloc_factor)
{
struct memtx_engine *memtx;
memtx = memtx_engine_new(snap_dirname, force_recovery,
tuple_arena_max_size,
objsize_min, dontdump,
- alloc_factor);
+ allocator, alloc_factor);
if (memtx == NULL)
diag_raise();
return memtx;
diff --git a/src/box/memtx_hash.c b/src/box/memtx_hash.c
index ed4dba90a..4d620e318 100644
--- a/src/box/memtx_hash.c
+++ b/src/box/memtx_hash.c
@@ -458,7 +458,7 @@ hash_snapshot_iterator_free(struct snapshot_iterator *iterator)
assert(iterator->free == hash_snapshot_iterator_free);
struct hash_snapshot_iterator *it =
(struct hash_snapshot_iterator *) iterator;
- memtx_leave_delayed_free_mode((struct memtx_engine *)
+ g_memtx_leave_delayed_free_mode((struct memtx_engine *)
it->index->base.engine);
light_index_iterator_destroy(&it->index->hash_table, &it->iterator);
index_unref(&it->index->base);
@@ -523,7 +523,7 @@ memtx_hash_index_create_snapshot_iterator(struct index *base)
index_ref(base);
light_index_iterator_begin(&index->hash_table, &it->iterator);
light_index_iterator_freeze(&index->hash_table, &it->iterator);
- memtx_enter_delayed_free_mode((struct memtx_engine *)base->engine);
+ g_memtx_enter_delayed_free_mode((struct memtx_engine *)base->engine);
return (struct snapshot_iterator *) it;
}
diff --git a/src/box/memtx_space.c b/src/box/memtx_space.c
index 73b4c450e..66950bb7e 100644
--- a/src/box/memtx_space.c
+++ b/src/box/memtx_space.c
@@ -327,7 +327,7 @@ memtx_space_execute_replace(struct space *space, struct txn *txn,
struct memtx_space *memtx_space = (struct memtx_space *)space;
struct txn_stmt *stmt = txn_current_stmt(txn);
enum dup_replace_mode mode = dup_replace_mode(request->type);
- stmt->new_tuple = memtx_tuple_new(space->format, request->tuple,
+ stmt->new_tuple = g_memtx_tuple_new(space->format, request->tuple,
request->tuple_end);
if (stmt->new_tuple == NULL)
return -1;
@@ -412,7 +412,7 @@ memtx_space_execute_update(struct space *space, struct txn *txn,
if (new_data == NULL)
return -1;
- stmt->new_tuple = memtx_tuple_new(format, new_data,
+ stmt->new_tuple = g_memtx_tuple_new(format, new_data,
new_data + new_size);
if (stmt->new_tuple == NULL)
return -1;
@@ -483,7 +483,7 @@ memtx_space_execute_upsert(struct space *space, struct txn *txn,
format, request->index_base) != 0) {
return -1;
}
- stmt->new_tuple = memtx_tuple_new(format, request->tuple,
+ stmt->new_tuple = g_memtx_tuple_new(format, request->tuple,
request->tuple_end);
if (stmt->new_tuple == NULL)
return -1;
@@ -507,7 +507,7 @@ memtx_space_execute_upsert(struct space *space, struct txn *txn,
if (new_data == NULL)
return -1;
- stmt->new_tuple = memtx_tuple_new(format, new_data,
+ stmt->new_tuple = g_memtx_tuple_new(format, new_data,
new_data + new_size);
if (stmt->new_tuple == NULL)
return -1;
@@ -559,14 +559,14 @@ memtx_space_ephemeral_replace(struct space *space, const char *tuple,
const char *tuple_end)
{
struct memtx_space *memtx_space = (struct memtx_space *)space;
- struct tuple *new_tuple = memtx_tuple_new(space->format, tuple,
+ struct tuple *new_tuple = g_memtx_tuple_new(space->format, tuple,
tuple_end);
if (new_tuple == NULL)
return -1;
struct tuple *old_tuple;
if (memtx_space->replace(space, NULL, new_tuple,
DUP_REPLACE_OR_INSERT, &old_tuple) != 0) {
- memtx_tuple_delete(space->format, new_tuple);
+ g_memtx_tuple_delete(space->format, new_tuple);
return -1;
}
if (old_tuple != NULL)
diff --git a/src/box/memtx_tree.cc b/src/box/memtx_tree.cc
index 44bdc86e6..0c73bd676 100644
--- a/src/box/memtx_tree.cc
+++ b/src/box/memtx_tree.cc
@@ -1441,7 +1441,7 @@ tree_snapshot_iterator_free(struct snapshot_iterator *iterator)
assert(iterator->free == &tree_snapshot_iterator_free<USE_HINT>);
struct tree_snapshot_iterator<USE_HINT> *it =
(struct tree_snapshot_iterator<USE_HINT> *)iterator;
- memtx_leave_delayed_free_mode((struct memtx_engine *)
+ g_memtx_leave_delayed_free_mode((struct memtx_engine *)
it->index->base.engine);
memtx_tree_iterator_destroy(&it->index->tree, &it->tree_iterator);
index_unref(&it->index->base);
@@ -1516,7 +1516,7 @@ memtx_tree_index_create_snapshot_iterator(struct index *base)
index_ref(base);
it->tree_iterator = memtx_tree_iterator_first(&index->tree);
memtx_tree_iterator_freeze(&index->tree, &it->tree_iterator);
- memtx_enter_delayed_free_mode((struct memtx_engine *)base->engine);
+ g_memtx_enter_delayed_free_mode((struct memtx_engine *)base->engine);
return (struct snapshot_iterator *) it;
}
diff --git a/src/box/system_allocator.h b/src/box/system_allocator.h
new file mode 100644
index 000000000..8d7bb8067
--- /dev/null
+++ b/src/box/system_allocator.h
@@ -0,0 +1,218 @@
+#pragma once
+/*
+ * Copyright 2010-2020, Tarantool AUTHORS, please see AUTHORS file.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the
+ * following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY <COPYRIGHT HOLDER> ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+ * <COPYRIGHT HOLDER> OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+ * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+#include <stdlib.h>
+#include <trivia/util.h>
+#include <trivia/config.h>
+
+#if defined(__cplusplus)
+extern "C" {
+#endif /* defined(__cplusplus) */
+
+#if HAVE_MALLOC_SIZE_DARWIN
+#include <malloc/malloc.h>
+static inline size_t
+portable_malloc_usable_size(void *p)
+{
+ return malloc_size(p);
+}
+#elif HAVE_MALLOC_USABLE_SIZE_BSD
+#include <malloc_np.h>
+static inline size_t
+portable_malloc_usable_size(void *p)
+{
+ return malloc_usable_size(p);
+}
+#elif HAVE_MALLOC_USABLE_SIZE_LINUX
+#include <malloc.h>
+static inline size_t
+portable_malloc_usable_size(void *p)
+{
+ return malloc_usable_size(p);
+}
+#else
+#error "Undefined system type"
+#endif
+
+/**
+ * Free mode
+ */
+enum system_free_mode {
+ /** Free objects immediately. */
+ SYSTEM_FREE,
+ /** Collect garbage after delayed free. */
+ SYSTEM_COLLECT_GARBAGE,
+ /** Postpone deletion of objects. */
+ SYSTEM_DELAYED_FREE,
+};
+
+struct system_alloc {
+ /**
+ * Bytes allocated by system allocator
+ */
+ uint64_t used_bytes;
+ /**
+ * Allocator quota
+ */
+ struct quota *quota;
+ /**
+ * Free mode.
+ */
+ enum system_free_mode free_mode;
+ /**
+ * List of pointers for delayed free.
+ */
+ struct lifo delayed;
+};
+
+struct system_stats {
+ size_t used;
+ size_t total;
+};
+
+enum system_opt {
+ SYSTEM_DELAYED_FREE_MODE
+};
+
+static inline void
+sysfree(struct system_alloc *alloc, void *ptr, MAYBE_UNUSED unsigned bytes)
+{
+ size_t size = portable_malloc_usable_size(ptr);
+ uint32_t s = size % QUOTA_UNIT_SIZE, units = size / QUOTA_UNIT_SIZE;
+ size_t used_bytes = pm_atomic_fetch_sub(&alloc->used_bytes, size);
+ if (small_align(used_bytes, QUOTA_UNIT_SIZE) >
+ small_align(used_bytes - s, QUOTA_UNIT_SIZE))
+ units++;
+ if (units > 0)
+ quota_release(alloc->quota, units * QUOTA_UNIT_SIZE);
+ free(ptr);
+}
+
+static inline void
+system_collect_garbage(struct system_alloc *alloc)
+{
+ if (alloc->free_mode != SYSTEM_COLLECT_GARBAGE)
+ return;
+
+ const int BATCH = 100;
+ if (!lifo_is_empty(&alloc->delayed)) {
+ for (int i = 0; i < BATCH; i++) {
+ void *item = lifo_pop(&alloc->delayed);
+ if (item == NULL)
+ break;
+ sysfree(alloc, item, 0 /* unused parameter */);
+ }
+ } else {
+ /* Finish garbage collection and switch to regular mode */
+ alloc->free_mode = SYSTEM_FREE;
+ }
+}
+
+static inline void
+system_alloc_setopt(struct system_alloc *alloc, enum system_opt opt, bool val)
+{
+ switch (opt) {
+ case SYSTEM_DELAYED_FREE_MODE:
+ alloc->free_mode = val ? SYSTEM_DELAYED_FREE :
+ SYSTEM_COLLECT_GARBAGE;
+ break;
+ default:
+ assert(false);
+ break;
+ }
+}
+
+static inline void
+system_stats(struct system_alloc *alloc, struct system_stats *totals)
+{
+ totals->used = pm_atomic_load_explicit(&alloc->used_bytes,
+ pm_memory_order_relaxed);
+ totals->total = quota_total(alloc->quota);
+}
+
+static inline void
+system_alloc_create(struct system_alloc *alloc, struct quota *quota)
+{
+ alloc->used_bytes = 0;
+ alloc->quota = quota;
+ lifo_init(&alloc->delayed);
+}
+
+static inline void
+system_alloc_destroy(MAYBE_UNUSED struct system_alloc *alloc)
+{
+
+}
+
+static inline void
+sysfree_delayed(struct system_alloc *alloc, void *ptr, unsigned bytes)
+{
+ if (alloc->free_mode == SYSTEM_DELAYED_FREE && ptr) {
+ lifo_push(&alloc->delayed, ptr);
+ } else {
+ sysfree(alloc, ptr, bytes);
+ }
+}
+
+static inline void *
+sysalloc(struct system_alloc *alloc, unsigned bytes)
+{
+ system_collect_garbage(alloc);
+
+ void *ptr = malloc(bytes);
+ if (!ptr)
+ return NULL;
+ size_t size = portable_malloc_usable_size(ptr);
+ uint32_t s = size % QUOTA_UNIT_SIZE, units = size / QUOTA_UNIT_SIZE;
+ while (1) {
+ size_t used_bytes = pm_atomic_load(&alloc->used_bytes);
+ if (small_align(used_bytes, QUOTA_UNIT_SIZE) <
+ small_align(used_bytes + s, QUOTA_UNIT_SIZE))
+ units++;
+ if (units > 0) {
+ if (quota_use(alloc->quota,
+ units * QUOTA_UNIT_SIZE) < 0) {
+ free(ptr);
+ return NULL;
+ }
+ }
+ if (pm_atomic_compare_exchange_strong(&alloc->used_bytes,
+ &used_bytes, used_bytes + size))
+ break;
+ if (units > 0)
+ quota_release(alloc->quota, units * QUOTA_UNIT_SIZE);
+ }
+ return ptr;
+}
+
+#if defined(__cplusplus)
+} /* extern "C" */
+#endif /* defined(__cplusplus) */
diff --git a/src/trivia/config.h.cmake b/src/trivia/config.h.cmake
index 89e0d39c6..107cd8049 100644
--- a/src/trivia/config.h.cmake
+++ b/src/trivia/config.h.cmake
@@ -169,6 +169,9 @@
#cmakedefine HAVE_POSIX_FADVISE 1
#cmakedefine HAVE_FALLOCATE 1
#cmakedefine HAVE_MREMAP 1
+#cmakedefine HAVE_MALLOC_USABLE_SIZE_LINUX 1
+#cmakedefine HAVE_MALLOC_USABLE_SIZE_BSD 1
+#cmakedefine HAVE_MALLOC_SIZE_DARWIN 1
#cmakedefine HAVE_SYNC_FILE_RANGE 1
#cmakedefine HAVE_MSG_NOSIGNAL 1
diff --git a/test/app-tap/init_script.result b/test/app-tap/init_script.result
index 72aa67db2..3b5211a90 100644
--- a/test/app-tap/init_script.result
+++ b/test/app-tap/init_script.result
@@ -3,6 +3,7 @@
--
box.cfg
+allocator:small
background:false
checkpoint_count:2
checkpoint_interval:3600
diff --git a/test/box/admin.result b/test/box/admin.result
index e05440f66..9e4813133 100644
--- a/test/box/admin.result
+++ b/test/box/admin.result
@@ -27,7 +27,9 @@ help()
...
cfg_filter(box.cfg)
---
-- - - background
+- - - allocator
+ - small
+ - - background
- false
- - checkpoint_count
- 2
diff --git a/test/box/cfg.result b/test/box/cfg.result
index 10fef006c..d23255872 100644
--- a/test/box/cfg.result
+++ b/test/box/cfg.result
@@ -15,7 +15,9 @@ box.cfg.nosuchoption = 1
| ...
cfg_filter(box.cfg)
| ---
- | - - - background
+ | - - - allocator
+ | - small
+ | - - background
| - false
| - - checkpoint_count
| - 2
@@ -128,7 +130,9 @@ box.cfg()
| ...
cfg_filter(box.cfg)
| ---
- | - - - background
+ | - - - allocator
+ | - small
+ | - - background
| - false
| - - checkpoint_count
| - 2
diff --git a/test/box/choose_memtx_allocator.lua b/test/box/choose_memtx_allocator.lua
new file mode 100644
index 000000000..77a0ec638
--- /dev/null
+++ b/test/box/choose_memtx_allocator.lua
@@ -0,0 +1,9 @@
+#!/usr/bin/env tarantool
+
+require('console').listen(os.getenv('ADMIN'))
+
+box.cfg({
+ listen = os.getenv("LISTEN"),
+ allocator=arg[1],
+ checkpoint_interval=10
+})
diff --git a/test/box/choose_memtx_allocator.result b/test/box/choose_memtx_allocator.result
new file mode 100644
index 000000000..dab316b93
--- /dev/null
+++ b/test/box/choose_memtx_allocator.result
@@ -0,0 +1,135 @@
+-- test-run result file version 2
+
+-- write data recover from latest snapshot
+env = require('test_run')
+ | ---
+ | ...
+test_run = env.new()
+ | ---
+ | ...
+test_run:cmd('create server test with script="box/choose_memtx_allocator.lua"')
+ | ---
+ | - true
+ | ...
+--test small allocator
+test_run:cmd('start server test with args="small"')
+ | ---
+ | - true
+ | ...
+test_run:cmd('switch test')
+ | ---
+ | - true
+ | ...
+space = box.schema.space.create('test')
+ | ---
+ | ...
+space:format({ {name = 'id', type = 'unsigned'}, {name = 'year', type = 'unsigned'} })
+ | ---
+ | ...
+s = space:create_index('primary', { parts = {'id'} })
+ | ---
+ | ...
+for key = 1, 1000 do space:insert({key, key + 1000}) end
+ | ---
+ | ...
+for key = 1, 1000 do space:replace({key, key + 5000}) end
+ | ---
+ | ...
+for key = 1, 1000 do space:delete(key) end
+ | ---
+ | ...
+space:drop()
+ | ---
+ | ...
+test_run:cmd('switch default')
+ | ---
+ | - true
+ | ...
+test_run:cmd('stop server test')
+ | ---
+ | - true
+ | ...
+--test system(malloc) allocator
+test_run:cmd('start server test with args="system"')
+ | ---
+ | - true
+ | ...
+test_run:cmd('switch test')
+ | ---
+ | - true
+ | ...
+space = box.schema.space.create('test')
+ | ---
+ | ...
+space:format({ {name = 'id', type = 'unsigned'}, {name = 'year', type = 'unsigned'} })
+ | ---
+ | ...
+s = space:create_index('primary', { parts = {'id'} })
+ | ---
+ | ...
+for key = 1, 500000 do space:insert({key, key + 1000}) end
+ | ---
+ | ...
+for key = 1, 500000 do space:replace({key, key + 5000}) end
+ | ---
+ | ...
+for key = 1, 500000 do space:delete(key) end
+ | ---
+ | ...
+space:drop()
+ | ---
+ | ...
+test_run:cmd('switch default')
+ | ---
+ | - true
+ | ...
+test_run:cmd('stop server test')
+ | ---
+ | - true
+ | ...
+--test default (small) allocator
+test_run:cmd('start server test')
+ | ---
+ | - true
+ | ...
+test_run:cmd('switch test')
+ | ---
+ | - true
+ | ...
+space = box.schema.space.create('test')
+ | ---
+ | ...
+space:format({ {name = 'id', type = 'unsigned'}, {name = 'year', type = 'unsigned'} })
+ | ---
+ | ...
+s = space:create_index('primary', { parts = {'id'} })
+ | ---
+ | ...
+for key = 1, 1000 do space:insert({key, key + 1000}) end
+ | ---
+ | ...
+for key = 1, 1000 do space:replace({key, key + 5000}) end
+ | ---
+ | ...
+for key = 1, 1000 do space:delete(key) end
+ | ---
+ | ...
+space:drop()
+ | ---
+ | ...
+test_run:cmd('switch default')
+ | ---
+ | - true
+ | ...
+test_run:cmd('stop server test')
+ | ---
+ | - true
+ | ...
+test_run:cmd('cleanup server test')
+ | ---
+ | - true
+ | ...
+test_run:cmd('delete server test')
+ | ---
+ | - true
+ | ...
diff --git a/test/box/choose_memtx_allocator.test.lua b/test/box/choose_memtx_allocator.test.lua
new file mode 100644
index 000000000..007b01d80
--- /dev/null
+++ b/test/box/choose_memtx_allocator.test.lua
@@ -0,0 +1,43 @@
+
+-- write data recover from latest snapshot
+env = require('test_run')
+test_run = env.new()
+test_run:cmd('create server test with script="box/choose_memtx_allocator.lua"')
+--test small allocator
+test_run:cmd('start server test with args="small"')
+test_run:cmd('switch test')
+space = box.schema.space.create('test')
+space:format({ {name = 'id', type = 'unsigned'}, {name = 'year', type = 'unsigned'} })
+s = space:create_index('primary', { parts = {'id'} })
+for key = 1, 1000 do space:insert({key, key + 1000}) end
+for key = 1, 1000 do space:replace({key, key + 5000}) end
+for key = 1, 1000 do space:delete(key) end
+space:drop()
+test_run:cmd('switch default')
+test_run:cmd('stop server test')
+--test system(malloc) allocator
+test_run:cmd('start server test with args="system"')
+test_run:cmd('switch test')
+space = box.schema.space.create('test')
+space:format({ {name = 'id', type = 'unsigned'}, {name = 'year', type = 'unsigned'} })
+s = space:create_index('primary', { parts = {'id'} })
+for key = 1, 500000 do space:insert({key, key + 1000}) end
+for key = 1, 500000 do space:replace({key, key + 5000}) end
+for key = 1, 500000 do space:delete(key) end
+space:drop()
+test_run:cmd('switch default')
+test_run:cmd('stop server test')
+--test default (small) allocator
+test_run:cmd('start server test')
+test_run:cmd('switch test')
+space = box.schema.space.create('test')
+space:format({ {name = 'id', type = 'unsigned'}, {name = 'year', type = 'unsigned'} })
+s = space:create_index('primary', { parts = {'id'} })
+for key = 1, 1000 do space:insert({key, key + 1000}) end
+for key = 1, 1000 do space:replace({key, key + 5000}) end
+for key = 1, 1000 do space:delete(key) end
+space:drop()
+test_run:cmd('switch default')
+test_run:cmd('stop server test')
+test_run:cmd('cleanup server test')
+test_run:cmd('delete server test')
--
2.20.1
^ permalink raw reply [flat|nested] only message in thread
only message in thread, other threads:[~2020-12-18 14:11 UTC | newest]
Thread overview: (only message) (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2020-12-18 14:11 [Tarantool-patches] [PATCH] Add new 'allocator' option in box.cfg (cpp version) mechanik20051988
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox