From: mechanik20051988 via Tarantool-patches <tarantool-patches@dev.tarantool.org>
To: v.shpilevoy@tarantool.org, alyapunov@tarantool.org
Cc: tarantool-patches@dev.tarantool.org
Subject: [Tarantool-patches] [PATCH v2 5/5] memtx: implement api for memory allocator selection
Date: Wed, 20 Jan 2021 10:53:43 +0300 [thread overview]
Message-ID: <d14adeb3f353fa329775386035c6d22e84e3f2b3.1611073501.git.mechanik20051988@tarantool.org> (raw)
In-Reply-To: <cover.1611073501.git.mechanik20051988@tarantool.org>
Added an allocator option in the.cfg window for {}, which provides
user the ability to select a memory allocator for memtx.
The available option values is "system" and "small". The default value
is "small".
Closes #5419
@TarantoolBot document
Title: Add new 'allocator' option to box.cfg{}
Add new 'allocator' option to box.cfg{} which allows to
select the appropriate allocator for memtx tuples if necessary.
Possible values are "system" for malloc allocator and
"small" for default small allocator.
---
src/box/box.cc | 1 +
src/box/lua/load_cfg.lua | 2 +
src/box/memtx_engine.cc | 31 +++--
src/box/memtx_engine.h | 6 +-
test/app-tap/init_script.result | 1 +
test/box/admin.result | 4 +-
test/box/cfg.result | 8 +-
test/box/choose_memtx_allocator.lua | 8 ++
test/box/choose_memtx_allocator.result | 147 +++++++++++++++++++++++
test/box/choose_memtx_allocator.test.lua | 46 +++++++
10 files changed, 240 insertions(+), 14 deletions(-)
create mode 100644 test/box/choose_memtx_allocator.lua
create mode 100644 test/box/choose_memtx_allocator.result
create mode 100644 test/box/choose_memtx_allocator.test.lua
diff --git a/src/box/box.cc b/src/box/box.cc
index 0bf7c175e..7e1b9d207 100644
--- a/src/box/box.cc
+++ b/src/box/box.cc
@@ -2542,6 +2542,7 @@ engine_init()
cfg_getd("memtx_memory"),
cfg_geti("memtx_min_tuple_size"),
cfg_geti("strip_core"),
+ cfg_gets("allocator"),
cfg_getd("slab_alloc_factor"));
engine_register((struct engine *)memtx);
box_set_memtx_max_tuple_size();
diff --git a/src/box/lua/load_cfg.lua b/src/box/lua/load_cfg.lua
index 574c8bef4..2fe8a5b6c 100644
--- a/src/box/lua/load_cfg.lua
+++ b/src/box/lua/load_cfg.lua
@@ -43,6 +43,7 @@ local default_cfg = {
memtx_min_tuple_size = 16,
memtx_max_tuple_size = 1024 * 1024,
slab_alloc_factor = 1.05,
+ allocator = "small",
work_dir = nil,
memtx_dir = ".",
wal_dir = ".",
@@ -124,6 +125,7 @@ local template_cfg = {
memtx_min_tuple_size = 'number',
memtx_max_tuple_size = 'number',
slab_alloc_factor = 'number',
+ allocator = 'string',
work_dir = 'string',
memtx_dir = 'string',
wal_dir = 'string',
diff --git a/src/box/memtx_engine.cc b/src/box/memtx_engine.cc
index ad904889a..285b49655 100644
--- a/src/box/memtx_engine.cc
+++ b/src/box/memtx_engine.cc
@@ -1092,7 +1092,7 @@ memtx_engine_gc_f(va_list va)
struct memtx_engine *
memtx_engine_new(const char *snap_dirname, bool force_recovery,
uint64_t tuple_arena_max_size, uint32_t objsize_min,
- bool dontdump, float alloc_factor)
+ bool dontdump, const char *allocator, float alloc_factor)
{
int64_t snap_signature;
struct memtx_engine *memtx = (struct memtx_engine *)calloc(1, sizeof(*memtx));
@@ -1102,13 +1102,28 @@ memtx_engine_new(const char *snap_dirname, bool force_recovery,
return NULL;
}
- memtx->allocator_type = MEMTX_SMALL_ALLOCATOR;
- MEMXT_TUPLE_FORMAT_VTAB(SmallAllocator)
- memtx_engine_vtab.memory_stat =
- memtx_engine_memory_stat<struct small_stats,
- struct mempool_stats,
- SmallAllocator,
- small_stats_noop_cb>;
+ assert(allocator != NULL);
+ if (!strcmp(allocator, "small")) {
+ memtx->allocator_type = MEMTX_SMALL_ALLOCATOR;
+ MEMXT_TUPLE_FORMAT_VTAB(SmallAllocator)
+ memtx_engine_vtab.memory_stat =
+ memtx_engine_memory_stat<struct small_stats,
+ struct mempool_stats,
+ SmallAllocator,
+ small_stats_noop_cb>;
+ } else if (!strcmp(allocator, "system")) {
+ memtx->allocator_type = MEMTX_SYSTEM_ALLOCATOR;
+ MEMXT_TUPLE_FORMAT_VTAB(SystemAllocator)
+ memtx_engine_vtab.memory_stat =
+ memtx_engine_memory_stat<struct system_stats,
+ struct system_stats,
+ SystemAllocator,
+ system_stats_noop_cb>;
+ } else {
+ diag_set(IllegalParams, "Invalid memory allocator name");
+ free(memtx);
+ return NULL;
+ }
xdir_create(&memtx->snap_dir, snap_dirname, SNAP, &INSTANCE_UUID,
&xlog_opts_default);
diff --git a/src/box/memtx_engine.h b/src/box/memtx_engine.h
index ba2394462..6edb8b373 100644
--- a/src/box/memtx_engine.h
+++ b/src/box/memtx_engine.h
@@ -218,7 +218,7 @@ struct memtx_engine *
memtx_engine_new(const char *snap_dirname, bool force_recovery,
uint64_t tuple_arena_max_size,
uint32_t objsize_min, bool dontdump,
- float alloc_factor);
+ const char *allocator, float alloc_factor);
int
memtx_engine_recover_snapshot(struct memtx_engine *memtx,
@@ -316,13 +316,13 @@ static inline struct memtx_engine *
memtx_engine_new_xc(const char *snap_dirname, bool force_recovery,
uint64_t tuple_arena_max_size,
uint32_t objsize_min, bool dontdump,
- float alloc_factor)
+ const char *allocator, float alloc_factor)
{
struct memtx_engine *memtx;
memtx = memtx_engine_new(snap_dirname, force_recovery,
tuple_arena_max_size,
objsize_min, dontdump,
- alloc_factor);
+ allocator, alloc_factor);
if (memtx == NULL)
diag_raise();
return memtx;
diff --git a/test/app-tap/init_script.result b/test/app-tap/init_script.result
index 16c5b01d2..cd5218e61 100644
--- a/test/app-tap/init_script.result
+++ b/test/app-tap/init_script.result
@@ -3,6 +3,7 @@
--
box.cfg
+allocator:small
background:false
checkpoint_count:2
checkpoint_interval:3600
diff --git a/test/box/admin.result b/test/box/admin.result
index 05debe673..ecea53957 100644
--- a/test/box/admin.result
+++ b/test/box/admin.result
@@ -27,7 +27,9 @@ help()
...
cfg_filter(box.cfg)
---
-- - - background
+- - - allocator
+ - small
+ - - background
- false
- - checkpoint_count
- 2
diff --git a/test/box/cfg.result b/test/box/cfg.result
index 22a720c2c..16b321008 100644
--- a/test/box/cfg.result
+++ b/test/box/cfg.result
@@ -15,7 +15,9 @@ box.cfg.nosuchoption = 1
| ...
cfg_filter(box.cfg)
| ---
- | - - - background
+ | - - - allocator
+ | - small
+ | - - background
| - false
| - - checkpoint_count
| - 2
@@ -130,7 +132,9 @@ box.cfg()
| ...
cfg_filter(box.cfg)
| ---
- | - - - background
+ | - - - allocator
+ | - small
+ | - - background
| - false
| - - checkpoint_count
| - 2
diff --git a/test/box/choose_memtx_allocator.lua b/test/box/choose_memtx_allocator.lua
new file mode 100644
index 000000000..2c173d9c5
--- /dev/null
+++ b/test/box/choose_memtx_allocator.lua
@@ -0,0 +1,8 @@
+#!/usr/bin/env tarantool
+
+require('console').listen(os.getenv('ADMIN'))
+
+box.cfg({
+ listen = os.getenv("LISTEN"),
+ allocator=arg[1],
+})
diff --git a/test/box/choose_memtx_allocator.result b/test/box/choose_memtx_allocator.result
new file mode 100644
index 000000000..0af720a4e
--- /dev/null
+++ b/test/box/choose_memtx_allocator.result
@@ -0,0 +1,147 @@
+-- test-run result file version 2
+
+-- write data recover from latest snapshot
+env = require('test_run')
+ | ---
+ | ...
+test_run = env.new()
+ | ---
+ | ...
+test_run:cmd('create server test with script="box/choose_memtx_allocator.lua"')
+ | ---
+ | - true
+ | ...
+--test small allocator
+test_run:cmd('start server test with args="small"')
+ | ---
+ | - true
+ | ...
+test_run:cmd('switch test')
+ | ---
+ | - true
+ | ...
+space = box.schema.space.create('test')
+ | ---
+ | ...
+space:format({ {name = 'id', type = 'unsigned'}, {name = 'year', type = 'unsigned'} })
+ | ---
+ | ...
+s = space:create_index('primary', { parts = {'id'} })
+ | ---
+ | ...
+for key = 1, 1000 do space:insert({key, key + 1000}) end
+ | ---
+ | ...
+for key = 1, 1000 do space:replace({key, key + 5000}) end
+ | ---
+ | ...
+box.snapshot()
+ | ---
+ | - ok
+ | ...
+for key = 1, 1000 do space:delete(key) end
+ | ---
+ | ...
+space:drop()
+ | ---
+ | ...
+test_run:cmd('switch default')
+ | ---
+ | - true
+ | ...
+test_run:cmd('stop server test')
+ | ---
+ | - true
+ | ...
+--test system(malloc) allocator
+test_run:cmd('start server test with args="system"')
+ | ---
+ | - true
+ | ...
+test_run:cmd('switch test')
+ | ---
+ | - true
+ | ...
+space = box.schema.space.create('test')
+ | ---
+ | ...
+space:format({ {name = 'id', type = 'unsigned'}, {name = 'year', type = 'unsigned'} })
+ | ---
+ | ...
+s = space:create_index('primary', { parts = {'id'} })
+ | ---
+ | ...
+for key = 1, 10000 do space:insert({key, key + 1000}) end
+ | ---
+ | ...
+box.snapshot()
+ | ---
+ | - ok
+ | ...
+for key = 1, 10000 do space:replace({key, key + 5000}) end
+ | ---
+ | ...
+box.snapshot()
+ | ---
+ | - ok
+ | ...
+for key = 1, 10000 do space:delete(key) end
+ | ---
+ | ...
+space:drop()
+ | ---
+ | ...
+test_run:cmd('switch default')
+ | ---
+ | - true
+ | ...
+test_run:cmd('stop server test')
+ | ---
+ | - true
+ | ...
+--test default (small) allocator
+test_run:cmd('start server test')
+ | ---
+ | - true
+ | ...
+test_run:cmd('switch test')
+ | ---
+ | - true
+ | ...
+space = box.schema.space.create('test')
+ | ---
+ | ...
+space:format({ {name = 'id', type = 'unsigned'}, {name = 'year', type = 'unsigned'} })
+ | ---
+ | ...
+s = space:create_index('primary', { parts = {'id'} })
+ | ---
+ | ...
+for key = 1, 1000 do space:insert({key, key + 1000}) end
+ | ---
+ | ...
+for key = 1, 1000 do space:replace({key, key + 5000}) end
+ | ---
+ | ...
+for key = 1, 1000 do space:delete(key) end
+ | ---
+ | ...
+space:drop()
+ | ---
+ | ...
+test_run:cmd('switch default')
+ | ---
+ | - true
+ | ...
+test_run:cmd('stop server test')
+ | ---
+ | - true
+ | ...
+test_run:cmd('cleanup server test')
+ | ---
+ | - true
+ | ...
+test_run:cmd('delete server test')
+ | ---
+ | - true
+ | ...
diff --git a/test/box/choose_memtx_allocator.test.lua b/test/box/choose_memtx_allocator.test.lua
new file mode 100644
index 000000000..678ff58d9
--- /dev/null
+++ b/test/box/choose_memtx_allocator.test.lua
@@ -0,0 +1,46 @@
+
+-- write data recover from latest snapshot
+env = require('test_run')
+test_run = env.new()
+test_run:cmd('create server test with script="box/choose_memtx_allocator.lua"')
+--test small allocator
+test_run:cmd('start server test with args="small"')
+test_run:cmd('switch test')
+space = box.schema.space.create('test')
+space:format({ {name = 'id', type = 'unsigned'}, {name = 'year', type = 'unsigned'} })
+s = space:create_index('primary', { parts = {'id'} })
+for key = 1, 1000 do space:insert({key, key + 1000}) end
+for key = 1, 1000 do space:replace({key, key + 5000}) end
+box.snapshot()
+for key = 1, 1000 do space:delete(key) end
+space:drop()
+test_run:cmd('switch default')
+test_run:cmd('stop server test')
+--test system(malloc) allocator
+test_run:cmd('start server test with args="system"')
+test_run:cmd('switch test')
+space = box.schema.space.create('test')
+space:format({ {name = 'id', type = 'unsigned'}, {name = 'year', type = 'unsigned'} })
+s = space:create_index('primary', { parts = {'id'} })
+for key = 1, 10000 do space:insert({key, key + 1000}) end
+box.snapshot()
+for key = 1, 10000 do space:replace({key, key + 5000}) end
+box.snapshot()
+for key = 1, 10000 do space:delete(key) end
+space:drop()
+test_run:cmd('switch default')
+test_run:cmd('stop server test')
+--test default (small) allocator
+test_run:cmd('start server test')
+test_run:cmd('switch test')
+space = box.schema.space.create('test')
+space:format({ {name = 'id', type = 'unsigned'}, {name = 'year', type = 'unsigned'} })
+s = space:create_index('primary', { parts = {'id'} })
+for key = 1, 1000 do space:insert({key, key + 1000}) end
+for key = 1, 1000 do space:replace({key, key + 5000}) end
+for key = 1, 1000 do space:delete(key) end
+space:drop()
+test_run:cmd('switch default')
+test_run:cmd('stop server test')
+test_run:cmd('cleanup server test')
+test_run:cmd('delete server test')
--
2.20.1
prev parent reply other threads:[~2021-01-20 7:56 UTC|newest]
Thread overview: 6+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-01-20 7:53 [Tarantool-patches] [PATCH v2 0/5] Choose allocator for memtx mechanik20051988 via Tarantool-patches
2021-01-20 7:53 ` [Tarantool-patches] [PATCH v2 1/5] test: add performance test for memtx allocator mechanik20051988 via Tarantool-patches
2021-01-20 7:53 ` [Tarantool-patches] [PATCH v2 2/5] memtx: changed some memtx files from .c to .cc mechanik20051988 via Tarantool-patches
2021-01-20 7:53 ` [Tarantool-patches] [PATCH v2 3/5] memtx: implement template tuple allocation mechanik20051988 via Tarantool-patches
2021-01-20 7:53 ` [Tarantool-patches] [PATCH v2 4/5] Implement system allocator, based on malloc mechanik20051988 via Tarantool-patches
2021-01-20 7:53 ` mechanik20051988 via Tarantool-patches [this message]
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=d14adeb3f353fa329775386035c6d22e84e3f2b3.1611073501.git.mechanik20051988@tarantool.org \
--to=tarantool-patches@dev.tarantool.org \
--cc=alyapunov@tarantool.org \
--cc=mechanik20051988@tarantool.org \
--cc=v.shpilevoy@tarantool.org \
--subject='Re: [Tarantool-patches] [PATCH v2 5/5] memtx: implement api for memory allocator selection' \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox