[Tarantool-patches] [PATCH v2 5/5] memtx: implement api for memory allocator selection

mechanik20051988 mechanik20051988 at tarantool.org
Wed Jan 20 10:53:43 MSK 2021


Added an allocator option in the.cfg window for {}, which provides
user the ability to select a memory allocator for memtx.
The available option values is "system" and "small". The default value
is "small".

Closes #5419

@TarantoolBot document
Title: Add new 'allocator' option to box.cfg{}
Add new 'allocator' option to box.cfg{} which allows to
select the appropriate allocator for memtx tuples if necessary.
Possible values are "system" for malloc allocator and
"small" for default small allocator.
---
 src/box/box.cc                           |   1 +
 src/box/lua/load_cfg.lua                 |   2 +
 src/box/memtx_engine.cc                  |  31 +++--
 src/box/memtx_engine.h                   |   6 +-
 test/app-tap/init_script.result          |   1 +
 test/box/admin.result                    |   4 +-
 test/box/cfg.result                      |   8 +-
 test/box/choose_memtx_allocator.lua      |   8 ++
 test/box/choose_memtx_allocator.result   | 147 +++++++++++++++++++++++
 test/box/choose_memtx_allocator.test.lua |  46 +++++++
 10 files changed, 240 insertions(+), 14 deletions(-)
 create mode 100644 test/box/choose_memtx_allocator.lua
 create mode 100644 test/box/choose_memtx_allocator.result
 create mode 100644 test/box/choose_memtx_allocator.test.lua

diff --git a/src/box/box.cc b/src/box/box.cc
index 0bf7c175e..7e1b9d207 100644
--- a/src/box/box.cc
+++ b/src/box/box.cc
@@ -2542,6 +2542,7 @@ engine_init()
 				    cfg_getd("memtx_memory"),
 				    cfg_geti("memtx_min_tuple_size"),
 				    cfg_geti("strip_core"),
+				    cfg_gets("allocator"),
 				    cfg_getd("slab_alloc_factor"));
 	engine_register((struct engine *)memtx);
 	box_set_memtx_max_tuple_size();
diff --git a/src/box/lua/load_cfg.lua b/src/box/lua/load_cfg.lua
index 574c8bef4..2fe8a5b6c 100644
--- a/src/box/lua/load_cfg.lua
+++ b/src/box/lua/load_cfg.lua
@@ -43,6 +43,7 @@ local default_cfg = {
     memtx_min_tuple_size = 16,
     memtx_max_tuple_size = 1024 * 1024,
     slab_alloc_factor   = 1.05,
+    allocator           = "small",
     work_dir            = nil,
     memtx_dir           = ".",
     wal_dir             = ".",
@@ -124,6 +125,7 @@ local template_cfg = {
     memtx_min_tuple_size  = 'number',
     memtx_max_tuple_size  = 'number',
     slab_alloc_factor   = 'number',
+    allocator           = 'string',
     work_dir            = 'string',
     memtx_dir            = 'string',
     wal_dir             = 'string',
diff --git a/src/box/memtx_engine.cc b/src/box/memtx_engine.cc
index ad904889a..285b49655 100644
--- a/src/box/memtx_engine.cc
+++ b/src/box/memtx_engine.cc
@@ -1092,7 +1092,7 @@ memtx_engine_gc_f(va_list va)
 struct memtx_engine *
 memtx_engine_new(const char *snap_dirname, bool force_recovery,
 		 uint64_t tuple_arena_max_size, uint32_t objsize_min,
-		 bool dontdump, float alloc_factor)
+		 bool dontdump, const char *allocator, float alloc_factor)
 {
 	int64_t snap_signature;
 	struct memtx_engine *memtx = (struct memtx_engine *)calloc(1, sizeof(*memtx));
@@ -1102,13 +1102,28 @@ memtx_engine_new(const char *snap_dirname, bool force_recovery,
 		return NULL;
 	}
 
-	memtx->allocator_type = MEMTX_SMALL_ALLOCATOR;
-	MEMXT_TUPLE_FORMAT_VTAB(SmallAllocator)
-	memtx_engine_vtab.memory_stat =
-		memtx_engine_memory_stat<struct small_stats,
-					 struct mempool_stats,
-					 SmallAllocator,
-					 small_stats_noop_cb>;
+	assert(allocator != NULL);
+	if (!strcmp(allocator, "small")) {
+		memtx->allocator_type = MEMTX_SMALL_ALLOCATOR;
+		MEMXT_TUPLE_FORMAT_VTAB(SmallAllocator)
+		memtx_engine_vtab.memory_stat =
+			memtx_engine_memory_stat<struct small_stats,
+						 struct mempool_stats,
+						 SmallAllocator,
+						 small_stats_noop_cb>;
+	} else if (!strcmp(allocator, "system")) {
+		memtx->allocator_type = MEMTX_SYSTEM_ALLOCATOR;
+		MEMXT_TUPLE_FORMAT_VTAB(SystemAllocator)
+		memtx_engine_vtab.memory_stat =
+			memtx_engine_memory_stat<struct system_stats,
+						 struct system_stats,
+						 SystemAllocator,
+						 system_stats_noop_cb>;
+	} else {
+		diag_set(IllegalParams, "Invalid memory allocator name");
+		free(memtx);
+		return NULL;
+	}
 
 	xdir_create(&memtx->snap_dir, snap_dirname, SNAP, &INSTANCE_UUID,
 		    &xlog_opts_default);
diff --git a/src/box/memtx_engine.h b/src/box/memtx_engine.h
index ba2394462..6edb8b373 100644
--- a/src/box/memtx_engine.h
+++ b/src/box/memtx_engine.h
@@ -218,7 +218,7 @@ struct memtx_engine *
 memtx_engine_new(const char *snap_dirname, bool force_recovery,
 		 uint64_t tuple_arena_max_size,
 		 uint32_t objsize_min, bool dontdump,
-		 float alloc_factor);
+		 const char *allocator, float alloc_factor);
 
 int
 memtx_engine_recover_snapshot(struct memtx_engine *memtx,
@@ -316,13 +316,13 @@ static inline struct memtx_engine *
 memtx_engine_new_xc(const char *snap_dirname, bool force_recovery,
 		    uint64_t tuple_arena_max_size,
 		    uint32_t objsize_min, bool dontdump,
-		    float alloc_factor)
+		    const char *allocator, float alloc_factor)
 {
 	struct memtx_engine *memtx;
 	memtx = memtx_engine_new(snap_dirname, force_recovery,
 				 tuple_arena_max_size,
 				 objsize_min, dontdump,
-				 alloc_factor);
+				 allocator, alloc_factor);
 	if (memtx == NULL)
 		diag_raise();
 	return memtx;
diff --git a/test/app-tap/init_script.result b/test/app-tap/init_script.result
index 16c5b01d2..cd5218e61 100644
--- a/test/app-tap/init_script.result
+++ b/test/app-tap/init_script.result
@@ -3,6 +3,7 @@
 --
 
 box.cfg
+allocator:small
 background:false
 checkpoint_count:2
 checkpoint_interval:3600
diff --git a/test/box/admin.result b/test/box/admin.result
index 05debe673..ecea53957 100644
--- a/test/box/admin.result
+++ b/test/box/admin.result
@@ -27,7 +27,9 @@ help()
 ...
 cfg_filter(box.cfg)
 ---
-- - - background
+- - - allocator
+    - small
+  - - background
     - false
   - - checkpoint_count
     - 2
diff --git a/test/box/cfg.result b/test/box/cfg.result
index 22a720c2c..16b321008 100644
--- a/test/box/cfg.result
+++ b/test/box/cfg.result
@@ -15,7 +15,9 @@ box.cfg.nosuchoption = 1
  | ...
 cfg_filter(box.cfg)
  | ---
- | - - - background
+ | - - - allocator
+ |     - small
+ |   - - background
  |     - false
  |   - - checkpoint_count
  |     - 2
@@ -130,7 +132,9 @@ box.cfg()
  | ...
 cfg_filter(box.cfg)
  | ---
- | - - - background
+ | - - - allocator
+ |     - small
+ |   - - background
  |     - false
  |   - - checkpoint_count
  |     - 2
diff --git a/test/box/choose_memtx_allocator.lua b/test/box/choose_memtx_allocator.lua
new file mode 100644
index 000000000..2c173d9c5
--- /dev/null
+++ b/test/box/choose_memtx_allocator.lua
@@ -0,0 +1,8 @@
+#!/usr/bin/env tarantool
+
+require('console').listen(os.getenv('ADMIN'))
+
+box.cfg({
+    listen = os.getenv("LISTEN"),
+    allocator=arg[1],
+})
diff --git a/test/box/choose_memtx_allocator.result b/test/box/choose_memtx_allocator.result
new file mode 100644
index 000000000..0af720a4e
--- /dev/null
+++ b/test/box/choose_memtx_allocator.result
@@ -0,0 +1,147 @@
+-- test-run result file version 2
+
+-- write data recover from latest snapshot
+env = require('test_run')
+ | ---
+ | ...
+test_run = env.new()
+ | ---
+ | ...
+test_run:cmd('create server test with script="box/choose_memtx_allocator.lua"')
+ | ---
+ | - true
+ | ...
+--test small allocator
+test_run:cmd('start server test with args="small"')
+ | ---
+ | - true
+ | ...
+test_run:cmd('switch test')
+ | ---
+ | - true
+ | ...
+space = box.schema.space.create('test')
+ | ---
+ | ...
+space:format({ {name = 'id', type = 'unsigned'}, {name = 'year', type = 'unsigned'} })
+ | ---
+ | ...
+s = space:create_index('primary', { parts = {'id'} })
+ | ---
+ | ...
+for key = 1, 1000 do space:insert({key, key + 1000}) end
+ | ---
+ | ...
+for key = 1, 1000 do space:replace({key, key + 5000}) end
+ | ---
+ | ...
+box.snapshot()
+ | ---
+ | - ok
+ | ...
+for key = 1, 1000 do space:delete(key) end
+ | ---
+ | ...
+space:drop()
+ | ---
+ | ...
+test_run:cmd('switch default')
+ | ---
+ | - true
+ | ...
+test_run:cmd('stop server test')
+ | ---
+ | - true
+ | ...
+--test system(malloc) allocator
+test_run:cmd('start server test with args="system"')
+ | ---
+ | - true
+ | ...
+test_run:cmd('switch test')
+ | ---
+ | - true
+ | ...
+space = box.schema.space.create('test')
+ | ---
+ | ...
+space:format({ {name = 'id', type = 'unsigned'}, {name = 'year', type = 'unsigned'} })
+ | ---
+ | ...
+s = space:create_index('primary', { parts = {'id'} })
+ | ---
+ | ...
+for key = 1, 10000 do space:insert({key, key + 1000}) end
+ | ---
+ | ...
+box.snapshot()
+ | ---
+ | - ok
+ | ...
+for key = 1, 10000 do space:replace({key, key + 5000}) end
+ | ---
+ | ...
+box.snapshot()
+ | ---
+ | - ok
+ | ...
+for key = 1, 10000 do space:delete(key) end
+ | ---
+ | ...
+space:drop()
+ | ---
+ | ...
+test_run:cmd('switch default')
+ | ---
+ | - true
+ | ...
+test_run:cmd('stop server test')
+ | ---
+ | - true
+ | ...
+--test default (small) allocator
+test_run:cmd('start server test')
+ | ---
+ | - true
+ | ...
+test_run:cmd('switch test')
+ | ---
+ | - true
+ | ...
+space = box.schema.space.create('test')
+ | ---
+ | ...
+space:format({ {name = 'id', type = 'unsigned'}, {name = 'year', type = 'unsigned'} })
+ | ---
+ | ...
+s = space:create_index('primary', { parts = {'id'} })
+ | ---
+ | ...
+for key = 1, 1000 do space:insert({key, key + 1000}) end
+ | ---
+ | ...
+for key = 1, 1000 do space:replace({key, key + 5000}) end
+ | ---
+ | ...
+for key = 1, 1000 do space:delete(key) end
+ | ---
+ | ...
+space:drop()
+ | ---
+ | ...
+test_run:cmd('switch default')
+ | ---
+ | - true
+ | ...
+test_run:cmd('stop server test')
+ | ---
+ | - true
+ | ...
+test_run:cmd('cleanup server test')
+ | ---
+ | - true
+ | ...
+test_run:cmd('delete server test')
+ | ---
+ | - true
+ | ...
diff --git a/test/box/choose_memtx_allocator.test.lua b/test/box/choose_memtx_allocator.test.lua
new file mode 100644
index 000000000..678ff58d9
--- /dev/null
+++ b/test/box/choose_memtx_allocator.test.lua
@@ -0,0 +1,46 @@
+
+-- write data recover from latest snapshot
+env = require('test_run')
+test_run = env.new()
+test_run:cmd('create server test with script="box/choose_memtx_allocator.lua"')
+--test small allocator
+test_run:cmd('start server test with args="small"')
+test_run:cmd('switch test')
+space = box.schema.space.create('test')
+space:format({ {name = 'id', type = 'unsigned'}, {name = 'year', type = 'unsigned'} })
+s = space:create_index('primary', { parts = {'id'} })
+for key = 1, 1000 do space:insert({key, key + 1000}) end
+for key = 1, 1000 do space:replace({key, key + 5000}) end
+box.snapshot()
+for key = 1, 1000 do space:delete(key) end
+space:drop()
+test_run:cmd('switch default')
+test_run:cmd('stop server test')
+--test system(malloc) allocator
+test_run:cmd('start server test with args="system"')
+test_run:cmd('switch test')
+space = box.schema.space.create('test')
+space:format({ {name = 'id', type = 'unsigned'}, {name = 'year', type = 'unsigned'} })
+s = space:create_index('primary', { parts = {'id'} })
+for key = 1, 10000 do space:insert({key, key + 1000}) end
+box.snapshot()
+for key = 1, 10000 do space:replace({key, key + 5000}) end
+box.snapshot()
+for key = 1, 10000 do space:delete(key) end
+space:drop()
+test_run:cmd('switch default')
+test_run:cmd('stop server test')
+--test default (small) allocator
+test_run:cmd('start server test')
+test_run:cmd('switch test')
+space = box.schema.space.create('test')
+space:format({ {name = 'id', type = 'unsigned'}, {name = 'year', type = 'unsigned'} })
+s = space:create_index('primary', { parts = {'id'} })
+for key = 1, 1000 do space:insert({key, key + 1000}) end
+for key = 1, 1000 do space:replace({key, key + 5000}) end
+for key = 1, 1000 do space:delete(key) end
+space:drop()
+test_run:cmd('switch default')
+test_run:cmd('stop server test')
+test_run:cmd('cleanup server test')
+test_run:cmd('delete server test')
-- 
2.20.1



More information about the Tarantool-patches mailing list