From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from [87.239.111.99] (localhost [127.0.0.1]) by dev.tarantool.org (Postfix) with ESMTP id 124DDBD3F8; Wed, 20 Jan 2021 10:56:41 +0300 (MSK) DKIM-Filter: OpenDKIM Filter v2.11.0 dev.tarantool.org 124DDBD3F8 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=tarantool.org; s=dev; t=1611129401; bh=hZItlgCE8+HXD2cF7JqDysCUv0J/Y6jZU9aDcRY44fk=; h=To:Date:In-Reply-To:References:Subject:List-Id:List-Unsubscribe: List-Archive:List-Post:List-Help:List-Subscribe:From:Reply-To:Cc: From; b=aReMJrLUBFdU7BhdbaRsFdVk3lo0OYXsCUIAhYXM7r9vKak+N92hZMO6Ekpgn15aC F0WM4S/KlZjv4Z505tG2N5QVOjRM9pzF0RF09o+3XpapVqqLvthbG/QqEcJ256l1wh j35aRLbjOAFmn1XTepFgFUC6d0tnCq0Ugnkv5QQY= Received: from smtp57.i.mail.ru (smtp57.i.mail.ru [217.69.128.37]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by dev.tarantool.org (Postfix) with ESMTPS id 43391BD3F8 for ; Wed, 20 Jan 2021 10:54:01 +0300 (MSK) DKIM-Filter: OpenDKIM Filter v2.11.0 dev.tarantool.org 43391BD3F8 Received: by smtp57.i.mail.ru with esmtpa (envelope-from ) id 1l28JU-00059t-8k; Wed, 20 Jan 2021 10:54:00 +0300 To: v.shpilevoy@tarantool.org, alyapunov@tarantool.org Date: Wed, 20 Jan 2021 10:53:43 +0300 Message-Id: X-Mailer: git-send-email 2.20.1 In-Reply-To: References: MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-7564579A: B8F34718100C35BD X-77F55803: 4F1203BC0FB41BD9D0E79FBC973162CD9B382B274FF537389656BC690FF7BBE200894C459B0CD1B969CB68AC38D63357CA017B7F43160A8F5A6B4612E3BC219B8DC7380D2C131FBD X-7FA49CB5: FF5795518A3D127A4AD6D5ED66289B5278DA827A17800CE77784844F620517D7EA1F7E6F0F101C67BD4B6F7A4D31EC0BCC500DACC3FED6E28638F802B75D45FF8AA50765F7900637B6346D94A32A1DEB8638F802B75D45FF5571747095F342E8C7A0BC55FA0FE5FC8504C28B02F6CFC22B14108C67B2298C663FAE35B2F2E9E9389733CBF5DBD5E913377AFFFEAFD269A417C69337E82CC2CC7F00164DA146DAFE8445B8C89999729449624AB7ADAF37F6B57BC7E64490611E7FA7ABCAF51C92A417C69337E82CC2CC7F00164DA146DA6F5DAA56C3B73B237318B6A418E8EAB8D32BA5DBAC0009BE9E8FC8737B5C22497257B14DC1D4E3CB76E601842F6C81A12EF20D2F80756B5F7E9C4E3C761E06A776E601842F6C81A127C277FBC8AE2E8BFF4145385FF5C9503AA81AA40904B5D9DBF02ECDB25306B2B25CBF701D1BE8734AD6D5ED66289B5278DA827A17800CE746D285266DF1BB7967F23339F89546C5A8DF7F3B2552694A6FED454B719173D6725E5C173C3A84C3809F89741C6E521635872C767BF85DA2F004C906525384306FED454B719173D6462275124DF8B9C923A316D17DDAC330E5BFE6E7EFDEDCD789D4C264860C145E X-B7AD71C0: 14C14B24D00AF5AC321EF223B8115265C77432E3705FE0C78E1CD14B953EB46D302B20359BD17239355D89D7DBCDD132 X-C1DE0DAB: C20DE7B7AB408E4181F030C43753B8186998911F362727C414F749A5E30D975C2549B6A8B600A67AC69A4F5309F1D0D73FBC941C333CAF5F9C2B6934AE262D3EE7EAB7254005DCED7532B743992DF240BDC6A1CF3F042BAD6DF99611D93F60EF6EA1BA7CA28B4A74699F904B3F4130E343918A1A30D5E7FCCB5012B2E24CD356 X-C8649E89: 4E36BF7865823D7055A7F0CF078B5EC49A30900B95165D3455AC8BF8E3153BA0195F69AB0F58A5584745BBF0230821A7C847B79ED5DC3BB7C81FD96A0A4E41391D7E09C32AA3244C38AD1CAF1E7B380B4C2F4F725D2A418B259227199D06760A927AC6DF5659F194 X-D57D3AED: 3ZO7eAau8CL7WIMRKs4sN3D3tLDjz0dLbV79QFUyzQ2Ujvy7cMT6pYYqY16iZVKkSc3dCLJ7zSJH7+u4VD18S7Vl4ZUrpaVfd2+vE6kuoey4m4VkSEu530nj6fImhcD4MUrOEAnl0W826KZ9Q+tr5ycPtXkTV4k65bRjmOUUP8cvGozZ33TWg5HZplvhhXbhDGzqmQDTd6OAevLeAnq3Ra9uf7zvY2zzsIhlcp/Y7m53TZgf2aB4JOg4gkr2bioj7AvRt3Uvx5QWBU21xR7PfQ== X-Mailru-Sender: 583F1D7ACE8F49BD901CF26B10B5FEED962675491849D24057664B99FB73BEA6D68D8472E842F4D9B79567116EAC6FCF4E830D9205DBEA545646F0D3C63A617F27ACC94E9A535D22112434F685709FCF0DA7A0AF5A3A8387 X-Mras: Ok Subject: [Tarantool-patches] [PATCH v2 5/5] memtx: implement api for memory allocator selection X-BeenThere: tarantool-patches@dev.tarantool.org X-Mailman-Version: 2.1.34 Precedence: list List-Id: Tarantool development patches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , From: mechanik20051988 via Tarantool-patches Reply-To: mechanik20051988 Cc: tarantool-patches@dev.tarantool.org Errors-To: tarantool-patches-bounces@dev.tarantool.org Sender: "Tarantool-patches" Added an allocator option in the.cfg window for {}, which provides user the ability to select a memory allocator for memtx. The available option values is "system" and "small". The default value is "small". Closes #5419 @TarantoolBot document Title: Add new 'allocator' option to box.cfg{} Add new 'allocator' option to box.cfg{} which allows to select the appropriate allocator for memtx tuples if necessary. Possible values are "system" for malloc allocator and "small" for default small allocator. --- src/box/box.cc | 1 + src/box/lua/load_cfg.lua | 2 + src/box/memtx_engine.cc | 31 +++-- src/box/memtx_engine.h | 6 +- test/app-tap/init_script.result | 1 + test/box/admin.result | 4 +- test/box/cfg.result | 8 +- test/box/choose_memtx_allocator.lua | 8 ++ test/box/choose_memtx_allocator.result | 147 +++++++++++++++++++++++ test/box/choose_memtx_allocator.test.lua | 46 +++++++ 10 files changed, 240 insertions(+), 14 deletions(-) create mode 100644 test/box/choose_memtx_allocator.lua create mode 100644 test/box/choose_memtx_allocator.result create mode 100644 test/box/choose_memtx_allocator.test.lua diff --git a/src/box/box.cc b/src/box/box.cc index 0bf7c175e..7e1b9d207 100644 --- a/src/box/box.cc +++ b/src/box/box.cc @@ -2542,6 +2542,7 @@ engine_init() cfg_getd("memtx_memory"), cfg_geti("memtx_min_tuple_size"), cfg_geti("strip_core"), + cfg_gets("allocator"), cfg_getd("slab_alloc_factor")); engine_register((struct engine *)memtx); box_set_memtx_max_tuple_size(); diff --git a/src/box/lua/load_cfg.lua b/src/box/lua/load_cfg.lua index 574c8bef4..2fe8a5b6c 100644 --- a/src/box/lua/load_cfg.lua +++ b/src/box/lua/load_cfg.lua @@ -43,6 +43,7 @@ local default_cfg = { memtx_min_tuple_size = 16, memtx_max_tuple_size = 1024 * 1024, slab_alloc_factor = 1.05, + allocator = "small", work_dir = nil, memtx_dir = ".", wal_dir = ".", @@ -124,6 +125,7 @@ local template_cfg = { memtx_min_tuple_size = 'number', memtx_max_tuple_size = 'number', slab_alloc_factor = 'number', + allocator = 'string', work_dir = 'string', memtx_dir = 'string', wal_dir = 'string', diff --git a/src/box/memtx_engine.cc b/src/box/memtx_engine.cc index ad904889a..285b49655 100644 --- a/src/box/memtx_engine.cc +++ b/src/box/memtx_engine.cc @@ -1092,7 +1092,7 @@ memtx_engine_gc_f(va_list va) struct memtx_engine * memtx_engine_new(const char *snap_dirname, bool force_recovery, uint64_t tuple_arena_max_size, uint32_t objsize_min, - bool dontdump, float alloc_factor) + bool dontdump, const char *allocator, float alloc_factor) { int64_t snap_signature; struct memtx_engine *memtx = (struct memtx_engine *)calloc(1, sizeof(*memtx)); @@ -1102,13 +1102,28 @@ memtx_engine_new(const char *snap_dirname, bool force_recovery, return NULL; } - memtx->allocator_type = MEMTX_SMALL_ALLOCATOR; - MEMXT_TUPLE_FORMAT_VTAB(SmallAllocator) - memtx_engine_vtab.memory_stat = - memtx_engine_memory_stat; + assert(allocator != NULL); + if (!strcmp(allocator, "small")) { + memtx->allocator_type = MEMTX_SMALL_ALLOCATOR; + MEMXT_TUPLE_FORMAT_VTAB(SmallAllocator) + memtx_engine_vtab.memory_stat = + memtx_engine_memory_stat; + } else if (!strcmp(allocator, "system")) { + memtx->allocator_type = MEMTX_SYSTEM_ALLOCATOR; + MEMXT_TUPLE_FORMAT_VTAB(SystemAllocator) + memtx_engine_vtab.memory_stat = + memtx_engine_memory_stat; + } else { + diag_set(IllegalParams, "Invalid memory allocator name"); + free(memtx); + return NULL; + } xdir_create(&memtx->snap_dir, snap_dirname, SNAP, &INSTANCE_UUID, &xlog_opts_default); diff --git a/src/box/memtx_engine.h b/src/box/memtx_engine.h index ba2394462..6edb8b373 100644 --- a/src/box/memtx_engine.h +++ b/src/box/memtx_engine.h @@ -218,7 +218,7 @@ struct memtx_engine * memtx_engine_new(const char *snap_dirname, bool force_recovery, uint64_t tuple_arena_max_size, uint32_t objsize_min, bool dontdump, - float alloc_factor); + const char *allocator, float alloc_factor); int memtx_engine_recover_snapshot(struct memtx_engine *memtx, @@ -316,13 +316,13 @@ static inline struct memtx_engine * memtx_engine_new_xc(const char *snap_dirname, bool force_recovery, uint64_t tuple_arena_max_size, uint32_t objsize_min, bool dontdump, - float alloc_factor) + const char *allocator, float alloc_factor) { struct memtx_engine *memtx; memtx = memtx_engine_new(snap_dirname, force_recovery, tuple_arena_max_size, objsize_min, dontdump, - alloc_factor); + allocator, alloc_factor); if (memtx == NULL) diag_raise(); return memtx; diff --git a/test/app-tap/init_script.result b/test/app-tap/init_script.result index 16c5b01d2..cd5218e61 100644 --- a/test/app-tap/init_script.result +++ b/test/app-tap/init_script.result @@ -3,6 +3,7 @@ -- box.cfg +allocator:small background:false checkpoint_count:2 checkpoint_interval:3600 diff --git a/test/box/admin.result b/test/box/admin.result index 05debe673..ecea53957 100644 --- a/test/box/admin.result +++ b/test/box/admin.result @@ -27,7 +27,9 @@ help() ... cfg_filter(box.cfg) --- -- - - background +- - - allocator + - small + - - background - false - - checkpoint_count - 2 diff --git a/test/box/cfg.result b/test/box/cfg.result index 22a720c2c..16b321008 100644 --- a/test/box/cfg.result +++ b/test/box/cfg.result @@ -15,7 +15,9 @@ box.cfg.nosuchoption = 1 | ... cfg_filter(box.cfg) | --- - | - - - background + | - - - allocator + | - small + | - - background | - false | - - checkpoint_count | - 2 @@ -130,7 +132,9 @@ box.cfg() | ... cfg_filter(box.cfg) | --- - | - - - background + | - - - allocator + | - small + | - - background | - false | - - checkpoint_count | - 2 diff --git a/test/box/choose_memtx_allocator.lua b/test/box/choose_memtx_allocator.lua new file mode 100644 index 000000000..2c173d9c5 --- /dev/null +++ b/test/box/choose_memtx_allocator.lua @@ -0,0 +1,8 @@ +#!/usr/bin/env tarantool + +require('console').listen(os.getenv('ADMIN')) + +box.cfg({ + listen = os.getenv("LISTEN"), + allocator=arg[1], +}) diff --git a/test/box/choose_memtx_allocator.result b/test/box/choose_memtx_allocator.result new file mode 100644 index 000000000..0af720a4e --- /dev/null +++ b/test/box/choose_memtx_allocator.result @@ -0,0 +1,147 @@ +-- test-run result file version 2 + +-- write data recover from latest snapshot +env = require('test_run') + | --- + | ... +test_run = env.new() + | --- + | ... +test_run:cmd('create server test with script="box/choose_memtx_allocator.lua"') + | --- + | - true + | ... +--test small allocator +test_run:cmd('start server test with args="small"') + | --- + | - true + | ... +test_run:cmd('switch test') + | --- + | - true + | ... +space = box.schema.space.create('test') + | --- + | ... +space:format({ {name = 'id', type = 'unsigned'}, {name = 'year', type = 'unsigned'} }) + | --- + | ... +s = space:create_index('primary', { parts = {'id'} }) + | --- + | ... +for key = 1, 1000 do space:insert({key, key + 1000}) end + | --- + | ... +for key = 1, 1000 do space:replace({key, key + 5000}) end + | --- + | ... +box.snapshot() + | --- + | - ok + | ... +for key = 1, 1000 do space:delete(key) end + | --- + | ... +space:drop() + | --- + | ... +test_run:cmd('switch default') + | --- + | - true + | ... +test_run:cmd('stop server test') + | --- + | - true + | ... +--test system(malloc) allocator +test_run:cmd('start server test with args="system"') + | --- + | - true + | ... +test_run:cmd('switch test') + | --- + | - true + | ... +space = box.schema.space.create('test') + | --- + | ... +space:format({ {name = 'id', type = 'unsigned'}, {name = 'year', type = 'unsigned'} }) + | --- + | ... +s = space:create_index('primary', { parts = {'id'} }) + | --- + | ... +for key = 1, 10000 do space:insert({key, key + 1000}) end + | --- + | ... +box.snapshot() + | --- + | - ok + | ... +for key = 1, 10000 do space:replace({key, key + 5000}) end + | --- + | ... +box.snapshot() + | --- + | - ok + | ... +for key = 1, 10000 do space:delete(key) end + | --- + | ... +space:drop() + | --- + | ... +test_run:cmd('switch default') + | --- + | - true + | ... +test_run:cmd('stop server test') + | --- + | - true + | ... +--test default (small) allocator +test_run:cmd('start server test') + | --- + | - true + | ... +test_run:cmd('switch test') + | --- + | - true + | ... +space = box.schema.space.create('test') + | --- + | ... +space:format({ {name = 'id', type = 'unsigned'}, {name = 'year', type = 'unsigned'} }) + | --- + | ... +s = space:create_index('primary', { parts = {'id'} }) + | --- + | ... +for key = 1, 1000 do space:insert({key, key + 1000}) end + | --- + | ... +for key = 1, 1000 do space:replace({key, key + 5000}) end + | --- + | ... +for key = 1, 1000 do space:delete(key) end + | --- + | ... +space:drop() + | --- + | ... +test_run:cmd('switch default') + | --- + | - true + | ... +test_run:cmd('stop server test') + | --- + | - true + | ... +test_run:cmd('cleanup server test') + | --- + | - true + | ... +test_run:cmd('delete server test') + | --- + | - true + | ... diff --git a/test/box/choose_memtx_allocator.test.lua b/test/box/choose_memtx_allocator.test.lua new file mode 100644 index 000000000..678ff58d9 --- /dev/null +++ b/test/box/choose_memtx_allocator.test.lua @@ -0,0 +1,46 @@ + +-- write data recover from latest snapshot +env = require('test_run') +test_run = env.new() +test_run:cmd('create server test with script="box/choose_memtx_allocator.lua"') +--test small allocator +test_run:cmd('start server test with args="small"') +test_run:cmd('switch test') +space = box.schema.space.create('test') +space:format({ {name = 'id', type = 'unsigned'}, {name = 'year', type = 'unsigned'} }) +s = space:create_index('primary', { parts = {'id'} }) +for key = 1, 1000 do space:insert({key, key + 1000}) end +for key = 1, 1000 do space:replace({key, key + 5000}) end +box.snapshot() +for key = 1, 1000 do space:delete(key) end +space:drop() +test_run:cmd('switch default') +test_run:cmd('stop server test') +--test system(malloc) allocator +test_run:cmd('start server test with args="system"') +test_run:cmd('switch test') +space = box.schema.space.create('test') +space:format({ {name = 'id', type = 'unsigned'}, {name = 'year', type = 'unsigned'} }) +s = space:create_index('primary', { parts = {'id'} }) +for key = 1, 10000 do space:insert({key, key + 1000}) end +box.snapshot() +for key = 1, 10000 do space:replace({key, key + 5000}) end +box.snapshot() +for key = 1, 10000 do space:delete(key) end +space:drop() +test_run:cmd('switch default') +test_run:cmd('stop server test') +--test default (small) allocator +test_run:cmd('start server test') +test_run:cmd('switch test') +space = box.schema.space.create('test') +space:format({ {name = 'id', type = 'unsigned'}, {name = 'year', type = 'unsigned'} }) +s = space:create_index('primary', { parts = {'id'} }) +for key = 1, 1000 do space:insert({key, key + 1000}) end +for key = 1, 1000 do space:replace({key, key + 5000}) end +for key = 1, 1000 do space:delete(key) end +space:drop() +test_run:cmd('switch default') +test_run:cmd('stop server test') +test_run:cmd('cleanup server test') +test_run:cmd('delete server test') -- 2.20.1