From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from smtpng3.m.smailru.net (smtpng3.m.smailru.net [94.100.177.149]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by dev.tarantool.org (Postfix) with ESMTPS id 657734696C9 for ; Fri, 14 Feb 2020 22:40:41 +0300 (MSK) From: Ilya Kosarev Date: Fri, 14 Feb 2020 22:40:33 +0300 Message-Id: <2df56691aa63211bf6f64e39711385806a11ba19.1581705010.git.i.kosarev@tarantool.org> In-Reply-To: References: In-Reply-To: References: Subject: [Tarantool-patches] [PATCH 3/4] memtx: use reserved slab for truncation List-Id: Tarantool development patches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , To: tarantool-patches@dev.tarantool.org Trying to perform space:truncate() while reaching memtx_memory limit we could experience slab allocator failure. This behavior seems to be quite surprising for users. Now we are using specifically preallocated on arena slab for truncation tuples in case we are running out of limit. --- src/box/box.cc | 27 ++++++- src/box/tuple.c | 4 ++ src/lib/core/memory.c | 1 + test/engine/engine.cfg | 3 + test/engine/low_memory.lua | 8 +++ test/engine/stress_truncate.result | 103 +++++++++++++++++++++++++++ test/engine/stress_truncate.test.lua | 52 ++++++++++++++ 7 files changed, 197 insertions(+), 1 deletion(-) create mode 100644 test/engine/low_memory.lua create mode 100644 test/engine/stress_truncate.result create mode 100644 test/engine/stress_truncate.test.lua diff --git a/src/box/box.cc b/src/box/box.cc index 1b2b27d61..eea4269e7 100644 --- a/src/box/box.cc +++ b/src/box/box.cc @@ -1321,9 +1321,34 @@ space_truncate(struct space *space) ops_buf_end = mp_encode_uint(ops_buf_end, 1); assert(ops_buf_end < buf + buf_size); + struct txn *txn = NULL; + struct txn_savepoint *txn_svp = NULL; + if (!box_txn()) { + txn = txn_begin(); + if (txn == NULL) + diag_raise(); + } else { + txn_svp = box_txn_savepoint(); + if (txn_svp == NULL) + diag_raise(); + } + struct space *truncate_space = space_cache_find_xc(BOX_TRUNCATE_ID); + ((struct memtx_engine *)truncate_space->engine)->arena.truncating = true; if (box_upsert(BOX_TRUNCATE_ID, 0, tuple_buf, tuple_buf_end, - ops_buf, ops_buf_end, 0, NULL) != 0) + ops_buf, ops_buf_end, 0, NULL) != 0) { + ((struct memtx_engine *)truncate_space->engine)->arena.truncating = false; + if (txn != NULL) + txn_rollback(txn); + else + box_txn_rollback_to_savepoint(txn_svp); + fiber_gc(); diag_raise(); + } + ((struct memtx_engine *)truncate_space->engine)->arena.truncating = false; + if (txn != NULL && txn_commit(txn) != 0) { + fiber_gc(); + diag_raise(); + } } int diff --git a/src/box/tuple.c b/src/box/tuple.c index 4d676b090..0e8aeb92e 100644 --- a/src/box/tuple.c +++ b/src/box/tuple.c @@ -342,6 +342,10 @@ tuple_arena_create(struct slab_arena *arena, struct quota *quota, say_info("mapping %zu bytes for %s tuple arena...", prealloc, arena_name); + if (!strncmp(arena_name, "memtx", 5)) + arena->trunc_alloc = slab_size; + else + arena->trunc_alloc = 0; if (slab_arena_create(arena, quota, prealloc, slab_size, flags) != 0) { if (errno == ENOMEM) { panic("failed to preallocate %zu bytes: Cannot "\ diff --git a/src/lib/core/memory.c b/src/lib/core/memory.c index f236c1887..0ce977d38 100644 --- a/src/lib/core/memory.c +++ b/src/lib/core/memory.c @@ -42,6 +42,7 @@ memory_init() quota_init(&runtime_quota, QUOTA_MAX); /* No limit on the runtime memory. */ + runtime.trunc_alloc = 0; slab_arena_create(&runtime, &runtime_quota, 0, SLAB_SIZE, SLAB_ARENA_PRIVATE); } diff --git a/test/engine/engine.cfg b/test/engine/engine.cfg index f1e7de274..4b9eeb5f5 100644 --- a/test/engine/engine.cfg +++ b/test/engine/engine.cfg @@ -5,6 +5,9 @@ }, "func_index.test.lua": { "memtx": {"engine": "memtx"} + }, + "stress_truncate.test.lua": { + "memtx": {"engine": "memtx"} } } diff --git a/test/engine/low_memory.lua b/test/engine/low_memory.lua new file mode 100644 index 000000000..46fd26d1b --- /dev/null +++ b/test/engine/low_memory.lua @@ -0,0 +1,8 @@ +#!/usr/bin/env tarantool +os = require('os') +box.cfg({ + listen = os.getenv("LISTEN"), + memtx_memory = 32 * 1024 * 1024, +}) + +require('console').listen(os.getenv('ADMIN')) diff --git a/test/engine/stress_truncate.result b/test/engine/stress_truncate.result new file mode 100644 index 000000000..c5308a0ac --- /dev/null +++ b/test/engine/stress_truncate.result @@ -0,0 +1,103 @@ +-- test-run result file version 2 +test_run = require('test_run').new() + | --- + | ... + + +test_run:cmd("create server master with script='engine/low_memory.lua'") + | --- + | - true + | ... +test_run:cmd('start server master') + | --- + | - true + | ... +test_run:cmd("switch master") + | --- + | - true + | ... + + +test_run:cmd("setopt delimiter ';'") + | --- + | - true + | ... +function create_space(name) + local space = box.schema.create_space(name) + space:format({ + { name = "id", type = "unsigned" }, + { name = "val", type = "str" } + }) + space:create_index('primary', { parts = { 'id' } }) + return space +end; + | --- + | ... + +function insert(space, i) + space:insert({ i, string.rep(string.char(32 + math.random(127-32)), math.random(1024)) }) +end; + | --- + | ... + +function fill_space(space, start) + local _, err = nil + local i = start + while err == nil do _, err = pcall(insert, space, i) i = i + 1 end +end; + | --- + | ... + +function stress_truncation(i, spaces) + local res, space = pcall(create_space, 'test' .. tostring(i)) + if res then spaces[i] = space return end + fill_space(box.space.test, box.space.test:len()) + for _, s in pairs(spaces) do s:truncate() end +end; + | --- + | ... +test_run:cmd("setopt delimiter ''"); + | --- + | - true + | ... + + +_ = create_space('test') + | --- + | ... +for i = 0, 27000 do insert(box.space.test, i) end + | --- + | ... + +spaces = {} + | --- + | ... +counter = 0 + | --- + | ... +status = true + | --- + | ... +res = nil + | --- + | ... +while counter < 80000 do status, res = pcall(stress_truncation, counter, spaces) counter = counter + 1 end + | --- + | ... +status + | --- + | - true + | ... +res + | --- + | - null + | ... + +-- Cleanup. +test_run:cmd('switch default') + | --- + | - true + | ... +test_run:drop_cluster({'master'}) + | --- + | ... diff --git a/test/engine/stress_truncate.test.lua b/test/engine/stress_truncate.test.lua new file mode 100644 index 000000000..761b2a9c2 --- /dev/null +++ b/test/engine/stress_truncate.test.lua @@ -0,0 +1,52 @@ +test_run = require('test_run').new() + + +test_run:cmd("create server master with script='engine/low_memory.lua'") +test_run:cmd('start server master') +test_run:cmd("switch master") + + +test_run:cmd("setopt delimiter ';'") +function create_space(name) + local space = box.schema.create_space(name) + space:format({ + { name = "id", type = "unsigned" }, + { name = "val", type = "str" } + }) + space:create_index('primary', { parts = { 'id' } }) + return space +end; + +function insert(space, i) + space:insert({ i, string.rep(string.char(32 + math.random(127-32)), math.random(1024)) }) +end; + +function fill_space(space, start) + local _, err = nil + local i = start + while err == nil do _, err = pcall(insert, space, i) i = i + 1 end +end; + +function stress_truncation(i, spaces) + local res, space = pcall(create_space, 'test' .. tostring(i)) + if res then spaces[i] = space return end + fill_space(box.space.test, box.space.test:len()) + for _, s in pairs(spaces) do s:truncate() end +end; +test_run:cmd("setopt delimiter ''"); + + +_ = create_space('test') +for i = 0, 27000 do insert(box.space.test, i) end + +spaces = {} +counter = 0 +status = true +res = nil +while counter < 80000 do status, res = pcall(stress_truncation, counter, spaces) counter = counter + 1 end +status +res + +-- Cleanup. +test_run:cmd('switch default') +test_run:drop_cluster({'master'}) -- 2.17.1