From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: From: Vladimir Davydov Subject: [PATCH v2 5/7] Introduce _vinyl_deferred_delete system space Date: Tue, 21 Aug 2018 14:15:38 +0300 Message-Id: <2af0dbcdd1daeffac95fa7d23a86efeec3990dd7.1534847663.git.vdavydov.dev@gmail.com> In-Reply-To: References: In-Reply-To: References: To: kostja@tarantool.org Cc: tarantool-patches@freelists.org List-ID: The space is a blackhole. It will be used for writing deferred DELETE statements generated by vinyl compaction tasks to WAL so that we can recover deferred DELETEs that hadn't been dumped to disk before the server was restarted. Since the new space doesn't depend on other system spaces, let's assign the minimal possible id to it, i.e. 257. Needed for #2129 --- src/box/bootstrap.snap | Bin 1540 -> 1607 bytes src/box/lua/space.cc | 2 ++ src/box/lua/upgrade.lua | 21 +++++++++++++++++++++ src/box/schema.cc | 27 ++++++++++++++++++++++++++- src/box/schema_def.h | 2 ++ src/box/vinyl.c | 34 ++++++++++++++++++++++++++++++++++ test/app-tap/tarantoolctl.test.lua | 2 +- test/box-py/bootstrap.result | 7 +++++-- test/box/access_misc.result | 5 ++++- test/box/access_sysview.result | 2 +- test/wal_off/alter.result | 2 +- test/xlog/upgrade.result | 7 +++++-- 12 files changed, 102 insertions(+), 9 deletions(-) diff --git a/src/box/bootstrap.snap b/src/box/bootstrap.snap index b610828c9c9ae9a22acdd8c150c16c6838b7a273..2c515e6e597e943f7538e24a582dc118c7569bc4 100644 GIT binary patch delta 1602 zcmV-I2EF-&495(R7=JM>GC4ObXJcV8GcjX0FbYX-b97;DV`VxZG&y8qHa9R~Ej2S_ zFfBAGp{7RQs7xsZ%K*gy*Z>~CwAvy7 z?~Tt7N3x%qhq3SVDsxTNesp!&m+yOci-6_?jNFZ!(QlwLN zVQtBo+iP2)|mxC<)u|l!N%j6V?jOt>3nBZCV`RiCaEz9HcL%1#O3+w z_`6Fgg#(+Vrp6zNqAK)U&+@Q$_KiC~d$7)}*#6abooVBDE;FK^1oHd}RTu+2pG)rI zwDQvGOMiimTU!KYD;$Dn*QDs(FZ>J+>AccED{^ObL}0Vj9C0hg`zXq=Cx)}p)%R;2 z@kA1&#oTciir!69T?YJ**h=T!`RoTM2<_>uZ=JaHtTQQUbqB>xjt)-6Ie}v$RU$Ih~up;if-rKZaZeE~&z5=hqM z%;$f18EvUVL7dF!`;PbPUtOM+3=LsYcOq<-n&)@yX>#`~RCOS1mYU@2)+)|}GVd|v zvVU1>{_4YA<~<7C?!6C{IP@=ejrWRH<~`oCq8z~XbF$Q6v(#jD62o?;55!3YqEw|+ z$%X`)ZcwDsr!tYcFT{z2yBizTmoj0fQePYnj|w}g4xBBA)!9kmDBYm3!PSVNQD0`b zRHD<$s9Mb3(`r6crUssW3K6&3_SnzQ4?xtm$l)ngA@VFb?%%|48z)R&e3v zaQr<~w-z6YTju3ZMUJy-VO+;T=hlbMT32kAnk7~sYjp822L=nTgN+`~jvV%~CTbM~qk%L7i*%bF!wUM^rVb6~+aDD1WHY zXk(~EMi2l10Z0G=21hB#iXIYx;5dqdFbZH821E!jh)x&*fWTr1iU4ZCL~}MKy2Y|v zJ^bl(UdTx}3KdD7L;W ztel6Vy@Cy?hNRhkp_231+>TD=HGg;x?NWpdPHRSKpb}gn|C(ibD3utwg&La(yhJCw z{svd9xuA&bGH@-T$1+n@u_(ma@h(VdM@2s~OyeFGhWCsL5x z*eTDDukK8TbMliI3kBu&wYwd~jh6D%9&Ws$`k=hOnH^_S1^k8aGzNaIynoviddCN} zDh(7ktzP#YX>f80=J1RRTle83@yc!;k4`2i0PwZVqmpO|*wUufJ8_s1`J-SBL*NUF z{*fY4*U~sf6XXXs2mo>}&=_hR?Mh+zXsVWS1PvTq#7U~e2h9pWND{IchE%_`P~dT4F*z~{Np5p=VQyn(Iv_b=V_`LAF*YqTV`gS8 zG-hTvEnzrjFfCy?W;Qf4VP!dGGGq!?Lu_wjYdRo%F*+bOeF_TIx(m9^1&9F7A|rpb zr2qf`001bpFZ}>e{VMI&<{eU?~Ai$VOI~v629dx2JaIm@kx4N&&zCxB$xlBK}6v zM#}nreKVfmYggvF>;bcooE<&pjzw|&ydgCd;B)Xi4C|Up=iRyONm;Mu=n31mId~$~K1OP2OJJrGIzc!y|v*5Q}mxHFpG|K|%#jeQ^A;E^l>* z3*Bp}NefgxX1+gm^KmUTgDMc1Jsb?P)oZC4!mL!?OgGcvU!7uNP_0u4V%4c@$$Tde zOCDy|bcHe0;hLW1g*=6kJQqvOo9n=`!@rT1v3PA=sEaGtUmrgF1V*+HwF$ z%o|ck2Ck)MDdN~U>lpNk)Rh9)Qd8p>$FU`TvVVQ}JNw3+pEy`&7|(v!cb#wKcU}Xc zztr*E^M9!-1a3aZxQyRo-mQ)X;JxvQVS5a1E-Z#;*X8(mp!hlb(b=Wn_TxWHa?e&p z0ms_NyVg8C=MtXRLX=hZO}2jlujiL*MCw|r<8%iFyl-urCG}bk_A!)lFN(Z z*9rxd2`ZdQ%ZOz}8m(9>QmHeYBvm9q6rtA9wbTrW=uAh+mpZa8Z9M*WuhEnWw8OPP z0Ken?`duE+M1Fc0sWuO;rRJ>TPnWxUKDEZdwbU$UUQcC(VtJFR2G>&aT`%TX-lUJ$ zzkkQk8h`!+Ak42Gb5O~I9X_GL4xhR(RGq@`<8YEmeGLBa%Ed{YaV#4wH7X6(wXrob zRx(mCOfa|?Bo}qm;#z9jPz5?Ii>k-mKV61Bw$iw6MNm)>lUj2vHAVFK{#e#!kBV!l ziDGG`ab1f2WXaFUg5t{}`TM8d6EBXt_J3uuRg$)9X&lOg&M*+4wU)S+njl=^il77Rj$ynhqj zxeG%W18Ui|uoin^FOJH&6BE>1-;h#Us!zr2@yPWOP}$xk=>TS*Ej6Q8ZgBx&}sXh}Wym z;Yvs=Xm6s&oTIkS5VXc28)t^2+kc4qR4r1#;*gdpU~^0s$pG8pm?G)gVGer7p5*j@ zrQSOro_cGm&fR}<@b8h^=iXW!%4reE=RqJFWsqt`1BcS0HC@As3ba#z^Ko9Pi8avYk$Oa!!S#& zSx3_!A8xyrpfT#3Y0zk_Hi}Y>p%$pI61F+5M3N&%2xKJAC;$CDD}<6?=$#5uoTZ5Z zCNj}EgB|fKv=$L@ag>MEJ_76DnN*W1Z`O`MZ9ptwx|K1yxMV1V1(Ltx%(4y7obk4E zW?=^8GIQ=kv4A0`wS7N1*>`Z!opukPx!_g@c{8|RJi<`!QS!z#<&+X#nnV8V6uo^% zFL^|hfwSZ-+|LAZ&d5i?Zt>-$tMH_1UnT!Yb{@SussOv@WfJ50JyfLCQd9Bw_N-dh k2=V>Mm|$S01}gnuSl5OyKJnJxL~`9Mur5?f$>nwg3PC diff --git a/src/box/lua/space.cc b/src/box/lua/space.cc index 580e0ea2..25b7e36d 100644 --- a/src/box/lua/space.cc +++ b/src/box/lua/space.cc @@ -518,6 +518,8 @@ box_lua_space_init(struct lua_State *L) lua_newtable(L); lua_setfield(L, -2, "schema"); lua_getfield(L, -1, "schema"); + lua_pushnumber(L, BOX_VINYL_DEFERRED_DELETE_ID); + lua_setfield(L, -2, "VINYL_DEFERRED_DELETE_ID"); lua_pushnumber(L, BOX_SCHEMA_ID); lua_setfield(L, -2, "SCHEMA_ID"); lua_pushnumber(L, BOX_SPACE_ID); diff --git a/src/box/lua/upgrade.lua b/src/box/lua/upgrade.lua index 0293f6ef..754685f8 100644 --- a/src/box/lua/upgrade.lua +++ b/src/box/lua/upgrade.lua @@ -964,6 +964,26 @@ local function upgrade_to_1_10_0() create_vsequence_space() end +-------------------------------------------------------------------------------- +--- Tarantool 1.10.2 +-------------------------------------------------------------------------------- +local function create_vinyl_deferred_delete_space() + local _space = box.space[box.schema.SPACE_ID] + local _vinyl_deferred_delete = box.space[box.schema.VINYL_DEFERRED_DELETE_ID] + + local format = {} + format[1] = {name = 'space_id', type = 'unsigned'} + format[2] = {name = 'lsn', type = 'unsigned'} + format[3] = {name = 'tuple', type = 'array'} + + log.info("create space _vinyl_deferred_delete") + _space:insert{_vinyl_deferred_delete.id, ADMIN, '_vinyl_deferred_delete', + 'blackhole', 0, {group_id = 1}, format} +end + +local function upgrade_to_1_10_2() + create_vinyl_deferred_delete_space() +end local function get_version() local version = box.space._schema:get{'version'} @@ -991,6 +1011,7 @@ local function upgrade(options) {version = mkversion(1, 7, 6), func = upgrade_to_1_7_6, auto = false}, {version = mkversion(1, 7, 7), func = upgrade_to_1_7_7, auto = true}, {version = mkversion(1, 10, 0), func = upgrade_to_1_10_0, auto = true}, + {version = mkversion(1, 10, 2), func = upgrade_to_1_10_2, auto = true}, } for _, handler in ipairs(handlers) do diff --git a/src/box/schema.cc b/src/box/schema.cc index 433f52c0..32669c69 100644 --- a/src/box/schema.cc +++ b/src/box/schema.cc @@ -37,7 +37,8 @@ #include "scoped_guard.h" #include "version.h" #include "user.h" -#include +#include "vclock.h" + /** * @module Data Dictionary * @@ -351,6 +352,30 @@ schema_init() FIELD_TYPE_UNSIGNED, false, NULL, COLL_NONE); sc_space_new(BOX_INDEX_ID, "_index", key_def, &alter_space_on_replace_index, &on_stmt_begin_index); + + /* + * _vinyl_deferred_delete - blackhole that is needed + * for writing deferred DELETE statements generated by + * vinyl compaction tasks to WAL. + */ + { + const char *engine = "blackhole"; + const char *name = "_vinyl_deferred_delete"; + struct space_opts opts = space_opts_default; + opts.group_id = GROUP_LOCAL; + struct space_def *def; + def = space_def_new_xc(BOX_VINYL_DEFERRED_DELETE_ID, ADMIN, 0, + name, strlen(name), engine, + strlen(engine), &opts, NULL, 0); + auto def_guard = make_scoped_guard([=] { + space_def_delete(def); + }); + RLIST_HEAD(key_list); + struct space *space = space_new_xc(def, &key_list); + space_cache_replace(space); + init_system_space(space); + trigger_run_xc(&on_alter_space, space); + } } void diff --git a/src/box/schema_def.h b/src/box/schema_def.h index 2edb8d37..9beed389 100644 --- a/src/box/schema_def.h +++ b/src/box/schema_def.h @@ -66,6 +66,8 @@ static_assert(BOX_INVALID_NAME_MAX <= BOX_NAME_MAX, enum { /** Start of the reserved range of system spaces. */ BOX_SYSTEM_ID_MIN = 256, + /** Space if of _vinyl_deferred_delete. */ + BOX_VINYL_DEFERRED_DELETE_ID = 257, /** Space id of _schema. */ BOX_SCHEMA_ID = 272, /** Space id of _collation. */ diff --git a/src/box/vinyl.c b/src/box/vinyl.c index fd14d1e7..18aa1ba5 100644 --- a/src/box/vinyl.c +++ b/src/box/vinyl.c @@ -65,6 +65,7 @@ #include "engine.h" #include "space.h" #include "index.h" +#include "schema.h" #include "xstream.h" #include "info.h" #include "column_mask.h" @@ -256,6 +257,8 @@ static const struct engine_vtab vinyl_engine_vtab; static const struct space_vtab vinyl_space_vtab; static const struct index_vtab vinyl_index_vtab; +static struct trigger on_replace_vinyl_deferred_delete; + /** * A quick intro into Vinyl cosmology and file format * -------------------------------------------------- @@ -2771,6 +2774,20 @@ vinyl_engine_abort_checkpoint(struct engine *engine) /** {{{ Recovery */ +/** + * Install trigger on the _vinyl_deferred_delete system space. + * Called on bootstrap and recovery. Note, this function can't + * be called from engine constructor, because the latter is + * invoked before the schema is initialized. + */ +static void +vy_set_deferred_delete_trigger(void) +{ + struct space *space = space_by_id(BOX_VINYL_DEFERRED_DELETE_ID); + assert(space != NULL); + trigger_add(&space->on_replace, &on_replace_vinyl_deferred_delete); +} + static int vinyl_engine_bootstrap(struct engine *engine) { @@ -2780,6 +2797,7 @@ vinyl_engine_bootstrap(struct engine *engine) return -1; vy_quota_set_limit(&e->quota, e->memory); e->status = VINYL_ONLINE; + vy_set_deferred_delete_trigger(); return 0; } @@ -2802,6 +2820,7 @@ vinyl_engine_begin_initial_recovery(struct engine *engine, vy_quota_set_limit(&e->quota, e->memory); e->status = VINYL_INITIAL_RECOVERY_REMOTE; } + vy_set_deferred_delete_trigger(); return 0; } @@ -4265,6 +4284,21 @@ vinyl_space_build_index(struct space *src_space, struct index *new_index, /* }}} Index build */ +/* {{{ Deferred DELETE handling */ + +static void +vy_deferred_delete_on_replace(struct trigger *trigger, void *event) +{ + (void)trigger; + (void)event; +} + +static struct trigger on_replace_vinyl_deferred_delete = { + RLIST_LINK_INITIALIZER, vy_deferred_delete_on_replace, NULL, NULL +}; + +/* }}} Deferred DELETE handling */ + static const struct engine_vtab vinyl_engine_vtab = { /* .shutdown = */ vinyl_engine_shutdown, /* .create_space = */ vinyl_engine_create_space, diff --git a/test/app-tap/tarantoolctl.test.lua b/test/app-tap/tarantoolctl.test.lua index ac9a208c..5a4a8450 100755 --- a/test/app-tap/tarantoolctl.test.lua +++ b/test/app-tap/tarantoolctl.test.lua @@ -346,7 +346,7 @@ do check_ctlcat_xlog(test_i, dir, "--from=3 --to=6 --format=json --show-system --replica 1", "\n", 3) check_ctlcat_xlog(test_i, dir, "--from=3 --to=6 --format=json --show-system --replica 1 --replica 2", "\n", 3) check_ctlcat_xlog(test_i, dir, "--from=3 --to=6 --format=json --show-system --replica 2", "\n", 0) - check_ctlcat_snap(test_i, dir, "--space=280", "---\n", 18) + check_ctlcat_snap(test_i, dir, "--space=280", "---\n", 19) check_ctlcat_snap(test_i, dir, "--space=288", "---\n", 43) end) end) diff --git a/test/box-py/bootstrap.result b/test/box-py/bootstrap.result index 16c2027c..336cd449 100644 --- a/test/box-py/bootstrap.result +++ b/test/box-py/bootstrap.result @@ -5,7 +5,7 @@ box.space._schema:select{} --- - - ['cluster', ''] - ['max_id', 511] - - ['version', 1, 10, 0] + - ['version', 1, 10, 2] ... box.space._cluster:select{} --- @@ -13,7 +13,10 @@ box.space._cluster:select{} ... box.space._space:select{} --- -- - [272, 1, '_schema', 'memtx', 0, {}, [{'type': 'string', 'name': 'key'}]] +- - [257, 1, '_vinyl_deferred_delete', 'blackhole', 0, {'group_id': 1}, [{'name': 'space_id', + 'type': 'unsigned'}, {'name': 'lsn', 'type': 'unsigned'}, {'name': 'tuple', + 'type': 'array'}]] + - [272, 1, '_schema', 'memtx', 0, {}, [{'type': 'string', 'name': 'key'}]] - [276, 1, '_collation', 'memtx', 0, {}, [{'name': 'id', 'type': 'unsigned'}, { 'name': 'name', 'type': 'string'}, {'name': 'owner', 'type': 'unsigned'}, {'name': 'type', 'type': 'string'}, {'name': 'locale', 'type': 'string'}, { diff --git a/test/box/access_misc.result b/test/box/access_misc.result index 2d87fa2d..4054313b 100644 --- a/test/box/access_misc.result +++ b/test/box/access_misc.result @@ -752,7 +752,10 @@ box.space._user:select() ... box.space._space:select() --- -- - [272, 1, '_schema', 'memtx', 0, {}, [{'type': 'string', 'name': 'key'}]] +- - [257, 1, '_vinyl_deferred_delete', 'blackhole', 0, {'group_id': 1}, [{'name': 'space_id', + 'type': 'unsigned'}, {'name': 'lsn', 'type': 'unsigned'}, {'name': 'tuple', + 'type': 'array'}]] + - [272, 1, '_schema', 'memtx', 0, {}, [{'type': 'string', 'name': 'key'}]] - [276, 1, '_collation', 'memtx', 0, {}, [{'name': 'id', 'type': 'unsigned'}, { 'name': 'name', 'type': 'string'}, {'name': 'owner', 'type': 'unsigned'}, {'name': 'type', 'type': 'string'}, {'name': 'locale', 'type': 'string'}, { diff --git a/test/box/access_sysview.result b/test/box/access_sysview.result index 20efd2bb..bc5e069f 100644 --- a/test/box/access_sysview.result +++ b/test/box/access_sysview.result @@ -230,7 +230,7 @@ box.session.su('guest') ... #box.space._vspace:select{} --- -- 19 +- 20 ... #box.space._vindex:select{} --- diff --git a/test/wal_off/alter.result b/test/wal_off/alter.result index afac1e55..f4703395 100644 --- a/test/wal_off/alter.result +++ b/test/wal_off/alter.result @@ -28,7 +28,7 @@ end; ... #spaces; --- -- 65515 +- 65513 ... -- cleanup for k, v in pairs(spaces) do diff --git a/test/xlog/upgrade.result b/test/xlog/upgrade.result index f02996bb..a411e0a7 100644 --- a/test/xlog/upgrade.result +++ b/test/xlog/upgrade.result @@ -36,11 +36,14 @@ box.space._schema:select() --- - - ['cluster', ''] - ['max_id', 513] - - ['version', 1, 10, 0] + - ['version', 1, 10, 2] ... box.space._space:select() --- -- - [272, 1, '_schema', 'memtx', 0, {}, [{'type': 'string', 'name': 'key'}]] +- - [257, 1, '_vinyl_deferred_delete', 'blackhole', 0, {'group_id': 1}, [{'name': 'space_id', + 'type': 'unsigned'}, {'name': 'lsn', 'type': 'unsigned'}, {'name': 'tuple', + 'type': 'array'}]] + - [272, 1, '_schema', 'memtx', 0, {}, [{'type': 'string', 'name': 'key'}]] - [276, 1, '_collation', 'memtx', 0, {}, [{'name': 'id', 'type': 'unsigned'}, { 'name': 'name', 'type': 'string'}, {'name': 'owner', 'type': 'unsigned'}, {'name': 'type', 'type': 'string'}, {'name': 'locale', 'type': 'string'}, { -- 2.11.0