From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: From: Vladimir Davydov Subject: [PATCH 24/25] Introduce _vinyl_deferred_delete system space Date: Fri, 27 Jul 2018 14:30:04 +0300 Message-Id: <920a0685fd01546464585090dadf57732fa7a696.1532689066.git.vdavydov.dev@gmail.com> In-Reply-To: References: In-Reply-To: References: To: kostja@tarantool.org Cc: tarantool-patches@freelists.org List-ID: The space is a blackhole. It will be used for writing deferred DELETE statements generated by vinyl compaction tasks to WAL so that we can recover deferred DELETEs that hadn't been dumped to disk before the server was restarted. Needed for #2129 --- src/box/bootstrap.snap | Bin 1540 -> 1605 bytes src/box/lua/space.cc | 2 ++ src/box/lua/upgrade.lua | 21 +++++++++++ src/box/schema.cc | 72 ++++++++++++++++++++++++++----------- src/box/schema_def.h | 2 ++ src/box/vinyl.c | 15 ++++++++ test/app-tap/tarantoolctl.test.lua | 2 +- test/box-py/bootstrap.result | 5 ++- test/box/access_misc.result | 3 ++ test/box/access_sysview.result | 2 +- test/box/alter.result | 4 +-- test/wal_off/alter.result | 2 +- test/xlog/upgrade.result | 5 ++- 13 files changed, 108 insertions(+), 27 deletions(-) diff --git a/src/box/bootstrap.snap b/src/box/bootstrap.snap index b610828c9c9ae9a22acdd8c150c16c6838b7a273..dd0766a1af6d2aa6577eb48ea3ccc876a5989b4b 100644 GIT binary patch delta 1600 zcmV-G2EX})48;tP7=JM>GBh+TXEZc7Ic76rV+u)bb97;DV`VxZGBq_fF)}t}EipJS zG%YkSFfc7*IWl7{Fg7z{V=-Z4Wi&7|3RXjGZ)0mZAbWiZ3e~y`y3Ga40M2jv`c0(( z00000D77#B08kY!0GdD~4oc8gQ2_v7Jir%5=(vCY{|RX2wtw1;q>_zSo~QIMph6SH zKt!gJkdm06?)@c64FXTjX{eWg+ICbh5T%X;t23>bPxYamS}UcL0?h!!0N4Pe0k%$3 zye}H>VkiV&9D+45*7s|ku>4-PGS}nn%SL{JbeQ`E!4P{yltsYzp#AG=p_b0O^V*BC zF7yKgWcKvezkka6dKc!Njjm9mAsi`yt&@!H`^sCixq3_g?)>JHu}4$^*g8odMQ)sE zL8JeLIP1|ygE$sz>m;cGs>96p!)!uqo#YWU`m)z^Uo~p$Bx%08F{Mygbz*&`n$u0H zWOM3HrHewHN~-SOm{2GbhAI0yA3VS0C?eKOeo4b}K@#9$7H2 zMr9V*I?1s=7lQT3@=6+`Ac#w_uUKYUP|trldo>D!fC$VZqAv$*o#Y5!SSjoG^N>&q z25gVIlF8X1#J2PE?$W+tX&>m-3pu^S z>3v4B)i0VABb&{jZyHOmLHQnj)HQ`mw zO;^G+s#U4$OsCSROPNaF4UGw9V?%>s3YBpfkE~mzGcNVw=;#Ovb$?;&BuVr6epuGyO~=+r0;rSDxKxY%;>b%{ zfd$T>_`8VO6%U462xo9LiZxniT&F_k*LO=wQ`kDm5iOQWjBzl>g$mq(#*XKS#ps>r zg6Cl_7!%WV@sfC5c*tizAB8&qu~pJJ<2n?!PBQ1l3mOesI?wFqF8O2rvo{7y*F5AS56Hs09NAqYq$Ri@=2Q zn^SU0?7$tk19zDLPcXS|7A2K651YnC&+)Pt0-b=I4#Uq63zL*AOnHplt&3F?)hC9u zS6aPkLp`%!61t`BSfQt)M|s|%Wq(|89EFUB15?lYW3oL#x^uqy_e!tt1X^qlS+CUk?yYz4UsCFV{tM`8d#X9-Rux+_T| zY3n`4W>nIM;)VgVnzK&QA0hB{Ee&Fnlyz_$s?9Em#!w3%4h-`|HWkXbH<*YM?}`3B z1Qa6cFSK9@3C`BFbfb(Nn18h+gw1i0G6qMfzamFw9pn@Bzx-ybPrOYa1JkKbq~ol_ z`z$aE#&Oi`lH$x*&+!XpK<`ZAj!Y(o+?Ji|Nw0$|$=OT*WC`XGk@tiP^&`yZRFYv$ z(@gA82zVGqoiK}wd+$)U4E&OpbR!cyI%6J*-C~xLe#w()a@GEk>^V(*r&WPUO-m#O y=K9b;Rxp^_zF*hsu14bDtvX3J(`rc2zai@m65FtDt0OJ=DvX{az(x+$5UuS!*y?2f delta 1534 zcmVdT4F*z~{Np5p=VQyn(Iv_b=V_`LAF*YqTV`gS8 zG-hTvEnzrjFfCy?W;Qf4VP!dGGGq!?Lu_wjYdRo%F*+bOeF_TIx(m9^1&9F7A|rpb zr2qf`001bpFZ}>e{VMI&<{eU?~Ai$VOI~v629dx2JaIm@kx4N&&zCxB$xlBK}6v zM#}nreKVfmYggvF>;bcooE<&pjzw|&ydgCd;B)Xi4C|Up=iRyONm;Mu=n31mId~$~K1OP2OJJrGIzc!y|v*5Q}mxHFpG|K|%#jeQ^A;E^l>* z3*Bp}NefgxX1+gm^KmUTgDMc1Jsb?P)oZC4!mL!?OgGcvU!7uNP_0u4V%4c@$$Tde zOCDy|bcHe0;hLW1g*=6kJQqvOo9n=`!@rT1v3PA=sEaGtUmrgF1V*+HwF$ z%o|ck2Ck)MDdN~U>lpNk)Rh9)Qd8p>$FU`TvVVQ}JNw3+pEy`&7|(v!cb#wKcU}Xc zztr*E^M9!-1a3aZxQyRo-mQ)X;JxvQVS5a1E-Z#;*X8(mp!hlb(b=Wn_TxWHa?e&p z0ms_NyVg8C=MtXRLX=hZO}2jlujiL*MCw|r<8%iFyl-urCG}bk_A!)lFN(Z z*9rxd2`ZdQ%ZOz}8m(9>QmHeYBvm9q6rtA9wbTrW=uAh+mpZa8Z9M*WuhEnWw8OPP z0Ken?`duE+M1Fc0sWuO;rRJ>TPnWxUKDEZdwbU$UUQcC(VtJFR2G>&aT`%TX-lUJ$ zzkkQk8h`!+Ak42Gb5O~I9X_GL4xhR(RGq@`<8YEmeGLBa%Ed{YaV#4wH7X6(wXrob zRx(mCOfa|?Bo}qm;#z9jPz5?Ii>k-mKV61Bw$iw6MNm)>lUj2vHAVFK{#e#!kBV!l ziDGG`ab1f2WXaFUg5t{}`TM8d6EBXt_J3uuRg$)9X&lOg&M*+4wU)S+njl=^il77Rj$ynhqj zxeG%W18Ui|uoin^FOJH&6BE>1-;h#Us!zr2@yPWOP}$xk=>TS*Ej6Q8ZgBx&}sXh}Wym z;Yvs=Xm6s&oTIkS5VXc28)t^2+kc4qR4r1#;*gdpU~^0s$pG8pm?G)gVGer7p5*j@ zrQSOro_cGm&fR}<@b8h^=iXW!%4reE=RqJFWsqt`1BcS0HC@As3ba#z^Ko9Pi8avYk$Oa!!S#& zSx3_!A8xyrpfT#3Y0zk_Hi}Y>p%$pI61F+5M3N&%2xKJAC;$CDD}<6?=$#5uoTZ5Z zCNj}EgB|fKv=$L@ag>MEJ_76DnN*W1Z`O`MZ9ptwx|K1yxMV1V1(Ltx%(4y7obk4E zW?=^8GIQ=kv4A0`wS7N1*>`Z!opukPx!_g@c{8|RJi<`!QS!z#<&+X#nnV8V6uo^% zFL^|hfwSZ-+|LAZ&d5i?Zt>-$tMH_1UnT!Yb{@SussOv@WfJ50JyfLCQd9Bw_N-dh k2=V>Mm|$S01}gnuSl5OyKJnJxL~`9Mur5?efFfv;Y7A diff --git a/src/box/lua/space.cc b/src/box/lua/space.cc index 580e0ea2..6e180ad9 100644 --- a/src/box/lua/space.cc +++ b/src/box/lua/space.cc @@ -554,6 +554,8 @@ box_lua_space_init(struct lua_State *L) lua_setfield(L, -2, "VSEQUENCE_ID"); lua_pushnumber(L, BOX_SPACE_SEQUENCE_ID); lua_setfield(L, -2, "SPACE_SEQUENCE_ID"); + lua_pushnumber(L, BOX_VINYL_DEFERRED_DELETE_ID); + lua_setfield(L, -2, "VINYL_DEFERRED_DELETE_ID"); lua_pushnumber(L, BOX_SYSTEM_ID_MIN); lua_setfield(L, -2, "SYSTEM_ID_MIN"); lua_pushnumber(L, BOX_SYSTEM_ID_MAX); diff --git a/src/box/lua/upgrade.lua b/src/box/lua/upgrade.lua index 0293f6ef..bdbd7d4a 100644 --- a/src/box/lua/upgrade.lua +++ b/src/box/lua/upgrade.lua @@ -964,6 +964,26 @@ local function upgrade_to_1_10_0() create_vsequence_space() end +-------------------------------------------------------------------------------- +--- Tarantool 1.10.1 +-------------------------------------------------------------------------------- +local function create_vinyl_deferred_delete_space() + local _space = box.space[box.schema.SPACE_ID] + local _vinyl_deferred_delete = box.space[box.schema.VINYL_DEFERRED_DELETE_ID] + + local format = {} + format[1] = {name = 'space_id', type = 'unsigned'} + format[2] = {name = 'lsn', type = 'unsigned'} + format[3] = {name = 'tuple', type = 'array'} + + log.info("create space _vinyl_deferred_delete") + _space:insert{_vinyl_deferred_delete.id, ADMIN, '_vinyl_deferred_delete', + 'blackhole', 0, {group_id = 1}, format} +end + +local function upgrade_to_1_10_1() + create_vinyl_deferred_delete_space() +end local function get_version() local version = box.space._schema:get{'version'} @@ -991,6 +1011,7 @@ local function upgrade(options) {version = mkversion(1, 7, 6), func = upgrade_to_1_7_6, auto = false}, {version = mkversion(1, 7, 7), func = upgrade_to_1_7_7, auto = true}, {version = mkversion(1, 10, 0), func = upgrade_to_1_10_0, auto = true}, + {version = mkversion(1, 10, 1), func = upgrade_to_1_10_1, auto = true}, } for _, handler in ipairs(handlers) do diff --git a/src/box/schema.cc b/src/box/schema.cc index 433f52c0..48644bbe 100644 --- a/src/box/schema.cc +++ b/src/box/schema.cc @@ -37,6 +37,7 @@ #include "scoped_guard.h" #include "version.h" #include "user.h" +#include "vclock.h" #include /** * @module Data Dictionary @@ -191,29 +192,16 @@ space_cache_replace(struct space *space) /** A wrapper around space_new() for data dictionary spaces. */ static void -sc_space_new(uint32_t id, const char *name, struct key_def *key_def, - struct trigger *replace_trigger, - struct trigger *stmt_begin_trigger) +do_sc_space_new(uint32_t id, const char *name, const char *engine, + const struct space_opts *opts, struct rlist *key_list, + struct trigger *replace_trigger, + struct trigger *stmt_begin_trigger) { - struct index_def *index_def = index_def_new(id, /* space id */ - 0 /* index id */, - "primary", /* name */ - strlen("primary"), - TREE /* index type */, - &index_opts_default, - key_def, NULL); - if (index_def == NULL) - diag_raise(); - auto index_def_guard = - make_scoped_guard([=] { index_def_delete(index_def); }); struct space_def *def = - space_def_new_xc(id, ADMIN, 0, name, strlen(name), "memtx", - strlen("memtx"), &space_opts_default, NULL, 0); + space_def_new_xc(id, ADMIN, 0, name, strlen(name), engine, + strlen(engine), opts, NULL, 0); auto def_guard = make_scoped_guard([=] { space_def_delete(def); }); - struct rlist key_list; - rlist_create(&key_list); - rlist_add_entry(&key_list, index_def, link); - struct space *space = space_new_xc(def, &key_list); + struct space *space = space_new_xc(def, key_list); (void) space_cache_replace(space); if (replace_trigger) trigger_add(&space->on_replace, replace_trigger); @@ -234,6 +222,40 @@ sc_space_new(uint32_t id, const char *name, struct key_def *key_def, trigger_run_xc(&on_alter_space, space); } +static void +sc_space_new(uint32_t id, const char *name, struct key_def *key_def, + struct trigger *replace_trigger, + struct trigger *stmt_begin_trigger) +{ + struct index_def *index_def = index_def_new(id, /* space id */ + 0 /* index id */, + "primary", /* name */ + strlen("primary"), + TREE /* index type */, + &index_opts_default, + key_def, NULL); + if (index_def == NULL) + diag_raise(); + auto index_def_guard = + make_scoped_guard([=] { index_def_delete(index_def); }); + struct rlist key_list; + rlist_create(&key_list); + rlist_add_entry(&key_list, index_def, link); + do_sc_space_new(id, name, "memtx", &space_opts_default, &key_list, + replace_trigger, stmt_begin_trigger); +} + +static void +sc_space_new_blackhole(uint32_t id, const char *name, uint32_t group_id, + struct trigger *replace_trigger) +{ + struct space_opts opts = space_opts_default; + opts.group_id = group_id; + RLIST_HEAD(key_list); + do_sc_space_new(id, name, "blackhole", &opts, &key_list, + replace_trigger, NULL); +} + uint32_t schema_find_id(uint32_t system_space_id, uint32_t index_id, const char *name, uint32_t len) @@ -351,6 +373,16 @@ schema_init() FIELD_TYPE_UNSIGNED, false, NULL, COLL_NONE); sc_space_new(BOX_INDEX_ID, "_index", key_def, &alter_space_on_replace_index, &on_stmt_begin_index); + + /* + * _vinyl_deferred_delete - blackhole that is needed + * for writing deferred DELETE statements generated by + * vinyl compaction tasks to WAL. + */ + extern struct trigger on_replace_vinyl_deferred_delete; + sc_space_new_blackhole(BOX_VINYL_DEFERRED_DELETE_ID, + "_vinyl_deferred_delete", GROUP_LOCAL, + &on_replace_vinyl_deferred_delete); } void diff --git a/src/box/schema_def.h b/src/box/schema_def.h index 2edb8d37..f6704bf8 100644 --- a/src/box/schema_def.h +++ b/src/box/schema_def.h @@ -102,6 +102,8 @@ enum { BOX_TRUNCATE_ID = 330, /** Space id of _space_sequence. */ BOX_SPACE_SEQUENCE_ID = 340, + /** Space if of _vinyl_deferred_delete. */ + BOX_VINYL_DEFERRED_DELETE_ID = 450, /** End of the reserved range of system spaces. */ BOX_SYSTEM_ID_MAX = 511, BOX_ID_NIL = 2147483647 diff --git a/src/box/vinyl.c b/src/box/vinyl.c index ddaa22bb..340da35b 100644 --- a/src/box/vinyl.c +++ b/src/box/vinyl.c @@ -4266,6 +4266,21 @@ vinyl_space_build_index(struct space *src_space, struct index *new_index, /* }}} Index build */ +/* {{{ Deferred DELETE handling */ + +static void +vy_deferred_delete_on_replace(struct trigger *trigger, void *event) +{ + (void)trigger; + (void)event; +} + +struct trigger on_replace_vinyl_deferred_delete = { + RLIST_LINK_INITIALIZER, vy_deferred_delete_on_replace, NULL, NULL +}; + +/* }}} Deferred DELETE handling */ + static const struct engine_vtab vinyl_engine_vtab = { /* .shutdown = */ vinyl_engine_shutdown, /* .create_space = */ vinyl_engine_create_space, diff --git a/test/app-tap/tarantoolctl.test.lua b/test/app-tap/tarantoolctl.test.lua index 6946c831..607dbf77 100755 --- a/test/app-tap/tarantoolctl.test.lua +++ b/test/app-tap/tarantoolctl.test.lua @@ -338,7 +338,7 @@ do check_ctlcat_xlog(test_i, dir, "--from=3 --to=6 --format=json --show-system --replica 1", "\n", 3) check_ctlcat_xlog(test_i, dir, "--from=3 --to=6 --format=json --show-system --replica 1 --replica 2", "\n", 3) check_ctlcat_xlog(test_i, dir, "--from=3 --to=6 --format=json --show-system --replica 2", "\n", 0) - check_ctlcat_snap(test_i, dir, "--space=280", "---\n", 18) + check_ctlcat_snap(test_i, dir, "--space=280", "---\n", 19) check_ctlcat_snap(test_i, dir, "--space=288", "---\n", 43) end) end) diff --git a/test/box-py/bootstrap.result b/test/box-py/bootstrap.result index 16c2027c..a83f5d06 100644 --- a/test/box-py/bootstrap.result +++ b/test/box-py/bootstrap.result @@ -5,7 +5,7 @@ box.space._schema:select{} --- - - ['cluster', ''] - ['max_id', 511] - - ['version', 1, 10, 0] + - ['version', 1, 10, 1] ... box.space._cluster:select{} --- @@ -68,6 +68,9 @@ box.space._space:select{} 'type': 'unsigned'}]] - [340, 1, '_space_sequence', 'memtx', 0, {}, [{'name': 'id', 'type': 'unsigned'}, {'name': 'sequence_id', 'type': 'unsigned'}, {'name': 'is_generated', 'type': 'boolean'}]] + - [450, 1, '_vinyl_deferred_delete', 'blackhole', 0, {'group_id': 1}, [{'name': 'space_id', + 'type': 'unsigned'}, {'name': 'lsn', 'type': 'unsigned'}, {'name': 'tuple', + 'type': 'array'}]] ... box.space._index:select{} --- diff --git a/test/box/access_misc.result b/test/box/access_misc.result index 2d87fa2d..b2850504 100644 --- a/test/box/access_misc.result +++ b/test/box/access_misc.result @@ -807,6 +807,9 @@ box.space._space:select() 'type': 'unsigned'}]] - [340, 1, '_space_sequence', 'memtx', 0, {}, [{'name': 'id', 'type': 'unsigned'}, {'name': 'sequence_id', 'type': 'unsigned'}, {'name': 'is_generated', 'type': 'boolean'}]] + - [450, 1, '_vinyl_deferred_delete', 'blackhole', 0, {'group_id': 1}, [{'name': 'space_id', + 'type': 'unsigned'}, {'name': 'lsn', 'type': 'unsigned'}, {'name': 'tuple', + 'type': 'array'}]] ... box.space._func:select() --- diff --git a/test/box/access_sysview.result b/test/box/access_sysview.result index 20efd2bb..bc5e069f 100644 --- a/test/box/access_sysview.result +++ b/test/box/access_sysview.result @@ -230,7 +230,7 @@ box.session.su('guest') ... #box.space._vspace:select{} --- -- 19 +- 20 ... #box.space._vindex:select{} --- diff --git a/test/box/alter.result b/test/box/alter.result index eb7014d8..5eb8229e 100644 --- a/test/box/alter.result +++ b/test/box/alter.result @@ -107,7 +107,7 @@ space = box.space[t[1]] ... space.id --- -- 341 +- 451 ... space.field_count --- @@ -152,7 +152,7 @@ space_deleted ... space:replace{0} --- -- error: Space '341' does not exist +- error: Space '451' does not exist ... _index:insert{_space.id, 0, 'primary', 'tree', 1, 1, 0, 'unsigned'} --- diff --git a/test/wal_off/alter.result b/test/wal_off/alter.result index afac1e55..f4703395 100644 --- a/test/wal_off/alter.result +++ b/test/wal_off/alter.result @@ -28,7 +28,7 @@ end; ... #spaces; --- -- 65515 +- 65513 ... -- cleanup for k, v in pairs(spaces) do diff --git a/test/xlog/upgrade.result b/test/xlog/upgrade.result index f02996bb..9862eb43 100644 --- a/test/xlog/upgrade.result +++ b/test/xlog/upgrade.result @@ -36,7 +36,7 @@ box.space._schema:select() --- - - ['cluster', ''] - ['max_id', 513] - - ['version', 1, 10, 0] + - ['version', 1, 10, 1] ... box.space._space:select() --- @@ -95,6 +95,9 @@ box.space._space:select() 'type': 'unsigned'}]] - [340, 1, '_space_sequence', 'memtx', 0, {}, [{'name': 'id', 'type': 'unsigned'}, {'name': 'sequence_id', 'type': 'unsigned'}, {'name': 'is_generated', 'type': 'boolean'}]] + - [450, 1, '_vinyl_deferred_delete', 'blackhole', 0, {'group_id': 1}, [{'name': 'space_id', + 'type': 'unsigned'}, {'name': 'lsn', 'type': 'unsigned'}, {'name': 'tuple', + 'type': 'array'}]] - [512, 1, 'distro', 'memtx', 0, {}, [{'name': 'os', 'type': 'str'}, {'name': 'dist', 'type': 'str'}, {'name': 'version', 'type': 'num'}, {'name': 'time', 'type': 'num'}]] - [513, 1, 'temporary', 'memtx', 0, {'temporary': true}, []] -- 2.11.0