[PATCH v2 5/7] Introduce _vinyl_deferred_delete system space

[Vladimir Davydov <vdavydov.dev@gmail.com>]

The space is a blackhole. It will be used for writing deferred DELETE
statements generated by vinyl compaction tasks to WAL so that we can
recover deferred DELETEs that hadn't been dumped to disk before the
server was restarted.

Since the new space doesn't depend on other system spaces, let's assign
the minimal possible id to it, i.e. 257.

Needed for #2129
---
 src/box/bootstrap.snap             | Bin 1540 -> 1607 bytes
 src/box/lua/space.cc               |   2 ++
 src/box/lua/upgrade.lua            |  21 +++++++++++++++++++++
 src/box/schema.cc                  |  27 ++++++++++++++++++++++++++-
 src/box/schema_def.h               |   2 ++
 src/box/vinyl.c                    |  34 ++++++++++++++++++++++++++++++++++
 test/app-tap/tarantoolctl.test.lua |   2 +-
 test/box-py/bootstrap.result       |   7 +++++--
 test/box/access_misc.result        |   5 ++++-
 test/box/access_sysview.result     |   2 +-
 test/wal_off/alter.result          |   2 +-
 test/xlog/upgrade.result           |   7 +++++--
 12 files changed, 102 insertions(+), 9 deletions(-)

diff --git a/src/box/bootstrap.snap b/src/box/bootstrap.snap
index b610828c9c9ae9a22acdd8c150c16c6838b7a273..2c515e6e597e943f7538e24a582dc118c7569bc4 100644
GIT binary patch
delta 1602
zcmV-I2EF-&495(R7=JM>GC4ObXJcV8GcjX0FbYX-b97;DV`VxZG&y8qHa9R~Ej2S_
zFfBA<WHBu`H#BA~HZeIdVmM}EH8Ny03RXjGZ)0mZAbWiZ3e~y`y3Ga60L}%iSbe1c
z00000D77#B08k|^0NOw-4oT1$Hvs_Cg%?~QI9ps-+!jp2m4A`gL}K&+2d!A(UZS6h
z7JH|uax=vw5&7K{{qQ7QrX-P`Z)s5?q@<S9O$+>Gp{7RQs7xsZ%K*gy*Z>~CwAvy7
z?~Tt7N3x%qhq3SVDsxTNesp!&m+yOc<h5^ME&CZ#zkEkh+^d=o{ZM56@NlRS`u^d6
z_xrsc{O^V?oqx;gq}EDQgc}9eEHzylxAGootiIB}JFod<?<TPTHcQPQM{=Zi5v%`+
zUe+Y725~OiW~pfnR4rz{zoZe_EH#TN(U)%zZq4dusc9491~i>i-6_?jNFZ!(QlwLN
zV<Lend?LNghf0Ae6^1JNJ10Yr_TpM9rBhmd1XvM*C4b8mDmhXBSo#ys$$Qq00TEA=
zV0mYire(9#B>QtBo+iP2)|mxC<)u|l!N%j6V?jOt>3nBZCV`RiCaEz9HcL%1#O3+w
z_`6Fgg#(+Vrp6zNqAK)U&+@Q$_KiC~d$7)}*#6abooVBDE;FK^1oHd}RTu+2pG)rI
zwDQvGOMiimTU!KYD;$Dn*QDs(FZ>J+>AccED{^ObL}0Vj9C0hg`zXq=Cx)}p)%R;2
z@kA1&#oTciir!69T?YJ**h=T!`RoTM2<_>uZ=JaHtTQQUbqB>xjt)-6Ie}v$RU$<q
zHR3h1C=nEu+AKA<*ANt{bW^e^ojOw&q60xwqJQatXg<u;%#b!qO`sVMT!$COL5Aif
zGZV8RMng;n7RM|K7+@~ISb*VPWL_$~#8^9`P+ve$>Ih~up;if-rKZaZeE~&z5=hqM
z%;$f18EvUVL7dF!`;PbPUtOM+3=LsYcOq<-n&)@yX>#`~RCOS1mYU@2)+)|}GVd|v
zvVU1>{_4YA<~<7C?!6C{IP@=ejrWRH<~`oCq8z~XbF$Q6v(#jD62o?;55!3YqEw|+
z$%X`)ZcwDsr!tYcFT{z2yBizTmoj0fQePYnj|w}g4xBBA)!9kmDBYm3!PSVNQD0`b
zRHD<$s9Mb3(<E546~>`r6crUssW3K6&3_SnzQ4?xtm$l)ngA@VFb?%%|48z)R&e3v
zaQr<~w-z6YTju3ZMUJy-VO+;T=hlbMT32kAnk7~sYjp822L=nTgN+`~<C@XC5eCnf
ztnx_-vp8wQEKkU1KPP3KKe;+9g>jvV%~CTbM~qk%L7i*%bF!wUM^rVb6~+aDD1WHY
zXk(~EMi2l10Z0G=21hB#iXIYx;5dqdFbZH821E!jh)x&*fWTr1iU4ZCL~}MKy2Y|v
z<S`LUiXpHBgJLM?b0-QD$QBJ*36Ze4iMI6}<vGSICLpHBU>J^bl(UdTx}3KdD7L;W
ztel6Vy@Cy?hNRhkp_231+>TD=HGg;x?NWpdPHRSKpb}gn|C(ibD3utwg&La(yhJCw
z{svd9xuA&<Jt-V@{esaOCqq17j>bGH@-T$1+n@u_(ma@h(VdM@2s~OyeFGhWCsL5x
z*eTDDukK8TbMliI3kBu&wYwd~jh6D%9&Ws$`k=hOnH^_S1^k8aGzNaIynoviddCN}
zDh(7ktzP#YX>f80=J1RRTle83@yc!;k4`2i0PwZVqmpO|*wUufJ8_s1`J-SBL*NUF
z{*fY4*U~sf6XXXs2mo>}&=_hR?Mh+zXsVWS1PvTq#7U~e2h9pWND{IchE%_`P~<No
zl{DB9!s?a?gOQ`upq3-H4u7_b+K+y-*uvE|0fp&a4gW<ECyqg$FEGx8Zo_~xK0Rl=
z%z(@R(sxv#Fyv@0+)qRz93-be0b~-~7eL+yE*Ot6!!Jm#JxzJaU5m=0nRZs*9MOv-
ztz_VrywM`a4W29~iYY~Ovrn4&YDz}J1HPMLSzuy9m%_3+F`7fPqdV9=l-qF!Q~S$`
z!J8=MqZ2hy2KGVtruZ^5^BOf+XNye<Sk}KC><l71vaH<^6n#a*CkYU*1Jw|%?e`|%
As{jB1

delta 1534
zcmV<a1p)fU41^4j7=JJ=G%zh^V>dT4F*z~{Np5p=VQyn(Iv_b=V_`LAF*YqTV`gS8
zG-hTvEnzrjFfCy?W;Qf4VP!dGGGq!?Lu_wjYdRo%F*+bOeF_TIx(m9^1&9F7A|rpb
zr2qf`001bpFZ}>e{VM<lJ=RLl7I6XqUp$Ch%_7lYJ_Lj_@PFoXP~+we(A76EO7~<&
z)F3;RA|;uhw(`4WgTRw>I&<{eU?~Ai$VOI~v629dx2JaIm@kx4N&&zCxB$xlBK}6v
zM#}nreKVfmYggvF>;bcooE<&pjzw|&ydgCd;B)Xi4C|Up=iRyONm;Mu=<J|49rmyj
z!~XSg?^aVP(0>n31mId~$~K1OP2OJJrGIzc!y|v*5Q}mxHFpG|K|%#jeQ^A;E^l>*
z3*Bp}NefgxX1+gm^KmUTgDMc1Jsb?P)oZC4!mL!?OgGcvU!7uNP_0u4V%4c@$$Tde
z<JJuBKo=+bJ1@oWHsew%rA$(CR4!bhCq$aiEJ3j}Cx2ttc)M<~yh-n@9Ru2aUZxgY
zOU+j=F>OCDy|bcHe0;hLW1g*=6kJQqvOo9n=`!@rT1v3PA=sEaGtUmrgF1V*+HwF$
z%o|ck2Ck)MDdN~U>lpNk)Rh9)Qd8p>$FU`TvVVQ}JNw3+pEy`&7|(v!cb#wKcU}Xc
zztr*E^M9!-1a3aZxQyRo-mQ)X;JxvQVS5a1E-Z#;*X8(mp!hlb(b=Wn_TxWHa?e&p
z0<NVdiD5jvlVdHR&M*6U`SOA@wvN<F<4PlXstt_|t42g+g<yeLeOP%|b(q;~x1wEl
zEj90~M>ms_NyVg8C=MtXRLX=hZO}2jlujiL*MCw|r<8%iFyl-urCG}bk_A!)lFN(Z
z*9rxd2`ZdQ%ZOz}8m(9>QmHeYBvm9q6rtA9wbTrW=uAh+mpZa8Z9M*WuhEnWw8OPP
z0Ken?`duE+M1Fc0sWuO;rRJ>TPnWxUKDEZdwbU$UUQcC(VtJFR2G>&aT`%TX-lUJ$
zzkkQk8h`!+Ak42Gb5O~I9X_GL4xhR(RGq@`<8YEmeGLBa%Ed{YaV#4wH7X6(wXrob
zRx(mCOfa|?Bo}qm;#z9jPz5?Ii>k-mKV61Bw$iw6MNm)>lUj2vHAVFK{#e#!kBV!l
ziDGG`ab1f2WXaFUg5t{}`TM8d6EBXt_J3uuRg$)9X&lOg&M*+4wU)S+njl=^i<EJZ
zX2ppwLyQ}LBXiMF<cgb*y<$jG_{C2oetkYX_VcpV`IoM<QW}RMaV<4zXt3a^V}T&3
z(OhIq3CsWhfB+}}AqGb&W<?JYfWSD4qc93!7zRWLI0_9I0RX{)ke~>l77Rj$ynhqj
zxeG%W18Ui|uoin^FOJH&6BE><l!R4e54%@5i^3tJ2=RjuhoRy}$qP!PWxP;CvBe2+
zB`AQ$m7GU4)NS@llAFhZ>1-;h#Us!zr2@yPWOP}$xk=>TS*Ej6Q8ZgBx&}sXh}Wym
z;Yvs=Xm6s&oTIkS5VXc28)t^2+kc4qR4r1#;*gdpU~^0s$pG8pm?G)gVGer7p5*j@
zrQSOro_cGm&fR}<@b8h^=iXW!%4reE=RqJFW<g443Z&4$8`^`Azzc12c=p(Su(w)T
z1)jmCmtK6UOn@l~=HxoFllhK1;>sqt`1BcS0HC@As3ba#z^Ko9Pi8avYk$Oa!!S#&
zSx3_!A8xyrpfT#3Y0zk_Hi}Y>p%$pI61F+5M3N&%2xKJAC;$CDD}<6?=$#5uoTZ5Z
zCNj}EgB|fKv=$L@ag>MEJ_76DnN*W1Z`O`MZ9ptwx|K1yxMV1V1(Ltx%(4y7obk4E
zW?=^8GIQ=kv4A0`wS7N1*>`Z!opukPx!_g@c{8|RJi<`!QS!z#<&+X#nnV8V6uo^%
zFL^|hfwSZ-+|LAZ&d5i?Zt>-$tMH_1UnT!Yb{@SussOv@WfJ50JyfLCQd9Bw_N-dh
k2=V>Mm|$S01}gnuSl5<E4okR>OyKJnJxL~`9Mur5?f$>nwg3PC

diff --git a/src/box/lua/space.cc b/src/box/lua/space.cc
index 580e0ea2..25b7e36d 100644
--- a/src/box/lua/space.cc
+++ b/src/box/lua/space.cc
@@ -518,6 +518,8 @@ box_lua_space_init(struct lua_State *L)
 	lua_newtable(L);
 	lua_setfield(L, -2, "schema");
 	lua_getfield(L, -1, "schema");
+	lua_pushnumber(L, BOX_VINYL_DEFERRED_DELETE_ID);
+	lua_setfield(L, -2, "VINYL_DEFERRED_DELETE_ID");
 	lua_pushnumber(L, BOX_SCHEMA_ID);
 	lua_setfield(L, -2, "SCHEMA_ID");
 	lua_pushnumber(L, BOX_SPACE_ID);
diff --git a/src/box/lua/upgrade.lua b/src/box/lua/upgrade.lua
index 0293f6ef..754685f8 100644
--- a/src/box/lua/upgrade.lua
+++ b/src/box/lua/upgrade.lua
@@ -964,6 +964,26 @@ local function upgrade_to_1_10_0()
     create_vsequence_space()
 end
 
+--------------------------------------------------------------------------------
+--- Tarantool 1.10.2
+--------------------------------------------------------------------------------
+local function create_vinyl_deferred_delete_space()
+    local _space = box.space[box.schema.SPACE_ID]
+    local _vinyl_deferred_delete = box.space[box.schema.VINYL_DEFERRED_DELETE_ID]
+
+    local format = {}
+    format[1] = {name = 'space_id', type = 'unsigned'}
+    format[2] = {name = 'lsn', type = 'unsigned'}
+    format[3] = {name = 'tuple', type = 'array'}
+
+    log.info("create space _vinyl_deferred_delete")
+    _space:insert{_vinyl_deferred_delete.id, ADMIN, '_vinyl_deferred_delete',
+                  'blackhole', 0, {group_id = 1}, format}
+end
+
+local function upgrade_to_1_10_2()
+    create_vinyl_deferred_delete_space()
+end
 
 local function get_version()
     local version = box.space._schema:get{'version'}
@@ -991,6 +1011,7 @@ local function upgrade(options)
         {version = mkversion(1, 7, 6), func = upgrade_to_1_7_6, auto = false},
         {version = mkversion(1, 7, 7), func = upgrade_to_1_7_7, auto = true},
         {version = mkversion(1, 10, 0), func = upgrade_to_1_10_0, auto = true},
+        {version = mkversion(1, 10, 2), func = upgrade_to_1_10_2, auto = true},
     }
 
     for _, handler in ipairs(handlers) do
diff --git a/src/box/schema.cc b/src/box/schema.cc
index 433f52c0..32669c69 100644
--- a/src/box/schema.cc
+++ b/src/box/schema.cc
@@ -37,7 +37,8 @@
 #include "scoped_guard.h"
 #include "version.h"
 #include "user.h"
-#include <stdio.h>
+#include "vclock.h"
+
 /**
  * @module Data Dictionary
  *
@@ -351,6 +352,30 @@ schema_init()
 			 FIELD_TYPE_UNSIGNED, false, NULL, COLL_NONE);
 	sc_space_new(BOX_INDEX_ID, "_index", key_def,
 		     &alter_space_on_replace_index, &on_stmt_begin_index);
+
+	/*
+	 * _vinyl_deferred_delete - blackhole that is needed
+	 * for writing deferred DELETE statements generated by
+	 * vinyl compaction tasks to WAL.
+	 */
+	{
+		const char *engine = "blackhole";
+		const char *name = "_vinyl_deferred_delete";
+		struct space_opts opts = space_opts_default;
+		opts.group_id = GROUP_LOCAL;
+		struct space_def *def;
+		def = space_def_new_xc(BOX_VINYL_DEFERRED_DELETE_ID, ADMIN, 0,
+				       name, strlen(name), engine,
+				       strlen(engine), &opts, NULL, 0);
+		auto def_guard = make_scoped_guard([=] {
+			space_def_delete(def);
+		});
+		RLIST_HEAD(key_list);
+		struct space *space = space_new_xc(def, &key_list);
+		space_cache_replace(space);
+		init_system_space(space);
+		trigger_run_xc(&on_alter_space, space);
+	}
 }
 
 void
diff --git a/src/box/schema_def.h b/src/box/schema_def.h
index 2edb8d37..9beed389 100644
--- a/src/box/schema_def.h
+++ b/src/box/schema_def.h
@@ -66,6 +66,8 @@ static_assert(BOX_INVALID_NAME_MAX <= BOX_NAME_MAX,
 enum {
 	/** Start of the reserved range of system spaces. */
 	BOX_SYSTEM_ID_MIN = 256,
+	/** Space if of _vinyl_deferred_delete. */
+	BOX_VINYL_DEFERRED_DELETE_ID = 257,
 	/** Space id of _schema. */
 	BOX_SCHEMA_ID = 272,
 	/** Space id of _collation. */
diff --git a/src/box/vinyl.c b/src/box/vinyl.c
index fd14d1e7..18aa1ba5 100644
--- a/src/box/vinyl.c
+++ b/src/box/vinyl.c
@@ -65,6 +65,7 @@
 #include "engine.h"
 #include "space.h"
 #include "index.h"
+#include "schema.h"
 #include "xstream.h"
 #include "info.h"
 #include "column_mask.h"
@@ -256,6 +257,8 @@ static const struct engine_vtab vinyl_engine_vtab;
 static const struct space_vtab vinyl_space_vtab;
 static const struct index_vtab vinyl_index_vtab;
 
+static struct trigger on_replace_vinyl_deferred_delete;
+
 /**
  * A quick intro into Vinyl cosmology and file format
  * --------------------------------------------------
@@ -2771,6 +2774,20 @@ vinyl_engine_abort_checkpoint(struct engine *engine)
 
 /** {{{ Recovery */
 
+/**
+ * Install trigger on the _vinyl_deferred_delete system space.
+ * Called on bootstrap and recovery. Note, this function can't
+ * be called from engine constructor, because the latter is
+ * invoked before the schema is initialized.
+ */
+static void
+vy_set_deferred_delete_trigger(void)
+{
+	struct space *space = space_by_id(BOX_VINYL_DEFERRED_DELETE_ID);
+	assert(space != NULL);
+	trigger_add(&space->on_replace, &on_replace_vinyl_deferred_delete);
+}
+
 static int
 vinyl_engine_bootstrap(struct engine *engine)
 {
@@ -2780,6 +2797,7 @@ vinyl_engine_bootstrap(struct engine *engine)
 		return -1;
 	vy_quota_set_limit(&e->quota, e->memory);
 	e->status = VINYL_ONLINE;
+	vy_set_deferred_delete_trigger();
 	return 0;
 }
 
@@ -2802,6 +2820,7 @@ vinyl_engine_begin_initial_recovery(struct engine *engine,
 		vy_quota_set_limit(&e->quota, e->memory);
 		e->status = VINYL_INITIAL_RECOVERY_REMOTE;
 	}
+	vy_set_deferred_delete_trigger();
 	return 0;
 }
 
@@ -4265,6 +4284,21 @@ vinyl_space_build_index(struct space *src_space, struct index *new_index,
 
 /* }}} Index build */
 
+/* {{{ Deferred DELETE handling */
+
+static void
+vy_deferred_delete_on_replace(struct trigger *trigger, void *event)
+{
+	(void)trigger;
+	(void)event;
+}
+
+static struct trigger on_replace_vinyl_deferred_delete = {
+	RLIST_LINK_INITIALIZER, vy_deferred_delete_on_replace, NULL, NULL
+};
+
+/* }}} Deferred DELETE handling */
+
 static const struct engine_vtab vinyl_engine_vtab = {
 	/* .shutdown = */ vinyl_engine_shutdown,
 	/* .create_space = */ vinyl_engine_create_space,
diff --git a/test/app-tap/tarantoolctl.test.lua b/test/app-tap/tarantoolctl.test.lua
index ac9a208c..5a4a8450 100755
--- a/test/app-tap/tarantoolctl.test.lua
+++ b/test/app-tap/tarantoolctl.test.lua
@@ -346,7 +346,7 @@ do
             check_ctlcat_xlog(test_i, dir, "--from=3 --to=6 --format=json --show-system --replica 1", "\n", 3)
             check_ctlcat_xlog(test_i, dir, "--from=3 --to=6 --format=json --show-system --replica 1 --replica 2", "\n", 3)
             check_ctlcat_xlog(test_i, dir, "--from=3 --to=6 --format=json --show-system --replica 2", "\n", 0)
-            check_ctlcat_snap(test_i, dir, "--space=280", "---\n", 18)
+            check_ctlcat_snap(test_i, dir, "--space=280", "---\n", 19)
             check_ctlcat_snap(test_i, dir, "--space=288", "---\n", 43)
         end)
     end)
diff --git a/test/box-py/bootstrap.result b/test/box-py/bootstrap.result
index 16c2027c..336cd449 100644
--- a/test/box-py/bootstrap.result
+++ b/test/box-py/bootstrap.result
@@ -5,7 +5,7 @@ box.space._schema:select{}
 ---
 - - ['cluster', '<cluster uuid>']
   - ['max_id', 511]
-  - ['version', 1, 10, 0]
+  - ['version', 1, 10, 2]
 ...
 box.space._cluster:select{}
 ---
@@ -13,7 +13,10 @@ box.space._cluster:select{}
 ...
 box.space._space:select{}
 ---
-- - [272, 1, '_schema', 'memtx', 0, {}, [{'type': 'string', 'name': 'key'}]]
+- - [257, 1, '_vinyl_deferred_delete', 'blackhole', 0, {'group_id': 1}, [{'name': 'space_id',
+        'type': 'unsigned'}, {'name': 'lsn', 'type': 'unsigned'}, {'name': 'tuple',
+        'type': 'array'}]]
+  - [272, 1, '_schema', 'memtx', 0, {}, [{'type': 'string', 'name': 'key'}]]
   - [276, 1, '_collation', 'memtx', 0, {}, [{'name': 'id', 'type': 'unsigned'}, {
         'name': 'name', 'type': 'string'}, {'name': 'owner', 'type': 'unsigned'},
       {'name': 'type', 'type': 'string'}, {'name': 'locale', 'type': 'string'}, {
diff --git a/test/box/access_misc.result b/test/box/access_misc.result
index 2d87fa2d..4054313b 100644
--- a/test/box/access_misc.result
+++ b/test/box/access_misc.result
@@ -752,7 +752,10 @@ box.space._user:select()
 ...
 box.space._space:select()
 ---
-- - [272, 1, '_schema', 'memtx', 0, {}, [{'type': 'string', 'name': 'key'}]]
+- - [257, 1, '_vinyl_deferred_delete', 'blackhole', 0, {'group_id': 1}, [{'name': 'space_id',
+        'type': 'unsigned'}, {'name': 'lsn', 'type': 'unsigned'}, {'name': 'tuple',
+        'type': 'array'}]]
+  - [272, 1, '_schema', 'memtx', 0, {}, [{'type': 'string', 'name': 'key'}]]
   - [276, 1, '_collation', 'memtx', 0, {}, [{'name': 'id', 'type': 'unsigned'}, {
         'name': 'name', 'type': 'string'}, {'name': 'owner', 'type': 'unsigned'},
       {'name': 'type', 'type': 'string'}, {'name': 'locale', 'type': 'string'}, {
diff --git a/test/box/access_sysview.result b/test/box/access_sysview.result
index 20efd2bb..bc5e069f 100644
--- a/test/box/access_sysview.result
+++ b/test/box/access_sysview.result
@@ -230,7 +230,7 @@ box.session.su('guest')
 ...
 #box.space._vspace:select{}
 ---
-- 19
+- 20
 ...
 #box.space._vindex:select{}
 ---
diff --git a/test/wal_off/alter.result b/test/wal_off/alter.result
index afac1e55..f4703395 100644
--- a/test/wal_off/alter.result
+++ b/test/wal_off/alter.result
@@ -28,7 +28,7 @@ end;
 ...
 #spaces;
 ---
-- 65515
+- 65513
 ...
 -- cleanup
 for k, v in pairs(spaces) do
diff --git a/test/xlog/upgrade.result b/test/xlog/upgrade.result
index f02996bb..a411e0a7 100644
--- a/test/xlog/upgrade.result
+++ b/test/xlog/upgrade.result
@@ -36,11 +36,14 @@ box.space._schema:select()
 ---
 - - ['cluster', '<server_uuid>']
   - ['max_id', 513]
-  - ['version', 1, 10, 0]
+  - ['version', 1, 10, 2]
 ...
 box.space._space:select()
 ---
-- - [272, 1, '_schema', 'memtx', 0, {}, [{'type': 'string', 'name': 'key'}]]
+- - [257, 1, '_vinyl_deferred_delete', 'blackhole', 0, {'group_id': 1}, [{'name': 'space_id',
+        'type': 'unsigned'}, {'name': 'lsn', 'type': 'unsigned'}, {'name': 'tuple',
+        'type': 'array'}]]
+  - [272, 1, '_schema', 'memtx', 0, {}, [{'type': 'string', 'name': 'key'}]]
   - [276, 1, '_collation', 'memtx', 0, {}, [{'name': 'id', 'type': 'unsigned'}, {
         'name': 'name', 'type': 'string'}, {'name': 'owner', 'type': 'unsigned'},
       {'name': 'type', 'type': 'string'}, {'name': 'locale', 'type': 'string'}, {
-- 
2.11.0