[tarantool-patches] [PATCH v1 1/1] box: option to start tarantool with no format checks.

[tarantool-patches] [PATCH v1 1/1] box: option to start tarantool with no format checks.

[imeevma]

It is a common case that an instance is running on a version
1.6.*, then is upgraded to 1.9 with box.schema.upgrade(). But
some of users has malformed space formats, and some of them got
and ignored errors on box.schema.upgrade(). Such half-upgraded
data can not be used to start a new version due to format
violations, and can not be rolled back because some of new system
spaces managed to be created. And we can not repair raw xlogs and
snapshots because we have no such tools and they are zipped.

So it would be useful to have a special box.cfg option
ignore_space_formats. It would allow to start an instance on
the spaces with malformed formats, fix them, create a snapshot
and then start with the formats turned on.

Closes #3605

@TarantoolBot document
Title: box.cfg option 'ignore_space_formats'
The option allows to turn off space formats validation before
the instance is started. Useful to fix malformed formats after
an upgrade from version < 1.7.5 to >= 1.7.5.
---
Branch: https://github.com/tarantool/tarantool/tree/imeevma/gh-3605-ignore-space-formats-option
Issue: https://github.com/tarantool/tarantool/issues/3605

 src/box/alter.cc                     | 14 ++++++-
 src/box/box.h                        |  1 +
 src/box/lua/load_cfg.lua             |  2 +
 src/main.cc                          |  1 +
 test/app-tap/init_script.result      | 77 ++++++++++++++++++------------------
 test/box-tap/ignore_formats.test.lua | 28 +++++++++++++
 test/box/admin.result                |  2 +
 test/box/cfg.result                  |  4 ++
 8 files changed, 90 insertions(+), 39 deletions(-)
 create mode 100755 test/box-tap/ignore_formats.test.lua

diff --git a/src/box/alter.cc b/src/box/alter.cc
index a6299a1..045bb1d 100644
--- a/src/box/alter.cc
+++ b/src/box/alter.cc
@@ -54,6 +54,18 @@
 #include "sequence.h"
 
 /**
+ * A flag to ignore space formats and do not validate tuples by
+ * them.
+ */
+static bool ignore_space_formats = false;
+
+void
+box_set_ignore_space_formats(bool value)
+{
+	ignore_space_formats = value;
+}
+
+/**
  * chap-sha1 of empty string, i.e.
  * base64_encode(sha1(sha1(""), 0)
  */
@@ -552,7 +564,7 @@ space_def_new_from_tuple(struct tuple *tuple, uint32_t errcode,
 	const char *space_opts;
 	struct field_def *fields;
 	uint32_t field_count;
-	if (dd_version_id >= version_id(1, 7, 6)) {
+	if (dd_version_id >= version_id(1, 7, 6) && !ignore_space_formats) {
 		/* Check space opts. */
 		space_opts =
 			tuple_field_with_type_xc(tuple, BOX_SPACE_FIELD_OPTS,
diff --git a/src/box/box.h b/src/box/box.h
index 9930d4a..67eb58a 100644
--- a/src/box/box.h
+++ b/src/box/box.h
@@ -199,6 +199,7 @@ void box_set_replication_sync_lag(void);
 void box_set_replication_sync_timeout(void);
 void box_set_replication_skip_conflict(void);
 void box_set_net_msg_max(void);
+void box_set_ignore_space_formats(bool value);
 
 extern "C" {
 #endif /* defined(__cplusplus) */
diff --git a/src/box/lua/load_cfg.lua b/src/box/lua/load_cfg.lua
index f62f4dc..8f330e4 100644
--- a/src/box/lua/load_cfg.lua
+++ b/src/box/lua/load_cfg.lua
@@ -66,6 +66,7 @@ local default_cfg = {
     username            = nil,
     coredump            = false,
     read_only           = false,
+    ignore_space_formats= false,
     hot_standby         = false,
     checkpoint_interval = 3600,
     checkpoint_count    = 2,
@@ -130,6 +131,7 @@ local template_cfg = {
     checkpoint_interval = 'number',
     checkpoint_count    = 'number',
     read_only           = 'boolean',
+    ignore_space_formats= 'boolean',
     hot_standby         = 'boolean',
     worker_pool_threads = 'number',
     replication_timeout = 'number',
diff --git a/src/main.cc b/src/main.cc
index a36a2b0..61f0911 100644
--- a/src/main.cc
+++ b/src/main.cc
@@ -522,6 +522,7 @@ load_cfg()
 
 	title_set_custom(cfg_gets("custom_proc_title"));
 	title_update();
+	box_set_ignore_space_formats(cfg_geti("ignore_space_formats") != 0);
 	box_cfg();
 }
 
diff --git a/test/app-tap/init_script.result b/test/app-tap/init_script.result
index a2191d1..3907ded 100644
--- a/test/app-tap/init_script.result
+++ b/test/app-tap/init_script.result
@@ -12,44 +12,45 @@ box.cfg
 7	feedback_interval:3600
 8	force_recovery:false
 9	hot_standby:false
-10	listen:port
-11	log:tarantool.log
-12	log_format:plain
-13	log_level:5
-14	log_nonblock:true
-15	memtx_dir:.
-16	memtx_max_tuple_size:1048576
-17	memtx_memory:107374182
-18	memtx_min_tuple_size:16
-19	net_msg_max:768
-20	pid_file:box.pid
-21	read_only:false
-22	readahead:16320
-23	replication_connect_timeout:30
-24	replication_skip_conflict:false
-25	replication_sync_lag:10
-26	replication_sync_timeout:300
-27	replication_timeout:1
-28	rows_per_wal:500000
-29	slab_alloc_factor:1.05
-30	too_long_threshold:0.5
-31	vinyl_bloom_fpr:0.05
-32	vinyl_cache:134217728
-33	vinyl_dir:.
-34	vinyl_max_tuple_size:1048576
-35	vinyl_memory:134217728
-36	vinyl_page_size:8192
-37	vinyl_range_size:1073741824
-38	vinyl_read_threads:1
-39	vinyl_run_count_per_level:2
-40	vinyl_run_size_ratio:3.5
-41	vinyl_timeout:60
-42	vinyl_write_threads:4
-43	wal_dir:.
-44	wal_dir_rescan_delay:2
-45	wal_max_size:268435456
-46	wal_mode:write
-47	worker_pool_threads:4
+10	ignore_space_formats:false
+11	listen:port
+12	log:tarantool.log
+13	log_format:plain
+14	log_level:5
+15	log_nonblock:true
+16	memtx_dir:.
+17	memtx_max_tuple_size:1048576
+18	memtx_memory:107374182
+19	memtx_min_tuple_size:16
+20	net_msg_max:768
+21	pid_file:box.pid
+22	read_only:false
+23	readahead:16320
+24	replication_connect_timeout:30
+25	replication_skip_conflict:false
+26	replication_sync_lag:10
+27	replication_sync_timeout:300
+28	replication_timeout:1
+29	rows_per_wal:500000
+30	slab_alloc_factor:1.05
+31	too_long_threshold:0.5
+32	vinyl_bloom_fpr:0.05
+33	vinyl_cache:134217728
+34	vinyl_dir:.
+35	vinyl_max_tuple_size:1048576
+36	vinyl_memory:134217728
+37	vinyl_page_size:8192
+38	vinyl_range_size:1073741824
+39	vinyl_read_threads:1
+40	vinyl_run_count_per_level:2
+41	vinyl_run_size_ratio:3.5
+42	vinyl_timeout:60
+43	vinyl_write_threads:4
+44	wal_dir:.
+45	wal_dir_rescan_delay:2
+46	wal_max_size:268435456
+47	wal_mode:write
+48	worker_pool_threads:4
 --
 -- Test insert from detached fiber
 --
diff --git a/test/box-tap/ignore_formats.test.lua b/test/box-tap/ignore_formats.test.lua
new file mode 100755
index 0000000..1e038d6
--- /dev/null
+++ b/test/box-tap/ignore_formats.test.lua
@@ -0,0 +1,28 @@
+#!/usr/bin/env tarantool
+--
+-- gh-3605: allow to ignore space formats via box.cfg option.
+--
+local tap = require('tap')
+local test = tap.test("ignore_formats")
+test:plan(3)
+
+box.cfg{ignore_space_formats=true}
+
+local format = {}
+format[1] = {'field1', 'unsigned'}
+format[2] = {'field2', 'unsigned'}
+
+local s = box.schema.create_space('t0', {format = format})
+local pk = s:create_index('pk')
+test:ok(pcall(s.replace, s, {1, 'string'}), 'can violate format')
+-- Is not a dynamic option.
+local ok, err = pcall(box.cfg, {ignore_space_formats = false})
+test:ok(not ok, "ignore_space_formats is not a dynamic option")
+test:ok(err:match("Can't set option"), "error message")
+
+s:drop()
+
+-- Cleanup xlog
+box.snapshot()
+
+os.exit(test:check() == true and 0 or 1)
diff --git a/test/box/admin.result b/test/box/admin.result
index 8048460..1465e58 100644
--- a/test/box/admin.result
+++ b/test/box/admin.result
@@ -36,6 +36,8 @@ cfg_filter(box.cfg)
     - false
   - - hot_standby
     - false
+  - - ignore_space_formats
+    - false
   - - listen
     - <hidden>
   - - log
diff --git a/test/box/cfg.result b/test/box/cfg.result
index 5150337..5108db3 100644
--- a/test/box/cfg.result
+++ b/test/box/cfg.result
@@ -32,6 +32,8 @@ cfg_filter(box.cfg)
     - false
   - - hot_standby
     - false
+  - - ignore_space_formats
+    - false
   - - listen
     - <hidden>
   - - log
@@ -133,6 +135,8 @@ cfg_filter(box.cfg)
     - false
   - - hot_standby
     - false
+  - - ignore_space_formats
+    - false
   - - listen
     - <hidden>
   - - log
-- 
2.7.4

[tarantool-patches] Re: [PATCH v1 1/1] box: option to start tarantool with no format checks.

[Vladislav Shpilevoy]

Hi! Thanks for the patch! LGTM.

On 21/09/2018 20:49, imeevma@tarantool.org wrote:
> It is a common case that an instance is running on a version
> 1.6.*, then is upgraded to 1.9 with box.schema.upgrade(). But
> some of users has malformed space formats, and some of them got
> and ignored errors on box.schema.upgrade(). Such half-upgraded
> data can not be used to start a new version due to format
> violations, and can not be rolled back because some of new system
> spaces managed to be created. And we can not repair raw xlogs and
> snapshots because we have no such tools and they are zipped.
> 
> So it would be useful to have a special box.cfg option
> ignore_space_formats. It would allow to start an instance on
> the spaces with malformed formats, fix them, create a snapshot
> and then start with the formats turned on.
> 
> Closes #3605
> 
> @TarantoolBot document
> Title: box.cfg option 'ignore_space_formats'
> The option allows to turn off space formats validation before
> the instance is started. Useful to fix malformed formats after
> an upgrade from version < 1.7.5 to >= 1.7.5.
> ---
> Branch: https://github.com/tarantool/tarantool/tree/imeevma/gh-3605-ignore-space-formats-option
> Issue: https://github.com/tarantool/tarantool/issues/3605
> 
>   src/box/alter.cc                     | 14 ++++++-
>   src/box/box.h                        |  1 +
>   src/box/lua/load_cfg.lua             |  2 +
>   src/main.cc                          |  1 +
>   test/app-tap/init_script.result      | 77 ++++++++++++++++++------------------
>   test/box-tap/ignore_formats.test.lua | 28 +++++++++++++
>   test/box/admin.result                |  2 +
>   test/box/cfg.result                  |  4 ++
>   8 files changed, 90 insertions(+), 39 deletions(-)
>   create mode 100755 test/box-tap/ignore_formats.test.lua
> 

[tarantool-patches] [PATCH v1 1/1] box: option to start tarantool with no format checks.

[imeevma]

Special option allows Tarantool create spaces with no
format checks. This is useful to fix malformed formats
after an upgrade from version < 1.7.5 to >= 1.7.5.

Closes #3605.
---
Branch: https://github.com/tarantool/tarantool/tree/imeevma/gh-3605-ignore-space-formats-option
Issue: https://github.com/tarantool/tarantool/issues/3605

 src/box/alter.cc                     | 14 ++++++-
 src/box/box.h                        |  1 +
 src/box/lua/cfg.cc                   | 13 +++++++
 src/box/lua/load_cfg.lua             |  3 ++
 src/main.cc                          |  1 +
 test/app-tap/init_script.result      | 71 ++++++++++++++++++------------------
 test/box-tap/ignore_formats.test.lua | 34 +++++++++++++++++
 test/box/admin.result                |  2 +
 test/box/cfg.result                  |  4 ++
 9 files changed, 107 insertions(+), 36 deletions(-)
 create mode 100755 test/box-tap/ignore_formats.test.lua

diff --git a/src/box/alter.cc b/src/box/alter.cc
index ec3b91b..b379dad 100644
--- a/src/box/alter.cc
+++ b/src/box/alter.cc
@@ -54,6 +54,18 @@
 #include "sequence.h"
 
 /**
+ * A flag to ignore space formats and do not validate tuples by
+ * them.
+ */
+static bool ignore_space_formats = false;
+
+void
+box_set_ignore_space_formats(bool value)
+{
+	ignore_space_formats = value;
+}
+
+/**
  * chap-sha1 of empty string, i.e.
  * base64_encode(sha1(sha1(""), 0)
  */
@@ -533,7 +545,7 @@ space_def_new_from_tuple(struct tuple *tuple, uint32_t errcode,
 	const char *space_opts;
 	struct field_def *fields;
 	uint32_t field_count;
-	if (dd_version_id >= version_id(1, 7, 6)) {
+	if (dd_version_id >= version_id(1, 7, 6) && !ignore_space_formats) {
 		/* Check space opts. */
 		space_opts =
 			tuple_field_with_type_xc(tuple, BOX_SPACE_FIELD_OPTS,
diff --git a/src/box/box.h b/src/box/box.h
index 9dfb3fd..d4043db 100644
--- a/src/box/box.h
+++ b/src/box/box.h
@@ -176,6 +176,7 @@ void box_set_vinyl_timeout(void);
 void box_set_replication_timeout(void);
 void box_set_replication_connect_timeout(void);
 void box_set_replication_connect_quorum(void);
+void box_set_ignore_space_formats(bool value);
 
 extern "C" {
 #endif /* defined(__cplusplus) */
diff --git a/src/box/lua/cfg.cc b/src/box/lua/cfg.cc
index 0ca1508..c4237ba 100644
--- a/src/box/lua/cfg.cc
+++ b/src/box/lua/cfg.cc
@@ -177,6 +177,18 @@ lbox_cfg_set_read_only(struct lua_State *L)
 }
 
 static int
+lbox_cfg_set_ignore_space_formats(struct lua_State *L)
+{
+	try {
+		box_set_ignore_space_formats(cfg_geti("ignore_space_"\
+						      "formats") != 0);
+	} catch (Exception *) {
+		luaT_error(L);
+	}
+	return 0;
+}
+
+static int
 lbox_cfg_set_memtx_max_tuple_size(struct lua_State *L)
 {
 	try {
@@ -279,6 +291,7 @@ box_lua_cfg_init(struct lua_State *L)
 		{"cfg_set_snap_io_rate_limit", lbox_cfg_set_snap_io_rate_limit},
 		{"cfg_set_checkpoint_count", lbox_cfg_set_checkpoint_count},
 		{"cfg_set_read_only", lbox_cfg_set_read_only},
+		{"cfg_set_ignore_space_formats", lbox_cfg_set_ignore_space_formats},
 		{"cfg_set_memtx_max_tuple_size", lbox_cfg_set_memtx_max_tuple_size},
 		{"cfg_set_vinyl_max_tuple_size", lbox_cfg_set_vinyl_max_tuple_size},
 		{"cfg_set_vinyl_cache", lbox_cfg_set_vinyl_cache},
diff --git a/src/box/lua/load_cfg.lua b/src/box/lua/load_cfg.lua
index 2a7142d..8dfcbf0 100644
--- a/src/box/lua/load_cfg.lua
+++ b/src/box/lua/load_cfg.lua
@@ -66,6 +66,7 @@ local default_cfg = {
     username            = nil,
     coredump            = false,
     read_only           = false,
+    ignore_space_formats= false,
     hot_standby         = false,
     checkpoint_interval = 3600,
     checkpoint_count    = 2,
@@ -124,6 +125,7 @@ local template_cfg = {
     checkpoint_interval = 'number',
     checkpoint_count    = 'number',
     read_only           = 'boolean',
+    ignore_space_formats= 'boolean',
     hot_standby         = 'boolean',
     worker_pool_threads = 'number',
     replication_timeout = 'number',
@@ -183,6 +185,7 @@ local dynamic_cfg = {
     too_long_threshold      = private.cfg_set_too_long_threshold,
     snap_io_rate_limit      = private.cfg_set_snap_io_rate_limit,
     read_only               = private.cfg_set_read_only,
+    ignore_space_formats    = private.cfg_set_ignore_space_formats,
     memtx_max_tuple_size    = private.cfg_set_memtx_max_tuple_size,
     vinyl_max_tuple_size    = private.cfg_set_vinyl_max_tuple_size,
     vinyl_cache             = private.cfg_set_vinyl_cache,
diff --git a/src/main.cc b/src/main.cc
index 1682bae..30cc78d 100644
--- a/src/main.cc
+++ b/src/main.cc
@@ -521,6 +521,7 @@ load_cfg()
 
 	title_set_custom(cfg_gets("custom_proc_title"));
 	title_update();
+	box_set_ignore_space_formats(cfg_geti("ignore_space_formats") != 0);
 	box_cfg();
 }
 
diff --git a/test/app-tap/init_script.result b/test/app-tap/init_script.result
index eea9f5b..f8b8994 100644
--- a/test/app-tap/init_script.result
+++ b/test/app-tap/init_script.result
@@ -9,41 +9,42 @@ box.cfg
 4	coredump:false
 5	force_recovery:false
 6	hot_standby:false
-7	listen:port
-8	log:tarantool.log
-9	log_format:plain
-10	log_level:5
-11	log_nonblock:true
-12	memtx_dir:.
-13	memtx_max_tuple_size:1048576
-14	memtx_memory:107374182
-15	memtx_min_tuple_size:16
-16	pid_file:box.pid
-17	read_only:false
-18	readahead:16320
-19	replication_connect_timeout:30
-20	replication_sync_lag:10
-21	replication_timeout:1
-22	rows_per_wal:500000
-23	slab_alloc_factor:1.05
-24	too_long_threshold:0.5
-25	vinyl_bloom_fpr:0.05
-26	vinyl_cache:134217728
-27	vinyl_dir:.
-28	vinyl_max_tuple_size:1048576
-29	vinyl_memory:134217728
-30	vinyl_page_size:8192
-31	vinyl_range_size:1073741824
-32	vinyl_read_threads:1
-33	vinyl_run_count_per_level:2
-34	vinyl_run_size_ratio:3.5
-35	vinyl_timeout:60
-36	vinyl_write_threads:2
-37	wal_dir:.
-38	wal_dir_rescan_delay:2
-39	wal_max_size:268435456
-40	wal_mode:write
-41	worker_pool_threads:4
+7	ignore_space_formats:false
+8	listen:port
+9	log:tarantool.log
+10	log_format:plain
+11	log_level:5
+12	log_nonblock:true
+13	memtx_dir:.
+14	memtx_max_tuple_size:1048576
+15	memtx_memory:107374182
+16	memtx_min_tuple_size:16
+17	pid_file:box.pid
+18	read_only:false
+19	readahead:16320
+20	replication_connect_timeout:30
+21	replication_sync_lag:10
+22	replication_timeout:1
+23	rows_per_wal:500000
+24	slab_alloc_factor:1.05
+25	too_long_threshold:0.5
+26	vinyl_bloom_fpr:0.05
+27	vinyl_cache:134217728
+28	vinyl_dir:.
+29	vinyl_max_tuple_size:1048576
+30	vinyl_memory:134217728
+31	vinyl_page_size:8192
+32	vinyl_range_size:1073741824
+33	vinyl_read_threads:1
+34	vinyl_run_count_per_level:2
+35	vinyl_run_size_ratio:3.5
+36	vinyl_timeout:60
+37	vinyl_write_threads:2
+38	wal_dir:.
+39	wal_dir_rescan_delay:2
+40	wal_max_size:268435456
+41	wal_mode:write
+42	worker_pool_threads:4
 --
 -- Test insert from detached fiber
 --
diff --git a/test/box-tap/ignore_formats.test.lua b/test/box-tap/ignore_formats.test.lua
new file mode 100755
index 0000000..84818ed
--- /dev/null
+++ b/test/box-tap/ignore_formats.test.lua
@@ -0,0 +1,34 @@
+#!/usr/bin/env tarantool
+--
+-- gh-3605: allow to ignore space formats via box.cfg option.
+--
+local tap = require('tap')
+local test = tap.test("ignore_formats")
+test:plan(3)
+
+local format = {}
+format[1] = {'field1', 'unsigned'}
+format[2] = {'field2', 'unsigned'}
+
+-- Default value for ignore_space_formats is 'false'
+box.cfg{}
+local s1 = box.schema.create_space('test1', {format = format})
+local pk1 = s1:create_index('pk')
+test:ok(not pcall(s1.replace, s1, {1, 'string'}), 'cannot violate format')
+
+-- Set ignore_space_formats = true.
+box.cfg{ignore_space_formats=true}
+local s2 = box.schema.create_space('test2', {format = format})
+local pk2 = s2:create_index('pk')
+test:ok(pcall(s2.replace, s2, {1, 'string'}), 'can violate format')
+
+-- Set ignore_space_formats = false.
+box.cfg{ignore_space_formats=false}
+local s3 = box.schema.create_space('test3', {format = format})
+local pk3 = s3:create_index('pk')
+test:ok(not pcall(s3.replace, s3, {1, 'string'}), 'cannot violate format')
+
+s1:drop()
+s2:drop()
+s3:drop()
+os.exit(test:check() == true and 0 or 1)
diff --git a/test/box/admin.result b/test/box/admin.result
index c3e318a..9200ed8 100644
--- a/test/box/admin.result
+++ b/test/box/admin.result
@@ -30,6 +30,8 @@ cfg_filter(box.cfg)
     - false
   - - hot_standby
     - false
+  - - ignore_space_formats
+    - false
   - - listen
     - <hidden>
   - - log
diff --git a/test/box/cfg.result b/test/box/cfg.result
index cda7aa0..fad4b87 100644
--- a/test/box/cfg.result
+++ b/test/box/cfg.result
@@ -26,6 +26,8 @@ cfg_filter(box.cfg)
     - false
   - - hot_standby
     - false
+  - - ignore_space_formats
+    - false
   - - listen
     - <hidden>
   - - log
@@ -115,6 +117,8 @@ cfg_filter(box.cfg)
     - false
   - - hot_standby
     - false
+  - - ignore_space_formats
+    - false
   - - listen
     - <hidden>
   - - log
-- 
2.7.4