From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from localhost (localhost [127.0.0.1]) by turing.freelists.org (Avenir Technologies Mail Multiplex) with ESMTP id 7C95B2BBF5 for ; Fri, 21 Sep 2018 13:49:53 -0400 (EDT) Received: from turing.freelists.org ([127.0.0.1]) by localhost (turing.freelists.org [127.0.0.1]) (amavisd-new, port 10024) with ESMTP id FkP24m__ZqdZ for ; Fri, 21 Sep 2018 13:49:53 -0400 (EDT) Received: from smtp46.i.mail.ru (smtp46.i.mail.ru [94.100.177.106]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by turing.freelists.org (Avenir Technologies Mail Multiplex) with ESMTPS id BDA352BA47 for ; Fri, 21 Sep 2018 13:49:52 -0400 (EDT) From: imeevma@tarantool.org Subject: [tarantool-patches] [PATCH v1 1/1] box: option to start tarantool with no format checks. Date: Fri, 21 Sep 2018 20:49:50 +0300 Message-Id: <5a0af01663110c2ac8b673ae61bce523673c3998.1537551740.git.imeevma@gmail.com> Sender: tarantool-patches-bounce@freelists.org Errors-to: tarantool-patches-bounce@freelists.org Reply-To: tarantool-patches@freelists.org List-help: List-unsubscribe: List-software: Ecartis version 1.0.0 List-Id: tarantool-patches List-subscribe: List-owner: List-post: List-archive: To: tarantool-patches@freelists.org, v.shpilevoy@tarantool.org It is a common case that an instance is running on a version 1.6.*, then is upgraded to 1.9 with box.schema.upgrade(). But some of users has malformed space formats, and some of them got and ignored errors on box.schema.upgrade(). Such half-upgraded data can not be used to start a new version due to format violations, and can not be rolled back because some of new system spaces managed to be created. And we can not repair raw xlogs and snapshots because we have no such tools and they are zipped. So it would be useful to have a special box.cfg option ignore_space_formats. It would allow to start an instance on the spaces with malformed formats, fix them, create a snapshot and then start with the formats turned on. Closes #3605 @TarantoolBot document Title: box.cfg option 'ignore_space_formats' The option allows to turn off space formats validation before the instance is started. Useful to fix malformed formats after an upgrade from version < 1.7.5 to >= 1.7.5. --- Branch: https://github.com/tarantool/tarantool/tree/imeevma/gh-3605-ignore-space-formats-option Issue: https://github.com/tarantool/tarantool/issues/3605 src/box/alter.cc | 14 ++++++- src/box/box.h | 1 + src/box/lua/load_cfg.lua | 2 + src/main.cc | 1 + test/app-tap/init_script.result | 77 ++++++++++++++++++------------------ test/box-tap/ignore_formats.test.lua | 28 +++++++++++++ test/box/admin.result | 2 + test/box/cfg.result | 4 ++ 8 files changed, 90 insertions(+), 39 deletions(-) create mode 100755 test/box-tap/ignore_formats.test.lua diff --git a/src/box/alter.cc b/src/box/alter.cc index a6299a1..045bb1d 100644 --- a/src/box/alter.cc +++ b/src/box/alter.cc @@ -54,6 +54,18 @@ #include "sequence.h" /** + * A flag to ignore space formats and do not validate tuples by + * them. + */ +static bool ignore_space_formats = false; + +void +box_set_ignore_space_formats(bool value) +{ + ignore_space_formats = value; +} + +/** * chap-sha1 of empty string, i.e. * base64_encode(sha1(sha1(""), 0) */ @@ -552,7 +564,7 @@ space_def_new_from_tuple(struct tuple *tuple, uint32_t errcode, const char *space_opts; struct field_def *fields; uint32_t field_count; - if (dd_version_id >= version_id(1, 7, 6)) { + if (dd_version_id >= version_id(1, 7, 6) && !ignore_space_formats) { /* Check space opts. */ space_opts = tuple_field_with_type_xc(tuple, BOX_SPACE_FIELD_OPTS, diff --git a/src/box/box.h b/src/box/box.h index 9930d4a..67eb58a 100644 --- a/src/box/box.h +++ b/src/box/box.h @@ -199,6 +199,7 @@ void box_set_replication_sync_lag(void); void box_set_replication_sync_timeout(void); void box_set_replication_skip_conflict(void); void box_set_net_msg_max(void); +void box_set_ignore_space_formats(bool value); extern "C" { #endif /* defined(__cplusplus) */ diff --git a/src/box/lua/load_cfg.lua b/src/box/lua/load_cfg.lua index f62f4dc..8f330e4 100644 --- a/src/box/lua/load_cfg.lua +++ b/src/box/lua/load_cfg.lua @@ -66,6 +66,7 @@ local default_cfg = { username = nil, coredump = false, read_only = false, + ignore_space_formats= false, hot_standby = false, checkpoint_interval = 3600, checkpoint_count = 2, @@ -130,6 +131,7 @@ local template_cfg = { checkpoint_interval = 'number', checkpoint_count = 'number', read_only = 'boolean', + ignore_space_formats= 'boolean', hot_standby = 'boolean', worker_pool_threads = 'number', replication_timeout = 'number', diff --git a/src/main.cc b/src/main.cc index a36a2b0..61f0911 100644 --- a/src/main.cc +++ b/src/main.cc @@ -522,6 +522,7 @@ load_cfg() title_set_custom(cfg_gets("custom_proc_title")); title_update(); + box_set_ignore_space_formats(cfg_geti("ignore_space_formats") != 0); box_cfg(); } diff --git a/test/app-tap/init_script.result b/test/app-tap/init_script.result index a2191d1..3907ded 100644 --- a/test/app-tap/init_script.result +++ b/test/app-tap/init_script.result @@ -12,44 +12,45 @@ box.cfg 7 feedback_interval:3600 8 force_recovery:false 9 hot_standby:false -10 listen:port -11 log:tarantool.log -12 log_format:plain -13 log_level:5 -14 log_nonblock:true -15 memtx_dir:. -16 memtx_max_tuple_size:1048576 -17 memtx_memory:107374182 -18 memtx_min_tuple_size:16 -19 net_msg_max:768 -20 pid_file:box.pid -21 read_only:false -22 readahead:16320 -23 replication_connect_timeout:30 -24 replication_skip_conflict:false -25 replication_sync_lag:10 -26 replication_sync_timeout:300 -27 replication_timeout:1 -28 rows_per_wal:500000 -29 slab_alloc_factor:1.05 -30 too_long_threshold:0.5 -31 vinyl_bloom_fpr:0.05 -32 vinyl_cache:134217728 -33 vinyl_dir:. -34 vinyl_max_tuple_size:1048576 -35 vinyl_memory:134217728 -36 vinyl_page_size:8192 -37 vinyl_range_size:1073741824 -38 vinyl_read_threads:1 -39 vinyl_run_count_per_level:2 -40 vinyl_run_size_ratio:3.5 -41 vinyl_timeout:60 -42 vinyl_write_threads:4 -43 wal_dir:. -44 wal_dir_rescan_delay:2 -45 wal_max_size:268435456 -46 wal_mode:write -47 worker_pool_threads:4 +10 ignore_space_formats:false +11 listen:port +12 log:tarantool.log +13 log_format:plain +14 log_level:5 +15 log_nonblock:true +16 memtx_dir:. +17 memtx_max_tuple_size:1048576 +18 memtx_memory:107374182 +19 memtx_min_tuple_size:16 +20 net_msg_max:768 +21 pid_file:box.pid +22 read_only:false +23 readahead:16320 +24 replication_connect_timeout:30 +25 replication_skip_conflict:false +26 replication_sync_lag:10 +27 replication_sync_timeout:300 +28 replication_timeout:1 +29 rows_per_wal:500000 +30 slab_alloc_factor:1.05 +31 too_long_threshold:0.5 +32 vinyl_bloom_fpr:0.05 +33 vinyl_cache:134217728 +34 vinyl_dir:. +35 vinyl_max_tuple_size:1048576 +36 vinyl_memory:134217728 +37 vinyl_page_size:8192 +38 vinyl_range_size:1073741824 +39 vinyl_read_threads:1 +40 vinyl_run_count_per_level:2 +41 vinyl_run_size_ratio:3.5 +42 vinyl_timeout:60 +43 vinyl_write_threads:4 +44 wal_dir:. +45 wal_dir_rescan_delay:2 +46 wal_max_size:268435456 +47 wal_mode:write +48 worker_pool_threads:4 -- -- Test insert from detached fiber -- diff --git a/test/box-tap/ignore_formats.test.lua b/test/box-tap/ignore_formats.test.lua new file mode 100755 index 0000000..1e038d6 --- /dev/null +++ b/test/box-tap/ignore_formats.test.lua @@ -0,0 +1,28 @@ +#!/usr/bin/env tarantool +-- +-- gh-3605: allow to ignore space formats via box.cfg option. +-- +local tap = require('tap') +local test = tap.test("ignore_formats") +test:plan(3) + +box.cfg{ignore_space_formats=true} + +local format = {} +format[1] = {'field1', 'unsigned'} +format[2] = {'field2', 'unsigned'} + +local s = box.schema.create_space('t0', {format = format}) +local pk = s:create_index('pk') +test:ok(pcall(s.replace, s, {1, 'string'}), 'can violate format') +-- Is not a dynamic option. +local ok, err = pcall(box.cfg, {ignore_space_formats = false}) +test:ok(not ok, "ignore_space_formats is not a dynamic option") +test:ok(err:match("Can't set option"), "error message") + +s:drop() + +-- Cleanup xlog +box.snapshot() + +os.exit(test:check() == true and 0 or 1) diff --git a/test/box/admin.result b/test/box/admin.result index 8048460..1465e58 100644 --- a/test/box/admin.result +++ b/test/box/admin.result @@ -36,6 +36,8 @@ cfg_filter(box.cfg) - false - - hot_standby - false + - - ignore_space_formats + - false - - listen - - - log diff --git a/test/box/cfg.result b/test/box/cfg.result index 5150337..5108db3 100644 --- a/test/box/cfg.result +++ b/test/box/cfg.result @@ -32,6 +32,8 @@ cfg_filter(box.cfg) - false - - hot_standby - false + - - ignore_space_formats + - false - - listen - - - log @@ -133,6 +135,8 @@ cfg_filter(box.cfg) - false - - hot_standby - false + - - ignore_space_formats + - false - - listen - - - log -- 2.7.4