* [tarantool-patches] [PATCH v3] vshard reload mechanism
@ 2018-07-26 8:27 AKhatskevich
2018-07-26 8:27 ` [tarantool-patches] [PATCH 1/3] Fix races related to object outdating AKhatskevich
` (2 more replies)
0 siblings, 3 replies; 4+ messages in thread
From: AKhatskevich @ 2018-07-26 8:27 UTC (permalink / raw)
To: v.shpilevoy, tarantool-patches
Issue1: https://github.com/tarantool/vshard/issues/112
Issue2: https://github.com/tarantool/vshard/issues/125
Branch: https://github.com/tarantool/vshard/tree/kh/gh-112-reload-mt-2
This patcheset improves vshard reload mechanism.
Changes since PATCH v2:
- Races related to object outdating are fixed.
AKhatskevich (3):
Fix races related to object outdating
tests: separate bootstrap routine to a lua_libs
Introduce storage reload evolution
.travis.yml | 2 +-
rpm/prebuild.sh | 2 +
test/lua_libs/bootstrap.lua | 50 +++++
test/lua_libs/git_util.lua | 51 +++++
test/lua_libs/util.lua | 20 ++
test/rebalancer/box_1_a.lua | 47 +---
test/rebalancer/errinj.result | 2 +-
test/rebalancer/errinj.test.lua | 2 +-
test/rebalancer/rebalancer.result | 2 +-
test/rebalancer/rebalancer.test.lua | 2 +-
test/rebalancer/rebalancer_lock_and_pin.result | 2 +-
test/rebalancer/rebalancer_lock_and_pin.test.lua | 2 +-
test/rebalancer/restart_during_rebalancing.result | 2 +-
.../rebalancer/restart_during_rebalancing.test.lua | 2 +-
test/rebalancer/stress_add_remove_rs.result | 2 +-
test/rebalancer/stress_add_remove_rs.test.lua | 2 +-
.../rebalancer/stress_add_remove_several_rs.result | 2 +-
.../stress_add_remove_several_rs.test.lua | 2 +-
test/rebalancer/suite.ini | 2 +-
test/reload_evolution/storage.result | 245 +++++++++++++++++++++
test/reload_evolution/storage.test.lua | 87 ++++++++
test/reload_evolution/storage_1_a.lua | 48 ++++
test/reload_evolution/storage_1_b.lua | 1 +
test/reload_evolution/storage_2_a.lua | 1 +
test/reload_evolution/storage_2_b.lua | 1 +
test/reload_evolution/suite.ini | 6 +
test/reload_evolution/test.lua | 9 +
test/unit/reload_evolution.result | 45 ++++
test/unit/reload_evolution.test.lua | 18 ++
vshard/replicaset.lua | 30 +--
vshard/router/init.lua | 58 +++--
vshard/storage/init.lua | 12 +
vshard/storage/reload_evolution.lua | 58 +++++
33 files changed, 722 insertions(+), 95 deletions(-)
create mode 100644 test/lua_libs/bootstrap.lua
create mode 100644 test/lua_libs/git_util.lua
create mode 100644 test/reload_evolution/storage.result
create mode 100644 test/reload_evolution/storage.test.lua
create mode 100755 test/reload_evolution/storage_1_a.lua
create mode 120000 test/reload_evolution/storage_1_b.lua
create mode 120000 test/reload_evolution/storage_2_a.lua
create mode 120000 test/reload_evolution/storage_2_b.lua
create mode 100644 test/reload_evolution/suite.ini
create mode 100644 test/reload_evolution/test.lua
create mode 100644 test/unit/reload_evolution.result
create mode 100644 test/unit/reload_evolution.test.lua
create mode 100644 vshard/storage/reload_evolution.lua
--
2.14.1
^ permalink raw reply [flat|nested] 4+ messages in thread
* [tarantool-patches] [PATCH 1/3] Fix races related to object outdating
2018-07-26 8:27 [tarantool-patches] [PATCH v3] vshard reload mechanism AKhatskevich
@ 2018-07-26 8:27 ` AKhatskevich
2018-07-26 8:27 ` [tarantool-patches] [PATCH 2/3] tests: separate bootstrap routine to a lua_libs AKhatskevich
2018-07-26 8:27 ` [tarantool-patches] [PATCH 3/3] Introduce storage reload evolution AKhatskevich
2 siblings, 0 replies; 4+ messages in thread
From: AKhatskevich @ 2018-07-26 8:27 UTC (permalink / raw)
To: v.shpilevoy, tarantool-patches
Reload/reconfigure may replace many of M fields during any yield.
Old objects should not be accessed after they are outdated.
This commit handles such cases within `vshard.router`.
---
vshard/replicaset.lua | 30 ++++++++++++++-----------
vshard/router/init.lua | 58 +++++++++++++++++++++++++++++--------------------
vshard/storage/init.lua | 1 +
3 files changed, 52 insertions(+), 37 deletions(-)
diff --git a/vshard/replicaset.lua b/vshard/replicaset.lua
index 6c8d477..87e26d3 100644
--- a/vshard/replicaset.lua
+++ b/vshard/replicaset.lua
@@ -340,16 +340,13 @@ local function replicaset_tostring(replicaset)
master)
end
-local outdate_replicasets
--
-- Copy netbox connections from old replica objects to new ones
-- and outdate old objects.
-- @param replicasets New replicasets
-- @param old_replicasets Replicasets and replicas to be outdated.
--- @param outdate_delay Number of seconds; delay to outdate
--- old objects.
--
-local function rebind_replicasets(replicasets, old_replicasets, outdate_delay)
+local function rebind_replicasets(replicasets, old_replicasets)
for replicaset_uuid, replicaset in pairs(replicasets) do
local old_replicaset = old_replicasets and
old_replicasets[replicaset_uuid]
@@ -370,9 +367,6 @@ local function rebind_replicasets(replicasets, old_replicasets, outdate_delay)
end
end
end
- if old_replicasets then
- util.async_task(outdate_delay, outdate_replicasets, old_replicasets)
- end
end
--
@@ -453,12 +447,7 @@ for fname, func in pairs(replica_mt.__index) do
outdated_replica_mt.__index[fname] = outdated_warning
end
---
--- Outdate replicaset and replica objects:
--- * Set outdated_metatables.
--- * Remove connections.
---
-outdate_replicasets = function(replicasets)
+local outdate_replicasets_internal = function(replicasets)
for _, replicaset in pairs(replicasets) do
setmetatable(replicaset, outdated_replicaset_mt)
for _, replica in pairs(replicaset.replicas) do
@@ -469,6 +458,20 @@ outdate_replicasets = function(replicasets)
log.info('Old replicaset and replica objects are outdated.')
end
+--
+-- Outdate replicaset and replica objects:
+-- * Set outdated_metatables.
+-- * Remove connections.
+-- @param replicasets Old replicasets to be outdated.
+-- @param outdate_delay Delay in seconds before the outdating.
+--
+local function outdate_replicasets(replicasets, outdate_delay)
+ if replicasets then
+ util.async_task(outdate_delay, outdate_replicasets_internal,
+ replicasets)
+ end
+end
+
--
-- Calculate for each replicaset its etalon bucket count.
-- Iterative algorithm is used to learn the best balance in a
@@ -650,4 +653,5 @@ return {
calculate_etalon_balance = cluster_calculate_etalon_balance,
wait_masters_connect = wait_masters_connect,
rebind_replicasets = rebind_replicasets,
+ outdate_replicasets = outdate_replicasets,
}
diff --git a/vshard/router/init.lua b/vshard/router/init.lua
index 142ddb6..1a0ed2f 100644
--- a/vshard/router/init.lua
+++ b/vshard/router/init.lua
@@ -52,9 +52,14 @@ if not M then
}
end
--- Set a replicaset by container of a bucket.
-local function bucket_set(bucket_id, replicaset)
- assert(replicaset)
+-- Set a bucket to a replicaset.
+local function bucket_set(bucket_id, rs_uuid)
+ local replicaset = M.replicasets[rs_uuid]
+ -- It is technically possible to delete a replicaset at the
+ -- same time when route to the bucket is discovered.
+ if not replicaset then
+ return nil, lerror.vshard(lerror.code.NO_ROUTE_TO_BUCKET, bucket_id)
+ end
local old_replicaset = M.route_map[bucket_id]
if old_replicaset ~= replicaset then
if old_replicaset then
@@ -63,6 +68,7 @@ local function bucket_set(bucket_id, replicaset)
replicaset.bucket_count = replicaset.bucket_count + 1
end
M.route_map[bucket_id] = replicaset
+ return replicaset
end
-- Remove a bucket from the cache.
@@ -88,15 +94,18 @@ local function bucket_discovery(bucket_id)
log.verbose("Discovering bucket %d", bucket_id)
local last_err = nil
local unreachable_uuid = nil
- for uuid, replicaset in pairs(M.replicasets) do
- local _, err =
- replicaset:callrw('vshard.storage.bucket_stat', {bucket_id})
- if err == nil then
- bucket_set(bucket_id, replicaset)
- return replicaset
- elseif err.code ~= lerror.code.WRONG_BUCKET then
- last_err = err
- unreachable_uuid = uuid
+ for uuid, _ in pairs(M.replicasets) do
+ -- Handle reload/reconfigure.
+ replicaset = M.replicasets[uuid]
+ if replicaset then
+ local _, err =
+ replicaset:callrw('vshard.storage.bucket_stat', {bucket_id})
+ if err == nil then
+ return bucket_set(bucket_id, replicaset.uuid)
+ elseif err.code ~= lerror.code.WRONG_BUCKET then
+ last_err = err
+ unreachable_uuid = uuid
+ end
end
end
local err = nil
@@ -262,13 +271,13 @@ local function router_call(bucket_id, mode, func, args, opts)
end
end
else
- bucket_set(bucket_id, replicaset)
+ replicaset = bucket_set(bucket_id, replicaset.uuid)
lfiber.yield()
-- Protect against infinite cycle in a
-- case of broken cluster, when a bucket
-- is sent on two replicasets to each
-- other.
- if lfiber.time() <= tend then
+ if replicaset and lfiber.time() <= tend then
goto replicaset_is_found
end
end
@@ -513,27 +522,28 @@ local function router_cfg(cfg)
end
box.cfg(box_cfg)
log.info("Box has been configured")
- M.connection_outdate_delay = cfg.connection_outdate_delay
- M.total_bucket_count = total_bucket_count
- M.collect_lua_garbage = collect_lua_garbage
- M.current_cfg = new_cfg
-- Move connections from an old configuration to a new one.
-- It must be done with no yields to prevent usage both of not
-- fully moved old replicasets, and not fully built new ones.
- lreplicaset.rebind_replicasets(new_replicasets, M.replicasets,
- M.connection_outdate_delay)
- M.replicasets = new_replicasets
+ lreplicaset.rebind_replicasets(new_replicasets, M.replicasets)
-- Now the new replicasets are fully built. Can establish
-- connections and yield.
for _, replicaset in pairs(new_replicasets) do
replicaset:connect_all()
end
+ lreplicaset.wait_masters_connect(new_replicasets)
+ lreplicaset.outdate_replicasets(M.replicasets, cfg.connection_outdate_delay)
+ M.connection_outdate_delay = cfg.connection_outdate_delay
+ M.total_bucket_count = total_bucket_count
+ M.collect_lua_garbage = collect_lua_garbage
+ M.current_cfg = cfg
+ M.replicasets = new_replicasets
-- Update existing route map in-place.
- for bucket, rs in pairs(M.route_map) do
+ local old_route_map = M.route_map
+ M.route_map = {}
+ for bucket, rs in pairs(old_route_map) do
M.route_map[bucket] = M.replicasets[rs.uuid]
end
-
- lreplicaset.wait_masters_connect(new_replicasets)
if M.failover_fiber == nil then
lfiber.create(util.reloadable_fiber_f, M, 'failover_f', 'Failover')
end
diff --git a/vshard/storage/init.lua b/vshard/storage/init.lua
index 07bd00c..c1df0e6 100644
--- a/vshard/storage/init.lua
+++ b/vshard/storage/init.lua
@@ -1620,6 +1620,7 @@ local function storage_cfg(cfg, this_replica_uuid)
box.once("vshard:storage:1", storage_schema_v1, uri.login, uri.password)
lreplicaset.rebind_replicasets(new_replicasets, M.replicasets)
+ lreplicaset.outdate_replicasets(M.replicasets)
M.replicasets = new_replicasets
M.this_replicaset = this_replicaset
M.this_replica = this_replica
--
2.14.1
^ permalink raw reply [flat|nested] 4+ messages in thread
* [tarantool-patches] [PATCH 2/3] tests: separate bootstrap routine to a lua_libs
2018-07-26 8:27 [tarantool-patches] [PATCH v3] vshard reload mechanism AKhatskevich
2018-07-26 8:27 ` [tarantool-patches] [PATCH 1/3] Fix races related to object outdating AKhatskevich
@ 2018-07-26 8:27 ` AKhatskevich
2018-07-26 8:27 ` [tarantool-patches] [PATCH 3/3] Introduce storage reload evolution AKhatskevich
2 siblings, 0 replies; 4+ messages in thread
From: AKhatskevich @ 2018-07-26 8:27 UTC (permalink / raw)
To: v.shpilevoy, tarantool-patches
What is moved to `test/lua_libs/bootstrap.lua`:
1. create schema
2. create main stored procedures
3. `wait_rebalancer_state` procedure
This code will be reused in further commits.
---
test/lua_libs/bootstrap.lua | 50 ++++++++++++++++++++++
test/rebalancer/box_1_a.lua | 47 ++------------------
test/rebalancer/errinj.result | 2 +-
test/rebalancer/errinj.test.lua | 2 +-
test/rebalancer/rebalancer.result | 2 +-
test/rebalancer/rebalancer.test.lua | 2 +-
test/rebalancer/rebalancer_lock_and_pin.result | 2 +-
test/rebalancer/rebalancer_lock_and_pin.test.lua | 2 +-
test/rebalancer/restart_during_rebalancing.result | 2 +-
.../rebalancer/restart_during_rebalancing.test.lua | 2 +-
test/rebalancer/stress_add_remove_rs.result | 2 +-
test/rebalancer/stress_add_remove_rs.test.lua | 2 +-
.../rebalancer/stress_add_remove_several_rs.result | 2 +-
.../stress_add_remove_several_rs.test.lua | 2 +-
test/rebalancer/suite.ini | 2 +-
15 files changed, 66 insertions(+), 57 deletions(-)
create mode 100644 test/lua_libs/bootstrap.lua
diff --git a/test/lua_libs/bootstrap.lua b/test/lua_libs/bootstrap.lua
new file mode 100644
index 0000000..62c2f78
--- /dev/null
+++ b/test/lua_libs/bootstrap.lua
@@ -0,0 +1,50 @@
+local log = require('log')
+
+function init_schema()
+ local format = {}
+ format[1] = {name = 'field', type = 'unsigned'}
+ format[2] = {name = 'bucket_id', type = 'unsigned'}
+ local s = box.schema.create_space('test', {format = format})
+ local pk = s:create_index('pk')
+ local bucket_id_idx =
+ s:create_index('vbucket', {parts = {'bucket_id'},
+ unique = false})
+end
+
+box.once('schema', function()
+ box.schema.func.create('do_replace')
+ box.schema.role.grant('public', 'execute', 'function', 'do_replace')
+ box.schema.func.create('do_select')
+ box.schema.role.grant('public', 'execute', 'function', 'do_select')
+ init_schema()
+end)
+
+function do_replace(...)
+ box.space.test:replace(...)
+ return true
+end
+
+function do_select(...)
+ return box.space.test:select(...)
+end
+
+function check_consistency()
+ for _, tuple in box.space.test:pairs() do
+ assert(box.space._bucket:get{tuple.bucket_id})
+ end
+ return true
+end
+
+--
+-- Wait a specified log message.
+-- Requirements:
+-- * Should be executed from a storage with a rebalancer.
+-- * NAME - global variable, name of instance should be set.
+function wait_rebalancer_state(state, test_run)
+ log.info(string.rep('a', 1000))
+ vshard.storage.rebalancer_wakeup()
+ while not test_run:grep_log(NAME, state, 1000) do
+ fiber.sleep(0.1)
+ vshard.storage.rebalancer_wakeup()
+ end
+end
diff --git a/test/rebalancer/box_1_a.lua b/test/rebalancer/box_1_a.lua
index 8fddcf0..2ca8306 100644
--- a/test/rebalancer/box_1_a.lua
+++ b/test/rebalancer/box_1_a.lua
@@ -2,7 +2,7 @@
-- Get instance name
require('strict').on()
local fio = require('fio')
-local NAME = fio.basename(arg[0], '.lua')
+NAME = fio.basename(arg[0], '.lua')
log = require('log')
require('console').listen(os.getenv('ADMIN'))
fiber = require('fiber')
@@ -23,40 +23,8 @@ if NAME == 'box_4_a' or NAME == 'box_4_b' or
end
vshard.storage.cfg(cfg, names.replica_uuid[NAME])
-function init_schema()
- local format = {}
- format[1] = {name = 'field', type = 'unsigned'}
- format[2] = {name = 'bucket_id', type = 'unsigned'}
- local s = box.schema.create_space('test', {format = format})
- local pk = s:create_index('pk')
- local bucket_id_idx =
- s:create_index('vbucket', {parts = {'bucket_id'},
- unique = false})
-end
-
-box.once('schema', function()
- box.schema.func.create('do_replace')
- box.schema.role.grant('public', 'execute', 'function', 'do_replace')
- box.schema.func.create('do_select')
- box.schema.role.grant('public', 'execute', 'function', 'do_select')
- init_schema()
-end)
-
-function do_replace(...)
- box.space.test:replace(...)
- return true
-end
-
-function do_select(...)
- return box.space.test:select(...)
-end
-
-function check_consistency()
- for _, tuple in box.space.test:pairs() do
- assert(box.space._bucket:get{tuple.bucket_id})
- end
- return true
-end
+-- Bootstrap storage.
+require('lua_libs.bootstrap')
function switch_rs1_master()
local replica_uuid = names.replica_uuid
@@ -68,12 +36,3 @@ end
function nullify_rs_weight()
cfg.sharding[names.rs_uuid[1]].weight = 0
end
-
-function wait_rebalancer_state(state, test_run)
- log.info(string.rep('a', 1000))
- vshard.storage.rebalancer_wakeup()
- while not test_run:grep_log(NAME, state, 1000) do
- fiber.sleep(0.1)
- vshard.storage.rebalancer_wakeup()
- end
-end
diff --git a/test/rebalancer/errinj.result b/test/rebalancer/errinj.result
index d09349e..826c2c6 100644
--- a/test/rebalancer/errinj.result
+++ b/test/rebalancer/errinj.result
@@ -13,7 +13,7 @@ test_run:create_cluster(REPLICASET_1, 'rebalancer')
test_run:create_cluster(REPLICASET_2, 'rebalancer')
---
...
-util = require('util')
+util = require('lua_libs.util')
---
...
util.wait_master(test_run, REPLICASET_1, 'box_1_a')
diff --git a/test/rebalancer/errinj.test.lua b/test/rebalancer/errinj.test.lua
index d6a2920..fc0730c 100644
--- a/test/rebalancer/errinj.test.lua
+++ b/test/rebalancer/errinj.test.lua
@@ -5,7 +5,7 @@ REPLICASET_2 = { 'box_2_a', 'box_2_b' }
test_run:create_cluster(REPLICASET_1, 'rebalancer')
test_run:create_cluster(REPLICASET_2, 'rebalancer')
-util = require('util')
+util = require('lua_libs.util')
util.wait_master(test_run, REPLICASET_1, 'box_1_a')
util.wait_master(test_run, REPLICASET_2, 'box_2_a')
diff --git a/test/rebalancer/rebalancer.result b/test/rebalancer/rebalancer.result
index 88cbaae..71e43e1 100644
--- a/test/rebalancer/rebalancer.result
+++ b/test/rebalancer/rebalancer.result
@@ -13,7 +13,7 @@ test_run:create_cluster(REPLICASET_1, 'rebalancer')
test_run:create_cluster(REPLICASET_2, 'rebalancer')
---
...
-util = require('util')
+util = require('lua_libs.util')
---
...
util.wait_master(test_run, REPLICASET_1, 'box_1_a')
diff --git a/test/rebalancer/rebalancer.test.lua b/test/rebalancer/rebalancer.test.lua
index 01f2061..1b7ddae 100644
--- a/test/rebalancer/rebalancer.test.lua
+++ b/test/rebalancer/rebalancer.test.lua
@@ -5,7 +5,7 @@ REPLICASET_2 = { 'box_2_a', 'box_2_b' }
test_run:create_cluster(REPLICASET_1, 'rebalancer')
test_run:create_cluster(REPLICASET_2, 'rebalancer')
-util = require('util')
+util = require('lua_libs.util')
util.wait_master(test_run, REPLICASET_1, 'box_1_a')
util.wait_master(test_run, REPLICASET_2, 'box_2_a')
diff --git a/test/rebalancer/rebalancer_lock_and_pin.result b/test/rebalancer/rebalancer_lock_and_pin.result
index dd9fe47..0f2921c 100644
--- a/test/rebalancer/rebalancer_lock_and_pin.result
+++ b/test/rebalancer/rebalancer_lock_and_pin.result
@@ -16,7 +16,7 @@ test_run:create_cluster(REPLICASET_1, 'rebalancer')
test_run:create_cluster(REPLICASET_2, 'rebalancer')
---
...
-util = require('util')
+util = require('lua_libs.util')
---
...
util.wait_master(test_run, REPLICASET_1, 'box_1_a')
diff --git a/test/rebalancer/rebalancer_lock_and_pin.test.lua b/test/rebalancer/rebalancer_lock_and_pin.test.lua
index fe866c4..3a2daa0 100644
--- a/test/rebalancer/rebalancer_lock_and_pin.test.lua
+++ b/test/rebalancer/rebalancer_lock_and_pin.test.lua
@@ -6,7 +6,7 @@ REPLICASET_3 = { 'box_3_a', 'box_3_b' }
test_run:create_cluster(REPLICASET_1, 'rebalancer')
test_run:create_cluster(REPLICASET_2, 'rebalancer')
-util = require('util')
+util = require('lua_libs.util')
util.wait_master(test_run, REPLICASET_1, 'box_1_a')
util.wait_master(test_run, REPLICASET_2, 'box_2_a')
diff --git a/test/rebalancer/restart_during_rebalancing.result b/test/rebalancer/restart_during_rebalancing.result
index d2b8a12..0eb0f2e 100644
--- a/test/rebalancer/restart_during_rebalancing.result
+++ b/test/rebalancer/restart_during_rebalancing.result
@@ -25,7 +25,7 @@ test_run:create_cluster(REPLICASET_3, 'rebalancer')
test_run:create_cluster(REPLICASET_4, 'rebalancer')
---
...
-util = require('util')
+util = require('lua_libs.util')
---
...
util.wait_master(test_run, REPLICASET_1, 'fullbox_1_a')
diff --git a/test/rebalancer/restart_during_rebalancing.test.lua b/test/rebalancer/restart_during_rebalancing.test.lua
index 5b1a8df..7b707ca 100644
--- a/test/rebalancer/restart_during_rebalancing.test.lua
+++ b/test/rebalancer/restart_during_rebalancing.test.lua
@@ -9,7 +9,7 @@ test_run:create_cluster(REPLICASET_1, 'rebalancer')
test_run:create_cluster(REPLICASET_2, 'rebalancer')
test_run:create_cluster(REPLICASET_3, 'rebalancer')
test_run:create_cluster(REPLICASET_4, 'rebalancer')
-util = require('util')
+util = require('lua_libs.util')
util.wait_master(test_run, REPLICASET_1, 'fullbox_1_a')
util.wait_master(test_run, REPLICASET_2, 'fullbox_2_a')
util.wait_master(test_run, REPLICASET_3, 'fullbox_3_a')
diff --git a/test/rebalancer/stress_add_remove_rs.result b/test/rebalancer/stress_add_remove_rs.result
index 8a955e2..10bcaac 100644
--- a/test/rebalancer/stress_add_remove_rs.result
+++ b/test/rebalancer/stress_add_remove_rs.result
@@ -16,7 +16,7 @@ test_run:create_cluster(REPLICASET_1, 'rebalancer')
test_run:create_cluster(REPLICASET_2, 'rebalancer')
---
...
-util = require('util')
+util = require('lua_libs.util')
---
...
util.wait_master(test_run, REPLICASET_1, 'box_1_a')
diff --git a/test/rebalancer/stress_add_remove_rs.test.lua b/test/rebalancer/stress_add_remove_rs.test.lua
index c80df40..b9bb027 100644
--- a/test/rebalancer/stress_add_remove_rs.test.lua
+++ b/test/rebalancer/stress_add_remove_rs.test.lua
@@ -6,7 +6,7 @@ REPLICASET_3 = { 'box_3_a', 'box_3_b' }
test_run:create_cluster(REPLICASET_1, 'rebalancer')
test_run:create_cluster(REPLICASET_2, 'rebalancer')
-util = require('util')
+util = require('lua_libs.util')
util.wait_master(test_run, REPLICASET_1, 'box_1_a')
util.wait_master(test_run, REPLICASET_2, 'box_2_a')
diff --git a/test/rebalancer/stress_add_remove_several_rs.result b/test/rebalancer/stress_add_remove_several_rs.result
index d6008b8..611362c 100644
--- a/test/rebalancer/stress_add_remove_several_rs.result
+++ b/test/rebalancer/stress_add_remove_several_rs.result
@@ -19,7 +19,7 @@ test_run:create_cluster(REPLICASET_1, 'rebalancer')
test_run:create_cluster(REPLICASET_2, 'rebalancer')
---
...
-util = require('util')
+util = require('lua_libs.util')
---
...
util.wait_master(test_run, REPLICASET_1, 'box_1_a')
diff --git a/test/rebalancer/stress_add_remove_several_rs.test.lua b/test/rebalancer/stress_add_remove_several_rs.test.lua
index 3cc105e..9acb8de 100644
--- a/test/rebalancer/stress_add_remove_several_rs.test.lua
+++ b/test/rebalancer/stress_add_remove_several_rs.test.lua
@@ -7,7 +7,7 @@ REPLICASET_4 = { 'box_4_a', 'box_4_b' }
test_run:create_cluster(REPLICASET_1, 'rebalancer')
test_run:create_cluster(REPLICASET_2, 'rebalancer')
-util = require('util')
+util = require('lua_libs.util')
util.wait_master(test_run, REPLICASET_1, 'box_1_a')
util.wait_master(test_run, REPLICASET_2, 'box_2_a')
diff --git a/test/rebalancer/suite.ini b/test/rebalancer/suite.ini
index afc5141..8689da5 100644
--- a/test/rebalancer/suite.ini
+++ b/test/rebalancer/suite.ini
@@ -4,6 +4,6 @@ description = Rebalancer tests
script = test.lua
is_parallel = False
release_disabled = errinj.test.lua
-lua_libs = ../lua_libs/util.lua config.lua names.lua router_1.lua
+lua_libs = ../lua_libs config.lua names.lua router_1.lua
box_1_a.lua box_1_b.lua box_2_a.lua box_2_b.lua
box_3_a.lua box_3_b.lua rebalancer_utils.lua
--
2.14.1
^ permalink raw reply [flat|nested] 4+ messages in thread
* [tarantool-patches] [PATCH 3/3] Introduce storage reload evolution
2018-07-26 8:27 [tarantool-patches] [PATCH v3] vshard reload mechanism AKhatskevich
2018-07-26 8:27 ` [tarantool-patches] [PATCH 1/3] Fix races related to object outdating AKhatskevich
2018-07-26 8:27 ` [tarantool-patches] [PATCH 2/3] tests: separate bootstrap routine to a lua_libs AKhatskevich
@ 2018-07-26 8:27 ` AKhatskevich
2 siblings, 0 replies; 4+ messages in thread
From: AKhatskevich @ 2018-07-26 8:27 UTC (permalink / raw)
To: v.shpilevoy, tarantool-patches
Changes:
1. Introduce storage reload evolution.
2. Setup cross-version reload testing.
1:
This mechanism updates Lua objects on reload in case they are
changed in a new vshard.storage version.
Since this commit, any change in vshard.storage.M has to be
reflected in vshard.storage.reload_evolution to guarantee
correct reload.
2:
The testing uses git infrastructure and is performed in the following
way:
1. Copy old version of vshard to a temp folder.
2. Run vshard on this code.
3. Checkout the latest version of the vshard sources.
4. Reload vshard storage.
5. Make sure it works (Perform simple tests).
Notes:
* this patch contains some legacy-driven decisions:
1. SOURCEDIR path retrieved differently in case of
packpack build.
2. git directory in the `reload_evolution/storage` test
is copied with respect to Centos 7 and `ro` mode of
SOURCEDIR.
Closes #112 #125
---
.travis.yml | 2 +-
rpm/prebuild.sh | 2 +
test/lua_libs/git_util.lua | 51 +++++++
test/lua_libs/util.lua | 20 +++
test/reload_evolution/storage.result | 245 +++++++++++++++++++++++++++++++++
test/reload_evolution/storage.test.lua | 87 ++++++++++++
test/reload_evolution/storage_1_a.lua | 48 +++++++
test/reload_evolution/storage_1_b.lua | 1 +
test/reload_evolution/storage_2_a.lua | 1 +
test/reload_evolution/storage_2_b.lua | 1 +
test/reload_evolution/suite.ini | 6 +
test/reload_evolution/test.lua | 9 ++
test/unit/reload_evolution.result | 45 ++++++
test/unit/reload_evolution.test.lua | 18 +++
vshard/storage/init.lua | 11 ++
vshard/storage/reload_evolution.lua | 58 ++++++++
16 files changed, 604 insertions(+), 1 deletion(-)
create mode 100644 test/lua_libs/git_util.lua
create mode 100644 test/reload_evolution/storage.result
create mode 100644 test/reload_evolution/storage.test.lua
create mode 100755 test/reload_evolution/storage_1_a.lua
create mode 120000 test/reload_evolution/storage_1_b.lua
create mode 120000 test/reload_evolution/storage_2_a.lua
create mode 120000 test/reload_evolution/storage_2_b.lua
create mode 100644 test/reload_evolution/suite.ini
create mode 100644 test/reload_evolution/test.lua
create mode 100644 test/unit/reload_evolution.result
create mode 100644 test/unit/reload_evolution.test.lua
create mode 100644 vshard/storage/reload_evolution.lua
diff --git a/.travis.yml b/.travis.yml
index 54bfe44..eff4a51 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -41,7 +41,7 @@ env:
script:
- git describe --long
- git clone https://github.com/packpack/packpack.git packpack
- - packpack/packpack
+ - packpack/packpack -e PACKPACK_GIT_SOURCEDIR=/source/
before_deploy:
- ls -l build/
diff --git a/rpm/prebuild.sh b/rpm/prebuild.sh
index 768b22b..554032b 100755
--- a/rpm/prebuild.sh
+++ b/rpm/prebuild.sh
@@ -1 +1,3 @@
curl -s https://packagecloud.io/install/repositories/tarantool/1_9/script.rpm.sh | sudo bash
+sudo yum -y install python-devel python-pip
+sudo pip install tarantool msgpack
diff --git a/test/lua_libs/git_util.lua b/test/lua_libs/git_util.lua
new file mode 100644
index 0000000..a75bb08
--- /dev/null
+++ b/test/lua_libs/git_util.lua
@@ -0,0 +1,51 @@
+--
+-- Lua bridge for some of the git commands.
+--
+local os = require('os')
+
+local temp_file = 'some_strange_rare_unique_file_name_for_git_util'
+
+--
+-- Exec a git command.
+-- @param params Table of parameters:
+-- * options - git options.
+-- * cmd - git command.
+-- * args - command arguments.
+-- * dir - working directory.
+-- * fout - write output to the file.
+local function exec_cmd(params)
+ local fout = params.fout
+ local shell_cmd = {'git'}
+ for _, param in pairs({'options', 'cmd', 'args'}) do
+ table.insert(shell_cmd, params[param])
+ end
+ if fout then
+ table.insert(shell_cmd, ' >' .. fout)
+ end
+ shell_cmd = table.concat(shell_cmd, ' ')
+ if params.dir then
+ shell_cmd = string.format('cd %s && %s', params.dir, shell_cmd)
+ end
+ local res = os.execute(shell_cmd)
+ assert(res == 0, 'Git cmd error: ' .. res)
+end
+
+local function log_hashes(params)
+ params.args = "--format='%h' " .. params.args
+ local local_temp_file = string.format('%s/%s', os.getenv('PWD'), temp_file)
+ params.fout = local_temp_file
+ params.cmd = 'log'
+ exec_cmd(params)
+ local lines = {}
+ for line in io.lines(local_temp_file) do
+ table.insert(lines, line)
+ end
+ os.remove(local_temp_file)
+ return lines
+end
+
+
+return {
+ exec_cmd = exec_cmd,
+ log_hashes = log_hashes
+}
diff --git a/test/lua_libs/util.lua b/test/lua_libs/util.lua
index f40d3a6..935ff41 100644
--- a/test/lua_libs/util.lua
+++ b/test/lua_libs/util.lua
@@ -1,5 +1,6 @@
local fiber = require('fiber')
local log = require('log')
+local fio = require('fio')
local function check_error(func, ...)
local pstatus, status, err = pcall(func, ...)
@@ -92,10 +93,29 @@ local function has_same_fields(etalon, data)
return true
end
+-- Git directory of the project. Used in evolution tests to
+-- fetch old versions of vshard.
+local SOURCEDIR = os.getenv('PACKPACK_GIT_SOURCEDIR')
+if not SOURCEDIR then
+ SOURCEDIR = os.getenv('SOURCEDIR')
+end
+if not SOURCEDIR then
+ local script_path = debug.getinfo(1).source:match("@?(.*/)")
+ script_path = fio.abspath(script_path)
+ SOURCEDIR = fio.abspath(script_path .. '/../../../')
+end
+
+local BUILDDIR = os.getenv('BUILDDIR')
+if not BUILDDIR then
+ BUILDDIR = SOURCEDIR
+end
+
return {
check_error = check_error,
shuffle_masters = shuffle_masters,
collect_timeouts = collect_timeouts,
wait_master = wait_master,
has_same_fields = has_same_fields,
+ SOURCEDIR = SOURCEDIR,
+ BUILDDIR = BUILDDIR,
}
diff --git a/test/reload_evolution/storage.result b/test/reload_evolution/storage.result
new file mode 100644
index 0000000..007192c
--- /dev/null
+++ b/test/reload_evolution/storage.result
@@ -0,0 +1,245 @@
+test_run = require('test_run').new()
+---
+...
+git_util = require('lua_libs.git_util')
+---
+...
+util = require('lua_libs.util')
+---
+...
+vshard_copy_path = util.BUILDDIR .. '/test/var/vshard_git_tree_copy'
+---
+...
+evolution_log = git_util.log_hashes({args='vshard/storage/reload_evolution.lua', dir=util.SOURCEDIR})
+---
+...
+-- Cleanup the directory after a previous build.
+_ = os.execute('rm -rf ' .. vshard_copy_path)
+---
+...
+-- 1. `git worktree` cannot be used because PACKPACK mounts
+-- `/source/` in `ro` mode.
+-- 2. Just `cp -rf` cannot be used due to a little different
+-- behavior in Centos 7.
+_ = os.execute('mkdir ' .. vshard_copy_path)
+---
+...
+_ = os.execute("cd " .. util.SOURCEDIR .. ' && cp -rf `ls -A --ignore=build` ' .. vshard_copy_path)
+---
+...
+-- Checkout the first commit with a reload_evolution mechanism.
+git_util.exec_cmd({cmd='checkout', args='-f', dir=vshard_copy_path})
+---
+...
+git_util.exec_cmd({cmd='checkout', args=evolution_log[#evolution_log] .. '~1', dir=vshard_copy_path})
+---
+...
+REPLICASET_1 = { 'storage_1_a', 'storage_1_b' }
+---
+...
+REPLICASET_2 = { 'storage_2_a', 'storage_2_b' }
+---
+...
+test_run:create_cluster(REPLICASET_1, 'reload_evolution')
+---
+...
+test_run:create_cluster(REPLICASET_2, 'reload_evolution')
+---
+...
+util = require('lua_libs.util')
+---
+...
+util.wait_master(test_run, REPLICASET_1, 'storage_1_a')
+---
+...
+util.wait_master(test_run, REPLICASET_2, 'storage_2_a')
+---
+...
+test_run:switch('storage_1_a')
+---
+- true
+...
+vshard.storage.bucket_force_create(1, vshard.consts.DEFAULT_BUCKET_COUNT / 2)
+---
+- true
+...
+bucket_id_to_move = vshard.consts.DEFAULT_BUCKET_COUNT
+---
+...
+test_run:switch('storage_2_a')
+---
+- true
+...
+fiber = require('fiber')
+---
+...
+vshard.storage.bucket_force_create(vshard.consts.DEFAULT_BUCKET_COUNT / 2 + 1, vshard.consts.DEFAULT_BUCKET_COUNT / 2)
+---
+- true
+...
+bucket_id_to_move = vshard.consts.DEFAULT_BUCKET_COUNT
+---
+...
+vshard.storage.internal.reload_version
+---
+- null
+...
+wait_rebalancer_state('The cluster is balanced ok', test_run)
+---
+...
+box.space.test:insert({42, bucket_id_to_move})
+---
+- [42, 3000]
+...
+test_run:switch('default')
+---
+- true
+...
+git_util.exec_cmd({cmd='checkout', args=evolution_log[1], dir=vshard_copy_path})
+---
+...
+test_run:switch('storage_2_a')
+---
+- true
+...
+package.loaded['vshard.storage'] = nil
+---
+...
+vshard.storage = require("vshard.storage")
+---
+...
+test_run:grep_log('storage_2_a', 'vshard.storage.reload_evolution: upgraded to') ~= nil
+---
+- true
+...
+vshard.storage.internal.reload_version
+---
+- 1
+...
+-- Make sure storage operates well.
+vshard.storage.bucket_force_drop(2000)
+---
+- true
+...
+vshard.storage.bucket_force_create(2000)
+---
+- true
+...
+vshard.storage.buckets_info()[2000]
+---
+- status: active
+ id: 2000
+...
+vshard.storage.call(bucket_id_to_move, 'read', 'do_select', {42})
+---
+- true
+- - [42, 3000]
+...
+vshard.storage.bucket_send(bucket_id_to_move, replicaset1_uuid)
+---
+- true
+...
+vshard.storage.garbage_collector_wakeup()
+---
+...
+fiber = require('fiber')
+---
+...
+while box.space._bucket:get({bucket_id_to_move}) do fiber.sleep(0.01) end
+---
+...
+test_run:switch('storage_1_a')
+---
+- true
+...
+vshard.storage.bucket_send(bucket_id_to_move, replicaset2_uuid)
+---
+- true
+...
+test_run:switch('storage_2_a')
+---
+- true
+...
+vshard.storage.call(bucket_id_to_move, 'read', 'do_select', {42})
+---
+- true
+- - [42, 3000]
+...
+-- Check info() does not fail.
+vshard.storage.info() ~= nil
+---
+- true
+...
+--
+-- Send buckets to create a disbalance. Wait until the rebalancer
+-- repairs it. Similar to `tests/rebalancer/rebalancer.test.lua`.
+--
+vshard.storage.rebalancer_disable()
+---
+...
+move_start = vshard.consts.DEFAULT_BUCKET_COUNT / 2 + 1
+---
+...
+move_cnt = 100
+---
+...
+assert(move_start + move_cnt < vshard.consts.DEFAULT_BUCKET_COUNT)
+---
+- true
+...
+for i = move_start, move_start + move_cnt - 1 do box.space._bucket:delete{i} end
+---
+...
+box.space._bucket.index.status:count({vshard.consts.BUCKET.ACTIVE})
+---
+- 1400
+...
+test_run:switch('storage_1_a')
+---
+- true
+...
+move_start = vshard.consts.DEFAULT_BUCKET_COUNT / 2 + 1
+---
+...
+move_cnt = 100
+---
+...
+vshard.storage.bucket_force_create(move_start, move_cnt)
+---
+- true
+...
+box.space._bucket.index.status:count({vshard.consts.BUCKET.ACTIVE})
+---
+- 1600
+...
+test_run:switch('storage_2_a')
+---
+- true
+...
+vshard.storage.rebalancer_enable()
+---
+...
+wait_rebalancer_state('Rebalance routes are sent', test_run)
+---
+...
+wait_rebalancer_state('The cluster is balanced ok', test_run)
+---
+...
+box.space._bucket.index.status:count({vshard.consts.BUCKET.ACTIVE})
+---
+- 1500
+...
+test_run:switch('default')
+---
+- true
+...
+test_run:drop_cluster(REPLICASET_2)
+---
+...
+test_run:drop_cluster(REPLICASET_1)
+---
+...
+test_run:cmd('clear filter')
+---
+- true
+...
diff --git a/test/reload_evolution/storage.test.lua b/test/reload_evolution/storage.test.lua
new file mode 100644
index 0000000..7af464b
--- /dev/null
+++ b/test/reload_evolution/storage.test.lua
@@ -0,0 +1,87 @@
+test_run = require('test_run').new()
+
+git_util = require('lua_libs.git_util')
+util = require('lua_libs.util')
+vshard_copy_path = util.BUILDDIR .. '/test/var/vshard_git_tree_copy'
+evolution_log = git_util.log_hashes({args='vshard/storage/reload_evolution.lua', dir=util.SOURCEDIR})
+-- Cleanup the directory after a previous build.
+_ = os.execute('rm -rf ' .. vshard_copy_path)
+-- 1. `git worktree` cannot be used because PACKPACK mounts
+-- `/source/` in `ro` mode.
+-- 2. Just `cp -rf` cannot be used due to a little different
+-- behavior in Centos 7.
+_ = os.execute('mkdir ' .. vshard_copy_path)
+_ = os.execute("cd " .. util.SOURCEDIR .. ' && cp -rf `ls -A --ignore=build` ' .. vshard_copy_path)
+-- Checkout the first commit with a reload_evolution mechanism.
+git_util.exec_cmd({cmd='checkout', args='-f', dir=vshard_copy_path})
+git_util.exec_cmd({cmd='checkout', args=evolution_log[#evolution_log] .. '~1', dir=vshard_copy_path})
+
+REPLICASET_1 = { 'storage_1_a', 'storage_1_b' }
+REPLICASET_2 = { 'storage_2_a', 'storage_2_b' }
+test_run:create_cluster(REPLICASET_1, 'reload_evolution')
+test_run:create_cluster(REPLICASET_2, 'reload_evolution')
+util = require('lua_libs.util')
+util.wait_master(test_run, REPLICASET_1, 'storage_1_a')
+util.wait_master(test_run, REPLICASET_2, 'storage_2_a')
+
+test_run:switch('storage_1_a')
+vshard.storage.bucket_force_create(1, vshard.consts.DEFAULT_BUCKET_COUNT / 2)
+bucket_id_to_move = vshard.consts.DEFAULT_BUCKET_COUNT
+
+test_run:switch('storage_2_a')
+fiber = require('fiber')
+vshard.storage.bucket_force_create(vshard.consts.DEFAULT_BUCKET_COUNT / 2 + 1, vshard.consts.DEFAULT_BUCKET_COUNT / 2)
+bucket_id_to_move = vshard.consts.DEFAULT_BUCKET_COUNT
+vshard.storage.internal.reload_version
+wait_rebalancer_state('The cluster is balanced ok', test_run)
+box.space.test:insert({42, bucket_id_to_move})
+
+test_run:switch('default')
+git_util.exec_cmd({cmd='checkout', args=evolution_log[1], dir=vshard_copy_path})
+
+test_run:switch('storage_2_a')
+package.loaded['vshard.storage'] = nil
+vshard.storage = require("vshard.storage")
+test_run:grep_log('storage_2_a', 'vshard.storage.reload_evolution: upgraded to') ~= nil
+vshard.storage.internal.reload_version
+-- Make sure storage operates well.
+vshard.storage.bucket_force_drop(2000)
+vshard.storage.bucket_force_create(2000)
+vshard.storage.buckets_info()[2000]
+vshard.storage.call(bucket_id_to_move, 'read', 'do_select', {42})
+vshard.storage.bucket_send(bucket_id_to_move, replicaset1_uuid)
+vshard.storage.garbage_collector_wakeup()
+fiber = require('fiber')
+while box.space._bucket:get({bucket_id_to_move}) do fiber.sleep(0.01) end
+test_run:switch('storage_1_a')
+vshard.storage.bucket_send(bucket_id_to_move, replicaset2_uuid)
+test_run:switch('storage_2_a')
+vshard.storage.call(bucket_id_to_move, 'read', 'do_select', {42})
+-- Check info() does not fail.
+vshard.storage.info() ~= nil
+
+--
+-- Send buckets to create a disbalance. Wait until the rebalancer
+-- repairs it. Similar to `tests/rebalancer/rebalancer.test.lua`.
+--
+vshard.storage.rebalancer_disable()
+move_start = vshard.consts.DEFAULT_BUCKET_COUNT / 2 + 1
+move_cnt = 100
+assert(move_start + move_cnt < vshard.consts.DEFAULT_BUCKET_COUNT)
+for i = move_start, move_start + move_cnt - 1 do box.space._bucket:delete{i} end
+box.space._bucket.index.status:count({vshard.consts.BUCKET.ACTIVE})
+test_run:switch('storage_1_a')
+move_start = vshard.consts.DEFAULT_BUCKET_COUNT / 2 + 1
+move_cnt = 100
+vshard.storage.bucket_force_create(move_start, move_cnt)
+box.space._bucket.index.status:count({vshard.consts.BUCKET.ACTIVE})
+test_run:switch('storage_2_a')
+vshard.storage.rebalancer_enable()
+wait_rebalancer_state('Rebalance routes are sent', test_run)
+wait_rebalancer_state('The cluster is balanced ok', test_run)
+box.space._bucket.index.status:count({vshard.consts.BUCKET.ACTIVE})
+
+test_run:switch('default')
+test_run:drop_cluster(REPLICASET_2)
+test_run:drop_cluster(REPLICASET_1)
+test_run:cmd('clear filter')
diff --git a/test/reload_evolution/storage_1_a.lua b/test/reload_evolution/storage_1_a.lua
new file mode 100755
index 0000000..f1a2981
--- /dev/null
+++ b/test/reload_evolution/storage_1_a.lua
@@ -0,0 +1,48 @@
+#!/usr/bin/env tarantool
+
+require('strict').on()
+
+local log = require('log')
+local fiber = require('fiber')
+local util = require('lua_libs.util')
+local fio = require('fio')
+
+-- Get instance name
+NAME = fio.basename(arg[0], '.lua')
+
+-- test-run gate.
+test_run = require('test_run').new()
+require('console').listen(os.getenv('ADMIN'))
+
+-- Run one storage on a different vshard version.
+-- To do that, place vshard src to
+-- BUILDDIR/test/var/vshard_git_tree_copy/.
+if NAME == 'storage_2_a' then
+ local script_path = debug.getinfo(1).source:match("@?(.*/)")
+ vshard_copy = util.BUILDDIR .. '/test/var/vshard_git_tree_copy'
+ package.path = string.format(
+ '%s/?.lua;%s/?/init.lua;%s',
+ vshard_copy, vshard_copy, package.path
+ )
+end
+
+-- Call a configuration provider
+cfg = require('localcfg')
+-- Name to uuid map
+names = {
+ ['storage_1_a'] = '8a274925-a26d-47fc-9e1b-af88ce939412',
+ ['storage_1_b'] = '3de2e3e1-9ebe-4d0d-abb1-26d301b84633',
+ ['storage_2_a'] = '1e02ae8a-afc0-4e91-ba34-843a356b8ed7',
+ ['storage_2_b'] = '001688c3-66f8-4a31-8e19-036c17d489c2',
+}
+
+replicaset1_uuid = 'cbf06940-0790-498b-948d-042b62cf3d29'
+replicaset2_uuid = 'ac522f65-aa94-4134-9f64-51ee384f1a54'
+replicasets = {replicaset1_uuid, replicaset2_uuid}
+
+-- Start the database with sharding
+vshard = require('vshard')
+vshard.storage.cfg(cfg, names[NAME])
+
+-- Bootstrap storage.
+require('lua_libs.bootstrap')
diff --git a/test/reload_evolution/storage_1_b.lua b/test/reload_evolution/storage_1_b.lua
new file mode 120000
index 0000000..02572da
--- /dev/null
+++ b/test/reload_evolution/storage_1_b.lua
@@ -0,0 +1 @@
+storage_1_a.lua
\ No newline at end of file
diff --git a/test/reload_evolution/storage_2_a.lua b/test/reload_evolution/storage_2_a.lua
new file mode 120000
index 0000000..02572da
--- /dev/null
+++ b/test/reload_evolution/storage_2_a.lua
@@ -0,0 +1 @@
+storage_1_a.lua
\ No newline at end of file
diff --git a/test/reload_evolution/storage_2_b.lua b/test/reload_evolution/storage_2_b.lua
new file mode 120000
index 0000000..02572da
--- /dev/null
+++ b/test/reload_evolution/storage_2_b.lua
@@ -0,0 +1 @@
+storage_1_a.lua
\ No newline at end of file
diff --git a/test/reload_evolution/suite.ini b/test/reload_evolution/suite.ini
new file mode 100644
index 0000000..5f55418
--- /dev/null
+++ b/test/reload_evolution/suite.ini
@@ -0,0 +1,6 @@
+[default]
+core = tarantool
+description = Reload evolution tests
+script = test.lua
+is_parallel = False
+lua_libs = ../lua_libs ../../example/localcfg.lua
diff --git a/test/reload_evolution/test.lua b/test/reload_evolution/test.lua
new file mode 100644
index 0000000..ad0543a
--- /dev/null
+++ b/test/reload_evolution/test.lua
@@ -0,0 +1,9 @@
+#!/usr/bin/env tarantool
+
+require('strict').on()
+
+box.cfg{
+ listen = os.getenv("LISTEN"),
+}
+
+require('console').listen(os.getenv('ADMIN'))
diff --git a/test/unit/reload_evolution.result b/test/unit/reload_evolution.result
new file mode 100644
index 0000000..10e606d
--- /dev/null
+++ b/test/unit/reload_evolution.result
@@ -0,0 +1,45 @@
+test_run = require('test_run').new()
+---
+...
+fiber = require('fiber')
+---
+...
+log = require('log')
+---
+...
+util = require('util')
+---
+...
+reload_evolution = require('vshard.storage.reload_evolution')
+---
+...
+-- Init with the latest version.
+fake_M = { reload_version = reload_evolution.version }
+---
+...
+-- Test reload to the same version.
+reload_evolution.upgrade(fake_M)
+---
+...
+test_run:grep_log('default', 'vshard.storage.evolution') == nil
+---
+- true
+...
+-- Test downgrage version.
+log.info(string.rep('a', 1000))
+---
+...
+fake_M.reload_version = fake_M.reload_version + 1
+---
+...
+err = util.check_error(reload_evolution.upgrade, fake_M)
+---
+...
+err:match('auto%-downgrade is not implemented')
+---
+- auto-downgrade is not implemented
+...
+test_run:grep_log('default', 'vshard.storage.evolution', 1000) ~= nil
+---
+- false
+...
diff --git a/test/unit/reload_evolution.test.lua b/test/unit/reload_evolution.test.lua
new file mode 100644
index 0000000..2e99152
--- /dev/null
+++ b/test/unit/reload_evolution.test.lua
@@ -0,0 +1,18 @@
+test_run = require('test_run').new()
+fiber = require('fiber')
+log = require('log')
+util = require('util')
+reload_evolution = require('vshard.storage.reload_evolution')
+-- Init with the latest version.
+fake_M = { reload_version = reload_evolution.version }
+
+-- Test reload to the same version.
+reload_evolution.upgrade(fake_M)
+test_run:grep_log('default', 'vshard.storage.evolution') == nil
+
+-- Test downgrage version.
+log.info(string.rep('a', 1000))
+fake_M.reload_version = fake_M.reload_version + 1
+err = util.check_error(reload_evolution.upgrade, fake_M)
+err:match('auto%-downgrade is not implemented')
+test_run:grep_log('default', 'vshard.storage.evolution', 1000) ~= nil
diff --git a/vshard/storage/init.lua b/vshard/storage/init.lua
index c1df0e6..c7d81e3 100644
--- a/vshard/storage/init.lua
+++ b/vshard/storage/init.lua
@@ -10,6 +10,7 @@ if rawget(_G, MODULE_INTERNALS) then
local vshard_modules = {
'vshard.consts', 'vshard.error', 'vshard.cfg',
'vshard.replicaset', 'vshard.util',
+ 'vshard.storage.reload_evolution'
}
for _, module in pairs(vshard_modules) do
package.loaded[module] = nil
@@ -20,12 +21,16 @@ local lerror = require('vshard.error')
local lcfg = require('vshard.cfg')
local lreplicaset = require('vshard.replicaset')
local util = require('vshard.util')
+local reload_evolution = require('vshard.storage.reload_evolution')
local M = rawget(_G, MODULE_INTERNALS)
if not M then
--
-- The module is loaded for the first time.
--
+ -- !!!WARNING: any change of this table must be reflected in
+ -- `vshard.storage.reload_evolution` module to guarantee
+ -- reloadability of the module.
M = {
---------------- Common module attributes ----------------
-- The last passed configuration.
@@ -105,6 +110,11 @@ if not M then
-- a destination replicaset must drop already received
-- data.
rebalancer_sending_bucket = 0,
+
+ ------------------------- Reload -------------------------
+ -- Version of the loaded module. This number is used on
+ -- reload to determine which upgrade scripts to run.
+ reload_version = reload_evolution.version,
}
end
@@ -1864,6 +1874,7 @@ end
if not rawget(_G, MODULE_INTERNALS) then
rawset(_G, MODULE_INTERNALS, M)
else
+ reload_evolution.upgrade(M)
storage_cfg(M.current_cfg, M.this_replica.uuid)
M.module_version = M.module_version + 1
end
diff --git a/vshard/storage/reload_evolution.lua b/vshard/storage/reload_evolution.lua
new file mode 100644
index 0000000..8502a33
--- /dev/null
+++ b/vshard/storage/reload_evolution.lua
@@ -0,0 +1,58 @@
+--
+-- This module is used to upgrade the vshard.storage on the fly.
+-- It updates internal Lua structures in case they are changed
+-- in a commit.
+--
+local log = require('log')
+
+--
+-- Array of upgrade functions.
+-- migrations[version] = function which upgrades module version
+-- from `version` to `version + 1`.
+--
+local migrations = {}
+
+-- Initialize reload_upgrade mechanism
+migrations[#migrations + 1] = function (M)
+ -- Code to update Lua objects.
+end
+
+--
+-- Perform an update based on a version stored in `M` (internals).
+-- @param M Old module internals which should be updated.
+--
+local function upgrade(M)
+ local start_version = M.reload_version or 1
+ if start_version > #migrations then
+ local err_msg = string.format(
+ 'vshard.storage.reload_evolution: ' ..
+ 'auto-downgrade is not implemented; ' ..
+ 'loaded version is %d, upgrade script version is %d',
+ start_version, #migrations
+ )
+ log.error(err_msg)
+ error(err_msg)
+ end
+ for i = start_version, #migrations do
+ local ok, err = pcall(migrations[i], M)
+ if ok then
+ log.info('vshard.storage.reload_evolution: upgraded to %d version',
+ i)
+ else
+ local err_msg = string.format(
+ 'vshard.storage.reload_evolution: ' ..
+ 'error during upgrade to %d version: %s', i, err
+ )
+ log.error(err_msg)
+ error(err_msg)
+ end
+ -- Update the version just after upgrade to have an
+ -- actual version in case of an error.
+ M.reload_version = i
+ end
+end
+
+return {
+ version = #migrations,
+ upgrade = upgrade,
+}
--
2.14.1
^ permalink raw reply [flat|nested] 4+ messages in thread
end of thread, other threads:[~2018-07-26 8:27 UTC | newest]
Thread overview: 4+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2018-07-26 8:27 [tarantool-patches] [PATCH v3] vshard reload mechanism AKhatskevich
2018-07-26 8:27 ` [tarantool-patches] [PATCH 1/3] Fix races related to object outdating AKhatskevich
2018-07-26 8:27 ` [tarantool-patches] [PATCH 2/3] tests: separate bootstrap routine to a lua_libs AKhatskevich
2018-07-26 8:27 ` [tarantool-patches] [PATCH 3/3] Introduce storage reload evolution AKhatskevich
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox