From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from localhost (localhost [127.0.0.1]) by turing.freelists.org (Avenir Technologies Mail Multiplex) with ESMTP id 4787F1FF70 for ; Mon, 23 Jul 2018 16:11:05 -0400 (EDT) Received: from turing.freelists.org ([127.0.0.1]) by localhost (turing.freelists.org [127.0.0.1]) (amavisd-new, port 10024) with ESMTP id bFLrsJkUVhaZ for ; Mon, 23 Jul 2018 16:11:05 -0400 (EDT) Received: from smtp20.mail.ru (smtp20.mail.ru [94.100.179.251]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by turing.freelists.org (Avenir Technologies Mail Multiplex) with ESMTPS id A1A1C1FE6E for ; Mon, 23 Jul 2018 16:11:04 -0400 (EDT) Subject: [tarantool-patches] Re: [PATCH 4/4] Introduce storage reload evolution References: From: Alex Khatskevich Message-ID: <3cde6575-8eca-ce7d-6c8f-ed2b5742a002@tarantool.org> Date: Mon, 23 Jul 2018 23:10:54 +0300 MIME-Version: 1.0 In-Reply-To: Content-Type: text/plain; charset="utf-8"; format="flowed" Content-Transfer-Encoding: 8bit Content-Language: en-US Sender: tarantool-patches-bounce@freelists.org Errors-to: tarantool-patches-bounce@freelists.org Reply-To: tarantool-patches@freelists.org List-help: List-unsubscribe: List-software: Ecartis version 1.0.0 List-Id: tarantool-patches List-subscribe: List-owner: List-post: List-archive: To: Vladislav Shpilevoy , tarantool-patches@freelists.org On 23.07.2018 17:44, Vladislav Shpilevoy wrote: > Thanks for the patch! See 4 comments below. > > On 23/07/2018 14:14, AKhatskevich wrote: >> Changes: >> 1. Introduce storage reload evolution. >> 2. Setup cross-version reload testing. >> >> 1: >> This mechanism updates Lua objects on reload in case they are >> changed in a new vshard.storage version. >> >> Since this commit, any change in vshard.storage.M has to be >> reflected in vshard.storage.reload_evolution to guarantee >> correct reload. >> >> 2: >> The testing uses git infrastructure and is performed in the following >> way: >> 1. Copy old version of vshard to a temp folder. >> 2. Run vshard on this code. >> 3. Checkout the latest version of the vshard sources. >> 4. Reload vshard storage. >> 5. Make sure it works (Perform simple tests). >> >> Notes: >> * this patch contains some legacy-driven decisions: >>    1. SOURCEDIR path retrieved differently in case of >>       packpack build. >>    2. git directory in the `reload_evolution/storage` test >>       is copied with respect to Centos 7 and `ro` mode of >>       SOURCEDIR. >> >> diff --git a/test/reload_evolution/storage.result >> b/test/reload_evolution/storage.result >> new file mode 100644 >> index 0000000..54ff6b7 >> --- /dev/null >> +++ b/test/reload_evolution/storage.result >> @@ -0,0 +1,248 @@ >> +test_run = require('test_run').new() >> +--- >> +... >> +git_util = require('lua_libs.git_util') >> +--- >> +... >> +util = require('lua_libs.util') >> +--- >> +... >> +vshard_copy_path = util.BUILDDIR .. '/test/var/vshard_git_tree_copy' >> +--- >> +... >> +evolution_log = >> git_util.log_hashes({args='vshard/storage/reload_evolution.lua', >> dir=util.SOURCEDIR}) >> +--- >> +... >> +-- Cleanup the directory after a previous build. >> +_ = os.execute('rm -rf ' .. vshard_copy_path) >> +--- >> +... >> +-- 1. `git worktree` cannot be used because PACKPACK mounts >> +-- `/source/` in `ro` mode. >> +-- 2. Just `cp -rf` cannot be used due to a little different >> +-- behavior in Centos 7. >> +_ = os.execute('mkdir ' .. vshard_copy_path) >> +--- >> +... >> +_ = os.execute("cd " .. util.SOURCEDIR .. ' && cp -rf `ls -A >> --ignore=build` ' .. vshard_copy_path) >> +--- >> +... >> +-- Checkout the first commit with a reload_evolution mechanism. >> +git_util.exec_cmd({cmd='checkout', args='-f', dir=vshard_copy_path}) >> +--- >> +... >> +git_util.exec_cmd({cmd='checkout', >> args=evolution_log[#evolution_log] .. '~1', dir=vshard_copy_path}) >> +--- >> +... >> +REPLICASET_1 = { 'storage_1_a', 'storage_1_b' } >> +--- >> +... >> +REPLICASET_2 = { 'storage_2_a', 'storage_2_b' } >> +--- >> +... >> +test_run:create_cluster(REPLICASET_1, 'reload_evolution') >> +--- >> +... >> +test_run:create_cluster(REPLICASET_2, 'reload_evolution') >> +--- >> +... >> +util = require('lua_libs.util') >> +--- >> +... >> +util.wait_master(test_run, REPLICASET_1, 'storage_1_a') >> +--- >> +... >> +util.wait_master(test_run, REPLICASET_2, 'storage_2_a') >> +--- >> +... >> +test_run:switch('storage_1_a') >> +--- >> +- true >> +... >> +vshard.storage.bucket_force_create(1, >> vshard.consts.DEFAULT_BUCKET_COUNT / 2) >> +--- >> +- true >> +... >> +bucket_id_to_move = vshard.consts.DEFAULT_BUCKET_COUNT >> +--- >> +... >> +test_run:switch('storage_2_a') >> +--- >> +- true >> +... >> +fiber = require('fiber') >> +--- >> +... >> +vshard.storage.bucket_force_create(vshard.consts.DEFAULT_BUCKET_COUNT >> / 2 + 1, vshard.consts.DEFAULT_BUCKET_COUNT / 2) >> +--- >> +- true >> +... >> +bucket_id_to_move = vshard.consts.DEFAULT_BUCKET_COUNT >> +--- >> +... >> +vshard.storage.internal.reload_evolution_version >> +--- >> +- null >> +... >> +box.space.test:insert({42, bucket_id_to_move}) >> +--- >> +- [42, 3000] >> +... >> +while test_run:grep_log('storage_2_a', 'The cluster is balanced ok') >> == nil do vshard.storage.rebalancer_wakeup() fiber.sleep(0.1) end > > 1. Now you have wait_rebalancer_state util from the previous commit. I thought that it is possible at this point that the cluster is already balanced. But it seems that it is close to impossible. Fixed > >> +--- >> +... >> +test_run:switch('default') >> +--- >> +- true >> +... >> +git_util.exec_cmd({cmd='checkout', args=evolution_log[1], >> dir=vshard_copy_path}) >> +--- >> +... >> +test_run:switch('storage_2_a') >> +--- >> +- true >> +... >> +package.loaded["vshard.storage"] = nil >> +--- >> +... >> +vshard.storage = require("vshard.storage") >> +--- >> +... >> +test_run:grep_log('storage_2_a', 'vshard.storage.reload_evolution: >> upgraded to') ~= nil >> +--- >> +- true >> +... >> +vshard.storage.internal.reload_evolution_version >> +--- >> +- 1 >> +... >> +-- Make sure storage operates well. >> +vshard.storage.bucket_force_drop(2000) >> +--- >> +- true >> +... >> +vshard.storage.bucket_force_create(2000) >> +--- >> +- true >> +... >> +vshard.storage.buckets_info()[2000] >> +--- >> +- status: active >> +  id: 2000 >> +... >> +vshard.storage.call(bucket_id_to_move, 'read', 'do_select', {42}) >> +--- >> +- true >> +- - [42, 3000] >> +... >> +vshard.storage.bucket_send(bucket_id_to_move, replicaset1_uuid) >> +--- >> +- true >> +... >> +vshard.storage.garbage_collector_wakeup() >> +--- >> +... >> +fiber = require('fiber') >> +--- >> +... >> +while box.space._bucket:get({bucket_id_to_move}) do >> fiber.sleep(0.01) end >> +--- >> +... >> +test_run:switch('storage_1_a') >> +--- >> +- true >> +... >> +vshard.storage.bucket_send(bucket_id_to_move, replicaset2_uuid) >> +--- >> +- true >> +... >> +test_run:switch('storage_2_a') >> +--- >> +- true >> +... >> +vshard.storage.call(bucket_id_to_move, 'read', 'do_select', {42}) >> +--- >> +- true >> +- - [42, 3000] >> +... >> +-- Check info() does not fail. >> +vshard.storage.info() ~= nil >> +--- >> +- true >> +... >> +-- >> +-- Send buckets to create a disbalance. Wait until the rebalancer >> +-- repairs it. Similar to `tests/rebalancer/rebalancer.test.lua`. >> +-- >> +vshard.storage.rebalancer_disable() >> +--- >> +... >> +move_start = vshard.consts.DEFAULT_BUCKET_COUNT / 2 + 1 >> +--- >> +... >> +move_cnt = 100 >> +--- >> +... >> +assert(move_start + move_cnt < vshard.consts.DEFAULT_BUCKET_COUNT) >> +--- >> +- true >> +... >> +for i = move_start, move_start + move_cnt - 1 do >> box.space._bucket:delete{i} end >> +--- >> +... >> +box.space._bucket.index.status:count({vshard.consts.BUCKET.ACTIVE}) >> +--- >> +- 1400 >> +... >> +test_run:switch('storage_1_a') >> +--- >> +- true >> +... >> +move_start = vshard.consts.DEFAULT_BUCKET_COUNT / 2 + 1 >> +--- >> +... >> +move_cnt = 100 >> +--- >> +... >> +vshard.storage.bucket_force_create(move_start, move_cnt) >> +--- >> +- true >> +... >> +box.space._bucket.index.status:count({vshard.consts.BUCKET.ACTIVE}) >> +--- >> +- 1600 >> +... >> +test_run:switch('storage_2_a') >> +--- >> +- true >> +... >> +vshard.storage.rebalancer_enable() >> +--- >> +... >> +vshard.storage.rebalancer_wakeup() > > 2. You do not need explicit rebalancer_wakeup. wait_rebalancer_state > calls it. Fixed > >> +--- >> +... >> +wait_rebalancer_state("Rebalance routes are sent", test_run) >> +--- >> +... >> +wait_rebalancer_state('The cluster is balanced ok', test_run) >> +--- >> +... >> +box.space._bucket.index.status:count({vshard.consts.BUCKET.ACTIVE}) >> +--- >> +- 1500 >> +... >> +test_run:switch('default') >> +--- >> +- true >> +... >> +test_run:drop_cluster(REPLICASET_2) >> +--- >> +... >> +test_run:drop_cluster(REPLICASET_1) >> +--- >> +... >> +test_run:cmd('clear filter') >> +--- >> +- true >> +... >> diff --git a/test/unit/reload_evolution.result >> b/test/unit/reload_evolution.result >> new file mode 100644 >> index 0000000..342ac24 >> --- /dev/null >> +++ b/test/unit/reload_evolution.result >> @@ -0,0 +1,45 @@ >> +test_run = require('test_run').new() >> +--- >> +... >> +fiber = require('fiber') >> +--- >> +... >> +log = require('log') >> +--- >> +... >> +util = require('util') >> +--- >> +... >> +reload_evolution = require('vshard.storage.reload_evolution') >> +--- >> +... >> +-- Init with the latest version. >> +fake_M = { reload_evolution_version = reload_evolution.version } >> +--- >> +... >> +-- Test reload to the same version. >> +reload_evolution.upgrade(fake_M) >> +--- >> +... >> +test_run:grep_log('default', 'vshard.storage.evolution') == nil >> +--- >> +- true >> +... >> +-- Test downgrage version. >> +log.info(string.rep('a', 1000)) >> +--- >> +... >> +fake_M.reload_evolution_version = fake_M.reload_evolution_version + 1 >> +--- >> +... >> +err = util.check_error(reload_evolution.upgrade, fake_M) >> +--- >> +... >> +err:match('auto%-downgrade is not implemented') >> +--- >> +- auto-downgrade is not implemented > > 3. Why do you need match? check_error output is ok already. And > what is 'auto%'? I see that you always print exactly "auto-downgrade" > in reload_evolution.lua. I need match to cut numbers which may change from the output. % is used to escape '-' (which is a special character. > >> +... >> +test_run:grep_log('default', 'vshard.storage.evolution', 1000) ~= nil >> +--- >> +- false >> +... >> @@ -105,6 +110,11 @@ if not M then >>           -- a destination replicaset must drop already received >>           -- data. >>           rebalancer_sending_bucket = 0, >> + >> +        ------------------------- Reload ------------------------- >> +        -- Version of the loaded module. This number is used on >> +        -- reload to determine which upgrade scripts to run. >> +        reload_evolution_version = reload_evolution.version, > > 4. Please, rename it to just 'version' or 'reload_version' or > 'module_version'. 'Reload_evolution_version' is too long and > complex. Renamed to reload_version. I like `reload_evolution_version` though. > >>       } >>   end Full diff: commit ad151e7c3fb4c15dd49859b28113195cf74ad418 Author: AKhatskevich Date:   Fri Jun 29 20:34:26 2018 +0300     Introduce storage reload evolution     Changes:     1. Introduce storage reload evolution.     2. Setup cross-version reload testing.     1:     This mechanism updates Lua objects on reload in case they are     changed in a new vshard.storage version.     Since this commit, any change in vshard.storage.M has to be     reflected in vshard.storage.reload_evolution to guarantee     correct reload.     2:     The testing uses git infrastructure and is performed in the following     way:     1. Copy old version of vshard to a temp folder.     2. Run vshard on this code.     3. Checkout the latest version of the vshard sources.     4. Reload vshard storage.     5. Make sure it works (Perform simple tests).     Notes:     * this patch contains some legacy-driven decisions:       1. SOURCEDIR path retrieved differently in case of          packpack build.       2. git directory in the `reload_evolution/storage` test          is copied with respect to Centos 7 and `ro` mode of          SOURCEDIR.     Closes #112 #125 diff --git a/.travis.yml b/.travis.yml index 54bfe44..eff4a51 100644 --- a/.travis.yml +++ b/.travis.yml @@ -41,7 +41,7 @@ env:  script:    - git describe --long    - git clone https://github.com/packpack/packpack.git packpack -  - packpack/packpack +  - packpack/packpack -e PACKPACK_GIT_SOURCEDIR=/source/  before_deploy:    - ls -l build/ diff --git a/rpm/prebuild.sh b/rpm/prebuild.sh index 768b22b..554032b 100755 --- a/rpm/prebuild.sh +++ b/rpm/prebuild.sh @@ -1 +1,3 @@  curl -s https://packagecloud.io/install/repositories/tarantool/1_9/script.rpm.sh | sudo bash +sudo yum -y install python-devel python-pip +sudo pip install tarantool msgpack diff --git a/test/lua_libs/git_util.lua b/test/lua_libs/git_util.lua new file mode 100644 index 0000000..a75bb08 --- /dev/null +++ b/test/lua_libs/git_util.lua @@ -0,0 +1,51 @@ +-- +-- Lua bridge for some of the git commands. +-- +local os = require('os') + +local temp_file = 'some_strange_rare_unique_file_name_for_git_util' + +-- +-- Exec a git command. +-- @param params Table of parameters: +--        * options - git options. +--        * cmd - git command. +--        * args - command arguments. +--        * dir - working directory. +--        * fout - write output to the file. +local function exec_cmd(params) +    local fout = params.fout +    local shell_cmd = {'git'} +    for _, param in pairs({'options', 'cmd', 'args'}) do +        table.insert(shell_cmd, params[param]) +    end +    if fout then +        table.insert(shell_cmd, ' >' .. fout) +    end +    shell_cmd = table.concat(shell_cmd, ' ') +    if params.dir then +        shell_cmd = string.format('cd %s && %s', params.dir, shell_cmd) +    end +    local res = os.execute(shell_cmd) +    assert(res == 0, 'Git cmd error: ' .. res) +end + +local function log_hashes(params) +    params.args = "--format='%h' " .. params.args +    local local_temp_file = string.format('%s/%s', os.getenv('PWD'), temp_file) +    params.fout = local_temp_file +    params.cmd = 'log' +    exec_cmd(params) +    local lines = {} +    for line in io.lines(local_temp_file) do +        table.insert(lines, line) +    end +    os.remove(local_temp_file) +    return lines +end + + +return { +    exec_cmd = exec_cmd, +    log_hashes = log_hashes +} diff --git a/test/lua_libs/util.lua b/test/lua_libs/util.lua index f40d3a6..935ff41 100644 --- a/test/lua_libs/util.lua +++ b/test/lua_libs/util.lua @@ -1,5 +1,6 @@  local fiber = require('fiber')  local log = require('log') +local fio = require('fio')  local function check_error(func, ...)      local pstatus, status, err = pcall(func, ...) @@ -92,10 +93,29 @@ local function has_same_fields(etalon, data)      return true  end +-- Git directory of the project. Used in evolution tests to +-- fetch old versions of vshard. +local SOURCEDIR = os.getenv('PACKPACK_GIT_SOURCEDIR') +if not SOURCEDIR then +    SOURCEDIR = os.getenv('SOURCEDIR') +end +if not SOURCEDIR then +    local script_path = debug.getinfo(1).source:match("@?(.*/)") +    script_path = fio.abspath(script_path) +    SOURCEDIR = fio.abspath(script_path .. '/../../../') +end + +local BUILDDIR = os.getenv('BUILDDIR') +if not BUILDDIR then +    BUILDDIR = SOURCEDIR +end +  return {      check_error = check_error,      shuffle_masters = shuffle_masters,      collect_timeouts = collect_timeouts,      wait_master = wait_master,      has_same_fields = has_same_fields, +    SOURCEDIR = SOURCEDIR, +    BUILDDIR = BUILDDIR,  } diff --git a/test/reload_evolution/storage.result b/test/reload_evolution/storage.result new file mode 100644 index 0000000..007192c --- /dev/null +++ b/test/reload_evolution/storage.result @@ -0,0 +1,245 @@ +test_run = require('test_run').new() +--- +... +git_util = require('lua_libs.git_util') +--- +... +util = require('lua_libs.util') +--- +... +vshard_copy_path = util.BUILDDIR .. '/test/var/vshard_git_tree_copy' +--- +... +evolution_log = git_util.log_hashes({args='vshard/storage/reload_evolution.lua', dir=util.SOURCEDIR}) +--- +... +-- Cleanup the directory after a previous build. +_ = os.execute('rm -rf ' .. vshard_copy_path) +--- +... +-- 1. `git worktree` cannot be used because PACKPACK mounts +-- `/source/` in `ro` mode. +-- 2. Just `cp -rf` cannot be used due to a little different +-- behavior in Centos 7. +_ = os.execute('mkdir ' .. vshard_copy_path) +--- +... +_ = os.execute("cd " .. util.SOURCEDIR .. ' && cp -rf `ls -A --ignore=build` ' .. vshard_copy_path) +--- +... +-- Checkout the first commit with a reload_evolution mechanism. +git_util.exec_cmd({cmd='checkout', args='-f', dir=vshard_copy_path}) +--- +... +git_util.exec_cmd({cmd='checkout', args=evolution_log[#evolution_log] .. '~1', dir=vshard_copy_path}) +--- +... +REPLICASET_1 = { 'storage_1_a', 'storage_1_b' } +--- +... +REPLICASET_2 = { 'storage_2_a', 'storage_2_b' } +--- +... +test_run:create_cluster(REPLICASET_1, 'reload_evolution') +--- +... +test_run:create_cluster(REPLICASET_2, 'reload_evolution') +--- +... +util = require('lua_libs.util') +--- +... +util.wait_master(test_run, REPLICASET_1, 'storage_1_a') +--- +... +util.wait_master(test_run, REPLICASET_2, 'storage_2_a') +--- +... +test_run:switch('storage_1_a') +--- +- true +... +vshard.storage.bucket_force_create(1, vshard.consts.DEFAULT_BUCKET_COUNT / 2) +--- +- true +... +bucket_id_to_move = vshard.consts.DEFAULT_BUCKET_COUNT +--- +... +test_run:switch('storage_2_a') +--- +- true +... +fiber = require('fiber') +--- +... +vshard.storage.bucket_force_create(vshard.consts.DEFAULT_BUCKET_COUNT / 2 + 1, vshard.consts.DEFAULT_BUCKET_COUNT / 2) +--- +- true +... +bucket_id_to_move = vshard.consts.DEFAULT_BUCKET_COUNT +--- +... +vshard.storage.internal.reload_version +--- +- null +... +wait_rebalancer_state('The cluster is balanced ok', test_run) +--- +... +box.space.test:insert({42, bucket_id_to_move}) +--- +- [42, 3000] +... +test_run:switch('default') +--- +- true +... +git_util.exec_cmd({cmd='checkout', args=evolution_log[1], dir=vshard_copy_path}) +--- +... +test_run:switch('storage_2_a') +--- +- true +... +package.loaded['vshard.storage'] = nil +--- +... +vshard.storage = require("vshard.storage") +--- +... +test_run:grep_log('storage_2_a', 'vshard.storage.reload_evolution: upgraded to') ~= nil +--- +- true +... +vshard.storage.internal.reload_version +--- +- 1 +... +-- Make sure storage operates well. +vshard.storage.bucket_force_drop(2000) +--- +- true +... +vshard.storage.bucket_force_create(2000) +--- +- true +... +vshard.storage.buckets_info()[2000] +--- +- status: active +  id: 2000 +... +vshard.storage.call(bucket_id_to_move, 'read', 'do_select', {42}) +--- +- true +- - [42, 3000] +... +vshard.storage.bucket_send(bucket_id_to_move, replicaset1_uuid) +--- +- true +... +vshard.storage.garbage_collector_wakeup() +--- +... +fiber = require('fiber') +--- +... +while box.space._bucket:get({bucket_id_to_move}) do fiber.sleep(0.01) end +--- +... +test_run:switch('storage_1_a') +--- +- true +... +vshard.storage.bucket_send(bucket_id_to_move, replicaset2_uuid) +--- +- true +... +test_run:switch('storage_2_a') +--- +- true +... +vshard.storage.call(bucket_id_to_move, 'read', 'do_select', {42}) +--- +- true +- - [42, 3000] +... +-- Check info() does not fail. +vshard.storage.info() ~= nil +--- +- true +... +-- +-- Send buckets to create a disbalance. Wait until the rebalancer +-- repairs it. Similar to `tests/rebalancer/rebalancer.test.lua`. +-- +vshard.storage.rebalancer_disable() +--- +... +move_start = vshard.consts.DEFAULT_BUCKET_COUNT / 2 + 1 +--- +... +move_cnt = 100 +--- +... +assert(move_start + move_cnt < vshard.consts.DEFAULT_BUCKET_COUNT) +--- +- true +... +for i = move_start, move_start + move_cnt - 1 do box.space._bucket:delete{i} end +--- +... +box.space._bucket.index.status:count({vshard.consts.BUCKET.ACTIVE}) +--- +- 1400 +... +test_run:switch('storage_1_a') +--- +- true +... +move_start = vshard.consts.DEFAULT_BUCKET_COUNT / 2 + 1 +--- +... +move_cnt = 100 +--- +... +vshard.storage.bucket_force_create(move_start, move_cnt) +--- +- true +... +box.space._bucket.index.status:count({vshard.consts.BUCKET.ACTIVE}) +--- +- 1600 +... +test_run:switch('storage_2_a') +--- +- true +... +vshard.storage.rebalancer_enable() +--- +... +wait_rebalancer_state('Rebalance routes are sent', test_run) +--- +... +wait_rebalancer_state('The cluster is balanced ok', test_run) +--- +... +box.space._bucket.index.status:count({vshard.consts.BUCKET.ACTIVE}) +--- +- 1500 +... +test_run:switch('default') +--- +- true +... +test_run:drop_cluster(REPLICASET_2) +--- +... +test_run:drop_cluster(REPLICASET_1) +--- +... +test_run:cmd('clear filter') +--- +- true +... diff --git a/test/reload_evolution/storage.test.lua b/test/reload_evolution/storage.test.lua new file mode 100644 index 0000000..7af464b --- /dev/null +++ b/test/reload_evolution/storage.test.lua @@ -0,0 +1,87 @@ +test_run = require('test_run').new() + +git_util = require('lua_libs.git_util') +util = require('lua_libs.util') +vshard_copy_path = util.BUILDDIR .. '/test/var/vshard_git_tree_copy' +evolution_log = git_util.log_hashes({args='vshard/storage/reload_evolution.lua', dir=util.SOURCEDIR}) +-- Cleanup the directory after a previous build. +_ = os.execute('rm -rf ' .. vshard_copy_path) +-- 1. `git worktree` cannot be used because PACKPACK mounts +-- `/source/` in `ro` mode. +-- 2. Just `cp -rf` cannot be used due to a little different +-- behavior in Centos 7. +_ = os.execute('mkdir ' .. vshard_copy_path) +_ = os.execute("cd " .. util.SOURCEDIR .. ' && cp -rf `ls -A --ignore=build` ' .. vshard_copy_path) +-- Checkout the first commit with a reload_evolution mechanism. +git_util.exec_cmd({cmd='checkout', args='-f', dir=vshard_copy_path}) +git_util.exec_cmd({cmd='checkout', args=evolution_log[#evolution_log] .. '~1', dir=vshard_copy_path}) + +REPLICASET_1 = { 'storage_1_a', 'storage_1_b' } +REPLICASET_2 = { 'storage_2_a', 'storage_2_b' } +test_run:create_cluster(REPLICASET_1, 'reload_evolution') +test_run:create_cluster(REPLICASET_2, 'reload_evolution') +util = require('lua_libs.util') +util.wait_master(test_run, REPLICASET_1, 'storage_1_a') +util.wait_master(test_run, REPLICASET_2, 'storage_2_a') + +test_run:switch('storage_1_a') +vshard.storage.bucket_force_create(1, vshard.consts.DEFAULT_BUCKET_COUNT / 2) +bucket_id_to_move = vshard.consts.DEFAULT_BUCKET_COUNT + +test_run:switch('storage_2_a') +fiber = require('fiber') +vshard.storage.bucket_force_create(vshard.consts.DEFAULT_BUCKET_COUNT / 2 + 1, vshard.consts.DEFAULT_BUCKET_COUNT / 2) +bucket_id_to_move = vshard.consts.DEFAULT_BUCKET_COUNT +vshard.storage.internal.reload_version +wait_rebalancer_state('The cluster is balanced ok', test_run) +box.space.test:insert({42, bucket_id_to_move}) + +test_run:switch('default') +git_util.exec_cmd({cmd='checkout', args=evolution_log[1], dir=vshard_copy_path}) + +test_run:switch('storage_2_a') +package.loaded['vshard.storage'] = nil +vshard.storage = require("vshard.storage") +test_run:grep_log('storage_2_a', 'vshard.storage.reload_evolution: upgraded to') ~= nil +vshard.storage.internal.reload_version +-- Make sure storage operates well. +vshard.storage.bucket_force_drop(2000) +vshard.storage.bucket_force_create(2000) +vshard.storage.buckets_info()[2000] +vshard.storage.call(bucket_id_to_move, 'read', 'do_select', {42}) +vshard.storage.bucket_send(bucket_id_to_move, replicaset1_uuid) +vshard.storage.garbage_collector_wakeup() +fiber = require('fiber') +while box.space._bucket:get({bucket_id_to_move}) do fiber.sleep(0.01) end +test_run:switch('storage_1_a') +vshard.storage.bucket_send(bucket_id_to_move, replicaset2_uuid) +test_run:switch('storage_2_a') +vshard.storage.call(bucket_id_to_move, 'read', 'do_select', {42}) +-- Check info() does not fail. +vshard.storage.info() ~= nil + +-- +-- Send buckets to create a disbalance. Wait until the rebalancer +-- repairs it. Similar to `tests/rebalancer/rebalancer.test.lua`. +-- +vshard.storage.rebalancer_disable() +move_start = vshard.consts.DEFAULT_BUCKET_COUNT / 2 + 1 +move_cnt = 100 +assert(move_start + move_cnt < vshard.consts.DEFAULT_BUCKET_COUNT) +for i = move_start, move_start + move_cnt - 1 do box.space._bucket:delete{i} end +box.space._bucket.index.status:count({vshard.consts.BUCKET.ACTIVE}) +test_run:switch('storage_1_a') +move_start = vshard.consts.DEFAULT_BUCKET_COUNT / 2 + 1 +move_cnt = 100 +vshard.storage.bucket_force_create(move_start, move_cnt) +box.space._bucket.index.status:count({vshard.consts.BUCKET.ACTIVE}) +test_run:switch('storage_2_a') +vshard.storage.rebalancer_enable() +wait_rebalancer_state('Rebalance routes are sent', test_run) +wait_rebalancer_state('The cluster is balanced ok', test_run) +box.space._bucket.index.status:count({vshard.consts.BUCKET.ACTIVE}) + +test_run:switch('default') +test_run:drop_cluster(REPLICASET_2) +test_run:drop_cluster(REPLICASET_1) +test_run:cmd('clear filter') diff --git a/test/reload_evolution/storage_1_a.lua b/test/reload_evolution/storage_1_a.lua new file mode 100755 index 0000000..f1a2981 --- /dev/null +++ b/test/reload_evolution/storage_1_a.lua @@ -0,0 +1,48 @@ +#!/usr/bin/env tarantool + +require('strict').on() + +local log = require('log') +local fiber = require('fiber') +local util = require('lua_libs.util') +local fio = require('fio') + +-- Get instance name +NAME = fio.basename(arg[0], '.lua') + +-- test-run gate. +test_run = require('test_run').new() +require('console').listen(os.getenv('ADMIN')) + +-- Run one storage on a different vshard version. +-- To do that, place vshard src to +-- BUILDDIR/test/var/vshard_git_tree_copy/. +if NAME == 'storage_2_a' then +    local script_path = debug.getinfo(1).source:match("@?(.*/)") +    vshard_copy = util.BUILDDIR .. '/test/var/vshard_git_tree_copy' +    package.path = string.format( +        '%s/?.lua;%s/?/init.lua;%s', +        vshard_copy, vshard_copy, package.path +    ) +end + +-- Call a configuration provider +cfg = require('localcfg') +-- Name to uuid map +names = { +    ['storage_1_a'] = '8a274925-a26d-47fc-9e1b-af88ce939412', +    ['storage_1_b'] = '3de2e3e1-9ebe-4d0d-abb1-26d301b84633', +    ['storage_2_a'] = '1e02ae8a-afc0-4e91-ba34-843a356b8ed7', +    ['storage_2_b'] = '001688c3-66f8-4a31-8e19-036c17d489c2', +} + +replicaset1_uuid = 'cbf06940-0790-498b-948d-042b62cf3d29' +replicaset2_uuid = 'ac522f65-aa94-4134-9f64-51ee384f1a54' +replicasets = {replicaset1_uuid, replicaset2_uuid} + +-- Start the database with sharding +vshard = require('vshard') +vshard.storage.cfg(cfg, names[NAME]) + +-- Bootstrap storage. +require('lua_libs.bootstrap') diff --git a/test/reload_evolution/storage_1_b.lua b/test/reload_evolution/storage_1_b.lua new file mode 120000 index 0000000..02572da --- /dev/null +++ b/test/reload_evolution/storage_1_b.lua @@ -0,0 +1 @@ +storage_1_a.lua \ No newline at end of file diff --git a/test/reload_evolution/storage_2_a.lua b/test/reload_evolution/storage_2_a.lua new file mode 120000 index 0000000..02572da --- /dev/null +++ b/test/reload_evolution/storage_2_a.lua @@ -0,0 +1 @@ +storage_1_a.lua \ No newline at end of file diff --git a/test/reload_evolution/storage_2_b.lua b/test/reload_evolution/storage_2_b.lua new file mode 120000 index 0000000..02572da --- /dev/null +++ b/test/reload_evolution/storage_2_b.lua @@ -0,0 +1 @@ +storage_1_a.lua \ No newline at end of file diff --git a/test/reload_evolution/suite.ini b/test/reload_evolution/suite.ini new file mode 100644 index 0000000..5f55418 --- /dev/null +++ b/test/reload_evolution/suite.ini @@ -0,0 +1,6 @@ +[default] +core = tarantool +description = Reload evolution tests +script = test.lua +is_parallel = False +lua_libs = ../lua_libs ../../example/localcfg.lua diff --git a/test/reload_evolution/test.lua b/test/reload_evolution/test.lua new file mode 100644 index 0000000..ad0543a --- /dev/null +++ b/test/reload_evolution/test.lua @@ -0,0 +1,9 @@ +#!/usr/bin/env tarantool + +require('strict').on() + +box.cfg{ +    listen = os.getenv("LISTEN"), +} + +require('console').listen(os.getenv('ADMIN')) diff --git a/test/unit/reload_evolution.result b/test/unit/reload_evolution.result new file mode 100644 index 0000000..342ac24 --- /dev/null +++ b/test/unit/reload_evolution.result @@ -0,0 +1,45 @@ +test_run = require('test_run').new() +--- +... +fiber = require('fiber') +--- +... +log = require('log') +--- +... +util = require('util') +--- +... +reload_evolution = require('vshard.storage.reload_evolution') +--- +... +-- Init with the latest version. +fake_M = { reload_evolution_version = reload_evolution.version } +--- +... +-- Test reload to the same version. +reload_evolution.upgrade(fake_M) +--- +... +test_run:grep_log('default', 'vshard.storage.evolution') == nil +--- +- true +... +-- Test downgrage version. +log.info(string.rep('a', 1000)) +--- +... +fake_M.reload_evolution_version = fake_M.reload_evolution_version + 1 +--- +... +err = util.check_error(reload_evolution.upgrade, fake_M) +--- +... +err:match('auto%-downgrade is not implemented') +--- +- auto-downgrade is not implemented +... +test_run:grep_log('default', 'vshard.storage.evolution', 1000) ~= nil +--- +- false +... diff --git a/test/unit/reload_evolution.test.lua b/test/unit/reload_evolution.test.lua new file mode 100644 index 0000000..b8a3ca8 --- /dev/null +++ b/test/unit/reload_evolution.test.lua @@ -0,0 +1,18 @@ +test_run = require('test_run').new() +fiber = require('fiber') +log = require('log') +util = require('util') +reload_evolution = require('vshard.storage.reload_evolution') +-- Init with the latest version. +fake_M = { reload_version = reload_evolution.version } + +-- Test reload to the same version. +reload_evolution.upgrade(fake_M) +test_run:grep_log('default', 'vshard.storage.evolution') == nil + +-- Test downgrage version. +log.info(string.rep('a', 1000)) +fake_M.reload_evolution_version = fake_M.reload_evolution_version + 1 +err = util.check_error(reload_evolution.upgrade, fake_M) +err:match('auto%-downgrade is not implemented') +test_run:grep_log('default', 'vshard.storage.evolution', 1000) ~= nil diff --git a/vshard/storage/init.lua b/vshard/storage/init.lua index 07bd00c..69e858a 100644 --- a/vshard/storage/init.lua +++ b/vshard/storage/init.lua @@ -10,6 +10,7 @@ if rawget(_G, MODULE_INTERNALS) then      local vshard_modules = {          'vshard.consts', 'vshard.error', 'vshard.cfg',          'vshard.replicaset', 'vshard.util', +        'vshard.storage.reload_evolution'      }      for _, module in pairs(vshard_modules) do          package.loaded[module] = nil @@ -20,12 +21,16 @@ local lerror = require('vshard.error')  local lcfg = require('vshard.cfg')  local lreplicaset = require('vshard.replicaset')  local util = require('vshard.util') +local reload_evolution = require('vshard.storage.reload_evolution')  local M = rawget(_G, MODULE_INTERNALS)  if not M then      --      -- The module is loaded for the first time.      -- +    -- !!!WARNING: any change of this table must be reflected in +    -- `vshard.storage.reload_evolution` module to guarantee +    -- reloadability of the module.      M = {          ---------------- Common module attributes ----------------          -- The last passed configuration. @@ -105,6 +110,11 @@ if not M then          -- a destination replicaset must drop already received          -- data.          rebalancer_sending_bucket = 0, + +        ------------------------- Reload ------------------------- +        -- Version of the loaded module. This number is used on +        -- reload to determine which upgrade scripts to run. +        reload_version = reload_evolution.version,      }  end @@ -1863,6 +1873,7 @@ end  if not rawget(_G, MODULE_INTERNALS) then      rawset(_G, MODULE_INTERNALS, M)  else +    reload_evolution.upgrade(M)      storage_cfg(M.current_cfg, M.this_replica.uuid)      M.module_version = M.module_version + 1  end diff --git a/vshard/storage/reload_evolution.lua b/vshard/storage/reload_evolution.lua new file mode 100644 index 0000000..8502a33 --- /dev/null +++ b/vshard/storage/reload_evolution.lua @@ -0,0 +1,58 @@ +-- +-- This module is used to upgrade the vshard.storage on the fly. +-- It updates internal Lua structures in case they are changed +-- in a commit. +-- +local log = require('log') + +-- +-- Array of upgrade functions. +-- migrations[version] = function which upgrades module version +-- from `version` to `version + 1`. +-- +local migrations = {} + +-- Initialize reload_upgrade mechanism +migrations[#migrations + 1] = function (M) +    -- Code to update Lua objects. +end + +-- +-- Perform an update based on a version stored in `M` (internals). +-- @param M Old module internals which should be updated. +-- +local function upgrade(M) +    local start_version = M.reload_version or 1 +    if start_version > #migrations then +        local err_msg = string.format( +            'vshard.storage.reload_evolution: ' .. +            'auto-downgrade is not implemented; ' .. +            'loaded version is %d, upgrade script version is %d', +            start_version, #migrations +        ) +        log.error(err_msg) +        error(err_msg) +    end +    for i = start_version, #migrations  do +        local ok, err = pcall(migrations[i], M) +        if ok then +            log.info('vshard.storage.reload_evolution: upgraded to %d version', +                     i) +        else +            local err_msg = string.format( +                'vshard.storage.reload_evolution: ' .. +                'error during upgrade to %d version: %s', i, err +            ) +            log.error(err_msg) +            error(err_msg) +        end +        -- Update the version just after upgrade to have an +        -- actual version in case of an error. +        M.reload_version = i +    end +end + +return { +    version = #migrations, +    upgrade = upgrade, +}