From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from localhost (localhost [127.0.0.1]) by turing.freelists.org (Avenir Technologies Mail Multiplex) with ESMTP id B862327581 for ; Wed, 1 Aug 2018 14:10:04 -0400 (EDT) Received: from turing.freelists.org ([127.0.0.1]) by localhost (turing.freelists.org [127.0.0.1]) (amavisd-new, port 10024) with ESMTP id raD7IS-_vhgg for ; Wed, 1 Aug 2018 14:10:04 -0400 (EDT) Received: from smtp34.i.mail.ru (smtp34.i.mail.ru [94.100.177.94]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by turing.freelists.org (Avenir Technologies Mail Multiplex) with ESMTPS id 633AD22C05 for ; Wed, 1 Aug 2018 14:10:04 -0400 (EDT) Subject: [tarantool-patches] Re: [PATCH 4/4] Introduce storage reload evolution References: <1e067dd68d69270ccf5aea2ab73548ebd9a0f2ad.1532940401.git.avkhatskevich@tarantool.org> <3d8a81f7-33a6-bf1d-809f-ae5b95730a64@tarantool.org> <83a958f1-e47c-cf5c-7996-3983b7a28b9f@tarantool.org> <264b6bab-ba8a-2bd4-f730-fe83ad52279b@tarantool.org> From: Alex Khatskevich Message-ID: Date: Wed, 1 Aug 2018 21:09:56 +0300 MIME-Version: 1.0 In-Reply-To: <264b6bab-ba8a-2bd4-f730-fe83ad52279b@tarantool.org> Content-Type: text/plain; charset="utf-8"; format="flowed" Content-Transfer-Encoding: 8bit Content-Language: en-US Sender: tarantool-patches-bounce@freelists.org Errors-to: tarantool-patches-bounce@freelists.org Reply-To: tarantool-patches@freelists.org List-help: List-unsubscribe: List-software: Ecartis version 1.0.0 List-Id: tarantool-patches List-subscribe: List-owner: List-post: List-archive: To: Vladislav Shpilevoy , tarantool-patches@freelists.org I have rebased this branch over a new master. On 01.08.2018 15:36, Vladislav Shpilevoy wrote: > Thanks for the patch! See 2 comments below. > >> diff --git a/test/lua_libs/git_util.lua b/test/lua_libs/git_util.lua >> new file mode 100644 >> index 0000000..8826c4c >> --- /dev/null >> +++ b/test/lua_libs/git_util.lua >> @@ -0,0 +1,49 @@ >> +-- >> +-- Lua bridge for some of the git commands. >> +-- >> +local os = require('os') >> + >> +-- >> +-- Exec a git command. >> +-- @param params Table of parameters: >> +--        * options - git options. >> +--        * cmd - git command. >> +--        * args - command arguments. >> +--        * dir - working directory. >> +--        * fout - write output to the file. >> +local function exec_cmd(params) > > 1. Lets rename it to just 'exec'. Obviously it executes > 'cmd'. Also lets make 'cmd' be first mandatory non-named > parameter and the second is params: args, dir etc. Fixed. > >     local function exec(cmd, params) > >> +    local fout = params.fout >> +    local shell_cmd = {'git'} >> +    for _, param in pairs({'options', 'cmd', 'args'}) do >> +        table.insert(shell_cmd, params[param]) >> +    end >> +    if fout then >> +        table.insert(shell_cmd, ' >' .. fout) >> +    end >> +    shell_cmd = table.concat(shell_cmd, ' ') >> +    if params.dir then >> +        shell_cmd = string.format('cd %s && %s', params.dir, shell_cmd) >> +    end >> +    local res = os.execute(shell_cmd) >> +    assert(res == 0, 'Git cmd error: ' .. res) >> +end >> + >> diff --git a/vshard/storage/reload_evolution.lua >> b/vshard/storage/reload_evolution.lua >> new file mode 100644 >> index 0000000..8502a33 >> --- /dev/null >> +++ b/vshard/storage/reload_evolution.lua >> @@ -0,0 +1,58 @@ >> +-- >> +-- This module is used to upgrade the vshard.storage on the fly. >> +-- It updates internal Lua structures in case they are changed >> +-- in a commit. >> +-- >> +local log = require('log') >> + >> +-- >> +-- Array of upgrade functions. >> +-- migrations[version] = function which upgrades module version >> +-- from `version` to `version + 1`. >> +-- >> +local migrations = {} >> + >> +-- Initialize reload_upgrade mechanism >> +migrations[#migrations + 1] = function (M) > > 2. Redundant white space after 'function'. Fixed. > >> +    -- Code to update Lua objects. >> +end >> + >> +-- >> +-- Perform an update based on a version stored in `M` (internals). >> +-- @param M Old module internals which should be updated. >> +-- >> +local function upgrade(M) >> +    local start_version = M.reload_version or 1 >> +    if start_version > #migrations then >> +        local err_msg = string.format( >> +            'vshard.storage.reload_evolution: ' .. >> +            'auto-downgrade is not implemented; ' .. >> +            'loaded version is %d, upgrade script version is %d', >> +            start_version, #migrations >> +        ) >> +        log.error(err_msg) >> +        error(err_msg) >> +    end >> +    for i = start_version, #migrations  do >> +        local ok, err = pcall(migrations[i], M) >> +        if ok then >> +            log.info('vshard.storage.reload_evolution: upgraded to >> %d version', >> +                     i) >> +        else >> +            local err_msg = string.format( >> +                'vshard.storage.reload_evolution: ' .. >> +                'error during upgrade to %d version: %s', i, err >> +            ) >> +            log.error(err_msg) >> +            error(err_msg) >> +        end >> +        -- Update the version just after upgrade to have an >> +        -- actual version in case of an error. >> +        M.reload_version = i >> +    end >> +end >> + >> +return { >> +    version = #migrations, >> +    upgrade = upgrade, >> +} > Here is a full diff for git_util.lua file commit 8fbf81cf46b579c158b2b5292a89a0b11ab75494 Author: AKhatskevich Date:   Fri Jun 29 20:34:26 2018 +0300     Introduce storage reload evolution     Changes:     1. Introduce storage reload evolution.     2. Setup cross-version reload testing.     1:     This mechanism updates Lua objects on reload in case they are     changed in a new vshard.storage version.     Since this commit, any change in vshard.storage.M has to be     reflected in vshard.storage.reload_evolution to guarantee     correct reload.     2:     The testing uses git infrastructure and is performed in the following     way:     1. Copy old version of vshard to a temp folder.     2. Run vshard on this code.     3. Checkout the latest version of the vshard sources.     4. Reload vshard storage.     5. Make sure it works (Perform simple tests).     Notes:     * this patch contains some legacy-driven decisions:       1. SOURCEDIR path retrieved differently in case of          packpack build.       2. git directory in the `reload_evolution/storage` test          is copied with respect to Centos 7 and `ro` mode of          SOURCEDIR.     Closes #112 #125 diff --git a/test/lua_libs/git_util.lua b/test/lua_libs/git_util.lua new file mode 100644 index 0000000..48f9827 --- /dev/null +++ b/test/lua_libs/git_util.lua @@ -0,0 +1,46 @@ +-- +-- Lua bridge for some of the git commands. +-- +local os = require('os') + +-- +-- Exec a git command. +-- @param cmd Git command to run. +-- @param params Table of parameters: +--        * options - git options. +--        * args - command arguments. +--        * dir - working directory. +--        * fout - write output to the file. +local function exec(cmd, params) +    params.options = params.options or '' +    params.args = params.args or '' +    local shell_cmd = string.format('git %s %s %s', params.options, cmd, +                                    params.args) +    if params.fout then +        shell_cmd = string.format('%s >%s', shell_cmd, params.fout) +    end +    if params.dir then +        shell_cmd = string.format('cd %s && %s', params.dir, shell_cmd) +    end +    local res = os.execute(shell_cmd) +    assert(res == 0, 'Git cmd error: ' .. res) +end + +local function log_hashes(params) +    params.args = "--format='%h' " .. params.args +    -- Store log to the file. +    local temp_file = os.tmpname() +    params.fout = temp_file +    exec('log', params) +    local lines = {} +    for line in io.lines(temp_file) do +        table.insert(lines, line) +    end +    os.remove(temp_file) +    return lines +end + +return { +    exec = exec, +    log_hashes = log_hashes +}