From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from smtp57.i.mail.ru (smtp57.i.mail.ru [217.69.128.37]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by dev.tarantool.org (Postfix) with ESMTPS id 6AEF745C305 for ; Tue, 8 Dec 2020 14:39:01 +0300 (MSK) From: Oleg Koshovetc References: <984618ac-e7ee-c7db-4c0d-f71fbba59210@tarantool.org> Message-ID: <747b240c-ca63-57c9-c935-bcc6da921de7@tarantool.org> Date: Tue, 8 Dec 2020 14:38:16 +0300 MIME-Version: 1.0 In-Reply-To: Content-Type: text/plain; charset="utf-8"; format="flowed" Content-Transfer-Encoding: 8bit Content-Language: en-US Subject: [Tarantool-patches] [PATCH 2/2] install: local benchmarks installation scripts List-Id: Tarantool development patches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , To: Alexander Turenko , Alexander Tikhonov Cc: tarantool-patches@dev.tarantool.org All of the following can be accessed on github https://github.com/tarantool/bench-run/pull/14 The 'install' command was added to ttbench. It clones benchmarks and installs them localy so the developers do not have to do it themselves. Also added the -s flag that suppresses all the output on any command ---  benches/cbench/install.sh     |   8 ++  benches/common.sh             |  18 +++-  benches/linkbench/install.sh  |   8 ++  benches/linkbench/run.sh      |   6 +-  benches/nosqlbench/install.sh |   8 ++  benches/sysbench/install.sh   |   8 ++  benches/tpcc/install.sh       |   8 ++  benches/tpch/install.sh       |   5 +  benches/ycsb/install.sh       |   6 ++  config.sh                     |  38 ++++---  ttbench                       | 232 ++++++++++++++++++++++++++++++++++--------  11 files changed, 290 insertions(+), 55 deletions(-)  create mode 100755 benches/cbench/install.sh  create mode 100755 benches/linkbench/install.sh  create mode 100755 benches/nosqlbench/install.sh  create mode 100755 benches/sysbench/install.sh  create mode 100755 benches/tpcc/install.sh  create mode 100755 benches/tpch/install.sh  create mode 100755 benches/ycsb/install.sh diff --git a/benches/cbench/install.sh b/benches/cbench/install.sh new file mode 100755 index 0000000..39046a0 --- /dev/null +++ b/benches/cbench/install.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -euo pipefail + +git clone https://github.com/tarantool/cbench.git "$PWD" + +cmake . -DCMAKE_BUILD_TYPE=RelWithDebInfo +make -j diff --git a/benches/common.sh b/benches/common.sh index 1116cc9..abc99d7 100644 --- a/benches/common.sh +++ b/benches/common.sh @@ -65,7 +65,9 @@ function kill_tarantool {  }  function error { -    echo "ERROR:" "$@" +    local _caller +    _caller=( $(caller 0) ) +    echo "${_caller[2]} line=${_caller[0]} fn=${_caller[1]} ERROR:" "$@"      exit 100  } @@ -97,7 +99,7 @@ function wait_for_tarantool_runnning {      local tt=0      while [ "$tt" -lt "$t" ]; do -        if echo 'if type(box.cfg) ~= "function" then return box.info().status end' | tarantoolctl connect "$creds" 2>/dev/null | grep -q 'running'; then +        if echo 'if type(box.cfg) ~= "function" then return box.info().status end' | "$TARANTOOLCTL_EXECUTABLE" connect "$creds" 2>/dev/null | grep -q 'running'; then              return 0          fi @@ -107,3 +109,15 @@ function wait_for_tarantool_runnning {      return 1  } + +function is_directory_empty { +    local d="$1" + +    [ ! -d "$d" ] && error "no such directory '$d'" + +    if find "$d/" -maxdepth 1 -mindepth 1 | grep -q .; then +        return 1 +    else +        return 0 +    fi +} diff --git a/benches/linkbench/install.sh b/benches/linkbench/install.sh new file mode 100755 index 0000000..2aa5386 --- /dev/null +++ b/benches/linkbench/install.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -euo pipefail + +git clone https://github.com/tarantool/linkbench.git -b update-fixes "$PWD" + +luarocks install https://raw.githubusercontent.com/tarantool/gperftools/master/rockspecs/gperftools-scm-1.rockspec \ +    --tree .rocks --lua-version 5.1 diff --git a/benches/linkbench/run.sh b/benches/linkbench/run.sh index c5f493e..739c46e 100755 --- a/benches/linkbench/run.sh +++ b/benches/linkbench/run.sh @@ -18,7 +18,11 @@ make -C src/tarantool -B  tfile=src/tarantool/app.lua  stop_and_clean_tarantool -maybe_under_numactl "${numaopts[@]}" -- tarantool "$tfile" 1>/dev/null 2>/dev/null & + +lua_path_prefix="$PWD/.rocks/share/lua/5.1" + +export LUA_PATH="$lua_path_prefix/?.lua;$lua_path_prefix/?/init.lua" +maybe_under_numactl "${numaopts[@]}" -- tarantool "$tfile" 1>ttserver.log 2>&1 &  cfgfile=config/LinkConfigTarantool.properties  sed "s/^maxid1 = .*/maxid1 = $LINKBENCH_WORKLOAD_SIZE/g" -i config/FBWorkload.properties diff --git a/benches/nosqlbench/install.sh b/benches/nosqlbench/install.sh new file mode 100755 index 0000000..5659613 --- /dev/null +++ b/benches/nosqlbench/install.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -euo pipefail + +git clone --recursive https://github.com/tarantool/nosqlbench.git "$PWD" +git submodule foreach --recursive +cmake . +make -j diff --git a/benches/sysbench/install.sh b/benches/sysbench/install.sh new file mode 100755 index 0000000..9678254 --- /dev/null +++ b/benches/sysbench/install.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -euo pipefail + +git clone https://github.com/tarantool/sysbench.git "$PWD" +./autogen.sh +./configure --with-tarantool --without-mysql +make -j diff --git a/benches/tpcc/install.sh b/benches/tpcc/install.sh new file mode 100755 index 0000000..d6e827e --- /dev/null +++ b/benches/tpcc/install.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -euo pipefail + +git clone https://github.com/tarantool/tpcc.git "$PWD" +cd src +make -j +cd .. diff --git a/benches/tpch/install.sh b/benches/tpch/install.sh new file mode 100755 index 0000000..2ee1e14 --- /dev/null +++ b/benches/tpch/install.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +set -euo pipefail + +git clone https://github.com/tarantool/tpch.git "$PWD" diff --git a/benches/ycsb/install.sh b/benches/ycsb/install.sh new file mode 100755 index 0000000..5c536ea --- /dev/null +++ b/benches/ycsb/install.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -euo pipefail + +git clone https://github.com/tarantool/YCSB.git "$PWD" +mvn clean package diff --git a/config.sh b/config.sh index 93d3fc5..bd6f558 100644 --- a/config.sh +++ b/config.sh @@ -1,41 +1,55 @@  # path to tarantool executable  export TARANTOOL_EXECUTABLE=tarantool +# path to tarantoolctl executable +export TARANTOOLCTL_EXECUTABLE=tarantoolctl +  # location of tests to be run +# all the tests will be copied and run in this directory  export BENCH_WORKDIR="$PWD/.benchdir" -# cbench parameters  export CBENCH_DIR="$HOME/work/cbench/" +export LINKBENCH_DIR="$HOME/work/linkbench/" +export NOSQLBENCH_DIR="$HOME/work/nosqlbench/" +export SYSBENCH_DIR="$HOME/work/sysbench/" +export TPCC_DIR="$HOME/work/tpcc/" +export TPCH_DIR="$HOME/work/tpch/" +export YCSB_DIR="$HOME/work/ycsb/" + +# ----------------- BENCHMARK PARAMS ----------------- +# Fill free to delete all the params as long as all +# the benchmarks have their own defaults set in run.sh + +# cbench parameters  export CBENCH_VINYL_WORKLOAD=50  export CBENCH_MEMTX_WORKLOAD=10000  # linkbench parameters -export LINKBENCH_DIR="$HOME/work/linkbench/"  export LINKBENCH_REQUESTERS=1  export LINKBENCH_REQUESTS=1000  export LINKBENCH_WORKLOAD_SIZE=1000 -# linkbench parameters -export NOSQLBENCH_DIR="$HOME/work/nosqlbench/" +# nosqlbench parameters +export NOSQLBENCH_TIMELIMIT=20000 +export NOSQLBENCH_BATCHCOUNT=10 +export NOSQLBENCH_RPS=20000 -# linkbench parameters -export SYSBENCH_DIR="$HOME/work/sysbench/" +# sysbench parameters  export SYSBENCH_RUNS=1  export SYSBENCH_TIME=5  # tpcc parameters -export TPCC_DIR="$HOME/work/tpcc/"  export TPCC_TIME=20  export TPCC_WARMUPTIME=5  export TPCC_WAREHOUSES=15 -export TPCC_FROMSNAPSHOT="/tmp/00000000000000033515.snap" +## possible to start tpcc benchmark from snapshot wich I higly recommend +## as tpcc_load takes a good portion of time to load the data +# export TPCC_FROMSNAPSHOT="/tmp/00000000000000033515.snap" -# tpcc parameters -export TPCH_DIR="$HOME/work/tpch/" +# tpch parameters  export TPCH_SKIP_SQLITE=1 -# tpcc parameters -export YCSB_DIR="$HOME/work/ycsb/" +# ycsb parameters  export YCSB_OPERATIONCOUNT=1000  export YCSB_RECORDCOUNT=1000  export YCSB_RUNS=1 diff --git a/ttbench b/ttbench index ac656f5..b856820 100755 --- a/ttbench +++ b/ttbench @@ -4,72 +4,174 @@ set -euo pipefail  TTBENCH_CONFIG=config.sh  TTBENCH_BENCHES=all +TTBENCH_SILENT= +TTBENCH_FORCE=  TTBENCH_COMMAND=  TTBENCH_BENCHDIR=benches -function error { -    echo "ERROR:" "$@" -    exit 100 -} +TTBENCH_ENV_OVERRIDES=() + +source "$TTBENCH_BENCHDIR/common.sh"  function print_help {      grep '^\s*# HELP' "$0" | sed 's/^\s*# HELP\s//g'  } -function list_benchmarks { +function maybe_silently { +    if [ -z "$TTBENCH_SILENT" ]; then +        "$@" +    else +        "$@" 1>/dev/null 2>/dev/null +    fi +} + +function list_all_benchmarks {      for d in $TTBENCH_BENCHDIR/*; do          sed "s#$TTBENCH_BENCHDIR/##" <<< "$d"      done | grep -v common.sh  } +function list_given_benchmarks { +    if [ "$TTBENCH_BENCHES" == all ]; then +        for b in $(list_all_benchmarks); do +            echo "$b" +        done +    else +        IFS=, read -ra benchlist <<< "$TTBENCH_BENCHES" +        for b in "${benchlist[@]}"; do +            if [ ! -d "$TTBENCH_BENCHDIR/$b" ]; then +                error "Unknown benchmark='$b', please run '$0 list-benchmarks'" +            fi +        done + +        for b in "${benchlist[@]}"; do +            echo "$b" +        done +    fi +} + +function get_bench_dir { +    local bench="$1" +    if benchdir=$(env | grep "${bench^^}_DIR" | sed 's/^.*=//'); then +        echo "$benchdir" +    else +        error "variable '${bench^^}_DIR' is not set in config" +    fi +} +  function run_bench {      local bench="$1"      local benchdir=      local workdir="${BENCH_WORKDIR:?}/$bench/" +    mkdir -p "$BENCH_WORKDIR"      cp "$TTBENCH_BENCHDIR/common.sh" "${BENCH_WORKDIR}"      export COMMON_FUNCTIONS="$BENCH_WORKDIR/common.sh" -    if benchdir=$(env | grep "${bench^^}_DIR" | sed 's/^.*=//'); then -        echo "Running $bench with workdir $benchdir" 1>&2 -        rm -rf "$workdir" -        mkdir -p "$workdir" - -        cp -r "$TTBENCH_BENCHDIR/$bench"/* "$workdir" -        cp -r "$benchdir"/* "$workdir" - -        local runner= -        runner=$(readlink -f "$TTBENCH_BENCHDIR/$bench/run.sh") -        pushd "$workdir" 1>/dev/null -        "$runner" -        popd 1>/dev/null -    else -        error "variable '${bench^^}_DIR' is not set in config, can't run benchmark" -    fi +    local benchdir= +    benchdir=$(get_bench_dir "$bench") + +    echo "Running $bench with workdir $benchdir" 1>&2 +    rm -rf "$workdir" +    mkdir -p "$workdir" + +    cp -r "$TTBENCH_BENCHDIR/$bench"/* "$workdir" +    cp -r "$benchdir"/* "$workdir" + +    # some of benchmarks have external dependencies that are stored in .rocks directory +    # but some of benchmark do not have them +    cp -r "$benchdir"/.rocks "$workdir" 2>/dev/null || true + +    local runner= +    runner=$(readlink -f "$TTBENCH_BENCHDIR/$bench/run.sh") +    pushd "$workdir" 1>/dev/null +    "$runner" +    popd 1>/dev/null  }  function run_benchmarks {      # must be set in config -    [ -z "$BENCH_WORKDIR" ] && error "BENCH_WORKDIR is not set" +    [ -z "$BENCH_WORKDIR" ]           && error "BENCH_WORKDIR is not set" +    [ -z "$TARANTOOL_EXECUTABLE" ]    && error "TARANTOOL_EXECUTABLE is not set" +    [ -z "$TARANTOOLCTL_EXECUTABLE" ] && error "TARANTOOLCTL_EXECUTABLE is not set" -    if [ "$TTBENCH_BENCHES" == all ]; then -        for b in $(list_benchmarks); do -            run_bench "$b" -        done -    else -        IFS=, read -ra benchlist <<< "$TTBENCH_BENCHES" -        # local benchlist=( $(sed 's/,/ /g' <<< "$TTBENCH_BENCHES") ); -        for b in "${benchlist[@]}"; do -            if [ ! -d "$TTBENCH_BENCHDIR/$b" ]; then -                error "Unknown benchmark='$b', please run '$0 list-benchmarks'" -            fi -        done +    for b in $(list_given_benchmarks); do +        run_bench "$b" +    done +} -        for b in "${benchlist[@]}"; do -            run_bench "$b" -        done +function install_benchmark { +    local bench="$1" + +    local benchdir= +    benchdir=$(get_bench_dir "$bench") + +    mkdir -p "$benchdir" + +    if ! is_directory_empty "$benchdir"; then +        if [ -z "$TTBENCH_FORCE" ]; then +            error "'$benchdir' is not empty, use --force flag to reinstall" +        fi +    fi + +    local bdir= +    bdir=$(realpath "$benchdir") + +    local install_path_ok= + +    if [[ "$bdir" == /home/* ]]; then +        install_path_ok=1 +    fi + +    if [[ "$bdir" == /opt/* ]]; then +        install_path_ok=1 +    fi + +    if [[ "$bdir" == /tmp/* ]]; then +        install_path_ok=1 +    fi + +    if [ -z "$install_path_ok" ]; then +        echo "ERROR: benchmark install directory='$bdir' is" +        echo "ERROR: neither under /home/" +        echo "ERROR: nor under /opt/" +        echo "ERROR: nor under /tmp/" +        echo "ERROR: any other installation path is considered to be unsafe" +        echo "ERROR: as it is required to run rm -rf on benchmark insall directory" +        echo "ERROR: if you think this is an error, consider patching $0 yourself" +        echo "ERROR: or installing the benchmarks manually" +        error "unsafe install directory='$bdir' detected"      fi + +    rm -rf "${bdir:?}" +    mkdir -p "$bdir" + +    local install_script="$TTBENCH_BENCHDIR/$bench/install.sh" +    [ ! -f "$install_script" ] && error +    install_script=$(readlink -f "$TTBENCH_BENCHDIR/$bench/install.sh") + +    pushd "$bdir" 1>/dev/null +    "$install_script" +    popd 1>/dev/null +} + +function run_install { +    local benchlist +    benchlist=( $(list_given_benchmarks) ) + +    # running this in single loop for a side effect where get_bench_dir +    # will raise an error if some env variables are not set +    for b in "${benchlist[@]}"; do +        get_bench_dir "$b" +        if [ ! -f "$TTBENCH_BENCHDIR/$b/install.sh" ]; then +            error "$TTBENCH_BENCHDIR/$b is missing install.sh script" +        fi +    done + +    for b in "${benchlist[@]}"; do +        install_benchmark "$b" +    done  }  # HELP ttbench - launch tarantool external benchmarks @@ -96,6 +198,20 @@ while [[ $# -gt 0 ]]; do              set -x              shift;; +        # HELP   -f, --force +        # HELP           if used with install command, will reinstall benchmarks +        # HELP +        -f|--force) +            TTBENCH_FORCE=1 +            shift;; + +        # HELP   -s, --silent +        # HELP           suppress any output, just give an exit code +        # HELP +        -s|--silent) +            TTBENCH_SILENT=1 +            shift;; +          # HELP   -b, --benchmarks 'bench1,bench2'          # HELP           comma separated list of benchmarks (defauts to 'all')          # HELP @@ -104,6 +220,25 @@ while [[ $# -gt 0 ]]; do              [ -z "$TTBENCH_BENCHES" ] && error "$option param can not be empty"              shift 2;; +        # HELP   -D, --define KEY=VALUE +        # HELP   -DKEY=VALUE +        # HELP           allows to define extra ENV variables +        # HELP           or rewrite ones in config file from command line +        # HELP +        -D|--define|-D*) +            case "$option" in +                -D|--define) +                    override="${2:-}" +                    [ -z "$override" ] && error "$option param can not be empty" +                    TTBENCH_ENV_OVERRIDES+=( "$override" ) +                    shift 2;; +                -D*) +                    override="${option#-D}" +                    [ -z "$override" ] && error "$option param can not be empty" +                    TTBENCH_ENV_OVERRIDES+=( "$override" ) +                    shift;; +            esac;; +          # HELP   -h, --help          # HELP           print help message          # HELP @@ -116,7 +251,18 @@ while [[ $# -gt 0 ]]; do      esac  done -[ -f "$TTBENCH_CONFIG" ] && source "$TTBENCH_CONFIG" +if [ -f "$TTBENCH_CONFIG" ]; then +    source "$TTBENCH_CONFIG" +else +    error "config file='$TTBENCH_CONFIG' is absent" +fi + +# Disgusting hack for empty arrays on bash < 4.4 +if [ 0 -ne "${#TTBENCH_ENV_OVERRIDES[@]}" ]; then +    for override in "${TTBENCH_ENV_OVERRIDES[@]}"; do +        export "$override" +    done +fi  TTBENCH_COMMAND="${1:-}" @@ -135,12 +281,18 @@ case "$TTBENCH_COMMAND" in      # HELP           lists all known benchmarks      # HELP      ls|list-benchmarks) -        list_benchmarks;; +        maybe_silently list_all_benchmarks;;      # HELP   run      # HELP           run benchmarks specified by --benchmarks flag +    # HELP      run) -        run_benchmarks;; +        maybe_silently run_benchmarks;; + +    # HELP   install +    # HELP           install benchmarks specified by --benchmarks flag +    install) +        maybe_silently run_install;;      *)          print_help -- 2.7.4