[Tarantool-patches] [PATCH 2/2] install: local benchmarks installation scripts

[Oleg Koshovetc <okoshovetc@tarantool.org>]

All of the following can be accessed on github

https://github.com/tarantool/bench-run/pull/14

The 'install' command was added to ttbench. It clones benchmarks and
installs them localy so the developers do not have to do it themselves.
Also added the -s flag that suppresses all the output on any command
---
  benches/cbench/install.sh     |   8 ++
  benches/common.sh             |  18 +++-
  benches/linkbench/install.sh  |   8 ++
  benches/linkbench/run.sh      |   6 +-
  benches/nosqlbench/install.sh |   8 ++
  benches/sysbench/install.sh   |   8 ++
  benches/tpcc/install.sh       |   8 ++
  benches/tpch/install.sh       |   5 +
  benches/ycsb/install.sh       |   6 ++
  config.sh                     |  38 ++++---
  ttbench                       | 232 
++++++++++++++++++++++++++++++++++--------
  11 files changed, 290 insertions(+), 55 deletions(-)
  create mode 100755 benches/cbench/install.sh
  create mode 100755 benches/linkbench/install.sh
  create mode 100755 benches/nosqlbench/install.sh
  create mode 100755 benches/sysbench/install.sh
  create mode 100755 benches/tpcc/install.sh
  create mode 100755 benches/tpch/install.sh
  create mode 100755 benches/ycsb/install.sh

diff --git a/benches/cbench/install.sh b/benches/cbench/install.sh
new file mode 100755
index 0000000..39046a0
--- /dev/null
+++ b/benches/cbench/install.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+git clone https://github.com/tarantool/cbench.git "$PWD"
+
+cmake . -DCMAKE_BUILD_TYPE=RelWithDebInfo
+make -j
diff --git a/benches/common.sh b/benches/common.sh
index 1116cc9..abc99d7 100644
--- a/benches/common.sh
+++ b/benches/common.sh
@@ -65,7 +65,9 @@ function kill_tarantool {
  }

  function error {
-    echo "ERROR:" "$@"
+    local _caller
+    _caller=( $(caller 0) )
+    echo "${_caller[2]} line=${_caller[0]} fn=${_caller[1]} ERROR:" "$@"
      exit 100
  }

@@ -97,7 +99,7 @@ function wait_for_tarantool_runnning {
      local tt=0

      while [ "$tt" -lt "$t" ]; do
-        if echo 'if type(box.cfg) ~= "function" then return 
box.info().status end' | tarantoolctl connect "$creds" 2>/dev/null | 
grep -q 'running'; then
+        if echo 'if type(box.cfg) ~= "function" then return 
box.info().status end' | "$TARANTOOLCTL_EXECUTABLE" connect "$creds" 
2>/dev/null | grep -q 'running'; then
              return 0
          fi

@@ -107,3 +109,15 @@ function wait_for_tarantool_runnning {

      return 1
  }
+
+function is_directory_empty {
+    local d="$1"
+
+    [ ! -d "$d" ] && error "no such directory '$d'"
+
+    if find "$d/" -maxdepth 1 -mindepth 1 | grep -q .; then
+        return 1
+    else
+        return 0
+    fi
+}
diff --git a/benches/linkbench/install.sh b/benches/linkbench/install.sh
new file mode 100755
index 0000000..2aa5386
--- /dev/null
+++ b/benches/linkbench/install.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+git clone https://github.com/tarantool/linkbench.git -b update-fixes "$PWD"
+
+luarocks install 
https://raw.githubusercontent.com/tarantool/gperftools/master/rockspecs/gperftools-scm-1.rockspec 
\
+    --tree .rocks --lua-version 5.1
diff --git a/benches/linkbench/run.sh b/benches/linkbench/run.sh
index c5f493e..739c46e 100755
--- a/benches/linkbench/run.sh
+++ b/benches/linkbench/run.sh
@@ -18,7 +18,11 @@ make -C src/tarantool -B
  tfile=src/tarantool/app.lua

  stop_and_clean_tarantool
-maybe_under_numactl "${numaopts[@]}" -- tarantool "$tfile" 1>/dev/null 
2>/dev/null &
+
+lua_path_prefix="$PWD/.rocks/share/lua/5.1"
+
+export LUA_PATH="$lua_path_prefix/?.lua;$lua_path_prefix/?/init.lua"
+maybe_under_numactl "${numaopts[@]}" -- tarantool "$tfile" 
1>ttserver.log 2>&1 &

  cfgfile=config/LinkConfigTarantool.properties
  sed "s/^maxid1 = .*/maxid1 = $LINKBENCH_WORKLOAD_SIZE/g" -i 
config/FBWorkload.properties
diff --git a/benches/nosqlbench/install.sh b/benches/nosqlbench/install.sh
new file mode 100755
index 0000000..5659613
--- /dev/null
+++ b/benches/nosqlbench/install.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+git clone --recursive https://github.com/tarantool/nosqlbench.git "$PWD"
+git submodule foreach --recursive
+cmake .
+make -j
diff --git a/benches/sysbench/install.sh b/benches/sysbench/install.sh
new file mode 100755
index 0000000..9678254
--- /dev/null
+++ b/benches/sysbench/install.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+git clone https://github.com/tarantool/sysbench.git "$PWD"
+./autogen.sh
+./configure --with-tarantool --without-mysql
+make -j
diff --git a/benches/tpcc/install.sh b/benches/tpcc/install.sh
new file mode 100755
index 0000000..d6e827e
--- /dev/null
+++ b/benches/tpcc/install.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+git clone https://github.com/tarantool/tpcc.git "$PWD"
+cd src
+make -j
+cd ..
diff --git a/benches/tpch/install.sh b/benches/tpch/install.sh
new file mode 100755
index 0000000..2ee1e14
--- /dev/null
+++ b/benches/tpch/install.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+git clone https://github.com/tarantool/tpch.git "$PWD"
diff --git a/benches/ycsb/install.sh b/benches/ycsb/install.sh
new file mode 100755
index 0000000..5c536ea
--- /dev/null
+++ b/benches/ycsb/install.sh
@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+git clone https://github.com/tarantool/YCSB.git "$PWD"
+mvn clean package
diff --git a/config.sh b/config.sh
index 93d3fc5..bd6f558 100644
--- a/config.sh
+++ b/config.sh
@@ -1,41 +1,55 @@
  # path to tarantool executable
  export TARANTOOL_EXECUTABLE=tarantool

+# path to tarantoolctl executable
+export TARANTOOLCTL_EXECUTABLE=tarantoolctl
+
  # location of tests to be run
+# all the tests will be copied and run in this directory
  export BENCH_WORKDIR="$PWD/.benchdir"

-# cbench parameters
  export CBENCH_DIR="$HOME/work/cbench/"
+export LINKBENCH_DIR="$HOME/work/linkbench/"
+export NOSQLBENCH_DIR="$HOME/work/nosqlbench/"
+export SYSBENCH_DIR="$HOME/work/sysbench/"
+export TPCC_DIR="$HOME/work/tpcc/"
+export TPCH_DIR="$HOME/work/tpch/"
+export YCSB_DIR="$HOME/work/ycsb/"
+
+# ----------------- BENCHMARK PARAMS -----------------
+# Fill free to delete all the params as long as all
+# the benchmarks have their own defaults set in run.sh
+
+# cbench parameters
  export CBENCH_VINYL_WORKLOAD=50
  export CBENCH_MEMTX_WORKLOAD=10000

  # linkbench parameters
-export LINKBENCH_DIR="$HOME/work/linkbench/"
  export LINKBENCH_REQUESTERS=1
  export LINKBENCH_REQUESTS=1000
  export LINKBENCH_WORKLOAD_SIZE=1000

-# linkbench parameters
-export NOSQLBENCH_DIR="$HOME/work/nosqlbench/"
+# nosqlbench parameters
+export NOSQLBENCH_TIMELIMIT=20000
+export NOSQLBENCH_BATCHCOUNT=10
+export NOSQLBENCH_RPS=20000

-# linkbench parameters
-export SYSBENCH_DIR="$HOME/work/sysbench/"
+# sysbench parameters
  export SYSBENCH_RUNS=1
  export SYSBENCH_TIME=5

  # tpcc parameters
-export TPCC_DIR="$HOME/work/tpcc/"
  export TPCC_TIME=20
  export TPCC_WARMUPTIME=5
  export TPCC_WAREHOUSES=15
-export TPCC_FROMSNAPSHOT="/tmp/00000000000000033515.snap"
+## possible to start tpcc benchmark from snapshot wich I higly recommend
+## as tpcc_load takes a good portion of time to load the data
+# export TPCC_FROMSNAPSHOT="/tmp/00000000000000033515.snap"

-# tpcc parameters
-export TPCH_DIR="$HOME/work/tpch/"
+# tpch parameters
  export TPCH_SKIP_SQLITE=1

-# tpcc parameters
-export YCSB_DIR="$HOME/work/ycsb/"
+# ycsb parameters
  export YCSB_OPERATIONCOUNT=1000
  export YCSB_RECORDCOUNT=1000
  export YCSB_RUNS=1
diff --git a/ttbench b/ttbench
index ac656f5..b856820 100755
--- a/ttbench
+++ b/ttbench
@@ -4,72 +4,174 @@ set -euo pipefail

  TTBENCH_CONFIG=config.sh
  TTBENCH_BENCHES=all
+TTBENCH_SILENT=
+TTBENCH_FORCE=
  TTBENCH_COMMAND=
  TTBENCH_BENCHDIR=benches

-function error {
-    echo "ERROR:" "$@"
-    exit 100
-}
+TTBENCH_ENV_OVERRIDES=()
+
+source "$TTBENCH_BENCHDIR/common.sh"

  function print_help {
      grep '^\s*# HELP' "$0" | sed 's/^\s*# HELP\s//g'
  }

-function list_benchmarks {
+function maybe_silently {
+    if [ -z "$TTBENCH_SILENT" ]; then
+        "$@"
+    else
+        "$@" 1>/dev/null 2>/dev/null
+    fi
+}
+
+function list_all_benchmarks {
      for d in $TTBENCH_BENCHDIR/*; do
          sed "s#$TTBENCH_BENCHDIR/##" <<< "$d"
      done | grep -v common.sh
  }

+function list_given_benchmarks {
+    if [ "$TTBENCH_BENCHES" == all ]; then
+        for b in $(list_all_benchmarks); do
+            echo "$b"
+        done
+    else
+        IFS=, read -ra benchlist <<< "$TTBENCH_BENCHES"
+        for b in "${benchlist[@]}"; do
+            if [ ! -d "$TTBENCH_BENCHDIR/$b" ]; then
+                error "Unknown benchmark='$b', please run '$0 
list-benchmarks'"
+            fi
+        done
+
+        for b in "${benchlist[@]}"; do
+            echo "$b"
+        done
+    fi
+}
+
+function get_bench_dir {
+    local bench="$1"
+    if benchdir=$(env | grep "${bench^^}_DIR" | sed 's/^.*=//'); then
+        echo "$benchdir"
+    else
+        error "variable '${bench^^}_DIR' is not set in config"
+    fi
+}
+
  function run_bench {
      local bench="$1"
      local benchdir=
      local workdir="${BENCH_WORKDIR:?}/$bench/"

+    mkdir -p "$BENCH_WORKDIR"
      cp "$TTBENCH_BENCHDIR/common.sh" "${BENCH_WORKDIR}"

      export COMMON_FUNCTIONS="$BENCH_WORKDIR/common.sh"

-    if benchdir=$(env | grep "${bench^^}_DIR" | sed 's/^.*=//'); then
-        echo "Running $bench with workdir $benchdir" 1>&2
-        rm -rf "$workdir"
-        mkdir -p "$workdir"
-
-        cp -r "$TTBENCH_BENCHDIR/$bench"/* "$workdir"
-        cp -r "$benchdir"/* "$workdir"
-
-        local runner=
-        runner=$(readlink -f "$TTBENCH_BENCHDIR/$bench/run.sh")
-        pushd "$workdir" 1>/dev/null
-        "$runner"
-        popd 1>/dev/null
-    else
-        error "variable '${bench^^}_DIR' is not set in config, can't 
run benchmark"
-    fi
+    local benchdir=
+    benchdir=$(get_bench_dir "$bench")
+
+    echo "Running $bench with workdir $benchdir" 1>&2
+    rm -rf "$workdir"
+    mkdir -p "$workdir"
+
+    cp -r "$TTBENCH_BENCHDIR/$bench"/* "$workdir"
+    cp -r "$benchdir"/* "$workdir"
+
+    # some of benchmarks have external dependencies that are stored in 
.rocks directory
+    # but some of benchmark do not have them
+    cp -r "$benchdir"/.rocks "$workdir" 2>/dev/null || true
+
+    local runner=
+    runner=$(readlink -f "$TTBENCH_BENCHDIR/$bench/run.sh")
+    pushd "$workdir" 1>/dev/null
+    "$runner"
+    popd 1>/dev/null
  }

  function run_benchmarks {
      # must be set in config
-    [ -z "$BENCH_WORKDIR" ] && error "BENCH_WORKDIR is not set"
+    [ -z "$BENCH_WORKDIR" ]           && error "BENCH_WORKDIR is not set"
+    [ -z "$TARANTOOL_EXECUTABLE" ]    && error "TARANTOOL_EXECUTABLE is 
not set"
+    [ -z "$TARANTOOLCTL_EXECUTABLE" ] && error "TARANTOOLCTL_EXECUTABLE 
is not set"

-    if [ "$TTBENCH_BENCHES" == all ]; then
-        for b in $(list_benchmarks); do
-            run_bench "$b"
-        done
-    else
-        IFS=, read -ra benchlist <<< "$TTBENCH_BENCHES"
-        # local benchlist=( $(sed 's/,/ /g' <<< "$TTBENCH_BENCHES") );
-        for b in "${benchlist[@]}"; do
-            if [ ! -d "$TTBENCH_BENCHDIR/$b" ]; then
-                error "Unknown benchmark='$b', please run '$0 
list-benchmarks'"
-            fi
-        done
+    for b in $(list_given_benchmarks); do
+        run_bench "$b"
+    done
+}

-        for b in "${benchlist[@]}"; do
-            run_bench "$b"
-        done
+function install_benchmark {
+    local bench="$1"
+
+    local benchdir=
+    benchdir=$(get_bench_dir "$bench")
+
+    mkdir -p "$benchdir"
+
+    if ! is_directory_empty "$benchdir"; then
+        if [ -z "$TTBENCH_FORCE" ]; then
+            error "'$benchdir' is not empty, use --force flag to reinstall"
+        fi
+    fi
+
+    local bdir=
+    bdir=$(realpath "$benchdir")
+
+    local install_path_ok=
+
+    if [[ "$bdir" == /home/* ]]; then
+        install_path_ok=1
+    fi
+
+    if [[ "$bdir" == /opt/* ]]; then
+        install_path_ok=1
+    fi
+
+    if [[ "$bdir" == /tmp/* ]]; then
+        install_path_ok=1
+    fi
+
+    if [ -z "$install_path_ok" ]; then
+        echo "ERROR: benchmark install directory='$bdir' is"
+        echo "ERROR: neither under /home/"
+        echo "ERROR: nor under /opt/"
+        echo "ERROR: nor under /tmp/"
+        echo "ERROR: any other installation path is considered to be 
unsafe"
+        echo "ERROR: as it is required to run rm -rf on benchmark 
insall directory"
+        echo "ERROR: if you think this is an error, consider patching 
$0 yourself"
+        echo "ERROR: or installing the benchmarks manually"
+        error "unsafe install directory='$bdir' detected"
      fi
+
+    rm -rf "${bdir:?}"
+    mkdir -p "$bdir"
+
+    local install_script="$TTBENCH_BENCHDIR/$bench/install.sh"
+    [ ! -f "$install_script" ] && error
+    install_script=$(readlink -f "$TTBENCH_BENCHDIR/$bench/install.sh")
+
+    pushd "$bdir" 1>/dev/null
+    "$install_script"
+    popd 1>/dev/null
+}
+
+function run_install {
+    local benchlist
+    benchlist=( $(list_given_benchmarks) )
+
+    # running this in single loop for a side effect where get_bench_dir
+    # will raise an error if some env variables are not set
+    for b in "${benchlist[@]}"; do
+        get_bench_dir "$b"
+        if [ ! -f "$TTBENCH_BENCHDIR/$b/install.sh" ]; then
+            error "$TTBENCH_BENCHDIR/$b is missing install.sh script"
+        fi
+    done
+
+    for b in "${benchlist[@]}"; do
+        install_benchmark "$b"
+    done
  }

  # HELP ttbench - launch tarantool external benchmarks
@@ -96,6 +198,20 @@ while [[ $# -gt 0 ]]; do
              set -x
              shift;;

+        # HELP   -f, --force
+        # HELP           if used with install command, will reinstall 
benchmarks
+        # HELP
+        -f|--force)
+            TTBENCH_FORCE=1
+            shift;;
+
+        # HELP   -s, --silent
+        # HELP           suppress any output, just give an exit code
+        # HELP
+        -s|--silent)
+            TTBENCH_SILENT=1
+            shift;;
+
          # HELP   -b, --benchmarks 'bench1,bench2'
          # HELP           comma separated list of benchmarks (defauts 
to 'all')
          # HELP
@@ -104,6 +220,25 @@ while [[ $# -gt 0 ]]; do
              [ -z "$TTBENCH_BENCHES" ] && error "$option param can not 
be empty"
              shift 2;;

+        # HELP   -D, --define KEY=VALUE
+        # HELP   -DKEY=VALUE
+        # HELP           allows to define extra ENV variables
+        # HELP           or rewrite ones in config file from command line
+        # HELP
+        -D|--define|-D*)
+            case "$option" in
+                -D|--define)
+                    override="${2:-}"
+                    [ -z "$override" ] && error "$option param can not 
be empty"
+                    TTBENCH_ENV_OVERRIDES+=( "$override" )
+                    shift 2;;
+                -D*)
+                    override="${option#-D}"
+                    [ -z "$override" ] && error "$option param can not 
be empty"
+                    TTBENCH_ENV_OVERRIDES+=( "$override" )
+                    shift;;
+            esac;;
+
          # HELP   -h, --help
          # HELP           print help message
          # HELP
@@ -116,7 +251,18 @@ while [[ $# -gt 0 ]]; do
      esac
  done

-[ -f "$TTBENCH_CONFIG" ] && source "$TTBENCH_CONFIG"
+if [ -f "$TTBENCH_CONFIG" ]; then
+    source "$TTBENCH_CONFIG"
+else
+    error "config file='$TTBENCH_CONFIG' is absent"
+fi
+
+# Disgusting hack for empty arrays on bash < 4.4
+if [ 0 -ne "${#TTBENCH_ENV_OVERRIDES[@]}" ]; then
+    for override in "${TTBENCH_ENV_OVERRIDES[@]}"; do
+        export "$override"
+    done
+fi

  TTBENCH_COMMAND="${1:-}"

@@ -135,12 +281,18 @@ case "$TTBENCH_COMMAND" in
      # HELP           lists all known benchmarks
      # HELP
      ls|list-benchmarks)
-        list_benchmarks;;
+        maybe_silently list_all_benchmarks;;

      # HELP   run
      # HELP           run benchmarks specified by --benchmarks flag
+    # HELP
      run)
-        run_benchmarks;;
+        maybe_silently run_benchmarks;;
+
+    # HELP   install
+    # HELP           install benchmarks specified by --benchmarks flag
+    install)
+        maybe_silently run_install;;

      *)
          print_help
-- 
2.7.4