[Tarantool-patches] [PATCH] benchmarks: runner for local benchmarks runs

[Oleg Koshovetc <okoshovetc@tarantool.org>]

https://github.com/tarantool/bench-run/compare/ttbench?expand=1

This runner allows to run various benchmarks on local machines (if all
the source code was already present and prebuilt on a machine). However
analyzing the results of benchmarks requires manual reading through
their logs

TODO: implement automatic local installation of benchmarks
TODO: implement tool to save and diff benchmarks' results
---
  benches/cbench/run.sh           |  35 ++++++++++
  benches/common.sh               | 109 +++++++++++++++++++++++++++++
  benches/linkbench/app.lua       |  17 +++++
  benches/linkbench/run.sh        |  46 +++++++++++++
  benches/nosqlbench/run.sh       |  57 +++++++++++++++
  benches/nosqlbench/tnt_hash.lua |  30 ++++++++
  benches/nosqlbench/tnt_tree.lua |  31 +++++++++
  benches/publication/publish.py  |  64 +++++++++++++++++
  benches/sysbench/run.sh         |  92 +++++++++++++++++++++++++
  benches/sysbench/tnt_srv.lua    |  28 ++++++++
  benches/tpcc/init_empty.lua     |   2 +
  benches/tpcc/init_not_empty.lua |   1 +
  benches/tpcc/run.sh             |  49 +++++++++++++
  benches/tpcc/schema.lua         | 144 
++++++++++++++++++++++++++++++++++++++
  benches/tpcc/server.lua         |   2 +
  benches/tpch/run.sh             |  51 ++++++++++++++
  benches/ycsb/run.sh             |  86 +++++++++++++++++++++++
  config.sh                       |  41 +++++++++++
  ttbench                         | 149 
++++++++++++++++++++++++++++++++++++++++
  19 files changed, 1034 insertions(+)
  create mode 100755 benches/cbench/run.sh
  create mode 100644 benches/common.sh
  create mode 100755 benches/linkbench/app.lua
  create mode 100755 benches/linkbench/run.sh
  create mode 100755 benches/nosqlbench/run.sh
  create mode 100644 benches/nosqlbench/tnt_hash.lua
  create mode 100644 benches/nosqlbench/tnt_tree.lua
  create mode 100755 benches/publication/publish.py
  create mode 100755 benches/sysbench/run.sh
  create mode 100644 benches/sysbench/tnt_srv.lua
  create mode 100644 benches/tpcc/init_empty.lua
  create mode 100644 benches/tpcc/init_not_empty.lua
  create mode 100755 benches/tpcc/run.sh
  create mode 100644 benches/tpcc/schema.lua
  create mode 100644 benches/tpcc/server.lua
  create mode 100755 benches/tpch/run.sh
  create mode 100755 benches/ycsb/run.sh
  create mode 100644 config.sh
  create mode 100755 ttbench

diff --git a/benches/cbench/run.sh b/benches/cbench/run.sh
new file mode 100755
index 0000000..09c4dee
--- /dev/null
+++ b/benches/cbench/run.sh
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+set -eu
+set -o pipefail
+
+source ../common.sh
+
+CBENCH_VINYL_WORKLOAD="${CBENCH_VINYL_WORKLOAD:-500}"
+CBENCH_MEMTX_WORKLOAD="${CBENCH_MEMTX_WORKLOAD:-1000000}"
+
+export LUA_CPATH="$PWD/?.so"
+export LUA_PATH="$PWD/?/init.lua"
+
+TAR_VER=$(get_tarantool_version)
+numaopts=(--membind=1 --cpunodebind=1 --physcpubind=11)
+
+stop_and_clean_tarantool
+maybe_drop_cache
+
+maybe_under_numactl "${numaopts[@]}" -- \
+    "$TARANTOOL_EXECUTABLE" cbench_runner.lua memtx 
"$CBENCH_MEMTX_WORKLOAD" 2>&1 | tee cbench_output_memtx.txt
+
+stop_and_clean_tarantool
+maybe_drop_cache
+
+maybe_under_numactl "${numaopts[@]}" -- \
+    "$TARANTOOL_EXECUTABLE" cbench_runner.lua vinyl fsync 
"$CBENCH_VINYL_WORKLOAD" 2>&1 | tee cbench_output_vinyl_fsync.txt
+
+stop_and_clean_tarantool
+maybe_drop_cache
+
+maybe_under_numactl "${numaopts[@]}" -- \
+    "$TARANTOOL_EXECUTABLE" cbench_runner.lua vinyl write 
"$CBENCH_VINYL_WORKLOAD" 2>&1 | tee cbench_output_vinyl_write.txt
+
+echo "$TAR_VER"
diff --git a/benches/common.sh b/benches/common.sh
new file mode 100644
index 0000000..1116cc9
--- /dev/null
+++ b/benches/common.sh
@@ -0,0 +1,109 @@
+# will not work if not under root
+# if benches are run localy by a developer droping caches might not be 
too important
+function maybe_drop_cache {
+    if test -w /proc/sys/vm/drop_caches; then
+        echo 3 > /proc/sys/vm/drop_caches
+    fi
+}
+
+function can_be_run_under_numa {
+    numactl "$@" true 1>/dev/null 2>/dev/null
+}
+
+# not every local machine has numa enabled on it
+# not every local machine has enough hardware units to run with 
intended configuration
+#
+# usage:
+# maybe_under_numactl numaoptions -- command to run
+# example:
+# maybe_under_numactl --membind=1 --cpunodebind=1 --physcpubind=11 -- 
echo qwe
+function maybe_under_numactl {
+    local numaoptions=()
+    local cmdoptions=()
+    local parsing_numa=1
+    for option in "$@"; do
+        if [ -n "$parsing_numa" ]; then
+            if [ "$option" == -- ]; then
+                parsing_numa=
+            else
+                numaoptions+=( "$option" )
+            fi
+        else
+            cmdoptions+=( "$option" )
+        fi
+    done
+
+    if which numactl 1>/dev/null 2>/dev/null; then
+        # check if it is even runable with given numactl options
+        if can_be_run_under_numa "${numaoptions[@]}"; then
+            numactl "${numaoptions[@]}" "${cmdoptions[@]}"
+        else
+            "${cmdoptions[@]}"
+        fi
+    else
+        "${cmdoptions[@]}"
+    fi
+}
+
+function get_tarantool_version {
+    "$TARANTOOL_EXECUTABLE" -v | grep -e "Tarantool" |  grep -oP '\s\K\S*'
+}
+
+function clean_tarantool {
+    rm -rf ./*.snap
+    rm -rf ./*.xlog
+    rm -rf ./*.vylog
+    rm -rf 5*
+}
+
+function sync_disk {
+    sync && echo "sync passed" || echo "sync failed with error: $?"
+}
+
+function kill_tarantool {
+    killall tarantool 2>/dev/null || true
+}
+
+function error {
+    echo "ERROR:" "$@"
+    exit 100
+}
+
+function stop_and_clean_tarantool {
+    kill_tarantool
+    clean_tarantool
+    sync_disk
+}
+
+function wait_for_file_release {
+    local f="$1"
+    local t="$2"
+    local tt=0
+    while [ "$tt" -lt "$t" ]; do
+        if ! lsof "$f" 1>/dev/null 2>/dev/null; then
+            return 0
+        fi
+
+        tt=$(( tt + 1 ))
+        sleep 1
+    done
+
+    return 1
+}
+
+function wait_for_tarantool_runnning {
+    local creds="$1"
+    local t="$2"
+    local tt=0
+
+    while [ "$tt" -lt "$t" ]; do
+        if echo 'if type(box.cfg) ~= "function" then return 
box.info().status end' | tarantoolctl connect "$creds" 2>/dev/null | 
grep -q 'running'; then
+            return 0
+        fi
+
+        tt=$(( tt + 1 ))
+        sleep 1
+    done
+
+    return 1
+}
diff --git a/benches/linkbench/app.lua b/benches/linkbench/app.lua
new file mode 100755
index 0000000..79252e0
--- /dev/null
+++ b/benches/linkbench/app.lua
@@ -0,0 +1,17 @@
+#!/usr/bin/env tarantool
+
+local path = require('fio').dirname(arg[0])
+package.path = path.."/?.lua;"..package.path
+package.cpath = path.."/?.so;"..package.cpath
+
+require('console').listen('unix/:./tarantool.sock')
+require('gperftools').cpu.start('tarantool.prof')
+
+box.cfg{
+    listen = 3301;
+    vinyl_memory = 256 * 1024 * 1024;
+    vinyl_cache =  256 * 1024 * 1024;
+    vinyl_read_threads = 1;
+}
+
+require('app_internal')
diff --git a/benches/linkbench/run.sh b/benches/linkbench/run.sh
new file mode 100755
index 0000000..c5f493e
--- /dev/null
+++ b/benches/linkbench/run.sh
@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+
+set -eu
+set -o pipefail
+
+source ../common.sh
+
+LINKBENCH_REQUESTS="${LINKBENCH_REQUESTS:-2000000}"
+LINKBENCH_REQUESTERS="${LINKBENCH_REQUESTERS:-1}"
+LINKBENCH_WORKLOAD_SIZE="${LINKBENCH_WORKLOAD_SIZE:-5000000}"
+
+TAR_VER=$(get_tarantool_version)
+numaopts=(--membind=1 --cpunodebind=1 '--physcpubind=6,7,8,9,10,11')
+
+mvn clean package -Dmaven.test.skip=true
+make -C src/tarantool -B
+
+tfile=src/tarantool/app.lua
+
+stop_and_clean_tarantool
+maybe_under_numactl "${numaopts[@]}" -- tarantool "$tfile" 1>/dev/null 
2>/dev/null &
+
+cfgfile=config/LinkConfigTarantool.properties
+sed "s/^maxid1 = .*/maxid1 = $LINKBENCH_WORKLOAD_SIZE/g" -i 
config/FBWorkload.properties
+sed "s/^requesters = .*/requesters = $LINKBENCH_REQUESTERS/g" -i "$cfgfile"
+sed "s/^requests = .*/requests = $LINKBENCH_REQUESTS/g" -i "$cfgfile"
+
+maybe_under_numactl "${numaopts[@]}" -- \
+    ./bin/linkbench -c "$cfgfile" -l 2>&1 | tee loading.res.txt
+
+sync_disk
+maybe_drop_cache
+
+maybe_under_numactl "${numaopts[@]}" -- \
+    ./bin/linkbench -c "$cfgfile" -r 2>&1 | tee linkbench_output.txt
+
+kill_tarantool
+
+grep "REQUEST PHASE COMPLETED" linkbench_output.txt | sed "s/.*second = 
/linkbench:/" | tee -a linkbench.ssd_result.txt
+echo "${TAR_VER}" | tee linkbench.ssd_t_version.txt
+
+echo "Tarantool TAG:"
+cat linkbench.ssd_t_version.txt
+echo "Overall results:"
+echo "================"
+cat linkbench.ssd_result.txt
diff --git a/benches/nosqlbench/run.sh b/benches/nosqlbench/run.sh
new file mode 100755
index 0000000..b092cfe
--- /dev/null
+++ b/benches/nosqlbench/run.sh
@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+
+set -eu
+set -o pipefail
+
+source ../common.sh
+
+# type=$1
+# if [ "$type" == "" ]; then
+#     type=hash
+# fi
+
+NOSQLBENCH_WORKLOAD="${NOSQLBENCH_WORKLOAD:-}"
+NOSQLBENCH_TIMELIMIT="${NOSQLBENCH_TIMELIMIT:-20000}"
+NOSQLBENCH_BATCHCOUNT="${NOSQLBENCH_BATCHCOUNT:-10}"
+NOSQLBENCH_RPS="${NOSQLBENCH_RPS:-20000}"
+
+TAR_VER=$(get_tarantool_version)
+numaopts=(--membind=1 --cpunodebind=1 '--physcpubind=6,7,8,9,10,11')
+
+function run_nosqlbench {
+    local type="$1"
+    stop_and_clean_tarantool
+
+    maybe_under_numactl "${numaopts[@]}" -- \
+        "$TARANTOOL_EXECUTABLE" "tnt_${type}.lua" 2>&1 &
+
+    config=nosqlbench.conf
+    cp src/nosqlbench.conf "$config"
+
+    sed  "s/port 3303/port 3301/" "$config" -i
+    sed  "s/benchmark 'no_limit'/benchmark 'time_limit'/" "$config" -i
+    sed  "s/time_limit 10/time_limit $NOSQLBENCH_TIMELIMIT/" "$config" -i
+    sed  "s/request_batch_count 1/request_batch_count 
$NOSQLBENCH_BATCHCOUNT/" "$config" -i
+    sed  "s/rps 12000/rps $NOSQLBENCH_RPS/" "$config" -i
+
+    sleep 5
+    echo "Run NB type='$type'"
+
+    # WARNING: don't try to save output from stderr - file will use the 
whole disk space !
+    (maybe_under_numactl "${numaopts[@]}" -- \
+        ./src/nb "$config") \
+        | grep -v "Warmup" \
+        | grep -v "Failed to allocate" >nosqlbench_output.txt \
+        || cat nosqlbench_output.txt
+
+    grep "TOTAL RPS STATISTICS:" nosqlbench_output.txt -A6 | \
+        awk -F "|" 'NR > 4 {print $2,":", $4}' > 
"noSQLbench.${type}_result.txt"
+    echo "${TAR_VER}" | tee "noSQLbench.${type}_t_version.txt"
+}
+
+if [ -z "$NOSQLBENCH_WORKLOAD" -o "$NOSQLBENCH_WORKLOAD" == all ]; then
+    run_nosqlbench tree
+    run_nosqlbench hash
+else
+    run_nosqlbench "$NOSQLBENCH_WORKLOAD"
+fi
diff --git a/benches/nosqlbench/tnt_hash.lua 
b/benches/nosqlbench/tnt_hash.lua
new file mode 100644
index 0000000..a616490
--- /dev/null
+++ b/benches/nosqlbench/tnt_hash.lua
@@ -0,0 +1,30 @@
+local console = require('console')
+console.listen("/tmp/tarantool-server.sock")
+
+box.cfg {
+    pid_file   = "./tarantool-server.pid",
+    log        = "./tarantool-server.log",
+    listen = 3301,
+    memtx_memory = 2000000000,
+    background = true,
+    checkpoint_interval = 0,
+    wal_mode = 'none',
+}
+
+s = box.schema.space.create('tester_nosqlbench')
+s:create_index('primary', {type = 'hash', parts = {1, 'unsigned'}})
+
+function try(f, catch_f)
+    local status, exception = pcall(f)
+    if not status then
+        catch_f(exception)
+    end
+end
+
+try(function()
+    box.schema.user.grant('guest', 'create,read,write,execute', 'universe')
+end, function(e)
+    print(e)
+end)
+
+
diff --git a/benches/nosqlbench/tnt_tree.lua 
b/benches/nosqlbench/tnt_tree.lua
new file mode 100644
index 0000000..57b646b
--- /dev/null
+++ b/benches/nosqlbench/tnt_tree.lua
@@ -0,0 +1,31 @@
+local console = require('console')
+console.listen("/tmp/tarantool-server.sock")
+
+--memtx_memory = 2000000000,
+box.cfg {
+    pid_file   = "./tarantool-server.pid",
+    log        = "./tarantool-server.log",
+    listen = 3301,
+    vinyl_memory = 107374182,
+    background = true,
+    checkpoint_interval = 0,
+    wal_mode = 'none',
+}
+
+s = box.schema.space.create('tester_nosqlbench')
+s:create_index('primary', {type = 'tree', parts = {1, 'unsigned'}})
+
+function try(f, catch_f)
+    local status, exception = pcall(f)
+    if not status then
+        catch_f(exception)
+    end
+end
+
+try(function()
+    box.schema.user.grant('guest', 'create,read,write,execute', 'universe')
+end, function(e)
+    print(e)
+end)
+
+
diff --git a/benches/publication/publish.py b/benches/publication/publish.py
new file mode 100755
index 0000000..42bc53a
--- /dev/null
+++ b/benches/publication/publish.py
@@ -0,0 +1,64 @@
+#!/usr/bin/env python
+import fnmatch
+import os
+from urllib import urlencode
+import requests
+
+
+def parse_bench(filename):
+    with open(filename) as raw_data:
+        return raw_data.readlines()
+
+
+def get_version(filename):
+    with open(filename) as raw_data:
+        version = raw_data.readlines()[-1]
+        return version.split()[0]
+
+
+def push_to_microb(server, token, name, value, version, tab):
+    uri = 'http://%s/push?%s' % (server, urlencode(dict(
+        key=token, name=name, param=value,
+        v=version, unit='trps', tab=tab
+    )))
+
+    r = requests.get(uri)
+    if r.status_code == 200:
+        print('Export complete')
+    else:
+        print('Export error http: %d' % r.status_code)
+        print('Export error text: %d' % r.text)
+
+
+def main():
+    if "MICROB_WEB_TOKEN" in os.environ and "MICROB_WEB_HOST" in 
os.environ:
+        bench = {}
+        res = []
+        current_data = {}
+        version = ''
+        for file in os.listdir('.'):
+            if fnmatch.fnmatch(file, '*_result.txt'):
+                values = parse_bench(file)
+                benchmark = file.split('_')[0]
+                version = get_version('{}_t_version.txt'.format(benchmark))
+                for value in values:
+                    test_name = value.split(':')[0]
+                    test_res = float(value.split(':')[1])
+                    res.append(test_res)
+                    push_to_microb(
+                        os.environ['MICROB_WEB_HOST'],
+                        os.environ['MICROB_WEB_TOKEN'],
+                        test_name,
+                        test_res,
+                        version,
+                        benchmark,
+                    )
+        print ("VERSION - ", version)
+    else:
+        print("MICROB params not specified")
+
+    return 0
+
+
+if __name__ == '__main__':
+    main()
diff --git a/benches/sysbench/run.sh b/benches/sysbench/run.sh
new file mode 100755
index 0000000..4b52385
--- /dev/null
+++ b/benches/sysbench/run.sh
@@ -0,0 +1,92 @@
+#!/usr/bin/env bash
+
+set -eu
+set -o pipefail
+
+source ../common.sh
+
+SYSBENCH_TESTS="${SYSBENCH_TESTS:-}"
+SYSBENCH_TIME="${SYSBENCH_TIME:-20}"
+SYSBENCH_WARMUPTIME="${SYSBENCH_WARMUPTIME:-5}"
+SYSBENCH_DBMS="${SYSBENCH_DBMS:-tarantool}"
+SYSBENCH_THREADS="${SYSBENCH_THREADS:-200}"
+SYSBENCH_RUNS="${SYSBENCH_RUNS:-10}"
+
+TAR_VER=$(get_tarantool_version)
+numaconf=(--membind=1 --cpunodebind=1 '--physcpubind=6,7,8,9,10,11')
+
+ARRAY_TESTS=(
+    "oltp_read_only"
+    "oltp_write_only"
+    "oltp_read_write"
+    "oltp_update_index"
+    "oltp_update_non_index"
+    "oltp_insert"
+    "oltp_delete"
+    "oltp_point_select"
+    "select_random_points"
+    "select_random_ranges"
+    # "bulk_insert"
+)
+
+opts=("--db-driver=${SYSBENCH_DBMS}" "--threads=${SYSBENCH_THREADS}")
+
+export LD_LIBRARY_PATH=/usr/local/lib
+
+if [ -n "$SYSBENCH_TESTS" -a "$SYSBENCH_TESTS" != all ]; then
+    IPS=, read -ra testlist <<< "$SYSBENCH_TESTS"
+    ARRAY_TESTS=$testlist
+fi
+
+for test in "${ARRAY_TESTS[@]}"; do
+    res=0
+    tlog=sysbench_${test}_results.txt
+    rm -f "$tlog"
+    maxres=0
+    for run in $(seq 1 "$SYSBENCH_RUNS"); do
+        echo "$run"
+        echo "------------ $test ------------ rerun: # $run ------------"
+
+        stop_and_clean_tarantool
+        wait_for_file_release /tmp/tarantool-server.sock 10
+        maybe_under_numactl "${numaconf[@]}" -- \
+            "$TARANTOOL_EXECUTABLE" tnt_srv.lua >tnt_server.txt 2>&1 \
+            || ( cat tnt_server.txt && false )
+
+        sleep 1
+
+        # ./run_tnt.sh >tnt_server.txt 2>&1 || ( cat tnt_server.txt && 
false )
+
+        ./src/sysbench "$test" "${opts[@]}" cleanup > sysbench_output.txt
+        ./src/sysbench "$test" "${opts[@]}" prepare >> sysbench_output.txt
+
+        maybe_under_numactl "${numaconf[@]}" -- \
+            ./src/sysbench "$test" "${opts[@]}" \
+                "--time=${SYSBENCH_TIME}" \
+                "--warmup-time=${SYSBENCH_WARMUPTIME}" \
+                run >> sysbench_output.txt
+
+        grep -e 'transactions:' sysbench_output.txt | grep -oP 
'\(\K\S*' | tee "$tlog"
+        tres=$(sed 's#^.*:##g' < "$tlog" | sed 's#\..*$##g')
+        echo ">>> $tres"
+        res=$(( res + tres ))
+        if [[ $tres -gt $maxres ]]; then maxres=$tres ; fi
+    done
+
+    res=$(( res / SYSBENCH_RUNS ))
+    echo "${test}: $res" >>Sysbench_result.txt
+
+    echo "Subtest '$test' results:"
+    echo "==============================="
+    echo "Average result: $res"
+    echo "Maximum result: $maxres"
+    printf "Deviations (AVG -> MAX): %.2f%%" "$(bc <<< "scale = 4; (1 - 
$res / $maxres) * 100")"
+done
+
+echo "${TAR_VER}" | tee Sysbench_t_version.txt
+
+echo "Tarantool TAG:"
+cat Sysbench_t_version.txt
+echo "Overall results:"
+echo "================"
+cat Sysbench_result.txt
diff --git a/benches/sysbench/tnt_srv.lua b/benches/sysbench/tnt_srv.lua
new file mode 100644
index 0000000..d84413d
--- /dev/null
+++ b/benches/sysbench/tnt_srv.lua
@@ -0,0 +1,28 @@
+local console = require('console')
+console.listen("/tmp/tarantool-server.sock")
+
+box.cfg {
+    pid_file   = "./tarantool-server.pid",
+    log        = "./tarantool-server.log",
+    listen = 3301,
+    memtx_memory = 2000000000,
+    background = true,
+    checkpoint_interval = 0,
+}
+
+
+function try(f, catch_f)
+    local status, exception = pcall(f)
+    if not status then
+        catch_f(exception)
+    end
+end
+
+try(function()
+    box.schema.user.grant('guest', 'create,read,write,execute', 'universe')
+end, function(e)
+    print(e)
+end)
+
+
+
diff --git a/benches/tpcc/init_empty.lua b/benches/tpcc/init_empty.lua
new file mode 100644
index 0000000..e29ff04
--- /dev/null
+++ b/benches/tpcc/init_empty.lua
@@ -0,0 +1,2 @@
+require 'server'
+require 'schema'
diff --git a/benches/tpcc/init_not_empty.lua 
b/benches/tpcc/init_not_empty.lua
new file mode 100644
index 0000000..b36eeec
--- /dev/null
+++ b/benches/tpcc/init_not_empty.lua
@@ -0,0 +1 @@
+require 'server'
diff --git a/benches/tpcc/run.sh b/benches/tpcc/run.sh
new file mode 100755
index 0000000..e391d5f
--- /dev/null
+++ b/benches/tpcc/run.sh
@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+set -eu
+set -o pipefail
+
+source ../common.sh
+
+TAR_VER=$(get_tarantool_version)
+numaopts=(--membind=1 --cpunodebind=1 '--physcpubind=6,7,8,9,10,11')
+
+TPCC_TIME="${TPCC_TIME:-1200}"
+TPCC_WARMUPTIME="${TPCC_WARMUPTIME:-10}"
+TPCC_WAREHOUSES="${TPCC_WAREHOUSES:-15}"
+TPCC_FROMSNAPSHOT="${TPCC_FROMSNAPSHOT:-}"
+
+killall tpcc_load 2>/dev/null || true
+stop_and_clean_tarantool
+
+tpcc_opts=(-h localhost -P 3301 -d tarantool -u root -p '' -w 
"$TPCC_WAREHOUSES")
+
+if [ -z "$TPCC_FROMSNAPSHOT" ]; then
+    maybe_under_numactl "${numaopts[@]}" \
+        -- "$TARANTOOL_EXECUTABLE" init_empty.lua &
+    wait_for_tarantool_runnning 3301 15
+
+    ./tpcc_load "${tpcc_opts[@]}"
+else
+    [ ! -f "$TPCC_FROMSNAPSHOT" ] && error "No such file: 
'$TPCC_FROMSNAPSHOT'"
+    cp "$TPCC_FROMSNAPSHOT" .
+
+    maybe_under_numactl "${numaopts[@]}" \
+        -- "$TARANTOOL_EXECUTABLE" init_not_empty.lua &
+    wait_for_tarantool_runnning 3301 60
+fi
+
+
+maybe_under_numactl "${numaopts[@]}" -- \
+    ./tpcc_start "${tpcc_opts[@]}" -r "$TPCC_WARMUPTIME" -l 
"$TPCC_TIME" -i "$TPCC_TIME" > tpcc_output.txt 2>/dev/null
+
+echo -n "tpcc:" | tee tpc.c_result.txt
+grep -e '<TpmC>' tpcc_output.txt | grep -oP '\K[0-9.]*' | tee -a 
tpc.c_result.txt
+cat tpcc_output.txt
+
+echo "${TAR_VER}" | tee tpc.c_t_version.txt
+
+echo "Tarantool TAG:"
+cat tpc.c_t_version.txt
+echo "Overall result:"
+echo "==============="
+cat tpc.c_result.txt
diff --git a/benches/tpcc/schema.lua b/benches/tpcc/schema.lua
new file mode 100644
index 0000000..dc9c0df
--- /dev/null
+++ b/benches/tpcc/schema.lua
@@ -0,0 +1,144 @@
+local sql_execute = box.sql and box.sql.execute or box.execute
+
+sql_execute("drop table if exists warehouse;")
+sql_execute("create table warehouse ( \
+w_id int not null, \
+w_name varchar(10), \
+w_street_1 varchar(20), \
+w_street_2 varchar(20), \
+w_city varchar(20), \
+w_state varchar(2), \
+w_zip varchar(9), \
+w_tax number, \
+w_ytd number, \
+primary key (w_id) )")
+
+sql_execute("drop table if exists district;")
+sql_execute("create table district ( \
+d_id int not null, \
+d_w_id int not null, \
+d_name varchar(10), \
+d_street_1 varchar(20), \
+d_street_2 varchar(20), \
+d_city varchar(20), \
+d_state varchar(2), \
+d_zip varchar(9), \
+d_tax number, \
+d_ytd number, \
+d_next_o_id int, \
+primary key (d_w_id, d_id), \
+FOREIGN KEY(d_w_id) REFERENCES warehouse(w_id) );")
+
+sql_execute("drop table if exists customer;")
+sql_execute("create table customer ( \
+c_id int not null, \
+c_d_id int not null, \
+c_w_id int not null, \
+c_first varchar(16), \
+c_middle varchar(2), \
+c_last varchar(16), \
+c_street_1 varchar(20), \
+c_street_2 varchar(20), \
+c_city varchar(20), \
+c_state varchar(2), \
+c_zip varchar(9), \
+c_phone varchar(16), \
+c_since varchar(100), \
+c_credit varchar(2), \
+c_credit_lim int, \
+c_discount number, \
+c_balance number, \
+c_ytd_payment number, \
+c_payment_cnt int, \
+c_delivery_cnt int, \
+c_data text, \
+PRIMARY KEY(c_w_id, c_d_id, c_id), \
+FOREIGN KEY(c_w_id,c_d_id) REFERENCES district(d_w_id,d_id) );")
+
+sql_execute("drop table if exists history;")
+sql_execute("create table history ( \
+_id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, \
+h_c_id int, \
+h_c_d_id int, \
+h_c_w_id int, \
+h_d_id int, \
+h_w_id int, \
+h_date varchar(100), \
+h_amount number, \
+h_data varchar(24), \
+FOREIGN KEY(h_c_w_id,h_c_d_id,h_c_id) REFERENCES 
customer(c_w_id,c_d_id,c_id), \
+FOREIGN KEY(h_w_id,h_d_id) REFERENCES district(d_w_id,d_id) );")
+
+sql_execute("drop table if exists orders;")
+sql_execute("create table orders ( \
+o_id int not null, \
+o_d_id int not null, \
+o_w_id int not null, \
+o_c_id int, \
+o_entry_d varchar(100), \
+o_carrier_id int, \
+o_ol_cnt int, \
+o_all_local int, \
+PRIMARY KEY(o_w_id, o_d_id, o_id), \
+FOREIGN KEY(o_w_id,o_d_id,o_c_id) REFERENCES 
customer(c_w_id,c_d_id,c_id) );")
+
+sql_execute("drop table if exists new_orders;")
+sql_execute("create table new_orders ( \
+no_o_id int not null, \
+no_d_id int not null, \
+no_w_id int not null, \
+PRIMARY KEY(no_w_id, no_d_id, no_o_id), \
+FOREIGN KEY(no_w_id,no_d_id,no_o_id) REFERENCES 
orders(o_w_id,o_d_id,o_id));")
+
+sql_execute("drop table if exists item;")
+sql_execute("create table item ( \
+i_id int not null, \
+i_im_id int, \
+i_name varchar(24), \
+i_price number, \
+i_data varchar(50), \
+PRIMARY KEY(i_id) );")
+
+sql_execute("drop table if exists stock;")
+sql_execute("create table stock ( \
+s_i_id int not null, \
+s_w_id int not null, \
+s_quantity int, \
+s_dist_01 varchar(24), \
+s_dist_02 varchar(24), \
+s_dist_03 varchar(24), \
+s_dist_04 varchar(24), \
+s_dist_05 varchar(24), \
+s_dist_06 varchar(24), \
+s_dist_07 varchar(24), \
+s_dist_08 varchar(24), \
+s_dist_09 varchar(24), \
+s_dist_10 varchar(24), \
+s_ytd varchar(9), \
+s_order_cnt int, \
+s_remote_cnt int, \
+s_data varchar(50), \
+PRIMARY KEY(s_w_id, s_i_id), \
+FOREIGN KEY(s_w_id) REFERENCES warehouse(w_id), \
+FOREIGN KEY(s_i_id) REFERENCES item(i_id) );")
+
+sql_execute("drop table if exists order_line;")
+sql_execute("create table order_line ( \
+ol_o_id int not null, \
+ol_d_id int not null, \
+ol_w_id int not null, \
+ol_number int not null, \
+ol_i_id int, \
+ol_supply_w_id int, \
+ol_delivery_d varchar(100), \
+ol_quantity int, \
+ol_amount number, \
+ol_dist_info varchar(24), \
+PRIMARY KEY(ol_w_id, ol_d_id, ol_o_id, ol_number), \
+FOREIGN KEY(ol_w_id,ol_d_id,ol_o_id) REFERENCES 
orders(o_w_id,o_d_id,o_id), \
+FOREIGN KEY(ol_supply_w_id,ol_i_id) REFERENCES stock(s_w_id,s_i_id) );")
+
+sql_execute("CREATE INDEX idx_customer ON customer 
(c_w_id,c_d_id,c_last,c_first);")
+sql_execute("CREATE INDEX idx_orders ON orders 
(o_w_id,o_d_id,o_c_id,o_id);")
+sql_execute("CREATE INDEX fkey_stock_2 ON stock (s_i_id);")
+sql_execute("CREATE INDEX fkey_order_line_2 ON order_line 
(ol_supply_w_id,ol_i_id);")
diff --git a/benches/tpcc/server.lua b/benches/tpcc/server.lua
new file mode 100644
index 0000000..17cda59
--- /dev/null
+++ b/benches/tpcc/server.lua
@@ -0,0 +1,2 @@
+box.cfg{listen = 3301, memtx_memory = 10 * 1024^3}
+box.schema.user.grant('guest', 'read,write,execute', 'universe', nil, { 
if_not_exists = true })
diff --git a/benches/tpch/run.sh b/benches/tpch/run.sh
new file mode 100755
index 0000000..e9c6ca4
--- /dev/null
+++ b/benches/tpch/run.sh
@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+set -eu
+set -o pipefail
+
+source ../common.sh
+
+TPCH_SKIP_SQLITE="${TPCH_SKIP_SQLITE:-}"
+
+TAR_VER=$(get_tarantool_version)
+numaopts=(--membind=1 --cpunodebind=1 '--physcpubind=6,7,8,9,10,11')
+numastr=''
+
+if can_be_run_under_numa "${numaopts[@]}"; then
+    numastr="numactl ${numaopts[*]}"
+fi
+
+if [ -n "$TPCH_SKIP_SQLITE" ]; then
+    make create_SQL_db
+
+    kill_tarantool
+    sync_disk
+    maybe_drop_cache
+
+    make bench-sqlite NUMAOPTS="$numastr"
+fi
+
+make create_TNT_db
+
+kill_tarantool
+sync_disk
+maybe_drop_cache
+
+make bench-tnt TARANTOOL="$TARANTOOL_EXECUTABLE" NUMAOPTS="$numastr"
+make report
+sed "/-2/d" bench-tnt.csv | sed "s/;/:/" | sed "s/-1/0/" | tee 
tpc.h_result.txt
+
+echo "${TAR_VER}" | tee tpc.h_t_version.txt
+
+echo "Tarantool TAG:"
+cat tpc.h_t_version.txt
+
+if [ -n "$TPCH_SKIP_SQLITE" ]; then
+    echo "Overall result SQL:"
+    echo "==================="
+    cat bench-sqlite.csv
+fi
+
+echo " "
+echo "Overall result TNT:"
+echo "==================="
+cat tpc.h_result.txt
diff --git a/benches/ycsb/run.sh b/benches/ycsb/run.sh
new file mode 100755
index 0000000..5083347
--- /dev/null
+++ b/benches/ycsb/run.sh
@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+
+set -eu
+set -o pipefail
+
+source ../common.sh
+
+YCSB_TYPES="${YCSB_TYPES:-all}"
+YCSB_RECORDCOUNT="${YCSB_RECORDCOUNT:-1000000}"
+YCSB_OPERATIONCOUNT="${YCSB_OPERATIONCOUNT:-1000000}"
+YCSB_MEMTXMEMORY="${YCSB_MEMTXMEMORY:-2000000000}"
+YCSB_RUNS="${YCSB_RUNS:-1}"
+YCSB_WORKLOADS="${YCSB_WORKLOADS:-all}"
+
+TAR_VER=$(get_tarantool_version)
+
+function run_ycsb {
+    local mode="$1"
+    local srvlua="tarantool/src/main/conf/tarantool-${mode}.lua"
+    sed "s/listen=.*/listen=3301,\n   memtx_memory = 
$YCSB_MEMTXMEMORY,/" -i "$srvlua"
+    sed 's/logger_nonblock.*//' -i "$srvlua"
+    sed 's/logger/log/' -i "$srvlua"
+    sed 's/read,write,execute/create,read,write,execute/' -i "$srvlua"
+    maybe_under_numactl "${numaopts[@]}" -- \
+        "$TARANTOOL_EXECUTABLE" "$srvlua" 2>&1 &
+    wait_for_tarantool_runnning 3301 10
+
+    local workloads=(a b c d e f)
+
+    if [ -n "$YCSB_WORKLOADS" -a "$YCSB_WORKLOADS" != "all" ]; then
+        IPS=, read -ra workloads <<< "$YCSB_WORKLOADS"
+    fi
+
+    for l in "${workloads[@]}"; do
+        echo "=============== $l"
+        for r in $( seq 1 "$YCSB_RUNS" ); do
+            local res="$plogs/run${l}_${r}"
+            echo "---------------- ${l}: $r"
+            echo "tarantool.port=3301" >> "workloads/workload${l}"
+            maybe_under_numactl "${numaopts[@]}" -- \
+                ./bin/ycsb load tarantool -s -P 
"workloads/workload${l}" > "${res}.load" 2>&1 || cat "${res}.load"
+            sync_disk
+            maybe_drop_cache
+
+            maybe_under_numactl "${numaopts[@]}" -- \
+                ./bin/ycsb run tarantool -s -P "workloads/workload${l}" 
 > "${res}.log" 2>&1 || cat "${res}.log"
+
+            grep Thro "${res}.log" | awk '{ print "Overall result: "$3 
}' | tee "${res}.txt"
+            sed "s#Overall result#$l $r#g" "${res}.txt" >> 
"${plogs}/ycsb.${mode}_result.txt"
+
+            stop_and_clean_tarantool
+        done
+    done
+}
+
+types=(hash tree)
+
+if [ -n "$YCSB_TYPES" -a "$YCSB_TYPES" != 'all' ]; then
+    IPS=, read -ra types <<< "$YCSB_TYPES"
+fi
+
+numaopts=(--membind=1 --cpunodebind=1 '--physcpubind=6,7,8,9,10,11')
+
+for f in workloads/workload[a-f] ; do
+    sed "s#recordcount=.*#recordcount=$YCSB_RECORDCOUNT#g" -i "$f"
+    sed "s#operationcount=.*#operationcount=$YCSB_OPERATIONCOUNT#g" -i "$f"
+    echo "tarantool.port=3301" >> "$f"
+done
+
+
+plogs=results
+rm -rf "$plogs"
+mkdir "$plogs"
+
+for t in "${types[@]}"; do
+    run_ycsb "$t"
+done
+
+echo "${TAR_VER}" | tee "ycsb.${mode}_t_version.txt"
+cp -f "${plogs}/ycsb.${mode}_result.txt" .
+
+echo "Tarantool TAG:"
+cat "ycsb.${mode}_t_version.txt"
+echo "Overall results:"
+echo "================"
+cat "ycsb.${mode}_result.txt"
diff --git a/config.sh b/config.sh
new file mode 100644
index 0000000..93d3fc5
--- /dev/null
+++ b/config.sh
@@ -0,0 +1,41 @@
+# path to tarantool executable
+export TARANTOOL_EXECUTABLE=tarantool
+
+# location of tests to be run
+export BENCH_WORKDIR="$PWD/.benchdir"
+
+# cbench parameters
+export CBENCH_DIR="$HOME/work/cbench/"
+export CBENCH_VINYL_WORKLOAD=50
+export CBENCH_MEMTX_WORKLOAD=10000
+
+# linkbench parameters
+export LINKBENCH_DIR="$HOME/work/linkbench/"
+export LINKBENCH_REQUESTERS=1
+export LINKBENCH_REQUESTS=1000
+export LINKBENCH_WORKLOAD_SIZE=1000
+
+# linkbench parameters
+export NOSQLBENCH_DIR="$HOME/work/nosqlbench/"
+
+# linkbench parameters
+export SYSBENCH_DIR="$HOME/work/sysbench/"
+export SYSBENCH_RUNS=1
+export SYSBENCH_TIME=5
+
+# tpcc parameters
+export TPCC_DIR="$HOME/work/tpcc/"
+export TPCC_TIME=20
+export TPCC_WARMUPTIME=5
+export TPCC_WAREHOUSES=15
+export TPCC_FROMSNAPSHOT="/tmp/00000000000000033515.snap"
+
+# tpcc parameters
+export TPCH_DIR="$HOME/work/tpch/"
+export TPCH_SKIP_SQLITE=1
+
+# tpcc parameters
+export YCSB_DIR="$HOME/work/ycsb/"
+export YCSB_OPERATIONCOUNT=1000
+export YCSB_RECORDCOUNT=1000
+export YCSB_RUNS=1
diff --git a/ttbench b/ttbench
new file mode 100755
index 0000000..ac656f5
--- /dev/null
+++ b/ttbench
@@ -0,0 +1,149 @@
+#!/bin/bash
+
+set -euo pipefail
+
+TTBENCH_CONFIG=config.sh
+TTBENCH_BENCHES=all
+TTBENCH_COMMAND=
+TTBENCH_BENCHDIR=benches
+
+function error {
+    echo "ERROR:" "$@"
+    exit 100
+}
+
+function print_help {
+    grep '^\s*# HELP' "$0" | sed 's/^\s*# HELP\s//g'
+}
+
+function list_benchmarks {
+    for d in $TTBENCH_BENCHDIR/*; do
+        sed "s#$TTBENCH_BENCHDIR/##" <<< "$d"
+    done | grep -v common.sh
+}
+
+function run_bench {
+    local bench="$1"
+    local benchdir=
+    local workdir="${BENCH_WORKDIR:?}/$bench/"
+
+    cp "$TTBENCH_BENCHDIR/common.sh" "${BENCH_WORKDIR}"
+
+    export COMMON_FUNCTIONS="$BENCH_WORKDIR/common.sh"
+
+    if benchdir=$(env | grep "${bench^^}_DIR" | sed 's/^.*=//'); then
+        echo "Running $bench with workdir $benchdir" 1>&2
+        rm -rf "$workdir"
+        mkdir -p "$workdir"
+
+        cp -r "$TTBENCH_BENCHDIR/$bench"/* "$workdir"
+        cp -r "$benchdir"/* "$workdir"
+
+        local runner=
+        runner=$(readlink -f "$TTBENCH_BENCHDIR/$bench/run.sh")
+        pushd "$workdir" 1>/dev/null
+        "$runner"
+        popd 1>/dev/null
+    else
+        error "variable '${bench^^}_DIR' is not set in config, can't 
run benchmark"
+    fi
+}
+
+function run_benchmarks {
+    # must be set in config
+    [ -z "$BENCH_WORKDIR" ] && error "BENCH_WORKDIR is not set"
+
+    if [ "$TTBENCH_BENCHES" == all ]; then
+        for b in $(list_benchmarks); do
+            run_bench "$b"
+        done
+    else
+        IFS=, read -ra benchlist <<< "$TTBENCH_BENCHES"
+        # local benchlist=( $(sed 's/,/ /g' <<< "$TTBENCH_BENCHES") );
+        for b in "${benchlist[@]}"; do
+            if [ ! -d "$TTBENCH_BENCHDIR/$b" ]; then
+                error "Unknown benchmark='$b', please run '$0 
list-benchmarks'"
+            fi
+        done
+
+        for b in "${benchlist[@]}"; do
+            run_bench "$b"
+        done
+    fi
+}
+
+# HELP ttbench - launch tarantool external benchmarks
+# HELP
+# HELP Usage: ttbench [OPTIONS]... COMMAND
+# HELP
+# HELP Available options are:
+while [[ $# -gt 0 ]]; do
+    option="$1"
+
+    case $option in
+        # HELP   -c, --config 'config_file'
+        # HELP           use given config file (defaults to config.sh)
+        # HELP
+        -c|--config)
+            TTBENCH_CONFIG="${2:-}"
+            [ -z "$TTBENCH_CONFIG" ] && error "$option param can not be 
empty"
+            shift 2;;
+
+        # HELP   -d, --debug
+        # HELP           sets -x flag on whole script
+        # HELP
+        -d|--debug)
+            set -x
+            shift;;
+
+        # HELP   -b, --benchmarks 'bench1,bench2'
+        # HELP           comma separated list of benchmarks (defauts to 
'all')
+        # HELP
+        -b|--benchmarks)
+            TTBENCH_BENCHES="${2:-}"
+            [ -z "$TTBENCH_BENCHES" ] && error "$option param can not 
be empty"
+            shift 2;;
+
+        # HELP   -h, --help
+        # HELP           print help message
+        # HELP
+        -h|--help)
+            print_help
+            exit 0;;
+
+        *)
+            break;;
+    esac
+done
+
+[ -f "$TTBENCH_CONFIG" ] && source "$TTBENCH_CONFIG"
+
+TTBENCH_COMMAND="${1:-}"
+
+if [ -z "$TTBENCH_COMMAND" ]; then
+    print_help
+    echo "ERROR: no COMMAND specified" 1>&2
+    exit 1
+fi
+
+shift
+
+# HELP
+# HELP Available commands are:
+case "$TTBENCH_COMMAND" in
+    # HELP   ls, list-benchmarks
+    # HELP           lists all known benchmarks
+    # HELP
+    ls|list-benchmarks)
+        list_benchmarks;;
+
+    # HELP   run
+    # HELP           run benchmarks specified by --benchmarks flag
+    run)
+        run_benchmarks;;
+
+    *)
+        print_help
+        echo "ERROR: Unknown COMMAND='$TTBENCH_COMMAND'" 1>&2
+        exit 1;;
+esac
-- 
2.7.4