[Tarantool-patches] [PATCH] benchmarks: runner for local benchmarks runs
Oleg Koshovetc
okoshovetc at tarantool.org
Wed Dec 2 20:08:52 MSK 2020
https://github.com/tarantool/bench-run/compare/ttbench?expand=1
This runner allows to run various benchmarks on local machines (if all
the source code was already present and prebuilt on a machine). However
analyzing the results of benchmarks requires manual reading through
their logs
TODO: implement automatic local installation of benchmarks
TODO: implement tool to save and diff benchmarks' results
---
benches/cbench/run.sh | 35 ++++++++++
benches/common.sh | 109 +++++++++++++++++++++++++++++
benches/linkbench/app.lua | 17 +++++
benches/linkbench/run.sh | 46 +++++++++++++
benches/nosqlbench/run.sh | 57 +++++++++++++++
benches/nosqlbench/tnt_hash.lua | 30 ++++++++
benches/nosqlbench/tnt_tree.lua | 31 +++++++++
benches/publication/publish.py | 64 +++++++++++++++++
benches/sysbench/run.sh | 92 +++++++++++++++++++++++++
benches/sysbench/tnt_srv.lua | 28 ++++++++
benches/tpcc/init_empty.lua | 2 +
benches/tpcc/init_not_empty.lua | 1 +
benches/tpcc/run.sh | 49 +++++++++++++
benches/tpcc/schema.lua | 144
++++++++++++++++++++++++++++++++++++++
benches/tpcc/server.lua | 2 +
benches/tpch/run.sh | 51 ++++++++++++++
benches/ycsb/run.sh | 86 +++++++++++++++++++++++
config.sh | 41 +++++++++++
ttbench | 149
++++++++++++++++++++++++++++++++++++++++
19 files changed, 1034 insertions(+)
create mode 100755 benches/cbench/run.sh
create mode 100644 benches/common.sh
create mode 100755 benches/linkbench/app.lua
create mode 100755 benches/linkbench/run.sh
create mode 100755 benches/nosqlbench/run.sh
create mode 100644 benches/nosqlbench/tnt_hash.lua
create mode 100644 benches/nosqlbench/tnt_tree.lua
create mode 100755 benches/publication/publish.py
create mode 100755 benches/sysbench/run.sh
create mode 100644 benches/sysbench/tnt_srv.lua
create mode 100644 benches/tpcc/init_empty.lua
create mode 100644 benches/tpcc/init_not_empty.lua
create mode 100755 benches/tpcc/run.sh
create mode 100644 benches/tpcc/schema.lua
create mode 100644 benches/tpcc/server.lua
create mode 100755 benches/tpch/run.sh
create mode 100755 benches/ycsb/run.sh
create mode 100644 config.sh
create mode 100755 ttbench
diff --git a/benches/cbench/run.sh b/benches/cbench/run.sh
new file mode 100755
index 0000000..09c4dee
--- /dev/null
+++ b/benches/cbench/run.sh
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+set -eu
+set -o pipefail
+
+source ../common.sh
+
+CBENCH_VINYL_WORKLOAD="${CBENCH_VINYL_WORKLOAD:-500}"
+CBENCH_MEMTX_WORKLOAD="${CBENCH_MEMTX_WORKLOAD:-1000000}"
+
+export LUA_CPATH="$PWD/?.so"
+export LUA_PATH="$PWD/?/init.lua"
+
+TAR_VER=$(get_tarantool_version)
+numaopts=(--membind=1 --cpunodebind=1 --physcpubind=11)
+
+stop_and_clean_tarantool
+maybe_drop_cache
+
+maybe_under_numactl "${numaopts[@]}" -- \
+ "$TARANTOOL_EXECUTABLE" cbench_runner.lua memtx
"$CBENCH_MEMTX_WORKLOAD" 2>&1 | tee cbench_output_memtx.txt
+
+stop_and_clean_tarantool
+maybe_drop_cache
+
+maybe_under_numactl "${numaopts[@]}" -- \
+ "$TARANTOOL_EXECUTABLE" cbench_runner.lua vinyl fsync
"$CBENCH_VINYL_WORKLOAD" 2>&1 | tee cbench_output_vinyl_fsync.txt
+
+stop_and_clean_tarantool
+maybe_drop_cache
+
+maybe_under_numactl "${numaopts[@]}" -- \
+ "$TARANTOOL_EXECUTABLE" cbench_runner.lua vinyl write
"$CBENCH_VINYL_WORKLOAD" 2>&1 | tee cbench_output_vinyl_write.txt
+
+echo "$TAR_VER"
diff --git a/benches/common.sh b/benches/common.sh
new file mode 100644
index 0000000..1116cc9
--- /dev/null
+++ b/benches/common.sh
@@ -0,0 +1,109 @@
+# will not work if not under root
+# if benches are run localy by a developer droping caches might not be
too important
+function maybe_drop_cache {
+ if test -w /proc/sys/vm/drop_caches; then
+ echo 3 > /proc/sys/vm/drop_caches
+ fi
+}
+
+function can_be_run_under_numa {
+ numactl "$@" true 1>/dev/null 2>/dev/null
+}
+
+# not every local machine has numa enabled on it
+# not every local machine has enough hardware units to run with
intended configuration
+#
+# usage:
+# maybe_under_numactl numaoptions -- command to run
+# example:
+# maybe_under_numactl --membind=1 --cpunodebind=1 --physcpubind=11 --
echo qwe
+function maybe_under_numactl {
+ local numaoptions=()
+ local cmdoptions=()
+ local parsing_numa=1
+ for option in "$@"; do
+ if [ -n "$parsing_numa" ]; then
+ if [ "$option" == -- ]; then
+ parsing_numa=
+ else
+ numaoptions+=( "$option" )
+ fi
+ else
+ cmdoptions+=( "$option" )
+ fi
+ done
+
+ if which numactl 1>/dev/null 2>/dev/null; then
+ # check if it is even runable with given numactl options
+ if can_be_run_under_numa "${numaoptions[@]}"; then
+ numactl "${numaoptions[@]}" "${cmdoptions[@]}"
+ else
+ "${cmdoptions[@]}"
+ fi
+ else
+ "${cmdoptions[@]}"
+ fi
+}
+
+function get_tarantool_version {
+ "$TARANTOOL_EXECUTABLE" -v | grep -e "Tarantool" | grep -oP '\s\K\S*'
+}
+
+function clean_tarantool {
+ rm -rf ./*.snap
+ rm -rf ./*.xlog
+ rm -rf ./*.vylog
+ rm -rf 5*
+}
+
+function sync_disk {
+ sync && echo "sync passed" || echo "sync failed with error: $?"
+}
+
+function kill_tarantool {
+ killall tarantool 2>/dev/null || true
+}
+
+function error {
+ echo "ERROR:" "$@"
+ exit 100
+}
+
+function stop_and_clean_tarantool {
+ kill_tarantool
+ clean_tarantool
+ sync_disk
+}
+
+function wait_for_file_release {
+ local f="$1"
+ local t="$2"
+ local tt=0
+ while [ "$tt" -lt "$t" ]; do
+ if ! lsof "$f" 1>/dev/null 2>/dev/null; then
+ return 0
+ fi
+
+ tt=$(( tt + 1 ))
+ sleep 1
+ done
+
+ return 1
+}
+
+function wait_for_tarantool_runnning {
+ local creds="$1"
+ local t="$2"
+ local tt=0
+
+ while [ "$tt" -lt "$t" ]; do
+ if echo 'if type(box.cfg) ~= "function" then return
box.info().status end' | tarantoolctl connect "$creds" 2>/dev/null |
grep -q 'running'; then
+ return 0
+ fi
+
+ tt=$(( tt + 1 ))
+ sleep 1
+ done
+
+ return 1
+}
diff --git a/benches/linkbench/app.lua b/benches/linkbench/app.lua
new file mode 100755
index 0000000..79252e0
--- /dev/null
+++ b/benches/linkbench/app.lua
@@ -0,0 +1,17 @@
+#!/usr/bin/env tarantool
+
+local path = require('fio').dirname(arg[0])
+package.path = path.."/?.lua;"..package.path
+package.cpath = path.."/?.so;"..package.cpath
+
+require('console').listen('unix/:./tarantool.sock')
+require('gperftools').cpu.start('tarantool.prof')
+
+box.cfg{
+ listen = 3301;
+ vinyl_memory = 256 * 1024 * 1024;
+ vinyl_cache = 256 * 1024 * 1024;
+ vinyl_read_threads = 1;
+}
+
+require('app_internal')
diff --git a/benches/linkbench/run.sh b/benches/linkbench/run.sh
new file mode 100755
index 0000000..c5f493e
--- /dev/null
+++ b/benches/linkbench/run.sh
@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+
+set -eu
+set -o pipefail
+
+source ../common.sh
+
+LINKBENCH_REQUESTS="${LINKBENCH_REQUESTS:-2000000}"
+LINKBENCH_REQUESTERS="${LINKBENCH_REQUESTERS:-1}"
+LINKBENCH_WORKLOAD_SIZE="${LINKBENCH_WORKLOAD_SIZE:-5000000}"
+
+TAR_VER=$(get_tarantool_version)
+numaopts=(--membind=1 --cpunodebind=1 '--physcpubind=6,7,8,9,10,11')
+
+mvn clean package -Dmaven.test.skip=true
+make -C src/tarantool -B
+
+tfile=src/tarantool/app.lua
+
+stop_and_clean_tarantool
+maybe_under_numactl "${numaopts[@]}" -- tarantool "$tfile" 1>/dev/null
2>/dev/null &
+
+cfgfile=config/LinkConfigTarantool.properties
+sed "s/^maxid1 = .*/maxid1 = $LINKBENCH_WORKLOAD_SIZE/g" -i
config/FBWorkload.properties
+sed "s/^requesters = .*/requesters = $LINKBENCH_REQUESTERS/g" -i "$cfgfile"
+sed "s/^requests = .*/requests = $LINKBENCH_REQUESTS/g" -i "$cfgfile"
+
+maybe_under_numactl "${numaopts[@]}" -- \
+ ./bin/linkbench -c "$cfgfile" -l 2>&1 | tee loading.res.txt
+
+sync_disk
+maybe_drop_cache
+
+maybe_under_numactl "${numaopts[@]}" -- \
+ ./bin/linkbench -c "$cfgfile" -r 2>&1 | tee linkbench_output.txt
+
+kill_tarantool
+
+grep "REQUEST PHASE COMPLETED" linkbench_output.txt | sed "s/.*second =
/linkbench:/" | tee -a linkbench.ssd_result.txt
+echo "${TAR_VER}" | tee linkbench.ssd_t_version.txt
+
+echo "Tarantool TAG:"
+cat linkbench.ssd_t_version.txt
+echo "Overall results:"
+echo "================"
+cat linkbench.ssd_result.txt
diff --git a/benches/nosqlbench/run.sh b/benches/nosqlbench/run.sh
new file mode 100755
index 0000000..b092cfe
--- /dev/null
+++ b/benches/nosqlbench/run.sh
@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+
+set -eu
+set -o pipefail
+
+source ../common.sh
+
+# type=$1
+# if [ "$type" == "" ]; then
+# type=hash
+# fi
+
+NOSQLBENCH_WORKLOAD="${NOSQLBENCH_WORKLOAD:-}"
+NOSQLBENCH_TIMELIMIT="${NOSQLBENCH_TIMELIMIT:-20000}"
+NOSQLBENCH_BATCHCOUNT="${NOSQLBENCH_BATCHCOUNT:-10}"
+NOSQLBENCH_RPS="${NOSQLBENCH_RPS:-20000}"
+
+TAR_VER=$(get_tarantool_version)
+numaopts=(--membind=1 --cpunodebind=1 '--physcpubind=6,7,8,9,10,11')
+
+function run_nosqlbench {
+ local type="$1"
+ stop_and_clean_tarantool
+
+ maybe_under_numactl "${numaopts[@]}" -- \
+ "$TARANTOOL_EXECUTABLE" "tnt_${type}.lua" 2>&1 &
+
+ config=nosqlbench.conf
+ cp src/nosqlbench.conf "$config"
+
+ sed "s/port 3303/port 3301/" "$config" -i
+ sed "s/benchmark 'no_limit'/benchmark 'time_limit'/" "$config" -i
+ sed "s/time_limit 10/time_limit $NOSQLBENCH_TIMELIMIT/" "$config" -i
+ sed "s/request_batch_count 1/request_batch_count
$NOSQLBENCH_BATCHCOUNT/" "$config" -i
+ sed "s/rps 12000/rps $NOSQLBENCH_RPS/" "$config" -i
+
+ sleep 5
+ echo "Run NB type='$type'"
+
+ # WARNING: don't try to save output from stderr - file will use the
whole disk space !
+ (maybe_under_numactl "${numaopts[@]}" -- \
+ ./src/nb "$config") \
+ | grep -v "Warmup" \
+ | grep -v "Failed to allocate" >nosqlbench_output.txt \
+ || cat nosqlbench_output.txt
+
+ grep "TOTAL RPS STATISTICS:" nosqlbench_output.txt -A6 | \
+ awk -F "|" 'NR > 4 {print $2,":", $4}' >
"noSQLbench.${type}_result.txt"
+ echo "${TAR_VER}" | tee "noSQLbench.${type}_t_version.txt"
+}
+
+if [ -z "$NOSQLBENCH_WORKLOAD" -o "$NOSQLBENCH_WORKLOAD" == all ]; then
+ run_nosqlbench tree
+ run_nosqlbench hash
+else
+ run_nosqlbench "$NOSQLBENCH_WORKLOAD"
+fi
diff --git a/benches/nosqlbench/tnt_hash.lua
b/benches/nosqlbench/tnt_hash.lua
new file mode 100644
index 0000000..a616490
--- /dev/null
+++ b/benches/nosqlbench/tnt_hash.lua
@@ -0,0 +1,30 @@
+local console = require('console')
+console.listen("/tmp/tarantool-server.sock")
+
+box.cfg {
+ pid_file = "./tarantool-server.pid",
+ log = "./tarantool-server.log",
+ listen = 3301,
+ memtx_memory = 2000000000,
+ background = true,
+ checkpoint_interval = 0,
+ wal_mode = 'none',
+}
+
+s = box.schema.space.create('tester_nosqlbench')
+s:create_index('primary', {type = 'hash', parts = {1, 'unsigned'}})
+
+function try(f, catch_f)
+ local status, exception = pcall(f)
+ if not status then
+ catch_f(exception)
+ end
+end
+
+try(function()
+ box.schema.user.grant('guest', 'create,read,write,execute', 'universe')
+end, function(e)
+ print(e)
+end)
+
+
diff --git a/benches/nosqlbench/tnt_tree.lua
b/benches/nosqlbench/tnt_tree.lua
new file mode 100644
index 0000000..57b646b
--- /dev/null
+++ b/benches/nosqlbench/tnt_tree.lua
@@ -0,0 +1,31 @@
+local console = require('console')
+console.listen("/tmp/tarantool-server.sock")
+
+--memtx_memory = 2000000000,
+box.cfg {
+ pid_file = "./tarantool-server.pid",
+ log = "./tarantool-server.log",
+ listen = 3301,
+ vinyl_memory = 107374182,
+ background = true,
+ checkpoint_interval = 0,
+ wal_mode = 'none',
+}
+
+s = box.schema.space.create('tester_nosqlbench')
+s:create_index('primary', {type = 'tree', parts = {1, 'unsigned'}})
+
+function try(f, catch_f)
+ local status, exception = pcall(f)
+ if not status then
+ catch_f(exception)
+ end
+end
+
+try(function()
+ box.schema.user.grant('guest', 'create,read,write,execute', 'universe')
+end, function(e)
+ print(e)
+end)
+
+
diff --git a/benches/publication/publish.py b/benches/publication/publish.py
new file mode 100755
index 0000000..42bc53a
--- /dev/null
+++ b/benches/publication/publish.py
@@ -0,0 +1,64 @@
+#!/usr/bin/env python
+import fnmatch
+import os
+from urllib import urlencode
+import requests
+
+
+def parse_bench(filename):
+ with open(filename) as raw_data:
+ return raw_data.readlines()
+
+
+def get_version(filename):
+ with open(filename) as raw_data:
+ version = raw_data.readlines()[-1]
+ return version.split()[0]
+
+
+def push_to_microb(server, token, name, value, version, tab):
+ uri = 'http://%s/push?%s' % (server, urlencode(dict(
+ key=token, name=name, param=value,
+ v=version, unit='trps', tab=tab
+ )))
+
+ r = requests.get(uri)
+ if r.status_code == 200:
+ print('Export complete')
+ else:
+ print('Export error http: %d' % r.status_code)
+ print('Export error text: %d' % r.text)
+
+
+def main():
+ if "MICROB_WEB_TOKEN" in os.environ and "MICROB_WEB_HOST" in
os.environ:
+ bench = {}
+ res = []
+ current_data = {}
+ version = ''
+ for file in os.listdir('.'):
+ if fnmatch.fnmatch(file, '*_result.txt'):
+ values = parse_bench(file)
+ benchmark = file.split('_')[0]
+ version = get_version('{}_t_version.txt'.format(benchmark))
+ for value in values:
+ test_name = value.split(':')[0]
+ test_res = float(value.split(':')[1])
+ res.append(test_res)
+ push_to_microb(
+ os.environ['MICROB_WEB_HOST'],
+ os.environ['MICROB_WEB_TOKEN'],
+ test_name,
+ test_res,
+ version,
+ benchmark,
+ )
+ print ("VERSION - ", version)
+ else:
+ print("MICROB params not specified")
+
+ return 0
+
+
+if __name__ == '__main__':
+ main()
diff --git a/benches/sysbench/run.sh b/benches/sysbench/run.sh
new file mode 100755
index 0000000..4b52385
--- /dev/null
+++ b/benches/sysbench/run.sh
@@ -0,0 +1,92 @@
+#!/usr/bin/env bash
+
+set -eu
+set -o pipefail
+
+source ../common.sh
+
+SYSBENCH_TESTS="${SYSBENCH_TESTS:-}"
+SYSBENCH_TIME="${SYSBENCH_TIME:-20}"
+SYSBENCH_WARMUPTIME="${SYSBENCH_WARMUPTIME:-5}"
+SYSBENCH_DBMS="${SYSBENCH_DBMS:-tarantool}"
+SYSBENCH_THREADS="${SYSBENCH_THREADS:-200}"
+SYSBENCH_RUNS="${SYSBENCH_RUNS:-10}"
+
+TAR_VER=$(get_tarantool_version)
+numaconf=(--membind=1 --cpunodebind=1 '--physcpubind=6,7,8,9,10,11')
+
+ARRAY_TESTS=(
+ "oltp_read_only"
+ "oltp_write_only"
+ "oltp_read_write"
+ "oltp_update_index"
+ "oltp_update_non_index"
+ "oltp_insert"
+ "oltp_delete"
+ "oltp_point_select"
+ "select_random_points"
+ "select_random_ranges"
+ # "bulk_insert"
+)
+
+opts=("--db-driver=${SYSBENCH_DBMS}" "--threads=${SYSBENCH_THREADS}")
+
+export LD_LIBRARY_PATH=/usr/local/lib
+
+if [ -n "$SYSBENCH_TESTS" -a "$SYSBENCH_TESTS" != all ]; then
+ IPS=, read -ra testlist <<< "$SYSBENCH_TESTS"
+ ARRAY_TESTS=$testlist
+fi
+
+for test in "${ARRAY_TESTS[@]}"; do
+ res=0
+ tlog=sysbench_${test}_results.txt
+ rm -f "$tlog"
+ maxres=0
+ for run in $(seq 1 "$SYSBENCH_RUNS"); do
+ echo "$run"
+ echo "------------ $test ------------ rerun: # $run ------------"
+
+ stop_and_clean_tarantool
+ wait_for_file_release /tmp/tarantool-server.sock 10
+ maybe_under_numactl "${numaconf[@]}" -- \
+ "$TARANTOOL_EXECUTABLE" tnt_srv.lua >tnt_server.txt 2>&1 \
+ || ( cat tnt_server.txt && false )
+
+ sleep 1
+
+ # ./run_tnt.sh >tnt_server.txt 2>&1 || ( cat tnt_server.txt &&
false )
+
+ ./src/sysbench "$test" "${opts[@]}" cleanup > sysbench_output.txt
+ ./src/sysbench "$test" "${opts[@]}" prepare >> sysbench_output.txt
+
+ maybe_under_numactl "${numaconf[@]}" -- \
+ ./src/sysbench "$test" "${opts[@]}" \
+ "--time=${SYSBENCH_TIME}" \
+ "--warmup-time=${SYSBENCH_WARMUPTIME}" \
+ run >> sysbench_output.txt
+
+ grep -e 'transactions:' sysbench_output.txt | grep -oP
'\(\K\S*' | tee "$tlog"
+ tres=$(sed 's#^.*:##g' < "$tlog" | sed 's#\..*$##g')
+ echo ">>> $tres"
+ res=$(( res + tres ))
+ if [[ $tres -gt $maxres ]]; then maxres=$tres ; fi
+ done
+
+ res=$(( res / SYSBENCH_RUNS ))
+ echo "${test}: $res" >>Sysbench_result.txt
+
+ echo "Subtest '$test' results:"
+ echo "==============================="
+ echo "Average result: $res"
+ echo "Maximum result: $maxres"
+ printf "Deviations (AVG -> MAX): %.2f%%" "$(bc <<< "scale = 4; (1 -
$res / $maxres) * 100")"
+done
+
+echo "${TAR_VER}" | tee Sysbench_t_version.txt
+
+echo "Tarantool TAG:"
+cat Sysbench_t_version.txt
+echo "Overall results:"
+echo "================"
+cat Sysbench_result.txt
diff --git a/benches/sysbench/tnt_srv.lua b/benches/sysbench/tnt_srv.lua
new file mode 100644
index 0000000..d84413d
--- /dev/null
+++ b/benches/sysbench/tnt_srv.lua
@@ -0,0 +1,28 @@
+local console = require('console')
+console.listen("/tmp/tarantool-server.sock")
+
+box.cfg {
+ pid_file = "./tarantool-server.pid",
+ log = "./tarantool-server.log",
+ listen = 3301,
+ memtx_memory = 2000000000,
+ background = true,
+ checkpoint_interval = 0,
+}
+
+
+function try(f, catch_f)
+ local status, exception = pcall(f)
+ if not status then
+ catch_f(exception)
+ end
+end
+
+try(function()
+ box.schema.user.grant('guest', 'create,read,write,execute', 'universe')
+end, function(e)
+ print(e)
+end)
+
+
+
diff --git a/benches/tpcc/init_empty.lua b/benches/tpcc/init_empty.lua
new file mode 100644
index 0000000..e29ff04
--- /dev/null
+++ b/benches/tpcc/init_empty.lua
@@ -0,0 +1,2 @@
+require 'server'
+require 'schema'
diff --git a/benches/tpcc/init_not_empty.lua
b/benches/tpcc/init_not_empty.lua
new file mode 100644
index 0000000..b36eeec
--- /dev/null
+++ b/benches/tpcc/init_not_empty.lua
@@ -0,0 +1 @@
+require 'server'
diff --git a/benches/tpcc/run.sh b/benches/tpcc/run.sh
new file mode 100755
index 0000000..e391d5f
--- /dev/null
+++ b/benches/tpcc/run.sh
@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+set -eu
+set -o pipefail
+
+source ../common.sh
+
+TAR_VER=$(get_tarantool_version)
+numaopts=(--membind=1 --cpunodebind=1 '--physcpubind=6,7,8,9,10,11')
+
+TPCC_TIME="${TPCC_TIME:-1200}"
+TPCC_WARMUPTIME="${TPCC_WARMUPTIME:-10}"
+TPCC_WAREHOUSES="${TPCC_WAREHOUSES:-15}"
+TPCC_FROMSNAPSHOT="${TPCC_FROMSNAPSHOT:-}"
+
+killall tpcc_load 2>/dev/null || true
+stop_and_clean_tarantool
+
+tpcc_opts=(-h localhost -P 3301 -d tarantool -u root -p '' -w
"$TPCC_WAREHOUSES")
+
+if [ -z "$TPCC_FROMSNAPSHOT" ]; then
+ maybe_under_numactl "${numaopts[@]}" \
+ -- "$TARANTOOL_EXECUTABLE" init_empty.lua &
+ wait_for_tarantool_runnning 3301 15
+
+ ./tpcc_load "${tpcc_opts[@]}"
+else
+ [ ! -f "$TPCC_FROMSNAPSHOT" ] && error "No such file:
'$TPCC_FROMSNAPSHOT'"
+ cp "$TPCC_FROMSNAPSHOT" .
+
+ maybe_under_numactl "${numaopts[@]}" \
+ -- "$TARANTOOL_EXECUTABLE" init_not_empty.lua &
+ wait_for_tarantool_runnning 3301 60
+fi
+
+
+maybe_under_numactl "${numaopts[@]}" -- \
+ ./tpcc_start "${tpcc_opts[@]}" -r "$TPCC_WARMUPTIME" -l
"$TPCC_TIME" -i "$TPCC_TIME" > tpcc_output.txt 2>/dev/null
+
+echo -n "tpcc:" | tee tpc.c_result.txt
+grep -e '<TpmC>' tpcc_output.txt | grep -oP '\K[0-9.]*' | tee -a
tpc.c_result.txt
+cat tpcc_output.txt
+
+echo "${TAR_VER}" | tee tpc.c_t_version.txt
+
+echo "Tarantool TAG:"
+cat tpc.c_t_version.txt
+echo "Overall result:"
+echo "==============="
+cat tpc.c_result.txt
diff --git a/benches/tpcc/schema.lua b/benches/tpcc/schema.lua
new file mode 100644
index 0000000..dc9c0df
--- /dev/null
+++ b/benches/tpcc/schema.lua
@@ -0,0 +1,144 @@
+local sql_execute = box.sql and box.sql.execute or box.execute
+
+sql_execute("drop table if exists warehouse;")
+sql_execute("create table warehouse ( \
+w_id int not null, \
+w_name varchar(10), \
+w_street_1 varchar(20), \
+w_street_2 varchar(20), \
+w_city varchar(20), \
+w_state varchar(2), \
+w_zip varchar(9), \
+w_tax number, \
+w_ytd number, \
+primary key (w_id) )")
+
+sql_execute("drop table if exists district;")
+sql_execute("create table district ( \
+d_id int not null, \
+d_w_id int not null, \
+d_name varchar(10), \
+d_street_1 varchar(20), \
+d_street_2 varchar(20), \
+d_city varchar(20), \
+d_state varchar(2), \
+d_zip varchar(9), \
+d_tax number, \
+d_ytd number, \
+d_next_o_id int, \
+primary key (d_w_id, d_id), \
+FOREIGN KEY(d_w_id) REFERENCES warehouse(w_id) );")
+
+sql_execute("drop table if exists customer;")
+sql_execute("create table customer ( \
+c_id int not null, \
+c_d_id int not null, \
+c_w_id int not null, \
+c_first varchar(16), \
+c_middle varchar(2), \
+c_last varchar(16), \
+c_street_1 varchar(20), \
+c_street_2 varchar(20), \
+c_city varchar(20), \
+c_state varchar(2), \
+c_zip varchar(9), \
+c_phone varchar(16), \
+c_since varchar(100), \
+c_credit varchar(2), \
+c_credit_lim int, \
+c_discount number, \
+c_balance number, \
+c_ytd_payment number, \
+c_payment_cnt int, \
+c_delivery_cnt int, \
+c_data text, \
+PRIMARY KEY(c_w_id, c_d_id, c_id), \
+FOREIGN KEY(c_w_id,c_d_id) REFERENCES district(d_w_id,d_id) );")
+
+sql_execute("drop table if exists history;")
+sql_execute("create table history ( \
+_id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, \
+h_c_id int, \
+h_c_d_id int, \
+h_c_w_id int, \
+h_d_id int, \
+h_w_id int, \
+h_date varchar(100), \
+h_amount number, \
+h_data varchar(24), \
+FOREIGN KEY(h_c_w_id,h_c_d_id,h_c_id) REFERENCES
customer(c_w_id,c_d_id,c_id), \
+FOREIGN KEY(h_w_id,h_d_id) REFERENCES district(d_w_id,d_id) );")
+
+sql_execute("drop table if exists orders;")
+sql_execute("create table orders ( \
+o_id int not null, \
+o_d_id int not null, \
+o_w_id int not null, \
+o_c_id int, \
+o_entry_d varchar(100), \
+o_carrier_id int, \
+o_ol_cnt int, \
+o_all_local int, \
+PRIMARY KEY(o_w_id, o_d_id, o_id), \
+FOREIGN KEY(o_w_id,o_d_id,o_c_id) REFERENCES
customer(c_w_id,c_d_id,c_id) );")
+
+sql_execute("drop table if exists new_orders;")
+sql_execute("create table new_orders ( \
+no_o_id int not null, \
+no_d_id int not null, \
+no_w_id int not null, \
+PRIMARY KEY(no_w_id, no_d_id, no_o_id), \
+FOREIGN KEY(no_w_id,no_d_id,no_o_id) REFERENCES
orders(o_w_id,o_d_id,o_id));")
+
+sql_execute("drop table if exists item;")
+sql_execute("create table item ( \
+i_id int not null, \
+i_im_id int, \
+i_name varchar(24), \
+i_price number, \
+i_data varchar(50), \
+PRIMARY KEY(i_id) );")
+
+sql_execute("drop table if exists stock;")
+sql_execute("create table stock ( \
+s_i_id int not null, \
+s_w_id int not null, \
+s_quantity int, \
+s_dist_01 varchar(24), \
+s_dist_02 varchar(24), \
+s_dist_03 varchar(24), \
+s_dist_04 varchar(24), \
+s_dist_05 varchar(24), \
+s_dist_06 varchar(24), \
+s_dist_07 varchar(24), \
+s_dist_08 varchar(24), \
+s_dist_09 varchar(24), \
+s_dist_10 varchar(24), \
+s_ytd varchar(9), \
+s_order_cnt int, \
+s_remote_cnt int, \
+s_data varchar(50), \
+PRIMARY KEY(s_w_id, s_i_id), \
+FOREIGN KEY(s_w_id) REFERENCES warehouse(w_id), \
+FOREIGN KEY(s_i_id) REFERENCES item(i_id) );")
+
+sql_execute("drop table if exists order_line;")
+sql_execute("create table order_line ( \
+ol_o_id int not null, \
+ol_d_id int not null, \
+ol_w_id int not null, \
+ol_number int not null, \
+ol_i_id int, \
+ol_supply_w_id int, \
+ol_delivery_d varchar(100), \
+ol_quantity int, \
+ol_amount number, \
+ol_dist_info varchar(24), \
+PRIMARY KEY(ol_w_id, ol_d_id, ol_o_id, ol_number), \
+FOREIGN KEY(ol_w_id,ol_d_id,ol_o_id) REFERENCES
orders(o_w_id,o_d_id,o_id), \
+FOREIGN KEY(ol_supply_w_id,ol_i_id) REFERENCES stock(s_w_id,s_i_id) );")
+
+sql_execute("CREATE INDEX idx_customer ON customer
(c_w_id,c_d_id,c_last,c_first);")
+sql_execute("CREATE INDEX idx_orders ON orders
(o_w_id,o_d_id,o_c_id,o_id);")
+sql_execute("CREATE INDEX fkey_stock_2 ON stock (s_i_id);")
+sql_execute("CREATE INDEX fkey_order_line_2 ON order_line
(ol_supply_w_id,ol_i_id);")
diff --git a/benches/tpcc/server.lua b/benches/tpcc/server.lua
new file mode 100644
index 0000000..17cda59
--- /dev/null
+++ b/benches/tpcc/server.lua
@@ -0,0 +1,2 @@
+box.cfg{listen = 3301, memtx_memory = 10 * 1024^3}
+box.schema.user.grant('guest', 'read,write,execute', 'universe', nil, {
if_not_exists = true })
diff --git a/benches/tpch/run.sh b/benches/tpch/run.sh
new file mode 100755
index 0000000..e9c6ca4
--- /dev/null
+++ b/benches/tpch/run.sh
@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+set -eu
+set -o pipefail
+
+source ../common.sh
+
+TPCH_SKIP_SQLITE="${TPCH_SKIP_SQLITE:-}"
+
+TAR_VER=$(get_tarantool_version)
+numaopts=(--membind=1 --cpunodebind=1 '--physcpubind=6,7,8,9,10,11')
+numastr=''
+
+if can_be_run_under_numa "${numaopts[@]}"; then
+ numastr="numactl ${numaopts[*]}"
+fi
+
+if [ -n "$TPCH_SKIP_SQLITE" ]; then
+ make create_SQL_db
+
+ kill_tarantool
+ sync_disk
+ maybe_drop_cache
+
+ make bench-sqlite NUMAOPTS="$numastr"
+fi
+
+make create_TNT_db
+
+kill_tarantool
+sync_disk
+maybe_drop_cache
+
+make bench-tnt TARANTOOL="$TARANTOOL_EXECUTABLE" NUMAOPTS="$numastr"
+make report
+sed "/-2/d" bench-tnt.csv | sed "s/;/:/" | sed "s/-1/0/" | tee
tpc.h_result.txt
+
+echo "${TAR_VER}" | tee tpc.h_t_version.txt
+
+echo "Tarantool TAG:"
+cat tpc.h_t_version.txt
+
+if [ -n "$TPCH_SKIP_SQLITE" ]; then
+ echo "Overall result SQL:"
+ echo "==================="
+ cat bench-sqlite.csv
+fi
+
+echo " "
+echo "Overall result TNT:"
+echo "==================="
+cat tpc.h_result.txt
diff --git a/benches/ycsb/run.sh b/benches/ycsb/run.sh
new file mode 100755
index 0000000..5083347
--- /dev/null
+++ b/benches/ycsb/run.sh
@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+
+set -eu
+set -o pipefail
+
+source ../common.sh
+
+YCSB_TYPES="${YCSB_TYPES:-all}"
+YCSB_RECORDCOUNT="${YCSB_RECORDCOUNT:-1000000}"
+YCSB_OPERATIONCOUNT="${YCSB_OPERATIONCOUNT:-1000000}"
+YCSB_MEMTXMEMORY="${YCSB_MEMTXMEMORY:-2000000000}"
+YCSB_RUNS="${YCSB_RUNS:-1}"
+YCSB_WORKLOADS="${YCSB_WORKLOADS:-all}"
+
+TAR_VER=$(get_tarantool_version)
+
+function run_ycsb {
+ local mode="$1"
+ local srvlua="tarantool/src/main/conf/tarantool-${mode}.lua"
+ sed "s/listen=.*/listen=3301,\n memtx_memory =
$YCSB_MEMTXMEMORY,/" -i "$srvlua"
+ sed 's/logger_nonblock.*//' -i "$srvlua"
+ sed 's/logger/log/' -i "$srvlua"
+ sed 's/read,write,execute/create,read,write,execute/' -i "$srvlua"
+ maybe_under_numactl "${numaopts[@]}" -- \
+ "$TARANTOOL_EXECUTABLE" "$srvlua" 2>&1 &
+ wait_for_tarantool_runnning 3301 10
+
+ local workloads=(a b c d e f)
+
+ if [ -n "$YCSB_WORKLOADS" -a "$YCSB_WORKLOADS" != "all" ]; then
+ IPS=, read -ra workloads <<< "$YCSB_WORKLOADS"
+ fi
+
+ for l in "${workloads[@]}"; do
+ echo "=============== $l"
+ for r in $( seq 1 "$YCSB_RUNS" ); do
+ local res="$plogs/run${l}_${r}"
+ echo "---------------- ${l}: $r"
+ echo "tarantool.port=3301" >> "workloads/workload${l}"
+ maybe_under_numactl "${numaopts[@]}" -- \
+ ./bin/ycsb load tarantool -s -P
"workloads/workload${l}" > "${res}.load" 2>&1 || cat "${res}.load"
+ sync_disk
+ maybe_drop_cache
+
+ maybe_under_numactl "${numaopts[@]}" -- \
+ ./bin/ycsb run tarantool -s -P "workloads/workload${l}"
> "${res}.log" 2>&1 || cat "${res}.log"
+
+ grep Thro "${res}.log" | awk '{ print "Overall result: "$3
}' | tee "${res}.txt"
+ sed "s#Overall result#$l $r#g" "${res}.txt" >>
"${plogs}/ycsb.${mode}_result.txt"
+
+ stop_and_clean_tarantool
+ done
+ done
+}
+
+types=(hash tree)
+
+if [ -n "$YCSB_TYPES" -a "$YCSB_TYPES" != 'all' ]; then
+ IPS=, read -ra types <<< "$YCSB_TYPES"
+fi
+
+numaopts=(--membind=1 --cpunodebind=1 '--physcpubind=6,7,8,9,10,11')
+
+for f in workloads/workload[a-f] ; do
+ sed "s#recordcount=.*#recordcount=$YCSB_RECORDCOUNT#g" -i "$f"
+ sed "s#operationcount=.*#operationcount=$YCSB_OPERATIONCOUNT#g" -i "$f"
+ echo "tarantool.port=3301" >> "$f"
+done
+
+
+plogs=results
+rm -rf "$plogs"
+mkdir "$plogs"
+
+for t in "${types[@]}"; do
+ run_ycsb "$t"
+done
+
+echo "${TAR_VER}" | tee "ycsb.${mode}_t_version.txt"
+cp -f "${plogs}/ycsb.${mode}_result.txt" .
+
+echo "Tarantool TAG:"
+cat "ycsb.${mode}_t_version.txt"
+echo "Overall results:"
+echo "================"
+cat "ycsb.${mode}_result.txt"
diff --git a/config.sh b/config.sh
new file mode 100644
index 0000000..93d3fc5
--- /dev/null
+++ b/config.sh
@@ -0,0 +1,41 @@
+# path to tarantool executable
+export TARANTOOL_EXECUTABLE=tarantool
+
+# location of tests to be run
+export BENCH_WORKDIR="$PWD/.benchdir"
+
+# cbench parameters
+export CBENCH_DIR="$HOME/work/cbench/"
+export CBENCH_VINYL_WORKLOAD=50
+export CBENCH_MEMTX_WORKLOAD=10000
+
+# linkbench parameters
+export LINKBENCH_DIR="$HOME/work/linkbench/"
+export LINKBENCH_REQUESTERS=1
+export LINKBENCH_REQUESTS=1000
+export LINKBENCH_WORKLOAD_SIZE=1000
+
+# linkbench parameters
+export NOSQLBENCH_DIR="$HOME/work/nosqlbench/"
+
+# linkbench parameters
+export SYSBENCH_DIR="$HOME/work/sysbench/"
+export SYSBENCH_RUNS=1
+export SYSBENCH_TIME=5
+
+# tpcc parameters
+export TPCC_DIR="$HOME/work/tpcc/"
+export TPCC_TIME=20
+export TPCC_WARMUPTIME=5
+export TPCC_WAREHOUSES=15
+export TPCC_FROMSNAPSHOT="/tmp/00000000000000033515.snap"
+
+# tpcc parameters
+export TPCH_DIR="$HOME/work/tpch/"
+export TPCH_SKIP_SQLITE=1
+
+# tpcc parameters
+export YCSB_DIR="$HOME/work/ycsb/"
+export YCSB_OPERATIONCOUNT=1000
+export YCSB_RECORDCOUNT=1000
+export YCSB_RUNS=1
diff --git a/ttbench b/ttbench
new file mode 100755
index 0000000..ac656f5
--- /dev/null
+++ b/ttbench
@@ -0,0 +1,149 @@
+#!/bin/bash
+
+set -euo pipefail
+
+TTBENCH_CONFIG=config.sh
+TTBENCH_BENCHES=all
+TTBENCH_COMMAND=
+TTBENCH_BENCHDIR=benches
+
+function error {
+ echo "ERROR:" "$@"
+ exit 100
+}
+
+function print_help {
+ grep '^\s*# HELP' "$0" | sed 's/^\s*# HELP\s//g'
+}
+
+function list_benchmarks {
+ for d in $TTBENCH_BENCHDIR/*; do
+ sed "s#$TTBENCH_BENCHDIR/##" <<< "$d"
+ done | grep -v common.sh
+}
+
+function run_bench {
+ local bench="$1"
+ local benchdir=
+ local workdir="${BENCH_WORKDIR:?}/$bench/"
+
+ cp "$TTBENCH_BENCHDIR/common.sh" "${BENCH_WORKDIR}"
+
+ export COMMON_FUNCTIONS="$BENCH_WORKDIR/common.sh"
+
+ if benchdir=$(env | grep "${bench^^}_DIR" | sed 's/^.*=//'); then
+ echo "Running $bench with workdir $benchdir" 1>&2
+ rm -rf "$workdir"
+ mkdir -p "$workdir"
+
+ cp -r "$TTBENCH_BENCHDIR/$bench"/* "$workdir"
+ cp -r "$benchdir"/* "$workdir"
+
+ local runner=
+ runner=$(readlink -f "$TTBENCH_BENCHDIR/$bench/run.sh")
+ pushd "$workdir" 1>/dev/null
+ "$runner"
+ popd 1>/dev/null
+ else
+ error "variable '${bench^^}_DIR' is not set in config, can't
run benchmark"
+ fi
+}
+
+function run_benchmarks {
+ # must be set in config
+ [ -z "$BENCH_WORKDIR" ] && error "BENCH_WORKDIR is not set"
+
+ if [ "$TTBENCH_BENCHES" == all ]; then
+ for b in $(list_benchmarks); do
+ run_bench "$b"
+ done
+ else
+ IFS=, read -ra benchlist <<< "$TTBENCH_BENCHES"
+ # local benchlist=( $(sed 's/,/ /g' <<< "$TTBENCH_BENCHES") );
+ for b in "${benchlist[@]}"; do
+ if [ ! -d "$TTBENCH_BENCHDIR/$b" ]; then
+ error "Unknown benchmark='$b', please run '$0
list-benchmarks'"
+ fi
+ done
+
+ for b in "${benchlist[@]}"; do
+ run_bench "$b"
+ done
+ fi
+}
+
+# HELP ttbench - launch tarantool external benchmarks
+# HELP
+# HELP Usage: ttbench [OPTIONS]... COMMAND
+# HELP
+# HELP Available options are:
+while [[ $# -gt 0 ]]; do
+ option="$1"
+
+ case $option in
+ # HELP -c, --config 'config_file'
+ # HELP use given config file (defaults to config.sh)
+ # HELP
+ -c|--config)
+ TTBENCH_CONFIG="${2:-}"
+ [ -z "$TTBENCH_CONFIG" ] && error "$option param can not be
empty"
+ shift 2;;
+
+ # HELP -d, --debug
+ # HELP sets -x flag on whole script
+ # HELP
+ -d|--debug)
+ set -x
+ shift;;
+
+ # HELP -b, --benchmarks 'bench1,bench2'
+ # HELP comma separated list of benchmarks (defauts to
'all')
+ # HELP
+ -b|--benchmarks)
+ TTBENCH_BENCHES="${2:-}"
+ [ -z "$TTBENCH_BENCHES" ] && error "$option param can not
be empty"
+ shift 2;;
+
+ # HELP -h, --help
+ # HELP print help message
+ # HELP
+ -h|--help)
+ print_help
+ exit 0;;
+
+ *)
+ break;;
+ esac
+done
+
+[ -f "$TTBENCH_CONFIG" ] && source "$TTBENCH_CONFIG"
+
+TTBENCH_COMMAND="${1:-}"
+
+if [ -z "$TTBENCH_COMMAND" ]; then
+ print_help
+ echo "ERROR: no COMMAND specified" 1>&2
+ exit 1
+fi
+
+shift
+
+# HELP
+# HELP Available commands are:
+case "$TTBENCH_COMMAND" in
+ # HELP ls, list-benchmarks
+ # HELP lists all known benchmarks
+ # HELP
+ ls|list-benchmarks)
+ list_benchmarks;;
+
+ # HELP run
+ # HELP run benchmarks specified by --benchmarks flag
+ run)
+ run_benchmarks;;
+
+ *)
+ print_help
+ echo "ERROR: Unknown COMMAND='$TTBENCH_COMMAND'" 1>&2
+ exit 1;;
+esac
--
2.7.4
More information about the Tarantool-patches
mailing list