[Tarantool-patches] [PATCH v2] bench-run: submit perf results into database

[Tarantool-patches] [PATCH v2] bench-run: submit perf results into database

[Oleg Piskunov]

[-- Attachment #1: Type: text/plain, Size: 19770 bytes --]



Resurrect bench.tarantool.org:
- submit perf results into bench.tarantool.org database
- refactoring bench running scripts
 
Fix docker files for perf testing:
- adding python "requests" module needed for publish script.
 
Closes #4865
 
Github: https://github.com/tarantool/bench-run/tree/opiskunov/gh-4865-submit-perf-results
Issue: https://github.com/tarantool/tarantool/issues/4865
---
 benchs/cbench/run.sh          | 60 +++++++++++++++++++++++++++++-----------
 benchs/linkbench/run.sh       | 26 ++++++++++++------
 benchs/nosqlbench/run.sh      | 46 +++++++++++++------------------
 benchs/publication/publish.py | 64 +++++++++++++++++++++++++++++++++++++++++++
 benchs/sysbench/run.sh        | 52 ++++++++++++++---------------------
 benchs/tpcc/run.sh            | 38 ++++++++++++++-----------
 benchs/ycsb/run.sh            | 42 ++++++++++++++++------------
 dockerfiles/ubuntu_benchs     |  3 ++
 8 files changed, 216 insertions(+), 115 deletions(-)
 create mode 100755 benchs/publication/publish.py
 
diff --git a/benchs/cbench/run.sh b/benchs/cbench/run.sh
index 4d07972..77b88e7 100755
--- a/benchs/cbench/run.sh
+++ b/benchs/cbench/run.sh
@@ -1,36 +1,64 @@
 #!/usr/bin/env bash
 
-kill `pidof tarantool`
-set -e
+set -eu
 set -o pipefail
 
 TAR_VER=$(tarantool -v | grep -e "Tarantool" |  grep -oP '\s\K\S*')
-echo ${TAR_VER} | tee cbench_t_version.txt
+numaopts="numactl --membind=1 --cpunodebind=1 --physcpubind=11"
+cbench_opts=500
 
 killall tarantool 2>/dev/null || true
 rm -rf 5* 0*
-sync
+sync && echo "sync passed" || echo "sync failed with error" $?
 echo 3 > /proc/sys/vm/drop_caches
-numactl --membind=1 --cpunodebind=1 --physcpubind=9 tarantool /opt/cbench/cbench_runner.lua memtx 2>&1 | tee cbench_output_memtx.txt
+$numaopts tarantool /opt/cbench/cbench_runner.lua memtx 2>&1 | tee cbench_output_memtx.txt
 
 killall tarantool 2>/dev/null || true
 rm -rf 5* 0*
-sync
+sync && echo "sync passed" || echo "sync failed with error" $?
 echo 3 > /proc/sys/vm/drop_caches
-numactl --membind=1 --cpunodebind=1 --physcpubind=10 tarantool /opt/cbench/cbench_runner.lua vinyl fsync 500 2>&1 | tee cbench_output_vinyl_fsync.txt
+$numaopts tarantool /opt/cbench/cbench_runner.lua vinyl fsync $cbench_opts 2>&1 | tee cbench_output_vinyl_fsync.txt
 
 killall tarantool 2>/dev/null || true
 rm -rf 5* 0*
-sync
+sync && echo "sync passed" || echo "sync failed with error" $?
 echo 3 > /proc/sys/vm/drop_caches
-numactl --membind=1 --cpunodebind=1 --physcpubind=11 tarantool /opt/cbench/cbench_runner.lua vinyl write 500 2>&1 | tee cbench_output_memtx_write.txt
-
-grep "^?tab" cbench_output_memtx.txt | sed "s/.*name=//"| sed "s/&param=/:/"| sed "s/cb\./cb\.memtx\./"| tee -a cbench_result.txt
-grep "^?tab" cbench_output_vinyl_fsync.txt | sed "s/.*name=//"| sed "s/&param=/:/"| sed "s/cb\./cb\.vinyl\.fsync\./"| tee -a cbench_result.txt
-grep "^?tab" cbench_output_memtx_write.txt | sed "s/.*name=//"| sed "s/&param=/:/"| sed "s/cb\./cb\.vinyl\.write\./"| tee -a cbench_result.txt
+$numaopts tarantool /opt/cbench/cbench_runner.lua vinyl write $cbench_opts 2>&1 | tee cbench_output_vinyl_write.txt
 
-#mv tarantool-server.log cbench_tarantool_server.log
+grep "^?tab=cbench.tree" cbench_output_memtx.txt | \
+  sed "s/.*name=//"| sed "s/&param=/:/"| sed "s/cb\./cb\.memtx\./"| \
+  tee -a cbench-memtx-tree_result.txt
+grep "^?tab=cbench.hash" cbench_output_memtx.txt | \
+  sed "s/.*name=//"| sed "s/&param=/:/"| sed "s/cb\./cb\.memtx\./"| \
+  tee -a cbench-memtx-hash_result.txt
+grep "^?tab" cbench_output_vinyl_fsync.txt | \
+  sed "s/.*name=//"| sed "s/&param=/:/"| sed "s/cb\./cb\.vinyl\.fsync\./"| \
+  tee -a cbench-vinyl-fsync_result.txt
+grep "^?tab" cbench_output_vinyl_write.txt | \
+  sed "s/.*name=//"| sed "s/&param=/:/"| sed "s/cb\./cb\.vinyl\.write\./"| \
+  tee -a cbench-vinyl-write_result.txt
 
-echo "RESULTS:"
-cat cbench_result.txt
+echo ${TAR_VER} | tee cbench-memtx-tree_t_version.txt
+echo ${TAR_VER} | tee cbench-memtx-hash_t_version.txt
+echo ${TAR_VER} | tee cbench-vinyl-fsync_t_version.txt
+echo ${TAR_VER} | tee cbench-vinyl-write_t_version.txt
+echo ${TAR_VER} | tee cbench_t_version.txt
 
+echo "Tarantool TAG:"
+cat cbench_t_version.txt
+echo "Overall results:"
+echo "================"
+echo "RESULTS (cbench-memtx-tree_result.txt):"
+cat cbench-memtx-tree_result.txt
+echo " "
+echo "RESULTS (cbench-memtx-hash_result.txt):"
+cat cbench-memtx-hash_result.txt
+echo " "
+echo "RESULTS (cbench-vinyl-fsync_result.txt):"
+cat cbench-vinyl-fsync_result.txt
+echo " "
+echo "RESULTS (cbench-vinyl-write_result.txt):"
+cat cbench-vinyl-write_result.txt
+echo " "
+echo "Publish data to bench database"
+/opt/bench-run/benchs/publication/publish.py
diff --git a/benchs/linkbench/run.sh b/benchs/linkbench/run.sh
index 6738436..aac7d3c 100755
--- a/benchs/linkbench/run.sh
+++ b/benchs/linkbench/run.sh
@@ -1,19 +1,16 @@
 #!/usr/bin/env bash
 
-kill `pidof tarantool`
-
-set -e
+set -eu
 set -o pipefail
 
 TAR_VER=$(tarantool -v | grep -e "Tarantool" |  grep -oP '\s\K\S*')
-echo ${TAR_VER} | tee linkbench_t_version.txt
+numaopts="numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11"
+base_dir=$(pwd)
 
 cd /opt/linkbench && mvn clean package -Dmaven.test.skip=true
 cd src/tarantool && make
 cd -
 
-numaopts="numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11"
-
 # use always newly created path specialy mounted if needed
 wpath=/builds/ws
 rm -rf $wpath
@@ -30,8 +27,21 @@ sed "s/^maxid1 = .*/maxid1 = 5000000/g" -i /opt/linkbench/config/FBWorkload.prop
 sed "s/^requesters = .*/requesters = 1/g" -i $cfgfile
 sed "s/^requests = .*/requests = 2000000/g" -i $cfgfile
 $numaopts /opt/linkbench/bin/linkbench -c $cfgfile -l 2>&1 | tee loading.res.txt
-sync
+
+sync && echo "sync passed" || echo "sync failed with error" $?
 echo 3 > /proc/sys/vm/drop_caches
 $numaopts /opt/linkbench/bin/linkbench -c $cfgfile -r 2>&1 | tee linkbench_output.txt
 
-grep "REQUEST PHASE COMPLETED" linkbench_output.txt | sed "s/.*second = /linkbench:/" | tee -a linkbench_result.txt
+grep "REQUEST PHASE COMPLETED" linkbench_output.txt | sed "s/.*second = /linkbench:/" | tee -a linkbench.ssd_result.txt
+echo ${TAR_VER} | tee linkbench.ssd_t_version.txt
+cp -f linkbench.ssd_t_version.txt $base_dir
+cp -f linkbench.ssd_result.txt $base_dir
+
+echo "Tarantool TAG:"
+cat linkbench.ssd_t_version.txt
+echo "Overall results:"
+echo "================"
+cat linkbench.ssd_result.txt
+echo " "
+echo "Publish data to bench database"
+/opt/bench-run/benchs/publication/publish.py
diff --git a/benchs/nosqlbench/run.sh b/benchs/nosqlbench/run.sh
index c9491f5..157f372 100755
--- a/benchs/nosqlbench/run.sh
+++ b/benchs/nosqlbench/run.sh
@@ -1,34 +1,22 @@
 #!/usr/bin/env bash
 
+set -eu
+set -o pipefail
+
 type=$1
 if [ "$type" == "" ]; then
     type=hash
 fi
 
 TAR_VER=$(tarantool -v | grep -e "Tarantool" |  grep -oP '\s\K\S*')
-echo ${TAR_VER} | tee nosqlbench_t_version.txt
-
-kill `pidof tarantool`
-
-#set -e
-#
-#set -o pipefail
+numaopts="numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11"
 
 killall tarantool 2>/dev/null || true
-rm -rf 0*.xlog 0*.snap
-sync
+sync && echo "sync passed" || echo "sync failed with error" $?
 echo 3 > /proc/sys/vm/drop_caches
 
-#cd /opt/nosqlbench/
-#cmake . &&  make -j
-#cd -
-#
-#tarantool -v
-#
-free -h
-numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11 tarantool `dirname $0`/tnt_${type}.lua 2>&1 &
+$numaopts tarantool `dirname $0`/tnt_${type}.lua 2>&1 &
 sed  "s/port 3303/port 3301/" /opt/nosqlbench/src/nosqlbench.conf -i
-#sed  "s/request_count 4000000/request_count 400000/" /opt/nosqlbench/src/nosqlbench.conf -i
 sed  "s/benchmark 'no_limit'/benchmark 'time_limit'/" /opt/nosqlbench/src/nosqlbench.conf -i
 sed  "s/time_limit 10/time_limit 2000/" /opt/nosqlbench/src/nosqlbench.conf -i
 sed  "s/request_batch_count 1/request_batch_count 10/" /opt/nosqlbench/src/nosqlbench.conf -i
@@ -36,14 +24,18 @@ sed  "s/rps 12000/rps 20000/" /opt/nosqlbench/src/nosqlbench.conf -i
 sleep 5
 echo "Run NB"
 # WARNING: don't try to save output from stderr - file will use the whole disk space !
-numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11 /opt/nosqlbench/src/nb /opt/nosqlbench/src/nosqlbench.conf | \
+$numaopts  /opt/nosqlbench/src/nb /opt/nosqlbench/src/nosqlbench.conf | \
     grep -v "Warmup" | grep -v "Failed to allocate" >nosqlbench_output.txt || cat nosqlbench_output.txt
 
-#echo "Latest 1000 lines:"
-#tail -1000 nosqlbench_output.txt
-
-echo "Getting results"
-grep "TOTAL RPS STATISTICS:" nosqlbench_output.txt -A6 | awk -F "|" 'NR > 4 {print $2,":", $4}' > nosqlbench_result.txt
-
-cat nosqlbench_result.txt
-
+grep "TOTAL RPS STATISTICS:" nosqlbench_output.txt -A6 | \
+  awk -F "|" 'NR > 4 {print $2,":", $4}' > noSQLbench.${type}_result.txt
+echo ${TAR_VER} | tee noSQLbench.${type}_t_version.txt
+
+echo "Tarantool TAG:"
+cat noSQLbench.${type}_t_version.txt
+echo "Overall results:"
+echo "================"
+cat noSQLbench.${type}_result.txt
+echo " "
+echo "Publish data to bench database"
+/opt/bench-run/benchs/publication/publish.py
diff --git a/benchs/publication/publish.py b/benchs/publication/publish.py
new file mode 100755
index 0000000..42bc53a
--- /dev/null
+++ b/benchs/publication/publish.py
@@ -0,0 +1,64 @@
+#!/usr/bin/env python
+import fnmatch
+import os
+from urllib import urlencode
+import requests
+
+
+def parse_bench(filename):
+    with open(filename) as raw_data:
+        return raw_data.readlines()
+
+
+def get_version(filename):
+    with open(filename) as raw_data:
+        version = raw_data.readlines()[-1]
+        return version.split()[0]
+
+
+def push_to_microb(server, token, name, value, version, tab):
+    uri = 'http://%s/push?%s' % (server, urlencode(dict(
+        key=token, name=name, param=value,
+        v=version, unit='trps', tab=tab
+    )))
+
+    r = requests.get(uri)
+    if r.status_code == 200:
+        print('Export complete')
+    else:
+        print('Export error http: %d' % r.status_code)
+        print('Export error text: %d' % r.text)
+
+
+def main():
+    if "MICROB_WEB_TOKEN" in os.environ and "MICROB_WEB_HOST" in os.environ:
+        bench = {}
+        res = []
+        current_data = {}
+        version = ''
+        for file in os.listdir('.'):
+            if fnmatch.fnmatch(file, '*_result.txt'):
+                values = parse_bench(file)
+                benchmark = file.split('_')[0]
+                version = get_version('{}_t_version.txt'.format(benchmark))
+                for value in values:
+                    test_name = value.split(':')[0]
+                    test_res = float(value.split(':')[1])
+                    res.append(test_res)
+                    push_to_microb(
+                        os.environ['MICROB_WEB_HOST'],
+                        os.environ['MICROB_WEB_TOKEN'],
+                        test_name,
+                        test_res,
+                        version,
+                        benchmark,
+                    )
+        print ("VERSION - ", version)
+    else:
+        print("MICROB params not specified")
+
+    return 0
+
+
+if __name__ == '__main__':
+    main()
diff --git a/benchs/sysbench/run.sh b/benchs/sysbench/run.sh
index 6a69d14..a9cf2f3 100755
--- a/benchs/sysbench/run.sh
+++ b/benchs/sysbench/run.sh
@@ -1,18 +1,12 @@
 #!/usr/bin/env bash
 
-runs=10
-if [ "$1" != "" ]; then
-    runs=$1
-fi
-
-kill `pidof tarantool`
-
-set -e
+set -eu
 set -o pipefail
 
+runs=10
+
 TAR_VER=$(tarantool -v | grep -e "Tarantool" |  grep -oP '\s\K\S*')
-echo ${TAR_VER} | tee sysbench_t_version.txt
-set -e
+numaconf="numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11"
 
 ARRAY_TESTS=(
     "oltp_read_only"
@@ -28,27 +22,17 @@ ARRAY_TESTS=(
 #    "bulk_insert"
 )
 
-if [ -n "${TEST}" ]; then ARRAY_TESTS=("${TEST}"); fi
-
-
-if [ ! -n "${WARMUP_TIME}" ]; then WARMUP_TIME=5; fi
-if [ ! -n "${TIME}" ]; then TIME=20; fi
-if [ ! -n "${DBMS}" ]; then DBMS="tarantool"; fi
-if [ ! -n "${THREADS}" ]; then THREADS=200; fi
-
-if [ -n "${USER}" ]; then USER=--${DBMS}-user=${USER}; fi
-if [ -n "${PASSWORD}" ]; then PASSWORD=--${DBMS}-password=${PASSWORD}; fi
-
-numaconf="--membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11"
+WARMUP_TIME=5
+TIME=20
+DBMS="tarantool"
+THREADS=200
 opts="--db-driver=${DBMS} --threads=${THREADS}"
 
 export LD_LIBRARY_PATH=/usr/local/lib
 
-rm -f sysbench_result.txt
-set -o pipefail
 for test in "${ARRAY_TESTS[@]}"; do
     res=0
-    tlog=sysbench_${test}_result.txt
+    tlog=sysbench_${test}_results.txt
     rm -f $tlog
     maxres=0
     for run in `eval echo {1..$runs}` ; do
@@ -58,10 +42,10 @@ for test in "${ARRAY_TESTS[@]}"; do
         sysbench $test $opts cleanup >sysbench_output.txt
         sysbench $test $opts prepare >>sysbench_output.txt
 
-        numactl $numaconf sysbench $test $opts \
+        $numaconf sysbench $test $opts \
             --time=${TIME} --warmup-time=${WARMUP_TIME} run >>sysbench_output.txt
 
-        numactl $numaconf sysbench $test $opts cleanup >>sysbench_output.txt
+        $numaconf sysbench $test $opts cleanup >>sysbench_output.txt
 
         cat sysbench_output.txt | grep -e 'transactions:' | grep -oP '\(\K\S*' | tee $tlog
         tres=`cat $tlog | sed 's#^.*:##g' | sed 's#\..*$##g'`
@@ -69,16 +53,22 @@ for test in "${ARRAY_TESTS[@]}"; do
         if [[ $tres -gt $maxres ]]; then maxres=$tres ; fi
     done
     res=$(($res/$runs))
-    echo "${test}: $res" >>sysbench_result.txt
+    echo "${test}: $res" >>Sysbench_result.txt
 
     echo "Subtest '$test' results:"
     echo "==============================="
     echo "Average result: $res"
     echo "Maximum result: $maxres"
-    printf "Diviations (AVG -> MAX): %.2f" `bc <<< "scale = 4; (1 - $res / $maxres) * 100"` ; echo %
+    printf "Deviations (AVG -> MAX): %.2f" `bc <<< "scale = 4; (1 - $res / $maxres) * 100"` ; echo %
 done
 
+echo ${TAR_VER} | tee Sysbench_t_version.txt
+
+echo "Tarantool TAG:"
+cat Sysbench_t_version.txt
 echo "Overall results:"
 echo "================"
-cat sysbench_result.txt
-
+cat Sysbench_result.txt
+echo " "
+echo "Publish data to bench database"
+/opt/bench-run/benchs/publication/publish.py
diff --git a/benchs/tpcc/run.sh b/benchs/tpcc/run.sh
index f1f60cc..fca7939 100755
--- a/benchs/tpcc/run.sh
+++ b/benchs/tpcc/run.sh
@@ -1,22 +1,20 @@
 #!/usr/bin/env bash
+set -eu
+set -o pipefail
 
 TAR_VER=$(tarantool -v | grep -e "Tarantool" |  grep -oP '\s\K\S*')
-echo ${TAR_VER} | tee tpcc_t_version.txt
-
-set -e
+numaopts="numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11"
+base_dir=$(pwd)
 
-if [ ! -n "${TIME}" ]; then TIME=1200; fi
-if [ ! -n "${WARMUP_TIME}" ]; then WARMUP_TIME=10; fi
-
-set -o pipefail
+TIME=1200
+WARMUP_TIME=10
 
 killall tarantool tpcc_load 2>/dev/null || true
-rm -rf 0*.xlog 0*.snap
-sync
+sync && echo "sync passed" || echo "sync failed with error" $?
 echo 3 > /proc/sys/vm/drop_caches
 
 sed 's#box.sql#box#g' -i /opt/tpcc/create_table.lua
-numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11 tarantool /opt/tpcc/create_table.lua &
+$numaopts tarantool /opt/tpcc/create_table.lua &
 sleep 5
 
 # Usage: tpcc_load -h server_host -P port -d database_name -u mysql_user
@@ -27,13 +25,21 @@ tpcc_opts="-h localhost -P 3301 -d tarantool -u root -p '' -w 15"
 cd /opt/tpcc
 . /opt/tpcc/load.sh tarantool 15
 
-numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11 \
-    /opt/tpcc/tpcc_start $tpcc_opts -r10 -l${TIME} -i${TIME} >tpcc_output.txt 2>/dev/null
-
-echo -n "tpcc:" | tee tpcc_result.txt
-cat tpcc_output.txt | grep -e '<TpmC>' | grep -oP '\K[0-9.]*' | tee -a tpcc_result.txt
+$numaopts /opt/tpcc/tpcc_start $tpcc_opts -r10 -l${TIME} -i${TIME} >tpcc_output.txt 2>/dev/null
 
+echo -n "tpcc:" | tee tpc.c_result.txt
+cat tpcc_output.txt | grep -e '<TpmC>' | grep -oP '\K[0-9.]*' | tee -a tpc.c_result.txt
 cat tpcc_output.txt
+
+echo ${TAR_VER} | tee tpc.c_t_version.txt
+cp -f tpc.c_t_version.txt $base_dir
+cp -f tpc.c_result.txt $base_dir
+
+echo "Tarantool TAG:"
+cat tpc.c_t_version.txt
 echo "Overall result:"
 echo "==============="
-cat tpcc_result.txt
+cat tpc.c_result.txt
+echo " "
+echo "Publish data to bench database"
+/opt/bench-run/benchs/publication/publish.py
diff --git a/benchs/ycsb/run.sh b/benchs/ycsb/run.sh
index 5ae1c75..e5219a1 100755
--- a/benchs/ycsb/run.sh
+++ b/benchs/ycsb/run.sh
@@ -1,23 +1,18 @@
 #!/usr/bin/env bash
 
+set -eu
+set -o pipefail
+
 mode=$1
-runs=$2
+runs=1
 
 if [ "$mode" == "" ]; then
     mode=hash
 fi
 
-if [ "$runs" == "" ]; then
-    runs=1
-fi
-
-kill `pidof tarantool`
-
-set -e
-set -o pipefail
-
 TAR_VER=$(tarantool -v | grep -e "Tarantool" |  grep -oP '\s\K\S*')
-echo ${TAR_VER} | tee ycsb_t_version.txt
+numaopts="numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11"
+base_dir=$(pwd)
 
 ws=/opt/ycsb
 cd $ws
@@ -33,7 +28,7 @@ sed 's/listen=.*/listen=3301,\n   memtx_memory = 2000000000,/' -i $srvlua
 sed 's/logger_nonblock.*//' -i $srvlua
 sed 's/logger/log/' -i $srvlua
 sed 's/read,write,execute/create,read,write,execute/' -i $srvlua
-numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11 tarantool $srvlua 2>&1 &
+$numaopts tarantool $srvlua 2>&1 &
 
 sleep 5
 plogs=$ws/results
@@ -45,12 +40,25 @@ for l in a b c d e f ; do
         res=$plogs/run${l}_${r}
         echo ---------------- ${l}: $r
         echo "tarantool.port=3301" >> $ws/workloads/workload${l}
-        numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11 bin/ycsb load tarantool -s -P workloads/workload${l} >${res}.load 2>&1 || cat ${res}.load
-       sync
+        $numaopts bin/ycsb load tarantool -s -P workloads/workload${l} >${res}.load 2>&1 || cat ${res}.load
+       sync && echo "sync passed" || echo "sync failed with error" $?
        echo 3 > /proc/sys/vm/drop_caches
-        numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11 bin/ycsb run tarantool -s -P workloads/workload${l} >${res}.log 2>&1 || cat ${res}.log
+        $numaopts bin/ycsb run tarantool -s -P workloads/workload${l} >${res}.log 2>&1 || cat ${res}.log
         grep Thro ${res}.log | awk '{ print "Overall result: "$3 }' | tee ${res}.txt
-        sed "s#Overall result#$l $r#g" ${res}.txt >>${plogs}/results.txt
+        sed "s#Overall result#$l $r#g" ${res}.txt >>${plogs}/ycsb.${mode}_result.txt
     done
 done
-cat ${plogs}/results.txt
+
+echo ${TAR_VER} | tee ycsb.${mode}_t_version.txt
+cp -f ${plogs}/ycsb.${mode}_result.txt .
+cp -f ycsb.${mode}_t_version.txt $base_dir
+cp -f ycsb.${mode}_result.txt $base_dir
+
+echo "Tarantool TAG:"
+cat ycsb.${mode}_t_version.txt
+echo "Overall results:"
+echo "================"
+cat ycsb.${mode}_result.txt
+echo " "
+echo "Publish data to bench database"
+/opt/bench-run/benchs/publication/publish.py
diff --git a/dockerfiles/ubuntu_benchs b/dockerfiles/ubuntu_benchs
index fd9acb4..947f082 100644
--- a/dockerfiles/ubuntu_benchs
+++ b/dockerfiles/ubuntu_benchs
@@ -73,3 +73,6 @@ RUN luarocks install \
 
 # benchmarks runners
 RUN git clone https://github.com/tarantool/bench-run.git /opt/bench-run
+
+# adding python dependency
+RUN pip install requests
--
1.8.3.1
 

[-- Attachment #2: Type: text/html, Size: 24944 bytes --]

Re: [Tarantool-patches] [PATCH v2] bench-run: submit perf results into database

[Sergey Bronnikov]

LGTM

On 14:23 Wed 29 Apr , Oleg Piskunov wrote:
> 
> 
> Resurrect bench.tarantool.org:
> - submit perf results into bench.tarantool.org database
> - refactoring bench running scripts
>  
> Fix docker files for perf testing:
> - adding python "requests" module needed for publish script.
>  
> Closes #4865
>  
> Github: https://github.com/tarantool/bench-run/tree/opiskunov/gh-4865-submit-perf-results
> Issue: https://github.com/tarantool/tarantool/issues/4865
> ---
>  benchs/cbench/run.sh          | 60 +++++++++++++++++++++++++++++-----------
>  benchs/linkbench/run.sh       | 26 ++++++++++++------
>  benchs/nosqlbench/run.sh      | 46 +++++++++++++------------------
>  benchs/publication/publish.py | 64 +++++++++++++++++++++++++++++++++++++++++++
>  benchs/sysbench/run.sh        | 52 ++++++++++++++---------------------
>  benchs/tpcc/run.sh            | 38 ++++++++++++++-----------
>  benchs/ycsb/run.sh            | 42 ++++++++++++++++------------
>  dockerfiles/ubuntu_benchs     |  3 ++
>  8 files changed, 216 insertions(+), 115 deletions(-)
>  create mode 100755 benchs/publication/publish.py
>  
> diff --git a/benchs/cbench/run.sh b/benchs/cbench/run.sh
> index 4d07972..77b88e7 100755
> --- a/benchs/cbench/run.sh
> +++ b/benchs/cbench/run.sh
> @@ -1,36 +1,64 @@
>  #!/usr/bin/env bash
>  
> -kill `pidof tarantool`
> -set -e
> +set -eu
>  set -o pipefail
>  
>  TAR_VER=$(tarantool -v | grep -e "Tarantool" |  grep -oP '\s\K\S*')
> -echo ${TAR_VER} | tee cbench_t_version.txt
> +numaopts="numactl --membind=1 --cpunodebind=1 --physcpubind=11"
> +cbench_opts=500
>  
>  killall tarantool 2>/dev/null || true
>  rm -rf 5* 0*
> -sync
> +sync && echo "sync passed" || echo "sync failed with error" $?
>  echo 3 > /proc/sys/vm/drop_caches
> -numactl --membind=1 --cpunodebind=1 --physcpubind=9 tarantool /opt/cbench/cbench_runner.lua memtx 2>&1 | tee cbench_output_memtx.txt
> +$numaopts tarantool /opt/cbench/cbench_runner.lua memtx 2>&1 | tee cbench_output_memtx.txt
>  
>  killall tarantool 2>/dev/null || true
>  rm -rf 5* 0*
> -sync
> +sync && echo "sync passed" || echo "sync failed with error" $?
>  echo 3 > /proc/sys/vm/drop_caches
> -numactl --membind=1 --cpunodebind=1 --physcpubind=10 tarantool /opt/cbench/cbench_runner.lua vinyl fsync 500 2>&1 | tee cbench_output_vinyl_fsync.txt
> +$numaopts tarantool /opt/cbench/cbench_runner.lua vinyl fsync $cbench_opts 2>&1 | tee cbench_output_vinyl_fsync.txt
>  
>  killall tarantool 2>/dev/null || true
>  rm -rf 5* 0*
> -sync
> +sync && echo "sync passed" || echo "sync failed with error" $?
>  echo 3 > /proc/sys/vm/drop_caches
> -numactl --membind=1 --cpunodebind=1 --physcpubind=11 tarantool /opt/cbench/cbench_runner.lua vinyl write 500 2>&1 | tee cbench_output_memtx_write.txt
> -
> -grep "^?tab" cbench_output_memtx.txt | sed "s/.*name=//"| sed "s/&param=/:/"| sed "s/cb\./cb\.memtx\./"| tee -a cbench_result.txt
> -grep "^?tab" cbench_output_vinyl_fsync.txt | sed "s/.*name=//"| sed "s/&param=/:/"| sed "s/cb\./cb\.vinyl\.fsync\./"| tee -a cbench_result.txt
> -grep "^?tab" cbench_output_memtx_write.txt | sed "s/.*name=//"| sed "s/&param=/:/"| sed "s/cb\./cb\.vinyl\.write\./"| tee -a cbench_result.txt
> +$numaopts tarantool /opt/cbench/cbench_runner.lua vinyl write $cbench_opts 2>&1 | tee cbench_output_vinyl_write.txt
>  
> -#mv tarantool-server.log cbench_tarantool_server.log
> +grep "^?tab=cbench.tree" cbench_output_memtx.txt | \
> +  sed "s/.*name=//"| sed "s/&param=/:/"| sed "s/cb\./cb\.memtx\./"| \
> +  tee -a cbench-memtx-tree_result.txt
> +grep "^?tab=cbench.hash" cbench_output_memtx.txt | \
> +  sed "s/.*name=//"| sed "s/&param=/:/"| sed "s/cb\./cb\.memtx\./"| \
> +  tee -a cbench-memtx-hash_result.txt
> +grep "^?tab" cbench_output_vinyl_fsync.txt | \
> +  sed "s/.*name=//"| sed "s/&param=/:/"| sed "s/cb\./cb\.vinyl\.fsync\./"| \
> +  tee -a cbench-vinyl-fsync_result.txt
> +grep "^?tab" cbench_output_vinyl_write.txt | \
> +  sed "s/.*name=//"| sed "s/&param=/:/"| sed "s/cb\./cb\.vinyl\.write\./"| \
> +  tee -a cbench-vinyl-write_result.txt
>  
> -echo "RESULTS:"
> -cat cbench_result.txt
> +echo ${TAR_VER} | tee cbench-memtx-tree_t_version.txt
> +echo ${TAR_VER} | tee cbench-memtx-hash_t_version.txt
> +echo ${TAR_VER} | tee cbench-vinyl-fsync_t_version.txt
> +echo ${TAR_VER} | tee cbench-vinyl-write_t_version.txt
> +echo ${TAR_VER} | tee cbench_t_version.txt
>  
> +echo "Tarantool TAG:"
> +cat cbench_t_version.txt
> +echo "Overall results:"
> +echo "================"
> +echo "RESULTS (cbench-memtx-tree_result.txt):"
> +cat cbench-memtx-tree_result.txt
> +echo " "
> +echo "RESULTS (cbench-memtx-hash_result.txt):"
> +cat cbench-memtx-hash_result.txt
> +echo " "
> +echo "RESULTS (cbench-vinyl-fsync_result.txt):"
> +cat cbench-vinyl-fsync_result.txt
> +echo " "
> +echo "RESULTS (cbench-vinyl-write_result.txt):"
> +cat cbench-vinyl-write_result.txt
> +echo " "
> +echo "Publish data to bench database"
> +/opt/bench-run/benchs/publication/publish.py
> diff --git a/benchs/linkbench/run.sh b/benchs/linkbench/run.sh
> index 6738436..aac7d3c 100755
> --- a/benchs/linkbench/run.sh
> +++ b/benchs/linkbench/run.sh
> @@ -1,19 +1,16 @@
>  #!/usr/bin/env bash
>  
> -kill `pidof tarantool`
> -
> -set -e
> +set -eu
>  set -o pipefail
>  
>  TAR_VER=$(tarantool -v | grep -e "Tarantool" |  grep -oP '\s\K\S*')
> -echo ${TAR_VER} | tee linkbench_t_version.txt
> +numaopts="numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11"
> +base_dir=$(pwd)
>  
>  cd /opt/linkbench && mvn clean package -Dmaven.test.skip=true
>  cd src/tarantool && make
>  cd -
>  
> -numaopts="numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11"
> -
>  # use always newly created path specialy mounted if needed
>  wpath=/builds/ws
>  rm -rf $wpath
> @@ -30,8 +27,21 @@ sed "s/^maxid1 = .*/maxid1 = 5000000/g" -i /opt/linkbench/config/FBWorkload.prop
>  sed "s/^requesters = .*/requesters = 1/g" -i $cfgfile
>  sed "s/^requests = .*/requests = 2000000/g" -i $cfgfile
>  $numaopts /opt/linkbench/bin/linkbench -c $cfgfile -l 2>&1 | tee loading.res.txt
> -sync
> +
> +sync && echo "sync passed" || echo "sync failed with error" $?
>  echo 3 > /proc/sys/vm/drop_caches
>  $numaopts /opt/linkbench/bin/linkbench -c $cfgfile -r 2>&1 | tee linkbench_output.txt
>  
> -grep "REQUEST PHASE COMPLETED" linkbench_output.txt | sed "s/.*second = /linkbench:/" | tee -a linkbench_result.txt
> +grep "REQUEST PHASE COMPLETED" linkbench_output.txt | sed "s/.*second = /linkbench:/" | tee -a linkbench.ssd_result.txt
> +echo ${TAR_VER} | tee linkbench.ssd_t_version.txt
> +cp -f linkbench.ssd_t_version.txt $base_dir
> +cp -f linkbench.ssd_result.txt $base_dir
> +
> +echo "Tarantool TAG:"
> +cat linkbench.ssd_t_version.txt
> +echo "Overall results:"
> +echo "================"
> +cat linkbench.ssd_result.txt
> +echo " "
> +echo "Publish data to bench database"
> +/opt/bench-run/benchs/publication/publish.py
> diff --git a/benchs/nosqlbench/run.sh b/benchs/nosqlbench/run.sh
> index c9491f5..157f372 100755
> --- a/benchs/nosqlbench/run.sh
> +++ b/benchs/nosqlbench/run.sh
> @@ -1,34 +1,22 @@
>  #!/usr/bin/env bash
>  
> +set -eu
> +set -o pipefail
> +
>  type=$1
>  if [ "$type" == "" ]; then
>      type=hash
>  fi
>  
>  TAR_VER=$(tarantool -v | grep -e "Tarantool" |  grep -oP '\s\K\S*')
> -echo ${TAR_VER} | tee nosqlbench_t_version.txt
> -
> -kill `pidof tarantool`
> -
> -#set -e
> -#
> -#set -o pipefail
> +numaopts="numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11"
>  
>  killall tarantool 2>/dev/null || true
> -rm -rf 0*.xlog 0*.snap
> -sync
> +sync && echo "sync passed" || echo "sync failed with error" $?
>  echo 3 > /proc/sys/vm/drop_caches
>  
> -#cd /opt/nosqlbench/
> -#cmake . &&  make -j
> -#cd -
> -#
> -#tarantool -v
> -#
> -free -h
> -numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11 tarantool `dirname $0`/tnt_${type}.lua 2>&1 &
> +$numaopts tarantool `dirname $0`/tnt_${type}.lua 2>&1 &
>  sed  "s/port 3303/port 3301/" /opt/nosqlbench/src/nosqlbench.conf -i
> -#sed  "s/request_count 4000000/request_count 400000/" /opt/nosqlbench/src/nosqlbench.conf -i
>  sed  "s/benchmark 'no_limit'/benchmark 'time_limit'/" /opt/nosqlbench/src/nosqlbench.conf -i
>  sed  "s/time_limit 10/time_limit 2000/" /opt/nosqlbench/src/nosqlbench.conf -i
>  sed  "s/request_batch_count 1/request_batch_count 10/" /opt/nosqlbench/src/nosqlbench.conf -i
> @@ -36,14 +24,18 @@ sed  "s/rps 12000/rps 20000/" /opt/nosqlbench/src/nosqlbench.conf -i
>  sleep 5
>  echo "Run NB"
>  # WARNING: don't try to save output from stderr - file will use the whole disk space !
> -numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11 /opt/nosqlbench/src/nb /opt/nosqlbench/src/nosqlbench.conf | \
> +$numaopts  /opt/nosqlbench/src/nb /opt/nosqlbench/src/nosqlbench.conf | \
>      grep -v "Warmup" | grep -v "Failed to allocate" >nosqlbench_output.txt || cat nosqlbench_output.txt
>  
> -#echo "Latest 1000 lines:"
> -#tail -1000 nosqlbench_output.txt
> -
> -echo "Getting results"
> -grep "TOTAL RPS STATISTICS:" nosqlbench_output.txt -A6 | awk -F "|" 'NR > 4 {print $2,":", $4}' > nosqlbench_result.txt
> -
> -cat nosqlbench_result.txt
> -
> +grep "TOTAL RPS STATISTICS:" nosqlbench_output.txt -A6 | \
> +  awk -F "|" 'NR > 4 {print $2,":", $4}' > noSQLbench.${type}_result.txt
> +echo ${TAR_VER} | tee noSQLbench.${type}_t_version.txt
> +
> +echo "Tarantool TAG:"
> +cat noSQLbench.${type}_t_version.txt
> +echo "Overall results:"
> +echo "================"
> +cat noSQLbench.${type}_result.txt
> +echo " "
> +echo "Publish data to bench database"
> +/opt/bench-run/benchs/publication/publish.py
> diff --git a/benchs/publication/publish.py b/benchs/publication/publish.py
> new file mode 100755
> index 0000000..42bc53a
> --- /dev/null
> +++ b/benchs/publication/publish.py
> @@ -0,0 +1,64 @@
> +#!/usr/bin/env python
> +import fnmatch
> +import os
> +from urllib import urlencode
> +import requests
> +
> +
> +def parse_bench(filename):
> +    with open(filename) as raw_data:
> +        return raw_data.readlines()
> +
> +
> +def get_version(filename):
> +    with open(filename) as raw_data:
> +        version = raw_data.readlines()[-1]
> +        return version.split()[0]
> +
> +
> +def push_to_microb(server, token, name, value, version, tab):
> +    uri = 'http://%s/push?%s' % (server, urlencode(dict(
> +        key=token, name=name, param=value,
> +        v=version, unit='trps', tab=tab
> +    )))
> +
> +    r = requests.get(uri)
> +    if r.status_code == 200:
> +        print('Export complete')
> +    else:
> +        print('Export error http: %d' % r.status_code)
> +        print('Export error text: %d' % r.text)
> +
> +
> +def main():
> +    if "MICROB_WEB_TOKEN" in os.environ and "MICROB_WEB_HOST" in os.environ:
> +        bench = {}
> +        res = []
> +        current_data = {}
> +        version = ''
> +        for file in os.listdir('.'):
> +            if fnmatch.fnmatch(file, '*_result.txt'):
> +                values = parse_bench(file)
> +                benchmark = file.split('_')[0]
> +                version = get_version('{}_t_version.txt'.format(benchmark))
> +                for value in values:
> +                    test_name = value.split(':')[0]
> +                    test_res = float(value.split(':')[1])
> +                    res.append(test_res)
> +                    push_to_microb(
> +                        os.environ['MICROB_WEB_HOST'],
> +                        os.environ['MICROB_WEB_TOKEN'],
> +                        test_name,
> +                        test_res,
> +                        version,
> +                        benchmark,
> +                    )
> +        print ("VERSION - ", version)
> +    else:
> +        print("MICROB params not specified")
> +
> +    return 0
> +
> +
> +if __name__ == '__main__':
> +    main()
> diff --git a/benchs/sysbench/run.sh b/benchs/sysbench/run.sh
> index 6a69d14..a9cf2f3 100755
> --- a/benchs/sysbench/run.sh
> +++ b/benchs/sysbench/run.sh
> @@ -1,18 +1,12 @@
>  #!/usr/bin/env bash
>  
> -runs=10
> -if [ "$1" != "" ]; then
> -    runs=$1
> -fi
> -
> -kill `pidof tarantool`
> -
> -set -e
> +set -eu
>  set -o pipefail
>  
> +runs=10
> +
>  TAR_VER=$(tarantool -v | grep -e "Tarantool" |  grep -oP '\s\K\S*')
> -echo ${TAR_VER} | tee sysbench_t_version.txt
> -set -e
> +numaconf="numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11"
>  
>  ARRAY_TESTS=(
>      "oltp_read_only"
> @@ -28,27 +22,17 @@ ARRAY_TESTS=(
>  #    "bulk_insert"
>  )
>  
> -if [ -n "${TEST}" ]; then ARRAY_TESTS=("${TEST}"); fi
> -
> -
> -if [ ! -n "${WARMUP_TIME}" ]; then WARMUP_TIME=5; fi
> -if [ ! -n "${TIME}" ]; then TIME=20; fi
> -if [ ! -n "${DBMS}" ]; then DBMS="tarantool"; fi
> -if [ ! -n "${THREADS}" ]; then THREADS=200; fi
> -
> -if [ -n "${USER}" ]; then USER=--${DBMS}-user=${USER}; fi
> -if [ -n "${PASSWORD}" ]; then PASSWORD=--${DBMS}-password=${PASSWORD}; fi
> -
> -numaconf="--membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11"
> +WARMUP_TIME=5
> +TIME=20
> +DBMS="tarantool"
> +THREADS=200
>  opts="--db-driver=${DBMS} --threads=${THREADS}"
>  
>  export LD_LIBRARY_PATH=/usr/local/lib
>  
> -rm -f sysbench_result.txt
> -set -o pipefail
>  for test in "${ARRAY_TESTS[@]}"; do
>      res=0
> -    tlog=sysbench_${test}_result.txt
> +    tlog=sysbench_${test}_results.txt
>      rm -f $tlog
>      maxres=0
>      for run in `eval echo {1..$runs}` ; do
> @@ -58,10 +42,10 @@ for test in "${ARRAY_TESTS[@]}"; do
>          sysbench $test $opts cleanup >sysbench_output.txt
>          sysbench $test $opts prepare >>sysbench_output.txt
>  
> -        numactl $numaconf sysbench $test $opts \
> +        $numaconf sysbench $test $opts \
>              --time=${TIME} --warmup-time=${WARMUP_TIME} run >>sysbench_output.txt
>  
> -        numactl $numaconf sysbench $test $opts cleanup >>sysbench_output.txt
> +        $numaconf sysbench $test $opts cleanup >>sysbench_output.txt
>  
>          cat sysbench_output.txt | grep -e 'transactions:' | grep -oP '\(\K\S*' | tee $tlog
>          tres=`cat $tlog | sed 's#^.*:##g' | sed 's#\..*$##g'`
> @@ -69,16 +53,22 @@ for test in "${ARRAY_TESTS[@]}"; do
>          if [[ $tres -gt $maxres ]]; then maxres=$tres ; fi
>      done
>      res=$(($res/$runs))
> -    echo "${test}: $res" >>sysbench_result.txt
> +    echo "${test}: $res" >>Sysbench_result.txt
>  
>      echo "Subtest '$test' results:"
>      echo "==============================="
>      echo "Average result: $res"
>      echo "Maximum result: $maxres"
> -    printf "Diviations (AVG -> MAX): %.2f" `bc <<< "scale = 4; (1 - $res / $maxres) * 100"` ; echo %
> +    printf "Deviations (AVG -> MAX): %.2f" `bc <<< "scale = 4; (1 - $res / $maxres) * 100"` ; echo %
>  done
>  
> +echo ${TAR_VER} | tee Sysbench_t_version.txt
> +
> +echo "Tarantool TAG:"
> +cat Sysbench_t_version.txt
>  echo "Overall results:"
>  echo "================"
> -cat sysbench_result.txt
> -
> +cat Sysbench_result.txt
> +echo " "
> +echo "Publish data to bench database"
> +/opt/bench-run/benchs/publication/publish.py
> diff --git a/benchs/tpcc/run.sh b/benchs/tpcc/run.sh
> index f1f60cc..fca7939 100755
> --- a/benchs/tpcc/run.sh
> +++ b/benchs/tpcc/run.sh
> @@ -1,22 +1,20 @@
>  #!/usr/bin/env bash
> +set -eu
> +set -o pipefail
>  
>  TAR_VER=$(tarantool -v | grep -e "Tarantool" |  grep -oP '\s\K\S*')
> -echo ${TAR_VER} | tee tpcc_t_version.txt
> -
> -set -e
> +numaopts="numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11"
> +base_dir=$(pwd)
>  
> -if [ ! -n "${TIME}" ]; then TIME=1200; fi
> -if [ ! -n "${WARMUP_TIME}" ]; then WARMUP_TIME=10; fi
> -
> -set -o pipefail
> +TIME=1200
> +WARMUP_TIME=10
>  
>  killall tarantool tpcc_load 2>/dev/null || true
> -rm -rf 0*.xlog 0*.snap
> -sync
> +sync && echo "sync passed" || echo "sync failed with error" $?
>  echo 3 > /proc/sys/vm/drop_caches
>  
>  sed 's#box.sql#box#g' -i /opt/tpcc/create_table.lua
> -numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11 tarantool /opt/tpcc/create_table.lua &
> +$numaopts tarantool /opt/tpcc/create_table.lua &
>  sleep 5
>  
>  # Usage: tpcc_load -h server_host -P port -d database_name -u mysql_user
> @@ -27,13 +25,21 @@ tpcc_opts="-h localhost -P 3301 -d tarantool -u root -p '' -w 15"
>  cd /opt/tpcc
>  . /opt/tpcc/load.sh tarantool 15
>  
> -numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11 \
> -    /opt/tpcc/tpcc_start $tpcc_opts -r10 -l${TIME} -i${TIME} >tpcc_output.txt 2>/dev/null
> -
> -echo -n "tpcc:" | tee tpcc_result.txt
> -cat tpcc_output.txt | grep -e '<TpmC>' | grep -oP '\K[0-9.]*' | tee -a tpcc_result.txt
> +$numaopts /opt/tpcc/tpcc_start $tpcc_opts -r10 -l${TIME} -i${TIME} >tpcc_output.txt 2>/dev/null
>  
> +echo -n "tpcc:" | tee tpc.c_result.txt
> +cat tpcc_output.txt | grep -e '<TpmC>' | grep -oP '\K[0-9.]*' | tee -a tpc.c_result.txt
>  cat tpcc_output.txt
> +
> +echo ${TAR_VER} | tee tpc.c_t_version.txt
> +cp -f tpc.c_t_version.txt $base_dir
> +cp -f tpc.c_result.txt $base_dir
> +
> +echo "Tarantool TAG:"
> +cat tpc.c_t_version.txt
>  echo "Overall result:"
>  echo "==============="
> -cat tpcc_result.txt
> +cat tpc.c_result.txt
> +echo " "
> +echo "Publish data to bench database"
> +/opt/bench-run/benchs/publication/publish.py
> diff --git a/benchs/ycsb/run.sh b/benchs/ycsb/run.sh
> index 5ae1c75..e5219a1 100755
> --- a/benchs/ycsb/run.sh
> +++ b/benchs/ycsb/run.sh
> @@ -1,23 +1,18 @@
>  #!/usr/bin/env bash
>  
> +set -eu
> +set -o pipefail
> +
>  mode=$1
> -runs=$2
> +runs=1
>  
>  if [ "$mode" == "" ]; then
>      mode=hash
>  fi
>  
> -if [ "$runs" == "" ]; then
> -    runs=1
> -fi
> -
> -kill `pidof tarantool`
> -
> -set -e
> -set -o pipefail
> -
>  TAR_VER=$(tarantool -v | grep -e "Tarantool" |  grep -oP '\s\K\S*')
> -echo ${TAR_VER} | tee ycsb_t_version.txt
> +numaopts="numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11"
> +base_dir=$(pwd)
>  
>  ws=/opt/ycsb
>  cd $ws
> @@ -33,7 +28,7 @@ sed 's/listen=.*/listen=3301,\n   memtx_memory = 2000000000,/' -i $srvlua
>  sed 's/logger_nonblock.*//' -i $srvlua
>  sed 's/logger/log/' -i $srvlua
>  sed 's/read,write,execute/create,read,write,execute/' -i $srvlua
> -numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11 tarantool $srvlua 2>&1 &
> +$numaopts tarantool $srvlua 2>&1 &
>  
>  sleep 5
>  plogs=$ws/results
> @@ -45,12 +40,25 @@ for l in a b c d e f ; do
>          res=$plogs/run${l}_${r}
>          echo ---------------- ${l}: $r
>          echo "tarantool.port=3301" >> $ws/workloads/workload${l}
> -        numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11 bin/ycsb load tarantool -s -P workloads/workload${l} >${res}.load 2>&1 || cat ${res}.load
> -       sync
> +        $numaopts bin/ycsb load tarantool -s -P workloads/workload${l} >${res}.load 2>&1 || cat ${res}.load
> +       sync && echo "sync passed" || echo "sync failed with error" $?
>         echo 3 > /proc/sys/vm/drop_caches
> -        numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11 bin/ycsb run tarantool -s -P workloads/workload${l} >${res}.log 2>&1 || cat ${res}.log
> +        $numaopts bin/ycsb run tarantool -s -P workloads/workload${l} >${res}.log 2>&1 || cat ${res}.log
>          grep Thro ${res}.log | awk '{ print "Overall result: "$3 }' | tee ${res}.txt
> -        sed "s#Overall result#$l $r#g" ${res}.txt >>${plogs}/results.txt
> +        sed "s#Overall result#$l $r#g" ${res}.txt >>${plogs}/ycsb.${mode}_result.txt
>      done
>  done
> -cat ${plogs}/results.txt
> +
> +echo ${TAR_VER} | tee ycsb.${mode}_t_version.txt
> +cp -f ${plogs}/ycsb.${mode}_result.txt .
> +cp -f ycsb.${mode}_t_version.txt $base_dir
> +cp -f ycsb.${mode}_result.txt $base_dir
> +
> +echo "Tarantool TAG:"
> +cat ycsb.${mode}_t_version.txt
> +echo "Overall results:"
> +echo "================"
> +cat ycsb.${mode}_result.txt
> +echo " "
> +echo "Publish data to bench database"
> +/opt/bench-run/benchs/publication/publish.py
> diff --git a/dockerfiles/ubuntu_benchs b/dockerfiles/ubuntu_benchs
> index fd9acb4..947f082 100644
> --- a/dockerfiles/ubuntu_benchs
> +++ b/dockerfiles/ubuntu_benchs
> @@ -73,3 +73,6 @@ RUN luarocks install \
>  
>  # benchmarks runners
>  RUN git clone https://github.com/tarantool/bench-run.git /opt/bench-run
> +
> +# adding python dependency
> +RUN pip install requests
> --
> 1.8.3.1
>  

-- 
sergeyb@

Re: [Tarantool-patches] [PATCH v2] bench-run: submit perf results into database

[Alexander V. Tikhonov]

Hi Oleg, thanks for the patch, LGTM, push into the master due to second
approvement.

On Wed, Apr 29, 2020 at 02:23:46PM +0300, Oleg Piskunov wrote:
> 
> 
> Resurrect bench.tarantool.org:
> - submit perf results into bench.tarantool.org database
> - refactoring bench running scripts
>  
> Fix docker files for perf testing:
> - adding python "requests" module needed for publish script.
>  
> Closes #4865
>  
> Github: https://github.com/tarantool/bench-run/tree/opiskunov/gh-4865-submit-perf-results
> Issue: https://github.com/tarantool/tarantool/issues/4865
> ---
>  benchs/cbench/run.sh          | 60 +++++++++++++++++++++++++++++-----------
>  benchs/linkbench/run.sh       | 26 ++++++++++++------
>  benchs/nosqlbench/run.sh      | 46 +++++++++++++------------------
>  benchs/publication/publish.py | 64 +++++++++++++++++++++++++++++++++++++++++++
>  benchs/sysbench/run.sh        | 52 ++++++++++++++---------------------
>  benchs/tpcc/run.sh            | 38 ++++++++++++++-----------
>  benchs/ycsb/run.sh            | 42 ++++++++++++++++------------
>  dockerfiles/ubuntu_benchs     |  3 ++
>  8 files changed, 216 insertions(+), 115 deletions(-)
>  create mode 100755 benchs/publication/publish.py
>  
> diff --git a/benchs/cbench/run.sh b/benchs/cbench/run.sh
> index 4d07972..77b88e7 100755
> --- a/benchs/cbench/run.sh
> +++ b/benchs/cbench/run.sh
> @@ -1,36 +1,64 @@
>  #!/usr/bin/env bash
>  
> -kill `pidof tarantool`
> -set -e
> +set -eu
>  set -o pipefail
>  
>  TAR_VER=$(tarantool -v | grep -e "Tarantool" |  grep -oP '\s\K\S*')
> -echo ${TAR_VER} | tee cbench_t_version.txt
> +numaopts="numactl --membind=1 --cpunodebind=1 --physcpubind=11"
> +cbench_opts=500
>  
>  killall tarantool 2>/dev/null || true
>  rm -rf 5* 0*
> -sync
> +sync && echo "sync passed" || echo "sync failed with error" $?
>  echo 3 > /proc/sys/vm/drop_caches
> -numactl --membind=1 --cpunodebind=1 --physcpubind=9 tarantool /opt/cbench/cbench_runner.lua memtx 2>&1 | tee cbench_output_memtx.txt
> +$numaopts tarantool /opt/cbench/cbench_runner.lua memtx 2>&1 | tee cbench_output_memtx.txt
>  
>  killall tarantool 2>/dev/null || true
>  rm -rf 5* 0*
> -sync
> +sync && echo "sync passed" || echo "sync failed with error" $?
>  echo 3 > /proc/sys/vm/drop_caches
> -numactl --membind=1 --cpunodebind=1 --physcpubind=10 tarantool /opt/cbench/cbench_runner.lua vinyl fsync 500 2>&1 | tee cbench_output_vinyl_fsync.txt
> +$numaopts tarantool /opt/cbench/cbench_runner.lua vinyl fsync $cbench_opts 2>&1 | tee cbench_output_vinyl_fsync.txt
>  
>  killall tarantool 2>/dev/null || true
>  rm -rf 5* 0*
> -sync
> +sync && echo "sync passed" || echo "sync failed with error" $?
>  echo 3 > /proc/sys/vm/drop_caches
> -numactl --membind=1 --cpunodebind=1 --physcpubind=11 tarantool /opt/cbench/cbench_runner.lua vinyl write 500 2>&1 | tee cbench_output_memtx_write.txt
> -
> -grep "^?tab" cbench_output_memtx.txt | sed "s/.*name=//"| sed "s/&param=/:/"| sed "s/cb\./cb\.memtx\./"| tee -a cbench_result.txt
> -grep "^?tab" cbench_output_vinyl_fsync.txt | sed "s/.*name=//"| sed "s/&param=/:/"| sed "s/cb\./cb\.vinyl\.fsync\./"| tee -a cbench_result.txt
> -grep "^?tab" cbench_output_memtx_write.txt | sed "s/.*name=//"| sed "s/&param=/:/"| sed "s/cb\./cb\.vinyl\.write\./"| tee -a cbench_result.txt
> +$numaopts tarantool /opt/cbench/cbench_runner.lua vinyl write $cbench_opts 2>&1 | tee cbench_output_vinyl_write.txt
>  
> -#mv tarantool-server.log cbench_tarantool_server.log
> +grep "^?tab=cbench.tree" cbench_output_memtx.txt | \
> +  sed "s/.*name=//"| sed "s/&param=/:/"| sed "s/cb\./cb\.memtx\./"| \
> +  tee -a cbench-memtx-tree_result.txt
> +grep "^?tab=cbench.hash" cbench_output_memtx.txt | \
> +  sed "s/.*name=//"| sed "s/&param=/:/"| sed "s/cb\./cb\.memtx\./"| \
> +  tee -a cbench-memtx-hash_result.txt
> +grep "^?tab" cbench_output_vinyl_fsync.txt | \
> +  sed "s/.*name=//"| sed "s/&param=/:/"| sed "s/cb\./cb\.vinyl\.fsync\./"| \
> +  tee -a cbench-vinyl-fsync_result.txt
> +grep "^?tab" cbench_output_vinyl_write.txt | \
> +  sed "s/.*name=//"| sed "s/&param=/:/"| sed "s/cb\./cb\.vinyl\.write\./"| \
> +  tee -a cbench-vinyl-write_result.txt
>  
> -echo "RESULTS:"
> -cat cbench_result.txt
> +echo ${TAR_VER} | tee cbench-memtx-tree_t_version.txt
> +echo ${TAR_VER} | tee cbench-memtx-hash_t_version.txt
> +echo ${TAR_VER} | tee cbench-vinyl-fsync_t_version.txt
> +echo ${TAR_VER} | tee cbench-vinyl-write_t_version.txt
> +echo ${TAR_VER} | tee cbench_t_version.txt
>  
> +echo "Tarantool TAG:"
> +cat cbench_t_version.txt
> +echo "Overall results:"
> +echo "================"
> +echo "RESULTS (cbench-memtx-tree_result.txt):"
> +cat cbench-memtx-tree_result.txt
> +echo " "
> +echo "RESULTS (cbench-memtx-hash_result.txt):"
> +cat cbench-memtx-hash_result.txt
> +echo " "
> +echo "RESULTS (cbench-vinyl-fsync_result.txt):"
> +cat cbench-vinyl-fsync_result.txt
> +echo " "
> +echo "RESULTS (cbench-vinyl-write_result.txt):"
> +cat cbench-vinyl-write_result.txt
> +echo " "
> +echo "Publish data to bench database"
> +/opt/bench-run/benchs/publication/publish.py
> diff --git a/benchs/linkbench/run.sh b/benchs/linkbench/run.sh
> index 6738436..aac7d3c 100755
> --- a/benchs/linkbench/run.sh
> +++ b/benchs/linkbench/run.sh
> @@ -1,19 +1,16 @@
>  #!/usr/bin/env bash
>  
> -kill `pidof tarantool`
> -
> -set -e
> +set -eu
>  set -o pipefail
>  
>  TAR_VER=$(tarantool -v | grep -e "Tarantool" |  grep -oP '\s\K\S*')
> -echo ${TAR_VER} | tee linkbench_t_version.txt
> +numaopts="numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11"
> +base_dir=$(pwd)
>  
>  cd /opt/linkbench && mvn clean package -Dmaven.test.skip=true
>  cd src/tarantool && make
>  cd -
>  
> -numaopts="numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11"
> -
>  # use always newly created path specialy mounted if needed
>  wpath=/builds/ws
>  rm -rf $wpath
> @@ -30,8 +27,21 @@ sed "s/^maxid1 = .*/maxid1 = 5000000/g" -i /opt/linkbench/config/FBWorkload.prop
>  sed "s/^requesters = .*/requesters = 1/g" -i $cfgfile
>  sed "s/^requests = .*/requests = 2000000/g" -i $cfgfile
>  $numaopts /opt/linkbench/bin/linkbench -c $cfgfile -l 2>&1 | tee loading.res.txt
> -sync
> +
> +sync && echo "sync passed" || echo "sync failed with error" $?
>  echo 3 > /proc/sys/vm/drop_caches
>  $numaopts /opt/linkbench/bin/linkbench -c $cfgfile -r 2>&1 | tee linkbench_output.txt
>  
> -grep "REQUEST PHASE COMPLETED" linkbench_output.txt | sed "s/.*second = /linkbench:/" | tee -a linkbench_result.txt
> +grep "REQUEST PHASE COMPLETED" linkbench_output.txt | sed "s/.*second = /linkbench:/" | tee -a linkbench.ssd_result.txt
> +echo ${TAR_VER} | tee linkbench.ssd_t_version.txt
> +cp -f linkbench.ssd_t_version.txt $base_dir
> +cp -f linkbench.ssd_result.txt $base_dir
> +
> +echo "Tarantool TAG:"
> +cat linkbench.ssd_t_version.txt
> +echo "Overall results:"
> +echo "================"
> +cat linkbench.ssd_result.txt
> +echo " "
> +echo "Publish data to bench database"
> +/opt/bench-run/benchs/publication/publish.py
> diff --git a/benchs/nosqlbench/run.sh b/benchs/nosqlbench/run.sh
> index c9491f5..157f372 100755
> --- a/benchs/nosqlbench/run.sh
> +++ b/benchs/nosqlbench/run.sh
> @@ -1,34 +1,22 @@
>  #!/usr/bin/env bash
>  
> +set -eu
> +set -o pipefail
> +
>  type=$1
>  if [ "$type" == "" ]; then
>      type=hash
>  fi
>  
>  TAR_VER=$(tarantool -v | grep -e "Tarantool" |  grep -oP '\s\K\S*')
> -echo ${TAR_VER} | tee nosqlbench_t_version.txt
> -
> -kill `pidof tarantool`
> -
> -#set -e
> -#
> -#set -o pipefail
> +numaopts="numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11"
>  
>  killall tarantool 2>/dev/null || true
> -rm -rf 0*.xlog 0*.snap
> -sync
> +sync && echo "sync passed" || echo "sync failed with error" $?
>  echo 3 > /proc/sys/vm/drop_caches
>  
> -#cd /opt/nosqlbench/
> -#cmake . &&  make -j
> -#cd -
> -#
> -#tarantool -v
> -#
> -free -h
> -numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11 tarantool `dirname $0`/tnt_${type}.lua 2>&1 &
> +$numaopts tarantool `dirname $0`/tnt_${type}.lua 2>&1 &
>  sed  "s/port 3303/port 3301/" /opt/nosqlbench/src/nosqlbench.conf -i
> -#sed  "s/request_count 4000000/request_count 400000/" /opt/nosqlbench/src/nosqlbench.conf -i
>  sed  "s/benchmark 'no_limit'/benchmark 'time_limit'/" /opt/nosqlbench/src/nosqlbench.conf -i
>  sed  "s/time_limit 10/time_limit 2000/" /opt/nosqlbench/src/nosqlbench.conf -i
>  sed  "s/request_batch_count 1/request_batch_count 10/" /opt/nosqlbench/src/nosqlbench.conf -i
> @@ -36,14 +24,18 @@ sed  "s/rps 12000/rps 20000/" /opt/nosqlbench/src/nosqlbench.conf -i
>  sleep 5
>  echo "Run NB"
>  # WARNING: don't try to save output from stderr - file will use the whole disk space !
> -numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11 /opt/nosqlbench/src/nb /opt/nosqlbench/src/nosqlbench.conf | \
> +$numaopts  /opt/nosqlbench/src/nb /opt/nosqlbench/src/nosqlbench.conf | \
>      grep -v "Warmup" | grep -v "Failed to allocate" >nosqlbench_output.txt || cat nosqlbench_output.txt
>  
> -#echo "Latest 1000 lines:"
> -#tail -1000 nosqlbench_output.txt
> -
> -echo "Getting results"
> -grep "TOTAL RPS STATISTICS:" nosqlbench_output.txt -A6 | awk -F "|" 'NR > 4 {print $2,":", $4}' > nosqlbench_result.txt
> -
> -cat nosqlbench_result.txt
> -
> +grep "TOTAL RPS STATISTICS:" nosqlbench_output.txt -A6 | \
> +  awk -F "|" 'NR > 4 {print $2,":", $4}' > noSQLbench.${type}_result.txt
> +echo ${TAR_VER} | tee noSQLbench.${type}_t_version.txt
> +
> +echo "Tarantool TAG:"
> +cat noSQLbench.${type}_t_version.txt
> +echo "Overall results:"
> +echo "================"
> +cat noSQLbench.${type}_result.txt
> +echo " "
> +echo "Publish data to bench database"
> +/opt/bench-run/benchs/publication/publish.py
> diff --git a/benchs/publication/publish.py b/benchs/publication/publish.py
> new file mode 100755
> index 0000000..42bc53a
> --- /dev/null
> +++ b/benchs/publication/publish.py
> @@ -0,0 +1,64 @@
> +#!/usr/bin/env python
> +import fnmatch
> +import os
> +from urllib import urlencode
> +import requests
> +
> +
> +def parse_bench(filename):
> +    with open(filename) as raw_data:
> +        return raw_data.readlines()
> +
> +
> +def get_version(filename):
> +    with open(filename) as raw_data:
> +        version = raw_data.readlines()[-1]
> +        return version.split()[0]
> +
> +
> +def push_to_microb(server, token, name, value, version, tab):
> +    uri = 'http://%s/push?%s' % (server, urlencode(dict(
> +        key=token, name=name, param=value,
> +        v=version, unit='trps', tab=tab
> +    )))
> +
> +    r = requests.get(uri)
> +    if r.status_code == 200:
> +        print('Export complete')
> +    else:
> +        print('Export error http: %d' % r.status_code)
> +        print('Export error text: %d' % r.text)
> +
> +
> +def main():
> +    if "MICROB_WEB_TOKEN" in os.environ and "MICROB_WEB_HOST" in os.environ:
> +        bench = {}
> +        res = []
> +        current_data = {}
> +        version = ''
> +        for file in os.listdir('.'):
> +            if fnmatch.fnmatch(file, '*_result.txt'):
> +                values = parse_bench(file)
> +                benchmark = file.split('_')[0]
> +                version = get_version('{}_t_version.txt'.format(benchmark))
> +                for value in values:
> +                    test_name = value.split(':')[0]
> +                    test_res = float(value.split(':')[1])
> +                    res.append(test_res)
> +                    push_to_microb(
> +                        os.environ['MICROB_WEB_HOST'],
> +                        os.environ['MICROB_WEB_TOKEN'],
> +                        test_name,
> +                        test_res,
> +                        version,
> +                        benchmark,
> +                    )
> +        print ("VERSION - ", version)
> +    else:
> +        print("MICROB params not specified")
> +
> +    return 0
> +
> +
> +if __name__ == '__main__':
> +    main()
> diff --git a/benchs/sysbench/run.sh b/benchs/sysbench/run.sh
> index 6a69d14..a9cf2f3 100755
> --- a/benchs/sysbench/run.sh
> +++ b/benchs/sysbench/run.sh
> @@ -1,18 +1,12 @@
>  #!/usr/bin/env bash
>  
> -runs=10
> -if [ "$1" != "" ]; then
> -    runs=$1
> -fi
> -
> -kill `pidof tarantool`
> -
> -set -e
> +set -eu
>  set -o pipefail
>  
> +runs=10
> +
>  TAR_VER=$(tarantool -v | grep -e "Tarantool" |  grep -oP '\s\K\S*')
> -echo ${TAR_VER} | tee sysbench_t_version.txt
> -set -e
> +numaconf="numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11"
>  
>  ARRAY_TESTS=(
>      "oltp_read_only"
> @@ -28,27 +22,17 @@ ARRAY_TESTS=(
>  #    "bulk_insert"
>  )
>  
> -if [ -n "${TEST}" ]; then ARRAY_TESTS=("${TEST}"); fi
> -
> -
> -if [ ! -n "${WARMUP_TIME}" ]; then WARMUP_TIME=5; fi
> -if [ ! -n "${TIME}" ]; then TIME=20; fi
> -if [ ! -n "${DBMS}" ]; then DBMS="tarantool"; fi
> -if [ ! -n "${THREADS}" ]; then THREADS=200; fi
> -
> -if [ -n "${USER}" ]; then USER=--${DBMS}-user=${USER}; fi
> -if [ -n "${PASSWORD}" ]; then PASSWORD=--${DBMS}-password=${PASSWORD}; fi
> -
> -numaconf="--membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11"
> +WARMUP_TIME=5
> +TIME=20
> +DBMS="tarantool"
> +THREADS=200
>  opts="--db-driver=${DBMS} --threads=${THREADS}"
>  
>  export LD_LIBRARY_PATH=/usr/local/lib
>  
> -rm -f sysbench_result.txt
> -set -o pipefail
>  for test in "${ARRAY_TESTS[@]}"; do
>      res=0
> -    tlog=sysbench_${test}_result.txt
> +    tlog=sysbench_${test}_results.txt
>      rm -f $tlog
>      maxres=0
>      for run in `eval echo {1..$runs}` ; do
> @@ -58,10 +42,10 @@ for test in "${ARRAY_TESTS[@]}"; do
>          sysbench $test $opts cleanup >sysbench_output.txt
>          sysbench $test $opts prepare >>sysbench_output.txt
>  
> -        numactl $numaconf sysbench $test $opts \
> +        $numaconf sysbench $test $opts \
>              --time=${TIME} --warmup-time=${WARMUP_TIME} run >>sysbench_output.txt
>  
> -        numactl $numaconf sysbench $test $opts cleanup >>sysbench_output.txt
> +        $numaconf sysbench $test $opts cleanup >>sysbench_output.txt
>  
>          cat sysbench_output.txt | grep -e 'transactions:' | grep -oP '\(\K\S*' | tee $tlog
>          tres=`cat $tlog | sed 's#^.*:##g' | sed 's#\..*$##g'`
> @@ -69,16 +53,22 @@ for test in "${ARRAY_TESTS[@]}"; do
>          if [[ $tres -gt $maxres ]]; then maxres=$tres ; fi
>      done
>      res=$(($res/$runs))
> -    echo "${test}: $res" >>sysbench_result.txt
> +    echo "${test}: $res" >>Sysbench_result.txt
>  
>      echo "Subtest '$test' results:"
>      echo "==============================="
>      echo "Average result: $res"
>      echo "Maximum result: $maxres"
> -    printf "Diviations (AVG -> MAX): %.2f" `bc <<< "scale = 4; (1 - $res / $maxres) * 100"` ; echo %
> +    printf "Deviations (AVG -> MAX): %.2f" `bc <<< "scale = 4; (1 - $res / $maxres) * 100"` ; echo %
>  done
>  
> +echo ${TAR_VER} | tee Sysbench_t_version.txt
> +
> +echo "Tarantool TAG:"
> +cat Sysbench_t_version.txt
>  echo "Overall results:"
>  echo "================"
> -cat sysbench_result.txt
> -
> +cat Sysbench_result.txt
> +echo " "
> +echo "Publish data to bench database"
> +/opt/bench-run/benchs/publication/publish.py
> diff --git a/benchs/tpcc/run.sh b/benchs/tpcc/run.sh
> index f1f60cc..fca7939 100755
> --- a/benchs/tpcc/run.sh
> +++ b/benchs/tpcc/run.sh
> @@ -1,22 +1,20 @@
>  #!/usr/bin/env bash
> +set -eu
> +set -o pipefail
>  
>  TAR_VER=$(tarantool -v | grep -e "Tarantool" |  grep -oP '\s\K\S*')
> -echo ${TAR_VER} | tee tpcc_t_version.txt
> -
> -set -e
> +numaopts="numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11"
> +base_dir=$(pwd)
>  
> -if [ ! -n "${TIME}" ]; then TIME=1200; fi
> -if [ ! -n "${WARMUP_TIME}" ]; then WARMUP_TIME=10; fi
> -
> -set -o pipefail
> +TIME=1200
> +WARMUP_TIME=10
>  
>  killall tarantool tpcc_load 2>/dev/null || true
> -rm -rf 0*.xlog 0*.snap
> -sync
> +sync && echo "sync passed" || echo "sync failed with error" $?
>  echo 3 > /proc/sys/vm/drop_caches
>  
>  sed 's#box.sql#box#g' -i /opt/tpcc/create_table.lua
> -numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11 tarantool /opt/tpcc/create_table.lua &
> +$numaopts tarantool /opt/tpcc/create_table.lua &
>  sleep 5
>  
>  # Usage: tpcc_load -h server_host -P port -d database_name -u mysql_user
> @@ -27,13 +25,21 @@ tpcc_opts="-h localhost -P 3301 -d tarantool -u root -p '' -w 15"
>  cd /opt/tpcc
>  . /opt/tpcc/load.sh tarantool 15
>  
> -numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11 \
> -    /opt/tpcc/tpcc_start $tpcc_opts -r10 -l${TIME} -i${TIME} >tpcc_output.txt 2>/dev/null
> -
> -echo -n "tpcc:" | tee tpcc_result.txt
> -cat tpcc_output.txt | grep -e '<TpmC>' | grep -oP '\K[0-9.]*' | tee -a tpcc_result.txt
> +$numaopts /opt/tpcc/tpcc_start $tpcc_opts -r10 -l${TIME} -i${TIME} >tpcc_output.txt 2>/dev/null
>  
> +echo -n "tpcc:" | tee tpc.c_result.txt
> +cat tpcc_output.txt | grep -e '<TpmC>' | grep -oP '\K[0-9.]*' | tee -a tpc.c_result.txt
>  cat tpcc_output.txt
> +
> +echo ${TAR_VER} | tee tpc.c_t_version.txt
> +cp -f tpc.c_t_version.txt $base_dir
> +cp -f tpc.c_result.txt $base_dir
> +
> +echo "Tarantool TAG:"
> +cat tpc.c_t_version.txt
>  echo "Overall result:"
>  echo "==============="
> -cat tpcc_result.txt
> +cat tpc.c_result.txt
> +echo " "
> +echo "Publish data to bench database"
> +/opt/bench-run/benchs/publication/publish.py
> diff --git a/benchs/ycsb/run.sh b/benchs/ycsb/run.sh
> index 5ae1c75..e5219a1 100755
> --- a/benchs/ycsb/run.sh
> +++ b/benchs/ycsb/run.sh
> @@ -1,23 +1,18 @@
>  #!/usr/bin/env bash
>  
> +set -eu
> +set -o pipefail
> +
>  mode=$1
> -runs=$2
> +runs=1
>  
>  if [ "$mode" == "" ]; then
>      mode=hash
>  fi
>  
> -if [ "$runs" == "" ]; then
> -    runs=1
> -fi
> -
> -kill `pidof tarantool`
> -
> -set -e
> -set -o pipefail
> -
>  TAR_VER=$(tarantool -v | grep -e "Tarantool" |  grep -oP '\s\K\S*')
> -echo ${TAR_VER} | tee ycsb_t_version.txt
> +numaopts="numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11"
> +base_dir=$(pwd)
>  
>  ws=/opt/ycsb
>  cd $ws
> @@ -33,7 +28,7 @@ sed 's/listen=.*/listen=3301,\n   memtx_memory = 2000000000,/' -i $srvlua
>  sed 's/logger_nonblock.*//' -i $srvlua
>  sed 's/logger/log/' -i $srvlua
>  sed 's/read,write,execute/create,read,write,execute/' -i $srvlua
> -numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11 tarantool $srvlua 2>&1 &
> +$numaopts tarantool $srvlua 2>&1 &
>  
>  sleep 5
>  plogs=$ws/results
> @@ -45,12 +40,25 @@ for l in a b c d e f ; do
>          res=$plogs/run${l}_${r}
>          echo ---------------- ${l}: $r
>          echo "tarantool.port=3301" >> $ws/workloads/workload${l}
> -        numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11 bin/ycsb load tarantool -s -P workloads/workload${l} >${res}.load 2>&1 || cat ${res}.load
> -       sync
> +        $numaopts bin/ycsb load tarantool -s -P workloads/workload${l} >${res}.load 2>&1 || cat ${res}.load
> +       sync && echo "sync passed" || echo "sync failed with error" $?
>         echo 3 > /proc/sys/vm/drop_caches
> -        numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11 bin/ycsb run tarantool -s -P workloads/workload${l} >${res}.log 2>&1 || cat ${res}.log
> +        $numaopts bin/ycsb run tarantool -s -P workloads/workload${l} >${res}.log 2>&1 || cat ${res}.log
>          grep Thro ${res}.log | awk '{ print "Overall result: "$3 }' | tee ${res}.txt
> -        sed "s#Overall result#$l $r#g" ${res}.txt >>${plogs}/results.txt
> +        sed "s#Overall result#$l $r#g" ${res}.txt >>${plogs}/ycsb.${mode}_result.txt
>      done
>  done
> -cat ${plogs}/results.txt
> +
> +echo ${TAR_VER} | tee ycsb.${mode}_t_version.txt
> +cp -f ${plogs}/ycsb.${mode}_result.txt .
> +cp -f ycsb.${mode}_t_version.txt $base_dir
> +cp -f ycsb.${mode}_result.txt $base_dir
> +
> +echo "Tarantool TAG:"
> +cat ycsb.${mode}_t_version.txt
> +echo "Overall results:"
> +echo "================"
> +cat ycsb.${mode}_result.txt
> +echo " "
> +echo "Publish data to bench database"
> +/opt/bench-run/benchs/publication/publish.py
> diff --git a/dockerfiles/ubuntu_benchs b/dockerfiles/ubuntu_benchs
> index fd9acb4..947f082 100644
> --- a/dockerfiles/ubuntu_benchs
> +++ b/dockerfiles/ubuntu_benchs
> @@ -73,3 +73,6 @@ RUN luarocks install \
>  
>  # benchmarks runners
>  RUN git clone https://github.com/tarantool/bench-run.git /opt/bench-run
> +
> +# adding python dependency
> +RUN pip install requests
> --
> 1.8.3.1
>