* [Tarantool-patches] [PATCH v1] bench-run: submit perf results into database
@ 2020-04-22 10:18 Олег Пискунов
2020-04-22 10:57 ` Alexander Tikhonov
2020-04-23 9:28 ` Sergey Bronnikov
0 siblings, 2 replies; 8+ messages in thread
From: Олег Пискунов @ 2020-04-22 10:18 UTC (permalink / raw)
To: tarantool-patches; +Cc: Oleg Piskunov
[-- Attachment #1: Type: text/plain, Size: 13050 bytes --]
Resurrect bench.tarantool.org:
- submit perf results into bench.tarantool.org database
Fix docker files for perf testing:
- adding python "requests" module needed for publish script.
Closes #4865
Github: https://github.com/tarantool/bench-run/tree/opiskunov/gh-4865-submit-perf-results
Issue: https://github.com/tarantool/tarantool/issues/4865
---
benchs/cbench/run.sh | 38 ++++++++++++++++++------
benchs/linkbench/run.sh | 13 +++++++--
benchs/nosqlbench/run.sh | 22 ++++++++------
benchs/publication/publish.py | 67 +++++++++++++++++++++++++++++++++++++++++++
benchs/sysbench/run.sh | 19 ++++++------
benchs/tpcc/run.sh | 21 ++++++++------
benchs/ycsb/run.sh | 16 +++++++++--
dockerfiles/ubuntu_benchs | 4 +--
8 files changed, 158 insertions(+), 42 deletions(-)
create mode 100755 benchs/publication/publish.py
diff --git a/benchs/cbench/run.sh b/benchs/cbench/run.sh
index 4d07972..c4867ad 100755
--- a/benchs/cbench/run.sh
+++ b/benchs/cbench/run.sh
@@ -1,11 +1,11 @@
#!/usr/bin/env bash
kill `pidof tarantool`
+
set -e
set -o pipefail
TAR_VER=$(tarantool -v | grep -e "Tarantool" | grep -oP '\s\K\S*')
-echo ${TAR_VER} | tee cbench_t_version.txt
killall tarantool 2>/dev/null || true
rm -rf 5* 0*
@@ -23,14 +23,34 @@ killall tarantool 2>/dev/null || true
rm -rf 5* 0*
sync
echo 3 > /proc/sys/vm/drop_caches
-numactl --membind=1 --cpunodebind=1 --physcpubind=11 tarantool /opt/cbench/cbench_runner.lua vinyl write 500 2>&1 | tee cbench_output_memtx_write.txt
+numactl --membind=1 --cpunodebind=1 --physcpubind=11 tarantool /opt/cbench/cbench_runner.lua vinyl write 500 2>&1 | tee cbench_output_vinyl_write.txt
-grep "^?tab" cbench_output_memtx.txt | sed "s/.*name=//"| sed "s/¶m=/:/"| sed "s/cb\./cb\.memtx\./"| tee -a cbench_result.txt
-grep "^?tab" cbench_output_vinyl_fsync.txt | sed "s/.*name=//"| sed "s/¶m=/:/"| sed "s/cb\./cb\.vinyl\.fsync\./"| tee -a cbench_result.txt
-grep "^?tab" cbench_output_memtx_write.txt | sed "s/.*name=//"| sed "s/¶m=/:/"| sed "s/cb\./cb\.vinyl\.write\./"| tee -a cbench_result.txt
+grep "^?tab=cbench.tree" cbench_output_memtx.txt | sed "s/.*name=//"| sed "s/¶m=/:/"| sed "s/cb\./cb\.memtx\./"| tee -a cbench-memtx-tree_result.txt
+grep "^?tab=cbench.hash" cbench_output_memtx.txt | sed "s/.*name=//"| sed "s/¶m=/:/"| sed "s/cb\./cb\.memtx\./"| tee -a cbench-memtx-hash_result.txt
+grep "^?tab" cbench_output_vinyl_fsync.txt | sed "s/.*name=//"| sed "s/¶m=/:/"| sed "s/cb\./cb\.vinyl\.fsync\./"| tee -a cbench-vinyl-fsync_result.txt
+grep "^?tab" cbench_output_vinyl_write.txt | sed "s/.*name=//"| sed "s/¶m=/:/"| sed "s/cb\./cb\.vinyl\.write\./"| tee -a cbench-vinyl-write_result.txt
-#mv tarantool-server.log cbench_tarantool_server.log
-
-echo "RESULTS:"
-cat cbench_result.txt
+echo ${TAR_VER} | tee cbench-memtx-tree_t_version.txt
+echo ${TAR_VER} | tee cbench-memtx-hash_t_version.txt
+echo ${TAR_VER} | tee cbench-vinyl-fsync_t_version.txt
+echo ${TAR_VER} | tee cbench-vinyl-write_t_version.txt
+echo ${TAR_VER} | tee cbench_t_version.txt
+echo "Tarantool TAG:"
+cat cbench_t_version.txt
+echo "Overall results:"
+echo "================"
+echo "RESULTS (cbench-memtx-tree_result.txt):"
+cat cbench-memtx-tree_result.txt
+echo " "
+echo "RESULTS (cbench-memtx-hash_result.txt):"
+cat cbench-memtx-hash_result.txt
+echo " "
+echo "RESULTS (cbench-vinyl-fsync_result.txt):"
+cat cbench-vinyl-fsync_result.txt
+echo " "
+echo "RESULTS (cbench-vinyl-write_result.txt):"
+cat cbench-vinyl-write_result.txt
+echo " "
+echo "Publish data to bench database"
+/opt/bench-run/benchs/publication/publish.py
diff --git a/benchs/linkbench/run.sh b/benchs/linkbench/run.sh
index 6738436..e7c77c3 100755
--- a/benchs/linkbench/run.sh
+++ b/benchs/linkbench/run.sh
@@ -6,7 +6,6 @@ set -e
set -o pipefail
TAR_VER=$(tarantool -v | grep -e "Tarantool" | grep -oP '\s\K\S*')
-echo ${TAR_VER} | tee linkbench_t_version.txt
cd /opt/linkbench && mvn clean package -Dmaven.test.skip=true
cd src/tarantool && make
@@ -34,4 +33,14 @@ sync
echo 3 > /proc/sys/vm/drop_caches
$numaopts /opt/linkbench/bin/linkbench -c $cfgfile -r 2>&1 | tee linkbench_output.txt
-grep "REQUEST PHASE COMPLETED" linkbench_output.txt | sed "s/.*second = /linkbench:/" | tee -a linkbench_result.txt
+grep "REQUEST PHASE COMPLETED" linkbench_output.txt | sed "s/.*second = /linkbench:/" | tee -a linkbench.ssd_result.txt
+echo ${TAR_VER} | tee linkbench.ssd_t_version.txt
+
+echo "Tarantool TAG:"
+cat linkbench.ssd_t_version.txt
+echo "Overall results:"
+echo "================"
+cat linkbench.ssd_result.txt
+echo " "
+echo "Publish data to bench database"
+/opt/bench-run/benchs/publication/publish.py
diff --git a/benchs/nosqlbench/run.sh b/benchs/nosqlbench/run.sh
index c9491f5..a68248e 100755
--- a/benchs/nosqlbench/run.sh
+++ b/benchs/nosqlbench/run.sh
@@ -6,13 +6,11 @@ if [ "$type" == "" ]; then
fi
TAR_VER=$(tarantool -v | grep -e "Tarantool" | grep -oP '\s\K\S*')
-echo ${TAR_VER} | tee nosqlbench_t_version.txt
kill `pidof tarantool`
-#set -e
-#
-#set -o pipefail
+set -e
+set -o pipefail
killall tarantool 2>/dev/null || true
rm -rf 0*.xlog 0*.snap
@@ -42,8 +40,14 @@ numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11 /opt/nosqlbench/
#echo "Latest 1000 lines:"
#tail -1000 nosqlbench_output.txt
-echo "Getting results"
-grep "TOTAL RPS STATISTICS:" nosqlbench_output.txt -A6 | awk -F "|" 'NR > 4 {print $2,":", $4}' > nosqlbench_result.txt
-
-cat nosqlbench_result.txt
-
+grep "TOTAL RPS STATISTICS:" nosqlbench_output.txt -A6 | awk -F "|" 'NR > 4 {print $2,":", $4}' > noSQLbench.${type}_result.txt
+echo ${TAR_VER} | tee noSQLbench.${type}_t_version.txt
+
+echo "Tarantool TAG:"
+cat noSQLbench.${type}_t_version.txt
+echo "Overall results:"
+echo "================"
+cat noSQLbench.${type}_result.txt
+echo " "
+echo "Publish data to bench database"
+/opt/bench-run/benchs/publication/publish.py
diff --git a/benchs/publication/publish.py b/benchs/publication/publish.py
new file mode 100755
index 0000000..d472fd7
--- /dev/null
+++ b/benchs/publication/publish.py
@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+import fnmatch
+import os
+from urllib import urlencode
+import requests
+
+
+def parse_bench(filename):
+ fileHandle = open(filename)
+ lastline = fileHandle.readlines()
+ fileHandle.close()
+ return lastline
+
+
+def get_version(filename):
+ fileHandle = open(filename)
+ lastline = fileHandle.readlines()[-1]
+ fileHandle.close()
+ return lastline.split()[0]
+
+
+def push_to_microb(server, token, name, value, version, tab):
+ uri = 'http://%s/push?%s' % (server, urlencode(dict(
+ key=token, name=name, param=value,
+ v=version, unit='trps', tab=tab
+ )))
+
+ r = requests.get(uri)
+ if r.status_code == 200:
+ print('Export complete')
+ else:
+ print('Export error http: %d' % r.status_code)
+ print('Export error text: %d' % r.text)
+
+
+def main():
+ if "MICROB_WEB_TOKEN" in os.environ and "MICROB_WEB_HOST" in os.environ:
+ bench = {}
+ res = []
+ current_data = {}
+ version = ''
+ for file in os.listdir('.'):
+ if fnmatch.fnmatch(file, '*_result.txt'):
+ values = parse_bench(file)
+ benchmark = file.split('_')[0]
+ version = get_version('{}_t_version.txt'.format(benchmark))
+ for value in values:
+ test_name = value.split(':')[0]
+ test_res = float(value.split(':')[1])
+ res.append(test_res)
+ push_to_microb(
+ os.environ['MICROB_WEB_HOST'],
+ os.environ['MICROB_WEB_TOKEN'],
+ test_name,
+ test_res,
+ version,
+ benchmark,
+ )
+ print ("VERSION - ", version)
+ else:
+ print("MICROB params not specified")
+
+ return 0
+
+
+if __name__ == '__main__':
+ main()
diff --git a/benchs/sysbench/run.sh b/benchs/sysbench/run.sh
index 6a69d14..ef0af43 100755
--- a/benchs/sysbench/run.sh
+++ b/benchs/sysbench/run.sh
@@ -11,8 +11,6 @@ set -e
set -o pipefail
TAR_VER=$(tarantool -v | grep -e "Tarantool" | grep -oP '\s\K\S*')
-echo ${TAR_VER} | tee sysbench_t_version.txt
-set -e
ARRAY_TESTS=(
"oltp_read_only"
@@ -44,11 +42,10 @@ opts="--db-driver=${DBMS} --threads=${THREADS}"
export LD_LIBRARY_PATH=/usr/local/lib
-rm -f sysbench_result.txt
-set -o pipefail
+rm -f Sysbench_result.txt
for test in "${ARRAY_TESTS[@]}"; do
res=0
- tlog=sysbench_${test}_result.txt
+ tlog=sysbench_${test}_results.txt
rm -f $tlog
maxres=0
for run in `eval echo {1..$runs}` ; do
@@ -69,7 +66,7 @@ for test in "${ARRAY_TESTS[@]}"; do
if [[ $tres -gt $maxres ]]; then maxres=$tres ; fi
done
res=$(($res/$runs))
- echo "${test}: $res" >>sysbench_result.txt
+ echo "${test}: $res" >>Sysbench_result.txt
echo "Subtest '$test' results:"
echo "==============================="
@@ -78,7 +75,13 @@ for test in "${ARRAY_TESTS[@]}"; do
printf "Diviations (AVG -> MAX): %.2f" `bc <<< "scale = 4; (1 - $res / $maxres) * 100"` ; echo %
done
+echo ${TAR_VER} | tee Sysbench_t_version.txt
+
+echo "Tarantool TAG:"
+cat Sysbench_t_version.txt
echo "Overall results:"
echo "================"
-cat sysbench_result.txt
-
+cat Sysbench_result.txt
+echo " "
+echo "Publish data to bench database"
+/opt/bench-run/benchs/publication/publish.py
diff --git a/benchs/tpcc/run.sh b/benchs/tpcc/run.sh
index f1f60cc..76754e5 100755
--- a/benchs/tpcc/run.sh
+++ b/benchs/tpcc/run.sh
@@ -1,15 +1,12 @@
#!/usr/bin/env bash
+set -e
+set -o pipefail
TAR_VER=$(tarantool -v | grep -e "Tarantool" | grep -oP '\s\K\S*')
-echo ${TAR_VER} | tee tpcc_t_version.txt
-
-set -e
if [ ! -n "${TIME}" ]; then TIME=1200; fi
if [ ! -n "${WARMUP_TIME}" ]; then WARMUP_TIME=10; fi
-set -o pipefail
-
killall tarantool tpcc_load 2>/dev/null || true
rm -rf 0*.xlog 0*.snap
sync
@@ -30,10 +27,16 @@ cd /opt/tpcc
numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11 \
/opt/tpcc/tpcc_start $tpcc_opts -r10 -l${TIME} -i${TIME} >tpcc_output.txt 2>/dev/null
-echo -n "tpcc:" | tee tpcc_result.txt
-cat tpcc_output.txt | grep -e '<TpmC>' | grep -oP '\K[0-9.]*' | tee -a tpcc_result.txt
-
+echo -n "tpcc:" | tee tpc.c_result.txt
+cat tpcc_output.txt | grep -e '<TpmC>' | grep -oP '\K[0-9.]*' | tee -a tpc.c_result.txt
cat tpcc_output.txt
+echo ${TAR_VER} | tee tpc.c_t_version.txt
+
+echo "Tarantool TAG:"
+cat tpc.c_t_version.txt
echo "Overall result:"
echo "==============="
-cat tpcc_result.txt
+cat tpc.c_result.txt
+echo " "
+echo "Publish data to bench database"
+/opt/bench-run/benchs/publication/publish.py
diff --git a/benchs/ycsb/run.sh b/benchs/ycsb/run.sh
index 5ae1c75..d2cf8af 100755
--- a/benchs/ycsb/run.sh
+++ b/benchs/ycsb/run.sh
@@ -17,7 +17,6 @@ set -e
set -o pipefail
TAR_VER=$(tarantool -v | grep -e "Tarantool" | grep -oP '\s\K\S*')
-echo ${TAR_VER} | tee ycsb_t_version.txt
ws=/opt/ycsb
cd $ws
@@ -50,7 +49,18 @@ for l in a b c d e f ; do
echo 3 > /proc/sys/vm/drop_caches
numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11 bin/ycsb run tarantool -s -P workloads/workload${l} >${res}.log 2>&1 || cat ${res}.log
grep Thro ${res}.log | awk '{ print "Overall result: "$3 }' | tee ${res}.txt
- sed "s#Overall result#$l $r#g" ${res}.txt >>${plogs}/results.txt
+ sed "s#Overall result#$l $r#g" ${res}.txt >>${plogs}/ycsb.${mode}_result.txt
done
done
-cat ${plogs}/results.txt
+
+echo ${TAR_VER} | tee ycsb.${mode}_t_version.txt
+cp -f ${plogs}/ycsb.${mode}_result.txt .
+
+echo "Tarantool TAG:"
+cat ycsb.${mode}_t_version.txt
+echo "Overall results:"
+echo "================"
+cat ycsb.${mode}_result.txt
+echo " "
+echo "Publish data to bench database"
+/opt/bench-run/benchs/publication/publish.py
diff --git a/dockerfiles/ubuntu_benchs b/dockerfiles/ubuntu_benchs
index fd9acb4..2c2e717 100644
--- a/dockerfiles/ubuntu_benchs
+++ b/dockerfiles/ubuntu_benchs
@@ -71,5 +71,5 @@ RUN luarocks install \
https://raw.githubusercontent.com/tarantool/gperftools/master/rockspecs/gperftools-scm-1.rockspec \
--local >build.log 2>&1 || ( cat build.log && false )
-# benchmarks runners
-RUN git clone https://github.com/tarantool/bench-run.git /opt/bench-run
+# adding python dependency
+RUN pip install requests
--
1.8.3.1
[-- Attachment #2: Type: text/html, Size: 16573 bytes --]
^ permalink raw reply [flat|nested] 8+ messages in thread
* Re: [Tarantool-patches] [PATCH v1] bench-run: submit perf results into database
2020-04-22 10:18 [Tarantool-patches] [PATCH v1] bench-run: submit perf results into database Олег Пискунов
@ 2020-04-22 10:57 ` Alexander Tikhonov
2020-04-23 9:28 ` Sergey Bronnikov
1 sibling, 0 replies; 8+ messages in thread
From: Alexander Tikhonov @ 2020-04-22 10:57 UTC (permalink / raw)
To: Олег
Пискунов
Cc: Oleg Piskunov, tarantool-patches
[-- Attachment #1: Type: text/plain, Size: 13697 bytes --]
Hi Oleg, thanks for the patch, I see that you just divided the
previous patch into this and the other one w/o changes, so LGTM.
>Среда, 22 апреля 2020, 13:18 +03:00 от Олег Пискунов <o.piskunov@corp.mail.ru>:
>
>Resurrect bench.tarantool.org:
>- submit perf results into bench.tarantool.org database
>
>Fix docker files for perf testing:
>- adding python "requests" module needed for publish script.
>
>Closes #4865
>
>Github: https://github.com/tarantool/bench-run/tree/opiskunov/gh-4865-submit-perf-results
>Issue: https://github.com/tarantool/tarantool/issues/4865
>---
> benchs/cbench/run.sh | 38 ++++++++++++++++++------
> benchs/linkbench/run.sh | 13 +++++++--
> benchs/nosqlbench/run.sh | 22 ++++++++------
> benchs/publication/publish.py | 67 +++++++++++++++++++++++++++++++++++++++++++
> benchs/sysbench/run.sh | 19 ++++++------
> benchs/tpcc/run.sh | 21 ++++++++------
> benchs/ycsb/run.sh | 16 +++++++++--
> dockerfiles/ubuntu_benchs | 4 +--
> 8 files changed, 158 insertions(+), 42 deletions(-)
> create mode 100755 benchs/publication/publish.py
>
>diff --git a/benchs/cbench/run.sh b/benchs/cbench/run.sh
>index 4d07972..c4867ad 100755
>--- a/benchs/cbench/run.sh
>+++ b/benchs/cbench/run.sh
>@@ -1,11 +1,11 @@
> #!/usr/bin/env bash
>
> kill `pidof tarantool`
>+
> set -e
> set -o pipefail
>
> TAR_VER=$(tarantool -v | grep -e "Tarantool" | grep -oP '\s\K\S*')
>-echo ${TAR_VER} | tee cbench_t_version.txt
>
> killall tarantool 2>/dev/null || true
> rm -rf 5* 0*
>@@ -23,14 +23,34 @@ killall tarantool 2>/dev/null || true
> rm -rf 5* 0*
> sync
> echo 3 > /proc/sys/vm/drop_caches
>-numactl --membind=1 --cpunodebind=1 --physcpubind=11 tarantool /opt/cbench/cbench_runner.lua vinyl write 500 2>&1 | tee cbench_output_memtx_write.txt
>+numactl --membind=1 --cpunodebind=1 --physcpubind=11 tarantool /opt/cbench/cbench_runner.lua vinyl write 500 2>&1 | tee cbench_output_vinyl_write.txt
>
>-grep "^?tab" cbench_output_memtx.txt | sed "s/.*name=//"| sed "s/¶m=/:/"| sed "s/cb\./cb\.memtx\./"| tee -a cbench_result.txt
>-grep "^?tab" cbench_output_vinyl_fsync.txt | sed "s/.*name=//"| sed "s/¶m=/:/"| sed "s/cb\./cb\.vinyl\.fsync\./"| tee -a cbench_result.txt
>-grep "^?tab" cbench_output_memtx_write.txt | sed "s/.*name=//"| sed "s/¶m=/:/"| sed "s/cb\./cb\.vinyl\.write\./"| tee -a cbench_result.txt
>+grep "^?tab=cbench.tree" cbench_output_memtx.txt | sed "s/.*name=//"| sed "s/¶m=/:/"| sed "s/cb\./cb\.memtx\./"| tee -a cbench-memtx-tree_result.txt
>+grep "^?tab=cbench.hash" cbench_output_memtx.txt | sed "s/.*name=//"| sed "s/¶m=/:/"| sed "s/cb\./cb\.memtx\./"| tee -a cbench-memtx-hash_result.txt
>+grep "^?tab" cbench_output_vinyl_fsync.txt | sed "s/.*name=//"| sed "s/¶m=/:/"| sed "s/cb\./cb\.vinyl\.fsync\./"| tee -a cbench-vinyl-fsync_result.txt
>+grep "^?tab" cbench_output_vinyl_write.txt | sed "s/.*name=//"| sed "s/¶m=/:/"| sed "s/cb\./cb\.vinyl\.write\./"| tee -a cbench-vinyl-write_result.txt
>
>-#mv tarantool-server.log cbench_tarantool_server.log
>-
>-echo "RESULTS:"
>-cat cbench_result.txt
>+echo ${TAR_VER} | tee cbench-memtx-tree_t_version.txt
>+echo ${TAR_VER} | tee cbench-memtx-hash_t_version.txt
>+echo ${TAR_VER} | tee cbench-vinyl-fsync_t_version.txt
>+echo ${TAR_VER} | tee cbench-vinyl-write_t_version.txt
>+echo ${TAR_VER} | tee cbench_t_version.txt
>
>+echo "Tarantool TAG:"
>+cat cbench_t_version.txt
>+echo "Overall results:"
>+echo "================"
>+echo "RESULTS (cbench-memtx-tree_result.txt):"
>+cat cbench-memtx-tree_result.txt
>+echo " "
>+echo "RESULTS (cbench-memtx-hash_result.txt):"
>+cat cbench-memtx-hash_result.txt
>+echo " "
>+echo "RESULTS (cbench-vinyl-fsync_result.txt):"
>+cat cbench-vinyl-fsync_result.txt
>+echo " "
>+echo "RESULTS (cbench-vinyl-write_result.txt):"
>+cat cbench-vinyl-write_result.txt
>+echo " "
>+echo "Publish data to bench database"
>+/opt/bench-run/benchs/publication/publish.py
>diff --git a/benchs/linkbench/run.sh b/benchs/linkbench/run.sh
>index 6738436..e7c77c3 100755
>--- a/benchs/linkbench/run.sh
>+++ b/benchs/linkbench/run.sh
>@@ -6,7 +6,6 @@ set -e
> set -o pipefail
>
> TAR_VER=$(tarantool -v | grep -e "Tarantool" | grep -oP '\s\K\S*')
>-echo ${TAR_VER} | tee linkbench_t_version.txt
>
> cd /opt/linkbench && mvn clean package -Dmaven.test.skip=true
> cd src/tarantool && make
>@@ -34,4 +33,14 @@ sync
> echo 3 > /proc/sys/vm/drop_caches
> $numaopts /opt/linkbench/bin/linkbench -c $cfgfile -r 2>&1 | tee linkbench_output.txt
>
>-grep "REQUEST PHASE COMPLETED" linkbench_output.txt | sed "s/.*second = /linkbench:/" | tee -a linkbench_result.txt
>+grep "REQUEST PHASE COMPLETED" linkbench_output.txt | sed "s/.*second = /linkbench:/" | tee -a linkbench.ssd_result.txt
>+echo ${TAR_VER} | tee linkbench.ssd_t_version.txt
>+
>+echo "Tarantool TAG:"
>+cat linkbench.ssd_t_version.txt
>+echo "Overall results:"
>+echo "================"
>+cat linkbench.ssd_result.txt
>+echo " "
>+echo "Publish data to bench database"
>+/opt/bench-run/benchs/publication/publish.py
>diff --git a/benchs/nosqlbench/run.sh b/benchs/nosqlbench/run.sh
>index c9491f5..a68248e 100755
>--- a/benchs/nosqlbench/run.sh
>+++ b/benchs/nosqlbench/run.sh
>@@ -6,13 +6,11 @@ if [ "$type" == "" ]; then
> fi
>
> TAR_VER=$(tarantool -v | grep -e "Tarantool" | grep -oP '\s\K\S*')
>-echo ${TAR_VER} | tee nosqlbench_t_version.txt
>
> kill `pidof tarantool`
>
>-#set -e
>-#
>-#set -o pipefail
>+set -e
>+set -o pipefail
>
> killall tarantool 2>/dev/null || true
> rm -rf 0*.xlog 0*.snap
>@@ -42,8 +40,14 @@ numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11 /opt/nosqlbench/
> #echo "Latest 1000 lines:"
> #tail -1000 nosqlbench_output.txt
>
>-echo "Getting results"
>-grep "TOTAL RPS STATISTICS:" nosqlbench_output.txt -A6 | awk -F "|" 'NR > 4 {print $2,":", $4}' > nosqlbench_result.txt
>-
>-cat nosqlbench_result.txt
>-
>+grep "TOTAL RPS STATISTICS:" nosqlbench_output.txt -A6 | awk -F "|" 'NR > 4 {print $2,":", $4}' > noSQLbench.${type}_result.txt
>+echo ${TAR_VER} | tee noSQLbench.${type}_t_version.txt
>+
>+echo "Tarantool TAG:"
>+cat noSQLbench.${type}_t_version.txt
>+echo "Overall results:"
>+echo "================"
>+cat noSQLbench.${type}_result.txt
>+echo " "
>+echo "Publish data to bench database"
>+/opt/bench-run/benchs/publication/publish.py
>diff --git a/benchs/publication/publish.py b/benchs/publication/publish.py
>new file mode 100755
>index 0000000..d472fd7
>--- /dev/null
>+++ b/benchs/publication/publish.py
>@@ -0,0 +1,67 @@
>+#!/usr/bin/env python
>+import fnmatch
>+import os
>+from urllib import urlencode
>+import requests
>+
>+
>+def parse_bench(filename):
>+ fileHandle = open(filename)
>+ lastline = fileHandle.readlines()
>+ fileHandle.close()
>+ return lastline
>+
>+
>+def get_version(filename):
>+ fileHandle = open(filename)
>+ lastline = fileHandle.readlines()[-1]
>+ fileHandle.close()
>+ return lastline.split()[0]
>+
>+
>+def push_to_microb(server, token, name, value, version, tab):
>+ uri = 'http://%s/push?%s' % (server, urlencode(dict(
>+ key=token, name=name, param=value,
>+ v=version, unit='trps', tab=tab
>+ )))
>+
>+ r = requests.get(uri)
>+ if r.status_code == 200:
>+ print('Export complete')
>+ else:
>+ print('Export error http: %d' % r.status_code)
>+ print('Export error text: %d' % r.text)
>+
>+
>+def main():
>+ if "MICROB_WEB_TOKEN" in os.environ and "MICROB_WEB_HOST" in os.environ:
>+ bench = {}
>+ res = []
>+ current_data = {}
>+ version = ''
>+ for file in os.listdir('.'):
>+ if fnmatch.fnmatch(file, '*_result.txt'):
>+ values = parse_bench(file)
>+ benchmark = file.split('_')[0]
>+ version = get_version('{}_t_version.txt'.format(benchmark))
>+ for value in values:
>+ test_name = value.split(':')[0]
>+ test_res = float(value.split(':')[1])
>+ res.append(test_res)
>+ push_to_microb(
>+ os.environ['MICROB_WEB_HOST'],
>+ os.environ['MICROB_WEB_TOKEN'],
>+ test_name,
>+ test_res,
>+ version,
>+ benchmark,
>+ )
>+ print ("VERSION - ", version)
>+ else:
>+ print("MICROB params not specified")
>+
>+ return 0
>+
>+
>+if __name__ == '__main__':
>+ main()
>diff --git a/benchs/sysbench/run.sh b/benchs/sysbench/run.sh
>index 6a69d14..ef0af43 100755
>--- a/benchs/sysbench/run.sh
>+++ b/benchs/sysbench/run.sh
>@@ -11,8 +11,6 @@ set -e
> set -o pipefail
>
> TAR_VER=$(tarantool -v | grep -e "Tarantool" | grep -oP '\s\K\S*')
>-echo ${TAR_VER} | tee sysbench_t_version.txt
>-set -e
>
> ARRAY_TESTS=(
> "oltp_read_only"
>@@ -44,11 +42,10 @@ opts="--db-driver=${DBMS} --threads=${THREADS}"
>
> export LD_LIBRARY_PATH=/usr/local/lib
>
>-rm -f sysbench_result.txt
>-set -o pipefail
>+rm -f Sysbench_result.txt
> for test in "${ARRAY_TESTS[@]}"; do
> res=0
>- tlog=sysbench_${test}_result.txt
>+ tlog=sysbench_${test}_results.txt
> rm -f $tlog
> maxres=0
> for run in `eval echo {1..$runs}` ; do
>@@ -69,7 +66,7 @@ for test in "${ARRAY_TESTS[@]}"; do
> if [[ $tres -gt $maxres ]]; then maxres=$tres ; fi
> done
> res=$(($res/$runs))
>- echo "${test}: $res" >>sysbench_result.txt
>+ echo "${test}: $res" >>Sysbench_result.txt
> echo "Subtest '$test' results:"
> echo "==============================="
>@@ -78,7 +75,13 @@ for test in "${ARRAY_TESTS[@]}"; do
> printf "Diviations (AVG -> MAX): %.2f" `bc <<< "scale = 4; (1 - $res / $maxres) * 100"` ; echo %
> done
>
>+echo ${TAR_VER} | tee Sysbench_t_version.txt
>+
>+echo "Tarantool TAG:"
>+cat Sysbench_t_version.txt
> echo "Overall results:"
> echo "================"
>-cat sysbench_result.txt
>-
>+cat Sysbench_result.txt
>+echo " "
>+echo "Publish data to bench database"
>+/opt/bench-run/benchs/publication/publish.py
>diff --git a/benchs/tpcc/run.sh b/benchs/tpcc/run.sh
>index f1f60cc..76754e5 100755
>--- a/benchs/tpcc/run.sh
>+++ b/benchs/tpcc/run.sh
>@@ -1,15 +1,12 @@
> #!/usr/bin/env bash
>+set -e
>+set -o pipefail
>
> TAR_VER=$(tarantool -v | grep -e "Tarantool" | grep -oP '\s\K\S*')
>-echo ${TAR_VER} | tee tpcc_t_version.txt
>-
>-set -e
>
> if [ ! -n "${TIME}" ]; then TIME=1200; fi
> if [ ! -n "${WARMUP_TIME}" ]; then WARMUP_TIME=10; fi
>
>-set -o pipefail
>-
> killall tarantool tpcc_load 2>/dev/null || true
> rm -rf 0*.xlog 0*.snap
> sync
>@@ -30,10 +27,16 @@ cd /opt/tpcc
> numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11 \
> /opt/tpcc/tpcc_start $tpcc_opts -r10 -l${TIME} -i${TIME} >tpcc_output.txt 2>/dev/null
>
>-echo -n "tpcc:" | tee tpcc_result.txt
>-cat tpcc_output.txt | grep -e '<TpmC>' | grep -oP '\K[0-9.]*' | tee -a tpcc_result.txt
>-
>+echo -n "tpcc:" | tee tpc.c_result.txt
>+cat tpcc_output.txt | grep -e '<TpmC>' | grep -oP '\K[0-9.]*' | tee -a tpc.c_result.txt
> cat tpcc_output.txt
>+echo ${TAR_VER} | tee tpc.c_t_version.txt
>+
>+echo "Tarantool TAG:"
>+cat tpc.c_t_version.txt
> echo "Overall result:"
> echo "==============="
>-cat tpcc_result.txt
>+cat tpc.c_result.txt
>+echo " "
>+echo "Publish data to bench database"
>+/opt/bench-run/benchs/publication/publish.py
>diff --git a/benchs/ycsb/run.sh b/benchs/ycsb/run.sh
>index 5ae1c75..d2cf8af 100755
>--- a/benchs/ycsb/run.sh
>+++ b/benchs/ycsb/run.sh
>@@ -17,7 +17,6 @@ set -e
> set -o pipefail
>
> TAR_VER=$(tarantool -v | grep -e "Tarantool" | grep -oP '\s\K\S*')
>-echo ${TAR_VER} | tee ycsb_t_version.txt
>
> ws=/opt/ycsb
> cd $ws
>@@ -50,7 +49,18 @@ for l in a b c d e f ; do
> echo 3 > /proc/sys/vm/drop_caches
> numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11 bin/ycsb run tarantool -s -P workloads/workload${l} >${res}.log 2>&1 || cat ${res}.log
> grep Thro ${res}.log | awk '{ print "Overall result: "$3 }' | tee ${res}.txt
>- sed "s#Overall result#$l $r#g" ${res}.txt >>${plogs}/results.txt
>+ sed "s#Overall result#$l $r#g" ${res}.txt >>${plogs}/ycsb.${mode}_result.txt
> done
> done
>-cat ${plogs}/results.txt
>+
>+echo ${TAR_VER} | tee ycsb.${mode}_t_version.txt
>+cp -f ${plogs}/ycsb.${mode}_result.txt .
>+
>+echo "Tarantool TAG:"
>+cat ycsb.${mode}_t_version.txt
>+echo "Overall results:"
>+echo "================"
>+cat ycsb.${mode}_result.txt
>+echo " "
>+echo "Publish data to bench database"
>+/opt/bench-run/benchs/publication/publish.py
>diff --git a/dockerfiles/ubuntu_benchs b/dockerfiles/ubuntu_benchs
>index fd9acb4..2c2e717 100644
>--- a/dockerfiles/ubuntu_benchs
>+++ b/dockerfiles/ubuntu_benchs
>@@ -71,5 +71,5 @@ RUN luarocks install \
> https://raw.githubusercontent.com/tarantool/gperftools/master/rockspecs/gperftools-scm-1.rockspec \
> --local >build.log 2>&1 || ( cat build.log && false )
>
>-# benchmarks runners
>-RUN git clone https://github.com/tarantool/bench-run.git /opt/bench-run
>+# adding python dependency
>+RUN pip install requests
>--
>1.8.3.1
>
--
Alexander Tikhonov
[-- Attachment #2: Type: text/html, Size: 17704 bytes --]
^ permalink raw reply [flat|nested] 8+ messages in thread
* Re: [Tarantool-patches] [PATCH v1] bench-run: submit perf results into database
2020-04-22 10:18 [Tarantool-patches] [PATCH v1] bench-run: submit perf results into database Олег Пискунов
2020-04-22 10:57 ` Alexander Tikhonov
@ 2020-04-23 9:28 ` Sergey Bronnikov
2020-04-27 12:28 ` Oleg Piskunov
1 sibling, 1 reply; 8+ messages in thread
From: Sergey Bronnikov @ 2020-04-23 9:28 UTC (permalink / raw)
To: Олег
Пискунов
Cc: Oleg Piskunov, tarantool-patches
Hello!
Oleg, thanks for the patch. See my 22 comments below.
On 13:18 Wed 22 Apr , Олег Пискунов wrote:
>
> Resurrect bench.tarantool.org:
> - submit perf results into bench.tarantool.org database
>
> Fix docker files for perf testing:
> - adding python "requests" module needed for publish script.
>
> Closes #4865
>
> Github: https://github.com/tarantool/bench-run/tree/opiskunov/gh-4865-submit-perf-results
> Issue: https://github.com/tarantool/tarantool/issues/4865
> ---
> benchs/cbench/run.sh | 38 ++++++++++++++++++------
> benchs/linkbench/run.sh | 13 +++++++--
> benchs/nosqlbench/run.sh | 22 ++++++++------
> benchs/publication/publish.py | 67 +++++++++++++++++++++++++++++++++++++++++++
> benchs/sysbench/run.sh | 19 ++++++------
> benchs/tpcc/run.sh | 21 ++++++++------
> benchs/ycsb/run.sh | 16 +++++++++--
> dockerfiles/ubuntu_benchs | 4 +--
> 8 files changed, 158 insertions(+), 42 deletions(-)
> create mode 100755 benchs/publication/publish.py
>
> diff --git a/benchs/cbench/run.sh b/benchs/cbench/run.sh
> index 4d07972..c4867ad 100755
> --- a/benchs/cbench/run.sh
> +++ b/benchs/cbench/run.sh
> @@ -1,11 +1,11 @@
> #!/usr/bin/env bash
>
> kill `pidof tarantool`
> +
> set -e
> set -o pipefail
>
> TAR_VER=$(tarantool -v | grep -e "Tarantool" | grep -oP '\s\K\S*')
> -echo ${TAR_VER} | tee cbench_t_version.txt
>
> killall tarantool 2>/dev/null || true
> rm -rf 5* 0*
> @@ -23,14 +23,34 @@ killall tarantool 2>/dev/null || true
> rm -rf 5* 0*
> sync
1. sync() can be asynchonous operation so we should wait it or somehow make
sure it has finished.
> echo 3 > /proc/sys/vm/drop_caches
> -numactl --membind=1 --cpunodebind=1 --physcpubind=11 tarantool /opt/cbench/cbench_runner.lua vinyl write 500 2>&1 | tee cbench_output_memtx_write.txt
> +numactl --membind=1 --cpunodebind=1 --physcpubind=11 tarantool /opt/cbench/cbench_runner.lua vinyl write 500 2>&1 | tee cbench_output_vinyl_write.txt
2. What does '500' mean? Let's move magic numbers to constants.
> -grep "^?tab" cbench_output_memtx.txt | sed "s/.*name=//"| sed "s/¶m=/:/"| sed "s/cb\./cb\.memtx\./"| tee -a cbench_result.txt
> -grep "^?tab" cbench_output_vinyl_fsync.txt | sed "s/.*name=//"| sed "s/¶m=/:/"| sed "s/cb\./cb\.vinyl\.fsync\./"| tee -a cbench_result.txt
> -grep "^?tab" cbench_output_memtx_write.txt | sed "s/.*name=//"| sed "s/¶m=/:/"| sed "s/cb\./cb\.vinyl\.write\./"| tee -a cbench_result.txt
> +grep "^?tab=cbench.tree" cbench_output_memtx.txt | sed "s/.*name=//"| sed "s/¶m=/:/"| sed "s/cb\./cb\.memtx\./"| tee -a cbench-memtx-tree_result.txt
> +grep "^?tab=cbench.hash" cbench_output_memtx.txt | sed "s/.*name=//"| sed "s/¶m=/:/"| sed "s/cb\./cb\.memtx\./"| tee -a cbench-memtx-hash_result.txt
> +grep "^?tab" cbench_output_vinyl_fsync.txt | sed "s/.*name=//"| sed "s/¶m=/:/"| sed "s/cb\./cb\.vinyl\.fsync\./"| tee -a cbench-vinyl-fsync_result.txt
> +grep "^?tab" cbench_output_vinyl_write.txt | sed "s/.*name=//"| sed "s/¶m=/:/"| sed "s/cb\./cb\.vinyl\.write\./"| tee -a cbench-vinyl-write_result.txt
3. As far as I understand cbench_runner.lua wrote two files
cbench_output_memtx_write.txt and cbench_output_vinyl_write.txt, you
pass filenames as arguments. On next step we should extract interested
numbers from these files. But cbench_output_memtx_write.txt is unused
here. And how other files were created: cbench_output_memtx.txt and
cbench_output_vinyl_fsync.txt?
4. Lines with grep/sed looks absolutely unreadable. Please split lines, like below:
grep "^?tab" cbench_output_memtx.txt |
sed "s/.*name=//"| sed "s/¶m=/:/"| sed "s/cb\./cb\.memtx\./" |
tee -a cbench_result.txt
grep "^?tab" cbench_output_vinyl_fsync.txt |
sed "s/.*name=//"| sed "s/¶m=/:/"| sed "s/cb\./cb\.vinyl\.fsync\./"|
tee -a cbench_result.txt
> -#mv tarantool-server.log cbench_tarantool_server.log
> -
> -echo "RESULTS:"
> -cat cbench_result.txt
> +echo ${TAR_VER} | tee cbench-memtx-tree_t_version.txt
> +echo ${TAR_VER} | tee cbench-memtx-hash_t_version.txt
> +echo ${TAR_VER} | tee cbench-vinyl-fsync_t_version.txt
> +echo ${TAR_VER} | tee cbench-vinyl-write_t_version.txt
> +echo ${TAR_VER} | tee cbench_t_version.txt
>
> +echo "Tarantool TAG:"
> +cat cbench_t_version.txt
> +echo "Overall results:"
> +echo "================"
> +echo "RESULTS (cbench-memtx-tree_result.txt):"
> +cat cbench-memtx-tree_result.txt
> +echo " "
> +echo "RESULTS (cbench-memtx-hash_result.txt):"
> +cat cbench-memtx-hash_result.txt
> +echo " "
> +echo "RESULTS (cbench-vinyl-fsync_result.txt):"
> +cat cbench-vinyl-fsync_result.txt
> +echo " "
> +echo "RESULTS (cbench-vinyl-write_result.txt):"
> +cat cbench-vinyl-write_result.txt
> +echo " "
> +echo "Publish data to bench database"
> +/opt/bench-run/benchs/publication/publish.py
5. Running of publish.py will broke if we will move repo to another
directory. Let's use relative path to publish.py.
> diff --git a/benchs/linkbench/run.sh b/benchs/linkbench/run.sh
> index 6738436..e7c77c3 100755
> --- a/benchs/linkbench/run.sh
> +++ b/benchs/linkbench/run.sh
> @@ -6,7 +6,6 @@ set -e
6. set -u
> set -o pipefail
>
> TAR_VER=$(tarantool -v | grep -e "Tarantool" | grep -oP '\s\K\S*')
> -echo ${TAR_VER} | tee linkbench_t_version.txt
>
> cd /opt/linkbench && mvn clean package -Dmaven.test.skip=true
> cd src/tarantool && make
> @@ -34,4 +33,14 @@ sync
7. sync() can be asynchonous operation, we should wait it to make sure
it has finished.
> echo 3 > /proc/sys/vm/drop_caches
> $numaopts /opt/linkbench/bin/linkbench -c $cfgfile -r 2>&1 | tee linkbench_output.txt
>
> -grep "REQUEST PHASE COMPLETED" linkbench_output.txt | sed "s/.*second = /linkbench:/" | tee -a linkbench_result.txt
> +grep "REQUEST PHASE COMPLETED" linkbench_output.txt | sed "s/.*second = /linkbench:/" | tee -a linkbench.ssd_result.txt
> +echo ${TAR_VER} | tee linkbench.ssd_t_version.txt
> +
> +echo "Tarantool TAG:"
> +cat linkbench.ssd_t_version.txt
> +echo "Overall results:"
> +echo "================"
> +cat linkbench.ssd_result.txt
> +echo " "
> +echo "Publish data to bench database"
> +/opt/bench-run/benchs/publication/publish.py
8. Running of publish.py will broke if we will move repo to another
directory. Let's use relative path to publish.py.
> diff --git a/benchs/nosqlbench/run.sh b/benchs/nosqlbench/run.sh
> index c9491f5..a68248e 100755
> --- a/benchs/nosqlbench/run.sh
> +++ b/benchs/nosqlbench/run.sh
> @@ -6,13 +6,11 @@ if [ "$type" == "" ]; then
> fi
>
> TAR_VER=$(tarantool -v | grep -e "Tarantool" | grep -oP '\s\K\S*')
> -echo ${TAR_VER} | tee nosqlbench_t_version.txt
>
> kill `pidof tarantool`
>
> -#set -e
> -#
> -#set -o pipefail
> +set -e
> +set -o pipefail
>
> killall tarantool 2>/dev/null || true
> rm -rf 0*.xlog 0*.snap
9. It is not clear why filemask includes zero at the beginning.
> @@ -42,8 +40,14 @@ numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11 /opt/nosqlbench/
> #echo "Latest 1000 lines:"
> #tail -1000 nosqlbench_output.txt
10. please remove commented lines or uncomment them
>
> -echo "Getting results"
> -grep "TOTAL RPS STATISTICS:" nosqlbench_output.txt -A6 | awk -F "|" 'NR > 4 {print $2,":", $4}' > nosqlbench_result.txt
> -
> -cat nosqlbench_result.txt
> -
> +grep "TOTAL RPS STATISTICS:" nosqlbench_output.txt -A6 | awk -F "|" 'NR > 4 {print $2,":", $4}' > noSQLbench.${type}_result.txt
> +echo ${TAR_VER} | tee noSQLbench.${type}_t_version.txt
> +
> +echo "Tarantool TAG:"
> +cat noSQLbench.${type}_t_version.txt
> +echo "Overall results:"
> +echo "================"
> +cat noSQLbench.${type}_result.txt
> +echo " "
> +echo "Publish data to bench database"
> +/opt/bench-run/benchs/publication/publish.py
> diff --git a/benchs/publication/publish.py b/benchs/publication/publish.py
> new file mode 100755
> index 0000000..d472fd7
> --- /dev/null
> +++ b/benchs/publication/publish.py
11. Running of publish.py will broke if we will move repo to another
directory. Let's use relative path to publish.py.
> @@ -0,0 +1,67 @@
> +#!/usr/bin/env python
> +import fnmatch
> +import os
> +from urllib import urlencode
> +import requests
> +
> +
> +def parse_bench(filename):
> + fileHandle = open(filename)
> + lastline = fileHandle.readlines()
> + fileHandle.close()
> + return lastline
12. you can implement the same body of function much simpler:
with open(filename) as raw_data:
return raw_data.readlines()
> +
> +
> +def get_version(filename):
> + fileHandle = open(filename)
> + lastline = fileHandle.readlines()[-1]
> + fileHandle.close()
> + return lastline.split()[0]
13. you can implement the same body of function much simpler:
with open(filename) as raw_data:
return raw_data.readlines()[-1][0]
> +def push_to_microb(server, token, name, value, version, tab):
> + uri = 'http://%s/push?%s' % (server, urlencode(dict(
> + key=token, name=name, param=value,
> + v=version, unit='trps', tab=tab
> + )))
> +
> + r = requests.get(uri)
> + if r.status_code == 200:
> + print('Export complete')
> + else:
> + print('Export error http: %d' % r.status_code)
> + print('Export error text: %d' % r.text)
> +
> +
> +def main():
> + if "MICROB_WEB_TOKEN" in os.environ and "MICROB_WEB_HOST" in os.environ:
> + bench = {}
> + res = []
> + current_data = {}
> + version = ''
> + for file in os.listdir('.'):
> + if fnmatch.fnmatch(file, '*_result.txt'):
> + values = parse_bench(file)
> + benchmark = file.split('_')[0]
> + version = get_version('{}_t_version.txt'.format(benchmark))
> + for value in values:
> + test_name = value.split(':')[0]
> + test_res = float(value.split(':')[1])
> + res.append(test_res)
> + push_to_microb(
> + os.environ['MICROB_WEB_HOST'],
> + os.environ['MICROB_WEB_TOKEN'],
> + test_name,
> + test_res,
> + version,
> + benchmark,
> + )
> + print ("VERSION - ", version)
> + else:
> + print("MICROB params not specified")
> +
> + return 0
> +
> +
> +if __name__ == '__main__':
> + main()
> diff --git a/benchs/sysbench/run.sh b/benchs/sysbench/run.sh
> index 6a69d14..ef0af43 100755
> --- a/benchs/sysbench/run.sh
> +++ b/benchs/sysbench/run.sh
> @@ -11,8 +11,6 @@ set -e
> set -o pipefail
>
> TAR_VER=$(tarantool -v | grep -e "Tarantool" | grep -oP '\s\K\S*')
> -echo ${TAR_VER} | tee sysbench_t_version.txt
> -set -e
>
> ARRAY_TESTS=(
> "oltp_read_only"
> @@ -44,11 +42,10 @@ opts="--db-driver=${DBMS} --threads=${THREADS}"
>
> export LD_LIBRARY_PATH=/usr/local/lib
>
> -rm -f sysbench_result.txt
> -set -o pipefail
> +rm -f Sysbench_result.txt
> for test in "${ARRAY_TESTS[@]}"; do
> res=0
> - tlog=sysbench_${test}_result.txt
> + tlog=sysbench_${test}_results.txt
> rm -f $tlog
> maxres=0
> for run in `eval echo {1..$runs}` ; do
> @@ -69,7 +66,7 @@ for test in "${ARRAY_TESTS[@]}"; do
> if [[ $tres -gt $maxres ]]; then maxres=$tres ; fi
> done
> res=$(($res/$runs))
> - echo "${test}: $res" >>sysbench_result.txt
> + echo "${test}: $res" >>Sysbench_result.txt
> echo "Subtest '$test' results:"
> echo "==============================="
> @@ -78,7 +75,13 @@ for test in "${ARRAY_TESTS[@]}"; do
> printf "Diviations (AVG -> MAX): %.2f" `bc <<< "scale = 4; (1 - $res / $maxres) * 100"` ; echo %
14. Deviations
> done
>
> +echo ${TAR_VER} | tee Sysbench_t_version.txt
> +
> +echo "Tarantool TAG:"
> +cat Sysbench_t_version.txt
> echo "Overall results:"
> echo "================"
> -cat sysbench_result.txt
> -
> +cat Sysbench_result.txt
> +echo " "
> +echo "Publish data to bench database"
> +/opt/bench-run/benchs/publication/publish.py
15. Running of publish.py will broke if we will move repo to another
directory. Let's use relative path to publish.py.
> diff --git a/benchs/tpcc/run.sh b/benchs/tpcc/run.sh
> index f1f60cc..76754e5 100755
> --- a/benchs/tpcc/run.sh
> +++ b/benchs/tpcc/run.sh
> @@ -1,15 +1,12 @@
> #!/usr/bin/env bash
> +set -e
16. set -u
> +set -o pipefail
>
> TAR_VER=$(tarantool -v | grep -e "Tarantool" | grep -oP '\s\K\S*')
> -echo ${TAR_VER} | tee tpcc_t_version.txt
> -
> -set -e
>
> if [ ! -n "${TIME}" ]; then TIME=1200; fi
> if [ ! -n "${WARMUP_TIME}" ]; then WARMUP_TIME=10; fi
>
> -set -o pipefail
> -
> killall tarantool tpcc_load 2>/dev/null || true
> rm -rf 0*.xlog 0*.snap
> sync
17. the same comment as above regarding sync() and filemask for *.xlog and *.snap
> @@ -30,10 +27,16 @@ cd /opt/tpcc
> numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11 \
> /opt/tpcc/tpcc_start $tpcc_opts -r10 -l${TIME} -i${TIME} >tpcc_output.txt 2>/dev/null
>
> -echo -n "tpcc:" | tee tpcc_result.txt
> -cat tpcc_output.txt | grep -e '<TpmC>' | grep -oP '\K[0-9.]*' | tee -a tpcc_result.txt
> -
> +echo -n "tpcc:" | tee tpc.c_result.txt
> +cat tpcc_output.txt | grep -e '<TpmC>' | grep -oP '\K[0-9.]*' | tee -a tpc.c_result.txt
> cat tpcc_output.txt
> +echo ${TAR_VER} | tee tpc.c_t_version.txt
> +
> +echo "Tarantool TAG:"
> +cat tpc.c_t_version.txt
> echo "Overall result:"
> echo "==============="
> -cat tpcc_result.txt
> +cat tpc.c_result.txt
> +echo " "
> +echo "Publish data to bench database"
> +/opt/bench-run/benchs/publication/publish.py
18. Running of publish.py will broke if we will move repo to another
directory. Let's use relative path to publish.py.
> diff --git a/benchs/ycsb/run.sh b/benchs/ycsb/run.sh
> index 5ae1c75..d2cf8af 100755
> --- a/benchs/ycsb/run.sh
> +++ b/benchs/ycsb/run.sh
> @@ -17,7 +17,6 @@ set -e
> set -o pipefail
>
> TAR_VER=$(tarantool -v | grep -e "Tarantool" | grep -oP '\s\K\S*')
> -echo ${TAR_VER} | tee ycsb_t_version.txt
>
> ws=/opt/ycsb
> cd $ws
> @@ -50,7 +49,18 @@ for l in a b c d e f ; do
> echo 3 > /proc/sys/vm/drop_caches
> numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11 bin/ycsb run tarantool -s -P workloads/workload${l} >${res}.log 2>&1 || cat ${res}.log
19. cbench script uses physcpubind equal to 1, why we set exact these
numbers and why physcpubind has not the same set of numbers?
20. What does "l" variable mean?
> grep Thro ${res}.log | awk '{ print "Overall result: "$3 }' | tee ${res}.txt
> - sed "s#Overall result#$l $r#g" ${res}.txt >>${plogs}/results.txt
> + sed "s#Overall result#$l $r#g" ${res}.txt >>${plogs}/ycsb.${mode}_result.txt
> done
> done
> -cat ${plogs}/results.txt
> +
> +echo ${TAR_VER} | tee ycsb.${mode}_t_version.txt
> +cp -f ${plogs}/ycsb.${mode}_result.txt .
> +
> +echo "Tarantool TAG:"
> +cat ycsb.${mode}_t_version.txt
> +echo "Overall results:"
> +echo "================"
> +cat ycsb.${mode}_result.txt
> +echo " "
> +echo "Publish data to bench database"
> +/opt/bench-run/benchs/publication/publish.py
21. Running of publish.py will broke if we will move repo to another
directory. Let's use relative path to publish.py.
> diff --git a/dockerfiles/ubuntu_benchs b/dockerfiles/ubuntu_benchs
> index fd9acb4..2c2e717 100644
> --- a/dockerfiles/ubuntu_benchs
> +++ b/dockerfiles/ubuntu_benchs
> @@ -71,5 +71,5 @@ RUN luarocks install \
> https://raw.githubusercontent.com/tarantool/gperftools/master/rockspecs/gperftools-scm-1.rockspec \
> --local >build.log 2>&1 || ( cat build.log && false )
22. Why do we need gperftools here?
> -# benchmarks runners
> -RUN git clone https://github.com/tarantool/bench-run.git /opt/bench-run
> +# adding python dependency
> +RUN pip install requests
> --
> 1.8.3.1
--
sergeyb@
^ permalink raw reply [flat|nested] 8+ messages in thread
* Re: [Tarantool-patches] [PATCH v1] bench-run: submit perf results into database
2020-04-23 9:28 ` Sergey Bronnikov
@ 2020-04-27 12:28 ` Oleg Piskunov
2020-04-29 10:07 ` Oleg Piskunov
0 siblings, 1 reply; 8+ messages in thread
From: Oleg Piskunov @ 2020-04-27 12:28 UTC (permalink / raw)
To: Sergey Bronnikov; +Cc: Oleg Piskunov, tarantool-patches
[-- Attachment #1: Type: text/plain, Size: 4002 bytes --]
Sergey, thanks a lot for the review.
>1. sync() can be asynchonous operation so we should wait it or somehow make
>sure it has finished.
>7. sync() can be asynchonous operation, we should wait it to make sure
>it has finished.
>17. the same comment as above regarding sync()
The «sync» behaviour can be asynchronous, however, modern Linux has been waiting for the recording to finish.
Any way I will add checking exit status from sync.
>2. What does '500' mean? Let's move magic numbers to constants.
Sure. Will fix it.
>3. As far as I understand cbench_runner.lua wrote two files
>cbench_output_memtx_write.txt and cbench_output_vinyl_write.txt, you
>pass filenames as arguments. On next step we should extract interested
>numbers from these files. But cbench_output_memtx_write.txt is unused
>here. And how other files were created: cbench_output_memtx.txt and
>cbench_output_vinyl_fsync.txt?
There are 3 runs for cbench with different parameters which produce 3 output files:
cbench_output_memtx.txt
cbench_output_vinyl_fsync.txt
cbench_output_vinyl_write.txt
which are parsed to 4 results files.
>4. Lines with grep/sed looks absolutely unreadable. Please split lines, like below:
> grep "^?tab" cbench_output_memtx.txt |
> sed "s/.*name=//"| sed "s/¶m=/:/"| sed "s/cb\./cb\.memtx\./" |
> tee -a cbench_result.txt
> grep "^?tab" cbench_output_vinyl_fsync.txt |
> sed "s/.*name=//"| sed "s/¶m=/:/"| sed "s/cb\./cb\.vinyl\.fsync\./"|
> tee -a cbench_result.txt
>6. set -u
>9. It is not clear why filemask includes zero at the beginning.
>10. please remove commented lines or uncomment them
>12. you can implement the same body of function much simpler:
> with open(filename) as raw_data:
> return raw_data.readlines()
>13. you can implement the same body of function much simpler:
> with open(filename) as raw_data:
> return raw_data.readlines()[-1][0] 14. Deviations
>16. set -u
>17(b). filemask for *.xlog and *.snap
Refactoring. I’ve added changes you suggested.
>5. Running of publish.py will broke if we will move repo to another
> directory. Let's use relative path to publish.py.
>8. Running of publish.py will broke if we will move repo to another
> directory. Let's use relative path to publish.py.
>11. Running of publish.py will broke if we will move repo to another
> directory. Let's use relative path to publish.py.
>15. Running of publish.py will broke if we will move repo to another
> directory. Let's use relative path to publish.py.
>18. Running of publish.py will broke if we will move repo to another
> directory. Let's use relative path to publish.py.
>21. Running of publish.py will broke if we will move repo to another
>directory. Let's use relative path to publish.py.
This absolute path from container root. As we agreed it will be kept.
>19. cbench script uses physcpubind equal to 1, why we set exact these
>numbers and why physcpubind has not the same set of numbers?
For performance testing we are using servers with 2 processors
and each processor have few (6) cores. For stable results we bind tasks
to single proc or even single core. The difference can be because
of bench requirement (as Tarantool can run multithread execution).
So this is still open question how to do correctly binding, but for now
I would suggest to keep this parameters unchanged.
>20. What does "l" variable mean?
This variables changes in range (for l in a b c d e f ; do) and runs different workloads:
-rw-r--r-- 1 root root 3014 Apr 24 07:24 workloada
-rw-r--r-- 1 root root 3050 Apr 24 07:24 workloadb
-rw-r--r-- 1 root root 3036 Apr 24 07:24 workloadc
-rw-r--r-- 1 root root 3399 Apr 24 07:24 workloadd
-rw-r--r-- 1 root root 3558 Apr 24 07:24 workloade
-rw-r--r-- 1 root root 3149 Apr 24 07:24 workloadf
>22. Why do we need gperftools here?
Possible artefact from debugging phase a long time ago. Doesn't need for perf testing anymore. Removing.
--
Oleg Piskunov
[-- Attachment #2: Type: text/html, Size: 5530 bytes --]
^ permalink raw reply [flat|nested] 8+ messages in thread
* Re: [Tarantool-patches] [PATCH v1] bench-run: submit perf results into database
2020-04-27 12:28 ` Oleg Piskunov
@ 2020-04-29 10:07 ` Oleg Piskunov
0 siblings, 0 replies; 8+ messages in thread
From: Oleg Piskunov @ 2020-04-29 10:07 UTC (permalink / raw)
To: Oleg Piskunov; +Cc: tarantool-patches
[-- Attachment #1: Type: text/plain, Size: 261 bytes --]
>
>>>22. Why do we need gperftools here?
>>Possible artefact from debugging phase a long time ago. Doesn't need for perf testing anymore. Removing.
>
>gperftoll needed for linkbench test. Will keep gperftools in docker file.
>
>--
>Oleg Piskunov
>
[-- Attachment #2: Type: text/html, Size: 891 bytes --]
^ permalink raw reply [flat|nested] 8+ messages in thread
* Re: [Tarantool-patches] [PATCH v1] bench-run: submit perf results into database
2020-04-20 17:42 ` Alexander Tikhonov
@ 2020-04-21 6:14 ` Oleg Piskunov
0 siblings, 0 replies; 8+ messages in thread
From: Oleg Piskunov @ 2020-04-21 6:14 UTC (permalink / raw)
To: Alexander Tikhonov
Cc: tarantool-patches,
Олег
Пискунов
[-- Attachment #1: Type: text/plain, Size: 16640 bytes --]
Hi Sasha, thanks for the review, I'm going to split and resend patch, I'll take into account your comments.
>Понедельник, 20 апреля 2020, 20:42 +03:00 от Alexander Tikhonov <avtikhon@tarantool.org>:
>
>Oleg, thanks for the patch, please check my questions below and minor corrections in the letter after:
>- as about scripts per benchmarks all changes are correct;
>- bench-run repository creation moved from dockerfile of the first stage to the second, may be it will be good to notice about it in the commit message;
>- fetching was specially set to bench-run while you already added ‘--no-cache’ option - fetching will never be used in such situation, due to ‘--no-cache’ option will block the docker from its cache use;
>- fetching was set to Tarantool sources, but it is not needed because again used ‘--no-cache’ option and the main cause that gitlab-ci fetches the Tarantool sources itself at the very start of each gitlab-ci job.
>
>>Воскресенье, 19 апреля 2020, 19:28 +03:00 от Олег Пискунов < o.piskunov@corp.mail.ru >:
>>
>>
>>Resurrect bench.tarantool.org:
>>- submit perf results into bench.tarantool.org database
>>
>>Fix docker files for perf testing:
>>- Disable cashing from docker file creation.
>>- adding python "requests" module needed for publish script.
>>
>>Closes #4865, #4870
>>---
>>
>>Github: https://github.com/tarantool/tarantool/tree/opiskunov/gh-4865-submit-perf-results
>Wrong link.
>>
>>Issue: https://github.com/tarantool/tarantool/issues/4865
>> https://github.com/tarantool/tarantool/issues/4870
>>---
>> benchs/cbench/run.sh | 38 ++++++++++++++++++------
>> benchs/linkbench/run.sh | 13 +++++++--
>> benchs/nosqlbench/run.sh | 22 ++++++++------
>> benchs/publication/publish.py | 67 +++++++++++++++++++++++++++++++++++++++++++
>> benchs/sysbench/run.sh | 19 ++++++------
>> benchs/tpcc/run.sh | 21 ++++++++------
>> benchs/ycsb/run.sh | 16 +++++++++--
>> dockerfiles/ubuntu_benchs | 4 +--
>> dockerfiles/ubuntu_tnt | 6 ++++
>> targets.mk | 2 +-
>> 10 files changed, 165 insertions(+), 43 deletions(-)
>> create mode 100755 benchs/publication/publish.py
>>
>>diff --git a/benchs/cbench/run.sh b/benchs/cbench/run.sh
>>index 4d07972..c4867ad 100755
>>--- a/benchs/cbench/run.sh
>>+++ b/benchs/cbench/run.sh
>>@@ -1,11 +1,11 @@
>> #!/usr/bin/env bash
>>
>> kill `pidof tarantool`
>>+
>> set -e
>> set -o pipefail
>>
>> TAR_VER=$(tarantool -v | grep -e "Tarantool" | grep -oP '\s\K\S*')
>>-echo ${TAR_VER} | tee cbench_t_version.txt
>>
>> killall tarantool 2>/dev/null || true
>> rm -rf 5* 0*
>>@@ -23,14 +23,34 @@ killall tarantool 2>/dev/null || true
>> rm -rf 5* 0*
>> sync
>> echo 3 > /proc/sys/vm/drop_caches
>>-numactl --membind=1 --cpunodebind=1 --physcpubind=11 tarantool /opt/cbench/cbench_runner.lua vinyl write 500 2>&1 | tee cbench_output_memtx_write.txt
>>+numactl --membind=1 --cpunodebind=1 --physcpubind=11 tarantool /opt/cbench/cbench_runner.lua vinyl write 500 2>&1 | tee cbench_output_vinyl_write.txt
>>
>>-grep "^?tab" cbench_output_memtx.txt | sed "s/.*name=//"| sed "s/¶m=/:/"| sed "s/cb\./cb\.memtx\./"| tee -a cbench_result.txt
>>-grep "^?tab" cbench_output_vinyl_fsync.txt | sed "s/.*name=//"| sed "s/¶m=/:/"| sed "s/cb\./cb\.vinyl\.fsync\./"| tee -a cbench_result.txt
>>-grep "^?tab" cbench_output_memtx_write.txt | sed "s/.*name=//"| sed "s/¶m=/:/"| sed "s/cb\./cb\.vinyl\.write\./"| tee -a cbench_result.txt
>>+grep "^?tab=cbench.tree" cbench_output_memtx.txt | sed "s/.*name=//"| sed "s/¶m=/:/"| sed "s/cb\./cb\.memtx\./"| tee -a cbench-memtx-tree_result.txt
>>+grep "^?tab=cbench.hash" cbench_output_memtx.txt | sed "s/.*name=//"| sed "s/¶m=/:/"| sed "s/cb\./cb\.memtx\./"| tee -a cbench-memtx-hash_result.txt
>>+grep "^?tab" cbench_output_vinyl_fsync.txt | sed "s/.*name=//"| sed "s/¶m=/:/"| sed "s/cb\./cb\.vinyl\.fsync\./"| tee -a cbench-vinyl-fsync_result.txt
>>+grep "^?tab" cbench_output_vinyl_write.txt | sed "s/.*name=//"| sed "s/¶m=/:/"| sed "s/cb\./cb\.vinyl\.write\./"| tee -a cbench-vinyl-write_result.txt
>>
>>-#mv tarantool-server.log cbench_tarantool_server.log
>>-
>>-echo "RESULTS:"
>>-cat cbench_result.txt
>>+echo ${TAR_VER} | tee cbench-memtx-tree_t_version.txt
>>+echo ${TAR_VER} | tee cbench-memtx-hash_t_version.txt
>>+echo ${TAR_VER} | tee cbench-vinyl-fsync_t_version.txt
>>+echo ${TAR_VER} | tee cbench-vinyl-write_t_version.txt
>>+echo ${TAR_VER} | tee cbench_t_version.txt
>>
>>+echo "Tarantool TAG:"
>>+cat cbench_t_version.txt
>>+echo "Overall results:"
>>+echo "================"
>>+echo "RESULTS (cbench-memtx-tree_result.txt):"
>>+cat cbench-memtx-tree_result.txt
>>+echo " "
>>+echo "RESULTS (cbench-memtx-hash_result.txt):"
>>+cat cbench-memtx-hash_result.txt
>>+echo " "
>>+echo "RESULTS (cbench-vinyl-fsync_result.txt):"
>>+cat cbench-vinyl-fsync_result.txt
>>+echo " "
>>+echo "RESULTS (cbench-vinyl-write_result.txt):"
>>+cat cbench-vinyl-write_result.txt
>>+echo " "
>>+echo "Publish data to bench database"
>>+/opt/bench-run/benchs/publication/publish.py
>>diff --git a/benchs/linkbench/run.sh b/benchs/linkbench/run.sh
>>index 6738436..e7c77c3 100755
>>--- a/benchs/linkbench/run.sh
>>+++ b/benchs/linkbench/run.sh
>>@@ -6,7 +6,6 @@ set -e
>> set -o pipefail
>>
>> TAR_VER=$(tarantool -v | grep -e "Tarantool" | grep -oP '\s\K\S*')
>>-echo ${TAR_VER} | tee linkbench_t_version.txt
>>
>> cd /opt/linkbench && mvn clean package -Dmaven.test.skip=true
>> cd src/tarantool && make
>>@@ -34,4 +33,14 @@ sync
>> echo 3 > /proc/sys/vm/drop_caches
>> $numaopts /opt/linkbench/bin/linkbench -c $cfgfile -r 2>&1 | tee linkbench_output.txt
>>
>>-grep "REQUEST PHASE COMPLETED" linkbench_output.txt | sed "s/.*second = /linkbench:/" | tee -a linkbench_result.txt
>>+grep "REQUEST PHASE COMPLETED" linkbench_output.txt | sed "s/.*second = /linkbench:/" | tee -a linkbench.ssd_result.txt
>>+echo ${TAR_VER} | tee linkbench.ssd_t_version.txt
>>+
>>+echo "Tarantool TAG:"
>>+cat linkbench.ssd_t_version.txt
>>+echo "Overall results:"
>>+echo "================"
>>+cat linkbench.ssd_result.txt
>>+echo " "
>>+echo "Publish data to bench database"
>>+/opt/bench-run/benchs/publication/publish.py
>>diff --git a/benchs/nosqlbench/run.sh b/benchs/nosqlbench/run.sh
>>index c9491f5..a68248e 100755
>>--- a/benchs/nosqlbench/run.sh
>>+++ b/benchs/nosqlbench/run.sh
>>@@ -6,13 +6,11 @@ if [ "$type" == "" ]; then
>> fi
>>
>> TAR_VER=$(tarantool -v | grep -e "Tarantool" | grep -oP '\s\K\S*')
>>-echo ${TAR_VER} | tee nosqlbench_t_version.txt
>>
>> kill `pidof tarantool`
>>
>>-#set -e
>>-#
>>-#set -o pipefail
>>+set -e
>>+set -o pipefail
>>
>> killall tarantool 2>/dev/null || true
>> rm -rf 0*.xlog 0*.snap
>>@@ -42,8 +40,14 @@ numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11 /opt/nosqlbench/
>> #echo "Latest 1000 lines:"
>> #tail -1000 nosqlbench_output.txt
>>
>>-echo "Getting results"
>>-grep "TOTAL RPS STATISTICS:" nosqlbench_output.txt -A6 | awk -F "|" 'NR > 4 {print $2,":", $4}' > nosqlbench_result.txt
>>-
>>-cat nosqlbench_result.txt
>>-
>>+grep "TOTAL RPS STATISTICS:" nosqlbench_output.txt -A6 | awk -F "|" 'NR > 4 {print $2,":", $4}' > noSQLbench.${type}_result.txt
>>+echo ${TAR_VER} | tee noSQLbench.${type}_t_version.txt
>>+
>>+echo "Tarantool TAG:"
>>+cat noSQLbench.${type}_t_version.txt
>>+echo "Overall results:"
>>+echo "================"
>>+cat noSQLbench.${type}_result.txt
>>+echo " "
>>+echo "Publish data to bench database"
>>+/opt/bench-run/benchs/publication/publish.py
>>diff --git a/benchs/publication/publish.py b/benchs/publication/publish.py
>>new file mode 100755
>>index 0000000..d472fd7
>>--- /dev/null
>>+++ b/benchs/publication/publish.py
>>@@ -0,0 +1,67 @@
>>+#!/usr/bin/env python
>>+import fnmatch
>>+import os
>>+from urllib import urlencode
>>+import requests
>>+
>>+
>>+def parse_bench(filename):
>>+ fileHandle = open(filename)
>>+ lastline = fileHandle.readlines()
>>+ fileHandle.close()
>>+ return lastline
>>+
>>+
>>+def get_version(filename):
>>+ fileHandle = open(filename)
>>+ lastline = fileHandle.readlines()[-1]
>>+ fileHandle.close()
>>+ return lastline.split()[0]
>>+
>>+
>>+def push_to_microb(server, token, name, value, version, tab):
>>+ uri = 'http://%s/push?%s' % (server, urlencode(dict(
>>+ key=token, name=name, param=value,
>>+ v=version, unit='trps', tab=tab
>>+ )))
>>+
>>+ r = requests.get(uri)
>>+ if r.status_code == 200:
>>+ print('Export complete')
>>+ else:
>>+ print('Export error http: %d' % r.status_code)
>>+ print('Export error text: %d' % r.text)
>>+
>>+
>>+def main():
>>+ if "MICROB_WEB_TOKEN" in os.environ and "MICROB_WEB_HOST" in os.environ:
>>+ bench = {}
>>+ res = []
>>+ current_data = {}
>>+ version = ''
>>+ for file in os.listdir('.'):
>>+ if fnmatch.fnmatch(file, '*_result.txt'):
>>+ values = parse_bench(file)
>>+ benchmark = file.split('_')[0]
>>+ version = get_version('{}_t_version.txt'.format(benchmark))
>>+ for value in values:
>>+ test_name = value.split(':')[0]
>>+ test_res = float(value.split(':')[1])
>>+ res.append(test_res)
>>+ push_to_microb(
>>+ os.environ['MICROB_WEB_HOST'],
>>+ os.environ['MICROB_WEB_TOKEN'],
>>+ test_name,
>>+ test_res,
>>+ version,
>>+ benchmark,
>>+ )
>>+ print ("VERSION - ", version)
>>+ else:
>>+ print("MICROB params not specified")
>>+
>>+ return 0
>>+
>>+
>>+if __name__ == '__main__':
>>+ main()
>>diff --git a/benchs/sysbench/run.sh b/benchs/sysbench/run.sh
>>index 6a69d14..ef0af43 100755
>>--- a/benchs/sysbench/run.sh
>>+++ b/benchs/sysbench/run.sh
>>@@ -11,8 +11,6 @@ set -e
>> set -o pipefail
>>
>> TAR_VER=$(tarantool -v | grep -e "Tarantool" | grep -oP '\s\K\S*')
>>-echo ${TAR_VER} | tee sysbench_t_version.txt
>>-set -e
>>
>> ARRAY_TESTS=(
>> "oltp_read_only"
>>@@ -44,11 +42,10 @@ opts="--db-driver=${DBMS} --threads=${THREADS}"
>>
>> export LD_LIBRARY_PATH=/usr/local/lib
>>
>>-rm -f sysbench_result.txt
>>-set -o pipefail
>>+rm -f Sysbench_result.txt
>> for test in "${ARRAY_TESTS[@]}"; do
>> res=0
>>- tlog=sysbench_${test}_result.txt
>>+ tlog=sysbench_${test}_results.txt
>> rm -f $tlog
>> maxres=0
>> for run in `eval echo {1..$runs}` ; do
>>@@ -69,7 +66,7 @@ for test in "${ARRAY_TESTS[@]}"; do
>> if [[ $tres -gt $maxres ]]; then maxres=$tres ; fi
>> done
>> res=$(($res/$runs))
>>- echo "${test}: $res" >>sysbench_result.txt
>>+ echo "${test}: $res" >>Sysbench_result.txt
>> echo "Subtest '$test' results:"
>> echo "==============================="
>>@@ -78,7 +75,13 @@ for test in "${ARRAY_TESTS[@]}"; do
>> printf "Diviations (AVG -> MAX): %.2f" `bc <<< "scale = 4; (1 - $res / $maxres) * 100"` ; echo %
>> done
>>
>>+echo ${TAR_VER} | tee Sysbench_t_version.txt
>>+
>>+echo "Tarantool TAG:"
>>+cat Sysbench_t_version.txt
>> echo "Overall results:"
>> echo "================"
>>-cat sysbench_result.txt
>>-
>>+cat Sysbench_result.txt
>>+echo " "
>>+echo "Publish data to bench database"
>>+/opt/bench-run/benchs/publication/publish.py
>>diff --git a/benchs/tpcc/run.sh b/benchs/tpcc/run.sh
>>index f1f60cc..76754e5 100755
>>--- a/benchs/tpcc/run.sh
>>+++ b/benchs/tpcc/run.sh
>>@@ -1,15 +1,12 @@
>> #!/usr/bin/env bash
>>+set -e
>>+set -o pipefail
>>
>> TAR_VER=$(tarantool -v | grep -e "Tarantool" | grep -oP '\s\K\S*')
>>-echo ${TAR_VER} | tee tpcc_t_version.txt
>>-
>>-set -e
>>
>> if [ ! -n "${TIME}" ]; then TIME=1200; fi
>> if [ ! -n "${WARMUP_TIME}" ]; then WARMUP_TIME=10; fi
>>
>>-set -o pipefail
>>-
>> killall tarantool tpcc_load 2>/dev/null || true
>> rm -rf 0*.xlog 0*.snap
>> sync
>>@@ -30,10 +27,16 @@ cd /opt/tpcc
>> numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11 \
>> /opt/tpcc/tpcc_start $tpcc_opts -r10 -l${TIME} -i${TIME} >tpcc_output.txt 2>/dev/null
>>
>>-echo -n "tpcc:" | tee tpcc_result.txt
>>-cat tpcc_output.txt | grep -e '<TpmC>' | grep -oP '\K[0-9.]*' | tee -a tpcc_result.txt
>>-
>>+echo -n "tpcc:" | tee tpc.c_result.txt
>>+cat tpcc_output.txt | grep -e '<TpmC>' | grep -oP '\K[0-9.]*' | tee -a tpc.c_result.txt
>> cat tpcc_output.txt
>>+echo ${TAR_VER} | tee tpc.c_t_version.txt
>>+
>>+echo "Tarantool TAG:"
>>+cat tpc.c_t_version.txt
>> echo "Overall result:"
>> echo "==============="
>>-cat tpcc_result.txt
>>+cat tpc.c_result.txt
>>+echo " "
>>+echo "Publish data to bench database"
>>+/opt/bench-run/benchs/publication/publish.py
>>diff --git a/benchs/ycsb/run.sh b/benchs/ycsb/run.sh
>>index 5ae1c75..d2cf8af 100755
>>--- a/benchs/ycsb/run.sh
>>+++ b/benchs/ycsb/run.sh
>>@@ -17,7 +17,6 @@ set -e
>> set -o pipefail
>>
>> TAR_VER=$(tarantool -v | grep -e "Tarantool" | grep -oP '\s\K\S*')
>>-echo ${TAR_VER} | tee ycsb_t_version.txt
>>
>> ws=/opt/ycsb
>> cd $ws
>>@@ -50,7 +49,18 @@ for l in a b c d e f ; do
>> echo 3 > /proc/sys/vm/drop_caches
>> numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11 bin/ycsb run tarantool -s -P workloads/workload${l} >${res}.log 2>&1 || cat ${res}.log
>> grep Thro ${res}.log | awk '{ print "Overall result: "$3 }' | tee ${res}.txt
>>- sed "s#Overall result#$l $r#g" ${res}.txt >>${plogs}/results.txt
>>+ sed "s#Overall result#$l $r#g" ${res}.txt >>${plogs}/ycsb.${mode}_result.txt
>> done
>> done
>>-cat ${plogs}/results.txt
>>+
>>+echo ${TAR_VER} | tee ycsb.${mode}_t_version.txt
>>+cp -f ${plogs}/ycsb.${mode}_result.txt .
>>+
>>+echo "Tarantool TAG:"
>>+cat ycsb.${mode}_t_version.txt
>>+echo "Overall results:"
>>+echo "================"
>>+cat ycsb.${mode}_result.txt
>>+echo " "
>>+echo "Publish data to bench database"
>>+/opt/bench-run/benchs/publication/publish.py
>>diff --git a/dockerfiles/ubuntu_benchs b/dockerfiles/ubuntu_benchs
>>index fd9acb4..2c2e717 100644
>>--- a/dockerfiles/ubuntu_benchs
>>+++ b/dockerfiles/ubuntu_benchs
>>@@ -71,5 +71,5 @@ RUN luarocks install \
>> https://raw.githubusercontent.com/tarantool/gperftools/master/rockspecs/gperftools-scm-1.rockspec \
>> --local >build.log 2>&1 || ( cat build.log && false )
>>
>>-# benchmarks runners
>>-RUN git clone https://github.com/tarantool/bench-run.git /opt/bench-run
>>+# adding python dependency
>>+RUN pip install requests
>>diff --git a/dockerfiles/ubuntu_tnt b/dockerfiles/ubuntu_tnt
>>index 361e9d6..c1657ff 100644
>>--- a/dockerfiles/ubuntu_tnt
>>+++ b/dockerfiles/ubuntu_tnt
>>@@ -3,11 +3,17 @@ FROM ${image_from}
>>
>> COPY . /opt/tarantool
>> WORKDIR /opt/tarantool
>>+RUN git fetch --all
>> RUN git submodule update --recursive --init --force
>> RUN ( cmake . -DENABLE_DIST=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo >build.log 2>&1 \
>> && make -j >>build.log 2>&1 && make install >>build.log 2>&1 ) || \
>> ( cat build.log && false )
>>
>>+# benchmarks runners
>>+RUN git clone https://github.com/tarantool/bench-run.git /opt/bench-run
>>+WORKDIR /opt/bench-run
>>+RUN git fetch --all
>>+
>> # cbench
>> RUN git clone https://github.com/tarantool/cbench.git /opt/cbench
>> WORKDIR /opt/cbench
>>diff --git a/targets.mk b/targets.mk
>>index 6cf36b6..581b722 100644
>>--- a/targets.mk
>>+++ b/targets.mk
>>@@ -21,7 +21,7 @@ prepare:
>> docker push ${IMAGE_PERF}
>> # build Tarantool and benchmarks with depends on Tarantool sources
>> ${DOCKERFILE_BUILD} --build-arg image_from=${IMAGE_PERF} \
>>- -t ${IMAGE_PERF_BUILT} -f bench-run/dockerfiles/ubuntu_tnt .
>>+ -t ${IMAGE_PERF_BUILT} --no-cache -f bench-run/dockerfiles/ubuntu_tnt .
>> docker push ${IMAGE_PERF_BUILT}
>>
>> # #####################################################
>>--
>>1.8.3.1
>>
>
>
>--
>Alexander Tikhonov
>
--
Oleg Piskunov
[-- Attachment #2: Type: text/html, Size: 22084 bytes --]
^ permalink raw reply [flat|nested] 8+ messages in thread
* Re: [Tarantool-patches] [PATCH v1] bench-run: submit perf results into database
2020-04-19 16:28 Олег Пискунов
@ 2020-04-20 17:42 ` Alexander Tikhonov
2020-04-21 6:14 ` Oleg Piskunov
0 siblings, 1 reply; 8+ messages in thread
From: Alexander Tikhonov @ 2020-04-20 17:42 UTC (permalink / raw)
To: Олег
Пискунов
Cc: Oleg Piskunov, tarantool-patches
[-- Attachment #1: Type: text/plain, Size: 15963 bytes --]
Oleg, thanks for the patch, please check my questions below and minor corrections in the letter after:
- as about scripts per benchmarks all changes are correct;
- bench-run repository creation moved from dockerfile of the first stage to the second, may be it will be good to notice about it in the commit message;
- fetching was specially set to bench-run while you already added ‘--no-cache’ option - fetching will never be used in such situation, due to ‘--no-cache’ option will block the docker from its cache use;
- fetching was set to Tarantool sources, but it is not needed because again used ‘--no-cache’ option and the main cause that gitlab-ci fetches the Tarantool sources itself at the very start of each gitlab-ci job.
>Воскресенье, 19 апреля 2020, 19:28 +03:00 от Олег Пискунов <o.piskunov@corp.mail.ru>:
>
>
>Resurrect bench.tarantool.org:
>- submit perf results into bench.tarantool.org database
>
>Fix docker files for perf testing:
>- Disable cashing from docker file creation.
>- adding python "requests" module needed for publish script.
>
>Closes #4865, #4870
>---
>
>Github: https://github.com/tarantool/tarantool/tree/opiskunov/gh-4865-submit-perf-results
Wrong link.
>
>Issue: https://github.com/tarantool/tarantool/issues/4865
> https://github.com/tarantool/tarantool/issues/4870
>---
> benchs/cbench/run.sh | 38 ++++++++++++++++++------
> benchs/linkbench/run.sh | 13 +++++++--
> benchs/nosqlbench/run.sh | 22 ++++++++------
> benchs/publication/publish.py | 67 +++++++++++++++++++++++++++++++++++++++++++
> benchs/sysbench/run.sh | 19 ++++++------
> benchs/tpcc/run.sh | 21 ++++++++------
> benchs/ycsb/run.sh | 16 +++++++++--
> dockerfiles/ubuntu_benchs | 4 +--
> dockerfiles/ubuntu_tnt | 6 ++++
> targets.mk | 2 +-
> 10 files changed, 165 insertions(+), 43 deletions(-)
> create mode 100755 benchs/publication/publish.py
>
>diff --git a/benchs/cbench/run.sh b/benchs/cbench/run.sh
>index 4d07972..c4867ad 100755
>--- a/benchs/cbench/run.sh
>+++ b/benchs/cbench/run.sh
>@@ -1,11 +1,11 @@
> #!/usr/bin/env bash
>
> kill `pidof tarantool`
>+
> set -e
> set -o pipefail
>
> TAR_VER=$(tarantool -v | grep -e "Tarantool" | grep -oP '\s\K\S*')
>-echo ${TAR_VER} | tee cbench_t_version.txt
>
> killall tarantool 2>/dev/null || true
> rm -rf 5* 0*
>@@ -23,14 +23,34 @@ killall tarantool 2>/dev/null || true
> rm -rf 5* 0*
> sync
> echo 3 > /proc/sys/vm/drop_caches
>-numactl --membind=1 --cpunodebind=1 --physcpubind=11 tarantool /opt/cbench/cbench_runner.lua vinyl write 500 2>&1 | tee cbench_output_memtx_write.txt
>+numactl --membind=1 --cpunodebind=1 --physcpubind=11 tarantool /opt/cbench/cbench_runner.lua vinyl write 500 2>&1 | tee cbench_output_vinyl_write.txt
>
>-grep "^?tab" cbench_output_memtx.txt | sed "s/.*name=//"| sed "s/¶m=/:/"| sed "s/cb\./cb\.memtx\./"| tee -a cbench_result.txt
>-grep "^?tab" cbench_output_vinyl_fsync.txt | sed "s/.*name=//"| sed "s/¶m=/:/"| sed "s/cb\./cb\.vinyl\.fsync\./"| tee -a cbench_result.txt
>-grep "^?tab" cbench_output_memtx_write.txt | sed "s/.*name=//"| sed "s/¶m=/:/"| sed "s/cb\./cb\.vinyl\.write\./"| tee -a cbench_result.txt
>+grep "^?tab=cbench.tree" cbench_output_memtx.txt | sed "s/.*name=//"| sed "s/¶m=/:/"| sed "s/cb\./cb\.memtx\./"| tee -a cbench-memtx-tree_result.txt
>+grep "^?tab=cbench.hash" cbench_output_memtx.txt | sed "s/.*name=//"| sed "s/¶m=/:/"| sed "s/cb\./cb\.memtx\./"| tee -a cbench-memtx-hash_result.txt
>+grep "^?tab" cbench_output_vinyl_fsync.txt | sed "s/.*name=//"| sed "s/¶m=/:/"| sed "s/cb\./cb\.vinyl\.fsync\./"| tee -a cbench-vinyl-fsync_result.txt
>+grep "^?tab" cbench_output_vinyl_write.txt | sed "s/.*name=//"| sed "s/¶m=/:/"| sed "s/cb\./cb\.vinyl\.write\./"| tee -a cbench-vinyl-write_result.txt
>
>-#mv tarantool-server.log cbench_tarantool_server.log
>-
>-echo "RESULTS:"
>-cat cbench_result.txt
>+echo ${TAR_VER} | tee cbench-memtx-tree_t_version.txt
>+echo ${TAR_VER} | tee cbench-memtx-hash_t_version.txt
>+echo ${TAR_VER} | tee cbench-vinyl-fsync_t_version.txt
>+echo ${TAR_VER} | tee cbench-vinyl-write_t_version.txt
>+echo ${TAR_VER} | tee cbench_t_version.txt
>
>+echo "Tarantool TAG:"
>+cat cbench_t_version.txt
>+echo "Overall results:"
>+echo "================"
>+echo "RESULTS (cbench-memtx-tree_result.txt):"
>+cat cbench-memtx-tree_result.txt
>+echo " "
>+echo "RESULTS (cbench-memtx-hash_result.txt):"
>+cat cbench-memtx-hash_result.txt
>+echo " "
>+echo "RESULTS (cbench-vinyl-fsync_result.txt):"
>+cat cbench-vinyl-fsync_result.txt
>+echo " "
>+echo "RESULTS (cbench-vinyl-write_result.txt):"
>+cat cbench-vinyl-write_result.txt
>+echo " "
>+echo "Publish data to bench database"
>+/opt/bench-run/benchs/publication/publish.py
>diff --git a/benchs/linkbench/run.sh b/benchs/linkbench/run.sh
>index 6738436..e7c77c3 100755
>--- a/benchs/linkbench/run.sh
>+++ b/benchs/linkbench/run.sh
>@@ -6,7 +6,6 @@ set -e
> set -o pipefail
>
> TAR_VER=$(tarantool -v | grep -e "Tarantool" | grep -oP '\s\K\S*')
>-echo ${TAR_VER} | tee linkbench_t_version.txt
>
> cd /opt/linkbench && mvn clean package -Dmaven.test.skip=true
> cd src/tarantool && make
>@@ -34,4 +33,14 @@ sync
> echo 3 > /proc/sys/vm/drop_caches
> $numaopts /opt/linkbench/bin/linkbench -c $cfgfile -r 2>&1 | tee linkbench_output.txt
>
>-grep "REQUEST PHASE COMPLETED" linkbench_output.txt | sed "s/.*second = /linkbench:/" | tee -a linkbench_result.txt
>+grep "REQUEST PHASE COMPLETED" linkbench_output.txt | sed "s/.*second = /linkbench:/" | tee -a linkbench.ssd_result.txt
>+echo ${TAR_VER} | tee linkbench.ssd_t_version.txt
>+
>+echo "Tarantool TAG:"
>+cat linkbench.ssd_t_version.txt
>+echo "Overall results:"
>+echo "================"
>+cat linkbench.ssd_result.txt
>+echo " "
>+echo "Publish data to bench database"
>+/opt/bench-run/benchs/publication/publish.py
>diff --git a/benchs/nosqlbench/run.sh b/benchs/nosqlbench/run.sh
>index c9491f5..a68248e 100755
>--- a/benchs/nosqlbench/run.sh
>+++ b/benchs/nosqlbench/run.sh
>@@ -6,13 +6,11 @@ if [ "$type" == "" ]; then
> fi
>
> TAR_VER=$(tarantool -v | grep -e "Tarantool" | grep -oP '\s\K\S*')
>-echo ${TAR_VER} | tee nosqlbench_t_version.txt
>
> kill `pidof tarantool`
>
>-#set -e
>-#
>-#set -o pipefail
>+set -e
>+set -o pipefail
>
> killall tarantool 2>/dev/null || true
> rm -rf 0*.xlog 0*.snap
>@@ -42,8 +40,14 @@ numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11 /opt/nosqlbench/
> #echo "Latest 1000 lines:"
> #tail -1000 nosqlbench_output.txt
>
>-echo "Getting results"
>-grep "TOTAL RPS STATISTICS:" nosqlbench_output.txt -A6 | awk -F "|" 'NR > 4 {print $2,":", $4}' > nosqlbench_result.txt
>-
>-cat nosqlbench_result.txt
>-
>+grep "TOTAL RPS STATISTICS:" nosqlbench_output.txt -A6 | awk -F "|" 'NR > 4 {print $2,":", $4}' > noSQLbench.${type}_result.txt
>+echo ${TAR_VER} | tee noSQLbench.${type}_t_version.txt
>+
>+echo "Tarantool TAG:"
>+cat noSQLbench.${type}_t_version.txt
>+echo "Overall results:"
>+echo "================"
>+cat noSQLbench.${type}_result.txt
>+echo " "
>+echo "Publish data to bench database"
>+/opt/bench-run/benchs/publication/publish.py
>diff --git a/benchs/publication/publish.py b/benchs/publication/publish.py
>new file mode 100755
>index 0000000..d472fd7
>--- /dev/null
>+++ b/benchs/publication/publish.py
>@@ -0,0 +1,67 @@
>+#!/usr/bin/env python
>+import fnmatch
>+import os
>+from urllib import urlencode
>+import requests
>+
>+
>+def parse_bench(filename):
>+ fileHandle = open(filename)
>+ lastline = fileHandle.readlines()
>+ fileHandle.close()
>+ return lastline
>+
>+
>+def get_version(filename):
>+ fileHandle = open(filename)
>+ lastline = fileHandle.readlines()[-1]
>+ fileHandle.close()
>+ return lastline.split()[0]
>+
>+
>+def push_to_microb(server, token, name, value, version, tab):
>+ uri = 'http://%s/push?%s' % (server, urlencode(dict(
>+ key=token, name=name, param=value,
>+ v=version, unit='trps', tab=tab
>+ )))
>+
>+ r = requests.get(uri)
>+ if r.status_code == 200:
>+ print('Export complete')
>+ else:
>+ print('Export error http: %d' % r.status_code)
>+ print('Export error text: %d' % r.text)
>+
>+
>+def main():
>+ if "MICROB_WEB_TOKEN" in os.environ and "MICROB_WEB_HOST" in os.environ:
>+ bench = {}
>+ res = []
>+ current_data = {}
>+ version = ''
>+ for file in os.listdir('.'):
>+ if fnmatch.fnmatch(file, '*_result.txt'):
>+ values = parse_bench(file)
>+ benchmark = file.split('_')[0]
>+ version = get_version('{}_t_version.txt'.format(benchmark))
>+ for value in values:
>+ test_name = value.split(':')[0]
>+ test_res = float(value.split(':')[1])
>+ res.append(test_res)
>+ push_to_microb(
>+ os.environ['MICROB_WEB_HOST'],
>+ os.environ['MICROB_WEB_TOKEN'],
>+ test_name,
>+ test_res,
>+ version,
>+ benchmark,
>+ )
>+ print ("VERSION - ", version)
>+ else:
>+ print("MICROB params not specified")
>+
>+ return 0
>+
>+
>+if __name__ == '__main__':
>+ main()
>diff --git a/benchs/sysbench/run.sh b/benchs/sysbench/run.sh
>index 6a69d14..ef0af43 100755
>--- a/benchs/sysbench/run.sh
>+++ b/benchs/sysbench/run.sh
>@@ -11,8 +11,6 @@ set -e
> set -o pipefail
>
> TAR_VER=$(tarantool -v | grep -e "Tarantool" | grep -oP '\s\K\S*')
>-echo ${TAR_VER} | tee sysbench_t_version.txt
>-set -e
>
> ARRAY_TESTS=(
> "oltp_read_only"
>@@ -44,11 +42,10 @@ opts="--db-driver=${DBMS} --threads=${THREADS}"
>
> export LD_LIBRARY_PATH=/usr/local/lib
>
>-rm -f sysbench_result.txt
>-set -o pipefail
>+rm -f Sysbench_result.txt
> for test in "${ARRAY_TESTS[@]}"; do
> res=0
>- tlog=sysbench_${test}_result.txt
>+ tlog=sysbench_${test}_results.txt
> rm -f $tlog
> maxres=0
> for run in `eval echo {1..$runs}` ; do
>@@ -69,7 +66,7 @@ for test in "${ARRAY_TESTS[@]}"; do
> if [[ $tres -gt $maxres ]]; then maxres=$tres ; fi
> done
> res=$(($res/$runs))
>- echo "${test}: $res" >>sysbench_result.txt
>+ echo "${test}: $res" >>Sysbench_result.txt
> echo "Subtest '$test' results:"
> echo "==============================="
>@@ -78,7 +75,13 @@ for test in "${ARRAY_TESTS[@]}"; do
> printf "Diviations (AVG -> MAX): %.2f" `bc <<< "scale = 4; (1 - $res / $maxres) * 100"` ; echo %
> done
>
>+echo ${TAR_VER} | tee Sysbench_t_version.txt
>+
>+echo "Tarantool TAG:"
>+cat Sysbench_t_version.txt
> echo "Overall results:"
> echo "================"
>-cat sysbench_result.txt
>-
>+cat Sysbench_result.txt
>+echo " "
>+echo "Publish data to bench database"
>+/opt/bench-run/benchs/publication/publish.py
>diff --git a/benchs/tpcc/run.sh b/benchs/tpcc/run.sh
>index f1f60cc..76754e5 100755
>--- a/benchs/tpcc/run.sh
>+++ b/benchs/tpcc/run.sh
>@@ -1,15 +1,12 @@
> #!/usr/bin/env bash
>+set -e
>+set -o pipefail
>
> TAR_VER=$(tarantool -v | grep -e "Tarantool" | grep -oP '\s\K\S*')
>-echo ${TAR_VER} | tee tpcc_t_version.txt
>-
>-set -e
>
> if [ ! -n "${TIME}" ]; then TIME=1200; fi
> if [ ! -n "${WARMUP_TIME}" ]; then WARMUP_TIME=10; fi
>
>-set -o pipefail
>-
> killall tarantool tpcc_load 2>/dev/null || true
> rm -rf 0*.xlog 0*.snap
> sync
>@@ -30,10 +27,16 @@ cd /opt/tpcc
> numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11 \
> /opt/tpcc/tpcc_start $tpcc_opts -r10 -l${TIME} -i${TIME} >tpcc_output.txt 2>/dev/null
>
>-echo -n "tpcc:" | tee tpcc_result.txt
>-cat tpcc_output.txt | grep -e '<TpmC>' | grep -oP '\K[0-9.]*' | tee -a tpcc_result.txt
>-
>+echo -n "tpcc:" | tee tpc.c_result.txt
>+cat tpcc_output.txt | grep -e '<TpmC>' | grep -oP '\K[0-9.]*' | tee -a tpc.c_result.txt
> cat tpcc_output.txt
>+echo ${TAR_VER} | tee tpc.c_t_version.txt
>+
>+echo "Tarantool TAG:"
>+cat tpc.c_t_version.txt
> echo "Overall result:"
> echo "==============="
>-cat tpcc_result.txt
>+cat tpc.c_result.txt
>+echo " "
>+echo "Publish data to bench database"
>+/opt/bench-run/benchs/publication/publish.py
>diff --git a/benchs/ycsb/run.sh b/benchs/ycsb/run.sh
>index 5ae1c75..d2cf8af 100755
>--- a/benchs/ycsb/run.sh
>+++ b/benchs/ycsb/run.sh
>@@ -17,7 +17,6 @@ set -e
> set -o pipefail
>
> TAR_VER=$(tarantool -v | grep -e "Tarantool" | grep -oP '\s\K\S*')
>-echo ${TAR_VER} | tee ycsb_t_version.txt
>
> ws=/opt/ycsb
> cd $ws
>@@ -50,7 +49,18 @@ for l in a b c d e f ; do
> echo 3 > /proc/sys/vm/drop_caches
> numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11 bin/ycsb run tarantool -s -P workloads/workload${l} >${res}.log 2>&1 || cat ${res}.log
> grep Thro ${res}.log | awk '{ print "Overall result: "$3 }' | tee ${res}.txt
>- sed "s#Overall result#$l $r#g" ${res}.txt >>${plogs}/results.txt
>+ sed "s#Overall result#$l $r#g" ${res}.txt >>${plogs}/ycsb.${mode}_result.txt
> done
> done
>-cat ${plogs}/results.txt
>+
>+echo ${TAR_VER} | tee ycsb.${mode}_t_version.txt
>+cp -f ${plogs}/ycsb.${mode}_result.txt .
>+
>+echo "Tarantool TAG:"
>+cat ycsb.${mode}_t_version.txt
>+echo "Overall results:"
>+echo "================"
>+cat ycsb.${mode}_result.txt
>+echo " "
>+echo "Publish data to bench database"
>+/opt/bench-run/benchs/publication/publish.py
>diff --git a/dockerfiles/ubuntu_benchs b/dockerfiles/ubuntu_benchs
>index fd9acb4..2c2e717 100644
>--- a/dockerfiles/ubuntu_benchs
>+++ b/dockerfiles/ubuntu_benchs
>@@ -71,5 +71,5 @@ RUN luarocks install \
> https://raw.githubusercontent.com/tarantool/gperftools/master/rockspecs/gperftools-scm-1.rockspec \
> --local >build.log 2>&1 || ( cat build.log && false )
>
>-# benchmarks runners
>-RUN git clone https://github.com/tarantool/bench-run.git /opt/bench-run
>+# adding python dependency
>+RUN pip install requests
>diff --git a/dockerfiles/ubuntu_tnt b/dockerfiles/ubuntu_tnt
>index 361e9d6..c1657ff 100644
>--- a/dockerfiles/ubuntu_tnt
>+++ b/dockerfiles/ubuntu_tnt
>@@ -3,11 +3,17 @@ FROM ${image_from}
>
> COPY . /opt/tarantool
> WORKDIR /opt/tarantool
>+RUN git fetch --all
> RUN git submodule update --recursive --init --force
> RUN ( cmake . -DENABLE_DIST=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo >build.log 2>&1 \
> && make -j >>build.log 2>&1 && make install >>build.log 2>&1 ) || \
> ( cat build.log && false )
>
>+# benchmarks runners
>+RUN git clone https://github.com/tarantool/bench-run.git /opt/bench-run
>+WORKDIR /opt/bench-run
>+RUN git fetch --all
>+
> # cbench
> RUN git clone https://github.com/tarantool/cbench.git /opt/cbench
> WORKDIR /opt/cbench
>diff --git a/targets.mk b/targets.mk
>index 6cf36b6..581b722 100644
>--- a/targets.mk
>+++ b/targets.mk
>@@ -21,7 +21,7 @@ prepare:
> docker push ${IMAGE_PERF}
> # build Tarantool and benchmarks with depends on Tarantool sources
> ${DOCKERFILE_BUILD} --build-arg image_from=${IMAGE_PERF} \
>- -t ${IMAGE_PERF_BUILT} -f bench-run/dockerfiles/ubuntu_tnt .
>+ -t ${IMAGE_PERF_BUILT} --no-cache -f bench-run/dockerfiles/ubuntu_tnt .
> docker push ${IMAGE_PERF_BUILT}
>
> # #####################################################
>--
>1.8.3.1
>
--
Alexander Tikhonov
[-- Attachment #2: Type: text/html, Size: 20955 bytes --]
^ permalink raw reply [flat|nested] 8+ messages in thread
* [Tarantool-patches] [PATCH v1] bench-run: submit perf results into database
@ 2020-04-19 16:28 Олег Пискунов
2020-04-20 17:42 ` Alexander Tikhonov
0 siblings, 1 reply; 8+ messages in thread
From: Олег Пискунов @ 2020-04-19 16:28 UTC (permalink / raw)
To: tarantool-patches; +Cc: Oleg Piskunov
[-- Attachment #1: Type: text/plain, Size: 14624 bytes --]
Resurrect bench.tarantool.org:
- submit perf results into bench.tarantool.org database
Fix docker files for perf testing:
- Disable cashing from docker file creation.
- adding python "requests" module needed for publish script.
Closes #4865, #4870
---
Github: https://github.com/tarantool/tarantool/tree/opiskunov/gh-4865-submit-perf-results
Issue: https://github.com/tarantool/tarantool/issues/4865
https://github.com/tarantool/tarantool/issues/4870
---
benchs/cbench/run.sh | 38 ++++++++++++++++++------
benchs/linkbench/run.sh | 13 +++++++--
benchs/nosqlbench/run.sh | 22 ++++++++------
benchs/publication/publish.py | 67 +++++++++++++++++++++++++++++++++++++++++++
benchs/sysbench/run.sh | 19 ++++++------
benchs/tpcc/run.sh | 21 ++++++++------
benchs/ycsb/run.sh | 16 +++++++++--
dockerfiles/ubuntu_benchs | 4 +--
dockerfiles/ubuntu_tnt | 6 ++++
targets.mk | 2 +-
10 files changed, 165 insertions(+), 43 deletions(-)
create mode 100755 benchs/publication/publish.py
diff --git a/benchs/cbench/run.sh b/benchs/cbench/run.sh
index 4d07972..c4867ad 100755
--- a/benchs/cbench/run.sh
+++ b/benchs/cbench/run.sh
@@ -1,11 +1,11 @@
#!/usr/bin/env bash
kill `pidof tarantool`
+
set -e
set -o pipefail
TAR_VER=$(tarantool -v | grep -e "Tarantool" | grep -oP '\s\K\S*')
-echo ${TAR_VER} | tee cbench_t_version.txt
killall tarantool 2>/dev/null || true
rm -rf 5* 0*
@@ -23,14 +23,34 @@ killall tarantool 2>/dev/null || true
rm -rf 5* 0*
sync
echo 3 > /proc/sys/vm/drop_caches
-numactl --membind=1 --cpunodebind=1 --physcpubind=11 tarantool /opt/cbench/cbench_runner.lua vinyl write 500 2>&1 | tee cbench_output_memtx_write.txt
+numactl --membind=1 --cpunodebind=1 --physcpubind=11 tarantool /opt/cbench/cbench_runner.lua vinyl write 500 2>&1 | tee cbench_output_vinyl_write.txt
-grep "^?tab" cbench_output_memtx.txt | sed "s/.*name=//"| sed "s/¶m=/:/"| sed "s/cb\./cb\.memtx\./"| tee -a cbench_result.txt
-grep "^?tab" cbench_output_vinyl_fsync.txt | sed "s/.*name=//"| sed "s/¶m=/:/"| sed "s/cb\./cb\.vinyl\.fsync\./"| tee -a cbench_result.txt
-grep "^?tab" cbench_output_memtx_write.txt | sed "s/.*name=//"| sed "s/¶m=/:/"| sed "s/cb\./cb\.vinyl\.write\./"| tee -a cbench_result.txt
+grep "^?tab=cbench.tree" cbench_output_memtx.txt | sed "s/.*name=//"| sed "s/¶m=/:/"| sed "s/cb\./cb\.memtx\./"| tee -a cbench-memtx-tree_result.txt
+grep "^?tab=cbench.hash" cbench_output_memtx.txt | sed "s/.*name=//"| sed "s/¶m=/:/"| sed "s/cb\./cb\.memtx\./"| tee -a cbench-memtx-hash_result.txt
+grep "^?tab" cbench_output_vinyl_fsync.txt | sed "s/.*name=//"| sed "s/¶m=/:/"| sed "s/cb\./cb\.vinyl\.fsync\./"| tee -a cbench-vinyl-fsync_result.txt
+grep "^?tab" cbench_output_vinyl_write.txt | sed "s/.*name=//"| sed "s/¶m=/:/"| sed "s/cb\./cb\.vinyl\.write\./"| tee -a cbench-vinyl-write_result.txt
-#mv tarantool-server.log cbench_tarantool_server.log
-
-echo "RESULTS:"
-cat cbench_result.txt
+echo ${TAR_VER} | tee cbench-memtx-tree_t_version.txt
+echo ${TAR_VER} | tee cbench-memtx-hash_t_version.txt
+echo ${TAR_VER} | tee cbench-vinyl-fsync_t_version.txt
+echo ${TAR_VER} | tee cbench-vinyl-write_t_version.txt
+echo ${TAR_VER} | tee cbench_t_version.txt
+echo "Tarantool TAG:"
+cat cbench_t_version.txt
+echo "Overall results:"
+echo "================"
+echo "RESULTS (cbench-memtx-tree_result.txt):"
+cat cbench-memtx-tree_result.txt
+echo " "
+echo "RESULTS (cbench-memtx-hash_result.txt):"
+cat cbench-memtx-hash_result.txt
+echo " "
+echo "RESULTS (cbench-vinyl-fsync_result.txt):"
+cat cbench-vinyl-fsync_result.txt
+echo " "
+echo "RESULTS (cbench-vinyl-write_result.txt):"
+cat cbench-vinyl-write_result.txt
+echo " "
+echo "Publish data to bench database"
+/opt/bench-run/benchs/publication/publish.py
diff --git a/benchs/linkbench/run.sh b/benchs/linkbench/run.sh
index 6738436..e7c77c3 100755
--- a/benchs/linkbench/run.sh
+++ b/benchs/linkbench/run.sh
@@ -6,7 +6,6 @@ set -e
set -o pipefail
TAR_VER=$(tarantool -v | grep -e "Tarantool" | grep -oP '\s\K\S*')
-echo ${TAR_VER} | tee linkbench_t_version.txt
cd /opt/linkbench && mvn clean package -Dmaven.test.skip=true
cd src/tarantool && make
@@ -34,4 +33,14 @@ sync
echo 3 > /proc/sys/vm/drop_caches
$numaopts /opt/linkbench/bin/linkbench -c $cfgfile -r 2>&1 | tee linkbench_output.txt
-grep "REQUEST PHASE COMPLETED" linkbench_output.txt | sed "s/.*second = /linkbench:/" | tee -a linkbench_result.txt
+grep "REQUEST PHASE COMPLETED" linkbench_output.txt | sed "s/.*second = /linkbench:/" | tee -a linkbench.ssd_result.txt
+echo ${TAR_VER} | tee linkbench.ssd_t_version.txt
+
+echo "Tarantool TAG:"
+cat linkbench.ssd_t_version.txt
+echo "Overall results:"
+echo "================"
+cat linkbench.ssd_result.txt
+echo " "
+echo "Publish data to bench database"
+/opt/bench-run/benchs/publication/publish.py
diff --git a/benchs/nosqlbench/run.sh b/benchs/nosqlbench/run.sh
index c9491f5..a68248e 100755
--- a/benchs/nosqlbench/run.sh
+++ b/benchs/nosqlbench/run.sh
@@ -6,13 +6,11 @@ if [ "$type" == "" ]; then
fi
TAR_VER=$(tarantool -v | grep -e "Tarantool" | grep -oP '\s\K\S*')
-echo ${TAR_VER} | tee nosqlbench_t_version.txt
kill `pidof tarantool`
-#set -e
-#
-#set -o pipefail
+set -e
+set -o pipefail
killall tarantool 2>/dev/null || true
rm -rf 0*.xlog 0*.snap
@@ -42,8 +40,14 @@ numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11 /opt/nosqlbench/
#echo "Latest 1000 lines:"
#tail -1000 nosqlbench_output.txt
-echo "Getting results"
-grep "TOTAL RPS STATISTICS:" nosqlbench_output.txt -A6 | awk -F "|" 'NR > 4 {print $2,":", $4}' > nosqlbench_result.txt
-
-cat nosqlbench_result.txt
-
+grep "TOTAL RPS STATISTICS:" nosqlbench_output.txt -A6 | awk -F "|" 'NR > 4 {print $2,":", $4}' > noSQLbench.${type}_result.txt
+echo ${TAR_VER} | tee noSQLbench.${type}_t_version.txt
+
+echo "Tarantool TAG:"
+cat noSQLbench.${type}_t_version.txt
+echo "Overall results:"
+echo "================"
+cat noSQLbench.${type}_result.txt
+echo " "
+echo "Publish data to bench database"
+/opt/bench-run/benchs/publication/publish.py
diff --git a/benchs/publication/publish.py b/benchs/publication/publish.py
new file mode 100755
index 0000000..d472fd7
--- /dev/null
+++ b/benchs/publication/publish.py
@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+import fnmatch
+import os
+from urllib import urlencode
+import requests
+
+
+def parse_bench(filename):
+ fileHandle = open(filename)
+ lastline = fileHandle.readlines()
+ fileHandle.close()
+ return lastline
+
+
+def get_version(filename):
+ fileHandle = open(filename)
+ lastline = fileHandle.readlines()[-1]
+ fileHandle.close()
+ return lastline.split()[0]
+
+
+def push_to_microb(server, token, name, value, version, tab):
+ uri = 'http://%s/push?%s' % (server, urlencode(dict(
+ key=token, name=name, param=value,
+ v=version, unit='trps', tab=tab
+ )))
+
+ r = requests.get(uri)
+ if r.status_code == 200:
+ print('Export complete')
+ else:
+ print('Export error http: %d' % r.status_code)
+ print('Export error text: %d' % r.text)
+
+
+def main():
+ if "MICROB_WEB_TOKEN" in os.environ and "MICROB_WEB_HOST" in os.environ:
+ bench = {}
+ res = []
+ current_data = {}
+ version = ''
+ for file in os.listdir('.'):
+ if fnmatch.fnmatch(file, '*_result.txt'):
+ values = parse_bench(file)
+ benchmark = file.split('_')[0]
+ version = get_version('{}_t_version.txt'.format(benchmark))
+ for value in values:
+ test_name = value.split(':')[0]
+ test_res = float(value.split(':')[1])
+ res.append(test_res)
+ push_to_microb(
+ os.environ['MICROB_WEB_HOST'],
+ os.environ['MICROB_WEB_TOKEN'],
+ test_name,
+ test_res,
+ version,
+ benchmark,
+ )
+ print ("VERSION - ", version)
+ else:
+ print("MICROB params not specified")
+
+ return 0
+
+
+if __name__ == '__main__':
+ main()
diff --git a/benchs/sysbench/run.sh b/benchs/sysbench/run.sh
index 6a69d14..ef0af43 100755
--- a/benchs/sysbench/run.sh
+++ b/benchs/sysbench/run.sh
@@ -11,8 +11,6 @@ set -e
set -o pipefail
TAR_VER=$(tarantool -v | grep -e "Tarantool" | grep -oP '\s\K\S*')
-echo ${TAR_VER} | tee sysbench_t_version.txt
-set -e
ARRAY_TESTS=(
"oltp_read_only"
@@ -44,11 +42,10 @@ opts="--db-driver=${DBMS} --threads=${THREADS}"
export LD_LIBRARY_PATH=/usr/local/lib
-rm -f sysbench_result.txt
-set -o pipefail
+rm -f Sysbench_result.txt
for test in "${ARRAY_TESTS[@]}"; do
res=0
- tlog=sysbench_${test}_result.txt
+ tlog=sysbench_${test}_results.txt
rm -f $tlog
maxres=0
for run in `eval echo {1..$runs}` ; do
@@ -69,7 +66,7 @@ for test in "${ARRAY_TESTS[@]}"; do
if [[ $tres -gt $maxres ]]; then maxres=$tres ; fi
done
res=$(($res/$runs))
- echo "${test}: $res" >>sysbench_result.txt
+ echo "${test}: $res" >>Sysbench_result.txt
echo "Subtest '$test' results:"
echo "==============================="
@@ -78,7 +75,13 @@ for test in "${ARRAY_TESTS[@]}"; do
printf "Diviations (AVG -> MAX): %.2f" `bc <<< "scale = 4; (1 - $res / $maxres) * 100"` ; echo %
done
+echo ${TAR_VER} | tee Sysbench_t_version.txt
+
+echo "Tarantool TAG:"
+cat Sysbench_t_version.txt
echo "Overall results:"
echo "================"
-cat sysbench_result.txt
-
+cat Sysbench_result.txt
+echo " "
+echo "Publish data to bench database"
+/opt/bench-run/benchs/publication/publish.py
diff --git a/benchs/tpcc/run.sh b/benchs/tpcc/run.sh
index f1f60cc..76754e5 100755
--- a/benchs/tpcc/run.sh
+++ b/benchs/tpcc/run.sh
@@ -1,15 +1,12 @@
#!/usr/bin/env bash
+set -e
+set -o pipefail
TAR_VER=$(tarantool -v | grep -e "Tarantool" | grep -oP '\s\K\S*')
-echo ${TAR_VER} | tee tpcc_t_version.txt
-
-set -e
if [ ! -n "${TIME}" ]; then TIME=1200; fi
if [ ! -n "${WARMUP_TIME}" ]; then WARMUP_TIME=10; fi
-set -o pipefail
-
killall tarantool tpcc_load 2>/dev/null || true
rm -rf 0*.xlog 0*.snap
sync
@@ -30,10 +27,16 @@ cd /opt/tpcc
numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11 \
/opt/tpcc/tpcc_start $tpcc_opts -r10 -l${TIME} -i${TIME} >tpcc_output.txt 2>/dev/null
-echo -n "tpcc:" | tee tpcc_result.txt
-cat tpcc_output.txt | grep -e '<TpmC>' | grep -oP '\K[0-9.]*' | tee -a tpcc_result.txt
-
+echo -n "tpcc:" | tee tpc.c_result.txt
+cat tpcc_output.txt | grep -e '<TpmC>' | grep -oP '\K[0-9.]*' | tee -a tpc.c_result.txt
cat tpcc_output.txt
+echo ${TAR_VER} | tee tpc.c_t_version.txt
+
+echo "Tarantool TAG:"
+cat tpc.c_t_version.txt
echo "Overall result:"
echo "==============="
-cat tpcc_result.txt
+cat tpc.c_result.txt
+echo " "
+echo "Publish data to bench database"
+/opt/bench-run/benchs/publication/publish.py
diff --git a/benchs/ycsb/run.sh b/benchs/ycsb/run.sh
index 5ae1c75..d2cf8af 100755
--- a/benchs/ycsb/run.sh
+++ b/benchs/ycsb/run.sh
@@ -17,7 +17,6 @@ set -e
set -o pipefail
TAR_VER=$(tarantool -v | grep -e "Tarantool" | grep -oP '\s\K\S*')
-echo ${TAR_VER} | tee ycsb_t_version.txt
ws=/opt/ycsb
cd $ws
@@ -50,7 +49,18 @@ for l in a b c d e f ; do
echo 3 > /proc/sys/vm/drop_caches
numactl --membind=1 --cpunodebind=1 --physcpubind=6,7,8,9,10,11 bin/ycsb run tarantool -s -P workloads/workload${l} >${res}.log 2>&1 || cat ${res}.log
grep Thro ${res}.log | awk '{ print "Overall result: "$3 }' | tee ${res}.txt
- sed "s#Overall result#$l $r#g" ${res}.txt >>${plogs}/results.txt
+ sed "s#Overall result#$l $r#g" ${res}.txt >>${plogs}/ycsb.${mode}_result.txt
done
done
-cat ${plogs}/results.txt
+
+echo ${TAR_VER} | tee ycsb.${mode}_t_version.txt
+cp -f ${plogs}/ycsb.${mode}_result.txt .
+
+echo "Tarantool TAG:"
+cat ycsb.${mode}_t_version.txt
+echo "Overall results:"
+echo "================"
+cat ycsb.${mode}_result.txt
+echo " "
+echo "Publish data to bench database"
+/opt/bench-run/benchs/publication/publish.py
diff --git a/dockerfiles/ubuntu_benchs b/dockerfiles/ubuntu_benchs
index fd9acb4..2c2e717 100644
--- a/dockerfiles/ubuntu_benchs
+++ b/dockerfiles/ubuntu_benchs
@@ -71,5 +71,5 @@ RUN luarocks install \
https://raw.githubusercontent.com/tarantool/gperftools/master/rockspecs/gperftools-scm-1.rockspec \
--local >build.log 2>&1 || ( cat build.log && false )
-# benchmarks runners
-RUN git clone https://github.com/tarantool/bench-run.git /opt/bench-run
+# adding python dependency
+RUN pip install requests
diff --git a/dockerfiles/ubuntu_tnt b/dockerfiles/ubuntu_tnt
index 361e9d6..c1657ff 100644
--- a/dockerfiles/ubuntu_tnt
+++ b/dockerfiles/ubuntu_tnt
@@ -3,11 +3,17 @@ FROM ${image_from}
COPY . /opt/tarantool
WORKDIR /opt/tarantool
+RUN git fetch --all
RUN git submodule update --recursive --init --force
RUN ( cmake . -DENABLE_DIST=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo >build.log 2>&1 \
&& make -j >>build.log 2>&1 && make install >>build.log 2>&1 ) || \
( cat build.log && false )
+# benchmarks runners
+RUN git clone https://github.com/tarantool/bench-run.git /opt/bench-run
+WORKDIR /opt/bench-run
+RUN git fetch --all
+
# cbench
RUN git clone https://github.com/tarantool/cbench.git /opt/cbench
WORKDIR /opt/cbench
diff --git a/targets.mk b/targets.mk
index 6cf36b6..581b722 100644
--- a/targets.mk
+++ b/targets.mk
@@ -21,7 +21,7 @@ prepare:
docker push ${IMAGE_PERF}
# build Tarantool and benchmarks with depends on Tarantool sources
${DOCKERFILE_BUILD} --build-arg image_from=${IMAGE_PERF} \
- -t ${IMAGE_PERF_BUILT} -f bench-run/dockerfiles/ubuntu_tnt .
+ -t ${IMAGE_PERF_BUILT} --no-cache -f bench-run/dockerfiles/ubuntu_tnt .
docker push ${IMAGE_PERF_BUILT}
# #####################################################
--
1.8.3.1
[-- Attachment #2: Type: text/html, Size: 18656 bytes --]
^ permalink raw reply [flat|nested] 8+ messages in thread
end of thread, other threads:[~2020-04-29 10:07 UTC | newest]
Thread overview: 8+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2020-04-22 10:18 [Tarantool-patches] [PATCH v1] bench-run: submit perf results into database Олег Пискунов
2020-04-22 10:57 ` Alexander Tikhonov
2020-04-23 9:28 ` Sergey Bronnikov
2020-04-27 12:28 ` Oleg Piskunov
2020-04-29 10:07 ` Oleg Piskunov
-- strict thread matches above, loose matches on Subject: below --
2020-04-19 16:28 Олег Пискунов
2020-04-20 17:42 ` Alexander Tikhonov
2020-04-21 6:14 ` Oleg Piskunov
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox