From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from smtp37.i.mail.ru (smtp37.i.mail.ru [94.100.177.97]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by dev.tarantool.org (Postfix) with ESMTPS id 4EEDD45C304 for ; Wed, 2 Dec 2020 20:09:35 +0300 (MSK) From: Oleg Koshovetc Message-ID: <984618ac-e7ee-c7db-4c0d-f71fbba59210@tarantool.org> Date: Wed, 2 Dec 2020 20:08:52 +0300 MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8"; format="flowed" Content-Transfer-Encoding: 8bit Content-Language: en-US Subject: [Tarantool-patches] [PATCH] benchmarks: runner for local benchmarks runs List-Id: Tarantool development patches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , To: Alexander Turenko , Alexander Tikhonov Cc: tarantool-patches@dev.tarantool.org https://github.com/tarantool/bench-run/compare/ttbench?expand=1 This runner allows to run various benchmarks on local machines (if all the source code was already present and prebuilt on a machine). However analyzing the results of benchmarks requires manual reading through their logs TODO: implement automatic local installation of benchmarks TODO: implement tool to save and diff benchmarks' results ---  benches/cbench/run.sh           |  35 ++++++++++  benches/common.sh               | 109 +++++++++++++++++++++++++++++  benches/linkbench/app.lua       |  17 +++++  benches/linkbench/run.sh        |  46 +++++++++++++  benches/nosqlbench/run.sh       |  57 +++++++++++++++  benches/nosqlbench/tnt_hash.lua |  30 ++++++++  benches/nosqlbench/tnt_tree.lua |  31 +++++++++  benches/publication/publish.py  |  64 +++++++++++++++++  benches/sysbench/run.sh         |  92 +++++++++++++++++++++++++  benches/sysbench/tnt_srv.lua    |  28 ++++++++  benches/tpcc/init_empty.lua     |   2 +  benches/tpcc/init_not_empty.lua |   1 +  benches/tpcc/run.sh             |  49 +++++++++++++  benches/tpcc/schema.lua         | 144 ++++++++++++++++++++++++++++++++++++++  benches/tpcc/server.lua         |   2 +  benches/tpch/run.sh             |  51 ++++++++++++++  benches/ycsb/run.sh             |  86 +++++++++++++++++++++++  config.sh                       |  41 +++++++++++  ttbench                         | 149 ++++++++++++++++++++++++++++++++++++++++  19 files changed, 1034 insertions(+)  create mode 100755 benches/cbench/run.sh  create mode 100644 benches/common.sh  create mode 100755 benches/linkbench/app.lua  create mode 100755 benches/linkbench/run.sh  create mode 100755 benches/nosqlbench/run.sh  create mode 100644 benches/nosqlbench/tnt_hash.lua  create mode 100644 benches/nosqlbench/tnt_tree.lua  create mode 100755 benches/publication/publish.py  create mode 100755 benches/sysbench/run.sh  create mode 100644 benches/sysbench/tnt_srv.lua  create mode 100644 benches/tpcc/init_empty.lua  create mode 100644 benches/tpcc/init_not_empty.lua  create mode 100755 benches/tpcc/run.sh  create mode 100644 benches/tpcc/schema.lua  create mode 100644 benches/tpcc/server.lua  create mode 100755 benches/tpch/run.sh  create mode 100755 benches/ycsb/run.sh  create mode 100644 config.sh  create mode 100755 ttbench diff --git a/benches/cbench/run.sh b/benches/cbench/run.sh new file mode 100755 index 0000000..09c4dee --- /dev/null +++ b/benches/cbench/run.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +source ../common.sh + +CBENCH_VINYL_WORKLOAD="${CBENCH_VINYL_WORKLOAD:-500}" +CBENCH_MEMTX_WORKLOAD="${CBENCH_MEMTX_WORKLOAD:-1000000}" + +export LUA_CPATH="$PWD/?.so" +export LUA_PATH="$PWD/?/init.lua" + +TAR_VER=$(get_tarantool_version) +numaopts=(--membind=1 --cpunodebind=1 --physcpubind=11) + +stop_and_clean_tarantool +maybe_drop_cache + +maybe_under_numactl "${numaopts[@]}" -- \ +    "$TARANTOOL_EXECUTABLE" cbench_runner.lua memtx "$CBENCH_MEMTX_WORKLOAD" 2>&1 | tee cbench_output_memtx.txt + +stop_and_clean_tarantool +maybe_drop_cache + +maybe_under_numactl "${numaopts[@]}" -- \ +    "$TARANTOOL_EXECUTABLE" cbench_runner.lua vinyl fsync "$CBENCH_VINYL_WORKLOAD" 2>&1 | tee cbench_output_vinyl_fsync.txt + +stop_and_clean_tarantool +maybe_drop_cache + +maybe_under_numactl "${numaopts[@]}" -- \ +    "$TARANTOOL_EXECUTABLE" cbench_runner.lua vinyl write "$CBENCH_VINYL_WORKLOAD" 2>&1 | tee cbench_output_vinyl_write.txt + +echo "$TAR_VER" diff --git a/benches/common.sh b/benches/common.sh new file mode 100644 index 0000000..1116cc9 --- /dev/null +++ b/benches/common.sh @@ -0,0 +1,109 @@ +# will not work if not under root +# if benches are run localy by a developer droping caches might not be too important +function maybe_drop_cache { +    if test -w /proc/sys/vm/drop_caches; then +        echo 3 > /proc/sys/vm/drop_caches +    fi +} + +function can_be_run_under_numa { +    numactl "$@" true 1>/dev/null 2>/dev/null +} + +# not every local machine has numa enabled on it +# not every local machine has enough hardware units to run with intended configuration +# +# usage: +# maybe_under_numactl numaoptions -- command to run +# example: +# maybe_under_numactl --membind=1 --cpunodebind=1 --physcpubind=11 -- echo qwe +function maybe_under_numactl { +    local numaoptions=() +    local cmdoptions=() +    local parsing_numa=1 +    for option in "$@"; do +        if [ -n "$parsing_numa" ]; then +            if [ "$option" == -- ]; then +                parsing_numa= +            else +                numaoptions+=( "$option" ) +            fi +        else +            cmdoptions+=( "$option" ) +        fi +    done + +    if which numactl 1>/dev/null 2>/dev/null; then +        # check if it is even runable with given numactl options +        if can_be_run_under_numa "${numaoptions[@]}"; then +            numactl "${numaoptions[@]}" "${cmdoptions[@]}" +        else +            "${cmdoptions[@]}" +        fi +    else +        "${cmdoptions[@]}" +    fi +} + +function get_tarantool_version { +    "$TARANTOOL_EXECUTABLE" -v | grep -e "Tarantool" |  grep -oP '\s\K\S*' +} + +function clean_tarantool { +    rm -rf ./*.snap +    rm -rf ./*.xlog +    rm -rf ./*.vylog +    rm -rf 5* +} + +function sync_disk { +    sync && echo "sync passed" || echo "sync failed with error: $?" +} + +function kill_tarantool { +    killall tarantool 2>/dev/null || true +} + +function error { +    echo "ERROR:" "$@" +    exit 100 +} + +function stop_and_clean_tarantool { +    kill_tarantool +    clean_tarantool +    sync_disk +} + +function wait_for_file_release { +    local f="$1" +    local t="$2" +    local tt=0 +    while [ "$tt" -lt "$t" ]; do +        if ! lsof "$f" 1>/dev/null 2>/dev/null; then +            return 0 +        fi + +        tt=$(( tt + 1 )) +        sleep 1 +    done + +    return 1 +} + +function wait_for_tarantool_runnning { +    local creds="$1" +    local t="$2" +    local tt=0 + +    while [ "$tt" -lt "$t" ]; do +        if echo 'if type(box.cfg) ~= "function" then return box.info().status end' | tarantoolctl connect "$creds" 2>/dev/null | grep -q 'running'; then +            return 0 +        fi + +        tt=$(( tt + 1 )) +        sleep 1 +    done + +    return 1 +} diff --git a/benches/linkbench/app.lua b/benches/linkbench/app.lua new file mode 100755 index 0000000..79252e0 --- /dev/null +++ b/benches/linkbench/app.lua @@ -0,0 +1,17 @@ +#!/usr/bin/env tarantool + +local path = require('fio').dirname(arg[0]) +package.path = path.."/?.lua;"..package.path +package.cpath = path.."/?.so;"..package.cpath + +require('console').listen('unix/:./tarantool.sock') +require('gperftools').cpu.start('tarantool.prof') + +box.cfg{ +    listen = 3301; +    vinyl_memory = 256 * 1024 * 1024; +    vinyl_cache =  256 * 1024 * 1024; +    vinyl_read_threads = 1; +} + +require('app_internal') diff --git a/benches/linkbench/run.sh b/benches/linkbench/run.sh new file mode 100755 index 0000000..c5f493e --- /dev/null +++ b/benches/linkbench/run.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +source ../common.sh + +LINKBENCH_REQUESTS="${LINKBENCH_REQUESTS:-2000000}" +LINKBENCH_REQUESTERS="${LINKBENCH_REQUESTERS:-1}" +LINKBENCH_WORKLOAD_SIZE="${LINKBENCH_WORKLOAD_SIZE:-5000000}" + +TAR_VER=$(get_tarantool_version) +numaopts=(--membind=1 --cpunodebind=1 '--physcpubind=6,7,8,9,10,11') + +mvn clean package -Dmaven.test.skip=true +make -C src/tarantool -B + +tfile=src/tarantool/app.lua + +stop_and_clean_tarantool +maybe_under_numactl "${numaopts[@]}" -- tarantool "$tfile" 1>/dev/null 2>/dev/null & + +cfgfile=config/LinkConfigTarantool.properties +sed "s/^maxid1 = .*/maxid1 = $LINKBENCH_WORKLOAD_SIZE/g" -i config/FBWorkload.properties +sed "s/^requesters = .*/requesters = $LINKBENCH_REQUESTERS/g" -i "$cfgfile" +sed "s/^requests = .*/requests = $LINKBENCH_REQUESTS/g" -i "$cfgfile" + +maybe_under_numactl "${numaopts[@]}" -- \ +    ./bin/linkbench -c "$cfgfile" -l 2>&1 | tee loading.res.txt + +sync_disk +maybe_drop_cache + +maybe_under_numactl "${numaopts[@]}" -- \ +    ./bin/linkbench -c "$cfgfile" -r 2>&1 | tee linkbench_output.txt + +kill_tarantool + +grep "REQUEST PHASE COMPLETED" linkbench_output.txt | sed "s/.*second = /linkbench:/" | tee -a linkbench.ssd_result.txt +echo "${TAR_VER}" | tee linkbench.ssd_t_version.txt + +echo "Tarantool TAG:" +cat linkbench.ssd_t_version.txt +echo "Overall results:" +echo "================" +cat linkbench.ssd_result.txt diff --git a/benches/nosqlbench/run.sh b/benches/nosqlbench/run.sh new file mode 100755 index 0000000..b092cfe --- /dev/null +++ b/benches/nosqlbench/run.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +source ../common.sh + +# type=$1 +# if [ "$type" == "" ]; then +#     type=hash +# fi + +NOSQLBENCH_WORKLOAD="${NOSQLBENCH_WORKLOAD:-}" +NOSQLBENCH_TIMELIMIT="${NOSQLBENCH_TIMELIMIT:-20000}" +NOSQLBENCH_BATCHCOUNT="${NOSQLBENCH_BATCHCOUNT:-10}" +NOSQLBENCH_RPS="${NOSQLBENCH_RPS:-20000}" + +TAR_VER=$(get_tarantool_version) +numaopts=(--membind=1 --cpunodebind=1 '--physcpubind=6,7,8,9,10,11') + +function run_nosqlbench { +    local type="$1" +    stop_and_clean_tarantool + +    maybe_under_numactl "${numaopts[@]}" -- \ +        "$TARANTOOL_EXECUTABLE" "tnt_${type}.lua" 2>&1 & + +    config=nosqlbench.conf +    cp src/nosqlbench.conf "$config" + +    sed  "s/port 3303/port 3301/" "$config" -i +    sed  "s/benchmark 'no_limit'/benchmark 'time_limit'/" "$config" -i +    sed  "s/time_limit 10/time_limit $NOSQLBENCH_TIMELIMIT/" "$config" -i +    sed  "s/request_batch_count 1/request_batch_count $NOSQLBENCH_BATCHCOUNT/" "$config" -i +    sed  "s/rps 12000/rps $NOSQLBENCH_RPS/" "$config" -i + +    sleep 5 +    echo "Run NB type='$type'" + +    # WARNING: don't try to save output from stderr - file will use the whole disk space ! +    (maybe_under_numactl "${numaopts[@]}" -- \ +        ./src/nb "$config") \ +        | grep -v "Warmup" \ +        | grep -v "Failed to allocate" >nosqlbench_output.txt \ +        || cat nosqlbench_output.txt + +    grep "TOTAL RPS STATISTICS:" nosqlbench_output.txt -A6 | \ +        awk -F "|" 'NR > 4 {print $2,":", $4}' > "noSQLbench.${type}_result.txt" +    echo "${TAR_VER}" | tee "noSQLbench.${type}_t_version.txt" +} + +if [ -z "$NOSQLBENCH_WORKLOAD" -o "$NOSQLBENCH_WORKLOAD" == all ]; then +    run_nosqlbench tree +    run_nosqlbench hash +else +    run_nosqlbench "$NOSQLBENCH_WORKLOAD" +fi diff --git a/benches/nosqlbench/tnt_hash.lua b/benches/nosqlbench/tnt_hash.lua new file mode 100644 index 0000000..a616490 --- /dev/null +++ b/benches/nosqlbench/tnt_hash.lua @@ -0,0 +1,30 @@ +local console = require('console') +console.listen("/tmp/tarantool-server.sock") + +box.cfg { +    pid_file   = "./tarantool-server.pid", +    log        = "./tarantool-server.log", +    listen = 3301, +    memtx_memory = 2000000000, +    background = true, +    checkpoint_interval = 0, +    wal_mode = 'none', +} + +s = box.schema.space.create('tester_nosqlbench') +s:create_index('primary', {type = 'hash', parts = {1, 'unsigned'}}) + +function try(f, catch_f) +    local status, exception = pcall(f) +    if not status then +        catch_f(exception) +    end +end + +try(function() +    box.schema.user.grant('guest', 'create,read,write,execute', 'universe') +end, function(e) +    print(e) +end) + + diff --git a/benches/nosqlbench/tnt_tree.lua b/benches/nosqlbench/tnt_tree.lua new file mode 100644 index 0000000..57b646b --- /dev/null +++ b/benches/nosqlbench/tnt_tree.lua @@ -0,0 +1,31 @@ +local console = require('console') +console.listen("/tmp/tarantool-server.sock") + +--memtx_memory = 2000000000, +box.cfg { +    pid_file   = "./tarantool-server.pid", +    log        = "./tarantool-server.log", +    listen = 3301, +    vinyl_memory = 107374182, +    background = true, +    checkpoint_interval = 0, +    wal_mode = 'none', +} + +s = box.schema.space.create('tester_nosqlbench') +s:create_index('primary', {type = 'tree', parts = {1, 'unsigned'}}) + +function try(f, catch_f) +    local status, exception = pcall(f) +    if not status then +        catch_f(exception) +    end +end + +try(function() +    box.schema.user.grant('guest', 'create,read,write,execute', 'universe') +end, function(e) +    print(e) +end) + + diff --git a/benches/publication/publish.py b/benches/publication/publish.py new file mode 100755 index 0000000..42bc53a --- /dev/null +++ b/benches/publication/publish.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python +import fnmatch +import os +from urllib import urlencode +import requests + + +def parse_bench(filename): +    with open(filename) as raw_data: +        return raw_data.readlines() + + +def get_version(filename): +    with open(filename) as raw_data: +        version = raw_data.readlines()[-1] +        return version.split()[0] + + +def push_to_microb(server, token, name, value, version, tab): +    uri = 'http://%s/push?%s' % (server, urlencode(dict( +        key=token, name=name, param=value, +        v=version, unit='trps', tab=tab +    ))) + +    r = requests.get(uri) +    if r.status_code == 200: +        print('Export complete') +    else: +        print('Export error http: %d' % r.status_code) +        print('Export error text: %d' % r.text) + + +def main(): +    if "MICROB_WEB_TOKEN" in os.environ and "MICROB_WEB_HOST" in os.environ: +        bench = {} +        res = [] +        current_data = {} +        version = '' +        for file in os.listdir('.'): +            if fnmatch.fnmatch(file, '*_result.txt'): +                values = parse_bench(file) +                benchmark = file.split('_')[0] +                version = get_version('{}_t_version.txt'.format(benchmark)) +                for value in values: +                    test_name = value.split(':')[0] +                    test_res = float(value.split(':')[1]) +                    res.append(test_res) +                    push_to_microb( +                        os.environ['MICROB_WEB_HOST'], +                        os.environ['MICROB_WEB_TOKEN'], +                        test_name, +                        test_res, +                        version, +                        benchmark, +                    ) +        print ("VERSION - ", version) +    else: +        print("MICROB params not specified") + +    return 0 + + +if __name__ == '__main__': +    main() diff --git a/benches/sysbench/run.sh b/benches/sysbench/run.sh new file mode 100755 index 0000000..4b52385 --- /dev/null +++ b/benches/sysbench/run.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +source ../common.sh + +SYSBENCH_TESTS="${SYSBENCH_TESTS:-}" +SYSBENCH_TIME="${SYSBENCH_TIME:-20}" +SYSBENCH_WARMUPTIME="${SYSBENCH_WARMUPTIME:-5}" +SYSBENCH_DBMS="${SYSBENCH_DBMS:-tarantool}" +SYSBENCH_THREADS="${SYSBENCH_THREADS:-200}" +SYSBENCH_RUNS="${SYSBENCH_RUNS:-10}" + +TAR_VER=$(get_tarantool_version) +numaconf=(--membind=1 --cpunodebind=1 '--physcpubind=6,7,8,9,10,11') + +ARRAY_TESTS=( +    "oltp_read_only" +    "oltp_write_only" +    "oltp_read_write" +    "oltp_update_index" +    "oltp_update_non_index" +    "oltp_insert" +    "oltp_delete" +    "oltp_point_select" +    "select_random_points" +    "select_random_ranges" +    # "bulk_insert" +) + +opts=("--db-driver=${SYSBENCH_DBMS}" "--threads=${SYSBENCH_THREADS}") + +export LD_LIBRARY_PATH=/usr/local/lib + +if [ -n "$SYSBENCH_TESTS" -a "$SYSBENCH_TESTS" != all ]; then +    IPS=, read -ra testlist <<< "$SYSBENCH_TESTS" +    ARRAY_TESTS=$testlist +fi + +for test in "${ARRAY_TESTS[@]}"; do +    res=0 +    tlog=sysbench_${test}_results.txt +    rm -f "$tlog" +    maxres=0 +    for run in $(seq 1 "$SYSBENCH_RUNS"); do +        echo "$run" +        echo "------------ $test ------------ rerun: # $run ------------" + +        stop_and_clean_tarantool +        wait_for_file_release /tmp/tarantool-server.sock 10 +        maybe_under_numactl "${numaconf[@]}" -- \ +            "$TARANTOOL_EXECUTABLE" tnt_srv.lua >tnt_server.txt 2>&1 \ +            || ( cat tnt_server.txt && false ) + +        sleep 1 + +        # ./run_tnt.sh >tnt_server.txt 2>&1 || ( cat tnt_server.txt && false ) + +        ./src/sysbench "$test" "${opts[@]}" cleanup > sysbench_output.txt +        ./src/sysbench "$test" "${opts[@]}" prepare >> sysbench_output.txt + +        maybe_under_numactl "${numaconf[@]}" -- \ +            ./src/sysbench "$test" "${opts[@]}" \ +                "--time=${SYSBENCH_TIME}" \ +                "--warmup-time=${SYSBENCH_WARMUPTIME}" \ +                run >> sysbench_output.txt + +        grep -e 'transactions:' sysbench_output.txt | grep -oP '\(\K\S*' | tee "$tlog" +        tres=$(sed 's#^.*:##g' < "$tlog" | sed 's#\..*$##g') +        echo ">>> $tres" +        res=$(( res + tres )) +        if [[ $tres -gt $maxres ]]; then maxres=$tres ; fi +    done + +    res=$(( res / SYSBENCH_RUNS )) +    echo "${test}: $res" >>Sysbench_result.txt + +    echo "Subtest '$test' results:" +    echo "===============================" +    echo "Average result: $res" +    echo "Maximum result: $maxres" +    printf "Deviations (AVG -> MAX): %.2f%%" "$(bc <<< "scale = 4; (1 - $res / $maxres) * 100")" +done + +echo "${TAR_VER}" | tee Sysbench_t_version.txt + +echo "Tarantool TAG:" +cat Sysbench_t_version.txt +echo "Overall results:" +echo "================" +cat Sysbench_result.txt diff --git a/benches/sysbench/tnt_srv.lua b/benches/sysbench/tnt_srv.lua new file mode 100644 index 0000000..d84413d --- /dev/null +++ b/benches/sysbench/tnt_srv.lua @@ -0,0 +1,28 @@ +local console = require('console') +console.listen("/tmp/tarantool-server.sock") + +box.cfg { +    pid_file   = "./tarantool-server.pid", +    log        = "./tarantool-server.log", +    listen = 3301, +    memtx_memory = 2000000000, +    background = true, +    checkpoint_interval = 0, +} + + +function try(f, catch_f) +    local status, exception = pcall(f) +    if not status then +        catch_f(exception) +    end +end + +try(function() +    box.schema.user.grant('guest', 'create,read,write,execute', 'universe') +end, function(e) +    print(e) +end) + + + diff --git a/benches/tpcc/init_empty.lua b/benches/tpcc/init_empty.lua new file mode 100644 index 0000000..e29ff04 --- /dev/null +++ b/benches/tpcc/init_empty.lua @@ -0,0 +1,2 @@ +require 'server' +require 'schema' diff --git a/benches/tpcc/init_not_empty.lua b/benches/tpcc/init_not_empty.lua new file mode 100644 index 0000000..b36eeec --- /dev/null +++ b/benches/tpcc/init_not_empty.lua @@ -0,0 +1 @@ +require 'server' diff --git a/benches/tpcc/run.sh b/benches/tpcc/run.sh new file mode 100755 index 0000000..e391d5f --- /dev/null +++ b/benches/tpcc/run.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +set -eu +set -o pipefail + +source ../common.sh + +TAR_VER=$(get_tarantool_version) +numaopts=(--membind=1 --cpunodebind=1 '--physcpubind=6,7,8,9,10,11') + +TPCC_TIME="${TPCC_TIME:-1200}" +TPCC_WARMUPTIME="${TPCC_WARMUPTIME:-10}" +TPCC_WAREHOUSES="${TPCC_WAREHOUSES:-15}" +TPCC_FROMSNAPSHOT="${TPCC_FROMSNAPSHOT:-}" + +killall tpcc_load 2>/dev/null || true +stop_and_clean_tarantool + +tpcc_opts=(-h localhost -P 3301 -d tarantool -u root -p '' -w "$TPCC_WAREHOUSES") + +if [ -z "$TPCC_FROMSNAPSHOT" ]; then +    maybe_under_numactl "${numaopts[@]}" \ +        -- "$TARANTOOL_EXECUTABLE" init_empty.lua & +    wait_for_tarantool_runnning 3301 15 + +    ./tpcc_load "${tpcc_opts[@]}" +else +    [ ! -f "$TPCC_FROMSNAPSHOT" ] && error "No such file: '$TPCC_FROMSNAPSHOT'" +    cp "$TPCC_FROMSNAPSHOT" . + +    maybe_under_numactl "${numaopts[@]}" \ +        -- "$TARANTOOL_EXECUTABLE" init_not_empty.lua & +    wait_for_tarantool_runnning 3301 60 +fi + + +maybe_under_numactl "${numaopts[@]}" -- \ +    ./tpcc_start "${tpcc_opts[@]}" -r "$TPCC_WARMUPTIME" -l "$TPCC_TIME" -i "$TPCC_TIME" > tpcc_output.txt 2>/dev/null + +echo -n "tpcc:" | tee tpc.c_result.txt +grep -e '' tpcc_output.txt | grep -oP '\K[0-9.]*' | tee -a tpc.c_result.txt +cat tpcc_output.txt + +echo "${TAR_VER}" | tee tpc.c_t_version.txt + +echo "Tarantool TAG:" +cat tpc.c_t_version.txt +echo "Overall result:" +echo "===============" +cat tpc.c_result.txt diff --git a/benches/tpcc/schema.lua b/benches/tpcc/schema.lua new file mode 100644 index 0000000..dc9c0df --- /dev/null +++ b/benches/tpcc/schema.lua @@ -0,0 +1,144 @@ +local sql_execute = box.sql and box.sql.execute or box.execute + +sql_execute("drop table if exists warehouse;") +sql_execute("create table warehouse ( \ +w_id int not null, \ +w_name varchar(10), \ +w_street_1 varchar(20), \ +w_street_2 varchar(20), \ +w_city varchar(20), \ +w_state varchar(2), \ +w_zip varchar(9), \ +w_tax number, \ +w_ytd number, \ +primary key (w_id) )") + +sql_execute("drop table if exists district;") +sql_execute("create table district ( \ +d_id int not null, \ +d_w_id int not null, \ +d_name varchar(10), \ +d_street_1 varchar(20), \ +d_street_2 varchar(20), \ +d_city varchar(20), \ +d_state varchar(2), \ +d_zip varchar(9), \ +d_tax number, \ +d_ytd number, \ +d_next_o_id int, \ +primary key (d_w_id, d_id), \ +FOREIGN KEY(d_w_id) REFERENCES warehouse(w_id) );") + +sql_execute("drop table if exists customer;") +sql_execute("create table customer ( \ +c_id int not null, \ +c_d_id int not null, \ +c_w_id int not null, \ +c_first varchar(16), \ +c_middle varchar(2), \ +c_last varchar(16), \ +c_street_1 varchar(20), \ +c_street_2 varchar(20), \ +c_city varchar(20), \ +c_state varchar(2), \ +c_zip varchar(9), \ +c_phone varchar(16), \ +c_since varchar(100), \ +c_credit varchar(2), \ +c_credit_lim int, \ +c_discount number, \ +c_balance number, \ +c_ytd_payment number, \ +c_payment_cnt int, \ +c_delivery_cnt int, \ +c_data text, \ +PRIMARY KEY(c_w_id, c_d_id, c_id), \ +FOREIGN KEY(c_w_id,c_d_id) REFERENCES district(d_w_id,d_id) );") + +sql_execute("drop table if exists history;") +sql_execute("create table history ( \ +_id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, \ +h_c_id int, \ +h_c_d_id int, \ +h_c_w_id int, \ +h_d_id int, \ +h_w_id int, \ +h_date varchar(100), \ +h_amount number, \ +h_data varchar(24), \ +FOREIGN KEY(h_c_w_id,h_c_d_id,h_c_id) REFERENCES customer(c_w_id,c_d_id,c_id), \ +FOREIGN KEY(h_w_id,h_d_id) REFERENCES district(d_w_id,d_id) );") + +sql_execute("drop table if exists orders;") +sql_execute("create table orders ( \ +o_id int not null, \ +o_d_id int not null, \ +o_w_id int not null, \ +o_c_id int, \ +o_entry_d varchar(100), \ +o_carrier_id int, \ +o_ol_cnt int, \ +o_all_local int, \ +PRIMARY KEY(o_w_id, o_d_id, o_id), \ +FOREIGN KEY(o_w_id,o_d_id,o_c_id) REFERENCES customer(c_w_id,c_d_id,c_id) );") + +sql_execute("drop table if exists new_orders;") +sql_execute("create table new_orders ( \ +no_o_id int not null, \ +no_d_id int not null, \ +no_w_id int not null, \ +PRIMARY KEY(no_w_id, no_d_id, no_o_id), \ +FOREIGN KEY(no_w_id,no_d_id,no_o_id) REFERENCES orders(o_w_id,o_d_id,o_id));") + +sql_execute("drop table if exists item;") +sql_execute("create table item ( \ +i_id int not null, \ +i_im_id int, \ +i_name varchar(24), \ +i_price number, \ +i_data varchar(50), \ +PRIMARY KEY(i_id) );") + +sql_execute("drop table if exists stock;") +sql_execute("create table stock ( \ +s_i_id int not null, \ +s_w_id int not null, \ +s_quantity int, \ +s_dist_01 varchar(24), \ +s_dist_02 varchar(24), \ +s_dist_03 varchar(24), \ +s_dist_04 varchar(24), \ +s_dist_05 varchar(24), \ +s_dist_06 varchar(24), \ +s_dist_07 varchar(24), \ +s_dist_08 varchar(24), \ +s_dist_09 varchar(24), \ +s_dist_10 varchar(24), \ +s_ytd varchar(9), \ +s_order_cnt int, \ +s_remote_cnt int, \ +s_data varchar(50), \ +PRIMARY KEY(s_w_id, s_i_id), \ +FOREIGN KEY(s_w_id) REFERENCES warehouse(w_id), \ +FOREIGN KEY(s_i_id) REFERENCES item(i_id) );") + +sql_execute("drop table if exists order_line;") +sql_execute("create table order_line ( \ +ol_o_id int not null, \ +ol_d_id int not null, \ +ol_w_id int not null, \ +ol_number int not null, \ +ol_i_id int, \ +ol_supply_w_id int, \ +ol_delivery_d varchar(100), \ +ol_quantity int, \ +ol_amount number, \ +ol_dist_info varchar(24), \ +PRIMARY KEY(ol_w_id, ol_d_id, ol_o_id, ol_number), \ +FOREIGN KEY(ol_w_id,ol_d_id,ol_o_id) REFERENCES orders(o_w_id,o_d_id,o_id), \ +FOREIGN KEY(ol_supply_w_id,ol_i_id) REFERENCES stock(s_w_id,s_i_id) );") + +sql_execute("CREATE INDEX idx_customer ON customer (c_w_id,c_d_id,c_last,c_first);") +sql_execute("CREATE INDEX idx_orders ON orders (o_w_id,o_d_id,o_c_id,o_id);") +sql_execute("CREATE INDEX fkey_stock_2 ON stock (s_i_id);") +sql_execute("CREATE INDEX fkey_order_line_2 ON order_line (ol_supply_w_id,ol_i_id);") diff --git a/benches/tpcc/server.lua b/benches/tpcc/server.lua new file mode 100644 index 0000000..17cda59 --- /dev/null +++ b/benches/tpcc/server.lua @@ -0,0 +1,2 @@ +box.cfg{listen = 3301, memtx_memory = 10 * 1024^3} +box.schema.user.grant('guest', 'read,write,execute', 'universe', nil, { if_not_exists = true }) diff --git a/benches/tpch/run.sh b/benches/tpch/run.sh new file mode 100755 index 0000000..e9c6ca4 --- /dev/null +++ b/benches/tpch/run.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +set -eu +set -o pipefail + +source ../common.sh + +TPCH_SKIP_SQLITE="${TPCH_SKIP_SQLITE:-}" + +TAR_VER=$(get_tarantool_version) +numaopts=(--membind=1 --cpunodebind=1 '--physcpubind=6,7,8,9,10,11') +numastr='' + +if can_be_run_under_numa "${numaopts[@]}"; then +    numastr="numactl ${numaopts[*]}" +fi + +if [ -n "$TPCH_SKIP_SQLITE" ]; then +    make create_SQL_db + +    kill_tarantool +    sync_disk +    maybe_drop_cache + +    make bench-sqlite NUMAOPTS="$numastr" +fi + +make create_TNT_db + +kill_tarantool +sync_disk +maybe_drop_cache + +make bench-tnt TARANTOOL="$TARANTOOL_EXECUTABLE" NUMAOPTS="$numastr" +make report +sed "/-2/d" bench-tnt.csv | sed "s/;/:/" | sed "s/-1/0/" | tee tpc.h_result.txt + +echo "${TAR_VER}" | tee tpc.h_t_version.txt + +echo "Tarantool TAG:" +cat tpc.h_t_version.txt + +if [ -n "$TPCH_SKIP_SQLITE" ]; then +    echo "Overall result SQL:" +    echo "===================" +    cat bench-sqlite.csv +fi + +echo " " +echo "Overall result TNT:" +echo "===================" +cat tpc.h_result.txt diff --git a/benches/ycsb/run.sh b/benches/ycsb/run.sh new file mode 100755 index 0000000..5083347 --- /dev/null +++ b/benches/ycsb/run.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +source ../common.sh + +YCSB_TYPES="${YCSB_TYPES:-all}" +YCSB_RECORDCOUNT="${YCSB_RECORDCOUNT:-1000000}" +YCSB_OPERATIONCOUNT="${YCSB_OPERATIONCOUNT:-1000000}" +YCSB_MEMTXMEMORY="${YCSB_MEMTXMEMORY:-2000000000}" +YCSB_RUNS="${YCSB_RUNS:-1}" +YCSB_WORKLOADS="${YCSB_WORKLOADS:-all}" + +TAR_VER=$(get_tarantool_version) + +function run_ycsb { +    local mode="$1" +    local srvlua="tarantool/src/main/conf/tarantool-${mode}.lua" +    sed "s/listen=.*/listen=3301,\n   memtx_memory = $YCSB_MEMTXMEMORY,/" -i "$srvlua" +    sed 's/logger_nonblock.*//' -i "$srvlua" +    sed 's/logger/log/' -i "$srvlua" +    sed 's/read,write,execute/create,read,write,execute/' -i "$srvlua" +    maybe_under_numactl "${numaopts[@]}" -- \ +        "$TARANTOOL_EXECUTABLE" "$srvlua" 2>&1 & +    wait_for_tarantool_runnning 3301 10 + +    local workloads=(a b c d e f) + +    if [ -n "$YCSB_WORKLOADS" -a "$YCSB_WORKLOADS" != "all" ]; then +        IPS=, read -ra workloads <<< "$YCSB_WORKLOADS" +    fi + +    for l in "${workloads[@]}"; do +        echo "=============== $l" +        for r in $( seq 1 "$YCSB_RUNS" ); do +            local res="$plogs/run${l}_${r}" +            echo "---------------- ${l}: $r" +            echo "tarantool.port=3301" >> "workloads/workload${l}" +            maybe_under_numactl "${numaopts[@]}" -- \ +                ./bin/ycsb load tarantool -s -P "workloads/workload${l}" > "${res}.load" 2>&1 || cat "${res}.load" +            sync_disk +            maybe_drop_cache + +            maybe_under_numactl "${numaopts[@]}" -- \ +                ./bin/ycsb run tarantool -s -P "workloads/workload${l}" > "${res}.log" 2>&1 || cat "${res}.log" + +            grep Thro "${res}.log" | awk '{ print "Overall result: "$3 }' | tee "${res}.txt" +            sed "s#Overall result#$l $r#g" "${res}.txt" >> "${plogs}/ycsb.${mode}_result.txt" + +            stop_and_clean_tarantool +        done +    done +} + +types=(hash tree) + +if [ -n "$YCSB_TYPES" -a "$YCSB_TYPES" != 'all' ]; then +    IPS=, read -ra types <<< "$YCSB_TYPES" +fi + +numaopts=(--membind=1 --cpunodebind=1 '--physcpubind=6,7,8,9,10,11') + +for f in workloads/workload[a-f] ; do +    sed "s#recordcount=.*#recordcount=$YCSB_RECORDCOUNT#g" -i "$f" +    sed "s#operationcount=.*#operationcount=$YCSB_OPERATIONCOUNT#g" -i "$f" +    echo "tarantool.port=3301" >> "$f" +done + + +plogs=results +rm -rf "$plogs" +mkdir "$plogs" + +for t in "${types[@]}"; do +    run_ycsb "$t" +done + +echo "${TAR_VER}" | tee "ycsb.${mode}_t_version.txt" +cp -f "${plogs}/ycsb.${mode}_result.txt" . + +echo "Tarantool TAG:" +cat "ycsb.${mode}_t_version.txt" +echo "Overall results:" +echo "================" +cat "ycsb.${mode}_result.txt" diff --git a/config.sh b/config.sh new file mode 100644 index 0000000..93d3fc5 --- /dev/null +++ b/config.sh @@ -0,0 +1,41 @@ +# path to tarantool executable +export TARANTOOL_EXECUTABLE=tarantool + +# location of tests to be run +export BENCH_WORKDIR="$PWD/.benchdir" + +# cbench parameters +export CBENCH_DIR="$HOME/work/cbench/" +export CBENCH_VINYL_WORKLOAD=50 +export CBENCH_MEMTX_WORKLOAD=10000 + +# linkbench parameters +export LINKBENCH_DIR="$HOME/work/linkbench/" +export LINKBENCH_REQUESTERS=1 +export LINKBENCH_REQUESTS=1000 +export LINKBENCH_WORKLOAD_SIZE=1000 + +# linkbench parameters +export NOSQLBENCH_DIR="$HOME/work/nosqlbench/" + +# linkbench parameters +export SYSBENCH_DIR="$HOME/work/sysbench/" +export SYSBENCH_RUNS=1 +export SYSBENCH_TIME=5 + +# tpcc parameters +export TPCC_DIR="$HOME/work/tpcc/" +export TPCC_TIME=20 +export TPCC_WARMUPTIME=5 +export TPCC_WAREHOUSES=15 +export TPCC_FROMSNAPSHOT="/tmp/00000000000000033515.snap" + +# tpcc parameters +export TPCH_DIR="$HOME/work/tpch/" +export TPCH_SKIP_SQLITE=1 + +# tpcc parameters +export YCSB_DIR="$HOME/work/ycsb/" +export YCSB_OPERATIONCOUNT=1000 +export YCSB_RECORDCOUNT=1000 +export YCSB_RUNS=1 diff --git a/ttbench b/ttbench new file mode 100755 index 0000000..ac656f5 --- /dev/null +++ b/ttbench @@ -0,0 +1,149 @@ +#!/bin/bash + +set -euo pipefail + +TTBENCH_CONFIG=config.sh +TTBENCH_BENCHES=all +TTBENCH_COMMAND= +TTBENCH_BENCHDIR=benches + +function error { +    echo "ERROR:" "$@" +    exit 100 +} + +function print_help { +    grep '^\s*# HELP' "$0" | sed 's/^\s*# HELP\s//g' +} + +function list_benchmarks { +    for d in $TTBENCH_BENCHDIR/*; do +        sed "s#$TTBENCH_BENCHDIR/##" <<< "$d" +    done | grep -v common.sh +} + +function run_bench { +    local bench="$1" +    local benchdir= +    local workdir="${BENCH_WORKDIR:?}/$bench/" + +    cp "$TTBENCH_BENCHDIR/common.sh" "${BENCH_WORKDIR}" + +    export COMMON_FUNCTIONS="$BENCH_WORKDIR/common.sh" + +    if benchdir=$(env | grep "${bench^^}_DIR" | sed 's/^.*=//'); then +        echo "Running $bench with workdir $benchdir" 1>&2 +        rm -rf "$workdir" +        mkdir -p "$workdir" + +        cp -r "$TTBENCH_BENCHDIR/$bench"/* "$workdir" +        cp -r "$benchdir"/* "$workdir" + +        local runner= +        runner=$(readlink -f "$TTBENCH_BENCHDIR/$bench/run.sh") +        pushd "$workdir" 1>/dev/null +        "$runner" +        popd 1>/dev/null +    else +        error "variable '${bench^^}_DIR' is not set in config, can't run benchmark" +    fi +} + +function run_benchmarks { +    # must be set in config +    [ -z "$BENCH_WORKDIR" ] && error "BENCH_WORKDIR is not set" + +    if [ "$TTBENCH_BENCHES" == all ]; then +        for b in $(list_benchmarks); do +            run_bench "$b" +        done +    else +        IFS=, read -ra benchlist <<< "$TTBENCH_BENCHES" +        # local benchlist=( $(sed 's/,/ /g' <<< "$TTBENCH_BENCHES") ); +        for b in "${benchlist[@]}"; do +            if [ ! -d "$TTBENCH_BENCHDIR/$b" ]; then +                error "Unknown benchmark='$b', please run '$0 list-benchmarks'" +            fi +        done + +        for b in "${benchlist[@]}"; do +            run_bench "$b" +        done +    fi +} + +# HELP ttbench - launch tarantool external benchmarks +# HELP +# HELP Usage: ttbench [OPTIONS]... COMMAND +# HELP +# HELP Available options are: +while [[ $# -gt 0 ]]; do +    option="$1" + +    case $option in +        # HELP   -c, --config 'config_file' +        # HELP           use given config file (defaults to config.sh) +        # HELP +        -c|--config) +            TTBENCH_CONFIG="${2:-}" +            [ -z "$TTBENCH_CONFIG" ] && error "$option param can not be empty" +            shift 2;; + +        # HELP   -d, --debug +        # HELP           sets -x flag on whole script +        # HELP +        -d|--debug) +            set -x +            shift;; + +        # HELP   -b, --benchmarks 'bench1,bench2' +        # HELP           comma separated list of benchmarks (defauts to 'all') +        # HELP +        -b|--benchmarks) +            TTBENCH_BENCHES="${2:-}" +            [ -z "$TTBENCH_BENCHES" ] && error "$option param can not be empty" +            shift 2;; + +        # HELP   -h, --help +        # HELP           print help message +        # HELP +        -h|--help) +            print_help +            exit 0;; + +        *) +            break;; +    esac +done + +[ -f "$TTBENCH_CONFIG" ] && source "$TTBENCH_CONFIG" + +TTBENCH_COMMAND="${1:-}" + +if [ -z "$TTBENCH_COMMAND" ]; then +    print_help +    echo "ERROR: no COMMAND specified" 1>&2 +    exit 1 +fi + +shift + +# HELP +# HELP Available commands are: +case "$TTBENCH_COMMAND" in +    # HELP   ls, list-benchmarks +    # HELP           lists all known benchmarks +    # HELP +    ls|list-benchmarks) +        list_benchmarks;; + +    # HELP   run +    # HELP           run benchmarks specified by --benchmarks flag +    run) +        run_benchmarks;; + +    *) +        print_help +        echo "ERROR: Unknown COMMAND='$TTBENCH_COMMAND'" 1>&2 +        exit 1;; +esac -- 2.7.4