From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from localhost (localhost [127.0.0.1]) by turing.freelists.org (Avenir Technologies Mail Multiplex) with ESMTP id D3A722B7C2 for ; Thu, 30 May 2019 06:14:07 -0400 (EDT) Received: from turing.freelists.org ([127.0.0.1]) by localhost (turing.freelists.org [127.0.0.1]) (amavisd-new, port 10024) with ESMTP id yAwuq0T9K1rh for ; Thu, 30 May 2019 06:14:07 -0400 (EDT) Received: from smtp34.i.mail.ru (smtp34.i.mail.ru [94.100.177.94]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by turing.freelists.org (Avenir Technologies Mail Multiplex) with ESMTPS id 01C502B79B for ; Thu, 30 May 2019 06:14:06 -0400 (EDT) From: "Alexander V. Tikhonov" Subject: [tarantool-patches] [PATCH v1] Gitlab-ci testing process implement Date: Thu, 30 May 2019 13:14:02 +0300 Message-Id: <189eee463e7af31f0784e840454a7096b111dd4c.1559211098.git.avtikhon@tarantool.org> Sender: tarantool-patches-bounce@freelists.org Errors-to: tarantool-patches-bounce@freelists.org Reply-To: tarantool-patches@freelists.org List-Help: List-Unsubscribe: List-software: Ecartis version 1.0.0 List-Id: tarantool-patches List-Subscribe: List-Owner: List-post: List-Archive: To: Alexander Turenko Cc: avtikhon , tarantool-patches@freelists.org From: avtikhon Complete testing process equal to the current travis-ci prepared. Closes #4156 --- Gitlab-ci: https://gitlab.com/tarantool/tarantool/pipelines/63905818 Issue: https://github.com/tarantool/tarantool/issues/4156 .gitlab-ci.yml | 554 ++++++++++++++++++++++++++++ .travis.mk | 43 ++- .travis.yml | 133 ------- images/Dockerfile.debian | 4 + images/Dockerfile.lto | 5 + images/Dockerfile.ubuntu_perf | 59 +++ images/Dockerfile.ubuntu_perf_build | 14 + perf/run-tarantool-server.sh | 13 + perf/tarantool-server.lua | 26 ++ test/box/suite.ini | 2 +- 10 files changed, 706 insertions(+), 147 deletions(-) create mode 100644 .gitlab-ci.yml delete mode 100644 .travis.yml create mode 100644 images/Dockerfile.debian create mode 100644 images/Dockerfile.lto create mode 100644 images/Dockerfile.ubuntu_perf create mode 100644 images/Dockerfile.ubuntu_perf_build create mode 100755 perf/run-tarantool-server.sh create mode 100644 perf/tarantool-server.lua diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 000000000..d37c4681e --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,554 @@ +stages: + - bootstrap + - test + - perf_deploy + +variables: + # Docker + DCKLGN: "docker login -u ${CI_REGISTRY_USER} -p ${CI_REGISTRY_PASSWORD} ${CI_REGISTRY}" + DCKBLD: "docker build --network=host" + DCKPSH: "docker push" + # Build + FLAG_LTO: "-DENABLE_LTO=ON" + GITCLN: "git submodule update --recursive --init --force" + TRAVMK: "make -f .travis.mk" + # Images + TVER: "master" + IMGUBB_BST: "${CI_REGISTRY}/${CI_PROJECT_PATH}:ubuntu-bionic_perf" + IMGUBB_BLT: "${CI_REGISTRY}/${CI_PROJECT_PATH}:ubuntu-bionic_perf_${TVER}" + IMGDBS_BST: "${CI_REGISTRY}/${CI_PROJECT_PATH}:debian-stretch_${TVER}" + IMGDBB_BST: "${CI_REGISTRY}/${CI_PROJECT_PATH}:debian-buster_${TVER}" + # OSX + OSX_SSH: "ssh -A tarantool@127.0.0.1 -p 2222" + OSX_SSHENV: "export PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin" + OSX_GITCHK: "git fetch -p && git checkout --force ${CI_BUILD_REF} && ${GITCLN}" + # Pack + PACKCLO: "git clone https://github.com/packpack/packpack.git packpack" + PACKMK: "make -f packpack/pack/Makefile -C . BUILDDIR=$PWD/build -j" + +# Bootstraps + +debian9: + stage: bootstrap + tags: + - bootstrap + before_script: + - ${DCKLGN} + script: + - cp .travis.mk images/ + - ${DCKBLD} -t ${IMGDBS_BST} -f images/Dockerfile.debian images + - ${DCKPSH} ${IMGDBS_BST} + +debian10_lto: + only: + refs: + - master + stage: bootstrap + tags: + - bootstrap + before_script: + - ${DCKLGN} + script: + - cp .travis.mk images/ + - ${DCKBLD} -t ${IMGDBB_BST} -f images/Dockerfile.lto images + - ${DCKPSH} ${IMGDBB_BST} + +# Tests + +release: + image: ${IMGDBS_BST} + stage: test + tags: + - docker_test + script: + - ${GITCLN} + - ${TRAVMK} runtest_ubuntu + +debug: + image: ${IMGDBS_BST} + stage: test + tags: + - docker_test + script: + - ${GITCLN} + - ${TRAVMK} runtest_cov_ubuntu + +release_clang: + image: ${IMGDBS_BST} + stage: test + tags: + - docker_test + variables: + CC: clang + CXX: clang++ + script: + - ${GITCLN} + - ${TRAVMK} runtest_ubuntu + +release_lto: + only: + refs: + - master + image: ${IMGDBB_BST} + stage: test + tags: + - docker_test + variables: + CMAKE_EXTRA_PARAMS: ${FLAG_LTO} + script: + - ${GITCLN} + - ${TRAVMK} runtest_ubuntu + +release_lto_clang8: + only: + refs: + - master + image: ${IMGDBB_BST} + stage: test + tags: + - docker_test + variables: + CC: clang-8 + CXX: clang++-8 + CMAKE_EXTRA_PARAMS: ${FLAG_LTO} + script: + - ${GITCLN} + - ${TRAVMK} runtest_ubuntu + +osx13_release: + only: + refs: + - master + stage: test + tags: + - osx + before_script: + - VBoxManage controlvm Sierra poweroff || true + - VBoxManage snapshot Sierra restore Sierra + - VBoxManage startvm Sierra --type headless + script: + - export + - ${OSX_SSH} "/bin/bash -c '${OSX_SSHENV} ; cd tarantool && ${OSX_GITCHK} && ${TRAVMK} test_osx'" + after_script: + - VBoxManage controlvm Sierra poweroff + +osx14_release: + stage: test + tags: + - osx + before_script: + - VBoxManage controlvm Mojave poweroff || true + - VBoxManage snapshot Mojave restore Mojave + - VBoxManage startvm Mojave --type headless + script: + - export + - ${OSX_SSH} "/bin/bash -c '${OSX_SSHENV} ; cd tarantool && ${OSX_GITCHK} && ${TRAVMK} test_osx'" + after_script: + - VBoxManage controlvm Mojave poweroff + +osx14_release_lto: + only: + refs: + - master + stage: test + tags: + - osx + before_script: + - VBoxManage controlvm Mojave poweroff || true + - VBoxManage snapshot Mojave restore Mojave + - VBoxManage startvm Mojave --type headless + script: + - export + - ${OSX_SSH} "/bin/bash -c '${OSX_SSHENV} ; export CMAKE_EXTRA_PARAMS=${FLAG_LTO} ; cd tarantool && ${OSX_GITCHK} && ${TRAVMK} test_osx'" + after_script: + - VBoxManage controlvm Mojave poweroff + +# Perf prepare + +perf_bootstrap: + only: + refs: + - master + - /.*-perf/ + - /.*-perflong/ + stage: test + tags: + - perf + before_script: + - ${DCKLGN} + script: + # !!! TEMPORARY !!! + - cp -rfp /home/aleks.tikhonov/Workspaces/runners images/. + - ${DCKBLD} -t ${IMGUBB_BST} -f images/Dockerfile.ubuntu_perf images + - ${DCKPSH} ${IMGUBB_BST} + - ${DCKBLD} -t ${IMGUBB_BLT} -f images/Dockerfile.ubuntu_perf_build . + - ${DCKPSH} ${IMGUBB_BLT} + +# Perf sysbench 1 *perf/*perflong + 2 *perflong + +perf_sysbench: + only: + refs: + - master + - /.*-perf/ + - /.*-perflong/ + image: ${IMGUBB_BLT} + stage: perf_deploy + tags: + - docker_perf + script: + - perf/run-tarantool-server.sh + - THREADS=4 /opt/runners/sysbench-benchmarking/run.sh + +perf_sysbench_1: + only: + refs: + - /.*-perflong/ + image: ${IMGUBB_BLT} + stage: perf_deploy + tags: + - docker_perf + script: + - perf/run-tarantool-server.sh + - THREADS=4 /opt/runners/sysbench-benchmarking/run.sh + +perf_sysbench_2: + only: + refs: + - /.*-perflong/ + image: ${IMGUBB_BLT} + stage: perf_deploy + tags: + - docker_perf + script: + - perf/run-tarantool-server.sh + - THREADS=4 /opt/runners/sysbench-benchmarking/run.sh + +# Perf tpcc 1 *perf/*perflong + 2 *perflong + +perf_tpcc: + only: + refs: + - master + - /.*-perf/ + - /.*-perflong/ + image: ${IMGUBB_BLT} + stage: perf_deploy + tags: + - docker_perf + script: + - /opt/runners/tpcc-benchmarking/run.sh + +perf_tpcc_1: + only: + refs: + - /.*-perflong/ + image: ${IMGUBB_BLT} + stage: perf_deploy + tags: + - docker_perf + script: + - /opt/runners/tpcc-benchmarking/run.sh + +perf_tpcc_2: + only: + refs: + - /.*-perflong/ + image: ${IMGUBB_BLT} + stage: perf_deploy + tags: + - docker_perf + script: + - /opt/runners/tpcc-benchmarking/run.sh + +# Perf ycsb 1 *perf + 1*5 *perflong + +perf_ycsb: + only: + refs: + - master + - /.*-perf/ + image: ${IMGUBB_BLT} + stage: perf_deploy + tags: + - docker_perf + script: + - /opt/runners/ycsb-benchmarking/run.sh + +perf_ycsb_1..5: + only: + refs: + - /.*-perflong/ + image: ${IMGUBB_BLT} + stage: perf_deploy + tags: + - docker_perf + script: + - /opt/runners/ycsb-benchmarking/run.sh 5 + +# Perf nosqlbench 1 *perf/*perflong + 4 *perflong + +perf_nosqlbench: + only: + refs: + - master + - /.*-perf/ + - /.*-perflong/ + image: ${IMGUBB_BLT} + stage: perf_deploy + tags: + - docker_perf + script: + - /opt/runners/nosqlbench-benchmarking/run.sh + +perf_nosqlbench_1: + only: + refs: + - /.*-perflong/ + image: ${IMGUBB_BLT} + stage: perf_deploy + tags: + - docker_perf + script: + - /opt/runners/nosqlbench-benchmarking/run.sh + +perf_nosqlbench_2: + only: + refs: + - /.*-perflong/ + image: ${IMGUBB_BLT} + stage: perf_deploy + tags: + - docker_perf + script: + - /opt/runners/nosqlbench-benchmarking/run.sh + +perf_nosqlbench_3: + only: + refs: + - /.*-perflong/ + image: ${IMGUBB_BLT} + stage: perf_deploy + tags: + - docker_perf + script: + - /opt/runners/nosqlbench-benchmarking/run.sh + +perf_nosqlbench_4: + only: + refs: + - /.*-perflong/ + image: ${IMGUBB_BLT} + stage: perf_deploy + tags: + - docker_perf + script: + - /opt/runners/nosqlbench-benchmarking/run.sh + +# Perf cbench 1 *perf/*perflong + 2 *perflong + +perf_cbench: + only: + refs: + - master + - /.*-perf/ + - /.*-perflong/ + image: ${IMGUBB_BLT} + stage: perf_deploy + tags: + - docker_perf + script: + - /opt/runners/cbench-benchmarking/run.sh + +perf_cbench_1: + only: + refs: + - /.*-perflong/ + image: ${IMGUBB_BLT} + stage: perf_deploy + tags: + - docker_perf + script: + - /opt/runners/cbench-benchmarking/run.sh + +perf_cbench_2: + only: + refs: + - /.*-perflong/ + image: ${IMGUBB_BLT} + stage: perf_deploy + tags: + - docker_perf + script: + - /opt/runners/cbench-benchmarking/run.sh + +# Packs + +pack_cent6: + only: + refs: + - master + image: packpack/packpack:el-6 + stage: perf_deploy + tags: + - deploy + script: + - git submodule update --recursive --init + - ${PACKCLO} + - ${PACKMK} + +pack_cent7: + only: + refs: + - master + image: packpack/packpack:el-7 + stage: perf_deploy + tags: + - deploy + script: + - ${GITCLN} + - ${PACKCLO} + - ${PACKMK} + +pack_fedora28: + only: + refs: + - master + image: packpack/packpack:fedora-28 + stage: perf_deploy + tags: + - deploy + script: + - ${GITCLN} + - ${PACKCLO} + - ${PACKMK} + +pack_fedora29: + only: + refs: + - master + image: packpack/packpack:fedora-29 + stage: perf_deploy + tags: + - deploy + script: + - ${GITCLN} + - ${PACKCLO} + - ${PACKMK} + +pack_fedora30: + only: + refs: + - master + image: packpack/packpack:fedora-30 + stage: perf_deploy + tags: + - deploy + script: + - ${GITCLN} + - ${PACKCLO} + - ${PACKMK} + +pack_ubuntu14: + only: + refs: + - master + image: packpack/packpack:ubuntu-trusty + stage: perf_deploy + tags: + - deploy + script: + - ${GITCLN} + - ${PACKCLO} + - ${PACKMK} + +pack_ubuntu16: + only: + refs: + - master + image: packpack/packpack:ubuntu-xenial + stage: perf_deploy + tags: + - deploy + script: + - ${GITCLN} + - ${PACKCLO} + - ${PACKMK} + +pack_ubuntu18: + only: + refs: + - master + image: packpack/packpack:ubuntu-bionic + stage: perf_deploy + tags: + - deploy + script: + - ${GITCLN} + - ${PACKCLO} + - ${PACKMK} + +pack_ubuntu1810: + only: + refs: + - master + image: packpack/packpack:ubuntu-cosmic + stage: perf_deploy + tags: + - deploy + script: + - ${GITCLN} + - ${PACKCLO} + - ${PACKMK} + +pack_ubuntu19: + only: + refs: + - master + image: packpack/packpack:ubuntu-disco + stage: perf_deploy + tags: + - deploy + script: + - ${GITCLN} + - ${PACKCLO} + - ${PACKMK} + +pack_debian8: + only: + refs: + - master + image: packpack/packpack:debian-jessie + stage: perf_deploy + tags: + - deploy + script: + - ${GITCLN} + - ${PACKCLO} + - ${PACKMK} + +pack_debian9: + only: + refs: + - master + image: packpack/packpack:debian-stretch + stage: perf_deploy + tags: + - deploy + script: + - ${GITCLN} + - ${PACKCLO} + - ${PACKMK} + +pack_debian10: + only: + refs: + - master + image: packpack/packpack:debian-buster + stage: perf_deploy + tags: + - deploy + script: + - ${GITCLN} + - ${PACKCLO} + - ${PACKMK} diff --git a/.travis.mk b/.travis.mk index 55bee9980..92d3e7e65 100644 --- a/.travis.mk +++ b/.travis.mk @@ -34,24 +34,36 @@ docker_%: make -f .travis.mk $(subst docker_,,$@) deps_ubuntu: - sudo apt-get update && apt-get install -y -f \ + apt-get -y update && apt-get install -y -f \ build-essential cmake coreutils sed \ libreadline-dev libncurses5-dev libyaml-dev libssl-dev \ libcurl4-openssl-dev libunwind-dev libicu-dev \ python python-pip python-setuptools python-dev \ python-msgpack python-yaml python-argparse python-six python-gevent \ - lcov ruby clang llvm llvm-dev + lcov ruby clang llvm llvm-dev mdm -test_ubuntu: deps_ubuntu +deps_buster_clang_8: + echo "deb http://apt.llvm.org/buster/ llvm-toolchain-buster-8 main" > /etc/apt/sources.list.d/clang_8.list + echo "deb-src http://apt.llvm.org/buster/ llvm-toolchain-buster-8 main" >> /etc/apt/sources.list.d/clang_8.list + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - + apt-get -y update + apt-get -y install clang-8 llvm-8 llvm-8-dev + +build_ubuntu: cmake . -DCMAKE_BUILD_TYPE=RelWithDebInfo -DENABLE_WERROR=ON ${CMAKE_EXTRA_PARAMS} - make -j8 - cd test && /usr/bin/python test-run.py --force -j 1 + make -j + +runtest_ubuntu: build_ubuntu + cd test && /usr/bin/python test-run.py --force -j $$((`ncpus`*2)) --no-output-timeout -1 + +test_ubuntu: deps_ubuntu runtest_ubuntu deps_osx: brew update - brew install openssl readline curl icu4c --force - curl --silent --show-error --retry 5 https://bootstrap.pypa.io/get-pip.py | python - pip install -r test-run/requirements.txt + brew install openssl readline curl icu4c cmake --force || true + curl --silent --show-error --retry 5 https://bootstrap.pypa.io/get-pip.py >get-pip.py + python get-pip.py --user + pip install --user --ignore-installed -r test-run/requirements.txt test_osx: deps_osx cmake . -DCMAKE_BUILD_TYPE=RelWithDebInfo -DENABLE_WERROR=ON ${CMAKE_EXTRA_PARAMS} @@ -61,14 +73,17 @@ test_osx: deps_osx sudo launchctl limit maxfiles 20480 || : ulimit -S -n 20480 || : ulimit -n - make -j8 - cd test && python test-run.py --force -j 1 unit/ app/ app-tap/ box/ box-tap/ + make -j + cd test && TRAVIS=True python test-run.py --force -j $$((`sysctl -n hw.ncpu`*2)) unit/ app/ app-tap/ box/ box-tap/ -coverage_ubuntu: deps_ubuntu +build_cov_ubuntu: cmake . -DCMAKE_BUILD_TYPE=Debug -DENABLE_GCOV=ON - make -j8 + find -name "*.gcda" -exec rm -rf {} \; + make -j + +runtest_cov_ubuntu: build_cov_ubuntu # Enable --long tests for coverage - cd test && /usr/bin/python test-run.py --force -j 1 --long + cd test && /usr/bin/python test-run.py --force -j $$((`ncpus`*2)) --long --no-output-timeout -1 lcov --compat-libtool --directory src/ --capture --output-file coverage.info.tmp lcov --compat-libtool --remove coverage.info.tmp 'tests/*' 'third_party/*' '/usr/*' \ --output-file coverage.info @@ -80,6 +95,8 @@ coverage_ubuntu: deps_ubuntu coveralls-lcov --service-name travis-ci --service-job-id $(TRAVIS_JOB_ID) --repo-token $(COVERALLS_TOKEN) coverage.info; \ fi; +coverage_ubuntu: deps_ubuntu runtest_cov_ubuntu + source: git clone https://github.com/packpack/packpack.git packpack TARBALL_COMPRESSOR=gz packpack/packpack tarball diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index e94d02ef5..000000000 --- a/.travis.yml +++ /dev/null @@ -1,133 +0,0 @@ -sudo: false -services: - - docker - -language: cpp - -# default values -os: linux -compiler: gcc - -osx_image: xcode10.2 - -cache: - directories: - - $HOME/.cache - -git: - depth: 100500 - -jobs: - include: - # Testing targets (just run tests on Debian Stretch or OS X). - - name: "RelWithDebInfo build + test (Linux, gcc)" - env: TARGET=test - - name: "RelWithDebInfo build + test (Linux, clang)" - env: TARGET=test - compiler: clang - - name: "RelWithDebInfo build + test (OS X Mojave 10.14)" - env: TARGET=test - os: osx - - name: "Debug build + test + coverage (Linux, gcc)" - env: TARGET=coverage - - name: "RelWithDebInfo build + test (OS X High Sierra 10.13)" - env: TARGET=test - os: osx - osx_image: xcode9.4 - if: branch = "master" - # Special targets (only LTO for now). - - name: "LTO build + test (Linux, gcc)" - env: > - TARGET=test - CMAKE_EXTRA_PARAMS=-DENABLE_LTO=ON - DOCKER_IMAGE=packpack/packpack:debian-buster - if: branch = "master" - - name: "LTO build + test (Linux, clang)" - env: > - TARGET=test - CMAKE_EXTRA_PARAMS=-DENABLE_LTO=ON - DOCKER_IMAGE=packpack/packpack:debian-buster - if: branch = "master" - compiler: clang - - name: "LTO build + test (OS X Mojave 10.14)" - os: osx - env: TARGET=test CMAKE_EXTRA_PARAMS=-DENABLE_LTO=ON - if: branch = "master" - # Deploy targets (they also catch distro-specific problems). - - name: "Create and deploy tarball" - env: TARGET=source - if: branch = "master" - - name: "CentOS 6 build + deploy RPM" - env: OS=el DIST=6 - if: branch = "master" - - name: "CentOS 7 build + test + deploy RPM" - env: OS=el DIST=7 - if: branch = "master" - - name: "Fedora 28 build + test + deploy RPM" - env: OS=fedora DIST=28 - if: branch = "master" - - name: "Fedora 29 build + test + deploy RPM" - env: OS=fedora DIST=29 - if: branch = "master" - - name: "Fedora 30 build + test + deploy RPM" - env: OS=fedora DIST=30 - if: branch = "master" - - name: "Ubuntu Trusty (14.04) build + deploy DEB" - env: OS=ubuntu DIST=trusty - if: branch = "master" - - name: "Ubuntu Xenial (16.04) build + deploy DEB" - env: OS=ubuntu DIST=xenial - if: branch = "master" - - name: "Ubuntu Bionic (18.04) build + deploy DEB" - env: OS=ubuntu DIST=bionic - if: branch = "master" - - name: "Ubuntu Cosmic (18.10) build + deploy DEB" - env: OS=ubuntu DIST=cosmic - if: branch = "master" - - name: "Ubuntu Disco (19.04) build + deploy DEB" - env: OS=ubuntu DIST=disco - if: branch = "master" - - name: "Debian Jessie (8) build + deploy DEB" - env: OS=debian DIST=jessie - if: branch = "master" - - name: "Debian Stretch (9) build + deploy DEB" - env: OS=debian DIST=stretch - if: branch = "master" - - name: "Debian Buster (10) build + deploy DEB" - env: OS=debian DIST=buster - if: branch = "master" - -script: - - make -f .travis.mk ${TARGET} - -before_deploy: - - ls -l build/ - -deploy: - # Deploy packages to PackageCloud - - provider: packagecloud - username: "tarantool" - repository: "2_2" - token: "${PACKAGECLOUD_TOKEN}" - dist: "${OS}/${DIST}" - package_glob: build/*.{rpm,deb,dsc} - skip_cleanup: true - on: - repo: tarantool/tarantool - branch: "master" - condition: -n "${OS}" && -n "${DIST}" && -n "${PACKAGECLOUD_TOKEN}" - # Deploy source tarballs to S3 - - provider: script - script: make -f .travis.mk source_deploy - skip_cleanup: true - on: - repo: tarantool/tarantool - branch: "master" - condition: "x${TARGET} = xsource" - -notifications: - email: - recipients: - - build@tarantool.org - on_success: change - on_failure: always diff --git a/images/Dockerfile.debian b/images/Dockerfile.debian new file mode 100644 index 000000000..eb5278343 --- /dev/null +++ b/images/Dockerfile.debian @@ -0,0 +1,4 @@ +FROM packpack/packpack:debian-stretch + +COPY .travis.mk . +RUN make -f .travis.mk deps_ubuntu diff --git a/images/Dockerfile.lto b/images/Dockerfile.lto new file mode 100644 index 000000000..54c585acc --- /dev/null +++ b/images/Dockerfile.lto @@ -0,0 +1,5 @@ +FROM packpack/packpack:debian-buster + +COPY .travis.mk . +RUN make -f .travis.mk deps_ubuntu +RUN make -f .travis.mk deps_buster_clang_8 diff --git a/images/Dockerfile.ubuntu_perf b/images/Dockerfile.ubuntu_perf new file mode 100644 index 000000000..0541746de --- /dev/null +++ b/images/Dockerfile.ubuntu_perf @@ -0,0 +1,59 @@ +FROM ubuntu:18.04 + +RUN apt-get -y update && apt-get install -y -f \ + numactl gcc libc6-dev zlib1g-dev make libmysqlclient-dev \ + ssh vim git dh-autoreconf pkg-config libicu-dev \ + build-essential cmake coreutils sed libreadline-dev \ + libncurses5-dev libyaml-dev libssl-dev libcurl4-openssl-dev \ + libunwind-dev python python-pip python-setuptools python-dev \ + python-msgpack python-yaml python-argparse python-six python-gevent \ + gdb net-tools + +# benchmark-runners +# !!! TEMPORARY !!! +#RUN git clone https://gitlab.com/tarantool/tarantool-benchmarks-runner.git \ +# /opt/runners && \ +COPY runners /opt/runners + +# tarantool-c +RUN git clone --recursive https://github.com/tarantool/tarantool-c.git \ + /opt/tarantool-c +WORKDIR /opt/tarantool-c/third_party/msgpuck/ +RUN cmake . && make && make install +WORKDIR /opt/tarantool-c +RUN cmake . && make -j && make install + +# msgpack-c +RUN git clone https://github.com/msgpack/msgpack-c.git /opt/msgpack-c +WORKDIR /opt/msgpack-c +RUN cmake . && make -j && make install + +# sysbench +RUN git clone https://github.com/iproha94/sysbench.git /opt/sysbench +WORKDIR /opt/sysbench +RUN ./autogen.sh && ./configure --with-tarantool --without-mysql && \ + make -j && make install && \ + sed 's# CHAR(# VARCHAR(#g' -i /opt/sysbench/src/lua/oltp_common.lua && \ + cp /opt/sysbench/src/lua/oltp_common.lua \ + /usr/local/share/sysbench/oltp_common.lua + +# tpcc-tarantool +RUN git clone https://github.com/tarantool/tpcc.git /opt/tpcc +WORKDIR /opt/tpcc/src +RUN make + +# YCSB +RUN git clone https://github.com/brianfrankcooper/YCSB.git /opt/ycsb +RUN apt-get update -y && apt-get install gtk-update-icon-cache=3.22.30-1ubuntu3 || true +RUN apt-get install --fix-missing -y -f default-jdk maven +WORKDIR /opt/ycsb +RUN mvn -pl com.yahoo.ycsb:tarantool-binding -am clean package + +# nosqlbench +RUN git clone --recursive https://github.com/tarantool/nosqlbench.git \ + /opt/nosqlbench +WORKDIR /opt/nosqlbench +RUN git submodule foreach --recursive git pull origin master +RUN apt-get install -y -f libev-dev +RUN cmake . && make -j + diff --git a/images/Dockerfile.ubuntu_perf_build b/images/Dockerfile.ubuntu_perf_build new file mode 100644 index 000000000..4f14b5469 --- /dev/null +++ b/images/Dockerfile.ubuntu_perf_build @@ -0,0 +1,14 @@ +FROM registry.gitlab.com/tarantool/tarantool:ubuntu-bionic_perf + +COPY . /opt/tarantool +WORKDIR /opt/tarantool +RUN git submodule update --recursive --init --force +RUN cmake . -DENABLE_DIST=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo >cmake.log && make -j >build.log && make install >install.log + +# cbench +RUN git clone https://github.com/tarantool/cbench.git /opt/cbench +WORKDIR /opt/cbench +RUN cmake . -DCMAKE_BUILD_TYPE=RelWithDebInfo && make && make install + +# reset the workdir to tarantool path +WORKDIR /opt/tarantool diff --git a/perf/run-tarantool-server.sh b/perf/run-tarantool-server.sh new file mode 100755 index 000000000..f43c2427f --- /dev/null +++ b/perf/run-tarantool-server.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +numactl --show +numactl --hardware +numactl --cpunodebind=1 --physcpubind=6,7,8,9,10,11 tarantool perf/tarantool-server.lua 2>&1 + +STATUS= +while [ ${#STATUS} -eq "0" ]; do + STATUS="$(echo box.info.status | tarantoolctl connect /tmp/tarantool-server.sock | grep -e "- running")" + echo "waiting load snapshot to tarantool..." + sleep 5 +done + diff --git a/perf/tarantool-server.lua b/perf/tarantool-server.lua new file mode 100644 index 000000000..a0de7d932 --- /dev/null +++ b/perf/tarantool-server.lua @@ -0,0 +1,26 @@ +local console = require('console') +console.listen("/tmp/tarantool-server.sock") + +box.cfg { + pid_file = "./tarantool-server.pid", + log = "./tarantool-server.log", + listen = 3301, + memtx_memory = 2000000000, + background = true, + checkpoint_interval = 0, +} + + +function try(f, catch_f) + local status, exception = pcall(f) + if not status then + catch_f(exception) + end +end + +try(function() + box.schema.user.grant('guest', 'create,read,write,execute', 'universe') +end, function(e) + print(e) +end) + diff --git a/test/box/suite.ini b/test/box/suite.ini index c7b75c173..5efad688f 100644 --- a/test/box/suite.ini +++ b/test/box/suite.ini @@ -2,7 +2,7 @@ core = tarantool description = Database tests script = box.lua -disabled = rtree_errinj.test.lua tuple_bench.test.lua +disabled = rtree_errinj.test.lua tuple_bench.test.lua on_shutdown.test.lua release_disabled = errinj.test.lua errinj_index.test.lua rtree_errinj.test.lua upsert_errinj.test.lua iproto_stress.test.lua lua_libs = lua/fifo.lua lua/utils.lua lua/bitset.lua lua/index_random_test.lua lua/push.lua lua/identifier.lua use_unix_sockets = True -- 2.17.1