Tarantool development patches archive
 help / color / mirror / Atom feed
* [tarantool-patches] [PATCH v1] Gitlab-ci testing process implement
@ 2019-05-30 10:14 Alexander V. Tikhonov
  2019-06-07 17:33 ` [tarantool-patches] " Alexander Turenko
  0 siblings, 1 reply; 2+ messages in thread
From: Alexander V. Tikhonov @ 2019-05-30 10:14 UTC (permalink / raw)
  To: Alexander Turenko; +Cc: avtikhon, tarantool-patches

From: avtikhon <avtikhon@gmail.com>

Complete testing process equal to the current travis-ci prepared.

Closes #4156
---

Gitlab-ci: https://gitlab.com/tarantool/tarantool/pipelines/63905818
Issue: https://github.com/tarantool/tarantool/issues/4156

 .gitlab-ci.yml                      | 554 ++++++++++++++++++++++++++++
 .travis.mk                          |  43 ++-
 .travis.yml                         | 133 -------
 images/Dockerfile.debian            |   4 +
 images/Dockerfile.lto               |   5 +
 images/Dockerfile.ubuntu_perf       |  59 +++
 images/Dockerfile.ubuntu_perf_build |  14 +
 perf/run-tarantool-server.sh        |  13 +
 perf/tarantool-server.lua           |  26 ++
 test/box/suite.ini                  |   2 +-
 10 files changed, 706 insertions(+), 147 deletions(-)
 create mode 100644 .gitlab-ci.yml
 delete mode 100644 .travis.yml
 create mode 100644 images/Dockerfile.debian
 create mode 100644 images/Dockerfile.lto
 create mode 100644 images/Dockerfile.ubuntu_perf
 create mode 100644 images/Dockerfile.ubuntu_perf_build
 create mode 100755 perf/run-tarantool-server.sh
 create mode 100644 perf/tarantool-server.lua

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
new file mode 100644
index 000000000..d37c4681e
--- /dev/null
+++ b/.gitlab-ci.yml
@@ -0,0 +1,554 @@
+stages:
+  - bootstrap
+  - test
+  - perf_deploy
+
+variables:
+  # Docker
+  DCKLGN: "docker login -u ${CI_REGISTRY_USER} -p ${CI_REGISTRY_PASSWORD} ${CI_REGISTRY}"
+  DCKBLD: "docker build --network=host"
+  DCKPSH: "docker push"
+  # Build
+  FLAG_LTO: "-DENABLE_LTO=ON"
+  GITCLN: "git submodule update --recursive --init --force"
+  TRAVMK: "make -f .travis.mk"
+  # Images
+  TVER: "master"
+  IMGUBB_BST: "${CI_REGISTRY}/${CI_PROJECT_PATH}:ubuntu-bionic_perf"
+  IMGUBB_BLT: "${CI_REGISTRY}/${CI_PROJECT_PATH}:ubuntu-bionic_perf_${TVER}"
+  IMGDBS_BST: "${CI_REGISTRY}/${CI_PROJECT_PATH}:debian-stretch_${TVER}"
+  IMGDBB_BST: "${CI_REGISTRY}/${CI_PROJECT_PATH}:debian-buster_${TVER}"
+  # OSX
+  OSX_SSH: "ssh -A tarantool@127.0.0.1 -p 2222"
+  OSX_SSHENV: "export PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin"
+  OSX_GITCHK: "git fetch -p && git checkout --force ${CI_BUILD_REF} && ${GITCLN}"
+  # Pack
+  PACKCLO: "git clone https://github.com/packpack/packpack.git packpack"
+  PACKMK: "make -f packpack/pack/Makefile -C . BUILDDIR=$PWD/build -j"
+
+# Bootstraps
+
+debian9:
+  stage: bootstrap
+  tags:
+    - bootstrap
+  before_script:
+    - ${DCKLGN}
+  script:
+    - cp .travis.mk images/
+    - ${DCKBLD} -t ${IMGDBS_BST} -f images/Dockerfile.debian images
+    - ${DCKPSH} ${IMGDBS_BST}
+
+debian10_lto:
+  only:
+    refs:
+      - master
+  stage: bootstrap
+  tags:
+    - bootstrap
+  before_script:
+    - ${DCKLGN}
+  script:
+    - cp .travis.mk images/
+    - ${DCKBLD} -t ${IMGDBB_BST} -f images/Dockerfile.lto images
+    - ${DCKPSH} ${IMGDBB_BST}
+
+# Tests
+
+release:
+  image: ${IMGDBS_BST}
+  stage: test
+  tags:
+    - docker_test
+  script:
+    - ${GITCLN}
+    - ${TRAVMK} runtest_ubuntu
+
+debug:
+  image: ${IMGDBS_BST}
+  stage: test
+  tags:
+    - docker_test
+  script:
+    - ${GITCLN}
+    - ${TRAVMK} runtest_cov_ubuntu
+
+release_clang:
+  image: ${IMGDBS_BST}
+  stage: test
+  tags:
+    - docker_test
+  variables:
+    CC: clang
+    CXX: clang++
+  script:
+    - ${GITCLN}
+    - ${TRAVMK} runtest_ubuntu
+
+release_lto:
+  only:
+    refs:
+      - master
+  image: ${IMGDBB_BST}
+  stage: test
+  tags:
+    - docker_test
+  variables:
+    CMAKE_EXTRA_PARAMS: ${FLAG_LTO}
+  script:
+    - ${GITCLN}
+    - ${TRAVMK} runtest_ubuntu
+
+release_lto_clang8:
+  only:
+    refs:
+      - master
+  image: ${IMGDBB_BST}
+  stage: test
+  tags:
+    - docker_test
+  variables:
+    CC: clang-8
+    CXX: clang++-8
+    CMAKE_EXTRA_PARAMS: ${FLAG_LTO}
+  script:
+    - ${GITCLN}
+    - ${TRAVMK} runtest_ubuntu
+
+osx13_release:
+  only:
+    refs:
+      - master
+  stage: test
+  tags:
+    - osx
+  before_script:
+    - VBoxManage controlvm Sierra poweroff || true
+    - VBoxManage snapshot Sierra restore Sierra
+    - VBoxManage startvm Sierra --type headless
+  script:
+    - export
+    - ${OSX_SSH} "/bin/bash -c '${OSX_SSHENV} ; cd tarantool && ${OSX_GITCHK} && ${TRAVMK} test_osx'"
+  after_script:
+    - VBoxManage controlvm Sierra poweroff
+
+osx14_release:
+  stage: test
+  tags:
+    - osx
+  before_script:
+    - VBoxManage controlvm Mojave poweroff || true
+    - VBoxManage snapshot Mojave restore Mojave
+    - VBoxManage startvm Mojave --type headless
+  script:
+    - export
+    - ${OSX_SSH} "/bin/bash -c '${OSX_SSHENV} ; cd tarantool && ${OSX_GITCHK} && ${TRAVMK} test_osx'"
+  after_script:
+    - VBoxManage controlvm Mojave poweroff
+
+osx14_release_lto:
+  only:
+    refs:
+      - master
+  stage: test
+  tags:
+    - osx
+  before_script:
+    - VBoxManage controlvm Mojave poweroff || true
+    - VBoxManage snapshot Mojave restore Mojave
+    - VBoxManage startvm Mojave --type headless
+  script:
+    - export
+    - ${OSX_SSH} "/bin/bash -c '${OSX_SSHENV} ; export CMAKE_EXTRA_PARAMS=${FLAG_LTO} ; cd tarantool && ${OSX_GITCHK} && ${TRAVMK} test_osx'"
+  after_script:
+    - VBoxManage controlvm Mojave poweroff
+
+# Perf prepare
+
+perf_bootstrap:
+  only:
+    refs:
+      - master
+      - /.*-perf/
+      - /.*-perflong/
+  stage: test
+  tags:
+    - perf
+  before_script:
+    - ${DCKLGN}
+  script:
+    # !!! TEMPORARY !!!
+    - cp -rfp /home/aleks.tikhonov/Workspaces/runners images/.
+    - ${DCKBLD} -t ${IMGUBB_BST} -f images/Dockerfile.ubuntu_perf images
+    - ${DCKPSH} ${IMGUBB_BST}
+    - ${DCKBLD} -t ${IMGUBB_BLT} -f images/Dockerfile.ubuntu_perf_build .
+    - ${DCKPSH} ${IMGUBB_BLT}
+
+# Perf sysbench 1 *perf/*perflong + 2 *perflong
+
+perf_sysbench:
+  only:
+    refs:
+      - master
+      - /.*-perf/
+      - /.*-perflong/
+  image: ${IMGUBB_BLT}
+  stage: perf_deploy
+  tags:
+    - docker_perf
+  script:
+    - perf/run-tarantool-server.sh
+    - THREADS=4 /opt/runners/sysbench-benchmarking/run.sh
+
+perf_sysbench_1:
+  only:
+    refs:
+      - /.*-perflong/
+  image: ${IMGUBB_BLT}
+  stage: perf_deploy
+  tags:
+    - docker_perf
+  script:
+    - perf/run-tarantool-server.sh
+    - THREADS=4 /opt/runners/sysbench-benchmarking/run.sh
+
+perf_sysbench_2:
+  only:
+    refs:
+      - /.*-perflong/
+  image: ${IMGUBB_BLT}
+  stage: perf_deploy
+  tags:
+    - docker_perf
+  script:
+    - perf/run-tarantool-server.sh
+    - THREADS=4 /opt/runners/sysbench-benchmarking/run.sh
+
+# Perf tpcc 1 *perf/*perflong + 2 *perflong
+
+perf_tpcc:
+  only:
+    refs:
+      - master
+      - /.*-perf/
+      - /.*-perflong/
+  image: ${IMGUBB_BLT}
+  stage: perf_deploy
+  tags:
+    - docker_perf
+  script:
+    - /opt/runners/tpcc-benchmarking/run.sh
+
+perf_tpcc_1:
+  only:
+    refs:
+      - /.*-perflong/
+  image: ${IMGUBB_BLT}
+  stage: perf_deploy
+  tags:
+    - docker_perf
+  script:
+    - /opt/runners/tpcc-benchmarking/run.sh
+
+perf_tpcc_2:
+  only:
+    refs:
+      - /.*-perflong/
+  image: ${IMGUBB_BLT}
+  stage: perf_deploy
+  tags:
+    - docker_perf
+  script:
+    - /opt/runners/tpcc-benchmarking/run.sh
+
+# Perf ycsb 1 *perf + 1*5 *perflong
+
+perf_ycsb:
+  only:
+    refs:
+      - master
+      - /.*-perf/
+  image: ${IMGUBB_BLT}
+  stage: perf_deploy
+  tags:
+    - docker_perf
+  script:
+    - /opt/runners/ycsb-benchmarking/run.sh
+
+perf_ycsb_1..5:
+  only:
+    refs:
+      - /.*-perflong/
+  image: ${IMGUBB_BLT}
+  stage: perf_deploy
+  tags:
+    - docker_perf
+  script:
+    - /opt/runners/ycsb-benchmarking/run.sh 5
+
+# Perf nosqlbench 1 *perf/*perflong + 4 *perflong
+
+perf_nosqlbench:
+  only:
+    refs:
+      - master
+      - /.*-perf/
+      - /.*-perflong/
+  image: ${IMGUBB_BLT}
+  stage: perf_deploy
+  tags:
+    - docker_perf
+  script:
+    - /opt/runners/nosqlbench-benchmarking/run.sh
+
+perf_nosqlbench_1:
+  only:
+    refs:
+      - /.*-perflong/
+  image: ${IMGUBB_BLT}
+  stage: perf_deploy
+  tags:
+    - docker_perf
+  script:
+    - /opt/runners/nosqlbench-benchmarking/run.sh
+
+perf_nosqlbench_2:
+  only:
+    refs:
+      - /.*-perflong/
+  image: ${IMGUBB_BLT}
+  stage: perf_deploy
+  tags:
+    - docker_perf
+  script:
+    - /opt/runners/nosqlbench-benchmarking/run.sh
+
+perf_nosqlbench_3:
+  only:
+    refs:
+      - /.*-perflong/
+  image: ${IMGUBB_BLT}
+  stage: perf_deploy
+  tags:
+    - docker_perf
+  script:
+    - /opt/runners/nosqlbench-benchmarking/run.sh
+
+perf_nosqlbench_4:
+  only:
+    refs:
+      - /.*-perflong/
+  image: ${IMGUBB_BLT}
+  stage: perf_deploy
+  tags:
+    - docker_perf
+  script:
+    - /opt/runners/nosqlbench-benchmarking/run.sh
+
+# Perf cbench 1 *perf/*perflong + 2 *perflong
+
+perf_cbench:
+  only:
+    refs:
+      - master
+      - /.*-perf/
+      - /.*-perflong/
+  image: ${IMGUBB_BLT}
+  stage: perf_deploy
+  tags:
+    - docker_perf
+  script:
+    - /opt/runners/cbench-benchmarking/run.sh
+
+perf_cbench_1:
+  only:
+    refs:
+      - /.*-perflong/
+  image: ${IMGUBB_BLT}
+  stage: perf_deploy
+  tags:
+    - docker_perf
+  script:
+    - /opt/runners/cbench-benchmarking/run.sh
+
+perf_cbench_2:
+  only:
+    refs:
+      - /.*-perflong/
+  image: ${IMGUBB_BLT}
+  stage: perf_deploy
+  tags:
+    - docker_perf
+  script:
+    - /opt/runners/cbench-benchmarking/run.sh
+
+# Packs
+
+pack_cent6:
+  only:
+    refs:
+      - master
+  image: packpack/packpack:el-6
+  stage: perf_deploy
+  tags:
+    - deploy
+  script:
+    - git submodule update --recursive --init
+    - ${PACKCLO}
+    - ${PACKMK}
+
+pack_cent7:
+  only:
+    refs:
+      - master
+  image: packpack/packpack:el-7
+  stage: perf_deploy
+  tags:
+    - deploy
+  script:
+    - ${GITCLN}
+    - ${PACKCLO}
+    - ${PACKMK}
+
+pack_fedora28:
+  only:
+    refs:
+      - master
+  image: packpack/packpack:fedora-28
+  stage: perf_deploy
+  tags:
+    - deploy
+  script:
+    - ${GITCLN}
+    - ${PACKCLO}
+    - ${PACKMK}
+
+pack_fedora29:
+  only:
+    refs:
+      - master
+  image: packpack/packpack:fedora-29
+  stage: perf_deploy
+  tags:
+    - deploy
+  script:
+    - ${GITCLN}
+    - ${PACKCLO}
+    - ${PACKMK}
+
+pack_fedora30:
+  only:
+    refs:
+      - master
+  image: packpack/packpack:fedora-30
+  stage: perf_deploy
+  tags:
+    - deploy
+  script:
+    - ${GITCLN}
+    - ${PACKCLO}
+    - ${PACKMK}
+
+pack_ubuntu14:
+  only:
+    refs:
+      - master
+  image: packpack/packpack:ubuntu-trusty
+  stage: perf_deploy
+  tags:
+    - deploy
+  script:
+    - ${GITCLN}
+    - ${PACKCLO}
+    - ${PACKMK}
+
+pack_ubuntu16:
+  only:
+    refs:
+      - master
+  image: packpack/packpack:ubuntu-xenial
+  stage: perf_deploy
+  tags:
+    - deploy
+  script:
+    - ${GITCLN}
+    - ${PACKCLO}
+    - ${PACKMK}
+
+pack_ubuntu18:
+  only:
+    refs:
+      - master
+  image: packpack/packpack:ubuntu-bionic
+  stage: perf_deploy
+  tags:
+    - deploy
+  script:
+    - ${GITCLN}
+    - ${PACKCLO}
+    - ${PACKMK}
+
+pack_ubuntu1810:
+  only:
+    refs:
+      - master
+  image: packpack/packpack:ubuntu-cosmic
+  stage: perf_deploy
+  tags:
+    - deploy
+  script:
+    - ${GITCLN}
+    - ${PACKCLO}
+    - ${PACKMK}
+
+pack_ubuntu19:
+  only:
+    refs:
+      - master
+  image: packpack/packpack:ubuntu-disco
+  stage: perf_deploy
+  tags:
+    - deploy
+  script:
+    - ${GITCLN}
+    - ${PACKCLO}
+    - ${PACKMK}
+
+pack_debian8:
+  only:
+    refs:
+      - master
+  image: packpack/packpack:debian-jessie
+  stage: perf_deploy
+  tags:
+    - deploy
+  script:
+    - ${GITCLN}
+    - ${PACKCLO}
+    - ${PACKMK}
+
+pack_debian9:
+  only:
+    refs:
+      - master
+  image: packpack/packpack:debian-stretch
+  stage: perf_deploy
+  tags:
+    - deploy
+  script:
+    - ${GITCLN}
+    - ${PACKCLO}
+    - ${PACKMK}
+
+pack_debian10:
+  only:
+    refs:
+      - master
+  image: packpack/packpack:debian-buster
+  stage: perf_deploy
+  tags:
+    - deploy
+  script:
+    - ${GITCLN}
+    - ${PACKCLO}
+    - ${PACKMK}
diff --git a/.travis.mk b/.travis.mk
index 55bee9980..92d3e7e65 100644
--- a/.travis.mk
+++ b/.travis.mk
@@ -34,24 +34,36 @@ docker_%:
 		make -f .travis.mk $(subst docker_,,$@)
 
 deps_ubuntu:
-	sudo apt-get update && apt-get install -y -f \
+	apt-get -y update && apt-get install -y -f \
 		build-essential cmake coreutils sed \
 		libreadline-dev libncurses5-dev libyaml-dev libssl-dev \
 		libcurl4-openssl-dev libunwind-dev libicu-dev \
 		python python-pip python-setuptools python-dev \
 		python-msgpack python-yaml python-argparse python-six python-gevent \
-		lcov ruby clang llvm llvm-dev
+		lcov ruby clang llvm llvm-dev mdm
 
-test_ubuntu: deps_ubuntu
+deps_buster_clang_8:
+	echo "deb http://apt.llvm.org/buster/ llvm-toolchain-buster-8 main" > /etc/apt/sources.list.d/clang_8.list
+	echo "deb-src http://apt.llvm.org/buster/ llvm-toolchain-buster-8 main" >> /etc/apt/sources.list.d/clang_8.list
+	wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
+	apt-get -y update
+	apt-get -y install clang-8 llvm-8 llvm-8-dev
+
+build_ubuntu:
 	cmake . -DCMAKE_BUILD_TYPE=RelWithDebInfo -DENABLE_WERROR=ON ${CMAKE_EXTRA_PARAMS}
-	make -j8
-	cd test && /usr/bin/python test-run.py --force -j 1
+	make -j
+
+runtest_ubuntu: build_ubuntu
+	cd test && /usr/bin/python test-run.py --force -j $$((`ncpus`*2)) --no-output-timeout -1
+
+test_ubuntu: deps_ubuntu runtest_ubuntu
 
 deps_osx:
 	brew update
-	brew install openssl readline curl icu4c --force
-	curl --silent --show-error --retry 5 https://bootstrap.pypa.io/get-pip.py | python
-	pip install -r test-run/requirements.txt
+	brew install openssl readline curl icu4c cmake --force || true
+	curl --silent --show-error --retry 5 https://bootstrap.pypa.io/get-pip.py >get-pip.py
+	python get-pip.py --user
+	pip install --user --ignore-installed -r test-run/requirements.txt
 
 test_osx: deps_osx
 	cmake . -DCMAKE_BUILD_TYPE=RelWithDebInfo -DENABLE_WERROR=ON ${CMAKE_EXTRA_PARAMS}
@@ -61,14 +73,17 @@ test_osx: deps_osx
 	sudo launchctl limit maxfiles 20480 || :
 	ulimit -S -n 20480 || :
 	ulimit -n
-	make -j8
-	cd test && python test-run.py --force -j 1 unit/ app/ app-tap/ box/ box-tap/
+	make -j
+	cd test && TRAVIS=True python test-run.py --force -j $$((`sysctl -n hw.ncpu`*2)) unit/ app/ app-tap/ box/ box-tap/
 
-coverage_ubuntu: deps_ubuntu
+build_cov_ubuntu:
 	cmake . -DCMAKE_BUILD_TYPE=Debug -DENABLE_GCOV=ON
-	make -j8
+	find -name "*.gcda" -exec rm -rf {} \;
+	make -j
+
+runtest_cov_ubuntu: build_cov_ubuntu
 	# Enable --long tests for coverage
-	cd test && /usr/bin/python test-run.py --force -j 1 --long
+	cd test && /usr/bin/python test-run.py --force -j $$((`ncpus`*2)) --long --no-output-timeout -1
 	lcov --compat-libtool --directory src/ --capture --output-file coverage.info.tmp
 	lcov --compat-libtool --remove coverage.info.tmp 'tests/*' 'third_party/*' '/usr/*' \
 		--output-file coverage.info
@@ -80,6 +95,8 @@ coverage_ubuntu: deps_ubuntu
 		coveralls-lcov --service-name travis-ci --service-job-id $(TRAVIS_JOB_ID) --repo-token $(COVERALLS_TOKEN) coverage.info; \
 	fi;
 
+coverage_ubuntu: deps_ubuntu runtest_cov_ubuntu
+
 source:
 	git clone https://github.com/packpack/packpack.git packpack
 	TARBALL_COMPRESSOR=gz packpack/packpack tarball
diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index e94d02ef5..000000000
--- a/.travis.yml
+++ /dev/null
@@ -1,133 +0,0 @@
-sudo: false
-services:
-  - docker
-
-language: cpp
-
-# default values
-os: linux
-compiler: gcc
-
-osx_image: xcode10.2
-
-cache:
-    directories:
-      - $HOME/.cache
-
-git:
-    depth: 100500
-
-jobs:
-    include:
-      # Testing targets (just run tests on Debian Stretch or OS X).
-      - name: "RelWithDebInfo build + test (Linux, gcc)"
-        env: TARGET=test
-      - name: "RelWithDebInfo build + test (Linux, clang)"
-        env: TARGET=test
-        compiler: clang
-      - name: "RelWithDebInfo build + test (OS X Mojave 10.14)"
-        env: TARGET=test
-        os: osx
-      - name: "Debug build + test + coverage (Linux, gcc)"
-        env: TARGET=coverage
-      - name: "RelWithDebInfo build + test (OS X High Sierra 10.13)"
-        env: TARGET=test
-        os: osx
-        osx_image: xcode9.4
-        if: branch = "master"
-      # Special targets (only LTO for now).
-      - name: "LTO build + test (Linux, gcc)"
-        env: >
-          TARGET=test
-          CMAKE_EXTRA_PARAMS=-DENABLE_LTO=ON
-          DOCKER_IMAGE=packpack/packpack:debian-buster
-        if: branch = "master"
-      - name: "LTO build + test (Linux, clang)"
-        env: >
-          TARGET=test
-          CMAKE_EXTRA_PARAMS=-DENABLE_LTO=ON
-          DOCKER_IMAGE=packpack/packpack:debian-buster
-        if: branch = "master"
-        compiler: clang
-      - name: "LTO build + test (OS X Mojave 10.14)"
-        os: osx
-        env: TARGET=test CMAKE_EXTRA_PARAMS=-DENABLE_LTO=ON
-        if: branch = "master"
-      # Deploy targets (they also catch distro-specific problems).
-      - name: "Create and deploy tarball"
-        env: TARGET=source
-        if: branch = "master"
-      - name: "CentOS 6 build + deploy RPM"
-        env: OS=el DIST=6
-        if: branch = "master"
-      - name: "CentOS 7 build + test + deploy RPM"
-        env: OS=el DIST=7
-        if: branch = "master"
-      - name: "Fedora 28 build + test + deploy RPM"
-        env: OS=fedora DIST=28
-        if: branch = "master"
-      - name: "Fedora 29 build + test + deploy RPM"
-        env: OS=fedora DIST=29
-        if: branch = "master"
-      - name: "Fedora 30 build + test + deploy RPM"
-        env: OS=fedora DIST=30
-        if: branch = "master"
-      - name: "Ubuntu Trusty (14.04) build + deploy DEB"
-        env: OS=ubuntu DIST=trusty
-        if: branch = "master"
-      - name: "Ubuntu Xenial (16.04) build + deploy DEB"
-        env: OS=ubuntu DIST=xenial
-        if: branch = "master"
-      - name: "Ubuntu Bionic (18.04) build + deploy DEB"
-        env: OS=ubuntu DIST=bionic
-        if: branch = "master"
-      - name: "Ubuntu Cosmic (18.10) build + deploy DEB"
-        env: OS=ubuntu DIST=cosmic
-        if: branch = "master"
-      - name: "Ubuntu Disco (19.04) build + deploy DEB"
-        env: OS=ubuntu DIST=disco
-        if: branch = "master"
-      - name: "Debian Jessie (8) build + deploy DEB"
-        env: OS=debian DIST=jessie
-        if: branch = "master"
-      - name: "Debian Stretch (9) build + deploy DEB"
-        env: OS=debian DIST=stretch
-        if: branch = "master"
-      - name: "Debian Buster (10) build + deploy DEB"
-        env: OS=debian DIST=buster
-        if: branch = "master"
-
-script:
-  - make -f .travis.mk ${TARGET}
-
-before_deploy:
-  - ls -l build/
-
-deploy:
-  # Deploy packages to PackageCloud
-  - provider: packagecloud
-    username: "tarantool"
-    repository: "2_2"
-    token: "${PACKAGECLOUD_TOKEN}"
-    dist: "${OS}/${DIST}"
-    package_glob: build/*.{rpm,deb,dsc}
-    skip_cleanup: true
-    on:
-      repo: tarantool/tarantool
-      branch: "master"
-      condition: -n "${OS}" && -n "${DIST}" && -n "${PACKAGECLOUD_TOKEN}"
-  # Deploy source tarballs to S3
-  - provider: script
-    script: make -f .travis.mk source_deploy
-    skip_cleanup: true
-    on:
-      repo: tarantool/tarantool
-      branch: "master"
-      condition: "x${TARGET} = xsource"
-
-notifications:
-  email:
-    recipients:
-      - build@tarantool.org
-    on_success: change
-    on_failure: always
diff --git a/images/Dockerfile.debian b/images/Dockerfile.debian
new file mode 100644
index 000000000..eb5278343
--- /dev/null
+++ b/images/Dockerfile.debian
@@ -0,0 +1,4 @@
+FROM packpack/packpack:debian-stretch
+
+COPY .travis.mk .
+RUN make -f .travis.mk deps_ubuntu
diff --git a/images/Dockerfile.lto b/images/Dockerfile.lto
new file mode 100644
index 000000000..54c585acc
--- /dev/null
+++ b/images/Dockerfile.lto
@@ -0,0 +1,5 @@
+FROM packpack/packpack:debian-buster
+
+COPY .travis.mk .
+RUN make -f .travis.mk deps_ubuntu
+RUN make -f .travis.mk deps_buster_clang_8
diff --git a/images/Dockerfile.ubuntu_perf b/images/Dockerfile.ubuntu_perf
new file mode 100644
index 000000000..0541746de
--- /dev/null
+++ b/images/Dockerfile.ubuntu_perf
@@ -0,0 +1,59 @@
+FROM ubuntu:18.04
+
+RUN apt-get -y update && apt-get install -y -f \
+    numactl gcc libc6-dev zlib1g-dev make libmysqlclient-dev \
+    ssh vim git dh-autoreconf pkg-config libicu-dev \
+    build-essential cmake coreutils sed libreadline-dev \
+    libncurses5-dev libyaml-dev libssl-dev libcurl4-openssl-dev \
+    libunwind-dev python python-pip python-setuptools python-dev \
+    python-msgpack python-yaml python-argparse python-six python-gevent \
+    gdb net-tools
+
+# benchmark-runners
+# !!! TEMPORARY !!!
+#RUN git clone https://gitlab.com/tarantool/tarantool-benchmarks-runner.git \
+#    /opt/runners && \
+COPY runners /opt/runners
+
+# tarantool-c
+RUN git clone --recursive https://github.com/tarantool/tarantool-c.git \
+    /opt/tarantool-c
+WORKDIR /opt/tarantool-c/third_party/msgpuck/
+RUN cmake . && make && make install
+WORKDIR /opt/tarantool-c
+RUN cmake . && make -j && make install
+
+# msgpack-c
+RUN git clone https://github.com/msgpack/msgpack-c.git /opt/msgpack-c
+WORKDIR /opt/msgpack-c
+RUN cmake . && make -j && make install
+
+# sysbench
+RUN git clone https://github.com/iproha94/sysbench.git /opt/sysbench
+WORKDIR /opt/sysbench
+RUN ./autogen.sh && ./configure --with-tarantool --without-mysql && \
+    make -j && make install && \
+    sed 's# CHAR(# VARCHAR(#g' -i /opt/sysbench/src/lua/oltp_common.lua && \
+    cp /opt/sysbench/src/lua/oltp_common.lua \
+        /usr/local/share/sysbench/oltp_common.lua
+
+# tpcc-tarantool
+RUN git clone https://github.com/tarantool/tpcc.git /opt/tpcc
+WORKDIR /opt/tpcc/src
+RUN make
+
+# YCSB
+RUN git clone https://github.com/brianfrankcooper/YCSB.git /opt/ycsb
+RUN apt-get update -y && apt-get install gtk-update-icon-cache=3.22.30-1ubuntu3 || true
+RUN apt-get install --fix-missing -y -f default-jdk maven
+WORKDIR /opt/ycsb
+RUN mvn -pl com.yahoo.ycsb:tarantool-binding -am clean package
+
+# nosqlbench
+RUN git clone --recursive https://github.com/tarantool/nosqlbench.git \
+        /opt/nosqlbench
+WORKDIR /opt/nosqlbench
+RUN git submodule foreach --recursive git pull origin master
+RUN apt-get install -y -f libev-dev
+RUN cmake . && make -j
+
diff --git a/images/Dockerfile.ubuntu_perf_build b/images/Dockerfile.ubuntu_perf_build
new file mode 100644
index 000000000..4f14b5469
--- /dev/null
+++ b/images/Dockerfile.ubuntu_perf_build
@@ -0,0 +1,14 @@
+FROM registry.gitlab.com/tarantool/tarantool:ubuntu-bionic_perf
+
+COPY . /opt/tarantool
+WORKDIR /opt/tarantool
+RUN git submodule update --recursive --init --force
+RUN cmake . -DENABLE_DIST=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo >cmake.log && make -j >build.log && make install >install.log
+
+# cbench
+RUN git clone https://github.com/tarantool/cbench.git /opt/cbench
+WORKDIR /opt/cbench
+RUN cmake . -DCMAKE_BUILD_TYPE=RelWithDebInfo && make && make install
+
+# reset the workdir to tarantool path
+WORKDIR /opt/tarantool
diff --git a/perf/run-tarantool-server.sh b/perf/run-tarantool-server.sh
new file mode 100755
index 000000000..f43c2427f
--- /dev/null
+++ b/perf/run-tarantool-server.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+numactl --show
+numactl --hardware
+numactl --cpunodebind=1 --physcpubind=6,7,8,9,10,11 tarantool perf/tarantool-server.lua 2>&1
+
+STATUS=
+while [ ${#STATUS} -eq "0" ]; do
+    STATUS="$(echo box.info.status | tarantoolctl connect /tmp/tarantool-server.sock | grep -e "- running")"
+    echo "waiting load snapshot to tarantool..."
+    sleep 5
+done
+
diff --git a/perf/tarantool-server.lua b/perf/tarantool-server.lua
new file mode 100644
index 000000000..a0de7d932
--- /dev/null
+++ b/perf/tarantool-server.lua
@@ -0,0 +1,26 @@
+local console = require('console')
+console.listen("/tmp/tarantool-server.sock")
+
+box.cfg {
+    pid_file   		= "./tarantool-server.pid",
+    log        		= "./tarantool-server.log",
+    listen 		= 3301,
+    memtx_memory 	= 2000000000,
+    background 		= true,
+    checkpoint_interval = 0,
+}
+
+
+function try(f, catch_f)
+    local status, exception = pcall(f)
+    if not status then
+        catch_f(exception)
+    end
+end
+
+try(function()
+    box.schema.user.grant('guest', 'create,read,write,execute', 'universe')
+end, function(e)
+    print(e)
+end)
+
diff --git a/test/box/suite.ini b/test/box/suite.ini
index c7b75c173..5efad688f 100644
--- a/test/box/suite.ini
+++ b/test/box/suite.ini
@@ -2,7 +2,7 @@
 core = tarantool
 description = Database tests
 script = box.lua
-disabled = rtree_errinj.test.lua tuple_bench.test.lua
+disabled = rtree_errinj.test.lua tuple_bench.test.lua on_shutdown.test.lua
 release_disabled = errinj.test.lua errinj_index.test.lua rtree_errinj.test.lua upsert_errinj.test.lua iproto_stress.test.lua
 lua_libs = lua/fifo.lua lua/utils.lua lua/bitset.lua lua/index_random_test.lua lua/push.lua lua/identifier.lua
 use_unix_sockets = True
-- 
2.17.1

^ permalink raw reply	[flat|nested] 2+ messages in thread

end of thread, other threads:[~2019-06-07 17:34 UTC | newest]

Thread overview: 2+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2019-05-30 10:14 [tarantool-patches] [PATCH v1] Gitlab-ci testing process implement Alexander V. Tikhonov
2019-06-07 17:33 ` [tarantool-patches] " Alexander Turenko

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox