From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from smtpng2.m.smailru.net (smtpng2.m.smailru.net [94.100.179.3]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by dev.tarantool.org (Postfix) with ESMTPS id 0081D46971A for ; Tue, 3 Dec 2019 13:34:09 +0300 (MSK) From: "Alexander V. Tikhonov" Date: Tue, 3 Dec 2019 13:34:08 +0300 Message-Id: Subject: [Tarantool-patches] [PATCH v1] gitlab-ci: add script for VBOX VMs orchestrate List-Id: Tarantool development patches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , To: Igor Munkin Cc: tarantool-patches@dev.tarantool.org Added 'tools/run_vm.sh' script for VBOX VMs orchestrate. It starts the VM on the local host at the previously prepared configurations. It uses the template in options: osx_13_*|osx_14_*|freebsd_12_* to be able to choose free VM from the list: osx_13_[1-X]|osx_14_*[1-X]|freebsd_12_[1-X] Gitlab-ci orchestrator chooses the host, where it's job can be run, so to make it always workable without delays in waiting the free VMs by the new local orchestrator script, the limit of the overall started jobs must be not bigger than requested template can find the appropriate VMs on the host. Each of the template can use up to X pre-configured VMs, so the gitlab-runner configuration file: /etc/gitlab-runner/config.toml has common configuration option: concurrent = X and each gitlab runner has limit option: limit = X X value should be manually found by testing at each used host. --- Github: https://github.com/tarantool/tarantool/tree/avtikhon/vms-orchestrate .gitlab-ci.yml | 30 ++------- .gitlab.mk | 15 +---- tools/run_vm.sh | 176 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 185 insertions(+), 36 deletions(-) create mode 100755 tools/run_vm.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cf13c382e..4542f8d75 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -42,10 +42,8 @@ variables: .vbox_template: &vbox_definition stage: test - before_script: - - ${GITLAB_MAKE} vms_start - after_script: - - ${GITLAB_MAKE} vms_shutdown + tags: + - vms_test # Tests @@ -93,47 +91,31 @@ release_asan_clang8: osx_13_release: <<: *release_only_definition <<: *vbox_definition - tags: - - vms_osx_13 variables: - VMS_NAME: 'osx_13' - VMS_USER: 'tarantool' - VMS_PORT: '2212' + VMS_NAME: 'osx_13_*' script: - ${GITLAB_MAKE} vms_test_osx osx_14_release: <<: *vbox_definition - tags: - - vms_osx_14 variables: - VMS_NAME: 'osx_14' - VMS_USER: 'tarantool' - VMS_PORT: '2222' + VMS_NAME: 'osx_14_*' script: - ${GITLAB_MAKE} vms_test_osx osx_14_release_lto: <<: *release_only_definition <<: *vbox_definition - tags: - - vms_osx_14 variables: EXTRA_ENV: "export CMAKE_EXTRA_PARAMS=-DENABLE_LTO=ON ;" - VMS_NAME: 'osx_14' - VMS_USER: 'tarantool' - VMS_PORT: '2222' + VMS_NAME: 'osx_14_*' script: - ${GITLAB_MAKE} vms_test_osx freebsd_12_release: <<: *vbox_definition - tags: - - vms_freebsd_12 variables: - VMS_NAME: 'freebsd_12' - VMS_USER: 'vagrant' - VMS_PORT: '2232' + VMS_NAME: 'freebsd_12_*' MAKE: 'gmake' script: - ${GITLAB_MAKE} vms_test_freebsd diff --git a/.gitlab.mk b/.gitlab.mk index 48a92e518..863d5fdf7 100644 --- a/.gitlab.mk +++ b/.gitlab.mk @@ -82,21 +82,12 @@ docker_bootstrap: # Run tests under a virtual machine # ################################# -vms_start: - VBoxManage controlvm ${VMS_NAME} poweroff || true - VBoxManage snapshot ${VMS_NAME} restore ${VMS_NAME} - VBoxManage startvm ${VMS_NAME} --type headless - vms_test_%: - tar czf - ../tarantool | ssh ${VMS_USER}@127.0.0.1 -p ${VMS_PORT} tar xzf - - ssh ${VMS_USER}@127.0.0.1 -p ${VMS_PORT} "/bin/bash -c \ - '${EXTRA_ENV} \ + ./tools/run_vm.sh -t=${VMS_NAME} -s -a=try_restore,start,login,stop \ + -c="${EXTRA_ENV} \ cd tarantool && \ ${GITLAB_MAKE} git_submodule_update && \ - ${TRAVIS_MAKE} $(subst vms_,,$@)'" - -vms_shutdown: - VBoxManage controlvm ${VMS_NAME} poweroff + ${TRAVIS_MAKE} $(subst vms_,,$@)" # ######################## # Build RPM / Deb packages diff --git a/tools/run_vm.sh b/tools/run_vm.sh new file mode 100755 index 000000000..abe409441 --- /dev/null +++ b/tools/run_vm.sh @@ -0,0 +1,176 @@ +#!/bin/bash + +set -e + +usage() { + echo "Usage: $0 -t= [-a=] [-c=]" +} + +NAME= +choose_vm() { + os_dist=$1 + all_vms=`VBoxManage list vms | awk -F '"' '{print $2}' | sort | grep "^${os_dist}_[1-9]*$" || true` + if [ "$all_vms" == "" ]; then + echo "ERROR: the VM for the given os=$os and dist=$dist by pattern '^${os_dist}_[1-9]*\$' is not available" + usage + exit 1 + fi + run_vms=`VBoxManage list runningvms | awk -F '"' '{print $2}' | sort | grep "^${os_dist}_[1-9]*$" || true` + for vm in $all_vms ; do + used=0 + for rvm in $run_vms ; do + if [ "$vm" == "$rvm" ]; then + used=1 + break + fi + done + + # let's setup the lock the file which will pause the + # next job till the current job will start the VM: + # lockfile will hold the current VM for 10 seconds + # to start up the VM in this delay, while next job + # will wait for 30 seconds for the lockfile released + # to recheck the VM, otherwise it will try to find + # the other free VM + lfile=/tmp/$vm + lockfile -1 -r 30 -l 10 $lfile + if [ "$used" == "0" ] && ! VBoxManage list runningvms | awk -F '"' '{print $2}' | grep "^${vm}$" ; then + NAME=$vm + echo "Found free VM: $NAME" + break + fi + done + if [ "$NAME" == "" ]; then + echo "WARNING: all of the appropriate VMs are in use!" + echo "VMs: $all_vms" + fi +} + +SCP_LOCAL_SOURCES=0 +for i in "$@" +do +case $i in + -s|--scp_local_sources) + SCP_LOCAL_SOURCES=1 + ;; + -t=*|--template=*) + TEMPLATE="${i#*=}" + shift # past argument=value + ;; + -a=*|--actions=*) + ACTIONS="${i#*=}" + shift # past argument=value + ;; + -c=*|--command=*) + COMMAND="${i#*=}" + shift # past argument=value + ;; + -h|--help) + usage + exit 0 + ;; + *) + usage + exit 1 + ;; +esac +done + +if [[ $TEMPLATE =~ .*_.*_\* ]]; then + while [ 1 ]; do + choose_vm $TEMPLATE + if [ "$NAME" != "" ]; then + break + fi + echo "INFO: wait a minute to recheck for the free VM ..." + sleep 60 + done +else + NAME=$TEMPLATE +fi + +if [ "$ACTIONS" == "" ]; then + ACTIONS=restore,start,login,stop + echo "WARNING: actions to start was not set: using default '$ACTIONS'" + echo +fi + +if [[ $ACTIONS =~ .*download.* ]]; then + echo "Images drives and configuration can be downloaded from:" + echo " https://mcs.mail.ru/app/services/storage/bucket/objects/packages/?Prefix=testing%2Fvms%2F" + + # download the files + for ftype in .ovf .mf -disk001.vmdk; do + file=${NAME}$ftype + wget -O - "https://download.tarantool.io/testing/vms/$file" >/tmp/$file + done +fi + +if [[ $ACTIONS =~ .*install.* ]]; then + echo "Unregister the old VM with the same name" + VBoxManage unregistervm $NAME --delete 2>/dev/null || true + + echo "Register the VM" + VBoxManage import /tmp/${NAME}.ovf +fi + +if [[ $ACTIONS =~ .*restore.* ]]; then + echo "Restore the virtual machine snapshot" + if ! VBoxManage snapshot $NAME restore $NAME ; then + if ! [[ $ACTIONS =~ .*try_restore.* ]]; then + exit 1 + fi + echo "WARNING: the restore action failed, but it was expected - continuing" + fi +fi + +if [[ $ACTIONS =~ .*start.* ]]; then + echo "Start the virtual machine without GUI" + VBoxManage startvm $NAME --type headless +fi + +exit_code=0 +if [[ $ACTIONS =~ .*login.* ]]; then + if [[ $NAME =~ freebsd_.* ]] ; then + VMUSER=vagrant + else + VMUSER=tarantool + fi + + host="${VMUSER}@127.0.0.1" + ssh="ssh $host -p "`VBoxManage showvminfo $NAME | grep "host port = " \ + | sed 's#.* host port = ##g' | sed 's#,.*##g'` + + echo "Try to ssh into the virtual machine" + i=0 + while [[ $i -lt 1000 ]] || exit 1 ; do + if $ssh "exit 0" ; then + break + fi + echo "The host is not ready, let's wait a little to recheck ..." + i=$(($i+1)) + sleep 10 + done + if [ "$SCP_LOCAL_SOURCES" == "1" ]; then + $ssh "cd ~ && rm -rf tarantool" + tar czf - ../tarantool | $ssh tar xzf - + fi + if [ "$COMMAND" == "" ]; then + $ssh || exit_code=$? + else + $ssh "$COMMAND" || exit_code=$? + fi + echo "The task exited with code: $exit_code" +fi + +if [[ $ACTIONS =~ .*snapshot.* ]]; then + echo "Take the virtual machine snapshot" + VBoxManage snapshot $NAME delete $NAME 2>/dev/null || true + VBoxManage snapshot $NAME take $NAME --pause +fi + +if [[ $ACTIONS =~ .*stop.* ]]; then + VBoxManage controlvm $NAME poweroff +fi + +exit $exit_code -- 2.17.1