Re: [Tarantool-patches] [PATCH v1] gitlab-ci: add script for VBOX VMs orchestrate

["Alexander Tikhonov" <avtikhon@tarantool.org>]

[-- Attachment #1: Type: text/plain, Size: 9249 bytes --]



Oleg,

Please review the patch.

>Вторник,  3 декабря 2019, 13:34 +03:00 от Alexander V. Tikhonov <avtikhon@tarantool.org>:
>
>Added 'tools/run_vm.sh' script for VBOX VMs orchestrate.
>It starts the VM on the local host at the previously
>prepared configurations. It uses the template in options:
>  osx_13_*|osx_14_*|freebsd_12_*
>to be able to choose free VM from the list:
>  osx_13_[1-X]|osx_14_*[1-X]|freebsd_12_[1-X]
>
>Gitlab-ci orchestrator chooses the host, where it's job can
>be run, so to make it always workable without delays in
>waiting the free VMs by the new local orchestrator script,
>the limit of the overall started jobs must be not bigger
>than requested template can find the appropriate VMs on the
>host. Each of the template can use up to X pre-configured
>VMs, so the gitlab-runner configuration file:
>  /etc/gitlab-runner/config.toml
>has common configuration option:
>  concurrent = X
>and each gitlab runner has limit option:
>  limit = X
>X value should be manually found by testing at each used host.
>---
>
>Github:  https://github.com/tarantool/tarantool/tree/avtikhon/vms-orchestrate
>
> .gitlab-ci.yml  |  30 ++-------
> .gitlab.mk      |  15 +----
> tools/run_vm.sh | 176 ++++++++++++++++++++++++++++++++++++++++++++++++
> 3 files changed, 185 insertions(+), 36 deletions(-)
> create mode 100755 tools/run_vm.sh
>
>diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
>index cf13c382e..4542f8d75 100644
>--- a/.gitlab-ci.yml
>+++ b/.gitlab-ci.yml
>@@ -42,10 +42,8 @@ variables:
> 
> .vbox_template: &vbox_definition
>   stage: test
>-  before_script:
>-    - ${GITLAB_MAKE} vms_start
>-  after_script:
>-    - ${GITLAB_MAKE} vms_shutdown
>+  tags:
>+    - vms_test
> 
> # Tests
> 
>@@ -93,47 +91,31 @@ release_asan_clang8:
> osx_13_release:
>   <<: *release_only_definition
>   <<: *vbox_definition
>-  tags:
>-    - vms_osx_13
>   variables:
>-    VMS_NAME: 'osx_13'
>-    VMS_USER: 'tarantool'
>-    VMS_PORT: '2212'
>+    VMS_NAME: 'osx_13_*'
>   script:
>     - ${GITLAB_MAKE} vms_test_osx
> 
> osx_14_release:
>   <<: *vbox_definition
>-  tags:
>-    - vms_osx_14
>   variables:
>-    VMS_NAME: 'osx_14'
>-    VMS_USER: 'tarantool'
>-    VMS_PORT: '2222'
>+    VMS_NAME: 'osx_14_*'
>   script:
>     - ${GITLAB_MAKE} vms_test_osx
> 
> osx_14_release_lto:
>   <<: *release_only_definition
>   <<: *vbox_definition
>-  tags:
>-    - vms_osx_14
>   variables:
>     EXTRA_ENV: "export CMAKE_EXTRA_PARAMS=-DENABLE_LTO=ON ;"
>-    VMS_NAME: 'osx_14'
>-    VMS_USER: 'tarantool'
>-    VMS_PORT: '2222'
>+    VMS_NAME: 'osx_14_*'
>   script:
>     - ${GITLAB_MAKE} vms_test_osx
> 
> freebsd_12_release:
>   <<: *vbox_definition
>-  tags:
>-    - vms_freebsd_12
>   variables:
>-    VMS_NAME: 'freebsd_12'
>-    VMS_USER: 'vagrant'
>-    VMS_PORT: '2232'
>+    VMS_NAME: 'freebsd_12_*'
>     MAKE: 'gmake'
>   script:
>     - ${GITLAB_MAKE} vms_test_freebsd
>diff --git a/.gitlab.mk b/.gitlab.mk
>index 48a92e518..863d5fdf7 100644
>--- a/.gitlab.mk
>+++ b/.gitlab.mk
>@@ -82,21 +82,12 @@ docker_bootstrap:
> # Run tests under a virtual machine
> # #################################
> 
>-vms_start:
>-	VBoxManage controlvm ${VMS_NAME} poweroff || true
>-	VBoxManage snapshot ${VMS_NAME} restore ${VMS_NAME}
>-	VBoxManage startvm ${VMS_NAME} --type headless
>-
> vms_test_%:
>-	tar czf - ../tarantool | ssh ${VMS_USER}@127.0.0.1 -p ${VMS_PORT} tar xzf -
>-	ssh ${VMS_USER}@127.0.0.1 -p ${VMS_PORT} "/bin/bash -c \
>-		'${EXTRA_ENV} \
>+	./tools/run_vm.sh -t=${VMS_NAME} -s -a=try_restore,start,login,stop \
>+		-c="${EXTRA_ENV} \
> 		cd tarantool && \
> 		${GITLAB_MAKE} git_submodule_update && \
>-		${TRAVIS_MAKE} $(subst vms_,,$@)'"
>-
>-vms_shutdown:
>-	VBoxManage controlvm ${VMS_NAME} poweroff
>+		${TRAVIS_MAKE} $(subst vms_,,$@)"
> 
> # ########################
> # Build RPM / Deb packages
>diff --git a/tools/run_vm.sh b/tools/run_vm.sh
>new file mode 100755
>index 000000000..abe409441
>--- /dev/null
>+++ b/tools/run_vm.sh
>@@ -0,0 +1,176 @@
>+#!/bin/bash
>+
>+set -e
>+
>+usage() {
>+    echo "Usage: $0 -t=<freebsd_12[_*]|osx_1[34][_*]> [-a=<download,install,restore,start,login,snapshot,stop>] [-c=<command to run in VM>]"
>+}
>+
>+NAME=
>+choose_vm() {
>+    os_dist=$1
>+    all_vms=`VBoxManage list vms | awk -F '"' '{print $2}' | sort | grep "^${os_dist}_[1-9]*$" || true`
>+    if [ "$all_vms" == "" ]; then
>+        echo "ERROR: the VM for the given os=$os and dist=$dist by pattern '^${os_dist}_[1-9]*\$' is not available"
>+        usage
>+        exit 1
>+    fi
>+    run_vms=`VBoxManage list runningvms | awk -F '"' '{print $2}' | sort | grep "^${os_dist}_[1-9]*$" || true`
>+    for vm in $all_vms ; do
>+        used=0
>+        for rvm in $run_vms ; do
>+            if [ "$vm" == "$rvm" ]; then
>+                used=1
>+                break
>+            fi
>+        done
>+
>+        # let's setup the lock the file which will pause the
>+        # next job till the current job will start the VM:
>+        # lockfile will hold the current VM for 10 seconds
>+        # to start up the VM in this delay, while next job
>+        # will wait for 30 seconds for the lockfile released
>+        # to recheck the VM, otherwise it will try to find
>+        # the other free VM
>+        lfile=/tmp/$vm
>+        lockfile -1 -r 30 -l 10 $lfile
>+        if [ "$used" == "0" ] && ! VBoxManage list runningvms | awk -F '"' '{print $2}' | grep "^${vm}$" ; then
>+            NAME=$vm
>+            echo "Found free VM: $NAME"
>+            break
>+        fi
>+    done
>+    if [ "$NAME" == "" ]; then
>+        echo "WARNING: all of the appropriate VMs are in use!"
>+        echo "VMs: $all_vms"
>+    fi
>+}
>+
>+SCP_LOCAL_SOURCES=0
>+for i in "$@"
>+do
>+case $i in
>+    -s|--scp_local_sources)
>+    SCP_LOCAL_SOURCES=1
>+    ;;
>+    -t=*|--template=*)
>+    TEMPLATE="${i#*=}"
>+    shift # past argument=value
>+    ;;
>+    -a=*|--actions=*)
>+    ACTIONS="${i#*=}"
>+    shift # past argument=value
>+    ;;
>+    -c=*|--command=*)
>+    COMMAND="${i#*=}"
>+    shift # past argument=value
>+    ;;
>+    -h|--help)
>+    usage
>+    exit 0
>+    ;;
>+    *)
>+    usage
>+    exit 1
>+    ;;
>+esac
>+done
>+
>+if [[ $TEMPLATE =~ .*_.*_\* ]]; then
>+    while [ 1 ]; do
>+        choose_vm $TEMPLATE
>+        if [ "$NAME" != "" ]; then
>+            break
>+        fi
>+        echo "INFO: wait a minute to recheck for the free VM ..."
>+        sleep 60
>+    done
>+else
>+    NAME=$TEMPLATE
>+fi
>+
>+if [ "$ACTIONS" == "" ]; then
>+    ACTIONS=restore,start,login,stop
>+    echo "WARNING: actions to start was not set: using default '$ACTIONS'"
>+    echo
>+fi
>+
>+if [[ $ACTIONS =~ .*download.* ]]; then
>+    echo "Images drives and configuration can be downloaded from:"
>+    echo "  https://mcs.mail.ru/app/services/storage/bucket/objects/packages/?Prefix=testing%2Fvms%2F "
>+
>+    # download the files
>+    for ftype in .ovf .mf -disk001.vmdk; do
>+        file=${NAME}$ftype
>+        wget -O - " https://download.tarantool.io/testing/vms/$file " >/tmp/$file
>+    done
>+fi
>+
>+if [[ $ACTIONS =~ .*install.* ]]; then
>+    echo "Unregister the old VM with the same name"
>+    VBoxManage unregistervm $NAME --delete 2>/dev/null || true
>+
>+    echo "Register the VM"
>+    VBoxManage import /tmp/${NAME}.ovf
>+fi
>+
>+if [[ $ACTIONS =~ .*restore.* ]]; then
>+    echo "Restore the virtual machine snapshot"
>+    if ! VBoxManage snapshot $NAME restore $NAME ; then
>+        if ! [[ $ACTIONS =~ .*try_restore.* ]]; then
>+            exit 1
>+        fi
>+        echo "WARNING: the restore action failed, but it was expected - continuing"
>+    fi
>+fi
>+
>+if [[ $ACTIONS =~ .*start.* ]]; then
>+    echo "Start the virtual machine without GUI"
>+    VBoxManage startvm $NAME --type headless
>+fi
>+
>+exit_code=0
>+if [[ $ACTIONS =~ .*login.* ]]; then
>+    if [[ $NAME =~ freebsd_.* ]] ; then
>+        VMUSER=vagrant
>+    else
>+        VMUSER=tarantool
>+    fi
>+
>+    host="${VMUSER}@127.0.0.1"
>+    ssh="ssh $host -p "`VBoxManage showvminfo $NAME | grep "host port = " \
>+	    | sed 's#.* host port = ##g' | sed 's#,.*##g'`
>+
>+    echo "Try to ssh into the virtual machine"
>+    i=0
>+    while [[ $i -lt 1000 ]] || exit 1 ; do
>+        if $ssh "exit 0" ; then
>+            break
>+        fi
>+        echo "The host is not ready, let's wait a little to recheck ..."
>+        i=$(($i+1))
>+        sleep 10
>+    done
>+    if [ "$SCP_LOCAL_SOURCES" == "1" ]; then
>+        $ssh "cd ~ && rm -rf tarantool"
>+        tar czf - ../tarantool | $ssh tar xzf -
>+    fi
>+    if [ "$COMMAND" == "" ]; then
>+        $ssh || exit_code=$?
>+    else
>+        $ssh "$COMMAND" || exit_code=$?
>+    fi
>+    echo "The task exited with code: $exit_code"
>+fi
>+
>+if [[ $ACTIONS =~ .*snapshot.* ]]; then
>+    echo "Take the virtual machine snapshot"
>+    VBoxManage snapshot $NAME delete $NAME 2>/dev/null || true
>+    VBoxManage snapshot $NAME take $NAME --pause
>+fi
>+
>+if [[ $ACTIONS =~ .*stop.* ]]; then
>+    VBoxManage controlvm $NAME poweroff
>+fi
>+
>+exit $exit_code
>-- 
>2.17.1
>


-- 
Alexander Tikhonov

[-- Attachment #2: Type: text/html, Size: 11419 bytes --]