[Tarantool-patches] [PATCH v1] gitlab-ci: add script for VBOX VMs orchestrate

["Alexander V. Tikhonov" <avtikhon@tarantool.org>]

Added 'tools/run_vm.sh' script for VBOX VMs orchestrate.
It starts the VM on the local host at the previously
prepared configurations. It uses the template in options:
  freebsd_12_*|<OS_DIST>_*
to be able to choose free VM from the list:
  freebsd_12_[1-X]|<OS_DIST>_[1-X]

Gitlab-ci orchestrator chooses the host, where it's job can
be run, so to make it always workable without delays in
waiting the free VMs by the new local orchestrator script,
the limit of the overall started jobs must be not bigger
than requested template can find the appropriate VMs on the
host. Each of the template can use up to X pre-configured
VMs, so the gitlab-runner configuration file:
  /etc/gitlab-runner/config.toml
has common configuration option:
  concurrent = X
and each gitlab runner has limit option:
  limit = X
X value should be manually found by testing at each used host.

Closes #5056
---

Github: https://github.com/tarantool/tarantool/tree/avtikhon/vms-orchestrate
Issue: https://github.com/tarantool/tarantool/issues/5056

 .gitlab-ci.yml  |  12 +---
 .gitlab.mk      |  15 +----
 tools/run_vm.sh | 176 ++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 182 insertions(+), 21 deletions(-)
 create mode 100755 tools/run_vm.sh

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 7705631dd..85877d304 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -84,10 +84,8 @@ variables:
 
 .vbox_template: &vbox_definition
   stage: test
-  before_script:
-    - ${GITLAB_MAKE} vms_start
-  after_script:
-    - ${GITLAB_MAKE} vms_shutdown
+  tags:
+    - vms_test
 
 .perf_docker_test_template: &perf_docker_test_definition
   <<: *perf_only_definition
@@ -177,12 +175,8 @@ osx_15_release_lto:
 
 freebsd_12_release:
   <<: *vbox_definition
-  tags:
-    - vms_freebsd_12
   variables:
-    VMS_NAME: 'freebsd_12'
-    VMS_USER: 'vagrant'
-    VMS_PORT: '2232'
+    VMS_NAME: 'freebsd_12_*'
     MAKE: 'gmake'
   script:
     - ${GITLAB_MAKE} vms_test_freebsd
diff --git a/.gitlab.mk b/.gitlab.mk
index f2a9b77fd..448d726ba 100644
--- a/.gitlab.mk
+++ b/.gitlab.mk
@@ -97,21 +97,12 @@ perf_cleanup: perf_clone_benchs_repo perf_cleanup_image
 # Run tests under a virtual machine
 # #################################
 
-vms_start:
-	VBoxManage controlvm ${VMS_NAME} poweroff || true
-	VBoxManage snapshot ${VMS_NAME} restore ${VMS_NAME}
-	VBoxManage startvm ${VMS_NAME} --type headless
-
 vms_test_%:
-	tar czf - ../tarantool | ssh ${VMS_USER}@127.0.0.1 -p ${VMS_PORT} tar xzf -
-	ssh ${VMS_USER}@127.0.0.1 -p ${VMS_PORT} "/bin/bash -c \
-		'${EXTRA_ENV} \
+	./tools/run_vm.sh -t=${VMS_NAME} -s -a=try_restore,start,login,stop \
+		-c="${EXTRA_ENV} \
 		cd tarantool && \
 		${GITLAB_MAKE} git_submodule_update && \
-		${TRAVIS_MAKE} $(subst vms_,,$@)'"
-
-vms_shutdown:
-	VBoxManage controlvm ${VMS_NAME} poweroff
+		${TRAVIS_MAKE} $(subst vms_,,$@)"
 
 # ######
 # Deploy
diff --git a/tools/run_vm.sh b/tools/run_vm.sh
new file mode 100755
index 000000000..1f8ae49b1
--- /dev/null
+++ b/tools/run_vm.sh
@@ -0,0 +1,176 @@
+#!/bin/bash
+
+set -e
+
+usage() {
+    echo "Usage: $0 -t=<freebsd_12[_*]|<OS_DIST>[_*]> [-a=<download,install,restore,start,login,snapshot,stop>] [-c=<command to run in VM>]"
+}
+
+NAME=
+choose_vm() {
+    os_dist=$1
+    all_vms=`VBoxManage list vms | awk -F '"' '{print $2}' | sort | grep "^${os_dist}_[1-9]*$" || true`
+    if [ "$all_vms" == "" ]; then
+        echo "ERROR: the VM for the given os=$os and dist=$dist by pattern '^${os_dist}_[1-9]*\$' is not available"
+        usage
+        exit 1
+    fi
+    run_vms=`VBoxManage list runningvms | awk -F '"' '{print $2}' | sort | grep "^${os_dist}_[1-9]*$" || true`
+    for vm in $all_vms ; do
+        used=0
+        for rvm in $run_vms ; do
+            if [ "$vm" == "$rvm" ]; then
+                used=1
+                break
+            fi
+        done
+
+        # let's setup the lock the file which will pause the
+        # next job till the current job will start the VM:
+        # lockfile will hold the current VM for 10 seconds
+        # to start up the VM in this delay, while next job
+        # will wait for 30 seconds for the lockfile released
+        # to recheck the VM, otherwise it will try to find
+        # the other free VM
+        lfile=/tmp/$vm
+        lockfile -1 -r 30 -l 10 $lfile
+        if [ "$used" == "0" ] && ! VBoxManage list runningvms | awk -F '"' '{print $2}' | grep "^${vm}$" ; then
+            NAME=$vm
+            echo "Found free VM: $NAME"
+            break
+        fi
+    done
+    if [ "$NAME" == "" ]; then
+        echo "WARNING: all of the appropriate VMs are in use!"
+        echo "VMs: $all_vms"
+    fi
+}
+
+SCP_LOCAL_SOURCES=0
+for i in "$@"
+do
+case $i in
+    -s|--scp_local_sources)
+    SCP_LOCAL_SOURCES=1
+    ;;
+    -t=*|--template=*)
+    TEMPLATE="${i#*=}"
+    shift # past argument=value
+    ;;
+    -a=*|--actions=*)
+    ACTIONS="${i#*=}"
+    shift # past argument=value
+    ;;
+    -c=*|--command=*)
+    COMMAND="${i#*=}"
+    shift # past argument=value
+    ;;
+    -h|--help)
+    usage
+    exit 0
+    ;;
+    *)
+    usage
+    exit 1
+    ;;
+esac
+done
+
+if [[ $TEMPLATE =~ .*_.*_\* ]]; then
+    while [ 1 ]; do
+        choose_vm $TEMPLATE
+        if [ "$NAME" != "" ]; then
+            break
+        fi
+        echo "INFO: wait a minute to recheck for the free VM ..."
+        sleep 60
+    done
+else
+    NAME=$TEMPLATE
+fi
+
+if [ "$ACTIONS" == "" ]; then
+    ACTIONS=restore,start,login,stop
+    echo "WARNING: actions to start was not set: using default '$ACTIONS'"
+    echo
+fi
+
+if [[ $ACTIONS =~ .*download.* ]]; then
+    echo "Images drives and configuration can be downloaded from:"
+    echo "    https://mcs.mail.ru/app/services/storage/bucket/objects/packages/?Prefix=testing%2Fvms%2F"
+
+    # download the files
+    for ftype in .ovf .mf -disk001.vmdk; do
+        file=${NAME}$ftype
+        wget -O - "https://download.tarantool.io/testing/vms/$file" >/tmp/$file
+    done
+fi
+
+if [[ $ACTIONS =~ .*install.* ]]; then
+    echo "Unregister the old VM with the same name"
+    VBoxManage unregistervm $NAME --delete 2>/dev/null || true
+
+    echo "Register the VM"
+    VBoxManage import /tmp/${NAME}.ovf
+fi
+
+if [[ $ACTIONS =~ .*restore.* ]]; then
+    echo "Restore the virtual machine snapshot"
+    if ! VBoxManage snapshot $NAME restore $NAME ; then
+        if ! [[ $ACTIONS =~ .*try_restore.* ]]; then
+            exit 1
+        fi
+        echo "WARNING: the restore action failed, but it was expected - continuing"
+    fi
+fi
+
+if [[ $ACTIONS =~ .*start.* ]]; then
+    echo "Start the virtual machine without GUI"
+    VBoxManage startvm $NAME --type headless
+fi
+
+exit_code=0
+if [[ $ACTIONS =~ .*login.* ]]; then
+    if [[ $NAME =~ freebsd_.* ]] ; then
+        VMUSER=vagrant
+    else
+        VMUSER=tarantool
+    fi
+
+    host="${VMUSER}@127.0.0.1"
+    ssh="ssh $host -p "`VBoxManage showvminfo $NAME | grep "host port = " \
+	    | sed 's#.* host port = ##g' | sed 's#,.*##g'`
+
+    echo "Try to ssh into the virtual machine"
+    i=0
+    while [[ $i -lt 1000 ]] || exit 1 ; do
+        if $ssh "exit 0" ; then
+            break
+        fi
+        echo "The host is not ready, let's wait a little to recheck ..."
+        i=$(($i+1))
+        sleep 10
+    done
+    if [ "$SCP_LOCAL_SOURCES" == "1" ]; then
+        $ssh "cd ~ && rm -rf tarantool"
+        tar czf - ../tarantool | $ssh tar xzf -
+    fi
+    if [ "$COMMAND" == "" ]; then
+        $ssh || exit_code=$?
+    else
+        $ssh "$COMMAND" || exit_code=$?
+    fi
+    echo "The task exited with code: $exit_code"
+fi
+
+if [[ $ACTIONS =~ .*snapshot.* ]]; then
+    echo "Take the virtual machine snapshot"
+    VBoxManage snapshot $NAME delete $NAME 2>/dev/null || true
+    VBoxManage snapshot $NAME take $NAME --pause
+fi
+
+if [[ $ACTIONS =~ .*stop.* ]]; then
+    VBoxManage controlvm $NAME poweroff
+fi
+
+exit $exit_code
-- 
2.17.1