[Tarantool-patches] [PATCH v1] gitlab-ci: set cleanup based on docker for all

[Tarantool-patches] [PATCH v1] gitlab-ci: set cleanup based on docker for all

[Alexander V. Tikhonov]

Set cleanup based on docker for all of the jobs to avoid of fails
when the job before change could use docker and reassigned the
temporary files in the working directory. If the docker is not
available than no need to use it for cleanup and cleanup runs
using shell. Gitlab-ci clean flags disabled and reorganized localy
as shown at [1].

Disabled gitlab-ci checkout default process as shown at [2] to be
able to fix repository before the checkout. Found that previously
run gitlab-ci jobs could change permissions files at repository
which broke the checkouts at the next jobs. Used for checkout
strategy [3], for submodule update strategy [4].

List of steps made localy instead of gitlab-ci preparations:

1. Check/clone the Tarantool repository with submodules.
2. For shell based jobs change ownership of all the sources
   to 'gitlab-runner' user. (NOTE: in Docker based jobs the
   'gitlab-runner' user is not known.)
3. Fetch Tarantool sources with branches and force checkout
   of the testing commit.
4. Update submodules recursively (use force where supports).
5. Cleanup all the sources from all files except from repository.

[1] https://docs.gitlab.com/ee/ci/yaml/README.html#git-clean-flags
[2] https://docs.gitlab.com/ee/ci/yaml/README.html#git-strategy
[3] https://docs.gitlab.com/ee/ci/yaml/README.html#git-checkout
[4] https://docs.gitlab.com/ee/ci/yaml/README.html#git-submodule-strategy

Follows up #5036
---

Github: https://github.com/tarantool/tarantool/tree/avtikhon/cleanup_gitlab-perf
Issue: https://github.com/tarantool/tarantool/issues/5036

 .gitlab-ci.yml | 53 +++++++++++++++++++++++++++++++++-----------------
 1 file changed, 35 insertions(+), 18 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ba88a26a9..ca8d7b252 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -4,18 +4,35 @@ stages:
   - perf
   - cleanup
 
+# 1. Git strategy disabled as shown at:
+#    https://docs.gitlab.com/ee/ci/yaml/README.html#git-strategy
+# 2. Git clean flags disabled as shown at:
+#    https://docs.gitlab.com/ee/ci/yaml/README.html#git-clean-flags
 variables:
   GITLAB_MAKE: "make -f .gitlab.mk"
-  GIT_CLEAN_COMMAND: "git clean -ffdx && git submodule foreach git clean -ffdx && git submodule foreach git status"
+  GIT_STRATEGY: none
   GIT_CLEAN_FLAGS: none
 
-.shell_before_script_template: &shell_cleanup_script
+# 1. Git checkout strategy used from:
+#    https://docs.gitlab.com/ee/ci/yaml/README.html#git-checkout
+# 2. Submodule strategy used from:
+#    https://docs.gitlab.com/ee/ci/yaml/README.html#git-submodule-strategy
+# List of steps:
+# 1) Check/clone the Tarantool repository with submodules.
+# 2) For shell based jobs change ownership of all the sources
+#    to gitlab-runner user (NOTE: in Docker based jobs the
+#    gitlab-runner user is not known).
+# 3) Fetch Tarantool sources with branches and force checkout
+#    of the testing commit.
+# 4) Update submodules recursively (use force where supports).
+# 5) Cleanup all the sources from all files except from repository.
+.docker_before_script_template: &job_ws_preparation_script
   before_script:
-    - /bin/bash -c "${GIT_CLEAN_COMMAND}"
-
-.docker_before_script_template: &docker_cleanup_script
-  before_script:
-    - docker run -w /source -v ${PWD}:/source -i packpack/packpack:el-7 /bin/bash -c "${GIT_CLEAN_COMMAND}"
+    - /bin/bash -c "pwd && ls -d .git || git clone --recurse-submodules ${CI_REPOSITORY_URL} ."
+    - /bin/bash -c "! docker -v || (uid=`id -u` ; gid=`id -g` ; docker run -w /source -v ${PWD}:/source -i packpack/packpack:el-7 /bin/bash -c \"chown -R \$uid:\$gid * .[^.]*\")"
+    - /bin/bash -c "git fetch -p && git checkout -f ${CI_COMMIT_SHORT_SHA}"
+    - /bin/bash -c "git submodule sync --recursive && git submodule update --force --init --recursive || git submodule update --recursive --init"
+    - /bin/bash -c "git clean -ffdx && git submodule foreach git clean -ffdx && git submodule foreach git status"
 
 # Jobs templates
 
@@ -46,21 +63,21 @@ variables:
   stage: test
   tags:
     - docker_test
-  <<: *shell_cleanup_script
+  <<: *job_ws_preparation_script
 
 .docker_test_clang8_template: &docker_test_clang8_definition
   image: "${CI_REGISTRY}/${CI_PROJECT_PATH}/testing/debian-buster:latest"
   stage: test
   tags:
     - docker_test
-  <<: *shell_cleanup_script
+  <<: *job_ws_preparation_script
 
 .pack_template: &pack_definition
   <<: *pack_only_definition
   stage: test
   tags:
     - deploy
-  <<: *docker_cleanup_script
+  <<: *job_ws_preparation_script
   script:
     - ${GITLAB_MAKE} package
 
@@ -69,7 +86,7 @@ variables:
   stage: test
   tags:
     - deploy_test
-  <<: *docker_cleanup_script
+  <<: *job_ws_preparation_script
   script:
     - ${GITLAB_MAKE} package
 
@@ -78,7 +95,7 @@ variables:
   stage: test
   tags:
     - deploy
-  <<: *docker_cleanup_script
+  <<: *job_ws_preparation_script
   script:
     - ${GITLAB_MAKE} deploy
 
@@ -87,13 +104,13 @@ variables:
   stage: test
   tags:
     - deploy_test
-  <<: *docker_cleanup_script
+  <<: *job_ws_preparation_script
   script:
     - ${GITLAB_MAKE} deploy
 
 .osx_template: &osx_definition
   stage: test
-  <<: *shell_cleanup_script
+  <<: *job_ws_preparation_script
   script:
     - ${GITLAB_MAKE} test_osx
 
@@ -114,14 +131,14 @@ variables:
     paths:
       - "*_result.txt"
       - "*_t_version.txt"
-  <<: *shell_cleanup_script
+  <<: *job_ws_preparation_script
   script:
     - ${GITLAB_MAKE} perf_run
 
 .perf_cleanup_definition: &perf_cleanup_definition
   <<: *perf_only_definition
   stage: cleanup
-  <<: *shell_cleanup_script
+  <<: *job_ws_preparation_script
   script:
     - ${GITLAB_MAKE} perf_cleanup
 
@@ -216,7 +233,7 @@ perf_bootstrap:
   stage: test
   tags:
     - deploy
-  <<: *shell_cleanup_script
+  <<: *job_ws_preparation_script
   script:
     - ${GITLAB_MAKE} perf_prepare
   after_script:
@@ -534,6 +551,6 @@ static_docker_build:
   stage: test
   tags:
     - deploy_test
-  <<: *docker_cleanup_script
+  <<: *job_ws_preparation_script
   script:
     - ${GITLAB_MAKE} test_static_docker_build
-- 
2.17.1

[Tarantool-patches] [PATCH v1] gitlab-ci: set cleanup based on docker for all

[Alexander V. Tikhonov]

Set cleanup based on docker for all of the jobs to avoid of fails
when the job before change could use docker and reassigned the
temporary files in the working directory. If the docker is not
available than no need to use it for cleanup and cleanup runs
using shell.

Follows up #5036
---

Github: https://github.com/tarantool/tarantool/tree/avtikhon/cleanup_gitlab-perf
Issue: https://github.com/tarantool/tarantool/issues/5036

 .gitlab-ci.yml | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ba88a26a9..1e6904eb6 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -9,13 +9,9 @@ variables:
   GIT_CLEAN_COMMAND: "git clean -ffdx && git submodule foreach git clean -ffdx && git submodule foreach git status"
   GIT_CLEAN_FLAGS: none
 
-.shell_before_script_template: &shell_cleanup_script
-  before_script:
-    - /bin/bash -c "${GIT_CLEAN_COMMAND}"
-
 .docker_before_script_template: &docker_cleanup_script
   before_script:
-    - docker run -w /source -v ${PWD}:/source -i packpack/packpack:el-7 /bin/bash -c "${GIT_CLEAN_COMMAND}"
+    - /bin/bash -c "docker run -w /source -v ${PWD}:/source -i packpack/packpack:el-7 /bin/bash -c \"${GIT_CLEAN_COMMAND}\" || ( ${GIT_CLEAN_COMMAND} )"
 
 # Jobs templates
 
@@ -46,14 +42,14 @@ variables:
   stage: test
   tags:
     - docker_test
-  <<: *shell_cleanup_script
+  <<: *docker_cleanup_script
 
 .docker_test_clang8_template: &docker_test_clang8_definition
   image: "${CI_REGISTRY}/${CI_PROJECT_PATH}/testing/debian-buster:latest"
   stage: test
   tags:
     - docker_test
-  <<: *shell_cleanup_script
+  <<: *docker_cleanup_script
 
 .pack_template: &pack_definition
   <<: *pack_only_definition
@@ -93,7 +89,7 @@ variables:
 
 .osx_template: &osx_definition
   stage: test
-  <<: *shell_cleanup_script
+  <<: *docker_cleanup_script
   script:
     - ${GITLAB_MAKE} test_osx
 
@@ -114,14 +110,14 @@ variables:
     paths:
       - "*_result.txt"
       - "*_t_version.txt"
-  <<: *shell_cleanup_script
+  <<: *docker_cleanup_script
   script:
     - ${GITLAB_MAKE} perf_run
 
 .perf_cleanup_definition: &perf_cleanup_definition
   <<: *perf_only_definition
   stage: cleanup
-  <<: *shell_cleanup_script
+  <<: *docker_cleanup_script
   script:
     - ${GITLAB_MAKE} perf_cleanup
 
@@ -216,7 +212,7 @@ perf_bootstrap:
   stage: test
   tags:
     - deploy
-  <<: *shell_cleanup_script
+  <<: *docker_cleanup_script
   script:
     - ${GITLAB_MAKE} perf_prepare
   after_script:
-- 
2.17.1