Re: [Tarantool-patches] [PATCH v1] ci: enhance jobs with jepsen tests

["Alexander V. Tikhonov" <avtikhon@tarantool.org>]

Hi Sergey, thanks for the patch. I see you created the new CI template
and that is really good to have for different jobs. Patch LGTM, just
nit: we used 'gitlab-ci:' instead of 'ci:' prefix in commit message.

On Mon, Oct 26, 2020 at 12:00:07PM +0300, sergeyb@tarantool.org wrote:
> From: Sergey Bronnikov <sergeyb@tarantool.org>
> 
> To run Jepsen tests in different configurations we need to parametrize run
> script by options, so lein options and number of nodes passed with environment
> variables. By default script runs testing with Tarantool built from latest
> commit.
> 
> Added these configurations:
> 
> - single instance
> - single instance with enabled TXM
> - cluster with enabled Raft
> - cluster with enabled Raft and TXM
> 
> Closes #5437
> 
> ---
> 
> Branch: https://github.com/tarantool/tarantool/tree/ligurio/gh-5437-enhance-jepsen-jobs
> CI:  https://gitlab.com/tarantool/tarantool/-/pipelines/206846941
> Issue: https://github.com/tarantool/tarantool/issues/5437
> 
>  .gitlab-ci.yml            | 43 ++++++++++++++++++++++++++++-----------
>  .travis.mk                |  2 +-
>  CMakeLists.txt            | 18 ++++++----------
>  tools/run-jepsen-tests.sh | 12 ++++++-----
>  4 files changed, 45 insertions(+), 30 deletions(-)
> 
> diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
> index dc58888b6..dda51efbd 100644
> --- a/.gitlab-ci.yml
> +++ b/.gitlab-ci.yml
> @@ -169,6 +169,19 @@ before_script:
>    script:
>      - ${GITLAB_MAKE} perf_cleanup
>  
> +.jepsen_test_definition: &jepsen_test_definition
> +  <<: *docker_test_definition
> +  script:
> +    - ${GITLAB_MAKE} test_jepsen
> +  stage: long_tests
> +  when: manual
> +  tags:
> +    - mcs_jepsen_docker
> +  artifacts:
> +    paths:
> +      - jepsen-tests-prefix/src/jepsen-tests/store
> +    expire_in: 6 month
> +
>  # Static Analysis
>  
>  luacheck:
> @@ -264,18 +277,24 @@ freebsd_12_release:
>      - ${GITLAB_MAKE} vms_start
>      - ${GITLAB_MAKE} vms_test_freebsd
>  
> -jepsen:
> -  <<: *docker_test_definition
> -  script:
> -    - ${GITLAB_MAKE} test_jepsen
> -  stage: long_tests
> -  when: manual
> -  tags:
> -    - mcs_jepsen_docker
> -  artifacts:
> -    paths:
> -      - jepsen-tests-prefix/src/jepsen-tests/store
> -    expire_in: 6 month
> +jepsen-single-instance:
> +  <<: *jepsen_test_definition
> +
> +jepsen-single-instance-txm:
> +  <<: *jepsen_test_definition
> +  variables:
> +    LEIN_OPT: '--mvcc'
> +
> +jepsen-cluster:
> +  <<: *jepsen_test_definition
> +  variables:
> +    INSTANCE_COUNT: '5'
> +
> +jepsen-cluster-txm:
> +  <<: *jepsen_test_definition
> +  variables:
> +    LEIN_OPT: '--mvcc'
> +    INSTANCE_COUNT: '5'
>  
>  # ####
>  # Perf
> diff --git a/.travis.mk b/.travis.mk
> index a44e1b0c5..238e31cda 100644
> --- a/.travis.mk
> +++ b/.travis.mk
> @@ -326,4 +326,4 @@ test_freebsd: deps_freebsd test_freebsd_no_deps
>  
>  test_jepsen: deps_debian_jepsen
>  	cmake . -DCMAKE_BUILD_TYPE=RelWithDebInfo -DENABLE_WERROR=ON -DWITH_JEPSEN=ON
> -	make jepsen-single
> +	make run-jepsen
> diff --git a/CMakeLists.txt b/CMakeLists.txt
> index 512f50e05..fa6818f8e 100644
> --- a/CMakeLists.txt
> +++ b/CMakeLists.txt
> @@ -175,21 +175,15 @@ if (WITH_JEPSEN)
>      )
>  
>      #
> -    # Enable 'make jepsen-*' targets.
> +    # Enable 'make run-jepsen' target.
>      #
>  
> -    add_custom_target(jepsen-single DEPENDS jepsen-tests)
> -    add_custom_command(TARGET jepsen-single
> +    add_custom_target(run-jepsen DEPENDS jepsen-tests)
> +    add_custom_command(TARGET run-jepsen
>          COMMAND ${BASH} ${PROJECT_SOURCE_DIR}/tools/run-jepsen-tests.sh
> -				${PROJECT_SOURCE_DIR} ${PROJECT_BINARY_DIR}
> -        COMMENT "Running Jepsen tests on a single Tarantool instance"
> -    )
> -
> -    add_custom_target(jepsen-cluster DEPENDS jepsen-tests)
> -    add_custom_command(TARGET jepsen-cluster
> -        COMMAND ${BASH} ${PROJECT_SOURCE_DIR}/tools/run-jepsen-tests.sh
> -				${PROJECT_SOURCE_DIR} ${PROJECT_BINARY_DIR} 5
> -        COMMENT "Running Jepsen tests on a cluster with 5 Tarantool instances"
> +				${PROJECT_SOURCE_DIR}
> +                                ${PROJECT_BINARY_DIR}
> +        COMMENT "Running Jepsen tests"
>      )
>  endif()
>  
> diff --git a/tools/run-jepsen-tests.sh b/tools/run-jepsen-tests.sh
> index a313b2990..26931eaae 100755
> --- a/tools/run-jepsen-tests.sh
> +++ b/tools/run-jepsen-tests.sh
> @@ -9,14 +9,14 @@
>  # 	TF_VAR_user_domain_id
>  # 	TF_VAR_keypair_name
>  # 	TF_VAR_ssh_key
> -# and three arguments: path to a Tarantool project source directory, path to a
> -# Tarantool project binary directory and number on instances (optional).
> +# and two arguments: path to a Tarantool project source directory and path to a
> +# Tarantool project binary directory. And optional use environment variables:
> +# LEIN_OPT and INSTANCE_COUNT.
>  
>  set -Eeo pipefail
>  
>  PROJECT_SOURCE_DIR=$1
>  PROJECT_BINARY_DIR=$2
> -INSTANCE_COUNT=$3
>  
>  TESTS_DIR="$PROJECT_BINARY_DIR/jepsen-tests-prefix/src/jepsen-tests/"
>  TERRAFORM_CONFIG="$PROJECT_SOURCE_DIR/extra/tf"
> @@ -24,7 +24,9 @@ TERRAFORM_STATE="$PROJECT_BINARY_DIR/terraform.tfstate"
>  SSH_KEY_FILENAME="$PROJECT_SOURCE_DIR/build/tf-cloud-init"
>  NODES_FILENAME="$PROJECT_SOURCE_DIR/build/nodes"
>  
> -LEIN_OPTIONS="--nodes-file $NODES_FILENAME --username ubuntu --workload register"
> +CI_COMMIT_SHA=$(git rev-parse HEAD)
> +
> +LEIN_OPTIONS="test-all --nodes-file $NODES_FILENAME --username ubuntu $LEIN_OPT --version $CI_COMMIT_SHA"
>  
>  [[ -z ${PROJECT_SOURCE_DIR} ]] && (echo "Please specify path to a project source directory"; exit 1)
>  [[ -z ${PROJECT_BINARY_DIR} ]] && (echo "Please specify path to a project binary directory"; exit 1)
> @@ -72,6 +74,6 @@ terraform apply -state=$TERRAFORM_STATE -auto-approve $TERRAFORM_CONFIG
>  terraform providers $TERRAFORM_CONFIG
>  terraform output -state=$TERRAFORM_STATE instance_names
>  terraform output -state=$TERRAFORM_STATE -json instance_ips | jq --raw-output '.[]' > $NODES_FILENAME
> -pushd $TESTS_DIR && lein run test $LEIN_OPTIONS
> +pushd $TESTS_DIR && lein run $LEIN_OPTIONS
>  popd
>  cleanup
> -- 
> 2.28.0
>