[Tarantool-patches] [PATCH luajit v3 2/3] ci: enabled sanitizer tests for macOS

[Evgeniy Temirgaleev via Tarantool-patches <tarantool-patches@dev.tarantool.org>]

From: Temir Galeev <temir.galeev@bk.ru>

The arm64 and x86_64 architectures with clang/gcc compiler were added
to the matrix.
---
 .../README.md                                 |   0
 .../action.yml                                |   8 +-
 .../actions/setup-sanitizers-macos/README.md  |  15 +++
 .../actions/setup-sanitizers-macos/action.yml |  73 +++++++++++++
 .github/workflows/sanitizers-testing.yml      | 101 ++++++++++++++++--
 5 files changed, 186 insertions(+), 11 deletions(-)
 rename .github/actions/{setup-sanitizers => setup-sanitizers-linux}/README.md (100%)
 rename .github/actions/{setup-sanitizers => setup-sanitizers-linux}/action.yml (76%)
 create mode 100644 .github/actions/setup-sanitizers-macos/README.md
 create mode 100644 .github/actions/setup-sanitizers-macos/action.yml

diff --git a/.github/actions/setup-sanitizers/README.md b/.github/actions/setup-sanitizers-linux/README.md
similarity index 100%
rename from .github/actions/setup-sanitizers/README.md
rename to .github/actions/setup-sanitizers-linux/README.md
diff --git a/.github/actions/setup-sanitizers/action.yml b/.github/actions/setup-sanitizers-linux/action.yml
similarity index 76%
rename from .github/actions/setup-sanitizers/action.yml
rename to .github/actions/setup-sanitizers-linux/action.yml
index 8642d553..18f5a75d 100644
--- a/.github/actions/setup-sanitizers/action.yml
+++ b/.github/actions/setup-sanitizers-linux/action.yml
@@ -20,13 +20,17 @@ runs:
     - name: Install build and test dependencies
       run: |
         apt -y update
+        echo Available compilers:
+        export CC_FAMILY=`echo ${CC_NAME} | sed 's/-.*$//'`
+        apt list | grep -Pe "^${CC_FAMILY}-[0-9]+/"
+        # Try to install
         apt -y install ${CC_NAME} libstdc++-10-dev cmake ninja-build make perl
       shell: bash
       env:
         CC_NAME: ${{ inputs.cc_name }}
-    - name: Set specific C compiler as a default toolchain
+    - name: Set specific C compiler as a default toolchain for cmake
       run: |
-        echo CC=${CC_NAME} | tee -a $GITHUB_ENV
+        echo CMAKE_C_COMPILER=${CC_NAME} | tee -a $GITHUB_ENV
       shell: bash
       env:
         CC_NAME: ${{ inputs.cc_name }}
diff --git a/.github/actions/setup-sanitizers-macos/README.md b/.github/actions/setup-sanitizers-macos/README.md
new file mode 100644
index 00000000..9ac5eb38
--- /dev/null
+++ b/.github/actions/setup-sanitizers-macos/README.md
@@ -0,0 +1,15 @@
+# Setup environment for sanitizers on macOS
+
+Action setups the environment on macOS runners (install requirements, setup the
+workflow environment, etc) for testing with sanitizers enabled.
+
+Requires input:
+- cc_name as versioned C compiler: gcc-ver or clang-ver.
+
+## How to use Github Action from Github workflow
+
+Add the following code to the running steps before LuaJIT configuration:
+```
+- uses: ./.github/actions/setup-sanitizers-macos
+  if: ${{ matrix.OS == 'macOS' }}
+```
diff --git a/.github/actions/setup-sanitizers-macos/action.yml b/.github/actions/setup-sanitizers-macos/action.yml
new file mode 100644
index 00000000..9836ea03
--- /dev/null
+++ b/.github/actions/setup-sanitizers-macos/action.yml
@@ -0,0 +1,73 @@
+name: Setup CI environment for testing with sanitizers on macOS
+description: Common part to tweak macOS CI runner environment for sanitizers
+inputs:
+  cc_name:
+    description: C compiler name (for example, gcc-12)
+    required: false
+    default: clang-21
+runs:
+  using: composite
+  steps:
+    - name: Get compiler version from cc_name
+      shell: bash
+      env:
+        CC_NAME: ${{ inputs.cc_name }}
+      run: |
+        echo CC_VERSION=`echo ${CC_NAME} | sed 's/.*-//'` | tee -a $GITHUB_ENV
+    - name: Get brew formula from cc_name
+      shell: bash
+      env:
+        CC_FORMULA_NAME: |-
+          ${{ case(
+            startsWith(inputs.cc_name, 'gcc'), 'gcc',
+            startsWith(inputs.cc_name, 'clang'), 'llvm',
+            'MISCONFIG'
+          ) }}
+      run: |
+        echo CC_FORMULA=${CC_FORMULA_NAME}@${CC_VERSION} | tee -a $GITHUB_ENV
+        echo Available formulas: `brew search ${CC_FORMULA_NAME}`
+    - name: Setup CI environment
+      uses: ./.github/actions/setup
+    - name: Set CMAKE_BUILD_PARALLEL_LEVEL
+      shell: bash
+      run: |
+        # Set CMAKE_BUILD_PARALLEL_LEVEL environment variable to
+        # limit the number of parallel jobs for build/test step.
+        NPROC=$(sysctl -n hw.logicalcpu 2>/dev/null)
+        echo CMAKE_BUILD_PARALLEL_LEVEL=$(($NPROC + 1)) | tee -a $GITHUB_ENV
+    - name: Set MACOSX_DEPLOYMENT_TARGERT
+      shell: bash
+      run: |
+        # Set required MACOSX_DEPLOYMENT_TARGERT environment
+        # variable for Makefile.original build.
+        # See https://github.com/LuaJIT/LuaJIT/issues/484,
+        # https://github.com/LuaJIT/LuaJIT/issues/653.
+        echo MACOSX_DEPLOYMENT_TARGET=$(sw_vers -productVersion) | tee -a $GITHUB_ENV
+    - name: Install build and test dependencies
+      shell: bash
+      run: |
+        # Install brew using the command from Homebrew repository
+        # instructions: https://github.com/Homebrew/install.
+        # XXX: 'echo' command below is required since brew
+        # installation script obliges the one to enter a newline
+        # for confirming the installation via Ruby script.
+        brew update ||
+          echo | /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
+        # Try to install the packages either upgrade it to avoid
+        # of fails if the package already exists with the previous
+        # version.
+        brew install --force ${CC_FORMULA} cmake make ninja perl ||
+          brew upgrade ${CC_FORMULA} cmake make ninja perl
+    - name: Set specific C compiler as a default toolchain for cmake
+      shell: bash
+      env:
+        CC_NAME: ${{ inputs.cc_name }}
+      run: |
+        echo CMAKE_C_COMPILER=${CC_NAME} | tee -a $GITHUB_ENV
+        echo CMAKE_PREFIX_PATH="`brew --prefix ${CC_FORMULA}`" | tee -a $GITHUB_ENV
+    - name: Log installed compilers
+      shell: bash
+      run: |
+        echo default clang: `clang --version`
+        echo default gcc: `gcc --version`
+        echo cmake compiler: `${CMAKE_PREFIX_PATH}/bin/${CMAKE_C_COMPILER} --version`
diff --git a/.github/workflows/sanitizers-testing.yml b/.github/workflows/sanitizers-testing.yml
index 4bf7d023..fe550b81 100644
--- a/.github/workflows/sanitizers-testing.yml
+++ b/.github/workflows/sanitizers-testing.yml
@@ -31,17 +31,41 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # XXX: Let's start with only Linux/x86_64
+        ARCH: [ARM64, x86_64]
         BUILDTYPE: [Debug, Release]
-        CC: [gcc-10, clang-11]
+        OS: [Linux, macOS]
+        # There are different top-level versions available for Linux and macOS runners.
+        CC: [gcc-10, clang-11, gcc-15, clang-21]
         include:
           - BUILDTYPE: Debug
             CMAKEFLAGS: -DCMAKE_BUILD_TYPE=Debug -DLUA_USE_ASSERT=ON -DLUA_USE_APICHECK=ON
           - BUILDTYPE: Release
             CMAKEFLAGS: -DCMAKE_BUILD_TYPE=RelWithDebInfo
-    runs-on: [self-hosted, regular, Linux, x86_64]
+        exclude:
+          # On current runners with Linux/ARM64 environment and
+          # with LUAJIT_USE_SYSMALLOC=ON the system allocator returns addresses
+          # with 48-bit set. Thus checkptrGC() fails with new Lua state pointer
+          # and luajit fails to start with 'cannot create state: not enough memory'
+          # error. So, we exclude these cases.
+          - ARCH: ARM64
+            OS: Linux
+          # Exclude nonsuitable OS/compiler pairs.
+          - OS: macOS
+            CC: gcc-10
+          - OS: macOS
+            CC: clang-11
+          - OS: Linux
+            CC: gcc-15
+          - OS: Linux
+            CC: clang-21
+          # Exclude macOS/ARM64/gcc case due to some tests are failed.
+          # Details: https://github.com/tarantool/tarantool/issues/13018
+          - ARCH: ARM64
+            OS: macOS
+            CC: gcc-15
+    runs-on: [self-hosted, regular, '${{ matrix.OS }}', '${{ matrix.ARCH }}']
     name: >
-      LuaJIT with ASan and UBSan (Linux/x86_64)
+      LuaJIT with ASan and UBSan (${{ matrix.OS }}/${{ matrix.ARCH }})
       ${{ matrix.BUILDTYPE }}
       CC:${{ matrix.CC }}
       GC64:ON SYSMALLOC:ON
@@ -51,7 +75,13 @@ jobs:
           fetch-depth: 0
           submodules: recursive
       - name: setup Linux for sanitizers
-        uses: ./.github/actions/setup-sanitizers
+        if: ${{ matrix.OS == 'Linux' }}
+        uses: ./.github/actions/setup-sanitizers-linux
+        with:
+          cc_name: ${{ matrix.CC }}
+      - name: setup macOS for sanitizers
+        if: ${{ matrix.OS == 'macOS' }}
+        uses: ./.github/actions/setup-sanitizers-macos
         with:
           cc_name: ${{ matrix.CC }}
       - name: configure
@@ -70,18 +100,46 @@ jobs:
           cmake -S . -B ${{ env.BUILDDIR }}
           -G Ninja
           ${{ matrix.CMAKEFLAGS }}
+          -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
+          -DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}
           -DLUAJIT_ENABLE_GC64=ON
           -DLUAJIT_USE_ASAN=ON
           -DLUAJIT_USE_SYSMALLOC=ON
           -DLUAJIT_USE_UBSAN=ON
+      - name: Check for possible compiler misconfig
+        working-directory: ${{ env.BUILDDIR }}
+        env:
+          CC_NAME: ${{ matrix.CC }}
+        run: grep CMakeCache.txt -e CMAKE_C_COMPILER:STRING | grep ${CC_NAME}
       - name: build
         run: cmake --build . --parallel
         working-directory: ${{ env.BUILDDIR }}
-      - name: test
+
+      # Enable as much checks as possible. See more info here:
+      # https://github.com/google/sanitizers/wiki/AddressSanitizerFlags,
+      # https://github.com/google/sanitizers/wiki/SanitizerCommonFlags.
+      - name: setup sanitizer options for Linux
+        if: ${{ matrix.OS == 'Linux' }}
+        env:
+          ASAN_OPTIONS: "                    \
+            detect_invalid_pointer_pairs=1:  \
+            detect_leaks=1:                  \
+            detect_stack_use_after_return=1: \
+            dump_instruction_bytes=1:        \
+            heap_profile=0:                  \
+            print_suppressions=0:            \
+            symbolize=1:                     \
+            unmap_shadow_on_exit=1:          \
+          "
+          UBSAN_OPTIONS: "
+            print_stacktrace=1 \
+          "
+        run: |
+          echo ASAN_OPTIONS=${ASAN_OPTIONS} | tee -a $GITHUB_ENV
+          echo UBSAN_OPTIONS=${UBSAN_OPTIONS} | tee -a $GITHUB_ENV
+      - name: setup sanitizer options for macOS (common)
+        if: ${{ matrix.OS == 'macOS' && matrix.ARCH != 'ARM64' && matrix.CC != 'clang-21' }}
         env:
-          # Enable as much checks as possible. See more info here:
-          # https://github.com/google/sanitizers/wiki/AddressSanitizerFlags,
-          # https://github.com/google/sanitizers/wiki/SanitizerCommonFlags.
           ASAN_OPTIONS: "                    \
             detect_invalid_pointer_pairs=1:  \
             detect_leaks=1:                  \
@@ -95,5 +153,30 @@ jobs:
           UBSAN_OPTIONS: "
             print_stacktrace=1 \
           "
+        run: |
+          echo ASAN_OPTIONS=${ASAN_OPTIONS} | tee -a $GITHUB_ENV
+          echo UBSAN_OPTIONS=${UBSAN_OPTIONS} | tee -a $GITHUB_ENV
+      - name: setup sanitizer options for macOS (ARM64/clang-21)
+        if: ${{ matrix.OS == 'macOS' && matrix.ARCH == 'ARM64' && matrix.CC == 'clang-21' }}
+        # detect_leaks is disabled due to some tests build is failed.
+        # Details: https://github.com/tarantool/tarantool/issues/13019
+        env:
+          ASAN_OPTIONS: "                    \
+            detect_invalid_pointer_pairs=1:  \
+            detect_leaks=0:                  \
+            detect_stack_use_after_return=1: \
+            dump_instruction_bytes=1:        \
+            heap_profile=0:                  \
+            print_suppressions=0:            \
+            symbolize=1:                     \
+            unmap_shadow_on_exit=1:          \
+          "
+          UBSAN_OPTIONS: "
+            print_stacktrace=1 \
+          "
+        run: |
+          echo ASAN_OPTIONS=${ASAN_OPTIONS} | tee -a $GITHUB_ENV
+          echo UBSAN_OPTIONS=${UBSAN_OPTIONS} | tee -a $GITHUB_ENV
+      - name: test
         run: cmake --build . --parallel --target LuaJIT-test
         working-directory: ${{ env.BUILDDIR }}
-- 
2.49.0