[Tarantool-patches] [PATCH luajit 1/1] asan: instrumented LuaJIT memory allocator

mandesero at gmail.com mandesero at gmail.com
Thu Aug 22 21:15:42 MSK 2024


From: Maksim Tiushev <mandesero at gmail.com>

This patch adds instrumentation of the internal LuaJIT memory allocator.
It now enables the detection of memory-related errors when using FFI and
within LuaJIT itself. This enhancement improves reliability and debugging
capabilities.

This patch introduces two scenarios for using ASAN with LuaJIT:
- LuaJIT using sysmalloc: `-DLUAJIT_USE_ASAN=ON`
- LuaJIT using internal memory allocator: `-DLUAJIT_USE_ASAN_HARDENING=ON`

If you want to skip tests when LuaJIT uses the internal memory allocator,
you can check the `LJ_ASAN_HARDENING` environment variable.

The test `test/tarantool-tests/lj-1034-tabov-error-frame.test.lua` has
been disabled under ASAN & LuaJIT's internal allocator due to consistently
failing with a timeout.

Part of #10231
---
 .github/workflows/sanitizers-testing.yml      |  20 +-
 CMakeLists.txt                                |  29 +-
 src/lj_alloc.c                                | 267 +++++++++++++++++-
 .../asan-alloc-instrumentation.test.c         | 141 +++++++++
 .../asan-mmap-instrumentation.test.c          | 128 +++++++++
 .../ASAN-left-memory-miss.test.lua            |  12 +
 .../ASAN-left-memory-miss/script.lua          |  19 ++
 .../ASAN-right-memory-miss.test.lua           |  12 +
 .../ASAN-right-memory-miss/script.lua         |  19 ++
 .../ASAN-use-after-free.test.lua              |  12 +
 .../ASAN-use-after-free/script.lua            |  24 ++
 test/tarantool-tests/CMakeLists.txt           |   2 +-
 .../lj-1034-tabov-error-frame.test.lua        |   2 +
 13 files changed, 669 insertions(+), 18 deletions(-)
 create mode 100644 test/tarantool-c-tests/asan-alloc-instrumentation.test.c
 create mode 100644 test/tarantool-c-tests/asan-mmap-instrumentation.test.c
 create mode 100644 test/tarantool-tests/ASAN-left-memory-miss.test.lua
 create mode 100644 test/tarantool-tests/ASAN-left-memory-miss/script.lua
 create mode 100644 test/tarantool-tests/ASAN-right-memory-miss.test.lua
 create mode 100644 test/tarantool-tests/ASAN-right-memory-miss/script.lua
 create mode 100644 test/tarantool-tests/ASAN-use-after-free.test.lua
 create mode 100644 test/tarantool-tests/ASAN-use-after-free/script.lua

diff --git a/.github/workflows/sanitizers-testing.yml b/.github/workflows/sanitizers-testing.yml
index 4bf7d023..5699f843 100644
--- a/.github/workflows/sanitizers-testing.yml
+++ b/.github/workflows/sanitizers-testing.yml
@@ -34,17 +34,23 @@ jobs:
         # XXX: Let's start with only Linux/x86_64
         BUILDTYPE: [Debug, Release]
         CC: [gcc-10, clang-11]
+        ALLOCATOR: [sysmalloc, dlmalloc]
         include:
           - BUILDTYPE: Debug
             CMAKEFLAGS: -DCMAKE_BUILD_TYPE=Debug -DLUA_USE_ASSERT=ON -DLUA_USE_APICHECK=ON
           - BUILDTYPE: Release
             CMAKEFLAGS: -DCMAKE_BUILD_TYPE=RelWithDebInfo
+          - ALLOCATOR: sysmalloc
+            ASANFLAGS: -DLUAJIT_USE_ASAN=ON -DLUAJIT_USE_SYSMALLOC=ON
+          - ALLOCATOR: dlmalloc
+            ASANFLAGS: -DLUAJIT_USE_ASAN_HARDENING=ON
     runs-on: [self-hosted, regular, Linux, x86_64]
     name: >
       LuaJIT with ASan and UBSan (Linux/x86_64)
       ${{ matrix.BUILDTYPE }}
       CC:${{ matrix.CC }}
-      GC64:ON SYSMALLOC:ON
+      GC64:ON
+      ALLOC=${{ matrix.ALLOCATOR }}
     steps:
       - uses: actions/checkout at v4
         with:
@@ -55,24 +61,18 @@ jobs:
         with:
           cc_name: ${{ matrix.CC }}
       - name: configure
-        # XXX: LuaJIT configuration requires a couple of tweaks:
-        # LUAJIT_USE_SYSMALLOC=ON: Unfortunately, internal LuaJIT
-        #   memory allocator is not instrumented yet, so to find
-        #   any memory errors it's better to build LuaJIT with
-        #   system provided memory allocator (i.e. run CMake
-        #   configuration phase with -DLUAJIT_USE_SYSMALLOC=ON).
-        #   For more info, see root CMakeLists.txt.
         # LUAJIT_ENABLE_GC64=ON: LUAJIT_USE_SYSMALLOC cannot be
         #   enabled on x64 without GC64, since realloc usually
         #   doesn't return addresses in the right address range.
+        #   Additionally, the ASAN instrumentation for LuaJIT's
+        #   internal memory allocator is only available for GC64.
         #   For more info, see root CMakeLists.txt.
         run: >
           cmake -S . -B ${{ env.BUILDDIR }}
           -G Ninja
           ${{ matrix.CMAKEFLAGS }}
           -DLUAJIT_ENABLE_GC64=ON
-          -DLUAJIT_USE_ASAN=ON
-          -DLUAJIT_USE_SYSMALLOC=ON
+          ${{ matrix.ASANFLAGS }}
           -DLUAJIT_USE_UBSAN=ON
       - name: build
         run: cmake --build . --parallel
diff --git a/CMakeLists.txt b/CMakeLists.txt
index aa2103b3..619babad 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -284,17 +284,34 @@ if(LUA_USE_ASSERT)
   AppendFlags(TARGET_C_FLAGS -DLUA_USE_ASSERT)
 endif()
 
+option(LUAJIT_USE_ASAN "Build LuaJIT with AddressSanitizer" OFF)
+# Same as LUAJIT_USE_ASAN, but using the internal LuaJIT memory
+# allocator instrumented with ASAN.
+option(LUAJIT_USE_ASAN_HARDENING "Build LuaJIT with an internal allocator with integrated AddressSanitizer" OFF)
+if(LUAJIT_USE_ASAN_HARDENING)
+  set(LUAJIT_USE_ASAN ON)
+  if(NOT LUAJIT_ENABLE_GC64)
+    message(FATAL_ERROR
+      "ASAN only with GC64."
+    )
+  endif()
+  AppendFlags(CMAKE_C_FLAGS
+    # Enable ASAN instrumentation of internal LuaJIT memory allocator
+    # see (src/lj_alloc.c)
+    -DLUAJIT_USE_ASAN_HARDENING
+  )
+endif()
+
 # Turn on AddressSanitizer support. As a result, all artefacts
 # (i.e. buildvm, LuaJIT, testing infrastructure) are built with
 # ASan enabled.
-option(LUAJIT_USE_ASAN "Build LuaJIT with AddressSanitizer" OFF)
 if(LUAJIT_USE_ASAN)
-  if(NOT LUAJIT_USE_SYSMALLOC)
+  if(NOT (LUAJIT_USE_SYSMALLOC OR LUAJIT_USE_ASAN_HARDENING))
     message(WARNING
-      "Unfortunately, internal LuaJIT memory allocator is not instrumented yet,"
-      " so to find any memory errors it's better to build LuaJIT with system"
-      " provided memory allocator (i.e. run CMake configuration phase with"
-      " -DLUAJIT_USE_SYSMALLOC=ON)."
+      "Run CMake configuration phase with -DLUAJIT_USE_SYSMALLOC=ON "
+      "to use system memory allocator or replace -DLUAJIT_USE_ASAN=ON "
+      "to -DLUAJIT_USE_ASAN_HARDENING=ON to use internal LuaJIT memory"
+      "allocator."
     )
   endif()
   # Use all recommendations described in AddressSanitize docs:
diff --git a/src/lj_alloc.c b/src/lj_alloc.c
index f82c9854..097a078d 100644
--- a/src/lj_alloc.c
+++ b/src/lj_alloc.c
@@ -226,6 +226,108 @@ static int CALL_MUNMAP(void *ptr, size_t size)
 
 #define LJ_ALLOC_MMAP_PROBE_LOWER	((uintptr_t)0x4000)
 
+#if LUAJIT_USE_ASAN_HARDENING
+
+/*
+** The work of asan (AddressSanitizer) is to detect memory errors during program execution.
+** One way to achieve this is by adding redzones around memory allocations. The redzone is a
+** specially allocated area of memory before and after the allocated block, which is filled
+** with a unique value. If the program tries to access memory outside of the allocation,
+** asan detects this attempt and generates an error message, allowing the developer to
+** detect and fix the issue early.
+**
+** - Original paper: https://www.usenix.org/system/files/conference/atc12/atc12-final39.pdf
+**
+** LuaJIT ASAN instrumentation (mmap and others):
+**
+** - Memory map around allocation:
+** -------------------------------------------------------------------------------------
+** .. .. | [f7]    ...    [f7] | [00]     ...     [0(0-7)] | [f7]    ...    [f7] | .. ..
+**       |    left redzone     |           data            |    right redzone    |
+**       |  REDZONE_SIZE bytes |          N bytes          |  REDZONE_SIZE bytes |
+** -------------------------------------------------------------------------------------
+**
+** left redzone:
+**  The first SIZE_T_SIZE bytes of the redzone contain the data size N, the next SIZE_T_SIZE bytes
+**  of the redzone contain the full size of the allocation, including the alignment of the size N
+**  and the size of the redzones themselves.
+*/
+
+#include <sanitizer/asan_interface.h>
+
+/**
+ *
+ * Memory map for 64-bit (shift = 3)
+ * The shadow address is calculated by (Mem >> shift) + 0x7fff8000
+ *
+ * [0x10007fff8000, 0x7fffffffffff]	HighMem
+ * [0x02008fff7000, 0x10007fff7fff]	HighShadow
+ * [0x00008fff7000, 0x02008fff6fff]	ShadowGap
+ * [0x00007fff8000, 0x00008fff6fff]	LowShadow
+ * [0x000000000000, 0x00007fff7fff]	LowMem
+ *
+ */
+
+/* Recommended redzone size from 16 to 2048 bytes (must be a a power of two)
+** https://github.com/google/sanitizers/wiki/AddressSanitizerFlags
+*/
+#define REDZONE_SIZE FOUR_SIZE_T_SIZES
+
+/* Total redzone size around allocation */
+#define TOTAL_REDZONE_SIZE (REDZONE_SIZE << 1)
+
+/* Multiple of the allocated memory size */
+#define SIZE_ALIGNMENT MALLOC_ALIGNMENT
+
+/**
+ * We can only use the address from HighMem, so we must force the system allocator (mmap)
+ * to return addresses starting from the lower bound of HighMem.
+ */
+static inline uintptr_t asan_lower_address()
+{
+  size_t shadow_scale;
+  size_t shadow_offset;
+  __asan_get_shadow_mapping(&shadow_scale, &shadow_offset);
+  return (uintptr_t)(shadow_offset + (1ULL << (LJ_ALLOC_MBITS - shadow_scale)));
+}
+
+/* Casting to the nearest multiple of SIZE_ALIGNMENT from above */
+#define ALIGN_SIZE(S, ALIGN)  ((size_t)(((S) + (ALIGN) - 1) & ~((ALIGN) - 1)))
+
+#define alloc2mem(p)		((void *)((char *)(p) + REDZONE_SIZE))
+#define mem2alloc(mem)		((void *)((char *)(mem) - REDZONE_SIZE))
+
+/* Add redzones around allocation and keep the memory size and poison size. */
+void *mark_memory_region(void *ptr, size_t msize, size_t psize)
+{
+  if (ptr == NULL)
+    return NULL;
+
+  ASAN_UNPOISON_MEMORY_REGION(ptr, TWO_SIZE_T_SIZES);
+  *((size_t *)(ptr)) = msize;
+  *((size_t *)(ptr) + 1) = psize;
+  ASAN_POISON_MEMORY_REGION(ptr, psize);
+  ptr = alloc2mem(ptr);
+  ASAN_UNPOISON_MEMORY_REGION(ptr, msize);
+  return ptr;
+}
+
+typedef enum {
+  MEM_SIZE,
+  POISON_SIZE
+} SizeType;
+
+size_t asan_get_size(void *ptr, SizeType type)
+{
+  size_t offset = (type == MEM_SIZE) ? 0 : SIZE_T_SIZE;
+  ASAN_UNPOISON_MEMORY_REGION(ptr - REDZONE_SIZE + offset, SIZE_T_SIZE);
+  size_t size = *((size_t *)(ptr - REDZONE_SIZE + offset));
+  ASAN_POISON_MEMORY_REGION(ptr - REDZONE_SIZE + offset, SIZE_T_SIZE);
+  return size;
+}
+
+#endif
+
 /* No point in a giant ifdef mess. Just try to open /dev/urandom.
 ** It doesn't really matter if this fails, since we get some ASLR bits from
 ** every unsuitable allocation, too. And we prefer linear allocation, anyway.
@@ -252,14 +354,28 @@ static void *mmap_probe(size_t size)
   static uintptr_t hint_prng = 0;
   int olderr = errno;
   int retry;
+#if LUAJIT_USE_ASAN_HARDENING
+  /* Save the request memory size */
+  size_t msize = size;
+  /* Total allocation size corresponds to the memory size and the size of redzones */
+  size = ALIGN_SIZE(size + TOTAL_REDZONE_SIZE, LJ_PAGESIZE);
+#endif
   for (retry = 0; retry < LJ_ALLOC_MMAP_PROBE_MAX; retry++) {
     void *p = mmap((void *)hint_addr, size, MMAP_PROT, MMAP_FLAGS_PROBE, -1, 0);
     uintptr_t addr = (uintptr_t)p;
+#if LUAJIT_USE_ASAN_HARDENING
+    if ((addr >> LJ_ALLOC_MBITS) == 0 && addr >= asan_lower_address() &&
+	((addr + size) >> LJ_ALLOC_MBITS) == 0) {
+#else
     if ((addr >> LJ_ALLOC_MBITS) == 0 && addr >= LJ_ALLOC_MMAP_PROBE_LOWER &&
 	((addr + size) >> LJ_ALLOC_MBITS) == 0) {
+#endif
       /* We got a suitable address. Bump the hint address. */
       hint_addr = addr + size;
       errno = olderr;
+#if LUAJIT_USE_ASAN_HARDENING
+      p = mark_memory_region(p, msize, size);
+#endif
       return p;
     }
     if (p != MFAIL) {
@@ -338,8 +454,19 @@ static void *mmap_map32(size_t size)
 static void *CALL_MMAP(size_t size)
 {
   int olderr = errno;
+#if LUAJIT_USE_ASAN_HARDENING
+  size_t msize = size;
+  size = ALIGN_SIZE(size + TOTAL_REDZONE_SIZE, LJ_PAGESIZE);
+#endif
+#if LUAJIT_USE_ASAN_HARDENING
+  void *ptr = mmap((void *)asan_lower_address(), size, MMAP_PROT, MMAP_FLAGS, -1, 0);
+#else
   void *ptr = mmap(NULL, size, MMAP_PROT, MMAP_FLAGS, -1, 0);
+#endif
   errno = olderr;
+#if LUAJIT_USE_ASAN_HARDENING
+  ptr = mark_memory_region(ptr, msize, size);
+#endif
   return ptr;
 }
 #endif
@@ -361,7 +488,18 @@ static void init_mmap(void)
 static int CALL_MUNMAP(void *ptr, size_t size)
 {
   int olderr = errno;
+#if LUAJIT_USE_ASAN_HARDENING
+  /* check that memory is not poisoned */
+  memmove(ptr, ptr, size);
+  size = asan_get_size(ptr, POISON_SIZE);
+  ptr = mem2alloc(ptr);
+#endif
   int ret = munmap(ptr, size);
+#if LUAJIT_USE_ASAN_HARDENING
+  if (ret == 0) {
+    ASAN_POISON_MEMORY_REGION(ptr, size);
+  }
+#endif
   errno = olderr;
   return ret;
 }
@@ -371,7 +509,31 @@ static int CALL_MUNMAP(void *ptr, size_t size)
 static void *CALL_MREMAP_(void *ptr, size_t osz, size_t nsz, int flags)
 {
   int olderr = errno;
+#if LUAJIT_USE_ASAN_HARDENING && !(LJ_64 && (!LJ_GC64 || LJ_TARGET_ARM64))
+  void *new_ptr = CALL_MMAP(nsz);
+  if (new_ptr != MFAIL) {
+    size_t oms = asan_get_size(ptr, MEM_SIZE);
+    memcpy(new_ptr, ptr, oms > nsz ? nsz : oms);
+    CALL_MUNMAP(ptr, osz);
+    ptr = new_ptr;
+  }
+#else
+
+#if LUAJIT_USE_ASAN_HARDENING
+  void *old_ptr = ptr;
+  size_t nms = nsz;
+  osz = asan_get_size(old_ptr, POISON_SIZE);
+  nsz = ALIGN_SIZE(nsz + TOTAL_REDZONE_SIZE, LJ_PAGESIZE);
+  ptr = mem2alloc(ptr);
+#endif
   ptr = mremap(ptr, osz, nsz, flags);
+#if LUAJIT_USE_ASAN_HARDENING
+  if (ptr != MFAIL) {
+    ASAN_POISON_MEMORY_REGION((void *)((char *)(old_ptr) - REDZONE_SIZE), osz);
+    ptr = mark_memory_region(ptr, nms, nsz);
+  }
+#endif
+#endif
   errno = olderr;
   return ptr;
 }
@@ -432,9 +594,15 @@ typedef unsigned int flag_t;           /* The type of various bit flag sets */
 #define MIN_CHUNK_SIZE\
   ((MCHUNK_SIZE + CHUNK_ALIGN_MASK) & ~CHUNK_ALIGN_MASK)
 
+#if LUAJIT_USE_ASAN_HARDENING
+/* conversion from malloc headers to user pointers, and back */
+#define chunk2mem(p)		((void *)((char *)(p) + TWO_SIZE_T_SIZES + REDZONE_SIZE))
+#define mem2chunk(mem)		((mchunkptr)((char *)(mem) - TWO_SIZE_T_SIZES - REDZONE_SIZE))
+#else
 /* conversion from malloc headers to user pointers, and back */
 #define chunk2mem(p)		((void *)((char *)(p) + TWO_SIZE_T_SIZES))
 #define mem2chunk(mem)		((mchunkptr)((char *)(mem) - TWO_SIZE_T_SIZES))
+#endif
 /* chunk associated with aligned address A */
 #define align_as_chunk(A)	(mchunkptr)((A) + align_offset(chunk2mem(A)))
 
@@ -837,7 +1005,13 @@ static int has_segment_link(mstate m, msegmentptr ss)
 
 static void *direct_alloc(size_t nb)
 {
+#if LUAJIT_USE_ASAN_HARDENING
+  nb += TOTAL_REDZONE_SIZE;
+#endif
   size_t mmsize = mmap_align(nb + SIX_SIZE_T_SIZES + CHUNK_ALIGN_MASK);
+#if LUAJIT_USE_ASAN_HARDENING
+  mmsize -= TOTAL_REDZONE_SIZE;
+#endif
   if (LJ_LIKELY(mmsize > nb)) {     /* Check for wrap around 0 */
     char *mm = (char *)(DIRECT_MMAP(mmsize));
     if (mm != CMFAIL) {
@@ -887,7 +1061,12 @@ static mchunkptr direct_resize(mchunkptr oldp, size_t nb)
 static void init_top(mstate m, mchunkptr p, size_t psize)
 {
   /* Ensure alignment */
-  size_t offset = align_offset(chunk2mem(p));
+  void *t = chunk2mem(p);
+#if LUAJIT_USE_ASAN_HARDENING
+  t = mem2alloc(t);
+#endif
+  size_t offset = align_offset(t);
+
   p = (mchunkptr)((char *)p + offset);
   psize -= offset;
 
@@ -949,6 +1128,9 @@ static void add_segment(mstate m, char *tbase, size_t tsize)
   /* Determine locations and sizes of segment, fenceposts, old top */
   char *old_top = (char *)m->top;
   msegmentptr oldsp = segment_holding(m, old_top);
+#if LUAJIT_USE_ASAN_HARDENING
+  ASAN_UNPOISON_MEMORY_REGION(oldsp, sizeof(struct malloc_segment));
+#endif
   char *old_end = oldsp->base + oldsp->size;
   size_t ssize = pad_request(sizeof(struct malloc_segment));
   char *rawsp = old_end - (ssize + FOUR_SIZE_T_SIZES + CHUNK_ALIGN_MASK);
@@ -957,6 +1139,9 @@ static void add_segment(mstate m, char *tbase, size_t tsize)
   char *csp = (asp < (old_top + MIN_CHUNK_SIZE))? old_top : asp;
   mchunkptr sp = (mchunkptr)csp;
   msegmentptr ss = (msegmentptr)(chunk2mem(sp));
+#if LUAJIT_USE_ASAN_HARDENING
+  ss = (msegmentptr)(mem2alloc(ss));
+#endif
   mchunkptr tnext = chunk_plus_offset(sp, ssize);
   mchunkptr p = tnext;
 
@@ -1006,7 +1191,13 @@ static void *alloc_sys(mstate m, size_t nb)
 
   {
     size_t req = nb + TOP_FOOT_SIZE + SIZE_T_ONE;
+#if LUAJIT_USE_ASAN_HARDENING
+    req += TOTAL_REDZONE_SIZE;
+#endif
     size_t rsize = granularity_align(req);
+#if LUAJIT_USE_ASAN_HARDENING
+    rsize -= TOTAL_REDZONE_SIZE;
+#endif
     if (LJ_LIKELY(rsize > nb)) { /* Fail if wraps around zero */
       char *mp = (char *)(CALL_MMAP(rsize));
       if (mp != CMFAIL) {
@@ -1238,21 +1429,33 @@ static void *tmalloc_small(mstate m, size_t nb)
 void *lj_alloc_create(void)
 {
   size_t tsize = DEFAULT_GRANULARITY;
+#if LUAJIT_USE_ASAN_HARDENING
+  tsize -= TOTAL_REDZONE_SIZE;
+#endif
   char *tbase;
   INIT_MMAP();
   tbase = (char *)(CALL_MMAP(tsize));
   if (tbase != CMFAIL) {
     size_t msize = pad_request(sizeof(struct malloc_state));
     mchunkptr mn;
+#if LUAJIT_USE_ASAN_HARDENING
+    mchunkptr msp = (mchunkptr)(tbase + align_offset(mem2alloc(chunk2mem(tbase))));
+    mstate m = (mstate)(mem2alloc(chunk2mem(msp)));
+#else
     mchunkptr msp = align_as_chunk(tbase);
     mstate m = (mstate)(chunk2mem(msp));
+#endif
     memset(m, 0, msize);
     msp->head = (msize|PINUSE_BIT|CINUSE_BIT);
     m->seg.base = tbase;
     m->seg.size = tsize;
     m->release_checks = MAX_RELEASE_CHECK_RATE;
     init_bins(m);
+#if LUAJIT_USE_ASAN_HARDENING
+    mn = next_chunk((mchunkptr)((char *)(m) - TWO_SIZE_T_SIZES));
+#else
     mn = next_chunk(mem2chunk(m));
+#endif
     init_top(m, mn, (size_t)((tbase + tsize) - (char *)mn) - TOP_FOOT_SIZE);
     return m;
   }
@@ -1267,12 +1470,22 @@ void lj_alloc_destroy(void *msp)
     char *base = sp->base;
     size_t size = sp->size;
     sp = sp->next;
+#if LUAJIT_USE_ASAN_HARDENING
+    ASAN_UNPOISON_MEMORY_REGION(base, size);
+#endif
     CALL_MUNMAP(base, size);
   }
 }
 
 static LJ_NOINLINE void *lj_alloc_malloc(void *msp, size_t nsize)
 {
+#if LUAJIT_USE_ASAN_HARDENING
+  if (nsize == 0)
+    nsize = MIN_CHUNK_SIZE;
+  size_t mem_size = nsize;
+  size_t poison_size = ALIGN_SIZE(nsize, SIZE_ALIGNMENT) + TOTAL_REDZONE_SIZE;
+  nsize = poison_size;
+#endif
   mstate ms = (mstate)msp;
   void *mem;
   size_t nb;
@@ -1291,6 +1504,9 @@ static LJ_NOINLINE void *lj_alloc_malloc(void *msp, size_t nsize)
       unlink_first_small_chunk(ms, b, p, idx);
       set_inuse_and_pinuse(ms, p, small_index2size(idx));
       mem = chunk2mem(p);
+#if LUAJIT_USE_ASAN_HARDENING
+      mem = mark_memory_region(mem2alloc(mem), mem_size, poison_size);
+#endif
       return mem;
     } else if (nb > ms->dvsize) {
       if (smallbits != 0) { /* Use chunk in next nonempty smallbin */
@@ -1312,8 +1528,14 @@ static LJ_NOINLINE void *lj_alloc_malloc(void *msp, size_t nsize)
 	  replace_dv(ms, r, rsize);
 	}
 	mem = chunk2mem(p);
+#if LUAJIT_USE_ASAN_HARDENING
+  mem = mark_memory_region(mem2alloc(mem), mem_size, poison_size);
+#endif
 	return mem;
       } else if (ms->treemap != 0 && (mem = tmalloc_small(ms, nb)) != 0) {
+#if LUAJIT_USE_ASAN_HARDENING
+  mem = mark_memory_region(mem2alloc(mem), mem_size, poison_size);
+#endif
 	return mem;
       }
     }
@@ -1322,6 +1544,9 @@ static LJ_NOINLINE void *lj_alloc_malloc(void *msp, size_t nsize)
   } else {
     nb = pad_request(nsize);
     if (ms->treemap != 0 && (mem = tmalloc_large(ms, nb)) != 0) {
+#if LUAJIT_USE_ASAN_HARDENING
+      mem = mark_memory_region(mem2alloc(mem), mem_size, poison_size);
+#endif
       return mem;
     }
   }
@@ -1341,6 +1566,9 @@ static LJ_NOINLINE void *lj_alloc_malloc(void *msp, size_t nsize)
       set_inuse_and_pinuse(ms, p, dvs);
     }
     mem = chunk2mem(p);
+#if LUAJIT_USE_ASAN_HARDENING
+    mem = mark_memory_region(mem2alloc(mem), mem_size, poison_size);
+#endif
     return mem;
   } else if (nb < ms->topsize) { /* Split top */
     size_t rsize = ms->topsize -= nb;
@@ -1349,13 +1577,30 @@ static LJ_NOINLINE void *lj_alloc_malloc(void *msp, size_t nsize)
     r->head = rsize | PINUSE_BIT;
     set_size_and_pinuse_of_inuse_chunk(ms, p, nb);
     mem = chunk2mem(p);
+#if LUAJIT_USE_ASAN_HARDENING
+    mem = mark_memory_region(mem2alloc(mem), mem_size, poison_size);
+#endif
     return mem;
   }
+#if LUAJIT_USE_ASAN_HARDENING
+  return mark_memory_region(mem2alloc(alloc_sys(ms, nb)), mem_size, poison_size);
+#else
   return alloc_sys(ms, nb);
+#endif
 }
 
 static LJ_NOINLINE void *lj_alloc_free(void *msp, void *ptr)
 {
+#if LUAJIT_USE_ASAN_HARDENING
+  if (ptr != 0) {    
+    size_t mem_size = asan_get_size(ptr, MEM_SIZE);
+    size_t poison_size = asan_get_size(ptr, POISON_SIZE);
+
+    memmove(ptr, ptr, mem_size);
+    ASAN_POISON_MEMORY_REGION(mem2alloc(ptr), poison_size);
+  }
+  return NULL;
+#else
   if (ptr != 0) {
     mchunkptr p = mem2chunk(ptr);
     mstate fm = (mstate)msp;
@@ -1423,10 +1668,29 @@ static LJ_NOINLINE void *lj_alloc_free(void *msp, void *ptr)
     }
   }
   return NULL;
+#endif
 }
 
 static LJ_NOINLINE void *lj_alloc_realloc(void *msp, void *ptr, size_t nsize)
 {
+#if LUAJIT_USE_ASAN_HARDENING
+  if (nsize >= MAX_REQUEST)
+    return NULL;
+
+  mstate m = (mstate)msp;
+
+  size_t mem_size = asan_get_size(ptr, MEM_SIZE);
+  size_t poison_size = asan_get_size(ptr, POISON_SIZE);
+
+  void *newmem = lj_alloc_malloc(m, nsize);
+
+  if (newmem == NULL)
+    return NULL;
+
+  memcpy(newmem, ptr, nsize > mem_size ? mem_size : nsize);
+  ASAN_POISON_MEMORY_REGION(mem2alloc(ptr), poison_size);
+  return newmem;
+#else
   if (nsize >= MAX_REQUEST) {
     return NULL;
   } else {
@@ -1473,6 +1737,7 @@ static LJ_NOINLINE void *lj_alloc_realloc(void *msp, void *ptr, size_t nsize)
       return newmem;
     }
   }
+#endif
 }
 
 void *lj_alloc_f(void *msp, void *ptr, size_t osize, size_t nsize)
diff --git a/test/tarantool-c-tests/asan-alloc-instrumentation.test.c b/test/tarantool-c-tests/asan-alloc-instrumentation.test.c
new file mode 100644
index 00000000..fbbd387b
--- /dev/null
+++ b/test/tarantool-c-tests/asan-alloc-instrumentation.test.c
@@ -0,0 +1,141 @@
+#include "lua.h"
+#include "test.h"
+#include "utils.h"
+#include "lj_alloc.c"
+#include "lj_gc.h"
+
+#if LUAJIT_USE_ASAN_HARDENING
+#include <sanitizer/asan_interface.h>
+
+#define IS_POISONED(ptr) __asan_address_is_poisoned(ptr)
+
+int IS_POISONED_REGION(void *ptr, size_t size)
+{
+    int res = 1;
+    int i = 0;
+    do {
+        res *= IS_POISONED(ptr + i);
+    } while (res == 1 && ++i < size);
+    return res;
+}
+#endif
+
+static lua_State *main_LS = NULL;
+static global_State *main_GS = NULL;
+
+static int small_malloc_test(void *test_state)
+{
+#if !LUAJIT_USE_ASAN_HARDENING || LUAJIT_USE_SYSMALLOC
+    UNUSED(test_state);
+    return skip("Requires build with ASAN");
+#else
+    int res = -1;
+    size_t size = 30;
+    void *p = lj_mem_new(main_LS, size);
+    size_t algn = (MALLOC_ALIGNMENT - size % MALLOC_ALIGNMENT) % MALLOC_ALIGNMENT;
+
+    if (IS_POISONED_REGION(p - REDZONE_SIZE, REDZONE_SIZE) &&
+        !IS_POISONED_REGION(p, size) &&
+        IS_POISONED_REGION(p + size, algn + REDZONE_SIZE))
+        res = TEST_EXIT_SUCCESS;
+
+    lj_mem_free(main_GS, p, size);
+	return res == TEST_EXIT_SUCCESS ? TEST_EXIT_SUCCESS : TEST_EXIT_FAILURE;
+#endif
+}
+
+static int large_malloc_test(void *test_state)
+{
+#if !LUAJIT_USE_ASAN_HARDENING || LUAJIT_USE_SYSMALLOC
+    UNUSED(test_state);
+    return skip("Requires build with ASAN");
+#else
+    int res = -1;
+    size_t size = 1234;
+    void *p = lj_mem_new(main_LS, size);
+    size_t algn = (MALLOC_ALIGNMENT - size % MALLOC_ALIGNMENT) % MALLOC_ALIGNMENT;
+
+    if (IS_POISONED_REGION(p - REDZONE_SIZE, REDZONE_SIZE) &&
+        !IS_POISONED_REGION(p, size) &&
+        IS_POISONED_REGION(p + size, algn + REDZONE_SIZE))
+        res = TEST_EXIT_SUCCESS;
+        
+    lj_mem_free(main_GS, p, size);
+	return res == TEST_EXIT_SUCCESS ? TEST_EXIT_SUCCESS : TEST_EXIT_FAILURE;
+#endif
+}
+
+static int free_test(void *test_state)
+{
+#if !LUAJIT_USE_ASAN_HARDENING || LUAJIT_USE_SYSMALLOC
+    UNUSED(test_state);
+    return skip("Requires build with ASAN");
+#else
+    size_t size = 1234;
+    void *p = lj_mem_new(main_LS, size);
+    size_t algn = (MALLOC_ALIGNMENT - size % MALLOC_ALIGNMENT) % MALLOC_ALIGNMENT;
+    lj_mem_free(main_GS, p, size);
+
+    if (IS_POISONED_REGION(p - REDZONE_SIZE, TOTAL_REDZONE_SIZE + size + algn))
+    {
+        return TEST_EXIT_SUCCESS;
+    }
+    return TEST_EXIT_FAILURE;
+#endif
+}
+
+static int realloc_test(void *test_state)
+{
+#if !LUAJIT_USE_ASAN_HARDENING || LUAJIT_USE_SYSMALLOC
+    UNUSED(test_state);
+    return skip("Requires build with ASAN");
+#else
+    int res = -1;
+    size_t size = 150;
+    size_t new_size = size * 2;
+    void *p = lj_mem_new(main_LS, size);
+    uint8_t *ptr = (uint8_t *)p;
+    size_t algn = (MALLOC_ALIGNMENT - size % MALLOC_ALIGNMENT) % MALLOC_ALIGNMENT;
+    size_t new_algn = (MALLOC_ALIGNMENT - new_size % MALLOC_ALIGNMENT) % MALLOC_ALIGNMENT;
+
+    for (size_t i = 0; i < size; ++i)
+    {
+        ptr[i] = i;
+    }
+
+    void *newptr = lj_mem_realloc(main_LS, p, size, new_size);
+
+    if (IS_POISONED_REGION(ptr - REDZONE_SIZE, TOTAL_REDZONE_SIZE + size + algn))
+    {
+        ASAN_UNPOISON_MEMORY_REGION(ptr, size);
+        if (memcmp(ptr, newptr, size) != 0)
+            res = TEST_EXIT_FAILURE;
+
+        if (IS_POISONED_REGION(newptr - REDZONE_SIZE, REDZONE_SIZE) &&
+            !IS_POISONED_REGION(newptr, new_size) &&
+            IS_POISONED_REGION(newptr + new_size, new_algn + REDZONE_SIZE))
+            res = TEST_EXIT_SUCCESS;
+    }
+        lj_mem_free(main_GS, newptr, new_size);
+	return res == TEST_EXIT_SUCCESS ? TEST_EXIT_SUCCESS : TEST_EXIT_FAILURE;
+#endif
+}
+
+int main(void)
+{
+    lua_State *L = utils_lua_init();
+    global_State *g = G(L);
+    main_LS = L;
+    main_GS = g;
+
+    const struct test_unit tgroup[] = {
+        test_unit_def(small_malloc_test),
+        test_unit_def(large_malloc_test),
+        test_unit_def(free_test),
+        test_unit_def(realloc_test),
+    };
+
+    const int test_result = test_run_group(tgroup, L);
+    utils_lua_close(L);
+    return test_result;
+}
\ No newline at end of file
diff --git a/test/tarantool-c-tests/asan-mmap-instrumentation.test.c b/test/tarantool-c-tests/asan-mmap-instrumentation.test.c
new file mode 100644
index 00000000..9eabadfb
--- /dev/null
+++ b/test/tarantool-c-tests/asan-mmap-instrumentation.test.c
@@ -0,0 +1,128 @@
+#include "lua.h"
+#include "test.h"
+#include "utils.h"
+#include "lj_alloc.c"
+#include "lj_gc.h"
+
+#if LUAJIT_USE_ASAN_HARDENING
+#include <sanitizer/asan_interface.h>
+
+#define MALLOC(size) mmap_probe(size)
+#define FREE(ptr, size) CALL_MUNMAP(ptr, size)
+#define REALLOC(ptr, osz, nsz) CALL_MREMAP(ptr, osz, nsz, CALL_MREMAP_MV)
+#define IS_POISONED(ptr) __asan_address_is_poisoned(ptr)
+
+
+int IS_POISONED_REGION(void *ptr, size_t size)
+{
+	int res = 1;
+	int i = 0;
+	do {
+		res *= IS_POISONED(ptr + i);
+	} while (res == 1 && ++i < size);
+	return res;
+}
+#endif
+
+static lua_State *main_LS = NULL;
+
+static int mmap_probe_test(void *test_state)
+{
+#if !LUAJIT_USE_ASAN_HARDENING || LUAJIT_USE_SYSMALLOC
+    UNUSED(test_state);
+    return skip("Requires build with ASAN");
+#else
+	int res = -1;
+	size_t size = DEFAULT_GRANULARITY - TOTAL_REDZONE_SIZE;
+	void *p = MALLOC(size);
+	size_t algn = ALIGN_SIZE(size, SIZE_ALIGNMENT) - size;
+
+	if (p == MFAIL) {
+		perror("mmap memory allocation error");
+		return TEST_EXIT_FAILURE;
+	}
+
+	if (IS_POISONED_REGION(p - REDZONE_SIZE, REDZONE_SIZE) &&
+	    !IS_POISONED_REGION(p, size) &&
+	    IS_POISONED_REGION(p + size, algn + REDZONE_SIZE))
+		res = TEST_EXIT_SUCCESS;
+	else
+		perror("Not correct poison and unpoison areas");
+	FREE(p, size);
+	return res == TEST_EXIT_SUCCESS ? TEST_EXIT_SUCCESS : TEST_EXIT_FAILURE;
+#endif
+}
+
+static int munmap_test(void *test_state)
+{
+#if !LUAJIT_USE_ASAN_HARDENING || LUAJIT_USE_SYSMALLOC
+    UNUSED(test_state);
+    return skip("Requires build with ASAN");
+#else
+	size_t size = DEFAULT_GRANULARITY - TOTAL_REDZONE_SIZE;
+	size_t algn = ALIGN_SIZE(size, SIZE_ALIGNMENT) - size;
+	void *p = MALLOC(size);
+
+	if (p == MFAIL) {
+		perror("mmap memory allocation error");
+		return TEST_EXIT_FAILURE;
+	}
+
+	FREE(p, size);
+	if (IS_POISONED_REGION(p - REDZONE_SIZE, TOTAL_REDZONE_SIZE + size + algn))
+		return TEST_EXIT_SUCCESS;
+	perror("Not correct poison and unpoison areas");
+	return TEST_EXIT_FAILURE;
+#endif
+}
+
+static int mremap_test(void *test_state)
+{
+#if !LUAJIT_USE_ASAN_HARDENING || LUAJIT_USE_SYSMALLOC
+    UNUSED(test_state);
+    return skip("Requires build with ASAN");
+#else
+	int res = -1;
+	size_t size = (DEFAULT_GRANULARITY >> 2) - TOTAL_REDZONE_SIZE;
+	size_t new_size = (DEFAULT_GRANULARITY >> 1) - TOTAL_REDZONE_SIZE;
+	void *p = MALLOC(size);
+
+	if (p == MFAIL) {
+		perror("mmap memory allocation error");
+		return TEST_EXIT_FAILURE;
+	}
+
+	void *newptr = REALLOC(p, size, new_size);
+	if (newptr == MFAIL) {
+		perror("mremap return MFAIL");
+		FREE(p, size);
+		return TEST_EXIT_FAILURE;
+	}
+
+	if (IS_POISONED_REGION(newptr - REDZONE_SIZE, REDZONE_SIZE) &&
+	    !IS_POISONED_REGION(newptr, new_size) &&
+	    IS_POISONED_REGION(newptr + new_size, REDZONE_SIZE))
+		res = TEST_EXIT_SUCCESS;
+	else
+		perror("Not correct poison and unpoison areas");
+
+	FREE(newptr, new_size);
+	return res == TEST_EXIT_SUCCESS ? TEST_EXIT_SUCCESS : TEST_EXIT_FAILURE;
+#endif
+}
+
+int main(void)
+{
+	lua_State *L = utils_lua_init();
+	main_LS = L;
+
+	const struct test_unit tgroup[] = {
+		test_unit_def(mmap_probe_test),
+		test_unit_def(munmap_test),
+		test_unit_def(mremap_test)
+	};
+
+	const int test_result = test_run_group(tgroup, L);
+	utils_lua_close(L);
+	return test_result;
+}
diff --git a/test/tarantool-tests/ASAN-left-memory-miss.test.lua b/test/tarantool-tests/ASAN-left-memory-miss.test.lua
new file mode 100644
index 00000000..94673a88
--- /dev/null
+++ b/test/tarantool-tests/ASAN-left-memory-miss.test.lua
@@ -0,0 +1,12 @@
+local tap = require('tap')
+local asan_hardening = os.getenv("LJ_ASAN_HARDENING")
+local test = tap.test('asan-left-memory-miss'):skipcond({
+  ['Test requires ASAN-HARDENING enabled'] = asan_hardening == 'OFF',
+})
+test:plan(1)
+
+local script = require('utils').exec.makecmd(arg, { redirect = '2>&1' })
+local output, status = script()
+
+test:ok(status == 1, output)
+test:done(true)
\ No newline at end of file
diff --git a/test/tarantool-tests/ASAN-left-memory-miss/script.lua b/test/tarantool-tests/ASAN-left-memory-miss/script.lua
new file mode 100644
index 00000000..45cfad6e
--- /dev/null
+++ b/test/tarantool-tests/ASAN-left-memory-miss/script.lua
@@ -0,0 +1,19 @@
+local ffi = require('ffi')
+
+ffi.cdef[[
+typedef struct {
+    uint8_t a;
+    uint8_t b;
+} TestStruct;
+]]
+
+local obj = ffi.new("TestStruct")
+local intPtr = ffi.cast("uint64_t *", obj)
+
+-- mem dump: ... RZ 00 00 02 RZ ... \\ RZ - redzone
+--               ^        ^
+--               |        |
+--             ptr[-3]   ptr
+
+-- error (left RZ)
+print(intPtr[-3])
\ No newline at end of file
diff --git a/test/tarantool-tests/ASAN-right-memory-miss.test.lua b/test/tarantool-tests/ASAN-right-memory-miss.test.lua
new file mode 100644
index 00000000..965a576b
--- /dev/null
+++ b/test/tarantool-tests/ASAN-right-memory-miss.test.lua
@@ -0,0 +1,12 @@
+local tap = require('tap')
+local asan_hardening = os.getenv("LJ_ASAN_HARDENING")
+local test = tap.test('asan-right-memory-miss'):skipcond({
+  ['Test requires ASAN-HARDENING enabled'] = asan_hardening == 'OFF',
+})
+test:plan(1)
+
+local script = require('utils').exec.makecmd(arg, { redirect = '2>&1' })
+local output, status = script()
+
+test:ok(status == 1, output)
+test:done(true)
\ No newline at end of file
diff --git a/test/tarantool-tests/ASAN-right-memory-miss/script.lua b/test/tarantool-tests/ASAN-right-memory-miss/script.lua
new file mode 100644
index 00000000..a0133524
--- /dev/null
+++ b/test/tarantool-tests/ASAN-right-memory-miss/script.lua
@@ -0,0 +1,19 @@
+local ffi = require('ffi')
+
+ffi.cdef[[
+typedef struct {
+    uint8_t a;
+    uint8_t b;
+} TestStruct;
+]]
+
+local obj = ffi.new("TestStruct")
+local intPtr = ffi.cast("uint64_t *", obj)
+
+-- mem dump: ... RZ 00 00 02 RZ ... \\ RZ - redzone
+--                        ^
+--                        |
+--                       ptr
+
+-- error (right RZ)
+print(intPtr[2])
\ No newline at end of file
diff --git a/test/tarantool-tests/ASAN-use-after-free.test.lua b/test/tarantool-tests/ASAN-use-after-free.test.lua
new file mode 100644
index 00000000..413e4a57
--- /dev/null
+++ b/test/tarantool-tests/ASAN-use-after-free.test.lua
@@ -0,0 +1,12 @@
+local tap = require('tap')
+local asan_hardening = os.getenv("LJ_ASAN_HARDENING")
+local test = tap.test('asan-use-after-free'):skipcond({
+  ['Test requires ASAN-HARDENING enabled'] = asan_hardening == 'OFF',
+})
+test:plan(1)
+
+local script = require('utils').exec.makecmd(arg, { redirect = '2>&1' })
+local output, status = script()
+
+test:ok(status == 1, output)
+test:done(true)
\ No newline at end of file
diff --git a/test/tarantool-tests/ASAN-use-after-free/script.lua b/test/tarantool-tests/ASAN-use-after-free/script.lua
new file mode 100644
index 00000000..ffbdaf79
--- /dev/null
+++ b/test/tarantool-tests/ASAN-use-after-free/script.lua
@@ -0,0 +1,24 @@
+local ffi = require('ffi')
+
+ffi.cdef[[
+typedef struct {
+    uint8_t a;
+    uint8_t b;
+} TestStruct;
+]]
+
+local obj = ffi.new("TestStruct")
+obj = ffi.gc(obj, nil)
+local intPtr = ffi.cast("uint8_t *", obj)
+obj = nil
+collectgarbage("collect")
+print(obj)
+
+-- after free
+-- mem dump: ... RZ RZ RZ RZ RZ ... \\ RZ - redzone
+--                        ^
+--                        |
+--                       ptr
+
+-- error
+print(intPtr[0])
diff --git a/test/tarantool-tests/CMakeLists.txt b/test/tarantool-tests/CMakeLists.txt
index e3750bf3..a1159ec2 100644
--- a/test/tarantool-tests/CMakeLists.txt
+++ b/test/tarantool-tests/CMakeLists.txt
@@ -130,7 +130,7 @@ foreach(test_path ${tests})
     # LUA_CPATH and LD_LIBRARY_PATH variables and also
     # dependencies list with libraries are set in scope of
     # BuildTestLib macro.
-    ENVIRONMENT "LUA_PATH=${LUA_PATH};LUA_CPATH=${LUA_CPATH};${LUA_TEST_ENV_MORE}"
+    ENVIRONMENT "LUA_PATH=${LUA_PATH};LUA_CPATH=${LUA_CPATH};${LUA_TEST_ENV_MORE};LJ_ASAN_HARDENING=${LUAJIT_USE_ASAN_HARDENING}"
     LABELS ${TEST_SUITE_NAME}
     DEPENDS tarantool-tests-deps
   )
diff --git a/test/tarantool-tests/lj-1034-tabov-error-frame.test.lua b/test/tarantool-tests/lj-1034-tabov-error-frame.test.lua
index 0e23fdb2..6925f0ab 100644
--- a/test/tarantool-tests/lj-1034-tabov-error-frame.test.lua
+++ b/test/tarantool-tests/lj-1034-tabov-error-frame.test.lua
@@ -1,9 +1,11 @@
 local tap = require('tap')
 local ffi = require('ffi')
+local asan_hardening = os.getenv("LJ_ASAN_HARDENING")
 local test = tap.test('lj-1034-tabov-error-frame'):skipcond({
   ['Test requires JIT enabled'] = not jit.status(),
   ['Test requires GC64 mode enabled'] = not ffi.abi('gc64'),
   ['Disabled on MacOS due to #8652'] = jit.os == 'OSX',
+  ['Test requires ASAN-HARDENING disabled'] = asan_hardening == 'ON',
 })
 
 -- XXX: The test for the problem uses the table of GC
-- 
2.34.1



More information about the Tarantool-patches mailing list