Re: [Tarantool-patches] [PATCH v2 luajit 6/7] test: skip flaky tests with enabled table bump

[Sergey Bronnikov via Tarantool-patches <tarantool-patches@dev.tarantool.org>]

[-- Attachment #1: Type: text/plain, Size: 11205 bytes --]

Hi, Sergey,

test statuses from TAP report produced by tests is not propagated to CTest.

I propose to disable these tests in CTest:


--- a/test/tarantool-tests/CMakeLists.txt
+++ b/test/tarantool-tests/CMakeLists.txt
@@ -166,3 +167,14 @@ if(LUAJIT_USE_ASAN AND CMAKE_C_COMPILER_ID STREQUAL 
"GNU")
      LD_PRELOAD ${LIB_ASAN}
    )
  endif()
+
+set_tests_properties(
+  test/tarantool-tests/profilers/gh-5688-tool-cli-flag.test.lua
+ test/tarantool-tests/profilers/gh-5813-resolving-of-c-symbols.test.lua
+ test/tarantool-tests/profilers/gh-5994-memprof-human-readable.test.lua
+ 
test/tarantool-tests/profilers/gh-7264-add-proto-trace-sysprof-default.test.lua
+ 
test/tarantool-tests/profilers/gh-9217-profile-parsers-error-handling.test.lua
+  test/tarantool-tests/profilers/misclib-memprof-lapi.test.lua
+  test/tarantool-tests/profilers/misclib-sysprof-lapi.test.lua
+  PROPERTIES DISABLED TRUE
+)


CTest will always report that these tests are disabled and we will never 
forget to enable them back:


100% tests passed, 0 tests failed out of 158

Label Time Summary:
tarantool-tests    =  14.66 sec*proc (165 tests)

Total Test time (real) =  14.73 sec

The following tests did not run:
         222 - 
test/tarantool-tests/profilers/gh-5688-tool-cli-flag.test.lua (Disabled)
         223 - 
test/tarantool-tests/profilers/gh-5813-resolving-of-c-symbols.test.lua 
(Disabled)
         224 - 
test/tarantool-tests/profilers/gh-5994-memprof-human-readable.test.lua 
(Disabled)
         225 - 
test/tarantool-tests/profilers/gh-7264-add-proto-trace-sysprof-default.test.lua 
(Disabled)
         226 - 
test/tarantool-tests/profilers/gh-9217-profile-parsers-error-handling.test.lua 
(Disabled)
         227 - 
test/tarantool-tests/profilers/misclib-memprof-lapi.test.lua (Disabled)
         228 - 
test/tarantool-tests/profilers/misclib-sysprof-lapi.test.lua (Disabled)


What do you think?


On 24.09.2024 13:29, Sergey Kaplun wrote:
> Profilers tests with LUAJIT_ENABLE_TABLE_BUMP are flaky due to the bug
> in the TDUP patching. Disable them for now.
> ---
>   test/tarantool-tests/CMakeLists.txt              |  6 ++++++
>   .../profilers/gh-5688-tool-cli-flag.test.lua     |  2 ++
>   .../gh-5813-resolving-of-c-symbols.test.lua      |  2 ++
>   .../gh-5994-memprof-human-readable.test.lua      |  2 ++
>   ...7264-add-proto-trace-sysprof-default.test.lua |  2 ++
>   ...-9217-profile-parsers-error-handling.test.lua |  2 ++
>   .../profilers/misclib-memprof-lapi.test.lua      | 16 +++++++++-------
>   .../profilers/misclib-sysprof-lapi.test.lua      |  2 ++
>   8 files changed, 27 insertions(+), 7 deletions(-)
>
> diff --git a/test/tarantool-tests/CMakeLists.txt b/test/tarantool-tests/CMakeLists.txt
> index d0964aed..a1aee3a2 100644
> --- a/test/tarantool-tests/CMakeLists.txt
> +++ b/test/tarantool-tests/CMakeLists.txt
> @@ -97,6 +97,12 @@ if(LUAJIT_USE_ASAN AND CMAKE_C_COMPILER_ID STREQUAL "GNU")
>     list(APPEND LUA_TEST_ENV_MORE LD_PRELOAD=${LIB_ASAN})
>   endif()
>   
> +# FIXME: This is needed for disabling some flaky tests (like
> +# profilers), until LuaJIT/LuaJIT#606 will not be resolved.
> +if(LUAJIT_ENABLE_TABLE_BUMP)
> +  list(APPEND LUA_TEST_ENV_MORE LUAJIT_TABLE_BUMP=1)
> +endif()
> +
>   set(TEST_SUITE_NAME "tarantool-tests")
>   
>   # XXX: The call produces both test and target
> diff --git a/test/tarantool-tests/profilers/gh-5688-tool-cli-flag.test.lua b/test/tarantool-tests/profilers/gh-5688-tool-cli-flag.test.lua
> index dd0fd260..f935dc5b 100644
> --- a/test/tarantool-tests/profilers/gh-5688-tool-cli-flag.test.lua
> +++ b/test/tarantool-tests/profilers/gh-5688-tool-cli-flag.test.lua
> @@ -5,6 +5,8 @@ local test = tap.test('gh-5688-tool-cli-flag'):skipcond({
>     ['Profile tools are implemented for Linux only'] = jit.os ~= 'Linux',
>     -- XXX: Tarantool integration is required to run this test properly.
>     ['No profile tools CLI option integration'] = _TARANTOOL,
> +  -- See alsohttps://github.com/LuaJIT/LuaJIT/issues/606.
> +  ['Disabled due to LuaJIT/LuaJIT#606'] = os.getenv('LUAJIT_TABLE_BUMP'),
>   })
>   
>   test:plan(3)
> diff --git a/test/tarantool-tests/profilers/gh-5813-resolving-of-c-symbols.test.lua b/test/tarantool-tests/profilers/gh-5813-resolving-of-c-symbols.test.lua
> index 1581ee4b..74bcd9e8 100644
> --- a/test/tarantool-tests/profilers/gh-5813-resolving-of-c-symbols.test.lua
> +++ b/test/tarantool-tests/profilers/gh-5813-resolving-of-c-symbols.test.lua
> @@ -3,6 +3,8 @@ local test = tap.test("gh-5813-resolving-of-c-symbols"):skipcond({
>     ["Memprof is implemented for x86_64 only"] = jit.arch ~= "x86" and
>                                                  jit.arch ~= "x64",
>     ["Memprof is implemented for Linux only"] = jit.os ~= "Linux",
> +  -- See alsohttps://github.com/LuaJIT/LuaJIT/issues/606.
> +  ["Disabled due to LuaJIT/LuaJIT#606"] = os.getenv("LUAJIT_TABLE_BUMP"),
>   })
>   
>   test:plan(5)
> diff --git a/test/tarantool-tests/profilers/gh-5994-memprof-human-readable.test.lua b/test/tarantool-tests/profilers/gh-5994-memprof-human-readable.test.lua
> index 7c3ff94d..f3041779 100644
> --- a/test/tarantool-tests/profilers/gh-5994-memprof-human-readable.test.lua
> +++ b/test/tarantool-tests/profilers/gh-5994-memprof-human-readable.test.lua
> @@ -5,6 +5,8 @@ local test = tap.test('gh-5994-memprof-human-readable'):skipcond({
>     ['Profile tools are implemented for Linux only'] = jit.os ~= 'Linux',
>     -- XXX: Tarantool integration is required to run this test properly.
>     ['No profile tools CLI option integration'] = _TARANTOOL,
> +  -- See alsohttps://github.com/LuaJIT/LuaJIT/issues/606.
> +  ['Disabled due to LuaJIT/LuaJIT#606'] = os.getenv('LUAJIT_TABLE_BUMP'),
>   })
>   
>   local utils = require('utils')
> diff --git a/test/tarantool-tests/profilers/gh-7264-add-proto-trace-sysprof-default.test.lua b/test/tarantool-tests/profilers/gh-7264-add-proto-trace-sysprof-default.test.lua
> index c1d68e3c..176c1a15 100644
> --- a/test/tarantool-tests/profilers/gh-7264-add-proto-trace-sysprof-default.test.lua
> +++ b/test/tarantool-tests/profilers/gh-7264-add-proto-trace-sysprof-default.test.lua
> @@ -4,6 +4,8 @@ local test = tap.test('gh-7264-add-proto-trace-sysprof-default'):skipcond({
>     ['Sysprof is implemented for x86_64 only'] = jit.arch ~= 'x86' and
>                                                  jit.arch ~= 'x64',
>     ['Sysprof is implemented for Linux only'] = jit.os ~= 'Linux',
> +  -- See alsohttps://github.com/LuaJIT/LuaJIT/issues/606.
> +  ['Disabled due to LuaJIT/LuaJIT#606'] = os.getenv('LUAJIT_TABLE_BUMP'),
>   })
>   
>   test:plan(2)
> diff --git a/test/tarantool-tests/profilers/gh-9217-profile-parsers-error-handling.test.lua b/test/tarantool-tests/profilers/gh-9217-profile-parsers-error-handling.test.lua
> index 92f9f59d..65c51198 100644
> --- a/test/tarantool-tests/profilers/gh-9217-profile-parsers-error-handling.test.lua
> +++ b/test/tarantool-tests/profilers/gh-9217-profile-parsers-error-handling.test.lua
> @@ -5,6 +5,8 @@ local test = tap.test('gh-9217-profile-parsers-error-handling'):skipcond({
>     ['Profile tools are implemented for Linux only'] = jit.os ~= 'Linux',
>     -- XXX: Tarantool integration is required to run this test properly.
>     ['No profile tools CLI option integration'] = _TARANTOOL,
> +  -- See alsohttps://github.com/LuaJIT/LuaJIT/issues/606.
> +  ['Disabled due to LuaJIT/LuaJIT#606'] = os.getenv('LUAJIT_TABLE_BUMP'),
>   })
>   
>   jit.off()
> diff --git a/test/tarantool-tests/profilers/misclib-memprof-lapi.test.lua b/test/tarantool-tests/profilers/misclib-memprof-lapi.test.lua
> index 728a7ab7..ce41e4d5 100644
> --- a/test/tarantool-tests/profilers/misclib-memprof-lapi.test.lua
> +++ b/test/tarantool-tests/profilers/misclib-memprof-lapi.test.lua
> @@ -6,6 +6,8 @@ local test = tap.test("misc-memprof-lapi"):skipcond({
>     ['Disabled on *BSD due to #4819'] = jit.os == 'BSD',
>     ["Memprof is implemented for x86_64 only"] = jit.arch ~= "x86" and
>                                                  jit.arch ~= "x64",
> +  -- See alsohttps://github.com/LuaJIT/LuaJIT/issues/606.
> +  ["Disabled due to LuaJIT/LuaJIT#606"] = os.getenv("LUAJIT_TABLE_BUMP"),
>   })
>   
>   test:plan(5)
> @@ -162,9 +164,9 @@ test:test("output", function(subtest)
>     -- one is the number of allocations. 1 event - allocation of
>     -- table by itself + 1 allocation of array part as far it is
>     -- bigger than LJ_MAX_COLOSIZE (16).
> -  subtest:ok(check_alloc_report(alloc, { line = 37, linedefined = 35 }, 2))
> +  subtest:ok(check_alloc_report(alloc, { line = 39, linedefined = 37 }, 2))
>     -- 20 strings allocations.
> -  subtest:ok(check_alloc_report(alloc, { line = 42, linedefined = 35 }, 20))
> +  subtest:ok(check_alloc_report(alloc, { line = 44, linedefined = 37 }, 20))
>   
>     -- Collect all previous allocated objects.
>     subtest:ok(free.INTERNAL == 22)
> @@ -172,8 +174,8 @@ test:test("output", function(subtest)
>     -- Tests for leak-only option.
>     -- See alsohttps://github.com/tarantool/tarantool/issues/5812.
>     local heap_delta = process.form_heap_delta(events)
> -  local tab_alloc_stats = heap_delta[form_source_line(37)]
> -  local str_alloc_stats = heap_delta[form_source_line(42)]
> +  local tab_alloc_stats = heap_delta[form_source_line(39)]
> +  local str_alloc_stats = heap_delta[form_source_line(44)]
>     subtest:ok(tab_alloc_stats.nalloc == tab_alloc_stats.nfree)
>     subtest:ok(tab_alloc_stats.dbytes == 0)
>     subtest:ok(str_alloc_stats.nalloc == str_alloc_stats.nfree)
> @@ -258,10 +260,10 @@ test:test("jit-output", function(subtest)
>     -- 10 allocations in interpreter mode, 1 allocation for a trace
>     -- recording and assembling and next 9 allocations will happen
>     -- while running the trace.
> -  subtest:ok(check_alloc_report(alloc, { line = 42, linedefined = 35 }, 11))
> -  subtest:ok(check_alloc_report(alloc, { traceno = 1, line = 40 }, 9))
> +  subtest:ok(check_alloc_report(alloc, { line = 44, linedefined = 37 }, 11))
> +  subtest:ok(check_alloc_report(alloc, { traceno = 1, line = 42 }, 9))
>     -- See same checks with jit.off().
> -  subtest:ok(check_alloc_report(alloc, { line = 37, linedefined = 35 }, 2))
> +  subtest:ok(check_alloc_report(alloc, { line = 39, linedefined = 37 }, 2))
>   
>     -- Restore default JIT settings.
>     jit.opt.start(unpack(jit_opt_default))
> diff --git a/test/tarantool-tests/profilers/misclib-sysprof-lapi.test.lua b/test/tarantool-tests/profilers/misclib-sysprof-lapi.test.lua
> index fdaed46a..26c277cd 100644
> --- a/test/tarantool-tests/profilers/misclib-sysprof-lapi.test.lua
> +++ b/test/tarantool-tests/profilers/misclib-sysprof-lapi.test.lua
> @@ -3,6 +3,8 @@ local test = tap.test("misc-sysprof-lapi"):skipcond({
>     ["Sysprof is implemented for x86_64 only"] = jit.arch ~= "x86" and
>                                                  jit.arch ~= "x64",
>     ["Sysprof is implemented for Linux only"] = jit.os ~= "Linux",
> +  -- See alsohttps://github.com/LuaJIT/LuaJIT/issues/606.
> +  ["Disabled due to LuaJIT/LuaJIT#606"] = os.getenv("LUAJIT_TABLE_BUMP"),
>   })
>   
>   test:plan(19)

[-- Attachment #2: Type: text/html, Size: 12579 bytes --]