[Tarantool-patches] [PATCH luajit 2/3] sysprof: fix crash on a failed stream flush

[Sergey Bronnikov via Tarantool-patches <tarantool-patches@dev.tarantool.org>]

From: Sergey Bronnikov <sergeyb@tarantool.org>

A writer may release the stream buffer and set its pointer to NULL
on an unrecoverable error, e.g., when the file system with the
profile is full. The write buffer stops the stream in this case,
but a caller of `wbuf_reserve()` still writes data using the
released buffer. This leads to a null pointer dereference inside
`lj_utils_write_uleb128()` when the flush is triggered while
streaming a sample.

Now `wbuf_reserve()` reports whether the stream was stopped during
the flush. All the write functions skip writing in this case,
including the loop pumping a large payload through the buffer.

Fixes tarantool/tarantool#11620
---
 src/lj_wbuf.c                                 |  20 +-
 .../gh-11620-sysprof-enospc-crash.test.c      | 180 ++++++++++++++++++
 test/tarantool-c-tests/unit-wbuf.test.c       | 102 ++++++++++
 3 files changed, 298 insertions(+), 4 deletions(-)
 create mode 100644 test/tarantool-c-tests/gh-11620-sysprof-enospc-crash.test.c

diff --git a/src/lj_wbuf.c b/src/lj_wbuf.c
index 0001a02e..0794b186 100644
--- a/src/lj_wbuf.c
+++ b/src/lj_wbuf.c
@@ -51,11 +51,18 @@ void LJ_FASTCALL lj_wbuf_terminate(struct lj_wbuf *buf)
   lj_wbuf_init(buf, NULL, NULL, NULL, 0);
 }
 
-static LJ_AINLINE void wbuf_reserve(struct lj_wbuf *buf, size_t n)
+/*
+** Reserve n bytes in the buffer. Returns non-zero if the space is reserved
+** and zero if the stream was stopped during the flush. In the latter case
+** the writer may release the buffer (buf->buf == NULL), so it must not be
+** used by the caller.
+*/
+static LJ_AINLINE int wbuf_reserve(struct lj_wbuf *buf, size_t n)
 {
   lj_assertX(n <= buf->size, "wbuf overflow");
   if (LJ_UNLIKELY(wbuf_left(buf) < n))
     lj_wbuf_flush(buf);
+  return !lj_wbuf_test_flag(buf, STREAM_STOP);
 }
 
 /* Writes a byte to the output buffer. */
@@ -63,7 +70,8 @@ void LJ_FASTCALL lj_wbuf_addbyte(struct lj_wbuf *buf, uint8_t b)
 {
   if (LJ_UNLIKELY(lj_wbuf_test_flag(buf, STREAM_STOP)))
     return;
-  wbuf_reserve(buf, sizeof(b));
+  if (LJ_UNLIKELY(!wbuf_reserve(buf, sizeof(b))))
+    return;
   *buf->pos++ = b;
 }
 
@@ -72,7 +80,8 @@ void LJ_FASTCALL lj_wbuf_addu64(struct lj_wbuf *buf, uint64_t n)
 {
   if (LJ_UNLIKELY(lj_wbuf_test_flag(buf, STREAM_STOP)))
     return;
-  wbuf_reserve(buf, LEB128_U64_MAXSIZE);
+  if (LJ_UNLIKELY(!wbuf_reserve(buf, LEB128_U64_MAXSIZE)))
+    return;
   buf->pos += (ptrdiff_t)lj_utils_write_uleb128(buf->pos, n);
 }
 
@@ -91,11 +100,14 @@ void lj_wbuf_addn(struct lj_wbuf *buf, const void *src, size_t n)
     memcpy(buf->pos, src, left);
     buf->pos += (ptrdiff_t)left;
     lj_wbuf_flush(buf);
+    if (LJ_UNLIKELY(lj_wbuf_test_flag(buf, STREAM_STOP)))
+      return;
     src = (uint8_t *)src + (ptrdiff_t)left;
     n -= left;
   }
 
-  wbuf_reserve(buf, n);
+  if (LJ_UNLIKELY(!wbuf_reserve(buf, n)))
+    return;
   memcpy(buf->pos, src, n);
   buf->pos += (ptrdiff_t)n;
 }
diff --git a/test/tarantool-c-tests/gh-11620-sysprof-enospc-crash.test.c b/test/tarantool-c-tests/gh-11620-sysprof-enospc-crash.test.c
new file mode 100644
index 00000000..0f5ffe53
--- /dev/null
+++ b/test/tarantool-c-tests/gh-11620-sysprof-enospc-crash.test.c
@@ -0,0 +1,180 @@
+/*
+** Testing of the sysprof behaviour when a stream write fails.
+**
+** A full file system makes the sysprof writer return an error and
+** release the stream buffer. The profiler must stop the stream
+** gracefully, return an error from luaM_sysprof_stop() and
+** release the resources.
+*/
+
+#include "lauxlib.h"
+#include "lua.h"
+
+/* Need for skipcond for OS and ARCH. */
+#include "lj_arch.h"
+
+#include "test.h"
+
+#if LJ_HASSYSPROF && !defined(LUAJIT_USE_VALGRIND)
+
+#include "lmisclib.h"
+#include "utils.h"
+
+#include <errno.h>
+#include <signal.h>
+#include <stdint.h>
+#include <string.h>
+#include <unistd.h>
+
+/*
+ * The buffer size is enough for the maximal ULEB128 value.
+ * A small buffer makes the profiler flush it right during the
+ * prologue or the first sample.
+ */
+#define STREAM_BUFFER_SIZE 10
+
+/* Do not let the timer interfere with the manually emitted signals. */
+#define SYSPROF_TEST_INTERVAL 1000
+
+struct sysprof_ctx {
+	/* Simulate write errors. */
+	int fail;
+	/* Number of on_stop callback calls. */
+	int on_stop_calls;
+};
+
+static struct sysprof_ctx ctx;
+
+/*
+ * The profiler skips the frames up to
+ * SYSPROF_HANDLER_STACK_DEPTH (6) while streaming the host stack,
+ * so provide the frames after that boundary.
+ */
+#define SYSPROF_TEST_FRAME_START 7
+#define SYSPROF_TEST_FRAME_COUNT 4
+
+static size_t failing_writer(const void **buf_addr, size_t len, void *opt)
+{
+	struct sysprof_ctx *ctx = opt;
+
+	if (ctx->fail) {
+		errno = ENOSPC;
+		*buf_addr = NULL;
+		return 0;
+	}
+
+	return len;
+}
+
+static int on_stop(void *opt, uint8_t *buf)
+{
+	struct sysprof_ctx *ctx = opt;
+
+	UNUSED(buf);
+	ctx->on_stop_calls++;
+	return 0;
+}
+
+static void backtracer(void *(*frame_writer)(int, void *))
+{
+	int frame_no;
+
+	for (frame_no = SYSPROF_TEST_FRAME_START;
+	     frame_no < SYSPROF_TEST_FRAME_START + SYSPROF_TEST_FRAME_COUNT;
+	     ++frame_no)
+		frame_writer(frame_no, (void *)(uintptr_t)frame_no);
+}
+
+static void sysprof_test_configure(struct luam_Sysprof_Options *opt,
+				   uint8_t *buf)
+{
+	memset(&ctx, 0, sizeof(ctx));
+	memset(opt, 0, sizeof(*opt));
+	opt->mode = LUAM_SYSPROF_CALLGRAPH;
+	opt->interval = SYSPROF_TEST_INTERVAL;
+	opt->buf = buf;
+	opt->len = STREAM_BUFFER_SIZE;
+	opt->ctx = &ctx;
+
+	assert_true(luaM_sysprof_set_writer(failing_writer) == PROFILE_SUCCESS);
+	assert_true(luaM_sysprof_set_on_stop(on_stop) == PROFILE_SUCCESS);
+	assert_true(luaM_sysprof_set_backtracer(backtracer) == PROFILE_SUCCESS);
+}
+
+static int sysprof_enospc_during_sampling(void *test_state)
+{
+	lua_State *L = test_state;
+	static uint8_t buf[STREAM_BUFFER_SIZE];
+	struct luam_Sysprof_Options opt;
+	struct luam_Sysprof_Counters counters = {};
+
+	sysprof_test_configure(&opt, buf);
+	assert_true(luaM_sysprof_start(L, &opt) == PROFILE_SUCCESS);
+	assert_int_equal(ctx.on_stop_calls, 0);
+
+	/* Let the next stream flush fail as if the disk is full. */
+	ctx.fail = 1;
+	assert_int_equal(kill(getpid(), SIGPROF), 0);
+
+	/* The stream is stopped gracefully, without a crash. */
+	errno = 0;
+	assert_int_equal(luaM_sysprof_stop(L), PROFILE_ERRIO);
+	assert_int_equal(errno, ENOSPC);
+
+	/* Counters are still available. */
+	assert_true(luaM_sysprof_report(&counters) == PROFILE_SUCCESS);
+	assert_true(counters.samples >= 1);
+
+	/* The profiler is stopped. */
+	assert_int_equal(luaM_sysprof_stop(L), PROFILE_ERRRUN);
+
+	return TEST_EXIT_SUCCESS;
+}
+
+static int sysprof_enospc_on_start(void *test_state)
+{
+	lua_State *L = test_state;
+	static uint8_t buf[STREAM_BUFFER_SIZE];
+	struct luam_Sysprof_Options opt;
+
+	sysprof_test_configure(&opt, buf);
+	/* The very first flush fails, so the prologue cannot be streamed. */
+	ctx.fail = 1;
+	errno = 0;
+	assert_int_equal(luaM_sysprof_start(L, &opt), PROFILE_ERRIO);
+	assert_int_equal(errno, ENOSPC);
+	/* The start error path releases the profiler resources. */
+	assert_int_equal(ctx.on_stop_calls, 1);
+
+	/* The profiler is not running. */
+	assert_int_equal(luaM_sysprof_stop(L), PROFILE_ERRRUN);
+
+	return TEST_EXIT_SUCCESS;
+}
+
+#endif /* LJ_HASSYSPROF && !defined(LUAJIT_USE_VALGRIND) */
+
+int main(void)
+{
+#if LUAJIT_USE_VALGRIND
+	return skip_all("Disabled due to #10803");
+#elif !LJ_HASSYSPROF
+	return skip_all("Sysprof is disabled");
+#else
+	if (LUAJIT_OS != LUAJIT_OS_LINUX)
+		return skip_all("Sysprof is implemented for Linux only");
+	if (LUAJIT_TARGET != LUAJIT_ARCH_X86
+	    && LUAJIT_TARGET != LUAJIT_ARCH_X64)
+		return skip_all("Sysprof is implemented for x86_64 only");
+
+	lua_State *L = utils_lua_init();
+
+	const struct test_unit tgroup[] = {
+		test_unit_def(sysprof_enospc_during_sampling),
+		test_unit_def(sysprof_enospc_on_start)
+	};
+	const int test_result = test_run_group(tgroup, L);
+	utils_lua_close(L);
+	return test_result;
+#endif /* LUAJIT_USE_VALGRIND */
+}
diff --git a/test/tarantool-c-tests/unit-wbuf.test.c b/test/tarantool-c-tests/unit-wbuf.test.c
index 4bee6243..c1b71ff3 100644
--- a/test/tarantool-c-tests/unit-wbuf.test.c
+++ b/test/tarantool-c-tests/unit-wbuf.test.c
@@ -367,6 +367,104 @@ static int test_terminate(void *state)
 ** this case, so the write functions must not touch the released
 ** buffer.
 */
+static int test_addbyte_failed_flush(void *state)
+{
+	uint8_t mem[BUFFER_SIZE];
+	struct lj_wbuf buf;
+	struct writer_ctx ctx;
+	size_t i;
+
+	UNUSED(state);
+
+	writer_ctx_init(&ctx, WMODE_FAIL);
+	lj_wbuf_init(&buf, test_writer, &ctx, mem, sizeof(mem));
+
+	/* Fill the buffer without flushes. */
+	for (i = 0; i < sizeof(mem); ++i)
+		lj_wbuf_addbyte(&buf, (uint8_t)i);
+	assert_sizet_equal(ctx.flushes, 0);
+	assert_sizet_equal(buf_len(&buf), sizeof(mem));
+
+	/* The buffer is full, so the byte below triggers a failed flush. */
+	lj_wbuf_addbyte(&buf, 0xaa);
+	assert_stream_stopped_with_error(&buf);
+	assert_sizet_equal(ctx.flushes, 1);
+
+	return TEST_EXIT_SUCCESS;
+}
+
+static int test_addu64_failed_flush(void *state)
+{
+	uint8_t mem[BUFFER_SIZE];
+	struct lj_wbuf buf;
+	struct writer_ctx ctx;
+
+	UNUSED(state);
+
+	writer_ctx_init(&ctx, WMODE_FAIL);
+	lj_wbuf_init(&buf, test_writer, &ctx, mem, sizeof(mem));
+
+	/* The maximal value takes exactly BUFFER_SIZE bytes. */
+	lj_wbuf_addu64(&buf, UINT64_MAX);
+	assert_sizet_equal(buf_len(&buf), sizeof(mem));
+
+	/* There is no space left, so the next write triggers a failed flush. */
+	lj_wbuf_addu64(&buf, 0);
+	assert_stream_stopped_with_error(&buf);
+	assert_sizet_equal(ctx.flushes, 1);
+
+	return TEST_EXIT_SUCCESS;
+}
+
+static int test_addn_failed_flush(void *state)
+{
+	uint8_t mem[BUFFER_SIZE];
+	uint8_t payload[BUFFER_SIZE];
+	struct lj_wbuf buf;
+	struct writer_ctx ctx;
+
+	UNUSED(state);
+
+	memset(payload, 'a', sizeof(payload));
+	writer_ctx_init(&ctx, WMODE_FAIL);
+	lj_wbuf_init(&buf, test_writer, &ctx, mem, sizeof(mem));
+
+	/* Fill the buffer without flushes. */
+	lj_wbuf_addn(&buf, payload, sizeof(payload));
+	assert_sizet_equal(ctx.flushes, 0);
+
+	/* There is no space left, so the next write triggers a failed flush. */
+	lj_wbuf_addn(&buf, "b", 1);
+	assert_stream_stopped_with_error(&buf);
+	assert_sizet_equal(ctx.flushes, 1);
+
+	return TEST_EXIT_SUCCESS;
+}
+
+static int test_addn_failed_flush_in_loop(void *state)
+{
+	uint8_t mem[BUFFER_SIZE];
+	uint8_t payload[PAYLOAD_SIZE];
+	struct lj_wbuf buf;
+	struct writer_ctx ctx;
+
+	UNUSED(state);
+
+	memset(payload, 'a', sizeof(payload));
+	writer_ctx_init(&ctx, WMODE_FAIL);
+	lj_wbuf_init(&buf, test_writer, &ctx, mem, sizeof(mem));
+
+	/*
+	 * The payload does not fit into the buffer, so it is pumped through
+	 * it. The flush fails in the middle of the pump loop.
+	 */
+	lj_wbuf_addn(&buf, payload, sizeof(payload));
+	assert_stream_stopped_with_error(&buf);
+	assert_sizet_equal(ctx.flushes, 1);
+
+	return TEST_EXIT_SUCCESS;
+}
+
 int main(void)
 {
 	const struct test_unit tgroup[] = {
@@ -381,6 +479,10 @@ int main(void)
 		test_unit_def(test_flush_stopped_stream),
 		test_unit_def(test_write_to_stopped_stream),
 		test_unit_def(test_terminate),
+		test_unit_def(test_addbyte_failed_flush),
+		test_unit_def(test_addu64_failed_flush),
+		test_unit_def(test_addn_failed_flush),
+		test_unit_def(test_addn_failed_flush_in_loop),
 	};
 	const int test_result = test_run_group(tgroup, NULL);
 
-- 
2.56.0