[PATCH 2/2] lib/core/fiber: Allow to extend default stack size

[Cyrill Gorcunov]

Recently we found that 64K stack we've been using
for years is not longer enough due to third party
libraries, so we extended its size.

Still there is no guarantee that even new size will
lasts forever and users might face same problem again,
waiting for new build where stack capacity increased.

Instead of this ping-pong game lets allow a user to
setup default stack size via FIBER_STACK_SIZE_DEFAULT
environment variable.

Note we fetch this value once on program startup and
because it is early init stage we can't use box
config engine.
---
 src/lib/core/fiber.c | 78 ++++++++++++++++++++++++++++++++++++++------
 1 file changed, 68 insertions(+), 10 deletions(-)

diff --git a/src/lib/core/fiber.c b/src/lib/core/fiber.c
index 922a0bfe8..d3cb18a15 100644
--- a/src/lib/core/fiber.c
+++ b/src/lib/core/fiber.c
@@ -107,18 +107,25 @@ pthread_t main_thread_id;
 static size_t page_size;
 static bool stack_growsdown;
 
+/** Indices in fiber stack sizes (FSS) */
 enum {
+	FSS_IDX_MINIMAL,
+	FSS_IDX_WATERMARK,
+	FSS_IDX_DEFAULT,
+};
+
+/** Fiber stack sizes */
+static size_t fiber_stack_size[] = {
 	/* The minimum allowable fiber stack size in bytes */
-	FIBER_STACK_SIZE_MINIMAL = 16384,
+	[FSS_IDX_MINIMAL]	= 16384,
+	/* Stack size watermark in bytes */
+	[FSS_IDX_WATERMARK]	= 65536,
 	/* Default fiber stack size in bytes */
-	FIBER_STACK_SIZE_DEFAULT = 524288,
-	/* Stack size watermark in bytes. */
-	FIBER_STACK_SIZE_WATERMARK = 65536,
+	[FSS_IDX_DEFAULT]	= 524288,
 };
 
 /** Default fiber attributes */
-static const struct fiber_attr fiber_attr_default = {
-       .stack_size = FIBER_STACK_SIZE_DEFAULT,
+static struct fiber_attr fiber_attr_default = {
        .flags = FIBER_DEFAULT_FLAGS
 };
 
@@ -172,13 +179,13 @@ fiber_attr_delete(struct fiber_attr *fiber_attr)
 int
 fiber_attr_setstacksize(struct fiber_attr *fiber_attr, size_t stack_size)
 {
-	if (stack_size < FIBER_STACK_SIZE_MINIMAL) {
+	if (stack_size < fiber_stack_size[FSS_IDX_MINIMAL]) {
 		errno = EINVAL;
 		diag_set(SystemError, "stack size is too small");
 		return -1;
 	}
 	fiber_attr->stack_size = stack_size;
-	if (stack_size != FIBER_STACK_SIZE_DEFAULT) {
+	if (stack_size != fiber_stack_size[FSS_IDX_DEFAULT]) {
 		fiber_attr->flags |= FIBER_CUSTOM_STACK;
 	} else {
 		fiber_attr->flags &= ~FIBER_CUSTOM_STACK;
@@ -844,11 +851,11 @@ fiber_stack_watermark_create(struct fiber *fiber)
 	size_t offset = rand() % POISON_OFF * sizeof(poison_pool[0]);
 	if (stack_growsdown) {
 		fiber->stack_watermark  = fiber->stack + fiber->stack_size;
-		fiber->stack_watermark -= FIBER_STACK_SIZE_WATERMARK;
+		fiber->stack_watermark -= fiber_stack_size[FSS_IDX_WATERMARK];
 		fiber->stack_watermark += offset;
 	} else {
 		fiber->stack_watermark  = fiber->stack;
-		fiber->stack_watermark += FIBER_STACK_SIZE_WATERMARK;
+		fiber->stack_watermark += fiber_stack_size[FSS_IDX_WATERMARK];
 		fiber->stack_watermark -= page_size;
 		fiber->stack_watermark += offset;
 	}
@@ -1383,6 +1390,56 @@ cord_slab_cache(void)
 	return &cord()->slabc;
 }
 
+/**
+ * Initialize early parameters of the stack
+ * and setup default variables.
+ */
+static void
+fiber_stack_init(void)
+{
+	static const char default_size_name[] = "FIBER_STACK_SIZE_DEFAULT";
+	char *env;
+	size_t i;
+
+	/*
+	 * Make sure sizes are ordered, we rely on this
+	 * to minimize rss pressure when releasing pages.
+	 */
+	for (i = 1; i < lengthof(fiber_stack_size); i++)
+		assert(fiber_stack_size[i-1] < fiber_stack_size[i]);
+
+	env = getenv(default_size_name);
+	if (env) {
+		size_t value = atol(env);
+		if (!value) {
+			say_error("invalid value in env variable %s, ignored",
+				  default_size_name);
+			goto out;
+		}
+
+		/*
+		 * Allow extend stack or shrink it down close
+		 * to watermark value. The reason why we don't
+		 * allow to shrink down to minimal stack size
+		 * is our shrinker engine which we try to keep
+		 * as simple as possible.
+		 */
+		if (value > fiber_stack_size[FSS_IDX_DEFAULT] ||
+		    value > fiber_stack_size[FSS_IDX_WATERMARK]) {
+			value = (size_t)page_align_up((void *)value);
+			fiber_stack_size[FSS_IDX_DEFAULT] = value;
+		}
+	}
+
+out:
+	/*
+	 * Once default stack size is known we should
+	 * update default attribute since it is not
+	 * a constant anymore.
+	 */
+	fiber_attr_default.stack_size = fiber_stack_size[FSS_IDX_DEFAULT];
+}
+
 static NOINLINE int
 is_stack_growsdown(void *prev_stack_frame)
 {
@@ -1394,6 +1451,7 @@ fiber_init(int (*invoke)(fiber_func f, va_list ap))
 {
 	page_size = sysconf(_SC_PAGESIZE);
 	stack_growsdown = is_stack_growsdown(__builtin_frame_address(0));
+	fiber_stack_init();
 	fiber_invoke = invoke;
 	main_thread_id = pthread_self();
 	main_cord.loop = ev_default_loop(EVFLAG_AUTO | EVFLAG_ALLOCFD);
-- 
2.20.1