[Tarantool-patches] [PATCH v3] luajit: proxy -j and -b flags

[Maxim Kokryashkin via Tarantool-patches <tarantool-patches@dev.tarantool.org>]

There are two flags in the LuaJIT useful for debugging and runtime
configuration purposes: `-j` and `-b`. However, if you want to
check the same Lua code from the Tarantool, you will need to make
some adjustments in the script itself, as those flags are not
present in the Tarantool's CLI.

This patch introduces those flags to the Tarantool, so debugging is
much more convenient now. Flags are working the same as they do in
the LuaJIT.

Closes #5541
---
CI branch: https://github.com/tarantool/tarantool/tree/fckxorg/gh-5541-proxy-luajit-flags-full-ci
Last review: https://lists.tarantool.org/tarantool-patches/YbBgatP6M9Qw2BMx@root/

> Why do we need ':' option here for j?

We need it because -j takes a command as an additional argument

> Why do you remove workaround from LuaJIT here?
> | $ src/tarantool -j -e ''
> | Can't open script : No such file or directory
>
> | $ luajit -j -e ''
> | luajit: unknown luaJIT command or jit.* modules not installed
>
> I mean the following part:
>
> | const char *cmd = argv[i] + 2;
> | if (*cmd == '\0') cmd = argv[++i];
> | lua_assert(cmd != NULL);
> | if (dojitcmd(L, cmd))
> |   return 1;
> | break;

IINM, if there is a colon in optstring, then getopt_long() will split
constructions like '-jon' into '-j' and 'on', so that workaround is not needed.

Let's wait for Igor's opinion.
>> diff --git a/src/lua/init.c b/src/lua/init.c
>> index f9738025d..54d07926a 100644
>> --- a/src/lua/init.c
>> +++ b/src/lua/init.c
>> @@ -313,6 +313,154 @@ skip:		base = (base == -1 ? 10 : base);
>>
>>  /* }}} */
>>
>> +static void l_message(const char *msg)
>> +{
>> +        printf("%s\n", msg);
>> +        fflush(stdout);
>> +}
>
>Side note: I'm not sure that this is really convinient way for Tarantool to
>report error this way. Wait for Igor's opinion.

 src/lua/init.c           | 199 +++++++++++++++++++++++++++++++++++++++
 src/lua/init.h           |   6 ++
 src/main.cc              | 101 ++++++++++++--------
 test/box-py/args.result  |  28 ++++++
 test/box-py/args.test.py |   6 ++
 5 files changed, 299 insertions(+), 41 deletions(-)

diff --git a/src/lua/init.c b/src/lua/init.c
index 5aaaf75f0..48d2b8bda 100644
--- a/src/lua/init.c
+++ b/src/lua/init.c
@@ -524,6 +524,164 @@ skip:		base = (base == -1 ? 10 : base);
 
 /* }}} */
 
+static void l_message(const char *msg)
+{
+	printf("%s\n", msg);
+	fflush(stdout);
+}
+
+/**
+ * This function gets an error message as its argument on
+ * the Lua stack and prints it to stdout. After that it
+ * pops the error message from the stack.
+ */
+static int report(lua_State *L, int status)
+{
+	if (status && !lua_isnil(L, -1)) {
+		const char *msg = lua_tostring(L, -1);
+		if (msg == NULL)
+			msg = "(error object is not a string)";
+		l_message(msg);
+		lua_pop(L, 1);
+	}
+	return status;
+}
+
+/**
+ * Load add-on module. This function has no
+ * effects on the Lua stack.
+ */
+static int loadjitmodule(lua_State *L)
+{
+	lua_getglobal(L, "require");
+	lua_pushliteral(L, "jit.");
+	lua_pushvalue(L, -3);
+	lua_concat(L, 2);
+	if (lua_pcall(L, 1, 1, 0) != LUA_OK) {
+		const char *msg = lua_tostring(L, -1);
+		if (msg && !strncmp(msg, "module ", 7))
+			goto nomodule;
+		return report(L, 1);
+	}
+	lua_getfield(L, -1, "start");
+	if (lua_isnil(L, -1)) {
+		nomodule:
+		l_message("unknown luaJIT command or jit.* modules not installed");
+		return 1;
+	}
+	/* Drop module table. */
+	lua_remove(L, -2);
+	return 0;
+}
+/**
+ * This function dumps bytecode for Lua script.
+ * Has no effect on the Lua stack.
+ */
+int dobytecode(va_list ap)
+{
+	char **argv = va_arg(ap, char **);
+	struct diag *diag = va_arg(ap, struct diag *);
+
+	int narg = 0;
+	bool aux_loop_is_run = false;
+
+	lua_pushliteral(tarantool_L, "bcsave");
+	if (loadjitmodule(tarantool_L) != 0)
+		goto error;
+	if (argv[0][2]) {
+		narg++;
+		argv[0][1] = '-';
+		lua_pushstring(tarantool_L, argv[0] + 1);
+	}
+
+	/*
+	 * Return control to tarantool_lua_run_script.
+	 * tarantool_lua_run_script then will start an auxiliary event
+	 * loop and re-schedule this fiber.
+	 */
+	fiber_sleep(0.0);
+	aux_loop_is_run = true;
+
+	for (argv++; *argv != NULL; narg++, argv++)
+		lua_pushstring(tarantool_L, *argv);
+	int res = lua_pcall(tarantool_L, narg, 0, 0);
+	if (res != LUA_OK)
+		goto error;
+
+	end:
+	/*
+	 * Auxiliary loop in tarantool_lua_run_script
+	 * should start (ev_run()) before this fiber
+	 * invokes ev_break().
+	 */
+	if (!aux_loop_is_run)
+		fiber_sleep(0.0);
+	ev_break(loop(), EVBREAK_ALL);
+	return 0;
+
+	error:
+	diag_set(LuajitError, lua_tostring(tarantool_L, -1));
+	diag_move(diag_get(), diag);
+	goto end;
+}
+
+/**
+ * Runs jit module command with options.
+ * Has no effect on the Lua stack.
+ */
+static int runcmdopt(lua_State *L, const char *opt)
+{
+	int narg = 0;
+	if (opt && *opt) {
+	/* This loop splits args for jit module. */
+		for (;;) {
+			const char *p = strchr(opt, ',');
+			narg++;
+			if (!p) break;
+			if (p == opt)
+				lua_pushnil(L);
+			else
+				lua_pushlstring(L, opt, (size_t)(p - opt));
+			opt = p + 1;
+		}
+		if (*opt)
+			lua_pushstring(L, opt);
+		else
+			lua_pushnil(L);
+	}
+	return report(L, lua_pcall(L, narg, 0, 0));
+}
+
+/**
+ * JIT engine control command: try jit
+ * library first or load add-on module.
+ * Removes jit module name from Lua stack.
+ */
+int dojitcmd(const char *cmd)
+{
+	const char *opt = strchr(cmd, '=');
+	lua_pushlstring(tarantool_L, cmd, opt ? (size_t)(opt - cmd) : strlen(cmd));
+	lua_getfield(tarantool_L, LUA_REGISTRYINDEX, "_LOADED");
+	/* Get jit.* module table. */
+	lua_getfield(tarantool_L, -1, "jit");
+	lua_remove(tarantool_L, -2);
+	lua_pushvalue(tarantool_L, -2);
+	/* Lookup library function. */
+	lua_gettable(tarantool_L, -2);
+	if (!lua_isfunction(tarantool_L, -1)) {
+		/* Drop non-function and jit.* table, keep module name. */
+		lua_pop(tarantool_L, 2);
+	if (loadjitmodule(tarantool_L) != 0)
+		return 1;
+	} else {
+		/* Drop jit.* table. */
+		lua_remove(tarantool_L, -2);
+	}
+	/* Drop module name. */
+	lua_remove(tarantool_L, -2);
+	return runcmdopt(tarantool_L, opt ? opt + 1 : opt);
+}
+
 /**
  * Original LuaJIT/Lua logic: <luajit/src/lib_package.c - function setpath>
  *
@@ -842,6 +1000,10 @@ run_script_f(va_list ap)
 			lua_setglobal(L, optv[i + 1]);
 			lua_settop(L, 0);
 			break;
+		case 'j':
+			if (dojitcmd(optv[i + 1]) != 0)
+				goto error;
+			break;
 		case 'e':
 			/*
 			 * Execute chunk
@@ -981,6 +1143,43 @@ tarantool_lua_run_script(char *path, bool interactive,
 	return diag_is_empty(diag_get()) ? 0 : -1;
 }
 
+int
+tarantool_lua_dump_bytecode(char **argv) {
+  /*
+   * Tarantool has its own definition of os.exit() so running
+   * lua code outside of a fiber causes panic.
+   */
+	script_fiber = fiber_new("bcsave", dobytecode);
+	if (script_fiber == NULL)
+		panic("%s", diag_last_error(diag_get())->errmsg);
+	script_fiber->storage.lua.stack = tarantool_L;
+	/*
+	 * Create a new diag on the stack. Don't pass fiber's diag, because it
+	 * might be overwritten by libev callbacks invoked in the scheduler
+	 * fiber (which is this), and therefore can't be used as a sign of fail
+	 * in the script itself.
+	 */
+	struct diag bc_diag;
+	diag_create(&bc_diag);
+	fiber_start(script_fiber, argv, &bc_diag);
+	/*
+	 * Run an auxiliary event loop to re-schedule run_script fiber.
+	 * When this fiber finishes, it will call ev_break to stop the loop.
+	 */
+	if (start_loop)
+		ev_run(loop(), 0);
+	/* The fiber running the startup script has ended. */
+	script_fiber = NULL;
+	diag_move(&bc_diag, diag_get());
+	diag_destroy(&bc_diag);
+	/*
+	 * Result can't be obtained via fiber_join - script fiber
+	 * never dies if os.exit() was called. This is why diag
+	 * is checked explicitly.
+	 */
+	return diag_is_empty(diag_get()) ? 0 : -1;
+}
+
 void
 tarantool_lua_free()
 {
diff --git a/src/lua/init.h b/src/lua/init.h
index 7fc0b1a31..a438ae23d 100644
--- a/src/lua/init.h
+++ b/src/lua/init.h
@@ -75,6 +75,12 @@ tarantool_lua_run_script(char *path, bool force_interactive,
 			 int optc, const char **optv,
 			 int argc, char **argv);
 
+/**
+ * Dump bytecode for a specified Lua script.
+ */
+int
+tarantool_lua_dump_bytecode(char **argv);
+
 extern char *history;
 
 struct slab_cache *
diff --git a/src/main.cc b/src/main.cc
index 316bbef99..0cce347a2 100644
--- a/src/main.cc
+++ b/src/main.cc
@@ -578,6 +578,8 @@ print_help(const char *program)
 	puts("  -v, --version\t\t\tprint program version and exit");
 	puts("  -e EXPR\t\t\texecute string 'EXPR'");
 	puts("  -l NAME\t\t\trequire library 'NAME'");
+	puts("  -j cmd \t\t\tperform LuaJIT control command");
+	puts("  -b ... \t\t\tsave or list bytecode");
 	puts("  -i\t\t\t\tenter interactive mode after executing 'SCRIPT'");
 	puts("  --\t\t\t\tstop handling options");
 	puts("  -\t\t\t\texecute stdin and stop handling options");
@@ -586,6 +588,9 @@ print_help(const char *program)
 	puts("to see online documentation, submit bugs or contribute a patch.");
 }
 
+#define O_INTERACTIVE 0x1
+#define O_BYTECODE    0x2
+
 int
 main(int argc, char **argv)
 {
@@ -597,7 +602,7 @@ main(int argc, char **argv)
 	fpconv_check();
 
 	/* Enter interactive mode after executing 'script' */
-	bool interactive = false;
+	uint32_t opt_mask = 0;
 	/* Lua interpeter options, e.g. -e and -l */
 	int optc = 0;
 	const char **optv = NULL;
@@ -608,38 +613,47 @@ main(int argc, char **argv)
 		{"version", no_argument, 0, 'v'},
 		{NULL, 0, 0, 0},
 	};
-	static const char *opts = "+hVvie:l:";
+	static const char *opts = "+hVvbij:e:l:";
 
 	int ch;
+	bool lj_arg = false;
 	while ((ch = getopt_long(argc, argv, opts, longopts, NULL)) != -1) {
 		switch (ch) {
-		case 'V':
-		case 'v':
-			print_version();
-			return 0;
-		case 'h':
-			print_help(basename(argv[0]));
-			return 0;
-		case 'i':
-			/* Force interactive mode */
-			interactive = true;
-			break;
-		case 'l':
-		case 'e':
-			/* Save Lua interepter options to optv as is */
-			if (optc == 0) {
-				optv = (const char **) calloc(argc,
-							      sizeof(optv[0]));
-				if (optv == NULL)
-					panic_syserror("No enough memory for arguments");
-			}
-			optv[optc++] = ch == 'l' ? "-l" : "-e";
-			optv[optc++] = optarg;
-			break;
-		default:
-			/* "invalid option" is printed by getopt */
-			return EX_USAGE;
+			case 'V':
+			case 'v':
+				print_version();
+				return 0;
+			case 'h':
+				print_help(basename(argv[0]));
+				return 0;
+			case 'i':
+				/* Force interactive mode */
+				opt_mask |= O_INTERACTIVE;
+				break;
+			case 'b':
+				opt_mask |= O_BYTECODE;
+				lj_arg = true;
+				break;
+			case 'j':
+			case 'l':
+			case 'e':
+				/* Save Lua interepter options to optv as is */
+				if (optc == 0) {
+					optv = (const char **) calloc(argc, sizeof(optv[0]));
+					if (optv == NULL)
+						panic_syserror("No enough memory for arguments");
+				}
+				if(ch == 'l') optv[optc++] = "-l";
+				else if(ch == 'j') optv[optc++] = "-j";
+				else optv[optc++] = "-e";
+				optv[optc++] = optarg;
+				break;
+			default:
+				/* "invalid option" is printed by getopt */
+				return EX_USAGE;
 		}
+
+		if(lj_arg) break;
 	}
 
 	/* Shift arguments */
@@ -647,18 +661,20 @@ main(int argc, char **argv)
 	for (int i = 1; i < argc; i++)
 		argv[i] = argv[optind + i - 1];
 
-	if (argc > 1 && strcmp(argv[1], "-") && access(argv[1], R_OK) != 0) {
-		/*
-		 * Somebody made a mistake in the file
-		 * name. Be nice: open the file to set
-		 * errno.
-		 */
-		int fd = open(argv[1], O_RDONLY);
-		int save_errno = errno;
-		if (fd >= 0)
-			close(fd);
-		printf("Can't open script %s: %s\n", argv[1], strerror(save_errno));
-		return save_errno;
+	if(!(opt_mask & O_BYTECODE) && argc > 1 &&
+	    strcmp(argv[1], "-") && access(argv[1], R_OK) != 0) {
+	    /*
+	    * Somebody made a mistake in the file
+	    * name. Be nice: open the file to set
+	    * errno.
+	    */
+	    int fd = open(argv[1], O_RDONLY);
+	    int save_errno = errno;
+	    if (fd >= 0)
+		    close(fd);
+	    printf("Can't open script %s: %s\n",
+		argv[1], strerror(save_errno));
+		    return save_errno;
 	}
 
 	argv = title_init(argc, argv);
@@ -749,13 +765,16 @@ main(int argc, char **argv)
 			panic("%s", "can't init event loop");
 
 		int events = ev_activecnt(loop());
+
+		if(opt_mask & O_BYTECODE)
+			return tarantool_lua_dump_bytecode(argv);
 		/*
 		 * Load user init script.  The script should have access
 		 * to Tarantool Lua API (box.cfg, box.fiber, etc...) that
 		 * is why script must run only after the server was fully
 		 * initialized.
 		 */
-		if (tarantool_lua_run_script(script, interactive, optc, optv,
+		if (tarantool_lua_run_script(script, opt_mask & O_INTERACTIVE, optc, optv,
 					     main_argc, main_argv) != 0)
 			diag_raise();
 		/*
diff --git a/test/box-py/args.result b/test/box-py/args.result
index 533e66eef..4788a300d 100644
--- a/test/box-py/args.result
+++ b/test/box-py/args.result
@@ -9,6 +9,8 @@ When no script name is provided, the server responds to:
   -v, --version			print program version and exit
   -e EXPR			execute string 'EXPR'
   -l NAME			require library 'NAME'
+  -j cmd 			perform LuaJIT control command
+  -b ... 			save or list bytecode
   -i				enter interactive mode after executing 'SCRIPT'
   --				stop handling options
   -				execute stdin and stop handling options
@@ -27,6 +29,8 @@ When no script name is provided, the server responds to:
   -v, --version			print program version and exit
   -e EXPR			execute string 'EXPR'
   -l NAME			require library 'NAME'
+  -j cmd 			perform LuaJIT control command
+  -b ... 			save or list bytecode
   -i				enter interactive mode after executing 'SCRIPT'
   --				stop handling options
   -				execute stdin and stop handling options
@@ -152,3 +156,27 @@ arg[2] => 2
 arg[3] => 3
 arg[4] => --help
 
+tarantool -b
+Save LuaJIT bytecode: luajit -b[options] input output
+  -l        Only list bytecode.
+  -s        Strip debug info (default).
+  -g        Keep debug info.
+  -n name   Set module name (default: auto-detect from input name).
+  -t type   Set output file type (default: auto-detect from output name).
+  -a arch   Override architecture for object files (default: native).
+  -o os     Override OS for object files (default: native).
+  -e chunk  Use chunk string as input.
+  --        Stop handling options.
+  -         Use stdin as input and/or stdout as output.
+
+File types: c h obj o raw (default)
+
+tarantool -bl -e 
+-- BYTECODE -- "":0-1
+0001    RET0     0   1
+
+
+tarantool -jon -e 
+
+tarantool -j on -e 
+
diff --git a/test/box-py/args.test.py b/test/box-py/args.test.py
index 152e4f236..647322d4a 100644
--- a/test/box-py/args.test.py
+++ b/test/box-py/args.test.py
@@ -60,4 +60,10 @@ server.test_option("-e \"print(rawget(_G, 'log') == nil)\" " + \
                    script + \
                    " 1 2 3 --help")
 
+server.test_option("-b")
+server.test_option("-bl -e ''")
+
+server.test_option("-jon -e ''")
+server.test_option("-j on -e ''")
+
 sys.stdout.clear_all_filters()
-- 
2.35.1