[Tarantool-patches] [PATCH 3/5] popen/fio: Merge popen engine into fio internal module

[Cyrill Gorcunov <gorcunov@gmail.com>]

In the patch we wrap popen calls with lua C interface
which will allow us to use popen inside plain lua code.
They are wrapped via coio_custom calls thus each routine
takes into account current libev priority state.

Worth to note again that the creation of a new popen object
may cause coio thread to stall until exec/exit called. Should
not be a huge problem, after all exec itself is extremelly
heavy syscall.

Part-of #4031

Signed-off-by: Cyrill Gorcunov <gorcunov@gmail.com>
---
 src/lib/core/coio_file.c | 155 ++++++++++++++++++++++++++
 src/lib/core/coio_file.h |  10 ++
 src/lua/fio.c            | 235 +++++++++++++++++++++++++++++++++++++++
 3 files changed, 400 insertions(+)

diff --git a/src/lib/core/coio_file.c b/src/lib/core/coio_file.c
index 62388344e..39e4ed193 100644
--- a/src/lib/core/coio_file.c
+++ b/src/lib/core/coio_file.c
@@ -30,6 +30,7 @@
  */
 #include "coio_file.h"
 #include "coio_task.h"
+#include "popen.h"
 #include "fiber.h"
 #include "say.h"
 #include "fio.h"
@@ -103,6 +104,16 @@ struct coio_file_task {
 			const char *source;
 			const char *dest;
 		} copyfile;
+
+		struct {
+			struct popen_handle *handle;
+			const char *command;
+			unsigned int flags;
+			int timeout_msecs;
+			size_t count;
+			int wstatus;
+			void *buf;
+		} popen;
 	};
 };
 
@@ -639,3 +650,147 @@ coio_utime(const char *pathname, double atime, double mtime)
 	eio_req *req = eio_utime(pathname, atime, mtime, 0, coio_complete, &eio);
 	return coio_wait_done(req, &eio);
 }
+
+static void
+do_coio_popen_create(eio_req *req)
+{
+	struct coio_file_task *eio = req->data;
+
+	eio->popen.handle = popen_create(eio->popen.command,
+					 eio->popen.flags);
+	req->result = eio->popen.handle ? 0 : -1;
+}
+
+void *
+coio_popen_create(const char *command, unsigned int flags)
+{
+	INIT_COEIO_FILE(eio);
+	eio.popen.command = command;
+	eio.popen.flags = flags;
+
+	eio_req *req = eio_custom(do_coio_popen_create,
+				  EIO_PRI_DEFAULT,
+				  coio_complete, &eio);
+	coio_wait_done(req, &eio);
+
+	return eio.popen.handle;
+}
+
+static void
+coio_do_popen_destroy(eio_req *req)
+{
+        struct coio_file_task *eio = req->data;
+	req->result = popen_destroy(eio->popen.handle);
+}
+
+int
+coio_popen_destroy(void *handle)
+{
+	INIT_COEIO_FILE(eio);
+	eio.popen.handle = handle;
+
+	eio_req *req = eio_custom(coio_do_popen_destroy,
+				  EIO_PRI_DEFAULT,
+				  coio_complete, &eio);
+	return coio_wait_done(req, &eio);
+}
+
+static void
+coio_do_popen_kill(eio_req *req)
+{
+        struct coio_file_task *eio = req->data;
+	req->result = popen_kill(eio->popen.handle);
+}
+
+int
+coio_popen_kill(void *handle)
+{
+	INIT_COEIO_FILE(eio);
+	eio.popen.handle = handle;
+
+	eio_req *req = eio_custom(coio_do_popen_kill,
+				  EIO_PRI_DEFAULT,
+				  coio_complete, &eio);
+	return coio_wait_done(req, &eio);
+}
+
+static void
+coio_do_popen_status(eio_req *req)
+{
+        struct coio_file_task *eio = req->data;
+	req->result = popen_wstatus(eio->popen.handle,
+				    &eio->popen.wstatus);
+}
+
+int
+coio_popen_status(void *handle, int *reason, int *exit_code)
+{
+	INIT_COEIO_FILE(eio);
+	eio.popen.handle = handle;
+
+	eio_req *req = eio_custom(coio_do_popen_status,
+				  EIO_PRI_DEFAULT,
+				  coio_complete, &eio);
+	int ret = coio_wait_done(req, &eio);
+	if (ret > 0) {
+		*reason = WIFEXITED(eio.popen.wstatus) ?  1 : 2;
+		*exit_code = WIFEXITED(eio.popen.wstatus) ?
+			WEXITSTATUS(eio.popen.wstatus) :
+			WTERMSIG(eio.popen.wstatus);
+	}
+	return ret;
+}
+
+static void
+coio_do_popen_read(eio_req *req)
+{
+	struct coio_file_task *eio = req->data;
+	req->result = popen_read_timeout(eio->popen.handle,
+					 eio->popen.buf,
+					 eio->popen.count,
+					 eio->popen.flags,
+					 eio->popen.timeout_msecs);
+}
+
+ssize_t
+coio_popen_read(void *handle, void *buf, size_t count,
+		unsigned int flags, int timeout_msecs)
+{
+	INIT_COEIO_FILE(eio);
+	eio.popen.timeout_msecs = timeout_msecs;
+	eio.popen.handle = handle;
+	eio.popen.flags	= flags;
+	eio.popen.count	= count;
+	eio.popen.buf = buf;
+
+	eio_req *req = eio_custom(coio_do_popen_read,
+				  EIO_PRI_DEFAULT,
+				  coio_complete, &eio);
+	return coio_wait_done(req, &eio);
+}
+
+static void
+coio_do_popen_write(eio_req *req)
+{
+	struct coio_file_task *eio = req->data;
+	req->result = popen_write(eio->popen.handle,
+				  eio->popen.buf,
+				  eio->popen.count,
+				  eio->popen.flags);
+}
+
+ssize_t
+coio_popen_write(void *handle, void *buf, size_t count,
+		 unsigned int flags)
+{
+	INIT_COEIO_FILE(eio);
+	eio.popen.handle = handle;
+	eio.popen.flags	= flags;
+	eio.popen.count	= count;
+	eio.popen.buf = buf;
+
+	eio_req *req = eio_custom(coio_do_popen_write,
+				  EIO_PRI_DEFAULT,
+				  coio_complete, &eio);
+	return coio_wait_done(req, &eio);
+}
diff --git a/src/lib/core/coio_file.h b/src/lib/core/coio_file.h
index ac7b1aacf..fb385737d 100644
--- a/src/lib/core/coio_file.h
+++ b/src/lib/core/coio_file.h
@@ -85,6 +85,16 @@ int	coio_tempdir(char *path, size_t path_len);
 int	coio_readdir(const char *path, char **buf);
 int	coio_copyfile(const char *source, const char *dest);
 int	coio_utime(const char *pathname, double atime, double mtime);
+
+void	*coio_popen_create(const char *command, unsigned int flags);
+int	coio_popen_destroy(void *handle);
+int	coio_popen_kill(void *handle);
+int	coio_popen_status(void *handle, int *reason, int *exit_code);
+ssize_t	coio_popen_read(void *handle, void *buf, size_t count,
+			unsigned int flags, int timeout_msecs);
+ssize_t coio_popen_write(void *handle, void *buf, size_t count,
+			 unsigned int flags);
+
 #if defined(__cplusplus)
 } /* extern "C" */
 #endif /* defined(__cplusplus) */
diff --git a/src/lua/fio.c b/src/lua/fio.c
index 33ccd5d71..3cf52f2d5 100644
--- a/src/lua/fio.c
+++ b/src/lua/fio.c
@@ -39,6 +39,7 @@
 #include <time.h>
 #include <errno.h>
 #include "coio_task.h"
+#include "popen.h"
 
 #include <lua.h>
 #include <lauxlib.h>
@@ -683,6 +684,233 @@ lbox_fio_utime(struct lua_State *L)
 	return lbox_fio_pushbool(L, coio_utime(pathname, atime, mtime) == 0);
 }
 
+/**
+ * lbox_fio_popen_create - creates a new popen handle and runs a command inside
+ * @command:	a command to run
+ * @flags:	child pipe end specification
+ *
+ * Returns pair @handle = data, @err = nil on success,
+ * @handle = nil, err ~= nil on error.
+ */
+static int
+lbox_fio_popen_create(struct lua_State *L)
+{
+	struct popen_handle *handle;
+	unsigned int flags;
+	const char *command;
+
+	if (lua_gettop(L) < 1) {
+usage:
+		luaL_error(L, "Usage: fio.popen(command [, rw, {opts}])");
+	}
+
+	command = lua_tostring(L, 1);
+	if (!command)
+		goto usage;
+	flags = lua_tonumber(L, 2);
+
+	handle = coio_popen_create(command, flags);
+	if (!handle)
+		return lbox_fio_pushsyserror(L);
+
+	lua_pushlightuserdata(L, handle);
+	return 1;
+}
+
+/**
+ * lbox_fio_popen_destroy - close a popen handle
+ * @handle:	a handle to close
+ *
+ * If there is a running child it get killed first.
+ *
+ * Returns true if a handle is closeed, false otherwise.
+ */
+static int
+lbox_fio_popen_destroy(struct lua_State *L)
+{
+	void *handle = lua_touserdata(L, 1);
+	return lbox_fio_pushbool(L, coio_popen_destroy(handle) == 0);
+}
+
+/**
+ * lbox_fio_popen_kill - kill popen's child process
+ * @handle:	a handle carries child process to kill
+ *
+ * Returns true if process is killed and false
+ * otherwise. Note the process is simply signaled
+ * and it doesn't mean it is killed immediately,
+ * Poll lbox_fio_pstatus if need to find out when
+ * exactly the child is reaped out.
+ */
+static int
+lbox_fio_popen_kill(struct lua_State *L)
+{
+	struct popen_handle *p = lua_touserdata(L, 1);
+	return lbox_fio_pushbool(L, coio_popen_kill(p) == 0);
+}
+
+/**
+ * lbox_fio_popen_status - fetch popen child process status
+ * @handle:	a handle to fetch status from
+ *
+ * Returns @err = nil, @reason = [1|2], @exit_code = 'number'
+ * on success, where reason = 1 means process has been calling
+ * exit(@exit_code); and reason = 2 when process has been killed
+ * by @exit_code signal (either explicitly via lbox_fio_pkill
+ * or implicitly by a third side, say node admin via kill command
+ * in a shell).
+ */
+static int
+lbox_fio_popen_status(struct lua_State *L)
+{
+	struct popen_handle *p = lua_touserdata(L, 1);
+	int reason = 0, exit_code = 0, ret;
+
+	ret = coio_popen_status(p, &reason, &exit_code);
+	if (ret < 0) {
+		return lbox_fio_push_error(L);
+	} else if (ret == 0) {
+		lua_pushnil(L);
+		return 1;
+	}
+
+	lua_pushnil(L);
+	lua_pushinteger(L, reason);
+	lua_pushinteger(L, exit_code);
+	return 3;
+}
+
+/**
+ * lbox_fio_popen_read - read data from a child peer
+ * @handle:		a handle of a child process
+ * @buf:		destination buffer
+ * @count:		number of bytes to read
+ * @timeout_msecs:	read timeout, in microseconds, -1 to not use
+ * @flags:		which peer to read (stdout,stderr)
+ *
+ * Returns @size = 'read bytes', @err = nil on success,
+ * @size = nil, @err = nil if timeout expired, and
+ * @size = nil, @err ~= nil on error.
+ */
+static int
+lbox_fio_popen_read(struct lua_State *L)
+{
+	struct popen_handle *handle = lua_touserdata(L, 1);
+	uint32_t ctypeid;
+	void *buf =  *(char **)luaL_checkcdata(L, 2, &ctypeid);
+	size_t count = lua_tonumber(L, 3);
+	int timeout_msecs = lua_tonumber(L, 4);
+	unsigned int flags = lua_tonumber(L, 5);
+	ssize_t ret;
+
+	ret = coio_popen_read(handle, buf, count, flags, timeout_msecs);
+	if (ret < 0) {
+		if (ret == -EAGAIN) {
+			lua_pushnil(L);
+			lua_pushnil(L);
+			return 2;
+		} else
+			return lbox_fio_pushsyserror(L);
+	}
+
+	lua_pushinteger(L, ret);
+	return 1;
+}
+
+/**
+ * lbox_fio_popen_write - write data to a child peer
+ * @handle:	a handle of a child process
+ * @buf:	source buffer
+ * @count:	number of bytes to write
+ * @flags:	which peer to write (stdin)
+ *
+ * Returns @size = 'bytes wrote', @err = nil on succes,
+ * @size = nil, @err ~= nil on error.
+ */
+static int
+lbox_fio_popen_write(struct lua_State *L)
+{
+	struct popen_handle *handle = lua_touserdata(L, 1);
+	void *buf = (void *)lua_tostring(L, 2);
+	uint32_t ctypeid = 0;
+	if (buf == NULL)
+		buf =  *(char **)luaL_checkcdata(L, 2, &ctypeid);
+	size_t count = lua_tonumber(L, 3);
+	unsigned int flags = lua_tonumber(L, 4);
+	ssize_t ret;
+
+	ret = coio_popen_write(handle, buf, count, flags);
+	if (ret < 0)
+		return lbox_fio_pushsyserror(L);
+
+	lua_pushinteger(L, ret);
+	return 1;
+}
+
+/**
+ * lbox_fio_popen_info - return information about popen handle
+ * @handle:	a handle of a child process
+ *
+ * Returns a @table ~= nil, @err = nil on success,
+ * @table = nil, @err ~= nil on error.
+ */
+static int
+lbox_fio_popen_info(struct lua_State *L)
+{
+	struct popen_handle *handle = lua_touserdata(L, 1);
+	int reason = 0, exit_code = 0, ret;
+	struct popen_stat st = { };
+	char *status;
+
+	if (popen_stat(handle, &st))
+		return lbox_fio_pushsyserror(L);
+
+	ret = coio_popen_status(handle, &reason, &exit_code);
+	if (ret < 0)
+		return lbox_fio_pushsyserror(L);
+
+	if (ret == 0)
+		status = "alive";
+	else if (ret == 1)
+		status = "exited";
+	else if (ret == 2)
+		status = "signaled";
+	else
+		status = "unknown";
+
+	lua_newtable(L);
+
+	lua_pushliteral(L, "pid");
+	lua_pushinteger(L, st.pid);
+	lua_settable(L, -3);
+
+	lua_pushliteral(L, "flags");
+	lua_pushinteger(L, st.flags);
+	lua_settable(L, -3);
+
+	lua_pushliteral(L, "state");
+	lua_pushstring(L, status);
+	lua_settable(L, -3);
+
+	lua_pushliteral(L, "exit_code");
+	lua_pushinteger(L, exit_code);
+	lua_settable(L, -3);
+
+	lua_pushliteral(L, "stdin");
+	lua_pushinteger(L, st.fds[POPEN_FLAG_FD_STDIN_BIT]);
+	lua_settable(L, -3);
+
+	lua_pushliteral(L, "stdout");
+	lua_pushinteger(L, st.fds[POPEN_FLAG_FD_STDOUT_BIT]);
+	lua_settable(L, -3);
+
+	lua_pushliteral(L, "stderr");
+	lua_pushinteger(L, st.fds[POPEN_FLAG_FD_STDERR_BIT]);
+	lua_settable(L, -3);
+
+	return 1;
+}
+
 void
 tarantool_lua_fio_init(struct lua_State *L)
 {
@@ -726,6 +954,13 @@ tarantool_lua_fio_init(struct lua_State *L)
 		{ "fstat",		lbox_fio_fstat			},
 		{ "copyfile",		lbox_fio_copyfile,		},
 		{ "utime",		lbox_fio_utime			},
+		{ "popen_create",	lbox_fio_popen_create		},
+		{ "popen_destroy",	lbox_fio_popen_destroy		},
+		{ "popen_kill",		lbox_fio_popen_kill		},
+		{ "popen_status",	lbox_fio_popen_status		},
+		{ "popen_read",		lbox_fio_popen_read		},
+		{ "popen_write",	lbox_fio_popen_write		},
+		{ "popen_info",		lbox_fio_popen_info		},
 		{ NULL,			NULL				}
 	};
 	luaL_register(L, NULL, internal_methods);
-- 
2.20.1