[Tarantool-patches] [PATCH v2 4/5] popen/fio: Add ability to run external programs
Cyrill Gorcunov
gorcunov at gmail.com
Tue Dec 10 12:48:54 MSK 2019
In the patch we add support for running external
programs via fio module.
The interface is the following:
- To create new popen object either call fio.popen,
or fio.run; the difference is that fio.popen uses
simplified arguments list (like fio.popen(command, "rw"))
while fio.run accepts a table of options which will
be extended in future if needed
- To read/write popen:read()/write() methods are supported,
and read() supports timeout argument which implemented
via internal poll() call and allows to test for data present
on a child peer
- To interrupt children work and exit one can use popen:term()
which is soft signal and can be ignored by a child, or
popen:kill() to force exit. Note that signals are not handled
immediately so the next method is needed
- To wait until child is terminated popen:wait() should be
used. It waits in a cycle so be carefull here -- if child
get signaled with popen:term() it might ignore the signal
and popen:wait() will spin forever
- To fetch information about current state of a child process
use popen:info() method
- Once popen no longer needed the popen:close() should be
called to free all resources associated with the object
Part-of #4031
Signed-off-by: Cyrill Gorcunov <gorcunov at gmail.com>
---
src/lua/fio.lua | 578 ++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 578 insertions(+)
diff --git a/src/lua/fio.lua b/src/lua/fio.lua
index cb224f3d0..b613cfe8c 100644
--- a/src/lua/fio.lua
+++ b/src/lua/fio.lua
@@ -5,6 +5,7 @@ local ffi = require('ffi')
local buffer = require('buffer')
local fiber = require('fiber')
local errno = require('errno')
+local bit = require('bit')
ffi.cdef[[
int umask(int mask);
@@ -519,4 +520,581 @@ fio.path.lexists = function(filename)
return fio.lstat(filename) ~= nil
end
+local popen_methods = { }
+
+--
+-- Make sure the bits are matching POPEN_FLAG_x
+-- flags in popen header. This is api but internal
+-- one, we should not make it visible to users.
+local popen_flag_bits = {
+ STDIN = 0,
+ STDOUT = 1,
+ STDERR = 2,
+
+ STDIN_DEVNULL = 3,
+ STDOUT_DEVNULL = 4,
+ STDERR_DEVNULL = 5,
+
+ STDIN_CLOSE = 6,
+ STDOUT_CLOSE = 7,
+ STDERR_CLOSE = 8,
+
+ SHELL = 9,
+}
+
+local popen_flags = {
+ NONE = 0,
+ STDIN = bit.lshift(1, popen_flag_bits.STDIN),
+ STDOUT = bit.lshift(1, popen_flag_bits.STDOUT),
+ STDERR = bit.lshift(1, popen_flag_bits.STDERR),
+
+ STDIN_DEVNULL = bit.lshift(1, popen_flag_bits.STDIN_DEVNULL),
+ STDOUT_DEVNULL = bit.lshift(1, popen_flag_bits.STDOUT_DEVNULL),
+ STDERR_DEVNULL = bit.lshift(1, popen_flag_bits.STDERR_DEVNULL),
+
+ STDIN_CLOSE = bit.lshift(1, popen_flag_bits.STDIN_CLOSE),
+ STDOUT_CLOSE = bit.lshift(1, popen_flag_bits.STDOUT_CLOSE),
+ STDERR_CLOSE = bit.lshift(1, popen_flag_bits.STDERR_CLOSE),
+
+ SHELL = bit.lshift(1, popen_flag_bits.SHELL),
+}
+
+--
+-- Get default flags for popen
+local function popen_default_flags()
+ local flags = popen_flags.NONE
+
+ -- default flags: close everything and use shell
+ flags = bit.bor(flags, popen_flags.STDIN_CLOSE)
+ flags = bit.bor(flags, popen_flags.STDOUT_CLOSE)
+ flags = bit.bor(flags, popen_flags.STDERR_CLOSE)
+ flags = bit.bor(flags, popen_flags.SHELL)
+
+ return flags
+end
+
+--
+-- Close a popen object and release all resources.
+-- In case if there is a running child process
+-- it will be killed.
+--
+-- Returns @ret = true if popen handle closed, and
+-- @ret = false, @err ~= nil on error.
+popen_methods.close = function(self)
+ local ret, err = internal.popen_delete(self.popen_handle)
+ if err ~= nil then
+ return false, err
+ end
+ self.popen_handle = nil
+ return true
+end
+
+--
+-- Kill a child process
+--
+-- Returns @ret = true on success,
+-- @ret = false, @err ~= nil on error.
+popen_methods.kill = function(self)
+ return internal.popen_kill(self.popen_handle)
+end
+
+--
+-- Terminate a child process
+--
+-- Returns @ret = true on success,
+-- @ret = false, @err ~= nil on error.
+popen_methods.terminate = function(self)
+ return internal.popen_term(self.popen_handle)
+end
+
+--
+-- Fetch a child process state
+--
+-- Returns @err = nil, @state = (1 - if exited,
+-- 2 - if killed by a signal) and @exit_code ~= nil,
+-- otherwise @err ~= nil.
+--
+-- FIXME: Should the @state be a named constant?
+popen_methods.state = function(self)
+ return internal.popen_state(self.popen_handle)
+end
+
+--
+-- Wait until a child process finished its work
+-- and exit then.
+--
+-- Since it goes via coio thread we can't simply
+-- call wait4 (or similar) directly to not block
+-- the whole i/o subsystem. Instead we do an explicit
+-- polling for status change with predefined period
+-- of sleeping.
+--
+-- Returns the same as popen_methods.status.
+popen_methods.wait = function(self)
+ local err, state, code
+ while true do
+ err, state, code = internal.popen_state(self.popen_handle)
+ if err or state then
+ break
+ end
+ fiber.sleep(self.wait_secs)
+ end
+ return err, state, code
+end
+
+--
+-- A map for popen option keys into tfn ('true|false|nil') values
+-- where bits are just set without additional manipulations.
+--
+-- For example stdin=true means to open stdin end for write,
+-- stdin=false to close the end, finally stdin[=nil] means to
+-- provide /dev/null into a child peer.
+local popen_opts_tfn = {
+ stdin = {
+ popen_flags.STDIN,
+ popen_flags.STDIN_CLOSE,
+ popen_flags.STDIN_DEVNULL,
+ },
+ stdout = {
+ popen_flags.STDOUT,
+ popen_flags.STDOUT_CLOSE,
+ popen_flags.STDOUT_DEVNULL,
+ },
+ stderr = {
+ popen_flags.STDERR,
+ popen_flags.STDERR_CLOSE,
+ popen_flags.STDERR_DEVNULL,
+ },
+}
+
+--
+-- A map for popen option keys into tf ('true|false') values
+-- where bits are set on 'true' and clear on 'false'.
+local popen_opts_tf = {
+ shell = {
+ popen_flags.SHELL,
+ },
+}
+
+--
+-- Parses flags options from popen_opts_tfn and
+-- popen_opts_tf tables.
+local function parse_popen_flags(epfx, flags, opts)
+ if opts == nil then
+ return flags
+ end
+ for k,v in pairs(opts) do
+ if popen_opts_tfn[k] == nil then
+ if popen_opts_tf[k] == nil then
+ error(string.format("%s: Unknown key %s", epfx, k))
+ end
+ if v == true then
+ flags = bit.bor(flags, popen_opts_tf[k][1])
+ elseif v == false then
+ flags = bit.band(flags, bit.bnot(popen_opts_tf[k][1]))
+ else
+ error(string.format("%s: Unknown value %s", epfx, v))
+ end
+ else
+ if v == true then
+ flags = bit.band(flags, bit.bnot(popen_opts_tfn[k][2]))
+ flags = bit.band(flags, bit.bnot(popen_opts_tfn[k][3]))
+ flags = bit.bor(flags, popen_opts_tfn[k][1])
+ elseif v == false then
+ flags = bit.band(flags, bit.bnot(popen_opts_tfn[k][1]))
+ flags = bit.band(flags, bit.bnot(popen_opts_tfn[k][3]))
+ flags = bit.bor(flags, popen_opts_tfn[k][2])
+ elseif v == "devnull" then
+ flags = bit.band(flags, bit.bnot(popen_opts_tfn[k][1]))
+ flags = bit.band(flags, bit.bnot(popen_opts_tfn[k][2]))
+ flags = bit.bor(flags, popen_opts_tfn[k][3])
+ else
+ error(string.format("%s: Unknown value %s", epfx, v))
+ end
+ end
+ end
+ return flags
+end
+
+--
+-- popen:read2 - read stream of a child with options
+-- @opts: options table
+--
+-- The options should have the following keys
+--
+-- @flags: stdout=true or stderr=true
+-- @buf: const_char_ptr_t buffer
+-- @size: size of the buffer
+-- @timeout_msecs: read timeout in microsecond
+--
+-- Returns @res = bytes, err = nil in case if read processed
+-- without errors, @res = nil, @err = nil if timeout elapsed
+-- and no data appeared on the peer, @res = nil, @err ~= nil
+-- otherwise.
+popen_methods.read2 = function(self, opts)
+ --
+ -- We can reuse parse_popen_flags since only
+ -- a few flags we're interesed in -- stdout/stderr
+ local flags = parse_popen_flags("fio.popen:read2",
+ popen_flags.NONE,
+ opts['flags'])
+ local timeout_msecs = -1
+ if opts['buf'] == nil then
+ error("fio.popen:read2 {'buf'} key is missed")
+ elseif opts['size'] == nil then
+ error("fio.popen:read2 {'size'} key is missed")
+ elseif opts['timeout_msecs'] ~= nil then
+ timeout_msecs = tonumber(opts['timeout_msecs'])
+ end
+
+ return internal.popen_read(self.popen_handle, opts['buf'],
+ tonumber(opts['size']),
+ timeout_msecs, flags)
+end
+
+-- popen:write2 - write data to a child with options
+-- @opts: options table
+--
+-- The options should have the following keys
+--
+-- @flags: stdin=true
+-- @buf: const_char_ptr_t buffer
+-- @size: size of the buffer
+--
+-- Returns @res = bytes, err = nil in case if write processed
+-- without errors, or @res = nil, @err ~= nil otherwise.
+popen_methods.write2 = function(self, opts)
+ local flags = parse_popen_flags("fio.popen:write2",
+ popen_flags.NONE,
+ opts['flags'])
+ if opts['buf'] == nil then
+ error("fio.popen:write2 {'buf'} key is missed")
+ elseif opts['size'] == nil then
+ error("fio.popen:write2 {'size'} key is missed")
+ end
+ return internal.popen_write(self.popen_handle, opts['buf'],
+ tonumber(opts['size']), flags)
+end
+
+--
+-- Write data to stdin of a child process
+-- @buf: a buffer to write
+-- @size: size of the buffer
+--
+-- Both parameters are optional and some
+-- string could be passed instead
+--
+-- write(str)
+-- write(buf, len)
+--
+-- Returns @res = bytes, @err = nil in case of success,
+-- @res = , @err ~= nil in case of error.
+popen_methods.write = function(self, buf, size)
+ if not ffi.istype(const_char_ptr_t, buf) then
+ buf = tostring(buf)
+ size = #buf
+ end
+ return self:write2({ buf = buf, size = size,
+ flags = { stdin = true }})
+end
+
+--
+-- popen:read - read the output stream in blocked mode
+-- @buf: a buffer to read to, optional
+-- @size: size of the buffer, optional
+-- @opts: options table, optional
+--
+-- The options may have the following keys:
+--
+-- @flags: stdout=true or stderr=true,
+-- to specify which stream to read;
+-- by default stdout is used
+-- @timeout_msecs: read timeout in microcesonds, when
+-- specified the read will be interrupted
+-- after specified time elapced, to make
+-- nonblocking read; de default the read
+-- is blocking operation
+--
+-- The following variants are accepted as well
+--
+-- read() -> str
+-- read(buf) -> len
+-- read(size) -> str
+-- read(buf, size) -> len
+--
+-- FIXME: Should we merged it with plain fio.write()?
+popen_methods.read = function(self, buf, size, opts)
+ local tmpbuf
+
+ -- read() or read(buf)
+ if (not ffi.istype(const_char_ptr_t, buf) and buf == nil) or
+ (ffi.istype(const_char_ptr_t, buf) and size == nil) then
+ -- read @self.read_buf_size bytes at once if nothing specified
+ -- FIXME: should it be configurable on popen creation?
+ size = self.read_buf_size
+ end
+
+ -- read(size)
+ if not ffi.istype(const_char_ptr_t, buf) then
+ size = buf or size
+ tmpbuf = buffer.ibuf()
+ buf = tmpbuf:reserve(size)
+ end
+
+ if opts == nil then
+ opts = {
+ timeout_msecs = -1,
+ flags = { stdout = true },
+ }
+ end
+
+ local res, err = self:read2({ buf = buf, size = size,
+ timeout_msecs = opts['timeout_msecs'],
+ flags = opts['flags'] })
+
+ if res == nil then
+ if tmpbuf ~= nil then
+ tmpbuf:recycle()
+ end
+ return nil, err
+ end
+
+ if tmpbuf ~= nil then
+ tmpbuf:alloc(res)
+ res = ffi.string(tmpbuf.rpos, tmpbuf:size())
+ tmpbuf:recycle()
+ end
+
+ return res
+end
+
+--
+-- popen:info -- obtain information about a popen object
+--
+-- Returns @info table, err == nil on success,
+-- @info = nil, @err ~= nil otherwise.
+--
+-- The @info table contains the following keys:
+--
+-- @pid: pid of a child process
+-- @flags: flags associated (popen_flags bitmap)
+-- @stdout: parent peer for stdout, -1 if closed
+-- @stderr: parent peer for stderr, -1 if closed
+-- @stdin: parent peer for stdin, -1 if closed
+-- @state: alive | exited | signaled | unknown
+-- @exit_code: exit code of a child process if been
+-- exited or killed by a signal
+--
+-- The child should never be in "unknown" state, reserved
+-- for unexpected errors.
+--
+popen_methods.info = function(self)
+ return internal.popen_info(self.popen_handle)
+end
+
+local popen_mt = { __index = popen_methods }
+
+--
+-- Parse "mode" string to flags
+local function parse_popen_mode(epfx, flags, mode)
+ if mode == nil then
+ return flags
+ end
+ for i = 1, #mode do
+ local c = mode:sub(i, i)
+ if c == 'r' then
+ flags = bit.band(flags, bit.bnot(popen_flags.STDOUT_CLOSE))
+ flags = bit.bor(flags, popen_flags.STDOUT)
+ elseif c == 'w' then
+ flags = bit.band(flags, bit.bnot(popen_flags.STDIN_CLOSE))
+ flags = bit.bor(flags, popen_flags.STDIN)
+ else
+ error(string.format("%s: Unknown mode %s", epfx, c))
+ end
+ end
+ return flags
+end
+
+--
+-- Create a new popen object from options
+local function popen_new(opts)
+ local handle, err = internal.popen_new(opts)
+ if err ~= nil then
+ return nil, err
+ end
+
+ local popen_handle = {
+ -- a handle itself for future use
+ popen_handle = handle,
+
+ -- sleeping period for the @wait method
+ wait_secs = 1,
+
+ -- size of a read buffer to allocate
+ -- in case of implicit read
+ read_buf_size = 512,
+ }
+
+ setmetatable(popen_handle, popen_mt)
+ return popen_handle
+end
+
+--
+-- fio.popen - create a new child process and execute a command inside
+-- @command: a command to run
+-- @mode: r - to grab child's stdout for read,
+-- w - to obtain stdin for write
+-- @...: optional parameters table:
+--
+-- stdin=true alias for @mode=w
+-- stdin=false to close STDIN_FILENO inside a child process [*]
+-- stdin="devnull" a child will get /dev/null as STDIN_FILENO
+--
+-- stdout=true alias for @mode=r
+-- stdout=false to close STDOUT_FILENO inside a child process [*]
+-- stdin="devnull" a child will get /dev/null as STDOUT_FILENO
+--
+-- stderr=true to read STDERR_FILENO from a child process
+-- stderr=false to close STDERR_FILENO inside a child process [*]
+-- stdin="devnull" a child will get /dev/null as STDERR_FILENO
+--
+-- shell=true runs a child process via "sh -c" [*]
+-- shell=false invokes a child process executable directly
+--
+-- [*] are default values
+--
+-- Examples:
+--
+-- popen = require('fio').popen("date", "r")
+-- runs date as "sh -c date" to read the output,
+-- closing all file descriptors except stdin inside
+-- a child process
+--
+-- popen = require('fio').popen("/usr/bin/date", "r", {shell=false})
+-- invokes date executable without shell to read the output
+--
+fio.popen = function(command, mode, ...)
+ local flags = popen_default_flags()
+
+ if type(command) ~= 'string' then
+ error("Usage: fio.popen(command[, rw, {opts}])")
+ end
+
+ -- Mode gives simplified flags
+ flags = parse_popen_mode("fio.popen", flags, mode)
+
+ -- Options table can override the mode option
+ if ... ~= nil then
+ flags = parse_popen_flags("fio.popen", flags, ...)
+ end
+
+ local opts = {
+ argv = { command },
+ argc = 1,
+ flags = flags,
+ envc = -1,
+ }
+
+ return popen_new(opts)
+end
+
+--
+-- fio.run - spawn a new process and run a command inside
+-- @opt: table of options
+--
+-- @opts carries of the following options:
+--
+-- @argv: an array of a program to run with
+-- command line options, mandatory
+--
+-- @env: an array of environment variables to
+-- be used inside a process, if not
+-- set then the current environment is
+-- inherited, if set to an empty array
+-- then the environment will be dropped
+--
+-- @mode: "r" to read from stdout, "w" to
+-- write to stdin of a process
+--
+-- @flags: a dictionary to describe communication pipes
+-- and other parameters of a process to run
+--
+-- stdin=true to write into STDIN_FILENO of a child process
+-- stdin=false to close STDIN_FILENO inside a child process [*]
+-- stdin="devnull" a child will get /dev/null as STDIN_FILENO
+--
+-- stdout=true to write into STDOUT_FILENO of a child process
+-- stdout=false to close STDOUT_FILENO inside a child process [*]
+-- stdin="devnull" a child will get /dev/null as STDOUT_FILENO
+--
+-- stderr=true to read STDERR_FILENO from a child process
+-- stderr=false to close STDERR_FILENO inside a child process [*]
+-- stdin="devnull" a child will get /dev/null as STDERR_FILENO
+--
+-- shell=true runs a child process via "sh -c" [*]
+-- shell=false invokes a child process executable directly
+--
+-- [*] are default values
+--
+-- Examples:
+--
+-- popen = require('fio').run({argv={"date"},flags={stdout=true}})
+-- popen:read()
+-- popen:close()
+--
+-- Execute 'date' command inside a shell, read the result
+-- and close the popen object
+--
+-- popen = require('fio').run({argv={"/usr/bin/echo", "-n", "hello"},
+-- flags={stdout=true, shell=false}})
+-- popen:read()
+-- popen:close()
+--
+-- Execute /usr/bin/echo with arguments '-n','hello' directly
+-- without using a shell, read the result and close the popen
+-- object
+--
+fio.run = function(opts)
+ local flags = popen_default_flags()
+
+ if opts == nil or type(opts) ~= 'table' then
+ error("Usage: fio.run({argv={},envp={},mode={},flags={}})")
+ end
+
+ -- Test for required arguments
+ if opts["argv"] == nil then
+ error(string.format("fio.run: argv key is missing"))
+ end
+
+ -- Prepare options
+ for k,v in pairs(opts) do
+ if k == "mode" then
+ flags = parse_popen_mode("fio.run", flags, v)
+ elseif k == "argv" then
+ if #v == 0 then
+ error(string.format("fio.run: argv key is empty"))
+ end
+ elseif k == "flags" then
+ flags = parse_popen_flags("fio.run", flags, opts["flags"])
+ end
+ end
+
+ -- Update parsed flags
+ opts["flags"] = flags
+
+ -- We will need a number of args for speed sake
+ opts["argc"] = #opts["argv"]
+
+ -- Same applies to environment (note though
+ -- that env={} is pretty valid and env=nil
+ -- as well.
+ if opts["env"] ~= nil then
+ opts["envc"] = #opts["env"]
+ else
+ opts["envc"] = -1
+ end
+
+ return popen_new(opts)
+end
+
return fio
--
2.20.1
More information about the Tarantool-patches
mailing list