[Tarantool-patches] [PATCH v2 4/5] popen/fio: Add ability to run external programs

Cyrill Gorcunov gorcunov at gmail.com
Tue Dec 10 12:48:54 MSK 2019


In the patch we add support for running external
programs via fio module.

The interface is the following:

 - To create new popen object either call fio.popen,
   or fio.run; the difference is that fio.popen uses
   simplified arguments list (like fio.popen(command, "rw"))
   while fio.run accepts a table of options which will
   be extended in future if needed

 - To read/write popen:read()/write() methods are supported,
   and read() supports timeout argument which implemented
   via internal poll() call and allows to test for data present
   on a child peer

 - To interrupt children work and exit one can use popen:term()
   which is soft signal and can be ignored by a child, or
   popen:kill() to force exit. Note that signals are not handled
   immediately so the next method is needed

 - To wait until child is terminated popen:wait() should be
   used. It waits in a cycle so be carefull here -- if child
   get signaled with popen:term() it might ignore the signal
   and popen:wait() will spin forever

 - To fetch information about current state of a child process
   use popen:info() method

 - Once popen no longer needed the popen:close() should be
   called to free all resources associated with the object

Part-of #4031

Signed-off-by: Cyrill Gorcunov <gorcunov at gmail.com>
---
 src/lua/fio.lua | 578 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 578 insertions(+)

diff --git a/src/lua/fio.lua b/src/lua/fio.lua
index cb224f3d0..b613cfe8c 100644
--- a/src/lua/fio.lua
+++ b/src/lua/fio.lua
@@ -5,6 +5,7 @@ local ffi = require('ffi')
 local buffer = require('buffer')
 local fiber = require('fiber')
 local errno = require('errno')
+local bit = require('bit')
 
 ffi.cdef[[
     int umask(int mask);
@@ -519,4 +520,581 @@ fio.path.lexists = function(filename)
     return fio.lstat(filename) ~= nil
 end
 
+local popen_methods = { }
+
+--
+-- Make sure the bits are matching POPEN_FLAG_x
+-- flags in popen header. This is api but internal
+-- one, we should not make it visible to users.
+local popen_flag_bits = {
+    STDIN           = 0,
+    STDOUT          = 1,
+    STDERR          = 2,
+
+    STDIN_DEVNULL   = 3,
+    STDOUT_DEVNULL  = 4,
+    STDERR_DEVNULL  = 5,
+
+    STDIN_CLOSE     = 6,
+    STDOUT_CLOSE    = 7,
+    STDERR_CLOSE    = 8,
+
+    SHELL           = 9,
+}
+
+local popen_flags = {
+    NONE            = 0,
+    STDIN           = bit.lshift(1, popen_flag_bits.STDIN),
+    STDOUT          = bit.lshift(1, popen_flag_bits.STDOUT),
+    STDERR          = bit.lshift(1, popen_flag_bits.STDERR),
+
+    STDIN_DEVNULL   = bit.lshift(1, popen_flag_bits.STDIN_DEVNULL),
+    STDOUT_DEVNULL  = bit.lshift(1, popen_flag_bits.STDOUT_DEVNULL),
+    STDERR_DEVNULL  = bit.lshift(1, popen_flag_bits.STDERR_DEVNULL),
+
+    STDIN_CLOSE     = bit.lshift(1, popen_flag_bits.STDIN_CLOSE),
+    STDOUT_CLOSE    = bit.lshift(1, popen_flag_bits.STDOUT_CLOSE),
+    STDERR_CLOSE    = bit.lshift(1, popen_flag_bits.STDERR_CLOSE),
+
+    SHELL           = bit.lshift(1, popen_flag_bits.SHELL),
+}
+
+--
+-- Get default flags for popen
+local function popen_default_flags()
+    local flags = popen_flags.NONE
+
+    -- default flags: close everything and use shell
+    flags = bit.bor(flags, popen_flags.STDIN_CLOSE)
+    flags = bit.bor(flags, popen_flags.STDOUT_CLOSE)
+    flags = bit.bor(flags, popen_flags.STDERR_CLOSE)
+    flags = bit.bor(flags, popen_flags.SHELL)
+
+    return flags
+end
+
+--
+-- Close a popen object and release all resources.
+-- In case if there is a running child process
+-- it will be killed.
+--
+-- Returns @ret = true if popen handle closed, and
+-- @ret = false, @err ~= nil on error.
+popen_methods.close = function(self)
+    local ret, err = internal.popen_delete(self.popen_handle)
+    if err ~= nil then
+	    return false, err
+    end
+    self.popen_handle = nil
+    return true
+end
+
+--
+-- Kill a child process
+--
+-- Returns @ret = true on success,
+-- @ret = false, @err ~= nil on error.
+popen_methods.kill = function(self)
+    return internal.popen_kill(self.popen_handle)
+end
+
+--
+-- Terminate a child process
+--
+-- Returns @ret = true on success,
+-- @ret = false, @err ~= nil on error.
+popen_methods.terminate = function(self)
+    return internal.popen_term(self.popen_handle)
+end
+
+--
+-- Fetch a child process state
+--
+-- Returns @err = nil, @state = (1 - if exited,
+-- 2 - if killed by a signal) and @exit_code ~= nil,
+-- otherwise @err ~= nil.
+--
+-- FIXME: Should the @state be a named constant?
+popen_methods.state = function(self)
+    return internal.popen_state(self.popen_handle)
+end
+
+--
+-- Wait until a child process finished its work
+-- and exit then.
+--
+-- Since it goes via coio thread we can't simply
+-- call wait4 (or similar) directly to not block
+-- the whole i/o subsystem. Instead we do an explicit
+-- polling for status change with predefined period
+-- of sleeping.
+--
+-- Returns the same as popen_methods.status.
+popen_methods.wait = function(self)
+    local err, state, code
+    while true do
+        err, state, code = internal.popen_state(self.popen_handle)
+        if err or state then
+            break
+        end
+        fiber.sleep(self.wait_secs)
+    end
+    return err, state, code
+end
+
+--
+-- A map for popen option keys into tfn ('true|false|nil') values
+-- where bits are just set without additional manipulations.
+--
+-- For example stdin=true means to open stdin end for write,
+-- stdin=false to close the end, finally stdin[=nil] means to
+-- provide /dev/null into a child peer.
+local popen_opts_tfn = {
+    stdin = {
+        popen_flags.STDIN,
+        popen_flags.STDIN_CLOSE,
+        popen_flags.STDIN_DEVNULL,
+    },
+    stdout = {
+        popen_flags.STDOUT,
+        popen_flags.STDOUT_CLOSE,
+        popen_flags.STDOUT_DEVNULL,
+    },
+    stderr = {
+        popen_flags.STDERR,
+        popen_flags.STDERR_CLOSE,
+        popen_flags.STDERR_DEVNULL,
+    },
+}
+
+--
+-- A map for popen option keys into tf ('true|false') values
+-- where bits are set on 'true' and clear on 'false'.
+local popen_opts_tf = {
+    shell = {
+        popen_flags.SHELL,
+    },
+}
+
+--
+-- Parses flags options from popen_opts_tfn and
+-- popen_opts_tf tables.
+local function parse_popen_flags(epfx, flags, opts)
+    if opts == nil then
+        return flags
+    end
+    for k,v in pairs(opts) do
+        if popen_opts_tfn[k] == nil then
+            if popen_opts_tf[k] == nil then
+                error(string.format("%s: Unknown key %s", epfx, k))
+            end
+            if v == true then
+                flags = bit.bor(flags, popen_opts_tf[k][1])
+            elseif v == false then
+                flags = bit.band(flags, bit.bnot(popen_opts_tf[k][1]))
+            else
+                error(string.format("%s: Unknown value %s", epfx, v))
+            end
+        else
+            if v == true then
+                flags = bit.band(flags, bit.bnot(popen_opts_tfn[k][2]))
+                flags = bit.band(flags, bit.bnot(popen_opts_tfn[k][3]))
+                flags = bit.bor(flags, popen_opts_tfn[k][1])
+            elseif v == false then
+                flags = bit.band(flags, bit.bnot(popen_opts_tfn[k][1]))
+                flags = bit.band(flags, bit.bnot(popen_opts_tfn[k][3]))
+                flags = bit.bor(flags, popen_opts_tfn[k][2])
+            elseif v == "devnull" then
+                flags = bit.band(flags, bit.bnot(popen_opts_tfn[k][1]))
+                flags = bit.band(flags, bit.bnot(popen_opts_tfn[k][2]))
+                flags = bit.bor(flags, popen_opts_tfn[k][3])
+            else
+                error(string.format("%s: Unknown value %s", epfx, v))
+            end
+        end
+    end
+    return flags
+end
+
+--
+-- popen:read2 - read stream of a child with options
+-- @opts:       options table
+--
+-- The options should have the following keys
+--
+-- @flags:          stdout=true or stderr=true
+-- @buf:            const_char_ptr_t buffer
+-- @size:           size of the buffer
+-- @timeout_msecs:  read timeout in microsecond
+--
+-- Returns @res = bytes, err = nil in case if read processed
+-- without errors, @res = nil, @err = nil if timeout elapsed
+-- and no data appeared on the peer, @res = nil, @err ~= nil
+-- otherwise.
+popen_methods.read2 = function(self, opts)
+    --
+    -- We can reuse parse_popen_flags since only
+    -- a few flags we're interesed in -- stdout/stderr
+    local flags = parse_popen_flags("fio.popen:read2",
+                                    popen_flags.NONE,
+                                    opts['flags'])
+    local timeout_msecs = -1
+    if opts['buf'] == nil then
+        error("fio.popen:read2 {'buf'} key is missed")
+    elseif opts['size'] == nil then
+        error("fio.popen:read2 {'size'} key is missed")
+    elseif opts['timeout_msecs'] ~= nil then
+        timeout_msecs = tonumber(opts['timeout_msecs'])
+    end
+
+    return internal.popen_read(self.popen_handle, opts['buf'],
+                               tonumber(opts['size']),
+                               timeout_msecs, flags)
+end
+
+-- popen:write2 - write data to a child with options
+-- @opts:       options table
+--
+-- The options should have the following keys
+--
+-- @flags:      stdin=true
+-- @buf:        const_char_ptr_t buffer
+-- @size:       size of the buffer
+--
+-- Returns @res = bytes, err = nil in case if write processed
+-- without errors, or @res = nil, @err ~= nil otherwise.
+popen_methods.write2 = function(self, opts)
+    local flags = parse_popen_flags("fio.popen:write2",
+                                    popen_flags.NONE,
+                                    opts['flags'])
+    if opts['buf'] == nil then
+        error("fio.popen:write2 {'buf'} key is missed")
+    elseif opts['size'] == nil then
+        error("fio.popen:write2 {'size'} key is missed")
+    end
+    return internal.popen_write(self.popen_handle, opts['buf'],
+                                tonumber(opts['size']), flags)
+end
+
+--
+-- Write data to stdin of a child process
+-- @buf:        a buffer to write
+-- @size:       size of the buffer
+--
+-- Both parameters are optional and some
+-- string could be passed instead
+--
+-- write(str)
+-- write(buf, len)
+--
+-- Returns @res = bytes, @err = nil in case of success,
+-- @res = , @err ~= nil in case of error.
+popen_methods.write = function(self, buf, size)
+    if not ffi.istype(const_char_ptr_t, buf) then
+        buf = tostring(buf)
+        size = #buf
+    end
+    return self:write2({ buf = buf, size = size,
+                       flags = { stdin = true }})
+end
+
+--
+-- popen:read - read the output stream in blocked mode
+-- @buf:            a buffer to read to, optional
+-- @size:           size of the buffer, optional
+-- @opts:           options table, optional
+--
+-- The options may have the following keys:
+--
+-- @flags:          stdout=true or stderr=true,
+--                  to specify which stream to read;
+--                  by default stdout is used
+-- @timeout_msecs:  read timeout in microcesonds, when
+--                  specified the read will be interrupted
+--                  after specified time elapced, to make
+--                  nonblocking read; de default the read
+--                  is blocking operation
+--
+-- The following variants are accepted as well
+--
+--  read()          -> str
+--  read(buf)       -> len
+--  read(size)      -> str
+--  read(buf, size) -> len
+--
+-- FIXME: Should we merged it with plain fio.write()?
+popen_methods.read = function(self, buf, size, opts)
+    local tmpbuf
+
+    -- read() or read(buf)
+    if (not ffi.istype(const_char_ptr_t, buf) and buf == nil) or
+        (ffi.istype(const_char_ptr_t, buf) and size == nil) then
+        -- read @self.read_buf_size bytes at once if nothing specified
+        -- FIXME: should it be configurable on popen creation?
+        size = self.read_buf_size
+    end
+
+    -- read(size)
+    if not ffi.istype(const_char_ptr_t, buf) then
+        size = buf or size
+        tmpbuf = buffer.ibuf()
+        buf = tmpbuf:reserve(size)
+    end
+
+    if opts == nil then
+        opts = {
+            timeout_msecs = -1,
+            flags = { stdout = true },
+        }
+    end
+
+    local res, err = self:read2({ buf = buf, size = size,
+                                timeout_msecs = opts['timeout_msecs'],
+                                flags = opts['flags'] })
+
+    if res == nil then
+        if tmpbuf ~= nil then
+            tmpbuf:recycle()
+        end
+        return nil, err
+    end
+
+    if tmpbuf ~= nil then
+        tmpbuf:alloc(res)
+        res = ffi.string(tmpbuf.rpos, tmpbuf:size())
+        tmpbuf:recycle()
+    end
+
+    return res
+end
+
+--
+-- popen:info -- obtain information about a popen object
+--
+-- Returns @info table, err == nil on success,
+-- @info = nil, @err ~= nil otherwise.
+--
+-- The @info table contains the following keys:
+--
+-- @pid:        pid of a child process
+-- @flags:      flags associated (popen_flags bitmap)
+-- @stdout:     parent peer for stdout, -1 if closed
+-- @stderr:     parent peer for stderr, -1 if closed
+-- @stdin:      parent peer for stdin, -1 if closed
+-- @state:      alive | exited | signaled | unknown
+-- @exit_code:  exit code of a child process if been
+--              exited or killed by a signal
+--
+-- The child should never be in "unknown" state, reserved
+-- for unexpected errors.
+--
+popen_methods.info = function(self)
+    return internal.popen_info(self.popen_handle)
+end
+
+local popen_mt = { __index = popen_methods }
+
+--
+-- Parse "mode" string to flags
+local function parse_popen_mode(epfx, flags, mode)
+    if mode == nil then
+        return flags
+    end
+    for i = 1, #mode do
+        local c = mode:sub(i, i)
+        if c == 'r' then
+            flags = bit.band(flags, bit.bnot(popen_flags.STDOUT_CLOSE))
+            flags = bit.bor(flags, popen_flags.STDOUT)
+        elseif c == 'w' then
+            flags = bit.band(flags, bit.bnot(popen_flags.STDIN_CLOSE))
+            flags = bit.bor(flags, popen_flags.STDIN)
+        else
+            error(string.format("%s: Unknown mode %s", epfx, c))
+        end
+    end
+    return flags
+end
+
+--
+-- Create a new popen object from options
+local function popen_new(opts)
+    local handle, err = internal.popen_new(opts)
+    if err ~= nil then
+        return nil, err
+    end
+
+    local popen_handle = {
+        -- a handle itself for future use
+        popen_handle = handle,
+
+        -- sleeping period for the @wait method
+        wait_secs = 1,
+
+        -- size of a read buffer to allocate
+        -- in case of implicit read
+        read_buf_size = 512,
+    }
+
+    setmetatable(popen_handle, popen_mt)
+    return popen_handle
+end
+
+--
+-- fio.popen - create a new child process and execute a command inside
+-- @command:    a command to run
+-- @mode:       r - to grab child's stdout for read,
+--              w - to obtain stdin for write
+-- @...:        optional parameters table:
+--
+--  stdin=true      alias for @mode=w
+--  stdin=false     to close STDIN_FILENO inside a child process [*]
+--  stdin="devnull" a child will get /dev/null as STDIN_FILENO
+--
+--  stdout=true     alias for @mode=r
+--  stdout=false    to close STDOUT_FILENO inside a child process [*]
+--  stdin="devnull" a child will get /dev/null as STDOUT_FILENO
+--
+--  stderr=true     to read STDERR_FILENO from a child process
+--  stderr=false    to close STDERR_FILENO inside a child process [*]
+--  stdin="devnull" a child will get /dev/null as STDERR_FILENO
+--
+--  shell=true      runs a child process via "sh -c" [*]
+--  shell=false     invokes a child process executable directly
+--
+-- [*] are default values
+--
+-- Examples:
+--
+--  popen = require('fio').popen("date", "r")
+--      runs date as "sh -c date" to read the output,
+--      closing all file descriptors except stdin inside
+--      a child process
+--
+--  popen = require('fio').popen("/usr/bin/date", "r", {shell=false})
+--      invokes date executable without shell to read the output
+--
+fio.popen = function(command, mode, ...)
+    local flags = popen_default_flags()
+
+    if type(command) ~= 'string' then
+        error("Usage: fio.popen(command[, rw, {opts}])")
+    end
+
+    -- Mode gives simplified flags
+    flags = parse_popen_mode("fio.popen", flags, mode)
+
+    -- Options table can override the mode option
+    if ... ~= nil then
+        flags = parse_popen_flags("fio.popen", flags, ...)
+    end
+
+    local opts = {
+        argv    = { command },
+        argc    = 1,
+        flags   = flags,
+        envc    = -1,
+    }
+
+    return popen_new(opts)
+end
+
+--
+-- fio.run - spawn a new process and run a command inside
+-- @opt:    table of options
+--
+-- @opts carries of the following options:
+--
+--  @argv:  an array of a program to run with
+--          command line options, mandatory
+--
+--  @env:   an array of environment variables to
+--          be used inside a process, if not
+--          set then the current environment is
+--          inherited, if set to an empty array
+--          then the environment will be dropped
+--
+--  @mode:  "r" to read from stdout, "w" to
+--          write to stdin of a process
+--
+--  @flags: a dictionary to describe communication pipes
+--          and other parameters of a process to run
+--
+--      stdin=true      to write into STDIN_FILENO of a child process
+--      stdin=false     to close STDIN_FILENO inside a child process [*]
+--      stdin="devnull" a child will get /dev/null as STDIN_FILENO
+--
+--      stdout=true     to write into STDOUT_FILENO of a child process
+--      stdout=false    to close STDOUT_FILENO inside a child process [*]
+--      stdin="devnull" a child will get /dev/null as STDOUT_FILENO
+--
+--      stderr=true     to read STDERR_FILENO from a child process
+--      stderr=false    to close STDERR_FILENO inside a child process [*]
+--      stdin="devnull" a child will get /dev/null as STDERR_FILENO
+--
+--      shell=true      runs a child process via "sh -c" [*]
+--      shell=false     invokes a child process executable directly
+--
+-- [*] are default values
+--
+-- Examples:
+--
+--  popen = require('fio').run({argv={"date"},flags={stdout=true}})
+--  popen:read()
+--  popen:close()
+--
+--      Execute 'date' command inside a shell, read the result
+--      and close the popen object
+--
+--  popen = require('fio').run({argv={"/usr/bin/echo", "-n", "hello"},
+--                             flags={stdout=true, shell=false}})
+--  popen:read()
+--  popen:close()
+--
+--      Execute /usr/bin/echo with arguments '-n','hello' directly
+--      without using a shell, read the result and close the popen
+--      object
+--
+fio.run = function(opts)
+    local flags = popen_default_flags()
+
+    if opts == nil or type(opts) ~= 'table' then
+        error("Usage: fio.run({argv={},envp={},mode={},flags={}})")
+    end
+
+    -- Test for required arguments
+    if opts["argv"] == nil then
+        error(string.format("fio.run: argv key is missing"))
+    end
+
+    -- Prepare options
+    for k,v in pairs(opts) do
+        if k == "mode" then
+            flags = parse_popen_mode("fio.run", flags, v)
+        elseif k == "argv" then
+            if #v == 0 then
+                error(string.format("fio.run: argv key is empty"))
+            end
+        elseif k == "flags" then
+            flags = parse_popen_flags("fio.run", flags, opts["flags"])
+        end
+    end
+
+    -- Update parsed flags
+    opts["flags"] = flags
+
+    -- We will need a number of args for speed sake
+    opts["argc"] = #opts["argv"]
+
+    -- Same applies to environment (note though
+    -- that env={} is pretty valid and env=nil
+    -- as well.
+    if opts["env"] ~= nil then
+        opts["envc"] = #opts["env"]
+    else
+        opts["envc"] = -1
+    end
+
+    return popen_new(opts)
+end
+
 return fio
-- 
2.20.1



More information about the Tarantool-patches mailing list