From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mail-lf1-f66.google.com (mail-lf1-f66.google.com [209.85.167.66]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by dev.tarantool.org (Postfix) with ESMTPS id AC65646971A for ; Tue, 10 Dec 2019 12:49:49 +0300 (MSK) Received: by mail-lf1-f66.google.com with SMTP id 15so13195150lfr.2 for ; Tue, 10 Dec 2019 01:49:49 -0800 (PST) From: Cyrill Gorcunov Date: Tue, 10 Dec 2019 12:48:54 +0300 Message-Id: <20191210094855.24953-5-gorcunov@gmail.com> In-Reply-To: <20191210094855.24953-1-gorcunov@gmail.com> References: <20191210094855.24953-1-gorcunov@gmail.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit Subject: [Tarantool-patches] [PATCH v2 4/5] popen/fio: Add ability to run external programs List-Id: Tarantool development patches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , To: tml In the patch we add support for running external programs via fio module. The interface is the following: - To create new popen object either call fio.popen, or fio.run; the difference is that fio.popen uses simplified arguments list (like fio.popen(command, "rw")) while fio.run accepts a table of options which will be extended in future if needed - To read/write popen:read()/write() methods are supported, and read() supports timeout argument which implemented via internal poll() call and allows to test for data present on a child peer - To interrupt children work and exit one can use popen:term() which is soft signal and can be ignored by a child, or popen:kill() to force exit. Note that signals are not handled immediately so the next method is needed - To wait until child is terminated popen:wait() should be used. It waits in a cycle so be carefull here -- if child get signaled with popen:term() it might ignore the signal and popen:wait() will spin forever - To fetch information about current state of a child process use popen:info() method - Once popen no longer needed the popen:close() should be called to free all resources associated with the object Part-of #4031 Signed-off-by: Cyrill Gorcunov --- src/lua/fio.lua | 578 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 578 insertions(+) diff --git a/src/lua/fio.lua b/src/lua/fio.lua index cb224f3d0..b613cfe8c 100644 --- a/src/lua/fio.lua +++ b/src/lua/fio.lua @@ -5,6 +5,7 @@ local ffi = require('ffi') local buffer = require('buffer') local fiber = require('fiber') local errno = require('errno') +local bit = require('bit') ffi.cdef[[ int umask(int mask); @@ -519,4 +520,581 @@ fio.path.lexists = function(filename) return fio.lstat(filename) ~= nil end +local popen_methods = { } + +-- +-- Make sure the bits are matching POPEN_FLAG_x +-- flags in popen header. This is api but internal +-- one, we should not make it visible to users. +local popen_flag_bits = { + STDIN = 0, + STDOUT = 1, + STDERR = 2, + + STDIN_DEVNULL = 3, + STDOUT_DEVNULL = 4, + STDERR_DEVNULL = 5, + + STDIN_CLOSE = 6, + STDOUT_CLOSE = 7, + STDERR_CLOSE = 8, + + SHELL = 9, +} + +local popen_flags = { + NONE = 0, + STDIN = bit.lshift(1, popen_flag_bits.STDIN), + STDOUT = bit.lshift(1, popen_flag_bits.STDOUT), + STDERR = bit.lshift(1, popen_flag_bits.STDERR), + + STDIN_DEVNULL = bit.lshift(1, popen_flag_bits.STDIN_DEVNULL), + STDOUT_DEVNULL = bit.lshift(1, popen_flag_bits.STDOUT_DEVNULL), + STDERR_DEVNULL = bit.lshift(1, popen_flag_bits.STDERR_DEVNULL), + + STDIN_CLOSE = bit.lshift(1, popen_flag_bits.STDIN_CLOSE), + STDOUT_CLOSE = bit.lshift(1, popen_flag_bits.STDOUT_CLOSE), + STDERR_CLOSE = bit.lshift(1, popen_flag_bits.STDERR_CLOSE), + + SHELL = bit.lshift(1, popen_flag_bits.SHELL), +} + +-- +-- Get default flags for popen +local function popen_default_flags() + local flags = popen_flags.NONE + + -- default flags: close everything and use shell + flags = bit.bor(flags, popen_flags.STDIN_CLOSE) + flags = bit.bor(flags, popen_flags.STDOUT_CLOSE) + flags = bit.bor(flags, popen_flags.STDERR_CLOSE) + flags = bit.bor(flags, popen_flags.SHELL) + + return flags +end + +-- +-- Close a popen object and release all resources. +-- In case if there is a running child process +-- it will be killed. +-- +-- Returns @ret = true if popen handle closed, and +-- @ret = false, @err ~= nil on error. +popen_methods.close = function(self) + local ret, err = internal.popen_delete(self.popen_handle) + if err ~= nil then + return false, err + end + self.popen_handle = nil + return true +end + +-- +-- Kill a child process +-- +-- Returns @ret = true on success, +-- @ret = false, @err ~= nil on error. +popen_methods.kill = function(self) + return internal.popen_kill(self.popen_handle) +end + +-- +-- Terminate a child process +-- +-- Returns @ret = true on success, +-- @ret = false, @err ~= nil on error. +popen_methods.terminate = function(self) + return internal.popen_term(self.popen_handle) +end + +-- +-- Fetch a child process state +-- +-- Returns @err = nil, @state = (1 - if exited, +-- 2 - if killed by a signal) and @exit_code ~= nil, +-- otherwise @err ~= nil. +-- +-- FIXME: Should the @state be a named constant? +popen_methods.state = function(self) + return internal.popen_state(self.popen_handle) +end + +-- +-- Wait until a child process finished its work +-- and exit then. +-- +-- Since it goes via coio thread we can't simply +-- call wait4 (or similar) directly to not block +-- the whole i/o subsystem. Instead we do an explicit +-- polling for status change with predefined period +-- of sleeping. +-- +-- Returns the same as popen_methods.status. +popen_methods.wait = function(self) + local err, state, code + while true do + err, state, code = internal.popen_state(self.popen_handle) + if err or state then + break + end + fiber.sleep(self.wait_secs) + end + return err, state, code +end + +-- +-- A map for popen option keys into tfn ('true|false|nil') values +-- where bits are just set without additional manipulations. +-- +-- For example stdin=true means to open stdin end for write, +-- stdin=false to close the end, finally stdin[=nil] means to +-- provide /dev/null into a child peer. +local popen_opts_tfn = { + stdin = { + popen_flags.STDIN, + popen_flags.STDIN_CLOSE, + popen_flags.STDIN_DEVNULL, + }, + stdout = { + popen_flags.STDOUT, + popen_flags.STDOUT_CLOSE, + popen_flags.STDOUT_DEVNULL, + }, + stderr = { + popen_flags.STDERR, + popen_flags.STDERR_CLOSE, + popen_flags.STDERR_DEVNULL, + }, +} + +-- +-- A map for popen option keys into tf ('true|false') values +-- where bits are set on 'true' and clear on 'false'. +local popen_opts_tf = { + shell = { + popen_flags.SHELL, + }, +} + +-- +-- Parses flags options from popen_opts_tfn and +-- popen_opts_tf tables. +local function parse_popen_flags(epfx, flags, opts) + if opts == nil then + return flags + end + for k,v in pairs(opts) do + if popen_opts_tfn[k] == nil then + if popen_opts_tf[k] == nil then + error(string.format("%s: Unknown key %s", epfx, k)) + end + if v == true then + flags = bit.bor(flags, popen_opts_tf[k][1]) + elseif v == false then + flags = bit.band(flags, bit.bnot(popen_opts_tf[k][1])) + else + error(string.format("%s: Unknown value %s", epfx, v)) + end + else + if v == true then + flags = bit.band(flags, bit.bnot(popen_opts_tfn[k][2])) + flags = bit.band(flags, bit.bnot(popen_opts_tfn[k][3])) + flags = bit.bor(flags, popen_opts_tfn[k][1]) + elseif v == false then + flags = bit.band(flags, bit.bnot(popen_opts_tfn[k][1])) + flags = bit.band(flags, bit.bnot(popen_opts_tfn[k][3])) + flags = bit.bor(flags, popen_opts_tfn[k][2]) + elseif v == "devnull" then + flags = bit.band(flags, bit.bnot(popen_opts_tfn[k][1])) + flags = bit.band(flags, bit.bnot(popen_opts_tfn[k][2])) + flags = bit.bor(flags, popen_opts_tfn[k][3]) + else + error(string.format("%s: Unknown value %s", epfx, v)) + end + end + end + return flags +end + +-- +-- popen:read2 - read stream of a child with options +-- @opts: options table +-- +-- The options should have the following keys +-- +-- @flags: stdout=true or stderr=true +-- @buf: const_char_ptr_t buffer +-- @size: size of the buffer +-- @timeout_msecs: read timeout in microsecond +-- +-- Returns @res = bytes, err = nil in case if read processed +-- without errors, @res = nil, @err = nil if timeout elapsed +-- and no data appeared on the peer, @res = nil, @err ~= nil +-- otherwise. +popen_methods.read2 = function(self, opts) + -- + -- We can reuse parse_popen_flags since only + -- a few flags we're interesed in -- stdout/stderr + local flags = parse_popen_flags("fio.popen:read2", + popen_flags.NONE, + opts['flags']) + local timeout_msecs = -1 + if opts['buf'] == nil then + error("fio.popen:read2 {'buf'} key is missed") + elseif opts['size'] == nil then + error("fio.popen:read2 {'size'} key is missed") + elseif opts['timeout_msecs'] ~= nil then + timeout_msecs = tonumber(opts['timeout_msecs']) + end + + return internal.popen_read(self.popen_handle, opts['buf'], + tonumber(opts['size']), + timeout_msecs, flags) +end + +-- popen:write2 - write data to a child with options +-- @opts: options table +-- +-- The options should have the following keys +-- +-- @flags: stdin=true +-- @buf: const_char_ptr_t buffer +-- @size: size of the buffer +-- +-- Returns @res = bytes, err = nil in case if write processed +-- without errors, or @res = nil, @err ~= nil otherwise. +popen_methods.write2 = function(self, opts) + local flags = parse_popen_flags("fio.popen:write2", + popen_flags.NONE, + opts['flags']) + if opts['buf'] == nil then + error("fio.popen:write2 {'buf'} key is missed") + elseif opts['size'] == nil then + error("fio.popen:write2 {'size'} key is missed") + end + return internal.popen_write(self.popen_handle, opts['buf'], + tonumber(opts['size']), flags) +end + +-- +-- Write data to stdin of a child process +-- @buf: a buffer to write +-- @size: size of the buffer +-- +-- Both parameters are optional and some +-- string could be passed instead +-- +-- write(str) +-- write(buf, len) +-- +-- Returns @res = bytes, @err = nil in case of success, +-- @res = , @err ~= nil in case of error. +popen_methods.write = function(self, buf, size) + if not ffi.istype(const_char_ptr_t, buf) then + buf = tostring(buf) + size = #buf + end + return self:write2({ buf = buf, size = size, + flags = { stdin = true }}) +end + +-- +-- popen:read - read the output stream in blocked mode +-- @buf: a buffer to read to, optional +-- @size: size of the buffer, optional +-- @opts: options table, optional +-- +-- The options may have the following keys: +-- +-- @flags: stdout=true or stderr=true, +-- to specify which stream to read; +-- by default stdout is used +-- @timeout_msecs: read timeout in microcesonds, when +-- specified the read will be interrupted +-- after specified time elapced, to make +-- nonblocking read; de default the read +-- is blocking operation +-- +-- The following variants are accepted as well +-- +-- read() -> str +-- read(buf) -> len +-- read(size) -> str +-- read(buf, size) -> len +-- +-- FIXME: Should we merged it with plain fio.write()? +popen_methods.read = function(self, buf, size, opts) + local tmpbuf + + -- read() or read(buf) + if (not ffi.istype(const_char_ptr_t, buf) and buf == nil) or + (ffi.istype(const_char_ptr_t, buf) and size == nil) then + -- read @self.read_buf_size bytes at once if nothing specified + -- FIXME: should it be configurable on popen creation? + size = self.read_buf_size + end + + -- read(size) + if not ffi.istype(const_char_ptr_t, buf) then + size = buf or size + tmpbuf = buffer.ibuf() + buf = tmpbuf:reserve(size) + end + + if opts == nil then + opts = { + timeout_msecs = -1, + flags = { stdout = true }, + } + end + + local res, err = self:read2({ buf = buf, size = size, + timeout_msecs = opts['timeout_msecs'], + flags = opts['flags'] }) + + if res == nil then + if tmpbuf ~= nil then + tmpbuf:recycle() + end + return nil, err + end + + if tmpbuf ~= nil then + tmpbuf:alloc(res) + res = ffi.string(tmpbuf.rpos, tmpbuf:size()) + tmpbuf:recycle() + end + + return res +end + +-- +-- popen:info -- obtain information about a popen object +-- +-- Returns @info table, err == nil on success, +-- @info = nil, @err ~= nil otherwise. +-- +-- The @info table contains the following keys: +-- +-- @pid: pid of a child process +-- @flags: flags associated (popen_flags bitmap) +-- @stdout: parent peer for stdout, -1 if closed +-- @stderr: parent peer for stderr, -1 if closed +-- @stdin: parent peer for stdin, -1 if closed +-- @state: alive | exited | signaled | unknown +-- @exit_code: exit code of a child process if been +-- exited or killed by a signal +-- +-- The child should never be in "unknown" state, reserved +-- for unexpected errors. +-- +popen_methods.info = function(self) + return internal.popen_info(self.popen_handle) +end + +local popen_mt = { __index = popen_methods } + +-- +-- Parse "mode" string to flags +local function parse_popen_mode(epfx, flags, mode) + if mode == nil then + return flags + end + for i = 1, #mode do + local c = mode:sub(i, i) + if c == 'r' then + flags = bit.band(flags, bit.bnot(popen_flags.STDOUT_CLOSE)) + flags = bit.bor(flags, popen_flags.STDOUT) + elseif c == 'w' then + flags = bit.band(flags, bit.bnot(popen_flags.STDIN_CLOSE)) + flags = bit.bor(flags, popen_flags.STDIN) + else + error(string.format("%s: Unknown mode %s", epfx, c)) + end + end + return flags +end + +-- +-- Create a new popen object from options +local function popen_new(opts) + local handle, err = internal.popen_new(opts) + if err ~= nil then + return nil, err + end + + local popen_handle = { + -- a handle itself for future use + popen_handle = handle, + + -- sleeping period for the @wait method + wait_secs = 1, + + -- size of a read buffer to allocate + -- in case of implicit read + read_buf_size = 512, + } + + setmetatable(popen_handle, popen_mt) + return popen_handle +end + +-- +-- fio.popen - create a new child process and execute a command inside +-- @command: a command to run +-- @mode: r - to grab child's stdout for read, +-- w - to obtain stdin for write +-- @...: optional parameters table: +-- +-- stdin=true alias for @mode=w +-- stdin=false to close STDIN_FILENO inside a child process [*] +-- stdin="devnull" a child will get /dev/null as STDIN_FILENO +-- +-- stdout=true alias for @mode=r +-- stdout=false to close STDOUT_FILENO inside a child process [*] +-- stdin="devnull" a child will get /dev/null as STDOUT_FILENO +-- +-- stderr=true to read STDERR_FILENO from a child process +-- stderr=false to close STDERR_FILENO inside a child process [*] +-- stdin="devnull" a child will get /dev/null as STDERR_FILENO +-- +-- shell=true runs a child process via "sh -c" [*] +-- shell=false invokes a child process executable directly +-- +-- [*] are default values +-- +-- Examples: +-- +-- popen = require('fio').popen("date", "r") +-- runs date as "sh -c date" to read the output, +-- closing all file descriptors except stdin inside +-- a child process +-- +-- popen = require('fio').popen("/usr/bin/date", "r", {shell=false}) +-- invokes date executable without shell to read the output +-- +fio.popen = function(command, mode, ...) + local flags = popen_default_flags() + + if type(command) ~= 'string' then + error("Usage: fio.popen(command[, rw, {opts}])") + end + + -- Mode gives simplified flags + flags = parse_popen_mode("fio.popen", flags, mode) + + -- Options table can override the mode option + if ... ~= nil then + flags = parse_popen_flags("fio.popen", flags, ...) + end + + local opts = { + argv = { command }, + argc = 1, + flags = flags, + envc = -1, + } + + return popen_new(opts) +end + +-- +-- fio.run - spawn a new process and run a command inside +-- @opt: table of options +-- +-- @opts carries of the following options: +-- +-- @argv: an array of a program to run with +-- command line options, mandatory +-- +-- @env: an array of environment variables to +-- be used inside a process, if not +-- set then the current environment is +-- inherited, if set to an empty array +-- then the environment will be dropped +-- +-- @mode: "r" to read from stdout, "w" to +-- write to stdin of a process +-- +-- @flags: a dictionary to describe communication pipes +-- and other parameters of a process to run +-- +-- stdin=true to write into STDIN_FILENO of a child process +-- stdin=false to close STDIN_FILENO inside a child process [*] +-- stdin="devnull" a child will get /dev/null as STDIN_FILENO +-- +-- stdout=true to write into STDOUT_FILENO of a child process +-- stdout=false to close STDOUT_FILENO inside a child process [*] +-- stdin="devnull" a child will get /dev/null as STDOUT_FILENO +-- +-- stderr=true to read STDERR_FILENO from a child process +-- stderr=false to close STDERR_FILENO inside a child process [*] +-- stdin="devnull" a child will get /dev/null as STDERR_FILENO +-- +-- shell=true runs a child process via "sh -c" [*] +-- shell=false invokes a child process executable directly +-- +-- [*] are default values +-- +-- Examples: +-- +-- popen = require('fio').run({argv={"date"},flags={stdout=true}}) +-- popen:read() +-- popen:close() +-- +-- Execute 'date' command inside a shell, read the result +-- and close the popen object +-- +-- popen = require('fio').run({argv={"/usr/bin/echo", "-n", "hello"}, +-- flags={stdout=true, shell=false}}) +-- popen:read() +-- popen:close() +-- +-- Execute /usr/bin/echo with arguments '-n','hello' directly +-- without using a shell, read the result and close the popen +-- object +-- +fio.run = function(opts) + local flags = popen_default_flags() + + if opts == nil or type(opts) ~= 'table' then + error("Usage: fio.run({argv={},envp={},mode={},flags={}})") + end + + -- Test for required arguments + if opts["argv"] == nil then + error(string.format("fio.run: argv key is missing")) + end + + -- Prepare options + for k,v in pairs(opts) do + if k == "mode" then + flags = parse_popen_mode("fio.run", flags, v) + elseif k == "argv" then + if #v == 0 then + error(string.format("fio.run: argv key is empty")) + end + elseif k == "flags" then + flags = parse_popen_flags("fio.run", flags, opts["flags"]) + end + end + + -- Update parsed flags + opts["flags"] = flags + + -- We will need a number of args for speed sake + opts["argc"] = #opts["argv"] + + -- Same applies to environment (note though + -- that env={} is pretty valid and env=nil + -- as well. + if opts["env"] ~= nil then + opts["envc"] = #opts["env"] + else + opts["envc"] = -1 + end + + return popen_new(opts) +end + return fio -- 2.20.1