From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mail-lf1-f67.google.com (mail-lf1-f67.google.com [209.85.167.67]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by dev.tarantool.org (Postfix) with ESMTPS id D7FD54696C1 for ; Thu, 28 Nov 2019 23:46:13 +0300 (MSK) Received: by mail-lf1-f67.google.com with SMTP id r15so18003046lff.2 for ; Thu, 28 Nov 2019 12:46:13 -0800 (PST) From: Cyrill Gorcunov Date: Thu, 28 Nov 2019 23:45:11 +0300 Message-Id: <20191128204512.19732-5-gorcunov@gmail.com> In-Reply-To: <20191128204512.19732-1-gorcunov@gmail.com> References: <20191128204512.19732-1-gorcunov@gmail.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit Subject: [Tarantool-patches] [PATCH 4/5] popen/fio: Implement lua interface for a popen object List-Id: Tarantool development patches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , To: tml In the patch we implement lua interface for popen management. It provides the following methods: - fio.popen() to create a new popen object which basically runs an external process - popen:info() to fetch information about a process - popen:read() to read stdout or stderr of a child process, including passing timeout in microseconds to interrupt read procedure if no data present on the other peer - popen:write() to write into stdin of a child process - popen:kill() to kill the process - popen:wait() to wait until the process get exited Part-of #4031 Signed-off-by: Cyrill Gorcunov --- src/lua/fio.lua | 434 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 434 insertions(+) diff --git a/src/lua/fio.lua b/src/lua/fio.lua index cb224f3d0..df436c6f1 100644 --- a/src/lua/fio.lua +++ b/src/lua/fio.lua @@ -5,6 +5,7 @@ local ffi = require('ffi') local buffer = require('buffer') local fiber = require('fiber') local errno = require('errno') +local bit = require('bit') ffi.cdef[[ int umask(int mask); @@ -519,4 +520,437 @@ fio.path.lexists = function(filename) return fio.lstat(filename) ~= nil end +local popen_methods = { } + +-- +-- Make sure the bits are matching POPEN_FLAG_x +-- flags in popen header. This is api but internal +-- one, we should not make it visible to users. +local popen_flag_bits = { + STDIN = 0, + STDOUT = 1, + STDERR = 2, + + STDIN_DEVNULL = 3, + STDOUT_DEVNULL = 4, + STDERR_DEVNULL = 5, + + STDIN_CLOSE = 6, + STDOUT_CLOSE = 7, + STDERR_CLOSE = 8, + + SHELL = 9, +} + +local popen_flags = { + NONE = 0, + STDIN = bit.lshift(1, popen_flag_bits.STDIN), + STDOUT = bit.lshift(1, popen_flag_bits.STDOUT), + STDERR = bit.lshift(1, popen_flag_bits.STDERR), + + STDIN_DEVNULL = bit.lshift(1, popen_flag_bits.STDIN_DEVNULL), + STDOUT_DEVNULL = bit.lshift(1, popen_flag_bits.STDOUT_DEVNULL), + STDERR_DEVNULL = bit.lshift(1, popen_flag_bits.STDERR_DEVNULL), + + STDIN_CLOSE = bit.lshift(1, popen_flag_bits.STDIN_CLOSE), + STDOUT_CLOSE = bit.lshift(1, popen_flag_bits.STDOUT_CLOSE), + STDERR_CLOSE = bit.lshift(1, popen_flag_bits.STDERR_CLOSE), + + SHELL = bit.lshift(1, popen_flag_bits.SHELL), +} + +-- +-- Close a popen object and release all resources. +-- In case if there is a running child process +-- it will be killed. +-- +-- Returns @ret = true if popen handle closed, and +-- @ret = false, @err ~= nil on error. +popen_methods.close = function(self) + local ret, err = internal.popen_destroy(self.popen_handle) + if ret == true and err == nil then + self.popen_handle = nil + end + return ret, err +end + +-- +-- Kill a child process +-- +-- Returns @ret = true on success, +-- @ret = false, @err ~= nil on error. +popen_methods.kill = function(self) + return internal.popen_kill(self.popen_handle) +end + +-- +-- Fetch a child process status +-- +-- Returns @err = nil, @reason = (1 - if exited, +-- 2 - if killed by a signal) and @exit_code ~= nil, +-- otherwise @err ~= nil. +-- +-- FIXME: Should the @reason be a named constant? +popen_methods.status = function(self) + return internal.popen_status(self.popen_handle) +end + +-- +-- Wait until a child process finished its work +-- and exit then. +-- +-- Since it goes via coio thread we can't simply +-- call wait4 (or similar) directly to not block +-- the whole i/o subsystem. Instead we do an explicit +-- polling for status change with predefined period +-- of sleeping. +-- +-- Returns the same as popen_methods.status. +popen_methods.wait = function(self) + local err, reason, code + while true do + err, reason, code = internal.popen_status(self.popen_handle) + if err or reason then + break + end + fiber.sleep(self.wait_secs) + end + return err, reason, code +end + +-- +-- A map for popen option keys into tfn ('true|false|nil') values +-- where bits are just set without additional manipulations. +-- +-- For example stdin=true means to open stdin end for write, +-- stdin=false to close the end, finally stdin[=nil] means to +-- provide /dev/null into a child peer. +local popen_opts_tfn = { + stdin = { + popen_flags.STDIN, + popen_flags.STDIN_CLOSE, + popen_flags.STDIN_DEVNULL, + }, + stdout = { + popen_flags.STDOUT, + popen_flags.STDOUT_CLOSE, + popen_flags.STDOUT_DEVNULL, + }, + stderr = { + popen_flags.STDERR, + popen_flags.STDERR_CLOSE, + popen_flags.STDERR_DEVNULL, + }, +} + +-- +-- A map for popen option keys into tf ('true|false') values +-- where bits are set on 'true' and clear on 'false'. +local popen_opts_tf = { + shell = { + popen_flags.SHELL, + }, +} + +-- +-- Parses opts table with keys from popen_opts_tfn. +-- Values could be true, false or nil. +local function parse_popen_opts(flags, opts) + if opts == nil then + return flags + end + for k,v in pairs(opts) do + if popen_opts_tfn[k] == nil then + if popen_opts_tf[k] == nil then + error(string.format("fio.popen: Unknown key %s", k)) + end + if v == true then + flags = bit.bor(flags, popen_opts_tf[k][1]) + elseif v == false then + flags = bit.band(flags, bit.bnot(popen_opts_tf[k][1])) + else + error(string.format("fio.popen: Unknown value %s", v)) + end + else + if v == true then + flags = bit.band(flags, bit.bnot(popen_opts_tfn[k][2])) + flags = bit.band(flags, bit.bnot(popen_opts_tfn[k][3])) + flags = bit.bor(flags, popen_opts_tfn[k][1]) + elseif v == false then + flags = bit.band(flags, bit.bnot(popen_opts_tfn[k][1])) + flags = bit.band(flags, bit.bnot(popen_opts_tfn[k][3])) + flags = bit.bor(flags, popen_opts_tfn[k][2]) + elseif v == "devnull" then + flags = bit.band(flags, bit.bnot(popen_opts_tfn[k][1])) + flags = bit.band(flags, bit.bnot(popen_opts_tfn[k][2])) + flags = bit.bor(flags, popen_opts_tfn[k][3]) + else + error(string.format("fio.popen: Unknown value %s", v)) + end + end + end + return flags +end + +-- +-- popen:read2 - read stream of a child with options +-- @opts: options table +-- +-- The options should have the following keys +-- +-- @flags: stdout=true or stderr=true +-- @buf: const_char_ptr_t buffer +-- @size: size of the buffer +-- @timeout_msecs: read timeout in microsecond +-- +-- Returns @res = bytes, err = nil in case if read processed +-- without errors, @res = nil, @err = nil if timeout elapsed +-- and no data appeared on the peer, @res = nil, @err ~= nil +-- otherwise. +popen_methods.read2 = function(self, opts) + -- + -- We can reuse parse_popen_opts since only + -- a few flags we're interesed in -- stdout/stderr + local flags = parse_popen_opts(popen_flags.NONE, opts['flags']) + local timeout_msecs = -1 + if opts['buf'] == nil then + error("fio.popen:read2 {'buf'} key is missed") + elseif opts['size'] == nil then + error("fio.popen:read2 {'size'} key is missed") + elseif opts['timeout_msecs'] ~= nil then + timeout_msecs = tonumber(opts['timeout_msecs']) + end + + return internal.popen_read(self.popen_handle, opts['buf'], + tonumber(opts['size']), + timeout_msecs, flags) +end + +-- popen:write2 - write data to a child with options +-- @opts: options table +-- +-- The options should have the following keys +-- +-- @flags: stdin=true +-- @buf: const_char_ptr_t buffer +-- @size: size of the buffer +-- +-- Returns @res = bytes, err = nil in case if write processed +-- without errors, or @res = nil, @err ~= nil otherwise. +popen_methods.write2 = function(self, opts) + local flags = parse_popen_opts(popen_flags.NONE, opts['flags']) + if opts['buf'] == nil then + error("fio.popen:write2 {'buf'} key is missed") + elseif opts['size'] == nil then + error("fio.popen:write2 {'size'} key is missed") + end + return internal.popen_write(self.popen_handle, opts['buf'], + tonumber(opts['size']), flags) +end + +-- +-- Write data to stdin of a child process +-- @buf: a buffer to write +-- @size: size of the buffer +-- +-- Both parameters are optional and some +-- string could be passed instead +-- +-- write(str) +-- write(buf, len) +-- +-- Returns @res = bytes, @err = nil in case of success, +-- @res = , @err ~= nil in case of error. +popen_methods.write = function(self, buf, size) + if not ffi.istype(const_char_ptr_t, buf) then + buf = tostring(buf) + size = #buf + end + return self:write2({ buf = buf, size = size, + flags = { stdin = true }}) +end + +-- +-- popen:read - read the output stream in blocked mode +-- @buf: a buffer to read to, optional +-- @size: size of the buffer, optional +-- @opts: options table, optional +-- +-- The options may have the following keys: +-- +-- @flags: stdout=true or stderr=true, +-- to specify which stream to read; +-- by default stdout is used +-- @timeout_msecs: read timeout in microcesonds, when +-- specified the read will be interrupted +-- after specified time elapced, to make +-- nonblocking read; de default the read +-- is blocking operation +-- +-- The following variants are accepted as well +-- +-- read() -> str +-- read(buf) -> len +-- read(size) -> str +-- read(buf, size) -> len +-- +-- FIXME: Should we merged it with plain fio.write()? +popen_methods.read = function(self, buf, size, opts) + local tmpbuf + + -- read() or read(buf) + if (not ffi.istype(const_char_ptr_t, buf) and buf == nil) or + (ffi.istype(const_char_ptr_t, buf) and size == nil) then + -- read @self.read_buf_size bytes at once if nothing specified + -- FIXME: should it be configurable on popen creation? + size = self.read_buf_size + end + + -- read(size) + if not ffi.istype(const_char_ptr_t, buf) then + size = buf or size + tmpbuf = buffer.ibuf() + buf = tmpbuf:reserve(size) + end + + if opts == nil then + opts = { + timeout_msecs = -1, + flags = { stdout = true }, + } + end + + local res, err = self:read2({ buf = buf, size = size, + timeout_msecs = opts['timeout_msecs'], + flags = opts['flags'] }) + + if res == nil then + if tmpbuf ~= nil then + tmpbuf:recycle() + end + return nil, err + end + + if tmpbuf ~= nil then + tmpbuf:alloc(res) + res = ffi.string(tmpbuf.rpos, tmpbuf:size()) + tmpbuf:recycle() + end + + return res +end + +-- +-- popen:info -- obtain information about a popen object +-- +-- Returns @info table, err == nil on success, +-- @info = nil, @err ~= nil otherwise. +-- +-- The @info table contains the following keys: +-- +-- @pid: pid of a child process +-- @flags: flags associated (popen_flags bitmap) +-- @stdout: parent peer for stdout, -1 if closed +-- @stderr: parent peer for stderr, -1 if closed +-- @stdin: parent peer for stdin, -1 if closed +-- @state: alive | exited | signaled | unknown +-- @exit_code: exit code of a child process if been +-- exited or killed by a signal +-- +-- The child should never be in "unknown" state, reserved +-- for unexpected errors. +-- +popen_methods.info = function(self) + return internal.popen_info(self.popen_handle) +end + +local popen_mt = { __index = popen_methods } + +-- +-- fio.popen - create a new child process and execute a command inside +-- @command: a command to run +-- @mode: r - to grab child's stdout for read, +-- w - to obtain stdin for write +-- @...: optional parameters table: +-- +-- stdin=true alias for @mode=w +-- stdin=false to close STDIN_FILENO inside a child process [*] +-- stdin="devnull" the child will get /dev/null as STDIN_FILENO +-- +-- stdout=true alias for @mode=r +-- stdout=false to close STDOUT_FILENO inside a child process [*] +-- stdin="devnull" the child will get /dev/null as STDOUT_FILENO +-- +-- stderr=true to read STDERR_FILENO from a child process +-- stderr=false to close STDERR_FILENO inside a child process [*] +-- stdin="devnull" the child will get /dev/null as STDERR_FILENO +-- +-- shell=true runs a child process via "sh -c" [*] +-- shell=false invokes a child process executable directly +-- +-- [*] are default values +-- +-- Examples: +-- +-- popen = require('fio').popen("date", "r") +-- runs date as "sh -c date" to read the output, +-- closing all file descriptors except stdin inside +-- a child process +-- +-- popen = require('fio').popen("/usr/bin/date", "r", {shell=false}) +-- invokes date executable without shell to read the output +-- +fio.popen = function(command, mode, ...) + local flags = popen_flags.NONE + + if type(command) ~= 'string' then + error("Usage: fio.popen(command[, rw, {opts}])") + end + + -- default flags: close everything and use shell + flags = bit.bor(flags, popen_flags.STDIN_CLOSE) + flags = bit.bor(flags, popen_flags.STDOUT_CLOSE) + flags = bit.bor(flags, popen_flags.STDERR_CLOSE) + flags = bit.bor(flags, popen_flags.SHELL) + + for i = 1, #mode do + local c = mode:sub(i, i) + if c == 'r' then + flags = bit.band(flags, bit.bnot(popen_flags.STDOUT_CLOSE)) + flags = bit.bor(flags, popen_flags.STDOUT) + elseif c == 'w' then + flags = bit.band(flags, bit.bnot(popen_flags.STDIN_CLOSE)) + flags = bit.bor(flags, popen_flags.STDIN) + else + error(string.format("fio.popen: Unknown mode %s", c)) + end + end + + -- + -- Options table can override the mode option + if ... ~= nil then + flags = parse_popen_opts(flags, ...) + end + + local handle, err = internal.popen_create(command, flags) + if err ~= nil then + return nil, err + end + + local popen_handle = { + -- a handle itself for future use + popen_handle = handle, + + -- sleeping period for the @wait method + wait_secs = 1, + + -- size of a read buffer to allocate + -- in case of implicit read + read_buf_size = 512, + } + + setmetatable(popen_handle, popen_mt) + return popen_handle +end + return fio -- 2.20.1