[PATCH v3 0/3] core/coio_file: Use eio_sendfile_sync for file copying

[PATCH v3 0/3] core/coio_file: Use eio_sendfile_sync for file copying

[Cyrill Gorcunov]

Please take a look once time permit.

v3:
 - Use off_t to count bytes transferred
 - Provide test with forcing 1K transfer at once

Cyrill Gorcunov (3):
  core/coio_file: Use eio_sendfile_sync instead of a chunk mode
  core/coio_file: Add ERRINJ_COIO_SENDFILE_CHUNK error injection
  test: app/fio -- Add ERRINJ_COIO_SENDFILE_CHUNK

 src/lib/core/coio_file.c | 33 ++++++++++++++++++---------------
 src/lib/core/errinj.h    |  1 +
 test/app/fio.result      | 18 ++++++++++++++++++
 test/app/fio.test.lua    |  5 +++++
 4 files changed, 42 insertions(+), 15 deletions(-)

-- 
2.20.1

[PATCH 1/3] core/coio_file: Use eio_sendfile_sync instead of a chunk mode

[Cyrill Gorcunov]

eio library provides a portable version of sendfile syscall
which works a way more efficient than explicit copying file
by 4K chunks.

Signed-off-by: Cyrill Gorcunov <gorcunov@gmail.com>
---
 src/lib/core/coio_file.c | 26 +++++++++++---------------
 1 file changed, 11 insertions(+), 15 deletions(-)

diff --git a/src/lib/core/coio_file.c b/src/lib/core/coio_file.c
index c5b2db781..20053558e 100644
--- a/src/lib/core/coio_file.c
+++ b/src/lib/core/coio_file.c
@@ -571,7 +571,7 @@ static void
 coio_do_copyfile(eio_req *req)
 {
 	struct coio_file_task *eio = (struct coio_file_task *)req->data;
-
+	off_t pos, ret, left;
 	struct stat st;
 	if (stat(eio->copyfile.source, &st) < 0) {
 		goto error;
@@ -588,22 +588,18 @@ coio_do_copyfile(eio_req *req)
 		goto error_dest;
 	}
 
-	enum { COPY_FILE_BUF_SIZE = 4096 };
-
-	char buf[COPY_FILE_BUF_SIZE];
-
-	while (true) {
-		ssize_t nread = fio_read(source_fd, buf, sizeof(buf));
-		if (nread < 0)
-			goto error_copy;
-
-		if (nread == 0)
-			break; /* eof */
-
-		ssize_t nwritten = fio_writen(dest_fd, buf, nread);
-		if (nwritten < 0)
+	for (left = st.st_size, pos = 0; left > 0;) {
+		ret = eio_sendfile_sync(dest_fd, source_fd, pos, left);
+		if (ret < 0) {
+			say_syserror("sendfile, [%s -> %s]",
+				     fio_filename(source_fd),
+				     fio_filename(dest_fd));
 			goto error_copy;
+		}
+		pos += ret;
+		left -= ret;
 	}
+
 	req->result = 0;
 	close(source_fd);
 	close(dest_fd);
-- 
2.20.1

Re: [PATCH 1/3] core/coio_file: Use eio_sendfile_sync instead of a chunk mode

[Vladimir Davydov]

On Tue, Apr 16, 2019 at 11:08:56PM +0300, Cyrill Gorcunov wrote:
> eio library provides a portable version of sendfile syscall
> which works a way more efficient than explicit copying file
> by 4K chunks.
> 
> Signed-off-by: Cyrill Gorcunov <gorcunov@gmail.com>
> ---
>  src/lib/core/coio_file.c | 26 +++++++++++---------------
>  1 file changed, 11 insertions(+), 15 deletions(-)

The patch looks fine, but we typically don't split tests from the code
when submitting a patch, i.e. I'd squash all the three patches in one.

Also, after having added a new error injection, you have to update
box/errinj.result. Please do.

Also, please check that app/fio.test.lua works fine when Tarantool is
built in Release mode. The thing is we typically disable tests using
error injections for Release builds, because they don't work there
(see suite.ini:release_disabled). I assume the test case you submitted
should pass even without the error injection, nevertheless better check
that.

> 
> diff --git a/src/lib/core/coio_file.c b/src/lib/core/coio_file.c
> index c5b2db781..20053558e 100644
> --- a/src/lib/core/coio_file.c
> +++ b/src/lib/core/coio_file.c
> @@ -571,7 +571,7 @@ static void
>  coio_do_copyfile(eio_req *req)
>  {
>  	struct coio_file_task *eio = (struct coio_file_task *)req->data;
> -
> +	off_t pos, ret, left;

Nit: We don't have a requirement to define all variables in the
beginning of a code block. I'd define these variables right before
the 'for' loop where they are used.

>  	struct stat st;
>  	if (stat(eio->copyfile.source, &st) < 0) {
>  		goto error;
> @@ -588,22 +588,18 @@ coio_do_copyfile(eio_req *req)
>  		goto error_dest;
>  	}
>  
> -	enum { COPY_FILE_BUF_SIZE = 4096 };
> -
> -	char buf[COPY_FILE_BUF_SIZE];
> -
> -	while (true) {
> -		ssize_t nread = fio_read(source_fd, buf, sizeof(buf));
> -		if (nread < 0)
> -			goto error_copy;
> -
> -		if (nread == 0)
> -			break; /* eof */
> -
> -		ssize_t nwritten = fio_writen(dest_fd, buf, nread);
> -		if (nwritten < 0)
> +	for (left = st.st_size, pos = 0; left > 0;) {
> +		ret = eio_sendfile_sync(dest_fd, source_fd, pos, left);
> +		if (ret < 0) {
> +			say_syserror("sendfile, [%s -> %s]",
> +				     fio_filename(source_fd),
> +				     fio_filename(dest_fd));
>  			goto error_copy;
> +		}
> +		pos += ret;
> +		left -= ret;
>  	}
> +
>  	req->result = 0;
>  	close(source_fd);
>  	close(dest_fd);

Re: [PATCH 1/3] core/coio_file: Use eio_sendfile_sync instead of a chunk mode

[Cyrill Gorcunov]

On Thu, Apr 18, 2019 at 09:41:55PM +0300, Vladimir Davydov wrote:
> 
> The patch looks fine, but we typically don't split tests from the code
> when submitting a patch, i.e. I'd squash all the three patches in one.

ok

> Also, after having added a new error injection, you have to update
> box/errinj.result. Please do.

Ah, will do, tahnks!

> Also, please check that app/fio.test.lua works fine when Tarantool is
> built in Release mode. The thing is we typically disable tests using
> error injections for Release builds, because they don't work there
> (see suite.ini:release_disabled). I assume the test case you submitted
> should pass even without the error injection, nevertheless better check
> that.

Sure, thanks!

> > diff --git a/src/lib/core/coio_file.c b/src/lib/core/coio_file.c
> > index c5b2db781..20053558e 100644
> > --- a/src/lib/core/coio_file.c
> > +++ b/src/lib/core/coio_file.c
> > @@ -571,7 +571,7 @@ static void
> >  coio_do_copyfile(eio_req *req)
> >  {
> >  	struct coio_file_task *eio = (struct coio_file_task *)req->data;
> > -
> > +	off_t pos, ret, left;
> 
> Nit: We don't have a requirement to define all variables in the
> beginning of a code block. I'd define these variables right before
> the 'for' loop where they are used.

Yes, I noticed that, though personally I prefer to declare them early
(after all compiler highly likely will sub stack space at the
 procedure's start). Actually "define right before use" has a caveat:
implicit vars shadowing which is not always warned. Thus if this is
a code style violation I will change it of course, hmm?

	Cyrill

[PATCH 2/3] core/coio_file: Add ERRINJ_COIO_SENDFILE_CHUNK error injection

[Cyrill Gorcunov]

This allows us to test sendfile in a cycle.
---
 src/lib/core/coio_file.c | 11 +++++++++--
 src/lib/core/errinj.h    |  1 +
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/src/lib/core/coio_file.c b/src/lib/core/coio_file.c
index 20053558e..3caf185a5 100644
--- a/src/lib/core/coio_file.c
+++ b/src/lib/core/coio_file.c
@@ -33,6 +33,7 @@
 #include "fiber.h"
 #include "say.h"
 #include "fio.h"
+#include "errinj.h"
 #include <stdio.h>
 #include <stdlib.h>
 #include <dirent.h>
@@ -570,8 +571,9 @@ coio_readdir(const char *dir_path, char **buf)
 static void
 coio_do_copyfile(eio_req *req)
 {
+	struct errinj *inj = errinj(ERRINJ_COIO_SENDFILE_CHUNK, ERRINJ_INT);
 	struct coio_file_task *eio = (struct coio_file_task *)req->data;
-	off_t pos, ret, left;
+	off_t pos, ret, left, chunk;
 	struct stat st;
 	if (stat(eio->copyfile.source, &st) < 0) {
 		goto error;
@@ -588,8 +590,13 @@ coio_do_copyfile(eio_req *req)
 		goto error_dest;
 	}
 
+	if (inj != NULL && inj->iparam > 0)
+		chunk = (off_t)inj->iparam;
+	else
+		chunk = st.st_size;
+
 	for (left = st.st_size, pos = 0; left > 0;) {
-		ret = eio_sendfile_sync(dest_fd, source_fd, pos, left);
+		ret = eio_sendfile_sync(dest_fd, source_fd, pos, chunk);
 		if (ret < 0) {
 			say_syserror("sendfile, [%s -> %s]",
 				     fio_filename(source_fd),
diff --git a/src/lib/core/errinj.h b/src/lib/core/errinj.h
index 4bca81caf..462c98464 100644
--- a/src/lib/core/errinj.h
+++ b/src/lib/core/errinj.h
@@ -129,6 +129,7 @@ struct errinj {
 	_(ERRINJ_MEMTX_DELAY_GC, ERRINJ_BOOL, {.bparam = false}) \
 	_(ERRINJ_SIO_READ_MAX, ERRINJ_INT, {.iparam = -1}) \
 	_(ERRINJ_SQL_NAME_NORMALIZATION, ERRINJ_BOOL, {.bparam = false}) \
+	_(ERRINJ_COIO_SENDFILE_CHUNK, ERRINJ_INT, {.iparam = -1}) \
 
 ENUM0(errinj_id, ERRINJ_LIST);
 extern struct errinj errinjs[];
-- 
2.20.1

[PATCH 3/3] test: app/fio -- Add ERRINJ_COIO_SENDFILE_CHUNK

[Cyrill Gorcunov]

To test sendfile work in a cycle -- force transferring
1024 bytes at once.
---
 test/app/fio.result   | 18 ++++++++++++++++++
 test/app/fio.test.lua |  5 +++++
 2 files changed, 23 insertions(+)

diff --git a/test/app/fio.result b/test/app/fio.result
index 486cb8043..d640ab596 100644
--- a/test/app/fio.result
+++ b/test/app/fio.result
@@ -718,6 +718,9 @@ file2 = fio.pathjoin(tmp2, 'file.2')
 file3 = fio.pathjoin(tree, 'file.3')
 ---
 ...
+file4 = fio.pathjoin(tree, 'file.4')
+---
+...
 fh1 = fio.open(file1, { 'O_RDWR', 'O_TRUNC', 'O_CREAT' }, 0777)
 ---
 ...
@@ -752,6 +755,21 @@ fio.stat(fio.pathjoin(tmp2, "file.1")) ~= nil
 ---
 - true
 ...
+errinj = box.error.injection
+---
+...
+errinj.set('ERRINJ_COIO_SENDFILE_CHUNK', 1024)
+---
+- ok
+...
+fio.copyfile(file1, file4)
+---
+- true
+...
+fio.stat(file1, file4) ~= nil
+---
+- true
+...
 res, err = fio.copyfile(fio.pathjoin(tmp1, 'not_exists.txt'), tmp1)
 ---
 ...
diff --git a/test/app/fio.test.lua b/test/app/fio.test.lua
index 9af37044d..87d578991 100644
--- a/test/app/fio.test.lua
+++ b/test/app/fio.test.lua
@@ -230,6 +230,7 @@ fio.mktree(tree2, 0777)
 file1 = fio.pathjoin(tmp1, 'file.1')
 file2 = fio.pathjoin(tmp2, 'file.2')
 file3 = fio.pathjoin(tree, 'file.3')
+file4 = fio.pathjoin(tree, 'file.4')
 
 fh1 = fio.open(file1, { 'O_RDWR', 'O_TRUNC', 'O_CREAT' }, 0777)
 fh1:write("gogo")
@@ -240,6 +241,10 @@ fh1:close()
 fio.symlink(file1, file3)
 fio.copyfile(file1, tmp2)
 fio.stat(fio.pathjoin(tmp2, "file.1")) ~= nil
+errinj = box.error.injection
+errinj.set('ERRINJ_COIO_SENDFILE_CHUNK', 1024)
+fio.copyfile(file1, file4)
+fio.stat(file1, file4) ~= nil
 
 res, err = fio.copyfile(fio.pathjoin(tmp1, 'not_exists.txt'), tmp1)
 res
-- 
2.20.1

[PATCH v4 3/3] test: app/fio -- Add ERRINJ_COIO_SENDFILE_CHUNK

[Cyrill Gorcunov]

To test sendfile work in a cycle -- force transferring 1 byte at once.
---
v4: These files are quite small so test with 1 byte chunk to force
    sendfile cycle.

 test/app/fio.result   | 23 +++++++++++++++++++++++
 test/app/fio.test.lua |  8 ++++++++
 2 files changed, 31 insertions(+)

diff --git a/test/app/fio.result b/test/app/fio.result
index 486cb8043..879e0a767 100644
--- a/test/app/fio.result
+++ b/test/app/fio.result
@@ -718,6 +718,9 @@ file2 = fio.pathjoin(tmp2, 'file.2')
 file3 = fio.pathjoin(tree, 'file.3')
 ---
 ...
+file4 = fio.pathjoin(tree, 'file.4')
+---
+...
 fh1 = fio.open(file1, { 'O_RDWR', 'O_TRUNC', 'O_CREAT' }, 0777)
 ---
 ...
@@ -752,6 +755,26 @@ fio.stat(fio.pathjoin(tmp2, "file.1")) ~= nil
 ---
 - true
 ...
+--- test copyfile to operate with one byte transfer
+errinj = box.error.injection
+---
+...
+errinj.set('ERRINJ_COIO_SENDFILE_CHUNK', 1)
+---
+- ok
+...
+fio.copyfile(file1, file4)
+---
+- true
+...
+fio.stat(file1, file4) ~= nil
+---
+- true
+...
+errinj.set('ERRINJ_COIO_SENDFILE_CHUNK', -1)
+---
+- ok
+...
 res, err = fio.copyfile(fio.pathjoin(tmp1, 'not_exists.txt'), tmp1)
 ---
 ...
diff --git a/test/app/fio.test.lua b/test/app/fio.test.lua
index 9af37044d..1255b2804 100644
--- a/test/app/fio.test.lua
+++ b/test/app/fio.test.lua
@@ -230,6 +230,7 @@ fio.mktree(tree2, 0777)
 file1 = fio.pathjoin(tmp1, 'file.1')
 file2 = fio.pathjoin(tmp2, 'file.2')
 file3 = fio.pathjoin(tree, 'file.3')
+file4 = fio.pathjoin(tree, 'file.4')
 
 fh1 = fio.open(file1, { 'O_RDWR', 'O_TRUNC', 'O_CREAT' }, 0777)
 fh1:write("gogo")
@@ -241,6 +242,13 @@ fio.symlink(file1, file3)
 fio.copyfile(file1, tmp2)
 fio.stat(fio.pathjoin(tmp2, "file.1")) ~= nil
 
+--- test copyfile to operate with one byte transfer
+errinj = box.error.injection
+errinj.set('ERRINJ_COIO_SENDFILE_CHUNK', 1)
+fio.copyfile(file1, file4)
+fio.stat(file1, file4) ~= nil
+errinj.set('ERRINJ_COIO_SENDFILE_CHUNK', -1)
+
 res, err = fio.copyfile(fio.pathjoin(tmp1, 'not_exists.txt'), tmp1)
 res
 err:match("failed to copy") ~= nil
-- 
2.20.1