[PATCH 2/2] wal: remove old xlog files asynchronously

[Vladimir Davydov <vdavydov.dev@gmail.com>]

In contrast to TX thread, WAL thread performs garbage collection
synchronously, blocking all concurrent writes. We expected file removal
to happen instantly so we didn't bother to offload this job to eio
threads. However, it turned out that sometimes removal of a single xlog
file can take 50 or even 100 ms. If there are a dozen files to be
removed, this means a second delay and 'too long WAL write' warnings.

To fix this issue, let's make WAL garbage collection fully asynchronous.
Simply submit a jobs to eio and assume it will successfully complete
sooner or later.  This means that if unlink() fails for some reason, we
will log an error and never retry file removal until the server is
restarted. Not a big deal. We can live with it assuming unlink() doesn't
normally fail.

Closes #3938
---
 src/box/memtx_engine.c |  2 +-
 src/box/vy_log.c       |  2 +-
 src/box/wal.c          |  3 ++-
 src/box/xlog.c         | 32 +++++++++++++++++++++-----------
 src/box/xlog.h         |  2 +-
 5 files changed, 26 insertions(+), 15 deletions(-)

diff --git a/src/box/memtx_engine.c b/src/box/memtx_engine.c
index c024fa1a..692e41ef 100644
--- a/src/box/memtx_engine.c
+++ b/src/box/memtx_engine.c
@@ -826,7 +826,7 @@ memtx_engine_collect_garbage(struct engine *engine, const struct vclock *vclock)
 {
 	struct memtx_engine *memtx = (struct memtx_engine *)engine;
 	xdir_collect_garbage(&memtx->snap_dir, vclock_sum(vclock),
-			     XDIR_GC_USE_COIO);
+			     XDIR_GC_ASYNC);
 }
 
 static int
diff --git a/src/box/vy_log.c b/src/box/vy_log.c
index d3fa0c7a..11c763ce 100644
--- a/src/box/vy_log.c
+++ b/src/box/vy_log.c
@@ -1074,7 +1074,7 @@ vy_log_collect_garbage(const struct vclock *vclock)
 	vclock = vy_log_prev_checkpoint(vclock);
 	if (vclock == NULL)
 		return;
-	xdir_collect_garbage(&vy_log.dir, vclock_sum(vclock), XDIR_GC_USE_COIO);
+	xdir_collect_garbage(&vy_log.dir, vclock_sum(vclock), XDIR_GC_ASYNC);
 }
 
 int64_t
diff --git a/src/box/wal.c b/src/box/wal.c
index 07ea76ed..b1da75fd 100644
--- a/src/box/wal.c
+++ b/src/box/wal.c
@@ -687,7 +687,8 @@ wal_collect_garbage_f(struct cbus_call_msg *data)
 		vclock = vclockset_psearch(&writer->wal_dir.index, vclock);
 	}
 	if (vclock != NULL)
-		xdir_collect_garbage(&writer->wal_dir, vclock_sum(vclock), 0);
+		xdir_collect_garbage(&writer->wal_dir, vclock_sum(vclock),
+				     XDIR_GC_ASYNC);
 
 	return 0;
 }
diff --git a/src/box/xlog.c b/src/box/xlog.c
index 6d2d96cd..bd5614f6 100644
--- a/src/box/xlog.c
+++ b/src/box/xlog.c
@@ -663,6 +663,24 @@ xdir_format_filename(struct xdir *dir, int64_t signature,
 	return filename;
 }
 
+static void
+xdir_say_gc(int result, int errorno, const char *filename)
+{
+	if (result == 0) {
+		say_info("removed %s", filename);
+	} else if (errorno != ENOENT) {
+		errno = errorno;
+		say_syserror("error while removing %s", filename);
+	}
+}
+
+static int
+xdir_complete_gc(eio_req *req)
+{
+	xdir_say_gc(req->result, req->errorno, EIO_PATH(req));
+	return 0;
+}
+
 void
 xdir_collect_garbage(struct xdir *dir, int64_t signature, unsigned flags)
 {
@@ -671,18 +689,10 @@ xdir_collect_garbage(struct xdir *dir, int64_t signature, unsigned flags)
 	       vclock_sum(vclock) < signature) {
 		char *filename = xdir_format_filename(dir, vclock_sum(vclock),
 						      NONE);
-		int rc;
-		if (flags & XDIR_GC_USE_COIO)
-			rc = coio_unlink(filename);
+		if (flags & XDIR_GC_ASYNC)
+			eio_unlink(filename, 0, xdir_complete_gc, NULL);
 		else
-			rc = unlink(filename);
-		if (rc < 0) {
-			if (errno != ENOENT) {
-				say_syserror("error while removing %s",
-					     filename);
-			}
-		} else
-			say_info("removed %s", filename);
+			xdir_say_gc(unlink(filename), errno, filename);
 		vclockset_remove(&dir->index, vclock);
 		free(vclock);
 
diff --git a/src/box/xlog.h b/src/box/xlog.h
index 918ec06d..6b45c860 100644
--- a/src/box/xlog.h
+++ b/src/box/xlog.h
@@ -200,7 +200,7 @@ enum {
 	 * Delete files in coio threads so as not to block
 	 * the caller thread.
 	 */
-	XDIR_GC_USE_COIO = 1 << 0,
+	XDIR_GC_ASYNC = 1 << 0,
 	/**
 	 * Return after removing a file.
 	 */
-- 
2.11.0