From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: From: Vladimir Davydov Subject: [PATCH 2/2] wal: remove old xlog files asynchronously Date: Fri, 25 Jan 2019 17:34:49 +0300 Message-Id: <8e429f4b7a335ff318ca86b1147b194d1ede9c02.1548425270.git.vdavydov.dev@gmail.com> In-Reply-To: <783662fb698347f972a57b925bf13f227b710d63.1548425270.git.vdavydov.dev@gmail.com> References: <783662fb698347f972a57b925bf13f227b710d63.1548425270.git.vdavydov.dev@gmail.com> In-Reply-To: <783662fb698347f972a57b925bf13f227b710d63.1548425270.git.vdavydov.dev@gmail.com> References: <783662fb698347f972a57b925bf13f227b710d63.1548425270.git.vdavydov.dev@gmail.com> To: tarantool-patches@freelists.org List-ID: In contrast to TX thread, WAL thread performs garbage collection synchronously, blocking all concurrent writes. We expected file removal to happen instantly so we didn't bother to offload this job to eio threads. However, it turned out that sometimes removal of a single xlog file can take 50 or even 100 ms. If there are a dozen files to be removed, this means a second delay and 'too long WAL write' warnings. To fix this issue, let's make WAL garbage collection fully asynchronous. Simply submit a jobs to eio and assume it will successfully complete sooner or later. This means that if unlink() fails for some reason, we will log an error and never retry file removal until the server is restarted. Not a big deal. We can live with it assuming unlink() doesn't normally fail. Closes #3938 --- src/box/memtx_engine.c | 2 +- src/box/vy_log.c | 2 +- src/box/wal.c | 3 ++- src/box/xlog.c | 32 +++++++++++++++++++++----------- src/box/xlog.h | 2 +- 5 files changed, 26 insertions(+), 15 deletions(-) diff --git a/src/box/memtx_engine.c b/src/box/memtx_engine.c index c024fa1a..692e41ef 100644 --- a/src/box/memtx_engine.c +++ b/src/box/memtx_engine.c @@ -826,7 +826,7 @@ memtx_engine_collect_garbage(struct engine *engine, const struct vclock *vclock) { struct memtx_engine *memtx = (struct memtx_engine *)engine; xdir_collect_garbage(&memtx->snap_dir, vclock_sum(vclock), - XDIR_GC_USE_COIO); + XDIR_GC_ASYNC); } static int diff --git a/src/box/vy_log.c b/src/box/vy_log.c index d3fa0c7a..11c763ce 100644 --- a/src/box/vy_log.c +++ b/src/box/vy_log.c @@ -1074,7 +1074,7 @@ vy_log_collect_garbage(const struct vclock *vclock) vclock = vy_log_prev_checkpoint(vclock); if (vclock == NULL) return; - xdir_collect_garbage(&vy_log.dir, vclock_sum(vclock), XDIR_GC_USE_COIO); + xdir_collect_garbage(&vy_log.dir, vclock_sum(vclock), XDIR_GC_ASYNC); } int64_t diff --git a/src/box/wal.c b/src/box/wal.c index 07ea76ed..b1da75fd 100644 --- a/src/box/wal.c +++ b/src/box/wal.c @@ -687,7 +687,8 @@ wal_collect_garbage_f(struct cbus_call_msg *data) vclock = vclockset_psearch(&writer->wal_dir.index, vclock); } if (vclock != NULL) - xdir_collect_garbage(&writer->wal_dir, vclock_sum(vclock), 0); + xdir_collect_garbage(&writer->wal_dir, vclock_sum(vclock), + XDIR_GC_ASYNC); return 0; } diff --git a/src/box/xlog.c b/src/box/xlog.c index 6d2d96cd..bd5614f6 100644 --- a/src/box/xlog.c +++ b/src/box/xlog.c @@ -663,6 +663,24 @@ xdir_format_filename(struct xdir *dir, int64_t signature, return filename; } +static void +xdir_say_gc(int result, int errorno, const char *filename) +{ + if (result == 0) { + say_info("removed %s", filename); + } else if (errorno != ENOENT) { + errno = errorno; + say_syserror("error while removing %s", filename); + } +} + +static int +xdir_complete_gc(eio_req *req) +{ + xdir_say_gc(req->result, req->errorno, EIO_PATH(req)); + return 0; +} + void xdir_collect_garbage(struct xdir *dir, int64_t signature, unsigned flags) { @@ -671,18 +689,10 @@ xdir_collect_garbage(struct xdir *dir, int64_t signature, unsigned flags) vclock_sum(vclock) < signature) { char *filename = xdir_format_filename(dir, vclock_sum(vclock), NONE); - int rc; - if (flags & XDIR_GC_USE_COIO) - rc = coio_unlink(filename); + if (flags & XDIR_GC_ASYNC) + eio_unlink(filename, 0, xdir_complete_gc, NULL); else - rc = unlink(filename); - if (rc < 0) { - if (errno != ENOENT) { - say_syserror("error while removing %s", - filename); - } - } else - say_info("removed %s", filename); + xdir_say_gc(unlink(filename), errno, filename); vclockset_remove(&dir->index, vclock); free(vclock); diff --git a/src/box/xlog.h b/src/box/xlog.h index 918ec06d..6b45c860 100644 --- a/src/box/xlog.h +++ b/src/box/xlog.h @@ -200,7 +200,7 @@ enum { * Delete files in coio threads so as not to block * the caller thread. */ - XDIR_GC_USE_COIO = 1 << 0, + XDIR_GC_ASYNC = 1 << 0, /** * Return after removing a file. */ -- 2.11.0