From 004ba76083d1702e2e18a5dbf1a990df55355666 Mon Sep 17 00:00:00 2001 From: Mikulas Patocka Date: Tue, 25 May 2021 15:53:32 -0400 Subject: [PATCH] dm: improve kcopyd latency Introduce DM_KCOPYD_EARLY_CALLBACK flag that targets may set when calling dm_kcopyd_copy(). When DM_KCOPYD_EARLY_CALLBACK is set the completion is called from the interrupt context instead of process context. This change doesn't increase interrupts, it just handles completion early without having to finish doing so in process context. Similar changes were done to improve latency in dm-crypt recently (to avoid workqueues and complete in interrupt context). Update both the DM thinp and cache targets to use this flag to reduce IO latency. Signed-off-by: Mikulas Patocka Signed-off-by: Mike Snitzer --- drivers/md/dm-cache-target.c | 11 +++++++---- drivers/md/dm-kcopyd.c | 13 +++++++++---- drivers/md/dm-thin.c | 2 +- include/linux/dm-kcopyd.h | 5 +++-- 4 files changed, 20 insertions(+), 11 deletions(-) diff --git a/drivers/md/dm-cache-target.c b/drivers/md/dm-cache-target.c index 6ab01ff25747..d62ec0380c39 100644 --- a/drivers/md/dm-cache-target.c +++ b/drivers/md/dm-cache-target.c @@ -1170,10 +1170,13 @@ static void copy(struct dm_cache_migration *mg, bool promote) c_region.sector = from_cblock(mg->op->cblock) * cache->sectors_per_block; c_region.count = cache->sectors_per_block; - if (promote) - dm_kcopyd_copy(cache->copier, &o_region, 1, &c_region, 0, copy_complete, &mg->k); - else - dm_kcopyd_copy(cache->copier, &c_region, 1, &o_region, 0, copy_complete, &mg->k); + if (promote) { + dm_kcopyd_copy(cache->copier, &o_region, 1, &c_region, + BIT(DM_KCOPYD_EARLY_CALLBACK), copy_complete, &mg->k); + } else { + dm_kcopyd_copy(cache->copier, &c_region, 1, &o_region, + BIT(DM_KCOPYD_EARLY_CALLBACK), copy_complete, &mg->k); + } } static void bio_drop_shared_lock(struct cache *cache, struct bio *bio) diff --git a/drivers/md/dm-kcopyd.c b/drivers/md/dm-kcopyd.c index e50625ce74ec..c91bf0a2369a 100644 --- a/drivers/md/dm-kcopyd.c +++ b/drivers/md/dm-kcopyd.c @@ -500,7 +500,8 @@ static int run_complete_job(struct kcopyd_job *job) mutex_destroy(&job->lock); mempool_free(job, &kc->job_pool); } - fn(read_err, write_err, context); + if (fn) + fn(read_err, write_err, context); if (atomic_dec_and_test(&kc->nr_jobs)) wake_up(&kc->destroyq); @@ -530,10 +531,13 @@ static void complete_io(unsigned long error, void *context) } } - if (op_is_write(job->rw)) + if (op_is_write(job->rw)) { + if (job->flags & BIT(DM_KCOPYD_EARLY_CALLBACK)) { + job->fn(job->read_err, job->write_err, job->context); + job->fn = NULL; + } push(&kc->complete_jobs, job); - - else { + } else { job->rw = WRITE; push(&kc->io_jobs, job); } @@ -732,6 +736,7 @@ static void segment_complete(int read_err, unsigned long write_err, sub_job->dests[i].count = count; } + sub_job->flags &= ~BIT(DM_KCOPYD_EARLY_CALLBACK); sub_job->fn = segment_complete; sub_job->context = sub_job; dispatch_job(sub_job); diff --git a/drivers/md/dm-thin.c b/drivers/md/dm-thin.c index 985baee3a678..031d60318e1e 100644 --- a/drivers/md/dm-thin.c +++ b/drivers/md/dm-thin.c @@ -1359,7 +1359,7 @@ static void schedule_copy(struct thin_c *tc, dm_block_t virt_block, to.count = len; dm_kcopyd_copy(pool->copier, &from, 1, &to, - 0, copy_complete, m); + BIT(DM_KCOPYD_EARLY_CALLBACK), copy_complete, m); /* * Do we need to zero a tail region? diff --git a/include/linux/dm-kcopyd.h b/include/linux/dm-kcopyd.h index e42de7750c88..55c388ab35a2 100644 --- a/include/linux/dm-kcopyd.h +++ b/include/linux/dm-kcopyd.h @@ -19,8 +19,9 @@ /* FIXME: make this configurable */ #define DM_KCOPYD_MAX_REGIONS 8 -#define DM_KCOPYD_IGNORE_ERROR 1 -#define DM_KCOPYD_WRITE_SEQ 2 +#define DM_KCOPYD_IGNORE_ERROR 1 +#define DM_KCOPYD_WRITE_SEQ 2 +#define DM_KCOPYD_EARLY_CALLBACK 3 struct dm_kcopyd_throttle { unsigned throttle; -- 2.50.1