From: Mikulas Patocka <mpatocka@redhat.com>
Date: Tue, 25 May 2021 19:53:32 +0000 (-0400)
Subject: dm: improve kcopyd latency
X-Git-Url: https://www.infradead.org/git/?a=commitdiff_plain;h=004ba76083d1702e2e18a5dbf1a990df55355666;p=users%2Fjedix%2Flinux-maple.git

dm: improve kcopyd latency

Introduce DM_KCOPYD_EARLY_CALLBACK flag that targets may set when
calling dm_kcopyd_copy(). When DM_KCOPYD_EARLY_CALLBACK is set the
completion is called from the interrupt context instead of process
context.

This change doesn't increase interrupts, it just handles completion
early without having to finish doing so in process context. Similar
changes were done to improve latency in dm-crypt recently (to avoid
workqueues and complete in interrupt context).

Update both the DM thinp and cache targets to use this flag to reduce
IO latency.

Signed-off-by: Mikulas Patocka <mpatocka@redhat.com>
Signed-off-by: Mike Snitzer <snitzer@redhat.com>
---

diff --git a/drivers/md/dm-cache-target.c b/drivers/md/dm-cache-target.c
index 6ab01ff257470..d62ec0380c398 100644
--- a/drivers/md/dm-cache-target.c
+++ b/drivers/md/dm-cache-target.c
@@ -1170,10 +1170,13 @@ static void copy(struct dm_cache_migration *mg, bool promote)
 	c_region.sector = from_cblock(mg->op->cblock) * cache->sectors_per_block;
 	c_region.count = cache->sectors_per_block;
 
-	if (promote)
-		dm_kcopyd_copy(cache->copier, &o_region, 1, &c_region, 0, copy_complete, &mg->k);
-	else
-		dm_kcopyd_copy(cache->copier, &c_region, 1, &o_region, 0, copy_complete, &mg->k);
+	if (promote) {
+		dm_kcopyd_copy(cache->copier, &o_region, 1, &c_region,
+			       BIT(DM_KCOPYD_EARLY_CALLBACK), copy_complete, &mg->k);
+	} else {
+		dm_kcopyd_copy(cache->copier, &c_region, 1, &o_region,
+			       BIT(DM_KCOPYD_EARLY_CALLBACK), copy_complete, &mg->k);
+	}
 }
 
 static void bio_drop_shared_lock(struct cache *cache, struct bio *bio)
diff --git a/drivers/md/dm-kcopyd.c b/drivers/md/dm-kcopyd.c
index e50625ce74ec5..c91bf0a2369a1 100644
--- a/drivers/md/dm-kcopyd.c
+++ b/drivers/md/dm-kcopyd.c
@@ -500,7 +500,8 @@ static int run_complete_job(struct kcopyd_job *job)
 		mutex_destroy(&job->lock);
 		mempool_free(job, &kc->job_pool);
 	}
-	fn(read_err, write_err, context);
+	if (fn)
+		fn(read_err, write_err, context);
 
 	if (atomic_dec_and_test(&kc->nr_jobs))
 		wake_up(&kc->destroyq);
@@ -530,10 +531,13 @@ static void complete_io(unsigned long error, void *context)
 		}
 	}
 
-	if (op_is_write(job->rw))
+	if (op_is_write(job->rw)) {
+		if (job->flags & BIT(DM_KCOPYD_EARLY_CALLBACK)) {
+			job->fn(job->read_err, job->write_err, job->context);
+			job->fn = NULL;
+		}
 		push(&kc->complete_jobs, job);
-
-	else {
+	} else {
 		job->rw = WRITE;
 		push(&kc->io_jobs, job);
 	}
@@ -732,6 +736,7 @@ static void segment_complete(int read_err, unsigned long write_err,
 			sub_job->dests[i].count = count;
 		}
 
+		sub_job->flags &= ~BIT(DM_KCOPYD_EARLY_CALLBACK);
 		sub_job->fn = segment_complete;
 		sub_job->context = sub_job;
 		dispatch_job(sub_job);
diff --git a/drivers/md/dm-thin.c b/drivers/md/dm-thin.c
index 985baee3a678e..031d60318e1e1 100644
--- a/drivers/md/dm-thin.c
+++ b/drivers/md/dm-thin.c
@@ -1359,7 +1359,7 @@ static void schedule_copy(struct thin_c *tc, dm_block_t virt_block,
 		to.count = len;
 
 		dm_kcopyd_copy(pool->copier, &from, 1, &to,
-			       0, copy_complete, m);
+			       BIT(DM_KCOPYD_EARLY_CALLBACK), copy_complete, m);
 
 		/*
 		 * Do we need to zero a tail region?
diff --git a/include/linux/dm-kcopyd.h b/include/linux/dm-kcopyd.h
index e42de7750c884..55c388ab35a2d 100644
--- a/include/linux/dm-kcopyd.h
+++ b/include/linux/dm-kcopyd.h
@@ -19,8 +19,9 @@
 /* FIXME: make this configurable */
 #define DM_KCOPYD_MAX_REGIONS 8
 
-#define DM_KCOPYD_IGNORE_ERROR 1
-#define DM_KCOPYD_WRITE_SEQ    2
+#define DM_KCOPYD_IGNORE_ERROR		1
+#define DM_KCOPYD_WRITE_SEQ		2
+#define DM_KCOPYD_EARLY_CALLBACK	3
 
 struct dm_kcopyd_throttle {
 	unsigned throttle;