dm: improve kcopyd latency

Introduce DM_KCOPYD_EARLY_CALLBACK flag that targets may set when
calling dm_kcopyd_copy(). When DM_KCOPYD_EARLY_CALLBACK is set the
completion is called from the interrupt context instead of process
context.

This change doesn't increase interrupts, it just handles completion
early without having to finish doing so in process context. Similar
changes were done to improve latency in dm-crypt recently (to avoid
workqueues and complete in interrupt context).

Update both the DM thinp and cache targets to use this flag to reduce
IO latency.

Signed-off-by: Mikulas Patocka <mpatocka@redhat.com>
Signed-off-by: Mike Snitzer <snitzer@redhat.com>

diff --git a/drivers/md/dm-cache-target.c b/drivers/md/dm-cache-target.c
index 6ab01ff2574707a4f46c4aac28435ec807a2c1ed..d62ec0380c3987a9b78d5f06e494e6f0beede86e 100644 (file)
--- a/drivers/md/dm-cache-target.c
+++ b/drivers/md/dm-cache-target.c
@@ -1170,10 +1170,13 @@ static void copy(struct dm_cache_migration *mg, bool promote)
        c_region.sector = from_cblock(mg->op->cblock) * cache->sectors_per_block;
        c_region.count = cache->sectors_per_block;
 
-       if (promote)
-               dm_kcopyd_copy(cache->copier, &o_region, 1, &c_region, 0, copy_complete, &mg->k);
-       else
-               dm_kcopyd_copy(cache->copier, &c_region, 1, &o_region, 0, copy_complete, &mg->k);
+       if (promote) {
+               dm_kcopyd_copy(cache->copier, &o_region, 1, &c_region,
+                              BIT(DM_KCOPYD_EARLY_CALLBACK), copy_complete, &mg->k);
+       } else {
+               dm_kcopyd_copy(cache->copier, &c_region, 1, &o_region,
+                              BIT(DM_KCOPYD_EARLY_CALLBACK), copy_complete, &mg->k);
+       }
 }
 
 static void bio_drop_shared_lock(struct cache *cache, struct bio *bio)

diff --git a/drivers/md/dm-kcopyd.c b/drivers/md/dm-kcopyd.c
index e50625ce74ec5df03433ce9422145159af591612..c91bf0a2369a1ba92f67540ac7644a61608a2661 100644 (file)
--- a/drivers/md/dm-kcopyd.c
+++ b/drivers/md/dm-kcopyd.c
@@ -500,7 +500,8 @@ static int run_complete_job(struct kcopyd_job *job)
                mutex_destroy(&job->lock);
                mempool_free(job, &kc->job_pool);
        }
-       fn(read_err, write_err, context);
+       if (fn)
+               fn(read_err, write_err, context);
 
        if (atomic_dec_and_test(&kc->nr_jobs))
                wake_up(&kc->destroyq);
@@ -530,10 +531,13 @@ static void complete_io(unsigned long error, void *context)
                }
        }
 
-       if (op_is_write(job->rw))
+       if (op_is_write(job->rw)) {
+               if (job->flags & BIT(DM_KCOPYD_EARLY_CALLBACK)) {
+                       job->fn(job->read_err, job->write_err, job->context);
+                       job->fn = NULL;
+               }
                push(&kc->complete_jobs, job);
-
-       else {
+       } else {
                job->rw = WRITE;
                push(&kc->io_jobs, job);
        }
@@ -732,6 +736,7 @@ static void segment_complete(int read_err, unsigned long write_err,
                        sub_job->dests[i].count = count;
                }
 
+               sub_job->flags &= ~BIT(DM_KCOPYD_EARLY_CALLBACK);
                sub_job->fn = segment_complete;
                sub_job->context = sub_job;
                dispatch_job(sub_job);

diff --git a/drivers/md/dm-thin.c b/drivers/md/dm-thin.c
index 985baee3a678e8fa57a7d3a21212d67c7514d182..031d60318e1e153b3f21dedc722f8559851e25c0 100644 (file)
--- a/drivers/md/dm-thin.c
+++ b/drivers/md/dm-thin.c
@@ -1359,7 +1359,7 @@ static void schedule_copy(struct thin_c *tc, dm_block_t virt_block,
                to.count = len;
 
                dm_kcopyd_copy(pool->copier, &from, 1, &to,
-                              0, copy_complete, m);
+                              BIT(DM_KCOPYD_EARLY_CALLBACK), copy_complete, m);
 
                /*
                 * Do we need to zero a tail region?

diff --git a/include/linux/dm-kcopyd.h b/include/linux/dm-kcopyd.h
index e42de7750c884ec6f7c56e250db963bbc154d9f9..55c388ab35a2d7d29157d1a6f1a3c8b73b68cd46 100644 (file)
--- a/include/linux/dm-kcopyd.h
+++ b/include/linux/dm-kcopyd.h
@@ -19,8 +19,9 @@
 /* FIXME: make this configurable */
 #define DM_KCOPYD_MAX_REGIONS 8
 
-#define DM_KCOPYD_IGNORE_ERROR 1
-#define DM_KCOPYD_WRITE_SEQ    2
+#define DM_KCOPYD_IGNORE_ERROR         1
+#define DM_KCOPYD_WRITE_SEQ            2
+#define DM_KCOPYD_EARLY_CALLBACK       3
 
 struct dm_kcopyd_throttle {
        unsigned throttle;