dm: optimize flushes

Device mapper sends flush bios to all the targets and the targets send it
to the underlying device. That may be inefficient, for example if a table
contains 10 linear targets pointing to the same physical device, then
device mapper would send 10 flush bios to that device - despite the fact
that only one bio would be sufficient.

This commit optimizes the flush behavior. It introduces a per-target
variable flush_bypasses_map - it is set when the target supports flush
optimization - currently, the dm-linear and dm-stripe targets support it.
When all the targets in a table have flush_bypasses_map,
flush_bypasses_map on the table is set. __send_empty_flush tests if the
table has flush_bypasses_map - and if it has, no flush bios are sent to
the targets via the "map" method and the list dm_table->devices is
iterated and the flush bios are sent to each member of the list.

Signed-off-by: Mikulas Patocka <mpatocka@redhat.com>
Reviewed-by: Mike Snitzer <snitzer@kernel.org>
Suggested-by: Yang Yang <yang.yang@vivo.com>

diff --git a/drivers/md/dm-core.h b/drivers/md/dm-core.h
index 14a44c0f82868bdb57f3a6a62f26d25c82b5a589..3637761f35853ccf86d740c5e3feaf7088613cd7 100644 (file)
--- a/drivers/md/dm-core.h
+++ b/drivers/md/dm-core.h
@@ -206,6 +206,8 @@ struct dm_table {
 
        bool integrity_supported:1;
        bool singleton:1;
+       /* set if all the targets in the table have "flush_bypasses_map" set */
+       bool flush_bypasses_map:1;
 
        /*
         * Indicates the rw permissions for the new logical device.  This

diff --git a/drivers/md/dm-linear.c b/drivers/md/dm-linear.c
index 2d3e186ca87e3c78404005dc08484fae90a9a2e2..49fb0f6841938deec0955fa638a3c958683c91af 100644 (file)
--- a/drivers/md/dm-linear.c
+++ b/drivers/md/dm-linear.c
@@ -62,6 +62,7 @@ static int linear_ctr(struct dm_target *ti, unsigned int argc, char **argv)
        ti->num_discard_bios = 1;
        ti->num_secure_erase_bios = 1;
        ti->num_write_zeroes_bios = 1;
+       ti->flush_bypasses_map = true;
        ti->private = lc;
        return 0;
 

diff --git a/drivers/md/dm-stripe.c b/drivers/md/dm-stripe.c
index 16b93ae51d964d8268e01eaa493684a6a70d18a3..ec908cf8cfdcf0875c3e21541d2ca65cd7e6641b 100644 (file)
--- a/drivers/md/dm-stripe.c
+++ b/drivers/md/dm-stripe.c
@@ -157,6 +157,7 @@ static int stripe_ctr(struct dm_target *ti, unsigned int argc, char **argv)
        ti->num_discard_bios = stripes;
        ti->num_secure_erase_bios = stripes;
        ti->num_write_zeroes_bios = stripes;
+       ti->flush_bypasses_map = true;
 
        sc->chunk_size = chunk_size;
        if (chunk_size & (chunk_size - 1))

diff --git a/drivers/md/dm-table.c b/drivers/md/dm-table.c
index 92d18dcdb3e9ea0df643a8d564028ecf6af15744..33b7a1844ed4788daf16f11257517fbe685f4ea4 100644 (file)
--- a/drivers/md/dm-table.c
+++ b/drivers/md/dm-table.c
@@ -160,6 +160,7 @@ int dm_table_create(struct dm_table **result, blk_mode_t mode,
        t->type = DM_TYPE_NONE;
        t->mode = mode;
        t->md = md;
+       t->flush_bypasses_map = true;
        *result = t;
        return 0;
 }
@@ -748,6 +749,9 @@ int dm_table_add_target(struct dm_table *t, const char *type,
        if (ti->limit_swap_bios && !static_key_enabled(&swap_bios_enabled.key))
                static_branch_enable(&swap_bios_enabled);
 
+       if (!ti->flush_bypasses_map)
+               t->flush_bypasses_map = false;
+
        return 0;
 
  bad:

diff --git a/drivers/md/dm.c b/drivers/md/dm.c
index 7d107ae06e1ae17e53898d1633b8020432c90c82..3763b2ce557b0befd349bbfe08c4948a8744314b 100644 (file)
--- a/drivers/md/dm.c
+++ b/drivers/md/dm.c
@@ -645,7 +645,7 @@ static struct bio *alloc_tio(struct clone_info *ci, struct dm_target *ti,
 
        /* Set default bdev, but target must bio_set_dev() before issuing IO */
        clone->bi_bdev = md->disk->part0;
-       if (unlikely(ti->needs_bio_set_dev))
+       if (likely(ti != NULL) && unlikely(ti->needs_bio_set_dev))
                bio_set_dev(clone, md->disk->part0);
 
        if (len) {
@@ -1107,7 +1107,7 @@ static void clone_endio(struct bio *bio)
        blk_status_t error = bio->bi_status;
        struct dm_target_io *tio = clone_to_tio(bio);
        struct dm_target *ti = tio->ti;
-       dm_endio_fn endio = ti->type->end_io;
+       dm_endio_fn endio = likely(ti != NULL) ? ti->type->end_io : NULL;
        struct dm_io *io = tio->io;
        struct mapped_device *md = io->md;
 
@@ -1154,7 +1154,7 @@ static void clone_endio(struct bio *bio)
        }
 
        if (static_branch_unlikely(&swap_bios_enabled) &&
-           unlikely(swap_bios_limit(ti, bio)))
+           likely(ti != NULL) && unlikely(swap_bios_limit(ti, bio)))
                up(&md->swap_bios_semaphore);
 
        free_tio(bio);
@@ -1553,17 +1553,43 @@ static void __send_empty_flush(struct clone_info *ci)
        ci->sector_count = 0;
        ci->io->tio.clone.bi_iter.bi_size = 0;
 
-       for (unsigned int i = 0; i < t->num_targets; i++) {
-               unsigned int bios;
-               struct dm_target *ti = dm_table_get_target(t, i);
+       if (!t->flush_bypasses_map) {
+               for (unsigned int i = 0; i < t->num_targets; i++) {
+                       unsigned int bios;
+                       struct dm_target *ti = dm_table_get_target(t, i);
 
-               if (unlikely(ti->num_flush_bios == 0))
-                       continue;
+                       if (unlikely(ti->num_flush_bios == 0))
+                               continue;
 
-               atomic_add(ti->num_flush_bios, &ci->io->io_count);
-               bios = __send_duplicate_bios(ci, ti, ti->num_flush_bios,
-                                            NULL, GFP_NOWAIT);
-               atomic_sub(ti->num_flush_bios - bios, &ci->io->io_count);
+                       atomic_add(ti->num_flush_bios, &ci->io->io_count);
+                       bios = __send_duplicate_bios(ci, ti, ti->num_flush_bios,
+                                                    NULL, GFP_NOWAIT);
+                       atomic_sub(ti->num_flush_bios - bios, &ci->io->io_count);
+               }
+       } else {
+               /*
+                * Note that there's no need to grab t->devices_lock here
+                * because the targets that support flush optimization don't
+                * modify the list of devices.
+                */
+               struct list_head *devices = dm_table_get_devices(t);
+               unsigned int len = 0;
+               struct dm_dev_internal *dd;
+               list_for_each_entry(dd, devices, list) {
+                       struct bio *clone;
+                       /*
+                        * Note that the structure dm_target_io is not
+                        * associated with any target (because the device may be
+                        * used by multiple targets), so we set tio->ti = NULL.
+                        * We must check for NULL in the I/O processing path, to
+                        * avoid NULL pointer dereference.
+                        */
+                       clone = alloc_tio(ci, NULL, 0, &len, GFP_NOIO);
+                       atomic_add(1, &ci->io->io_count);
+                       bio_set_dev(clone, dd->dm_dev->bdev);
+                       clone->bi_end_io = clone_endio;
+                       dm_submit_bio_remap(clone, NULL);
+               }
        }
 
        /*

diff --git a/include/linux/device-mapper.h b/include/linux/device-mapper.h
index 82b2195efaca782fc06b7018edda51311297ac87..3611b230d0aa24977f7d23dade06eb7516c2b43e 100644 (file)
--- a/include/linux/device-mapper.h
+++ b/include/linux/device-mapper.h
@@ -397,6 +397,21 @@ struct dm_target {
         * bio_set_dev(). NOTE: ideally a target should _not_ need this.
         */
        bool needs_bio_set_dev:1;
+
+       /*
+        * Set if the target supports flush optimization. If all the targets in
+        * a table have flush_bypasses_map set, the dm core will not send
+        * flushes to the targets via a ->map method. It will iterate over
+        * dm_table->devices and send flushes to the devices directly. This
+        * optimization reduces the number of flushes being sent when multiple
+        * targets in a table use the same underlying device.
+        *
+        * This optimization may be enabled on targets that just pass the
+        * flushes to the underlying devices without performing any other
+        * actions on the flush request. Currently, dm-linear and dm-stripe
+        * support it.
+        */
+       bool flush_bypasses_map:1;
 };
 
 void *dm_per_bio_data(struct bio *bio, size_t data_size);