mm: compaction: introduce sync-light migration for use by compaction

This patch adds a lightweight sync migrate operation MIGRATE_SYNC_LIGHT
mode that avoids writing back pages to backing storage.  Async compaction
maps to MIGRATE_ASYNC while sync compaction maps to MIGRATE_SYNC_LIGHT.
For other migrate_pages users such as memory hotplug, MIGRATE_SYNC is
used.

This avoids sync compaction stalling for an excessive length of time,
particularly when copying files to a USB stick where there might be a
large number of dirty pages backed by a filesystem that does not support
->writepages.

[aarcange@redhat.com: This patch is heavily based on Andrea's work]
[akpm@linux-foundation.org: fix fs/nfs/write.c build]
[akpm@linux-foundation.org: fix fs/btrfs/disk-io.c build]
Signed-off-by: Mel Gorman <mgorman@suse.de>
Reviewed-by: Rik van Riel <riel@redhat.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Dave Jones <davej@redhat.com>
Cc: Jan Kara <jack@suse.cz>
Cc: Andy Isaacson <adi@hexapodia.org>
Cc: Nai Xia <nai.xia@gmail.com>
Cc: Johannes Weiner <jweiner@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c
index 1375494c8cb60c691a555b947f022bb8d0a439a7..d8525662ca7a721b18e197191816ea16b095eb19 100644 (file)
--- a/fs/btrfs/disk-io.c
+++ b/fs/btrfs/disk-io.c
@@ -872,7 +872,8 @@ static int btree_submit_bio_hook(struct inode *inode, int rw, struct bio *bio,
 
 #ifdef CONFIG_MIGRATION
 static int btree_migratepage(struct address_space *mapping,
-                       struct page *newpage, struct page *page, bool sync)
+                       struct page *newpage, struct page *page,
+                       enum migrate_mode mode)
 {
        /*
         * we can't safely write a btree page from here,
@@ -887,7 +888,7 @@ static int btree_migratepage(struct address_space *mapping,
        if (page_has_private(page) &&
            !try_to_release_page(page, GFP_KERNEL))
                return -EAGAIN;
-       return migrate_page(mapping, newpage, page, sync);
+       return migrate_page(mapping, newpage, page, mode);
 }
 #endif
 

diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c
index 06fd4608a990d15a3135156455a9ea0f349583da..1e85a7ac021759da6a3ad118b5c1734db0bd1cc0 100644 (file)
--- a/fs/hugetlbfs/inode.c
+++ b/fs/hugetlbfs/inode.c
@@ -584,7 +584,7 @@ static int hugetlbfs_set_page_dirty(struct page *page)
 
 static int hugetlbfs_migrate_page(struct address_space *mapping,
                                struct page *newpage, struct page *page,
-                               bool sync)
+                               enum migrate_mode mode)
 {
        int rc;
 

diff --git a/fs/nfs/internal.h b/fs/nfs/internal.h
index 114398a158304bb932b66b15d7ecf838c503f12a..8102db9b926c2eb56d9035126fa27b33058d711f 100644 (file)
--- a/fs/nfs/internal.h
+++ b/fs/nfs/internal.h
@@ -332,7 +332,7 @@ void nfs_commit_release_pages(struct nfs_write_data *data);
 
 #ifdef CONFIG_MIGRATION
 extern int nfs_migrate_page(struct address_space *,
-               struct page *, struct page *, bool);
+               struct page *, struct page *, enum migrate_mode);
 #else
 #define nfs_migrate_page NULL
 #endif

diff --git a/fs/nfs/write.c b/fs/nfs/write.c
index 889e98bc5a21bec07f3ea0d25159b9bf1a6503cc..834f0fe96f89f4acf707df504e1a244fc56d466b 100644 (file)
--- a/fs/nfs/write.c
+++ b/fs/nfs/write.c
@@ -1688,7 +1688,7 @@ out_error:
 
 #ifdef CONFIG_MIGRATION
 int nfs_migrate_page(struct address_space *mapping, struct page *newpage,
-               struct page *page, bool sync)
+               struct page *page, enum migrate_mode mode)
 {
        /*
         * If PagePrivate is set, then the page is currently associated with
@@ -1703,7 +1703,7 @@ int nfs_migrate_page(struct address_space *mapping, struct page *newpage,
 
        nfs_fscache_release_page(page, GFP_KERNEL);
 
-       return migrate_page(mapping, newpage, page, sync);
+       return migrate_page(mapping, newpage, page, mode);
 }
 #endif
 

diff --git a/include/linux/fs.h b/include/linux/fs.h
index b92b73d0b2b9e4e840769a29e367cfd72d552e9c..e694bd4434a417241613b12d93b2009598e76bc6 100644 (file)
--- a/include/linux/fs.h
+++ b/include/linux/fs.h
@@ -525,6 +525,7 @@ enum positive_aop_returns {
 struct page;
 struct address_space;
 struct writeback_control;
+enum migrate_mode;
 
 struct iov_iter {
        const struct iovec *iov;
@@ -614,7 +615,7 @@ struct address_space_operations {
         * is false, it must not block.
         */
        int (*migratepage) (struct address_space *,
-                       struct page *, struct page *, bool);
+                       struct page *, struct page *, enum migrate_mode);
        int (*launder_page) (struct page *);
        int (*is_partially_uptodate) (struct page *, read_descriptor_t *,
                                        unsigned long);
@@ -2540,7 +2541,8 @@ extern int generic_check_addressable(unsigned, u64);
 
 #ifdef CONFIG_MIGRATION
 extern int buffer_migrate_page(struct address_space *,
-                               struct page *, struct page *, bool);
+                               struct page *, struct page *,
+                               enum migrate_mode);
 #else
 #define buffer_migrate_page NULL
 #endif

diff --git a/include/linux/migrate.h b/include/linux/migrate.h
index 14e6d2a884752db4d74843d6b171731e76468af9..eaf867412f7adf26e96a3ad4eb3f002a418434e4 100644 (file)
--- a/include/linux/migrate.h
+++ b/include/linux/migrate.h
@@ -6,18 +6,31 @@
 
 typedef struct page *new_page_t(struct page *, unsigned long private, int **);
 
+/*
+ * MIGRATE_ASYNC means never block
+ * MIGRATE_SYNC_LIGHT in the current implementation means to allow blocking
+ *     on most operations but not ->writepage as the potential stall time
+ *     is too significant
+ * MIGRATE_SYNC will block when migrating pages
+ */
+enum migrate_mode {
+       MIGRATE_ASYNC,
+       MIGRATE_SYNC_LIGHT,
+       MIGRATE_SYNC,
+};
+
 #ifdef CONFIG_MIGRATION
 #define PAGE_MIGRATION 1
 
 extern void putback_lru_pages(struct list_head *l);
 extern int migrate_page(struct address_space *,
-                       struct page *, struct page *, bool);
+                       struct page *, struct page *, enum migrate_mode);
 extern int migrate_pages(struct list_head *l, new_page_t x,
                        unsigned long private, bool offlining,
-                       bool sync);
+                       enum migrate_mode mode);
 extern int migrate_huge_pages(struct list_head *l, new_page_t x,
                        unsigned long private, bool offlining,
-                       bool sync);
+                       enum migrate_mode mode);
 
 extern int fail_migrate_page(struct address_space *,
                        struct page *, struct page *);
@@ -36,10 +49,10 @@ extern int migrate_huge_page_move_mapping(struct address_space *mapping,
 static inline void putback_lru_pages(struct list_head *l) {}
 static inline int migrate_pages(struct list_head *l, new_page_t x,
                unsigned long private, bool offlining,
-               bool sync) { return -ENOSYS; }
+               enum migrate_mode mode) { return -ENOSYS; }
 static inline int migrate_huge_pages(struct list_head *l, new_page_t x,
                unsigned long private, bool offlining,
-               bool sync) { return -ENOSYS; }
+               enum migrate_mode mode) { return -ENOSYS; }
 
 static inline int migrate_prep(void) { return -ENOSYS; }
 static inline int migrate_prep_local(void) { return -ENOSYS; }

diff --git a/mm/compaction.c b/mm/compaction.c
index fb291585e1bf0e049c9eee1a01740e1cdc8f6b2c..71a58f67f4817720a4eca34678b2353fa96863e6 100644 (file)
--- a/mm/compaction.c
+++ b/mm/compaction.c
@@ -557,7 +557,7 @@ static int compact_zone(struct zone *zone, struct compact_control *cc)
                nr_migrate = cc->nr_migratepages;
                err = migrate_pages(&cc->migratepages, compaction_alloc,
                                (unsigned long)cc, false,
-                               cc->sync);
+                               cc->sync ? MIGRATE_SYNC_LIGHT : MIGRATE_ASYNC);
                update_nr_listpages(cc);
                nr_remaining = cc->nr_migratepages;
 

diff --git a/mm/memory-failure.c b/mm/memory-failure.c
index 06d3479513aaa2a3eeff26f0bf25e71c13695fdd..56080ea361406090860493861d5253dc314debc1 100644 (file)
--- a/mm/memory-failure.c
+++ b/mm/memory-failure.c
@@ -1557,7 +1557,7 @@ int soft_offline_page(struct page *page, int flags)
                                            page_is_file_cache(page));
                list_add(&page->lru, &pagelist);
                ret = migrate_pages(&pagelist, new_page, MPOL_MF_MOVE_ALL,
-                                                               0, true);
+                                                       0, MIGRATE_SYNC);
                if (ret) {
                        putback_lru_pages(&pagelist);
                        pr_info("soft offline: %#lx: migration failed %d, type %lx\n",

diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c
index 2168489c0bc9f309ff3347d6dac05a09039ad7d2..6629fafd6ce4a65eae9421dd9d68b25805bca993 100644 (file)
--- a/mm/memory_hotplug.c
+++ b/mm/memory_hotplug.c
@@ -809,7 +809,7 @@ do_migrate_range(unsigned long start_pfn, unsigned long end_pfn)
                }
                /* this function returns # of failed pages */
                ret = migrate_pages(&source, hotremove_migrate_alloc, 0,
-                                                               true, true);
+                                                       true, MIGRATE_SYNC);
                if (ret)
                        putback_lru_pages(&source);
        }

diff --git a/mm/mempolicy.c b/mm/mempolicy.c
index e3d58f088466c3b798f9017b76351295313d2771..06b145fb64ab59a399344bc73ca277be0e39ed7b 100644 (file)
--- a/mm/mempolicy.c
+++ b/mm/mempolicy.c
@@ -942,7 +942,7 @@ static int migrate_to_node(struct mm_struct *mm, int source, int dest,
 
        if (!list_empty(&pagelist)) {
                err = migrate_pages(&pagelist, new_node_page, dest,
-                                                               false, true);
+                                                       false, MIGRATE_SYNC);
                if (err)
                        putback_lru_pages(&pagelist);
        }

diff --git a/mm/migrate.c b/mm/migrate.c
index 4e86f3bacb852e1a4c09f8e9ad3fa2a68fb2973a..9871a56d82c30b1390ea268fddd9b3bc4b83c8da 100644 (file)
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -218,12 +218,13 @@ out:
 
 #ifdef CONFIG_BLOCK
 /* Returns true if all buffers are successfully locked */
-static bool buffer_migrate_lock_buffers(struct buffer_head *head, bool sync)
+static bool buffer_migrate_lock_buffers(struct buffer_head *head,
+                                                       enum migrate_mode mode)
 {
        struct buffer_head *bh = head;
 
        /* Simple case, sync compaction */
-       if (sync) {
+       if (mode != MIGRATE_ASYNC) {
                do {
                        get_bh(bh);
                        lock_buffer(bh);
@@ -259,7 +260,7 @@ static bool buffer_migrate_lock_buffers(struct buffer_head *head, bool sync)
 }
 #else
 static inline bool buffer_migrate_lock_buffers(struct buffer_head *head,
-                                                               bool sync)
+                                                       enum migrate_mode mode)
 {
        return true;
 }
@@ -275,7 +276,7 @@ static inline bool buffer_migrate_lock_buffers(struct buffer_head *head,
  */
 static int migrate_page_move_mapping(struct address_space *mapping,
                struct page *newpage, struct page *page,
-               struct buffer_head *head, bool sync)
+               struct buffer_head *head, enum migrate_mode mode)
 {
        int expected_count;
        void **pslot;
@@ -311,7 +312,8 @@ static int migrate_page_move_mapping(struct address_space *mapping,
         * the mapping back due to an elevated page count, we would have to
         * block waiting on other references to be dropped.
         */
-       if (!sync && head && !buffer_migrate_lock_buffers(head, sync)) {
+       if (mode == MIGRATE_ASYNC && head &&
+                       !buffer_migrate_lock_buffers(head, mode)) {
                page_unfreeze_refs(page, expected_count);
                spin_unlock_irq(&mapping->tree_lock);
                return -EAGAIN;
@@ -472,13 +474,14 @@ EXPORT_SYMBOL(fail_migrate_page);
  * Pages are locked upon entry and exit.
  */
 int migrate_page(struct address_space *mapping,
-               struct page *newpage, struct page *page, bool sync)
+               struct page *newpage, struct page *page,
+               enum migrate_mode mode)
 {
        int rc;
 
        BUG_ON(PageWriteback(page));    /* Writeback must be complete */
 
-       rc = migrate_page_move_mapping(mapping, newpage, page, NULL, sync);
+       rc = migrate_page_move_mapping(mapping, newpage, page, NULL, mode);
 
        if (rc)
                return rc;
@@ -495,17 +498,17 @@ EXPORT_SYMBOL(migrate_page);
  * exist.
  */
 int buffer_migrate_page(struct address_space *mapping,
-               struct page *newpage, struct page *page, bool sync)
+               struct page *newpage, struct page *page, enum migrate_mode mode)
 {
        struct buffer_head *bh, *head;
        int rc;
 
        if (!page_has_buffers(page))
-               return migrate_page(mapping, newpage, page, sync);
+               return migrate_page(mapping, newpage, page, mode);
 
        head = page_buffers(page);
 
-       rc = migrate_page_move_mapping(mapping, newpage, page, head, sync);
+       rc = migrate_page_move_mapping(mapping, newpage, page, head, mode);
 
        if (rc)
                return rc;
@@ -515,8 +518,8 @@ int buffer_migrate_page(struct address_space *mapping,
         * with an IRQ-safe spinlock held. In the sync case, the buffers
         * need to be locked now
         */
-       if (sync)
-               BUG_ON(!buffer_migrate_lock_buffers(head, sync));
+       if (mode != MIGRATE_ASYNC)
+               BUG_ON(!buffer_migrate_lock_buffers(head, mode));
 
        ClearPagePrivate(page);
        set_page_private(newpage, page_private(page));
@@ -593,10 +596,11 @@ static int writeout(struct address_space *mapping, struct page *page)
  * Default handling if a filesystem does not provide a migration function.
  */
 static int fallback_migrate_page(struct address_space *mapping,
-       struct page *newpage, struct page *page, bool sync)
+       struct page *newpage, struct page *page, enum migrate_mode mode)
 {
        if (PageDirty(page)) {
-               if (!sync)
+               /* Only writeback pages in full synchronous migration */
+               if (mode != MIGRATE_SYNC)
                        return -EBUSY;
                return writeout(mapping, page);
        }
@@ -609,7 +613,7 @@ static int fallback_migrate_page(struct address_space *mapping,
            !try_to_release_page(page, GFP_KERNEL))
                return -EAGAIN;
 
-       return migrate_page(mapping, newpage, page, sync);
+       return migrate_page(mapping, newpage, page, mode);
 }
 
 /*
@@ -624,7 +628,7 @@ static int fallback_migrate_page(struct address_space *mapping,
  *  == 0 - success
  */
 static int move_to_new_page(struct page *newpage, struct page *page,
-                                       int remap_swapcache, bool sync)
+                               int remap_swapcache, enum migrate_mode mode)
 {
        struct address_space *mapping;
        int rc;
@@ -645,7 +649,7 @@ static int move_to_new_page(struct page *newpage, struct page *page,
 
        mapping = page_mapping(page);
        if (!mapping)
-               rc = migrate_page(mapping, newpage, page, sync);
+               rc = migrate_page(mapping, newpage, page, mode);
        else if (mapping->a_ops->migratepage)
                /*
                 * Most pages have a mapping and most filesystems provide a
@@ -654,9 +658,9 @@ static int move_to_new_page(struct page *newpage, struct page *page,
                 * is the most common path for page migration.
                 */
                rc = mapping->a_ops->migratepage(mapping,
-                                               newpage, page, sync);
+                                               newpage, page, mode);
        else
-               rc = fallback_migrate_page(mapping, newpage, page, sync);
+               rc = fallback_migrate_page(mapping, newpage, page, mode);
 
        if (rc) {
                newpage->mapping = NULL;
@@ -671,7 +675,7 @@ static int move_to_new_page(struct page *newpage, struct page *page,
 }
 
 static int __unmap_and_move(struct page *page, struct page *newpage,
-                               int force, bool offlining, bool sync)
+                       int force, bool offlining, enum migrate_mode mode)
 {
        int rc = -EAGAIN;
        int remap_swapcache = 1;
@@ -680,7 +684,7 @@ static int __unmap_and_move(struct page *page, struct page *newpage,
        struct anon_vma *anon_vma = NULL;
 
        if (!trylock_page(page)) {
-               if (!force || !sync)
+               if (!force || mode == MIGRATE_ASYNC)
                        goto out;
 
                /*
@@ -726,10 +730,12 @@ static int __unmap_and_move(struct page *page, struct page *newpage,
 
        if (PageWriteback(page)) {
                /*
-                * For !sync, there is no point retrying as the retry loop
-                * is expected to be too short for PageWriteback to be cleared
+                * Only in the case of a full syncronous migration is it
+                * necessary to wait for PageWriteback. In the async case,
+                * the retry loop is too short and in the sync-light case,
+                * the overhead of stalling is too much
                 */
-               if (!sync) {
+               if (mode != MIGRATE_SYNC) {
                        rc = -EBUSY;
                        goto uncharge;
                }
@@ -800,7 +806,7 @@ static int __unmap_and_move(struct page *page, struct page *newpage,
 
 skip_unmap:
        if (!page_mapped(page))
-               rc = move_to_new_page(newpage, page, remap_swapcache, sync);
+               rc = move_to_new_page(newpage, page, remap_swapcache, mode);
 
        if (rc && remap_swapcache)
                remove_migration_ptes(page, page);
@@ -823,7 +829,8 @@ out:
  * to the newly allocated page in newpage.
  */
 static int unmap_and_move(new_page_t get_new_page, unsigned long private,
-                       struct page *page, int force, bool offlining, bool sync)
+                       struct page *page, int force, bool offlining,
+                       enum migrate_mode mode)
 {
        int rc = 0;
        int *result = NULL;
@@ -843,7 +850,7 @@ static int unmap_and_move(new_page_t get_new_page, unsigned long private,
                if (unlikely(split_huge_page(page)))
                        goto out;
 
-       rc = __unmap_and_move(page, newpage, force, offlining, sync);
+       rc = __unmap_and_move(page, newpage, force, offlining, mode);
 out:
        if (rc != -EAGAIN) {
                /*
@@ -891,7 +898,8 @@ out:
  */
 static int unmap_and_move_huge_page(new_page_t get_new_page,
                                unsigned long private, struct page *hpage,
-                               int force, bool offlining, bool sync)
+                               int force, bool offlining,
+                               enum migrate_mode mode)
 {
        int rc = 0;
        int *result = NULL;
@@ -904,7 +912,7 @@ static int unmap_and_move_huge_page(new_page_t get_new_page,
        rc = -EAGAIN;
 
        if (!trylock_page(hpage)) {
-               if (!force || !sync)
+               if (!force || mode != MIGRATE_SYNC)
                        goto out;
                lock_page(hpage);
        }
@@ -915,7 +923,7 @@ static int unmap_and_move_huge_page(new_page_t get_new_page,
        try_to_unmap(hpage, TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS);
 
        if (!page_mapped(hpage))
-               rc = move_to_new_page(new_hpage, hpage, 1, sync);
+               rc = move_to_new_page(new_hpage, hpage, 1, mode);
 
        if (rc)
                remove_migration_ptes(hpage, hpage);
@@ -958,7 +966,7 @@ out:
  */
 int migrate_pages(struct list_head *from,
                new_page_t get_new_page, unsigned long private, bool offlining,
-               bool sync)
+               enum migrate_mode mode)
 {
        int retry = 1;
        int nr_failed = 0;
@@ -979,7 +987,7 @@ int migrate_pages(struct list_head *from,
 
                        rc = unmap_and_move(get_new_page, private,
                                                page, pass > 2, offlining,
-                                               sync);
+                                               mode);
 
                        switch(rc) {
                        case -ENOMEM:
@@ -1009,7 +1017,7 @@ out:
 
 int migrate_huge_pages(struct list_head *from,
                new_page_t get_new_page, unsigned long private, bool offlining,
-               bool sync)
+               enum migrate_mode mode)
 {
        int retry = 1;
        int nr_failed = 0;
@@ -1026,7 +1034,7 @@ int migrate_huge_pages(struct list_head *from,
 
                        rc = unmap_and_move_huge_page(get_new_page,
                                        private, page, pass > 2, offlining,
-                                       sync);
+                                       mode);
 
                        switch(rc) {
                        case -ENOMEM:
@@ -1155,7 +1163,7 @@ set_status:
        err = 0;
        if (!list_empty(&pagelist)) {
                err = migrate_pages(&pagelist, new_page_node,
-                               (unsigned long)pm, 0, true);
+                               (unsigned long)pm, 0, MIGRATE_SYNC);
                if (err)
                        putback_lru_pages(&pagelist);
        }