mm: add replace_page_cache_page() function

This function basically does:

     remove_from_page_cache(old);
     page_cache_release(old);
     add_to_page_cache_locked(new);

Except it does this atomically, so there's no possibility for the "add" to
fail because of a race.

If memory cgroups are enabled, then the memory cgroup charge is also moved
from the old page to the new.

This function is currently used by fuse to move pages into the page cache
on read, instead of copying the page contents.

[minchan.kim@gmail.com: add freepage() hook to replace_page_cache_page()]
Signed-off-by: Miklos Szeredi <mszeredi@suse.cz>
Acked-by: Rik van Riel <riel@redhat.com>
Acked-by: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Minchan Kim <minchan.kim@gmail.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c
index 213d3cf4f5e942f323f7078fafb752c7f1c2bfe8..640fc229df10323b6c9fc94b0bda452157bf1ef0 100644 (file)
--- a/fs/fuse/dev.c
+++ b/fs/fuse/dev.c
@@ -737,14 +737,12 @@ static int fuse_try_move_page(struct fuse_copy_state *cs, struct page **pagep)
        if (WARN_ON(PageMlocked(oldpage)))
                goto out_fallback_unlock;
 
-       remove_from_page_cache(oldpage);
-       page_cache_release(oldpage);
-
-       err = add_to_page_cache_locked(newpage, mapping, index, GFP_KERNEL);
+       err = replace_page_cache_page(oldpage, newpage, GFP_KERNEL);
        if (err) {
-               printk(KERN_WARNING "fuse_try_move_page: failed to add page");
-               goto out_fallback_unlock;
+               unlock_page(newpage);
+               return err;
        }
+
        page_cache_get(newpage);
 
        if (!(buf->flags & PIPE_BUF_FLAG_LRU))

diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index f512e189be5a496b2aa1b555c1fd34e056fc81e6..a1a1e5384f6e426c143ad6b3374de23e5c30e02a 100644 (file)
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -96,7 +96,7 @@ extern struct cgroup_subsys_state *mem_cgroup_css(struct mem_cgroup *mem);
 
 extern int
 mem_cgroup_prepare_migration(struct page *page,
-       struct page *newpage, struct mem_cgroup **ptr);
+       struct page *newpage, struct mem_cgroup **ptr, gfp_t gfp_mask);
 extern void mem_cgroup_end_migration(struct mem_cgroup *mem,
        struct page *oldpage, struct page *newpage, bool migration_ok);
 
@@ -249,7 +249,7 @@ static inline struct cgroup_subsys_state *mem_cgroup_css(struct mem_cgroup *mem)
 
 static inline int
 mem_cgroup_prepare_migration(struct page *page, struct page *newpage,
-       struct mem_cgroup **ptr)
+       struct mem_cgroup **ptr, gfp_t gfp_mask)
 {
        return 0;
 }

diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h
index 9c66e994540f6d91750e6d78b30201c0156e0de9..26946ad483bf812718d960956c48317fff37bf02 100644 (file)
--- a/include/linux/pagemap.h
+++ b/include/linux/pagemap.h
@@ -457,6 +457,7 @@ int add_to_page_cache_lru(struct page *page, struct address_space *mapping,
                                pgoff_t index, gfp_t gfp_mask);
 extern void remove_from_page_cache(struct page *page);
 extern void __remove_from_page_cache(struct page *page);
+int replace_page_cache_page(struct page *old, struct page *new, gfp_t gfp_mask);
 
 /*
  * Like add_to_page_cache_locked, but used to add newly allocated pages:

diff --git a/mm/filemap.c b/mm/filemap.c
index 312b6eb7843098834dc447d2d1bceb90e1dba170..c1459f2cdb5e50b4206f3f479a716f8f481b65c8 100644 (file)
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -386,6 +386,76 @@ int filemap_write_and_wait_range(struct address_space *mapping,
 }
 EXPORT_SYMBOL(filemap_write_and_wait_range);
 
+/**
+ * replace_page_cache_page - replace a pagecache page with a new one
+ * @old:       page to be replaced
+ * @new:       page to replace with
+ * @gfp_mask:  allocation mode
+ *
+ * This function replaces a page in the pagecache with a new one.  On
+ * success it acquires the pagecache reference for the new page and
+ * drops it for the old page.  Both the old and new pages must be
+ * locked.  This function does not add the new page to the LRU, the
+ * caller must do that.
+ *
+ * The remove + add is atomic.  The only way this function can fail is
+ * memory allocation failure.
+ */
+int replace_page_cache_page(struct page *old, struct page *new, gfp_t gfp_mask)
+{
+       int error;
+       struct mem_cgroup *memcg = NULL;
+
+       VM_BUG_ON(!PageLocked(old));
+       VM_BUG_ON(!PageLocked(new));
+       VM_BUG_ON(new->mapping);
+
+       /*
+        * This is not page migration, but prepare_migration and
+        * end_migration does enough work for charge replacement.
+        *
+        * In the longer term we probably want a specialized function
+        * for moving the charge from old to new in a more efficient
+        * manner.
+        */
+       error = mem_cgroup_prepare_migration(old, new, &memcg, gfp_mask);
+       if (error)
+               return error;
+
+       error = radix_tree_preload(gfp_mask & ~__GFP_HIGHMEM);
+       if (!error) {
+               struct address_space *mapping = old->mapping;
+               void (*freepage)(struct page *);
+
+               pgoff_t offset = old->index;
+               freepage = mapping->a_ops->freepage;
+
+               page_cache_get(new);
+               new->mapping = mapping;
+               new->index = offset;
+
+               spin_lock_irq(&mapping->tree_lock);
+               __remove_from_page_cache(old);
+               error = radix_tree_insert(&mapping->page_tree, offset, new);
+               BUG_ON(error);
+               mapping->nrpages++;
+               __inc_zone_page_state(new, NR_FILE_PAGES);
+               if (PageSwapBacked(new))
+                       __inc_zone_page_state(new, NR_SHMEM);
+               spin_unlock_irq(&mapping->tree_lock);
+               radix_tree_preload_end();
+               if (freepage)
+                       freepage(old);
+               page_cache_release(old);
+               mem_cgroup_end_migration(memcg, old, new, true);
+       } else {
+               mem_cgroup_end_migration(memcg, old, new, false);
+       }
+
+       return error;
+}
+EXPORT_SYMBOL_GPL(replace_page_cache_page);
+
 /**
  * add_to_page_cache_locked - add a locked page to the pagecache
  * @page:      page to add

diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index da53a252b259f0f36f553e9646187db18d49d76a..6ef5c53dffcb30a42d76789f7c409f2a35dd5a5a 100644 (file)
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -2883,7 +2883,7 @@ static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
  * page belongs to.
  */
 int mem_cgroup_prepare_migration(struct page *page,
-       struct page *newpage, struct mem_cgroup **ptr)
+       struct page *newpage, struct mem_cgroup **ptr, gfp_t gfp_mask)
 {
        struct page_cgroup *pc;
        struct mem_cgroup *mem = NULL;
@@ -2940,7 +2940,7 @@ int mem_cgroup_prepare_migration(struct page *page,
                return 0;
 
        *ptr = mem;
-       ret = __mem_cgroup_try_charge(NULL, GFP_KERNEL, ptr, false, PAGE_SIZE);
+       ret = __mem_cgroup_try_charge(NULL, gfp_mask, ptr, false, PAGE_SIZE);
        css_put(&mem->css);/* drop extra refcnt */
        if (ret || *ptr == NULL) {
                if (PageAnon(page)) {

diff --git a/mm/migrate.c b/mm/migrate.c
index 352de555626c4434471a53e29bee7a02516adfe3..8aacce3af8cd5b30526cf39b724c5be06c6cb518 100644 (file)
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -678,7 +678,7 @@ static int unmap_and_move(new_page_t get_new_page, unsigned long private,
        }
 
        /* charge against new page */
-       charge = mem_cgroup_prepare_migration(page, newpage, &mem);
+       charge = mem_cgroup_prepare_migration(page, newpage, &mem, GFP_KERNEL);
        if (charge == -ENOMEM) {
                rc = -ENOMEM;
                goto unlock;