mm: introduce memalloc_noreclaim_{save,restore}

The previous patch ("mm: prevent potential recursive reclaim due to
clearing PF_MEMALLOC") has shown that simply setting and clearing
PF_MEMALLOC in current->flags can result in wrongly clearing a
pre-existing PF_MEMALLOC flag and potentially lead to recursive reclaim.
Let's introduce helpers that support proper nesting by saving the
previous stat of the flag, similar to the existing memalloc_noio_* and
memalloc_nofs_* helpers.  Convert existing setting/clearing of
PF_MEMALLOC within mm to the new helpers.

There are no known issues with the converted code, but the change makes
it more robust.

Link: http://lkml.kernel.org/r/20170405074700.29871-3-vbabka@suse.cz
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Suggested-by: Michal Hocko <mhocko@suse.com>
Acked-by: Michal Hocko <mhocko@suse.com>
Acked-by: Hillf Danton <hillf.zj@alibaba-inc.com>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Andrey Ryabinin <aryabinin@virtuozzo.com>
Cc: Boris Brezillon <boris.brezillon@free-electrons.com>
Cc: Chris Leech <cleech@redhat.com>
Cc: "David S. Miller" <davem@davemloft.net>
Cc: Eric Dumazet <edumazet@google.com>
Cc: Josef Bacik <jbacik@fb.com>
Cc: Lee Duncan <lduncan@suse.com>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Richard Weinberger <richard@nod.at>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/include/linux/sched/mm.h b/include/linux/sched/mm.h
index 9daabe138c9905fea8d596281afabdd3a206fa78..2b24a6974847664165bb323aa00c8fae6f3506b0 100644 (file)
--- a/include/linux/sched/mm.h
+++ b/include/linux/sched/mm.h
@@ -191,4 +191,16 @@ static inline void memalloc_nofs_restore(unsigned int flags)
        current->flags = (current->flags & ~PF_MEMALLOC_NOFS) | flags;
 }
 
+static inline unsigned int memalloc_noreclaim_save(void)
+{
+       unsigned int flags = current->flags & PF_MEMALLOC;
+       current->flags |= PF_MEMALLOC;
+       return flags;
+}
+
+static inline void memalloc_noreclaim_restore(unsigned int flags)
+{
+       current->flags = (current->flags & ~PF_MEMALLOC) | flags;
+}
+
 #endif /* _LINUX_SCHED_MM_H */

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 1daf509722c7fc0063bb2823a69ec3d6f4612fe6..f9e450c6b6e414d61b00d5a61be9cdea3b773e1b 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -3283,15 +3283,15 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
                enum compact_priority prio, enum compact_result *compact_result)
 {
        struct page *page;
-       unsigned int noreclaim_flag = current->flags & PF_MEMALLOC;
+       unsigned int noreclaim_flag;
 
        if (!order)
                return NULL;
 
-       current->flags |= PF_MEMALLOC;
+       noreclaim_flag = memalloc_noreclaim_save();
        *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac,
                                                                        prio);
-       current->flags = (current->flags & ~PF_MEMALLOC) | noreclaim_flag;
+       memalloc_noreclaim_restore(noreclaim_flag);
 
        if (*compact_result <= COMPACT_INACTIVE)
                return NULL;
@@ -3438,12 +3438,13 @@ __perform_reclaim(gfp_t gfp_mask, unsigned int order,
 {
        struct reclaim_state reclaim_state;
        int progress;
+       unsigned int noreclaim_flag;
 
        cond_resched();
 
        /* We now go into synchronous reclaim */
        cpuset_memory_pressure_bump();
-       current->flags |= PF_MEMALLOC;
+       noreclaim_flag = memalloc_noreclaim_save();
        lockdep_set_current_reclaim_state(gfp_mask);
        reclaim_state.reclaimed_slab = 0;
        current->reclaim_state = &reclaim_state;
@@ -3453,7 +3454,7 @@ __perform_reclaim(gfp_t gfp_mask, unsigned int order,
 
        current->reclaim_state = NULL;
        lockdep_clear_current_reclaim_state();
-       current->flags &= ~PF_MEMALLOC;
+       memalloc_noreclaim_restore(noreclaim_flag);
 
        cond_resched();
 

diff --git a/mm/vmscan.c b/mm/vmscan.c
index 4e7ed65842aff72beb90815262dacd29987cbcf8..2f45c0520f43fbcb5696063172cd281553302f9d 100644 (file)
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -3036,6 +3036,7 @@ unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg,
        struct zonelist *zonelist;
        unsigned long nr_reclaimed;
        int nid;
+       unsigned int noreclaim_flag;
        struct scan_control sc = {
                .nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX),
                .gfp_mask = (current_gfp_context(gfp_mask) & GFP_RECLAIM_MASK) |
@@ -3062,9 +3063,9 @@ unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg,
                                            sc.gfp_mask,
                                            sc.reclaim_idx);
 
-       current->flags |= PF_MEMALLOC;
+       noreclaim_flag = memalloc_noreclaim_save();
        nr_reclaimed = do_try_to_free_pages(zonelist, &sc);
-       current->flags &= ~PF_MEMALLOC;
+       memalloc_noreclaim_restore(noreclaim_flag);
 
        trace_mm_vmscan_memcg_reclaim_end(nr_reclaimed);
 
@@ -3589,8 +3590,9 @@ unsigned long shrink_all_memory(unsigned long nr_to_reclaim)
        struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask);
        struct task_struct *p = current;
        unsigned long nr_reclaimed;
+       unsigned int noreclaim_flag;
 
-       p->flags |= PF_MEMALLOC;
+       noreclaim_flag = memalloc_noreclaim_save();
        lockdep_set_current_reclaim_state(sc.gfp_mask);
        reclaim_state.reclaimed_slab = 0;
        p->reclaim_state = &reclaim_state;
@@ -3599,7 +3601,7 @@ unsigned long shrink_all_memory(unsigned long nr_to_reclaim)
 
        p->reclaim_state = NULL;
        lockdep_clear_current_reclaim_state();
-       p->flags &= ~PF_MEMALLOC;
+       memalloc_noreclaim_restore(noreclaim_flag);
 
        return nr_reclaimed;
 }
@@ -3764,6 +3766,7 @@ static int __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned in
        struct task_struct *p = current;
        struct reclaim_state reclaim_state;
        int classzone_idx = gfp_zone(gfp_mask);
+       unsigned int noreclaim_flag;
        struct scan_control sc = {
                .nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX),
                .gfp_mask = (gfp_mask = current_gfp_context(gfp_mask)),
@@ -3781,7 +3784,8 @@ static int __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned in
         * and we also need to be able to write out pages for RECLAIM_WRITE
         * and RECLAIM_UNMAP.
         */
-       p->flags |= PF_MEMALLOC | PF_SWAPWRITE;
+       noreclaim_flag = memalloc_noreclaim_save();
+       p->flags |= PF_SWAPWRITE;
        lockdep_set_current_reclaim_state(gfp_mask);
        reclaim_state.reclaimed_slab = 0;
        p->reclaim_state = &reclaim_state;
@@ -3797,7 +3801,8 @@ static int __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned in
        }
 
        p->reclaim_state = NULL;
-       current->flags &= ~(PF_MEMALLOC | PF_SWAPWRITE);
+       current->flags &= ~PF_SWAPWRITE;
+       memalloc_noreclaim_restore(noreclaim_flag);
        lockdep_clear_current_reclaim_state();
        return sc.nr_reclaimed >= nr_pages;
 }