mm/page_alloc: explicitly define how __GFP_HIGH non-blocking allocations accesses...

GFP_ATOMIC allocations get flagged ALLOC_HARDER which is a vague
description.  In preparation for the removal of GFP_ATOMIC redefine
__GFP_ATOMIC to simply mean non-blocking and renaming ALLOC_HARDER to
ALLOC_NON_BLOCK accordingly.  __GFP_HIGH is required for access to
reserves but non-blocking is granted more access.  For example, GFP_NOWAIT
is non-blocking but has no special access to reserves.  A __GFP_NOFAIL
blocking allocation is granted access similar to __GFP_HIGH if the only
alternative is an OOM kill.

Link: https://lkml.kernel.org/r/20230113111217.14134-6-mgorman@techsingularity.net
Signed-off-by: Mel Gorman <mgorman@techsingularity.net>
Acked-by: Michal Hocko <mhocko@suse.com>
Cc: Matthew Wilcox <willy@infradead.org>
Cc: NeilBrown <neilb@suse.de>
Cc: Thierry Reding <thierry.reding@gmail.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/mm/internal.h b/mm/internal.h
index f7c3bc80c475b471cca8cae5c28ff186c7740e2f..b0b88a95347fac510feee56850741867ab09d05a 100644 (file)
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -778,7 +778,10 @@ unsigned int reclaim_clean_pages_from_list(struct zone *zone,
 #define ALLOC_OOM              ALLOC_NO_WATERMARKS
 #endif
 
-#define ALLOC_HARDER            0x10 /* try to alloc harder */
+#define ALLOC_NON_BLOCK                 0x10 /* Caller cannot block. Allow access
+                                      * to 25% of the min watermark or
+                                      * 62.5% if __GFP_HIGH is set.
+                                      */
 #define ALLOC_MIN_RESERVE       0x20 /* __GFP_HIGH set. Allow access to 50%
                                       * of the min watermark.
                                       */
@@ -793,7 +796,7 @@ unsigned int reclaim_clean_pages_from_list(struct zone *zone,
 #define ALLOC_KSWAPD           0x800 /* allow waking of kswapd, __GFP_KSWAPD_RECLAIM set */
 
 /* Flags that allow allocations below the min watermark. */
-#define ALLOC_RESERVES (ALLOC_HARDER|ALLOC_MIN_RESERVE|ALLOC_HIGHATOMIC|ALLOC_OOM)
+#define ALLOC_RESERVES (ALLOC_NON_BLOCK|ALLOC_MIN_RESERVE|ALLOC_HIGHATOMIC|ALLOC_OOM)
 
 enum ttu_flags;
 struct tlbflush_unmap_batch;

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index ffe2151d11eea10abf01e42aac231adb448b55b8..18ca33a1945d663ac4020719ac983887d96a1f10 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -4007,18 +4007,19 @@ bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
                 * __GFP_HIGH allows access to 50% of the min reserve as well
                 * as OOM.
                 */
-               if (alloc_flags & ALLOC_MIN_RESERVE)
+               if (alloc_flags & ALLOC_MIN_RESERVE) {
                        min -= min / 2;
 
-               /*
-                * Non-blocking allocations can access some of the reserve
-                * with more access if also __GFP_HIGH. The reasoning is that
-                * a non-blocking caller may incur a more severe penalty
-                * if it cannot get memory quickly, particularly if it's
-                * also __GFP_HIGH.
-                */
-               if (alloc_flags & ALLOC_HARDER)
-                       min -= min / 4;
+                       /*
+                        * Non-blocking allocations (e.g. GFP_ATOMIC) can
+                        * access more reserves than just __GFP_HIGH. Other
+                        * non-blocking allocations requests such as GFP_NOWAIT
+                        * or (GFP_KERNEL & ~__GFP_DIRECT_RECLAIM) do not get
+                        * access to the min reserve.
+                        */
+                       if (alloc_flags & ALLOC_NON_BLOCK)
+                               min -= min / 4;
+               }
 
                /*
                 * OOM victims can try even harder than the normal reserve
@@ -4869,28 +4870,30 @@ gfp_to_alloc_flags(gfp_t gfp_mask, unsigned int order)
         * The caller may dip into page reserves a bit more if the caller
         * cannot run direct reclaim, or if the caller has realtime scheduling
         * policy or is asking for __GFP_HIGH memory.  GFP_ATOMIC requests will
-        * set both ALLOC_HARDER (__GFP_ATOMIC) and ALLOC_MIN_RESERVE(__GFP_HIGH).
+        * set both ALLOC_NON_BLOCK and ALLOC_MIN_RESERVE(__GFP_HIGH).
         */
        alloc_flags |= (__force int)
                (gfp_mask & (__GFP_HIGH | __GFP_KSWAPD_RECLAIM));
 
-       if (gfp_mask & __GFP_ATOMIC) {
+       if (!(gfp_mask & __GFP_DIRECT_RECLAIM)) {
                /*
                 * Not worth trying to allocate harder for __GFP_NOMEMALLOC even
                 * if it can't schedule.
                 */
                if (!(gfp_mask & __GFP_NOMEMALLOC)) {
-                       alloc_flags |= ALLOC_HARDER;
+                       alloc_flags |= ALLOC_NON_BLOCK;
 
                        if (order > 0)
                                alloc_flags |= ALLOC_HIGHATOMIC;
                }
 
                /*
-                * Ignore cpuset mems for GFP_ATOMIC rather than fail, see the
-                * comment for __cpuset_node_allowed().
+                * Ignore cpuset mems for non-blocking __GFP_HIGH (probably
+                * GFP_ATOMIC) rather than fail, see the comment for
+                * __cpuset_node_allowed().
                 */
-               alloc_flags &= ~ALLOC_CPUSET;
+               if (alloc_flags & ALLOC_MIN_RESERVE)
+                       alloc_flags &= ~ALLOC_CPUSET;
        } else if (unlikely(rt_task(current)) && in_task())
                alloc_flags |= ALLOC_MIN_RESERVE;
 
@@ -5321,12 +5324,13 @@ nopage:
                WARN_ON_ONCE_GFP(costly_order, gfp_mask);
 
                /*
-                * Help non-failing allocations by giving them access to memory
-                * reserves but do not use ALLOC_NO_WATERMARKS because this
+                * Help non-failing allocations by giving some access to memory
+                * reserves normally used for high priority non-blocking
+                * allocations but do not use ALLOC_NO_WATERMARKS because this
                 * could deplete whole memory reserves which would just make
-                * the situation worse
+                * the situation worse.
                 */
-               page = __alloc_pages_cpuset_fallback(gfp_mask, order, ALLOC_HARDER, ac);
+               page = __alloc_pages_cpuset_fallback(gfp_mask, order, ALLOC_MIN_RESERVE, ac);
                if (page)
                        goto got_pg;