mm/vmalloc: handle non-blocking GFP in __vmalloc_area_node()

Make __vmalloc_area_node() respect non-blocking GFP masks such as
GFP_ATOMIC and GFP_NOWAIT.

- Add memalloc_apply_gfp_scope()/memalloc_restore_scope()
  helpers to apply a proper scope.
- Apply memalloc_apply_gfp_scope()/memalloc_restore_scope()
  around vmap_pages_range() for page table setup.
- Set "nofail" to false if a non-blocking mask is used, as
  they are mutually exclusive.

This is particularly important for page table allocations that internally
use GFP_PGTABLE_KERNEL, which may sleep unless such scope restrictions are
applied.  For example:

<snip>
__pte_alloc_kernel()
  pte_alloc_one_kernel(&init_mm);
    pagetable_alloc_noprof(GFP_PGTABLE_KERNEL & ~__GFP_HIGHMEM, 0);
<snip>

Note: in most cases, PTE entries are established only up to the level
required by current vmap space usage, meaning the page tables are
typically fully populated during the mapping process.

Link: https://lkml.kernel.org/r/20251007122035.56347-6-urezki@gmail.com
Signed-off-by: Uladzislau Rezki (Sony) <urezki@gmail.com>
Acked-by: Michal Hocko <mhocko@suse.com>
Reviewed-by: Baoquan He <bhe@redhat.com>
Cc: Alexander Potapenko <glider@google.com>
Cc: Andrey Ryabinin <ryabinin.a.a@gmail.com>
Cc: Marco Elver <elver@google.com>
Cc: Michal Hocko <mhocko@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h
index 1e43181369f17da80cb05b44c568fc2196994620..e8e94f90d68665d2f53ba09bad0e0a2b312c762a 100644 (file)
--- a/include/linux/vmalloc.h
+++ b/include/linux/vmalloc.h
@@ -332,4 +332,6 @@ bool vmalloc_dump_obj(void *object);
 static inline bool vmalloc_dump_obj(void *object) { return false; }
 #endif
 
+unsigned int memalloc_apply_gfp_scope(gfp_t gfp_mask);
+void memalloc_restore_scope(unsigned int flags);
 #endif /* _LINUX_VMALLOC_H */

diff --git a/mm/vmalloc.c b/mm/vmalloc.c
index 9e29dd767c41cd25bdce31596d2a5c8991d03b44..d8bcd87239b59ab018ffb17506965e0b5ae3f3bf 100644 (file)
--- a/mm/vmalloc.c
+++ b/mm/vmalloc.c
@@ -3716,6 +3716,42 @@ static void defer_vm_area_cleanup(struct vm_struct *area)
                schedule_work(&cleanup_vm_area);
 }
 
+/*
+ * Page tables allocations ignore external GFP. Enforces it by
+ * the memalloc scope API. It is used by vmalloc internals and
+ * KASAN shadow population only.
+ *
+ * GFP to scope mapping:
+ *
+ * non-blocking (no __GFP_DIRECT_RECLAIM) - memalloc_noreclaim_save()
+ * GFP_NOFS - memalloc_nofs_save()
+ * GFP_NOIO - memalloc_noio_save()
+ *
+ * Returns a flag cookie to pair with restore.
+ */
+unsigned int
+memalloc_apply_gfp_scope(gfp_t gfp_mask)
+{
+       unsigned int flags = 0;
+
+       if (!gfpflags_allow_blocking(gfp_mask))
+               flags = memalloc_noreclaim_save();
+       else if ((gfp_mask & (__GFP_FS | __GFP_IO)) == __GFP_IO)
+               flags = memalloc_nofs_save();
+       else if ((gfp_mask & (__GFP_FS | __GFP_IO)) == 0)
+               flags = memalloc_noio_save();
+
+       /* 0 - no scope applied. */
+       return flags;
+}
+
+void
+memalloc_restore_scope(unsigned int flags)
+{
+       if (flags)
+               memalloc_flags_restore(flags);
+}
+
 static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
                                 pgprot_t prot, unsigned int page_shift,
                                 int node)
@@ -3732,6 +3768,10 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
 
        array_size = (unsigned long)nr_small_pages * sizeof(struct page *);
 
+       /* __GFP_NOFAIL and "noblock" flags are mutually exclusive. */
+       if (!gfpflags_allow_blocking(gfp_mask))
+               nofail = false;
+
        if (!(gfp_mask & (GFP_DMA | GFP_DMA32)))
                gfp_mask |= __GFP_HIGHMEM;
 
@@ -3797,22 +3837,14 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
         * page tables allocations ignore external gfp mask, enforce it
         * by the scope API
         */
-       if ((gfp_mask & (__GFP_FS | __GFP_IO)) == __GFP_IO)
-               flags = memalloc_nofs_save();
-       else if ((gfp_mask & (__GFP_FS | __GFP_IO)) == 0)
-               flags = memalloc_noio_save();
-
+       flags = memalloc_apply_gfp_scope(gfp_mask);
        do {
                ret = vmap_pages_range(addr, addr + size, prot, area->pages,
                        page_shift);
                if (nofail && (ret < 0))
                        schedule_timeout_uninterruptible(1);
        } while (nofail && (ret < 0));
-
-       if ((gfp_mask & (__GFP_FS | __GFP_IO)) == __GFP_IO)
-               memalloc_nofs_restore(flags);
-       else if ((gfp_mask & (__GFP_FS | __GFP_IO)) == 0)
-               memalloc_noio_restore(flags);
+       memalloc_restore_scope(flags);
 
        if (ret < 0) {
                warn_alloc(gfp_mask, NULL,