arm64: mm: use enum to identify pgtable level instead of *_SHIFT

Commit 90292aca9854 ("arm64: mm: use appropriate ctors for page tables")
introduced pgtable ctor calls in pgd_pgtable_alloc().  To identify the
pgtable level and call the appropriate ctor, the *_SHIFT value associated
with the pgtable level is used.  However, those values do not
unambiguously identify a level, because if a given level is folded, the
*_SHIFT value will be equal to that of the upper level (e.g.  PMD_SHIFT ==
PUD_SHIFT if PMD is folded).

As things stand, there is probably not much damaged done by calling the
ctor for a different level, and ARCH_ENABLE_SPLIT_PMD_PTLOCK is only
selected if PMD isn't folded (so we don't needlessly initialise
pmd_ptlock).  Still, this is pretty confusing, and it would get even more
confusing when adding ctor calls for the remaining levels.

Let's simplify all this by using an enum to identify the pgtable level
instead; this way folding becomes irrelevant.  This is inspired by one of
the m68k pgtable allocators (arch/m68k/include/asm/motorola_pgalloc.h).

Link: https://lkml.kernel.org/r/20250408095222.860601-9-kevin.brodsky@arm.com
Signed-off-by: Kevin Brodsky <kevin.brodsky@arm.com>
Cc: Albert Ou <aou@eecs.berkeley.edu>
Cc: Andreas Larsson <andreas@gaisler.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: David S. Miller <davem@davemloft.net>
Cc: Geert Uytterhoeven <geert@linux-m68k.org>
Cc: Linus Waleij <linus.walleij@linaro.org>
Cc: Madhavan Srinivasan <maddy@linux.ibm.com>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Mike Rapoport <rppt@kernel.org>
Cc: Palmer Dabbelt <palmer@dabbelt.com>
Cc: Paul Walmsley <paul.walmsley@sifive.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Qi Zheng <zhengqi.arch@bytedance.com>
Cc: Ryan Roberts <ryan.roberts@arm.com>
Cc: Will Deacon <will@kernel.org>
Cc: <x86@kernel.org>
Cc: Yang Shi <yang@os.amperecomputing.com>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: Alexander Gordeev <agordeev@linux.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index 8c5c471cfb064851c92c9131519155e02d081714..eca324b3a6fceacaf9136fab4fac83956e7ad07e 100644 (file)
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -46,6 +46,13 @@
 #define NO_CONT_MAPPINGS       BIT(1)
 #define NO_EXEC_MAPPINGS       BIT(2)  /* assumes FEAT_HPDS is not used */
 
+enum pgtable_type {
+       TABLE_PTE,
+       TABLE_PMD,
+       TABLE_PUD,
+       TABLE_P4D,
+};
+
 u64 kimage_voffset __ro_after_init;
 EXPORT_SYMBOL(kimage_voffset);
 
@@ -107,7 +114,7 @@ pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
 }
 EXPORT_SYMBOL(phys_mem_access_prot);
 
-static phys_addr_t __init early_pgtable_alloc(int shift)
+static phys_addr_t __init early_pgtable_alloc(enum pgtable_type pgtable_type)
 {
        phys_addr_t phys;
 
@@ -192,7 +199,7 @@ static void init_pte(pte_t *ptep, unsigned long addr, unsigned long end,
 static void alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr,
                                unsigned long end, phys_addr_t phys,
                                pgprot_t prot,
-                               phys_addr_t (*pgtable_alloc)(int),
+                               phys_addr_t (*pgtable_alloc)(enum pgtable_type),
                                int flags)
 {
        unsigned long next;
@@ -207,7 +214,7 @@ static void alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr,
                if (flags & NO_EXEC_MAPPINGS)
                        pmdval |= PMD_TABLE_PXN;
                BUG_ON(!pgtable_alloc);
-               pte_phys = pgtable_alloc(PAGE_SHIFT);
+               pte_phys = pgtable_alloc(TABLE_PTE);
                ptep = pte_set_fixmap(pte_phys);
                init_clear_pgtable(ptep);
                ptep += pte_index(addr);
@@ -243,7 +250,7 @@ static void alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr,
 
 static void init_pmd(pmd_t *pmdp, unsigned long addr, unsigned long end,
                     phys_addr_t phys, pgprot_t prot,
-                    phys_addr_t (*pgtable_alloc)(int), int flags)
+                    phys_addr_t (*pgtable_alloc)(enum pgtable_type), int flags)
 {
        unsigned long next;
 
@@ -277,7 +284,8 @@ static void init_pmd(pmd_t *pmdp, unsigned long addr, unsigned long end,
 static void alloc_init_cont_pmd(pud_t *pudp, unsigned long addr,
                                unsigned long end, phys_addr_t phys,
                                pgprot_t prot,
-                               phys_addr_t (*pgtable_alloc)(int), int flags)
+                               phys_addr_t (*pgtable_alloc)(enum pgtable_type),
+                               int flags)
 {
        unsigned long next;
        pud_t pud = READ_ONCE(*pudp);
@@ -294,7 +302,7 @@ static void alloc_init_cont_pmd(pud_t *pudp, unsigned long addr,
                if (flags & NO_EXEC_MAPPINGS)
                        pudval |= PUD_TABLE_PXN;
                BUG_ON(!pgtable_alloc);
-               pmd_phys = pgtable_alloc(PMD_SHIFT);
+               pmd_phys = pgtable_alloc(TABLE_PMD);
                pmdp = pmd_set_fixmap(pmd_phys);
                init_clear_pgtable(pmdp);
                pmdp += pmd_index(addr);
@@ -325,7 +333,7 @@ static void alloc_init_cont_pmd(pud_t *pudp, unsigned long addr,
 
 static void alloc_init_pud(p4d_t *p4dp, unsigned long addr, unsigned long end,
                           phys_addr_t phys, pgprot_t prot,
-                          phys_addr_t (*pgtable_alloc)(int),
+                          phys_addr_t (*pgtable_alloc)(enum pgtable_type),
                           int flags)
 {
        unsigned long next;
@@ -339,7 +347,7 @@ static void alloc_init_pud(p4d_t *p4dp, unsigned long addr, unsigned long end,
                if (flags & NO_EXEC_MAPPINGS)
                        p4dval |= P4D_TABLE_PXN;
                BUG_ON(!pgtable_alloc);
-               pud_phys = pgtable_alloc(PUD_SHIFT);
+               pud_phys = pgtable_alloc(TABLE_PUD);
                pudp = pud_set_fixmap(pud_phys);
                init_clear_pgtable(pudp);
                pudp += pud_index(addr);
@@ -383,7 +391,7 @@ static void alloc_init_pud(p4d_t *p4dp, unsigned long addr, unsigned long end,
 
 static void alloc_init_p4d(pgd_t *pgdp, unsigned long addr, unsigned long end,
                           phys_addr_t phys, pgprot_t prot,
-                          phys_addr_t (*pgtable_alloc)(int),
+                          phys_addr_t (*pgtable_alloc)(enum pgtable_type),
                           int flags)
 {
        unsigned long next;
@@ -397,7 +405,7 @@ static void alloc_init_p4d(pgd_t *pgdp, unsigned long addr, unsigned long end,
                if (flags & NO_EXEC_MAPPINGS)
                        pgdval |= PGD_TABLE_PXN;
                BUG_ON(!pgtable_alloc);
-               p4d_phys = pgtable_alloc(P4D_SHIFT);
+               p4d_phys = pgtable_alloc(TABLE_P4D);
                p4dp = p4d_set_fixmap(p4d_phys);
                init_clear_pgtable(p4dp);
                p4dp += p4d_index(addr);
@@ -427,7 +435,7 @@ static void alloc_init_p4d(pgd_t *pgdp, unsigned long addr, unsigned long end,
 static void __create_pgd_mapping_locked(pgd_t *pgdir, phys_addr_t phys,
                                        unsigned long virt, phys_addr_t size,
                                        pgprot_t prot,
-                                       phys_addr_t (*pgtable_alloc)(int),
+                                       phys_addr_t (*pgtable_alloc)(enum pgtable_type),
                                        int flags)
 {
        unsigned long addr, end, next;
@@ -455,7 +463,7 @@ static void __create_pgd_mapping_locked(pgd_t *pgdir, phys_addr_t phys,
 static void __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys,
                                 unsigned long virt, phys_addr_t size,
                                 pgprot_t prot,
-                                phys_addr_t (*pgtable_alloc)(int),
+                                phys_addr_t (*pgtable_alloc)(enum pgtable_type),
                                 int flags)
 {
        mutex_lock(&fixmap_lock);
@@ -468,10 +476,11 @@ static void __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys,
 extern __alias(__create_pgd_mapping_locked)
 void create_kpti_ng_temp_pgd(pgd_t *pgdir, phys_addr_t phys, unsigned long virt,
                             phys_addr_t size, pgprot_t prot,
-                            phys_addr_t (*pgtable_alloc)(int), int flags);
+                            phys_addr_t (*pgtable_alloc)(enum pgtable_type),
+                            int flags);
 #endif
 
-static phys_addr_t __pgd_pgtable_alloc(int shift)
+static phys_addr_t __pgd_pgtable_alloc(enum pgtable_type pgtable_type)
 {
        /* Page is zeroed by init_clear_pgtable() so don't duplicate effort. */
        void *ptr = (void *)__get_free_page(GFP_PGTABLE_KERNEL & ~__GFP_ZERO);
@@ -480,23 +489,26 @@ static phys_addr_t __pgd_pgtable_alloc(int shift)
        return __pa(ptr);
 }
 
-static phys_addr_t pgd_pgtable_alloc(int shift)
+static phys_addr_t pgd_pgtable_alloc(enum pgtable_type pgtable_type)
 {
-       phys_addr_t pa = __pgd_pgtable_alloc(shift);
+       phys_addr_t pa = __pgd_pgtable_alloc(pgtable_type);
        struct ptdesc *ptdesc = page_ptdesc(phys_to_page(pa));
 
        /*
         * Call proper page table ctor in case later we need to
         * call core mm functions like apply_to_page_range() on
         * this pre-allocated page table.
-        *
-        * We don't select ARCH_ENABLE_SPLIT_PMD_PTLOCK if pmd is
-        * folded, and if so pagetable_pte_ctor() becomes nop.
         */
-       if (shift == PAGE_SHIFT)
+       switch (pgtable_type) {
+       case TABLE_PTE:
                BUG_ON(!pagetable_pte_ctor(NULL, ptdesc));
-       else if (shift == PMD_SHIFT)
+               break;
+       case TABLE_PMD:
                BUG_ON(!pagetable_pmd_ctor(NULL, ptdesc));
+               break;
+       default:
+               break;
+       }
 
        return pa;
 }