mm: introduce ctor/dtor at PGD level

Following on from the introduction of P4D-level ctor/dtor, let's finish
the job and introduce ctor/dtor at PGD level.  The incurred improvement in
page accounting is minimal - the main motivation is to create a single,
generic place where construction/destruction hooks can be added for all
page table pages.

This patch should cover all architectures and all configurations where
PGDs are one or more regular pages.  This excludes any configuration where
PGDs are allocated from a kmem_cache object.

Link: https://lkml.kernel.org/r/20250103184415.2744423-7-kevin.brodsky@arm.com
Signed-off-by: Kevin Brodsky <kevin.brodsky@arm.com>
Acked-by: Dave Hansen <dave.hansen@linux.intel.com>
Acked-by: Qi Zheng <zhengqi.arch@bytedance.com>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Linus Walleij <linus.walleij@linaro.org>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Mike Rapoport (Microsoft) <rppt@kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Ryan Roberts <ryan.roberts@arm.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Will Deacon <will@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/arch/m68k/include/asm/mcf_pgalloc.h b/arch/m68k/include/asm/mcf_pgalloc.h
index 22d6c1fcabfb4f4330d5fe71833845b27dc1ca07..4c648b51e7fd5e369024e5705b0baa827163729f 100644 (file)
--- a/arch/m68k/include/asm/mcf_pgalloc.h
+++ b/arch/m68k/include/asm/mcf_pgalloc.h
@@ -73,7 +73,7 @@ static inline void pte_free(struct mm_struct *mm, pgtable_t pgtable)
 
 static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)
 {
-       pagetable_free(virt_to_ptdesc(pgd));
+       pagetable_dtor_free(virt_to_ptdesc(pgd));
 }
 
 static inline pgd_t *pgd_alloc(struct mm_struct *mm)
@@ -84,6 +84,7 @@ static inline pgd_t *pgd_alloc(struct mm_struct *mm)
 
        if (!ptdesc)
                return NULL;
+       pagetable_pgd_ctor(ptdesc);
        new_pgd = ptdesc_address(ptdesc);
 
        memcpy(new_pgd, swapper_pg_dir, PTRS_PER_PGD * sizeof(pgd_t));

diff --git a/arch/m68k/mm/motorola.c b/arch/m68k/mm/motorola.c
index 12de00df53c475c3c1506c7307eff5d8754e90e3..0fb8b098d9266fec4ea77717679dcf9b793f5fed 100644 (file)
--- a/arch/m68k/mm/motorola.c
+++ b/arch/m68k/mm/motorola.c
@@ -169,6 +169,9 @@ void *get_pointer_table(int type)
                case TABLE_PMD:
                        pagetable_pmd_ctor(virt_to_ptdesc(page));
                        break;
+               case TABLE_PGD:
+                       pagetable_pgd_ctor(virt_to_ptdesc(page));
+                       break;
                }
 
                mmu_page_ctor(page);
@@ -207,8 +210,7 @@ int free_pointer_table(void *table, int type)
                /* all tables in page are free, free page */
                list_del(dp);
                mmu_page_dtor((void *)page);
-               if (type == TABLE_PTE || type == TABLE_PMD)
-                       pagetable_dtor(virt_to_ptdesc((void *)page));
+               pagetable_dtor(virt_to_ptdesc((void *)page));
                free_page (page);
                return 1;
        } else if (ptable_list[type].next != dp) {

diff --git a/arch/s390/include/asm/pgalloc.h b/arch/s390/include/asm/pgalloc.h
index 5fced6d3c36b0bd748dbd33960439a2ddc534999..b19b6ed2ab533a91ff476dfd36430d6538e1f1aa 100644 (file)
--- a/arch/s390/include/asm/pgalloc.h
+++ b/arch/s390/include/asm/pgalloc.h
@@ -130,11 +130,18 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
 
 static inline pgd_t *pgd_alloc(struct mm_struct *mm)
 {
-       return (pgd_t *) crst_table_alloc(mm);
+       unsigned long *table = crst_table_alloc(mm);
+
+       if (!table)
+               return NULL;
+       pagetable_pgd_ctor(virt_to_ptdesc(table));
+
+       return (pgd_t *) table;
 }
 
 static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)
 {
+       pagetable_dtor(virt_to_ptdesc(pgd));
        crst_table_free(mm, (unsigned long *) pgd);
 }
 

diff --git a/include/asm-generic/pgalloc.h b/include/asm-generic/pgalloc.h
index de4df14158e6762ab657b3a89ca70f5073051883..892ece4558a209ec0059b53db4c3cdac308db89c 100644 (file)
--- a/include/asm-generic/pgalloc.h
+++ b/include/asm-generic/pgalloc.h
@@ -271,6 +271,7 @@ static inline pgd_t *__pgd_alloc_noprof(struct mm_struct *mm, unsigned int order
        if (!ptdesc)
                return NULL;
 
+       pagetable_pgd_ctor(ptdesc);
        return ptdesc_address(ptdesc);
 }
 #define __pgd_alloc(...)       alloc_hooks(__pgd_alloc_noprof(__VA_ARGS__))
@@ -280,7 +281,7 @@ static inline void __pgd_free(struct mm_struct *mm, pgd_t *pgd)
        struct ptdesc *ptdesc = virt_to_ptdesc(pgd);
 
        BUG_ON((unsigned long)pgd & (PAGE_SIZE-1));
-       pagetable_free(ptdesc);
+       pagetable_dtor_free(ptdesc);
 }
 
 #ifndef __HAVE_ARCH_PGD_FREE

diff --git a/include/linux/mm.h b/include/linux/mm.h
index 2949b58fd633ed8ccfaaee5cfe3d463d854f899a..3550cbeed48813f0888c87de38a4ebfddf8a7e55 100644 (file)
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -3159,6 +3159,11 @@ static inline void pagetable_p4d_ctor(struct ptdesc *ptdesc)
        __pagetable_ctor(ptdesc);
 }
 
+static inline void pagetable_pgd_ctor(struct ptdesc *ptdesc)
+{
+       __pagetable_ctor(ptdesc);
+}
+
 extern void __init pagecache_init(void);
 extern void free_initmem(void);