asm-generic: pgalloc: provide generic __pgd_{alloc,free}

We already have a generic implementation of alloc/free up to P4D level, as
well as pgd_free().  Let's finish the work and add a generic PGD-level
alloc helper as well.

Unlike at lower levels, almost all architectures need some specific magic
at PGD level (typically initialising PGD entries), so introducing a
generic pgd_alloc() isn't worth it.  Instead we introduce two new helpers,
__pgd_alloc() and __pgd_free(), and make use of them in the arch-specific
pgd_alloc() and pgd_free() wherever possible.  To accommodate as many arch
as possible, __pgd_alloc() takes a page allocation order.

Because pagetable_alloc() allocates zeroed pages, explicit zeroing in
pgd_alloc() becomes redundant and we can get rid of it.  Some trivial
implementations of pgd_free() also become unnecessary once __pgd_alloc()
is used; remove them.

Another small improvement is consistent accounting of PGD pages by using
GFP_PGTABLE_{USER,KERNEL} as appropriate.

Not all PGD allocations can be handled by the generic helpers.  In
particular, multiple architectures allocate PGDs from a kmem_cache, and
those PGDs may not be page-sized.

Link: https://lkml.kernel.org/r/20250103184415.2744423-6-kevin.brodsky@arm.com
Signed-off-by: Kevin Brodsky <kevin.brodsky@arm.com>
Acked-by: Dave Hansen <dave.hansen@linux.intel.com>
Acked-by: Qi Zheng <zhengqi.arch@bytedance.com>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Linus Walleij <linus.walleij@linaro.org>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Mike Rapoport (Microsoft) <rppt@kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Ryan Roberts <ryan.roberts@arm.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Will Deacon <will@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/arch/alpha/mm/init.c b/arch/alpha/mm/init.c
index 4fe618446e4cf99caff7e965e81ceb2bf74679d1..61c2198b1359cf81190b934c54a21b8de60d02e7 100644 (file)
--- a/arch/alpha/mm/init.c
+++ b/arch/alpha/mm/init.c
@@ -42,7 +42,7 @@ pgd_alloc(struct mm_struct *mm)
 {
        pgd_t *ret, *init;
 
-       ret = (pgd_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO);
+       ret = __pgd_alloc(mm, 0);
        init = pgd_offset(&init_mm, 0UL);
        if (ret) {
 #ifdef CONFIG_ALPHA_LARGE_VMALLOC

diff --git a/arch/arc/include/asm/pgalloc.h b/arch/arc/include/asm/pgalloc.h
index 096b8ef58edb92a35cc3b6d6614b61ca9cf1b4fa..dfae070fe8d556234c5e4c7b65db4193dd050c3f 100644 (file)
--- a/arch/arc/include/asm/pgalloc.h
+++ b/arch/arc/include/asm/pgalloc.h
@@ -53,19 +53,14 @@ static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, pgtable_t pte_
 
 static inline pgd_t *pgd_alloc(struct mm_struct *mm)
 {
-       pgd_t *ret = (pgd_t *) __get_free_page(GFP_KERNEL);
+       pgd_t *ret = __pgd_alloc(mm, 0);
 
        if (ret) {
                int num, num2;
-               num = USER_PTRS_PER_PGD + USER_KERNEL_GUTTER / PGDIR_SIZE;
-               memzero(ret, num * sizeof(pgd_t));
 
+               num = USER_PTRS_PER_PGD + USER_KERNEL_GUTTER / PGDIR_SIZE;
                num2 = VMALLOC_SIZE / PGDIR_SIZE;
                memcpy(ret + num, swapper_pg_dir + num, num2 * sizeof(pgd_t));
-
-               memzero(ret + num + num2,
-                              (PTRS_PER_PGD - num - num2) * sizeof(pgd_t));
-
        }
        return ret;
 }

diff --git a/arch/arm/mm/pgd.c b/arch/arm/mm/pgd.c
index 2a1077747758bad7a2da3fd83fb6402595210964..4eb81b7ed03afbd914b35ae146527e701f150de0 100644 (file)
--- a/arch/arm/mm/pgd.c
+++ b/arch/arm/mm/pgd.c
@@ -17,11 +17,11 @@
 #include "mm.h"
 
 #ifdef CONFIG_ARM_LPAE
-#define _pgd_alloc(mm)         kmalloc_array(PTRS_PER_PGD, sizeof(pgd_t), GFP_KERNEL)
+#define _pgd_alloc(mm)         kmalloc_array(PTRS_PER_PGD, sizeof(pgd_t), GFP_KERNEL | __GFP_ZERO)
 #define _pgd_free(mm, pgd)     kfree(pgd)
 #else
-#define _pgd_alloc(mm)         (pgd_t *)__get_free_pages(GFP_KERNEL, 2)
-#define _pgd_free(mm, pgd)     free_pages((unsigned long)pgd, 2)
+#define _pgd_alloc(mm)         __pgd_alloc(mm, 2)
+#define _pgd_free(mm, pgd)     __pgd_free(mm, pgd)
 #endif
 
 /*
@@ -39,8 +39,6 @@ pgd_t *pgd_alloc(struct mm_struct *mm)
        if (!new_pgd)
                goto no_pgd;
 
-       memset(new_pgd, 0, USER_PTRS_PER_PGD * sizeof(pgd_t));
-
        /*
         * Copy over the kernel and IO PGD entries
         */

diff --git a/arch/arm64/mm/pgd.c b/arch/arm64/mm/pgd.c
index 0c501cabc23846c4c420c3fc04ac52c578ff3086..8160cff35089d2d3c3d68b7a6b9637c2bd1f2609 100644 (file)
--- a/arch/arm64/mm/pgd.c
+++ b/arch/arm64/mm/pgd.c
@@ -33,7 +33,7 @@ pgd_t *pgd_alloc(struct mm_struct *mm)
        gfp_t gfp = GFP_PGTABLE_USER;
 
        if (pgdir_is_page_size())
-               return (pgd_t *)__get_free_page(gfp);
+               return __pgd_alloc(mm, 0);
        else
                return kmem_cache_alloc(pgd_cache, gfp);
 }
@@ -41,7 +41,7 @@ pgd_t *pgd_alloc(struct mm_struct *mm)
 void pgd_free(struct mm_struct *mm, pgd_t *pgd)
 {
        if (pgdir_is_page_size())
-               free_page((unsigned long)pgd);
+               __pgd_free(mm, pgd);
        else
                kmem_cache_free(pgd_cache, pgd);
 }

diff --git a/arch/csky/include/asm/pgalloc.h b/arch/csky/include/asm/pgalloc.h
index f1ce5b7b28f22c17ada61ae251c7f990f7fc7a87..bf8400c28b5a3cc04b101af2b50c86f1d68a54ae 100644 (file)
--- a/arch/csky/include/asm/pgalloc.h
+++ b/arch/csky/include/asm/pgalloc.h
@@ -44,7 +44,7 @@ static inline pgd_t *pgd_alloc(struct mm_struct *mm)
        pgd_t *ret;
        pgd_t *init;
 
-       ret = (pgd_t *) __get_free_page(GFP_KERNEL);
+       ret = __pgd_alloc(mm, 0);
        if (ret) {
                init = pgd_offset(&init_mm, 0UL);
                pgd_init((unsigned long *)ret);

diff --git a/arch/hexagon/include/asm/pgalloc.h b/arch/hexagon/include/asm/pgalloc.h
index 40e42a0e71673da607412579d55db9cc03556eb8..1ee5f5f157ca75a2bc5b7d80694d0ed632ad7702 100644 (file)
--- a/arch/hexagon/include/asm/pgalloc.h
+++ b/arch/hexagon/include/asm/pgalloc.h
@@ -22,7 +22,7 @@ static inline pgd_t *pgd_alloc(struct mm_struct *mm)
 {
        pgd_t *pgd;
 
-       pgd = (pgd_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO);
+       pgd = __pgd_alloc(mm, 0);
 
        /*
         * There may be better ways to do this, but to ensure

diff --git a/arch/loongarch/mm/pgtable.c b/arch/loongarch/mm/pgtable.c
index 3fa69b23ff8400198337f0073ef42b8c40e5b374..22a94bb3e6e8b8b89cabba03b6bc806323660375 100644 (file)
--- a/arch/loongarch/mm/pgtable.c
+++ b/arch/loongarch/mm/pgtable.c
@@ -23,11 +23,10 @@ EXPORT_SYMBOL(tlb_virt_to_page);
 
 pgd_t *pgd_alloc(struct mm_struct *mm)
 {
-       pgd_t *init, *ret = NULL;
-       struct ptdesc *ptdesc = pagetable_alloc(GFP_KERNEL & ~__GFP_HIGHMEM, 0);
+       pgd_t *init, *ret;
 
-       if (ptdesc) {
-               ret = (pgd_t *)ptdesc_address(ptdesc);
+       ret = __pgd_alloc(mm, 0);
+       if (ret) {
                init = pgd_offset(&init_mm, 0UL);
                pgd_init(ret);
                memcpy(ret + USER_PTRS_PER_PGD, init + USER_PTRS_PER_PGD,

diff --git a/arch/m68k/include/asm/sun3_pgalloc.h b/arch/m68k/include/asm/sun3_pgalloc.h
index 2b626cb3ad0aedd8ac4d24c619dbccbb14b5026f..f1ae4ed890db511b045e6798ed24c83f727ca280 100644 (file)
--- a/arch/m68k/include/asm/sun3_pgalloc.h
+++ b/arch/m68k/include/asm/sun3_pgalloc.h
@@ -43,7 +43,7 @@ static inline pgd_t * pgd_alloc(struct mm_struct *mm)
 {
        pgd_t *new_pgd;
 
-       new_pgd = (pgd_t *)get_zeroed_page(GFP_KERNEL);
+       new_pgd = __pgd_alloc(mm, 0);
        memcpy(new_pgd, swapper_pg_dir, PAGE_SIZE);
        memset(new_pgd, 0, (PAGE_OFFSET >> PGDIR_SHIFT));
        return new_pgd;

diff --git a/arch/microblaze/include/asm/pgalloc.h b/arch/microblaze/include/asm/pgalloc.h
index 6c33b05f730ff01846ce8b8f9a22449e2eb74090..084a8a0dc239525dae6550967e514d02bc9c2da5 100644 (file)
--- a/arch/microblaze/include/asm/pgalloc.h
+++ b/arch/microblaze/include/asm/pgalloc.h
@@ -21,12 +21,7 @@
 
 extern void __bad_pte(pmd_t *pmd);
 
-static inline pgd_t *get_pgd(void)
-{
-       return (pgd_t *)__get_free_pages(GFP_KERNEL|__GFP_ZERO, 0);
-}
-
-#define pgd_alloc(mm)          get_pgd()
+#define pgd_alloc(mm)          __pgd_alloc(mm, 0)
 
 extern pte_t *pte_alloc_one_kernel(struct mm_struct *mm);
 

diff --git a/arch/mips/include/asm/pgalloc.h b/arch/mips/include/asm/pgalloc.h
index 36d9805033c4b3ca0b897ab9c071ece41a37af08..26c7a6ede983c706b4ce2edfcafbc698b6d8b6f5 100644 (file)
--- a/arch/mips/include/asm/pgalloc.h
+++ b/arch/mips/include/asm/pgalloc.h
@@ -15,7 +15,6 @@
 
 #define __HAVE_ARCH_PMD_ALLOC_ONE
 #define __HAVE_ARCH_PUD_ALLOC_ONE
-#define __HAVE_ARCH_PGD_FREE
 #include <asm-generic/pgalloc.h>
 
 static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd,
@@ -49,11 +48,6 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
 extern void pgd_init(void *addr);
 extern pgd_t *pgd_alloc(struct mm_struct *mm);
 
-static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)
-{
-       pagetable_free(virt_to_ptdesc(pgd));
-}
-
 #define __pte_free_tlb(tlb, pte, address)                      \
 do {                                                           \
        pagetable_dtor(page_ptdesc(pte));                       \

diff --git a/arch/mips/mm/pgtable.c b/arch/mips/mm/pgtable.c
index 1506e458040d48c1ffd2a33abce68c50b67a7641..10835414819f3c7883642149004e1ec5c8860ca8 100644 (file)
--- a/arch/mips/mm/pgtable.c
+++ b/arch/mips/mm/pgtable.c
@@ -10,12 +10,10 @@
 
 pgd_t *pgd_alloc(struct mm_struct *mm)
 {
-       pgd_t *init, *ret = NULL;
-       struct ptdesc *ptdesc = pagetable_alloc(GFP_KERNEL & ~__GFP_HIGHMEM,
-                       PGD_TABLE_ORDER);
+       pgd_t *init, *ret;
 
-       if (ptdesc) {
-               ret = ptdesc_address(ptdesc);
+       ret = __pgd_alloc(mm, PGD_TABLE_ORDER);
+       if (ret) {
                init = pgd_offset(&init_mm, 0UL);
                pgd_init(ret);
                memcpy(ret + USER_PTRS_PER_PGD, init + USER_PTRS_PER_PGD,

diff --git a/arch/nios2/mm/pgtable.c b/arch/nios2/mm/pgtable.c
index 7c76e8a7447a608644a5764e5ab1aef5c238c152..6470ed3787826d2e685557e2f78951addc433849 100644 (file)
--- a/arch/nios2/mm/pgtable.c
+++ b/arch/nios2/mm/pgtable.c
@@ -11,6 +11,7 @@
 #include <linux/sched.h>
 
 #include <asm/cpuinfo.h>
+#include <asm/pgalloc.h>
 
 /* pteaddr:
  *   ptbase | vpn* | zero
@@ -54,7 +55,7 @@ pgd_t *pgd_alloc(struct mm_struct *mm)
 {
        pgd_t *ret, *init;
 
-       ret = (pgd_t *) __get_free_page(GFP_KERNEL);
+       ret = __pgd_alloc(mm, 0);
        if (ret) {
                init = pgd_offset(&init_mm, 0UL);
                pgd_init(ret);

diff --git a/arch/openrisc/include/asm/pgalloc.h b/arch/openrisc/include/asm/pgalloc.h
index 596e2355824e3e79af6f15619a0307cbe9ca4578..3372f4e6ab4b557e00365886eaf49264e79f230d 100644 (file)
--- a/arch/openrisc/include/asm/pgalloc.h
+++ b/arch/openrisc/include/asm/pgalloc.h
@@ -41,15 +41,13 @@ static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd,
  */
 static inline pgd_t *pgd_alloc(struct mm_struct *mm)
 {
-       pgd_t *ret = (pgd_t *)__get_free_page(GFP_KERNEL);
+       pgd_t *ret = __pgd_alloc(mm, 0);
 
-       if (ret) {
-               memset(ret, 0, USER_PTRS_PER_PGD * sizeof(pgd_t));
+       if (ret)
                memcpy(ret + USER_PTRS_PER_PGD,
                       swapper_pg_dir + USER_PTRS_PER_PGD,
                       (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t));
 
-       }
        return ret;
 }
 

diff --git a/arch/parisc/include/asm/pgalloc.h b/arch/parisc/include/asm/pgalloc.h
index 3e8dbd79670b1859f9e8de4ea29b5200c56fa8d7..2ca74a56415ca759aab552877bf0c784cbbf2c4d 100644 (file)
--- a/arch/parisc/include/asm/pgalloc.h
+++ b/arch/parisc/include/asm/pgalloc.h
@@ -11,26 +11,12 @@
 #include <asm/cache.h>
 
 #define __HAVE_ARCH_PMD_ALLOC_ONE
-#define __HAVE_ARCH_PGD_FREE
 #include <asm-generic/pgalloc.h>
 
 /* Allocate the top level pgd (page directory) */
 static inline pgd_t *pgd_alloc(struct mm_struct *mm)
 {
-       pgd_t *pgd;
-
-       pgd = (pgd_t *) __get_free_pages(GFP_KERNEL, PGD_TABLE_ORDER);
-       if (unlikely(pgd == NULL))
-               return NULL;
-
-       memset(pgd, 0, PAGE_SIZE << PGD_TABLE_ORDER);
-
-       return pgd;
-}
-
-static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)
-{
-       free_pages((unsigned long)pgd, PGD_TABLE_ORDER);
+       return __pgd_alloc(mm, PGD_TABLE_ORDER);
 }
 
 #if CONFIG_PGTABLE_LEVELS == 3

diff --git a/arch/riscv/include/asm/pgalloc.h b/arch/riscv/include/asm/pgalloc.h
index c8907b83171153bd7d3fa2a5197f278e9af69b3f..3e2aebea6312dbac6cb12678ddd472a38a57071c 100644 (file)
--- a/arch/riscv/include/asm/pgalloc.h
+++ b/arch/riscv/include/asm/pgalloc.h
@@ -130,9 +130,8 @@ static inline pgd_t *pgd_alloc(struct mm_struct *mm)
 {
        pgd_t *pgd;
 
-       pgd = (pgd_t *)__get_free_page(GFP_KERNEL);
+       pgd = __pgd_alloc(mm, 0);
        if (likely(pgd != NULL)) {
-               memset(pgd, 0, USER_PTRS_PER_PGD * sizeof(pgd_t));
                /* Copy kernel mappings */
                sync_kernel_mappings(pgd);
        }

diff --git a/arch/um/kernel/mem.c b/arch/um/kernel/mem.c
index 53248ed04771d1795342568981e2c8d15a1fd343..d9881290749376834423582cb91f4d96e596f047 100644 (file)
--- a/arch/um/kernel/mem.c
+++ b/arch/um/kernel/mem.c
@@ -214,14 +214,13 @@ void free_initmem(void)
 
 pgd_t *pgd_alloc(struct mm_struct *mm)
 {
-       pgd_t *pgd = (pgd_t *)__get_free_page(GFP_KERNEL);
+       pgd_t *pgd = __pgd_alloc(mm, 0);
 
-       if (pgd) {
-               memset(pgd, 0, USER_PTRS_PER_PGD * sizeof(pgd_t));
+       if (pgd)
                memcpy(pgd + USER_PTRS_PER_PGD,
                       swapper_pg_dir + USER_PTRS_PER_PGD,
                       (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t));
-       }
+
        return pgd;
 }
 

diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c
index c02aa0427a6a59f113d637ad47359cdb6ba7c006..1fef5ad32d5a8be139b1d93aa84291a93bf13245 100644 (file)
--- a/arch/x86/mm/pgtable.c
+++ b/arch/x86/mm/pgtable.c
@@ -397,15 +397,14 @@ void __init pgtable_cache_init(void)
                                      SLAB_PANIC, NULL);
 }
 
-static inline pgd_t *_pgd_alloc(void)
+static inline pgd_t *_pgd_alloc(struct mm_struct *mm)
 {
        /*
         * If no SHARED_KERNEL_PMD, PAE kernel is running as a Xen domain.
         * We allocate one page for pgd.
         */
        if (!SHARED_KERNEL_PMD)
-               return (pgd_t *)__get_free_pages(GFP_PGTABLE_USER,
-                                                PGD_ALLOCATION_ORDER);
+               return __pgd_alloc(mm, PGD_ALLOCATION_ORDER);
 
        /*
         * Now PAE kernel is not running as a Xen domain. We can allocate
@@ -414,24 +413,23 @@ static inline pgd_t *_pgd_alloc(void)
        return kmem_cache_alloc(pgd_cache, GFP_PGTABLE_USER);
 }
 
-static inline void _pgd_free(pgd_t *pgd)
+static inline void _pgd_free(struct mm_struct *mm, pgd_t *pgd)
 {
        if (!SHARED_KERNEL_PMD)
-               free_pages((unsigned long)pgd, PGD_ALLOCATION_ORDER);
+               __pgd_free(mm, pgd);
        else
                kmem_cache_free(pgd_cache, pgd);
 }
 #else
 
-static inline pgd_t *_pgd_alloc(void)
+static inline pgd_t *_pgd_alloc(struct mm_struct *mm)
 {
-       return (pgd_t *)__get_free_pages(GFP_PGTABLE_USER,
-                                        PGD_ALLOCATION_ORDER);
+       return __pgd_alloc(mm, PGD_ALLOCATION_ORDER);
 }
 
-static inline void _pgd_free(pgd_t *pgd)
+static inline void _pgd_free(struct mm_struct *mm, pgd_t *pgd)
 {
-       free_pages((unsigned long)pgd, PGD_ALLOCATION_ORDER);
+       __pgd_free(mm, pgd);
 }
 #endif /* CONFIG_X86_PAE */
 
@@ -441,7 +439,7 @@ pgd_t *pgd_alloc(struct mm_struct *mm)
        pmd_t *u_pmds[MAX_PREALLOCATED_USER_PMDS];
        pmd_t *pmds[MAX_PREALLOCATED_PMDS];
 
-       pgd = _pgd_alloc();
+       pgd = _pgd_alloc(mm);
 
        if (pgd == NULL)
                goto out;
@@ -484,7 +482,7 @@ out_free_pmds:
        if (sizeof(pmds) != 0)
                free_pmds(mm, pmds, PREALLOCATED_PMDS);
 out_free_pgd:
-       _pgd_free(pgd);
+       _pgd_free(mm, pgd);
 out:
        return NULL;
 }
@@ -494,7 +492,7 @@ void pgd_free(struct mm_struct *mm, pgd_t *pgd)
        pgd_mop_up_pmds(mm, pgd);
        pgd_dtor(pgd);
        paravirt_pgd_free(mm, pgd);
-       _pgd_free(pgd);
+       _pgd_free(mm, pgd);
 }
 
 /*

diff --git a/arch/xtensa/include/asm/pgalloc.h b/arch/xtensa/include/asm/pgalloc.h
index 7fc0f9126dd3545d7203947dbdfe0f65f3f87a58..1919ee9c3dd6f90849fb1a8fe3541da11b95f0e7 100644 (file)
--- a/arch/xtensa/include/asm/pgalloc.h
+++ b/arch/xtensa/include/asm/pgalloc.h
@@ -29,7 +29,7 @@
 static inline pgd_t*
 pgd_alloc(struct mm_struct *mm)
 {
-       return (pgd_t*) __get_free_page(GFP_KERNEL | __GFP_ZERO);
+       return __pgd_alloc(mm, 0);
 }
 
 static inline void ptes_clear(pte_t *ptep)

diff --git a/include/asm-generic/pgalloc.h b/include/asm-generic/pgalloc.h
index e3977ddca15e476e9db3f81b895796421ff69a0a..de4df14158e6762ab657b3a89ca70f5073051883 100644 (file)
--- a/include/asm-generic/pgalloc.h
+++ b/include/asm-generic/pgalloc.h
@@ -258,10 +258,35 @@ static inline void p4d_free(struct mm_struct *mm, p4d_t *p4d)
 
 #endif /* CONFIG_PGTABLE_LEVELS > 4 */
 
+static inline pgd_t *__pgd_alloc_noprof(struct mm_struct *mm, unsigned int order)
+{
+       gfp_t gfp = GFP_PGTABLE_USER;
+       struct ptdesc *ptdesc;
+
+       if (mm == &init_mm)
+               gfp = GFP_PGTABLE_KERNEL;
+       gfp &= ~__GFP_HIGHMEM;
+
+       ptdesc = pagetable_alloc_noprof(gfp, order);
+       if (!ptdesc)
+               return NULL;
+
+       return ptdesc_address(ptdesc);
+}
+#define __pgd_alloc(...)       alloc_hooks(__pgd_alloc_noprof(__VA_ARGS__))
+
+static inline void __pgd_free(struct mm_struct *mm, pgd_t *pgd)
+{
+       struct ptdesc *ptdesc = virt_to_ptdesc(pgd);
+
+       BUG_ON((unsigned long)pgd & (PAGE_SIZE-1));
+       pagetable_free(ptdesc);
+}
+
 #ifndef __HAVE_ARCH_PGD_FREE
 static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)
 {
-       pagetable_free(virt_to_ptdesc(pgd));
+       __pgd_free(mm, pgd);
 }
 #endif