powerpc/book3s: Use config independent helpers for page table walk

Even when we have HugeTLB and THP disabled, kernel linear map can still be
mapped with hugepages. This is only an issue with radix translation because hash
MMU doesn't map kernel linear range in linux page table and other kernel
map areas are not mapped using hugepage.

Add config independent helpers and put WARN_ON() when we don't expect things
to be mapped via hugepages.

Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.ibm.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>

diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h
index beabb099c04000acf3de74871b875e2563423c4f..62e6ea0a7650a5230f3674be0924c5039e01dfbf 100644 (file)
--- a/arch/powerpc/include/asm/book3s/64/pgtable.h
+++ b/arch/powerpc/include/asm/book3s/64/pgtable.h
@@ -1350,5 +1350,26 @@ static inline bool is_pte_rw_upgrade(unsigned long old_val, unsigned long new_va
        return false;
 }
 
+/*
+ * Like pmd_huge() and pmd_large(), but works regardless of config options
+ */
+#define pmd_is_leaf pmd_is_leaf
+static inline bool pmd_is_leaf(pmd_t pmd)
+{
+       return !!(pmd_raw(pmd) & cpu_to_be64(_PAGE_PTE));
+}
+
+#define pud_is_leaf pud_is_leaf
+static inline bool pud_is_leaf(pud_t pud)
+{
+       return !!(pud_raw(pud) & cpu_to_be64(_PAGE_PTE));
+}
+
+#define pgd_is_leaf pgd_is_leaf
+static inline bool pgd_is_leaf(pgd_t pgd)
+{
+       return !!(pgd_raw(pgd) & cpu_to_be64(_PAGE_PTE));
+}
+
 #endif /* __ASSEMBLY__ */
 #endif /* _ASM_POWERPC_BOOK3S_64_PGTABLE_H_ */

diff --git a/arch/powerpc/include/asm/pgtable.h b/arch/powerpc/include/asm/pgtable.h
index 3f53be60fb014928440ba5e3f5e7e9a985b9499e..bf7d771f342eea2a3e47cd68e950a198f78f8827 100644 (file)
--- a/arch/powerpc/include/asm/pgtable.h
+++ b/arch/powerpc/include/asm/pgtable.h
@@ -140,6 +140,30 @@ static inline void pte_frag_set(mm_context_t *ctx, void *p)
 }
 #endif
 
+#ifndef pmd_is_leaf
+#define pmd_is_leaf pmd_is_leaf
+static inline bool pmd_is_leaf(pmd_t pmd)
+{
+       return false;
+}
+#endif
+
+#ifndef pud_is_leaf
+#define pud_is_leaf pud_is_leaf
+static inline bool pud_is_leaf(pud_t pud)
+{
+       return false;
+}
+#endif
+
+#ifndef pgd_is_leaf
+#define pgd_is_leaf pgd_is_leaf
+static inline bool pgd_is_leaf(pgd_t pgd)
+{
+       return false;
+}
+#endif
+
 #endif /* __ASSEMBLY__ */
 
 #endif /* _ASM_POWERPC_PGTABLE_H */

diff --git a/arch/powerpc/include/asm/pte-walk.h b/arch/powerpc/include/asm/pte-walk.h
index 2d633e9d686c4f9d3b4ea39ebd160d93806467a7..33fa5dd8ee6a7d7e90f5ae0c087037c4098ce904 100644 (file)
--- a/arch/powerpc/include/asm/pte-walk.h
+++ b/arch/powerpc/include/asm/pte-walk.h
@@ -10,8 +10,20 @@ extern pte_t *__find_linux_pte(pgd_t *pgdir, unsigned long ea,
 static inline pte_t *find_linux_pte(pgd_t *pgdir, unsigned long ea,
                                    bool *is_thp, unsigned *hshift)
 {
+       pte_t *pte;
+
        VM_WARN(!arch_irqs_disabled(), "%s called with irq enabled\n", __func__);
-       return __find_linux_pte(pgdir, ea, is_thp, hshift);
+       pte = __find_linux_pte(pgdir, ea, is_thp, hshift);
+
+#if defined(CONFIG_DEBUG_VM) &&                                                \
+       !(defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE))
+       /*
+        * We should not find huge page if these configs are not enabled.
+        */
+       if (hshift)
+               WARN_ON(*hshift);
+#endif
+       return pte;
 }
 
 static inline pte_t *find_init_mm_pte(unsigned long ea, unsigned *hshift)
@@ -26,10 +38,22 @@ static inline pte_t *find_init_mm_pte(unsigned long ea, unsigned *hshift)
 static inline pte_t *find_current_mm_pte(pgd_t *pgdir, unsigned long ea,
                                         bool *is_thp, unsigned *hshift)
 {
+       pte_t *pte;
+
        VM_WARN(!arch_irqs_disabled(), "%s called with irq enabled\n", __func__);
        VM_WARN(pgdir != current->mm->pgd,
                "%s lock less page table lookup called on wrong mm\n", __func__);
-       return __find_linux_pte(pgdir, ea, is_thp, hshift);
+       pte = __find_linux_pte(pgdir, ea, is_thp, hshift);
+
+#if defined(CONFIG_DEBUG_VM) &&                                                \
+       !(defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE))
+       /*
+        * We should not find huge page if these configs are not enabled.
+        */
+       if (hshift)
+               WARN_ON(*hshift);
+#endif
+       return pte;
 }
 
 #endif /* _ASM_POWERPC_PTE_WALK_H */

diff --git a/arch/powerpc/kvm/book3s_64_mmu_radix.c b/arch/powerpc/kvm/book3s_64_mmu_radix.c
index f55ef071883f13166662ca006e10d314d70a135c..91efee7f0329a9753c0b6f835485187bc9643403 100644 (file)
--- a/arch/powerpc/kvm/book3s_64_mmu_radix.c
+++ b/arch/powerpc/kvm/book3s_64_mmu_radix.c
@@ -363,12 +363,6 @@ static void kvmppc_pte_free(pte_t *ptep)
        kmem_cache_free(kvm_pte_cache, ptep);
 }
 
-/* Like pmd_huge() and pmd_large(), but works regardless of config options */
-static inline int pmd_is_leaf(pmd_t pmd)
-{
-       return !!(pmd_val(pmd) & _PAGE_PTE);
-}
-
 static pmd_t *kvmppc_pmd_alloc(void)
 {
        return kmem_cache_alloc(kvm_pmd_cache, GFP_KERNEL);
@@ -489,7 +483,7 @@ static void kvmppc_unmap_free_pud(struct kvm *kvm, pud_t *pud,
        for (iu = 0; iu < PTRS_PER_PUD; ++iu, ++p) {
                if (!pud_present(*p))
                        continue;
-               if (pud_huge(*p)) {
+               if (pud_is_leaf(*p)) {
                        pud_clear(p);
                } else {
                        pmd_t *pmd;
@@ -588,7 +582,7 @@ int kvmppc_create_pte(struct kvm *kvm, pgd_t *pgtable, pte_t pte,
                new_pud = pud_alloc_one(kvm->mm, gpa);
 
        pmd = NULL;
-       if (pud && pud_present(*pud) && !pud_huge(*pud))
+       if (pud && pud_present(*pud) && !pud_is_leaf(*pud))
                pmd = pmd_offset(pud, gpa);
        else if (level <= 1)
                new_pmd = kvmppc_pmd_alloc();
@@ -611,7 +605,7 @@ int kvmppc_create_pte(struct kvm *kvm, pgd_t *pgtable, pte_t pte,
                new_pud = NULL;
        }
        pud = pud_offset(pgd, gpa);
-       if (pud_huge(*pud)) {
+       if (pud_is_leaf(*pud)) {
                unsigned long hgpa = gpa & PUD_MASK;
 
                /* Check if we raced and someone else has set the same thing */

diff --git a/arch/powerpc/mm/book3s64/radix_pgtable.c b/arch/powerpc/mm/book3s64/radix_pgtable.c
index b573fa024b3291b2ce9a0184fdb5d6e980d7d1b0..e92c6472a20c0f634fbd0845d45811c780c429a6 100644 (file)
--- a/arch/powerpc/mm/book3s64/radix_pgtable.c
+++ b/arch/powerpc/mm/book3s64/radix_pgtable.c
@@ -203,14 +203,14 @@ void radix__change_memory_range(unsigned long start, unsigned long end,
                pudp = pud_alloc(&init_mm, pgdp, idx);
                if (!pudp)
                        continue;
-               if (pud_huge(*pudp)) {
+               if (pud_is_leaf(*pudp)) {
                        ptep = (pte_t *)pudp;
                        goto update_the_pte;
                }
                pmdp = pmd_alloc(&init_mm, pudp, idx);
                if (!pmdp)
                        continue;
-               if (pmd_huge(*pmdp)) {
+               if (pmd_is_leaf(*pmdp)) {
                        ptep = pmdp_ptep(pmdp);
                        goto update_the_pte;
                }
@@ -835,7 +835,7 @@ static void remove_pmd_table(pmd_t *pmd_start, unsigned long addr,
                if (!pmd_present(*pmd))
                        continue;
 
-               if (pmd_huge(*pmd)) {
+               if (pmd_is_leaf(*pmd)) {
                        split_kernel_mapping(addr, end, PMD_SIZE, (pte_t *)pmd);
                        continue;
                }
@@ -860,7 +860,7 @@ static void remove_pud_table(pud_t *pud_start, unsigned long addr,
                if (!pud_present(*pud))
                        continue;
 
-               if (pud_huge(*pud)) {
+               if (pud_is_leaf(*pud)) {
                        split_kernel_mapping(addr, end, PUD_SIZE, (pte_t *)pud);
                        continue;
                }
@@ -886,7 +886,7 @@ static void __meminit remove_pagetable(unsigned long start, unsigned long end)
                if (!pgd_present(*pgd))
                        continue;
 
-               if (pgd_huge(*pgd)) {
+               if (pgd_is_leaf(*pgd)) {
                        split_kernel_mapping(addr, end, PGDIR_SIZE, (pte_t *)pgd);
                        continue;
                }

diff --git a/arch/powerpc/mm/pgtable.c b/arch/powerpc/mm/pgtable.c
index 533fc6fa672670614c193339f2dd4285963781b3..2029e585e5c33162b586a8dfa1c3e949d0c525b5 100644 (file)
--- a/arch/powerpc/mm/pgtable.c
+++ b/arch/powerpc/mm/pgtable.c
@@ -340,10 +340,11 @@ pte_t *__find_linux_pte(pgd_t *pgdir, unsigned long ea,
        if (pgd_none(pgd))
                return NULL;
 
-       if (pgd_huge(pgd)) {
+       if (pgd_is_leaf(pgd)) {
                ret_pte = (pte_t *)pgdp;
                goto out;
        }
+
        if (is_hugepd(__hugepd(pgd_val(pgd)))) {
                hpdp = (hugepd_t *)&pgd;
                goto out_huge;
@@ -361,14 +362,16 @@ pte_t *__find_linux_pte(pgd_t *pgdir, unsigned long ea,
        if (pud_none(pud))
                return NULL;
 
-       if (pud_huge(pud)) {
+       if (pud_is_leaf(pud)) {
                ret_pte = (pte_t *)pudp;
                goto out;
        }
+
        if (is_hugepd(__hugepd(pud_val(pud)))) {
                hpdp = (hugepd_t *)&pud;
                goto out_huge;
        }
+
        pdshift = PMD_SHIFT;
        pmdp = pmd_offset(&pud, ea);
        pmd  = READ_ONCE(*pmdp);
@@ -397,15 +400,12 @@ pte_t *__find_linux_pte(pgd_t *pgdir, unsigned long ea,
                ret_pte = (pte_t *)pmdp;
                goto out;
        }
-       /*
-        * pmd_large check below will handle the swap pmd pte
-        * we need to do both the check because they are config
-        * dependent.
-        */
-       if (pmd_huge(pmd) || pmd_large(pmd)) {
+
+       if (pmd_is_leaf(pmd)) {
                ret_pte = (pte_t *)pmdp;
                goto out;
        }
+
        if (is_hugepd(__hugepd(pmd_val(pmd)))) {
                hpdp = (hugepd_t *)&pmd;
                goto out_huge;

diff --git a/arch/powerpc/mm/pgtable_64.c b/arch/powerpc/mm/pgtable_64.c
index 63cd811306435184b0545cf618fa34dc975914ad..2892246a6fef48a2c3c54d384fccbc7804f5bb90 100644 (file)
--- a/arch/powerpc/mm/pgtable_64.c
+++ b/arch/powerpc/mm/pgtable_64.c
@@ -309,16 +309,20 @@ EXPORT_SYMBOL(__iounmap_at);
 /* 4 level page table */
 struct page *pgd_page(pgd_t pgd)
 {
-       if (pgd_huge(pgd))
+       if (pgd_is_leaf(pgd)) {
+               VM_WARN_ON(!pgd_huge(pgd));
                return pte_page(pgd_pte(pgd));
+       }
        return virt_to_page(pgd_page_vaddr(pgd));
 }
 #endif
 
 struct page *pud_page(pud_t pud)
 {
-       if (pud_huge(pud))
+       if (pud_is_leaf(pud)) {
+               VM_WARN_ON(!pud_huge(pud));
                return pte_page(pud_pte(pud));
+       }
        return virt_to_page(pud_page_vaddr(pud));
 }
 
@@ -328,8 +332,10 @@ struct page *pud_page(pud_t pud)
  */
 struct page *pmd_page(pmd_t pmd)
 {
-       if (pmd_large(pmd) || pmd_huge(pmd) || pmd_devmap(pmd))
+       if (pmd_is_leaf(pmd)) {
+               VM_WARN_ON(!(pmd_large(pmd) || pmd_huge(pmd) || pmd_devmap(pmd)));
                return pte_page(pmd_pte(pmd));
+       }
        return virt_to_page(pmd_page_vaddr(pmd));
 }
 

diff --git a/arch/powerpc/mm/ptdump/ptdump.c b/arch/powerpc/mm/ptdump/ptdump.c
index 646876d9da640fac314b7f1120af7f368439b588..abe60d25b4e6f7ac476e10feaf15b3d9321773d5 100644 (file)
--- a/arch/powerpc/mm/ptdump/ptdump.c
+++ b/arch/powerpc/mm/ptdump/ptdump.c
@@ -277,7 +277,7 @@ static void walk_pmd(struct pg_state *st, pud_t *pud, unsigned long start)
 
        for (i = 0; i < PTRS_PER_PMD; i++, pmd++) {
                addr = start + i * PMD_SIZE;
-               if (!pmd_none(*pmd) && !pmd_huge(*pmd))
+               if (!pmd_none(*pmd) && !pmd_is_leaf(*pmd))
                        /* pmd exists */
                        walk_pte(st, pmd, addr);
                else
@@ -293,7 +293,7 @@ static void walk_pud(struct pg_state *st, pgd_t *pgd, unsigned long start)
 
        for (i = 0; i < PTRS_PER_PUD; i++, pud++) {
                addr = start + i * PUD_SIZE;
-               if (!pud_none(*pud) && !pud_huge(*pud))
+               if (!pud_none(*pud) && !pud_is_leaf(*pud))
                        /* pud exists */
                        walk_pmd(st, pud, addr);
                else
@@ -314,7 +314,7 @@ static void walk_pagetables(struct pg_state *st)
         * the hash pagetable.
         */
        for (i = 0; i < PTRS_PER_PGD; i++, pgd++, addr += PGDIR_SIZE) {
-               if (!pgd_none(*pgd) && !pgd_huge(*pgd))
+               if (!pgd_none(*pgd) && !pgd_is_leaf(*pgd))
                        /* pgd exists */
                        walk_pud(st, pgd, addr);
                else

diff --git a/arch/powerpc/xmon/xmon.c b/arch/powerpc/xmon/xmon.c
index f879e9fe9733581124bcfce0b3682a73df0231d6..2ec20a5bb556bdaf85f54856471084cb276c1fb2 100644 (file)
--- a/arch/powerpc/xmon/xmon.c
+++ b/arch/powerpc/xmon/xmon.c
@@ -3098,7 +3098,7 @@ static void show_pte(unsigned long addr)
 
        printf("pgd  @ 0x%px\n", pgdir);
 
-       if (pgd_huge(*pgdp)) {
+       if (pgd_is_leaf(*pgdp)) {
                format_pte(pgdp, pgd_val(*pgdp));
                return;
        }
@@ -3111,7 +3111,7 @@ static void show_pte(unsigned long addr)
                return;
        }
 
-       if (pud_huge(*pudp)) {
+       if (pud_is_leaf(*pudp)) {
                format_pte(pudp, pud_val(*pudp));
                return;
        }
@@ -3125,7 +3125,7 @@ static void show_pte(unsigned long addr)
                return;
        }
 
-       if (pmd_huge(*pmdp)) {
+       if (pmd_is_leaf(*pmdp)) {
                format_pte(pmdp, pmd_val(*pmdp));
                return;
        }