LoongArch: Set initial pte entry with PAGE_GLOBAL for kernel space

There are two pages in one TLB entry on LoongArch system. For kernel
space, it requires both two pte entries (buddies) with PAGE_GLOBAL bit
set, otherwise HW treats it as non-global tlb, there will be potential
problems if tlb entry for kernel space is not global. Such as fail to
flush kernel tlb with the function local_flush_tlb_kernel_range() which
supposed only flush tlb with global bit.

Kernel address space areas include percpu, vmalloc, vmemmap, fixmap and
kasan areas. For these areas both two consecutive page table entries
should be enabled with PAGE_GLOBAL bit. So with function set_pte() and
pte_clear(), pte buddy entry is checked and set besides its own pte
entry. However it is not atomic operation to set both two pte entries,
there is problem with test_vmalloc test case.

So function kernel_pte_init() is added to init a pte table when it is
created for kernel address space, and the default initial pte value is
PAGE_GLOBAL rather than zero at beginning. Then only its own pte entry
need update with function set_pte() and pte_clear(), nothing to do with
the pte buddy entry.

Signed-off-by: Bibo Mao <maobibo@loongson.cn>
Signed-off-by: Huacai Chen <chenhuacai@loongson.cn>

diff --git a/arch/loongarch/include/asm/pgalloc.h b/arch/loongarch/include/asm/pgalloc.h
index 4e2d6b7ca2eebf5ba14bfa46ae2699e37716d210..a7b9c9e73593d2e4a7d79134ad3b70f3415787c4 100644 (file)
--- a/arch/loongarch/include/asm/pgalloc.h
+++ b/arch/loongarch/include/asm/pgalloc.h
@@ -10,6 +10,7 @@
 
 #define __HAVE_ARCH_PMD_ALLOC_ONE
 #define __HAVE_ARCH_PUD_ALLOC_ONE
+#define __HAVE_ARCH_PTE_ALLOC_ONE_KERNEL
 #include <asm-generic/pgalloc.h>
 
 static inline void pmd_populate_kernel(struct mm_struct *mm,
@@ -44,6 +45,16 @@ extern void pagetable_init(void);
 
 extern pgd_t *pgd_alloc(struct mm_struct *mm);
 
+static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm)
+{
+       pte_t *pte = __pte_alloc_one_kernel(mm);
+
+       if (pte)
+               kernel_pte_init(pte);
+
+       return pte;
+}
+
 #define __pte_free_tlb(tlb, pte, address)                      \
 do {                                                           \
        pagetable_pte_dtor(page_ptdesc(pte));                   \

diff --git a/arch/loongarch/include/asm/pgtable.h b/arch/loongarch/include/asm/pgtable.h
index 9965f52ef65b6f2d24da9dbeae306c5f9ca278a6..20714b73f14c8eef9f5a3461905e02e549e22f34 100644 (file)
--- a/arch/loongarch/include/asm/pgtable.h
+++ b/arch/loongarch/include/asm/pgtable.h
@@ -269,6 +269,7 @@ extern void set_pmd_at(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp, pm
 extern void pgd_init(void *addr);
 extern void pud_init(void *addr);
 extern void pmd_init(void *addr);
+extern void kernel_pte_init(void *addr);
 
 /*
  * Encode/decode swap entries and swap PTEs. Swap PTEs are all PTEs that
@@ -325,39 +326,17 @@ static inline void set_pte(pte_t *ptep, pte_t pteval)
 {
        WRITE_ONCE(*ptep, pteval);
 
-       if (pte_val(pteval) & _PAGE_GLOBAL) {
-               pte_t *buddy = ptep_buddy(ptep);
-               /*
-                * Make sure the buddy is global too (if it's !none,
-                * it better already be global)
-                */
-               if (pte_none(ptep_get(buddy))) {
 #ifdef CONFIG_SMP
-                       /*
-                        * For SMP, multiple CPUs can race, so we need
-                        * to do this atomically.
-                        */
-                       __asm__ __volatile__(
-                       __AMOR "$zero, %[global], %[buddy] \n"
-                       : [buddy] "+ZB" (buddy->pte)
-                       : [global] "r" (_PAGE_GLOBAL)
-                       : "memory");
-
-                       DBAR(0b11000); /* o_wrw = 0b11000 */
-#else /* !CONFIG_SMP */
-                       WRITE_ONCE(*buddy, __pte(pte_val(ptep_get(buddy)) | _PAGE_GLOBAL));
-#endif /* CONFIG_SMP */
-               }
-       }
+       if (pte_val(pteval) & _PAGE_GLOBAL)
+               DBAR(0b11000); /* o_wrw = 0b11000 */
+#endif
 }
 
 static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
 {
-       /* Preserve global status for the pair */
-       if (pte_val(ptep_get(ptep_buddy(ptep))) & _PAGE_GLOBAL)
-               set_pte(ptep, __pte(_PAGE_GLOBAL));
-       else
-               set_pte(ptep, __pte(0));
+       pte_t pte = ptep_get(ptep);
+       pte_val(pte) &= _PAGE_GLOBAL;
+       set_pte(ptep, pte);
 }
 
 #define PGD_T_LOG2     (__builtin_ffs(sizeof(pgd_t)) - 1)

diff --git a/arch/loongarch/mm/init.c b/arch/loongarch/mm/init.c
index 8a87a482c8f44bf085ae792da33342bb35b9b010..188b52bbb254273674eb931bf97e9e250fa30024 100644 (file)
--- a/arch/loongarch/mm/init.c
+++ b/arch/loongarch/mm/init.c
@@ -201,7 +201,9 @@ pte_t * __init populate_kernel_pte(unsigned long addr)
                pte = memblock_alloc(PAGE_SIZE, PAGE_SIZE);
                if (!pte)
                        panic("%s: Failed to allocate memory\n", __func__);
+
                pmd_populate_kernel(&init_mm, pmd, pte);
+               kernel_pte_init(pte);
        }
 
        return pte_offset_kernel(pmd, addr);

diff --git a/arch/loongarch/mm/pgtable.c b/arch/loongarch/mm/pgtable.c
index eb6a29b491a72be73f64a472d57fe3c2856bc9c1..3fa69b23ff8400198337f0073ef42b8c40e5b374 100644 (file)
--- a/arch/loongarch/mm/pgtable.c
+++ b/arch/loongarch/mm/pgtable.c
@@ -116,6 +116,26 @@ void pud_init(void *addr)
 EXPORT_SYMBOL_GPL(pud_init);
 #endif
 
+void kernel_pte_init(void *addr)
+{
+       unsigned long *p, *end;
+
+       p = (unsigned long *)addr;
+       end = p + PTRS_PER_PTE;
+
+       do {
+               p[0] = _PAGE_GLOBAL;
+               p[1] = _PAGE_GLOBAL;
+               p[2] = _PAGE_GLOBAL;
+               p[3] = _PAGE_GLOBAL;
+               p[4] = _PAGE_GLOBAL;
+               p += 8;
+               p[-3] = _PAGE_GLOBAL;
+               p[-2] = _PAGE_GLOBAL;
+               p[-1] = _PAGE_GLOBAL;
+       } while (p != end);
+}
+
 pmd_t mk_pmd(struct page *page, pgprot_t prot)
 {
        pmd_t pmd;

diff --git a/include/linux/mm.h b/include/linux/mm.h
index ecf63d2b058255e1855fd35d7838674443b516f9..61fff5d34ed532fe03144208aee618e0dd0a18ec 100644 (file)
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -3818,8 +3818,9 @@ void *sparse_buffer_alloc(unsigned long size);
 struct page * __populate_section_memmap(unsigned long pfn,
                unsigned long nr_pages, int nid, struct vmem_altmap *altmap,
                struct dev_pagemap *pgmap);
-void pmd_init(void *addr);
 void pud_init(void *addr);
+void pmd_init(void *addr);
+void kernel_pte_init(void *addr);
 pgd_t *vmemmap_pgd_populate(unsigned long addr, int node);
 p4d_t *vmemmap_p4d_populate(pgd_t *pgd, unsigned long addr, int node);
 pud_t *vmemmap_pud_populate(p4d_t *p4d, unsigned long addr, int node);

diff --git a/mm/kasan/init.c b/mm/kasan/init.c
index 89895f38f722423c02744f359799db4bb7f12dc2..ac607c306292f406bfdc88dcb379b3c7ecda0036 100644 (file)
--- a/mm/kasan/init.c
+++ b/mm/kasan/init.c
@@ -106,6 +106,10 @@ static void __ref zero_pte_populate(pmd_t *pmd, unsigned long addr,
        }
 }
 
+void __weak __meminit kernel_pte_init(void *addr)
+{
+}
+
 static int __ref zero_pmd_populate(pud_t *pud, unsigned long addr,
                                unsigned long end)
 {
@@ -126,8 +130,10 @@ static int __ref zero_pmd_populate(pud_t *pud, unsigned long addr,
 
                        if (slab_is_available())
                                p = pte_alloc_one_kernel(&init_mm);
-                       else
+                       else {
                                p = early_alloc(PAGE_SIZE, NUMA_NO_NODE);
+                               kernel_pte_init(p);
+                       }
                        if (!p)
                                return -ENOMEM;
 

diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c
index edcc7a6b0f6f2072e730e45a43a96eefded18302..c0388b2e959da6c374d3736dbdb20bd41996c5f1 100644 (file)
--- a/mm/sparse-vmemmap.c
+++ b/mm/sparse-vmemmap.c
@@ -184,6 +184,10 @@ static void * __meminit vmemmap_alloc_block_zero(unsigned long size, int node)
        return p;
 }
 
+void __weak __meminit kernel_pte_init(void *addr)
+{
+}
+
 pmd_t * __meminit vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node)
 {
        pmd_t *pmd = pmd_offset(pud, addr);
@@ -191,6 +195,7 @@ pmd_t * __meminit vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node)
                void *p = vmemmap_alloc_block_zero(PAGE_SIZE, node);
                if (!p)
                        return NULL;
+               kernel_pte_init(p);
                pmd_populate_kernel(&init_mm, pmd, p);
        }
        return pmd;