sparc64: 5-Level page table support for sparc

Extended Page table to 5-Level for sparc.

Orabug: 26076110
Orabug: 25808647

Signed-off-by: Vijay Kumar <vijay.ac.kumar@oracle.com>
Reviewed-by: Liam R. Howlett <Liam.Howlett@Oracle.com>
Reviewed-by: Bob Picco <bob.picco@oracle.com>
Signed-off-by: Shannon Nelson <shannon.nelson@oracle.com>

diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig
index 33b47002d4ab77c50ee9cd9b4e32cab7ece151ac..3bbce983243cf472ef28f5455ed8f55c526e8f70 100644 (file)
--- a/arch/sparc/Kconfig
+++ b/arch/sparc/Kconfig
@@ -161,7 +161,7 @@ config ARCH_SUPPORTS_DEBUG_PAGEALLOC
        def_bool y if SPARC64
 
 config PGTABLE_LEVELS
-       default 4 if 64BIT
+       default 5 if 64BIT
        default 3
 
 config ARCH_SUPPORTS_DTRACE

diff --git a/arch/sparc/include/asm/page_64.h b/arch/sparc/include/asm/page_64.h
index 969f39aadda2abe7f2836ecf3e06837dce492a64..e04f805f208c93459027e76320b6b7500639ced5 100644 (file)
--- a/arch/sparc/include/asm/page_64.h
+++ b/arch/sparc/include/asm/page_64.h
@@ -62,6 +62,7 @@ typedef struct { unsigned long pte; } pte_t;
 typedef struct { unsigned long iopte; } iopte_t;
 typedef struct { unsigned long pmd; } pmd_t;
 typedef struct { unsigned long pud; } pud_t;
+typedef struct { unsigned long p4d; } p4d_t;
 typedef struct { unsigned long pgd; } pgd_t;
 typedef struct { unsigned long pgprot; } pgprot_t;
 
@@ -69,6 +70,7 @@ typedef struct { unsigned long pgprot; } pgprot_t;
 #define iopte_val(x)   ((x).iopte)
 #define pmd_val(x)      ((x).pmd)
 #define pud_val(x)      ((x).pud)
+#define p4d_val(x)      ((x).p4d)
 #define pgd_val(x)     ((x).pgd)
 #define pgprot_val(x)  ((x).pgprot)
 
@@ -76,6 +78,7 @@ typedef struct { unsigned long pgprot; } pgprot_t;
 #define __iopte(x)     ((iopte_t) { (x) } )
 #define __pmd(x)        ((pmd_t) { (x) } )
 #define __pud(x)        ((pud_t) { (x) } )
+#define __p4d(x)        ((p4d_t) { (x) } )
 #define __pgd(x)       ((pgd_t) { (x) } )
 #define __pgprot(x)    ((pgprot_t) { (x) } )
 
@@ -85,6 +88,7 @@ typedef unsigned long pte_t;
 typedef unsigned long iopte_t;
 typedef unsigned long pmd_t;
 typedef unsigned long pud_t;
+typedef unsigned long p4d_t;
 typedef unsigned long pgd_t;
 typedef unsigned long pgprot_t;
 
@@ -92,6 +96,7 @@ typedef unsigned long pgprot_t;
 #define iopte_val(x)   (x)
 #define pmd_val(x)      (x)
 #define pud_val(x)      (x)
+#define p4d_val(x)      (x)
 #define pgd_val(x)     (x)
 #define pgprot_val(x)  (x)
 
@@ -99,6 +104,7 @@ typedef unsigned long pgprot_t;
 #define __iopte(x)     (x)
 #define __pmd(x)        (x)
 #define __pud(x)        (x)
+#define __p4d(x)        (x)
 #define __pgd(x)       (x)
 #define __pgprot(x)    (x)
 

diff --git a/arch/sparc/include/asm/pgalloc_64.h b/arch/sparc/include/asm/pgalloc_64.h
index 5e3187185b4a8b9c3f22d4a67ea538a12fdcd9e7..85a90a4dac8715138dca3ea490d93d8d3de1a842 100644 (file)
--- a/arch/sparc/include/asm/pgalloc_64.h
+++ b/arch/sparc/include/asm/pgalloc_64.h
@@ -15,12 +15,12 @@
 
 extern struct kmem_cache *pgtable_cache;
 
-static inline void __pgd_populate(pgd_t *pgd, pud_t *pud)
+static inline void __pgd_populate(pgd_t *pgd, p4d_t *p4d)
 {
-       pgd_set(pgd, pud);
+       pgd_set(pgd, p4d);
 }
+#define pgd_populate(MM, PGD, P4D)     __pgd_populate(PGD, P4D)
 
-#define pgd_populate(MM, PGD, PUD)     __pgd_populate(PGD, PUD)
 
 static inline pgd_t *pgd_alloc(struct mm_struct *mm)
 {
@@ -32,6 +32,22 @@ static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)
        kmem_cache_free(pgtable_cache, pgd);
 }
 
+static inline void __p4d_populate(p4d_t *p4d, pud_t *pud)
+{
+       p4d_set(p4d, pud);
+}
+#define p4d_populate(MM, P4D, PUD)     __p4d_populate(P4D, PUD)
+
+static inline p4d_t *p4d_alloc_one(struct mm_struct *mm, unsigned long addr)
+{
+       return kmem_cache_alloc(pgtable_cache, GFP_KERNEL);
+}
+
+static inline void p4d_free(struct mm_struct *mm, p4d_t *p4d)
+{
+       kmem_cache_free(pgtable_cache, p4d);
+}
+
 static inline void __pud_populate(pud_t *pud, pmd_t *pmd)
 {
        pud_set(pud, pmd);
@@ -117,4 +133,7 @@ static inline void __pte_free_tlb(struct mmu_gather *tlb, pte_t *pte,
 #define __pud_free_tlb(tlb, pud, addr)               \
        pgtable_free_tlb(tlb, pud, false)
 
+#define __p4d_free_tlb(tlb, p4d, addr)               \
+       pgtable_free_tlb(tlb, p4d, false)
+
 #endif /* _SPARC64_PGALLOC_H */

diff --git a/arch/sparc/include/asm/pgtable_64.h b/arch/sparc/include/asm/pgtable_64.h
index 101e3f5dfa97b0e29358b16e338dc6e8b4278c15..85ad9de833e6a4ae6d5a49d3dab9b73b96710d1c 100644 (file)
--- a/arch/sparc/include/asm/pgtable_64.h
+++ b/arch/sparc/include/asm/pgtable_64.h
@@ -12,7 +12,6 @@
  * the SpitFire page tables.
  */
 
-#include <asm-generic/5level-fixup.h>
 #include <linux/compiler.h>
 #include <linux/const.h>
 #include <asm/types.h>
@@ -63,8 +62,16 @@
 #define PUD_MASK       (~(PUD_SIZE-1))
 #define PUD_BITS       (PAGE_SHIFT - 3)
 
-/* PGDIR_SHIFT determines what a fourth-level page table entry can map */
-#define PGDIR_SHIFT    (PUD_SHIFT + PUD_BITS)
+/* P4D_SHIFT determines the size of the area a fourth-level page
+ * table can map
+*/
+#define P4D_SHIFT      (PUD_SHIFT + PUD_BITS)
+#define P4D_SIZE       (_AC(1,UL) << P4D_SHIFT)
+#define P4D_MASK       (~(P4D_SIZE-1))
+#define P4D_BITS       (PAGE_SHIFT - 3)
+
+/* PGDIR_SHIFT determines what a fifth-level page table entry can map */
+#define PGDIR_SHIFT    (P4D_SHIFT + P4D_BITS)
 #define PGDIR_SIZE     (_AC(1,UL) << PGDIR_SHIFT)
 #define PGDIR_MASK     (~(PGDIR_SIZE-1))
 #define PGDIR_BITS     (PAGE_SHIFT - 3)
@@ -73,7 +80,7 @@
 #error MAX_PHYS_ADDRESS_BITS exceeds what kernel page tables can support
 #endif
 
-#if (PGDIR_SHIFT + PGDIR_BITS) != 53
+#if (PGDIR_SHIFT + PGDIR_BITS) != 63
 #error Page table parameters do not cover virtual address space properly.
 #endif
 
@@ -95,6 +102,7 @@ bool kern_addr_valid(unsigned long addr);
 #define PTRS_PER_PTE   (1UL << (PAGE_SHIFT-3))
 #define PTRS_PER_PMD   (1UL << PMD_BITS)
 #define PTRS_PER_PUD   (1UL << PUD_BITS)
+#define PTRS_PER_P4D   (1UL << P4D_BITS)
 #define PTRS_PER_PGD   (1UL << PGDIR_BITS)
 
 /* Kernel has a separate 44bit address space. */
@@ -106,6 +114,9 @@ bool kern_addr_valid(unsigned long addr);
 #define pud_ERROR(e)                                                   \
        pr_err("%s:%d: bad pud %p(%016lx) seen at (%pS)\n",             \
               __FILE__, __LINE__, &(e), pud_val(e), __builtin_return_address(0))
+#define p4d_ERROR(e)                                                   \
+       pr_err("%s:%d: bad p4d %p(%016lx) seen at (%pS)\n",             \
+              __FILE__, __LINE__, &(e), p4d_val(e), __builtin_return_address(0))
 #define pgd_ERROR(e)                                                   \
        pr_err("%s:%d: bad pgd %p(%016lx) seen at (%pS)\n",             \
               __FILE__, __LINE__, &(e), pgd_val(e), __builtin_return_address(0))
@@ -817,6 +828,10 @@ static inline int pmd_present(pmd_t pmd)
 
 #define pud_bad(pud)                   (pud_val(pud) & ~PAGE_MASK)
 
+#define p4d_none(p4d)                  (!p4d_val(p4d))
+
+#define p4d_bad(p4d)                   (p4d_val(p4d) & ~PAGE_MASK)
+
 #define pgd_none(pgd)                  (!pgd_val(pgd))
 
 #define pgd_bad(pgd)                   (pgd_val(pgd) & ~PAGE_MASK)
@@ -850,13 +865,24 @@ static inline unsigned long __pmd_page(pmd_t pmd)
 
        return ((unsigned long) __va(pfn << PAGE_SHIFT));
 }
+
 #define pmd_page(pmd)                  virt_to_page((void *)__pmd_page(pmd))
+#define pmd_clear(pmdp)                        (pmd_val(*(pmdp)) = 0UL)
 #define pud_page_vaddr(pud)            \
        ((unsigned long) __va(pud_val(pud)))
 #define pud_page(pud)                  virt_to_page((void *)pud_page_vaddr(pud))
-#define pmd_clear(pmdp)                        (pmd_val(*(pmdp)) = 0UL)
 #define pud_present(pud)               (pud_val(pud) != 0U)
 #define pud_clear(pudp)                        (pud_val(*(pudp)) = 0UL)
+
+#define p4d_set(p4dp, pudp)    \
+       (p4d_val(*(p4dp)) = (__pa((unsigned long) (pudp))))
+
+#define p4d_page_vaddr(p4d)             \
+       ((unsigned long) __va(p4d_val(p4d)))
+#define p4d_page(p4d)                   virt_to_page((void *)p4d_page_vaddr(p4d))
+#define p4d_present(p4d)                (p4d_val(p4d) != 0U)
+#define p4d_clear(p4dp)                 (p4d_val(*(p4dp)) = 0UL)
+
 #define pgd_page_vaddr(pgd)            \
        ((unsigned long) __va(pgd_val(pgd)))
 #define pgd_present(pgd)               (pgd_val(pgd) != 0U)
@@ -879,8 +905,8 @@ static inline unsigned long pud_pfn(pud_t pud)
 /* Same in both SUN4V and SUN4U.  */
 #define pte_none(pte)                  (!pte_val(pte))
 
-#define pgd_set(pgdp, pudp)    \
-       (pgd_val(*(pgdp)) = (__pa((unsigned long) (pudp))))
+#define pgd_set(pgdp, p4dp)    \
+       (pgd_val(*(pgdp)) = (__pa((unsigned long) (p4dp))))
 
 /* to find an entry in a page-table-directory. */
 #define pgd_index(address)     (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1))
@@ -889,10 +915,15 @@ static inline unsigned long pud_pfn(pud_t pud)
 /* to find an entry in a kernel page-table-directory */
 #define pgd_offset_k(address) pgd_offset(&init_mm, address)
 
+/* Find an entry in the fourt-level page table.. */
+#define p4d_index(address)     (((address) >> P4D_SHIFT) & (PTRS_PER_P4D - 1))
+#define p4d_offset(pgdp, address)      \
+       ((p4d_t *) pgd_page_vaddr(*(pgdp)) + p4d_index(address))
+
 /* Find an entry in the third-level page table.. */
 #define pud_index(address)     (((address) >> PUD_SHIFT) & (PTRS_PER_PUD - 1))
-#define pud_offset(pgdp, address)      \
-       ((pud_t *) pgd_page_vaddr(*(pgdp)) + pud_index(address))
+#define pud_offset(p4dp, address)      \
+       ((pud_t *) p4d_page_vaddr(*(p4dp)) + pud_index(address))
 
 /* Find an entry in the second-level page table.. */
 #define pmd_offset(pudp, address)      \

diff --git a/arch/sparc/include/asm/tsb.h b/arch/sparc/include/asm/tsb.h
index 32258e08da035f018df2915bf9935556556628bb..d2e79744d901842ed435f8e9098d39be74d81741 100644 (file)
--- a/arch/sparc/include/asm/tsb.h
+++ b/arch/sparc/include/asm/tsb.h
@@ -159,6 +159,11 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end;
        srlx            REG2, 64 - PAGE_SHIFT, REG2; \
        andn            REG2, 0x7, REG2; \
        ldx             [REG1 + REG2], REG1; \
+       brz,pn          REG1, FAIL_LABEL; \
+        sllx           VADDR, 64 - (P4D_SHIFT + P4D_BITS), REG2; \
+       srlx            REG2, 64 - PAGE_SHIFT, REG2; \
+       andn            REG2, 0x7, REG2; \
+       ldxa            [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \
        brz,pn          REG1, FAIL_LABEL; \
         sllx           VADDR, 64 - (PUD_SHIFT + PUD_BITS), REG2; \
        srlx            REG2, 64 - PAGE_SHIFT, REG2; \
@@ -237,6 +242,11 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end;
        srlx            REG2, 64 - PAGE_SHIFT, REG2; \
        andn            REG2, 0x7, REG2; \
        ldxa            [PHYS_PGD + REG2] ASI_PHYS_USE_EC, REG1; \
+       brz,pn          REG1, FAIL_LABEL; \
+        sllx           VADDR, 64 - (P4D_SHIFT + P4D_BITS), REG2; \
+       srlx            REG2, 64 - PAGE_SHIFT, REG2; \
+       andn            REG2, 0x7, REG2; \
+       ldxa            [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \
        brz,pn          REG1, FAIL_LABEL; \
         sllx           VADDR, 64 - (PUD_SHIFT + PUD_BITS), REG2; \
        srlx            REG2, 64 - PAGE_SHIFT, REG2; \

diff --git a/arch/sparc/kernel/dtrace_util.c b/arch/sparc/kernel/dtrace_util.c
index 47e19175cd7b7e903931cd599eb81fa49634159e..40bee4da976ffccee97e143bb44adbd492c9c334 100644 (file)
--- a/arch/sparc/kernel/dtrace_util.c
+++ b/arch/sparc/kernel/dtrace_util.c
@@ -83,6 +83,7 @@ int dtrace_user_addr_is_exec(uintptr_t addr)
 {
        struct mm_struct        *mm = current->mm;
        pgd_t                   pgd;
+       p4d_t                   p4d;
        pud_t                   pud;
        pmd_t                   pmd;
        unsigned long           flags;
@@ -99,7 +100,11 @@ int dtrace_user_addr_is_exec(uintptr_t addr)
        if (pgd_none(pgd))
                goto out;
 
-       pud = *pud_offset(&pgd, addr);
+       p4d = *p4d_offset(&pgd, addr);
+       if (p4d_none(p4d))
+               goto out;
+
+       pud = *pud_offset(&p4d, addr);
        if (pud_none(pud))
                goto out;
 

diff --git a/arch/sparc/kernel/signal32.c b/arch/sparc/kernel/signal32.c
index 2b42d4ee238202963254587f19d8728be72157ed..cd130aa7b74e88910dc44be16f98cd4bda482fc9 100644 (file)
--- a/arch/sparc/kernel/signal32.c
+++ b/arch/sparc/kernel/signal32.c
@@ -369,6 +369,7 @@ static void flush_signal_insns(unsigned long address)
        unsigned long pstate, paddr;
        pte_t *ptep, pte;
        pgd_t *pgdp;
+       p4d_t *p4dp;
        pud_t *pudp;
        pmd_t *pmdp;
 
@@ -388,7 +389,10 @@ static void flush_signal_insns(unsigned long address)
        pgdp = pgd_offset(current->mm, address);
        if (pgd_none(*pgdp))
                goto out_irqs_on;
-       pudp = pud_offset(pgdp, address);
+       p4dp = p4d_offset(pgdp, address);
+       if (p4d_none(*p4dp))
+               goto out_irqs_on;
+       pudp = pud_offset(p4dp, address);
        if (pud_none(*pudp))
                goto out_irqs_on;
        pmdp = pmd_offset(pudp, address);

diff --git a/arch/sparc/kernel/smp_64.c b/arch/sparc/kernel/smp_64.c
index 1044b774514c076bce6f5bbc6a601b70e5443fd8..3210a3db13cd0acea7d301ef08789f43dc1f81ac 100644 (file)
--- a/arch/sparc/kernel/smp_64.c
+++ b/arch/sparc/kernel/smp_64.c
@@ -1616,17 +1616,26 @@ static int __init pcpu_cpu_distance(unsigned int from, unsigned int to)
 static void __init pcpu_populate_pte(unsigned long addr)
 {
        pgd_t *pgd = pgd_offset_k(addr);
+       p4d_t *p4d;
        pud_t *pud;
        pmd_t *pmd;
 
        if (pgd_none(*pgd)) {
-               pud_t *new;
+               p4d_t *new;
 
                new = __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);
                pgd_populate(&init_mm, pgd, new);
        }
 
-       pud = pud_offset(pgd, addr);
+       p4d = p4d_offset(pgd, addr);
+       if (p4d_none(*p4d)) {
+               pud_t *new;
+
+               new = __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);
+               p4d_populate(&init_mm, p4d, new);
+       }
+
+       pud = pud_offset(p4d, addr);
        if (pud_none(*pud)) {
                pmd_t *new;
 

diff --git a/arch/sparc/mm/fault_64.c b/arch/sparc/mm/fault_64.c
index 1cc8ef2e6aad0932081658e1c0fb987f40fa68a1..02cbe9512131babcff6dd08e10672c3bb9c3677a 100644 (file)
--- a/arch/sparc/mm/fault_64.c
+++ b/arch/sparc/mm/fault_64.c
@@ -94,6 +94,7 @@ static void __kprobes bad_kernel_pc(struct pt_regs *regs, unsigned long vaddr)
 static unsigned int get_user_insn(unsigned long tpc)
 {
        pgd_t *pgdp = pgd_offset(current->mm, tpc);
+       p4d_t *p4dp;
        pud_t *pudp;
        pmd_t *pmdp;
        pte_t *ptep, pte;
@@ -102,7 +103,12 @@ static unsigned int get_user_insn(unsigned long tpc)
 
        if (pgd_none(*pgdp) || unlikely(pgd_bad(*pgdp)))
                goto out;
-       pudp = pud_offset(pgdp, tpc);
+
+       p4dp = p4d_offset(pgdp, tpc);
+       if (p4d_none(*p4dp) || unlikely(p4d_bad(*p4dp)))
+               goto out;
+
+       pudp = pud_offset(p4dp, tpc);
        if (pud_none(*pudp) || unlikely(pud_bad(*pudp)))
                goto out;
 

diff --git a/arch/sparc/mm/gup.c b/arch/sparc/mm/gup.c
index 4b92a00112a7bd2aae97296effe2bdfdc9be194a..9c962f905a9eda01af460208cfb2bf7eea54c581 100644 (file)
--- a/arch/sparc/mm/gup.c
+++ b/arch/sparc/mm/gup.c
@@ -141,13 +141,13 @@ static int gup_pmd_range(pud_t pud, unsigned long addr, unsigned long end,
        return 1;
 }
 
-static int gup_pud_range(pgd_t pgd, unsigned long addr, unsigned long end,
+static int gup_pud_range(p4d_t p4d, unsigned long addr, unsigned long end,
                int write, struct page **pages, int *nr)
 {
        unsigned long next;
        pud_t *pudp;
 
-       pudp = pud_offset(&pgd, addr);
+       pudp = pud_offset(&p4d, addr);
        do {
                pud_t pud = *pudp;
 
@@ -161,6 +161,26 @@ static int gup_pud_range(pgd_t pgd, unsigned long addr, unsigned long end,
        return 1;
 }
 
+static int gup_p4d_range(pgd_t pgd, unsigned long addr, unsigned long end,
+                        int write, struct page **pages, int *nr)
+{
+       unsigned long next;
+       p4d_t *p4dp;
+
+       p4dp = p4d_offset(&pgd, addr);
+       do {
+               p4d_t p4d = *p4dp;
+
+               next = p4d_addr_end(addr, end);
+               if (p4d_none(p4d))
+                       return 0;
+               if (!gup_pud_range(p4d, addr, next, write, pages, nr))
+                       return 0;
+       } while (p4dp++, addr = next, addr != end);
+
+       return 1;
+}
+
 int __get_user_pages_fast(unsigned long start, int nr_pages, int write,
                          struct page **pages)
 {
@@ -201,7 +221,7 @@ int __get_user_pages_fast(unsigned long start, int nr_pages, int write,
                next = pgd_addr_end(addr, end);
                if (pgd_none(pgd))
                        break;
-               if (!gup_pud_range(pgd, addr, next, write, pages, &nr))
+               if (!gup_p4d_range(pgd, addr, next, write, pages, &nr))
                        break;
        } while (pgdp++, addr = next, addr != end);
        local_irq_restore(flags);
@@ -267,7 +287,7 @@ int get_user_pages_fast(unsigned long start, int nr_pages, int write,
                next = pgd_addr_end(addr, end);
                if (pgd_none(pgd))
                        goto slow;
-               if (!gup_pud_range(pgd, addr, next, write, pages, &nr))
+               if (!gup_p4d_range(pgd, addr, next, write, pages, &nr))
                        goto slow;
        } while (pgdp++, addr = next, addr != end);
 

diff --git a/arch/sparc/mm/hugetlbpage.c b/arch/sparc/mm/hugetlbpage.c
index d56e58792227365c49a8df4fd380626c77bbd8af..3eab5ac44b2fa8e9e402f6e629a7aa5ba38ee656 100644 (file)
--- a/arch/sparc/mm/hugetlbpage.c
+++ b/arch/sparc/mm/hugetlbpage.c
@@ -269,21 +269,25 @@ pte_t *huge_pte_alloc(struct mm_struct *mm,
                        unsigned long addr, unsigned long sz)
 {
        pgd_t *pgd;
+       p4d_t *p4d;
        pud_t *pud;
        pmd_t *pmd;
        pte_t *pte = NULL;
 
        pgd = pgd_offset(mm, addr);
-       pud = pud_alloc(mm, pgd, addr);
-       if (pud) {
-               pmd = pmd_alloc(mm, pud, addr);
-               if (!pmd)
-                       return NULL;
-
-               if (sz >= PMD_SIZE)
-                       pte = (pte_t *)pmd;
-               else
-                       pte = pte_alloc_map(mm, NULL, pmd, addr);
+       p4d = p4d_alloc(mm, pgd, addr);
+       if (p4d) {
+               pud = pud_alloc(mm, p4d, addr);
+               if (pud) {
+                       pmd = pmd_alloc(mm, pud, addr);
+                       if (!pmd)
+                               return NULL;
+
+                       if (sz >= PMD_SIZE)
+                               pte = (pte_t *)pmd;
+                       else
+                               pte = pte_alloc_map(mm, NULL, pmd, addr);
+               }
        }
 
        return pte;
@@ -292,20 +296,24 @@ pte_t *huge_pte_alloc(struct mm_struct *mm,
 pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
 {
        pgd_t *pgd;
+       p4d_t *p4d;
        pud_t *pud;
        pmd_t *pmd;
        pte_t *pte = NULL;
 
        pgd = pgd_offset(mm, addr);
        if (!pgd_none(*pgd)) {
-               pud = pud_offset(pgd, addr);
-               if (!pud_none(*pud)) {
-                       pmd = pmd_offset(pud, addr);
-                       if (!pmd_none(*pmd)) {
-                               if (is_hugetlb_pmd(*pmd))
-                                       pte = (pte_t *)pmd;
-                               else
-                                       pte = pte_offset_map(pmd, addr);
+               p4d = p4d_offset(pgd, addr);
+               if (!p4d_none(*p4d)) {
+                       pud = pud_offset(p4d, addr);
+                       if (!pud_none(*pud)) {
+                               pmd = pmd_offset(pud, addr);
+                               if (!pmd_none(*pmd)) {
+                                       if (is_hugetlb_pmd(*pmd))
+                                               pte = (pte_t *)pmd;
+                                       else
+                                               pte = pte_offset_map(pmd, addr);
+                               }
                        }
                }
        }
@@ -432,7 +440,7 @@ static void hugetlb_free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
        mm_dec_nr_pmds(tlb->mm);
 }
 
-static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
+static void hugetlb_free_pud_range(struct mmu_gather *tlb, p4d_t *p4d,
                                   unsigned long addr, unsigned long end,
                                   unsigned long floor, unsigned long ceiling)
 {
@@ -441,7 +449,7 @@ static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
        unsigned long start;
 
        start = addr;
-       pud = pud_offset(pgd, addr);
+       pud = pud_offset(p4d, addr);
        do {
                next = pud_addr_end(addr, end);
                if (pud_none_or_clear_bad(pud))
@@ -450,6 +458,40 @@ static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
                                       ceiling);
        } while (pud++, addr = next, addr != end);
 
+       start &= P4D_MASK;
+       if (start < floor)
+               return;
+       if (ceiling) {
+               ceiling &= P4D_MASK;
+               if (!ceiling)
+                       return;
+       }
+       if (end - 1 > ceiling - 1)
+               return;
+
+       pud = pud_offset(p4d, start);
+       p4d_clear(p4d);
+       pud_free_tlb(tlb, pud, start);
+}
+
+static void hugetlb_free_p4d_range(struct mmu_gather *tlb, pgd_t *pgd,
+                                  unsigned long addr, unsigned long end,
+                                  unsigned long floor, unsigned long ceiling)
+{
+       unsigned long next;
+       unsigned long start;
+       p4d_t *p4d;
+
+       start = addr;
+       p4d = p4d_offset(pgd, addr);
+       do {
+               next = p4d_addr_end(addr, end);
+               if (p4d_none_or_clear_bad(p4d))
+                       continue;
+               hugetlb_free_pud_range(tlb, p4d, addr, next, floor,
+                               ceiling);
+       } while (p4d++, addr = next, addr != end);
+
        start &= PGDIR_MASK;
        if (start < floor)
                return;
@@ -461,9 +503,9 @@ static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
        if (end - 1 > ceiling - 1)
                return;
 
-       pud = pud_offset(pgd, start);
+       p4d = p4d_offset(pgd, start);
        pgd_clear(pgd);
-       pud_free_tlb(tlb, pud, start);
+       p4d_free_tlb(tlb, p4d, start);
 }
 
 void hugetlb_free_pgd_range(struct mmu_gather *tlb,
@@ -494,6 +536,6 @@ void hugetlb_free_pgd_range(struct mmu_gather *tlb,
                next = pgd_addr_end(addr, end);
                if (pgd_none_or_clear_bad(pgd))
                        continue;
-               hugetlb_free_pud_range(tlb, pgd, addr, next, floor, ceiling);
+               hugetlb_free_p4d_range(tlb, pgd, addr, next, floor, ceiling);
        } while (pgd++, addr = next, addr != end);
 }

diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c
index 329eed4f10af491d872882c0e93c7fa350d8ad04..93a739ce1e252355c168ca80252413ae2d3408fe 100644 (file)
--- a/arch/sparc/mm/init_64.c
+++ b/arch/sparc/mm/init_64.c
@@ -582,7 +582,8 @@ void __kprobes flush_icache_range(unsigned long start, unsigned long end)
                                paddr = kaddr & mask;
                        else {
                                pgd_t *pgdp = pgd_offset_k(kaddr);
-                               pud_t *pudp = pud_offset(pgdp, kaddr);
+                               p4d_t *p4dp = p4d_offset(pgdp, kaddr);
+                               pud_t *pudp = pud_offset(p4dp, kaddr);
                                pmd_t *pmdp = pmd_offset(pudp, kaddr);
                                pte_t *ptep = pte_offset_kernel(pmdp, kaddr);
 
@@ -1818,6 +1819,7 @@ static unsigned long max_phys_bits = 40;
 bool kern_addr_valid(unsigned long addr)
 {
        pgd_t *pgd;
+       p4d_t *p4d;
        pud_t *pud;
        pmd_t *pmd;
        pte_t *pte;
@@ -1839,7 +1841,11 @@ bool kern_addr_valid(unsigned long addr)
        if (pgd_none(*pgd))
                return 0;
 
-       pud = pud_offset(pgd, addr);
+       p4d = p4d_offset(pgd, addr);
+       if (p4d_none(*p4d))
+               return 0;
+
+       pud = pud_offset(p4d, addr);
        if (pud_none(*pud))
                return 0;
 
@@ -1965,18 +1971,27 @@ static unsigned long __ref kernel_map_range(unsigned long pstart,
        while (vstart < vend) {
                unsigned long this_end, paddr = __pa(vstart);
                pgd_t *pgd = pgd_offset_k(vstart);
+               p4d_t *p4d;
                pud_t *pud;
                pmd_t *pmd;
                pte_t *pte;
 
                if (pgd_none(*pgd)) {
-                       pud_t *new;
+                       p4d_t *new;
 
                        new = __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);
                        alloc_bytes += PAGE_SIZE;
                        pgd_populate(&init_mm, pgd, new);
                }
-               pud = pud_offset(pgd, vstart);
+               p4d = p4d_offset(pgd, vstart);
+               if (p4d_none(*p4d)) {
+                       pud_t *new;
+
+                       new = __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);
+                       alloc_bytes += PAGE_SIZE;
+                       p4d_populate(&init_mm, p4d, new);
+               }
+               pud = pud_offset(p4d, vstart);
                if (pud_none(*pud)) {
                        pmd_t *new;
 
@@ -2837,18 +2852,28 @@ int __meminit vmemmap_populate(unsigned long vstart, unsigned long vend,
        for (; vstart < vend; vstart += PMD_SIZE) {
                pgd_t *pgd = pgd_offset_k(vstart);
                unsigned long pte;
+               p4d_t *p4d;
                pud_t *pud;
                pmd_t *pmd;
 
                if (pgd_none(*pgd)) {
-                       pud_t *new = vmemmap_alloc_block(PAGE_SIZE, node);
+                       p4d_t *new = vmemmap_alloc_block(PAGE_SIZE, node);
 
                        if (!new)
                                return -ENOMEM;
                        pgd_populate(&init_mm, pgd, new);
                }
 
-               pud = pud_offset(pgd, vstart);
+               p4d = p4d_offset(pgd, vstart);
+               if (p4d_none(*p4d)) {
+                       pud_t *new = vmemmap_alloc_block(PAGE_SIZE, node);
+
+                       if (!new)
+                               return -ENOMEM;
+                       p4d_populate(&init_mm, p4d, new);
+               }
+
+               pud = pud_offset(p4d, vstart);
                if (pud_none(*pud)) {
                        pmd_t *new = vmemmap_alloc_block(PAGE_SIZE, node);
 

diff --git a/drivers/infiniband/hw/sif/sif_spt.c b/drivers/infiniband/hw/sif/sif_spt.c
index a2faa4b0fca55d1fb6cc8af47fafd1a1df01f372..55a1fae79cb8cdb01d855acae14f96250694459a 100644 (file)
--- a/drivers/infiniband/hw/sif/sif_spt.c
+++ b/drivers/infiniband/hw/sif/sif_spt.c
@@ -75,6 +75,7 @@ static int sif_set_mmu_ctx(struct sif_dev *sdev, struct sif_mmu_ctx *ctx,
                        struct sif_mem *mem, bool write)
 {
        pgd_t *pgd;
+       p4d_t *p4d;
        pud_t *pud;
        pmd_t *pmd;
        pte_t *ptep, pte;
@@ -95,7 +96,12 @@ static int sif_set_mmu_ctx(struct sif_dev *sdev, struct sif_mmu_ctx *ctx,
 
        ctx->pt = (void *)pgd; /* Misuse pt to save the pointer to avoid going via mm at dealloc time */
        ctx->mt = SIFMT_ZERO;
-       pud = pud_offset(pgd, start);
+
+       p4d = p4d_offset(pgd, start);
+       if (p4d_none(*p4d))
+               goto err;
+
+       pud = pud_offset(p4d, start);
        if (pud_none(*pud))
                goto err;
 
@@ -203,6 +209,7 @@ void sif_spt_unmap_gva_ctx(struct sif_dev *sdev, struct sif_mmu_ctx *sctx)
        u64 start = sctx->base;
        u64 len = sctx->size;
        pgd_t *pgd;
+       p4d_t *p4d;
        pud_t *pud;
        pmd_t *pmd;
        pte_t *ptep, pte;
@@ -223,7 +230,13 @@ void sif_spt_unmap_gva_ctx(struct sif_dev *sdev, struct sif_mmu_ctx *sctx)
                goto out;
        }
 
-       pud = pud_offset(pgd, start);
+       p4d = p4d_offset(pgd, start);
+       if (p4d_none(*p4d)) {
+               sif_log(sdev, SIF_MMU, "Table entry(p4d) already freed");
+               goto out;
+       }
+
+       pud = pud_offset(p4d, start);
        if (pud_none(*pud)) {
                sif_log(sdev, SIF_MMU, "Table entry(pud) already freed");
                goto out;