powerpc/mm: Add new "set" flag argument to pte/pmd update function

pte_update() is a powerpc-ism used to change the bits of a PTE
when the access permission is being restricted (a flush is
potentially needed).

It uses atomic operations on when needed and handles the hash
synchronization on hash based processors.

It is currently only used to clear PTE bits and so the current
implementation doesn't provide a way to also set PTE bits.

The new _PAGE_NUMA bit, when set, is actually restricting access
so it must use that function too, so this change adds the ability
for pte_update() to also set bits.

We will use this later to set the _PAGE_NUMA bit.

Acked-by: Mel Gorman <mgorman@suse.de>
Acked-by: Rik van Riel <riel@redhat.com>
Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>

diff --git a/arch/powerpc/include/asm/hugetlb.h b/arch/powerpc/include/asm/hugetlb.h
index d750336b171db4cf2c75f7e1fb95cbf6bcba8d74..623f2971ce0ed8d4d947642c862d4dce7640d28d 100644 (file)
--- a/arch/powerpc/include/asm/hugetlb.h
+++ b/arch/powerpc/include/asm/hugetlb.h
@@ -127,7 +127,7 @@ static inline pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
                                            unsigned long addr, pte_t *ptep)
 {
 #ifdef CONFIG_PPC64
-       return __pte(pte_update(mm, addr, ptep, ~0UL, 1));
+       return __pte(pte_update(mm, addr, ptep, ~0UL, 0, 1));
 #else
        return __pte(pte_update(ptep, ~0UL, 0));
 #endif

diff --git a/arch/powerpc/include/asm/pgtable-ppc64.h b/arch/powerpc/include/asm/pgtable-ppc64.h
index bc141c950b1e6c8128769b786df189eb275f583f..eb9261024f5192386dec97bff1ee10e62da4b7c3 100644 (file)
--- a/arch/powerpc/include/asm/pgtable-ppc64.h
+++ b/arch/powerpc/include/asm/pgtable-ppc64.h
@@ -195,6 +195,7 @@ extern void hpte_need_flush(struct mm_struct *mm, unsigned long addr,
 static inline unsigned long pte_update(struct mm_struct *mm,
                                       unsigned long addr,
                                       pte_t *ptep, unsigned long clr,
+                                      unsigned long set,
                                       int huge)
 {
 #ifdef PTE_ATOMIC_UPDATES
@@ -205,14 +206,15 @@ static inline unsigned long pte_update(struct mm_struct *mm,
        andi.   %1,%0,%6\n\
        bne-    1b \n\
        andc    %1,%0,%4 \n\
+       or      %1,%1,%7\n\
        stdcx.  %1,0,%3 \n\
        bne-    1b"
        : "=&r" (old), "=&r" (tmp), "=m" (*ptep)
-       : "r" (ptep), "r" (clr), "m" (*ptep), "i" (_PAGE_BUSY)
+       : "r" (ptep), "r" (clr), "m" (*ptep), "i" (_PAGE_BUSY), "r" (set)
        : "cc" );
 #else
        unsigned long old = pte_val(*ptep);
-       *ptep = __pte(old & ~clr);
+       *ptep = __pte((old & ~clr) | set);
 #endif
        /* huge pages use the old page table lock */
        if (!huge)
@@ -231,9 +233,9 @@ static inline int __ptep_test_and_clear_young(struct mm_struct *mm,
 {
        unsigned long old;
 
-               if ((pte_val(*ptep) & (_PAGE_ACCESSED | _PAGE_HASHPTE)) == 0)
+       if ((pte_val(*ptep) & (_PAGE_ACCESSED | _PAGE_HASHPTE)) == 0)
                return 0;
-       old = pte_update(mm, addr, ptep, _PAGE_ACCESSED, 0);
+       old = pte_update(mm, addr, ptep, _PAGE_ACCESSED, 0, 0);
        return (old & _PAGE_ACCESSED) != 0;
 }
 #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
@@ -252,7 +254,7 @@ static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr,
        if ((pte_val(*ptep) & _PAGE_RW) == 0)
                return;
 
-       pte_update(mm, addr, ptep, _PAGE_RW, 0);
+       pte_update(mm, addr, ptep, _PAGE_RW, 0, 0);
 }
 
 static inline void huge_ptep_set_wrprotect(struct mm_struct *mm,
@@ -261,7 +263,7 @@ static inline void huge_ptep_set_wrprotect(struct mm_struct *mm,
        if ((pte_val(*ptep) & _PAGE_RW) == 0)
                return;
 
-       pte_update(mm, addr, ptep, _PAGE_RW, 1);
+       pte_update(mm, addr, ptep, _PAGE_RW, 0, 1);
 }
 
 /*
@@ -284,14 +286,14 @@ static inline void huge_ptep_set_wrprotect(struct mm_struct *mm,
 static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
                                       unsigned long addr, pte_t *ptep)
 {
-       unsigned long old = pte_update(mm, addr, ptep, ~0UL, 0);
+       unsigned long old = pte_update(mm, addr, ptep, ~0UL, 0, 0);
        return __pte(old);
 }
 
 static inline void pte_clear(struct mm_struct *mm, unsigned long addr,
                             pte_t * ptep)
 {
-       pte_update(mm, addr, ptep, ~0UL, 0);
+       pte_update(mm, addr, ptep, ~0UL, 0, 0);
 }
 
 
@@ -506,7 +508,9 @@ extern int pmdp_set_access_flags(struct vm_area_struct *vma,
 
 extern unsigned long pmd_hugepage_update(struct mm_struct *mm,
                                         unsigned long addr,
-                                        pmd_t *pmdp, unsigned long clr);
+                                        pmd_t *pmdp,
+                                        unsigned long clr,
+                                        unsigned long set);
 
 static inline int __pmdp_test_and_clear_young(struct mm_struct *mm,
                                              unsigned long addr, pmd_t *pmdp)
@@ -515,7 +519,7 @@ static inline int __pmdp_test_and_clear_young(struct mm_struct *mm,
 
        if ((pmd_val(*pmdp) & (_PAGE_ACCESSED | _PAGE_HASHPTE)) == 0)
                return 0;
-       old = pmd_hugepage_update(mm, addr, pmdp, _PAGE_ACCESSED);
+       old = pmd_hugepage_update(mm, addr, pmdp, _PAGE_ACCESSED, 0);
        return ((old & _PAGE_ACCESSED) != 0);
 }
 
@@ -542,7 +546,7 @@ static inline void pmdp_set_wrprotect(struct mm_struct *mm, unsigned long addr,
        if ((pmd_val(*pmdp) & _PAGE_RW) == 0)
                return;
 
-       pmd_hugepage_update(mm, addr, pmdp, _PAGE_RW);
+       pmd_hugepage_update(mm, addr, pmdp, _PAGE_RW, 0);
 }
 
 #define __HAVE_ARCH_PMDP_SPLITTING_FLUSH

diff --git a/arch/powerpc/mm/pgtable_64.c b/arch/powerpc/mm/pgtable_64.c
index 65b7b65e8708bd1867861f4b519d207b25beb5a7..62bf5e8e78daaaf5051fb6e5727bfd99be81a088 100644 (file)
--- a/arch/powerpc/mm/pgtable_64.c
+++ b/arch/powerpc/mm/pgtable_64.c
@@ -510,7 +510,8 @@ int pmdp_set_access_flags(struct vm_area_struct *vma, unsigned long address,
 }
 
 unsigned long pmd_hugepage_update(struct mm_struct *mm, unsigned long addr,
-                                 pmd_t *pmdp, unsigned long clr)
+                                 pmd_t *pmdp, unsigned long clr,
+                                 unsigned long set)
 {
 
        unsigned long old, tmp;
@@ -526,14 +527,15 @@ unsigned long pmd_hugepage_update(struct mm_struct *mm, unsigned long addr,
                andi.   %1,%0,%6\n\
                bne-    1b \n\
                andc    %1,%0,%4 \n\
+               or      %1,%1,%7\n\
                stdcx.  %1,0,%3 \n\
                bne-    1b"
        : "=&r" (old), "=&r" (tmp), "=m" (*pmdp)
-       : "r" (pmdp), "r" (clr), "m" (*pmdp), "i" (_PAGE_BUSY)
+       : "r" (pmdp), "r" (clr), "m" (*pmdp), "i" (_PAGE_BUSY), "r" (set)
        : "cc" );
 #else
        old = pmd_val(*pmdp);
-       *pmdp = __pmd(old & ~clr);
+       *pmdp = __pmd((old & ~clr) | set);
 #endif
        if (old & _PAGE_HASHPTE)
                hpte_do_hugepage_flush(mm, addr, pmdp);
@@ -708,7 +710,7 @@ void set_pmd_at(struct mm_struct *mm, unsigned long addr,
 void pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
                     pmd_t *pmdp)
 {
-       pmd_hugepage_update(vma->vm_mm, address, pmdp, _PAGE_PRESENT);
+       pmd_hugepage_update(vma->vm_mm, address, pmdp, _PAGE_PRESENT, 0);
 }
 
 /*
@@ -835,7 +837,7 @@ pmd_t pmdp_get_and_clear(struct mm_struct *mm,
        unsigned long old;
        pgtable_t *pgtable_slot;
 
-       old = pmd_hugepage_update(mm, addr, pmdp, ~0UL);
+       old = pmd_hugepage_update(mm, addr, pmdp, ~0UL, 0);
        old_pmd = __pmd(old);
        /*
         * We have pmd == none and we are holding page_table_lock.

diff --git a/arch/powerpc/mm/subpage-prot.c b/arch/powerpc/mm/subpage-prot.c
index a770df2dae7050bafd28daf1c354165113e378f9..6c0b1f5f8d2ccef746ccddfed70684c45f3adec2 100644 (file)
--- a/arch/powerpc/mm/subpage-prot.c
+++ b/arch/powerpc/mm/subpage-prot.c
@@ -78,7 +78,7 @@ static void hpte_flush_range(struct mm_struct *mm, unsigned long addr,
        pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
        arch_enter_lazy_mmu_mode();
        for (; npages > 0; --npages) {
-               pte_update(mm, addr, pte, 0, 0);
+               pte_update(mm, addr, pte, 0, 0, 0);
                addr += PAGE_SIZE;
                ++pte;
        }