x86/speculation/l1tf: Protect PROT_NONE PTEs against speculation

When PTEs are set to PROT_NONE the kernel just clears the Present bit and
preserves the PFN, which creates attack surface for L1TF speculation
speculation attacks.

This is important inside guests, because L1TF speculation bypasses physical
page remapping. While the host has its own migitations preventing leaking
data from other VMs into the guest, this would still risk leaking the wrong
page inside the current guest.

This uses the same technique as Linus' swap entry patch: while an entry is
is in PROTNONE state invert the complete PFN part part of it. This ensures
that the the highest bit will point to non existing memory.

The invert is done by pte/pmd_modify and pfn/pmd/pud_pte for PROTNONE and
pte/pmd/pud_pfn undo it.

This assume that no code path touches the PFN part of a PTE directly
without using these primitives.

This doesn't handle the case that MMIO is on the top of the CPU physical
memory. If such an MMIO region was exposed by an unpriviledged driver for
mmap it would be possible to attack some real memory.  However this
situation is all rather unlikely.

For 32bit non PAE the inversion is not done because there are really not
enough bits to protect anything.

Q: Why does the guest need to be protected when the HyperVisor already has
   L1TF mitigations?

A: Here's an example:

   Physical pages 1 2 get mapped into a guest as
   GPA 1 -> PA 2
   GPA 2 -> PA 1
   through EPT.

   The L1TF speculation ignores the EPT remapping.

   Now the guest kernel maps GPA 1 to process A and GPA 2 to process B, and
   they belong to different users and should be isolated.

   A sets the GPA 1 PA 2 PTE to PROT_NONE to bypass the EPT remapping and
   gets read access to the underlying physical page. Which in this case
   points to PA 2, so it can read process B's data, if it happened to be in
   L1, so isolation inside the guest is broken.

   There's nothing the hypervisor can do about this. This mitigation has to
   be done in the guest itself.

[ tglx: Massaged changelog ]

Signed-off-by: Andi Kleen <ak@linux.intel.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Josh Poimboeuf <jpoimboe@redhat.com>
Acked-by: Michal Hocko <mhocko@suse.com>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: Dave Hansen <dave.hansen@intel.com>
Orabug: 28220674
CVE: CVE-2018-3620

(cherry picked from commit 6b28baca9b1f0d4a42b865da7a05b1c81424bd5c)

Signed-off-by: Mihai Carabas <mihai.carabas@oracle.com>
Reviewed-by: Darren Kenny <darren.kenny@oracle.com>
Reviewed-by: Boris Ostrovsky <boris.ostrovsky@oracle.com>
Conflicts:
	arch/x86/include/asm/pgtable.h
[We do not have the wrapper check_pgprot (which was just adding some debug
config and calls massage_pgprot). Just use massage_pgprot.]

Conflicts:
	arch/x86/include/asm/pgtable-3level.h
	arch/x86/include/asm/pgtable.h
	arch/x86/include/asm/pgtable_64.h
pgtable-3level.h: contextual
pgtable.h: contextual as we didn't have some of the functions
pgtable_64.h: contextual

diff --git a/arch/x86/include/asm/page_types.h b/arch/x86/include/asm/page_types.h
index c7c712f2648b001b84f06d7ef683239fd09dd2cc..1d115e4a8a7973e34f652b5ed3d89dd4f9a4a5d3 100644 (file)
--- a/arch/x86/include/asm/page_types.h
+++ b/arch/x86/include/asm/page_types.h
@@ -9,6 +9,9 @@
 #define PAGE_SIZE      (_AC(1,UL) << PAGE_SHIFT)
 #define PAGE_MASK      (~(PAGE_SIZE-1))
 
+#define PMD_PAGE_SIZE          (_AC(1, UL) << PMD_SHIFT)
+#define PMD_PAGE_MASK          (~(PMD_PAGE_SIZE-1))
+
 #define __PHYSICAL_MASK                ((phys_addr_t)((1ULL << __PHYSICAL_MASK_SHIFT) - 1))
 #define __VIRTUAL_MASK         ((1UL << __VIRTUAL_MASK_SHIFT) - 1)
 
@@ -16,9 +19,7 @@
    virtual addresses are 32-bits but physical addresses are larger
    (ie, 32-bit PAE). */
 #define PHYSICAL_PAGE_MASK     (((signed long)PAGE_MASK) & __PHYSICAL_MASK)
-
-#define PMD_PAGE_SIZE          (_AC(1, UL) << PMD_SHIFT)
-#define PMD_PAGE_MASK          (~(PMD_PAGE_SIZE-1))
+#define PHYSICAL_PMD_PAGE_MASK (((signed long)PMD_PAGE_MASK) & __PHYSICAL_MASK)
 
 #define HPAGE_SHIFT            PMD_SHIFT
 #define HPAGE_SIZE             (_AC(1,UL) << HPAGE_SHIFT)

diff --git a/arch/x86/include/asm/pgtable-2level.h b/arch/x86/include/asm/pgtable-2level.h
index fd74a11959de0db5f4c6026f522decc3f40ea7e0..89c50332a71eda6c3bc4f16e8da1d7af36e9db5f 100644 (file)
--- a/arch/x86/include/asm/pgtable-2level.h
+++ b/arch/x86/include/asm/pgtable-2level.h
@@ -77,4 +77,21 @@ static inline unsigned long pte_bitop(unsigned long value, unsigned int rightshi
 #define __pte_to_swp_entry(pte)                ((swp_entry_t) { (pte).pte_low })
 #define __swp_entry_to_pte(x)          ((pte_t) { .pte = (x).val })
 
+/* No inverted PFNs on 2 level page tables */
+
+static inline u64 protnone_mask(u64 val)
+{
+       return 0;
+}
+
+static inline u64 flip_protnone_guard(u64 oldval, u64 val, u64 mask)
+{
+       return val;
+}
+
+static inline bool __pte_needs_invert(u64 val)
+{
+       return false;
+}
+
 #endif /* _ASM_X86_PGTABLE_2LEVEL_H */

diff --git a/arch/x86/include/asm/pgtable-3level.h b/arch/x86/include/asm/pgtable-3level.h
index cdaa58c9b39ed3a31054686f68f934e488990e2f..0c89891c7b4489f21df354181b5eaf1a92d973f5 100644 (file)
--- a/arch/x86/include/asm/pgtable-3level.h
+++ b/arch/x86/include/asm/pgtable-3level.h
@@ -184,4 +184,6 @@ static inline pmd_t native_pmdp_get_and_clear(pmd_t *pmdp)
 #define __pte_to_swp_entry(pte)                ((swp_entry_t){ (pte).pte_high })
 #define __swp_entry_to_pte(x)          ((pte_t){ { .pte_high = (x).val } })
 
+#include <asm/pgtable-invert.h>
+
 #endif /* _ASM_X86_PGTABLE_3LEVEL_H */

diff --git a/arch/x86/include/asm/pgtable-invert.h b/arch/x86/include/asm/pgtable-invert.h
new file mode 100644 (file)
index 0000000..1775641
--- /dev/null
+++ b/arch/x86/include/asm/pgtable-invert.h
@@ -0,0 +1,32 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_PGTABLE_INVERT_H
+#define _ASM_PGTABLE_INVERT_H 1
+
+#ifndef __ASSEMBLY__
+
+static inline bool __pte_needs_invert(u64 val)
+{
+       return (val & (_PAGE_PRESENT|_PAGE_PROTNONE)) == _PAGE_PROTNONE;
+}
+
+/* Get a mask to xor with the page table entry to get the correct pfn. */
+static inline u64 protnone_mask(u64 val)
+{
+       return __pte_needs_invert(val) ?  ~0ull : 0;
+}
+
+static inline u64 flip_protnone_guard(u64 oldval, u64 val, u64 mask)
+{
+       /*
+        * When a PTE transitions from NONE to !NONE or vice-versa
+        * invert the PFN part to stop speculation.
+        * pte_pfn undoes this when needed.
+        */
+       if (__pte_needs_invert(oldval) != __pte_needs_invert(val))
+               val = (val & ~mask) | (~val & mask);
+       return val;
+}
+
+#endif /* __ASSEMBLY__ */
+
+#endif

diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h
index 7844dff019fb5986f55fe6d0bae020c28150c4be..f14cf1d69cd0c933b3d007a5e80901e277a3b40b 100644 (file)
--- a/arch/x86/include/asm/pgtable.h
+++ b/arch/x86/include/asm/pgtable.h
@@ -141,19 +141,29 @@ static inline int pte_special(pte_t pte)
        return pte_flags(pte) & _PAGE_SPECIAL;
 }
 
+/* Entries that were set to PROT_NONE are inverted */
+
+static inline u64 protnone_mask(u64 val);
+
 static inline unsigned long pte_pfn(pte_t pte)
 {
-       return (pte_val(pte) & PTE_PFN_MASK) >> PAGE_SHIFT;
+       unsigned long pfn = pte_val(pte);
+       pfn ^= protnone_mask(pfn);
+       return (pfn & PTE_PFN_MASK) >> PAGE_SHIFT;
 }
 
 static inline unsigned long pmd_pfn(pmd_t pmd)
 {
-       return (pmd_val(pmd) & PTE_PFN_MASK) >> PAGE_SHIFT;
+       unsigned long pfn = pmd_val(pmd);
+       pfn ^= protnone_mask(pfn);
+       return (pfn & PTE_PFN_MASK) >> PAGE_SHIFT;
 }
 
 static inline unsigned long pud_pfn(pud_t pud)
 {
-       return (pud_val(pud) & PTE_PFN_MASK) >> PAGE_SHIFT;
+       unsigned long pfn = pud_val(pud);
+       pfn ^= protnone_mask(pfn);
+       return (pfn & PTE_PFN_MASK) >> PAGE_SHIFT;
 }
 
 #define pte_page(pte)  pfn_to_page(pte_pfn(pte))
@@ -342,19 +352,25 @@ static inline pgprotval_t massage_pgprot(pgprot_t pgprot)
 
 static inline pte_t pfn_pte(unsigned long page_nr, pgprot_t pgprot)
 {
-       return __pte(((phys_addr_t)page_nr << PAGE_SHIFT) |
-                    massage_pgprot(pgprot));
+       phys_addr_t pfn = page_nr << PAGE_SHIFT;
+       pfn ^= protnone_mask(pgprot_val(pgprot));
+       pfn &= PTE_PFN_MASK;
+       return __pte(pfn | massage_pgprot(pgprot));
 }
 
 static inline pmd_t pfn_pmd(unsigned long page_nr, pgprot_t pgprot)
 {
-       return __pmd(((phys_addr_t)page_nr << PAGE_SHIFT) |
-                    massage_pgprot(pgprot));
+       phys_addr_t pfn = page_nr << PAGE_SHIFT;
+       pfn ^= protnone_mask(pgprot_val(pgprot));
+       pfn &= PHYSICAL_PMD_PAGE_MASK;
+       return __pmd(pfn | massage_pgprot(pgprot));
 }
 
+static inline u64 flip_protnone_guard(u64 oldval, u64 val, u64 mask);
+
 static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
 {
-       pteval_t val = pte_val(pte);
+       pteval_t val = pte_val(pte), oldval = val;
 
        /*
         * Chop off the NX bit (if present), and add the NX portion of
@@ -362,17 +378,17 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
         */
        val &= _PAGE_CHG_MASK;
        val |= massage_pgprot(newprot) & ~_PAGE_CHG_MASK;
-
+       val = flip_protnone_guard(oldval, val, PTE_PFN_MASK);
        return __pte(val);
 }
 
 static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
 {
-       pmdval_t val = pmd_val(pmd);
+       pmdval_t val = pmd_val(pmd), oldval = val;
 
        val &= _HPAGE_CHG_MASK;
        val |= massage_pgprot(newprot) & ~_HPAGE_CHG_MASK;
-
+       val = flip_protnone_guard(oldval, val, PHYSICAL_PMD_PAGE_MASK);
        return __pmd(val);
 }
 
@@ -509,7 +525,7 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd)
  * Currently stuck as a macro due to indirect forward reference to
  * linux/mmzone.h's __section_mem_map_addr() definition:
  */
-#define pmd_page(pmd)  pfn_to_page((pmd_val(pmd) & PTE_PFN_MASK) >> PAGE_SHIFT)
+#define pmd_page(pmd)  pfn_to_page(pmd_pfn(pmd))
 
 /*
  * the pmd page can be thought of an array like this: pmd_t[PTRS_PER_PMD]

diff --git a/arch/x86/include/asm/pgtable_64.h b/arch/x86/include/asm/pgtable_64.h
index 4adbd61c077c5d4511e3e57c56b7162646a0098f..f358d23372639d628e984da162c9c4fba3f0ea19 100644 (file)
--- a/arch/x86/include/asm/pgtable_64.h
+++ b/arch/x86/include/asm/pgtable_64.h
@@ -214,6 +214,8 @@ extern void cleanup_highmap(void);
 extern void init_extra_mapping_uc(unsigned long phys, unsigned long size);
 extern void init_extra_mapping_wb(unsigned long phys, unsigned long size);
 
+#include <asm/pgtable-invert.h>
+
 #endif /* !__ASSEMBLY__ */
 
 #endif /* _ASM_X86_PGTABLE_64_H */