x86/mm: Decouple dynamic __PHYSICAL_MASK from AMD SME

AMD SME claims one bit from physical address to indicate whether the
page is encrypted or not. To achieve that we clear out the bit from
__PHYSICAL_MASK.

The capability to adjust __PHYSICAL_MASK is required beyond AMD SME.
For instance for upcoming Intel Multi-Key Total Memory Encryption.

Factor it out into a separate feature with own Kconfig handle.

It also helps with overhead of AMD SME. It saves more than 3k in .text
on defconfig + AMD_MEM_ENCRYPT:

	add/remove: 3/2 grow/shrink: 5/110 up/down: 189/-3753 (-3564)

We would need to return to this once we have infrastructure to patch
constants in code. That's good candidate for it.

Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Tom Lendacky <thomas.lendacky@amd.com>
Cc: linux-mm@kvack.org
Cc: "H. Peter Anvin" <hpa@zytor.com>
Link: https://lkml.kernel.org/r/20180518113028.79825-1-kirill.shutemov@linux.intel.com

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index c07f492b871a8bf0f47cfb7be03a004dd740c2c1..43a8fc4762965268be346919a7d8ae31ff914efd 100644 (file)
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -337,6 +337,9 @@ config ARCH_SUPPORTS_UPROBES
 config FIX_EARLYCON_MEM
        def_bool y
 
+config DYNAMIC_PHYSICAL_MASK
+       bool
+
 config PGTABLE_LEVELS
        int
        default 5 if X86_5LEVEL
@@ -1508,6 +1511,7 @@ config ARCH_HAS_MEM_ENCRYPT
 config AMD_MEM_ENCRYPT
        bool "AMD Secure Memory Encryption (SME) support"
        depends on X86_64 && CPU_SUP_AMD
+       select DYNAMIC_PHYSICAL_MASK
        ---help---
          Say yes to enable support for the encryption of system memory.
          This requires an AMD processor that supports Secure Memory

diff --git a/arch/x86/boot/compressed/kaslr_64.c b/arch/x86/boot/compressed/kaslr_64.c
index 522d11431433afd85eb4742c8afe9ab717420041..748456c365f4691af041753c63d6991f9bc8b4b8 100644 (file)
--- a/arch/x86/boot/compressed/kaslr_64.c
+++ b/arch/x86/boot/compressed/kaslr_64.c
@@ -69,6 +69,8 @@ static struct alloc_pgt_data pgt_data;
 /* The top level page table entry pointer. */
 static unsigned long top_level_pgt;
 
+phys_addr_t physical_mask = (1ULL << __PHYSICAL_MASK_SHIFT) - 1;
+
 /*
  * Mapping information structure passed to kernel_ident_mapping_init().
  * Due to relocation, pointers must be assigned at run time not build time.
@@ -81,6 +83,9 @@ void initialize_identity_maps(void)
        /* If running as an SEV guest, the encryption mask is required. */
        set_sev_encryption_mask();
 
+       /* Exclude the encryption mask from __PHYSICAL_MASK */
+       physical_mask &= ~sme_me_mask;
+
        /* Init mapping_info with run-time function/buffer pointers. */
        mapping_info.alloc_pgt_page = alloc_pgt_page;
        mapping_info.context = &pgt_data;

diff --git a/arch/x86/include/asm/page_types.h b/arch/x86/include/asm/page_types.h
index 1e53560a84bb8ce12a5332a5a7f5983b0b07b4b0..c85e15010f48da7ece9023ac836dafe5b21a5cf6 100644 (file)
--- a/arch/x86/include/asm/page_types.h
+++ b/arch/x86/include/asm/page_types.h
@@ -17,7 +17,6 @@
 #define PUD_PAGE_SIZE          (_AC(1, UL) << PUD_SHIFT)
 #define PUD_PAGE_MASK          (~(PUD_PAGE_SIZE-1))
 
-#define __PHYSICAL_MASK                ((phys_addr_t)(__sme_clr((1ULL << __PHYSICAL_MASK_SHIFT) - 1)))
 #define __VIRTUAL_MASK         ((1UL << __VIRTUAL_MASK_SHIFT) - 1)
 
 /* Cast *PAGE_MASK to a signed type so that it is sign-extended if
@@ -55,6 +54,13 @@
 
 #ifndef __ASSEMBLY__
 
+#ifdef CONFIG_DYNAMIC_PHYSICAL_MASK
+extern phys_addr_t physical_mask;
+#define __PHYSICAL_MASK                physical_mask
+#else
+#define __PHYSICAL_MASK                ((phys_addr_t)((1ULL << __PHYSICAL_MASK_SHIFT) - 1))
+#endif
+
 extern int devmem_is_allowed(unsigned long pagenr);
 
 extern unsigned long max_low_pfn_mapped;

diff --git a/arch/x86/mm/mem_encrypt_identity.c b/arch/x86/mm/mem_encrypt_identity.c
index 1b2197d13832e6307ae8a8f97cf5ba0d93e3a1ed..7ae36868aed256e44749bdd7f3692dc423574f7c 100644 (file)
--- a/arch/x86/mm/mem_encrypt_identity.c
+++ b/arch/x86/mm/mem_encrypt_identity.c
@@ -527,6 +527,7 @@ void __init sme_enable(struct boot_params *bp)
                /* SEV state cannot be controlled by a command line option */
                sme_me_mask = me_mask;
                sev_enabled = true;
+               physical_mask &= ~sme_me_mask;
                return;
        }
 
@@ -561,4 +562,6 @@ void __init sme_enable(struct boot_params *bp)
                sme_me_mask = 0;
        else
                sme_me_mask = active_by_default ? me_mask : 0;
+
+       physical_mask &= ~sme_me_mask;
 }

diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c
index ffc8c13c50e4491941a1f795e2339964305e51da..3ca59cf7a7f9406090e68366e8944c85ed506ff5 100644 (file)
--- a/arch/x86/mm/pgtable.c
+++ b/arch/x86/mm/pgtable.c
@@ -8,6 +8,11 @@
 #include <asm/fixmap.h>
 #include <asm/mtrr.h>
 
+#ifdef CONFIG_DYNAMIC_PHYSICAL_MASK
+phys_addr_t physical_mask __ro_after_init = (1ULL << __PHYSICAL_MASK_SHIFT) - 1;
+EXPORT_SYMBOL(physical_mask);
+#endif
+
 #define PGALLOC_GFP (GFP_KERNEL_ACCOUNT | __GFP_ZERO)
 
 #ifdef CONFIG_HIGHPTE