iommu/amd: Allow the AMD IOMMU to work with memory encryption

The IOMMU is programmed with physical addresses for the various tables
and buffers that are used to communicate between the device and the
driver. When the driver allocates this memory it is encrypted. In order
for the IOMMU to access the memory as encrypted the encryption mask needs
to be included in these physical addresses during configuration.

The PTE entries created by the IOMMU should also include the encryption
mask so that when the device behind the IOMMU performs a DMA, the DMA
will be performed to encrypted memory.

Signed-off-by: Tom Lendacky <thomas.lendacky@amd.com>
Reviewed-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Borislav Petkov <bp@suse.de>
Acked-by: Joerg Roedel <jroedel@suse.de>
Cc: <iommu@lists.linux-foundation.org>
Cc: Alexander Potapenko <glider@google.com>
Cc: Andrey Ryabinin <aryabinin@virtuozzo.com>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Brijesh Singh <brijesh.singh@amd.com>
Cc: Dave Young <dyoung@redhat.com>
Cc: Dmitry Vyukov <dvyukov@google.com>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Larry Woodman <lwoodman@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Matt Fleming <matt@codeblueprint.co.uk>
Cc: Michael S. Tsirkin <mst@redhat.com>
Cc: Paolo Bonzini <pbonzini@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Radim Krčmář <rkrcmar@redhat.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Toshimitsu Kani <toshi.kani@hpe.com>
Cc: kasan-dev@googlegroups.com
Cc: kvm@vger.kernel.org
Cc: linux-arch@vger.kernel.org
Cc: linux-doc@vger.kernel.org
Cc: linux-efi@vger.kernel.org
Cc: linux-mm@kvack.org
Link: http://lkml.kernel.org/r/3053631ea25ba8b1601c351cb7c541c496f6d9bc.1500319216.git.thomas.lendacky@amd.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>

diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c
index 688e77576e5a50b3f2f137eec72cd721d6bf96fa..abc6ca65a47b0352b48444f64da8ca58ce5c3518 100644 (file)
--- a/drivers/iommu/amd_iommu.c
+++ b/drivers/iommu/amd_iommu.c
@@ -575,7 +575,7 @@ static void dump_dte_entry(u16 devid)
 
 static void dump_command(unsigned long phys_addr)
 {
-       struct iommu_cmd *cmd = phys_to_virt(phys_addr);
+       struct iommu_cmd *cmd = iommu_phys_to_virt(phys_addr);
        int i;
 
        for (i = 0; i < 4; ++i)
@@ -919,11 +919,13 @@ static void copy_cmd_to_buffer(struct amd_iommu *iommu,
 
 static void build_completion_wait(struct iommu_cmd *cmd, u64 address)
 {
+       u64 paddr = iommu_virt_to_phys((void *)address);
+
        WARN_ON(address & 0x7ULL);
 
        memset(cmd, 0, sizeof(*cmd));
-       cmd->data[0] = lower_32_bits(__pa(address)) | CMD_COMPL_WAIT_STORE_MASK;
-       cmd->data[1] = upper_32_bits(__pa(address));
+       cmd->data[0] = lower_32_bits(paddr) | CMD_COMPL_WAIT_STORE_MASK;
+       cmd->data[1] = upper_32_bits(paddr);
        cmd->data[2] = 1;
        CMD_SET_TYPE(cmd, CMD_COMPL_WAIT);
 }
@@ -1383,7 +1385,7 @@ static bool increase_address_space(struct protection_domain *domain,
                return false;
 
        *pte             = PM_LEVEL_PDE(domain->mode,
-                                       virt_to_phys(domain->pt_root));
+                                       iommu_virt_to_phys(domain->pt_root));
        domain->pt_root  = pte;
        domain->mode    += 1;
        domain->updated  = true;
@@ -1420,7 +1422,7 @@ static u64 *alloc_pte(struct protection_domain *domain,
                        if (!page)
                                return NULL;
 
-                       __npte = PM_LEVEL_PDE(level, virt_to_phys(page));
+                       __npte = PM_LEVEL_PDE(level, iommu_virt_to_phys(page));
 
                        /* pte could have been changed somewhere. */
                        if (cmpxchg64(pte, __pte, __npte) != __pte) {
@@ -1536,10 +1538,10 @@ static int iommu_map_page(struct protection_domain *dom,
                        return -EBUSY;
 
        if (count > 1) {
-               __pte = PAGE_SIZE_PTE(phys_addr, page_size);
+               __pte = PAGE_SIZE_PTE(__sme_set(phys_addr), page_size);
                __pte |= PM_LEVEL_ENC(7) | IOMMU_PTE_P | IOMMU_PTE_FC;
        } else
-               __pte = phys_addr | IOMMU_PTE_P | IOMMU_PTE_FC;
+               __pte = __sme_set(phys_addr) | IOMMU_PTE_P | IOMMU_PTE_FC;
 
        if (prot & IOMMU_PROT_IR)
                __pte |= IOMMU_PTE_IR;
@@ -1755,7 +1757,7 @@ static void free_gcr3_tbl_level1(u64 *tbl)
                if (!(tbl[i] & GCR3_VALID))
                        continue;
 
-               ptr = __va(tbl[i] & PAGE_MASK);
+               ptr = iommu_phys_to_virt(tbl[i] & PAGE_MASK);
 
                free_page((unsigned long)ptr);
        }
@@ -1770,7 +1772,7 @@ static void free_gcr3_tbl_level2(u64 *tbl)
                if (!(tbl[i] & GCR3_VALID))
                        continue;
 
-               ptr = __va(tbl[i] & PAGE_MASK);
+               ptr = iommu_phys_to_virt(tbl[i] & PAGE_MASK);
 
                free_gcr3_tbl_level1(ptr);
        }
@@ -2049,7 +2051,7 @@ static void set_dte_entry(u16 devid, struct protection_domain *domain, bool ats)
        u64 flags = 0;
 
        if (domain->mode != PAGE_MODE_NONE)
-               pte_root = virt_to_phys(domain->pt_root);
+               pte_root = iommu_virt_to_phys(domain->pt_root);
 
        pte_root |= (domain->mode & DEV_ENTRY_MODE_MASK)
                    << DEV_ENTRY_MODE_SHIFT;
@@ -2061,7 +2063,7 @@ static void set_dte_entry(u16 devid, struct protection_domain *domain, bool ats)
                flags |= DTE_FLAG_IOTLB;
 
        if (domain->flags & PD_IOMMUV2_MASK) {
-               u64 gcr3 = __pa(domain->gcr3_tbl);
+               u64 gcr3 = iommu_virt_to_phys(domain->gcr3_tbl);
                u64 glx  = domain->glx;
                u64 tmp;
 
@@ -3606,10 +3608,10 @@ static u64 *__get_gcr3_pte(u64 *root, int level, int pasid, bool alloc)
                        if (root == NULL)
                                return NULL;
 
-                       *pte = __pa(root) | GCR3_VALID;
+                       *pte = iommu_virt_to_phys(root) | GCR3_VALID;
                }
 
-               root = __va(*pte & PAGE_MASK);
+               root = iommu_phys_to_virt(*pte & PAGE_MASK);
 
                level -= 1;
        }
@@ -3788,7 +3790,7 @@ static void set_dte_irq_entry(u16 devid, struct irq_remap_table *table)
 
        dte     = amd_iommu_dev_table[devid].data[2];
        dte     &= ~DTE_IRQ_PHYS_ADDR_MASK;
-       dte     |= virt_to_phys(table->table);
+       dte     |= iommu_virt_to_phys(table->table);
        dte     |= DTE_IRQ_REMAP_INTCTL;
        dte     |= DTE_IRQ_TABLE_LEN;
        dte     |= DTE_IRQ_REMAP_ENABLE;

diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c
index 5cc597b383c7208d69e83d8ec0646e936817d1b4..42ff9a86c1023d5d8c80f9953f55114dca69b863 100644 (file)
--- a/drivers/iommu/amd_iommu_init.c
+++ b/drivers/iommu/amd_iommu_init.c
@@ -30,6 +30,7 @@
 #include <linux/iommu.h>
 #include <linux/kmemleak.h>
 #include <linux/crash_dump.h>
+#include <linux/mem_encrypt.h>
 #include <asm/pci-direct.h>
 #include <asm/iommu.h>
 #include <asm/gart.h>
@@ -348,7 +349,7 @@ static void iommu_set_device_table(struct amd_iommu *iommu)
 
        BUG_ON(iommu->mmio_base == NULL);
 
-       entry = virt_to_phys(amd_iommu_dev_table);
+       entry = iommu_virt_to_phys(amd_iommu_dev_table);
        entry |= (dev_table_size >> 12) - 1;
        memcpy_toio(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET,
                        &entry, sizeof(entry));
@@ -606,7 +607,7 @@ static void iommu_enable_command_buffer(struct amd_iommu *iommu)
 
        BUG_ON(iommu->cmd_buf == NULL);
 
-       entry = (u64)virt_to_phys(iommu->cmd_buf);
+       entry = iommu_virt_to_phys(iommu->cmd_buf);
        entry |= MMIO_CMD_SIZE_512;
 
        memcpy_toio(iommu->mmio_base + MMIO_CMD_BUF_OFFSET,
@@ -635,7 +636,7 @@ static void iommu_enable_event_buffer(struct amd_iommu *iommu)
 
        BUG_ON(iommu->evt_buf == NULL);
 
-       entry = (u64)virt_to_phys(iommu->evt_buf) | EVT_LEN_MASK;
+       entry = iommu_virt_to_phys(iommu->evt_buf) | EVT_LEN_MASK;
 
        memcpy_toio(iommu->mmio_base + MMIO_EVT_BUF_OFFSET,
                    &entry, sizeof(entry));
@@ -668,7 +669,7 @@ static void iommu_enable_ppr_log(struct amd_iommu *iommu)
        if (iommu->ppr_log == NULL)
                return;
 
-       entry = (u64)virt_to_phys(iommu->ppr_log) | PPR_LOG_SIZE_512;
+       entry = iommu_virt_to_phys(iommu->ppr_log) | PPR_LOG_SIZE_512;
 
        memcpy_toio(iommu->mmio_base + MMIO_PPR_LOG_OFFSET,
                    &entry, sizeof(entry));
@@ -748,10 +749,10 @@ static int iommu_init_ga_log(struct amd_iommu *iommu)
        if (!iommu->ga_log_tail)
                goto err_out;
 
-       entry = (u64)virt_to_phys(iommu->ga_log) | GA_LOG_SIZE_512;
+       entry = iommu_virt_to_phys(iommu->ga_log) | GA_LOG_SIZE_512;
        memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_BASE_OFFSET,
                    &entry, sizeof(entry));
-       entry = ((u64)virt_to_phys(iommu->ga_log) & 0xFFFFFFFFFFFFFULL) & ~7ULL;
+       entry = (iommu_virt_to_phys(iommu->ga_log) & 0xFFFFFFFFFFFFFULL) & ~7ULL;
        memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_TAIL_OFFSET,
                    &entry, sizeof(entry));
        writel(0x00, iommu->mmio_base + MMIO_GA_HEAD_OFFSET);
@@ -2564,6 +2565,24 @@ static int __init amd_iommu_init(void)
        return ret;
 }
 
+static bool amd_iommu_sme_check(void)
+{
+       if (!sme_active() || (boot_cpu_data.x86 != 0x17))
+               return true;
+
+       /* For Fam17h, a specific level of support is required */
+       if (boot_cpu_data.microcode >= 0x08001205)
+               return true;
+
+       if ((boot_cpu_data.microcode >= 0x08001126) &&
+           (boot_cpu_data.microcode <= 0x080011ff))
+               return true;
+
+       pr_notice("AMD-Vi: IOMMU not currently supported when SME is active\n");
+
+       return false;
+}
+
 /****************************************************************************
  *
  * Early detect code. This code runs at IOMMU detection time in the DMA
@@ -2578,6 +2597,9 @@ int __init amd_iommu_detect(void)
        if (no_iommu || (iommu_detected && !gart_iommu_aperture))
                return -ENODEV;
 
+       if (!amd_iommu_sme_check())
+               return -ENODEV;
+
        ret = iommu_go_to_state(IOMMU_IVRS_DETECTED);
        if (ret)
                return ret;

diff --git a/drivers/iommu/amd_iommu_proto.h b/drivers/iommu/amd_iommu_proto.h
index 466260f8a1df37bb79738d4c8f91b90568cdd7c6..3f12fb2338ea5bbc1850f779fd98fff92808d7c0 100644 (file)
--- a/drivers/iommu/amd_iommu_proto.h
+++ b/drivers/iommu/amd_iommu_proto.h
@@ -87,4 +87,14 @@ static inline bool iommu_feature(struct amd_iommu *iommu, u64 f)
        return !!(iommu->features & f);
 }
 
+static inline u64 iommu_virt_to_phys(void *vaddr)
+{
+       return (u64)__sme_set(virt_to_phys(vaddr));
+}
+
+static inline void *iommu_phys_to_virt(unsigned long paddr)
+{
+       return phys_to_virt(__sme_clr(paddr));
+}
+
 #endif /* _ASM_X86_AMD_IOMMU_PROTO_H  */

diff --git a/drivers/iommu/amd_iommu_types.h b/drivers/iommu/amd_iommu_types.h
index 294a409e283b7ae4b52c59350756983711bba9d1..8591f43c467c9c93aa1c3dcf5d8e64d2f3bcdbd5 100644 (file)
--- a/drivers/iommu/amd_iommu_types.h
+++ b/drivers/iommu/amd_iommu_types.h
@@ -344,7 +344,7 @@
 
 #define IOMMU_PAGE_MASK (((1ULL << 52) - 1) & ~0xfffULL)
 #define IOMMU_PTE_PRESENT(pte) ((pte) & IOMMU_PTE_P)
-#define IOMMU_PTE_PAGE(pte) (phys_to_virt((pte) & IOMMU_PAGE_MASK))
+#define IOMMU_PTE_PAGE(pte) (iommu_phys_to_virt((pte) & IOMMU_PAGE_MASK))
 #define IOMMU_PTE_MODE(pte) (((pte) >> 9) & 0x07)
 
 #define IOMMU_PROT_MASK 0x03