iommu/amd: Reduce amount of MMIO when submitting commands

As newer, higher speed devices are developed, perf data shows that the
amount of MMIO that is performed when submitting commands to the IOMMU
causes performance issues. Currently, the command submission path reads
the command buffer head and tail pointers and then writes the tail
pointer once the command is ready.

The tail pointer is only ever updated by the driver so it can be tracked
by the driver without having to read it from the hardware.

The head pointer is updated by the hardware, but can be read
opportunistically. Reading the head pointer only when it appears that
there might not be room in the command buffer and then re-checking the
available space reduces the number of times the head pointer has to be
read.

Signed-off-by: Tom Lendacky <thomas.lendacky@amd.com>
Signed-off-by: Joerg Roedel <jroedel@suse.de>

diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c
index d7748955184b5577b7ee2f12b0c45ca9f21db389..d81c895ff4f43e0b51046064280eb41b1bffd270 100644 (file)
--- a/drivers/iommu/amd_iommu.c
+++ b/drivers/iommu/amd_iommu.c
@@ -874,19 +874,20 @@ static int wait_on_sem(volatile u64 *sem)
 }
 
 static void copy_cmd_to_buffer(struct amd_iommu *iommu,
-                              struct iommu_cmd *cmd,
-                              u32 tail)
+                              struct iommu_cmd *cmd)
 {
        u8 *target;
 
-       target = iommu->cmd_buf + tail;
-       tail   = (tail + sizeof(*cmd)) % CMD_BUFFER_SIZE;
+       target = iommu->cmd_buf + iommu->cmd_buf_tail;
+
+       iommu->cmd_buf_tail += sizeof(*cmd);
+       iommu->cmd_buf_tail %= CMD_BUFFER_SIZE;
 
        /* Copy command to buffer */
        memcpy(target, cmd, sizeof(*cmd));
 
        /* Tell the IOMMU about it */
-       writel(tail, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
+       writel(iommu->cmd_buf_tail, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
 }
 
 static void build_completion_wait(struct iommu_cmd *cmd, u64 address)
@@ -1044,23 +1045,31 @@ static int __iommu_queue_command_sync(struct amd_iommu *iommu,
                                      struct iommu_cmd *cmd,
                                      bool sync)
 {
-       u32 left, tail, head, next_tail;
+       bool read_head = true;
+       u32 left, next_tail;
 
+       next_tail = (iommu->cmd_buf_tail + sizeof(*cmd)) % CMD_BUFFER_SIZE;
 again:
-
-       head      = readl(iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);
-       tail      = readl(iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
-       next_tail = (tail + sizeof(*cmd)) % CMD_BUFFER_SIZE;
-       left      = (head - next_tail) % CMD_BUFFER_SIZE;
+       left      = (iommu->cmd_buf_head - next_tail) % CMD_BUFFER_SIZE;
 
        if (left <= 0x20) {
                struct iommu_cmd sync_cmd;
                int ret;
 
+               if (read_head) {
+                       /* Update head and recheck remaining space */
+                       iommu->cmd_buf_head = readl(iommu->mmio_base +
+                                                   MMIO_CMD_HEAD_OFFSET);
+                       read_head = false;
+                       goto again;
+               }
+
+               read_head = true;
+
                iommu->cmd_sem = 0;
 
                build_completion_wait(&sync_cmd, (u64)&iommu->cmd_sem);
-               copy_cmd_to_buffer(iommu, &sync_cmd, tail);
+               copy_cmd_to_buffer(iommu, &sync_cmd);
 
                if ((ret = wait_on_sem(&iommu->cmd_sem)) != 0)
                        return ret;
@@ -1068,7 +1077,7 @@ again:
                goto again;
        }
 
-       copy_cmd_to_buffer(iommu, cmd, tail);
+       copy_cmd_to_buffer(iommu, cmd);
 
        /* We need to sync now to make sure all commands are processed */
        iommu->need_sync = sync;

diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c
index 5a11328f4d9854457f5e078ab0b3dd29aff04b6e..3fa7e3b35507d1f14406c16635a43188f1ec0bfd 100644 (file)
--- a/drivers/iommu/amd_iommu_init.c
+++ b/drivers/iommu/amd_iommu_init.c
@@ -588,6 +588,8 @@ void amd_iommu_reset_cmd_buffer(struct amd_iommu *iommu)
 
        writel(0x00, iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);
        writel(0x00, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
+       iommu->cmd_buf_head = 0;
+       iommu->cmd_buf_tail = 0;
 
        iommu_feature_enable(iommu, CONTROL_CMDBUF_EN);
 }

diff --git a/drivers/iommu/amd_iommu_types.h b/drivers/iommu/amd_iommu_types.h
index 4de8f4160bb81592bea7dd0aaa6e06aebcd777cf..6960d7db2fabebcb792b4b03c8f2ce3d0b1485fb 100644 (file)
--- a/drivers/iommu/amd_iommu_types.h
+++ b/drivers/iommu/amd_iommu_types.h
@@ -516,6 +516,8 @@ struct amd_iommu {
 
        /* command buffer virtual address */
        u8 *cmd_buf;
+       u32 cmd_buf_head;
+       u32 cmd_buf_tail;
 
        /* event buffer virtual address */
        u8 *evt_buf;