accel/ivpu: Implement support for preemption buffers

Allocate per-context preemption buffers that are required by HWS.

There are two preemption buffers:
  * primary - allocated in user memory range (PIOVA accessible)
  * secondary - allocated in shave memory range

Signed-off-by: Wachowski, Karol <karol.wachowski@intel.com>
Signed-off-by: Jacek Lawrynowicz <jacek.lawrynowicz@linux.intel.com>
Reviewed-by: Jeffrey Hugo <quic_jhugo@quicinc.com>
Link: https://patchwork.freedesktop.org/patch/msgid/20240513120431.3187212-5-jacek.lawrynowicz@linux.intel.com

diff --git a/drivers/accel/ivpu/ivpu_drv.h b/drivers/accel/ivpu/ivpu_drv.h
index aafc5c3e9041e65d847babef35664e6745847c4c..f500b2d9245269d32e76583cb8cac405f454642f 100644 (file)
--- a/drivers/accel/ivpu/ivpu_drv.h
+++ b/drivers/accel/ivpu/ivpu_drv.h
@@ -170,6 +170,7 @@ extern bool ivpu_disable_mmu_cont_pages;
 #define IVPU_TEST_MODE_NULL_SUBMISSION    BIT(2)
 #define IVPU_TEST_MODE_D0I3_MSG_DISABLE   BIT(4)
 #define IVPU_TEST_MODE_D0I3_MSG_ENABLE    BIT(5)
+#define IVPU_TEST_MODE_PREEMPTION_DISABLE BIT(6)
 extern int ivpu_test_mode;
 
 struct ivpu_file_priv *ivpu_file_priv_get(struct ivpu_file_priv *file_priv);

diff --git a/drivers/accel/ivpu/ivpu_fw.c b/drivers/accel/ivpu/ivpu_fw.c
index 1457300828bf15ae3ca466a1a77fd457b8a2a3c5..29ecf7db238b3d14e01790dd0137e52b3e954924 100644 (file)
--- a/drivers/accel/ivpu/ivpu_fw.c
+++ b/drivers/accel/ivpu/ivpu_fw.c
@@ -200,6 +200,9 @@ static int ivpu_fw_parse(struct ivpu_device *vdev)
 
        fw->dvfs_mode = 0;
 
+       fw->primary_preempt_buf_size = fw_hdr->preemption_buffer_1_size;
+       fw->secondary_preempt_buf_size = fw_hdr->preemption_buffer_2_size;
+
        ivpu_dbg(vdev, FW_BOOT, "Size: file %lu image %u runtime %u shavenn %u\n",
                 fw->file->size, fw->image_size, fw->runtime_size, fw->shave_nn_size);
        ivpu_dbg(vdev, FW_BOOT, "Address: runtime 0x%llx, load 0x%llx, entry point 0x%llx\n",

diff --git a/drivers/accel/ivpu/ivpu_fw.h b/drivers/accel/ivpu/ivpu_fw.h
index 66b60fa161b522150abddbfa20c9ddd207fd5ada..66fc7da3ab0f3fba4493675d2ca2ec0ed60d721d 100644 (file)
--- a/drivers/accel/ivpu/ivpu_fw.h
+++ b/drivers/accel/ivpu/ivpu_fw.h
@@ -28,6 +28,8 @@ struct ivpu_fw_info {
        u32 trace_destination_mask;
        u64 trace_hw_component_mask;
        u32 dvfs_mode;
+       u32 primary_preempt_buf_size;
+       u32 secondary_preempt_buf_size;
 };
 
 int ivpu_fw_init(struct ivpu_device *vdev);

diff --git a/drivers/accel/ivpu/ivpu_job.c b/drivers/accel/ivpu/ivpu_job.c
index b56035de1a597b4c511e6bad09cefa8490c612db..3ef9d8022c9c3fcddbcc65050522494f150b0575 100644 (file)
--- a/drivers/accel/ivpu/ivpu_job.c
+++ b/drivers/accel/ivpu/ivpu_job.c
@@ -12,11 +12,13 @@
 #include <uapi/drm/ivpu_accel.h>
 
 #include "ivpu_drv.h"
+#include "ivpu_fw.h"
 #include "ivpu_hw.h"
 #include "ivpu_ipc.h"
 #include "ivpu_job.h"
 #include "ivpu_jsm_msg.h"
 #include "ivpu_pm.h"
+#include "vpu_boot_api.h"
 
 #define CMD_BUF_IDX         0
 #define JOB_ID_JOB_MASK             GENMASK(7, 0)
@@ -28,6 +30,53 @@ static void ivpu_cmdq_ring_db(struct ivpu_device *vdev, struct ivpu_cmdq *cmdq)
        ivpu_hw_reg_db_set(vdev, cmdq->db_id);
 }
 
+static int ivpu_preemption_buffers_create(struct ivpu_device *vdev,
+                                         struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq)
+{
+       u64 primary_size = ALIGN(vdev->fw->primary_preempt_buf_size, PAGE_SIZE);
+       u64 secondary_size = ALIGN(vdev->fw->secondary_preempt_buf_size, PAGE_SIZE);
+       struct ivpu_addr_range range;
+
+       if (vdev->hw->sched_mode != VPU_SCHEDULING_MODE_HW)
+               return 0;
+
+       range.start = vdev->hw->ranges.user.end - (primary_size * IVPU_NUM_CMDQS_PER_CTX);
+       range.end = vdev->hw->ranges.user.end;
+       cmdq->primary_preempt_buf = ivpu_bo_create(vdev, &file_priv->ctx, &range, primary_size,
+                                                  DRM_IVPU_BO_WC);
+       if (!cmdq->primary_preempt_buf) {
+               ivpu_err(vdev, "Failed to create primary preemption buffer\n");
+               return -ENOMEM;
+       }
+
+       range.start = vdev->hw->ranges.shave.end - (secondary_size * IVPU_NUM_CMDQS_PER_CTX);
+       range.end = vdev->hw->ranges.shave.end;
+       cmdq->secondary_preempt_buf = ivpu_bo_create(vdev, &file_priv->ctx, &range, secondary_size,
+                                                    DRM_IVPU_BO_WC);
+       if (!cmdq->secondary_preempt_buf) {
+               ivpu_err(vdev, "Failed to create secondary preemption buffer\n");
+               goto err_free_primary;
+       }
+
+       return 0;
+
+err_free_primary:
+       ivpu_bo_free(cmdq->primary_preempt_buf);
+       return -ENOMEM;
+}
+
+static void ivpu_preemption_buffers_free(struct ivpu_device *vdev,
+                                        struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq)
+{
+       if (vdev->hw->sched_mode != VPU_SCHEDULING_MODE_HW)
+               return;
+
+       drm_WARN_ON(&vdev->drm, !cmdq->primary_preempt_buf);
+       drm_WARN_ON(&vdev->drm, !cmdq->secondary_preempt_buf);
+       ivpu_bo_free(cmdq->primary_preempt_buf);
+       ivpu_bo_free(cmdq->secondary_preempt_buf);
+}
+
 static struct ivpu_cmdq *ivpu_cmdq_alloc(struct ivpu_file_priv *file_priv, u16 engine)
 {
        struct xa_limit db_xa_limit = {.max = IVPU_MAX_DB, .min = IVPU_MIN_DB};
@@ -50,6 +99,10 @@ static struct ivpu_cmdq *ivpu_cmdq_alloc(struct ivpu_file_priv *file_priv, u16 e
        if (!cmdq->mem)
                goto err_erase_xa;
 
+       ret = ivpu_preemption_buffers_create(vdev, file_priv, cmdq);
+       if (ret)
+               goto err_free_cmdq_mem;
+
        cmdq->entry_count = (u32)((ivpu_bo_size(cmdq->mem) - sizeof(struct vpu_job_queue_header)) /
                                  sizeof(struct vpu_job_queue_entry));
 
@@ -62,6 +115,8 @@ static struct ivpu_cmdq *ivpu_cmdq_alloc(struct ivpu_file_priv *file_priv, u16 e
 
        return cmdq;
 
+err_free_cmdq_mem:
+       ivpu_bo_free(cmdq->mem);
 err_erase_xa:
        xa_erase(&vdev->db_xa, cmdq->db_id);
 err_free_cmdq:
@@ -74,6 +129,7 @@ static void ivpu_cmdq_free(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *c
        if (!cmdq)
                return;
 
+       ivpu_preemption_buffers_free(file_priv->vdev, file_priv, cmdq);
        ivpu_bo_free(cmdq->mem);
        xa_erase(&file_priv->vdev->db_xa, cmdq->db_id);
        kfree(cmdq);
@@ -207,6 +263,15 @@ static int ivpu_cmdq_push_job(struct ivpu_cmdq *cmdq, struct ivpu_job *job)
        entry->flags = 0;
        if (unlikely(ivpu_test_mode & IVPU_TEST_MODE_NULL_SUBMISSION))
                entry->flags = VPU_JOB_FLAGS_NULL_SUBMISSION_MASK;
+
+       if (vdev->hw->sched_mode == VPU_SCHEDULING_MODE_HW &&
+           (unlikely(!(ivpu_test_mode & IVPU_TEST_MODE_PREEMPTION_DISABLE)))) {
+               entry->primary_preempt_buf_addr = cmdq->primary_preempt_buf->vpu_addr;
+               entry->primary_preempt_buf_size = ivpu_bo_size(cmdq->primary_preempt_buf);
+               entry->secondary_preempt_buf_addr = cmdq->secondary_preempt_buf->vpu_addr;
+               entry->secondary_preempt_buf_size = ivpu_bo_size(cmdq->secondary_preempt_buf);
+       }
+
        wmb(); /* Ensure that tail is updated after filling entry */
        header->tail = next_entry;
        wmb(); /* Flush WC buffer for jobq header */

diff --git a/drivers/accel/ivpu/ivpu_job.h b/drivers/accel/ivpu/ivpu_job.h
index ca4984071cc76b17d858ae86929a48a9cea39c88..e50002b5788c40910d01d6d4729d4bb1e1a1402d 100644 (file)
--- a/drivers/accel/ivpu/ivpu_job.h
+++ b/drivers/accel/ivpu/ivpu_job.h
@@ -24,6 +24,8 @@ struct ivpu_file_priv;
  */
 struct ivpu_cmdq {
        struct vpu_job_queue *jobq;
+       struct ivpu_bo *primary_preempt_buf;
+       struct ivpu_bo *secondary_preempt_buf;
        struct ivpu_bo *mem;
        u32 entry_count;
        u32 db_id;