accel/ivpu: Implement support for hardware scheduler

Add support for HWS (hardware scheduler). It is disabled by default.
The sched_mode module param can be used to enable it.

Each context has multiple command queues with different priorities and
HWS enables priority based execution on the HW/FW side.

The driver in HWS mode has to send a couple additional messages to
initialize HWS and describe command queue priorities.

Signed-off-by: Wachowski, Karol <karol.wachowski@intel.com>
Signed-off-by: Jacek Lawrynowicz <jacek.lawrynowicz@linux.intel.com>
Link: https://patchwork.freedesktop.org/patch/msgid/20240513120431.3187212-7-jacek.lawrynowicz@linux.intel.com

diff --git a/drivers/accel/ivpu/ivpu_drv.c b/drivers/accel/ivpu/ivpu_drv.c
index 8d80052182f0194aa13a9253fd99f8df037edcf5..ca4fcef7edf5d6298aa9fb9953671a8782543078 100644 (file)
--- a/drivers/accel/ivpu/ivpu_drv.c
+++ b/drivers/accel/ivpu/ivpu_drv.c
@@ -78,7 +78,6 @@ static void file_priv_unbind(struct ivpu_device *vdev, struct ivpu_file_priv *fi
                ivpu_dbg(vdev, FILE, "file_priv unbind: ctx %u\n", file_priv->ctx.id);
 
                ivpu_cmdq_release_all_locked(file_priv);
-               ivpu_jsm_context_release(vdev, file_priv->ctx.id);
                ivpu_bo_unbind_all_bos_from_context(vdev, &file_priv->ctx);
                ivpu_mmu_user_context_fini(vdev, &file_priv->ctx);
                file_priv->bound = false;
@@ -327,6 +326,21 @@ static int ivpu_wait_for_ready(struct ivpu_device *vdev)
        return ret;
 }
 
+static int ivpu_hw_sched_init(struct ivpu_device *vdev)
+{
+       int ret = 0;
+
+       if (vdev->hw->sched_mode == VPU_SCHEDULING_MODE_HW) {
+               ret = ivpu_jsm_hws_setup_priority_bands(vdev);
+               if (ret) {
+                       ivpu_err(vdev, "Failed to enable hw scheduler: %d", ret);
+                       return ret;
+               }
+       }
+
+       return ret;
+}
+
 /**
  * ivpu_boot() - Start VPU firmware
  * @vdev: VPU device
@@ -360,6 +374,10 @@ int ivpu_boot(struct ivpu_device *vdev)
        enable_irq(vdev->irq);
        ivpu_hw_irq_enable(vdev);
        ivpu_ipc_enable(vdev);
+
+       if (ivpu_fw_is_cold_boot(vdev))
+               return ivpu_hw_sched_init(vdev);
+
        return 0;
 }
 

diff --git a/drivers/accel/ivpu/ivpu_fw.c b/drivers/accel/ivpu/ivpu_fw.c
index 29ecf7db238b3d14e01790dd0137e52b3e954924..427cd72bd34fb29953ba81c49713a8251bf350a7 100644 (file)
--- a/drivers/accel/ivpu/ivpu_fw.c
+++ b/drivers/accel/ivpu/ivpu_fw.c
@@ -44,6 +44,8 @@
 #define IVPU_FW_CHECK_API_VER_LT(vdev, fw_hdr, name, major, minor) \
        ivpu_fw_check_api_ver_lt(vdev, fw_hdr, #name, VPU_##name##_API_VER_INDEX, major, minor)
 
+#define IVPU_FOCUS_PRESENT_TIMER_MS 1000
+
 static char *ivpu_firmware;
 module_param_named_unsafe(firmware, ivpu_firmware, charp, 0644);
 MODULE_PARM_DESC(firmware, "NPU firmware binary in /lib/firmware/..");
@@ -467,6 +469,8 @@ static void ivpu_fw_boot_params_print(struct ivpu_device *vdev, struct vpu_boot_
                 boot_params->punit_telemetry_sram_size);
        ivpu_dbg(vdev, FW_BOOT, "boot_params.vpu_telemetry_enable = 0x%x\n",
                 boot_params->vpu_telemetry_enable);
+       ivpu_dbg(vdev, FW_BOOT, "boot_params.vpu_scheduling_mode = 0x%x\n",
+                boot_params->vpu_scheduling_mode);
        ivpu_dbg(vdev, FW_BOOT, "boot_params.dvfs_mode = %u\n",
                 boot_params->dvfs_mode);
        ivpu_dbg(vdev, FW_BOOT, "boot_params.d0i3_delayed_entry = %d\n",
@@ -567,6 +571,9 @@ void ivpu_fw_boot_params_setup(struct ivpu_device *vdev, struct vpu_boot_params
        boot_params->punit_telemetry_sram_base = ivpu_hw_reg_telemetry_offset_get(vdev);
        boot_params->punit_telemetry_sram_size = ivpu_hw_reg_telemetry_size_get(vdev);
        boot_params->vpu_telemetry_enable = ivpu_hw_reg_telemetry_enable_get(vdev);
+       boot_params->vpu_scheduling_mode = vdev->hw->sched_mode;
+       if (vdev->hw->sched_mode == VPU_SCHEDULING_MODE_HW)
+               boot_params->vpu_focus_present_timer_ms = IVPU_FOCUS_PRESENT_TIMER_MS;
        boot_params->dvfs_mode = vdev->fw->dvfs_mode;
        if (!IVPU_WA(disable_d0i3_msg))
                boot_params->d0i3_delayed_entry = 1;

diff --git a/drivers/accel/ivpu/ivpu_job.c b/drivers/accel/ivpu/ivpu_job.c
index 3ef9d8022c9c3fcddbcc65050522494f150b0575..1d7b4388eb3bf6a7f12ebfdacd63ad91c1f84488 100644 (file)
--- a/drivers/accel/ivpu/ivpu_job.c
+++ b/drivers/accel/ivpu/ivpu_job.c
@@ -77,11 +77,10 @@ static void ivpu_preemption_buffers_free(struct ivpu_device *vdev,
        ivpu_bo_free(cmdq->secondary_preempt_buf);
 }
 
-static struct ivpu_cmdq *ivpu_cmdq_alloc(struct ivpu_file_priv *file_priv, u16 engine)
+static struct ivpu_cmdq *ivpu_cmdq_alloc(struct ivpu_file_priv *file_priv)
 {
        struct xa_limit db_xa_limit = {.max = IVPU_MAX_DB, .min = IVPU_MIN_DB};
        struct ivpu_device *vdev = file_priv->vdev;
-       struct vpu_job_queue_header *jobq_header;
        struct ivpu_cmdq *cmdq;
        int ret;
 
@@ -103,16 +102,6 @@ static struct ivpu_cmdq *ivpu_cmdq_alloc(struct ivpu_file_priv *file_priv, u16 e
        if (ret)
                goto err_free_cmdq_mem;
 
-       cmdq->entry_count = (u32)((ivpu_bo_size(cmdq->mem) - sizeof(struct vpu_job_queue_header)) /
-                                 sizeof(struct vpu_job_queue_entry));
-
-       cmdq->jobq = (struct vpu_job_queue *)ivpu_bo_vaddr(cmdq->mem);
-       jobq_header = &cmdq->jobq->header;
-       jobq_header->engine_idx = engine;
-       jobq_header->head = 0;
-       jobq_header->tail = 0;
-       wmb(); /* Flush WC buffer for jobq->header */
-
        return cmdq;
 
 err_free_cmdq_mem:
@@ -135,33 +124,126 @@ static void ivpu_cmdq_free(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *c
        kfree(cmdq);
 }
 
+static int ivpu_hws_cmdq_init(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq, u16 engine,
+                             u8 priority)
+{
+       struct ivpu_device *vdev = file_priv->vdev;
+       int ret;
+
+       ret = ivpu_jsm_hws_create_cmdq(vdev, file_priv->ctx.id, file_priv->ctx.id, cmdq->db_id,
+                                      task_pid_nr(current), engine,
+                                      cmdq->mem->vpu_addr, ivpu_bo_size(cmdq->mem));
+       if (ret)
+               return ret;
+
+       ret = ivpu_jsm_hws_set_context_sched_properties(vdev, file_priv->ctx.id, cmdq->db_id,
+                                                       priority);
+       if (ret)
+               return ret;
+
+       return 0;
+}
+
+static int ivpu_register_db(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq)
+{
+       struct ivpu_device *vdev = file_priv->vdev;
+       int ret;
+
+       if (vdev->hw->sched_mode == VPU_SCHEDULING_MODE_HW)
+               ret = ivpu_jsm_hws_register_db(vdev, file_priv->ctx.id, cmdq->db_id, cmdq->db_id,
+                                              cmdq->mem->vpu_addr, ivpu_bo_size(cmdq->mem));
+       else
+               ret = ivpu_jsm_register_db(vdev, file_priv->ctx.id, cmdq->db_id,
+                                          cmdq->mem->vpu_addr, ivpu_bo_size(cmdq->mem));
+
+       if (!ret)
+               ivpu_dbg(vdev, JOB, "DB %d registered to ctx %d\n", cmdq->db_id, file_priv->ctx.id);
+
+       return ret;
+}
+
+static int
+ivpu_cmdq_init(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq, u16 engine, u8 priority)
+{
+       struct ivpu_device *vdev = file_priv->vdev;
+       struct vpu_job_queue_header *jobq_header;
+       int ret;
+
+       lockdep_assert_held(&file_priv->lock);
+
+       if (cmdq->db_registered)
+               return 0;
+
+       cmdq->entry_count = (u32)((ivpu_bo_size(cmdq->mem) - sizeof(struct vpu_job_queue_header)) /
+                                 sizeof(struct vpu_job_queue_entry));
+
+       cmdq->jobq = (struct vpu_job_queue *)ivpu_bo_vaddr(cmdq->mem);
+       jobq_header = &cmdq->jobq->header;
+       jobq_header->engine_idx = engine;
+       jobq_header->head = 0;
+       jobq_header->tail = 0;
+       wmb(); /* Flush WC buffer for jobq->header */
+
+       if (vdev->hw->sched_mode == VPU_SCHEDULING_MODE_HW) {
+               ret = ivpu_hws_cmdq_init(file_priv, cmdq, engine, priority);
+               if (ret)
+                       return ret;
+       }
+
+       ret = ivpu_register_db(file_priv, cmdq);
+       if (ret)
+               return ret;
+
+       cmdq->db_registered = true;
+
+       return 0;
+}
+
+static int ivpu_cmdq_fini(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq)
+{
+       struct ivpu_device *vdev = file_priv->vdev;
+       int ret;
+
+       lockdep_assert_held(&file_priv->lock);
+
+       if (!cmdq->db_registered)
+               return 0;
+
+       cmdq->db_registered = false;
+
+       if (vdev->hw->sched_mode == VPU_SCHEDULING_MODE_HW) {
+               ret = ivpu_jsm_hws_destroy_cmdq(vdev, file_priv->ctx.id, cmdq->db_id);
+               if (!ret)
+                       ivpu_dbg(vdev, JOB, "Command queue %d destroyed\n", cmdq->db_id);
+       }
+
+       ret = ivpu_jsm_unregister_db(vdev, cmdq->db_id);
+       if (!ret)
+               ivpu_dbg(vdev, JOB, "DB %d unregistered\n", cmdq->db_id);
+
+       return 0;
+}
+
 static struct ivpu_cmdq *ivpu_cmdq_acquire(struct ivpu_file_priv *file_priv, u16 engine,
                                           u8 priority)
 {
        int cmdq_idx = IVPU_CMDQ_INDEX(engine, priority);
        struct ivpu_cmdq *cmdq = file_priv->cmdq[cmdq_idx];
-       struct ivpu_device *vdev = file_priv->vdev;
        int ret;
 
        lockdep_assert_held(&file_priv->lock);
 
        if (!cmdq) {
-               cmdq = ivpu_cmdq_alloc(file_priv, engine);
+               cmdq = ivpu_cmdq_alloc(file_priv);
                if (!cmdq)
                        return NULL;
                file_priv->cmdq[cmdq_idx] = cmdq;
        }
 
-       if (cmdq->db_registered)
-               return cmdq;
-
-       ret = ivpu_jsm_register_db(vdev, file_priv->ctx.id, cmdq->db_id,
-                                  cmdq->mem->vpu_addr, ivpu_bo_size(cmdq->mem));
+       ret = ivpu_cmdq_init(file_priv, cmdq, engine, priority);
        if (ret)
                return NULL;
 
-       cmdq->db_registered = true;
-
        return cmdq;
 }
 
@@ -174,9 +256,7 @@ static void ivpu_cmdq_release_locked(struct ivpu_file_priv *file_priv, u16 engin
 
        if (cmdq) {
                file_priv->cmdq[cmdq_idx] = NULL;
-               if (cmdq->db_registered)
-                       ivpu_jsm_unregister_db(file_priv->vdev, cmdq->db_id);
-
+               ivpu_cmdq_fini(file_priv, cmdq);
                ivpu_cmdq_free(file_priv, cmdq);
        }
 }
@@ -194,36 +274,27 @@ void ivpu_cmdq_release_all_locked(struct ivpu_file_priv *file_priv)
 }
 
 /*
- * Mark the doorbell as unregistered and reset job queue pointers.
+ * Mark the doorbell as unregistered
  * This function needs to be called when the VPU hardware is restarted
  * and FW loses job queue state. The next time job queue is used it
  * will be registered again.
  */
-static void ivpu_cmdq_reset_locked(struct ivpu_file_priv *file_priv, u16 engine, u8 priority)
-{
-       int cmdq_idx = IVPU_CMDQ_INDEX(engine, priority);
-       struct ivpu_cmdq *cmdq = file_priv->cmdq[cmdq_idx];
-
-       lockdep_assert_held(&file_priv->lock);
-
-       if (cmdq) {
-               cmdq->db_registered = false;
-               cmdq->jobq->header.head = 0;
-               cmdq->jobq->header.tail = 0;
-               wmb(); /* Flush WC buffer for jobq header */
-       }
-}
-
-static void ivpu_cmdq_reset_all(struct ivpu_file_priv *file_priv)
+static void ivpu_cmdq_reset(struct ivpu_file_priv *file_priv)
 {
        u16 engine;
        u8 priority;
 
        mutex_lock(&file_priv->lock);
 
-       for (engine = 0; engine < IVPU_NUM_ENGINES; engine++)
-               for (priority = 0; priority < IVPU_NUM_PRIORITIES; priority++)
-                       ivpu_cmdq_reset_locked(file_priv, engine, priority);
+       for (engine = 0; engine < IVPU_NUM_ENGINES; engine++) {
+               for (priority = 0; priority < IVPU_NUM_PRIORITIES; priority++) {
+                       int cmdq_idx = IVPU_CMDQ_INDEX(engine, priority);
+                       struct ivpu_cmdq *cmdq = file_priv->cmdq[cmdq_idx];
+
+                       if (cmdq)
+                               cmdq->db_registered = false;
+               }
+       }
 
        mutex_unlock(&file_priv->lock);
 }
@@ -236,10 +307,9 @@ void ivpu_cmdq_reset_all_contexts(struct ivpu_device *vdev)
        mutex_lock(&vdev->context_list_lock);
 
        xa_for_each(&vdev->context_xa, ctx_id, file_priv)
-               ivpu_cmdq_reset_all(file_priv);
+               ivpu_cmdq_reset(file_priv);
 
        mutex_unlock(&vdev->context_list_lock);
-
 }
 
 static int ivpu_cmdq_push_job(struct ivpu_cmdq *cmdq, struct ivpu_job *job)