accel/ivpu: Implement DCT handling

When host system is under heavy load and the NPU is already running
on the lowest frequency, PUNIT may request Duty Cycle Throttling (DCT).
This will further reduce NPU power usage.

PUNIT requests DCT mode using Survabilty IRQ and mailbox register.
The driver then issues a JSM message to the FW that enables
the DCT mode. If the NPU resets while in DCT mode, the driver request
DCT mode during FW boot.

Also add debugfs "dct" file that allows to set arbitrary DCT percentage,
which is used by driver tests.

Signed-off-by: Jacek Lawrynowicz <jacek.lawrynowicz@linux.intel.com>
Reviewed-by: Wachowski, Karol <karol.wachowski@intel.com>
Link: https://patchwork.freedesktop.org/patch/msgid/20240611120433.1012423-7-jacek.lawrynowicz@linux.intel.com

diff --git a/drivers/accel/ivpu/ivpu_debugfs.c b/drivers/accel/ivpu/ivpu_debugfs.c
index 10d6408c98317eac88ce457cd060cf2a188a7c8e..6f86f8df30db0fc263f13a7a4bc98d9e1d3d687b 100644 (file)
--- a/drivers/accel/ivpu/ivpu_debugfs.c
+++ b/drivers/accel/ivpu/ivpu_debugfs.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
- * Copyright (C) 2020-2023 Intel Corporation
+ * Copyright (C) 2020-2024 Intel Corporation
  */
 
 #include <linux/debugfs.h>
@@ -381,6 +381,39 @@ static const struct file_operations ivpu_resume_engine_fops = {
        .write = ivpu_resume_engine_fn,
 };
 
+static int dct_active_get(void *data, u64 *active_percent)
+{
+       struct ivpu_device *vdev = data;
+
+       *active_percent = vdev->pm->dct_active_percent;
+
+       return 0;
+}
+
+static int dct_active_set(void *data, u64 active_percent)
+{
+       struct ivpu_device *vdev = data;
+       int ret;
+
+       if (active_percent > 100)
+               return -EINVAL;
+
+       ret = ivpu_rpm_get(vdev);
+       if (ret)
+               return ret;
+
+       if (active_percent)
+               ret = ivpu_pm_dct_enable(vdev, active_percent);
+       else
+               ret = ivpu_pm_dct_disable(vdev);
+
+       ivpu_rpm_put(vdev);
+
+       return ret;
+}
+
+DEFINE_DEBUGFS_ATTRIBUTE(ivpu_dct_fops, dct_active_get, dct_active_set, "%llu\n");
+
 void ivpu_debugfs_init(struct ivpu_device *vdev)
 {
        struct dentry *debugfs_root = vdev->drm.debugfs_root;
@@ -409,7 +442,9 @@ void ivpu_debugfs_init(struct ivpu_device *vdev)
        debugfs_create_file("resume_engine", 0200, debugfs_root, vdev,
                            &ivpu_resume_engine_fops);
 
-       if (ivpu_hw_ip_gen(vdev) >= IVPU_HW_IP_40XX)
+       if (ivpu_hw_ip_gen(vdev) >= IVPU_HW_IP_40XX) {
                debugfs_create_file("fw_profiling_freq_drive", 0200,
                                    debugfs_root, vdev, &fw_profiling_freq_fops);
+               debugfs_create_file("dct", 0644, debugfs_root, vdev, &ivpu_dct_fops);
+       }
 }

diff --git a/drivers/accel/ivpu/ivpu_drv.c b/drivers/accel/ivpu/ivpu_drv.c
index 3ce12b66f699293bfb1fefe730bf9502bb36eb19..ce91eafe5746f7510caf188a02b334a054f991df 100644 (file)
--- a/drivers/accel/ivpu/ivpu_drv.c
+++ b/drivers/accel/ivpu/ivpu_drv.c
@@ -391,8 +391,13 @@ int ivpu_boot(struct ivpu_device *vdev)
        ivpu_hw_irq_enable(vdev);
        ivpu_ipc_enable(vdev);
 
-       if (ivpu_fw_is_cold_boot(vdev))
+       if (ivpu_fw_is_cold_boot(vdev)) {
+               ret = ivpu_pm_dct_init(vdev);
+               if (ret)
+                       return ret;
+
                return ivpu_hw_sched_init(vdev);
+       }
 
        return 0;
 }
@@ -482,6 +487,9 @@ static irqreturn_t ivpu_irq_thread_handler(int irq, void *arg)
                case IVPU_HW_IRQ_SRC_MMU_EVTQ:
                        ivpu_context_abort_invalid(vdev);
                        break;
+               case IVPU_HW_IRQ_SRC_DCT:
+                       ivpu_pm_dct_irq_thread_handler(vdev);
+                       break;
                default:
                        ivpu_err_ratelimited(vdev, "Unknown IRQ source: %u\n", irq_src);
                        break;

diff --git a/drivers/accel/ivpu/ivpu_hw.h b/drivers/accel/ivpu/ivpu_hw.h
index 1a55538aafdcdd22828e580b454b78c61b3814b4..1c0c98e3afb88d78fc514989cd8c7ca09d83737b 100644 (file)
--- a/drivers/accel/ivpu/ivpu_hw.h
+++ b/drivers/accel/ivpu/ivpu_hw.h
@@ -16,6 +16,7 @@
 
 #define IVPU_HW_IRQ_SRC_IPC 1
 #define IVPU_HW_IRQ_SRC_MMU_EVTQ 2
+#define IVPU_HW_IRQ_SRC_DCT 3
 
 struct ivpu_addr_range {
        resource_size_t start;

diff --git a/drivers/accel/ivpu/ivpu_hw_btrs.c b/drivers/accel/ivpu/ivpu_hw_btrs.c
index 123c1f2fd554b127f87d4d1166e87838539fd31c..745e5248803daf66d652eda98876e825ff922067 100644 (file)
--- a/drivers/accel/ivpu/ivpu_hw_btrs.c
+++ b/drivers/accel/ivpu/ivpu_hw_btrs.c
@@ -643,8 +643,11 @@ bool ivpu_hw_btrs_irq_handler_lnl(struct ivpu_device *vdev, int irq)
        if (!status)
                return false;
 
-       if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, SURV_ERR, status))
+       if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, SURV_ERR, status)) {
                ivpu_dbg(vdev, IRQ, "Survivability IRQ\n");
+               if (!kfifo_put(&vdev->hw->irq.fifo, IVPU_HW_IRQ_SRC_DCT))
+                       ivpu_err_ratelimited(vdev, "IRQ FIFO full\n");
+       }
 
        if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, FREQ_CHANGE, status))
                ivpu_dbg(vdev, IRQ, "FREQ_CHANGE irq: %08x", REGB_RD32(VPU_HW_BTRS_LNL_PLL_FREQ));
@@ -694,21 +697,40 @@ bool ivpu_hw_btrs_irq_handler_lnl(struct ivpu_device *vdev, int irq)
        return true;
 }
 
-static void dct_drive_40xx(struct ivpu_device *vdev, u32 dct_val)
+int ivpu_hw_btrs_dct_get_request(struct ivpu_device *vdev, bool *enable)
 {
-       u32 val = REGB_RD32(VPU_HW_BTRS_LNL_PCODE_MAILBOX);
+       u32 val = REGB_RD32(VPU_HW_BTRS_LNL_PCODE_MAILBOX_SHADOW);
+       u32 cmd = REG_GET_FLD(VPU_HW_BTRS_LNL_PCODE_MAILBOX_SHADOW, CMD, val);
+       u32 param1 = REG_GET_FLD(VPU_HW_BTRS_LNL_PCODE_MAILBOX_SHADOW, PARAM1, val);
 
-       val = REG_SET_FLD_NUM(VPU_HW_BTRS_LNL_PCODE_MAILBOX, CMD, DCT_REQ, val);
-       val = REG_SET_FLD_NUM(VPU_HW_BTRS_LNL_PCODE_MAILBOX, PARAM1,
-                             dct_val ? DCT_ENABLE : DCT_DISABLE, val);
-       val = REG_SET_FLD_NUM(VPU_HW_BTRS_LNL_PCODE_MAILBOX, PARAM2, dct_val, val);
+       if (cmd != DCT_REQ) {
+               ivpu_err_ratelimited(vdev, "Unsupported PCODE command: 0x%x\n", cmd);
+               return -EBADR;
+       }
 
-       REGB_WR32(VPU_HW_BTRS_LNL_PCODE_MAILBOX, val);
+       switch (param1) {
+       case DCT_ENABLE:
+               *enable = true;
+               return 0;
+       case DCT_DISABLE:
+               *enable = false;
+               return 0;
+       default:
+               ivpu_err_ratelimited(vdev, "Invalid PARAM1 value: %u\n", param1);
+               return -EINVAL;
+       }
 }
 
-void ivpu_hw_btrs_dct_drive(struct ivpu_device *vdev, u32 dct_val)
+void ivpu_hw_btrs_dct_set_status(struct ivpu_device *vdev, bool enable, u32 active_percent)
 {
-       return dct_drive_40xx(vdev, dct_val);
+       u32 val = 0;
+       u32 cmd = enable ? DCT_ENABLE : DCT_DISABLE;
+
+       val = REG_SET_FLD_NUM(VPU_HW_BTRS_LNL_PCODE_MAILBOX_STATUS, CMD, DCT_REQ, val);
+       val = REG_SET_FLD_NUM(VPU_HW_BTRS_LNL_PCODE_MAILBOX_STATUS, PARAM1, cmd, val);
+       val = REG_SET_FLD_NUM(VPU_HW_BTRS_LNL_PCODE_MAILBOX_STATUS, PARAM2, active_percent, val);
+
+       REGB_WR32(VPU_HW_BTRS_LNL_PCODE_MAILBOX_STATUS, val);
 }
 
 static u32 pll_ratio_to_freq_mtl(u32 ratio, u32 config)

diff --git a/drivers/accel/ivpu/ivpu_hw_btrs.h b/drivers/accel/ivpu/ivpu_hw_btrs.h
index b3e3ae2aa578411ce39bfb3e84e65d77a1388a76..04f14f50fed62e1e9c1608987c576c0d02d0b177 100644 (file)
--- a/drivers/accel/ivpu/ivpu_hw_btrs.h
+++ b/drivers/accel/ivpu/ivpu_hw_btrs.h
@@ -15,6 +15,9 @@
 #define PLL_PROFILING_FREQ_HIGH      400000000
 #define PLL_RATIO_TO_FREQ(x)         ((x) * PLL_REF_CLK_FREQ)
 
+#define DCT_DEFAULT_ACTIVE_PERCENT 15u
+#define DCT_PERIOD_US             35300u
+
 int ivpu_hw_btrs_info_init(struct ivpu_device *vdev);
 void ivpu_hw_btrs_freq_ratios_init(struct ivpu_device *vdev);
 int ivpu_hw_btrs_irqs_clear_with_0_mtl(struct ivpu_device *vdev);
@@ -31,7 +34,8 @@ void ivpu_hw_btrs_ats_print_lnl(struct ivpu_device *vdev);
 void ivpu_hw_btrs_clock_relinquish_disable_lnl(struct ivpu_device *vdev);
 bool ivpu_hw_btrs_irq_handler_mtl(struct ivpu_device *vdev, int irq);
 bool ivpu_hw_btrs_irq_handler_lnl(struct ivpu_device *vdev, int irq);
-void ivpu_hw_btrs_dct_drive(struct ivpu_device *vdev, u32 dct_val);
+int ivpu_hw_btrs_dct_get_request(struct ivpu_device *vdev, bool *enable);
+void ivpu_hw_btrs_dct_set_status(struct ivpu_device *vdev, bool enable, u32 dct_percent);
 u32 ivpu_hw_btrs_pll_freq_get(struct ivpu_device *vdev);
 u32 ivpu_hw_btrs_ratio_to_freq(struct ivpu_device *vdev, u32 ratio);
 u32 ivpu_hw_btrs_telemetry_offset_get(struct ivpu_device *vdev);

diff --git a/drivers/accel/ivpu/ivpu_hw_btrs_lnl_reg.h b/drivers/accel/ivpu/ivpu_hw_btrs_lnl_reg.h
index 93733bde02b0de4daf13cdeff1127f8d9af0c68f..fc51f3098f97895a24665565fe03cfa4256b28e5 100644 (file)
--- a/drivers/accel/ivpu/ivpu_hw_btrs_lnl_reg.h
+++ b/drivers/accel/ivpu/ivpu_hw_btrs_lnl_reg.h
@@ -44,11 +44,11 @@
 #define VPU_HW_BTRS_LNL_IMR_ERR_CFI1_HIGH                      0x0000005cu
 #define VPU_HW_BTRS_LNL_IMR_ERR_CFI1_CLEAR                     0x00000060u
 
-#define VPU_HW_BTRS_LNL_PCODE_MAILBOX                          0x00000070u
-#define VPU_HW_BTRS_LNL_PCODE_MAILBOX_CMD_MASK                 GENMASK(7, 0)
-#define VPU_HW_BTRS_LNL_PCODE_MAILBOX_PARAM1_MASK              GENMASK(15, 8)
-#define VPU_HW_BTRS_LNL_PCODE_MAILBOX_PARAM2_MASK              GENMASK(23, 16)
-#define VPU_HW_BTRS_LNL_PCODE_MAILBOX_PARAM3_MASK              GENMASK(31, 24)
+#define VPU_HW_BTRS_LNL_PCODE_MAILBOX_STATUS                   0x00000070u
+#define VPU_HW_BTRS_LNL_PCODE_MAILBOX_STATUS_CMD_MASK          GENMASK(7, 0)
+#define VPU_HW_BTRS_LNL_PCODE_MAILBOX_STATUS_PARAM1_MASK       GENMASK(15, 8)
+#define VPU_HW_BTRS_LNL_PCODE_MAILBOX_STATUS_PARAM2_MASK       GENMASK(23, 16)
+#define VPU_HW_BTRS_LNL_PCODE_MAILBOX_STATUS_PARAM3_MASK       GENMASK(31, 24)
 
 #define VPU_HW_BTRS_LNL_PCODE_MAILBOX_SHADOW                   0x00000074u
 #define VPU_HW_BTRS_LNL_PCODE_MAILBOX_SHADOW_CMD_MASK          GENMASK(7, 0)

diff --git a/drivers/accel/ivpu/ivpu_jsm_msg.c b/drivers/accel/ivpu/ivpu_jsm_msg.c
index 3179e80de1ec220bccde5e729cb86d5f0393332c..216aa61ba2096cc552e8bdeea604b13bf82f21e0 100644 (file)
--- a/drivers/accel/ivpu/ivpu_jsm_msg.c
+++ b/drivers/accel/ivpu/ivpu_jsm_msg.c
@@ -543,3 +543,26 @@ int ivpu_jsm_metric_streamer_info(struct ivpu_device *vdev, u64 metric_group_mas
 
        return ret;
 }
+
+int ivpu_jsm_dct_enable(struct ivpu_device *vdev, u32 active_us, u32 inactive_us)
+{
+       struct vpu_jsm_msg req = { .type = VPU_JSM_MSG_DCT_ENABLE };
+       struct vpu_jsm_msg resp;
+
+       req.payload.pwr_dct_control.dct_active_us = active_us;
+       req.payload.pwr_dct_control.dct_inactive_us = inactive_us;
+
+       return ivpu_ipc_send_receive_active(vdev, &req, VPU_JSM_MSG_DCT_ENABLE_DONE,
+                                           &resp, VPU_IPC_CHAN_ASYNC_CMD,
+                                           vdev->timeout.jsm);
+}
+
+int ivpu_jsm_dct_disable(struct ivpu_device *vdev)
+{
+       struct vpu_jsm_msg req = { .type = VPU_JSM_MSG_DCT_DISABLE };
+       struct vpu_jsm_msg resp;
+
+       return ivpu_ipc_send_receive_active(vdev, &req, VPU_JSM_MSG_DCT_DISABLE_DONE,
+                                           &resp, VPU_IPC_CHAN_ASYNC_CMD,
+                                           vdev->timeout.jsm);
+}

diff --git a/drivers/accel/ivpu/ivpu_jsm_msg.h b/drivers/accel/ivpu/ivpu_jsm_msg.h
index 060363409fb3e22b682df091e60dfbff805325a6..e4e42c0ff6e65614b36eb7ff52395d6628d9b0e1 100644 (file)
--- a/drivers/accel/ivpu/ivpu_jsm_msg.h
+++ b/drivers/accel/ivpu/ivpu_jsm_msg.h
@@ -41,4 +41,6 @@ int ivpu_jsm_metric_streamer_update(struct ivpu_device *vdev, u64 metric_group_m
                                    u64 buffer_addr, u64 buffer_size, u64 *bytes_written);
 int ivpu_jsm_metric_streamer_info(struct ivpu_device *vdev, u64 metric_group_mask, u64 buffer_addr,
                                  u64 buffer_size, u32 *sample_size, u64 *info_size);
+int ivpu_jsm_dct_enable(struct ivpu_device *vdev, u32 active_us, u32 inactive_us);
+int ivpu_jsm_dct_disable(struct ivpu_device *vdev);
 #endif

diff --git a/drivers/accel/ivpu/ivpu_pm.c b/drivers/accel/ivpu/ivpu_pm.c
index 9d5f500afd200a9f6574480153731022497a167a..602fa4e65c226c38ed0b0ebf0d93baf23f1bc010 100644 (file)
--- a/drivers/accel/ivpu/ivpu_pm.c
+++ b/drivers/accel/ivpu/ivpu_pm.c
@@ -245,7 +245,7 @@ int ivpu_pm_runtime_suspend_cb(struct device *dev)
 
        ivpu_dbg(vdev, PM, "Runtime suspend..\n");
 
-       is_idle = ivpu_hw_is_idle(vdev);
+       is_idle = ivpu_hw_is_idle(vdev) || vdev->pm->dct_active_percent;
        if (!is_idle)
                ivpu_err(vdev, "NPU is not idle before autosuspend\n");
 
@@ -397,3 +397,68 @@ void ivpu_pm_disable(struct ivpu_device *vdev)
        pm_runtime_get_noresume(vdev->drm.dev);
        pm_runtime_forbid(vdev->drm.dev);
 }
+
+int ivpu_pm_dct_init(struct ivpu_device *vdev)
+{
+       if (vdev->pm->dct_active_percent)
+               return ivpu_pm_dct_enable(vdev, vdev->pm->dct_active_percent);
+
+       return 0;
+}
+
+int ivpu_pm_dct_enable(struct ivpu_device *vdev, u8 active_percent)
+{
+       u32 active_us, inactive_us;
+       int ret;
+
+       if (active_percent == 0 || active_percent > 100)
+               return -EINVAL;
+
+       active_us = (DCT_PERIOD_US * active_percent) / 100;
+       inactive_us = DCT_PERIOD_US - active_us;
+
+       ret = ivpu_jsm_dct_enable(vdev, active_us, inactive_us);
+       if (ret) {
+               ivpu_err_ratelimited(vdev, "Filed to enable DCT: %d\n", ret);
+               return ret;
+       }
+
+       vdev->pm->dct_active_percent = active_percent;
+
+       ivpu_dbg(vdev, PM, "DCT set to %u%% (D0: %uus, D0i2: %uus)\n",
+                active_percent, active_us, inactive_us);
+       return 0;
+}
+
+int ivpu_pm_dct_disable(struct ivpu_device *vdev)
+{
+       int ret;
+
+       ret = ivpu_jsm_dct_disable(vdev);
+       if (ret) {
+               ivpu_err_ratelimited(vdev, "Filed to disable DCT: %d\n", ret);
+               return ret;
+       }
+
+       vdev->pm->dct_active_percent = 0;
+
+       ivpu_dbg(vdev, PM, "DCT disabled\n");
+       return 0;
+}
+
+void ivpu_pm_dct_irq_thread_handler(struct ivpu_device *vdev)
+{
+       bool enable;
+       int ret;
+
+       if (ivpu_hw_btrs_dct_get_request(vdev, &enable))
+               return;
+
+       if (vdev->pm->dct_active_percent)
+               ret = ivpu_pm_dct_enable(vdev, DCT_DEFAULT_ACTIVE_PERCENT);
+       else
+               ret = ivpu_pm_dct_disable(vdev);
+
+       if (!ret)
+               ivpu_hw_btrs_dct_set_status(vdev, enable, vdev->pm->dct_active_percent);
+}

diff --git a/drivers/accel/ivpu/ivpu_pm.h b/drivers/accel/ivpu/ivpu_pm.h
index e524412765be4a5accf95773b6f1cb84ff227169..c08a3ddf09c153ee5e709b4a7df21ff9d8678c72 100644 (file)
--- a/drivers/accel/ivpu/ivpu_pm.h
+++ b/drivers/accel/ivpu/ivpu_pm.h
@@ -19,6 +19,7 @@ struct ivpu_pm_info {
        atomic_t reset_counter;
        atomic_t reset_pending;
        bool is_warmboot;
+       u8 dct_active_percent;
 };
 
 void ivpu_pm_init(struct ivpu_device *vdev);
@@ -42,4 +43,9 @@ void ivpu_pm_trigger_recovery(struct ivpu_device *vdev, const char *reason);
 void ivpu_start_job_timeout_detection(struct ivpu_device *vdev);
 void ivpu_stop_job_timeout_detection(struct ivpu_device *vdev);
 
+int ivpu_pm_dct_init(struct ivpu_device *vdev);
+int ivpu_pm_dct_enable(struct ivpu_device *vdev, u8 active_percent);
+int ivpu_pm_dct_disable(struct ivpu_device *vdev);
+void ivpu_pm_dct_irq_thread_handler(struct ivpu_device *vdev);
+
 #endif /* __IVPU_PM_H__ */