accel/ivpu: Implement DCT handling

When host system is under heavy load and the NPU is already running
on the lowest frequency, PUNIT may request Duty Cycle Throttling (DCT).
This will further reduce NPU power usage.

PUNIT requests DCT mode using Survabilty IRQ and mailbox register.
The driver then issues a JSM message to the FW that enables
the DCT mode. If the NPU resets while in DCT mode, the driver request
DCT mode during FW boot.

Also add debugfs "dct" file that allows to set arbitrary DCT percentage,
which is used by driver tests.

Signed-off-by: Jacek Lawrynowicz <jacek.lawrynowicz@linux.intel.com>
Reviewed-by: Wachowski, Karol <karol.wachowski@intel.com>
Link: https://patchwork.freedesktop.org/patch/msgid/20240611120433.1012423-7-jacek.lawrynowicz@linux.intel.com

diff --git a/drivers/accel/ivpu/ivpu_debugfs.c b/drivers/accel/ivpu/ivpu_debugfs.c
index 10d6408..6f86f8d 100644 (file)
--- a/drivers/accel/ivpu/ivpu_debugfs.c
+++ b/drivers/accel/ivpu/ivpu_debugfs.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
- * Copyright (C) 2020-2023 Intel Corporation
+ * Copyright (C) 2020-2024 Intel Corporation
  */
 
 #include <linux/debugfs.h>
@@ -381,6 +381,39 @@ static const struct file_operations ivpu_resume_engine_fops = {
        .write = ivpu_resume_engine_fn,
 };
 
+static int dct_active_get(void *data, u64 *active_percent)
+{
+       struct ivpu_device *vdev = data;
+
+       *active_percent = vdev->pm->dct_active_percent;
+
+       return 0;
+}
+
+static int dct_active_set(void *data, u64 active_percent)
+{
+       struct ivpu_device *vdev = data;
+       int ret;
+
+       if (active_percent > 100)
+               return -EINVAL;
+
+       ret = ivpu_rpm_get(vdev);
+       if (ret)
+               return ret;
+
+       if (active_percent)
+               ret = ivpu_pm_dct_enable(vdev, active_percent);
+       else
+               ret = ivpu_pm_dct_disable(vdev);
+
+       ivpu_rpm_put(vdev);
+
+       return ret;
+}
+
+DEFINE_DEBUGFS_ATTRIBUTE(ivpu_dct_fops, dct_active_get, dct_active_set, "%llu\n");
+
 void ivpu_debugfs_init(struct ivpu_device *vdev)
 {
        struct dentry *debugfs_root = vdev->drm.debugfs_root;
@@ -409,7 +442,9 @@ void ivpu_debugfs_init(struct ivpu_device *vdev)
        debugfs_create_file("resume_engine", 0200, debugfs_root, vdev,
                            &ivpu_resume_engine_fops);
 
-       if (ivpu_hw_ip_gen(vdev) >= IVPU_HW_IP_40XX)
+       if (ivpu_hw_ip_gen(vdev) >= IVPU_HW_IP_40XX) {
                debugfs_create_file("fw_profiling_freq_drive", 0200,
                                    debugfs_root, vdev, &fw_profiling_freq_fops);
+               debugfs_create_file("dct", 0644, debugfs_root, vdev, &ivpu_dct_fops);
+       }
 }

diff --git a/drivers/accel/ivpu/ivpu_drv.c b/drivers/accel/ivpu/ivpu_drv.c
index 3ce12b6..ce91eaf 100644 (file)
--- a/drivers/accel/ivpu/ivpu_drv.c
+++ b/drivers/accel/ivpu/ivpu_drv.c
@@ -391,8 +391,13 @@ int ivpu_boot(struct ivpu_device *vdev)
        ivpu_hw_irq_enable(vdev);
        ivpu_ipc_enable(vdev);
 
-       if (ivpu_fw_is_cold_boot(vdev))
+       if (ivpu_fw_is_cold_boot(vdev)) {
+               ret = ivpu_pm_dct_init(vdev);
+               if (ret)
+                       return ret;
+
                return ivpu_hw_sched_init(vdev);
+       }
 
        return 0;
 }
@@ -482,6 +487,9 @@ static irqreturn_t ivpu_irq_thread_handler(int irq, void *arg)
                case IVPU_HW_IRQ_SRC_MMU_EVTQ:
                        ivpu_context_abort_invalid(vdev);
                        break;
+               case IVPU_HW_IRQ_SRC_DCT:
+                       ivpu_pm_dct_irq_thread_handler(vdev);
+                       break;
                default:
                        ivpu_err_ratelimited(vdev, "Unknown IRQ source: %u\n", irq_src);
                        break;

diff --git a/drivers/accel/ivpu/ivpu_hw.h b/drivers/accel/ivpu/ivpu_hw.h
index 1a55538..1c0c98e 100644 (file)
--- a/drivers/accel/ivpu/ivpu_hw.h
+++ b/drivers/accel/ivpu/ivpu_hw.h
@@ -16,6 +16,7 @@
 
 #define IVPU_HW_IRQ_SRC_IPC 1
 #define IVPU_HW_IRQ_SRC_MMU_EVTQ 2
+#define IVPU_HW_IRQ_SRC_DCT 3
 
 struct ivpu_addr_range {
        resource_size_t start;

diff --git a/drivers/accel/ivpu/ivpu_hw_btrs.c b/drivers/accel/ivpu/ivpu_hw_btrs.c
index 123c1f2..745e524 100644 (file)
--- a/drivers/accel/ivpu/ivpu_hw_btrs.c
+++ b/drivers/accel/ivpu/ivpu_hw_btrs.c
@@ -643,8 +643,11 @@ bool ivpu_hw_btrs_irq_handler_lnl(struct ivpu_device *vdev, int irq)
        if (!status)
                return false;
 
-       if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, SURV_ERR, status))
+       if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, SURV_ERR, status)) {
                ivpu_dbg(vdev, IRQ, "Survivability IRQ\n");
+               if (!kfifo_put(&vdev->hw->irq.fifo, IVPU_HW_IRQ_SRC_DCT))
+                       ivpu_err_ratelimited(vdev, "IRQ FIFO full\n");
+       }
 
        if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, FREQ_CHANGE, status))
                ivpu_dbg(vdev, IRQ, "FREQ_CHANGE irq: %08x", REGB_RD32(VPU_HW_BTRS_LNL_PLL_FREQ));
@@ -694,21 +697,40 @@ bool ivpu_hw_btrs_irq_handler_lnl(struct ivpu_device *vdev, int irq)
        return true;
 }
 
-static void dct_drive_40xx(struct ivpu_device *vdev, u32 dct_val)
+int ivpu_hw_btrs_dct_get_request(struct ivpu_device *vdev, bool *enable)
 {
-       u32 val = REGB_RD32(VPU_HW_BTRS_LNL_PCODE_MAILBOX);
+       u32 val = REGB_RD32(VPU_HW_BTRS_LNL_PCODE_MAILBOX_SHADOW);
+       u32 cmd = REG_GET_FLD(VPU_HW_BTRS_LNL_PCODE_MAILBOX_SHADOW, CMD, val);
+       u32 param1 = REG_GET_FLD(VPU_HW_BTRS_LNL_PCODE_MAILBOX_SHADOW, PARAM1, val);
 
-       val = REG_SET_FLD_NUM(VPU_HW_BTRS_LNL_PCODE_MAILBOX, CMD, DCT_REQ, val);
-       val = REG_SET_FLD_NUM(VPU_HW_BTRS_LNL_PCODE_MAILBOX, PARAM1,
-                             dct_val ? DCT_ENABLE : DCT_DISABLE, val);
-       val = REG_SET_FLD_NUM(VPU_HW_BTRS_LNL_PCODE_MAILBOX, PARAM2, dct_val, val);
+       if (cmd != DCT_REQ) {
+               ivpu_err_ratelimited(vdev, "Unsupported PCODE command: 0x%x\n", cmd);
+               return -EBADR;
+       }
 
-       REGB_WR32(VPU_HW_BTRS_LNL_PCODE_MAILBOX, val);
+       switch (param1) {
+       case DCT_ENABLE:
+               *enable = true;
+               return 0;
+       case DCT_DISABLE:
+               *enable = false;
+               return 0;
+       default:
+               ivpu_err_ratelimited(vdev, "Invalid PARAM1 value: %u\n", param1);
+               return -EINVAL;
+       }
 }
 
-void ivpu_hw_btrs_dct_drive(struct ivpu_device *vdev, u32 dct_val)
+void ivpu_hw_btrs_dct_set_status(struct ivpu_device *vdev, bool enable, u32 active_percent)
 {
-       return dct_drive_40xx(vdev, dct_val);
+       u32 val = 0;
+       u32 cmd = enable ? DCT_ENABLE : DCT_DISABLE;
+
+       val = REG_SET_FLD_NUM(VPU_HW_BTRS_LNL_PCODE_MAILBOX_STATUS, CMD, DCT_REQ, val);
+       val = REG_SET_FLD_NUM(VPU_HW_BTRS_LNL_PCODE_MAILBOX_STATUS, PARAM1, cmd, val);
+       val = REG_SET_FLD_NUM(VPU_HW_BTRS_LNL_PCODE_MAILBOX_STATUS, PARAM2, active_percent, val);
+
+       REGB_WR32(VPU_HW_BTRS_LNL_PCODE_MAILBOX_STATUS, val);
 }
 
 static u32 pll_ratio_to_freq_mtl(u32 ratio, u32 config)

diff --git a/drivers/accel/ivpu/ivpu_hw_btrs.h b/drivers/accel/ivpu/ivpu_hw_btrs.h
index b3e3ae2..04f14f5 100644 (file)
--- a/drivers/accel/ivpu/ivpu_hw_btrs.h
+++ b/drivers/accel/ivpu/ivpu_hw_btrs.h
@@ -15,6 +15,9 @@
 #define PLL_PROFILING_FREQ_HIGH      400000000
 #define PLL_RATIO_TO_FREQ(x)         ((x) * PLL_REF_CLK_FREQ)
 
+#define DCT_DEFAULT_ACTIVE_PERCENT 15u
+#define DCT_PERIOD_US             35300u
+
 int ivpu_hw_btrs_info_init(struct ivpu_device *vdev);
 void ivpu_hw_btrs_freq_ratios_init(struct ivpu_device *vdev);
 int ivpu_hw_btrs_irqs_clear_with_0_mtl(struct ivpu_device *vdev);
@@ -31,7 +34,8 @@ void ivpu_hw_btrs_ats_print_lnl(struct ivpu_device *vdev);
 void ivpu_hw_btrs_clock_relinquish_disable_lnl(struct ivpu_device *vdev);
 bool ivpu_hw_btrs_irq_handler_mtl(struct ivpu_device *vdev, int irq);
 bool ivpu_hw_btrs_irq_handler_lnl(struct ivpu_device *vdev, int irq);
-void ivpu_hw_btrs_dct_drive(struct ivpu_device *vdev, u32 dct_val);
+int ivpu_hw_btrs_dct_get_request(struct ivpu_device *vdev, bool *enable);
+void ivpu_hw_btrs_dct_set_status(struct ivpu_device *vdev, bool enable, u32 dct_percent);
 u32 ivpu_hw_btrs_pll_freq_get(struct ivpu_device *vdev);
 u32 ivpu_hw_btrs_ratio_to_freq(struct ivpu_device *vdev, u32 ratio);
 u32 ivpu_hw_btrs_telemetry_offset_get(struct ivpu_device *vdev);

diff --git a/drivers/accel/ivpu/ivpu_hw_btrs_lnl_reg.h b/drivers/accel/ivpu/ivpu_hw_btrs_lnl_reg.h
index 93733bd..fc51f30 100644 (file)
--- a/drivers/accel/ivpu/ivpu_hw_btrs_lnl_reg.h
+++ b/drivers/accel/ivpu/ivpu_hw_btrs_lnl_reg.h
@@ -44,11 +44,11 @@
 #define VPU_HW_BTRS_LNL_IMR_ERR_CFI1_HIGH                      0x0000005cu
 #define VPU_HW_BTRS_LNL_IMR_ERR_CFI1_CLEAR                     0x00000060u
 
-#define VPU_HW_BTRS_LNL_PCODE_MAILBOX                          0x00000070u
-#define VPU_HW_BTRS_LNL_PCODE_MAILBOX_CMD_MASK                 GENMASK(7, 0)
-#define VPU_HW_BTRS_LNL_PCODE_MAILBOX_PARAM1_MASK              GENMASK(15, 8)
-#define VPU_HW_BTRS_LNL_PCODE_MAILBOX_PARAM2_MASK              GENMASK(23, 16)
-#define VPU_HW_BTRS_LNL_PCODE_MAILBOX_PARAM3_MASK              GENMASK(31, 24)
+#define VPU_HW_BTRS_LNL_PCODE_MAILBOX_STATUS                   0x00000070u
+#define VPU_HW_BTRS_LNL_PCODE_MAILBOX_STATUS_CMD_MASK          GENMASK(7, 0)
+#define VPU_HW_BTRS_LNL_PCODE_MAILBOX_STATUS_PARAM1_MASK       GENMASK(15, 8)
+#define VPU_HW_BTRS_LNL_PCODE_MAILBOX_STATUS_PARAM2_MASK       GENMASK(23, 16)
+#define VPU_HW_BTRS_LNL_PCODE_MAILBOX_STATUS_PARAM3_MASK       GENMASK(31, 24)
 
 #define VPU_HW_BTRS_LNL_PCODE_MAILBOX_SHADOW                   0x00000074u
 #define VPU_HW_BTRS_LNL_PCODE_MAILBOX_SHADOW_CMD_MASK          GENMASK(7, 0)

diff --git a/drivers/accel/ivpu/ivpu_jsm_msg.c b/drivers/accel/ivpu/ivpu_jsm_msg.c
index 3179e80..216aa61 100644 (file)
--- a/drivers/accel/ivpu/ivpu_jsm_msg.c
+++ b/drivers/accel/ivpu/ivpu_jsm_msg.c
@@ -543,3 +543,26 @@ int ivpu_jsm_metric_streamer_info(struct ivpu_device *vdev, u64 metric_group_mas
 
        return ret;
 }
+
+int ivpu_jsm_dct_enable(struct ivpu_device *vdev, u32 active_us, u32 inactive_us)
+{
+       struct vpu_jsm_msg req = { .type = VPU_JSM_MSG_DCT_ENABLE };
+       struct vpu_jsm_msg resp;
+
+       req.payload.pwr_dct_control.dct_active_us = active_us;
+       req.payload.pwr_dct_control.dct_inactive_us = inactive_us;
+
+       return ivpu_ipc_send_receive_active(vdev, &req, VPU_JSM_MSG_DCT_ENABLE_DONE,
+                                           &resp, VPU_IPC_CHAN_ASYNC_CMD,
+                                           vdev->timeout.jsm);
+}
+
+int ivpu_jsm_dct_disable(struct ivpu_device *vdev)
+{
+       struct vpu_jsm_msg req = { .type = VPU_JSM_MSG_DCT_DISABLE };
+       struct vpu_jsm_msg resp;
+
+       return ivpu_ipc_send_receive_active(vdev, &req, VPU_JSM_MSG_DCT_DISABLE_DONE,
+                                           &resp, VPU_IPC_CHAN_ASYNC_CMD,
+                                           vdev->timeout.jsm);
+}

diff --git a/drivers/accel/ivpu/ivpu_jsm_msg.h b/drivers/accel/ivpu/ivpu_jsm_msg.h
index 0603634..e4e42c0 100644 (file)
--- a/drivers/accel/ivpu/ivpu_jsm_msg.h
+++ b/drivers/accel/ivpu/ivpu_jsm_msg.h
@@ -41,4 +41,6 @@ int ivpu_jsm_metric_streamer_update(struct ivpu_device *vdev, u64 metric_group_m
                                    u64 buffer_addr, u64 buffer_size, u64 *bytes_written);
 int ivpu_jsm_metric_streamer_info(struct ivpu_device *vdev, u64 metric_group_mask, u64 buffer_addr,
                                  u64 buffer_size, u32 *sample_size, u64 *info_size);
+int ivpu_jsm_dct_enable(struct ivpu_device *vdev, u32 active_us, u32 inactive_us);
+int ivpu_jsm_dct_disable(struct ivpu_device *vdev);
 #endif

diff --git a/drivers/accel/ivpu/ivpu_pm.c b/drivers/accel/ivpu/ivpu_pm.c
index 9d5f500..602fa4e 100644 (file)
--- a/drivers/accel/ivpu/ivpu_pm.c
+++ b/drivers/accel/ivpu/ivpu_pm.c
@@ -245,7 +245,7 @@ int ivpu_pm_runtime_suspend_cb(struct device *dev)
 
        ivpu_dbg(vdev, PM, "Runtime suspend..\n");
 
-       is_idle = ivpu_hw_is_idle(vdev);
+       is_idle = ivpu_hw_is_idle(vdev) || vdev->pm->dct_active_percent;
        if (!is_idle)
                ivpu_err(vdev, "NPU is not idle before autosuspend\n");
 
@@ -397,3 +397,68 @@ void ivpu_pm_disable(struct ivpu_device *vdev)
        pm_runtime_get_noresume(vdev->drm.dev);
        pm_runtime_forbid(vdev->drm.dev);
 }
+
+int ivpu_pm_dct_init(struct ivpu_device *vdev)
+{
+       if (vdev->pm->dct_active_percent)
+               return ivpu_pm_dct_enable(vdev, vdev->pm->dct_active_percent);
+
+       return 0;
+}
+
+int ivpu_pm_dct_enable(struct ivpu_device *vdev, u8 active_percent)
+{
+       u32 active_us, inactive_us;
+       int ret;
+
+       if (active_percent == 0 || active_percent > 100)
+               return -EINVAL;
+
+       active_us = (DCT_PERIOD_US * active_percent) / 100;
+       inactive_us = DCT_PERIOD_US - active_us;
+
+       ret = ivpu_jsm_dct_enable(vdev, active_us, inactive_us);
+       if (ret) {
+               ivpu_err_ratelimited(vdev, "Filed to enable DCT: %d\n", ret);
+               return ret;
+       }
+
+       vdev->pm->dct_active_percent = active_percent;
+
+       ivpu_dbg(vdev, PM, "DCT set to %u%% (D0: %uus, D0i2: %uus)\n",
+                active_percent, active_us, inactive_us);
+       return 0;
+}
+
+int ivpu_pm_dct_disable(struct ivpu_device *vdev)
+{
+       int ret;
+
+       ret = ivpu_jsm_dct_disable(vdev);
+       if (ret) {
+               ivpu_err_ratelimited(vdev, "Filed to disable DCT: %d\n", ret);
+               return ret;
+       }
+
+       vdev->pm->dct_active_percent = 0;
+
+       ivpu_dbg(vdev, PM, "DCT disabled\n");
+       return 0;
+}
+
+void ivpu_pm_dct_irq_thread_handler(struct ivpu_device *vdev)
+{
+       bool enable;
+       int ret;
+
+       if (ivpu_hw_btrs_dct_get_request(vdev, &enable))
+               return;
+
+       if (vdev->pm->dct_active_percent)
+               ret = ivpu_pm_dct_enable(vdev, DCT_DEFAULT_ACTIVE_PERCENT);
+       else
+               ret = ivpu_pm_dct_disable(vdev);
+
+       if (!ret)
+               ivpu_hw_btrs_dct_set_status(vdev, enable, vdev->pm->dct_active_percent);
+}

diff --git a/drivers/accel/ivpu/ivpu_pm.h b/drivers/accel/ivpu/ivpu_pm.h
index e524412..c08a3dd 100644 (file)
--- a/drivers/accel/ivpu/ivpu_pm.h
+++ b/drivers/accel/ivpu/ivpu_pm.h
@@ -19,6 +19,7 @@ struct ivpu_pm_info {
        atomic_t reset_counter;
        atomic_t reset_pending;
        bool is_warmboot;
+       u8 dct_active_percent;
 };
 
 void ivpu_pm_init(struct ivpu_device *vdev);
@@ -42,4 +43,9 @@ void ivpu_pm_trigger_recovery(struct ivpu_device *vdev, const char *reason);
 void ivpu_start_job_timeout_detection(struct ivpu_device *vdev);
 void ivpu_stop_job_timeout_detection(struct ivpu_device *vdev);
 
+int ivpu_pm_dct_init(struct ivpu_device *vdev);
+int ivpu_pm_dct_enable(struct ivpu_device *vdev, u8 active_percent);
+int ivpu_pm_dct_disable(struct ivpu_device *vdev);
+void ivpu_pm_dct_irq_thread_handler(struct ivpu_device *vdev);
+
 #endif /* __IVPU_PM_H__ */