1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (c) 2012-2016, The Linux Foundation. All rights reserved.
4 * Copyright (C) 2017 Linaro Ltd.
7 #include <linux/delay.h>
8 #include <linux/device.h>
9 #include <linux/dma-mapping.h>
10 #include <linux/interrupt.h>
11 #include <linux/iopoll.h>
12 #include <linux/kernel.h>
13 #include <linux/slab.h>
18 #include "hfi_venus.h"
19 #include "hfi_venus_io.h"
22 #define HFI_MASK_QHDR_TX_TYPE 0xff000000
23 #define HFI_MASK_QHDR_RX_TYPE 0x00ff0000
24 #define HFI_MASK_QHDR_PRI_TYPE 0x0000ff00
25 #define HFI_MASK_QHDR_ID_TYPE 0x000000ff
27 #define HFI_HOST_TO_CTRL_CMD_Q 0
28 #define HFI_CTRL_TO_HOST_MSG_Q 1
29 #define HFI_CTRL_TO_HOST_DBG_Q 2
30 #define HFI_MASK_QHDR_STATUS 0x000000ff
33 #define IFACEQ_CMD_IDX 0
34 #define IFACEQ_MSG_IDX 1
35 #define IFACEQ_DBG_IDX 2
36 #define IFACEQ_MAX_BUF_COUNT 50
37 #define IFACEQ_MAX_PARALLEL_CLNTS 16
38 #define IFACEQ_DFLT_QHDR 0x01010000
40 #define POLL_INTERVAL_US 50
42 #define IFACEQ_MAX_PKT_SIZE 1024
43 #define IFACEQ_MED_PKT_SIZE 768
44 #define IFACEQ_MIN_PKT_SIZE 8
45 #define IFACEQ_VAR_SMALL_PKT_SIZE 100
46 #define IFACEQ_VAR_LARGE_PKT_SIZE 512
47 #define IFACEQ_VAR_HUGE_PKT_SIZE (1024 * 12)
49 struct hfi_queue_table_header {
58 struct hfi_queue_header {
75 #define IFACEQ_TABLE_SIZE \
76 (sizeof(struct hfi_queue_table_header) + \
77 sizeof(struct hfi_queue_header) * IFACEQ_NUM)
79 #define IFACEQ_QUEUE_SIZE (IFACEQ_MAX_PKT_SIZE * \
80 IFACEQ_MAX_BUF_COUNT * IFACEQ_MAX_PARALLEL_CLNTS)
82 #define IFACEQ_GET_QHDR_START_ADDR(ptr, i) \
83 (void *)(((ptr) + sizeof(struct hfi_queue_table_header)) + \
84 ((i) * sizeof(struct hfi_queue_header)))
86 #define QDSS_SIZE SZ_4K
87 #define SFR_SIZE SZ_4K
89 (IFACEQ_TABLE_SIZE + (IFACEQ_QUEUE_SIZE * IFACEQ_NUM))
91 #define ALIGNED_QDSS_SIZE ALIGN(QDSS_SIZE, SZ_4K)
92 #define ALIGNED_SFR_SIZE ALIGN(SFR_SIZE, SZ_4K)
93 #define ALIGNED_QUEUE_SIZE ALIGN(QUEUE_SIZE, SZ_4K)
94 #define SHARED_QSIZE ALIGN(ALIGNED_SFR_SIZE + ALIGNED_QUEUE_SIZE + \
95 ALIGNED_QDSS_SIZE, SZ_1M)
98 dma_addr_t da; /* device address */
99 void *kva; /* kernel virtual address */
105 struct hfi_queue_header *qhdr;
106 struct mem_desc qmem;
110 VENUS_STATE_DEINIT = 1,
114 struct venus_hfi_device {
115 struct venus_core *core;
117 u32 last_packet_type;
120 enum venus_state state;
121 /* serialize read / write to the shared memory */
123 struct completion pwr_collapse_prep;
124 struct completion release_resource;
125 struct mem_desc ifaceq_table;
127 struct iface_queue queues[IFACEQ_NUM];
128 u8 pkt_buf[IFACEQ_VAR_HUGE_PKT_SIZE];
129 u8 dbg_buf[IFACEQ_VAR_HUGE_PKT_SIZE];
132 static bool venus_pkt_debug;
133 static int venus_fw_debug = HFI_DEBUG_MSG_ERROR | HFI_DEBUG_MSG_FATAL;
134 static bool venus_sys_idle_indicator;
135 static bool venus_fw_low_power_mode = true;
136 static int venus_hw_rsp_timeout = 1000;
137 static bool venus_fw_coverage;
139 static void venus_set_state(struct venus_hfi_device *hdev,
140 enum venus_state state)
142 mutex_lock(&hdev->lock);
144 mutex_unlock(&hdev->lock);
147 static bool venus_is_valid_state(struct venus_hfi_device *hdev)
149 return hdev->state != VENUS_STATE_DEINIT;
152 static void venus_dump_packet(struct venus_hfi_device *hdev, const void *packet)
154 size_t pkt_size = *(u32 *)packet;
156 if (!venus_pkt_debug)
159 print_hex_dump(KERN_DEBUG, "", DUMP_PREFIX_OFFSET, 16, 1, packet,
163 static int venus_write_queue(struct venus_hfi_device *hdev,
164 struct iface_queue *queue,
165 void *packet, u32 *rx_req)
167 struct hfi_queue_header *qhdr;
168 u32 dwords, new_wr_idx;
169 u32 empty_space, rd_idx, wr_idx, qsize;
172 if (!queue->qmem.kva)
179 venus_dump_packet(hdev, packet);
181 dwords = (*(u32 *)packet) >> 2;
185 rd_idx = qhdr->read_idx;
186 wr_idx = qhdr->write_idx;
187 qsize = qhdr->q_size;
188 /* ensure rd/wr indices's are read from memory */
191 if (wr_idx >= rd_idx)
192 empty_space = qsize - (wr_idx - rd_idx);
194 empty_space = rd_idx - wr_idx;
196 if (empty_space <= dwords) {
198 /* ensure tx_req is updated in memory */
204 /* ensure tx_req is updated in memory */
207 new_wr_idx = wr_idx + dwords;
208 wr_ptr = (u32 *)(queue->qmem.kva + (wr_idx << 2));
209 if (new_wr_idx < qsize) {
210 memcpy(wr_ptr, packet, dwords << 2);
215 len = (dwords - new_wr_idx) << 2;
216 memcpy(wr_ptr, packet, len);
217 memcpy(queue->qmem.kva, packet + len, new_wr_idx << 2);
220 /* make sure packet is written before updating the write index */
223 qhdr->write_idx = new_wr_idx;
224 *rx_req = qhdr->rx_req ? 1 : 0;
226 /* make sure write index is updated before an interrupt is raised */
232 static int venus_read_queue(struct venus_hfi_device *hdev,
233 struct iface_queue *queue, void *pkt, u32 *tx_req)
235 struct hfi_queue_header *qhdr;
236 u32 dwords, new_rd_idx;
237 u32 rd_idx, wr_idx, type, qsize;
239 u32 recv_request = 0;
242 if (!queue->qmem.kva)
250 rd_idx = qhdr->read_idx;
251 wr_idx = qhdr->write_idx;
252 qsize = qhdr->q_size;
254 /* make sure data is valid before using it */
258 * Do not set receive request for debug queue, if set, Venus generates
259 * interrupt for debug messages even when there is no response message
260 * available. In general debug queue will not become full as it is being
261 * emptied out for every interrupt from Venus. Venus will anyway
262 * generates interrupt if it is full.
264 if (type & HFI_CTRL_TO_HOST_MSG_Q)
267 if (rd_idx == wr_idx) {
268 qhdr->rx_req = recv_request;
270 /* update rx_req field in memory */
275 rd_ptr = (u32 *)(queue->qmem.kva + (rd_idx << 2));
276 dwords = *rd_ptr >> 2;
280 new_rd_idx = rd_idx + dwords;
281 if (((dwords << 2) <= IFACEQ_VAR_HUGE_PKT_SIZE) && rd_idx <= qsize) {
282 if (new_rd_idx < qsize) {
283 memcpy(pkt, rd_ptr, dwords << 2);
288 len = (dwords - new_rd_idx) << 2;
289 memcpy(pkt, rd_ptr, len);
290 memcpy(pkt + len, queue->qmem.kva, new_rd_idx << 2);
293 /* bad packet received, dropping */
294 new_rd_idx = qhdr->write_idx;
298 /* ensure the packet is read before updating read index */
301 qhdr->read_idx = new_rd_idx;
302 /* ensure updating read index */
305 rd_idx = qhdr->read_idx;
306 wr_idx = qhdr->write_idx;
307 /* ensure rd/wr indices are read from memory */
310 if (rd_idx != wr_idx)
313 qhdr->rx_req = recv_request;
315 *tx_req = qhdr->tx_req ? 1 : 0;
317 /* ensure rx_req is stored to memory and tx_req is loaded from memory */
320 venus_dump_packet(hdev, pkt);
325 static int venus_alloc(struct venus_hfi_device *hdev, struct mem_desc *desc,
328 struct device *dev = hdev->core->dev;
330 desc->attrs = DMA_ATTR_WRITE_COMBINE;
331 desc->size = ALIGN(size, SZ_4K);
333 desc->kva = dma_alloc_attrs(dev, desc->size, &desc->da, GFP_KERNEL,
341 static void venus_free(struct venus_hfi_device *hdev, struct mem_desc *mem)
343 struct device *dev = hdev->core->dev;
345 dma_free_attrs(dev, mem->size, mem->kva, mem->da, mem->attrs);
348 static void venus_writel(struct venus_hfi_device *hdev, u32 reg, u32 value)
350 writel(value, hdev->core->base + reg);
353 static u32 venus_readl(struct venus_hfi_device *hdev, u32 reg)
355 return readl(hdev->core->base + reg);
358 static void venus_set_registers(struct venus_hfi_device *hdev)
360 const struct venus_resources *res = hdev->core->res;
361 const struct reg_val *tbl = res->reg_tbl;
362 unsigned int count = res->reg_tbl_size;
365 for (i = 0; i < count; i++)
366 venus_writel(hdev, tbl[i].reg, tbl[i].value);
369 static void venus_soft_int(struct venus_hfi_device *hdev)
371 venus_writel(hdev, CPU_IC_SOFTINT, BIT(CPU_IC_SOFTINT_H2A_SHIFT));
374 static int venus_iface_cmdq_write_nolock(struct venus_hfi_device *hdev,
377 struct device *dev = hdev->core->dev;
378 struct hfi_pkt_hdr *cmd_packet;
379 struct iface_queue *queue;
383 if (!venus_is_valid_state(hdev))
386 cmd_packet = (struct hfi_pkt_hdr *)pkt;
387 hdev->last_packet_type = cmd_packet->pkt_type;
389 queue = &hdev->queues[IFACEQ_CMD_IDX];
391 ret = venus_write_queue(hdev, queue, pkt, &rx_req);
393 dev_err(dev, "write to iface cmd queue failed (%d)\n", ret);
398 venus_soft_int(hdev);
403 static int venus_iface_cmdq_write(struct venus_hfi_device *hdev, void *pkt)
407 mutex_lock(&hdev->lock);
408 ret = venus_iface_cmdq_write_nolock(hdev, pkt);
409 mutex_unlock(&hdev->lock);
414 static int venus_hfi_core_set_resource(struct venus_core *core, u32 id,
415 u32 size, u32 addr, void *cookie)
417 struct venus_hfi_device *hdev = to_hfi_priv(core);
418 struct hfi_sys_set_resource_pkt *pkt;
419 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
422 if (id == VIDC_RESOURCE_NONE)
425 pkt = (struct hfi_sys_set_resource_pkt *)packet;
427 ret = pkt_sys_set_resource(pkt, id, size, addr, cookie);
431 ret = venus_iface_cmdq_write(hdev, pkt);
438 static int venus_boot_core(struct venus_hfi_device *hdev)
440 struct device *dev = hdev->core->dev;
441 static const unsigned int max_tries = 100;
443 unsigned int count = 0;
446 venus_writel(hdev, VIDC_CTRL_INIT, BIT(VIDC_CTRL_INIT_CTRL_SHIFT));
447 venus_writel(hdev, WRAPPER_INTR_MASK, WRAPPER_INTR_MASK_A2HVCODEC_MASK);
448 venus_writel(hdev, CPU_CS_SCIACMDARG3, 1);
450 while (!ctrl_status && count < max_tries) {
451 ctrl_status = venus_readl(hdev, CPU_CS_SCIACMDARG0);
452 if ((ctrl_status & CPU_CS_SCIACMDARG0_ERROR_STATUS_MASK) == 4) {
453 dev_err(dev, "invalid setting for UC_REGION\n");
458 usleep_range(500, 1000);
462 if (count >= max_tries)
468 static u32 venus_hwversion(struct venus_hfi_device *hdev)
470 struct device *dev = hdev->core->dev;
471 u32 ver = venus_readl(hdev, WRAPPER_HW_VERSION);
472 u32 major, minor, step;
474 major = ver & WRAPPER_HW_VERSION_MAJOR_VERSION_MASK;
475 major = major >> WRAPPER_HW_VERSION_MAJOR_VERSION_SHIFT;
476 minor = ver & WRAPPER_HW_VERSION_MINOR_VERSION_MASK;
477 minor = minor >> WRAPPER_HW_VERSION_MINOR_VERSION_SHIFT;
478 step = ver & WRAPPER_HW_VERSION_STEP_VERSION_MASK;
480 dev_dbg(dev, "venus hw version %x.%x.%x\n", major, minor, step);
485 static int venus_run(struct venus_hfi_device *hdev)
487 struct device *dev = hdev->core->dev;
491 * Re-program all of the registers that get reset as a result of
492 * regulator_disable() and _enable()
494 venus_set_registers(hdev);
496 venus_writel(hdev, UC_REGION_ADDR, hdev->ifaceq_table.da);
497 venus_writel(hdev, UC_REGION_SIZE, SHARED_QSIZE);
498 venus_writel(hdev, CPU_CS_SCIACMDARG2, hdev->ifaceq_table.da);
499 venus_writel(hdev, CPU_CS_SCIACMDARG1, 0x01);
501 venus_writel(hdev, SFR_ADDR, hdev->sfr.da);
503 ret = venus_boot_core(hdev);
505 dev_err(dev, "failed to reset venus core\n");
509 venus_hwversion(hdev);
514 static int venus_halt_axi(struct venus_hfi_device *hdev)
516 void __iomem *base = hdev->core->base;
517 struct device *dev = hdev->core->dev;
521 if (IS_V4(hdev->core)) {
522 val = venus_readl(hdev, WRAPPER_CPU_AXI_HALT);
523 val |= WRAPPER_CPU_AXI_HALT_HALT;
524 venus_writel(hdev, WRAPPER_CPU_AXI_HALT, val);
526 ret = readl_poll_timeout(base + WRAPPER_CPU_AXI_HALT_STATUS,
528 val & WRAPPER_CPU_AXI_HALT_STATUS_IDLE,
530 VBIF_AXI_HALT_ACK_TIMEOUT_US);
532 dev_err(dev, "AXI bus port halt timeout\n");
539 /* Halt AXI and AXI IMEM VBIF Access */
540 val = venus_readl(hdev, VBIF_AXI_HALT_CTRL0);
541 val |= VBIF_AXI_HALT_CTRL0_HALT_REQ;
542 venus_writel(hdev, VBIF_AXI_HALT_CTRL0, val);
544 /* Request for AXI bus port halt */
545 ret = readl_poll_timeout(base + VBIF_AXI_HALT_CTRL1, val,
546 val & VBIF_AXI_HALT_CTRL1_HALT_ACK,
548 VBIF_AXI_HALT_ACK_TIMEOUT_US);
550 dev_err(dev, "AXI bus port halt timeout\n");
557 static int venus_power_off(struct venus_hfi_device *hdev)
561 if (!hdev->power_enabled)
564 ret = venus_set_hw_state_suspend(hdev->core);
568 ret = venus_halt_axi(hdev);
572 hdev->power_enabled = false;
577 static int venus_power_on(struct venus_hfi_device *hdev)
581 if (hdev->power_enabled)
584 ret = venus_set_hw_state_resume(hdev->core);
588 ret = venus_run(hdev);
592 hdev->power_enabled = true;
597 venus_set_hw_state_suspend(hdev->core);
599 hdev->power_enabled = false;
603 static int venus_iface_msgq_read_nolock(struct venus_hfi_device *hdev,
606 struct iface_queue *queue;
610 if (!venus_is_valid_state(hdev))
613 queue = &hdev->queues[IFACEQ_MSG_IDX];
615 ret = venus_read_queue(hdev, queue, pkt, &tx_req);
620 venus_soft_int(hdev);
625 static int venus_iface_msgq_read(struct venus_hfi_device *hdev, void *pkt)
629 mutex_lock(&hdev->lock);
630 ret = venus_iface_msgq_read_nolock(hdev, pkt);
631 mutex_unlock(&hdev->lock);
636 static int venus_iface_dbgq_read_nolock(struct venus_hfi_device *hdev,
639 struct iface_queue *queue;
643 ret = venus_is_valid_state(hdev);
647 queue = &hdev->queues[IFACEQ_DBG_IDX];
649 ret = venus_read_queue(hdev, queue, pkt, &tx_req);
654 venus_soft_int(hdev);
659 static int venus_iface_dbgq_read(struct venus_hfi_device *hdev, void *pkt)
666 mutex_lock(&hdev->lock);
667 ret = venus_iface_dbgq_read_nolock(hdev, pkt);
668 mutex_unlock(&hdev->lock);
673 static void venus_set_qhdr_defaults(struct hfi_queue_header *qhdr)
676 qhdr->type = IFACEQ_DFLT_QHDR;
677 qhdr->q_size = IFACEQ_QUEUE_SIZE / 4;
683 qhdr->rx_irq_status = 0;
684 qhdr->tx_irq_status = 0;
689 static void venus_interface_queues_release(struct venus_hfi_device *hdev)
691 mutex_lock(&hdev->lock);
693 venus_free(hdev, &hdev->ifaceq_table);
694 venus_free(hdev, &hdev->sfr);
696 memset(hdev->queues, 0, sizeof(hdev->queues));
697 memset(&hdev->ifaceq_table, 0, sizeof(hdev->ifaceq_table));
698 memset(&hdev->sfr, 0, sizeof(hdev->sfr));
700 mutex_unlock(&hdev->lock);
703 static int venus_interface_queues_init(struct venus_hfi_device *hdev)
705 struct hfi_queue_table_header *tbl_hdr;
706 struct iface_queue *queue;
708 struct mem_desc desc = {0};
713 ret = venus_alloc(hdev, &desc, ALIGNED_QUEUE_SIZE);
717 hdev->ifaceq_table = desc;
718 offset = IFACEQ_TABLE_SIZE;
720 for (i = 0; i < IFACEQ_NUM; i++) {
721 queue = &hdev->queues[i];
722 queue->qmem.da = desc.da + offset;
723 queue->qmem.kva = desc.kva + offset;
724 queue->qmem.size = IFACEQ_QUEUE_SIZE;
725 offset += queue->qmem.size;
727 IFACEQ_GET_QHDR_START_ADDR(hdev->ifaceq_table.kva, i);
729 venus_set_qhdr_defaults(queue->qhdr);
731 queue->qhdr->start_addr = queue->qmem.da;
733 if (i == IFACEQ_CMD_IDX)
734 queue->qhdr->type |= HFI_HOST_TO_CTRL_CMD_Q;
735 else if (i == IFACEQ_MSG_IDX)
736 queue->qhdr->type |= HFI_CTRL_TO_HOST_MSG_Q;
737 else if (i == IFACEQ_DBG_IDX)
738 queue->qhdr->type |= HFI_CTRL_TO_HOST_DBG_Q;
741 tbl_hdr = hdev->ifaceq_table.kva;
742 tbl_hdr->version = 0;
743 tbl_hdr->size = IFACEQ_TABLE_SIZE;
744 tbl_hdr->qhdr0_offset = sizeof(struct hfi_queue_table_header);
745 tbl_hdr->qhdr_size = sizeof(struct hfi_queue_header);
746 tbl_hdr->num_q = IFACEQ_NUM;
747 tbl_hdr->num_active_q = IFACEQ_NUM;
750 * Set receive request to zero on debug queue as there is no
751 * need of interrupt from video hardware for debug messages
753 queue = &hdev->queues[IFACEQ_DBG_IDX];
754 queue->qhdr->rx_req = 0;
756 ret = venus_alloc(hdev, &desc, ALIGNED_SFR_SIZE);
762 sfr->buf_size = ALIGNED_SFR_SIZE;
765 /* ensure table and queue header structs are settled in memory */
771 static int venus_sys_set_debug(struct venus_hfi_device *hdev, u32 debug)
773 struct hfi_sys_set_property_pkt *pkt;
774 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
777 pkt = (struct hfi_sys_set_property_pkt *)packet;
779 pkt_sys_debug_config(pkt, HFI_DEBUG_MODE_QUEUE, debug);
781 ret = venus_iface_cmdq_write(hdev, pkt);
788 static int venus_sys_set_coverage(struct venus_hfi_device *hdev, u32 mode)
790 struct hfi_sys_set_property_pkt *pkt;
791 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
794 pkt = (struct hfi_sys_set_property_pkt *)packet;
796 pkt_sys_coverage_config(pkt, mode);
798 ret = venus_iface_cmdq_write(hdev, pkt);
805 static int venus_sys_set_idle_message(struct venus_hfi_device *hdev,
808 struct hfi_sys_set_property_pkt *pkt;
809 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
815 pkt = (struct hfi_sys_set_property_pkt *)packet;
817 pkt_sys_idle_indicator(pkt, enable);
819 ret = venus_iface_cmdq_write(hdev, pkt);
826 static int venus_sys_set_power_control(struct venus_hfi_device *hdev,
829 struct hfi_sys_set_property_pkt *pkt;
830 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
833 pkt = (struct hfi_sys_set_property_pkt *)packet;
835 pkt_sys_power_control(pkt, enable);
837 ret = venus_iface_cmdq_write(hdev, pkt);
844 static int venus_get_queue_size(struct venus_hfi_device *hdev,
847 struct hfi_queue_header *qhdr;
849 if (index >= IFACEQ_NUM)
852 qhdr = hdev->queues[index].qhdr;
856 return abs(qhdr->read_idx - qhdr->write_idx);
859 static int venus_sys_set_default_properties(struct venus_hfi_device *hdev)
861 struct device *dev = hdev->core->dev;
864 ret = venus_sys_set_debug(hdev, venus_fw_debug);
866 dev_warn(dev, "setting fw debug msg ON failed (%d)\n", ret);
869 * Idle indicator is disabled by default on some 4xx firmware versions,
870 * enable it explicitly in order to make suspend functional by checking
871 * WFI (wait-for-interrupt) bit.
873 if (IS_V4(hdev->core))
874 venus_sys_idle_indicator = true;
876 ret = venus_sys_set_idle_message(hdev, venus_sys_idle_indicator);
878 dev_warn(dev, "setting idle response ON failed (%d)\n", ret);
880 ret = venus_sys_set_power_control(hdev, venus_fw_low_power_mode);
882 dev_warn(dev, "setting hw power collapse ON failed (%d)\n",
888 static int venus_session_cmd(struct venus_inst *inst, u32 pkt_type)
890 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
891 struct hfi_session_pkt pkt;
893 pkt_session_cmd(&pkt, pkt_type, inst);
895 return venus_iface_cmdq_write(hdev, &pkt);
898 static void venus_flush_debug_queue(struct venus_hfi_device *hdev)
900 struct device *dev = hdev->core->dev;
901 void *packet = hdev->dbg_buf;
903 while (!venus_iface_dbgq_read(hdev, packet)) {
904 struct hfi_msg_sys_coverage_pkt *pkt = packet;
906 if (pkt->hdr.pkt_type != HFI_MSG_SYS_COV) {
907 struct hfi_msg_sys_debug_pkt *pkt = packet;
909 dev_dbg(dev, "%s", pkt->msg_data);
914 static int venus_prepare_power_collapse(struct venus_hfi_device *hdev,
917 unsigned long timeout = msecs_to_jiffies(venus_hw_rsp_timeout);
918 struct hfi_sys_pc_prep_pkt pkt;
921 init_completion(&hdev->pwr_collapse_prep);
923 pkt_sys_pc_prep(&pkt);
925 ret = venus_iface_cmdq_write(hdev, &pkt);
932 ret = wait_for_completion_timeout(&hdev->pwr_collapse_prep, timeout);
934 venus_flush_debug_queue(hdev);
941 static int venus_are_queues_empty(struct venus_hfi_device *hdev)
945 ret1 = venus_get_queue_size(hdev, IFACEQ_MSG_IDX);
949 ret2 = venus_get_queue_size(hdev, IFACEQ_CMD_IDX);
959 static void venus_sfr_print(struct venus_hfi_device *hdev)
961 struct device *dev = hdev->core->dev;
962 struct hfi_sfr *sfr = hdev->sfr.kva;
968 p = memchr(sfr->data, '\0', sfr->buf_size);
970 * SFR isn't guaranteed to be NULL terminated since SYS_ERROR indicates
971 * that Venus is in the process of crashing.
974 sfr->data[sfr->buf_size - 1] = '\0';
976 dev_err_ratelimited(dev, "SFR message from FW: %s\n", sfr->data);
979 static void venus_process_msg_sys_error(struct venus_hfi_device *hdev,
982 struct hfi_msg_event_notify_pkt *event_pkt = packet;
984 if (event_pkt->event_id != HFI_EVENT_SYS_ERROR)
987 venus_set_state(hdev, VENUS_STATE_DEINIT);
990 * Once SYS_ERROR received from HW, it is safe to halt the AXI.
991 * With SYS_ERROR, Venus FW may have crashed and HW might be
992 * active and causing unnecessary transactions. Hence it is
993 * safe to stop all AXI transactions from venus subsystem.
995 venus_halt_axi(hdev);
996 venus_sfr_print(hdev);
999 static irqreturn_t venus_isr_thread(struct venus_core *core)
1001 struct venus_hfi_device *hdev = to_hfi_priv(core);
1002 const struct venus_resources *res;
1009 res = hdev->core->res;
1010 pkt = hdev->pkt_buf;
1012 if (hdev->irq_status & WRAPPER_INTR_STATUS_A2HWD_MASK) {
1013 venus_sfr_print(hdev);
1014 hfi_process_watchdog_timeout(core);
1017 while (!venus_iface_msgq_read(hdev, pkt)) {
1018 msg_ret = hfi_process_msg_packet(core, pkt);
1020 case HFI_MSG_EVENT_NOTIFY:
1021 venus_process_msg_sys_error(hdev, pkt);
1023 case HFI_MSG_SYS_INIT:
1024 venus_hfi_core_set_resource(core, res->vmem_id,
1029 case HFI_MSG_SYS_RELEASE_RESOURCE:
1030 complete(&hdev->release_resource);
1032 case HFI_MSG_SYS_PC_PREP:
1033 complete(&hdev->pwr_collapse_prep);
1040 venus_flush_debug_queue(hdev);
1045 static irqreturn_t venus_isr(struct venus_core *core)
1047 struct venus_hfi_device *hdev = to_hfi_priv(core);
1053 status = venus_readl(hdev, WRAPPER_INTR_STATUS);
1055 if (status & WRAPPER_INTR_STATUS_A2H_MASK ||
1056 status & WRAPPER_INTR_STATUS_A2HWD_MASK ||
1057 status & CPU_CS_SCIACMDARG0_INIT_IDLE_MSG_MASK)
1058 hdev->irq_status = status;
1060 venus_writel(hdev, CPU_CS_A2HSOFTINTCLR, 1);
1061 venus_writel(hdev, WRAPPER_INTR_CLEAR, status);
1063 return IRQ_WAKE_THREAD;
1066 static int venus_core_init(struct venus_core *core)
1068 struct venus_hfi_device *hdev = to_hfi_priv(core);
1069 struct device *dev = core->dev;
1070 struct hfi_sys_get_property_pkt version_pkt;
1071 struct hfi_sys_init_pkt pkt;
1074 pkt_sys_init(&pkt, HFI_VIDEO_ARCH_OX);
1076 venus_set_state(hdev, VENUS_STATE_INIT);
1078 ret = venus_iface_cmdq_write(hdev, &pkt);
1082 pkt_sys_image_version(&version_pkt);
1084 ret = venus_iface_cmdq_write(hdev, &version_pkt);
1086 dev_warn(dev, "failed to send image version pkt to fw\n");
1088 ret = venus_sys_set_default_properties(hdev);
1095 static int venus_core_deinit(struct venus_core *core)
1097 struct venus_hfi_device *hdev = to_hfi_priv(core);
1099 venus_set_state(hdev, VENUS_STATE_DEINIT);
1100 hdev->suspended = true;
1101 hdev->power_enabled = false;
1106 static int venus_core_ping(struct venus_core *core, u32 cookie)
1108 struct venus_hfi_device *hdev = to_hfi_priv(core);
1109 struct hfi_sys_ping_pkt pkt;
1111 pkt_sys_ping(&pkt, cookie);
1113 return venus_iface_cmdq_write(hdev, &pkt);
1116 static int venus_core_trigger_ssr(struct venus_core *core, u32 trigger_type)
1118 struct venus_hfi_device *hdev = to_hfi_priv(core);
1119 struct hfi_sys_test_ssr_pkt pkt;
1122 ret = pkt_sys_ssr_cmd(&pkt, trigger_type);
1126 return venus_iface_cmdq_write(hdev, &pkt);
1129 static int venus_session_init(struct venus_inst *inst, u32 session_type,
1132 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1133 struct hfi_session_init_pkt pkt;
1136 ret = pkt_session_init(&pkt, inst, session_type, codec);
1140 ret = venus_iface_cmdq_write(hdev, &pkt);
1147 venus_flush_debug_queue(hdev);
1151 static int venus_session_end(struct venus_inst *inst)
1153 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1154 struct device *dev = hdev->core->dev;
1156 if (venus_fw_coverage) {
1157 if (venus_sys_set_coverage(hdev, venus_fw_coverage))
1158 dev_warn(dev, "fw coverage msg ON failed\n");
1161 return venus_session_cmd(inst, HFI_CMD_SYS_SESSION_END);
1164 static int venus_session_abort(struct venus_inst *inst)
1166 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1168 venus_flush_debug_queue(hdev);
1170 return venus_session_cmd(inst, HFI_CMD_SYS_SESSION_ABORT);
1173 static int venus_session_flush(struct venus_inst *inst, u32 flush_mode)
1175 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1176 struct hfi_session_flush_pkt pkt;
1179 ret = pkt_session_flush(&pkt, inst, flush_mode);
1183 return venus_iface_cmdq_write(hdev, &pkt);
1186 static int venus_session_start(struct venus_inst *inst)
1188 return venus_session_cmd(inst, HFI_CMD_SESSION_START);
1191 static int venus_session_stop(struct venus_inst *inst)
1193 return venus_session_cmd(inst, HFI_CMD_SESSION_STOP);
1196 static int venus_session_continue(struct venus_inst *inst)
1198 return venus_session_cmd(inst, HFI_CMD_SESSION_CONTINUE);
1201 static int venus_session_etb(struct venus_inst *inst,
1202 struct hfi_frame_data *in_frame)
1204 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1205 u32 session_type = inst->session_type;
1208 if (session_type == VIDC_SESSION_TYPE_DEC) {
1209 struct hfi_session_empty_buffer_compressed_pkt pkt;
1211 ret = pkt_session_etb_decoder(&pkt, inst, in_frame);
1215 ret = venus_iface_cmdq_write(hdev, &pkt);
1216 } else if (session_type == VIDC_SESSION_TYPE_ENC) {
1217 struct hfi_session_empty_buffer_uncompressed_plane0_pkt pkt;
1219 ret = pkt_session_etb_encoder(&pkt, inst, in_frame);
1223 ret = venus_iface_cmdq_write(hdev, &pkt);
1231 static int venus_session_ftb(struct venus_inst *inst,
1232 struct hfi_frame_data *out_frame)
1234 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1235 struct hfi_session_fill_buffer_pkt pkt;
1238 ret = pkt_session_ftb(&pkt, inst, out_frame);
1242 return venus_iface_cmdq_write(hdev, &pkt);
1245 static int venus_session_set_buffers(struct venus_inst *inst,
1246 struct hfi_buffer_desc *bd)
1248 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1249 struct hfi_session_set_buffers_pkt *pkt;
1250 u8 packet[IFACEQ_VAR_LARGE_PKT_SIZE];
1253 if (bd->buffer_type == HFI_BUFFER_INPUT)
1256 pkt = (struct hfi_session_set_buffers_pkt *)packet;
1258 ret = pkt_session_set_buffers(pkt, inst, bd);
1262 return venus_iface_cmdq_write(hdev, pkt);
1265 static int venus_session_unset_buffers(struct venus_inst *inst,
1266 struct hfi_buffer_desc *bd)
1268 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1269 struct hfi_session_release_buffer_pkt *pkt;
1270 u8 packet[IFACEQ_VAR_LARGE_PKT_SIZE];
1273 if (bd->buffer_type == HFI_BUFFER_INPUT)
1276 pkt = (struct hfi_session_release_buffer_pkt *)packet;
1278 ret = pkt_session_unset_buffers(pkt, inst, bd);
1282 return venus_iface_cmdq_write(hdev, pkt);
1285 static int venus_session_load_res(struct venus_inst *inst)
1287 return venus_session_cmd(inst, HFI_CMD_SESSION_LOAD_RESOURCES);
1290 static int venus_session_release_res(struct venus_inst *inst)
1292 return venus_session_cmd(inst, HFI_CMD_SESSION_RELEASE_RESOURCES);
1295 static int venus_session_parse_seq_hdr(struct venus_inst *inst, u32 seq_hdr,
1298 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1299 struct hfi_session_parse_sequence_header_pkt *pkt;
1300 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
1303 pkt = (struct hfi_session_parse_sequence_header_pkt *)packet;
1305 ret = pkt_session_parse_seq_header(pkt, inst, seq_hdr, seq_hdr_len);
1309 ret = venus_iface_cmdq_write(hdev, pkt);
1316 static int venus_session_get_seq_hdr(struct venus_inst *inst, u32 seq_hdr,
1319 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1320 struct hfi_session_get_sequence_header_pkt *pkt;
1321 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
1324 pkt = (struct hfi_session_get_sequence_header_pkt *)packet;
1326 ret = pkt_session_get_seq_hdr(pkt, inst, seq_hdr, seq_hdr_len);
1330 return venus_iface_cmdq_write(hdev, pkt);
1333 static int venus_session_set_property(struct venus_inst *inst, u32 ptype,
1336 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1337 struct hfi_session_set_property_pkt *pkt;
1338 u8 packet[IFACEQ_VAR_LARGE_PKT_SIZE];
1341 pkt = (struct hfi_session_set_property_pkt *)packet;
1343 ret = pkt_session_set_property(pkt, inst, ptype, pdata);
1344 if (ret == -ENOTSUPP)
1349 return venus_iface_cmdq_write(hdev, pkt);
1352 static int venus_session_get_property(struct venus_inst *inst, u32 ptype)
1354 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1355 struct hfi_session_get_property_pkt pkt;
1358 ret = pkt_session_get_property(&pkt, inst, ptype);
1362 return venus_iface_cmdq_write(hdev, &pkt);
1365 static int venus_resume(struct venus_core *core)
1367 struct venus_hfi_device *hdev = to_hfi_priv(core);
1370 mutex_lock(&hdev->lock);
1372 if (!hdev->suspended)
1375 ret = venus_power_on(hdev);
1379 hdev->suspended = false;
1381 mutex_unlock(&hdev->lock);
1386 static int venus_suspend_1xx(struct venus_core *core)
1388 struct venus_hfi_device *hdev = to_hfi_priv(core);
1389 struct device *dev = core->dev;
1393 if (!hdev->power_enabled || hdev->suspended)
1396 mutex_lock(&hdev->lock);
1397 ret = venus_is_valid_state(hdev);
1398 mutex_unlock(&hdev->lock);
1401 dev_err(dev, "bad state, cannot suspend\n");
1405 ret = venus_prepare_power_collapse(hdev, true);
1407 dev_err(dev, "prepare for power collapse fail (%d)\n", ret);
1411 mutex_lock(&hdev->lock);
1413 if (hdev->last_packet_type != HFI_CMD_SYS_PC_PREP) {
1414 mutex_unlock(&hdev->lock);
1418 ret = venus_are_queues_empty(hdev);
1419 if (ret < 0 || !ret) {
1420 mutex_unlock(&hdev->lock);
1424 ctrl_status = venus_readl(hdev, CPU_CS_SCIACMDARG0);
1425 if (!(ctrl_status & CPU_CS_SCIACMDARG0_PC_READY)) {
1426 mutex_unlock(&hdev->lock);
1430 ret = venus_power_off(hdev);
1432 mutex_unlock(&hdev->lock);
1436 hdev->suspended = true;
1438 mutex_unlock(&hdev->lock);
1443 static bool venus_cpu_and_video_core_idle(struct venus_hfi_device *hdev)
1445 u32 ctrl_status, cpu_status;
1447 cpu_status = venus_readl(hdev, WRAPPER_CPU_STATUS);
1448 ctrl_status = venus_readl(hdev, CPU_CS_SCIACMDARG0);
1450 if (cpu_status & WRAPPER_CPU_STATUS_WFI &&
1451 ctrl_status & CPU_CS_SCIACMDARG0_INIT_IDLE_MSG_MASK)
1457 static bool venus_cpu_idle_and_pc_ready(struct venus_hfi_device *hdev)
1459 u32 ctrl_status, cpu_status;
1461 cpu_status = venus_readl(hdev, WRAPPER_CPU_STATUS);
1462 ctrl_status = venus_readl(hdev, CPU_CS_SCIACMDARG0);
1464 if (cpu_status & WRAPPER_CPU_STATUS_WFI &&
1465 ctrl_status & CPU_CS_SCIACMDARG0_PC_READY)
1471 static int venus_suspend_3xx(struct venus_core *core)
1473 struct venus_hfi_device *hdev = to_hfi_priv(core);
1474 struct device *dev = core->dev;
1479 if (!hdev->power_enabled || hdev->suspended)
1482 mutex_lock(&hdev->lock);
1483 ret = venus_is_valid_state(hdev);
1484 mutex_unlock(&hdev->lock);
1487 dev_err(dev, "bad state, cannot suspend\n");
1491 ctrl_status = venus_readl(hdev, CPU_CS_SCIACMDARG0);
1492 if (ctrl_status & CPU_CS_SCIACMDARG0_PC_READY)
1496 * Power collapse sequence for Venus 3xx and 4xx versions:
1497 * 1. Check for ARM9 and video core to be idle by checking WFI bit
1498 * (bit 0) in CPU status register and by checking Idle (bit 30) in
1499 * Control status register for video core.
1500 * 2. Send a command to prepare for power collapse.
1501 * 3. Check for WFI and PC_READY bits.
1503 ret = readx_poll_timeout(venus_cpu_and_video_core_idle, hdev, val, val,
1508 ret = venus_prepare_power_collapse(hdev, false);
1510 dev_err(dev, "prepare for power collapse fail (%d)\n", ret);
1514 ret = readx_poll_timeout(venus_cpu_idle_and_pc_ready, hdev, val, val,
1520 mutex_lock(&hdev->lock);
1522 ret = venus_power_off(hdev);
1524 dev_err(dev, "venus_power_off (%d)\n", ret);
1525 mutex_unlock(&hdev->lock);
1529 hdev->suspended = true;
1531 mutex_unlock(&hdev->lock);
1536 static int venus_suspend(struct venus_core *core)
1538 if (IS_V3(core) || IS_V4(core))
1539 return venus_suspend_3xx(core);
1541 return venus_suspend_1xx(core);
1544 static const struct hfi_ops venus_hfi_ops = {
1545 .core_init = venus_core_init,
1546 .core_deinit = venus_core_deinit,
1547 .core_ping = venus_core_ping,
1548 .core_trigger_ssr = venus_core_trigger_ssr,
1550 .session_init = venus_session_init,
1551 .session_end = venus_session_end,
1552 .session_abort = venus_session_abort,
1553 .session_flush = venus_session_flush,
1554 .session_start = venus_session_start,
1555 .session_stop = venus_session_stop,
1556 .session_continue = venus_session_continue,
1557 .session_etb = venus_session_etb,
1558 .session_ftb = venus_session_ftb,
1559 .session_set_buffers = venus_session_set_buffers,
1560 .session_unset_buffers = venus_session_unset_buffers,
1561 .session_load_res = venus_session_load_res,
1562 .session_release_res = venus_session_release_res,
1563 .session_parse_seq_hdr = venus_session_parse_seq_hdr,
1564 .session_get_seq_hdr = venus_session_get_seq_hdr,
1565 .session_set_property = venus_session_set_property,
1566 .session_get_property = venus_session_get_property,
1568 .resume = venus_resume,
1569 .suspend = venus_suspend,
1572 .isr_thread = venus_isr_thread,
1575 void venus_hfi_destroy(struct venus_core *core)
1577 struct venus_hfi_device *hdev = to_hfi_priv(core);
1579 venus_interface_queues_release(hdev);
1580 mutex_destroy(&hdev->lock);
1586 int venus_hfi_create(struct venus_core *core)
1588 struct venus_hfi_device *hdev;
1591 hdev = kzalloc(sizeof(*hdev), GFP_KERNEL);
1595 mutex_init(&hdev->lock);
1598 hdev->suspended = true;
1600 core->ops = &venus_hfi_ops;
1601 core->core_caps = ENC_ROTATION_CAPABILITY | ENC_SCALING_CAPABILITY |
1602 ENC_DEINTERLACE_CAPABILITY |
1603 DEC_MULTI_STREAM_CAPABILITY;
1605 ret = venus_interface_queues_init(hdev);