1 // SPDX-License-Identifier: GPL-2.0
4 * Copyright 2016-2022 HabanaLabs, Ltd.
8 #include "habanalabs.h"
9 #include "../include/common/hl_boot_if.h"
11 #include <linux/firmware.h>
12 #include <linux/crc32.h>
13 #include <linux/slab.h>
14 #include <linux/ctype.h>
16 #define FW_FILE_MAX_SIZE 0x1400000 /* maximum size of 20MB */
18 static char *extract_fw_ver_from_str(const char *fw_str)
20 char *str, *fw_ver, *whitespace;
22 fw_ver = kmalloc(16, GFP_KERNEL);
26 str = strnstr(fw_str, "fw-", VERSION_MAX_LEN);
30 /* Skip the fw- part */
33 /* Copy until the next whitespace */
34 whitespace = strnstr(str, " ", 15);
38 strscpy(fw_ver, str, whitespace - str + 1);
47 static int hl_request_fw(struct hl_device *hdev,
48 const struct firmware **firmware_p,
54 rc = request_firmware(firmware_p, fw_name, hdev->dev);
56 dev_err(hdev->dev, "Firmware file %s is not found! (error %d)\n",
61 fw_size = (*firmware_p)->size;
62 if ((fw_size % 4) != 0) {
63 dev_err(hdev->dev, "Illegal %s firmware size %zu\n",
69 dev_dbg(hdev->dev, "%s firmware size == %zu\n", fw_name, fw_size);
71 if (fw_size > FW_FILE_MAX_SIZE) {
73 "FW file size %zu exceeds maximum of %u bytes\n",
74 fw_size, FW_FILE_MAX_SIZE);
82 release_firmware(*firmware_p);
88 * hl_release_firmware() - release FW
92 * note: this inline function added to serve as a comprehensive mirror for the
93 * hl_request_fw function.
95 static inline void hl_release_firmware(const struct firmware *fw)
101 * hl_fw_copy_fw_to_device() - copy FW to device
103 * @hdev: pointer to hl_device structure.
105 * @dst: IO memory mapped address space to copy firmware to
106 * @src_offset: offset in src FW to copy from
107 * @size: amount of bytes to copy (0 to copy the whole binary)
109 * actual copy of FW binary data to device, shared by static and dynamic loaders
111 static int hl_fw_copy_fw_to_device(struct hl_device *hdev,
112 const struct firmware *fw, void __iomem *dst,
113 u32 src_offset, u32 size)
117 /* size 0 indicates to copy the whole file */
121 if (src_offset + size > fw->size) {
123 "size to copy(%u) and offset(%u) are invalid\n",
128 fw_data = (const void *) fw->data;
130 memcpy_toio(dst, fw_data + src_offset, size);
135 * hl_fw_copy_msg_to_device() - copy message to device
137 * @hdev: pointer to hl_device structure.
139 * @dst: IO memory mapped address space to copy firmware to
140 * @src_offset: offset in src message to copy from
141 * @size: amount of bytes to copy (0 to copy the whole binary)
143 * actual copy of message data to device.
145 static int hl_fw_copy_msg_to_device(struct hl_device *hdev,
146 struct lkd_msg_comms *msg, void __iomem *dst,
147 u32 src_offset, u32 size)
151 /* size 0 indicates to copy the whole file */
153 size = sizeof(struct lkd_msg_comms);
155 if (src_offset + size > sizeof(struct lkd_msg_comms)) {
157 "size to copy(%u) and offset(%u) are invalid\n",
162 msg_data = (void *) msg;
164 memcpy_toio(dst, msg_data + src_offset, size);
170 * hl_fw_load_fw_to_device() - Load F/W code to device's memory.
172 * @hdev: pointer to hl_device structure.
173 * @fw_name: the firmware image name
174 * @dst: IO memory mapped address space to copy firmware to
175 * @src_offset: offset in src FW to copy from
176 * @size: amount of bytes to copy (0 to copy the whole binary)
178 * Copy fw code from firmware file to device memory.
180 * Return: 0 on success, non-zero for failure.
182 int hl_fw_load_fw_to_device(struct hl_device *hdev, const char *fw_name,
183 void __iomem *dst, u32 src_offset, u32 size)
185 const struct firmware *fw;
188 rc = hl_request_fw(hdev, &fw, fw_name);
192 rc = hl_fw_copy_fw_to_device(hdev, fw, dst, src_offset, size);
194 hl_release_firmware(fw);
198 int hl_fw_send_pci_access_msg(struct hl_device *hdev, u32 opcode)
200 struct cpucp_packet pkt = {};
202 pkt.ctl = cpu_to_le32(opcode << CPUCP_PKT_CTL_OPCODE_SHIFT);
204 return hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt,
205 sizeof(pkt), 0, NULL);
208 int hl_fw_send_cpu_message(struct hl_device *hdev, u32 hw_queue_id, u32 *msg,
209 u16 len, u32 timeout, u64 *result)
211 struct hl_hw_queue *queue = &hdev->kernel_queues[hw_queue_id];
212 struct asic_fixed_properties *prop = &hdev->asic_prop;
213 struct cpucp_packet *pkt;
214 dma_addr_t pkt_dma_addr;
215 struct hl_bd *sent_bd;
216 u32 tmp, expected_ack_val, pi;
219 pkt = hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev, len,
223 "Failed to allocate DMA memory for packet to CPU\n");
227 memcpy(pkt, msg, len);
229 mutex_lock(&hdev->send_cpu_message_lock);
234 if (hdev->device_cpu_disabled) {
239 /* set fence to a non valid value */
240 pkt->fence = cpu_to_le32(UINT_MAX);
244 * The CPU queue is a synchronous queue with an effective depth of
245 * a single entry (although it is allocated with room for multiple
246 * entries). We lock on it using 'send_cpu_message_lock' which
247 * serializes accesses to the CPU queue.
248 * Which means that we don't need to lock the access to the entire H/W
249 * queues module when submitting a JOB to the CPU queue.
251 hl_hw_queue_submit_bd(hdev, queue, hl_queue_inc_ptr(queue->pi), len, pkt_dma_addr);
253 if (prop->fw_app_cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_PKT_PI_ACK_EN)
254 expected_ack_val = queue->pi;
256 expected_ack_val = CPUCP_PACKET_FENCE_VAL;
258 rc = hl_poll_timeout_memory(hdev, &pkt->fence, tmp,
259 (tmp == expected_ack_val), 1000,
262 hl_hw_queue_inc_ci_kernel(hdev, hw_queue_id);
264 if (rc == -ETIMEDOUT) {
265 dev_err(hdev->dev, "Device CPU packet timeout (0x%x)\n", tmp);
266 hdev->device_cpu_disabled = true;
270 tmp = le32_to_cpu(pkt->ctl);
272 rc = (tmp & CPUCP_PKT_CTL_RC_MASK) >> CPUCP_PKT_CTL_RC_SHIFT;
274 dev_err(hdev->dev, "F/W ERROR %d for CPU packet %d\n",
276 (tmp & CPUCP_PKT_CTL_OPCODE_MASK)
277 >> CPUCP_PKT_CTL_OPCODE_SHIFT);
280 *result = le64_to_cpu(pkt->result);
283 /* Scrub previous buffer descriptor 'ctl' field which contains the
284 * previous PI value written during packet submission.
285 * We must do this or else F/W can read an old value upon queue wraparound.
287 sent_bd = queue->kernel_address;
288 sent_bd += hl_pi_2_offset(pi);
289 sent_bd->ctl = cpu_to_le32(UINT_MAX);
292 mutex_unlock(&hdev->send_cpu_message_lock);
294 hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev, len, pkt);
299 int hl_fw_unmask_irq(struct hl_device *hdev, u16 event_type)
301 struct cpucp_packet pkt;
305 memset(&pkt, 0, sizeof(pkt));
307 pkt.ctl = cpu_to_le32(CPUCP_PACKET_UNMASK_RAZWI_IRQ <<
308 CPUCP_PKT_CTL_OPCODE_SHIFT);
309 pkt.value = cpu_to_le64(event_type);
311 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
315 dev_err(hdev->dev, "failed to unmask RAZWI IRQ %d", event_type);
320 int hl_fw_unmask_irq_arr(struct hl_device *hdev, const u32 *irq_arr,
323 struct cpucp_unmask_irq_arr_packet *pkt;
324 size_t total_pkt_size;
328 total_pkt_size = sizeof(struct cpucp_unmask_irq_arr_packet) +
331 /* data should be aligned to 8 bytes in order to CPU-CP to copy it */
332 total_pkt_size = (total_pkt_size + 0x7) & ~0x7;
334 /* total_pkt_size is casted to u16 later on */
335 if (total_pkt_size > USHRT_MAX) {
336 dev_err(hdev->dev, "too many elements in IRQ array\n");
340 pkt = kzalloc(total_pkt_size, GFP_KERNEL);
344 pkt->length = cpu_to_le32(irq_arr_size / sizeof(irq_arr[0]));
345 memcpy(&pkt->irqs, irq_arr, irq_arr_size);
347 pkt->cpucp_pkt.ctl = cpu_to_le32(CPUCP_PACKET_UNMASK_RAZWI_IRQ_ARRAY <<
348 CPUCP_PKT_CTL_OPCODE_SHIFT);
350 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) pkt,
351 total_pkt_size, 0, &result);
354 dev_err(hdev->dev, "failed to unmask IRQ array\n");
361 int hl_fw_test_cpu_queue(struct hl_device *hdev)
363 struct cpucp_packet test_pkt = {};
367 test_pkt.ctl = cpu_to_le32(CPUCP_PACKET_TEST <<
368 CPUCP_PKT_CTL_OPCODE_SHIFT);
369 test_pkt.value = cpu_to_le64(CPUCP_PACKET_FENCE_VAL);
371 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &test_pkt,
372 sizeof(test_pkt), 0, &result);
375 if (result != CPUCP_PACKET_FENCE_VAL)
377 "CPU queue test failed (%#08llx)\n", result);
379 dev_err(hdev->dev, "CPU queue test failed, error %d\n", rc);
385 void *hl_fw_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size,
386 dma_addr_t *dma_handle)
390 kernel_addr = gen_pool_alloc(hdev->cpu_accessible_dma_pool, size);
392 *dma_handle = hdev->cpu_accessible_dma_address +
393 (kernel_addr - (u64) (uintptr_t) hdev->cpu_accessible_dma_mem);
395 return (void *) (uintptr_t) kernel_addr;
398 void hl_fw_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size,
401 gen_pool_free(hdev->cpu_accessible_dma_pool, (u64) (uintptr_t) vaddr,
405 int hl_fw_send_heartbeat(struct hl_device *hdev)
407 struct cpucp_packet hb_pkt;
411 memset(&hb_pkt, 0, sizeof(hb_pkt));
412 hb_pkt.ctl = cpu_to_le32(CPUCP_PACKET_TEST <<
413 CPUCP_PKT_CTL_OPCODE_SHIFT);
414 hb_pkt.value = cpu_to_le64(CPUCP_PACKET_FENCE_VAL);
416 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &hb_pkt,
417 sizeof(hb_pkt), 0, &result);
419 if ((rc) || (result != CPUCP_PACKET_FENCE_VAL))
422 if (le32_to_cpu(hb_pkt.status_mask) &
423 CPUCP_PKT_HB_STATUS_EQ_FAULT_MASK) {
424 dev_warn(hdev->dev, "FW reported EQ fault during heartbeat\n");
431 static bool fw_report_boot_dev0(struct hl_device *hdev, u32 err_val,
434 bool err_exists = false;
436 if (!(err_val & CPU_BOOT_ERR0_ENABLED))
439 if (err_val & CPU_BOOT_ERR0_DRAM_INIT_FAIL) {
441 "Device boot error - DRAM initialization failed\n");
445 if (err_val & CPU_BOOT_ERR0_FIT_CORRUPTED) {
446 dev_err(hdev->dev, "Device boot error - FIT image corrupted\n");
450 if (err_val & CPU_BOOT_ERR0_TS_INIT_FAIL) {
452 "Device boot error - Thermal Sensor initialization failed\n");
456 if (err_val & CPU_BOOT_ERR0_BMC_WAIT_SKIPPED) {
457 if (hdev->bmc_enable) {
459 "Device boot error - Skipped waiting for BMC\n");
463 "Device boot message - Skipped waiting for BMC\n");
464 /* This is an info so we don't want it to disable the
467 err_val &= ~CPU_BOOT_ERR0_BMC_WAIT_SKIPPED;
471 if (err_val & CPU_BOOT_ERR0_NIC_DATA_NOT_RDY) {
473 "Device boot error - Serdes data from BMC not available\n");
477 if (err_val & CPU_BOOT_ERR0_NIC_FW_FAIL) {
479 "Device boot error - NIC F/W initialization failed\n");
483 if (err_val & CPU_BOOT_ERR0_SECURITY_NOT_RDY) {
485 "Device boot warning - security not ready\n");
489 if (err_val & CPU_BOOT_ERR0_SECURITY_FAIL) {
490 dev_err(hdev->dev, "Device boot error - security failure\n");
494 if (err_val & CPU_BOOT_ERR0_EFUSE_FAIL) {
495 dev_err(hdev->dev, "Device boot error - eFuse failure\n");
499 if (err_val & CPU_BOOT_ERR0_SEC_IMG_VER_FAIL) {
500 dev_err(hdev->dev, "Device boot error - Failed to load preboot secondary image\n");
504 if (err_val & CPU_BOOT_ERR0_PLL_FAIL) {
505 dev_err(hdev->dev, "Device boot error - PLL failure\n");
509 if (err_val & CPU_BOOT_ERR0_DEVICE_UNUSABLE_FAIL) {
510 /* Ignore this bit, don't prevent driver loading */
511 dev_dbg(hdev->dev, "device unusable status is set\n");
512 err_val &= ~CPU_BOOT_ERR0_DEVICE_UNUSABLE_FAIL;
515 if (sts_val & CPU_BOOT_DEV_STS0_ENABLED)
516 dev_dbg(hdev->dev, "Device status0 %#x\n", sts_val);
518 /* All warnings should go here in order not to reach the unknown error validation */
519 if (err_val & CPU_BOOT_ERR0_DRAM_SKIPPED) {
521 "Device boot warning - Skipped DRAM initialization\n");
522 /* This is a warning so we don't want it to disable the
525 err_val &= ~CPU_BOOT_ERR0_DRAM_SKIPPED;
528 if (err_val & CPU_BOOT_ERR0_PRI_IMG_VER_FAIL) {
530 "Device boot warning - Failed to load preboot primary image\n");
531 /* This is a warning so we don't want it to disable the
532 * device as we have a secondary preboot image
534 err_val &= ~CPU_BOOT_ERR0_PRI_IMG_VER_FAIL;
537 if (err_val & CPU_BOOT_ERR0_TPM_FAIL) {
539 "Device boot warning - TPM failure\n");
540 /* This is a warning so we don't want it to disable the
543 err_val &= ~CPU_BOOT_ERR0_TPM_FAIL;
546 if (!err_exists && (err_val & ~CPU_BOOT_ERR0_ENABLED)) {
548 "Device boot error - unknown ERR0 error 0x%08x\n", err_val);
552 /* return error only if it's in the predefined mask */
553 if (err_exists && ((err_val & ~CPU_BOOT_ERR0_ENABLED) &
554 lower_32_bits(hdev->boot_error_status_mask)))
560 /* placeholder for ERR1 as no errors defined there yet */
561 static bool fw_report_boot_dev1(struct hl_device *hdev, u32 err_val,
565 * keep this variable to preserve the logic of the function.
566 * this way it would require less modifications when error will be
569 bool err_exists = false;
571 if (!(err_val & CPU_BOOT_ERR1_ENABLED))
574 if (sts_val & CPU_BOOT_DEV_STS1_ENABLED)
575 dev_dbg(hdev->dev, "Device status1 %#x\n", sts_val);
577 if (!err_exists && (err_val & ~CPU_BOOT_ERR1_ENABLED)) {
579 "Device boot error - unknown ERR1 error 0x%08x\n",
584 /* return error only if it's in the predefined mask */
585 if (err_exists && ((err_val & ~CPU_BOOT_ERR1_ENABLED) &
586 upper_32_bits(hdev->boot_error_status_mask)))
592 static int fw_read_errors(struct hl_device *hdev, u32 boot_err0_reg,
593 u32 boot_err1_reg, u32 cpu_boot_dev_status0_reg,
594 u32 cpu_boot_dev_status1_reg)
596 u32 err_val, status_val;
597 bool err_exists = false;
599 /* Some of the firmware status codes are deprecated in newer f/w
600 * versions. In those versions, the errors are reported
601 * in different registers. Therefore, we need to check those
602 * registers and print the exact errors. Moreover, there
603 * may be multiple errors, so we need to report on each error
604 * separately. Some of the error codes might indicate a state
605 * that is not an error per-se, but it is an error in production
608 err_val = RREG32(boot_err0_reg);
609 status_val = RREG32(cpu_boot_dev_status0_reg);
610 err_exists = fw_report_boot_dev0(hdev, err_val, status_val);
612 err_val = RREG32(boot_err1_reg);
613 status_val = RREG32(cpu_boot_dev_status1_reg);
614 err_exists |= fw_report_boot_dev1(hdev, err_val, status_val);
622 int hl_fw_cpucp_info_get(struct hl_device *hdev,
623 u32 sts_boot_dev_sts0_reg,
624 u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg,
627 struct asic_fixed_properties *prop = &hdev->asic_prop;
628 struct cpucp_packet pkt = {};
629 dma_addr_t cpucp_info_dma_addr;
630 void *cpucp_info_cpu_addr;
635 cpucp_info_cpu_addr =
636 hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev,
637 sizeof(struct cpucp_info),
638 &cpucp_info_dma_addr);
639 if (!cpucp_info_cpu_addr) {
641 "Failed to allocate DMA memory for CPU-CP info packet\n");
645 memset(cpucp_info_cpu_addr, 0, sizeof(struct cpucp_info));
647 pkt.ctl = cpu_to_le32(CPUCP_PACKET_INFO_GET <<
648 CPUCP_PKT_CTL_OPCODE_SHIFT);
649 pkt.addr = cpu_to_le64(cpucp_info_dma_addr);
650 pkt.data_max_size = cpu_to_le32(sizeof(struct cpucp_info));
652 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
653 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
656 "Failed to handle CPU-CP info pkt, error %d\n", rc);
660 rc = fw_read_errors(hdev, boot_err0_reg, boot_err1_reg,
661 sts_boot_dev_sts0_reg, sts_boot_dev_sts1_reg);
663 dev_err(hdev->dev, "Errors in device boot\n");
667 memcpy(&prop->cpucp_info, cpucp_info_cpu_addr,
668 sizeof(prop->cpucp_info));
670 rc = hl_build_hwmon_channel_info(hdev, prop->cpucp_info.sensors);
673 "Failed to build hwmon channel info, error %d\n", rc);
678 kernel_ver = extract_fw_ver_from_str(prop->cpucp_info.kernel_version);
680 dev_info(hdev->dev, "Linux version %s", kernel_ver);
684 /* assume EQ code doesn't need to check eqe index */
685 hdev->event_queue.check_eqe_index = false;
687 /* Read FW application security bits again */
688 if (prop->fw_cpu_boot_dev_sts0_valid) {
689 prop->fw_app_cpu_boot_dev_sts0 = RREG32(sts_boot_dev_sts0_reg);
690 if (prop->fw_app_cpu_boot_dev_sts0 &
691 CPU_BOOT_DEV_STS0_EQ_INDEX_EN)
692 hdev->event_queue.check_eqe_index = true;
695 if (prop->fw_cpu_boot_dev_sts1_valid)
696 prop->fw_app_cpu_boot_dev_sts1 = RREG32(sts_boot_dev_sts1_reg);
699 hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev,
700 sizeof(struct cpucp_info), cpucp_info_cpu_addr);
705 static int hl_fw_send_msi_info_msg(struct hl_device *hdev)
707 struct cpucp_array_data_packet *pkt;
708 size_t total_pkt_size, data_size;
712 /* skip sending this info for unsupported ASICs */
713 if (!hdev->asic_funcs->get_msi_info)
716 data_size = CPUCP_NUM_OF_MSI_TYPES * sizeof(u32);
717 total_pkt_size = sizeof(struct cpucp_array_data_packet) + data_size;
719 /* data should be aligned to 8 bytes in order to CPU-CP to copy it */
720 total_pkt_size = (total_pkt_size + 0x7) & ~0x7;
722 /* total_pkt_size is casted to u16 later on */
723 if (total_pkt_size > USHRT_MAX) {
724 dev_err(hdev->dev, "CPUCP array data is too big\n");
728 pkt = kzalloc(total_pkt_size, GFP_KERNEL);
732 pkt->length = cpu_to_le32(CPUCP_NUM_OF_MSI_TYPES);
734 memset((void *) &pkt->data, 0xFF, data_size);
735 hdev->asic_funcs->get_msi_info(pkt->data);
737 pkt->cpucp_pkt.ctl = cpu_to_le32(CPUCP_PACKET_MSI_INFO_SET <<
738 CPUCP_PKT_CTL_OPCODE_SHIFT);
740 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *)pkt,
741 total_pkt_size, 0, &result);
744 * in case packet result is invalid it means that FW does not support
745 * this feature and will use default/hard coded MSI values. no reason
748 if (rc && result == cpucp_packet_invalid)
752 dev_err(hdev->dev, "failed to send CPUCP array data\n");
759 int hl_fw_cpucp_handshake(struct hl_device *hdev,
760 u32 sts_boot_dev_sts0_reg,
761 u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg,
766 rc = hl_fw_cpucp_info_get(hdev, sts_boot_dev_sts0_reg,
767 sts_boot_dev_sts1_reg, boot_err0_reg,
772 return hl_fw_send_msi_info_msg(hdev);
775 int hl_fw_get_eeprom_data(struct hl_device *hdev, void *data, size_t max_size)
777 struct cpucp_packet pkt = {};
778 void *eeprom_info_cpu_addr;
779 dma_addr_t eeprom_info_dma_addr;
783 eeprom_info_cpu_addr =
784 hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev,
785 max_size, &eeprom_info_dma_addr);
786 if (!eeprom_info_cpu_addr) {
788 "Failed to allocate DMA memory for CPU-CP EEPROM packet\n");
792 memset(eeprom_info_cpu_addr, 0, max_size);
794 pkt.ctl = cpu_to_le32(CPUCP_PACKET_EEPROM_DATA_GET <<
795 CPUCP_PKT_CTL_OPCODE_SHIFT);
796 pkt.addr = cpu_to_le64(eeprom_info_dma_addr);
797 pkt.data_max_size = cpu_to_le32(max_size);
799 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
800 HL_CPUCP_EEPROM_TIMEOUT_USEC, &result);
804 "Failed to handle CPU-CP EEPROM packet, error %d\n",
809 /* result contains the actual size */
810 memcpy(data, eeprom_info_cpu_addr, min((size_t)result, max_size));
813 hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev, max_size,
814 eeprom_info_cpu_addr);
819 int hl_fw_cpucp_pci_counters_get(struct hl_device *hdev,
820 struct hl_info_pci_counters *counters)
822 struct cpucp_packet pkt = {};
826 pkt.ctl = cpu_to_le32(CPUCP_PACKET_PCIE_THROUGHPUT_GET <<
827 CPUCP_PKT_CTL_OPCODE_SHIFT);
829 /* Fetch PCI rx counter */
830 pkt.index = cpu_to_le32(cpucp_pcie_throughput_rx);
831 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
832 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
835 "Failed to handle CPU-CP PCI info pkt, error %d\n", rc);
838 counters->rx_throughput = result;
840 memset(&pkt, 0, sizeof(pkt));
841 pkt.ctl = cpu_to_le32(CPUCP_PACKET_PCIE_THROUGHPUT_GET <<
842 CPUCP_PKT_CTL_OPCODE_SHIFT);
844 /* Fetch PCI tx counter */
845 pkt.index = cpu_to_le32(cpucp_pcie_throughput_tx);
846 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
847 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
850 "Failed to handle CPU-CP PCI info pkt, error %d\n", rc);
853 counters->tx_throughput = result;
855 /* Fetch PCI replay counter */
856 memset(&pkt, 0, sizeof(pkt));
857 pkt.ctl = cpu_to_le32(CPUCP_PACKET_PCIE_REPLAY_CNT_GET <<
858 CPUCP_PKT_CTL_OPCODE_SHIFT);
860 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
861 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
864 "Failed to handle CPU-CP PCI info pkt, error %d\n", rc);
867 counters->replay_cnt = (u32) result;
872 int hl_fw_cpucp_total_energy_get(struct hl_device *hdev, u64 *total_energy)
874 struct cpucp_packet pkt = {};
878 pkt.ctl = cpu_to_le32(CPUCP_PACKET_TOTAL_ENERGY_GET <<
879 CPUCP_PKT_CTL_OPCODE_SHIFT);
881 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
882 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
885 "Failed to handle CpuCP total energy pkt, error %d\n",
890 *total_energy = result;
895 int get_used_pll_index(struct hl_device *hdev, u32 input_pll_index,
896 enum pll_index *pll_index)
898 struct asic_fixed_properties *prop = &hdev->asic_prop;
899 u8 pll_byte, pll_bit_off;
903 dynamic_pll = !!(prop->fw_app_cpu_boot_dev_sts0 &
904 CPU_BOOT_DEV_STS0_DYN_PLL_EN);
908 * in case we are working with legacy FW (each asic has unique
909 * PLL numbering) use the driver based index as they are
910 * aligned with fw legacy numbering
912 *pll_index = input_pll_index;
916 /* retrieve a FW compatible PLL index based on
917 * ASIC specific user request
919 fw_pll_idx = hdev->asic_funcs->map_pll_idx_to_fw_idx(input_pll_index);
920 if (fw_pll_idx < 0) {
921 dev_err(hdev->dev, "Invalid PLL index (%u) error %d\n",
922 input_pll_index, fw_pll_idx);
926 /* PLL map is a u8 array */
927 pll_byte = prop->cpucp_info.pll_map[fw_pll_idx >> 3];
928 pll_bit_off = fw_pll_idx & 0x7;
930 if (!(pll_byte & BIT(pll_bit_off))) {
931 dev_err(hdev->dev, "PLL index %d is not supported\n",
936 *pll_index = fw_pll_idx;
941 int hl_fw_cpucp_pll_info_get(struct hl_device *hdev, u32 pll_index,
944 struct cpucp_packet pkt;
945 enum pll_index used_pll_idx;
949 rc = get_used_pll_index(hdev, pll_index, &used_pll_idx);
953 memset(&pkt, 0, sizeof(pkt));
955 pkt.ctl = cpu_to_le32(CPUCP_PACKET_PLL_INFO_GET <<
956 CPUCP_PKT_CTL_OPCODE_SHIFT);
957 pkt.pll_type = __cpu_to_le16((u16)used_pll_idx);
959 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
960 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
962 dev_err(hdev->dev, "Failed to read PLL info, error %d\n", rc);
964 pll_freq_arr[0] = FIELD_GET(CPUCP_PKT_RES_PLL_OUT0_MASK, result);
965 pll_freq_arr[1] = FIELD_GET(CPUCP_PKT_RES_PLL_OUT1_MASK, result);
966 pll_freq_arr[2] = FIELD_GET(CPUCP_PKT_RES_PLL_OUT2_MASK, result);
967 pll_freq_arr[3] = FIELD_GET(CPUCP_PKT_RES_PLL_OUT3_MASK, result);
972 int hl_fw_cpucp_power_get(struct hl_device *hdev, u64 *power)
974 struct cpucp_packet pkt;
978 memset(&pkt, 0, sizeof(pkt));
980 pkt.ctl = cpu_to_le32(CPUCP_PACKET_POWER_GET <<
981 CPUCP_PKT_CTL_OPCODE_SHIFT);
982 pkt.type = cpu_to_le16(CPUCP_POWER_INPUT);
984 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
985 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
987 dev_err(hdev->dev, "Failed to read power, error %d\n", rc);
996 int hl_fw_dram_replaced_row_get(struct hl_device *hdev,
997 struct cpucp_hbm_row_info *info)
999 struct cpucp_hbm_row_info *cpucp_repl_rows_info_cpu_addr;
1000 dma_addr_t cpucp_repl_rows_info_dma_addr;
1001 struct cpucp_packet pkt = {};
1005 cpucp_repl_rows_info_cpu_addr =
1006 hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev,
1007 sizeof(struct cpucp_hbm_row_info),
1008 &cpucp_repl_rows_info_dma_addr);
1009 if (!cpucp_repl_rows_info_cpu_addr) {
1011 "Failed to allocate DMA memory for CPU-CP replaced rows info packet\n");
1015 memset(cpucp_repl_rows_info_cpu_addr, 0, sizeof(struct cpucp_hbm_row_info));
1017 pkt.ctl = cpu_to_le32(CPUCP_PACKET_HBM_REPLACED_ROWS_INFO_GET <<
1018 CPUCP_PKT_CTL_OPCODE_SHIFT);
1019 pkt.addr = cpu_to_le64(cpucp_repl_rows_info_dma_addr);
1020 pkt.data_max_size = cpu_to_le32(sizeof(struct cpucp_hbm_row_info));
1022 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
1023 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
1026 "Failed to handle CPU-CP replaced rows info pkt, error %d\n", rc);
1030 memcpy(info, cpucp_repl_rows_info_cpu_addr, sizeof(*info));
1033 hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev,
1034 sizeof(struct cpucp_hbm_row_info),
1035 cpucp_repl_rows_info_cpu_addr);
1040 int hl_fw_dram_pending_row_get(struct hl_device *hdev, u32 *pend_rows_num)
1042 struct cpucp_packet pkt;
1046 memset(&pkt, 0, sizeof(pkt));
1048 pkt.ctl = cpu_to_le32(CPUCP_PACKET_HBM_PENDING_ROWS_STATUS << CPUCP_PKT_CTL_OPCODE_SHIFT);
1050 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, &result);
1053 "Failed to handle CPU-CP pending rows info pkt, error %d\n", rc);
1057 *pend_rows_num = (u32) result;
1062 int hl_fw_cpucp_engine_core_asid_set(struct hl_device *hdev, u32 asid)
1064 struct cpucp_packet pkt;
1067 memset(&pkt, 0, sizeof(pkt));
1069 pkt.ctl = cpu_to_le32(CPUCP_PACKET_ENGINE_CORE_ASID_SET << CPUCP_PKT_CTL_OPCODE_SHIFT);
1070 pkt.value = cpu_to_le64(asid);
1072 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
1073 HL_CPUCP_INFO_TIMEOUT_USEC, NULL);
1076 "Failed on ASID configuration request for engine core, error %d\n",
1082 void hl_fw_ask_hard_reset_without_linux(struct hl_device *hdev)
1084 struct static_fw_load_mgr *static_loader =
1085 &hdev->fw_loader.static_loader;
1088 if (hdev->asic_prop.dynamic_fw_load) {
1089 rc = hl_fw_dynamic_send_protocol_cmd(hdev, &hdev->fw_loader,
1090 COMMS_RST_DEV, 0, false,
1091 hdev->fw_loader.cpu_timeout);
1093 dev_warn(hdev->dev, "Failed sending COMMS_RST_DEV\n");
1095 WREG32(static_loader->kmd_msg_to_cpu_reg, KMD_MSG_RST_DEV);
1099 void hl_fw_ask_halt_machine_without_linux(struct hl_device *hdev)
1101 struct static_fw_load_mgr *static_loader =
1102 &hdev->fw_loader.static_loader;
1105 if (hdev->device_cpu_is_halted)
1108 /* Stop device CPU to make sure nothing bad happens */
1109 if (hdev->asic_prop.dynamic_fw_load) {
1110 rc = hl_fw_dynamic_send_protocol_cmd(hdev, &hdev->fw_loader,
1111 COMMS_GOTO_WFE, 0, true,
1112 hdev->fw_loader.cpu_timeout);
1114 dev_warn(hdev->dev, "Failed sending COMMS_GOTO_WFE\n");
1116 WREG32(static_loader->kmd_msg_to_cpu_reg, KMD_MSG_GOTO_WFE);
1117 msleep(static_loader->cpu_reset_wait_msec);
1119 /* Must clear this register in order to prevent preboot
1120 * from reading WFE after reboot
1122 WREG32(static_loader->kmd_msg_to_cpu_reg, KMD_MSG_NA);
1125 hdev->device_cpu_is_halted = true;
1128 static void detect_cpu_boot_status(struct hl_device *hdev, u32 status)
1130 /* Some of the status codes below are deprecated in newer f/w
1131 * versions but we keep them here for backward compatibility
1134 case CPU_BOOT_STATUS_NA:
1136 "Device boot progress - BTL/ROM did NOT run\n");
1138 case CPU_BOOT_STATUS_IN_WFE:
1140 "Device boot progress - Stuck inside WFE loop\n");
1142 case CPU_BOOT_STATUS_IN_BTL:
1144 "Device boot progress - Stuck in BTL\n");
1146 case CPU_BOOT_STATUS_IN_PREBOOT:
1148 "Device boot progress - Stuck in Preboot\n");
1150 case CPU_BOOT_STATUS_IN_SPL:
1152 "Device boot progress - Stuck in SPL\n");
1154 case CPU_BOOT_STATUS_IN_UBOOT:
1156 "Device boot progress - Stuck in u-boot\n");
1158 case CPU_BOOT_STATUS_DRAM_INIT_FAIL:
1160 "Device boot progress - DRAM initialization failed\n");
1162 case CPU_BOOT_STATUS_UBOOT_NOT_READY:
1164 "Device boot progress - Cannot boot\n");
1166 case CPU_BOOT_STATUS_TS_INIT_FAIL:
1168 "Device boot progress - Thermal Sensor initialization failed\n");
1170 case CPU_BOOT_STATUS_SECURITY_READY:
1172 "Device boot progress - Stuck in preboot after security initialization\n");
1176 "Device boot progress - Invalid status code %d\n",
1182 static int hl_fw_read_preboot_caps(struct hl_device *hdev,
1183 u32 cpu_boot_status_reg,
1184 u32 sts_boot_dev_sts0_reg,
1185 u32 sts_boot_dev_sts1_reg,
1186 u32 boot_err0_reg, u32 boot_err1_reg,
1189 struct asic_fixed_properties *prop = &hdev->asic_prop;
1190 u32 status, reg_val;
1193 /* Need to check two possible scenarios:
1195 * CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT - for newer firmwares where
1196 * the preboot is waiting for the boot fit
1198 * All other status values - for older firmwares where the uboot was
1199 * loaded from the FLASH
1201 rc = hl_poll_timeout(
1203 cpu_boot_status_reg,
1205 (status == CPU_BOOT_STATUS_IN_UBOOT) ||
1206 (status == CPU_BOOT_STATUS_DRAM_RDY) ||
1207 (status == CPU_BOOT_STATUS_NIC_FW_RDY) ||
1208 (status == CPU_BOOT_STATUS_READY_TO_BOOT) ||
1209 (status == CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT),
1210 hdev->fw_poll_interval_usec,
1214 dev_err(hdev->dev, "CPU boot ready status timeout\n");
1215 detect_cpu_boot_status(hdev, status);
1217 /* If we read all FF, then something is totally wrong, no point
1218 * of reading specific errors
1221 fw_read_errors(hdev, boot_err0_reg, boot_err1_reg,
1222 sts_boot_dev_sts0_reg,
1223 sts_boot_dev_sts1_reg);
1228 * the registers DEV_STS* contain FW capabilities/features.
1229 * We can rely on this registers only if bit CPU_BOOT_DEV_STS*_ENABLED
1231 * In the first read of this register we store the value of this
1232 * register ONLY if the register is enabled (which will be propagated
1233 * to next stages) and also mark the register as valid.
1234 * In case it is not enabled the stored value will be left 0- all
1235 * caps/features are off
1237 reg_val = RREG32(sts_boot_dev_sts0_reg);
1238 if (reg_val & CPU_BOOT_DEV_STS0_ENABLED) {
1239 prop->fw_cpu_boot_dev_sts0_valid = true;
1240 prop->fw_preboot_cpu_boot_dev_sts0 = reg_val;
1243 reg_val = RREG32(sts_boot_dev_sts1_reg);
1244 if (reg_val & CPU_BOOT_DEV_STS1_ENABLED) {
1245 prop->fw_cpu_boot_dev_sts1_valid = true;
1246 prop->fw_preboot_cpu_boot_dev_sts1 = reg_val;
1249 prop->dynamic_fw_load = !!(prop->fw_preboot_cpu_boot_dev_sts0 &
1250 CPU_BOOT_DEV_STS0_FW_LD_COM_EN);
1252 /* initialize FW loader once we know what load protocol is used */
1253 hdev->asic_funcs->init_firmware_loader(hdev);
1255 dev_dbg(hdev->dev, "Attempting %s FW load\n",
1256 prop->dynamic_fw_load ? "dynamic" : "legacy");
1260 static int hl_fw_static_read_device_fw_version(struct hl_device *hdev,
1261 enum hl_fw_component fwc)
1263 struct asic_fixed_properties *prop = &hdev->asic_prop;
1264 struct fw_load_mgr *fw_loader = &hdev->fw_loader;
1265 struct static_fw_load_mgr *static_loader;
1266 char *dest, *boot_ver, *preboot_ver;
1271 static_loader = &hdev->fw_loader.static_loader;
1274 case FW_COMP_BOOT_FIT:
1275 ver_off = RREG32(static_loader->boot_fit_version_offset_reg);
1276 dest = prop->uboot_ver;
1278 limit = static_loader->boot_fit_version_max_off;
1280 case FW_COMP_PREBOOT:
1281 ver_off = RREG32(static_loader->preboot_version_offset_reg);
1282 dest = prop->preboot_ver;
1284 limit = static_loader->preboot_version_max_off;
1287 dev_warn(hdev->dev, "Undefined FW component: %d\n", fwc);
1291 ver_off &= static_loader->sram_offset_mask;
1293 if (ver_off < limit) {
1295 hdev->pcie_bar[fw_loader->sram_bar_id] + ver_off,
1298 dev_err(hdev->dev, "%s version offset (0x%x) is above SRAM\n",
1300 strscpy(dest, "unavailable", VERSION_MAX_LEN);
1304 if (fwc == FW_COMP_BOOT_FIT) {
1305 boot_ver = extract_fw_ver_from_str(prop->uboot_ver);
1307 dev_info(hdev->dev, "boot-fit version %s\n", boot_ver);
1310 } else if (fwc == FW_COMP_PREBOOT) {
1311 preboot_ver = strnstr(prop->preboot_ver, "Preboot",
1313 if (preboot_ver && preboot_ver != prop->preboot_ver) {
1314 strscpy(btl_ver, prop->preboot_ver,
1315 min((int) (preboot_ver - prop->preboot_ver),
1317 dev_info(hdev->dev, "%s\n", btl_ver);
1320 preboot_ver = extract_fw_ver_from_str(prop->preboot_ver);
1322 dev_info(hdev->dev, "preboot version %s\n",
1332 * hl_fw_preboot_update_state - update internal data structures during
1333 * handshake with preboot
1336 * @hdev: pointer to the habanalabs device structure
1338 * @return 0 on success, otherwise non-zero error code
1340 static void hl_fw_preboot_update_state(struct hl_device *hdev)
1342 struct asic_fixed_properties *prop = &hdev->asic_prop;
1343 u32 cpu_boot_dev_sts0, cpu_boot_dev_sts1;
1345 cpu_boot_dev_sts0 = prop->fw_preboot_cpu_boot_dev_sts0;
1346 cpu_boot_dev_sts1 = prop->fw_preboot_cpu_boot_dev_sts1;
1348 /* We read boot_dev_sts registers multiple times during boot:
1349 * 1. preboot - a. Check whether the security status bits are valid
1350 * b. Check whether fw security is enabled
1351 * c. Check whether hard reset is done by preboot
1352 * 2. boot cpu - a. Fetch boot cpu security status
1353 * b. Check whether hard reset is done by boot cpu
1354 * 3. FW application - a. Fetch fw application security status
1355 * b. Check whether hard reset is done by fw app
1357 prop->hard_reset_done_by_fw = !!(cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_FW_HARD_RST_EN);
1359 dev_dbg(hdev->dev, "Firmware preboot boot device status0 %#x\n",
1362 dev_dbg(hdev->dev, "Firmware preboot boot device status1 %#x\n",
1365 dev_dbg(hdev->dev, "Firmware preboot hard-reset is %s\n",
1366 prop->hard_reset_done_by_fw ? "enabled" : "disabled");
1368 dev_dbg(hdev->dev, "firmware-level security is %s\n",
1369 prop->fw_security_enabled ? "enabled" : "disabled");
1371 dev_dbg(hdev->dev, "GIC controller is %s\n",
1372 prop->gic_interrupts_enable ? "enabled" : "disabled");
1375 static int hl_fw_static_read_preboot_status(struct hl_device *hdev)
1379 rc = hl_fw_static_read_device_fw_version(hdev, FW_COMP_PREBOOT);
1386 int hl_fw_read_preboot_status(struct hl_device *hdev, u32 cpu_boot_status_reg,
1387 u32 sts_boot_dev_sts0_reg,
1388 u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg,
1389 u32 boot_err1_reg, u32 timeout)
1393 if (!(hdev->fw_components & FW_TYPE_PREBOOT_CPU))
1397 * In order to determine boot method (static VS dymanic) we need to
1398 * read the boot caps register
1400 rc = hl_fw_read_preboot_caps(hdev, cpu_boot_status_reg,
1401 sts_boot_dev_sts0_reg,
1402 sts_boot_dev_sts1_reg, boot_err0_reg,
1403 boot_err1_reg, timeout);
1407 hl_fw_preboot_update_state(hdev);
1409 /* no need to read preboot status in dynamic load */
1410 if (hdev->asic_prop.dynamic_fw_load)
1413 return hl_fw_static_read_preboot_status(hdev);
1416 /* associate string with COMM status */
1417 static char *hl_dynamic_fw_status_str[COMMS_STS_INVLD_LAST] = {
1418 [COMMS_STS_NOOP] = "NOOP",
1419 [COMMS_STS_ACK] = "ACK",
1420 [COMMS_STS_OK] = "OK",
1421 [COMMS_STS_ERR] = "ERR",
1422 [COMMS_STS_VALID_ERR] = "VALID_ERR",
1423 [COMMS_STS_TIMEOUT_ERR] = "TIMEOUT_ERR",
1427 * hl_fw_dynamic_report_error_status - report error status
1429 * @hdev: pointer to the habanalabs device structure
1430 * @status: value of FW status register
1431 * @expected_status: the expected status
1433 static void hl_fw_dynamic_report_error_status(struct hl_device *hdev,
1435 enum comms_sts expected_status)
1437 enum comms_sts comm_status =
1438 FIELD_GET(COMMS_STATUS_STATUS_MASK, status);
1440 if (comm_status < COMMS_STS_INVLD_LAST)
1441 dev_err(hdev->dev, "Device status %s, expected status: %s\n",
1442 hl_dynamic_fw_status_str[comm_status],
1443 hl_dynamic_fw_status_str[expected_status]);
1445 dev_err(hdev->dev, "Device status unknown %d, expected status: %s\n",
1447 hl_dynamic_fw_status_str[expected_status]);
1451 * hl_fw_dynamic_send_cmd - send LKD to FW cmd
1453 * @hdev: pointer to the habanalabs device structure
1454 * @fw_loader: managing structure for loading device's FW
1455 * @cmd: LKD to FW cmd code
1456 * @size: size of next FW component to be loaded (0 if not necessary)
1458 * LDK to FW exact command layout is defined at struct comms_command.
1459 * note: the size argument is used only when the next FW component should be
1460 * loaded, otherwise it shall be 0. the size is used by the FW in later
1461 * protocol stages and when sending only indicating the amount of memory
1462 * to be allocated by the FW to receive the next boot component.
1464 static void hl_fw_dynamic_send_cmd(struct hl_device *hdev,
1465 struct fw_load_mgr *fw_loader,
1466 enum comms_cmd cmd, unsigned int size)
1468 struct cpu_dyn_regs *dyn_regs;
1471 dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs;
1473 val = FIELD_PREP(COMMS_COMMAND_CMD_MASK, cmd);
1474 val |= FIELD_PREP(COMMS_COMMAND_SIZE_MASK, size);
1476 WREG32(le32_to_cpu(dyn_regs->kmd_msg_to_cpu), val);
1480 * hl_fw_dynamic_extract_fw_response - update the FW response
1482 * @hdev: pointer to the habanalabs device structure
1483 * @fw_loader: managing structure for loading device's FW
1484 * @response: FW response
1485 * @status: the status read from CPU status register
1487 * @return 0 on success, otherwise non-zero error code
1489 static int hl_fw_dynamic_extract_fw_response(struct hl_device *hdev,
1490 struct fw_load_mgr *fw_loader,
1491 struct fw_response *response,
1494 response->status = FIELD_GET(COMMS_STATUS_STATUS_MASK, status);
1495 response->ram_offset = FIELD_GET(COMMS_STATUS_OFFSET_MASK, status) <<
1496 COMMS_STATUS_OFFSET_ALIGN_SHIFT;
1497 response->ram_type = FIELD_GET(COMMS_STATUS_RAM_TYPE_MASK, status);
1499 if ((response->ram_type != COMMS_SRAM) &&
1500 (response->ram_type != COMMS_DRAM)) {
1501 dev_err(hdev->dev, "FW status: invalid RAM type %u\n",
1502 response->ram_type);
1510 * hl_fw_dynamic_wait_for_status - wait for status in dynamic FW load
1512 * @hdev: pointer to the habanalabs device structure
1513 * @fw_loader: managing structure for loading device's FW
1514 * @expected_status: expected status to wait for
1515 * @timeout: timeout for status wait
1517 * @return 0 on success, otherwise non-zero error code
1519 * waiting for status from FW include polling the FW status register until
1520 * expected status is received or timeout occurs (whatever occurs first).
1522 static int hl_fw_dynamic_wait_for_status(struct hl_device *hdev,
1523 struct fw_load_mgr *fw_loader,
1524 enum comms_sts expected_status,
1527 struct cpu_dyn_regs *dyn_regs;
1531 dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs;
1533 /* Wait for expected status */
1534 rc = hl_poll_timeout(
1536 le32_to_cpu(dyn_regs->cpu_cmd_status_to_host),
1538 FIELD_GET(COMMS_STATUS_STATUS_MASK, status) == expected_status,
1539 hdev->fw_poll_interval_usec,
1543 hl_fw_dynamic_report_error_status(hdev, status,
1549 * skip storing FW response for NOOP to preserve the actual desired
1552 if (expected_status == COMMS_STS_NOOP)
1555 rc = hl_fw_dynamic_extract_fw_response(hdev, fw_loader,
1556 &fw_loader->dynamic_loader.response,
1562 * hl_fw_dynamic_send_clear_cmd - send clear command to FW
1564 * @hdev: pointer to the habanalabs device structure
1565 * @fw_loader: managing structure for loading device's FW
1567 * @return 0 on success, otherwise non-zero error code
1569 * after command cycle between LKD to FW CPU (i.e. LKD got an expected status
1570 * from FW) we need to clear the CPU status register in order to avoid garbage
1571 * between command cycles.
1572 * This is done by sending clear command and polling the CPU to LKD status
1573 * register to hold the status NOOP
1575 static int hl_fw_dynamic_send_clear_cmd(struct hl_device *hdev,
1576 struct fw_load_mgr *fw_loader)
1578 hl_fw_dynamic_send_cmd(hdev, fw_loader, COMMS_CLR_STS, 0);
1580 return hl_fw_dynamic_wait_for_status(hdev, fw_loader, COMMS_STS_NOOP,
1581 fw_loader->cpu_timeout);
1585 * hl_fw_dynamic_send_protocol_cmd - send LKD to FW cmd and wait for ACK
1587 * @hdev: pointer to the habanalabs device structure
1588 * @fw_loader: managing structure for loading device's FW
1589 * @cmd: LKD to FW cmd code
1590 * @size: size of next FW component to be loaded (0 if not necessary)
1591 * @wait_ok: if true also wait for OK response from FW
1592 * @timeout: timeout for status wait
1594 * @return 0 on success, otherwise non-zero error code
1597 * when sending protocol command we have the following steps:
1598 * - send clear (clear command and verify clear status register)
1599 * - send the actual protocol command
1600 * - wait for ACK on the protocol command
1603 * if, in addition, the specific protocol command should wait for OK then:
1609 * send clear: this is necessary in order to clear the status register to avoid
1610 * leftovers between command
1611 * NOOP command: necessary to avoid loop on the clear command by the FW
1613 int hl_fw_dynamic_send_protocol_cmd(struct hl_device *hdev,
1614 struct fw_load_mgr *fw_loader,
1615 enum comms_cmd cmd, unsigned int size,
1616 bool wait_ok, u32 timeout)
1620 /* first send clear command to clean former commands */
1621 rc = hl_fw_dynamic_send_clear_cmd(hdev, fw_loader);
1623 /* send the actual command */
1624 hl_fw_dynamic_send_cmd(hdev, fw_loader, cmd, size);
1626 /* wait for ACK for the command */
1627 rc = hl_fw_dynamic_wait_for_status(hdev, fw_loader, COMMS_STS_ACK,
1632 /* clear command to prepare for NOOP command */
1633 rc = hl_fw_dynamic_send_clear_cmd(hdev, fw_loader);
1637 /* send the actual NOOP command */
1638 hl_fw_dynamic_send_cmd(hdev, fw_loader, COMMS_NOOP, 0);
1643 rc = hl_fw_dynamic_wait_for_status(hdev, fw_loader, COMMS_STS_OK,
1648 /* clear command to prepare for NOOP command */
1649 rc = hl_fw_dynamic_send_clear_cmd(hdev, fw_loader);
1653 /* send the actual NOOP command */
1654 hl_fw_dynamic_send_cmd(hdev, fw_loader, COMMS_NOOP, 0);
1660 * hl_fw_compat_crc32 - CRC compatible with FW
1662 * @data: pointer to the data
1663 * @size: size of the data
1665 * @return the CRC32 result
1667 * NOTE: kernel's CRC32 differ's from standard CRC32 calculation.
1668 * in order to be aligned we need to flip the bits of both the input
1669 * initial CRC and kernel's CRC32 result.
1670 * in addition both sides use initial CRC of 0,
1672 static u32 hl_fw_compat_crc32(u8 *data, size_t size)
1674 return ~crc32_le(~((u32)0), data, size);
1678 * hl_fw_dynamic_validate_memory_bound - validate memory bounds for memory
1679 * transfer (image or descriptor) between
1682 * @hdev: pointer to the habanalabs device structure
1683 * @addr: device address of memory transfer
1684 * @size: memory transter size
1685 * @region: PCI memory region
1687 * @return 0 on success, otherwise non-zero error code
1689 static int hl_fw_dynamic_validate_memory_bound(struct hl_device *hdev,
1690 u64 addr, size_t size,
1691 struct pci_mem_region *region)
1695 /* now make sure that the memory transfer is within region's bounds */
1696 end_addr = addr + size;
1697 if (end_addr >= region->region_base + region->region_size) {
1699 "dynamic FW load: memory transfer end address out of memory region bounds. addr: %llx\n",
1705 * now make sure memory transfer is within predefined BAR bounds.
1706 * this is to make sure we do not need to set the bar (e.g. for DRAM
1709 if (end_addr >= region->region_base - region->offset_in_bar +
1712 "FW image beyond PCI BAR bounds\n");
1720 * hl_fw_dynamic_validate_descriptor - validate FW descriptor
1722 * @hdev: pointer to the habanalabs device structure
1723 * @fw_loader: managing structure for loading device's FW
1724 * @fw_desc: the descriptor form FW
1726 * @return 0 on success, otherwise non-zero error code
1728 static int hl_fw_dynamic_validate_descriptor(struct hl_device *hdev,
1729 struct fw_load_mgr *fw_loader,
1730 struct lkd_fw_comms_desc *fw_desc)
1732 struct pci_mem_region *region;
1733 enum pci_region region_id;
1740 if (le32_to_cpu(fw_desc->header.magic) != HL_COMMS_DESC_MAGIC) {
1741 dev_err(hdev->dev, "Invalid magic for dynamic FW descriptor (%x)\n",
1742 fw_desc->header.magic);
1746 if (fw_desc->header.version != HL_COMMS_DESC_VER) {
1747 dev_err(hdev->dev, "Invalid version for dynamic FW descriptor (%x)\n",
1748 fw_desc->header.version);
1753 * calc CRC32 of data without header.
1754 * note that no alignment/stride address issues here as all structures
1757 data_size = sizeof(struct lkd_fw_comms_desc) -
1758 sizeof(struct comms_desc_header);
1759 data_ptr = (u8 *)fw_desc + sizeof(struct comms_desc_header);
1761 if (le16_to_cpu(fw_desc->header.size) != data_size) {
1763 "Invalid descriptor size 0x%x, expected size 0x%zx\n",
1764 le16_to_cpu(fw_desc->header.size), data_size);
1768 data_crc32 = hl_fw_compat_crc32(data_ptr, data_size);
1770 if (data_crc32 != le32_to_cpu(fw_desc->header.crc32)) {
1772 "CRC32 mismatch for dynamic FW descriptor (%x:%x)\n",
1773 data_crc32, fw_desc->header.crc32);
1777 /* find memory region to which to copy the image */
1778 addr = le64_to_cpu(fw_desc->img_addr);
1779 region_id = hl_get_pci_memory_region(hdev, addr);
1780 if ((region_id != PCI_REGION_SRAM) &&
1781 ((region_id != PCI_REGION_DRAM))) {
1783 "Invalid region to copy FW image address=%llx\n", addr);
1787 region = &hdev->pci_mem_region[region_id];
1789 /* store the region for the copy stage */
1790 fw_loader->dynamic_loader.image_region = region;
1793 * here we know that the start address is valid, now make sure that the
1794 * image is within region's bounds
1796 rc = hl_fw_dynamic_validate_memory_bound(hdev, addr,
1797 fw_loader->dynamic_loader.fw_image_size,
1801 "invalid mem transfer request for FW image\n");
1805 /* here we can mark the descriptor as valid as the content has been validated */
1806 fw_loader->dynamic_loader.fw_desc_valid = true;
1811 static int hl_fw_dynamic_validate_response(struct hl_device *hdev,
1812 struct fw_response *response,
1813 struct pci_mem_region *region)
1818 device_addr = region->region_base + response->ram_offset;
1821 * validate that the descriptor is within region's bounds
1822 * Note that as the start address was supplied according to the RAM
1823 * type- testing only the end address is enough
1825 rc = hl_fw_dynamic_validate_memory_bound(hdev, device_addr,
1826 sizeof(struct lkd_fw_comms_desc),
1832 * hl_fw_dynamic_read_and_validate_descriptor - read and validate FW descriptor
1834 * @hdev: pointer to the habanalabs device structure
1835 * @fw_loader: managing structure for loading device's FW
1837 * @return 0 on success, otherwise non-zero error code
1839 static int hl_fw_dynamic_read_and_validate_descriptor(struct hl_device *hdev,
1840 struct fw_load_mgr *fw_loader)
1842 struct lkd_fw_comms_desc *fw_desc;
1843 struct pci_mem_region *region;
1844 struct fw_response *response;
1845 enum pci_region region_id;
1849 fw_desc = &fw_loader->dynamic_loader.comm_desc;
1850 response = &fw_loader->dynamic_loader.response;
1852 region_id = (response->ram_type == COMMS_SRAM) ?
1853 PCI_REGION_SRAM : PCI_REGION_DRAM;
1855 region = &hdev->pci_mem_region[region_id];
1857 rc = hl_fw_dynamic_validate_response(hdev, response, region);
1860 "invalid mem transfer request for FW descriptor\n");
1865 * extract address to copy the descriptor from
1866 * in addition, as the descriptor value is going to be over-ridden by new data- we mark it
1868 * it will be marked again as valid once validated
1870 fw_loader->dynamic_loader.fw_desc_valid = false;
1871 src = hdev->pcie_bar[region->bar_id] + region->offset_in_bar +
1872 response->ram_offset;
1873 memcpy_fromio(fw_desc, src, sizeof(struct lkd_fw_comms_desc));
1875 return hl_fw_dynamic_validate_descriptor(hdev, fw_loader, fw_desc);
1879 * hl_fw_dynamic_request_descriptor - handshake with CPU to get FW descriptor
1881 * @hdev: pointer to the habanalabs device structure
1882 * @fw_loader: managing structure for loading device's FW
1883 * @next_image_size: size to allocate for next FW component
1885 * @return 0 on success, otherwise non-zero error code
1887 static int hl_fw_dynamic_request_descriptor(struct hl_device *hdev,
1888 struct fw_load_mgr *fw_loader,
1889 size_t next_image_size)
1893 rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_PREP_DESC,
1894 next_image_size, true,
1895 fw_loader->cpu_timeout);
1899 return hl_fw_dynamic_read_and_validate_descriptor(hdev, fw_loader);
1903 * hl_fw_dynamic_read_device_fw_version - read FW version to exposed properties
1905 * @hdev: pointer to the habanalabs device structure
1906 * @fwc: the firmware component
1907 * @fw_version: fw component's version string
1909 static void hl_fw_dynamic_read_device_fw_version(struct hl_device *hdev,
1910 enum hl_fw_component fwc,
1911 const char *fw_version)
1913 struct asic_fixed_properties *prop = &hdev->asic_prop;
1914 char *preboot_ver, *boot_ver;
1918 case FW_COMP_BOOT_FIT:
1919 strscpy(prop->uboot_ver, fw_version, VERSION_MAX_LEN);
1920 boot_ver = extract_fw_ver_from_str(prop->uboot_ver);
1922 dev_info(hdev->dev, "boot-fit version %s\n", boot_ver);
1927 case FW_COMP_PREBOOT:
1928 strscpy(prop->preboot_ver, fw_version, VERSION_MAX_LEN);
1929 preboot_ver = strnstr(prop->preboot_ver, "Preboot",
1931 if (preboot_ver && preboot_ver != prop->preboot_ver) {
1932 strscpy(btl_ver, prop->preboot_ver,
1933 min((int) (preboot_ver - prop->preboot_ver),
1935 dev_info(hdev->dev, "%s\n", btl_ver);
1938 preboot_ver = extract_fw_ver_from_str(prop->preboot_ver);
1940 dev_info(hdev->dev, "preboot version %s\n",
1947 dev_warn(hdev->dev, "Undefined FW component: %d\n", fwc);
1953 * hl_fw_dynamic_copy_image - copy image to memory allocated by the FW
1955 * @hdev: pointer to the habanalabs device structure
1956 * @fw: fw descriptor
1957 * @fw_loader: managing structure for loading device's FW
1959 static int hl_fw_dynamic_copy_image(struct hl_device *hdev,
1960 const struct firmware *fw,
1961 struct fw_load_mgr *fw_loader)
1963 struct lkd_fw_comms_desc *fw_desc;
1964 struct pci_mem_region *region;
1969 fw_desc = &fw_loader->dynamic_loader.comm_desc;
1970 addr = le64_to_cpu(fw_desc->img_addr);
1972 /* find memory region to which to copy the image */
1973 region = fw_loader->dynamic_loader.image_region;
1975 dest = hdev->pcie_bar[region->bar_id] + region->offset_in_bar +
1976 (addr - region->region_base);
1978 rc = hl_fw_copy_fw_to_device(hdev, fw, dest,
1979 fw_loader->boot_fit_img.src_off,
1980 fw_loader->boot_fit_img.copy_size);
1986 * hl_fw_dynamic_copy_msg - copy msg to memory allocated by the FW
1988 * @hdev: pointer to the habanalabs device structure
1990 * @fw_loader: managing structure for loading device's FW
1992 static int hl_fw_dynamic_copy_msg(struct hl_device *hdev,
1993 struct lkd_msg_comms *msg, struct fw_load_mgr *fw_loader)
1995 struct lkd_fw_comms_desc *fw_desc;
1996 struct pci_mem_region *region;
2001 fw_desc = &fw_loader->dynamic_loader.comm_desc;
2002 addr = le64_to_cpu(fw_desc->img_addr);
2004 /* find memory region to which to copy the image */
2005 region = fw_loader->dynamic_loader.image_region;
2007 dest = hdev->pcie_bar[region->bar_id] + region->offset_in_bar +
2008 (addr - region->region_base);
2010 rc = hl_fw_copy_msg_to_device(hdev, msg, dest, 0, 0);
2016 * hl_fw_boot_fit_update_state - update internal data structures after boot-fit
2019 * @hdev: pointer to the habanalabs device structure
2020 * @cpu_boot_dev_sts0_reg: register holding CPU boot dev status 0
2021 * @cpu_boot_dev_sts1_reg: register holding CPU boot dev status 1
2023 * @return 0 on success, otherwise non-zero error code
2025 static void hl_fw_boot_fit_update_state(struct hl_device *hdev,
2026 u32 cpu_boot_dev_sts0_reg,
2027 u32 cpu_boot_dev_sts1_reg)
2029 struct asic_fixed_properties *prop = &hdev->asic_prop;
2031 hdev->fw_loader.fw_comp_loaded |= FW_TYPE_BOOT_CPU;
2033 /* Read boot_cpu status bits */
2034 if (prop->fw_preboot_cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_ENABLED) {
2035 prop->fw_bootfit_cpu_boot_dev_sts0 =
2036 RREG32(cpu_boot_dev_sts0_reg);
2038 prop->hard_reset_done_by_fw = !!(prop->fw_bootfit_cpu_boot_dev_sts0 &
2039 CPU_BOOT_DEV_STS0_FW_HARD_RST_EN);
2041 dev_dbg(hdev->dev, "Firmware boot CPU status0 %#x\n",
2042 prop->fw_bootfit_cpu_boot_dev_sts0);
2045 if (prop->fw_cpu_boot_dev_sts1_valid) {
2046 prop->fw_bootfit_cpu_boot_dev_sts1 =
2047 RREG32(cpu_boot_dev_sts1_reg);
2049 dev_dbg(hdev->dev, "Firmware boot CPU status1 %#x\n",
2050 prop->fw_bootfit_cpu_boot_dev_sts1);
2053 dev_dbg(hdev->dev, "Firmware boot CPU hard-reset is %s\n",
2054 prop->hard_reset_done_by_fw ? "enabled" : "disabled");
2057 static void hl_fw_dynamic_update_linux_interrupt_if(struct hl_device *hdev)
2059 struct cpu_dyn_regs *dyn_regs =
2060 &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
2062 /* Check whether all 3 interrupt interfaces are set, if not use a
2065 if (!hdev->asic_prop.gic_interrupts_enable &&
2066 !(hdev->asic_prop.fw_app_cpu_boot_dev_sts0 &
2067 CPU_BOOT_DEV_STS0_MULTI_IRQ_POLL_EN)) {
2068 dyn_regs->gic_host_halt_irq = dyn_regs->gic_host_pi_upd_irq;
2069 dyn_regs->gic_host_ints_irq = dyn_regs->gic_host_pi_upd_irq;
2072 "Using a single interrupt interface towards cpucp");
2076 * hl_fw_dynamic_load_image - load FW image using dynamic protocol
2078 * @hdev: pointer to the habanalabs device structure
2079 * @fw_loader: managing structure for loading device's FW
2080 * @load_fwc: the FW component to be loaded
2081 * @img_ld_timeout: image load timeout
2083 * @return 0 on success, otherwise non-zero error code
2085 static int hl_fw_dynamic_load_image(struct hl_device *hdev,
2086 struct fw_load_mgr *fw_loader,
2087 enum hl_fw_component load_fwc,
2090 enum hl_fw_component cur_fwc;
2091 const struct firmware *fw;
2096 * when loading image we have one of 2 scenarios:
2097 * 1. current FW component is preboot and we want to load boot-fit
2098 * 2. current FW component is boot-fit and we want to load linux
2100 if (load_fwc == FW_COMP_BOOT_FIT) {
2101 cur_fwc = FW_COMP_PREBOOT;
2102 fw_name = fw_loader->boot_fit_img.image_name;
2104 cur_fwc = FW_COMP_BOOT_FIT;
2105 fw_name = fw_loader->linux_img.image_name;
2108 /* request FW in order to communicate to FW the size to be allocated */
2109 rc = hl_request_fw(hdev, &fw, fw_name);
2113 /* store the image size for future validation */
2114 fw_loader->dynamic_loader.fw_image_size = fw->size;
2116 rc = hl_fw_dynamic_request_descriptor(hdev, fw_loader, fw->size);
2120 /* read preboot version */
2121 hl_fw_dynamic_read_device_fw_version(hdev, cur_fwc,
2122 fw_loader->dynamic_loader.comm_desc.cur_fw_ver);
2125 /* update state according to boot stage */
2126 if (cur_fwc == FW_COMP_BOOT_FIT) {
2127 struct cpu_dyn_regs *dyn_regs;
2129 dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs;
2130 hl_fw_boot_fit_update_state(hdev,
2131 le32_to_cpu(dyn_regs->cpu_boot_dev_sts0),
2132 le32_to_cpu(dyn_regs->cpu_boot_dev_sts1));
2135 /* copy boot fit to space allocated by FW */
2136 rc = hl_fw_dynamic_copy_image(hdev, fw, fw_loader);
2140 rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_DATA_RDY,
2142 fw_loader->cpu_timeout);
2146 rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_EXEC,
2151 hl_release_firmware(fw);
2155 static int hl_fw_dynamic_wait_for_boot_fit_active(struct hl_device *hdev,
2156 struct fw_load_mgr *fw_loader)
2158 struct dynamic_fw_load_mgr *dyn_loader;
2162 dyn_loader = &fw_loader->dynamic_loader;
2165 * Make sure CPU boot-loader is running
2166 * Note that the CPU_BOOT_STATUS_SRAM_AVAIL is generally set by Linux
2167 * yet there is a debug scenario in which we loading uboot (without Linux)
2168 * which at later stage is relocated to DRAM. In this case we expect
2169 * uboot to set the CPU_BOOT_STATUS_SRAM_AVAIL and so we add it to the
2172 rc = hl_poll_timeout(
2174 le32_to_cpu(dyn_loader->comm_desc.cpu_dyn_regs.cpu_boot_status),
2176 (status == CPU_BOOT_STATUS_READY_TO_BOOT) ||
2177 (status == CPU_BOOT_STATUS_SRAM_AVAIL),
2178 hdev->fw_poll_interval_usec,
2179 dyn_loader->wait_for_bl_timeout);
2181 dev_err(hdev->dev, "failed to wait for boot\n");
2185 dev_dbg(hdev->dev, "uboot status = %d\n", status);
2189 static int hl_fw_dynamic_wait_for_linux_active(struct hl_device *hdev,
2190 struct fw_load_mgr *fw_loader)
2192 struct dynamic_fw_load_mgr *dyn_loader;
2196 dyn_loader = &fw_loader->dynamic_loader;
2198 /* Make sure CPU linux is running */
2200 rc = hl_poll_timeout(
2202 le32_to_cpu(dyn_loader->comm_desc.cpu_dyn_regs.cpu_boot_status),
2204 (status == CPU_BOOT_STATUS_SRAM_AVAIL),
2205 hdev->fw_poll_interval_usec,
2206 fw_loader->cpu_timeout);
2208 dev_err(hdev->dev, "failed to wait for Linux\n");
2212 dev_dbg(hdev->dev, "Boot status = %d\n", status);
2217 * hl_fw_linux_update_state - update internal data structures after Linux
2219 * Note: Linux initialization is comprised mainly
2220 * of two stages - loading kernel (SRAM_AVAIL)
2222 * Therefore reading boot device status in any of
2223 * these stages might result in different values.
2225 * @hdev: pointer to the habanalabs device structure
2226 * @cpu_boot_dev_sts0_reg: register holding CPU boot dev status 0
2227 * @cpu_boot_dev_sts1_reg: register holding CPU boot dev status 1
2229 * @return 0 on success, otherwise non-zero error code
2231 static void hl_fw_linux_update_state(struct hl_device *hdev,
2232 u32 cpu_boot_dev_sts0_reg,
2233 u32 cpu_boot_dev_sts1_reg)
2235 struct asic_fixed_properties *prop = &hdev->asic_prop;
2237 hdev->fw_loader.fw_comp_loaded |= FW_TYPE_LINUX;
2239 /* Read FW application security bits */
2240 if (prop->fw_cpu_boot_dev_sts0_valid) {
2241 prop->fw_app_cpu_boot_dev_sts0 = RREG32(cpu_boot_dev_sts0_reg);
2243 prop->hard_reset_done_by_fw = !!(prop->fw_app_cpu_boot_dev_sts0 &
2244 CPU_BOOT_DEV_STS0_FW_HARD_RST_EN);
2246 if (prop->fw_app_cpu_boot_dev_sts0 &
2247 CPU_BOOT_DEV_STS0_GIC_PRIVILEGED_EN)
2248 prop->gic_interrupts_enable = false;
2251 "Firmware application CPU status0 %#x\n",
2252 prop->fw_app_cpu_boot_dev_sts0);
2254 dev_dbg(hdev->dev, "GIC controller is %s\n",
2255 prop->gic_interrupts_enable ?
2256 "enabled" : "disabled");
2259 if (prop->fw_cpu_boot_dev_sts1_valid) {
2260 prop->fw_app_cpu_boot_dev_sts1 = RREG32(cpu_boot_dev_sts1_reg);
2263 "Firmware application CPU status1 %#x\n",
2264 prop->fw_app_cpu_boot_dev_sts1);
2267 dev_dbg(hdev->dev, "Firmware application CPU hard-reset is %s\n",
2268 prop->hard_reset_done_by_fw ? "enabled" : "disabled");
2270 dev_info(hdev->dev, "Successfully loaded firmware to device\n");
2274 * hl_fw_dynamic_send_msg - send a COMMS message with attached data
2276 * @hdev: pointer to the habanalabs device structure
2277 * @fw_loader: managing structure for loading device's FW
2278 * @msg_type: message type
2279 * @data: data to be sent
2281 * @return 0 on success, otherwise non-zero error code
2283 static int hl_fw_dynamic_send_msg(struct hl_device *hdev,
2284 struct fw_load_mgr *fw_loader, u8 msg_type, void *data)
2286 struct lkd_msg_comms msg;
2289 memset(&msg, 0, sizeof(msg));
2291 /* create message to be sent */
2292 msg.header.type = msg_type;
2293 msg.header.size = cpu_to_le16(sizeof(struct comms_msg_header));
2294 msg.header.magic = cpu_to_le32(HL_COMMS_MSG_MAGIC);
2297 case HL_COMMS_RESET_CAUSE_TYPE:
2298 msg.reset_cause = *(__u8 *) data;
2302 "Send COMMS message - invalid message type %u\n",
2307 rc = hl_fw_dynamic_request_descriptor(hdev, fw_loader,
2308 sizeof(struct lkd_msg_comms));
2312 /* copy message to space allocated by FW */
2313 rc = hl_fw_dynamic_copy_msg(hdev, &msg, fw_loader);
2317 rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_DATA_RDY,
2319 fw_loader->cpu_timeout);
2323 rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_EXEC,
2325 fw_loader->cpu_timeout);
2333 * hl_fw_dynamic_init_cpu - initialize the device CPU using dynamic protocol
2335 * @hdev: pointer to the habanalabs device structure
2336 * @fw_loader: managing structure for loading device's FW
2338 * @return 0 on success, otherwise non-zero error code
2340 * brief: the dynamic protocol is master (LKD) slave (FW CPU) protocol.
2341 * the communication is done using registers:
2342 * - LKD command register
2343 * - FW status register
2344 * the protocol is race free. this goal is achieved by splitting the requests
2345 * and response to known synchronization points between the LKD and the FW.
2346 * each response to LKD request is known and bound to a predefined timeout.
2347 * in case of timeout expiration without the desired status from FW- the
2348 * protocol (and hence the boot) will fail.
2350 static int hl_fw_dynamic_init_cpu(struct hl_device *hdev,
2351 struct fw_load_mgr *fw_loader)
2353 struct cpu_dyn_regs *dyn_regs;
2357 "Loading firmware to device, may take some time...\n");
2359 /* initialize FW descriptor as invalid */
2360 fw_loader->dynamic_loader.fw_desc_valid = false;
2363 * In this stage, "cpu_dyn_regs" contains only LKD's hard coded values!
2364 * It will be updated from FW after hl_fw_dynamic_request_descriptor().
2366 dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs;
2368 rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_RST_STATE,
2370 fw_loader->cpu_timeout);
2374 if (hdev->reset_info.curr_reset_cause) {
2375 rc = hl_fw_dynamic_send_msg(hdev, fw_loader,
2376 HL_COMMS_RESET_CAUSE_TYPE, &hdev->reset_info.curr_reset_cause);
2380 /* Clear current reset cause */
2381 hdev->reset_info.curr_reset_cause = HL_RESET_CAUSE_UNKNOWN;
2384 if (!(hdev->fw_components & FW_TYPE_BOOT_CPU)) {
2385 rc = hl_fw_dynamic_request_descriptor(hdev, fw_loader, 0);
2389 /* read preboot version */
2390 hl_fw_dynamic_read_device_fw_version(hdev, FW_COMP_PREBOOT,
2391 fw_loader->dynamic_loader.comm_desc.cur_fw_ver);
2395 /* load boot fit to FW */
2396 rc = hl_fw_dynamic_load_image(hdev, fw_loader, FW_COMP_BOOT_FIT,
2397 fw_loader->boot_fit_timeout);
2399 dev_err(hdev->dev, "failed to load boot fit\n");
2404 * when testing FW load (without Linux) on PLDM we don't want to
2405 * wait until boot fit is active as it may take several hours.
2406 * instead, we load the bootfit and let it do all initializations in
2409 if (hdev->pldm && !(hdev->fw_components & FW_TYPE_LINUX))
2412 rc = hl_fw_dynamic_wait_for_boot_fit_active(hdev, fw_loader);
2416 /* Enable DRAM scrambling before Linux boot and after successful
2419 hdev->asic_funcs->init_cpu_scrambler_dram(hdev);
2421 if (!(hdev->fw_components & FW_TYPE_LINUX)) {
2422 dev_info(hdev->dev, "Skip loading Linux F/W\n");
2426 if (fw_loader->skip_bmc) {
2427 rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader,
2430 fw_loader->cpu_timeout);
2432 dev_err(hdev->dev, "failed to load boot fit\n");
2437 /* load Linux image to FW */
2438 rc = hl_fw_dynamic_load_image(hdev, fw_loader, FW_COMP_LINUX,
2439 fw_loader->cpu_timeout);
2441 dev_err(hdev->dev, "failed to load Linux\n");
2445 rc = hl_fw_dynamic_wait_for_linux_active(hdev, fw_loader);
2449 hl_fw_linux_update_state(hdev, le32_to_cpu(dyn_regs->cpu_boot_dev_sts0),
2450 le32_to_cpu(dyn_regs->cpu_boot_dev_sts1));
2452 hl_fw_dynamic_update_linux_interrupt_if(hdev);
2457 if (fw_loader->dynamic_loader.fw_desc_valid)
2458 fw_read_errors(hdev, le32_to_cpu(dyn_regs->cpu_boot_err0),
2459 le32_to_cpu(dyn_regs->cpu_boot_err1),
2460 le32_to_cpu(dyn_regs->cpu_boot_dev_sts0),
2461 le32_to_cpu(dyn_regs->cpu_boot_dev_sts1));
2466 * hl_fw_static_init_cpu - initialize the device CPU using static protocol
2468 * @hdev: pointer to the habanalabs device structure
2469 * @fw_loader: managing structure for loading device's FW
2471 * @return 0 on success, otherwise non-zero error code
2473 static int hl_fw_static_init_cpu(struct hl_device *hdev,
2474 struct fw_load_mgr *fw_loader)
2476 u32 cpu_msg_status_reg, cpu_timeout, msg_to_cpu_reg, status;
2477 u32 cpu_boot_dev_status0_reg, cpu_boot_dev_status1_reg;
2478 struct static_fw_load_mgr *static_loader;
2479 u32 cpu_boot_status_reg;
2482 if (!(hdev->fw_components & FW_TYPE_BOOT_CPU))
2485 /* init common loader parameters */
2486 cpu_timeout = fw_loader->cpu_timeout;
2488 /* init static loader parameters */
2489 static_loader = &fw_loader->static_loader;
2490 cpu_msg_status_reg = static_loader->cpu_cmd_status_to_host_reg;
2491 msg_to_cpu_reg = static_loader->kmd_msg_to_cpu_reg;
2492 cpu_boot_dev_status0_reg = static_loader->cpu_boot_dev_status0_reg;
2493 cpu_boot_dev_status1_reg = static_loader->cpu_boot_dev_status1_reg;
2494 cpu_boot_status_reg = static_loader->cpu_boot_status_reg;
2496 dev_info(hdev->dev, "Going to wait for device boot (up to %lds)\n",
2497 cpu_timeout / USEC_PER_SEC);
2499 /* Wait for boot FIT request */
2500 rc = hl_poll_timeout(
2502 cpu_boot_status_reg,
2504 status == CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT,
2505 hdev->fw_poll_interval_usec,
2506 fw_loader->boot_fit_timeout);
2510 "No boot fit request received, resuming boot\n");
2512 rc = hdev->asic_funcs->load_boot_fit_to_device(hdev);
2516 /* Clear device CPU message status */
2517 WREG32(cpu_msg_status_reg, CPU_MSG_CLR);
2519 /* Signal device CPU that boot loader is ready */
2520 WREG32(msg_to_cpu_reg, KMD_MSG_FIT_RDY);
2522 /* Poll for CPU device ack */
2523 rc = hl_poll_timeout(
2527 status == CPU_MSG_OK,
2528 hdev->fw_poll_interval_usec,
2529 fw_loader->boot_fit_timeout);
2533 "Timeout waiting for boot fit load ack\n");
2538 WREG32(msg_to_cpu_reg, KMD_MSG_NA);
2542 * Make sure CPU boot-loader is running
2543 * Note that the CPU_BOOT_STATUS_SRAM_AVAIL is generally set by Linux
2544 * yet there is a debug scenario in which we loading uboot (without Linux)
2545 * which at later stage is relocated to DRAM. In this case we expect
2546 * uboot to set the CPU_BOOT_STATUS_SRAM_AVAIL and so we add it to the
2549 rc = hl_poll_timeout(
2551 cpu_boot_status_reg,
2553 (status == CPU_BOOT_STATUS_DRAM_RDY) ||
2554 (status == CPU_BOOT_STATUS_NIC_FW_RDY) ||
2555 (status == CPU_BOOT_STATUS_READY_TO_BOOT) ||
2556 (status == CPU_BOOT_STATUS_SRAM_AVAIL),
2557 hdev->fw_poll_interval_usec,
2560 dev_dbg(hdev->dev, "uboot status = %d\n", status);
2562 /* Read U-Boot version now in case we will later fail */
2563 hl_fw_static_read_device_fw_version(hdev, FW_COMP_BOOT_FIT);
2565 /* update state according to boot stage */
2566 hl_fw_boot_fit_update_state(hdev, cpu_boot_dev_status0_reg,
2567 cpu_boot_dev_status1_reg);
2570 detect_cpu_boot_status(hdev, status);
2575 /* Enable DRAM scrambling before Linux boot and after successful
2578 hdev->asic_funcs->init_cpu_scrambler_dram(hdev);
2580 if (!(hdev->fw_components & FW_TYPE_LINUX)) {
2581 dev_info(hdev->dev, "Skip loading Linux F/W\n");
2586 if (status == CPU_BOOT_STATUS_SRAM_AVAIL) {
2592 "Loading firmware to device, may take some time...\n");
2594 rc = hdev->asic_funcs->load_firmware_to_device(hdev);
2598 if (fw_loader->skip_bmc) {
2599 WREG32(msg_to_cpu_reg, KMD_MSG_SKIP_BMC);
2601 rc = hl_poll_timeout(
2603 cpu_boot_status_reg,
2605 (status == CPU_BOOT_STATUS_BMC_WAITING_SKIPPED),
2606 hdev->fw_poll_interval_usec,
2611 "Failed to get ACK on skipping BMC, %d\n",
2613 WREG32(msg_to_cpu_reg, KMD_MSG_NA);
2619 WREG32(msg_to_cpu_reg, KMD_MSG_FIT_RDY);
2621 rc = hl_poll_timeout(
2623 cpu_boot_status_reg,
2625 (status == CPU_BOOT_STATUS_SRAM_AVAIL),
2626 hdev->fw_poll_interval_usec,
2630 WREG32(msg_to_cpu_reg, KMD_MSG_NA);
2633 if (status == CPU_BOOT_STATUS_FIT_CORRUPTED)
2635 "Device reports FIT image is corrupted\n");
2638 "Failed to load firmware to device, %d\n",
2645 rc = fw_read_errors(hdev, fw_loader->static_loader.boot_err0_reg,
2646 fw_loader->static_loader.boot_err1_reg,
2647 cpu_boot_dev_status0_reg,
2648 cpu_boot_dev_status1_reg);
2652 hl_fw_linux_update_state(hdev, cpu_boot_dev_status0_reg,
2653 cpu_boot_dev_status1_reg);
2658 fw_read_errors(hdev, fw_loader->static_loader.boot_err0_reg,
2659 fw_loader->static_loader.boot_err1_reg,
2660 cpu_boot_dev_status0_reg,
2661 cpu_boot_dev_status1_reg);
2667 * hl_fw_init_cpu - initialize the device CPU
2669 * @hdev: pointer to the habanalabs device structure
2671 * @return 0 on success, otherwise non-zero error code
2673 * perform necessary initializations for device's CPU. takes into account if
2674 * init protocol is static or dynamic.
2676 int hl_fw_init_cpu(struct hl_device *hdev)
2678 struct asic_fixed_properties *prop = &hdev->asic_prop;
2679 struct fw_load_mgr *fw_loader = &hdev->fw_loader;
2681 return prop->dynamic_fw_load ?
2682 hl_fw_dynamic_init_cpu(hdev, fw_loader) :
2683 hl_fw_static_init_cpu(hdev, fw_loader);
2686 void hl_fw_set_pll_profile(struct hl_device *hdev, enum hl_pll_frequency freq)
2688 hl_fw_set_frequency(hdev, hdev->asic_prop.clk_pll_index,
2689 hdev->asic_prop.max_freq_value);
2692 int hl_fw_get_clk_rate(struct hl_device *hdev, u32 *cur_clk, u32 *max_clk)
2696 if (!hl_device_operational(hdev, NULL))
2705 value = hl_fw_get_frequency(hdev, hdev->asic_prop.clk_pll_index, false);
2708 dev_err(hdev->dev, "Failed to retrieve device max clock %ld\n", value);
2712 *max_clk = (value / 1000 / 1000);
2714 value = hl_fw_get_frequency(hdev, hdev->asic_prop.clk_pll_index, true);
2717 dev_err(hdev->dev, "Failed to retrieve device current clock %ld\n", value);
2721 *cur_clk = (value / 1000 / 1000);
2726 long hl_fw_get_frequency(struct hl_device *hdev, u32 pll_index, bool curr)
2728 struct cpucp_packet pkt;
2733 rc = get_used_pll_index(hdev, pll_index, &used_pll_idx);
2737 memset(&pkt, 0, sizeof(pkt));
2740 pkt.ctl = cpu_to_le32(CPUCP_PACKET_FREQUENCY_CURR_GET <<
2741 CPUCP_PKT_CTL_OPCODE_SHIFT);
2743 pkt.ctl = cpu_to_le32(CPUCP_PACKET_FREQUENCY_GET << CPUCP_PKT_CTL_OPCODE_SHIFT);
2745 pkt.pll_index = cpu_to_le32((u32)used_pll_idx);
2747 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, &result);
2750 dev_err(hdev->dev, "Failed to get frequency of PLL %d, error %d\n",
2755 return (long) result;
2758 void hl_fw_set_frequency(struct hl_device *hdev, u32 pll_index, u64 freq)
2760 struct cpucp_packet pkt;
2764 rc = get_used_pll_index(hdev, pll_index, &used_pll_idx);
2768 memset(&pkt, 0, sizeof(pkt));
2770 pkt.ctl = cpu_to_le32(CPUCP_PACKET_FREQUENCY_SET << CPUCP_PKT_CTL_OPCODE_SHIFT);
2771 pkt.pll_index = cpu_to_le32((u32)used_pll_idx);
2772 pkt.value = cpu_to_le64(freq);
2774 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, NULL);
2777 dev_err(hdev->dev, "Failed to set frequency to PLL %d, error %d\n",
2781 u64 hl_fw_get_max_power(struct hl_device *hdev)
2783 struct cpucp_packet pkt;
2787 memset(&pkt, 0, sizeof(pkt));
2789 pkt.ctl = cpu_to_le32(CPUCP_PACKET_MAX_POWER_GET << CPUCP_PKT_CTL_OPCODE_SHIFT);
2791 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, &result);
2794 dev_err(hdev->dev, "Failed to get max power, error %d\n", rc);
2801 void hl_fw_set_max_power(struct hl_device *hdev)
2803 struct cpucp_packet pkt;
2806 /* TODO: remove this after simulator supports this packet */
2810 memset(&pkt, 0, sizeof(pkt));
2812 pkt.ctl = cpu_to_le32(CPUCP_PACKET_MAX_POWER_SET << CPUCP_PKT_CTL_OPCODE_SHIFT);
2813 pkt.value = cpu_to_le64(hdev->max_power);
2815 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, NULL);
2818 dev_err(hdev->dev, "Failed to set max power, error %d\n", rc);