1 // SPDX-License-Identifier: GPL-2.0
4 * Copyright 2016-2021 HabanaLabs, Ltd.
8 #include "habanalabs.h"
9 #include "../include/common/hl_boot_if.h"
11 #include <linux/firmware.h>
12 #include <linux/crc32.h>
13 #include <linux/slab.h>
14 #include <linux/ctype.h>
16 #define FW_FILE_MAX_SIZE 0x1400000 /* maximum size of 20MB */
18 #define FW_CPU_STATUS_POLL_INTERVAL_USEC 10000
20 static char *extract_fw_ver_from_str(const char *fw_str)
22 char *str, *fw_ver, *whitespace;
24 fw_ver = kmalloc(16, GFP_KERNEL);
28 str = strnstr(fw_str, "fw-", VERSION_MAX_LEN);
32 /* Skip the fw- part */
35 /* Copy until the next whitespace */
36 whitespace = strnstr(str, " ", 15);
40 strscpy(fw_ver, str, whitespace - str + 1);
49 static int hl_request_fw(struct hl_device *hdev,
50 const struct firmware **firmware_p,
56 rc = request_firmware(firmware_p, fw_name, hdev->dev);
58 dev_err(hdev->dev, "Firmware file %s is not found! (error %d)\n",
63 fw_size = (*firmware_p)->size;
64 if ((fw_size % 4) != 0) {
65 dev_err(hdev->dev, "Illegal %s firmware size %zu\n",
71 dev_dbg(hdev->dev, "%s firmware size == %zu\n", fw_name, fw_size);
73 if (fw_size > FW_FILE_MAX_SIZE) {
75 "FW file size %zu exceeds maximum of %u bytes\n",
76 fw_size, FW_FILE_MAX_SIZE);
84 release_firmware(*firmware_p);
90 * hl_release_firmware() - release FW
94 * note: this inline function added to serve as a comprehensive mirror for the
95 * hl_request_fw function.
97 static inline void hl_release_firmware(const struct firmware *fw)
103 * hl_fw_copy_fw_to_device() - copy FW to device
105 * @hdev: pointer to hl_device structure.
107 * @dst: IO memory mapped address space to copy firmware to
108 * @src_offset: offset in src FW to copy from
109 * @size: amount of bytes to copy (0 to copy the whole binary)
111 * actual copy of FW binary data to device, shared by static and dynamic loaders
113 static int hl_fw_copy_fw_to_device(struct hl_device *hdev,
114 const struct firmware *fw, void __iomem *dst,
115 u32 src_offset, u32 size)
119 /* size 0 indicates to copy the whole file */
123 if (src_offset + size > fw->size) {
125 "size to copy(%u) and offset(%u) are invalid\n",
130 fw_data = (const void *) fw->data;
132 memcpy_toio(dst, fw_data + src_offset, size);
137 * hl_fw_copy_msg_to_device() - copy message to device
139 * @hdev: pointer to hl_device structure.
141 * @dst: IO memory mapped address space to copy firmware to
142 * @src_offset: offset in src message to copy from
143 * @size: amount of bytes to copy (0 to copy the whole binary)
145 * actual copy of message data to device.
147 static int hl_fw_copy_msg_to_device(struct hl_device *hdev,
148 struct lkd_msg_comms *msg, void __iomem *dst,
149 u32 src_offset, u32 size)
153 /* size 0 indicates to copy the whole file */
155 size = sizeof(struct lkd_msg_comms);
157 if (src_offset + size > sizeof(struct lkd_msg_comms)) {
159 "size to copy(%u) and offset(%u) are invalid\n",
164 msg_data = (void *) msg;
166 memcpy_toio(dst, msg_data + src_offset, size);
172 * hl_fw_load_fw_to_device() - Load F/W code to device's memory.
174 * @hdev: pointer to hl_device structure.
175 * @fw_name: the firmware image name
176 * @dst: IO memory mapped address space to copy firmware to
177 * @src_offset: offset in src FW to copy from
178 * @size: amount of bytes to copy (0 to copy the whole binary)
180 * Copy fw code from firmware file to device memory.
182 * Return: 0 on success, non-zero for failure.
184 int hl_fw_load_fw_to_device(struct hl_device *hdev, const char *fw_name,
185 void __iomem *dst, u32 src_offset, u32 size)
187 const struct firmware *fw;
190 rc = hl_request_fw(hdev, &fw, fw_name);
194 rc = hl_fw_copy_fw_to_device(hdev, fw, dst, src_offset, size);
196 hl_release_firmware(fw);
200 int hl_fw_send_pci_access_msg(struct hl_device *hdev, u32 opcode)
202 struct cpucp_packet pkt = {};
204 pkt.ctl = cpu_to_le32(opcode << CPUCP_PKT_CTL_OPCODE_SHIFT);
206 return hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt,
207 sizeof(pkt), 0, NULL);
210 int hl_fw_send_cpu_message(struct hl_device *hdev, u32 hw_queue_id, u32 *msg,
211 u16 len, u32 timeout, u64 *result)
213 struct hl_hw_queue *queue = &hdev->kernel_queues[hw_queue_id];
214 struct asic_fixed_properties *prop = &hdev->asic_prop;
215 struct cpucp_packet *pkt;
216 dma_addr_t pkt_dma_addr;
217 u32 tmp, expected_ack_val;
220 pkt = hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev, len,
224 "Failed to allocate DMA memory for packet to CPU\n");
228 memcpy(pkt, msg, len);
230 mutex_lock(&hdev->send_cpu_message_lock);
235 if (hdev->device_cpu_disabled) {
240 /* set fence to a non valid value */
241 pkt->fence = cpu_to_le32(UINT_MAX);
244 * The CPU queue is a synchronous queue with an effective depth of
245 * a single entry (although it is allocated with room for multiple
246 * entries). We lock on it using 'send_cpu_message_lock' which
247 * serializes accesses to the CPU queue.
248 * Which means that we don't need to lock the access to the entire H/W
249 * queues module when submitting a JOB to the CPU queue.
251 hl_hw_queue_submit_bd(hdev, queue, 0, len, pkt_dma_addr);
253 if (prop->fw_app_cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_PKT_PI_ACK_EN)
254 expected_ack_val = queue->pi;
256 expected_ack_val = CPUCP_PACKET_FENCE_VAL;
258 rc = hl_poll_timeout_memory(hdev, &pkt->fence, tmp,
259 (tmp == expected_ack_val), 1000,
262 hl_hw_queue_inc_ci_kernel(hdev, hw_queue_id);
264 if (rc == -ETIMEDOUT) {
265 dev_err(hdev->dev, "Device CPU packet timeout (0x%x)\n", tmp);
266 hdev->device_cpu_disabled = true;
270 tmp = le32_to_cpu(pkt->ctl);
272 rc = (tmp & CPUCP_PKT_CTL_RC_MASK) >> CPUCP_PKT_CTL_RC_SHIFT;
274 dev_err(hdev->dev, "F/W ERROR %d for CPU packet %d\n",
276 (tmp & CPUCP_PKT_CTL_OPCODE_MASK)
277 >> CPUCP_PKT_CTL_OPCODE_SHIFT);
280 *result = le64_to_cpu(pkt->result);
284 mutex_unlock(&hdev->send_cpu_message_lock);
286 hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev, len, pkt);
291 int hl_fw_unmask_irq(struct hl_device *hdev, u16 event_type)
293 struct cpucp_packet pkt;
297 memset(&pkt, 0, sizeof(pkt));
299 pkt.ctl = cpu_to_le32(CPUCP_PACKET_UNMASK_RAZWI_IRQ <<
300 CPUCP_PKT_CTL_OPCODE_SHIFT);
301 pkt.value = cpu_to_le64(event_type);
303 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
307 dev_err(hdev->dev, "failed to unmask RAZWI IRQ %d", event_type);
312 int hl_fw_unmask_irq_arr(struct hl_device *hdev, const u32 *irq_arr,
315 struct cpucp_unmask_irq_arr_packet *pkt;
316 size_t total_pkt_size;
320 total_pkt_size = sizeof(struct cpucp_unmask_irq_arr_packet) +
323 /* data should be aligned to 8 bytes in order to CPU-CP to copy it */
324 total_pkt_size = (total_pkt_size + 0x7) & ~0x7;
326 /* total_pkt_size is casted to u16 later on */
327 if (total_pkt_size > USHRT_MAX) {
328 dev_err(hdev->dev, "too many elements in IRQ array\n");
332 pkt = kzalloc(total_pkt_size, GFP_KERNEL);
336 pkt->length = cpu_to_le32(irq_arr_size / sizeof(irq_arr[0]));
337 memcpy(&pkt->irqs, irq_arr, irq_arr_size);
339 pkt->cpucp_pkt.ctl = cpu_to_le32(CPUCP_PACKET_UNMASK_RAZWI_IRQ_ARRAY <<
340 CPUCP_PKT_CTL_OPCODE_SHIFT);
342 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) pkt,
343 total_pkt_size, 0, &result);
346 dev_err(hdev->dev, "failed to unmask IRQ array\n");
353 int hl_fw_test_cpu_queue(struct hl_device *hdev)
355 struct cpucp_packet test_pkt = {};
359 test_pkt.ctl = cpu_to_le32(CPUCP_PACKET_TEST <<
360 CPUCP_PKT_CTL_OPCODE_SHIFT);
361 test_pkt.value = cpu_to_le64(CPUCP_PACKET_FENCE_VAL);
363 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &test_pkt,
364 sizeof(test_pkt), 0, &result);
367 if (result != CPUCP_PACKET_FENCE_VAL)
369 "CPU queue test failed (%#08llx)\n", result);
371 dev_err(hdev->dev, "CPU queue test failed, error %d\n", rc);
377 void *hl_fw_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size,
378 dma_addr_t *dma_handle)
382 kernel_addr = gen_pool_alloc(hdev->cpu_accessible_dma_pool, size);
384 *dma_handle = hdev->cpu_accessible_dma_address +
385 (kernel_addr - (u64) (uintptr_t) hdev->cpu_accessible_dma_mem);
387 return (void *) (uintptr_t) kernel_addr;
390 void hl_fw_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size,
393 gen_pool_free(hdev->cpu_accessible_dma_pool, (u64) (uintptr_t) vaddr,
397 int hl_fw_send_heartbeat(struct hl_device *hdev)
399 struct cpucp_packet hb_pkt;
403 memset(&hb_pkt, 0, sizeof(hb_pkt));
404 hb_pkt.ctl = cpu_to_le32(CPUCP_PACKET_TEST <<
405 CPUCP_PKT_CTL_OPCODE_SHIFT);
406 hb_pkt.value = cpu_to_le64(CPUCP_PACKET_FENCE_VAL);
408 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &hb_pkt,
409 sizeof(hb_pkt), 0, &result);
411 if ((rc) || (result != CPUCP_PACKET_FENCE_VAL))
414 if (le32_to_cpu(hb_pkt.status_mask) &
415 CPUCP_PKT_HB_STATUS_EQ_FAULT_MASK) {
416 dev_warn(hdev->dev, "FW reported EQ fault during heartbeat\n");
423 static bool fw_report_boot_dev0(struct hl_device *hdev, u32 err_val,
426 bool err_exists = false;
428 if (!(err_val & CPU_BOOT_ERR0_ENABLED))
431 if (err_val & CPU_BOOT_ERR0_DRAM_INIT_FAIL) {
433 "Device boot error - DRAM initialization failed\n");
437 if (err_val & CPU_BOOT_ERR0_FIT_CORRUPTED) {
438 dev_err(hdev->dev, "Device boot error - FIT image corrupted\n");
442 if (err_val & CPU_BOOT_ERR0_TS_INIT_FAIL) {
444 "Device boot error - Thermal Sensor initialization failed\n");
448 if (err_val & CPU_BOOT_ERR0_DRAM_SKIPPED) {
450 "Device boot warning - Skipped DRAM initialization\n");
451 /* This is a warning so we don't want it to disable the
454 err_val &= ~CPU_BOOT_ERR0_DRAM_SKIPPED;
457 if (err_val & CPU_BOOT_ERR0_BMC_WAIT_SKIPPED) {
458 if (hdev->bmc_enable) {
460 "Device boot error - Skipped waiting for BMC\n");
464 "Device boot message - Skipped waiting for BMC\n");
465 /* This is an info so we don't want it to disable the
468 err_val &= ~CPU_BOOT_ERR0_BMC_WAIT_SKIPPED;
472 if (err_val & CPU_BOOT_ERR0_NIC_DATA_NOT_RDY) {
474 "Device boot error - Serdes data from BMC not available\n");
478 if (err_val & CPU_BOOT_ERR0_NIC_FW_FAIL) {
480 "Device boot error - NIC F/W initialization failed\n");
484 if (err_val & CPU_BOOT_ERR0_SECURITY_NOT_RDY) {
486 "Device boot warning - security not ready\n");
490 if (err_val & CPU_BOOT_ERR0_SECURITY_FAIL) {
491 dev_err(hdev->dev, "Device boot error - security failure\n");
495 if (err_val & CPU_BOOT_ERR0_EFUSE_FAIL) {
496 dev_err(hdev->dev, "Device boot error - eFuse failure\n");
500 if (err_val & CPU_BOOT_ERR0_PRI_IMG_VER_FAIL) {
502 "Device boot warning - Failed to load preboot primary image\n");
503 /* This is a warning so we don't want it to disable the
504 * device as we have a secondary preboot image
506 err_val &= ~CPU_BOOT_ERR0_PRI_IMG_VER_FAIL;
509 if (err_val & CPU_BOOT_ERR0_SEC_IMG_VER_FAIL) {
510 dev_err(hdev->dev, "Device boot error - Failed to load preboot secondary image\n");
514 if (err_val & CPU_BOOT_ERR0_PLL_FAIL) {
515 dev_err(hdev->dev, "Device boot error - PLL failure\n");
519 if (err_val & CPU_BOOT_ERR0_DEVICE_UNUSABLE_FAIL) {
520 /* Ignore this bit, don't prevent driver loading */
521 dev_dbg(hdev->dev, "device unusable status is set\n");
522 err_val &= ~CPU_BOOT_ERR0_DEVICE_UNUSABLE_FAIL;
525 if (sts_val & CPU_BOOT_DEV_STS0_ENABLED)
526 dev_dbg(hdev->dev, "Device status0 %#x\n", sts_val);
528 if (!err_exists && (err_val & ~CPU_BOOT_ERR0_ENABLED)) {
530 "Device boot error - unknown ERR0 error 0x%08x\n", err_val);
534 /* return error only if it's in the predefined mask */
535 if (err_exists && ((err_val & ~CPU_BOOT_ERR0_ENABLED) &
536 lower_32_bits(hdev->boot_error_status_mask)))
542 /* placeholder for ERR1 as no errors defined there yet */
543 static bool fw_report_boot_dev1(struct hl_device *hdev, u32 err_val,
547 * keep this variable to preserve the logic of the function.
548 * this way it would require less modifications when error will be
551 bool err_exists = false;
553 if (!(err_val & CPU_BOOT_ERR1_ENABLED))
556 if (sts_val & CPU_BOOT_DEV_STS1_ENABLED)
557 dev_dbg(hdev->dev, "Device status1 %#x\n", sts_val);
559 if (!err_exists && (err_val & ~CPU_BOOT_ERR1_ENABLED)) {
561 "Device boot error - unknown ERR1 error 0x%08x\n",
566 /* return error only if it's in the predefined mask */
567 if (err_exists && ((err_val & ~CPU_BOOT_ERR1_ENABLED) &
568 upper_32_bits(hdev->boot_error_status_mask)))
574 static int fw_read_errors(struct hl_device *hdev, u32 boot_err0_reg,
575 u32 boot_err1_reg, u32 cpu_boot_dev_status0_reg,
576 u32 cpu_boot_dev_status1_reg)
578 u32 err_val, status_val;
579 bool err_exists = false;
581 /* Some of the firmware status codes are deprecated in newer f/w
582 * versions. In those versions, the errors are reported
583 * in different registers. Therefore, we need to check those
584 * registers and print the exact errors. Moreover, there
585 * may be multiple errors, so we need to report on each error
586 * separately. Some of the error codes might indicate a state
587 * that is not an error per-se, but it is an error in production
590 err_val = RREG32(boot_err0_reg);
591 status_val = RREG32(cpu_boot_dev_status0_reg);
592 err_exists = fw_report_boot_dev0(hdev, err_val, status_val);
594 err_val = RREG32(boot_err1_reg);
595 status_val = RREG32(cpu_boot_dev_status1_reg);
596 err_exists |= fw_report_boot_dev1(hdev, err_val, status_val);
604 int hl_fw_cpucp_info_get(struct hl_device *hdev,
605 u32 sts_boot_dev_sts0_reg,
606 u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg,
609 struct asic_fixed_properties *prop = &hdev->asic_prop;
610 struct cpucp_packet pkt = {};
611 dma_addr_t cpucp_info_dma_addr;
612 void *cpucp_info_cpu_addr;
617 cpucp_info_cpu_addr =
618 hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev,
619 sizeof(struct cpucp_info),
620 &cpucp_info_dma_addr);
621 if (!cpucp_info_cpu_addr) {
623 "Failed to allocate DMA memory for CPU-CP info packet\n");
627 memset(cpucp_info_cpu_addr, 0, sizeof(struct cpucp_info));
629 pkt.ctl = cpu_to_le32(CPUCP_PACKET_INFO_GET <<
630 CPUCP_PKT_CTL_OPCODE_SHIFT);
631 pkt.addr = cpu_to_le64(cpucp_info_dma_addr);
632 pkt.data_max_size = cpu_to_le32(sizeof(struct cpucp_info));
634 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
635 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
638 "Failed to handle CPU-CP info pkt, error %d\n", rc);
642 rc = fw_read_errors(hdev, boot_err0_reg, boot_err1_reg,
643 sts_boot_dev_sts0_reg, sts_boot_dev_sts1_reg);
645 dev_err(hdev->dev, "Errors in device boot\n");
649 memcpy(&prop->cpucp_info, cpucp_info_cpu_addr,
650 sizeof(prop->cpucp_info));
652 rc = hl_build_hwmon_channel_info(hdev, prop->cpucp_info.sensors);
655 "Failed to build hwmon channel info, error %d\n", rc);
660 kernel_ver = extract_fw_ver_from_str(prop->cpucp_info.kernel_version);
662 dev_info(hdev->dev, "Linux version %s", kernel_ver);
666 /* assume EQ code doesn't need to check eqe index */
667 hdev->event_queue.check_eqe_index = false;
669 /* Read FW application security bits again */
670 if (prop->fw_cpu_boot_dev_sts0_valid) {
671 prop->fw_app_cpu_boot_dev_sts0 = RREG32(sts_boot_dev_sts0_reg);
672 if (prop->fw_app_cpu_boot_dev_sts0 &
673 CPU_BOOT_DEV_STS0_EQ_INDEX_EN)
674 hdev->event_queue.check_eqe_index = true;
677 if (prop->fw_cpu_boot_dev_sts1_valid)
678 prop->fw_app_cpu_boot_dev_sts1 = RREG32(sts_boot_dev_sts1_reg);
681 hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev,
682 sizeof(struct cpucp_info), cpucp_info_cpu_addr);
687 static int hl_fw_send_msi_info_msg(struct hl_device *hdev)
689 struct cpucp_array_data_packet *pkt;
690 size_t total_pkt_size, data_size;
694 /* skip sending this info for unsupported ASICs */
695 if (!hdev->asic_funcs->get_msi_info)
698 data_size = CPUCP_NUM_OF_MSI_TYPES * sizeof(u32);
699 total_pkt_size = sizeof(struct cpucp_array_data_packet) + data_size;
701 /* data should be aligned to 8 bytes in order to CPU-CP to copy it */
702 total_pkt_size = (total_pkt_size + 0x7) & ~0x7;
704 /* total_pkt_size is casted to u16 later on */
705 if (total_pkt_size > USHRT_MAX) {
706 dev_err(hdev->dev, "CPUCP array data is too big\n");
710 pkt = kzalloc(total_pkt_size, GFP_KERNEL);
714 pkt->length = cpu_to_le32(CPUCP_NUM_OF_MSI_TYPES);
716 memset((void *) &pkt->data, 0xFF, data_size);
717 hdev->asic_funcs->get_msi_info(pkt->data);
719 pkt->cpucp_pkt.ctl = cpu_to_le32(CPUCP_PACKET_MSI_INFO_SET <<
720 CPUCP_PKT_CTL_OPCODE_SHIFT);
722 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *)pkt,
723 total_pkt_size, 0, &result);
726 * in case packet result is invalid it means that FW does not support
727 * this feature and will use default/hard coded MSI values. no reason
730 if (rc && result == cpucp_packet_invalid)
734 dev_err(hdev->dev, "failed to send CPUCP array data\n");
741 int hl_fw_cpucp_handshake(struct hl_device *hdev,
742 u32 sts_boot_dev_sts0_reg,
743 u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg,
748 rc = hl_fw_cpucp_info_get(hdev, sts_boot_dev_sts0_reg,
749 sts_boot_dev_sts1_reg, boot_err0_reg,
754 return hl_fw_send_msi_info_msg(hdev);
757 int hl_fw_get_eeprom_data(struct hl_device *hdev, void *data, size_t max_size)
759 struct cpucp_packet pkt = {};
760 void *eeprom_info_cpu_addr;
761 dma_addr_t eeprom_info_dma_addr;
765 eeprom_info_cpu_addr =
766 hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev,
767 max_size, &eeprom_info_dma_addr);
768 if (!eeprom_info_cpu_addr) {
770 "Failed to allocate DMA memory for CPU-CP EEPROM packet\n");
774 memset(eeprom_info_cpu_addr, 0, max_size);
776 pkt.ctl = cpu_to_le32(CPUCP_PACKET_EEPROM_DATA_GET <<
777 CPUCP_PKT_CTL_OPCODE_SHIFT);
778 pkt.addr = cpu_to_le64(eeprom_info_dma_addr);
779 pkt.data_max_size = cpu_to_le32(max_size);
781 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
782 HL_CPUCP_EEPROM_TIMEOUT_USEC, &result);
786 "Failed to handle CPU-CP EEPROM packet, error %d\n",
791 /* result contains the actual size */
792 memcpy(data, eeprom_info_cpu_addr, min((size_t)result, max_size));
795 hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev, max_size,
796 eeprom_info_cpu_addr);
801 int hl_fw_cpucp_pci_counters_get(struct hl_device *hdev,
802 struct hl_info_pci_counters *counters)
804 struct cpucp_packet pkt = {};
808 pkt.ctl = cpu_to_le32(CPUCP_PACKET_PCIE_THROUGHPUT_GET <<
809 CPUCP_PKT_CTL_OPCODE_SHIFT);
811 /* Fetch PCI rx counter */
812 pkt.index = cpu_to_le32(cpucp_pcie_throughput_rx);
813 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
814 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
817 "Failed to handle CPU-CP PCI info pkt, error %d\n", rc);
820 counters->rx_throughput = result;
822 memset(&pkt, 0, sizeof(pkt));
823 pkt.ctl = cpu_to_le32(CPUCP_PACKET_PCIE_THROUGHPUT_GET <<
824 CPUCP_PKT_CTL_OPCODE_SHIFT);
826 /* Fetch PCI tx counter */
827 pkt.index = cpu_to_le32(cpucp_pcie_throughput_tx);
828 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
829 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
832 "Failed to handle CPU-CP PCI info pkt, error %d\n", rc);
835 counters->tx_throughput = result;
837 /* Fetch PCI replay counter */
838 memset(&pkt, 0, sizeof(pkt));
839 pkt.ctl = cpu_to_le32(CPUCP_PACKET_PCIE_REPLAY_CNT_GET <<
840 CPUCP_PKT_CTL_OPCODE_SHIFT);
842 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
843 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
846 "Failed to handle CPU-CP PCI info pkt, error %d\n", rc);
849 counters->replay_cnt = (u32) result;
854 int hl_fw_cpucp_total_energy_get(struct hl_device *hdev, u64 *total_energy)
856 struct cpucp_packet pkt = {};
860 pkt.ctl = cpu_to_le32(CPUCP_PACKET_TOTAL_ENERGY_GET <<
861 CPUCP_PKT_CTL_OPCODE_SHIFT);
863 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
864 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
867 "Failed to handle CpuCP total energy pkt, error %d\n",
872 *total_energy = result;
877 int get_used_pll_index(struct hl_device *hdev, u32 input_pll_index,
878 enum pll_index *pll_index)
880 struct asic_fixed_properties *prop = &hdev->asic_prop;
881 u8 pll_byte, pll_bit_off;
885 dynamic_pll = !!(prop->fw_app_cpu_boot_dev_sts0 &
886 CPU_BOOT_DEV_STS0_DYN_PLL_EN);
890 * in case we are working with legacy FW (each asic has unique
891 * PLL numbering) use the driver based index as they are
892 * aligned with fw legacy numbering
894 *pll_index = input_pll_index;
898 /* retrieve a FW compatible PLL index based on
899 * ASIC specific user request
901 fw_pll_idx = hdev->asic_funcs->map_pll_idx_to_fw_idx(input_pll_index);
902 if (fw_pll_idx < 0) {
903 dev_err(hdev->dev, "Invalid PLL index (%u) error %d\n",
904 input_pll_index, fw_pll_idx);
908 /* PLL map is a u8 array */
909 pll_byte = prop->cpucp_info.pll_map[fw_pll_idx >> 3];
910 pll_bit_off = fw_pll_idx & 0x7;
912 if (!(pll_byte & BIT(pll_bit_off))) {
913 dev_err(hdev->dev, "PLL index %d is not supported\n",
918 *pll_index = fw_pll_idx;
923 int hl_fw_cpucp_pll_info_get(struct hl_device *hdev, u32 pll_index,
926 struct cpucp_packet pkt;
927 enum pll_index used_pll_idx;
931 rc = get_used_pll_index(hdev, pll_index, &used_pll_idx);
935 memset(&pkt, 0, sizeof(pkt));
937 pkt.ctl = cpu_to_le32(CPUCP_PACKET_PLL_INFO_GET <<
938 CPUCP_PKT_CTL_OPCODE_SHIFT);
939 pkt.pll_type = __cpu_to_le16((u16)used_pll_idx);
941 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
942 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
944 dev_err(hdev->dev, "Failed to read PLL info, error %d\n", rc);
946 pll_freq_arr[0] = FIELD_GET(CPUCP_PKT_RES_PLL_OUT0_MASK, result);
947 pll_freq_arr[1] = FIELD_GET(CPUCP_PKT_RES_PLL_OUT1_MASK, result);
948 pll_freq_arr[2] = FIELD_GET(CPUCP_PKT_RES_PLL_OUT2_MASK, result);
949 pll_freq_arr[3] = FIELD_GET(CPUCP_PKT_RES_PLL_OUT3_MASK, result);
954 int hl_fw_cpucp_power_get(struct hl_device *hdev, u64 *power)
956 struct cpucp_packet pkt;
960 memset(&pkt, 0, sizeof(pkt));
962 pkt.ctl = cpu_to_le32(CPUCP_PACKET_POWER_GET <<
963 CPUCP_PKT_CTL_OPCODE_SHIFT);
965 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
966 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
968 dev_err(hdev->dev, "Failed to read power, error %d\n", rc);
977 void hl_fw_ask_hard_reset_without_linux(struct hl_device *hdev)
979 struct static_fw_load_mgr *static_loader =
980 &hdev->fw_loader.static_loader;
983 if (hdev->asic_prop.dynamic_fw_load) {
984 rc = hl_fw_dynamic_send_protocol_cmd(hdev, &hdev->fw_loader,
985 COMMS_RST_DEV, 0, false,
986 hdev->fw_loader.cpu_timeout);
988 dev_warn(hdev->dev, "Failed sending COMMS_RST_DEV\n");
990 WREG32(static_loader->kmd_msg_to_cpu_reg, KMD_MSG_RST_DEV);
994 void hl_fw_ask_halt_machine_without_linux(struct hl_device *hdev)
996 struct static_fw_load_mgr *static_loader =
997 &hdev->fw_loader.static_loader;
1000 if (hdev->device_cpu_is_halted)
1003 /* Stop device CPU to make sure nothing bad happens */
1004 if (hdev->asic_prop.dynamic_fw_load) {
1005 rc = hl_fw_dynamic_send_protocol_cmd(hdev, &hdev->fw_loader,
1006 COMMS_GOTO_WFE, 0, true,
1007 hdev->fw_loader.cpu_timeout);
1009 dev_warn(hdev->dev, "Failed sending COMMS_GOTO_WFE\n");
1011 WREG32(static_loader->kmd_msg_to_cpu_reg, KMD_MSG_GOTO_WFE);
1012 msleep(static_loader->cpu_reset_wait_msec);
1014 /* Must clear this register in order to prevent preboot
1015 * from reading WFE after reboot
1017 WREG32(static_loader->kmd_msg_to_cpu_reg, KMD_MSG_NA);
1020 hdev->device_cpu_is_halted = true;
1023 static void detect_cpu_boot_status(struct hl_device *hdev, u32 status)
1025 /* Some of the status codes below are deprecated in newer f/w
1026 * versions but we keep them here for backward compatibility
1029 case CPU_BOOT_STATUS_NA:
1031 "Device boot progress - BTL did NOT run\n");
1033 case CPU_BOOT_STATUS_IN_WFE:
1035 "Device boot progress - Stuck inside WFE loop\n");
1037 case CPU_BOOT_STATUS_IN_BTL:
1039 "Device boot progress - Stuck in BTL\n");
1041 case CPU_BOOT_STATUS_IN_PREBOOT:
1043 "Device boot progress - Stuck in Preboot\n");
1045 case CPU_BOOT_STATUS_IN_SPL:
1047 "Device boot progress - Stuck in SPL\n");
1049 case CPU_BOOT_STATUS_IN_UBOOT:
1051 "Device boot progress - Stuck in u-boot\n");
1053 case CPU_BOOT_STATUS_DRAM_INIT_FAIL:
1055 "Device boot progress - DRAM initialization failed\n");
1057 case CPU_BOOT_STATUS_UBOOT_NOT_READY:
1059 "Device boot progress - Cannot boot\n");
1061 case CPU_BOOT_STATUS_TS_INIT_FAIL:
1063 "Device boot progress - Thermal Sensor initialization failed\n");
1065 case CPU_BOOT_STATUS_SECURITY_READY:
1067 "Device boot progress - Stuck in preboot after security initialization\n");
1071 "Device boot progress - Invalid status code %d\n",
1077 static int hl_fw_read_preboot_caps(struct hl_device *hdev,
1078 u32 cpu_boot_status_reg,
1079 u32 sts_boot_dev_sts0_reg,
1080 u32 sts_boot_dev_sts1_reg,
1081 u32 boot_err0_reg, u32 boot_err1_reg,
1084 struct asic_fixed_properties *prop = &hdev->asic_prop;
1085 u32 status, reg_val;
1088 /* Need to check two possible scenarios:
1090 * CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT - for newer firmwares where
1091 * the preboot is waiting for the boot fit
1093 * All other status values - for older firmwares where the uboot was
1094 * loaded from the FLASH
1096 rc = hl_poll_timeout(
1098 cpu_boot_status_reg,
1100 (status == CPU_BOOT_STATUS_IN_UBOOT) ||
1101 (status == CPU_BOOT_STATUS_DRAM_RDY) ||
1102 (status == CPU_BOOT_STATUS_NIC_FW_RDY) ||
1103 (status == CPU_BOOT_STATUS_READY_TO_BOOT) ||
1104 (status == CPU_BOOT_STATUS_SRAM_AVAIL) ||
1105 (status == CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT),
1106 FW_CPU_STATUS_POLL_INTERVAL_USEC,
1110 dev_err(hdev->dev, "CPU boot ready status timeout\n");
1111 detect_cpu_boot_status(hdev, status);
1113 /* If we read all FF, then something is totally wrong, no point
1114 * of reading specific errors
1117 fw_read_errors(hdev, boot_err0_reg, boot_err1_reg,
1118 sts_boot_dev_sts0_reg,
1119 sts_boot_dev_sts1_reg);
1124 * the registers DEV_STS* contain FW capabilities/features.
1125 * We can rely on this registers only if bit CPU_BOOT_DEV_STS*_ENABLED
1127 * In the first read of this register we store the value of this
1128 * register ONLY if the register is enabled (which will be propagated
1129 * to next stages) and also mark the register as valid.
1130 * In case it is not enabled the stored value will be left 0- all
1131 * caps/features are off
1133 reg_val = RREG32(sts_boot_dev_sts0_reg);
1134 if (reg_val & CPU_BOOT_DEV_STS0_ENABLED) {
1135 prop->fw_cpu_boot_dev_sts0_valid = true;
1136 prop->fw_preboot_cpu_boot_dev_sts0 = reg_val;
1139 reg_val = RREG32(sts_boot_dev_sts1_reg);
1140 if (reg_val & CPU_BOOT_DEV_STS1_ENABLED) {
1141 prop->fw_cpu_boot_dev_sts1_valid = true;
1142 prop->fw_preboot_cpu_boot_dev_sts1 = reg_val;
1145 prop->dynamic_fw_load = !!(prop->fw_preboot_cpu_boot_dev_sts0 &
1146 CPU_BOOT_DEV_STS0_FW_LD_COM_EN);
1148 /* initialize FW loader once we know what load protocol is used */
1149 hdev->asic_funcs->init_firmware_loader(hdev);
1151 dev_dbg(hdev->dev, "Attempting %s FW load\n",
1152 prop->dynamic_fw_load ? "dynamic" : "legacy");
1156 static int hl_fw_static_read_device_fw_version(struct hl_device *hdev,
1157 enum hl_fw_component fwc)
1159 struct asic_fixed_properties *prop = &hdev->asic_prop;
1160 struct fw_load_mgr *fw_loader = &hdev->fw_loader;
1161 struct static_fw_load_mgr *static_loader;
1162 char *dest, *boot_ver, *preboot_ver;
1167 static_loader = &hdev->fw_loader.static_loader;
1170 case FW_COMP_BOOT_FIT:
1171 ver_off = RREG32(static_loader->boot_fit_version_offset_reg);
1172 dest = prop->uboot_ver;
1174 limit = static_loader->boot_fit_version_max_off;
1176 case FW_COMP_PREBOOT:
1177 ver_off = RREG32(static_loader->preboot_version_offset_reg);
1178 dest = prop->preboot_ver;
1180 limit = static_loader->preboot_version_max_off;
1183 dev_warn(hdev->dev, "Undefined FW component: %d\n", fwc);
1187 ver_off &= static_loader->sram_offset_mask;
1189 if (ver_off < limit) {
1191 hdev->pcie_bar[fw_loader->sram_bar_id] + ver_off,
1194 dev_err(hdev->dev, "%s version offset (0x%x) is above SRAM\n",
1196 strscpy(dest, "unavailable", VERSION_MAX_LEN);
1200 if (fwc == FW_COMP_BOOT_FIT) {
1201 boot_ver = extract_fw_ver_from_str(prop->uboot_ver);
1203 dev_info(hdev->dev, "boot-fit version %s\n", boot_ver);
1206 } else if (fwc == FW_COMP_PREBOOT) {
1207 preboot_ver = strnstr(prop->preboot_ver, "Preboot",
1209 if (preboot_ver && preboot_ver != prop->preboot_ver) {
1210 strscpy(btl_ver, prop->preboot_ver,
1211 min((int) (preboot_ver - prop->preboot_ver),
1213 dev_info(hdev->dev, "%s\n", btl_ver);
1216 preboot_ver = extract_fw_ver_from_str(prop->preboot_ver);
1218 dev_info(hdev->dev, "preboot version %s\n",
1228 * hl_fw_preboot_update_state - update internal data structures during
1229 * handshake with preboot
1232 * @hdev: pointer to the habanalabs device structure
1234 * @return 0 on success, otherwise non-zero error code
1236 static void hl_fw_preboot_update_state(struct hl_device *hdev)
1238 struct asic_fixed_properties *prop = &hdev->asic_prop;
1239 u32 cpu_boot_dev_sts0, cpu_boot_dev_sts1;
1241 cpu_boot_dev_sts0 = prop->fw_preboot_cpu_boot_dev_sts0;
1242 cpu_boot_dev_sts1 = prop->fw_preboot_cpu_boot_dev_sts1;
1244 /* We read boot_dev_sts registers multiple times during boot:
1245 * 1. preboot - a. Check whether the security status bits are valid
1246 * b. Check whether fw security is enabled
1247 * c. Check whether hard reset is done by preboot
1248 * 2. boot cpu - a. Fetch boot cpu security status
1249 * b. Check whether hard reset is done by boot cpu
1250 * 3. FW application - a. Fetch fw application security status
1251 * b. Check whether hard reset is done by fw app
1253 prop->hard_reset_done_by_fw =
1254 !!(cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_FW_HARD_RST_EN);
1256 dev_dbg(hdev->dev, "Firmware preboot boot device status0 %#x\n",
1259 dev_dbg(hdev->dev, "Firmware preboot boot device status1 %#x\n",
1262 dev_dbg(hdev->dev, "Firmware preboot hard-reset is %s\n",
1263 prop->hard_reset_done_by_fw ? "enabled" : "disabled");
1265 dev_dbg(hdev->dev, "firmware-level security is %s\n",
1266 prop->fw_security_enabled ? "enabled" : "disabled");
1268 dev_dbg(hdev->dev, "GIC controller is %s\n",
1269 prop->gic_interrupts_enable ? "enabled" : "disabled");
1272 static int hl_fw_static_read_preboot_status(struct hl_device *hdev)
1276 rc = hl_fw_static_read_device_fw_version(hdev, FW_COMP_PREBOOT);
1283 int hl_fw_read_preboot_status(struct hl_device *hdev, u32 cpu_boot_status_reg,
1284 u32 sts_boot_dev_sts0_reg,
1285 u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg,
1286 u32 boot_err1_reg, u32 timeout)
1290 /* pldm was added for cases in which we use preboot on pldm and want
1291 * to load boot fit, but we can't wait for preboot because it runs
1294 if (!(hdev->fw_components & FW_TYPE_PREBOOT_CPU) || hdev->pldm)
1298 * In order to determine boot method (static VS dymanic) we need to
1299 * read the boot caps register
1301 rc = hl_fw_read_preboot_caps(hdev, cpu_boot_status_reg,
1302 sts_boot_dev_sts0_reg,
1303 sts_boot_dev_sts1_reg, boot_err0_reg,
1304 boot_err1_reg, timeout);
1308 hl_fw_preboot_update_state(hdev);
1310 /* no need to read preboot status in dynamic load */
1311 if (hdev->asic_prop.dynamic_fw_load)
1314 return hl_fw_static_read_preboot_status(hdev);
1317 /* associate string with COMM status */
1318 static char *hl_dynamic_fw_status_str[COMMS_STS_INVLD_LAST] = {
1319 [COMMS_STS_NOOP] = "NOOP",
1320 [COMMS_STS_ACK] = "ACK",
1321 [COMMS_STS_OK] = "OK",
1322 [COMMS_STS_ERR] = "ERR",
1323 [COMMS_STS_VALID_ERR] = "VALID_ERR",
1324 [COMMS_STS_TIMEOUT_ERR] = "TIMEOUT_ERR",
1328 * hl_fw_dynamic_report_error_status - report error status
1330 * @hdev: pointer to the habanalabs device structure
1331 * @status: value of FW status register
1332 * @expected_status: the expected status
1334 static void hl_fw_dynamic_report_error_status(struct hl_device *hdev,
1336 enum comms_sts expected_status)
1338 enum comms_sts comm_status =
1339 FIELD_GET(COMMS_STATUS_STATUS_MASK, status);
1341 if (comm_status < COMMS_STS_INVLD_LAST)
1342 dev_err(hdev->dev, "Device status %s, expected status: %s\n",
1343 hl_dynamic_fw_status_str[comm_status],
1344 hl_dynamic_fw_status_str[expected_status]);
1346 dev_err(hdev->dev, "Device status unknown %d, expected status: %s\n",
1348 hl_dynamic_fw_status_str[expected_status]);
1352 * hl_fw_dynamic_send_cmd - send LKD to FW cmd
1354 * @hdev: pointer to the habanalabs device structure
1355 * @fw_loader: managing structure for loading device's FW
1356 * @cmd: LKD to FW cmd code
1357 * @size: size of next FW component to be loaded (0 if not necessary)
1359 * LDK to FW exact command layout is defined at struct comms_command.
1360 * note: the size argument is used only when the next FW component should be
1361 * loaded, otherwise it shall be 0. the size is used by the FW in later
1362 * protocol stages and when sending only indicating the amount of memory
1363 * to be allocated by the FW to receive the next boot component.
1365 static void hl_fw_dynamic_send_cmd(struct hl_device *hdev,
1366 struct fw_load_mgr *fw_loader,
1367 enum comms_cmd cmd, unsigned int size)
1369 struct cpu_dyn_regs *dyn_regs;
1372 dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs;
1374 val = FIELD_PREP(COMMS_COMMAND_CMD_MASK, cmd);
1375 val |= FIELD_PREP(COMMS_COMMAND_SIZE_MASK, size);
1377 WREG32(le32_to_cpu(dyn_regs->kmd_msg_to_cpu), val);
1381 * hl_fw_dynamic_extract_fw_response - update the FW response
1383 * @hdev: pointer to the habanalabs device structure
1384 * @fw_loader: managing structure for loading device's FW
1385 * @response: FW response
1386 * @status: the status read from CPU status register
1388 * @return 0 on success, otherwise non-zero error code
1390 static int hl_fw_dynamic_extract_fw_response(struct hl_device *hdev,
1391 struct fw_load_mgr *fw_loader,
1392 struct fw_response *response,
1395 response->status = FIELD_GET(COMMS_STATUS_STATUS_MASK, status);
1396 response->ram_offset = FIELD_GET(COMMS_STATUS_OFFSET_MASK, status) <<
1397 COMMS_STATUS_OFFSET_ALIGN_SHIFT;
1398 response->ram_type = FIELD_GET(COMMS_STATUS_RAM_TYPE_MASK, status);
1400 if ((response->ram_type != COMMS_SRAM) &&
1401 (response->ram_type != COMMS_DRAM)) {
1402 dev_err(hdev->dev, "FW status: invalid RAM type %u\n",
1403 response->ram_type);
1411 * hl_fw_dynamic_wait_for_status - wait for status in dynamic FW load
1413 * @hdev: pointer to the habanalabs device structure
1414 * @fw_loader: managing structure for loading device's FW
1415 * @expected_status: expected status to wait for
1416 * @timeout: timeout for status wait
1418 * @return 0 on success, otherwise non-zero error code
1420 * waiting for status from FW include polling the FW status register until
1421 * expected status is received or timeout occurs (whatever occurs first).
1423 static int hl_fw_dynamic_wait_for_status(struct hl_device *hdev,
1424 struct fw_load_mgr *fw_loader,
1425 enum comms_sts expected_status,
1428 struct cpu_dyn_regs *dyn_regs;
1432 dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs;
1434 /* Wait for expected status */
1435 rc = hl_poll_timeout(
1437 le32_to_cpu(dyn_regs->cpu_cmd_status_to_host),
1439 FIELD_GET(COMMS_STATUS_STATUS_MASK, status) == expected_status,
1440 FW_CPU_STATUS_POLL_INTERVAL_USEC,
1444 hl_fw_dynamic_report_error_status(hdev, status,
1450 * skip storing FW response for NOOP to preserve the actual desired
1453 if (expected_status == COMMS_STS_NOOP)
1456 rc = hl_fw_dynamic_extract_fw_response(hdev, fw_loader,
1457 &fw_loader->dynamic_loader.response,
1463 * hl_fw_dynamic_send_clear_cmd - send clear command to FW
1465 * @hdev: pointer to the habanalabs device structure
1466 * @fw_loader: managing structure for loading device's FW
1468 * @return 0 on success, otherwise non-zero error code
1470 * after command cycle between LKD to FW CPU (i.e. LKD got an expected status
1471 * from FW) we need to clear the CPU status register in order to avoid garbage
1472 * between command cycles.
1473 * This is done by sending clear command and polling the CPU to LKD status
1474 * register to hold the status NOOP
1476 static int hl_fw_dynamic_send_clear_cmd(struct hl_device *hdev,
1477 struct fw_load_mgr *fw_loader)
1479 hl_fw_dynamic_send_cmd(hdev, fw_loader, COMMS_CLR_STS, 0);
1481 return hl_fw_dynamic_wait_for_status(hdev, fw_loader, COMMS_STS_NOOP,
1482 fw_loader->cpu_timeout);
1486 * hl_fw_dynamic_send_protocol_cmd - send LKD to FW cmd and wait for ACK
1488 * @hdev: pointer to the habanalabs device structure
1489 * @fw_loader: managing structure for loading device's FW
1490 * @cmd: LKD to FW cmd code
1491 * @size: size of next FW component to be loaded (0 if not necessary)
1492 * @wait_ok: if true also wait for OK response from FW
1493 * @timeout: timeout for status wait
1495 * @return 0 on success, otherwise non-zero error code
1498 * when sending protocol command we have the following steps:
1499 * - send clear (clear command and verify clear status register)
1500 * - send the actual protocol command
1501 * - wait for ACK on the protocol command
1504 * if, in addition, the specific protocol command should wait for OK then:
1510 * send clear: this is necessary in order to clear the status register to avoid
1511 * leftovers between command
1512 * NOOP command: necessary to avoid loop on the clear command by the FW
1514 int hl_fw_dynamic_send_protocol_cmd(struct hl_device *hdev,
1515 struct fw_load_mgr *fw_loader,
1516 enum comms_cmd cmd, unsigned int size,
1517 bool wait_ok, u32 timeout)
1521 /* first send clear command to clean former commands */
1522 rc = hl_fw_dynamic_send_clear_cmd(hdev, fw_loader);
1524 /* send the actual command */
1525 hl_fw_dynamic_send_cmd(hdev, fw_loader, cmd, size);
1527 /* wait for ACK for the command */
1528 rc = hl_fw_dynamic_wait_for_status(hdev, fw_loader, COMMS_STS_ACK,
1533 /* clear command to prepare for NOOP command */
1534 rc = hl_fw_dynamic_send_clear_cmd(hdev, fw_loader);
1538 /* send the actual NOOP command */
1539 hl_fw_dynamic_send_cmd(hdev, fw_loader, COMMS_NOOP, 0);
1544 rc = hl_fw_dynamic_wait_for_status(hdev, fw_loader, COMMS_STS_OK,
1549 /* clear command to prepare for NOOP command */
1550 rc = hl_fw_dynamic_send_clear_cmd(hdev, fw_loader);
1554 /* send the actual NOOP command */
1555 hl_fw_dynamic_send_cmd(hdev, fw_loader, COMMS_NOOP, 0);
1561 * hl_fw_compat_crc32 - CRC compatible with FW
1563 * @data: pointer to the data
1564 * @size: size of the data
1566 * @return the CRC32 result
1568 * NOTE: kernel's CRC32 differ's from standard CRC32 calculation.
1569 * in order to be aligned we need to flip the bits of both the input
1570 * initial CRC and kernel's CRC32 result.
1571 * in addition both sides use initial CRC of 0,
1573 static u32 hl_fw_compat_crc32(u8 *data, size_t size)
1575 return ~crc32_le(~((u32)0), data, size);
1579 * hl_fw_dynamic_validate_memory_bound - validate memory bounds for memory
1580 * transfer (image or descriptor) between
1583 * @hdev: pointer to the habanalabs device structure
1584 * @addr: device address of memory transfer
1585 * @size: memory transter size
1586 * @region: PCI memory region
1588 * @return 0 on success, otherwise non-zero error code
1590 static int hl_fw_dynamic_validate_memory_bound(struct hl_device *hdev,
1591 u64 addr, size_t size,
1592 struct pci_mem_region *region)
1596 /* now make sure that the memory transfer is within region's bounds */
1597 end_addr = addr + size;
1598 if (end_addr >= region->region_base + region->region_size) {
1600 "dynamic FW load: memory transfer end address out of memory region bounds. addr: %llx\n",
1606 * now make sure memory transfer is within predefined BAR bounds.
1607 * this is to make sure we do not need to set the bar (e.g. for DRAM
1610 if (end_addr >= region->region_base - region->offset_in_bar +
1613 "FW image beyond PCI BAR bounds\n");
1621 * hl_fw_dynamic_validate_descriptor - validate FW descriptor
1623 * @hdev: pointer to the habanalabs device structure
1624 * @fw_loader: managing structure for loading device's FW
1625 * @fw_desc: the descriptor form FW
1627 * @return 0 on success, otherwise non-zero error code
1629 static int hl_fw_dynamic_validate_descriptor(struct hl_device *hdev,
1630 struct fw_load_mgr *fw_loader,
1631 struct lkd_fw_comms_desc *fw_desc)
1633 struct pci_mem_region *region;
1634 enum pci_region region_id;
1641 if (le32_to_cpu(fw_desc->header.magic) != HL_COMMS_DESC_MAGIC) {
1642 dev_err(hdev->dev, "Invalid magic for dynamic FW descriptor (%x)\n",
1643 fw_desc->header.magic);
1647 if (fw_desc->header.version != HL_COMMS_DESC_VER) {
1648 dev_err(hdev->dev, "Invalid version for dynamic FW descriptor (%x)\n",
1649 fw_desc->header.version);
1654 * calc CRC32 of data without header.
1655 * note that no alignment/stride address issues here as all structures
1658 data_size = sizeof(struct lkd_fw_comms_desc) -
1659 sizeof(struct comms_desc_header);
1660 data_ptr = (u8 *)fw_desc + sizeof(struct comms_desc_header);
1662 if (le16_to_cpu(fw_desc->header.size) != data_size) {
1664 "Invalid descriptor size 0x%x, expected size 0x%zx\n",
1665 le16_to_cpu(fw_desc->header.size), data_size);
1669 data_crc32 = hl_fw_compat_crc32(data_ptr, data_size);
1671 if (data_crc32 != le32_to_cpu(fw_desc->header.crc32)) {
1673 "CRC32 mismatch for dynamic FW descriptor (%x:%x)\n",
1674 data_crc32, fw_desc->header.crc32);
1678 /* find memory region to which to copy the image */
1679 addr = le64_to_cpu(fw_desc->img_addr);
1680 region_id = hl_get_pci_memory_region(hdev, addr);
1681 if ((region_id != PCI_REGION_SRAM) &&
1682 ((region_id != PCI_REGION_DRAM))) {
1684 "Invalid region to copy FW image address=%llx\n", addr);
1688 region = &hdev->pci_mem_region[region_id];
1690 /* store the region for the copy stage */
1691 fw_loader->dynamic_loader.image_region = region;
1694 * here we know that the start address is valid, now make sure that the
1695 * image is within region's bounds
1697 rc = hl_fw_dynamic_validate_memory_bound(hdev, addr,
1698 fw_loader->dynamic_loader.fw_image_size,
1702 "invalid mem transfer request for FW image\n");
1709 static int hl_fw_dynamic_validate_response(struct hl_device *hdev,
1710 struct fw_response *response,
1711 struct pci_mem_region *region)
1716 device_addr = region->region_base + response->ram_offset;
1719 * validate that the descriptor is within region's bounds
1720 * Note that as the start address was supplied according to the RAM
1721 * type- testing only the end address is enough
1723 rc = hl_fw_dynamic_validate_memory_bound(hdev, device_addr,
1724 sizeof(struct lkd_fw_comms_desc),
1730 * hl_fw_dynamic_read_and_validate_descriptor - read and validate FW descriptor
1732 * @hdev: pointer to the habanalabs device structure
1733 * @fw_loader: managing structure for loading device's FW
1735 * @return 0 on success, otherwise non-zero error code
1737 static int hl_fw_dynamic_read_and_validate_descriptor(struct hl_device *hdev,
1738 struct fw_load_mgr *fw_loader)
1740 struct lkd_fw_comms_desc *fw_desc;
1741 struct pci_mem_region *region;
1742 struct fw_response *response;
1743 enum pci_region region_id;
1747 fw_desc = &fw_loader->dynamic_loader.comm_desc;
1748 response = &fw_loader->dynamic_loader.response;
1750 region_id = (response->ram_type == COMMS_SRAM) ?
1751 PCI_REGION_SRAM : PCI_REGION_DRAM;
1753 region = &hdev->pci_mem_region[region_id];
1755 rc = hl_fw_dynamic_validate_response(hdev, response, region);
1758 "invalid mem transfer request for FW descriptor\n");
1762 /* extract address copy the descriptor from */
1763 src = hdev->pcie_bar[region->bar_id] + region->offset_in_bar +
1764 response->ram_offset;
1765 memcpy_fromio(fw_desc, src, sizeof(struct lkd_fw_comms_desc));
1767 return hl_fw_dynamic_validate_descriptor(hdev, fw_loader, fw_desc);
1771 * hl_fw_dynamic_request_descriptor - handshake with CPU to get FW descriptor
1773 * @hdev: pointer to the habanalabs device structure
1774 * @fw_loader: managing structure for loading device's FW
1775 * @next_image_size: size to allocate for next FW component
1777 * @return 0 on success, otherwise non-zero error code
1779 static int hl_fw_dynamic_request_descriptor(struct hl_device *hdev,
1780 struct fw_load_mgr *fw_loader,
1781 size_t next_image_size)
1785 rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_PREP_DESC,
1786 next_image_size, true,
1787 fw_loader->cpu_timeout);
1791 return hl_fw_dynamic_read_and_validate_descriptor(hdev, fw_loader);
1795 * hl_fw_dynamic_read_device_fw_version - read FW version to exposed properties
1797 * @hdev: pointer to the habanalabs device structure
1798 * @fwc: the firmware component
1799 * @fw_version: fw component's version string
1801 static void hl_fw_dynamic_read_device_fw_version(struct hl_device *hdev,
1802 enum hl_fw_component fwc,
1803 const char *fw_version)
1805 struct asic_fixed_properties *prop = &hdev->asic_prop;
1806 char *preboot_ver, *boot_ver;
1810 case FW_COMP_BOOT_FIT:
1811 strscpy(prop->uboot_ver, fw_version, VERSION_MAX_LEN);
1812 boot_ver = extract_fw_ver_from_str(prop->uboot_ver);
1814 dev_info(hdev->dev, "boot-fit version %s\n", boot_ver);
1819 case FW_COMP_PREBOOT:
1820 strscpy(prop->preboot_ver, fw_version, VERSION_MAX_LEN);
1821 preboot_ver = strnstr(prop->preboot_ver, "Preboot",
1823 if (preboot_ver && preboot_ver != prop->preboot_ver) {
1824 strscpy(btl_ver, prop->preboot_ver,
1825 min((int) (preboot_ver - prop->preboot_ver),
1827 dev_info(hdev->dev, "%s\n", btl_ver);
1830 preboot_ver = extract_fw_ver_from_str(prop->preboot_ver);
1832 dev_info(hdev->dev, "preboot version %s\n",
1839 dev_warn(hdev->dev, "Undefined FW component: %d\n", fwc);
1845 * hl_fw_dynamic_copy_image - copy image to memory allocated by the FW
1847 * @hdev: pointer to the habanalabs device structure
1848 * @fw: fw descriptor
1849 * @fw_loader: managing structure for loading device's FW
1851 static int hl_fw_dynamic_copy_image(struct hl_device *hdev,
1852 const struct firmware *fw,
1853 struct fw_load_mgr *fw_loader)
1855 struct lkd_fw_comms_desc *fw_desc;
1856 struct pci_mem_region *region;
1861 fw_desc = &fw_loader->dynamic_loader.comm_desc;
1862 addr = le64_to_cpu(fw_desc->img_addr);
1864 /* find memory region to which to copy the image */
1865 region = fw_loader->dynamic_loader.image_region;
1867 dest = hdev->pcie_bar[region->bar_id] + region->offset_in_bar +
1868 (addr - region->region_base);
1870 rc = hl_fw_copy_fw_to_device(hdev, fw, dest,
1871 fw_loader->boot_fit_img.src_off,
1872 fw_loader->boot_fit_img.copy_size);
1878 * hl_fw_dynamic_copy_msg - copy msg to memory allocated by the FW
1880 * @hdev: pointer to the habanalabs device structure
1882 * @fw_loader: managing structure for loading device's FW
1884 static int hl_fw_dynamic_copy_msg(struct hl_device *hdev,
1885 struct lkd_msg_comms *msg, struct fw_load_mgr *fw_loader)
1887 struct lkd_fw_comms_desc *fw_desc;
1888 struct pci_mem_region *region;
1893 fw_desc = &fw_loader->dynamic_loader.comm_desc;
1894 addr = le64_to_cpu(fw_desc->img_addr);
1896 /* find memory region to which to copy the image */
1897 region = fw_loader->dynamic_loader.image_region;
1899 dest = hdev->pcie_bar[region->bar_id] + region->offset_in_bar +
1900 (addr - region->region_base);
1902 rc = hl_fw_copy_msg_to_device(hdev, msg, dest, 0, 0);
1908 * hl_fw_boot_fit_update_state - update internal data structures after boot-fit
1911 * @hdev: pointer to the habanalabs device structure
1912 * @cpu_boot_dev_sts0_reg: register holding CPU boot dev status 0
1913 * @cpu_boot_dev_sts1_reg: register holding CPU boot dev status 1
1915 * @return 0 on success, otherwise non-zero error code
1917 static void hl_fw_boot_fit_update_state(struct hl_device *hdev,
1918 u32 cpu_boot_dev_sts0_reg,
1919 u32 cpu_boot_dev_sts1_reg)
1921 struct asic_fixed_properties *prop = &hdev->asic_prop;
1923 /* Clear reset status since we need to read it again from boot CPU */
1924 prop->hard_reset_done_by_fw = false;
1926 /* Read boot_cpu status bits */
1927 if (prop->fw_preboot_cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_ENABLED) {
1928 prop->fw_bootfit_cpu_boot_dev_sts0 =
1929 RREG32(cpu_boot_dev_sts0_reg);
1931 if (prop->fw_bootfit_cpu_boot_dev_sts0 &
1932 CPU_BOOT_DEV_STS0_FW_HARD_RST_EN)
1933 prop->hard_reset_done_by_fw = true;
1935 dev_dbg(hdev->dev, "Firmware boot CPU status0 %#x\n",
1936 prop->fw_bootfit_cpu_boot_dev_sts0);
1939 if (prop->fw_cpu_boot_dev_sts1_valid) {
1940 prop->fw_bootfit_cpu_boot_dev_sts1 =
1941 RREG32(cpu_boot_dev_sts1_reg);
1943 dev_dbg(hdev->dev, "Firmware boot CPU status1 %#x\n",
1944 prop->fw_bootfit_cpu_boot_dev_sts1);
1947 dev_dbg(hdev->dev, "Firmware boot CPU hard-reset is %s\n",
1948 prop->hard_reset_done_by_fw ? "enabled" : "disabled");
1951 static void hl_fw_dynamic_update_linux_interrupt_if(struct hl_device *hdev)
1953 struct cpu_dyn_regs *dyn_regs =
1954 &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
1956 /* Check whether all 3 interrupt interfaces are set, if not use a
1959 if (!hdev->asic_prop.gic_interrupts_enable &&
1960 !(hdev->asic_prop.fw_app_cpu_boot_dev_sts0 &
1961 CPU_BOOT_DEV_STS0_MULTI_IRQ_POLL_EN)) {
1962 dyn_regs->gic_host_halt_irq = dyn_regs->gic_host_pi_upd_irq;
1963 dyn_regs->gic_host_ints_irq = dyn_regs->gic_host_pi_upd_irq;
1966 "Using a single interrupt interface towards cpucp");
1970 * hl_fw_dynamic_load_image - load FW image using dynamic protocol
1972 * @hdev: pointer to the habanalabs device structure
1973 * @fw_loader: managing structure for loading device's FW
1974 * @load_fwc: the FW component to be loaded
1975 * @img_ld_timeout: image load timeout
1977 * @return 0 on success, otherwise non-zero error code
1979 static int hl_fw_dynamic_load_image(struct hl_device *hdev,
1980 struct fw_load_mgr *fw_loader,
1981 enum hl_fw_component load_fwc,
1984 enum hl_fw_component cur_fwc;
1985 const struct firmware *fw;
1990 * when loading image we have one of 2 scenarios:
1991 * 1. current FW component is preboot and we want to load boot-fit
1992 * 2. current FW component is boot-fit and we want to load linux
1994 if (load_fwc == FW_COMP_BOOT_FIT) {
1995 cur_fwc = FW_COMP_PREBOOT;
1996 fw_name = fw_loader->boot_fit_img.image_name;
1998 cur_fwc = FW_COMP_BOOT_FIT;
1999 fw_name = fw_loader->linux_img.image_name;
2002 /* request FW in order to communicate to FW the size to be allocated */
2003 rc = hl_request_fw(hdev, &fw, fw_name);
2007 /* store the image size for future validation */
2008 fw_loader->dynamic_loader.fw_image_size = fw->size;
2010 rc = hl_fw_dynamic_request_descriptor(hdev, fw_loader, fw->size);
2014 /* read preboot version */
2015 hl_fw_dynamic_read_device_fw_version(hdev, cur_fwc,
2016 fw_loader->dynamic_loader.comm_desc.cur_fw_ver);
2019 /* update state according to boot stage */
2020 if (cur_fwc == FW_COMP_BOOT_FIT) {
2021 struct cpu_dyn_regs *dyn_regs;
2023 dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs;
2024 hl_fw_boot_fit_update_state(hdev,
2025 le32_to_cpu(dyn_regs->cpu_boot_dev_sts0),
2026 le32_to_cpu(dyn_regs->cpu_boot_dev_sts1));
2029 /* copy boot fit to space allocated by FW */
2030 rc = hl_fw_dynamic_copy_image(hdev, fw, fw_loader);
2034 rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_DATA_RDY,
2036 fw_loader->cpu_timeout);
2040 rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_EXEC,
2045 hl_release_firmware(fw);
2049 static int hl_fw_dynamic_wait_for_boot_fit_active(struct hl_device *hdev,
2050 struct fw_load_mgr *fw_loader)
2052 struct dynamic_fw_load_mgr *dyn_loader;
2056 dyn_loader = &fw_loader->dynamic_loader;
2058 /* Make sure CPU boot-loader is running */
2059 rc = hl_poll_timeout(
2061 le32_to_cpu(dyn_loader->comm_desc.cpu_dyn_regs.cpu_boot_status),
2063 (status == CPU_BOOT_STATUS_NIC_FW_RDY) ||
2064 (status == CPU_BOOT_STATUS_READY_TO_BOOT),
2065 FW_CPU_STATUS_POLL_INTERVAL_USEC,
2066 dyn_loader->wait_for_bl_timeout);
2068 dev_err(hdev->dev, "failed to wait for boot\n");
2072 dev_dbg(hdev->dev, "uboot status = %d\n", status);
2076 static int hl_fw_dynamic_wait_for_linux_active(struct hl_device *hdev,
2077 struct fw_load_mgr *fw_loader)
2079 struct dynamic_fw_load_mgr *dyn_loader;
2083 dyn_loader = &fw_loader->dynamic_loader;
2085 /* Make sure CPU boot-loader is running */
2087 rc = hl_poll_timeout(
2089 le32_to_cpu(dyn_loader->comm_desc.cpu_dyn_regs.cpu_boot_status),
2091 (status == CPU_BOOT_STATUS_SRAM_AVAIL),
2092 FW_CPU_STATUS_POLL_INTERVAL_USEC,
2093 fw_loader->cpu_timeout);
2095 dev_err(hdev->dev, "failed to wait for Linux\n");
2099 dev_dbg(hdev->dev, "Boot status = %d\n", status);
2104 * hl_fw_linux_update_state - update internal data structures after Linux
2106 * Note: Linux initialization is comprised mainly
2107 * of two stages - loading kernel (SRAM_AVAIL)
2109 * Therefore reading boot device status in any of
2110 * these stages might result in different values.
2112 * @hdev: pointer to the habanalabs device structure
2113 * @cpu_boot_dev_sts0_reg: register holding CPU boot dev status 0
2114 * @cpu_boot_dev_sts1_reg: register holding CPU boot dev status 1
2116 * @return 0 on success, otherwise non-zero error code
2118 static void hl_fw_linux_update_state(struct hl_device *hdev,
2119 u32 cpu_boot_dev_sts0_reg,
2120 u32 cpu_boot_dev_sts1_reg)
2122 struct asic_fixed_properties *prop = &hdev->asic_prop;
2124 hdev->fw_loader.linux_loaded = true;
2126 /* Clear reset status since we need to read again from app */
2127 prop->hard_reset_done_by_fw = false;
2129 /* Read FW application security bits */
2130 if (prop->fw_cpu_boot_dev_sts0_valid) {
2131 prop->fw_app_cpu_boot_dev_sts0 = RREG32(cpu_boot_dev_sts0_reg);
2133 if (prop->fw_app_cpu_boot_dev_sts0 &
2134 CPU_BOOT_DEV_STS0_FW_HARD_RST_EN)
2135 prop->hard_reset_done_by_fw = true;
2137 if (prop->fw_app_cpu_boot_dev_sts0 &
2138 CPU_BOOT_DEV_STS0_GIC_PRIVILEGED_EN)
2139 prop->gic_interrupts_enable = false;
2142 "Firmware application CPU status0 %#x\n",
2143 prop->fw_app_cpu_boot_dev_sts0);
2145 dev_dbg(hdev->dev, "GIC controller is %s\n",
2146 prop->gic_interrupts_enable ?
2147 "enabled" : "disabled");
2150 if (prop->fw_cpu_boot_dev_sts1_valid) {
2151 prop->fw_app_cpu_boot_dev_sts1 = RREG32(cpu_boot_dev_sts1_reg);
2154 "Firmware application CPU status1 %#x\n",
2155 prop->fw_app_cpu_boot_dev_sts1);
2158 dev_dbg(hdev->dev, "Firmware application CPU hard-reset is %s\n",
2159 prop->hard_reset_done_by_fw ? "enabled" : "disabled");
2161 dev_info(hdev->dev, "Successfully loaded firmware to device\n");
2165 * hl_fw_dynamic_report_reset_cause - send a COMMS message with the cause
2166 * of the newly triggered hard reset
2168 * @hdev: pointer to the habanalabs device structure
2169 * @fw_loader: managing structure for loading device's FW
2170 * @reset_cause: enumerated cause for the recent hard reset
2172 * @return 0 on success, otherwise non-zero error code
2174 static int hl_fw_dynamic_report_reset_cause(struct hl_device *hdev,
2175 struct fw_load_mgr *fw_loader,
2176 enum comms_reset_cause reset_cause)
2178 struct lkd_msg_comms msg;
2181 memset(&msg, 0, sizeof(msg));
2183 /* create message to be sent */
2184 msg.header.type = HL_COMMS_RESET_CAUSE_TYPE;
2185 msg.header.size = cpu_to_le16(sizeof(struct comms_msg_header));
2186 msg.header.magic = cpu_to_le32(HL_COMMS_MSG_MAGIC);
2188 msg.reset_cause = reset_cause;
2190 rc = hl_fw_dynamic_request_descriptor(hdev, fw_loader,
2191 sizeof(struct lkd_msg_comms));
2195 /* copy message to space allocated by FW */
2196 rc = hl_fw_dynamic_copy_msg(hdev, &msg, fw_loader);
2200 rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_DATA_RDY,
2202 fw_loader->cpu_timeout);
2206 rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_EXEC,
2208 fw_loader->cpu_timeout);
2216 * hl_fw_dynamic_init_cpu - initialize the device CPU using dynamic protocol
2218 * @hdev: pointer to the habanalabs device structure
2219 * @fw_loader: managing structure for loading device's FW
2221 * @return 0 on success, otherwise non-zero error code
2223 * brief: the dynamic protocol is master (LKD) slave (FW CPU) protocol.
2224 * the communication is done using registers:
2225 * - LKD command register
2226 * - FW status register
2227 * the protocol is race free. this goal is achieved by splitting the requests
2228 * and response to known synchronization points between the LKD and the FW.
2229 * each response to LKD request is known and bound to a predefined timeout.
2230 * in case of timeout expiration without the desired status from FW- the
2231 * protocol (and hence the boot) will fail.
2233 static int hl_fw_dynamic_init_cpu(struct hl_device *hdev,
2234 struct fw_load_mgr *fw_loader)
2236 struct cpu_dyn_regs *dyn_regs;
2240 "Loading firmware to device, may take some time...\n");
2243 * In this stage, "cpu_dyn_regs" contains only LKD's hard coded values!
2244 * It will be updated from FW after hl_fw_dynamic_request_descriptor().
2246 dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs;
2248 rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_RST_STATE,
2250 fw_loader->cpu_timeout);
2254 if (hdev->curr_reset_cause) {
2255 rc = hl_fw_dynamic_report_reset_cause(hdev, fw_loader,
2256 hdev->curr_reset_cause);
2260 /* Clear current reset cause */
2261 hdev->curr_reset_cause = HL_RESET_CAUSE_UNKNOWN;
2264 if (!(hdev->fw_components & FW_TYPE_BOOT_CPU)) {
2265 rc = hl_fw_dynamic_request_descriptor(hdev, fw_loader, 0);
2269 /* read preboot version */
2270 hl_fw_dynamic_read_device_fw_version(hdev, FW_COMP_PREBOOT,
2271 fw_loader->dynamic_loader.comm_desc.cur_fw_ver);
2275 /* load boot fit to FW */
2276 rc = hl_fw_dynamic_load_image(hdev, fw_loader, FW_COMP_BOOT_FIT,
2277 fw_loader->boot_fit_timeout);
2279 dev_err(hdev->dev, "failed to load boot fit\n");
2283 rc = hl_fw_dynamic_wait_for_boot_fit_active(hdev, fw_loader);
2287 /* Enable DRAM scrambling before Linux boot and after successful
2290 hdev->asic_funcs->init_cpu_scrambler_dram(hdev);
2292 if (!(hdev->fw_components & FW_TYPE_LINUX)) {
2293 dev_info(hdev->dev, "Skip loading Linux F/W\n");
2297 if (fw_loader->skip_bmc) {
2298 rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader,
2301 fw_loader->cpu_timeout);
2303 dev_err(hdev->dev, "failed to load boot fit\n");
2308 /* load Linux image to FW */
2309 rc = hl_fw_dynamic_load_image(hdev, fw_loader, FW_COMP_LINUX,
2310 fw_loader->cpu_timeout);
2312 dev_err(hdev->dev, "failed to load Linux\n");
2316 rc = hl_fw_dynamic_wait_for_linux_active(hdev, fw_loader);
2320 hl_fw_linux_update_state(hdev, le32_to_cpu(dyn_regs->cpu_boot_dev_sts0),
2321 le32_to_cpu(dyn_regs->cpu_boot_dev_sts1));
2323 hl_fw_dynamic_update_linux_interrupt_if(hdev);
2328 fw_read_errors(hdev, le32_to_cpu(dyn_regs->cpu_boot_err0),
2329 le32_to_cpu(dyn_regs->cpu_boot_err1),
2330 le32_to_cpu(dyn_regs->cpu_boot_dev_sts0),
2331 le32_to_cpu(dyn_regs->cpu_boot_dev_sts1));
2336 * hl_fw_static_init_cpu - initialize the device CPU using static protocol
2338 * @hdev: pointer to the habanalabs device structure
2339 * @fw_loader: managing structure for loading device's FW
2341 * @return 0 on success, otherwise non-zero error code
2343 static int hl_fw_static_init_cpu(struct hl_device *hdev,
2344 struct fw_load_mgr *fw_loader)
2346 u32 cpu_msg_status_reg, cpu_timeout, msg_to_cpu_reg, status;
2347 u32 cpu_boot_dev_status0_reg, cpu_boot_dev_status1_reg;
2348 struct static_fw_load_mgr *static_loader;
2349 u32 cpu_boot_status_reg;
2352 if (!(hdev->fw_components & FW_TYPE_BOOT_CPU))
2355 /* init common loader parameters */
2356 cpu_timeout = fw_loader->cpu_timeout;
2358 /* init static loader parameters */
2359 static_loader = &fw_loader->static_loader;
2360 cpu_msg_status_reg = static_loader->cpu_cmd_status_to_host_reg;
2361 msg_to_cpu_reg = static_loader->kmd_msg_to_cpu_reg;
2362 cpu_boot_dev_status0_reg = static_loader->cpu_boot_dev_status0_reg;
2363 cpu_boot_dev_status1_reg = static_loader->cpu_boot_dev_status1_reg;
2364 cpu_boot_status_reg = static_loader->cpu_boot_status_reg;
2366 dev_info(hdev->dev, "Going to wait for device boot (up to %lds)\n",
2367 cpu_timeout / USEC_PER_SEC);
2369 /* Wait for boot FIT request */
2370 rc = hl_poll_timeout(
2372 cpu_boot_status_reg,
2374 status == CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT,
2375 FW_CPU_STATUS_POLL_INTERVAL_USEC,
2376 fw_loader->boot_fit_timeout);
2380 "No boot fit request received, resuming boot\n");
2382 rc = hdev->asic_funcs->load_boot_fit_to_device(hdev);
2386 /* Clear device CPU message status */
2387 WREG32(cpu_msg_status_reg, CPU_MSG_CLR);
2389 /* Signal device CPU that boot loader is ready */
2390 WREG32(msg_to_cpu_reg, KMD_MSG_FIT_RDY);
2392 /* Poll for CPU device ack */
2393 rc = hl_poll_timeout(
2397 status == CPU_MSG_OK,
2398 FW_CPU_STATUS_POLL_INTERVAL_USEC,
2399 fw_loader->boot_fit_timeout);
2403 "Timeout waiting for boot fit load ack\n");
2408 WREG32(msg_to_cpu_reg, KMD_MSG_NA);
2411 /* Make sure CPU boot-loader is running */
2412 rc = hl_poll_timeout(
2414 cpu_boot_status_reg,
2416 (status == CPU_BOOT_STATUS_DRAM_RDY) ||
2417 (status == CPU_BOOT_STATUS_NIC_FW_RDY) ||
2418 (status == CPU_BOOT_STATUS_READY_TO_BOOT) ||
2419 (status == CPU_BOOT_STATUS_SRAM_AVAIL),
2420 FW_CPU_STATUS_POLL_INTERVAL_USEC,
2423 dev_dbg(hdev->dev, "uboot status = %d\n", status);
2425 /* Read U-Boot version now in case we will later fail */
2426 hl_fw_static_read_device_fw_version(hdev, FW_COMP_BOOT_FIT);
2428 /* update state according to boot stage */
2429 hl_fw_boot_fit_update_state(hdev, cpu_boot_dev_status0_reg,
2430 cpu_boot_dev_status1_reg);
2433 detect_cpu_boot_status(hdev, status);
2438 /* Enable DRAM scrambling before Linux boot and after successful
2441 hdev->asic_funcs->init_cpu_scrambler_dram(hdev);
2443 if (!(hdev->fw_components & FW_TYPE_LINUX)) {
2444 dev_info(hdev->dev, "Skip loading Linux F/W\n");
2449 if (status == CPU_BOOT_STATUS_SRAM_AVAIL) {
2455 "Loading firmware to device, may take some time...\n");
2457 rc = hdev->asic_funcs->load_firmware_to_device(hdev);
2461 if (fw_loader->skip_bmc) {
2462 WREG32(msg_to_cpu_reg, KMD_MSG_SKIP_BMC);
2464 rc = hl_poll_timeout(
2466 cpu_boot_status_reg,
2468 (status == CPU_BOOT_STATUS_BMC_WAITING_SKIPPED),
2469 FW_CPU_STATUS_POLL_INTERVAL_USEC,
2474 "Failed to get ACK on skipping BMC, %d\n",
2476 WREG32(msg_to_cpu_reg, KMD_MSG_NA);
2482 WREG32(msg_to_cpu_reg, KMD_MSG_FIT_RDY);
2484 rc = hl_poll_timeout(
2486 cpu_boot_status_reg,
2488 (status == CPU_BOOT_STATUS_SRAM_AVAIL),
2489 FW_CPU_STATUS_POLL_INTERVAL_USEC,
2493 WREG32(msg_to_cpu_reg, KMD_MSG_NA);
2496 if (status == CPU_BOOT_STATUS_FIT_CORRUPTED)
2498 "Device reports FIT image is corrupted\n");
2501 "Failed to load firmware to device, %d\n",
2508 rc = fw_read_errors(hdev, fw_loader->static_loader.boot_err0_reg,
2509 fw_loader->static_loader.boot_err1_reg,
2510 cpu_boot_dev_status0_reg,
2511 cpu_boot_dev_status1_reg);
2515 hl_fw_linux_update_state(hdev, cpu_boot_dev_status0_reg,
2516 cpu_boot_dev_status1_reg);
2521 fw_read_errors(hdev, fw_loader->static_loader.boot_err0_reg,
2522 fw_loader->static_loader.boot_err1_reg,
2523 cpu_boot_dev_status0_reg,
2524 cpu_boot_dev_status1_reg);
2530 * hl_fw_init_cpu - initialize the device CPU
2532 * @hdev: pointer to the habanalabs device structure
2534 * @return 0 on success, otherwise non-zero error code
2536 * perform necessary initializations for device's CPU. takes into account if
2537 * init protocol is static or dynamic.
2539 int hl_fw_init_cpu(struct hl_device *hdev)
2541 struct asic_fixed_properties *prop = &hdev->asic_prop;
2542 struct fw_load_mgr *fw_loader = &hdev->fw_loader;
2544 return prop->dynamic_fw_load ?
2545 hl_fw_dynamic_init_cpu(hdev, fw_loader) :
2546 hl_fw_static_init_cpu(hdev, fw_loader);