1 // SPDX-License-Identifier: GPL-2.0
4 * Copyright 2016-2019 HabanaLabs, Ltd.
8 #include "habanalabs.h"
9 #include "../include/common/hl_boot_if.h"
11 #include <linux/firmware.h>
12 #include <linux/crc32.h>
13 #include <linux/slab.h>
14 #include <linux/ctype.h>
16 #define FW_FILE_MAX_SIZE 0x1400000 /* maximum size of 20MB */
18 #define FW_CPU_STATUS_POLL_INTERVAL_USEC 10000
20 static char *extract_fw_ver_from_str(const char *fw_str)
22 char *str, *fw_ver, *whitespace;
24 fw_ver = kmalloc(16, GFP_KERNEL);
28 str = strnstr(fw_str, "fw-", VERSION_MAX_LEN);
32 /* Skip the fw- part */
35 /* Copy until the next whitespace */
36 whitespace = strnstr(str, " ", 15);
40 strscpy(fw_ver, str, whitespace - str + 1);
49 static int hl_request_fw(struct hl_device *hdev,
50 const struct firmware **firmware_p,
56 rc = request_firmware(firmware_p, fw_name, hdev->dev);
58 dev_err(hdev->dev, "Firmware file %s is not found! (error %d)\n",
63 fw_size = (*firmware_p)->size;
64 if ((fw_size % 4) != 0) {
65 dev_err(hdev->dev, "Illegal %s firmware size %zu\n",
71 dev_dbg(hdev->dev, "%s firmware size == %zu\n", fw_name, fw_size);
73 if (fw_size > FW_FILE_MAX_SIZE) {
75 "FW file size %zu exceeds maximum of %u bytes\n",
76 fw_size, FW_FILE_MAX_SIZE);
84 release_firmware(*firmware_p);
90 * hl_release_firmware() - release FW
94 * note: this inline function added to serve as a comprehensive mirror for the
95 * hl_request_fw function.
97 static inline void hl_release_firmware(const struct firmware *fw)
103 * hl_fw_copy_fw_to_device() - copy FW to device
105 * @hdev: pointer to hl_device structure.
107 * @dst: IO memory mapped address space to copy firmware to
108 * @src_offset: offset in src FW to copy from
109 * @size: amount of bytes to copy (0 to copy the whole binary)
111 * actual copy of FW binary data to device, shared by static and dynamic loaders
113 static int hl_fw_copy_fw_to_device(struct hl_device *hdev,
114 const struct firmware *fw, void __iomem *dst,
115 u32 src_offset, u32 size)
119 /* size 0 indicates to copy the whole file */
123 if (src_offset + size > fw->size) {
125 "size to copy(%u) and offset(%u) are invalid\n",
130 fw_data = (const void *) fw->data;
132 memcpy_toio(dst, fw_data + src_offset, size);
137 * hl_fw_copy_msg_to_device() - copy message to device
139 * @hdev: pointer to hl_device structure.
141 * @dst: IO memory mapped address space to copy firmware to
142 * @src_offset: offset in src message to copy from
143 * @size: amount of bytes to copy (0 to copy the whole binary)
145 * actual copy of message data to device.
147 static int hl_fw_copy_msg_to_device(struct hl_device *hdev,
148 struct lkd_msg_comms *msg, void __iomem *dst,
149 u32 src_offset, u32 size)
153 /* size 0 indicates to copy the whole file */
155 size = sizeof(struct lkd_msg_comms);
157 if (src_offset + size > sizeof(struct lkd_msg_comms)) {
159 "size to copy(%u) and offset(%u) are invalid\n",
164 msg_data = (void *) msg;
166 memcpy_toio(dst, msg_data + src_offset, size);
172 * hl_fw_load_fw_to_device() - Load F/W code to device's memory.
174 * @hdev: pointer to hl_device structure.
175 * @fw_name: the firmware image name
176 * @dst: IO memory mapped address space to copy firmware to
177 * @src_offset: offset in src FW to copy from
178 * @size: amount of bytes to copy (0 to copy the whole binary)
180 * Copy fw code from firmware file to device memory.
182 * Return: 0 on success, non-zero for failure.
184 int hl_fw_load_fw_to_device(struct hl_device *hdev, const char *fw_name,
185 void __iomem *dst, u32 src_offset, u32 size)
187 const struct firmware *fw;
190 rc = hl_request_fw(hdev, &fw, fw_name);
194 rc = hl_fw_copy_fw_to_device(hdev, fw, dst, src_offset, size);
196 hl_release_firmware(fw);
200 int hl_fw_send_pci_access_msg(struct hl_device *hdev, u32 opcode)
202 struct cpucp_packet pkt = {};
204 pkt.ctl = cpu_to_le32(opcode << CPUCP_PKT_CTL_OPCODE_SHIFT);
206 return hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt,
207 sizeof(pkt), 0, NULL);
210 int hl_fw_send_cpu_message(struct hl_device *hdev, u32 hw_queue_id, u32 *msg,
211 u16 len, u32 timeout, u64 *result)
213 struct hl_hw_queue *queue = &hdev->kernel_queues[hw_queue_id];
214 struct asic_fixed_properties *prop = &hdev->asic_prop;
215 struct cpucp_packet *pkt;
216 dma_addr_t pkt_dma_addr;
217 u32 tmp, expected_ack_val;
220 pkt = hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev, len,
224 "Failed to allocate DMA memory for packet to CPU\n");
228 memcpy(pkt, msg, len);
230 mutex_lock(&hdev->send_cpu_message_lock);
235 if (hdev->device_cpu_disabled) {
240 /* set fence to a non valid value */
241 pkt->fence = cpu_to_le32(UINT_MAX);
243 rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id, len, pkt_dma_addr);
245 dev_err(hdev->dev, "Failed to send CB on CPU PQ (%d)\n", rc);
249 if (prop->fw_app_cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_PKT_PI_ACK_EN)
250 expected_ack_val = queue->pi;
252 expected_ack_val = CPUCP_PACKET_FENCE_VAL;
254 rc = hl_poll_timeout_memory(hdev, &pkt->fence, tmp,
255 (tmp == expected_ack_val), 1000,
258 hl_hw_queue_inc_ci_kernel(hdev, hw_queue_id);
260 if (rc == -ETIMEDOUT) {
261 dev_err(hdev->dev, "Device CPU packet timeout (0x%x)\n", tmp);
262 hdev->device_cpu_disabled = true;
266 tmp = le32_to_cpu(pkt->ctl);
268 rc = (tmp & CPUCP_PKT_CTL_RC_MASK) >> CPUCP_PKT_CTL_RC_SHIFT;
270 dev_err(hdev->dev, "F/W ERROR %d for CPU packet %d\n",
272 (tmp & CPUCP_PKT_CTL_OPCODE_MASK)
273 >> CPUCP_PKT_CTL_OPCODE_SHIFT);
276 *result = le64_to_cpu(pkt->result);
280 mutex_unlock(&hdev->send_cpu_message_lock);
282 hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev, len, pkt);
287 int hl_fw_unmask_irq(struct hl_device *hdev, u16 event_type)
289 struct cpucp_packet pkt;
293 memset(&pkt, 0, sizeof(pkt));
295 pkt.ctl = cpu_to_le32(CPUCP_PACKET_UNMASK_RAZWI_IRQ <<
296 CPUCP_PKT_CTL_OPCODE_SHIFT);
297 pkt.value = cpu_to_le64(event_type);
299 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
303 dev_err(hdev->dev, "failed to unmask RAZWI IRQ %d", event_type);
308 int hl_fw_unmask_irq_arr(struct hl_device *hdev, const u32 *irq_arr,
311 struct cpucp_unmask_irq_arr_packet *pkt;
312 size_t total_pkt_size;
316 total_pkt_size = sizeof(struct cpucp_unmask_irq_arr_packet) +
319 /* data should be aligned to 8 bytes in order to CPU-CP to copy it */
320 total_pkt_size = (total_pkt_size + 0x7) & ~0x7;
322 /* total_pkt_size is casted to u16 later on */
323 if (total_pkt_size > USHRT_MAX) {
324 dev_err(hdev->dev, "too many elements in IRQ array\n");
328 pkt = kzalloc(total_pkt_size, GFP_KERNEL);
332 pkt->length = cpu_to_le32(irq_arr_size / sizeof(irq_arr[0]));
333 memcpy(&pkt->irqs, irq_arr, irq_arr_size);
335 pkt->cpucp_pkt.ctl = cpu_to_le32(CPUCP_PACKET_UNMASK_RAZWI_IRQ_ARRAY <<
336 CPUCP_PKT_CTL_OPCODE_SHIFT);
338 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) pkt,
339 total_pkt_size, 0, &result);
342 dev_err(hdev->dev, "failed to unmask IRQ array\n");
349 int hl_fw_test_cpu_queue(struct hl_device *hdev)
351 struct cpucp_packet test_pkt = {};
355 test_pkt.ctl = cpu_to_le32(CPUCP_PACKET_TEST <<
356 CPUCP_PKT_CTL_OPCODE_SHIFT);
357 test_pkt.value = cpu_to_le64(CPUCP_PACKET_FENCE_VAL);
359 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &test_pkt,
360 sizeof(test_pkt), 0, &result);
363 if (result != CPUCP_PACKET_FENCE_VAL)
365 "CPU queue test failed (%#08llx)\n", result);
367 dev_err(hdev->dev, "CPU queue test failed, error %d\n", rc);
373 void *hl_fw_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size,
374 dma_addr_t *dma_handle)
378 kernel_addr = gen_pool_alloc(hdev->cpu_accessible_dma_pool, size);
380 *dma_handle = hdev->cpu_accessible_dma_address +
381 (kernel_addr - (u64) (uintptr_t) hdev->cpu_accessible_dma_mem);
383 return (void *) (uintptr_t) kernel_addr;
386 void hl_fw_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size,
389 gen_pool_free(hdev->cpu_accessible_dma_pool, (u64) (uintptr_t) vaddr,
393 int hl_fw_send_heartbeat(struct hl_device *hdev)
395 struct cpucp_packet hb_pkt;
399 memset(&hb_pkt, 0, sizeof(hb_pkt));
400 hb_pkt.ctl = cpu_to_le32(CPUCP_PACKET_TEST <<
401 CPUCP_PKT_CTL_OPCODE_SHIFT);
402 hb_pkt.value = cpu_to_le64(CPUCP_PACKET_FENCE_VAL);
404 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &hb_pkt,
405 sizeof(hb_pkt), 0, &result);
407 if ((rc) || (result != CPUCP_PACKET_FENCE_VAL))
410 if (le32_to_cpu(hb_pkt.status_mask) &
411 CPUCP_PKT_HB_STATUS_EQ_FAULT_MASK) {
412 dev_warn(hdev->dev, "FW reported EQ fault during heartbeat\n");
419 static bool fw_report_boot_dev0(struct hl_device *hdev, u32 err_val,
422 bool err_exists = false;
424 if (!(err_val & CPU_BOOT_ERR0_ENABLED))
427 if (err_val & CPU_BOOT_ERR0_DRAM_INIT_FAIL) {
429 "Device boot error - DRAM initialization failed\n");
433 if (err_val & CPU_BOOT_ERR0_FIT_CORRUPTED) {
434 dev_err(hdev->dev, "Device boot error - FIT image corrupted\n");
438 if (err_val & CPU_BOOT_ERR0_TS_INIT_FAIL) {
440 "Device boot error - Thermal Sensor initialization failed\n");
444 if (err_val & CPU_BOOT_ERR0_DRAM_SKIPPED) {
446 "Device boot warning - Skipped DRAM initialization\n");
447 /* This is a warning so we don't want it to disable the
450 err_val &= ~CPU_BOOT_ERR0_DRAM_SKIPPED;
453 if (err_val & CPU_BOOT_ERR0_BMC_WAIT_SKIPPED) {
454 if (hdev->bmc_enable) {
456 "Device boot error - Skipped waiting for BMC\n");
460 "Device boot message - Skipped waiting for BMC\n");
461 /* This is an info so we don't want it to disable the
464 err_val &= ~CPU_BOOT_ERR0_BMC_WAIT_SKIPPED;
468 if (err_val & CPU_BOOT_ERR0_NIC_DATA_NOT_RDY) {
470 "Device boot error - Serdes data from BMC not available\n");
474 if (err_val & CPU_BOOT_ERR0_NIC_FW_FAIL) {
476 "Device boot error - NIC F/W initialization failed\n");
480 if (err_val & CPU_BOOT_ERR0_SECURITY_NOT_RDY) {
482 "Device boot warning - security not ready\n");
486 if (err_val & CPU_BOOT_ERR0_SECURITY_FAIL) {
487 dev_err(hdev->dev, "Device boot error - security failure\n");
491 if (err_val & CPU_BOOT_ERR0_EFUSE_FAIL) {
492 dev_err(hdev->dev, "Device boot error - eFuse failure\n");
496 if (err_val & CPU_BOOT_ERR0_PRI_IMG_VER_FAIL) {
498 "Device boot warning - Failed to load preboot primary image\n");
499 /* This is a warning so we don't want it to disable the
500 * device as we have a secondary preboot image
502 err_val &= ~CPU_BOOT_ERR0_PRI_IMG_VER_FAIL;
505 if (err_val & CPU_BOOT_ERR0_SEC_IMG_VER_FAIL) {
506 dev_err(hdev->dev, "Device boot error - Failed to load preboot secondary image\n");
510 if (err_val & CPU_BOOT_ERR0_PLL_FAIL) {
511 dev_err(hdev->dev, "Device boot error - PLL failure\n");
515 if (err_val & CPU_BOOT_ERR0_DEVICE_UNUSABLE_FAIL) {
516 /* Ignore this bit, don't prevent driver loading */
517 dev_dbg(hdev->dev, "device unusable status is set\n");
518 err_val &= ~CPU_BOOT_ERR0_DEVICE_UNUSABLE_FAIL;
521 if (sts_val & CPU_BOOT_DEV_STS0_ENABLED)
522 dev_dbg(hdev->dev, "Device status0 %#x\n", sts_val);
524 if (!err_exists && (err_val & ~CPU_BOOT_ERR0_ENABLED)) {
526 "Device boot error - unknown ERR0 error 0x%08x\n", err_val);
530 /* return error only if it's in the predefined mask */
531 if (err_exists && ((err_val & ~CPU_BOOT_ERR0_ENABLED) &
532 lower_32_bits(hdev->boot_error_status_mask)))
538 /* placeholder for ERR1 as no errors defined there yet */
539 static bool fw_report_boot_dev1(struct hl_device *hdev, u32 err_val,
543 * keep this variable to preserve the logic of the function.
544 * this way it would require less modifications when error will be
547 bool err_exists = false;
549 if (!(err_val & CPU_BOOT_ERR1_ENABLED))
552 if (sts_val & CPU_BOOT_DEV_STS1_ENABLED)
553 dev_dbg(hdev->dev, "Device status1 %#x\n", sts_val);
555 if (!err_exists && (err_val & ~CPU_BOOT_ERR1_ENABLED)) {
557 "Device boot error - unknown ERR1 error 0x%08x\n",
562 /* return error only if it's in the predefined mask */
563 if (err_exists && ((err_val & ~CPU_BOOT_ERR1_ENABLED) &
564 upper_32_bits(hdev->boot_error_status_mask)))
570 static int fw_read_errors(struct hl_device *hdev, u32 boot_err0_reg,
571 u32 boot_err1_reg, u32 cpu_boot_dev_status0_reg,
572 u32 cpu_boot_dev_status1_reg)
574 u32 err_val, status_val;
575 bool err_exists = false;
577 /* Some of the firmware status codes are deprecated in newer f/w
578 * versions. In those versions, the errors are reported
579 * in different registers. Therefore, we need to check those
580 * registers and print the exact errors. Moreover, there
581 * may be multiple errors, so we need to report on each error
582 * separately. Some of the error codes might indicate a state
583 * that is not an error per-se, but it is an error in production
586 err_val = RREG32(boot_err0_reg);
587 status_val = RREG32(cpu_boot_dev_status0_reg);
588 err_exists = fw_report_boot_dev0(hdev, err_val, status_val);
590 err_val = RREG32(boot_err1_reg);
591 status_val = RREG32(cpu_boot_dev_status1_reg);
592 err_exists |= fw_report_boot_dev1(hdev, err_val, status_val);
600 int hl_fw_cpucp_info_get(struct hl_device *hdev,
601 u32 sts_boot_dev_sts0_reg,
602 u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg,
605 struct asic_fixed_properties *prop = &hdev->asic_prop;
606 struct cpucp_packet pkt = {};
607 dma_addr_t cpucp_info_dma_addr;
608 void *cpucp_info_cpu_addr;
613 cpucp_info_cpu_addr =
614 hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev,
615 sizeof(struct cpucp_info),
616 &cpucp_info_dma_addr);
617 if (!cpucp_info_cpu_addr) {
619 "Failed to allocate DMA memory for CPU-CP info packet\n");
623 memset(cpucp_info_cpu_addr, 0, sizeof(struct cpucp_info));
625 pkt.ctl = cpu_to_le32(CPUCP_PACKET_INFO_GET <<
626 CPUCP_PKT_CTL_OPCODE_SHIFT);
627 pkt.addr = cpu_to_le64(cpucp_info_dma_addr);
628 pkt.data_max_size = cpu_to_le32(sizeof(struct cpucp_info));
630 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
631 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
634 "Failed to handle CPU-CP info pkt, error %d\n", rc);
638 rc = fw_read_errors(hdev, boot_err0_reg, boot_err1_reg,
639 sts_boot_dev_sts0_reg, sts_boot_dev_sts1_reg);
641 dev_err(hdev->dev, "Errors in device boot\n");
645 memcpy(&prop->cpucp_info, cpucp_info_cpu_addr,
646 sizeof(prop->cpucp_info));
648 rc = hl_build_hwmon_channel_info(hdev, prop->cpucp_info.sensors);
651 "Failed to build hwmon channel info, error %d\n", rc);
656 kernel_ver = extract_fw_ver_from_str(prop->cpucp_info.kernel_version);
658 dev_info(hdev->dev, "Linux version %s", kernel_ver);
662 /* assume EQ code doesn't need to check eqe index */
663 hdev->event_queue.check_eqe_index = false;
665 /* Read FW application security bits again */
666 if (hdev->asic_prop.fw_cpu_boot_dev_sts0_valid) {
667 hdev->asic_prop.fw_app_cpu_boot_dev_sts0 =
668 RREG32(sts_boot_dev_sts0_reg);
669 if (hdev->asic_prop.fw_app_cpu_boot_dev_sts0 &
670 CPU_BOOT_DEV_STS0_EQ_INDEX_EN)
671 hdev->event_queue.check_eqe_index = true;
674 if (hdev->asic_prop.fw_cpu_boot_dev_sts1_valid)
675 hdev->asic_prop.fw_app_cpu_boot_dev_sts1 =
676 RREG32(sts_boot_dev_sts1_reg);
679 hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev,
680 sizeof(struct cpucp_info), cpucp_info_cpu_addr);
685 static int hl_fw_send_msi_info_msg(struct hl_device *hdev)
687 struct cpucp_array_data_packet *pkt;
688 size_t total_pkt_size, data_size;
692 /* skip sending this info for unsupported ASICs */
693 if (!hdev->asic_funcs->get_msi_info)
696 data_size = CPUCP_NUM_OF_MSI_TYPES * sizeof(u32);
697 total_pkt_size = sizeof(struct cpucp_array_data_packet) + data_size;
699 /* data should be aligned to 8 bytes in order to CPU-CP to copy it */
700 total_pkt_size = (total_pkt_size + 0x7) & ~0x7;
702 /* total_pkt_size is casted to u16 later on */
703 if (total_pkt_size > USHRT_MAX) {
704 dev_err(hdev->dev, "CPUCP array data is too big\n");
708 pkt = kzalloc(total_pkt_size, GFP_KERNEL);
712 pkt->length = cpu_to_le32(CPUCP_NUM_OF_MSI_TYPES);
714 memset((void *) &pkt->data, 0xFF, data_size);
715 hdev->asic_funcs->get_msi_info(pkt->data);
717 pkt->cpucp_pkt.ctl = cpu_to_le32(CPUCP_PACKET_MSI_INFO_SET <<
718 CPUCP_PKT_CTL_OPCODE_SHIFT);
720 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *)pkt,
721 total_pkt_size, 0, &result);
724 * in case packet result is invalid it means that FW does not support
725 * this feature and will use default/hard coded MSI values. no reason
728 if (rc && result == cpucp_packet_invalid)
732 dev_err(hdev->dev, "failed to send CPUCP array data\n");
739 int hl_fw_cpucp_handshake(struct hl_device *hdev,
740 u32 sts_boot_dev_sts0_reg,
741 u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg,
746 rc = hl_fw_cpucp_info_get(hdev, sts_boot_dev_sts0_reg,
747 sts_boot_dev_sts1_reg, boot_err0_reg,
752 return hl_fw_send_msi_info_msg(hdev);
755 int hl_fw_get_eeprom_data(struct hl_device *hdev, void *data, size_t max_size)
757 struct cpucp_packet pkt = {};
758 void *eeprom_info_cpu_addr;
759 dma_addr_t eeprom_info_dma_addr;
763 eeprom_info_cpu_addr =
764 hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev,
765 max_size, &eeprom_info_dma_addr);
766 if (!eeprom_info_cpu_addr) {
768 "Failed to allocate DMA memory for CPU-CP EEPROM packet\n");
772 memset(eeprom_info_cpu_addr, 0, max_size);
774 pkt.ctl = cpu_to_le32(CPUCP_PACKET_EEPROM_DATA_GET <<
775 CPUCP_PKT_CTL_OPCODE_SHIFT);
776 pkt.addr = cpu_to_le64(eeprom_info_dma_addr);
777 pkt.data_max_size = cpu_to_le32(max_size);
779 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
780 HL_CPUCP_EEPROM_TIMEOUT_USEC, &result);
784 "Failed to handle CPU-CP EEPROM packet, error %d\n",
789 /* result contains the actual size */
790 memcpy(data, eeprom_info_cpu_addr, min((size_t)result, max_size));
793 hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev, max_size,
794 eeprom_info_cpu_addr);
799 int hl_fw_cpucp_pci_counters_get(struct hl_device *hdev,
800 struct hl_info_pci_counters *counters)
802 struct cpucp_packet pkt = {};
806 pkt.ctl = cpu_to_le32(CPUCP_PACKET_PCIE_THROUGHPUT_GET <<
807 CPUCP_PKT_CTL_OPCODE_SHIFT);
809 /* Fetch PCI rx counter */
810 pkt.index = cpu_to_le32(cpucp_pcie_throughput_rx);
811 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
812 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
815 "Failed to handle CPU-CP PCI info pkt, error %d\n", rc);
818 counters->rx_throughput = result;
820 memset(&pkt, 0, sizeof(pkt));
821 pkt.ctl = cpu_to_le32(CPUCP_PACKET_PCIE_THROUGHPUT_GET <<
822 CPUCP_PKT_CTL_OPCODE_SHIFT);
824 /* Fetch PCI tx counter */
825 pkt.index = cpu_to_le32(cpucp_pcie_throughput_tx);
826 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
827 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
830 "Failed to handle CPU-CP PCI info pkt, error %d\n", rc);
833 counters->tx_throughput = result;
835 /* Fetch PCI replay counter */
836 memset(&pkt, 0, sizeof(pkt));
837 pkt.ctl = cpu_to_le32(CPUCP_PACKET_PCIE_REPLAY_CNT_GET <<
838 CPUCP_PKT_CTL_OPCODE_SHIFT);
840 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
841 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
844 "Failed to handle CPU-CP PCI info pkt, error %d\n", rc);
847 counters->replay_cnt = (u32) result;
852 int hl_fw_cpucp_total_energy_get(struct hl_device *hdev, u64 *total_energy)
854 struct cpucp_packet pkt = {};
858 pkt.ctl = cpu_to_le32(CPUCP_PACKET_TOTAL_ENERGY_GET <<
859 CPUCP_PKT_CTL_OPCODE_SHIFT);
861 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
862 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
865 "Failed to handle CpuCP total energy pkt, error %d\n",
870 *total_energy = result;
875 int get_used_pll_index(struct hl_device *hdev, u32 input_pll_index,
876 enum pll_index *pll_index)
878 struct asic_fixed_properties *prop = &hdev->asic_prop;
879 u8 pll_byte, pll_bit_off;
883 dynamic_pll = !!(prop->fw_app_cpu_boot_dev_sts0 &
884 CPU_BOOT_DEV_STS0_DYN_PLL_EN);
888 * in case we are working with legacy FW (each asic has unique
889 * PLL numbering) use the driver based index as they are
890 * aligned with fw legacy numbering
892 *pll_index = input_pll_index;
896 /* retrieve a FW compatible PLL index based on
897 * ASIC specific user request
899 fw_pll_idx = hdev->asic_funcs->map_pll_idx_to_fw_idx(input_pll_index);
900 if (fw_pll_idx < 0) {
901 dev_err(hdev->dev, "Invalid PLL index (%u) error %d\n",
902 input_pll_index, fw_pll_idx);
906 /* PLL map is a u8 array */
907 pll_byte = prop->cpucp_info.pll_map[fw_pll_idx >> 3];
908 pll_bit_off = fw_pll_idx & 0x7;
910 if (!(pll_byte & BIT(pll_bit_off))) {
911 dev_err(hdev->dev, "PLL index %d is not supported\n",
916 *pll_index = fw_pll_idx;
921 int hl_fw_cpucp_pll_info_get(struct hl_device *hdev, u32 pll_index,
924 struct cpucp_packet pkt;
925 enum pll_index used_pll_idx;
929 rc = get_used_pll_index(hdev, pll_index, &used_pll_idx);
933 memset(&pkt, 0, sizeof(pkt));
935 pkt.ctl = cpu_to_le32(CPUCP_PACKET_PLL_INFO_GET <<
936 CPUCP_PKT_CTL_OPCODE_SHIFT);
937 pkt.pll_type = __cpu_to_le16((u16)used_pll_idx);
939 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
940 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
942 dev_err(hdev->dev, "Failed to read PLL info, error %d\n", rc);
944 pll_freq_arr[0] = FIELD_GET(CPUCP_PKT_RES_PLL_OUT0_MASK, result);
945 pll_freq_arr[1] = FIELD_GET(CPUCP_PKT_RES_PLL_OUT1_MASK, result);
946 pll_freq_arr[2] = FIELD_GET(CPUCP_PKT_RES_PLL_OUT2_MASK, result);
947 pll_freq_arr[3] = FIELD_GET(CPUCP_PKT_RES_PLL_OUT3_MASK, result);
952 int hl_fw_cpucp_power_get(struct hl_device *hdev, u64 *power)
954 struct cpucp_packet pkt;
958 memset(&pkt, 0, sizeof(pkt));
960 pkt.ctl = cpu_to_le32(CPUCP_PACKET_POWER_GET <<
961 CPUCP_PKT_CTL_OPCODE_SHIFT);
963 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
964 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
966 dev_err(hdev->dev, "Failed to read power, error %d\n", rc);
975 void hl_fw_ask_hard_reset_without_linux(struct hl_device *hdev)
977 struct static_fw_load_mgr *static_loader =
978 &hdev->fw_loader.static_loader;
981 if (hdev->asic_prop.dynamic_fw_load) {
982 rc = hl_fw_dynamic_send_protocol_cmd(hdev, &hdev->fw_loader,
983 COMMS_RST_DEV, 0, false,
984 hdev->fw_loader.cpu_timeout);
986 dev_warn(hdev->dev, "Failed sending COMMS_RST_DEV\n");
988 WREG32(static_loader->kmd_msg_to_cpu_reg, KMD_MSG_RST_DEV);
992 void hl_fw_ask_halt_machine_without_linux(struct hl_device *hdev)
994 struct static_fw_load_mgr *static_loader =
995 &hdev->fw_loader.static_loader;
998 if (hdev->device_cpu_is_halted)
1001 /* Stop device CPU to make sure nothing bad happens */
1002 if (hdev->asic_prop.dynamic_fw_load) {
1003 rc = hl_fw_dynamic_send_protocol_cmd(hdev, &hdev->fw_loader,
1004 COMMS_GOTO_WFE, 0, true,
1005 hdev->fw_loader.cpu_timeout);
1007 dev_warn(hdev->dev, "Failed sending COMMS_GOTO_WFE\n");
1009 WREG32(static_loader->kmd_msg_to_cpu_reg, KMD_MSG_GOTO_WFE);
1010 msleep(static_loader->cpu_reset_wait_msec);
1013 hdev->device_cpu_is_halted = true;
1016 static void detect_cpu_boot_status(struct hl_device *hdev, u32 status)
1018 /* Some of the status codes below are deprecated in newer f/w
1019 * versions but we keep them here for backward compatibility
1022 case CPU_BOOT_STATUS_NA:
1024 "Device boot progress - BTL did NOT run\n");
1026 case CPU_BOOT_STATUS_IN_WFE:
1028 "Device boot progress - Stuck inside WFE loop\n");
1030 case CPU_BOOT_STATUS_IN_BTL:
1032 "Device boot progress - Stuck in BTL\n");
1034 case CPU_BOOT_STATUS_IN_PREBOOT:
1036 "Device boot progress - Stuck in Preboot\n");
1038 case CPU_BOOT_STATUS_IN_SPL:
1040 "Device boot progress - Stuck in SPL\n");
1042 case CPU_BOOT_STATUS_IN_UBOOT:
1044 "Device boot progress - Stuck in u-boot\n");
1046 case CPU_BOOT_STATUS_DRAM_INIT_FAIL:
1048 "Device boot progress - DRAM initialization failed\n");
1050 case CPU_BOOT_STATUS_UBOOT_NOT_READY:
1052 "Device boot progress - Cannot boot\n");
1054 case CPU_BOOT_STATUS_TS_INIT_FAIL:
1056 "Device boot progress - Thermal Sensor initialization failed\n");
1060 "Device boot progress - Invalid status code %d\n",
1066 static int hl_fw_read_preboot_caps(struct hl_device *hdev,
1067 u32 cpu_boot_status_reg,
1068 u32 sts_boot_dev_sts0_reg,
1069 u32 sts_boot_dev_sts1_reg,
1070 u32 boot_err0_reg, u32 boot_err1_reg,
1073 struct asic_fixed_properties *prop = &hdev->asic_prop;
1074 u32 status, reg_val;
1077 /* Need to check two possible scenarios:
1079 * CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT - for newer firmwares where
1080 * the preboot is waiting for the boot fit
1082 * All other status values - for older firmwares where the uboot was
1083 * loaded from the FLASH
1085 rc = hl_poll_timeout(
1087 cpu_boot_status_reg,
1089 (status == CPU_BOOT_STATUS_IN_UBOOT) ||
1090 (status == CPU_BOOT_STATUS_DRAM_RDY) ||
1091 (status == CPU_BOOT_STATUS_NIC_FW_RDY) ||
1092 (status == CPU_BOOT_STATUS_READY_TO_BOOT) ||
1093 (status == CPU_BOOT_STATUS_SRAM_AVAIL) ||
1094 (status == CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT),
1095 FW_CPU_STATUS_POLL_INTERVAL_USEC,
1099 dev_err(hdev->dev, "CPU boot ready status timeout\n");
1100 detect_cpu_boot_status(hdev, status);
1102 /* If we read all FF, then something is totally wrong, no point
1103 * of reading specific errors
1106 fw_read_errors(hdev, boot_err0_reg, boot_err1_reg,
1107 sts_boot_dev_sts0_reg,
1108 sts_boot_dev_sts1_reg);
1113 * the registers DEV_STS* contain FW capabilities/features.
1114 * We can rely on this registers only if bit CPU_BOOT_DEV_STS*_ENABLED
1116 * In the first read of this register we store the value of this
1117 * register ONLY if the register is enabled (which will be propagated
1118 * to next stages) and also mark the register as valid.
1119 * In case it is not enabled the stored value will be left 0- all
1120 * caps/features are off
1122 reg_val = RREG32(sts_boot_dev_sts0_reg);
1123 if (reg_val & CPU_BOOT_DEV_STS0_ENABLED) {
1124 prop->fw_cpu_boot_dev_sts0_valid = true;
1125 prop->fw_preboot_cpu_boot_dev_sts0 = reg_val;
1128 reg_val = RREG32(sts_boot_dev_sts1_reg);
1129 if (reg_val & CPU_BOOT_DEV_STS1_ENABLED) {
1130 prop->fw_cpu_boot_dev_sts1_valid = true;
1131 prop->fw_preboot_cpu_boot_dev_sts1 = reg_val;
1134 prop->dynamic_fw_load = !!(prop->fw_preboot_cpu_boot_dev_sts0 &
1135 CPU_BOOT_DEV_STS0_FW_LD_COM_EN);
1137 /* initialize FW loader once we know what load protocol is used */
1138 hdev->asic_funcs->init_firmware_loader(hdev);
1140 dev_dbg(hdev->dev, "Attempting %s FW load\n",
1141 prop->dynamic_fw_load ? "dynamic" : "legacy");
1145 static int hl_fw_static_read_device_fw_version(struct hl_device *hdev,
1146 enum hl_fw_component fwc)
1148 struct asic_fixed_properties *prop = &hdev->asic_prop;
1149 struct fw_load_mgr *fw_loader = &hdev->fw_loader;
1150 struct static_fw_load_mgr *static_loader;
1151 char *dest, *boot_ver, *preboot_ver;
1156 static_loader = &hdev->fw_loader.static_loader;
1159 case FW_COMP_BOOT_FIT:
1160 ver_off = RREG32(static_loader->boot_fit_version_offset_reg);
1161 dest = prop->uboot_ver;
1163 limit = static_loader->boot_fit_version_max_off;
1165 case FW_COMP_PREBOOT:
1166 ver_off = RREG32(static_loader->preboot_version_offset_reg);
1167 dest = prop->preboot_ver;
1169 limit = static_loader->preboot_version_max_off;
1172 dev_warn(hdev->dev, "Undefined FW component: %d\n", fwc);
1176 ver_off &= static_loader->sram_offset_mask;
1178 if (ver_off < limit) {
1180 hdev->pcie_bar[fw_loader->sram_bar_id] + ver_off,
1183 dev_err(hdev->dev, "%s version offset (0x%x) is above SRAM\n",
1185 strscpy(dest, "unavailable", VERSION_MAX_LEN);
1189 if (fwc == FW_COMP_BOOT_FIT) {
1190 boot_ver = extract_fw_ver_from_str(prop->uboot_ver);
1192 dev_info(hdev->dev, "boot-fit version %s\n", boot_ver);
1195 } else if (fwc == FW_COMP_PREBOOT) {
1196 preboot_ver = strnstr(prop->preboot_ver, "Preboot",
1198 if (preboot_ver && preboot_ver != prop->preboot_ver) {
1199 strscpy(btl_ver, prop->preboot_ver,
1200 min((int) (preboot_ver - prop->preboot_ver),
1202 dev_info(hdev->dev, "%s\n", btl_ver);
1205 preboot_ver = extract_fw_ver_from_str(prop->preboot_ver);
1207 dev_info(hdev->dev, "preboot version %s\n",
1217 * hl_fw_preboot_update_state - update internal data structures during
1218 * handshake with preboot
1221 * @hdev: pointer to the habanalabs device structure
1223 * @return 0 on success, otherwise non-zero error code
1225 static void hl_fw_preboot_update_state(struct hl_device *hdev)
1227 struct asic_fixed_properties *prop = &hdev->asic_prop;
1228 u32 cpu_boot_dev_sts0, cpu_boot_dev_sts1;
1230 cpu_boot_dev_sts0 = prop->fw_preboot_cpu_boot_dev_sts0;
1231 cpu_boot_dev_sts1 = prop->fw_preboot_cpu_boot_dev_sts1;
1233 /* We read boot_dev_sts registers multiple times during boot:
1234 * 1. preboot - a. Check whether the security status bits are valid
1235 * b. Check whether fw security is enabled
1236 * c. Check whether hard reset is done by preboot
1237 * 2. boot cpu - a. Fetch boot cpu security status
1238 * b. Check whether hard reset is done by boot cpu
1239 * 3. FW application - a. Fetch fw application security status
1240 * b. Check whether hard reset is done by fw app
1243 * Check security status bit (CPU_BOOT_DEV_STS0_ENABLED). If set, then-
1244 * check security enabled bit (CPU_BOOT_DEV_STS0_SECURITY_EN)
1245 * If set, then mark GIC controller to be disabled.
1247 prop->hard_reset_done_by_fw =
1248 !!(cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_FW_HARD_RST_EN);
1250 dev_dbg(hdev->dev, "Firmware preboot boot device status0 %#x\n",
1253 dev_dbg(hdev->dev, "Firmware preboot boot device status1 %#x\n",
1256 dev_dbg(hdev->dev, "Firmware preboot hard-reset is %s\n",
1257 prop->hard_reset_done_by_fw ? "enabled" : "disabled");
1259 dev_dbg(hdev->dev, "firmware-level security is %s\n",
1260 prop->fw_security_enabled ? "enabled" : "disabled");
1262 dev_dbg(hdev->dev, "GIC controller is %s\n",
1263 prop->gic_interrupts_enable ? "enabled" : "disabled");
1266 static int hl_fw_static_read_preboot_status(struct hl_device *hdev)
1270 rc = hl_fw_static_read_device_fw_version(hdev, FW_COMP_PREBOOT);
1277 int hl_fw_read_preboot_status(struct hl_device *hdev, u32 cpu_boot_status_reg,
1278 u32 sts_boot_dev_sts0_reg,
1279 u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg,
1280 u32 boot_err1_reg, u32 timeout)
1284 /* pldm was added for cases in which we use preboot on pldm and want
1285 * to load boot fit, but we can't wait for preboot because it runs
1288 if (!(hdev->fw_components & FW_TYPE_PREBOOT_CPU) || hdev->pldm)
1292 * In order to determine boot method (static VS dymanic) we need to
1293 * read the boot caps register
1295 rc = hl_fw_read_preboot_caps(hdev, cpu_boot_status_reg,
1296 sts_boot_dev_sts0_reg,
1297 sts_boot_dev_sts1_reg, boot_err0_reg,
1298 boot_err1_reg, timeout);
1302 hl_fw_preboot_update_state(hdev);
1304 /* no need to read preboot status in dynamic load */
1305 if (hdev->asic_prop.dynamic_fw_load)
1308 return hl_fw_static_read_preboot_status(hdev);
1311 /* associate string with COMM status */
1312 static char *hl_dynamic_fw_status_str[COMMS_STS_INVLD_LAST] = {
1313 [COMMS_STS_NOOP] = "NOOP",
1314 [COMMS_STS_ACK] = "ACK",
1315 [COMMS_STS_OK] = "OK",
1316 [COMMS_STS_ERR] = "ERR",
1317 [COMMS_STS_VALID_ERR] = "VALID_ERR",
1318 [COMMS_STS_TIMEOUT_ERR] = "TIMEOUT_ERR",
1322 * hl_fw_dynamic_report_error_status - report error status
1324 * @hdev: pointer to the habanalabs device structure
1325 * @status: value of FW status register
1326 * @expected_status: the expected status
1328 static void hl_fw_dynamic_report_error_status(struct hl_device *hdev,
1330 enum comms_sts expected_status)
1332 enum comms_sts comm_status =
1333 FIELD_GET(COMMS_STATUS_STATUS_MASK, status);
1335 if (comm_status < COMMS_STS_INVLD_LAST)
1336 dev_err(hdev->dev, "Device status %s, expected status: %s\n",
1337 hl_dynamic_fw_status_str[comm_status],
1338 hl_dynamic_fw_status_str[expected_status]);
1340 dev_err(hdev->dev, "Device status unknown %d, expected status: %s\n",
1342 hl_dynamic_fw_status_str[expected_status]);
1346 * hl_fw_dynamic_send_cmd - send LKD to FW cmd
1348 * @hdev: pointer to the habanalabs device structure
1349 * @fw_loader: managing structure for loading device's FW
1350 * @cmd: LKD to FW cmd code
1351 * @size: size of next FW component to be loaded (0 if not necessary)
1353 * LDK to FW exact command layout is defined at struct comms_command.
1354 * note: the size argument is used only when the next FW component should be
1355 * loaded, otherwise it shall be 0. the size is used by the FW in later
1356 * protocol stages and when sending only indicating the amount of memory
1357 * to be allocated by the FW to receive the next boot component.
1359 static void hl_fw_dynamic_send_cmd(struct hl_device *hdev,
1360 struct fw_load_mgr *fw_loader,
1361 enum comms_cmd cmd, unsigned int size)
1363 struct cpu_dyn_regs *dyn_regs;
1366 dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs;
1368 val = FIELD_PREP(COMMS_COMMAND_CMD_MASK, cmd);
1369 val |= FIELD_PREP(COMMS_COMMAND_SIZE_MASK, size);
1371 WREG32(le32_to_cpu(dyn_regs->kmd_msg_to_cpu), val);
1375 * hl_fw_dynamic_extract_fw_response - update the FW response
1377 * @hdev: pointer to the habanalabs device structure
1378 * @fw_loader: managing structure for loading device's FW
1379 * @response: FW response
1380 * @status: the status read from CPU status register
1382 * @return 0 on success, otherwise non-zero error code
1384 static int hl_fw_dynamic_extract_fw_response(struct hl_device *hdev,
1385 struct fw_load_mgr *fw_loader,
1386 struct fw_response *response,
1389 response->status = FIELD_GET(COMMS_STATUS_STATUS_MASK, status);
1390 response->ram_offset = FIELD_GET(COMMS_STATUS_OFFSET_MASK, status) <<
1391 COMMS_STATUS_OFFSET_ALIGN_SHIFT;
1392 response->ram_type = FIELD_GET(COMMS_STATUS_RAM_TYPE_MASK, status);
1394 if ((response->ram_type != COMMS_SRAM) &&
1395 (response->ram_type != COMMS_DRAM)) {
1396 dev_err(hdev->dev, "FW status: invalid RAM type %u\n",
1397 response->ram_type);
1405 * hl_fw_dynamic_wait_for_status - wait for status in dynamic FW load
1407 * @hdev: pointer to the habanalabs device structure
1408 * @fw_loader: managing structure for loading device's FW
1409 * @expected_status: expected status to wait for
1410 * @timeout: timeout for status wait
1412 * @return 0 on success, otherwise non-zero error code
1414 * waiting for status from FW include polling the FW status register until
1415 * expected status is received or timeout occurs (whatever occurs first).
1417 static int hl_fw_dynamic_wait_for_status(struct hl_device *hdev,
1418 struct fw_load_mgr *fw_loader,
1419 enum comms_sts expected_status,
1422 struct cpu_dyn_regs *dyn_regs;
1426 dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs;
1428 /* Wait for expected status */
1429 rc = hl_poll_timeout(
1431 le32_to_cpu(dyn_regs->cpu_cmd_status_to_host),
1433 FIELD_GET(COMMS_STATUS_STATUS_MASK, status) == expected_status,
1434 FW_CPU_STATUS_POLL_INTERVAL_USEC,
1438 hl_fw_dynamic_report_error_status(hdev, status,
1444 * skip storing FW response for NOOP to preserve the actual desired
1447 if (expected_status == COMMS_STS_NOOP)
1450 rc = hl_fw_dynamic_extract_fw_response(hdev, fw_loader,
1451 &fw_loader->dynamic_loader.response,
1457 * hl_fw_dynamic_send_clear_cmd - send clear command to FW
1459 * @hdev: pointer to the habanalabs device structure
1460 * @fw_loader: managing structure for loading device's FW
1462 * @return 0 on success, otherwise non-zero error code
1464 * after command cycle between LKD to FW CPU (i.e. LKD got an expected status
1465 * from FW) we need to clear the CPU status register in order to avoid garbage
1466 * between command cycles.
1467 * This is done by sending clear command and polling the CPU to LKD status
1468 * register to hold the status NOOP
1470 static int hl_fw_dynamic_send_clear_cmd(struct hl_device *hdev,
1471 struct fw_load_mgr *fw_loader)
1473 hl_fw_dynamic_send_cmd(hdev, fw_loader, COMMS_CLR_STS, 0);
1475 return hl_fw_dynamic_wait_for_status(hdev, fw_loader, COMMS_STS_NOOP,
1476 fw_loader->cpu_timeout);
1480 * hl_fw_dynamic_send_protocol_cmd - send LKD to FW cmd and wait for ACK
1482 * @hdev: pointer to the habanalabs device structure
1483 * @fw_loader: managing structure for loading device's FW
1484 * @cmd: LKD to FW cmd code
1485 * @size: size of next FW component to be loaded (0 if not necessary)
1486 * @wait_ok: if true also wait for OK response from FW
1487 * @timeout: timeout for status wait
1489 * @return 0 on success, otherwise non-zero error code
1492 * when sending protocol command we have the following steps:
1493 * - send clear (clear command and verify clear status register)
1494 * - send the actual protocol command
1495 * - wait for ACK on the protocol command
1498 * if, in addition, the specific protocol command should wait for OK then:
1504 * send clear: this is necessary in order to clear the status register to avoid
1505 * leftovers between command
1506 * NOOP command: necessary to avoid loop on the clear command by the FW
1508 int hl_fw_dynamic_send_protocol_cmd(struct hl_device *hdev,
1509 struct fw_load_mgr *fw_loader,
1510 enum comms_cmd cmd, unsigned int size,
1511 bool wait_ok, u32 timeout)
1515 /* first send clear command to clean former commands */
1516 rc = hl_fw_dynamic_send_clear_cmd(hdev, fw_loader);
1518 /* send the actual command */
1519 hl_fw_dynamic_send_cmd(hdev, fw_loader, cmd, size);
1521 /* wait for ACK for the command */
1522 rc = hl_fw_dynamic_wait_for_status(hdev, fw_loader, COMMS_STS_ACK,
1527 /* clear command to prepare for NOOP command */
1528 rc = hl_fw_dynamic_send_clear_cmd(hdev, fw_loader);
1532 /* send the actual NOOP command */
1533 hl_fw_dynamic_send_cmd(hdev, fw_loader, COMMS_NOOP, 0);
1538 rc = hl_fw_dynamic_wait_for_status(hdev, fw_loader, COMMS_STS_OK,
1543 /* clear command to prepare for NOOP command */
1544 rc = hl_fw_dynamic_send_clear_cmd(hdev, fw_loader);
1548 /* send the actual NOOP command */
1549 hl_fw_dynamic_send_cmd(hdev, fw_loader, COMMS_NOOP, 0);
1555 * hl_fw_compat_crc32 - CRC compatible with FW
1557 * @data: pointer to the data
1558 * @size: size of the data
1560 * @return the CRC32 result
1562 * NOTE: kernel's CRC32 differ's from standard CRC32 calculation.
1563 * in order to be aligned we need to flip the bits of both the input
1564 * initial CRC and kernel's CRC32 result.
1565 * in addition both sides use initial CRC of 0,
1567 static u32 hl_fw_compat_crc32(u8 *data, size_t size)
1569 return ~crc32_le(~((u32)0), data, size);
1573 * hl_fw_dynamic_validate_memory_bound - validate memory bounds for memory
1574 * transfer (image or descriptor) between
1577 * @hdev: pointer to the habanalabs device structure
1578 * @addr: device address of memory transfer
1579 * @size: memory transter size
1580 * @region: PCI memory region
1582 * @return 0 on success, otherwise non-zero error code
1584 static int hl_fw_dynamic_validate_memory_bound(struct hl_device *hdev,
1585 u64 addr, size_t size,
1586 struct pci_mem_region *region)
1590 /* now make sure that the memory transfer is within region's bounds */
1591 end_addr = addr + size;
1592 if (end_addr >= region->region_base + region->region_size) {
1594 "dynamic FW load: memory transfer end address out of memory region bounds. addr: %llx\n",
1600 * now make sure memory transfer is within predefined BAR bounds.
1601 * this is to make sure we do not need to set the bar (e.g. for DRAM
1604 if (end_addr >= region->region_base - region->offset_in_bar +
1607 "FW image beyond PCI BAR bounds\n");
1615 * hl_fw_dynamic_validate_descriptor - validate FW descriptor
1617 * @hdev: pointer to the habanalabs device structure
1618 * @fw_loader: managing structure for loading device's FW
1619 * @fw_desc: the descriptor form FW
1621 * @return 0 on success, otherwise non-zero error code
1623 static int hl_fw_dynamic_validate_descriptor(struct hl_device *hdev,
1624 struct fw_load_mgr *fw_loader,
1625 struct lkd_fw_comms_desc *fw_desc)
1627 struct pci_mem_region *region;
1628 enum pci_region region_id;
1635 if (le32_to_cpu(fw_desc->header.magic) != HL_COMMS_DESC_MAGIC) {
1636 dev_err(hdev->dev, "Invalid magic for dynamic FW descriptor (%x)\n",
1637 fw_desc->header.magic);
1641 if (fw_desc->header.version != HL_COMMS_DESC_VER) {
1642 dev_err(hdev->dev, "Invalid version for dynamic FW descriptor (%x)\n",
1643 fw_desc->header.version);
1648 * calc CRC32 of data without header.
1649 * note that no alignment/stride address issues here as all structures
1652 data_size = sizeof(struct lkd_fw_comms_desc) -
1653 sizeof(struct comms_desc_header);
1654 data_ptr = (u8 *)fw_desc + sizeof(struct comms_desc_header);
1656 if (le16_to_cpu(fw_desc->header.size) != data_size) {
1658 "Invalid descriptor size 0x%x, expected size 0x%zx\n",
1659 le16_to_cpu(fw_desc->header.size), data_size);
1663 data_crc32 = hl_fw_compat_crc32(data_ptr, data_size);
1665 if (data_crc32 != le32_to_cpu(fw_desc->header.crc32)) {
1667 "CRC32 mismatch for dynamic FW descriptor (%x:%x)\n",
1668 data_crc32, fw_desc->header.crc32);
1672 /* find memory region to which to copy the image */
1673 addr = le64_to_cpu(fw_desc->img_addr);
1674 region_id = hl_get_pci_memory_region(hdev, addr);
1675 if ((region_id != PCI_REGION_SRAM) &&
1676 ((region_id != PCI_REGION_DRAM))) {
1678 "Invalid region to copy FW image address=%llx\n", addr);
1682 region = &hdev->pci_mem_region[region_id];
1684 /* store the region for the copy stage */
1685 fw_loader->dynamic_loader.image_region = region;
1688 * here we know that the start address is valid, now make sure that the
1689 * image is within region's bounds
1691 rc = hl_fw_dynamic_validate_memory_bound(hdev, addr,
1692 fw_loader->dynamic_loader.fw_image_size,
1696 "invalid mem transfer request for FW image\n");
1703 static int hl_fw_dynamic_validate_response(struct hl_device *hdev,
1704 struct fw_response *response,
1705 struct pci_mem_region *region)
1710 device_addr = region->region_base + response->ram_offset;
1713 * validate that the descriptor is within region's bounds
1714 * Note that as the start address was supplied according to the RAM
1715 * type- testing only the end address is enough
1717 rc = hl_fw_dynamic_validate_memory_bound(hdev, device_addr,
1718 sizeof(struct lkd_fw_comms_desc),
1724 * hl_fw_dynamic_read_and_validate_descriptor - read and validate FW descriptor
1726 * @hdev: pointer to the habanalabs device structure
1727 * @fw_loader: managing structure for loading device's FW
1729 * @return 0 on success, otherwise non-zero error code
1731 static int hl_fw_dynamic_read_and_validate_descriptor(struct hl_device *hdev,
1732 struct fw_load_mgr *fw_loader)
1734 struct lkd_fw_comms_desc *fw_desc;
1735 struct pci_mem_region *region;
1736 struct fw_response *response;
1737 enum pci_region region_id;
1741 fw_desc = &fw_loader->dynamic_loader.comm_desc;
1742 response = &fw_loader->dynamic_loader.response;
1744 region_id = (response->ram_type == COMMS_SRAM) ?
1745 PCI_REGION_SRAM : PCI_REGION_DRAM;
1747 region = &hdev->pci_mem_region[region_id];
1749 rc = hl_fw_dynamic_validate_response(hdev, response, region);
1752 "invalid mem transfer request for FW descriptor\n");
1756 /* extract address copy the descriptor from */
1757 src = hdev->pcie_bar[region->bar_id] + region->offset_in_bar +
1758 response->ram_offset;
1759 memcpy_fromio(fw_desc, src, sizeof(struct lkd_fw_comms_desc));
1761 return hl_fw_dynamic_validate_descriptor(hdev, fw_loader, fw_desc);
1765 * hl_fw_dynamic_request_descriptor - handshake with CPU to get FW descriptor
1767 * @hdev: pointer to the habanalabs device structure
1768 * @fw_loader: managing structure for loading device's FW
1769 * @next_image_size: size to allocate for next FW component
1771 * @return 0 on success, otherwise non-zero error code
1773 static int hl_fw_dynamic_request_descriptor(struct hl_device *hdev,
1774 struct fw_load_mgr *fw_loader,
1775 size_t next_image_size)
1779 rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_PREP_DESC,
1780 next_image_size, true,
1781 fw_loader->cpu_timeout);
1785 return hl_fw_dynamic_read_and_validate_descriptor(hdev, fw_loader);
1789 * hl_fw_dynamic_read_device_fw_version - read FW version to exposed properties
1791 * @hdev: pointer to the habanalabs device structure
1792 * @fwc: the firmware component
1793 * @fw_version: fw component's version string
1795 static void hl_fw_dynamic_read_device_fw_version(struct hl_device *hdev,
1796 enum hl_fw_component fwc,
1797 const char *fw_version)
1799 struct asic_fixed_properties *prop = &hdev->asic_prop;
1800 char *preboot_ver, *boot_ver;
1804 case FW_COMP_BOOT_FIT:
1805 strscpy(prop->uboot_ver, fw_version, VERSION_MAX_LEN);
1806 boot_ver = extract_fw_ver_from_str(prop->uboot_ver);
1808 dev_info(hdev->dev, "boot-fit version %s\n", boot_ver);
1813 case FW_COMP_PREBOOT:
1814 strscpy(prop->preboot_ver, fw_version, VERSION_MAX_LEN);
1815 preboot_ver = strnstr(prop->preboot_ver, "Preboot",
1817 if (preboot_ver && preboot_ver != prop->preboot_ver) {
1818 strscpy(btl_ver, prop->preboot_ver,
1819 min((int) (preboot_ver - prop->preboot_ver),
1821 dev_info(hdev->dev, "%s\n", btl_ver);
1824 preboot_ver = extract_fw_ver_from_str(prop->preboot_ver);
1826 dev_info(hdev->dev, "preboot version %s\n",
1833 dev_warn(hdev->dev, "Undefined FW component: %d\n", fwc);
1839 * hl_fw_dynamic_copy_image - copy image to memory allocated by the FW
1841 * @hdev: pointer to the habanalabs device structure
1842 * @fw: fw descriptor
1843 * @fw_loader: managing structure for loading device's FW
1845 static int hl_fw_dynamic_copy_image(struct hl_device *hdev,
1846 const struct firmware *fw,
1847 struct fw_load_mgr *fw_loader)
1849 struct lkd_fw_comms_desc *fw_desc;
1850 struct pci_mem_region *region;
1855 fw_desc = &fw_loader->dynamic_loader.comm_desc;
1856 addr = le64_to_cpu(fw_desc->img_addr);
1858 /* find memory region to which to copy the image */
1859 region = fw_loader->dynamic_loader.image_region;
1861 dest = hdev->pcie_bar[region->bar_id] + region->offset_in_bar +
1862 (addr - region->region_base);
1864 rc = hl_fw_copy_fw_to_device(hdev, fw, dest,
1865 fw_loader->boot_fit_img.src_off,
1866 fw_loader->boot_fit_img.copy_size);
1872 * hl_fw_dynamic_copy_msg - copy msg to memory allocated by the FW
1874 * @hdev: pointer to the habanalabs device structure
1876 * @fw_loader: managing structure for loading device's FW
1878 static int hl_fw_dynamic_copy_msg(struct hl_device *hdev,
1879 struct lkd_msg_comms *msg, struct fw_load_mgr *fw_loader)
1881 struct lkd_fw_comms_desc *fw_desc;
1882 struct pci_mem_region *region;
1887 fw_desc = &fw_loader->dynamic_loader.comm_desc;
1888 addr = le64_to_cpu(fw_desc->img_addr);
1890 /* find memory region to which to copy the image */
1891 region = fw_loader->dynamic_loader.image_region;
1893 dest = hdev->pcie_bar[region->bar_id] + region->offset_in_bar +
1894 (addr - region->region_base);
1896 rc = hl_fw_copy_msg_to_device(hdev, msg, dest, 0, 0);
1902 * hl_fw_boot_fit_update_state - update internal data structures after boot-fit
1905 * @hdev: pointer to the habanalabs device structure
1906 * @cpu_boot_dev_sts0_reg: register holding CPU boot dev status 0
1907 * @cpu_boot_dev_sts1_reg: register holding CPU boot dev status 1
1909 * @return 0 on success, otherwise non-zero error code
1911 static void hl_fw_boot_fit_update_state(struct hl_device *hdev,
1912 u32 cpu_boot_dev_sts0_reg,
1913 u32 cpu_boot_dev_sts1_reg)
1915 struct asic_fixed_properties *prop = &hdev->asic_prop;
1917 /* Clear reset status since we need to read it again from boot CPU */
1918 prop->hard_reset_done_by_fw = false;
1920 /* Read boot_cpu status bits */
1921 if (prop->fw_preboot_cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_ENABLED) {
1922 prop->fw_bootfit_cpu_boot_dev_sts0 =
1923 RREG32(cpu_boot_dev_sts0_reg);
1925 if (prop->fw_bootfit_cpu_boot_dev_sts0 &
1926 CPU_BOOT_DEV_STS0_FW_HARD_RST_EN)
1927 prop->hard_reset_done_by_fw = true;
1929 dev_dbg(hdev->dev, "Firmware boot CPU status0 %#x\n",
1930 prop->fw_bootfit_cpu_boot_dev_sts0);
1933 if (prop->fw_cpu_boot_dev_sts1_valid) {
1934 prop->fw_bootfit_cpu_boot_dev_sts1 =
1935 RREG32(cpu_boot_dev_sts1_reg);
1937 dev_dbg(hdev->dev, "Firmware boot CPU status1 %#x\n",
1938 prop->fw_bootfit_cpu_boot_dev_sts1);
1941 dev_dbg(hdev->dev, "Firmware boot CPU hard-reset is %s\n",
1942 prop->hard_reset_done_by_fw ? "enabled" : "disabled");
1945 static void hl_fw_dynamic_update_linux_interrupt_if(struct hl_device *hdev)
1947 struct cpu_dyn_regs *dyn_regs =
1948 &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
1950 /* Check whether all 3 interrupt interfaces are set, if not use a
1953 if (!hdev->asic_prop.gic_interrupts_enable &&
1954 !(hdev->asic_prop.fw_app_cpu_boot_dev_sts0 &
1955 CPU_BOOT_DEV_STS0_MULTI_IRQ_POLL_EN)) {
1956 dyn_regs->gic_host_halt_irq = dyn_regs->gic_host_irq_ctrl;
1957 dyn_regs->gic_host_ints_irq = dyn_regs->gic_host_irq_ctrl;
1960 "Using a single interrupt interface towards cpucp");
1964 * hl_fw_dynamic_load_image - load FW image using dynamic protocol
1966 * @hdev: pointer to the habanalabs device structure
1967 * @fw_loader: managing structure for loading device's FW
1968 * @load_fwc: the FW component to be loaded
1969 * @img_ld_timeout: image load timeout
1971 * @return 0 on success, otherwise non-zero error code
1973 static int hl_fw_dynamic_load_image(struct hl_device *hdev,
1974 struct fw_load_mgr *fw_loader,
1975 enum hl_fw_component load_fwc,
1978 enum hl_fw_component cur_fwc;
1979 const struct firmware *fw;
1984 * when loading image we have one of 2 scenarios:
1985 * 1. current FW component is preboot and we want to load boot-fit
1986 * 2. current FW component is boot-fit and we want to load linux
1988 if (load_fwc == FW_COMP_BOOT_FIT) {
1989 cur_fwc = FW_COMP_PREBOOT;
1990 fw_name = fw_loader->boot_fit_img.image_name;
1992 cur_fwc = FW_COMP_BOOT_FIT;
1993 fw_name = fw_loader->linux_img.image_name;
1996 /* request FW in order to communicate to FW the size to be allocated */
1997 rc = hl_request_fw(hdev, &fw, fw_name);
2001 /* store the image size for future validation */
2002 fw_loader->dynamic_loader.fw_image_size = fw->size;
2004 rc = hl_fw_dynamic_request_descriptor(hdev, fw_loader, fw->size);
2008 /* read preboot version */
2009 hl_fw_dynamic_read_device_fw_version(hdev, cur_fwc,
2010 fw_loader->dynamic_loader.comm_desc.cur_fw_ver);
2013 /* update state according to boot stage */
2014 if (cur_fwc == FW_COMP_BOOT_FIT) {
2015 struct cpu_dyn_regs *dyn_regs;
2017 dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs;
2018 hl_fw_boot_fit_update_state(hdev,
2019 le32_to_cpu(dyn_regs->cpu_boot_dev_sts0),
2020 le32_to_cpu(dyn_regs->cpu_boot_dev_sts1));
2023 /* copy boot fit to space allocated by FW */
2024 rc = hl_fw_dynamic_copy_image(hdev, fw, fw_loader);
2028 rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_DATA_RDY,
2030 fw_loader->cpu_timeout);
2034 rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_EXEC,
2039 hl_release_firmware(fw);
2043 static int hl_fw_dynamic_wait_for_boot_fit_active(struct hl_device *hdev,
2044 struct fw_load_mgr *fw_loader)
2046 struct dynamic_fw_load_mgr *dyn_loader;
2050 dyn_loader = &fw_loader->dynamic_loader;
2052 /* Make sure CPU boot-loader is running */
2053 rc = hl_poll_timeout(
2055 le32_to_cpu(dyn_loader->comm_desc.cpu_dyn_regs.cpu_boot_status),
2057 (status == CPU_BOOT_STATUS_NIC_FW_RDY) ||
2058 (status == CPU_BOOT_STATUS_READY_TO_BOOT),
2059 FW_CPU_STATUS_POLL_INTERVAL_USEC,
2060 dyn_loader->wait_for_bl_timeout);
2062 dev_err(hdev->dev, "failed to wait for boot\n");
2066 dev_dbg(hdev->dev, "uboot status = %d\n", status);
2070 static int hl_fw_dynamic_wait_for_linux_active(struct hl_device *hdev,
2071 struct fw_load_mgr *fw_loader)
2073 struct dynamic_fw_load_mgr *dyn_loader;
2077 dyn_loader = &fw_loader->dynamic_loader;
2079 /* Make sure CPU boot-loader is running */
2081 rc = hl_poll_timeout(
2083 le32_to_cpu(dyn_loader->comm_desc.cpu_dyn_regs.cpu_boot_status),
2085 (status == CPU_BOOT_STATUS_SRAM_AVAIL),
2086 FW_CPU_STATUS_POLL_INTERVAL_USEC,
2087 fw_loader->cpu_timeout);
2089 dev_err(hdev->dev, "failed to wait for Linux\n");
2093 dev_dbg(hdev->dev, "Boot status = %d\n", status);
2098 * hl_fw_linux_update_state - update internal data structures after Linux
2100 * Note: Linux initialization is comprised mainly
2101 * of two stages - loading kernel (SRAM_AVAIL)
2103 * Therefore reading boot device status in any of
2104 * these stages might result in different values.
2106 * @hdev: pointer to the habanalabs device structure
2107 * @cpu_boot_dev_sts0_reg: register holding CPU boot dev status 0
2108 * @cpu_boot_dev_sts1_reg: register holding CPU boot dev status 1
2110 * @return 0 on success, otherwise non-zero error code
2112 static void hl_fw_linux_update_state(struct hl_device *hdev,
2113 u32 cpu_boot_dev_sts0_reg,
2114 u32 cpu_boot_dev_sts1_reg)
2116 struct asic_fixed_properties *prop = &hdev->asic_prop;
2118 hdev->fw_loader.linux_loaded = true;
2120 /* Clear reset status since we need to read again from app */
2121 prop->hard_reset_done_by_fw = false;
2123 /* Read FW application security bits */
2124 if (prop->fw_cpu_boot_dev_sts0_valid) {
2125 prop->fw_app_cpu_boot_dev_sts0 =
2126 RREG32(cpu_boot_dev_sts0_reg);
2128 if (prop->fw_app_cpu_boot_dev_sts0 &
2129 CPU_BOOT_DEV_STS0_FW_HARD_RST_EN)
2130 prop->hard_reset_done_by_fw = true;
2132 if (prop->fw_app_cpu_boot_dev_sts0 &
2133 CPU_BOOT_DEV_STS0_GIC_PRIVILEGED_EN)
2134 prop->gic_interrupts_enable = false;
2137 "Firmware application CPU status0 %#x\n",
2138 prop->fw_app_cpu_boot_dev_sts0);
2140 dev_dbg(hdev->dev, "GIC controller is %s\n",
2141 prop->gic_interrupts_enable ?
2142 "enabled" : "disabled");
2145 if (prop->fw_cpu_boot_dev_sts1_valid) {
2146 prop->fw_app_cpu_boot_dev_sts1 =
2147 RREG32(cpu_boot_dev_sts1_reg);
2150 "Firmware application CPU status1 %#x\n",
2151 prop->fw_app_cpu_boot_dev_sts1);
2154 dev_dbg(hdev->dev, "Firmware application CPU hard-reset is %s\n",
2155 prop->hard_reset_done_by_fw ? "enabled" : "disabled");
2157 dev_info(hdev->dev, "Successfully loaded firmware to device\n");
2161 * hl_fw_dynamic_report_reset_cause - send a COMMS message with the cause
2162 * of the newly triggered hard reset
2164 * @hdev: pointer to the habanalabs device structure
2165 * @fw_loader: managing structure for loading device's FW
2166 * @reset_cause: enumerated cause for the recent hard reset
2168 * @return 0 on success, otherwise non-zero error code
2170 static int hl_fw_dynamic_report_reset_cause(struct hl_device *hdev,
2171 struct fw_load_mgr *fw_loader,
2172 enum comms_reset_cause reset_cause)
2174 struct lkd_msg_comms msg;
2177 memset(&msg, 0, sizeof(msg));
2179 /* create message to be sent */
2180 msg.header.type = HL_COMMS_RESET_CAUSE_TYPE;
2181 msg.header.size = cpu_to_le16(sizeof(struct comms_msg_header));
2182 msg.header.magic = cpu_to_le32(HL_COMMS_MSG_MAGIC);
2184 msg.reset_cause = reset_cause;
2186 rc = hl_fw_dynamic_request_descriptor(hdev, fw_loader,
2187 sizeof(struct lkd_msg_comms));
2191 /* copy message to space allocated by FW */
2192 rc = hl_fw_dynamic_copy_msg(hdev, &msg, fw_loader);
2196 rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_DATA_RDY,
2198 fw_loader->cpu_timeout);
2202 rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_EXEC,
2204 fw_loader->cpu_timeout);
2212 * hl_fw_dynamic_init_cpu - initialize the device CPU using dynamic protocol
2214 * @hdev: pointer to the habanalabs device structure
2215 * @fw_loader: managing structure for loading device's FW
2217 * @return 0 on success, otherwise non-zero error code
2219 * brief: the dynamic protocol is master (LKD) slave (FW CPU) protocol.
2220 * the communication is done using registers:
2221 * - LKD command register
2222 * - FW status register
2223 * the protocol is race free. this goal is achieved by splitting the requests
2224 * and response to known synchronization points between the LKD and the FW.
2225 * each response to LKD request is known and bound to a predefined timeout.
2226 * in case of timeout expiration without the desired status from FW- the
2227 * protocol (and hence the boot) will fail.
2229 static int hl_fw_dynamic_init_cpu(struct hl_device *hdev,
2230 struct fw_load_mgr *fw_loader)
2232 struct cpu_dyn_regs *dyn_regs;
2236 "Loading firmware to device, may take some time...\n");
2238 dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs;
2240 rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_RST_STATE,
2242 fw_loader->cpu_timeout);
2246 if (hdev->curr_reset_cause) {
2247 rc = hl_fw_dynamic_report_reset_cause(hdev, fw_loader,
2248 hdev->curr_reset_cause);
2252 /* Clear current reset cause */
2253 hdev->curr_reset_cause = HL_RESET_CAUSE_UNKNOWN;
2256 if (!(hdev->fw_components & FW_TYPE_BOOT_CPU)) {
2257 rc = hl_fw_dynamic_request_descriptor(hdev, fw_loader, 0);
2261 /* read preboot version */
2262 hl_fw_dynamic_read_device_fw_version(hdev, FW_COMP_PREBOOT,
2263 fw_loader->dynamic_loader.comm_desc.cur_fw_ver);
2267 /* load boot fit to FW */
2268 rc = hl_fw_dynamic_load_image(hdev, fw_loader, FW_COMP_BOOT_FIT,
2269 fw_loader->boot_fit_timeout);
2271 dev_err(hdev->dev, "failed to load boot fit\n");
2275 rc = hl_fw_dynamic_wait_for_boot_fit_active(hdev, fw_loader);
2279 /* Enable DRAM scrambling before Linux boot and after successful
2282 hdev->asic_funcs->init_cpu_scrambler_dram(hdev);
2284 if (!(hdev->fw_components & FW_TYPE_LINUX)) {
2285 dev_info(hdev->dev, "Skip loading Linux F/W\n");
2289 if (fw_loader->skip_bmc) {
2290 rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader,
2293 fw_loader->cpu_timeout);
2295 dev_err(hdev->dev, "failed to load boot fit\n");
2300 /* load Linux image to FW */
2301 rc = hl_fw_dynamic_load_image(hdev, fw_loader, FW_COMP_LINUX,
2302 fw_loader->cpu_timeout);
2304 dev_err(hdev->dev, "failed to load Linux\n");
2308 rc = hl_fw_dynamic_wait_for_linux_active(hdev, fw_loader);
2312 hl_fw_linux_update_state(hdev, le32_to_cpu(dyn_regs->cpu_boot_dev_sts0),
2313 le32_to_cpu(dyn_regs->cpu_boot_dev_sts1));
2315 hl_fw_dynamic_update_linux_interrupt_if(hdev);
2320 fw_read_errors(hdev, le32_to_cpu(dyn_regs->cpu_boot_err0),
2321 le32_to_cpu(dyn_regs->cpu_boot_err1),
2322 le32_to_cpu(dyn_regs->cpu_boot_dev_sts0),
2323 le32_to_cpu(dyn_regs->cpu_boot_dev_sts1));
2328 * hl_fw_static_init_cpu - initialize the device CPU using static protocol
2330 * @hdev: pointer to the habanalabs device structure
2331 * @fw_loader: managing structure for loading device's FW
2333 * @return 0 on success, otherwise non-zero error code
2335 static int hl_fw_static_init_cpu(struct hl_device *hdev,
2336 struct fw_load_mgr *fw_loader)
2338 u32 cpu_msg_status_reg, cpu_timeout, msg_to_cpu_reg, status;
2339 u32 cpu_boot_dev_status0_reg, cpu_boot_dev_status1_reg;
2340 struct static_fw_load_mgr *static_loader;
2341 u32 cpu_boot_status_reg;
2344 if (!(hdev->fw_components & FW_TYPE_BOOT_CPU))
2347 /* init common loader parameters */
2348 cpu_timeout = fw_loader->cpu_timeout;
2350 /* init static loader parameters */
2351 static_loader = &fw_loader->static_loader;
2352 cpu_msg_status_reg = static_loader->cpu_cmd_status_to_host_reg;
2353 msg_to_cpu_reg = static_loader->kmd_msg_to_cpu_reg;
2354 cpu_boot_dev_status0_reg = static_loader->cpu_boot_dev_status0_reg;
2355 cpu_boot_dev_status1_reg = static_loader->cpu_boot_dev_status1_reg;
2356 cpu_boot_status_reg = static_loader->cpu_boot_status_reg;
2358 dev_info(hdev->dev, "Going to wait for device boot (up to %lds)\n",
2359 cpu_timeout / USEC_PER_SEC);
2361 /* Wait for boot FIT request */
2362 rc = hl_poll_timeout(
2364 cpu_boot_status_reg,
2366 status == CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT,
2367 FW_CPU_STATUS_POLL_INTERVAL_USEC,
2368 fw_loader->boot_fit_timeout);
2372 "No boot fit request received, resuming boot\n");
2374 rc = hdev->asic_funcs->load_boot_fit_to_device(hdev);
2378 /* Clear device CPU message status */
2379 WREG32(cpu_msg_status_reg, CPU_MSG_CLR);
2381 /* Signal device CPU that boot loader is ready */
2382 WREG32(msg_to_cpu_reg, KMD_MSG_FIT_RDY);
2384 /* Poll for CPU device ack */
2385 rc = hl_poll_timeout(
2389 status == CPU_MSG_OK,
2390 FW_CPU_STATUS_POLL_INTERVAL_USEC,
2391 fw_loader->boot_fit_timeout);
2395 "Timeout waiting for boot fit load ack\n");
2400 WREG32(msg_to_cpu_reg, KMD_MSG_NA);
2403 /* Make sure CPU boot-loader is running */
2404 rc = hl_poll_timeout(
2406 cpu_boot_status_reg,
2408 (status == CPU_BOOT_STATUS_DRAM_RDY) ||
2409 (status == CPU_BOOT_STATUS_NIC_FW_RDY) ||
2410 (status == CPU_BOOT_STATUS_READY_TO_BOOT) ||
2411 (status == CPU_BOOT_STATUS_SRAM_AVAIL),
2412 FW_CPU_STATUS_POLL_INTERVAL_USEC,
2415 dev_dbg(hdev->dev, "uboot status = %d\n", status);
2417 /* Read U-Boot version now in case we will later fail */
2418 hl_fw_static_read_device_fw_version(hdev, FW_COMP_BOOT_FIT);
2420 /* update state according to boot stage */
2421 hl_fw_boot_fit_update_state(hdev, cpu_boot_dev_status0_reg,
2422 cpu_boot_dev_status1_reg);
2425 detect_cpu_boot_status(hdev, status);
2430 /* Enable DRAM scrambling before Linux boot and after successful
2433 hdev->asic_funcs->init_cpu_scrambler_dram(hdev);
2435 if (!(hdev->fw_components & FW_TYPE_LINUX)) {
2436 dev_info(hdev->dev, "Skip loading Linux F/W\n");
2441 if (status == CPU_BOOT_STATUS_SRAM_AVAIL) {
2447 "Loading firmware to device, may take some time...\n");
2449 rc = hdev->asic_funcs->load_firmware_to_device(hdev);
2453 if (fw_loader->skip_bmc) {
2454 WREG32(msg_to_cpu_reg, KMD_MSG_SKIP_BMC);
2456 rc = hl_poll_timeout(
2458 cpu_boot_status_reg,
2460 (status == CPU_BOOT_STATUS_BMC_WAITING_SKIPPED),
2461 FW_CPU_STATUS_POLL_INTERVAL_USEC,
2466 "Failed to get ACK on skipping BMC, %d\n",
2468 WREG32(msg_to_cpu_reg, KMD_MSG_NA);
2474 WREG32(msg_to_cpu_reg, KMD_MSG_FIT_RDY);
2476 rc = hl_poll_timeout(
2478 cpu_boot_status_reg,
2480 (status == CPU_BOOT_STATUS_SRAM_AVAIL),
2481 FW_CPU_STATUS_POLL_INTERVAL_USEC,
2485 WREG32(msg_to_cpu_reg, KMD_MSG_NA);
2488 if (status == CPU_BOOT_STATUS_FIT_CORRUPTED)
2490 "Device reports FIT image is corrupted\n");
2493 "Failed to load firmware to device, %d\n",
2500 rc = fw_read_errors(hdev, fw_loader->static_loader.boot_err0_reg,
2501 fw_loader->static_loader.boot_err1_reg,
2502 cpu_boot_dev_status0_reg,
2503 cpu_boot_dev_status1_reg);
2507 hl_fw_linux_update_state(hdev, cpu_boot_dev_status0_reg,
2508 cpu_boot_dev_status1_reg);
2513 fw_read_errors(hdev, fw_loader->static_loader.boot_err0_reg,
2514 fw_loader->static_loader.boot_err1_reg,
2515 cpu_boot_dev_status0_reg,
2516 cpu_boot_dev_status1_reg);
2522 * hl_fw_init_cpu - initialize the device CPU
2524 * @hdev: pointer to the habanalabs device structure
2526 * @return 0 on success, otherwise non-zero error code
2528 * perform necessary initializations for device's CPU. takes into account if
2529 * init protocol is static or dynamic.
2531 int hl_fw_init_cpu(struct hl_device *hdev)
2533 struct asic_fixed_properties *prop = &hdev->asic_prop;
2534 struct fw_load_mgr *fw_loader = &hdev->fw_loader;
2536 return prop->dynamic_fw_load ?
2537 hl_fw_dynamic_init_cpu(hdev, fw_loader) :
2538 hl_fw_static_init_cpu(hdev, fw_loader);