1 // SPDX-License-Identifier: GPL-2.0
4 * Copyright 2016-2019 HabanaLabs, Ltd.
8 #include "habanalabs.h"
9 #include "../include/common/hl_boot_if.h"
11 #include <linux/firmware.h>
12 #include <linux/genalloc.h>
13 #include <linux/io-64-nonatomic-lo-hi.h>
14 #include <linux/slab.h>
16 #define FW_FILE_MAX_SIZE 0x1400000 /* maximum size of 20MB */
18 * hl_fw_load_fw_to_device() - Load F/W code to device's memory.
20 * @hdev: pointer to hl_device structure.
21 * @fw_name: the firmware image name
22 * @dst: IO memory mapped address space to copy firmware to
24 * Copy fw code from firmware file to device memory.
26 * Return: 0 on success, non-zero for failure.
28 int hl_fw_load_fw_to_device(struct hl_device *hdev, const char *fw_name,
31 const struct firmware *fw;
36 rc = request_firmware(&fw, fw_name, hdev->dev);
38 dev_err(hdev->dev, "Firmware file %s is not found!\n", fw_name);
43 if ((fw_size % 4) != 0) {
44 dev_err(hdev->dev, "Illegal %s firmware size %zu\n",
50 dev_dbg(hdev->dev, "%s firmware size == %zu\n", fw_name, fw_size);
52 if (fw_size > FW_FILE_MAX_SIZE) {
54 "FW file size %zu exceeds maximum of %u bytes\n",
55 fw_size, FW_FILE_MAX_SIZE);
60 fw_data = (const u64 *) fw->data;
62 memcpy_toio(dst, fw_data, fw_size);
69 int hl_fw_send_pci_access_msg(struct hl_device *hdev, u32 opcode)
71 struct cpucp_packet pkt = {};
73 pkt.ctl = cpu_to_le32(opcode << CPUCP_PKT_CTL_OPCODE_SHIFT);
75 return hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt,
76 sizeof(pkt), 0, NULL);
79 int hl_fw_send_cpu_message(struct hl_device *hdev, u32 hw_queue_id, u32 *msg,
80 u16 len, u32 timeout, long *result)
82 struct cpucp_packet *pkt;
83 dma_addr_t pkt_dma_addr;
87 pkt = hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev, len,
91 "Failed to allocate DMA memory for packet to CPU\n");
95 memcpy(pkt, msg, len);
97 mutex_lock(&hdev->send_cpu_message_lock);
102 if (hdev->device_cpu_disabled) {
107 rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id, len, pkt_dma_addr);
109 dev_err(hdev->dev, "Failed to send CB on CPU PQ (%d)\n", rc);
113 rc = hl_poll_timeout_memory(hdev, &pkt->fence, tmp,
114 (tmp == CPUCP_PACKET_FENCE_VAL), 1000,
117 hl_hw_queue_inc_ci_kernel(hdev, hw_queue_id);
119 if (rc == -ETIMEDOUT) {
120 dev_err(hdev->dev, "Device CPU packet timeout (0x%x)\n", tmp);
121 hdev->device_cpu_disabled = true;
125 tmp = le32_to_cpu(pkt->ctl);
127 rc = (tmp & CPUCP_PKT_CTL_RC_MASK) >> CPUCP_PKT_CTL_RC_SHIFT;
129 dev_err(hdev->dev, "F/W ERROR %d for CPU packet %d\n",
131 (tmp & CPUCP_PKT_CTL_OPCODE_MASK)
132 >> CPUCP_PKT_CTL_OPCODE_SHIFT);
135 *result = (long) le64_to_cpu(pkt->result);
139 mutex_unlock(&hdev->send_cpu_message_lock);
141 hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev, len, pkt);
146 int hl_fw_unmask_irq(struct hl_device *hdev, u16 event_type)
148 struct cpucp_packet pkt;
152 memset(&pkt, 0, sizeof(pkt));
154 pkt.ctl = cpu_to_le32(CPUCP_PACKET_UNMASK_RAZWI_IRQ <<
155 CPUCP_PKT_CTL_OPCODE_SHIFT);
156 pkt.value = cpu_to_le64(event_type);
158 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
162 dev_err(hdev->dev, "failed to unmask RAZWI IRQ %d", event_type);
167 int hl_fw_unmask_irq_arr(struct hl_device *hdev, const u32 *irq_arr,
170 struct cpucp_unmask_irq_arr_packet *pkt;
171 size_t total_pkt_size;
175 total_pkt_size = sizeof(struct cpucp_unmask_irq_arr_packet) +
178 /* data should be aligned to 8 bytes in order to CPU-CP to copy it */
179 total_pkt_size = (total_pkt_size + 0x7) & ~0x7;
181 /* total_pkt_size is casted to u16 later on */
182 if (total_pkt_size > USHRT_MAX) {
183 dev_err(hdev->dev, "too many elements in IRQ array\n");
187 pkt = kzalloc(total_pkt_size, GFP_KERNEL);
191 pkt->length = cpu_to_le32(irq_arr_size / sizeof(irq_arr[0]));
192 memcpy(&pkt->irqs, irq_arr, irq_arr_size);
194 pkt->cpucp_pkt.ctl = cpu_to_le32(CPUCP_PACKET_UNMASK_RAZWI_IRQ_ARRAY <<
195 CPUCP_PKT_CTL_OPCODE_SHIFT);
197 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) pkt,
198 total_pkt_size, 0, &result);
201 dev_err(hdev->dev, "failed to unmask IRQ array\n");
208 int hl_fw_test_cpu_queue(struct hl_device *hdev)
210 struct cpucp_packet test_pkt = {};
214 test_pkt.ctl = cpu_to_le32(CPUCP_PACKET_TEST <<
215 CPUCP_PKT_CTL_OPCODE_SHIFT);
216 test_pkt.value = cpu_to_le64(CPUCP_PACKET_FENCE_VAL);
218 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &test_pkt,
219 sizeof(test_pkt), 0, &result);
222 if (result != CPUCP_PACKET_FENCE_VAL)
224 "CPU queue test failed (0x%08lX)\n", result);
226 dev_err(hdev->dev, "CPU queue test failed, error %d\n", rc);
232 void *hl_fw_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size,
233 dma_addr_t *dma_handle)
237 kernel_addr = gen_pool_alloc(hdev->cpu_accessible_dma_pool, size);
239 *dma_handle = hdev->cpu_accessible_dma_address +
240 (kernel_addr - (u64) (uintptr_t) hdev->cpu_accessible_dma_mem);
242 return (void *) (uintptr_t) kernel_addr;
245 void hl_fw_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size,
248 gen_pool_free(hdev->cpu_accessible_dma_pool, (u64) (uintptr_t) vaddr,
252 int hl_fw_send_heartbeat(struct hl_device *hdev)
254 struct cpucp_packet hb_pkt = {};
258 hb_pkt.ctl = cpu_to_le32(CPUCP_PACKET_TEST <<
259 CPUCP_PKT_CTL_OPCODE_SHIFT);
260 hb_pkt.value = cpu_to_le64(CPUCP_PACKET_FENCE_VAL);
262 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &hb_pkt,
263 sizeof(hb_pkt), 0, &result);
265 if ((rc) || (result != CPUCP_PACKET_FENCE_VAL))
271 int hl_fw_cpucp_info_get(struct hl_device *hdev)
273 struct asic_fixed_properties *prop = &hdev->asic_prop;
274 struct cpucp_packet pkt = {};
275 void *cpucp_info_cpu_addr;
276 dma_addr_t cpucp_info_dma_addr;
280 cpucp_info_cpu_addr =
281 hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev,
282 sizeof(struct cpucp_info),
283 &cpucp_info_dma_addr);
284 if (!cpucp_info_cpu_addr) {
286 "Failed to allocate DMA memory for CPU-CP info packet\n");
290 memset(cpucp_info_cpu_addr, 0, sizeof(struct cpucp_info));
292 pkt.ctl = cpu_to_le32(CPUCP_PACKET_INFO_GET <<
293 CPUCP_PKT_CTL_OPCODE_SHIFT);
294 pkt.addr = cpu_to_le64(cpucp_info_dma_addr);
295 pkt.data_max_size = cpu_to_le32(sizeof(struct cpucp_info));
297 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
298 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
301 "Failed to handle CPU-CP info pkt, error %d\n", rc);
305 memcpy(&prop->cpucp_info, cpucp_info_cpu_addr,
306 sizeof(prop->cpucp_info));
308 rc = hl_build_hwmon_channel_info(hdev, prop->cpucp_info.sensors);
311 "Failed to build hwmon channel info, error %d\n", rc);
317 hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev,
318 sizeof(struct cpucp_info), cpucp_info_cpu_addr);
323 int hl_fw_get_eeprom_data(struct hl_device *hdev, void *data, size_t max_size)
325 struct cpucp_packet pkt = {};
326 void *eeprom_info_cpu_addr;
327 dma_addr_t eeprom_info_dma_addr;
331 eeprom_info_cpu_addr =
332 hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev,
333 max_size, &eeprom_info_dma_addr);
334 if (!eeprom_info_cpu_addr) {
336 "Failed to allocate DMA memory for CPU-CP EEPROM packet\n");
340 memset(eeprom_info_cpu_addr, 0, max_size);
342 pkt.ctl = cpu_to_le32(CPUCP_PACKET_EEPROM_DATA_GET <<
343 CPUCP_PKT_CTL_OPCODE_SHIFT);
344 pkt.addr = cpu_to_le64(eeprom_info_dma_addr);
345 pkt.data_max_size = cpu_to_le32(max_size);
347 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
348 HL_CPUCP_EEPROM_TIMEOUT_USEC, &result);
352 "Failed to handle CPU-CP EEPROM packet, error %d\n",
357 /* result contains the actual size */
358 memcpy(data, eeprom_info_cpu_addr, min((size_t)result, max_size));
361 hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev, max_size,
362 eeprom_info_cpu_addr);
367 int hl_fw_cpucp_pci_counters_get(struct hl_device *hdev,
368 struct hl_info_pci_counters *counters)
370 struct cpucp_packet pkt = {};
374 pkt.ctl = cpu_to_le32(CPUCP_PACKET_PCIE_THROUGHPUT_GET <<
375 CPUCP_PKT_CTL_OPCODE_SHIFT);
377 /* Fetch PCI rx counter */
378 pkt.index = cpu_to_le32(cpucp_pcie_throughput_rx);
379 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
380 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
383 "Failed to handle CPU-CP PCI info pkt, error %d\n", rc);
386 counters->rx_throughput = result;
388 /* Fetch PCI tx counter */
389 pkt.index = cpu_to_le32(cpucp_pcie_throughput_tx);
390 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
391 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
394 "Failed to handle CPU-CP PCI info pkt, error %d\n", rc);
397 counters->tx_throughput = result;
399 /* Fetch PCI replay counter */
400 pkt.ctl = cpu_to_le32(CPUCP_PACKET_PCIE_REPLAY_CNT_GET <<
401 CPUCP_PKT_CTL_OPCODE_SHIFT);
403 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
404 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
407 "Failed to handle CPU-CP PCI info pkt, error %d\n", rc);
410 counters->replay_cnt = (u32) result;
415 int hl_fw_cpucp_total_energy_get(struct hl_device *hdev, u64 *total_energy)
417 struct cpucp_packet pkt = {};
421 pkt.ctl = cpu_to_le32(CPUCP_PACKET_TOTAL_ENERGY_GET <<
422 CPUCP_PKT_CTL_OPCODE_SHIFT);
424 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
425 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
428 "Failed to handle CpuCP total energy pkt, error %d\n",
433 *total_energy = result;
438 static void fw_read_errors(struct hl_device *hdev, u32 boot_err0_reg)
442 /* Some of the firmware status codes are deprecated in newer f/w
443 * versions. In those versions, the errors are reported
444 * in different registers. Therefore, we need to check those
445 * registers and print the exact errors. Moreover, there
446 * may be multiple errors, so we need to report on each error
447 * separately. Some of the error codes might indicate a state
448 * that is not an error per-se, but it is an error in production
451 err_val = RREG32(boot_err0_reg);
452 if (!(err_val & CPU_BOOT_ERR0_ENABLED))
455 if (err_val & CPU_BOOT_ERR0_DRAM_INIT_FAIL)
457 "Device boot error - DRAM initialization failed\n");
458 if (err_val & CPU_BOOT_ERR0_FIT_CORRUPTED)
459 dev_err(hdev->dev, "Device boot error - FIT image corrupted\n");
460 if (err_val & CPU_BOOT_ERR0_TS_INIT_FAIL)
462 "Device boot error - Thermal Sensor initialization failed\n");
463 if (err_val & CPU_BOOT_ERR0_DRAM_SKIPPED)
465 "Device boot warning - Skipped DRAM initialization\n");
466 if (err_val & CPU_BOOT_ERR0_BMC_WAIT_SKIPPED)
468 "Device boot error - Skipped waiting for BMC\n");
469 if (err_val & CPU_BOOT_ERR0_NIC_DATA_NOT_RDY)
471 "Device boot error - Serdes data from BMC not available\n");
472 if (err_val & CPU_BOOT_ERR0_NIC_FW_FAIL)
474 "Device boot error - NIC F/W initialization failed\n");
477 static void detect_cpu_boot_status(struct hl_device *hdev, u32 status)
479 /* Some of the status codes below are deprecated in newer f/w
480 * versions but we keep them here for backward compatibility
483 case CPU_BOOT_STATUS_NA:
485 "Device boot error - BTL did NOT run\n");
487 case CPU_BOOT_STATUS_IN_WFE:
489 "Device boot error - Stuck inside WFE loop\n");
491 case CPU_BOOT_STATUS_IN_BTL:
493 "Device boot error - Stuck in BTL\n");
495 case CPU_BOOT_STATUS_IN_PREBOOT:
497 "Device boot error - Stuck in Preboot\n");
499 case CPU_BOOT_STATUS_IN_SPL:
501 "Device boot error - Stuck in SPL\n");
503 case CPU_BOOT_STATUS_IN_UBOOT:
505 "Device boot error - Stuck in u-boot\n");
507 case CPU_BOOT_STATUS_DRAM_INIT_FAIL:
509 "Device boot error - DRAM initialization failed\n");
511 case CPU_BOOT_STATUS_UBOOT_NOT_READY:
513 "Device boot error - u-boot stopped by user\n");
515 case CPU_BOOT_STATUS_TS_INIT_FAIL:
517 "Device boot error - Thermal Sensor initialization failed\n");
521 "Device boot error - Invalid status code %d\n",
527 int hl_fw_read_preboot_ver(struct hl_device *hdev, u32 cpu_boot_status_reg,
528 u32 boot_err0_reg, u32 timeout)
533 if (!hdev->cpu_enable)
536 /* Need to check two possible scenarios:
538 * CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT - for newer firmwares where
539 * the preboot is waiting for the boot fit
541 * All other status values - for older firmwares where the uboot was
542 * loaded from the FLASH
544 rc = hl_poll_timeout(
548 (status == CPU_BOOT_STATUS_IN_UBOOT) ||
549 (status == CPU_BOOT_STATUS_DRAM_RDY) ||
550 (status == CPU_BOOT_STATUS_NIC_FW_RDY) ||
551 (status == CPU_BOOT_STATUS_READY_TO_BOOT) ||
552 (status == CPU_BOOT_STATUS_SRAM_AVAIL) ||
553 (status == CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT),
558 dev_err(hdev->dev, "Failed to read preboot version\n");
559 detect_cpu_boot_status(hdev, status);
560 fw_read_errors(hdev, boot_err0_reg);
564 hdev->asic_funcs->read_device_fw_version(hdev, FW_COMP_PREBOOT);
569 int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg,
570 u32 msg_to_cpu_reg, u32 cpu_msg_status_reg,
571 u32 boot_err0_reg, bool skip_bmc,
572 u32 cpu_timeout, u32 boot_fit_timeout)
577 dev_info(hdev->dev, "Going to wait for device boot (up to %lds)\n",
578 cpu_timeout / USEC_PER_SEC);
580 /* Wait for boot FIT request */
581 rc = hl_poll_timeout(
585 status == CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT,
591 "No boot fit request received, resuming boot\n");
593 rc = hdev->asic_funcs->load_boot_fit_to_device(hdev);
597 /* Clear device CPU message status */
598 WREG32(cpu_msg_status_reg, CPU_MSG_CLR);
600 /* Signal device CPU that boot loader is ready */
601 WREG32(msg_to_cpu_reg, KMD_MSG_FIT_RDY);
603 /* Poll for CPU device ack */
604 rc = hl_poll_timeout(
608 status == CPU_MSG_OK,
614 "Timeout waiting for boot fit load ack\n");
619 WREG32(msg_to_cpu_reg, KMD_MSG_NA);
622 /* Make sure CPU boot-loader is running */
623 rc = hl_poll_timeout(
627 (status == CPU_BOOT_STATUS_DRAM_RDY) ||
628 (status == CPU_BOOT_STATUS_NIC_FW_RDY) ||
629 (status == CPU_BOOT_STATUS_READY_TO_BOOT) ||
630 (status == CPU_BOOT_STATUS_SRAM_AVAIL),
634 /* Read U-Boot version now in case we will later fail */
635 hdev->asic_funcs->read_device_fw_version(hdev, FW_COMP_UBOOT);
638 detect_cpu_boot_status(hdev, status);
643 if (!hdev->fw_loading) {
644 dev_info(hdev->dev, "Skip loading FW\n");
648 if (status == CPU_BOOT_STATUS_SRAM_AVAIL)
652 "Loading firmware to device, may take some time...\n");
654 rc = hdev->asic_funcs->load_firmware_to_device(hdev);
659 WREG32(msg_to_cpu_reg, KMD_MSG_SKIP_BMC);
661 rc = hl_poll_timeout(
665 (status == CPU_BOOT_STATUS_BMC_WAITING_SKIPPED),
671 "Failed to get ACK on skipping BMC, %d\n",
673 WREG32(msg_to_cpu_reg, KMD_MSG_NA);
679 WREG32(msg_to_cpu_reg, KMD_MSG_FIT_RDY);
681 rc = hl_poll_timeout(
685 (status == CPU_BOOT_STATUS_SRAM_AVAIL),
690 WREG32(msg_to_cpu_reg, KMD_MSG_NA);
693 if (status == CPU_BOOT_STATUS_FIT_CORRUPTED)
695 "Device reports FIT image is corrupted\n");
698 "Failed to load firmware to device, %d\n",
705 dev_info(hdev->dev, "Successfully loaded firmware to device\n");
708 fw_read_errors(hdev, boot_err0_reg);