drivers/misc/habanalabs/common/firmware_if.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 /*
   4  * Copyright 2016-2019 HabanaLabs, Ltd.
   5  * All Rights Reserved.
   6  */
   7
   8 #include "habanalabs.h"
   9 #include "../include/common/hl_boot_if.h"
  10
  11 #include <linux/firmware.h>
  12 #include <linux/crc32.h>
  13 #include <linux/slab.h>
  14 #include <linux/ctype.h>
  15
  16 #define FW_FILE_MAX_SIZE                0x1400000 /* maximum size of 20MB */
  17
  18 #define FW_CPU_STATUS_POLL_INTERVAL_USEC        10000
  19
  20 static char *extract_fw_ver_from_str(const char *fw_str)
  21 {
  22         char *str, *fw_ver, *whitespace;
  23
  24         fw_ver = kmalloc(16, GFP_KERNEL);
  25         if (!fw_ver)
  26                 return NULL;
  27
  28         str = strnstr(fw_str, "fw-", VERSION_MAX_LEN);
  29         if (!str)
  30                 goto free_fw_ver;
  31
  32         /* Skip the fw- part */
  33         str += 3;
  34
  35         /* Copy until the next whitespace */
  36         whitespace =  strnstr(str, " ", 15);
  37         if (!whitespace)
  38                 goto free_fw_ver;
  39
  40         strscpy(fw_ver, str, whitespace - str + 1);
  41
  42         return fw_ver;
  43
  44 free_fw_ver:
  45         kfree(fw_ver);
  46         return NULL;
  47 }
  48
  49 static int hl_request_fw(struct hl_device *hdev,
  50                                 const struct firmware **firmware_p,
  51                                 const char *fw_name)
  52 {
  53         size_t fw_size;
  54         int rc;
  55
  56         rc = request_firmware(firmware_p, fw_name, hdev->dev);
  57         if (rc) {
  58                 dev_err(hdev->dev, "Firmware file %s is not found! (error %d)\n",
  59                                 fw_name, rc);
  60                 goto out;
  61         }
  62
  63         fw_size = (*firmware_p)->size;
  64         if ((fw_size % 4) != 0) {
  65                 dev_err(hdev->dev, "Illegal %s firmware size %zu\n",
  66                                 fw_name, fw_size);
  67                 rc = -EINVAL;
  68                 goto release_fw;
  69         }
  70
  71         dev_dbg(hdev->dev, "%s firmware size == %zu\n", fw_name, fw_size);
  72
  73         if (fw_size > FW_FILE_MAX_SIZE) {
  74                 dev_err(hdev->dev,
  75                         "FW file size %zu exceeds maximum of %u bytes\n",
  76                         fw_size, FW_FILE_MAX_SIZE);
  77                 rc = -EINVAL;
  78                 goto release_fw;
  79         }
  80
  81         return 0;
  82
  83 release_fw:
  84         release_firmware(*firmware_p);
  85 out:
  86         return rc;
  87 }
  88
  89 /**
  90  * hl_release_firmware() - release FW
  91  *
  92  * @fw: fw descriptor
  93  *
  94  * note: this inline function added to serve as a comprehensive mirror for the
  95  *       hl_request_fw function.
  96  */
  97 static inline void hl_release_firmware(const struct firmware *fw)
  98 {
  99         release_firmware(fw);
 100 }
 101
 102 /**
 103  * hl_fw_copy_fw_to_device() - copy FW to device
 104  *
 105  * @hdev: pointer to hl_device structure.
 106  * @fw: fw descriptor
 107  * @dst: IO memory mapped address space to copy firmware to
 108  * @src_offset: offset in src FW to copy from
 109  * @size: amount of bytes to copy (0 to copy the whole binary)
 110  *
 111  * actual copy of FW binary data to device, shared by static and dynamic loaders
 112  */
 113 static int hl_fw_copy_fw_to_device(struct hl_device *hdev,
 114                                 const struct firmware *fw, void __iomem *dst,
 115                                 u32 src_offset, u32 size)
 116 {
 117         const void *fw_data;
 118
 119         /* size 0 indicates to copy the whole file */
 120         if (!size)
 121                 size = fw->size;
 122
 123         if (src_offset + size > fw->size) {
 124                 dev_err(hdev->dev,
 125                         "size to copy(%u) and offset(%u) are invalid\n",
 126                         size, src_offset);
 127                 return -EINVAL;
 128         }
 129
 130         fw_data = (const void *) fw->data;
 131
 132         memcpy_toio(dst, fw_data + src_offset, size);
 133         return 0;
 134 }
 135
 136 /**
 137  * hl_fw_copy_msg_to_device() - copy message to device
 138  *
 139  * @hdev: pointer to hl_device structure.
 140  * @msg: message
 141  * @dst: IO memory mapped address space to copy firmware to
 142  * @src_offset: offset in src message to copy from
 143  * @size: amount of bytes to copy (0 to copy the whole binary)
 144  *
 145  * actual copy of message data to device.
 146  */
 147 static int hl_fw_copy_msg_to_device(struct hl_device *hdev,
 148                 struct lkd_msg_comms *msg, void __iomem *dst,
 149                 u32 src_offset, u32 size)
 150 {
 151         void *msg_data;
 152
 153         /* size 0 indicates to copy the whole file */
 154         if (!size)
 155                 size = sizeof(struct lkd_msg_comms);
 156
 157         if (src_offset + size > sizeof(struct lkd_msg_comms)) {
 158                 dev_err(hdev->dev,
 159                         "size to copy(%u) and offset(%u) are invalid\n",
 160                         size, src_offset);
 161                 return -EINVAL;
 162         }
 163
 164         msg_data = (void *) msg;
 165
 166         memcpy_toio(dst, msg_data + src_offset, size);
 167
 168         return 0;
 169 }
 170
 171 /**
 172  * hl_fw_load_fw_to_device() - Load F/W code to device's memory.
 173  *
 174  * @hdev: pointer to hl_device structure.
 175  * @fw_name: the firmware image name
 176  * @dst: IO memory mapped address space to copy firmware to
 177  * @src_offset: offset in src FW to copy from
 178  * @size: amount of bytes to copy (0 to copy the whole binary)
 179  *
 180  * Copy fw code from firmware file to device memory.
 181  *
 182  * Return: 0 on success, non-zero for failure.
 183  */
 184 int hl_fw_load_fw_to_device(struct hl_device *hdev, const char *fw_name,
 185                                 void __iomem *dst, u32 src_offset, u32 size)
 186 {
 187         const struct firmware *fw;
 188         int rc;
 189
 190         rc = hl_request_fw(hdev, &fw, fw_name);
 191         if (rc)
 192                 return rc;
 193
 194         rc = hl_fw_copy_fw_to_device(hdev, fw, dst, src_offset, size);
 195
 196         hl_release_firmware(fw);
 197         return rc;
 198 }
 199
 200 int hl_fw_send_pci_access_msg(struct hl_device *hdev, u32 opcode)
 201 {
 202         struct cpucp_packet pkt = {};
 203
 204         pkt.ctl = cpu_to_le32(opcode << CPUCP_PKT_CTL_OPCODE_SHIFT);
 205
 206         return hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt,
 207                                                 sizeof(pkt), 0, NULL);
 208 }
 209
 210 int hl_fw_send_cpu_message(struct hl_device *hdev, u32 hw_queue_id, u32 *msg,
 211                                 u16 len, u32 timeout, u64 *result)
 212 {
 213         struct hl_hw_queue *queue = &hdev->kernel_queues[hw_queue_id];
 214         struct asic_fixed_properties *prop = &hdev->asic_prop;
 215         struct cpucp_packet *pkt;
 216         dma_addr_t pkt_dma_addr;
 217         u32 tmp, expected_ack_val;
 218         int rc = 0;
 219
 220         pkt = hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev, len,
 221                                                                 &pkt_dma_addr);
 222         if (!pkt) {
 223                 dev_err(hdev->dev,
 224                         "Failed to allocate DMA memory for packet to CPU\n");
 225                 return -ENOMEM;
 226         }
 227
 228         memcpy(pkt, msg, len);
 229
 230         mutex_lock(&hdev->send_cpu_message_lock);
 231
 232         if (hdev->disabled)
 233                 goto out;
 234
 235         if (hdev->device_cpu_disabled) {
 236                 rc = -EIO;
 237                 goto out;
 238         }
 239
 240         /* set fence to a non valid value */
 241         pkt->fence = cpu_to_le32(UINT_MAX);
 242
 243         rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id, len, pkt_dma_addr);
 244         if (rc) {
 245                 dev_err(hdev->dev, "Failed to send CB on CPU PQ (%d)\n", rc);
 246                 goto out;
 247         }
 248
 249         if (prop->fw_app_cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_PKT_PI_ACK_EN)
 250                 expected_ack_val = queue->pi;
 251         else
 252                 expected_ack_val = CPUCP_PACKET_FENCE_VAL;
 253
 254         rc = hl_poll_timeout_memory(hdev, &pkt->fence, tmp,
 255                                 (tmp == expected_ack_val), 1000,
 256                                 timeout, true);
 257
 258         hl_hw_queue_inc_ci_kernel(hdev, hw_queue_id);
 259
 260         if (rc == -ETIMEDOUT) {
 261                 dev_err(hdev->dev, "Device CPU packet timeout (0x%x)\n", tmp);
 262                 hdev->device_cpu_disabled = true;
 263                 goto out;
 264         }
 265
 266         tmp = le32_to_cpu(pkt->ctl);
 267
 268         rc = (tmp & CPUCP_PKT_CTL_RC_MASK) >> CPUCP_PKT_CTL_RC_SHIFT;
 269         if (rc) {
 270                 dev_err(hdev->dev, "F/W ERROR %d for CPU packet %d\n",
 271                         rc,
 272                         (tmp & CPUCP_PKT_CTL_OPCODE_MASK)
 273                                                 >> CPUCP_PKT_CTL_OPCODE_SHIFT);
 274                 rc = -EIO;
 275         } else if (result) {
 276                 *result = le64_to_cpu(pkt->result);
 277         }
 278
 279 out:
 280         mutex_unlock(&hdev->send_cpu_message_lock);
 281
 282         hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev, len, pkt);
 283
 284         return rc;
 285 }
 286
 287 int hl_fw_unmask_irq(struct hl_device *hdev, u16 event_type)
 288 {
 289         struct cpucp_packet pkt;
 290         u64 result;
 291         int rc;
 292
 293         memset(&pkt, 0, sizeof(pkt));
 294
 295         pkt.ctl = cpu_to_le32(CPUCP_PACKET_UNMASK_RAZWI_IRQ <<
 296                                 CPUCP_PKT_CTL_OPCODE_SHIFT);
 297         pkt.value = cpu_to_le64(event_type);
 298
 299         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
 300                                                 0, &result);
 301
 302         if (rc)
 303                 dev_err(hdev->dev, "failed to unmask RAZWI IRQ %d", event_type);
 304
 305         return rc;
 306 }
 307
 308 int hl_fw_unmask_irq_arr(struct hl_device *hdev, const u32 *irq_arr,
 309                 size_t irq_arr_size)
 310 {
 311         struct cpucp_unmask_irq_arr_packet *pkt;
 312         size_t total_pkt_size;
 313         u64 result;
 314         int rc;
 315
 316         total_pkt_size = sizeof(struct cpucp_unmask_irq_arr_packet) +
 317                         irq_arr_size;
 318
 319         /* data should be aligned to 8 bytes in order to CPU-CP to copy it */
 320         total_pkt_size = (total_pkt_size + 0x7) & ~0x7;
 321
 322         /* total_pkt_size is casted to u16 later on */
 323         if (total_pkt_size > USHRT_MAX) {
 324                 dev_err(hdev->dev, "too many elements in IRQ array\n");
 325                 return -EINVAL;
 326         }
 327
 328         pkt = kzalloc(total_pkt_size, GFP_KERNEL);
 329         if (!pkt)
 330                 return -ENOMEM;
 331
 332         pkt->length = cpu_to_le32(irq_arr_size / sizeof(irq_arr[0]));
 333         memcpy(&pkt->irqs, irq_arr, irq_arr_size);
 334
 335         pkt->cpucp_pkt.ctl = cpu_to_le32(CPUCP_PACKET_UNMASK_RAZWI_IRQ_ARRAY <<
 336                                                 CPUCP_PKT_CTL_OPCODE_SHIFT);
 337
 338         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) pkt,
 339                                                 total_pkt_size, 0, &result);
 340
 341         if (rc)
 342                 dev_err(hdev->dev, "failed to unmask IRQ array\n");
 343
 344         kfree(pkt);
 345
 346         return rc;
 347 }
 348
 349 int hl_fw_test_cpu_queue(struct hl_device *hdev)
 350 {
 351         struct cpucp_packet test_pkt = {};
 352         u64 result;
 353         int rc;
 354
 355         test_pkt.ctl = cpu_to_le32(CPUCP_PACKET_TEST <<
 356                                         CPUCP_PKT_CTL_OPCODE_SHIFT);
 357         test_pkt.value = cpu_to_le64(CPUCP_PACKET_FENCE_VAL);
 358
 359         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &test_pkt,
 360                                                 sizeof(test_pkt), 0, &result);
 361
 362         if (!rc) {
 363                 if (result != CPUCP_PACKET_FENCE_VAL)
 364                         dev_err(hdev->dev,
 365                                 "CPU queue test failed (%#08llx)\n", result);
 366         } else {
 367                 dev_err(hdev->dev, "CPU queue test failed, error %d\n", rc);
 368         }
 369
 370         return rc;
 371 }
 372
 373 void *hl_fw_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size,
 374                                                 dma_addr_t *dma_handle)
 375 {
 376         u64 kernel_addr;
 377
 378         kernel_addr = gen_pool_alloc(hdev->cpu_accessible_dma_pool, size);
 379
 380         *dma_handle = hdev->cpu_accessible_dma_address +
 381                 (kernel_addr - (u64) (uintptr_t) hdev->cpu_accessible_dma_mem);
 382
 383         return (void *) (uintptr_t) kernel_addr;
 384 }
 385
 386 void hl_fw_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size,
 387                                         void *vaddr)
 388 {
 389         gen_pool_free(hdev->cpu_accessible_dma_pool, (u64) (uintptr_t) vaddr,
 390                         size);
 391 }
 392
 393 int hl_fw_send_heartbeat(struct hl_device *hdev)
 394 {
 395         struct cpucp_packet hb_pkt;
 396         u64 result;
 397         int rc;
 398
 399         memset(&hb_pkt, 0, sizeof(hb_pkt));
 400         hb_pkt.ctl = cpu_to_le32(CPUCP_PACKET_TEST <<
 401                                         CPUCP_PKT_CTL_OPCODE_SHIFT);
 402         hb_pkt.value = cpu_to_le64(CPUCP_PACKET_FENCE_VAL);
 403
 404         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &hb_pkt,
 405                                                 sizeof(hb_pkt), 0, &result);
 406
 407         if ((rc) || (result != CPUCP_PACKET_FENCE_VAL))
 408                 return -EIO;
 409
 410         if (le32_to_cpu(hb_pkt.status_mask) &
 411                                         CPUCP_PKT_HB_STATUS_EQ_FAULT_MASK) {
 412                 dev_warn(hdev->dev, "FW reported EQ fault during heartbeat\n");
 413                 rc = -EIO;
 414         }
 415
 416         return rc;
 417 }
 418
 419 static bool fw_report_boot_dev0(struct hl_device *hdev, u32 err_val,
 420                                                                 u32 sts_val)
 421 {
 422         bool err_exists = false;
 423
 424         if (!(err_val & CPU_BOOT_ERR0_ENABLED))
 425                 return false;
 426
 427         if (err_val & CPU_BOOT_ERR0_DRAM_INIT_FAIL) {
 428                 dev_err(hdev->dev,
 429                         "Device boot error - DRAM initialization failed\n");
 430                 err_exists = true;
 431         }
 432
 433         if (err_val & CPU_BOOT_ERR0_FIT_CORRUPTED) {
 434                 dev_err(hdev->dev, "Device boot error - FIT image corrupted\n");
 435                 err_exists = true;
 436         }
 437
 438         if (err_val & CPU_BOOT_ERR0_TS_INIT_FAIL) {
 439                 dev_err(hdev->dev,
 440                         "Device boot error - Thermal Sensor initialization failed\n");
 441                 err_exists = true;
 442         }
 443
 444         if (err_val & CPU_BOOT_ERR0_DRAM_SKIPPED) {
 445                 dev_warn(hdev->dev,
 446                         "Device boot warning - Skipped DRAM initialization\n");
 447                 /* This is a warning so we don't want it to disable the
 448                  * device
 449                  */
 450                 err_val &= ~CPU_BOOT_ERR0_DRAM_SKIPPED;
 451         }
 452
 453         if (err_val & CPU_BOOT_ERR0_BMC_WAIT_SKIPPED) {
 454                 if (hdev->bmc_enable) {
 455                         dev_err(hdev->dev,
 456                                 "Device boot error - Skipped waiting for BMC\n");
 457                         err_exists = true;
 458                 } else {
 459                         dev_info(hdev->dev,
 460                                 "Device boot message - Skipped waiting for BMC\n");
 461                         /* This is an info so we don't want it to disable the
 462                          * device
 463                          */
 464                         err_val &= ~CPU_BOOT_ERR0_BMC_WAIT_SKIPPED;
 465                 }
 466         }
 467
 468         if (err_val & CPU_BOOT_ERR0_NIC_DATA_NOT_RDY) {
 469                 dev_err(hdev->dev,
 470                         "Device boot error - Serdes data from BMC not available\n");
 471                 err_exists = true;
 472         }
 473
 474         if (err_val & CPU_BOOT_ERR0_NIC_FW_FAIL) {
 475                 dev_err(hdev->dev,
 476                         "Device boot error - NIC F/W initialization failed\n");
 477                 err_exists = true;
 478         }
 479
 480         if (err_val & CPU_BOOT_ERR0_SECURITY_NOT_RDY) {
 481                 dev_err(hdev->dev,
 482                         "Device boot warning - security not ready\n");
 483                 err_exists = true;
 484         }
 485
 486         if (err_val & CPU_BOOT_ERR0_SECURITY_FAIL) {
 487                 dev_err(hdev->dev, "Device boot error - security failure\n");
 488                 err_exists = true;
 489         }
 490
 491         if (err_val & CPU_BOOT_ERR0_EFUSE_FAIL) {
 492                 dev_err(hdev->dev, "Device boot error - eFuse failure\n");
 493                 err_exists = true;
 494         }
 495
 496         if (err_val & CPU_BOOT_ERR0_PRI_IMG_VER_FAIL) {
 497                 dev_warn(hdev->dev,
 498                         "Device boot warning - Failed to load preboot primary image\n");
 499                 /* This is a warning so we don't want it to disable the
 500                  * device as we have a secondary preboot image
 501                  */
 502                 err_val &= ~CPU_BOOT_ERR0_PRI_IMG_VER_FAIL;
 503         }
 504
 505         if (err_val & CPU_BOOT_ERR0_SEC_IMG_VER_FAIL) {
 506                 dev_err(hdev->dev, "Device boot error - Failed to load preboot secondary image\n");
 507                 err_exists = true;
 508         }
 509
 510         if (err_val & CPU_BOOT_ERR0_PLL_FAIL) {
 511                 dev_err(hdev->dev, "Device boot error - PLL failure\n");
 512                 err_exists = true;
 513         }
 514
 515         if (err_val & CPU_BOOT_ERR0_DEVICE_UNUSABLE_FAIL) {
 516                 /* Ignore this bit, don't prevent driver loading */
 517                 dev_dbg(hdev->dev, "device unusable status is set\n");
 518                 err_val &= ~CPU_BOOT_ERR0_DEVICE_UNUSABLE_FAIL;
 519         }
 520
 521         if (sts_val & CPU_BOOT_DEV_STS0_ENABLED)
 522                 dev_dbg(hdev->dev, "Device status0 %#x\n", sts_val);
 523
 524         if (!err_exists && (err_val & ~CPU_BOOT_ERR0_ENABLED)) {
 525                 dev_err(hdev->dev,
 526                         "Device boot error - unknown ERR0 error 0x%08x\n", err_val);
 527                 err_exists = true;
 528         }
 529
 530         /* return error only if it's in the predefined mask */
 531         if (err_exists && ((err_val & ~CPU_BOOT_ERR0_ENABLED) &
 532                                 lower_32_bits(hdev->boot_error_status_mask)))
 533                 return true;
 534
 535         return false;
 536 }
 537
 538 /* placeholder for ERR1 as no errors defined there yet */
 539 static bool fw_report_boot_dev1(struct hl_device *hdev, u32 err_val,
 540                                                                 u32 sts_val)
 541 {
 542         /*
 543          * keep this variable to preserve the logic of the function.
 544          * this way it would require less modifications when error will be
 545          * added to DEV_ERR1
 546          */
 547         bool err_exists = false;
 548
 549         if (!(err_val & CPU_BOOT_ERR1_ENABLED))
 550                 return false;
 551
 552         if (sts_val & CPU_BOOT_DEV_STS1_ENABLED)
 553                 dev_dbg(hdev->dev, "Device status1 %#x\n", sts_val);
 554
 555         if (!err_exists && (err_val & ~CPU_BOOT_ERR1_ENABLED)) {
 556                 dev_err(hdev->dev,
 557                         "Device boot error - unknown ERR1 error 0x%08x\n",
 558                                                                 err_val);
 559                 err_exists = true;
 560         }
 561
 562         /* return error only if it's in the predefined mask */
 563         if (err_exists && ((err_val & ~CPU_BOOT_ERR1_ENABLED) &
 564                                 upper_32_bits(hdev->boot_error_status_mask)))
 565                 return true;
 566
 567         return false;
 568 }
 569
 570 static int fw_read_errors(struct hl_device *hdev, u32 boot_err0_reg,
 571                                 u32 boot_err1_reg, u32 cpu_boot_dev_status0_reg,
 572                                 u32 cpu_boot_dev_status1_reg)
 573 {
 574         u32 err_val, status_val;
 575         bool err_exists = false;
 576
 577         /* Some of the firmware status codes are deprecated in newer f/w
 578          * versions. In those versions, the errors are reported
 579          * in different registers. Therefore, we need to check those
 580          * registers and print the exact errors. Moreover, there
 581          * may be multiple errors, so we need to report on each error
 582          * separately. Some of the error codes might indicate a state
 583          * that is not an error per-se, but it is an error in production
 584          * environment
 585          */
 586         err_val = RREG32(boot_err0_reg);
 587         status_val = RREG32(cpu_boot_dev_status0_reg);
 588         err_exists = fw_report_boot_dev0(hdev, err_val, status_val);
 589
 590         err_val = RREG32(boot_err1_reg);
 591         status_val = RREG32(cpu_boot_dev_status1_reg);
 592         err_exists |= fw_report_boot_dev1(hdev, err_val, status_val);
 593
 594         if (err_exists)
 595                 return -EIO;
 596
 597         return 0;
 598 }
 599
 600 int hl_fw_cpucp_info_get(struct hl_device *hdev,
 601                                 u32 sts_boot_dev_sts0_reg,
 602                                 u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg,
 603                                 u32 boot_err1_reg)
 604 {
 605         struct asic_fixed_properties *prop = &hdev->asic_prop;
 606         struct cpucp_packet pkt = {};
 607         dma_addr_t cpucp_info_dma_addr;
 608         void *cpucp_info_cpu_addr;
 609         char *kernel_ver;
 610         u64 result;
 611         int rc;
 612
 613         cpucp_info_cpu_addr =
 614                         hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev,
 615                                         sizeof(struct cpucp_info),
 616                                         &cpucp_info_dma_addr);
 617         if (!cpucp_info_cpu_addr) {
 618                 dev_err(hdev->dev,
 619                         "Failed to allocate DMA memory for CPU-CP info packet\n");
 620                 return -ENOMEM;
 621         }
 622
 623         memset(cpucp_info_cpu_addr, 0, sizeof(struct cpucp_info));
 624
 625         pkt.ctl = cpu_to_le32(CPUCP_PACKET_INFO_GET <<
 626                                 CPUCP_PKT_CTL_OPCODE_SHIFT);
 627         pkt.addr = cpu_to_le64(cpucp_info_dma_addr);
 628         pkt.data_max_size = cpu_to_le32(sizeof(struct cpucp_info));
 629
 630         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
 631                                         HL_CPUCP_INFO_TIMEOUT_USEC, &result);
 632         if (rc) {
 633                 dev_err(hdev->dev,
 634                         "Failed to handle CPU-CP info pkt, error %d\n", rc);
 635                 goto out;
 636         }
 637
 638         rc = fw_read_errors(hdev, boot_err0_reg, boot_err1_reg,
 639                                 sts_boot_dev_sts0_reg, sts_boot_dev_sts1_reg);
 640         if (rc) {
 641                 dev_err(hdev->dev, "Errors in device boot\n");
 642                 goto out;
 643         }
 644
 645         memcpy(&prop->cpucp_info, cpucp_info_cpu_addr,
 646                         sizeof(prop->cpucp_info));
 647
 648         rc = hl_build_hwmon_channel_info(hdev, prop->cpucp_info.sensors);
 649         if (rc) {
 650                 dev_err(hdev->dev,
 651                         "Failed to build hwmon channel info, error %d\n", rc);
 652                 rc = -EFAULT;
 653                 goto out;
 654         }
 655
 656         kernel_ver = extract_fw_ver_from_str(prop->cpucp_info.kernel_version);
 657         if (kernel_ver) {
 658                 dev_info(hdev->dev, "Linux version %s", kernel_ver);
 659                 kfree(kernel_ver);
 660         }
 661
 662         /* assume EQ code doesn't need to check eqe index */
 663         hdev->event_queue.check_eqe_index = false;
 664
 665         /* Read FW application security bits again */
 666         if (hdev->asic_prop.fw_cpu_boot_dev_sts0_valid) {
 667                 hdev->asic_prop.fw_app_cpu_boot_dev_sts0 =
 668                                                 RREG32(sts_boot_dev_sts0_reg);
 669                 if (hdev->asic_prop.fw_app_cpu_boot_dev_sts0 &
 670                                 CPU_BOOT_DEV_STS0_EQ_INDEX_EN)
 671                         hdev->event_queue.check_eqe_index = true;
 672         }
 673
 674         if (hdev->asic_prop.fw_cpu_boot_dev_sts1_valid)
 675                 hdev->asic_prop.fw_app_cpu_boot_dev_sts1 =
 676                                                 RREG32(sts_boot_dev_sts1_reg);
 677
 678 out:
 679         hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev,
 680                         sizeof(struct cpucp_info), cpucp_info_cpu_addr);
 681
 682         return rc;
 683 }
 684
 685 static int hl_fw_send_msi_info_msg(struct hl_device *hdev)
 686 {
 687         struct cpucp_array_data_packet *pkt;
 688         size_t total_pkt_size, data_size;
 689         u64 result;
 690         int rc;
 691
 692         /* skip sending this info for unsupported ASICs */
 693         if (!hdev->asic_funcs->get_msi_info)
 694                 return 0;
 695
 696         data_size = CPUCP_NUM_OF_MSI_TYPES * sizeof(u32);
 697         total_pkt_size = sizeof(struct cpucp_array_data_packet) + data_size;
 698
 699         /* data should be aligned to 8 bytes in order to CPU-CP to copy it */
 700         total_pkt_size = (total_pkt_size + 0x7) & ~0x7;
 701
 702         /* total_pkt_size is casted to u16 later on */
 703         if (total_pkt_size > USHRT_MAX) {
 704                 dev_err(hdev->dev, "CPUCP array data is too big\n");
 705                 return -EINVAL;
 706         }
 707
 708         pkt = kzalloc(total_pkt_size, GFP_KERNEL);
 709         if (!pkt)
 710                 return -ENOMEM;
 711
 712         pkt->length = cpu_to_le32(CPUCP_NUM_OF_MSI_TYPES);
 713
 714         memset((void *) &pkt->data, 0xFF, data_size);
 715         hdev->asic_funcs->get_msi_info(pkt->data);
 716
 717         pkt->cpucp_pkt.ctl = cpu_to_le32(CPUCP_PACKET_MSI_INFO_SET <<
 718                                                 CPUCP_PKT_CTL_OPCODE_SHIFT);
 719
 720         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *)pkt,
 721                                                 total_pkt_size, 0, &result);
 722
 723         /*
 724          * in case packet result is invalid it means that FW does not support
 725          * this feature and will use default/hard coded MSI values. no reason
 726          * to stop the boot
 727          */
 728         if (rc && result == cpucp_packet_invalid)
 729                 rc = 0;
 730
 731         if (rc)
 732                 dev_err(hdev->dev, "failed to send CPUCP array data\n");
 733
 734         kfree(pkt);
 735
 736         return rc;
 737 }
 738
 739 int hl_fw_cpucp_handshake(struct hl_device *hdev,
 740                                 u32 sts_boot_dev_sts0_reg,
 741                                 u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg,
 742                                 u32 boot_err1_reg)
 743 {
 744         int rc;
 745
 746         rc = hl_fw_cpucp_info_get(hdev, sts_boot_dev_sts0_reg,
 747                                         sts_boot_dev_sts1_reg, boot_err0_reg,
 748                                         boot_err1_reg);
 749         if (rc)
 750                 return rc;
 751
 752         return hl_fw_send_msi_info_msg(hdev);
 753 }
 754
 755 int hl_fw_get_eeprom_data(struct hl_device *hdev, void *data, size_t max_size)
 756 {
 757         struct cpucp_packet pkt = {};
 758         void *eeprom_info_cpu_addr;
 759         dma_addr_t eeprom_info_dma_addr;
 760         u64 result;
 761         int rc;
 762
 763         eeprom_info_cpu_addr =
 764                         hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev,
 765                                         max_size, &eeprom_info_dma_addr);
 766         if (!eeprom_info_cpu_addr) {
 767                 dev_err(hdev->dev,
 768                         "Failed to allocate DMA memory for CPU-CP EEPROM packet\n");
 769                 return -ENOMEM;
 770         }
 771
 772         memset(eeprom_info_cpu_addr, 0, max_size);
 773
 774         pkt.ctl = cpu_to_le32(CPUCP_PACKET_EEPROM_DATA_GET <<
 775                                 CPUCP_PKT_CTL_OPCODE_SHIFT);
 776         pkt.addr = cpu_to_le64(eeprom_info_dma_addr);
 777         pkt.data_max_size = cpu_to_le32(max_size);
 778
 779         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
 780                         HL_CPUCP_EEPROM_TIMEOUT_USEC, &result);
 781
 782         if (rc) {
 783                 dev_err(hdev->dev,
 784                         "Failed to handle CPU-CP EEPROM packet, error %d\n",
 785                         rc);
 786                 goto out;
 787         }
 788
 789         /* result contains the actual size */
 790         memcpy(data, eeprom_info_cpu_addr, min((size_t)result, max_size));
 791
 792 out:
 793         hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev, max_size,
 794                         eeprom_info_cpu_addr);
 795
 796         return rc;
 797 }
 798
 799 int hl_fw_cpucp_pci_counters_get(struct hl_device *hdev,
 800                 struct hl_info_pci_counters *counters)
 801 {
 802         struct cpucp_packet pkt = {};
 803         u64 result;
 804         int rc;
 805
 806         pkt.ctl = cpu_to_le32(CPUCP_PACKET_PCIE_THROUGHPUT_GET <<
 807                         CPUCP_PKT_CTL_OPCODE_SHIFT);
 808
 809         /* Fetch PCI rx counter */
 810         pkt.index = cpu_to_le32(cpucp_pcie_throughput_rx);
 811         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
 812                                         HL_CPUCP_INFO_TIMEOUT_USEC, &result);
 813         if (rc) {
 814                 dev_err(hdev->dev,
 815                         "Failed to handle CPU-CP PCI info pkt, error %d\n", rc);
 816                 return rc;
 817         }
 818         counters->rx_throughput = result;
 819
 820         memset(&pkt, 0, sizeof(pkt));
 821         pkt.ctl = cpu_to_le32(CPUCP_PACKET_PCIE_THROUGHPUT_GET <<
 822                         CPUCP_PKT_CTL_OPCODE_SHIFT);
 823
 824         /* Fetch PCI tx counter */
 825         pkt.index = cpu_to_le32(cpucp_pcie_throughput_tx);
 826         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
 827                                         HL_CPUCP_INFO_TIMEOUT_USEC, &result);
 828         if (rc) {
 829                 dev_err(hdev->dev,
 830                         "Failed to handle CPU-CP PCI info pkt, error %d\n", rc);
 831                 return rc;
 832         }
 833         counters->tx_throughput = result;
 834
 835         /* Fetch PCI replay counter */
 836         memset(&pkt, 0, sizeof(pkt));
 837         pkt.ctl = cpu_to_le32(CPUCP_PACKET_PCIE_REPLAY_CNT_GET <<
 838                         CPUCP_PKT_CTL_OPCODE_SHIFT);
 839
 840         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
 841                         HL_CPUCP_INFO_TIMEOUT_USEC, &result);
 842         if (rc) {
 843                 dev_err(hdev->dev,
 844                         "Failed to handle CPU-CP PCI info pkt, error %d\n", rc);
 845                 return rc;
 846         }
 847         counters->replay_cnt = (u32) result;
 848
 849         return rc;
 850 }
 851
 852 int hl_fw_cpucp_total_energy_get(struct hl_device *hdev, u64 *total_energy)
 853 {
 854         struct cpucp_packet pkt = {};
 855         u64 result;
 856         int rc;
 857
 858         pkt.ctl = cpu_to_le32(CPUCP_PACKET_TOTAL_ENERGY_GET <<
 859                                 CPUCP_PKT_CTL_OPCODE_SHIFT);
 860
 861         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
 862                                         HL_CPUCP_INFO_TIMEOUT_USEC, &result);
 863         if (rc) {
 864                 dev_err(hdev->dev,
 865                         "Failed to handle CpuCP total energy pkt, error %d\n",
 866                                 rc);
 867                 return rc;
 868         }
 869
 870         *total_energy = result;
 871
 872         return rc;
 873 }
 874
 875 int get_used_pll_index(struct hl_device *hdev, u32 input_pll_index,
 876                                                 enum pll_index *pll_index)
 877 {
 878         struct asic_fixed_properties *prop = &hdev->asic_prop;
 879         u8 pll_byte, pll_bit_off;
 880         bool dynamic_pll;
 881         int fw_pll_idx;
 882
 883         dynamic_pll = !!(prop->fw_app_cpu_boot_dev_sts0 &
 884                                                 CPU_BOOT_DEV_STS0_DYN_PLL_EN);
 885
 886         if (!dynamic_pll) {
 887                 /*
 888                  * in case we are working with legacy FW (each asic has unique
 889                  * PLL numbering) use the driver based index as they are
 890                  * aligned with fw legacy numbering
 891                  */
 892                 *pll_index = input_pll_index;
 893                 return 0;
 894         }
 895
 896         /* retrieve a FW compatible PLL index based on
 897          * ASIC specific user request
 898          */
 899         fw_pll_idx = hdev->asic_funcs->map_pll_idx_to_fw_idx(input_pll_index);
 900         if (fw_pll_idx < 0) {
 901                 dev_err(hdev->dev, "Invalid PLL index (%u) error %d\n",
 902                         input_pll_index, fw_pll_idx);
 903                 return -EINVAL;
 904         }
 905
 906         /* PLL map is a u8 array */
 907         pll_byte = prop->cpucp_info.pll_map[fw_pll_idx >> 3];
 908         pll_bit_off = fw_pll_idx & 0x7;
 909
 910         if (!(pll_byte & BIT(pll_bit_off))) {
 911                 dev_err(hdev->dev, "PLL index %d is not supported\n",
 912                         fw_pll_idx);
 913                 return -EINVAL;
 914         }
 915
 916         *pll_index = fw_pll_idx;
 917
 918         return 0;
 919 }
 920
 921 int hl_fw_cpucp_pll_info_get(struct hl_device *hdev, u32 pll_index,
 922                 u16 *pll_freq_arr)
 923 {
 924         struct cpucp_packet pkt;
 925         enum pll_index used_pll_idx;
 926         u64 result;
 927         int rc;
 928
 929         rc = get_used_pll_index(hdev, pll_index, &used_pll_idx);
 930         if (rc)
 931                 return rc;
 932
 933         memset(&pkt, 0, sizeof(pkt));
 934
 935         pkt.ctl = cpu_to_le32(CPUCP_PACKET_PLL_INFO_GET <<
 936                                 CPUCP_PKT_CTL_OPCODE_SHIFT);
 937         pkt.pll_type = __cpu_to_le16((u16)used_pll_idx);
 938
 939         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
 940                         HL_CPUCP_INFO_TIMEOUT_USEC, &result);
 941         if (rc)
 942                 dev_err(hdev->dev, "Failed to read PLL info, error %d\n", rc);
 943
 944         pll_freq_arr[0] = FIELD_GET(CPUCP_PKT_RES_PLL_OUT0_MASK, result);
 945         pll_freq_arr[1] = FIELD_GET(CPUCP_PKT_RES_PLL_OUT1_MASK, result);
 946         pll_freq_arr[2] = FIELD_GET(CPUCP_PKT_RES_PLL_OUT2_MASK, result);
 947         pll_freq_arr[3] = FIELD_GET(CPUCP_PKT_RES_PLL_OUT3_MASK, result);
 948
 949         return rc;
 950 }
 951
 952 int hl_fw_cpucp_power_get(struct hl_device *hdev, u64 *power)
 953 {
 954         struct cpucp_packet pkt;
 955         u64 result;
 956         int rc;
 957
 958         memset(&pkt, 0, sizeof(pkt));
 959
 960         pkt.ctl = cpu_to_le32(CPUCP_PACKET_POWER_GET <<
 961                                 CPUCP_PKT_CTL_OPCODE_SHIFT);
 962
 963         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
 964                         HL_CPUCP_INFO_TIMEOUT_USEC, &result);
 965         if (rc) {
 966                 dev_err(hdev->dev, "Failed to read power, error %d\n", rc);
 967                 return rc;
 968         }
 969
 970         *power = result;
 971
 972         return rc;
 973 }
 974
 975 void hl_fw_ask_hard_reset_without_linux(struct hl_device *hdev)
 976 {
 977         struct static_fw_load_mgr *static_loader =
 978                         &hdev->fw_loader.static_loader;
 979         int rc;
 980
 981         if (hdev->asic_prop.dynamic_fw_load) {
 982                 rc = hl_fw_dynamic_send_protocol_cmd(hdev, &hdev->fw_loader,
 983                                 COMMS_RST_DEV, 0, false,
 984                                 hdev->fw_loader.cpu_timeout);
 985                 if (rc)
 986                         dev_warn(hdev->dev, "Failed sending COMMS_RST_DEV\n");
 987         } else {
 988                 WREG32(static_loader->kmd_msg_to_cpu_reg, KMD_MSG_RST_DEV);
 989         }
 990 }
 991
 992 void hl_fw_ask_halt_machine_without_linux(struct hl_device *hdev)
 993 {
 994         struct static_fw_load_mgr *static_loader =
 995                         &hdev->fw_loader.static_loader;
 996         int rc;
 997
 998         if (hdev->device_cpu_is_halted)
 999                 return;
1000
1001         /* Stop device CPU to make sure nothing bad happens */
1002         if (hdev->asic_prop.dynamic_fw_load) {
1003                 rc = hl_fw_dynamic_send_protocol_cmd(hdev, &hdev->fw_loader,
1004                                 COMMS_GOTO_WFE, 0, true,
1005                                 hdev->fw_loader.cpu_timeout);
1006                 if (rc)
1007                         dev_warn(hdev->dev, "Failed sending COMMS_GOTO_WFE\n");
1008         } else {
1009                 WREG32(static_loader->kmd_msg_to_cpu_reg, KMD_MSG_GOTO_WFE);
1010                 msleep(static_loader->cpu_reset_wait_msec);
1011         }
1012
1013         hdev->device_cpu_is_halted = true;
1014 }
1015
1016 static void detect_cpu_boot_status(struct hl_device *hdev, u32 status)
1017 {
1018         /* Some of the status codes below are deprecated in newer f/w
1019          * versions but we keep them here for backward compatibility
1020          */
1021         switch (status) {
1022         case CPU_BOOT_STATUS_NA:
1023                 dev_err(hdev->dev,
1024                         "Device boot progress - BTL did NOT run\n");
1025                 break;
1026         case CPU_BOOT_STATUS_IN_WFE:
1027                 dev_err(hdev->dev,
1028                         "Device boot progress - Stuck inside WFE loop\n");
1029                 break;
1030         case CPU_BOOT_STATUS_IN_BTL:
1031                 dev_err(hdev->dev,
1032                         "Device boot progress - Stuck in BTL\n");
1033                 break;
1034         case CPU_BOOT_STATUS_IN_PREBOOT:
1035                 dev_err(hdev->dev,
1036                         "Device boot progress - Stuck in Preboot\n");
1037                 break;
1038         case CPU_BOOT_STATUS_IN_SPL:
1039                 dev_err(hdev->dev,
1040                         "Device boot progress - Stuck in SPL\n");
1041                 break;
1042         case CPU_BOOT_STATUS_IN_UBOOT:
1043                 dev_err(hdev->dev,
1044                         "Device boot progress - Stuck in u-boot\n");
1045                 break;
1046         case CPU_BOOT_STATUS_DRAM_INIT_FAIL:
1047                 dev_err(hdev->dev,
1048                         "Device boot progress - DRAM initialization failed\n");
1049                 break;
1050         case CPU_BOOT_STATUS_UBOOT_NOT_READY:
1051                 dev_err(hdev->dev,
1052                         "Device boot progress - Cannot boot\n");
1053                 break;
1054         case CPU_BOOT_STATUS_TS_INIT_FAIL:
1055                 dev_err(hdev->dev,
1056                         "Device boot progress - Thermal Sensor initialization failed\n");
1057                 break;
1058         default:
1059                 dev_err(hdev->dev,
1060                         "Device boot progress - Invalid status code %d\n",
1061                         status);
1062                 break;
1063         }
1064 }
1065
1066 static int hl_fw_read_preboot_caps(struct hl_device *hdev,
1067                                         u32 cpu_boot_status_reg,
1068                                         u32 sts_boot_dev_sts0_reg,
1069                                         u32 sts_boot_dev_sts1_reg,
1070                                         u32 boot_err0_reg, u32 boot_err1_reg,
1071                                         u32 timeout)
1072 {
1073         struct asic_fixed_properties *prop = &hdev->asic_prop;
1074         u32 status, reg_val;
1075         int rc;
1076
1077         /* Need to check two possible scenarios:
1078          *
1079          * CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT - for newer firmwares where
1080          * the preboot is waiting for the boot fit
1081          *
1082          * All other status values - for older firmwares where the uboot was
1083          * loaded from the FLASH
1084          */
1085         rc = hl_poll_timeout(
1086                 hdev,
1087                 cpu_boot_status_reg,
1088                 status,
1089                 (status == CPU_BOOT_STATUS_IN_UBOOT) ||
1090                 (status == CPU_BOOT_STATUS_DRAM_RDY) ||
1091                 (status == CPU_BOOT_STATUS_NIC_FW_RDY) ||
1092                 (status == CPU_BOOT_STATUS_READY_TO_BOOT) ||
1093                 (status == CPU_BOOT_STATUS_SRAM_AVAIL) ||
1094                 (status == CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT),
1095                 FW_CPU_STATUS_POLL_INTERVAL_USEC,
1096                 timeout);
1097
1098         if (rc) {
1099                 dev_err(hdev->dev, "CPU boot ready status timeout\n");
1100                 detect_cpu_boot_status(hdev, status);
1101
1102                 /* If we read all FF, then something is totally wrong, no point
1103                  * of reading specific errors
1104                  */
1105                 if (status != -1)
1106                         fw_read_errors(hdev, boot_err0_reg, boot_err1_reg,
1107                                                         sts_boot_dev_sts0_reg,
1108                                                         sts_boot_dev_sts1_reg);
1109                 return -EIO;
1110         }
1111
1112         /*
1113          * the registers DEV_STS* contain FW capabilities/features.
1114          * We can rely on this registers only if bit CPU_BOOT_DEV_STS*_ENABLED
1115          * is set.
1116          * In the first read of this register we store the value of this
1117          * register ONLY if the register is enabled (which will be propagated
1118          * to next stages) and also mark the register as valid.
1119          * In case it is not enabled the stored value will be left 0- all
1120          * caps/features are off
1121          */
1122         reg_val = RREG32(sts_boot_dev_sts0_reg);
1123         if (reg_val & CPU_BOOT_DEV_STS0_ENABLED) {
1124                 prop->fw_cpu_boot_dev_sts0_valid = true;
1125                 prop->fw_preboot_cpu_boot_dev_sts0 = reg_val;
1126         }
1127
1128         reg_val = RREG32(sts_boot_dev_sts1_reg);
1129         if (reg_val & CPU_BOOT_DEV_STS1_ENABLED) {
1130                 prop->fw_cpu_boot_dev_sts1_valid = true;
1131                 prop->fw_preboot_cpu_boot_dev_sts1 = reg_val;
1132         }
1133
1134         prop->dynamic_fw_load = !!(prop->fw_preboot_cpu_boot_dev_sts0 &
1135                                                 CPU_BOOT_DEV_STS0_FW_LD_COM_EN);
1136
1137         /* initialize FW loader once we know what load protocol is used */
1138         hdev->asic_funcs->init_firmware_loader(hdev);
1139
1140         dev_dbg(hdev->dev, "Attempting %s FW load\n",
1141                         prop->dynamic_fw_load ? "dynamic" : "legacy");
1142         return 0;
1143 }
1144
1145 static int hl_fw_static_read_device_fw_version(struct hl_device *hdev,
1146                                         enum hl_fw_component fwc)
1147 {
1148         struct asic_fixed_properties *prop = &hdev->asic_prop;
1149         struct fw_load_mgr *fw_loader = &hdev->fw_loader;
1150         struct static_fw_load_mgr *static_loader;
1151         char *dest, *boot_ver, *preboot_ver;
1152         u32 ver_off, limit;
1153         const char *name;
1154         char btl_ver[32];
1155
1156         static_loader = &hdev->fw_loader.static_loader;
1157
1158         switch (fwc) {
1159         case FW_COMP_BOOT_FIT:
1160                 ver_off = RREG32(static_loader->boot_fit_version_offset_reg);
1161                 dest = prop->uboot_ver;
1162                 name = "Boot-fit";
1163                 limit = static_loader->boot_fit_version_max_off;
1164                 break;
1165         case FW_COMP_PREBOOT:
1166                 ver_off = RREG32(static_loader->preboot_version_offset_reg);
1167                 dest = prop->preboot_ver;
1168                 name = "Preboot";
1169                 limit = static_loader->preboot_version_max_off;
1170                 break;
1171         default:
1172                 dev_warn(hdev->dev, "Undefined FW component: %d\n", fwc);
1173                 return -EIO;
1174         }
1175
1176         ver_off &= static_loader->sram_offset_mask;
1177
1178         if (ver_off < limit) {
1179                 memcpy_fromio(dest,
1180                         hdev->pcie_bar[fw_loader->sram_bar_id] + ver_off,
1181                         VERSION_MAX_LEN);
1182         } else {
1183                 dev_err(hdev->dev, "%s version offset (0x%x) is above SRAM\n",
1184                                                                 name, ver_off);
1185                 strscpy(dest, "unavailable", VERSION_MAX_LEN);
1186                 return -EIO;
1187         }
1188
1189         if (fwc == FW_COMP_BOOT_FIT) {
1190                 boot_ver = extract_fw_ver_from_str(prop->uboot_ver);
1191                 if (boot_ver) {
1192                         dev_info(hdev->dev, "boot-fit version %s\n", boot_ver);
1193                         kfree(boot_ver);
1194                 }
1195         } else if (fwc == FW_COMP_PREBOOT) {
1196                 preboot_ver = strnstr(prop->preboot_ver, "Preboot",
1197                                                 VERSION_MAX_LEN);
1198                 if (preboot_ver && preboot_ver != prop->preboot_ver) {
1199                         strscpy(btl_ver, prop->preboot_ver,
1200                                 min((int) (preboot_ver - prop->preboot_ver),
1201                                                                         31));
1202                         dev_info(hdev->dev, "%s\n", btl_ver);
1203                 }
1204
1205                 preboot_ver = extract_fw_ver_from_str(prop->preboot_ver);
1206                 if (preboot_ver) {
1207                         dev_info(hdev->dev, "preboot version %s\n",
1208                                                                 preboot_ver);
1209                         kfree(preboot_ver);
1210                 }
1211         }
1212
1213         return 0;
1214 }
1215
1216 /**
1217  * hl_fw_preboot_update_state - update internal data structures during
1218  *                              handshake with preboot
1219  *
1220  *
1221  * @hdev: pointer to the habanalabs device structure
1222  *
1223  * @return 0 on success, otherwise non-zero error code
1224  */
1225 static void hl_fw_preboot_update_state(struct hl_device *hdev)
1226 {
1227         struct asic_fixed_properties *prop = &hdev->asic_prop;
1228         u32 cpu_boot_dev_sts0, cpu_boot_dev_sts1;
1229
1230         cpu_boot_dev_sts0 = prop->fw_preboot_cpu_boot_dev_sts0;
1231         cpu_boot_dev_sts1 = prop->fw_preboot_cpu_boot_dev_sts1;
1232
1233         /* We read boot_dev_sts registers multiple times during boot:
1234          * 1. preboot - a. Check whether the security status bits are valid
1235          *              b. Check whether fw security is enabled
1236          *              c. Check whether hard reset is done by preboot
1237          * 2. boot cpu - a. Fetch boot cpu security status
1238          *               b. Check whether hard reset is done by boot cpu
1239          * 3. FW application - a. Fetch fw application security status
1240          *                     b. Check whether hard reset is done by fw app
1241          *
1242          * Preboot:
1243          * Check security status bit (CPU_BOOT_DEV_STS0_ENABLED). If set, then-
1244          * check security enabled bit (CPU_BOOT_DEV_STS0_SECURITY_EN)
1245          * If set, then mark GIC controller to be disabled.
1246          */
1247         prop->hard_reset_done_by_fw =
1248                 !!(cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_FW_HARD_RST_EN);
1249
1250         dev_dbg(hdev->dev, "Firmware preboot boot device status0 %#x\n",
1251                                                         cpu_boot_dev_sts0);
1252
1253         dev_dbg(hdev->dev, "Firmware preboot boot device status1 %#x\n",
1254                                                         cpu_boot_dev_sts1);
1255
1256         dev_dbg(hdev->dev, "Firmware preboot hard-reset is %s\n",
1257                         prop->hard_reset_done_by_fw ? "enabled" : "disabled");
1258
1259         dev_dbg(hdev->dev, "firmware-level security is %s\n",
1260                         prop->fw_security_enabled ? "enabled" : "disabled");
1261
1262         dev_dbg(hdev->dev, "GIC controller is %s\n",
1263                         prop->gic_interrupts_enable ? "enabled" : "disabled");
1264 }
1265
1266 static int hl_fw_static_read_preboot_status(struct hl_device *hdev)
1267 {
1268         int rc;
1269
1270         rc = hl_fw_static_read_device_fw_version(hdev, FW_COMP_PREBOOT);
1271         if (rc)
1272                 return rc;
1273
1274         return 0;
1275 }
1276
1277 int hl_fw_read_preboot_status(struct hl_device *hdev, u32 cpu_boot_status_reg,
1278                                 u32 sts_boot_dev_sts0_reg,
1279                                 u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg,
1280                                 u32 boot_err1_reg, u32 timeout)
1281 {
1282         int rc;
1283
1284         /* pldm was added for cases in which we use preboot on pldm and want
1285          * to load boot fit, but we can't wait for preboot because it runs
1286          * very slowly
1287          */
1288         if (!(hdev->fw_components & FW_TYPE_PREBOOT_CPU) || hdev->pldm)
1289                 return 0;
1290
1291         /*
1292          * In order to determine boot method (static VS dymanic) we need to
1293          * read the boot caps register
1294          */
1295         rc = hl_fw_read_preboot_caps(hdev, cpu_boot_status_reg,
1296                                         sts_boot_dev_sts0_reg,
1297                                         sts_boot_dev_sts1_reg, boot_err0_reg,
1298                                         boot_err1_reg, timeout);
1299         if (rc)
1300                 return rc;
1301
1302         hl_fw_preboot_update_state(hdev);
1303
1304         /* no need to read preboot status in dynamic load */
1305         if (hdev->asic_prop.dynamic_fw_load)
1306                 return 0;
1307
1308         return hl_fw_static_read_preboot_status(hdev);
1309 }
1310
1311 /* associate string with COMM status */
1312 static char *hl_dynamic_fw_status_str[COMMS_STS_INVLD_LAST] = {
1313         [COMMS_STS_NOOP] = "NOOP",
1314         [COMMS_STS_ACK] = "ACK",
1315         [COMMS_STS_OK] = "OK",
1316         [COMMS_STS_ERR] = "ERR",
1317         [COMMS_STS_VALID_ERR] = "VALID_ERR",
1318         [COMMS_STS_TIMEOUT_ERR] = "TIMEOUT_ERR",
1319 };
1320
1321 /**
1322  * hl_fw_dynamic_report_error_status - report error status
1323  *
1324  * @hdev: pointer to the habanalabs device structure
1325  * @status: value of FW status register
1326  * @expected_status: the expected status
1327  */
1328 static void hl_fw_dynamic_report_error_status(struct hl_device *hdev,
1329                                                 u32 status,
1330                                                 enum comms_sts expected_status)
1331 {
1332         enum comms_sts comm_status =
1333                                 FIELD_GET(COMMS_STATUS_STATUS_MASK, status);
1334
1335         if (comm_status < COMMS_STS_INVLD_LAST)
1336                 dev_err(hdev->dev, "Device status %s, expected status: %s\n",
1337                                 hl_dynamic_fw_status_str[comm_status],
1338                                 hl_dynamic_fw_status_str[expected_status]);
1339         else
1340                 dev_err(hdev->dev, "Device status unknown %d, expected status: %s\n",
1341                                 comm_status,
1342                                 hl_dynamic_fw_status_str[expected_status]);
1343 }
1344
1345 /**
1346  * hl_fw_dynamic_send_cmd - send LKD to FW cmd
1347  *
1348  * @hdev: pointer to the habanalabs device structure
1349  * @fw_loader: managing structure for loading device's FW
1350  * @cmd: LKD to FW cmd code
1351  * @size: size of next FW component to be loaded (0 if not necessary)
1352  *
1353  * LDK to FW exact command layout is defined at struct comms_command.
1354  * note: the size argument is used only when the next FW component should be
1355  *       loaded, otherwise it shall be 0. the size is used by the FW in later
1356  *       protocol stages and when sending only indicating the amount of memory
1357  *       to be allocated by the FW to receive the next boot component.
1358  */
1359 static void hl_fw_dynamic_send_cmd(struct hl_device *hdev,
1360                                 struct fw_load_mgr *fw_loader,
1361                                 enum comms_cmd cmd, unsigned int size)
1362 {
1363         struct cpu_dyn_regs *dyn_regs;
1364         u32 val;
1365
1366         dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs;
1367
1368         val = FIELD_PREP(COMMS_COMMAND_CMD_MASK, cmd);
1369         val |= FIELD_PREP(COMMS_COMMAND_SIZE_MASK, size);
1370
1371         WREG32(le32_to_cpu(dyn_regs->kmd_msg_to_cpu), val);
1372 }
1373
1374 /**
1375  * hl_fw_dynamic_extract_fw_response - update the FW response
1376  *
1377  * @hdev: pointer to the habanalabs device structure
1378  * @fw_loader: managing structure for loading device's FW
1379  * @response: FW response
1380  * @status: the status read from CPU status register
1381  *
1382  * @return 0 on success, otherwise non-zero error code
1383  */
1384 static int hl_fw_dynamic_extract_fw_response(struct hl_device *hdev,
1385                                                 struct fw_load_mgr *fw_loader,
1386                                                 struct fw_response *response,
1387                                                 u32 status)
1388 {
1389         response->status = FIELD_GET(COMMS_STATUS_STATUS_MASK, status);
1390         response->ram_offset = FIELD_GET(COMMS_STATUS_OFFSET_MASK, status) <<
1391                                                 COMMS_STATUS_OFFSET_ALIGN_SHIFT;
1392         response->ram_type = FIELD_GET(COMMS_STATUS_RAM_TYPE_MASK, status);
1393
1394         if ((response->ram_type != COMMS_SRAM) &&
1395                                         (response->ram_type != COMMS_DRAM)) {
1396                 dev_err(hdev->dev, "FW status: invalid RAM type %u\n",
1397                                                         response->ram_type);
1398                 return -EIO;
1399         }
1400
1401         return 0;
1402 }
1403
1404 /**
1405  * hl_fw_dynamic_wait_for_status - wait for status in dynamic FW load
1406  *
1407  * @hdev: pointer to the habanalabs device structure
1408  * @fw_loader: managing structure for loading device's FW
1409  * @expected_status: expected status to wait for
1410  * @timeout: timeout for status wait
1411  *
1412  * @return 0 on success, otherwise non-zero error code
1413  *
1414  * waiting for status from FW include polling the FW status register until
1415  * expected status is received or timeout occurs (whatever occurs first).
1416  */
1417 static int hl_fw_dynamic_wait_for_status(struct hl_device *hdev,
1418                                                 struct fw_load_mgr *fw_loader,
1419                                                 enum comms_sts expected_status,
1420                                                 u32 timeout)
1421 {
1422         struct cpu_dyn_regs *dyn_regs;
1423         u32 status;
1424         int rc;
1425
1426         dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs;
1427
1428         /* Wait for expected status */
1429         rc = hl_poll_timeout(
1430                 hdev,
1431                 le32_to_cpu(dyn_regs->cpu_cmd_status_to_host),
1432                 status,
1433                 FIELD_GET(COMMS_STATUS_STATUS_MASK, status) == expected_status,
1434                 FW_CPU_STATUS_POLL_INTERVAL_USEC,
1435                 timeout);
1436
1437         if (rc) {
1438                 hl_fw_dynamic_report_error_status(hdev, status,
1439                                                         expected_status);
1440                 return -EIO;
1441         }
1442
1443         /*
1444          * skip storing FW response for NOOP to preserve the actual desired
1445          * FW status
1446          */
1447         if (expected_status == COMMS_STS_NOOP)
1448                 return 0;
1449
1450         rc = hl_fw_dynamic_extract_fw_response(hdev, fw_loader,
1451                                         &fw_loader->dynamic_loader.response,
1452                                         status);
1453         return rc;
1454 }
1455
1456 /**
1457  * hl_fw_dynamic_send_clear_cmd - send clear command to FW
1458  *
1459  * @hdev: pointer to the habanalabs device structure
1460  * @fw_loader: managing structure for loading device's FW
1461  *
1462  * @return 0 on success, otherwise non-zero error code
1463  *
1464  * after command cycle between LKD to FW CPU (i.e. LKD got an expected status
1465  * from FW) we need to clear the CPU status register in order to avoid garbage
1466  * between command cycles.
1467  * This is done by sending clear command and polling the CPU to LKD status
1468  * register to hold the status NOOP
1469  */
1470 static int hl_fw_dynamic_send_clear_cmd(struct hl_device *hdev,
1471                                                 struct fw_load_mgr *fw_loader)
1472 {
1473         hl_fw_dynamic_send_cmd(hdev, fw_loader, COMMS_CLR_STS, 0);
1474
1475         return hl_fw_dynamic_wait_for_status(hdev, fw_loader, COMMS_STS_NOOP,
1476                                                         fw_loader->cpu_timeout);
1477 }
1478
1479 /**
1480  * hl_fw_dynamic_send_protocol_cmd - send LKD to FW cmd and wait for ACK
1481  *
1482  * @hdev: pointer to the habanalabs device structure
1483  * @fw_loader: managing structure for loading device's FW
1484  * @cmd: LKD to FW cmd code
1485  * @size: size of next FW component to be loaded (0 if not necessary)
1486  * @wait_ok: if true also wait for OK response from FW
1487  * @timeout: timeout for status wait
1488  *
1489  * @return 0 on success, otherwise non-zero error code
1490  *
1491  * brief:
1492  * when sending protocol command we have the following steps:
1493  * - send clear (clear command and verify clear status register)
1494  * - send the actual protocol command
1495  * - wait for ACK on the protocol command
1496  * - send clear
1497  * - send NOOP
1498  * if, in addition, the specific protocol command should wait for OK then:
1499  * - wait for OK
1500  * - send clear
1501  * - send NOOP
1502  *
1503  * NOTES:
1504  * send clear: this is necessary in order to clear the status register to avoid
1505  *             leftovers between command
1506  * NOOP command: necessary to avoid loop on the clear command by the FW
1507  */
1508 int hl_fw_dynamic_send_protocol_cmd(struct hl_device *hdev,
1509                                 struct fw_load_mgr *fw_loader,
1510                                 enum comms_cmd cmd, unsigned int size,
1511                                 bool wait_ok, u32 timeout)
1512 {
1513         int rc;
1514
1515         /* first send clear command to clean former commands */
1516         rc = hl_fw_dynamic_send_clear_cmd(hdev, fw_loader);
1517
1518         /* send the actual command */
1519         hl_fw_dynamic_send_cmd(hdev, fw_loader, cmd, size);
1520
1521         /* wait for ACK for the command */
1522         rc = hl_fw_dynamic_wait_for_status(hdev, fw_loader, COMMS_STS_ACK,
1523                                                                 timeout);
1524         if (rc)
1525                 return rc;
1526
1527         /* clear command to prepare for NOOP command */
1528         rc = hl_fw_dynamic_send_clear_cmd(hdev, fw_loader);
1529         if (rc)
1530                 return rc;
1531
1532         /* send the actual NOOP command */
1533         hl_fw_dynamic_send_cmd(hdev, fw_loader, COMMS_NOOP, 0);
1534
1535         if (!wait_ok)
1536                 return 0;
1537
1538         rc = hl_fw_dynamic_wait_for_status(hdev, fw_loader, COMMS_STS_OK,
1539                                                                 timeout);
1540         if (rc)
1541                 return rc;
1542
1543         /* clear command to prepare for NOOP command */
1544         rc = hl_fw_dynamic_send_clear_cmd(hdev, fw_loader);
1545         if (rc)
1546                 return rc;
1547
1548         /* send the actual NOOP command */
1549         hl_fw_dynamic_send_cmd(hdev, fw_loader, COMMS_NOOP, 0);
1550
1551         return 0;
1552 }
1553
1554 /**
1555  * hl_fw_compat_crc32 - CRC compatible with FW
1556  *
1557  * @data: pointer to the data
1558  * @size: size of the data
1559  *
1560  * @return the CRC32 result
1561  *
1562  * NOTE: kernel's CRC32 differ's from standard CRC32 calculation.
1563  *       in order to be aligned we need to flip the bits of both the input
1564  *       initial CRC and kernel's CRC32 result.
1565  *       in addition both sides use initial CRC of 0,
1566  */
1567 static u32 hl_fw_compat_crc32(u8 *data, size_t size)
1568 {
1569         return ~crc32_le(~((u32)0), data, size);
1570 }
1571
1572 /**
1573  * hl_fw_dynamic_validate_memory_bound - validate memory bounds for memory
1574  *                                        transfer (image or descriptor) between
1575  *                                        host and FW
1576  *
1577  * @hdev: pointer to the habanalabs device structure
1578  * @addr: device address of memory transfer
1579  * @size: memory transter size
1580  * @region: PCI memory region
1581  *
1582  * @return 0 on success, otherwise non-zero error code
1583  */
1584 static int hl_fw_dynamic_validate_memory_bound(struct hl_device *hdev,
1585                                                 u64 addr, size_t size,
1586                                                 struct pci_mem_region *region)
1587 {
1588         u64 end_addr;
1589
1590         /* now make sure that the memory transfer is within region's bounds */
1591         end_addr = addr + size;
1592         if (end_addr >= region->region_base + region->region_size) {
1593                 dev_err(hdev->dev,
1594                         "dynamic FW load: memory transfer end address out of memory region bounds. addr: %llx\n",
1595                                                         end_addr);
1596                 return -EIO;
1597         }
1598
1599         /*
1600          * now make sure memory transfer is within predefined BAR bounds.
1601          * this is to make sure we do not need to set the bar (e.g. for DRAM
1602          * memory transfers)
1603          */
1604         if (end_addr >= region->region_base - region->offset_in_bar +
1605                                                         region->bar_size) {
1606                 dev_err(hdev->dev,
1607                         "FW image beyond PCI BAR bounds\n");
1608                 return -EIO;
1609         }
1610
1611         return 0;
1612 }
1613
1614 /**
1615  * hl_fw_dynamic_validate_descriptor - validate FW descriptor
1616  *
1617  * @hdev: pointer to the habanalabs device structure
1618  * @fw_loader: managing structure for loading device's FW
1619  * @fw_desc: the descriptor form FW
1620  *
1621  * @return 0 on success, otherwise non-zero error code
1622  */
1623 static int hl_fw_dynamic_validate_descriptor(struct hl_device *hdev,
1624                                         struct fw_load_mgr *fw_loader,
1625                                         struct lkd_fw_comms_desc *fw_desc)
1626 {
1627         struct pci_mem_region *region;
1628         enum pci_region region_id;
1629         size_t data_size;
1630         u32 data_crc32;
1631         u8 *data_ptr;
1632         u64 addr;
1633         int rc;
1634
1635         if (le32_to_cpu(fw_desc->header.magic) != HL_COMMS_DESC_MAGIC) {
1636                 dev_err(hdev->dev, "Invalid magic for dynamic FW descriptor (%x)\n",
1637                                 fw_desc->header.magic);
1638                 return -EIO;
1639         }
1640
1641         if (fw_desc->header.version != HL_COMMS_DESC_VER) {
1642                 dev_err(hdev->dev, "Invalid version for dynamic FW descriptor (%x)\n",
1643                                 fw_desc->header.version);
1644                 return -EIO;
1645         }
1646
1647         /*
1648          * calc CRC32 of data without header.
1649          * note that no alignment/stride address issues here as all structures
1650          * are 64 bit padded
1651          */
1652         data_size = sizeof(struct lkd_fw_comms_desc) -
1653                                         sizeof(struct comms_desc_header);
1654         data_ptr = (u8 *)fw_desc + sizeof(struct comms_desc_header);
1655
1656         if (le16_to_cpu(fw_desc->header.size) != data_size) {
1657                 dev_err(hdev->dev,
1658                         "Invalid descriptor size 0x%x, expected size 0x%zx\n",
1659                                 le16_to_cpu(fw_desc->header.size), data_size);
1660                 return -EIO;
1661         }
1662
1663         data_crc32 = hl_fw_compat_crc32(data_ptr, data_size);
1664
1665         if (data_crc32 != le32_to_cpu(fw_desc->header.crc32)) {
1666                 dev_err(hdev->dev,
1667                         "CRC32 mismatch for dynamic FW descriptor (%x:%x)\n",
1668                                         data_crc32, fw_desc->header.crc32);
1669                 return -EIO;
1670         }
1671
1672         /* find memory region to which to copy the image */
1673         addr = le64_to_cpu(fw_desc->img_addr);
1674         region_id = hl_get_pci_memory_region(hdev, addr);
1675         if ((region_id != PCI_REGION_SRAM) &&
1676                         ((region_id != PCI_REGION_DRAM))) {
1677                 dev_err(hdev->dev,
1678                         "Invalid region to copy FW image address=%llx\n", addr);
1679                 return -EIO;
1680         }
1681
1682         region = &hdev->pci_mem_region[region_id];
1683
1684         /* store the region for the copy stage */
1685         fw_loader->dynamic_loader.image_region = region;
1686
1687         /*
1688          * here we know that the start address is valid, now make sure that the
1689          * image is within region's bounds
1690          */
1691         rc = hl_fw_dynamic_validate_memory_bound(hdev, addr,
1692                                         fw_loader->dynamic_loader.fw_image_size,
1693                                         region);
1694         if (rc) {
1695                 dev_err(hdev->dev,
1696                         "invalid mem transfer request for FW image\n");
1697                 return rc;
1698         }
1699
1700         return 0;
1701 }
1702
1703 static int hl_fw_dynamic_validate_response(struct hl_device *hdev,
1704                                                 struct fw_response *response,
1705                                                 struct pci_mem_region *region)
1706 {
1707         u64 device_addr;
1708         int rc;
1709
1710         device_addr = region->region_base + response->ram_offset;
1711
1712         /*
1713          * validate that the descriptor is within region's bounds
1714          * Note that as the start address was supplied according to the RAM
1715          * type- testing only the end address is enough
1716          */
1717         rc = hl_fw_dynamic_validate_memory_bound(hdev, device_addr,
1718                                         sizeof(struct lkd_fw_comms_desc),
1719                                         region);
1720         return rc;
1721 }
1722
1723 /**
1724  * hl_fw_dynamic_read_and_validate_descriptor - read and validate FW descriptor
1725  *
1726  * @hdev: pointer to the habanalabs device structure
1727  * @fw_loader: managing structure for loading device's FW
1728  *
1729  * @return 0 on success, otherwise non-zero error code
1730  */
1731 static int hl_fw_dynamic_read_and_validate_descriptor(struct hl_device *hdev,
1732                                                 struct fw_load_mgr *fw_loader)
1733 {
1734         struct lkd_fw_comms_desc *fw_desc;
1735         struct pci_mem_region *region;
1736         struct fw_response *response;
1737         enum pci_region region_id;
1738         void __iomem *src;
1739         int rc;
1740
1741         fw_desc = &fw_loader->dynamic_loader.comm_desc;
1742         response = &fw_loader->dynamic_loader.response;
1743
1744         region_id = (response->ram_type == COMMS_SRAM) ?
1745                                         PCI_REGION_SRAM : PCI_REGION_DRAM;
1746
1747         region = &hdev->pci_mem_region[region_id];
1748
1749         rc = hl_fw_dynamic_validate_response(hdev, response, region);
1750         if (rc) {
1751                 dev_err(hdev->dev,
1752                         "invalid mem transfer request for FW descriptor\n");
1753                 return rc;
1754         }
1755
1756         /* extract address copy the descriptor from */
1757         src = hdev->pcie_bar[region->bar_id] + region->offset_in_bar +
1758                                                         response->ram_offset;
1759         memcpy_fromio(fw_desc, src, sizeof(struct lkd_fw_comms_desc));
1760
1761         return hl_fw_dynamic_validate_descriptor(hdev, fw_loader, fw_desc);
1762 }
1763
1764 /**
1765  * hl_fw_dynamic_request_descriptor - handshake with CPU to get FW descriptor
1766  *
1767  * @hdev: pointer to the habanalabs device structure
1768  * @fw_loader: managing structure for loading device's FW
1769  * @next_image_size: size to allocate for next FW component
1770  *
1771  * @return 0 on success, otherwise non-zero error code
1772  */
1773 static int hl_fw_dynamic_request_descriptor(struct hl_device *hdev,
1774                                                 struct fw_load_mgr *fw_loader,
1775                                                 size_t next_image_size)
1776 {
1777         int rc;
1778
1779         rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_PREP_DESC,
1780                                                 next_image_size, true,
1781                                                 fw_loader->cpu_timeout);
1782         if (rc)
1783                 return rc;
1784
1785         return hl_fw_dynamic_read_and_validate_descriptor(hdev, fw_loader);
1786 }
1787
1788 /**
1789  * hl_fw_dynamic_read_device_fw_version - read FW version to exposed properties
1790  *
1791  * @hdev: pointer to the habanalabs device structure
1792  * @fwc: the firmware component
1793  * @fw_version: fw component's version string
1794  */
1795 static void hl_fw_dynamic_read_device_fw_version(struct hl_device *hdev,
1796                                         enum hl_fw_component fwc,
1797                                         const char *fw_version)
1798 {
1799         struct asic_fixed_properties *prop = &hdev->asic_prop;
1800         char *preboot_ver, *boot_ver;
1801         char btl_ver[32];
1802
1803         switch (fwc) {
1804         case FW_COMP_BOOT_FIT:
1805                 strscpy(prop->uboot_ver, fw_version, VERSION_MAX_LEN);
1806                 boot_ver = extract_fw_ver_from_str(prop->uboot_ver);
1807                 if (boot_ver) {
1808                         dev_info(hdev->dev, "boot-fit version %s\n", boot_ver);
1809                         kfree(boot_ver);
1810                 }
1811
1812                 break;
1813         case FW_COMP_PREBOOT:
1814                 strscpy(prop->preboot_ver, fw_version, VERSION_MAX_LEN);
1815                 preboot_ver = strnstr(prop->preboot_ver, "Preboot",
1816                                                 VERSION_MAX_LEN);
1817                 if (preboot_ver && preboot_ver != prop->preboot_ver) {
1818                         strscpy(btl_ver, prop->preboot_ver,
1819                                 min((int) (preboot_ver - prop->preboot_ver),
1820                                                                         31));
1821                         dev_info(hdev->dev, "%s\n", btl_ver);
1822                 }
1823
1824                 preboot_ver = extract_fw_ver_from_str(prop->preboot_ver);
1825                 if (preboot_ver) {
1826                         dev_info(hdev->dev, "preboot version %s\n",
1827                                                                 preboot_ver);
1828                         kfree(preboot_ver);
1829                 }
1830
1831                 break;
1832         default:
1833                 dev_warn(hdev->dev, "Undefined FW component: %d\n", fwc);
1834                 return;
1835         }
1836 }
1837
1838 /**
1839  * hl_fw_dynamic_copy_image - copy image to memory allocated by the FW
1840  *
1841  * @hdev: pointer to the habanalabs device structure
1842  * @fw: fw descriptor
1843  * @fw_loader: managing structure for loading device's FW
1844  */
1845 static int hl_fw_dynamic_copy_image(struct hl_device *hdev,
1846                                                 const struct firmware *fw,
1847                                                 struct fw_load_mgr *fw_loader)
1848 {
1849         struct lkd_fw_comms_desc *fw_desc;
1850         struct pci_mem_region *region;
1851         void __iomem *dest;
1852         u64 addr;
1853         int rc;
1854
1855         fw_desc = &fw_loader->dynamic_loader.comm_desc;
1856         addr = le64_to_cpu(fw_desc->img_addr);
1857
1858         /* find memory region to which to copy the image */
1859         region = fw_loader->dynamic_loader.image_region;
1860
1861         dest = hdev->pcie_bar[region->bar_id] + region->offset_in_bar +
1862                                         (addr - region->region_base);
1863
1864         rc = hl_fw_copy_fw_to_device(hdev, fw, dest,
1865                                         fw_loader->boot_fit_img.src_off,
1866                                         fw_loader->boot_fit_img.copy_size);
1867
1868         return rc;
1869 }
1870
1871 /**
1872  * hl_fw_dynamic_copy_msg - copy msg to memory allocated by the FW
1873  *
1874  * @hdev: pointer to the habanalabs device structure
1875  * @msg: message
1876  * @fw_loader: managing structure for loading device's FW
1877  */
1878 static int hl_fw_dynamic_copy_msg(struct hl_device *hdev,
1879                 struct lkd_msg_comms *msg, struct fw_load_mgr *fw_loader)
1880 {
1881         struct lkd_fw_comms_desc *fw_desc;
1882         struct pci_mem_region *region;
1883         void __iomem *dest;
1884         u64 addr;
1885         int rc;
1886
1887         fw_desc = &fw_loader->dynamic_loader.comm_desc;
1888         addr = le64_to_cpu(fw_desc->img_addr);
1889
1890         /* find memory region to which to copy the image */
1891         region = fw_loader->dynamic_loader.image_region;
1892
1893         dest = hdev->pcie_bar[region->bar_id] + region->offset_in_bar +
1894                                         (addr - region->region_base);
1895
1896         rc = hl_fw_copy_msg_to_device(hdev, msg, dest, 0, 0);
1897
1898         return rc;
1899 }
1900
1901 /**
1902  * hl_fw_boot_fit_update_state - update internal data structures after boot-fit
1903  *                               is loaded
1904  *
1905  * @hdev: pointer to the habanalabs device structure
1906  * @cpu_boot_dev_sts0_reg: register holding CPU boot dev status 0
1907  * @cpu_boot_dev_sts1_reg: register holding CPU boot dev status 1
1908  *
1909  * @return 0 on success, otherwise non-zero error code
1910  */
1911 static void hl_fw_boot_fit_update_state(struct hl_device *hdev,
1912                                                 u32 cpu_boot_dev_sts0_reg,
1913                                                 u32 cpu_boot_dev_sts1_reg)
1914 {
1915         struct asic_fixed_properties *prop = &hdev->asic_prop;
1916
1917         /* Clear reset status since we need to read it again from boot CPU */
1918         prop->hard_reset_done_by_fw = false;
1919
1920         /* Read boot_cpu status bits */
1921         if (prop->fw_preboot_cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_ENABLED) {
1922                 prop->fw_bootfit_cpu_boot_dev_sts0 =
1923                                 RREG32(cpu_boot_dev_sts0_reg);
1924
1925                 if (prop->fw_bootfit_cpu_boot_dev_sts0 &
1926                                 CPU_BOOT_DEV_STS0_FW_HARD_RST_EN)
1927                         prop->hard_reset_done_by_fw = true;
1928
1929                 dev_dbg(hdev->dev, "Firmware boot CPU status0 %#x\n",
1930                                         prop->fw_bootfit_cpu_boot_dev_sts0);
1931         }
1932
1933         if (prop->fw_cpu_boot_dev_sts1_valid) {
1934                 prop->fw_bootfit_cpu_boot_dev_sts1 =
1935                                 RREG32(cpu_boot_dev_sts1_reg);
1936
1937                 dev_dbg(hdev->dev, "Firmware boot CPU status1 %#x\n",
1938                                         prop->fw_bootfit_cpu_boot_dev_sts1);
1939         }
1940
1941         dev_dbg(hdev->dev, "Firmware boot CPU hard-reset is %s\n",
1942                         prop->hard_reset_done_by_fw ? "enabled" : "disabled");
1943 }
1944
1945 static void hl_fw_dynamic_update_linux_interrupt_if(struct hl_device *hdev)
1946 {
1947         struct cpu_dyn_regs *dyn_regs =
1948                         &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
1949
1950         /* Check whether all 3 interrupt interfaces are set, if not use a
1951          * single interface
1952          */
1953         if (!hdev->asic_prop.gic_interrupts_enable &&
1954                         !(hdev->asic_prop.fw_app_cpu_boot_dev_sts0 &
1955                                 CPU_BOOT_DEV_STS0_MULTI_IRQ_POLL_EN)) {
1956                 dyn_regs->gic_host_halt_irq = dyn_regs->gic_host_irq_ctrl;
1957                 dyn_regs->gic_host_ints_irq = dyn_regs->gic_host_irq_ctrl;
1958
1959                 dev_warn(hdev->dev,
1960                         "Using a single interrupt interface towards cpucp");
1961         }
1962 }
1963 /**
1964  * hl_fw_dynamic_load_image - load FW image using dynamic protocol
1965  *
1966  * @hdev: pointer to the habanalabs device structure
1967  * @fw_loader: managing structure for loading device's FW
1968  * @load_fwc: the FW component to be loaded
1969  * @img_ld_timeout: image load timeout
1970  *
1971  * @return 0 on success, otherwise non-zero error code
1972  */
1973 static int hl_fw_dynamic_load_image(struct hl_device *hdev,
1974                                                 struct fw_load_mgr *fw_loader,
1975                                                 enum hl_fw_component load_fwc,
1976                                                 u32 img_ld_timeout)
1977 {
1978         enum hl_fw_component cur_fwc;
1979         const struct firmware *fw;
1980         char *fw_name;
1981         int rc = 0;
1982
1983         /*
1984          * when loading image we have one of 2 scenarios:
1985          * 1. current FW component is preboot and we want to load boot-fit
1986          * 2. current FW component is boot-fit and we want to load linux
1987          */
1988         if (load_fwc == FW_COMP_BOOT_FIT) {
1989                 cur_fwc = FW_COMP_PREBOOT;
1990                 fw_name = fw_loader->boot_fit_img.image_name;
1991         } else {
1992                 cur_fwc = FW_COMP_BOOT_FIT;
1993                 fw_name = fw_loader->linux_img.image_name;
1994         }
1995
1996         /* request FW in order to communicate to FW the size to be allocated */
1997         rc = hl_request_fw(hdev, &fw, fw_name);
1998         if (rc)
1999                 return rc;
2000
2001         /* store the image size for future validation */
2002         fw_loader->dynamic_loader.fw_image_size = fw->size;
2003
2004         rc = hl_fw_dynamic_request_descriptor(hdev, fw_loader, fw->size);
2005         if (rc)
2006                 goto release_fw;
2007
2008         /* read preboot version */
2009         hl_fw_dynamic_read_device_fw_version(hdev, cur_fwc,
2010                                 fw_loader->dynamic_loader.comm_desc.cur_fw_ver);
2011
2012
2013         /* update state according to boot stage */
2014         if (cur_fwc == FW_COMP_BOOT_FIT) {
2015                 struct cpu_dyn_regs *dyn_regs;
2016
2017                 dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs;
2018                 hl_fw_boot_fit_update_state(hdev,
2019                                 le32_to_cpu(dyn_regs->cpu_boot_dev_sts0),
2020                                 le32_to_cpu(dyn_regs->cpu_boot_dev_sts1));
2021         }
2022
2023         /* copy boot fit to space allocated by FW */
2024         rc = hl_fw_dynamic_copy_image(hdev, fw, fw_loader);
2025         if (rc)
2026                 goto release_fw;
2027
2028         rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_DATA_RDY,
2029                                                 0, true,
2030                                                 fw_loader->cpu_timeout);
2031         if (rc)
2032                 goto release_fw;
2033
2034         rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_EXEC,
2035                                                 0, false,
2036                                                 img_ld_timeout);
2037
2038 release_fw:
2039         hl_release_firmware(fw);
2040         return rc;
2041 }
2042
2043 static int hl_fw_dynamic_wait_for_boot_fit_active(struct hl_device *hdev,
2044                                         struct fw_load_mgr *fw_loader)
2045 {
2046         struct dynamic_fw_load_mgr *dyn_loader;
2047         u32 status;
2048         int rc;
2049
2050         dyn_loader = &fw_loader->dynamic_loader;
2051
2052         /* Make sure CPU boot-loader is running */
2053         rc = hl_poll_timeout(
2054                 hdev,
2055                 le32_to_cpu(dyn_loader->comm_desc.cpu_dyn_regs.cpu_boot_status),
2056                 status,
2057                 (status == CPU_BOOT_STATUS_NIC_FW_RDY) ||
2058                 (status == CPU_BOOT_STATUS_READY_TO_BOOT),
2059                 FW_CPU_STATUS_POLL_INTERVAL_USEC,
2060                 dyn_loader->wait_for_bl_timeout);
2061         if (rc) {
2062                 dev_err(hdev->dev, "failed to wait for boot\n");
2063                 return rc;
2064         }
2065
2066         dev_dbg(hdev->dev, "uboot status = %d\n", status);
2067         return 0;
2068 }
2069
2070 static int hl_fw_dynamic_wait_for_linux_active(struct hl_device *hdev,
2071                                                 struct fw_load_mgr *fw_loader)
2072 {
2073         struct dynamic_fw_load_mgr *dyn_loader;
2074         u32 status;
2075         int rc;
2076
2077         dyn_loader = &fw_loader->dynamic_loader;
2078
2079         /* Make sure CPU boot-loader is running */
2080
2081         rc = hl_poll_timeout(
2082                 hdev,
2083                 le32_to_cpu(dyn_loader->comm_desc.cpu_dyn_regs.cpu_boot_status),
2084                 status,
2085                 (status == CPU_BOOT_STATUS_SRAM_AVAIL),
2086                 FW_CPU_STATUS_POLL_INTERVAL_USEC,
2087                 fw_loader->cpu_timeout);
2088         if (rc) {
2089                 dev_err(hdev->dev, "failed to wait for Linux\n");
2090                 return rc;
2091         }
2092
2093         dev_dbg(hdev->dev, "Boot status = %d\n", status);
2094         return 0;
2095 }
2096
2097 /**
2098  * hl_fw_linux_update_state -   update internal data structures after Linux
2099  *                              is loaded.
2100  *                              Note: Linux initialization is comprised mainly
2101  *                              of two stages - loading kernel (SRAM_AVAIL)
2102  *                              & loading ARMCP.
2103  *                              Therefore reading boot device status in any of
2104  *                              these stages might result in different values.
2105  *
2106  * @hdev: pointer to the habanalabs device structure
2107  * @cpu_boot_dev_sts0_reg: register holding CPU boot dev status 0
2108  * @cpu_boot_dev_sts1_reg: register holding CPU boot dev status 1
2109  *
2110  * @return 0 on success, otherwise non-zero error code
2111  */
2112 static void hl_fw_linux_update_state(struct hl_device *hdev,
2113                                                 u32 cpu_boot_dev_sts0_reg,
2114                                                 u32 cpu_boot_dev_sts1_reg)
2115 {
2116         struct asic_fixed_properties *prop = &hdev->asic_prop;
2117
2118         hdev->fw_loader.linux_loaded = true;
2119
2120         /* Clear reset status since we need to read again from app */
2121         prop->hard_reset_done_by_fw = false;
2122
2123         /* Read FW application security bits */
2124         if (prop->fw_cpu_boot_dev_sts0_valid) {
2125                 prop->fw_app_cpu_boot_dev_sts0 =
2126                                 RREG32(cpu_boot_dev_sts0_reg);
2127
2128                 if (prop->fw_app_cpu_boot_dev_sts0 &
2129                                 CPU_BOOT_DEV_STS0_FW_HARD_RST_EN)
2130                         prop->hard_reset_done_by_fw = true;
2131
2132                 if (prop->fw_app_cpu_boot_dev_sts0 &
2133                                 CPU_BOOT_DEV_STS0_GIC_PRIVILEGED_EN)
2134                         prop->gic_interrupts_enable = false;
2135
2136                 dev_dbg(hdev->dev,
2137                         "Firmware application CPU status0 %#x\n",
2138                         prop->fw_app_cpu_boot_dev_sts0);
2139
2140                 dev_dbg(hdev->dev, "GIC controller is %s\n",
2141                                 prop->gic_interrupts_enable ?
2142                                                 "enabled" : "disabled");
2143         }
2144
2145         if (prop->fw_cpu_boot_dev_sts1_valid) {
2146                 prop->fw_app_cpu_boot_dev_sts1 =
2147                                 RREG32(cpu_boot_dev_sts1_reg);
2148
2149                 dev_dbg(hdev->dev,
2150                         "Firmware application CPU status1 %#x\n",
2151                         prop->fw_app_cpu_boot_dev_sts1);
2152         }
2153
2154         dev_dbg(hdev->dev, "Firmware application CPU hard-reset is %s\n",
2155                         prop->hard_reset_done_by_fw ? "enabled" : "disabled");
2156
2157         dev_info(hdev->dev, "Successfully loaded firmware to device\n");
2158 }
2159
2160 /**
2161  * hl_fw_dynamic_report_reset_cause - send a COMMS message with the cause
2162  *                                    of the newly triggered hard reset
2163  *
2164  * @hdev: pointer to the habanalabs device structure
2165  * @fw_loader: managing structure for loading device's FW
2166  * @reset_cause: enumerated cause for the recent hard reset
2167  *
2168  * @return 0 on success, otherwise non-zero error code
2169  */
2170 static int hl_fw_dynamic_report_reset_cause(struct hl_device *hdev,
2171                 struct fw_load_mgr *fw_loader,
2172                 enum comms_reset_cause reset_cause)
2173 {
2174         struct lkd_msg_comms msg;
2175         int rc;
2176
2177         memset(&msg, 0, sizeof(msg));
2178
2179         /* create message to be sent */
2180         msg.header.type = HL_COMMS_RESET_CAUSE_TYPE;
2181         msg.header.size = cpu_to_le16(sizeof(struct comms_msg_header));
2182         msg.header.magic = cpu_to_le32(HL_COMMS_MSG_MAGIC);
2183
2184         msg.reset_cause = reset_cause;
2185
2186         rc = hl_fw_dynamic_request_descriptor(hdev, fw_loader,
2187                         sizeof(struct lkd_msg_comms));
2188         if (rc)
2189                 return rc;
2190
2191         /* copy message to space allocated by FW */
2192         rc = hl_fw_dynamic_copy_msg(hdev, &msg, fw_loader);
2193         if (rc)
2194                 return rc;
2195
2196         rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_DATA_RDY,
2197                                                 0, true,
2198                                                 fw_loader->cpu_timeout);
2199         if (rc)
2200                 return rc;
2201
2202         rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_EXEC,
2203                                                 0, true,
2204                                                 fw_loader->cpu_timeout);
2205         if (rc)
2206                 return rc;
2207
2208         return 0;
2209 }
2210
2211 /**
2212  * hl_fw_dynamic_init_cpu - initialize the device CPU using dynamic protocol
2213  *
2214  * @hdev: pointer to the habanalabs device structure
2215  * @fw_loader: managing structure for loading device's FW
2216  *
2217  * @return 0 on success, otherwise non-zero error code
2218  *
2219  * brief: the dynamic protocol is master (LKD) slave (FW CPU) protocol.
2220  * the communication is done using registers:
2221  * - LKD command register
2222  * - FW status register
2223  * the protocol is race free. this goal is achieved by splitting the requests
2224  * and response to known synchronization points between the LKD and the FW.
2225  * each response to LKD request is known and bound to a predefined timeout.
2226  * in case of timeout expiration without the desired status from FW- the
2227  * protocol (and hence the boot) will fail.
2228  */
2229 static int hl_fw_dynamic_init_cpu(struct hl_device *hdev,
2230                                         struct fw_load_mgr *fw_loader)
2231 {
2232         struct cpu_dyn_regs *dyn_regs;
2233         int rc;
2234
2235         dev_info(hdev->dev,
2236                 "Loading firmware to device, may take some time...\n");
2237
2238         dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs;
2239
2240         rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_RST_STATE,
2241                                                 0, true,
2242                                                 fw_loader->cpu_timeout);
2243         if (rc)
2244                 goto protocol_err;
2245
2246         if (hdev->curr_reset_cause) {
2247                 rc = hl_fw_dynamic_report_reset_cause(hdev, fw_loader,
2248                                 hdev->curr_reset_cause);
2249                 if (rc)
2250                         goto protocol_err;
2251
2252                 /* Clear current reset cause */
2253                 hdev->curr_reset_cause = HL_RESET_CAUSE_UNKNOWN;
2254         }
2255
2256         if (!(hdev->fw_components & FW_TYPE_BOOT_CPU)) {
2257                 rc = hl_fw_dynamic_request_descriptor(hdev, fw_loader, 0);
2258                 if (rc)
2259                         goto protocol_err;
2260
2261                 /* read preboot version */
2262                 hl_fw_dynamic_read_device_fw_version(hdev, FW_COMP_PREBOOT,
2263                                 fw_loader->dynamic_loader.comm_desc.cur_fw_ver);
2264                 return 0;
2265         }
2266
2267         /* load boot fit to FW */
2268         rc = hl_fw_dynamic_load_image(hdev, fw_loader, FW_COMP_BOOT_FIT,
2269                                                 fw_loader->boot_fit_timeout);
2270         if (rc) {
2271                 dev_err(hdev->dev, "failed to load boot fit\n");
2272                 goto protocol_err;
2273         }
2274
2275         rc = hl_fw_dynamic_wait_for_boot_fit_active(hdev, fw_loader);
2276         if (rc)
2277                 goto protocol_err;
2278
2279         /* Enable DRAM scrambling before Linux boot and after successful
2280          *  UBoot
2281          */
2282         hdev->asic_funcs->init_cpu_scrambler_dram(hdev);
2283
2284         if (!(hdev->fw_components & FW_TYPE_LINUX)) {
2285                 dev_info(hdev->dev, "Skip loading Linux F/W\n");
2286                 return 0;
2287         }
2288
2289         if (fw_loader->skip_bmc) {
2290                 rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader,
2291                                                         COMMS_SKIP_BMC, 0,
2292                                                         true,
2293                                                         fw_loader->cpu_timeout);
2294                 if (rc) {
2295                         dev_err(hdev->dev, "failed to load boot fit\n");
2296                         goto protocol_err;
2297                 }
2298         }
2299
2300         /* load Linux image to FW */
2301         rc = hl_fw_dynamic_load_image(hdev, fw_loader, FW_COMP_LINUX,
2302                                                         fw_loader->cpu_timeout);
2303         if (rc) {
2304                 dev_err(hdev->dev, "failed to load Linux\n");
2305                 goto protocol_err;
2306         }
2307
2308         rc = hl_fw_dynamic_wait_for_linux_active(hdev, fw_loader);
2309         if (rc)
2310                 goto protocol_err;
2311
2312         hl_fw_linux_update_state(hdev, le32_to_cpu(dyn_regs->cpu_boot_dev_sts0),
2313                                 le32_to_cpu(dyn_regs->cpu_boot_dev_sts1));
2314
2315         hl_fw_dynamic_update_linux_interrupt_if(hdev);
2316
2317         return 0;
2318
2319 protocol_err:
2320         fw_read_errors(hdev, le32_to_cpu(dyn_regs->cpu_boot_err0),
2321                                 le32_to_cpu(dyn_regs->cpu_boot_err1),
2322                                 le32_to_cpu(dyn_regs->cpu_boot_dev_sts0),
2323                                 le32_to_cpu(dyn_regs->cpu_boot_dev_sts1));
2324         return rc;
2325 }
2326
2327 /**
2328  * hl_fw_static_init_cpu - initialize the device CPU using static protocol
2329  *
2330  * @hdev: pointer to the habanalabs device structure
2331  * @fw_loader: managing structure for loading device's FW
2332  *
2333  * @return 0 on success, otherwise non-zero error code
2334  */
2335 static int hl_fw_static_init_cpu(struct hl_device *hdev,
2336                                         struct fw_load_mgr *fw_loader)
2337 {
2338         u32 cpu_msg_status_reg, cpu_timeout, msg_to_cpu_reg, status;
2339         u32 cpu_boot_dev_status0_reg, cpu_boot_dev_status1_reg;
2340         struct static_fw_load_mgr *static_loader;
2341         u32 cpu_boot_status_reg;
2342         int rc;
2343
2344         if (!(hdev->fw_components & FW_TYPE_BOOT_CPU))
2345                 return 0;
2346
2347         /* init common loader parameters */
2348         cpu_timeout = fw_loader->cpu_timeout;
2349
2350         /* init static loader parameters */
2351         static_loader = &fw_loader->static_loader;
2352         cpu_msg_status_reg = static_loader->cpu_cmd_status_to_host_reg;
2353         msg_to_cpu_reg = static_loader->kmd_msg_to_cpu_reg;
2354         cpu_boot_dev_status0_reg = static_loader->cpu_boot_dev_status0_reg;
2355         cpu_boot_dev_status1_reg = static_loader->cpu_boot_dev_status1_reg;
2356         cpu_boot_status_reg = static_loader->cpu_boot_status_reg;
2357
2358         dev_info(hdev->dev, "Going to wait for device boot (up to %lds)\n",
2359                 cpu_timeout / USEC_PER_SEC);
2360
2361         /* Wait for boot FIT request */
2362         rc = hl_poll_timeout(
2363                 hdev,
2364                 cpu_boot_status_reg,
2365                 status,
2366                 status == CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT,
2367                 FW_CPU_STATUS_POLL_INTERVAL_USEC,
2368                 fw_loader->boot_fit_timeout);
2369
2370         if (rc) {
2371                 dev_dbg(hdev->dev,
2372                         "No boot fit request received, resuming boot\n");
2373         } else {
2374                 rc = hdev->asic_funcs->load_boot_fit_to_device(hdev);
2375                 if (rc)
2376                         goto out;
2377
2378                 /* Clear device CPU message status */
2379                 WREG32(cpu_msg_status_reg, CPU_MSG_CLR);
2380
2381                 /* Signal device CPU that boot loader is ready */
2382                 WREG32(msg_to_cpu_reg, KMD_MSG_FIT_RDY);
2383
2384                 /* Poll for CPU device ack */
2385                 rc = hl_poll_timeout(
2386                         hdev,
2387                         cpu_msg_status_reg,
2388                         status,
2389                         status == CPU_MSG_OK,
2390                         FW_CPU_STATUS_POLL_INTERVAL_USEC,
2391                         fw_loader->boot_fit_timeout);
2392
2393                 if (rc) {
2394                         dev_err(hdev->dev,
2395                                 "Timeout waiting for boot fit load ack\n");
2396                         goto out;
2397                 }
2398
2399                 /* Clear message */
2400                 WREG32(msg_to_cpu_reg, KMD_MSG_NA);
2401         }
2402
2403         /* Make sure CPU boot-loader is running */
2404         rc = hl_poll_timeout(
2405                 hdev,
2406                 cpu_boot_status_reg,
2407                 status,
2408                 (status == CPU_BOOT_STATUS_DRAM_RDY) ||
2409                 (status == CPU_BOOT_STATUS_NIC_FW_RDY) ||
2410                 (status == CPU_BOOT_STATUS_READY_TO_BOOT) ||
2411                 (status == CPU_BOOT_STATUS_SRAM_AVAIL),
2412                 FW_CPU_STATUS_POLL_INTERVAL_USEC,
2413                 cpu_timeout);
2414
2415         dev_dbg(hdev->dev, "uboot status = %d\n", status);
2416
2417         /* Read U-Boot version now in case we will later fail */
2418         hl_fw_static_read_device_fw_version(hdev, FW_COMP_BOOT_FIT);
2419
2420         /* update state according to boot stage */
2421         hl_fw_boot_fit_update_state(hdev, cpu_boot_dev_status0_reg,
2422                                                 cpu_boot_dev_status1_reg);
2423
2424         if (rc) {
2425                 detect_cpu_boot_status(hdev, status);
2426                 rc = -EIO;
2427                 goto out;
2428         }
2429
2430         /* Enable DRAM scrambling before Linux boot and after successful
2431          *  UBoot
2432          */
2433         hdev->asic_funcs->init_cpu_scrambler_dram(hdev);
2434
2435         if (!(hdev->fw_components & FW_TYPE_LINUX)) {
2436                 dev_info(hdev->dev, "Skip loading Linux F/W\n");
2437                 rc = 0;
2438                 goto out;
2439         }
2440
2441         if (status == CPU_BOOT_STATUS_SRAM_AVAIL) {
2442                 rc = 0;
2443                 goto out;
2444         }
2445
2446         dev_info(hdev->dev,
2447                 "Loading firmware to device, may take some time...\n");
2448
2449         rc = hdev->asic_funcs->load_firmware_to_device(hdev);
2450         if (rc)
2451                 goto out;
2452
2453         if (fw_loader->skip_bmc) {
2454                 WREG32(msg_to_cpu_reg, KMD_MSG_SKIP_BMC);
2455
2456                 rc = hl_poll_timeout(
2457                         hdev,
2458                         cpu_boot_status_reg,
2459                         status,
2460                         (status == CPU_BOOT_STATUS_BMC_WAITING_SKIPPED),
2461                         FW_CPU_STATUS_POLL_INTERVAL_USEC,
2462                         cpu_timeout);
2463
2464                 if (rc) {
2465                         dev_err(hdev->dev,
2466                                 "Failed to get ACK on skipping BMC, %d\n",
2467                                 status);
2468                         WREG32(msg_to_cpu_reg, KMD_MSG_NA);
2469                         rc = -EIO;
2470                         goto out;
2471                 }
2472         }
2473
2474         WREG32(msg_to_cpu_reg, KMD_MSG_FIT_RDY);
2475
2476         rc = hl_poll_timeout(
2477                 hdev,
2478                 cpu_boot_status_reg,
2479                 status,
2480                 (status == CPU_BOOT_STATUS_SRAM_AVAIL),
2481                 FW_CPU_STATUS_POLL_INTERVAL_USEC,
2482                 cpu_timeout);
2483
2484         /* Clear message */
2485         WREG32(msg_to_cpu_reg, KMD_MSG_NA);
2486
2487         if (rc) {
2488                 if (status == CPU_BOOT_STATUS_FIT_CORRUPTED)
2489                         dev_err(hdev->dev,
2490                                 "Device reports FIT image is corrupted\n");
2491                 else
2492                         dev_err(hdev->dev,
2493                                 "Failed to load firmware to device, %d\n",
2494                                 status);
2495
2496                 rc = -EIO;
2497                 goto out;
2498         }
2499
2500         rc = fw_read_errors(hdev, fw_loader->static_loader.boot_err0_reg,
2501                                         fw_loader->static_loader.boot_err1_reg,
2502                                         cpu_boot_dev_status0_reg,
2503                                         cpu_boot_dev_status1_reg);
2504         if (rc)
2505                 return rc;
2506
2507         hl_fw_linux_update_state(hdev, cpu_boot_dev_status0_reg,
2508                                                 cpu_boot_dev_status1_reg);
2509
2510         return 0;
2511
2512 out:
2513         fw_read_errors(hdev, fw_loader->static_loader.boot_err0_reg,
2514                                         fw_loader->static_loader.boot_err1_reg,
2515                                         cpu_boot_dev_status0_reg,
2516                                         cpu_boot_dev_status1_reg);
2517
2518         return rc;
2519 }
2520
2521 /**
2522  * hl_fw_init_cpu - initialize the device CPU
2523  *
2524  * @hdev: pointer to the habanalabs device structure
2525  *
2526  * @return 0 on success, otherwise non-zero error code
2527  *
2528  * perform necessary initializations for device's CPU. takes into account if
2529  * init protocol is static or dynamic.
2530  */
2531 int hl_fw_init_cpu(struct hl_device *hdev)
2532 {
2533         struct asic_fixed_properties *prop = &hdev->asic_prop;
2534         struct fw_load_mgr *fw_loader = &hdev->fw_loader;
2535
2536         return  prop->dynamic_fw_load ?
2537                         hl_fw_dynamic_init_cpu(hdev, fw_loader) :
2538                         hl_fw_static_init_cpu(hdev, fw_loader);
2539 }