drivers/misc/habanalabs/common/firmware_if.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 /*
   4  * Copyright 2016-2022 HabanaLabs, Ltd.
   5  * All Rights Reserved.
   6  */
   7
   8 #include "habanalabs.h"
   9 #include "../include/common/hl_boot_if.h"
  10
  11 #include <linux/firmware.h>
  12 #include <linux/crc32.h>
  13 #include <linux/slab.h>
  14 #include <linux/ctype.h>
  15
  16 #define FW_FILE_MAX_SIZE                0x1400000 /* maximum size of 20MB */
  17
  18 static char *extract_fw_ver_from_str(const char *fw_str)
  19 {
  20         char *str, *fw_ver, *whitespace;
  21
  22         fw_ver = kmalloc(16, GFP_KERNEL);
  23         if (!fw_ver)
  24                 return NULL;
  25
  26         str = strnstr(fw_str, "fw-", VERSION_MAX_LEN);
  27         if (!str)
  28                 goto free_fw_ver;
  29
  30         /* Skip the fw- part */
  31         str += 3;
  32
  33         /* Copy until the next whitespace */
  34         whitespace =  strnstr(str, " ", 15);
  35         if (!whitespace)
  36                 goto free_fw_ver;
  37
  38         strscpy(fw_ver, str, whitespace - str + 1);
  39
  40         return fw_ver;
  41
  42 free_fw_ver:
  43         kfree(fw_ver);
  44         return NULL;
  45 }
  46
  47 static int hl_request_fw(struct hl_device *hdev,
  48                                 const struct firmware **firmware_p,
  49                                 const char *fw_name)
  50 {
  51         size_t fw_size;
  52         int rc;
  53
  54         rc = request_firmware(firmware_p, fw_name, hdev->dev);
  55         if (rc) {
  56                 dev_err(hdev->dev, "Firmware file %s is not found! (error %d)\n",
  57                                 fw_name, rc);
  58                 goto out;
  59         }
  60
  61         fw_size = (*firmware_p)->size;
  62         if ((fw_size % 4) != 0) {
  63                 dev_err(hdev->dev, "Illegal %s firmware size %zu\n",
  64                                 fw_name, fw_size);
  65                 rc = -EINVAL;
  66                 goto release_fw;
  67         }
  68
  69         dev_dbg(hdev->dev, "%s firmware size == %zu\n", fw_name, fw_size);
  70
  71         if (fw_size > FW_FILE_MAX_SIZE) {
  72                 dev_err(hdev->dev,
  73                         "FW file size %zu exceeds maximum of %u bytes\n",
  74                         fw_size, FW_FILE_MAX_SIZE);
  75                 rc = -EINVAL;
  76                 goto release_fw;
  77         }
  78
  79         return 0;
  80
  81 release_fw:
  82         release_firmware(*firmware_p);
  83 out:
  84         return rc;
  85 }
  86
  87 /**
  88  * hl_release_firmware() - release FW
  89  *
  90  * @fw: fw descriptor
  91  *
  92  * note: this inline function added to serve as a comprehensive mirror for the
  93  *       hl_request_fw function.
  94  */
  95 static inline void hl_release_firmware(const struct firmware *fw)
  96 {
  97         release_firmware(fw);
  98 }
  99
 100 /**
 101  * hl_fw_copy_fw_to_device() - copy FW to device
 102  *
 103  * @hdev: pointer to hl_device structure.
 104  * @fw: fw descriptor
 105  * @dst: IO memory mapped address space to copy firmware to
 106  * @src_offset: offset in src FW to copy from
 107  * @size: amount of bytes to copy (0 to copy the whole binary)
 108  *
 109  * actual copy of FW binary data to device, shared by static and dynamic loaders
 110  */
 111 static int hl_fw_copy_fw_to_device(struct hl_device *hdev,
 112                                 const struct firmware *fw, void __iomem *dst,
 113                                 u32 src_offset, u32 size)
 114 {
 115         const void *fw_data;
 116
 117         /* size 0 indicates to copy the whole file */
 118         if (!size)
 119                 size = fw->size;
 120
 121         if (src_offset + size > fw->size) {
 122                 dev_err(hdev->dev,
 123                         "size to copy(%u) and offset(%u) are invalid\n",
 124                         size, src_offset);
 125                 return -EINVAL;
 126         }
 127
 128         fw_data = (const void *) fw->data;
 129
 130         memcpy_toio(dst, fw_data + src_offset, size);
 131         return 0;
 132 }
 133
 134 /**
 135  * hl_fw_copy_msg_to_device() - copy message to device
 136  *
 137  * @hdev: pointer to hl_device structure.
 138  * @msg: message
 139  * @dst: IO memory mapped address space to copy firmware to
 140  * @src_offset: offset in src message to copy from
 141  * @size: amount of bytes to copy (0 to copy the whole binary)
 142  *
 143  * actual copy of message data to device.
 144  */
 145 static int hl_fw_copy_msg_to_device(struct hl_device *hdev,
 146                 struct lkd_msg_comms *msg, void __iomem *dst,
 147                 u32 src_offset, u32 size)
 148 {
 149         void *msg_data;
 150
 151         /* size 0 indicates to copy the whole file */
 152         if (!size)
 153                 size = sizeof(struct lkd_msg_comms);
 154
 155         if (src_offset + size > sizeof(struct lkd_msg_comms)) {
 156                 dev_err(hdev->dev,
 157                         "size to copy(%u) and offset(%u) are invalid\n",
 158                         size, src_offset);
 159                 return -EINVAL;
 160         }
 161
 162         msg_data = (void *) msg;
 163
 164         memcpy_toio(dst, msg_data + src_offset, size);
 165
 166         return 0;
 167 }
 168
 169 /**
 170  * hl_fw_load_fw_to_device() - Load F/W code to device's memory.
 171  *
 172  * @hdev: pointer to hl_device structure.
 173  * @fw_name: the firmware image name
 174  * @dst: IO memory mapped address space to copy firmware to
 175  * @src_offset: offset in src FW to copy from
 176  * @size: amount of bytes to copy (0 to copy the whole binary)
 177  *
 178  * Copy fw code from firmware file to device memory.
 179  *
 180  * Return: 0 on success, non-zero for failure.
 181  */
 182 int hl_fw_load_fw_to_device(struct hl_device *hdev, const char *fw_name,
 183                                 void __iomem *dst, u32 src_offset, u32 size)
 184 {
 185         const struct firmware *fw;
 186         int rc;
 187
 188         rc = hl_request_fw(hdev, &fw, fw_name);
 189         if (rc)
 190                 return rc;
 191
 192         rc = hl_fw_copy_fw_to_device(hdev, fw, dst, src_offset, size);
 193
 194         hl_release_firmware(fw);
 195         return rc;
 196 }
 197
 198 int hl_fw_send_pci_access_msg(struct hl_device *hdev, u32 opcode)
 199 {
 200         struct cpucp_packet pkt = {};
 201
 202         pkt.ctl = cpu_to_le32(opcode << CPUCP_PKT_CTL_OPCODE_SHIFT);
 203
 204         return hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt,
 205                                                 sizeof(pkt), 0, NULL);
 206 }
 207
 208 int hl_fw_send_cpu_message(struct hl_device *hdev, u32 hw_queue_id, u32 *msg,
 209                                 u16 len, u32 timeout, u64 *result)
 210 {
 211         struct hl_hw_queue *queue = &hdev->kernel_queues[hw_queue_id];
 212         struct asic_fixed_properties *prop = &hdev->asic_prop;
 213         struct cpucp_packet *pkt;
 214         dma_addr_t pkt_dma_addr;
 215         struct hl_bd *sent_bd;
 216         u32 tmp, expected_ack_val, pi;
 217         int rc = 0;
 218
 219         pkt = hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev, len,
 220                                                                 &pkt_dma_addr);
 221         if (!pkt) {
 222                 dev_err(hdev->dev,
 223                         "Failed to allocate DMA memory for packet to CPU\n");
 224                 return -ENOMEM;
 225         }
 226
 227         memcpy(pkt, msg, len);
 228
 229         mutex_lock(&hdev->send_cpu_message_lock);
 230
 231         if (hdev->disabled)
 232                 goto out;
 233
 234         if (hdev->device_cpu_disabled) {
 235                 rc = -EIO;
 236                 goto out;
 237         }
 238
 239         /* set fence to a non valid value */
 240         pkt->fence = cpu_to_le32(UINT_MAX);
 241         pi = queue->pi;
 242
 243         /*
 244          * The CPU queue is a synchronous queue with an effective depth of
 245          * a single entry (although it is allocated with room for multiple
 246          * entries). We lock on it using 'send_cpu_message_lock' which
 247          * serializes accesses to the CPU queue.
 248          * Which means that we don't need to lock the access to the entire H/W
 249          * queues module when submitting a JOB to the CPU queue.
 250          */
 251         hl_hw_queue_submit_bd(hdev, queue, hl_queue_inc_ptr(queue->pi), len, pkt_dma_addr);
 252
 253         if (prop->fw_app_cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_PKT_PI_ACK_EN)
 254                 expected_ack_val = queue->pi;
 255         else
 256                 expected_ack_val = CPUCP_PACKET_FENCE_VAL;
 257
 258         rc = hl_poll_timeout_memory(hdev, &pkt->fence, tmp,
 259                                 (tmp == expected_ack_val), 1000,
 260                                 timeout, true);
 261
 262         hl_hw_queue_inc_ci_kernel(hdev, hw_queue_id);
 263
 264         if (rc == -ETIMEDOUT) {
 265                 dev_err(hdev->dev, "Device CPU packet timeout (0x%x)\n", tmp);
 266                 hdev->device_cpu_disabled = true;
 267                 goto out;
 268         }
 269
 270         tmp = le32_to_cpu(pkt->ctl);
 271
 272         rc = (tmp & CPUCP_PKT_CTL_RC_MASK) >> CPUCP_PKT_CTL_RC_SHIFT;
 273         if (rc) {
 274                 dev_err(hdev->dev, "F/W ERROR %d for CPU packet %d\n",
 275                         rc,
 276                         (tmp & CPUCP_PKT_CTL_OPCODE_MASK)
 277                                                 >> CPUCP_PKT_CTL_OPCODE_SHIFT);
 278                 rc = -EIO;
 279         } else if (result) {
 280                 *result = le64_to_cpu(pkt->result);
 281         }
 282
 283         /* Scrub previous buffer descriptor 'ctl' field which contains the
 284          * previous PI value written during packet submission.
 285          * We must do this or else F/W can read an old value upon queue wraparound.
 286          */
 287         sent_bd = queue->kernel_address;
 288         sent_bd += hl_pi_2_offset(pi);
 289         sent_bd->ctl = cpu_to_le32(UINT_MAX);
 290
 291 out:
 292         mutex_unlock(&hdev->send_cpu_message_lock);
 293
 294         hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev, len, pkt);
 295
 296         return rc;
 297 }
 298
 299 int hl_fw_unmask_irq(struct hl_device *hdev, u16 event_type)
 300 {
 301         struct cpucp_packet pkt;
 302         u64 result;
 303         int rc;
 304
 305         memset(&pkt, 0, sizeof(pkt));
 306
 307         pkt.ctl = cpu_to_le32(CPUCP_PACKET_UNMASK_RAZWI_IRQ <<
 308                                 CPUCP_PKT_CTL_OPCODE_SHIFT);
 309         pkt.value = cpu_to_le64(event_type);
 310
 311         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
 312                                                 0, &result);
 313
 314         if (rc)
 315                 dev_err(hdev->dev, "failed to unmask RAZWI IRQ %d", event_type);
 316
 317         return rc;
 318 }
 319
 320 int hl_fw_unmask_irq_arr(struct hl_device *hdev, const u32 *irq_arr,
 321                 size_t irq_arr_size)
 322 {
 323         struct cpucp_unmask_irq_arr_packet *pkt;
 324         size_t total_pkt_size;
 325         u64 result;
 326         int rc;
 327
 328         total_pkt_size = sizeof(struct cpucp_unmask_irq_arr_packet) +
 329                         irq_arr_size;
 330
 331         /* data should be aligned to 8 bytes in order to CPU-CP to copy it */
 332         total_pkt_size = (total_pkt_size + 0x7) & ~0x7;
 333
 334         /* total_pkt_size is casted to u16 later on */
 335         if (total_pkt_size > USHRT_MAX) {
 336                 dev_err(hdev->dev, "too many elements in IRQ array\n");
 337                 return -EINVAL;
 338         }
 339
 340         pkt = kzalloc(total_pkt_size, GFP_KERNEL);
 341         if (!pkt)
 342                 return -ENOMEM;
 343
 344         pkt->length = cpu_to_le32(irq_arr_size / sizeof(irq_arr[0]));
 345         memcpy(&pkt->irqs, irq_arr, irq_arr_size);
 346
 347         pkt->cpucp_pkt.ctl = cpu_to_le32(CPUCP_PACKET_UNMASK_RAZWI_IRQ_ARRAY <<
 348                                                 CPUCP_PKT_CTL_OPCODE_SHIFT);
 349
 350         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) pkt,
 351                                                 total_pkt_size, 0, &result);
 352
 353         if (rc)
 354                 dev_err(hdev->dev, "failed to unmask IRQ array\n");
 355
 356         kfree(pkt);
 357
 358         return rc;
 359 }
 360
 361 int hl_fw_test_cpu_queue(struct hl_device *hdev)
 362 {
 363         struct cpucp_packet test_pkt = {};
 364         u64 result;
 365         int rc;
 366
 367         test_pkt.ctl = cpu_to_le32(CPUCP_PACKET_TEST <<
 368                                         CPUCP_PKT_CTL_OPCODE_SHIFT);
 369         test_pkt.value = cpu_to_le64(CPUCP_PACKET_FENCE_VAL);
 370
 371         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &test_pkt,
 372                                                 sizeof(test_pkt), 0, &result);
 373
 374         if (!rc) {
 375                 if (result != CPUCP_PACKET_FENCE_VAL)
 376                         dev_err(hdev->dev,
 377                                 "CPU queue test failed (%#08llx)\n", result);
 378         } else {
 379                 dev_err(hdev->dev, "CPU queue test failed, error %d\n", rc);
 380         }
 381
 382         return rc;
 383 }
 384
 385 void *hl_fw_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size,
 386                                                 dma_addr_t *dma_handle)
 387 {
 388         u64 kernel_addr;
 389
 390         kernel_addr = gen_pool_alloc(hdev->cpu_accessible_dma_pool, size);
 391
 392         *dma_handle = hdev->cpu_accessible_dma_address +
 393                 (kernel_addr - (u64) (uintptr_t) hdev->cpu_accessible_dma_mem);
 394
 395         return (void *) (uintptr_t) kernel_addr;
 396 }
 397
 398 void hl_fw_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size,
 399                                         void *vaddr)
 400 {
 401         gen_pool_free(hdev->cpu_accessible_dma_pool, (u64) (uintptr_t) vaddr,
 402                         size);
 403 }
 404
 405 int hl_fw_send_heartbeat(struct hl_device *hdev)
 406 {
 407         struct cpucp_packet hb_pkt;
 408         u64 result;
 409         int rc;
 410
 411         memset(&hb_pkt, 0, sizeof(hb_pkt));
 412         hb_pkt.ctl = cpu_to_le32(CPUCP_PACKET_TEST <<
 413                                         CPUCP_PKT_CTL_OPCODE_SHIFT);
 414         hb_pkt.value = cpu_to_le64(CPUCP_PACKET_FENCE_VAL);
 415
 416         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &hb_pkt,
 417                                                 sizeof(hb_pkt), 0, &result);
 418
 419         if ((rc) || (result != CPUCP_PACKET_FENCE_VAL))
 420                 return -EIO;
 421
 422         if (le32_to_cpu(hb_pkt.status_mask) &
 423                                         CPUCP_PKT_HB_STATUS_EQ_FAULT_MASK) {
 424                 dev_warn(hdev->dev, "FW reported EQ fault during heartbeat\n");
 425                 rc = -EIO;
 426         }
 427
 428         return rc;
 429 }
 430
 431 static bool fw_report_boot_dev0(struct hl_device *hdev, u32 err_val,
 432                                                                 u32 sts_val)
 433 {
 434         bool err_exists = false;
 435
 436         if (!(err_val & CPU_BOOT_ERR0_ENABLED))
 437                 return false;
 438
 439         if (err_val & CPU_BOOT_ERR0_DRAM_INIT_FAIL) {
 440                 dev_err(hdev->dev,
 441                         "Device boot error - DRAM initialization failed\n");
 442                 err_exists = true;
 443         }
 444
 445         if (err_val & CPU_BOOT_ERR0_FIT_CORRUPTED) {
 446                 dev_err(hdev->dev, "Device boot error - FIT image corrupted\n");
 447                 err_exists = true;
 448         }
 449
 450         if (err_val & CPU_BOOT_ERR0_TS_INIT_FAIL) {
 451                 dev_err(hdev->dev,
 452                         "Device boot error - Thermal Sensor initialization failed\n");
 453                 err_exists = true;
 454         }
 455
 456         if (err_val & CPU_BOOT_ERR0_BMC_WAIT_SKIPPED) {
 457                 if (hdev->bmc_enable) {
 458                         dev_err(hdev->dev,
 459                                 "Device boot error - Skipped waiting for BMC\n");
 460                         err_exists = true;
 461                 } else {
 462                         dev_info(hdev->dev,
 463                                 "Device boot message - Skipped waiting for BMC\n");
 464                         /* This is an info so we don't want it to disable the
 465                          * device
 466                          */
 467                         err_val &= ~CPU_BOOT_ERR0_BMC_WAIT_SKIPPED;
 468                 }
 469         }
 470
 471         if (err_val & CPU_BOOT_ERR0_NIC_DATA_NOT_RDY) {
 472                 dev_err(hdev->dev,
 473                         "Device boot error - Serdes data from BMC not available\n");
 474                 err_exists = true;
 475         }
 476
 477         if (err_val & CPU_BOOT_ERR0_NIC_FW_FAIL) {
 478                 dev_err(hdev->dev,
 479                         "Device boot error - NIC F/W initialization failed\n");
 480                 err_exists = true;
 481         }
 482
 483         if (err_val & CPU_BOOT_ERR0_SECURITY_NOT_RDY) {
 484                 dev_err(hdev->dev,
 485                         "Device boot warning - security not ready\n");
 486                 err_exists = true;
 487         }
 488
 489         if (err_val & CPU_BOOT_ERR0_SECURITY_FAIL) {
 490                 dev_err(hdev->dev, "Device boot error - security failure\n");
 491                 err_exists = true;
 492         }
 493
 494         if (err_val & CPU_BOOT_ERR0_EFUSE_FAIL) {
 495                 dev_err(hdev->dev, "Device boot error - eFuse failure\n");
 496                 err_exists = true;
 497         }
 498
 499         if (err_val & CPU_BOOT_ERR0_SEC_IMG_VER_FAIL) {
 500                 dev_err(hdev->dev, "Device boot error - Failed to load preboot secondary image\n");
 501                 err_exists = true;
 502         }
 503
 504         if (err_val & CPU_BOOT_ERR0_PLL_FAIL) {
 505                 dev_err(hdev->dev, "Device boot error - PLL failure\n");
 506                 err_exists = true;
 507         }
 508
 509         if (err_val & CPU_BOOT_ERR0_DEVICE_UNUSABLE_FAIL) {
 510                 /* Ignore this bit, don't prevent driver loading */
 511                 dev_dbg(hdev->dev, "device unusable status is set\n");
 512                 err_val &= ~CPU_BOOT_ERR0_DEVICE_UNUSABLE_FAIL;
 513         }
 514
 515         if (sts_val & CPU_BOOT_DEV_STS0_ENABLED)
 516                 dev_dbg(hdev->dev, "Device status0 %#x\n", sts_val);
 517
 518         /* All warnings should go here in order not to reach the unknown error validation */
 519         if (err_val & CPU_BOOT_ERR0_DRAM_SKIPPED) {
 520                 dev_warn(hdev->dev,
 521                         "Device boot warning - Skipped DRAM initialization\n");
 522                 /* This is a warning so we don't want it to disable the
 523                  * device
 524                  */
 525                 err_val &= ~CPU_BOOT_ERR0_DRAM_SKIPPED;
 526         }
 527
 528         if (err_val & CPU_BOOT_ERR0_PRI_IMG_VER_FAIL) {
 529                 dev_warn(hdev->dev,
 530                         "Device boot warning - Failed to load preboot primary image\n");
 531                 /* This is a warning so we don't want it to disable the
 532                  * device as we have a secondary preboot image
 533                  */
 534                 err_val &= ~CPU_BOOT_ERR0_PRI_IMG_VER_FAIL;
 535         }
 536
 537         if (err_val & CPU_BOOT_ERR0_TPM_FAIL) {
 538                 dev_warn(hdev->dev,
 539                         "Device boot warning - TPM failure\n");
 540                 /* This is a warning so we don't want it to disable the
 541                  * device
 542                  */
 543                 err_val &= ~CPU_BOOT_ERR0_TPM_FAIL;
 544         }
 545
 546         if (!err_exists && (err_val & ~CPU_BOOT_ERR0_ENABLED)) {
 547                 dev_err(hdev->dev,
 548                         "Device boot error - unknown ERR0 error 0x%08x\n", err_val);
 549                 err_exists = true;
 550         }
 551
 552         /* return error only if it's in the predefined mask */
 553         if (err_exists && ((err_val & ~CPU_BOOT_ERR0_ENABLED) &
 554                                 lower_32_bits(hdev->boot_error_status_mask)))
 555                 return true;
 556
 557         return false;
 558 }
 559
 560 /* placeholder for ERR1 as no errors defined there yet */
 561 static bool fw_report_boot_dev1(struct hl_device *hdev, u32 err_val,
 562                                                                 u32 sts_val)
 563 {
 564         /*
 565          * keep this variable to preserve the logic of the function.
 566          * this way it would require less modifications when error will be
 567          * added to DEV_ERR1
 568          */
 569         bool err_exists = false;
 570
 571         if (!(err_val & CPU_BOOT_ERR1_ENABLED))
 572                 return false;
 573
 574         if (sts_val & CPU_BOOT_DEV_STS1_ENABLED)
 575                 dev_dbg(hdev->dev, "Device status1 %#x\n", sts_val);
 576
 577         if (!err_exists && (err_val & ~CPU_BOOT_ERR1_ENABLED)) {
 578                 dev_err(hdev->dev,
 579                         "Device boot error - unknown ERR1 error 0x%08x\n",
 580                                                                 err_val);
 581                 err_exists = true;
 582         }
 583
 584         /* return error only if it's in the predefined mask */
 585         if (err_exists && ((err_val & ~CPU_BOOT_ERR1_ENABLED) &
 586                                 upper_32_bits(hdev->boot_error_status_mask)))
 587                 return true;
 588
 589         return false;
 590 }
 591
 592 static int fw_read_errors(struct hl_device *hdev, u32 boot_err0_reg,
 593                                 u32 boot_err1_reg, u32 cpu_boot_dev_status0_reg,
 594                                 u32 cpu_boot_dev_status1_reg)
 595 {
 596         u32 err_val, status_val;
 597         bool err_exists = false;
 598
 599         /* Some of the firmware status codes are deprecated in newer f/w
 600          * versions. In those versions, the errors are reported
 601          * in different registers. Therefore, we need to check those
 602          * registers and print the exact errors. Moreover, there
 603          * may be multiple errors, so we need to report on each error
 604          * separately. Some of the error codes might indicate a state
 605          * that is not an error per-se, but it is an error in production
 606          * environment
 607          */
 608         err_val = RREG32(boot_err0_reg);
 609         status_val = RREG32(cpu_boot_dev_status0_reg);
 610         err_exists = fw_report_boot_dev0(hdev, err_val, status_val);
 611
 612         err_val = RREG32(boot_err1_reg);
 613         status_val = RREG32(cpu_boot_dev_status1_reg);
 614         err_exists |= fw_report_boot_dev1(hdev, err_val, status_val);
 615
 616         if (err_exists)
 617                 return -EIO;
 618
 619         return 0;
 620 }
 621
 622 int hl_fw_cpucp_info_get(struct hl_device *hdev,
 623                                 u32 sts_boot_dev_sts0_reg,
 624                                 u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg,
 625                                 u32 boot_err1_reg)
 626 {
 627         struct asic_fixed_properties *prop = &hdev->asic_prop;
 628         struct cpucp_packet pkt = {};
 629         dma_addr_t cpucp_info_dma_addr;
 630         void *cpucp_info_cpu_addr;
 631         char *kernel_ver;
 632         u64 result;
 633         int rc;
 634
 635         cpucp_info_cpu_addr =
 636                         hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev,
 637                                         sizeof(struct cpucp_info),
 638                                         &cpucp_info_dma_addr);
 639         if (!cpucp_info_cpu_addr) {
 640                 dev_err(hdev->dev,
 641                         "Failed to allocate DMA memory for CPU-CP info packet\n");
 642                 return -ENOMEM;
 643         }
 644
 645         memset(cpucp_info_cpu_addr, 0, sizeof(struct cpucp_info));
 646
 647         pkt.ctl = cpu_to_le32(CPUCP_PACKET_INFO_GET <<
 648                                 CPUCP_PKT_CTL_OPCODE_SHIFT);
 649         pkt.addr = cpu_to_le64(cpucp_info_dma_addr);
 650         pkt.data_max_size = cpu_to_le32(sizeof(struct cpucp_info));
 651
 652         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
 653                                         HL_CPUCP_INFO_TIMEOUT_USEC, &result);
 654         if (rc) {
 655                 dev_err(hdev->dev,
 656                         "Failed to handle CPU-CP info pkt, error %d\n", rc);
 657                 goto out;
 658         }
 659
 660         rc = fw_read_errors(hdev, boot_err0_reg, boot_err1_reg,
 661                                 sts_boot_dev_sts0_reg, sts_boot_dev_sts1_reg);
 662         if (rc) {
 663                 dev_err(hdev->dev, "Errors in device boot\n");
 664                 goto out;
 665         }
 666
 667         memcpy(&prop->cpucp_info, cpucp_info_cpu_addr,
 668                         sizeof(prop->cpucp_info));
 669
 670         rc = hl_build_hwmon_channel_info(hdev, prop->cpucp_info.sensors);
 671         if (rc) {
 672                 dev_err(hdev->dev,
 673                         "Failed to build hwmon channel info, error %d\n", rc);
 674                 rc = -EFAULT;
 675                 goto out;
 676         }
 677
 678         kernel_ver = extract_fw_ver_from_str(prop->cpucp_info.kernel_version);
 679         if (kernel_ver) {
 680                 dev_info(hdev->dev, "Linux version %s", kernel_ver);
 681                 kfree(kernel_ver);
 682         }
 683
 684         /* assume EQ code doesn't need to check eqe index */
 685         hdev->event_queue.check_eqe_index = false;
 686
 687         /* Read FW application security bits again */
 688         if (prop->fw_cpu_boot_dev_sts0_valid) {
 689                 prop->fw_app_cpu_boot_dev_sts0 = RREG32(sts_boot_dev_sts0_reg);
 690                 if (prop->fw_app_cpu_boot_dev_sts0 &
 691                                 CPU_BOOT_DEV_STS0_EQ_INDEX_EN)
 692                         hdev->event_queue.check_eqe_index = true;
 693         }
 694
 695         if (prop->fw_cpu_boot_dev_sts1_valid)
 696                 prop->fw_app_cpu_boot_dev_sts1 = RREG32(sts_boot_dev_sts1_reg);
 697
 698 out:
 699         hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev,
 700                         sizeof(struct cpucp_info), cpucp_info_cpu_addr);
 701
 702         return rc;
 703 }
 704
 705 static int hl_fw_send_msi_info_msg(struct hl_device *hdev)
 706 {
 707         struct cpucp_array_data_packet *pkt;
 708         size_t total_pkt_size, data_size;
 709         u64 result;
 710         int rc;
 711
 712         /* skip sending this info for unsupported ASICs */
 713         if (!hdev->asic_funcs->get_msi_info)
 714                 return 0;
 715
 716         data_size = CPUCP_NUM_OF_MSI_TYPES * sizeof(u32);
 717         total_pkt_size = sizeof(struct cpucp_array_data_packet) + data_size;
 718
 719         /* data should be aligned to 8 bytes in order to CPU-CP to copy it */
 720         total_pkt_size = (total_pkt_size + 0x7) & ~0x7;
 721
 722         /* total_pkt_size is casted to u16 later on */
 723         if (total_pkt_size > USHRT_MAX) {
 724                 dev_err(hdev->dev, "CPUCP array data is too big\n");
 725                 return -EINVAL;
 726         }
 727
 728         pkt = kzalloc(total_pkt_size, GFP_KERNEL);
 729         if (!pkt)
 730                 return -ENOMEM;
 731
 732         pkt->length = cpu_to_le32(CPUCP_NUM_OF_MSI_TYPES);
 733
 734         memset((void *) &pkt->data, 0xFF, data_size);
 735         hdev->asic_funcs->get_msi_info(pkt->data);
 736
 737         pkt->cpucp_pkt.ctl = cpu_to_le32(CPUCP_PACKET_MSI_INFO_SET <<
 738                                                 CPUCP_PKT_CTL_OPCODE_SHIFT);
 739
 740         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *)pkt,
 741                                                 total_pkt_size, 0, &result);
 742
 743         /*
 744          * in case packet result is invalid it means that FW does not support
 745          * this feature and will use default/hard coded MSI values. no reason
 746          * to stop the boot
 747          */
 748         if (rc && result == cpucp_packet_invalid)
 749                 rc = 0;
 750
 751         if (rc)
 752                 dev_err(hdev->dev, "failed to send CPUCP array data\n");
 753
 754         kfree(pkt);
 755
 756         return rc;
 757 }
 758
 759 int hl_fw_cpucp_handshake(struct hl_device *hdev,
 760                                 u32 sts_boot_dev_sts0_reg,
 761                                 u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg,
 762                                 u32 boot_err1_reg)
 763 {
 764         int rc;
 765
 766         rc = hl_fw_cpucp_info_get(hdev, sts_boot_dev_sts0_reg,
 767                                         sts_boot_dev_sts1_reg, boot_err0_reg,
 768                                         boot_err1_reg);
 769         if (rc)
 770                 return rc;
 771
 772         return hl_fw_send_msi_info_msg(hdev);
 773 }
 774
 775 int hl_fw_get_eeprom_data(struct hl_device *hdev, void *data, size_t max_size)
 776 {
 777         struct cpucp_packet pkt = {};
 778         void *eeprom_info_cpu_addr;
 779         dma_addr_t eeprom_info_dma_addr;
 780         u64 result;
 781         int rc;
 782
 783         eeprom_info_cpu_addr =
 784                         hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev,
 785                                         max_size, &eeprom_info_dma_addr);
 786         if (!eeprom_info_cpu_addr) {
 787                 dev_err(hdev->dev,
 788                         "Failed to allocate DMA memory for CPU-CP EEPROM packet\n");
 789                 return -ENOMEM;
 790         }
 791
 792         memset(eeprom_info_cpu_addr, 0, max_size);
 793
 794         pkt.ctl = cpu_to_le32(CPUCP_PACKET_EEPROM_DATA_GET <<
 795                                 CPUCP_PKT_CTL_OPCODE_SHIFT);
 796         pkt.addr = cpu_to_le64(eeprom_info_dma_addr);
 797         pkt.data_max_size = cpu_to_le32(max_size);
 798
 799         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
 800                         HL_CPUCP_EEPROM_TIMEOUT_USEC, &result);
 801
 802         if (rc) {
 803                 dev_err(hdev->dev,
 804                         "Failed to handle CPU-CP EEPROM packet, error %d\n",
 805                         rc);
 806                 goto out;
 807         }
 808
 809         /* result contains the actual size */
 810         memcpy(data, eeprom_info_cpu_addr, min((size_t)result, max_size));
 811
 812 out:
 813         hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev, max_size,
 814                         eeprom_info_cpu_addr);
 815
 816         return rc;
 817 }
 818
 819 int hl_fw_cpucp_pci_counters_get(struct hl_device *hdev,
 820                 struct hl_info_pci_counters *counters)
 821 {
 822         struct cpucp_packet pkt = {};
 823         u64 result;
 824         int rc;
 825
 826         pkt.ctl = cpu_to_le32(CPUCP_PACKET_PCIE_THROUGHPUT_GET <<
 827                         CPUCP_PKT_CTL_OPCODE_SHIFT);
 828
 829         /* Fetch PCI rx counter */
 830         pkt.index = cpu_to_le32(cpucp_pcie_throughput_rx);
 831         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
 832                                         HL_CPUCP_INFO_TIMEOUT_USEC, &result);
 833         if (rc) {
 834                 dev_err(hdev->dev,
 835                         "Failed to handle CPU-CP PCI info pkt, error %d\n", rc);
 836                 return rc;
 837         }
 838         counters->rx_throughput = result;
 839
 840         memset(&pkt, 0, sizeof(pkt));
 841         pkt.ctl = cpu_to_le32(CPUCP_PACKET_PCIE_THROUGHPUT_GET <<
 842                         CPUCP_PKT_CTL_OPCODE_SHIFT);
 843
 844         /* Fetch PCI tx counter */
 845         pkt.index = cpu_to_le32(cpucp_pcie_throughput_tx);
 846         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
 847                                         HL_CPUCP_INFO_TIMEOUT_USEC, &result);
 848         if (rc) {
 849                 dev_err(hdev->dev,
 850                         "Failed to handle CPU-CP PCI info pkt, error %d\n", rc);
 851                 return rc;
 852         }
 853         counters->tx_throughput = result;
 854
 855         /* Fetch PCI replay counter */
 856         memset(&pkt, 0, sizeof(pkt));
 857         pkt.ctl = cpu_to_le32(CPUCP_PACKET_PCIE_REPLAY_CNT_GET <<
 858                         CPUCP_PKT_CTL_OPCODE_SHIFT);
 859
 860         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
 861                         HL_CPUCP_INFO_TIMEOUT_USEC, &result);
 862         if (rc) {
 863                 dev_err(hdev->dev,
 864                         "Failed to handle CPU-CP PCI info pkt, error %d\n", rc);
 865                 return rc;
 866         }
 867         counters->replay_cnt = (u32) result;
 868
 869         return rc;
 870 }
 871
 872 int hl_fw_cpucp_total_energy_get(struct hl_device *hdev, u64 *total_energy)
 873 {
 874         struct cpucp_packet pkt = {};
 875         u64 result;
 876         int rc;
 877
 878         pkt.ctl = cpu_to_le32(CPUCP_PACKET_TOTAL_ENERGY_GET <<
 879                                 CPUCP_PKT_CTL_OPCODE_SHIFT);
 880
 881         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
 882                                         HL_CPUCP_INFO_TIMEOUT_USEC, &result);
 883         if (rc) {
 884                 dev_err(hdev->dev,
 885                         "Failed to handle CpuCP total energy pkt, error %d\n",
 886                                 rc);
 887                 return rc;
 888         }
 889
 890         *total_energy = result;
 891
 892         return rc;
 893 }
 894
 895 int get_used_pll_index(struct hl_device *hdev, u32 input_pll_index,
 896                                                 enum pll_index *pll_index)
 897 {
 898         struct asic_fixed_properties *prop = &hdev->asic_prop;
 899         u8 pll_byte, pll_bit_off;
 900         bool dynamic_pll;
 901         int fw_pll_idx;
 902
 903         dynamic_pll = !!(prop->fw_app_cpu_boot_dev_sts0 &
 904                                                 CPU_BOOT_DEV_STS0_DYN_PLL_EN);
 905
 906         if (!dynamic_pll) {
 907                 /*
 908                  * in case we are working with legacy FW (each asic has unique
 909                  * PLL numbering) use the driver based index as they are
 910                  * aligned with fw legacy numbering
 911                  */
 912                 *pll_index = input_pll_index;
 913                 return 0;
 914         }
 915
 916         /* retrieve a FW compatible PLL index based on
 917          * ASIC specific user request
 918          */
 919         fw_pll_idx = hdev->asic_funcs->map_pll_idx_to_fw_idx(input_pll_index);
 920         if (fw_pll_idx < 0) {
 921                 dev_err(hdev->dev, "Invalid PLL index (%u) error %d\n",
 922                         input_pll_index, fw_pll_idx);
 923                 return -EINVAL;
 924         }
 925
 926         /* PLL map is a u8 array */
 927         pll_byte = prop->cpucp_info.pll_map[fw_pll_idx >> 3];
 928         pll_bit_off = fw_pll_idx & 0x7;
 929
 930         if (!(pll_byte & BIT(pll_bit_off))) {
 931                 dev_err(hdev->dev, "PLL index %d is not supported\n",
 932                         fw_pll_idx);
 933                 return -EINVAL;
 934         }
 935
 936         *pll_index = fw_pll_idx;
 937
 938         return 0;
 939 }
 940
 941 int hl_fw_cpucp_pll_info_get(struct hl_device *hdev, u32 pll_index,
 942                 u16 *pll_freq_arr)
 943 {
 944         struct cpucp_packet pkt;
 945         enum pll_index used_pll_idx;
 946         u64 result;
 947         int rc;
 948
 949         rc = get_used_pll_index(hdev, pll_index, &used_pll_idx);
 950         if (rc)
 951                 return rc;
 952
 953         memset(&pkt, 0, sizeof(pkt));
 954
 955         pkt.ctl = cpu_to_le32(CPUCP_PACKET_PLL_INFO_GET <<
 956                                 CPUCP_PKT_CTL_OPCODE_SHIFT);
 957         pkt.pll_type = __cpu_to_le16((u16)used_pll_idx);
 958
 959         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
 960                         HL_CPUCP_INFO_TIMEOUT_USEC, &result);
 961         if (rc)
 962                 dev_err(hdev->dev, "Failed to read PLL info, error %d\n", rc);
 963
 964         pll_freq_arr[0] = FIELD_GET(CPUCP_PKT_RES_PLL_OUT0_MASK, result);
 965         pll_freq_arr[1] = FIELD_GET(CPUCP_PKT_RES_PLL_OUT1_MASK, result);
 966         pll_freq_arr[2] = FIELD_GET(CPUCP_PKT_RES_PLL_OUT2_MASK, result);
 967         pll_freq_arr[3] = FIELD_GET(CPUCP_PKT_RES_PLL_OUT3_MASK, result);
 968
 969         return rc;
 970 }
 971
 972 int hl_fw_cpucp_power_get(struct hl_device *hdev, u64 *power)
 973 {
 974         struct cpucp_packet pkt;
 975         u64 result;
 976         int rc;
 977
 978         memset(&pkt, 0, sizeof(pkt));
 979
 980         pkt.ctl = cpu_to_le32(CPUCP_PACKET_POWER_GET <<
 981                                 CPUCP_PKT_CTL_OPCODE_SHIFT);
 982         pkt.type = cpu_to_le16(CPUCP_POWER_INPUT);
 983
 984         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
 985                         HL_CPUCP_INFO_TIMEOUT_USEC, &result);
 986         if (rc) {
 987                 dev_err(hdev->dev, "Failed to read power, error %d\n", rc);
 988                 return rc;
 989         }
 990
 991         *power = result;
 992
 993         return rc;
 994 }
 995
 996 int hl_fw_dram_replaced_row_get(struct hl_device *hdev,
 997                                 struct cpucp_hbm_row_info *info)
 998 {
 999         struct cpucp_hbm_row_info *cpucp_repl_rows_info_cpu_addr;
1000         dma_addr_t cpucp_repl_rows_info_dma_addr;
1001         struct cpucp_packet pkt = {};
1002         u64 result;
1003         int rc;
1004
1005         cpucp_repl_rows_info_cpu_addr =
1006                         hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev,
1007                                         sizeof(struct cpucp_hbm_row_info),
1008                                         &cpucp_repl_rows_info_dma_addr);
1009         if (!cpucp_repl_rows_info_cpu_addr) {
1010                 dev_err(hdev->dev,
1011                         "Failed to allocate DMA memory for CPU-CP replaced rows info packet\n");
1012                 return -ENOMEM;
1013         }
1014
1015         memset(cpucp_repl_rows_info_cpu_addr, 0, sizeof(struct cpucp_hbm_row_info));
1016
1017         pkt.ctl = cpu_to_le32(CPUCP_PACKET_HBM_REPLACED_ROWS_INFO_GET <<
1018                                         CPUCP_PKT_CTL_OPCODE_SHIFT);
1019         pkt.addr = cpu_to_le64(cpucp_repl_rows_info_dma_addr);
1020         pkt.data_max_size = cpu_to_le32(sizeof(struct cpucp_hbm_row_info));
1021
1022         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
1023                                         HL_CPUCP_INFO_TIMEOUT_USEC, &result);
1024         if (rc) {
1025                 dev_err(hdev->dev,
1026                         "Failed to handle CPU-CP replaced rows info pkt, error %d\n", rc);
1027                 goto out;
1028         }
1029
1030         memcpy(info, cpucp_repl_rows_info_cpu_addr, sizeof(*info));
1031
1032 out:
1033         hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev,
1034                                         sizeof(struct cpucp_hbm_row_info),
1035                                         cpucp_repl_rows_info_cpu_addr);
1036
1037         return rc;
1038 }
1039
1040 int hl_fw_dram_pending_row_get(struct hl_device *hdev, u32 *pend_rows_num)
1041 {
1042         struct cpucp_packet pkt;
1043         u64 result;
1044         int rc;
1045
1046         memset(&pkt, 0, sizeof(pkt));
1047
1048         pkt.ctl = cpu_to_le32(CPUCP_PACKET_HBM_PENDING_ROWS_STATUS << CPUCP_PKT_CTL_OPCODE_SHIFT);
1049
1050         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, &result);
1051         if (rc) {
1052                 dev_err(hdev->dev,
1053                                 "Failed to handle CPU-CP pending rows info pkt, error %d\n", rc);
1054                 goto out;
1055         }
1056
1057         *pend_rows_num = (u32) result;
1058 out:
1059         return rc;
1060 }
1061
1062 int hl_fw_cpucp_engine_core_asid_set(struct hl_device *hdev, u32 asid)
1063 {
1064         struct cpucp_packet pkt;
1065         int rc;
1066
1067         memset(&pkt, 0, sizeof(pkt));
1068
1069         pkt.ctl = cpu_to_le32(CPUCP_PACKET_ENGINE_CORE_ASID_SET << CPUCP_PKT_CTL_OPCODE_SHIFT);
1070         pkt.value = cpu_to_le64(asid);
1071
1072         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
1073                                                 HL_CPUCP_INFO_TIMEOUT_USEC, NULL);
1074         if (rc)
1075                 dev_err(hdev->dev,
1076                         "Failed on ASID configuration request for engine core, error %d\n",
1077                         rc);
1078
1079         return rc;
1080 }
1081
1082 void hl_fw_ask_hard_reset_without_linux(struct hl_device *hdev)
1083 {
1084         struct static_fw_load_mgr *static_loader =
1085                         &hdev->fw_loader.static_loader;
1086         int rc;
1087
1088         if (hdev->asic_prop.dynamic_fw_load) {
1089                 rc = hl_fw_dynamic_send_protocol_cmd(hdev, &hdev->fw_loader,
1090                                 COMMS_RST_DEV, 0, false,
1091                                 hdev->fw_loader.cpu_timeout);
1092                 if (rc)
1093                         dev_warn(hdev->dev, "Failed sending COMMS_RST_DEV\n");
1094         } else {
1095                 WREG32(static_loader->kmd_msg_to_cpu_reg, KMD_MSG_RST_DEV);
1096         }
1097 }
1098
1099 void hl_fw_ask_halt_machine_without_linux(struct hl_device *hdev)
1100 {
1101         struct static_fw_load_mgr *static_loader =
1102                         &hdev->fw_loader.static_loader;
1103         int rc;
1104
1105         if (hdev->device_cpu_is_halted)
1106                 return;
1107
1108         /* Stop device CPU to make sure nothing bad happens */
1109         if (hdev->asic_prop.dynamic_fw_load) {
1110                 rc = hl_fw_dynamic_send_protocol_cmd(hdev, &hdev->fw_loader,
1111                                 COMMS_GOTO_WFE, 0, true,
1112                                 hdev->fw_loader.cpu_timeout);
1113                 if (rc)
1114                         dev_warn(hdev->dev, "Failed sending COMMS_GOTO_WFE\n");
1115         } else {
1116                 WREG32(static_loader->kmd_msg_to_cpu_reg, KMD_MSG_GOTO_WFE);
1117                 msleep(static_loader->cpu_reset_wait_msec);
1118
1119                 /* Must clear this register in order to prevent preboot
1120                  * from reading WFE after reboot
1121                  */
1122                 WREG32(static_loader->kmd_msg_to_cpu_reg, KMD_MSG_NA);
1123         }
1124
1125         hdev->device_cpu_is_halted = true;
1126 }
1127
1128 static void detect_cpu_boot_status(struct hl_device *hdev, u32 status)
1129 {
1130         /* Some of the status codes below are deprecated in newer f/w
1131          * versions but we keep them here for backward compatibility
1132          */
1133         switch (status) {
1134         case CPU_BOOT_STATUS_NA:
1135                 dev_err(hdev->dev,
1136                         "Device boot progress - BTL/ROM did NOT run\n");
1137                 break;
1138         case CPU_BOOT_STATUS_IN_WFE:
1139                 dev_err(hdev->dev,
1140                         "Device boot progress - Stuck inside WFE loop\n");
1141                 break;
1142         case CPU_BOOT_STATUS_IN_BTL:
1143                 dev_err(hdev->dev,
1144                         "Device boot progress - Stuck in BTL\n");
1145                 break;
1146         case CPU_BOOT_STATUS_IN_PREBOOT:
1147                 dev_err(hdev->dev,
1148                         "Device boot progress - Stuck in Preboot\n");
1149                 break;
1150         case CPU_BOOT_STATUS_IN_SPL:
1151                 dev_err(hdev->dev,
1152                         "Device boot progress - Stuck in SPL\n");
1153                 break;
1154         case CPU_BOOT_STATUS_IN_UBOOT:
1155                 dev_err(hdev->dev,
1156                         "Device boot progress - Stuck in u-boot\n");
1157                 break;
1158         case CPU_BOOT_STATUS_DRAM_INIT_FAIL:
1159                 dev_err(hdev->dev,
1160                         "Device boot progress - DRAM initialization failed\n");
1161                 break;
1162         case CPU_BOOT_STATUS_UBOOT_NOT_READY:
1163                 dev_err(hdev->dev,
1164                         "Device boot progress - Cannot boot\n");
1165                 break;
1166         case CPU_BOOT_STATUS_TS_INIT_FAIL:
1167                 dev_err(hdev->dev,
1168                         "Device boot progress - Thermal Sensor initialization failed\n");
1169                 break;
1170         case CPU_BOOT_STATUS_SECURITY_READY:
1171                 dev_err(hdev->dev,
1172                         "Device boot progress - Stuck in preboot after security initialization\n");
1173                 break;
1174         default:
1175                 dev_err(hdev->dev,
1176                         "Device boot progress - Invalid status code %d\n",
1177                         status);
1178                 break;
1179         }
1180 }
1181
1182 static int hl_fw_read_preboot_caps(struct hl_device *hdev,
1183                                         u32 cpu_boot_status_reg,
1184                                         u32 sts_boot_dev_sts0_reg,
1185                                         u32 sts_boot_dev_sts1_reg,
1186                                         u32 boot_err0_reg, u32 boot_err1_reg,
1187                                         u32 timeout)
1188 {
1189         struct asic_fixed_properties *prop = &hdev->asic_prop;
1190         u32 status, reg_val;
1191         int rc;
1192
1193         /* Need to check two possible scenarios:
1194          *
1195          * CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT - for newer firmwares where
1196          * the preboot is waiting for the boot fit
1197          *
1198          * All other status values - for older firmwares where the uboot was
1199          * loaded from the FLASH
1200          */
1201         rc = hl_poll_timeout(
1202                 hdev,
1203                 cpu_boot_status_reg,
1204                 status,
1205                 (status == CPU_BOOT_STATUS_IN_UBOOT) ||
1206                 (status == CPU_BOOT_STATUS_DRAM_RDY) ||
1207                 (status == CPU_BOOT_STATUS_NIC_FW_RDY) ||
1208                 (status == CPU_BOOT_STATUS_READY_TO_BOOT) ||
1209                 (status == CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT),
1210                 hdev->fw_poll_interval_usec,
1211                 timeout);
1212
1213         if (rc) {
1214                 dev_err(hdev->dev, "CPU boot ready status timeout\n");
1215                 detect_cpu_boot_status(hdev, status);
1216
1217                 /* If we read all FF, then something is totally wrong, no point
1218                  * of reading specific errors
1219                  */
1220                 if (status != -1)
1221                         fw_read_errors(hdev, boot_err0_reg, boot_err1_reg,
1222                                                         sts_boot_dev_sts0_reg,
1223                                                         sts_boot_dev_sts1_reg);
1224                 return -EIO;
1225         }
1226
1227         /*
1228          * the registers DEV_STS* contain FW capabilities/features.
1229          * We can rely on this registers only if bit CPU_BOOT_DEV_STS*_ENABLED
1230          * is set.
1231          * In the first read of this register we store the value of this
1232          * register ONLY if the register is enabled (which will be propagated
1233          * to next stages) and also mark the register as valid.
1234          * In case it is not enabled the stored value will be left 0- all
1235          * caps/features are off
1236          */
1237         reg_val = RREG32(sts_boot_dev_sts0_reg);
1238         if (reg_val & CPU_BOOT_DEV_STS0_ENABLED) {
1239                 prop->fw_cpu_boot_dev_sts0_valid = true;
1240                 prop->fw_preboot_cpu_boot_dev_sts0 = reg_val;
1241         }
1242
1243         reg_val = RREG32(sts_boot_dev_sts1_reg);
1244         if (reg_val & CPU_BOOT_DEV_STS1_ENABLED) {
1245                 prop->fw_cpu_boot_dev_sts1_valid = true;
1246                 prop->fw_preboot_cpu_boot_dev_sts1 = reg_val;
1247         }
1248
1249         prop->dynamic_fw_load = !!(prop->fw_preboot_cpu_boot_dev_sts0 &
1250                                                 CPU_BOOT_DEV_STS0_FW_LD_COM_EN);
1251
1252         /* initialize FW loader once we know what load protocol is used */
1253         hdev->asic_funcs->init_firmware_loader(hdev);
1254
1255         dev_dbg(hdev->dev, "Attempting %s FW load\n",
1256                         prop->dynamic_fw_load ? "dynamic" : "legacy");
1257         return 0;
1258 }
1259
1260 static int hl_fw_static_read_device_fw_version(struct hl_device *hdev,
1261                                         enum hl_fw_component fwc)
1262 {
1263         struct asic_fixed_properties *prop = &hdev->asic_prop;
1264         struct fw_load_mgr *fw_loader = &hdev->fw_loader;
1265         struct static_fw_load_mgr *static_loader;
1266         char *dest, *boot_ver, *preboot_ver;
1267         u32 ver_off, limit;
1268         const char *name;
1269         char btl_ver[32];
1270
1271         static_loader = &hdev->fw_loader.static_loader;
1272
1273         switch (fwc) {
1274         case FW_COMP_BOOT_FIT:
1275                 ver_off = RREG32(static_loader->boot_fit_version_offset_reg);
1276                 dest = prop->uboot_ver;
1277                 name = "Boot-fit";
1278                 limit = static_loader->boot_fit_version_max_off;
1279                 break;
1280         case FW_COMP_PREBOOT:
1281                 ver_off = RREG32(static_loader->preboot_version_offset_reg);
1282                 dest = prop->preboot_ver;
1283                 name = "Preboot";
1284                 limit = static_loader->preboot_version_max_off;
1285                 break;
1286         default:
1287                 dev_warn(hdev->dev, "Undefined FW component: %d\n", fwc);
1288                 return -EIO;
1289         }
1290
1291         ver_off &= static_loader->sram_offset_mask;
1292
1293         if (ver_off < limit) {
1294                 memcpy_fromio(dest,
1295                         hdev->pcie_bar[fw_loader->sram_bar_id] + ver_off,
1296                         VERSION_MAX_LEN);
1297         } else {
1298                 dev_err(hdev->dev, "%s version offset (0x%x) is above SRAM\n",
1299                                                                 name, ver_off);
1300                 strscpy(dest, "unavailable", VERSION_MAX_LEN);
1301                 return -EIO;
1302         }
1303
1304         if (fwc == FW_COMP_BOOT_FIT) {
1305                 boot_ver = extract_fw_ver_from_str(prop->uboot_ver);
1306                 if (boot_ver) {
1307                         dev_info(hdev->dev, "boot-fit version %s\n", boot_ver);
1308                         kfree(boot_ver);
1309                 }
1310         } else if (fwc == FW_COMP_PREBOOT) {
1311                 preboot_ver = strnstr(prop->preboot_ver, "Preboot",
1312                                                 VERSION_MAX_LEN);
1313                 if (preboot_ver && preboot_ver != prop->preboot_ver) {
1314                         strscpy(btl_ver, prop->preboot_ver,
1315                                 min((int) (preboot_ver - prop->preboot_ver),
1316                                                                         31));
1317                         dev_info(hdev->dev, "%s\n", btl_ver);
1318                 }
1319
1320                 preboot_ver = extract_fw_ver_from_str(prop->preboot_ver);
1321                 if (preboot_ver) {
1322                         dev_info(hdev->dev, "preboot version %s\n",
1323                                                                 preboot_ver);
1324                         kfree(preboot_ver);
1325                 }
1326         }
1327
1328         return 0;
1329 }
1330
1331 /**
1332  * hl_fw_preboot_update_state - update internal data structures during
1333  *                              handshake with preboot
1334  *
1335  *
1336  * @hdev: pointer to the habanalabs device structure
1337  *
1338  * @return 0 on success, otherwise non-zero error code
1339  */
1340 static void hl_fw_preboot_update_state(struct hl_device *hdev)
1341 {
1342         struct asic_fixed_properties *prop = &hdev->asic_prop;
1343         u32 cpu_boot_dev_sts0, cpu_boot_dev_sts1;
1344
1345         cpu_boot_dev_sts0 = prop->fw_preboot_cpu_boot_dev_sts0;
1346         cpu_boot_dev_sts1 = prop->fw_preboot_cpu_boot_dev_sts1;
1347
1348         /* We read boot_dev_sts registers multiple times during boot:
1349          * 1. preboot - a. Check whether the security status bits are valid
1350          *              b. Check whether fw security is enabled
1351          *              c. Check whether hard reset is done by preboot
1352          * 2. boot cpu - a. Fetch boot cpu security status
1353          *               b. Check whether hard reset is done by boot cpu
1354          * 3. FW application - a. Fetch fw application security status
1355          *                     b. Check whether hard reset is done by fw app
1356          */
1357         prop->hard_reset_done_by_fw = !!(cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_FW_HARD_RST_EN);
1358
1359         dev_dbg(hdev->dev, "Firmware preboot boot device status0 %#x\n",
1360                                                         cpu_boot_dev_sts0);
1361
1362         dev_dbg(hdev->dev, "Firmware preboot boot device status1 %#x\n",
1363                                                         cpu_boot_dev_sts1);
1364
1365         dev_dbg(hdev->dev, "Firmware preboot hard-reset is %s\n",
1366                         prop->hard_reset_done_by_fw ? "enabled" : "disabled");
1367
1368         dev_dbg(hdev->dev, "firmware-level security is %s\n",
1369                         prop->fw_security_enabled ? "enabled" : "disabled");
1370
1371         dev_dbg(hdev->dev, "GIC controller is %s\n",
1372                         prop->gic_interrupts_enable ? "enabled" : "disabled");
1373 }
1374
1375 static int hl_fw_static_read_preboot_status(struct hl_device *hdev)
1376 {
1377         int rc;
1378
1379         rc = hl_fw_static_read_device_fw_version(hdev, FW_COMP_PREBOOT);
1380         if (rc)
1381                 return rc;
1382
1383         return 0;
1384 }
1385
1386 int hl_fw_read_preboot_status(struct hl_device *hdev, u32 cpu_boot_status_reg,
1387                                 u32 sts_boot_dev_sts0_reg,
1388                                 u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg,
1389                                 u32 boot_err1_reg, u32 timeout)
1390 {
1391         int rc;
1392
1393         if (!(hdev->fw_components & FW_TYPE_PREBOOT_CPU))
1394                 return 0;
1395
1396         /*
1397          * In order to determine boot method (static VS dymanic) we need to
1398          * read the boot caps register
1399          */
1400         rc = hl_fw_read_preboot_caps(hdev, cpu_boot_status_reg,
1401                                         sts_boot_dev_sts0_reg,
1402                                         sts_boot_dev_sts1_reg, boot_err0_reg,
1403                                         boot_err1_reg, timeout);
1404         if (rc)
1405                 return rc;
1406
1407         hl_fw_preboot_update_state(hdev);
1408
1409         /* no need to read preboot status in dynamic load */
1410         if (hdev->asic_prop.dynamic_fw_load)
1411                 return 0;
1412
1413         return hl_fw_static_read_preboot_status(hdev);
1414 }
1415
1416 /* associate string with COMM status */
1417 static char *hl_dynamic_fw_status_str[COMMS_STS_INVLD_LAST] = {
1418         [COMMS_STS_NOOP] = "NOOP",
1419         [COMMS_STS_ACK] = "ACK",
1420         [COMMS_STS_OK] = "OK",
1421         [COMMS_STS_ERR] = "ERR",
1422         [COMMS_STS_VALID_ERR] = "VALID_ERR",
1423         [COMMS_STS_TIMEOUT_ERR] = "TIMEOUT_ERR",
1424 };
1425
1426 /**
1427  * hl_fw_dynamic_report_error_status - report error status
1428  *
1429  * @hdev: pointer to the habanalabs device structure
1430  * @status: value of FW status register
1431  * @expected_status: the expected status
1432  */
1433 static void hl_fw_dynamic_report_error_status(struct hl_device *hdev,
1434                                                 u32 status,
1435                                                 enum comms_sts expected_status)
1436 {
1437         enum comms_sts comm_status =
1438                                 FIELD_GET(COMMS_STATUS_STATUS_MASK, status);
1439
1440         if (comm_status < COMMS_STS_INVLD_LAST)
1441                 dev_err(hdev->dev, "Device status %s, expected status: %s\n",
1442                                 hl_dynamic_fw_status_str[comm_status],
1443                                 hl_dynamic_fw_status_str[expected_status]);
1444         else
1445                 dev_err(hdev->dev, "Device status unknown %d, expected status: %s\n",
1446                                 comm_status,
1447                                 hl_dynamic_fw_status_str[expected_status]);
1448 }
1449
1450 /**
1451  * hl_fw_dynamic_send_cmd - send LKD to FW cmd
1452  *
1453  * @hdev: pointer to the habanalabs device structure
1454  * @fw_loader: managing structure for loading device's FW
1455  * @cmd: LKD to FW cmd code
1456  * @size: size of next FW component to be loaded (0 if not necessary)
1457  *
1458  * LDK to FW exact command layout is defined at struct comms_command.
1459  * note: the size argument is used only when the next FW component should be
1460  *       loaded, otherwise it shall be 0. the size is used by the FW in later
1461  *       protocol stages and when sending only indicating the amount of memory
1462  *       to be allocated by the FW to receive the next boot component.
1463  */
1464 static void hl_fw_dynamic_send_cmd(struct hl_device *hdev,
1465                                 struct fw_load_mgr *fw_loader,
1466                                 enum comms_cmd cmd, unsigned int size)
1467 {
1468         struct cpu_dyn_regs *dyn_regs;
1469         u32 val;
1470
1471         dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs;
1472
1473         val = FIELD_PREP(COMMS_COMMAND_CMD_MASK, cmd);
1474         val |= FIELD_PREP(COMMS_COMMAND_SIZE_MASK, size);
1475
1476         WREG32(le32_to_cpu(dyn_regs->kmd_msg_to_cpu), val);
1477 }
1478
1479 /**
1480  * hl_fw_dynamic_extract_fw_response - update the FW response
1481  *
1482  * @hdev: pointer to the habanalabs device structure
1483  * @fw_loader: managing structure for loading device's FW
1484  * @response: FW response
1485  * @status: the status read from CPU status register
1486  *
1487  * @return 0 on success, otherwise non-zero error code
1488  */
1489 static int hl_fw_dynamic_extract_fw_response(struct hl_device *hdev,
1490                                                 struct fw_load_mgr *fw_loader,
1491                                                 struct fw_response *response,
1492                                                 u32 status)
1493 {
1494         response->status = FIELD_GET(COMMS_STATUS_STATUS_MASK, status);
1495         response->ram_offset = FIELD_GET(COMMS_STATUS_OFFSET_MASK, status) <<
1496                                                 COMMS_STATUS_OFFSET_ALIGN_SHIFT;
1497         response->ram_type = FIELD_GET(COMMS_STATUS_RAM_TYPE_MASK, status);
1498
1499         if ((response->ram_type != COMMS_SRAM) &&
1500                                         (response->ram_type != COMMS_DRAM)) {
1501                 dev_err(hdev->dev, "FW status: invalid RAM type %u\n",
1502                                                         response->ram_type);
1503                 return -EIO;
1504         }
1505
1506         return 0;
1507 }
1508
1509 /**
1510  * hl_fw_dynamic_wait_for_status - wait for status in dynamic FW load
1511  *
1512  * @hdev: pointer to the habanalabs device structure
1513  * @fw_loader: managing structure for loading device's FW
1514  * @expected_status: expected status to wait for
1515  * @timeout: timeout for status wait
1516  *
1517  * @return 0 on success, otherwise non-zero error code
1518  *
1519  * waiting for status from FW include polling the FW status register until
1520  * expected status is received or timeout occurs (whatever occurs first).
1521  */
1522 static int hl_fw_dynamic_wait_for_status(struct hl_device *hdev,
1523                                                 struct fw_load_mgr *fw_loader,
1524                                                 enum comms_sts expected_status,
1525                                                 u32 timeout)
1526 {
1527         struct cpu_dyn_regs *dyn_regs;
1528         u32 status;
1529         int rc;
1530
1531         dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs;
1532
1533         /* Wait for expected status */
1534         rc = hl_poll_timeout(
1535                 hdev,
1536                 le32_to_cpu(dyn_regs->cpu_cmd_status_to_host),
1537                 status,
1538                 FIELD_GET(COMMS_STATUS_STATUS_MASK, status) == expected_status,
1539                 hdev->fw_poll_interval_usec,
1540                 timeout);
1541
1542         if (rc) {
1543                 hl_fw_dynamic_report_error_status(hdev, status,
1544                                                         expected_status);
1545                 return -EIO;
1546         }
1547
1548         /*
1549          * skip storing FW response for NOOP to preserve the actual desired
1550          * FW status
1551          */
1552         if (expected_status == COMMS_STS_NOOP)
1553                 return 0;
1554
1555         rc = hl_fw_dynamic_extract_fw_response(hdev, fw_loader,
1556                                         &fw_loader->dynamic_loader.response,
1557                                         status);
1558         return rc;
1559 }
1560
1561 /**
1562  * hl_fw_dynamic_send_clear_cmd - send clear command to FW
1563  *
1564  * @hdev: pointer to the habanalabs device structure
1565  * @fw_loader: managing structure for loading device's FW
1566  *
1567  * @return 0 on success, otherwise non-zero error code
1568  *
1569  * after command cycle between LKD to FW CPU (i.e. LKD got an expected status
1570  * from FW) we need to clear the CPU status register in order to avoid garbage
1571  * between command cycles.
1572  * This is done by sending clear command and polling the CPU to LKD status
1573  * register to hold the status NOOP
1574  */
1575 static int hl_fw_dynamic_send_clear_cmd(struct hl_device *hdev,
1576                                                 struct fw_load_mgr *fw_loader)
1577 {
1578         hl_fw_dynamic_send_cmd(hdev, fw_loader, COMMS_CLR_STS, 0);
1579
1580         return hl_fw_dynamic_wait_for_status(hdev, fw_loader, COMMS_STS_NOOP,
1581                                                         fw_loader->cpu_timeout);
1582 }
1583
1584 /**
1585  * hl_fw_dynamic_send_protocol_cmd - send LKD to FW cmd and wait for ACK
1586  *
1587  * @hdev: pointer to the habanalabs device structure
1588  * @fw_loader: managing structure for loading device's FW
1589  * @cmd: LKD to FW cmd code
1590  * @size: size of next FW component to be loaded (0 if not necessary)
1591  * @wait_ok: if true also wait for OK response from FW
1592  * @timeout: timeout for status wait
1593  *
1594  * @return 0 on success, otherwise non-zero error code
1595  *
1596  * brief:
1597  * when sending protocol command we have the following steps:
1598  * - send clear (clear command and verify clear status register)
1599  * - send the actual protocol command
1600  * - wait for ACK on the protocol command
1601  * - send clear
1602  * - send NOOP
1603  * if, in addition, the specific protocol command should wait for OK then:
1604  * - wait for OK
1605  * - send clear
1606  * - send NOOP
1607  *
1608  * NOTES:
1609  * send clear: this is necessary in order to clear the status register to avoid
1610  *             leftovers between command
1611  * NOOP command: necessary to avoid loop on the clear command by the FW
1612  */
1613 int hl_fw_dynamic_send_protocol_cmd(struct hl_device *hdev,
1614                                 struct fw_load_mgr *fw_loader,
1615                                 enum comms_cmd cmd, unsigned int size,
1616                                 bool wait_ok, u32 timeout)
1617 {
1618         int rc;
1619
1620         /* first send clear command to clean former commands */
1621         rc = hl_fw_dynamic_send_clear_cmd(hdev, fw_loader);
1622
1623         /* send the actual command */
1624         hl_fw_dynamic_send_cmd(hdev, fw_loader, cmd, size);
1625
1626         /* wait for ACK for the command */
1627         rc = hl_fw_dynamic_wait_for_status(hdev, fw_loader, COMMS_STS_ACK,
1628                                                                 timeout);
1629         if (rc)
1630                 return rc;
1631
1632         /* clear command to prepare for NOOP command */
1633         rc = hl_fw_dynamic_send_clear_cmd(hdev, fw_loader);
1634         if (rc)
1635                 return rc;
1636
1637         /* send the actual NOOP command */
1638         hl_fw_dynamic_send_cmd(hdev, fw_loader, COMMS_NOOP, 0);
1639
1640         if (!wait_ok)
1641                 return 0;
1642
1643         rc = hl_fw_dynamic_wait_for_status(hdev, fw_loader, COMMS_STS_OK,
1644                                                                 timeout);
1645         if (rc)
1646                 return rc;
1647
1648         /* clear command to prepare for NOOP command */
1649         rc = hl_fw_dynamic_send_clear_cmd(hdev, fw_loader);
1650         if (rc)
1651                 return rc;
1652
1653         /* send the actual NOOP command */
1654         hl_fw_dynamic_send_cmd(hdev, fw_loader, COMMS_NOOP, 0);
1655
1656         return 0;
1657 }
1658
1659 /**
1660  * hl_fw_compat_crc32 - CRC compatible with FW
1661  *
1662  * @data: pointer to the data
1663  * @size: size of the data
1664  *
1665  * @return the CRC32 result
1666  *
1667  * NOTE: kernel's CRC32 differ's from standard CRC32 calculation.
1668  *       in order to be aligned we need to flip the bits of both the input
1669  *       initial CRC and kernel's CRC32 result.
1670  *       in addition both sides use initial CRC of 0,
1671  */
1672 static u32 hl_fw_compat_crc32(u8 *data, size_t size)
1673 {
1674         return ~crc32_le(~((u32)0), data, size);
1675 }
1676
1677 /**
1678  * hl_fw_dynamic_validate_memory_bound - validate memory bounds for memory
1679  *                                        transfer (image or descriptor) between
1680  *                                        host and FW
1681  *
1682  * @hdev: pointer to the habanalabs device structure
1683  * @addr: device address of memory transfer
1684  * @size: memory transter size
1685  * @region: PCI memory region
1686  *
1687  * @return 0 on success, otherwise non-zero error code
1688  */
1689 static int hl_fw_dynamic_validate_memory_bound(struct hl_device *hdev,
1690                                                 u64 addr, size_t size,
1691                                                 struct pci_mem_region *region)
1692 {
1693         u64 end_addr;
1694
1695         /* now make sure that the memory transfer is within region's bounds */
1696         end_addr = addr + size;
1697         if (end_addr >= region->region_base + region->region_size) {
1698                 dev_err(hdev->dev,
1699                         "dynamic FW load: memory transfer end address out of memory region bounds. addr: %llx\n",
1700                                                         end_addr);
1701                 return -EIO;
1702         }
1703
1704         /*
1705          * now make sure memory transfer is within predefined BAR bounds.
1706          * this is to make sure we do not need to set the bar (e.g. for DRAM
1707          * memory transfers)
1708          */
1709         if (end_addr >= region->region_base - region->offset_in_bar +
1710                                                         region->bar_size) {
1711                 dev_err(hdev->dev,
1712                         "FW image beyond PCI BAR bounds\n");
1713                 return -EIO;
1714         }
1715
1716         return 0;
1717 }
1718
1719 /**
1720  * hl_fw_dynamic_validate_descriptor - validate FW descriptor
1721  *
1722  * @hdev: pointer to the habanalabs device structure
1723  * @fw_loader: managing structure for loading device's FW
1724  * @fw_desc: the descriptor form FW
1725  *
1726  * @return 0 on success, otherwise non-zero error code
1727  */
1728 static int hl_fw_dynamic_validate_descriptor(struct hl_device *hdev,
1729                                         struct fw_load_mgr *fw_loader,
1730                                         struct lkd_fw_comms_desc *fw_desc)
1731 {
1732         struct pci_mem_region *region;
1733         enum pci_region region_id;
1734         size_t data_size;
1735         u32 data_crc32;
1736         u8 *data_ptr;
1737         u64 addr;
1738         int rc;
1739
1740         if (le32_to_cpu(fw_desc->header.magic) != HL_COMMS_DESC_MAGIC) {
1741                 dev_err(hdev->dev, "Invalid magic for dynamic FW descriptor (%x)\n",
1742                                 fw_desc->header.magic);
1743                 return -EIO;
1744         }
1745
1746         if (fw_desc->header.version != HL_COMMS_DESC_VER) {
1747                 dev_err(hdev->dev, "Invalid version for dynamic FW descriptor (%x)\n",
1748                                 fw_desc->header.version);
1749                 return -EIO;
1750         }
1751
1752         /*
1753          * calc CRC32 of data without header.
1754          * note that no alignment/stride address issues here as all structures
1755          * are 64 bit padded
1756          */
1757         data_size = sizeof(struct lkd_fw_comms_desc) -
1758                                         sizeof(struct comms_desc_header);
1759         data_ptr = (u8 *)fw_desc + sizeof(struct comms_desc_header);
1760
1761         if (le16_to_cpu(fw_desc->header.size) != data_size) {
1762                 dev_err(hdev->dev,
1763                         "Invalid descriptor size 0x%x, expected size 0x%zx\n",
1764                                 le16_to_cpu(fw_desc->header.size), data_size);
1765                 return -EIO;
1766         }
1767
1768         data_crc32 = hl_fw_compat_crc32(data_ptr, data_size);
1769
1770         if (data_crc32 != le32_to_cpu(fw_desc->header.crc32)) {
1771                 dev_err(hdev->dev,
1772                         "CRC32 mismatch for dynamic FW descriptor (%x:%x)\n",
1773                                         data_crc32, fw_desc->header.crc32);
1774                 return -EIO;
1775         }
1776
1777         /* find memory region to which to copy the image */
1778         addr = le64_to_cpu(fw_desc->img_addr);
1779         region_id = hl_get_pci_memory_region(hdev, addr);
1780         if ((region_id != PCI_REGION_SRAM) &&
1781                         ((region_id != PCI_REGION_DRAM))) {
1782                 dev_err(hdev->dev,
1783                         "Invalid region to copy FW image address=%llx\n", addr);
1784                 return -EIO;
1785         }
1786
1787         region = &hdev->pci_mem_region[region_id];
1788
1789         /* store the region for the copy stage */
1790         fw_loader->dynamic_loader.image_region = region;
1791
1792         /*
1793          * here we know that the start address is valid, now make sure that the
1794          * image is within region's bounds
1795          */
1796         rc = hl_fw_dynamic_validate_memory_bound(hdev, addr,
1797                                         fw_loader->dynamic_loader.fw_image_size,
1798                                         region);
1799         if (rc) {
1800                 dev_err(hdev->dev,
1801                         "invalid mem transfer request for FW image\n");
1802                 return rc;
1803         }
1804
1805         /* here we can mark the descriptor as valid as the content has been validated */
1806         fw_loader->dynamic_loader.fw_desc_valid = true;
1807
1808         return 0;
1809 }
1810
1811 static int hl_fw_dynamic_validate_response(struct hl_device *hdev,
1812                                                 struct fw_response *response,
1813                                                 struct pci_mem_region *region)
1814 {
1815         u64 device_addr;
1816         int rc;
1817
1818         device_addr = region->region_base + response->ram_offset;
1819
1820         /*
1821          * validate that the descriptor is within region's bounds
1822          * Note that as the start address was supplied according to the RAM
1823          * type- testing only the end address is enough
1824          */
1825         rc = hl_fw_dynamic_validate_memory_bound(hdev, device_addr,
1826                                         sizeof(struct lkd_fw_comms_desc),
1827                                         region);
1828         return rc;
1829 }
1830
1831 /**
1832  * hl_fw_dynamic_read_and_validate_descriptor - read and validate FW descriptor
1833  *
1834  * @hdev: pointer to the habanalabs device structure
1835  * @fw_loader: managing structure for loading device's FW
1836  *
1837  * @return 0 on success, otherwise non-zero error code
1838  */
1839 static int hl_fw_dynamic_read_and_validate_descriptor(struct hl_device *hdev,
1840                                                 struct fw_load_mgr *fw_loader)
1841 {
1842         struct lkd_fw_comms_desc *fw_desc;
1843         struct pci_mem_region *region;
1844         struct fw_response *response;
1845         enum pci_region region_id;
1846         void __iomem *src;
1847         int rc;
1848
1849         fw_desc = &fw_loader->dynamic_loader.comm_desc;
1850         response = &fw_loader->dynamic_loader.response;
1851
1852         region_id = (response->ram_type == COMMS_SRAM) ?
1853                                         PCI_REGION_SRAM : PCI_REGION_DRAM;
1854
1855         region = &hdev->pci_mem_region[region_id];
1856
1857         rc = hl_fw_dynamic_validate_response(hdev, response, region);
1858         if (rc) {
1859                 dev_err(hdev->dev,
1860                         "invalid mem transfer request for FW descriptor\n");
1861                 return rc;
1862         }
1863
1864         /*
1865          * extract address to copy the descriptor from
1866          * in addition, as the descriptor value is going to be over-ridden by new data- we mark it
1867          * as invalid.
1868          * it will be marked again as valid once validated
1869          */
1870         fw_loader->dynamic_loader.fw_desc_valid = false;
1871         src = hdev->pcie_bar[region->bar_id] + region->offset_in_bar +
1872                                                         response->ram_offset;
1873         memcpy_fromio(fw_desc, src, sizeof(struct lkd_fw_comms_desc));
1874
1875         return hl_fw_dynamic_validate_descriptor(hdev, fw_loader, fw_desc);
1876 }
1877
1878 /**
1879  * hl_fw_dynamic_request_descriptor - handshake with CPU to get FW descriptor
1880  *
1881  * @hdev: pointer to the habanalabs device structure
1882  * @fw_loader: managing structure for loading device's FW
1883  * @next_image_size: size to allocate for next FW component
1884  *
1885  * @return 0 on success, otherwise non-zero error code
1886  */
1887 static int hl_fw_dynamic_request_descriptor(struct hl_device *hdev,
1888                                                 struct fw_load_mgr *fw_loader,
1889                                                 size_t next_image_size)
1890 {
1891         int rc;
1892
1893         rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_PREP_DESC,
1894                                                 next_image_size, true,
1895                                                 fw_loader->cpu_timeout);
1896         if (rc)
1897                 return rc;
1898
1899         return hl_fw_dynamic_read_and_validate_descriptor(hdev, fw_loader);
1900 }
1901
1902 /**
1903  * hl_fw_dynamic_read_device_fw_version - read FW version to exposed properties
1904  *
1905  * @hdev: pointer to the habanalabs device structure
1906  * @fwc: the firmware component
1907  * @fw_version: fw component's version string
1908  */
1909 static void hl_fw_dynamic_read_device_fw_version(struct hl_device *hdev,
1910                                         enum hl_fw_component fwc,
1911                                         const char *fw_version)
1912 {
1913         struct asic_fixed_properties *prop = &hdev->asic_prop;
1914         char *preboot_ver, *boot_ver;
1915         char btl_ver[32];
1916
1917         switch (fwc) {
1918         case FW_COMP_BOOT_FIT:
1919                 strscpy(prop->uboot_ver, fw_version, VERSION_MAX_LEN);
1920                 boot_ver = extract_fw_ver_from_str(prop->uboot_ver);
1921                 if (boot_ver) {
1922                         dev_info(hdev->dev, "boot-fit version %s\n", boot_ver);
1923                         kfree(boot_ver);
1924                 }
1925
1926                 break;
1927         case FW_COMP_PREBOOT:
1928                 strscpy(prop->preboot_ver, fw_version, VERSION_MAX_LEN);
1929                 preboot_ver = strnstr(prop->preboot_ver, "Preboot",
1930                                                 VERSION_MAX_LEN);
1931                 if (preboot_ver && preboot_ver != prop->preboot_ver) {
1932                         strscpy(btl_ver, prop->preboot_ver,
1933                                 min((int) (preboot_ver - prop->preboot_ver),
1934                                                                         31));
1935                         dev_info(hdev->dev, "%s\n", btl_ver);
1936                 }
1937
1938                 preboot_ver = extract_fw_ver_from_str(prop->preboot_ver);
1939                 if (preboot_ver) {
1940                         dev_info(hdev->dev, "preboot version %s\n",
1941                                                                 preboot_ver);
1942                         kfree(preboot_ver);
1943                 }
1944
1945                 break;
1946         default:
1947                 dev_warn(hdev->dev, "Undefined FW component: %d\n", fwc);
1948                 return;
1949         }
1950 }
1951
1952 /**
1953  * hl_fw_dynamic_copy_image - copy image to memory allocated by the FW
1954  *
1955  * @hdev: pointer to the habanalabs device structure
1956  * @fw: fw descriptor
1957  * @fw_loader: managing structure for loading device's FW
1958  */
1959 static int hl_fw_dynamic_copy_image(struct hl_device *hdev,
1960                                                 const struct firmware *fw,
1961                                                 struct fw_load_mgr *fw_loader)
1962 {
1963         struct lkd_fw_comms_desc *fw_desc;
1964         struct pci_mem_region *region;
1965         void __iomem *dest;
1966         u64 addr;
1967         int rc;
1968
1969         fw_desc = &fw_loader->dynamic_loader.comm_desc;
1970         addr = le64_to_cpu(fw_desc->img_addr);
1971
1972         /* find memory region to which to copy the image */
1973         region = fw_loader->dynamic_loader.image_region;
1974
1975         dest = hdev->pcie_bar[region->bar_id] + region->offset_in_bar +
1976                                         (addr - region->region_base);
1977
1978         rc = hl_fw_copy_fw_to_device(hdev, fw, dest,
1979                                         fw_loader->boot_fit_img.src_off,
1980                                         fw_loader->boot_fit_img.copy_size);
1981
1982         return rc;
1983 }
1984
1985 /**
1986  * hl_fw_dynamic_copy_msg - copy msg to memory allocated by the FW
1987  *
1988  * @hdev: pointer to the habanalabs device structure
1989  * @msg: message
1990  * @fw_loader: managing structure for loading device's FW
1991  */
1992 static int hl_fw_dynamic_copy_msg(struct hl_device *hdev,
1993                 struct lkd_msg_comms *msg, struct fw_load_mgr *fw_loader)
1994 {
1995         struct lkd_fw_comms_desc *fw_desc;
1996         struct pci_mem_region *region;
1997         void __iomem *dest;
1998         u64 addr;
1999         int rc;
2000
2001         fw_desc = &fw_loader->dynamic_loader.comm_desc;
2002         addr = le64_to_cpu(fw_desc->img_addr);
2003
2004         /* find memory region to which to copy the image */
2005         region = fw_loader->dynamic_loader.image_region;
2006
2007         dest = hdev->pcie_bar[region->bar_id] + region->offset_in_bar +
2008                                         (addr - region->region_base);
2009
2010         rc = hl_fw_copy_msg_to_device(hdev, msg, dest, 0, 0);
2011
2012         return rc;
2013 }
2014
2015 /**
2016  * hl_fw_boot_fit_update_state - update internal data structures after boot-fit
2017  *                               is loaded
2018  *
2019  * @hdev: pointer to the habanalabs device structure
2020  * @cpu_boot_dev_sts0_reg: register holding CPU boot dev status 0
2021  * @cpu_boot_dev_sts1_reg: register holding CPU boot dev status 1
2022  *
2023  * @return 0 on success, otherwise non-zero error code
2024  */
2025 static void hl_fw_boot_fit_update_state(struct hl_device *hdev,
2026                                                 u32 cpu_boot_dev_sts0_reg,
2027                                                 u32 cpu_boot_dev_sts1_reg)
2028 {
2029         struct asic_fixed_properties *prop = &hdev->asic_prop;
2030
2031         hdev->fw_loader.fw_comp_loaded |= FW_TYPE_BOOT_CPU;
2032
2033         /* Read boot_cpu status bits */
2034         if (prop->fw_preboot_cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_ENABLED) {
2035                 prop->fw_bootfit_cpu_boot_dev_sts0 =
2036                                 RREG32(cpu_boot_dev_sts0_reg);
2037
2038                 prop->hard_reset_done_by_fw = !!(prop->fw_bootfit_cpu_boot_dev_sts0 &
2039                                                         CPU_BOOT_DEV_STS0_FW_HARD_RST_EN);
2040
2041                 dev_dbg(hdev->dev, "Firmware boot CPU status0 %#x\n",
2042                                         prop->fw_bootfit_cpu_boot_dev_sts0);
2043         }
2044
2045         if (prop->fw_cpu_boot_dev_sts1_valid) {
2046                 prop->fw_bootfit_cpu_boot_dev_sts1 =
2047                                 RREG32(cpu_boot_dev_sts1_reg);
2048
2049                 dev_dbg(hdev->dev, "Firmware boot CPU status1 %#x\n",
2050                                         prop->fw_bootfit_cpu_boot_dev_sts1);
2051         }
2052
2053         dev_dbg(hdev->dev, "Firmware boot CPU hard-reset is %s\n",
2054                         prop->hard_reset_done_by_fw ? "enabled" : "disabled");
2055 }
2056
2057 static void hl_fw_dynamic_update_linux_interrupt_if(struct hl_device *hdev)
2058 {
2059         struct cpu_dyn_regs *dyn_regs =
2060                         &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
2061
2062         /* Check whether all 3 interrupt interfaces are set, if not use a
2063          * single interface
2064          */
2065         if (!hdev->asic_prop.gic_interrupts_enable &&
2066                         !(hdev->asic_prop.fw_app_cpu_boot_dev_sts0 &
2067                                 CPU_BOOT_DEV_STS0_MULTI_IRQ_POLL_EN)) {
2068                 dyn_regs->gic_host_halt_irq = dyn_regs->gic_host_pi_upd_irq;
2069                 dyn_regs->gic_host_ints_irq = dyn_regs->gic_host_pi_upd_irq;
2070
2071                 dev_warn(hdev->dev,
2072                         "Using a single interrupt interface towards cpucp");
2073         }
2074 }
2075 /**
2076  * hl_fw_dynamic_load_image - load FW image using dynamic protocol
2077  *
2078  * @hdev: pointer to the habanalabs device structure
2079  * @fw_loader: managing structure for loading device's FW
2080  * @load_fwc: the FW component to be loaded
2081  * @img_ld_timeout: image load timeout
2082  *
2083  * @return 0 on success, otherwise non-zero error code
2084  */
2085 static int hl_fw_dynamic_load_image(struct hl_device *hdev,
2086                                                 struct fw_load_mgr *fw_loader,
2087                                                 enum hl_fw_component load_fwc,
2088                                                 u32 img_ld_timeout)
2089 {
2090         enum hl_fw_component cur_fwc;
2091         const struct firmware *fw;
2092         char *fw_name;
2093         int rc = 0;
2094
2095         /*
2096          * when loading image we have one of 2 scenarios:
2097          * 1. current FW component is preboot and we want to load boot-fit
2098          * 2. current FW component is boot-fit and we want to load linux
2099          */
2100         if (load_fwc == FW_COMP_BOOT_FIT) {
2101                 cur_fwc = FW_COMP_PREBOOT;
2102                 fw_name = fw_loader->boot_fit_img.image_name;
2103         } else {
2104                 cur_fwc = FW_COMP_BOOT_FIT;
2105                 fw_name = fw_loader->linux_img.image_name;
2106         }
2107
2108         /* request FW in order to communicate to FW the size to be allocated */
2109         rc = hl_request_fw(hdev, &fw, fw_name);
2110         if (rc)
2111                 return rc;
2112
2113         /* store the image size for future validation */
2114         fw_loader->dynamic_loader.fw_image_size = fw->size;
2115
2116         rc = hl_fw_dynamic_request_descriptor(hdev, fw_loader, fw->size);
2117         if (rc)
2118                 goto release_fw;
2119
2120         /* read preboot version */
2121         hl_fw_dynamic_read_device_fw_version(hdev, cur_fwc,
2122                                 fw_loader->dynamic_loader.comm_desc.cur_fw_ver);
2123
2124
2125         /* update state according to boot stage */
2126         if (cur_fwc == FW_COMP_BOOT_FIT) {
2127                 struct cpu_dyn_regs *dyn_regs;
2128
2129                 dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs;
2130                 hl_fw_boot_fit_update_state(hdev,
2131                                 le32_to_cpu(dyn_regs->cpu_boot_dev_sts0),
2132                                 le32_to_cpu(dyn_regs->cpu_boot_dev_sts1));
2133         }
2134
2135         /* copy boot fit to space allocated by FW */
2136         rc = hl_fw_dynamic_copy_image(hdev, fw, fw_loader);
2137         if (rc)
2138                 goto release_fw;
2139
2140         rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_DATA_RDY,
2141                                                 0, true,
2142                                                 fw_loader->cpu_timeout);
2143         if (rc)
2144                 goto release_fw;
2145
2146         rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_EXEC,
2147                                                 0, false,
2148                                                 img_ld_timeout);
2149
2150 release_fw:
2151         hl_release_firmware(fw);
2152         return rc;
2153 }
2154
2155 static int hl_fw_dynamic_wait_for_boot_fit_active(struct hl_device *hdev,
2156                                         struct fw_load_mgr *fw_loader)
2157 {
2158         struct dynamic_fw_load_mgr *dyn_loader;
2159         u32 status;
2160         int rc;
2161
2162         dyn_loader = &fw_loader->dynamic_loader;
2163
2164         /*
2165          * Make sure CPU boot-loader is running
2166          * Note that the CPU_BOOT_STATUS_SRAM_AVAIL is generally set by Linux
2167          * yet there is a debug scenario in which we loading uboot (without Linux)
2168          * which at later stage is relocated to DRAM. In this case we expect
2169          * uboot to set the CPU_BOOT_STATUS_SRAM_AVAIL and so we add it to the
2170          * poll flags
2171          */
2172         rc = hl_poll_timeout(
2173                 hdev,
2174                 le32_to_cpu(dyn_loader->comm_desc.cpu_dyn_regs.cpu_boot_status),
2175                 status,
2176                 (status == CPU_BOOT_STATUS_READY_TO_BOOT) ||
2177                 (status == CPU_BOOT_STATUS_SRAM_AVAIL),
2178                 hdev->fw_poll_interval_usec,
2179                 dyn_loader->wait_for_bl_timeout);
2180         if (rc) {
2181                 dev_err(hdev->dev, "failed to wait for boot\n");
2182                 return rc;
2183         }
2184
2185         dev_dbg(hdev->dev, "uboot status = %d\n", status);
2186         return 0;
2187 }
2188
2189 static int hl_fw_dynamic_wait_for_linux_active(struct hl_device *hdev,
2190                                                 struct fw_load_mgr *fw_loader)
2191 {
2192         struct dynamic_fw_load_mgr *dyn_loader;
2193         u32 status;
2194         int rc;
2195
2196         dyn_loader = &fw_loader->dynamic_loader;
2197
2198         /* Make sure CPU linux is running */
2199
2200         rc = hl_poll_timeout(
2201                 hdev,
2202                 le32_to_cpu(dyn_loader->comm_desc.cpu_dyn_regs.cpu_boot_status),
2203                 status,
2204                 (status == CPU_BOOT_STATUS_SRAM_AVAIL),
2205                 hdev->fw_poll_interval_usec,
2206                 fw_loader->cpu_timeout);
2207         if (rc) {
2208                 dev_err(hdev->dev, "failed to wait for Linux\n");
2209                 return rc;
2210         }
2211
2212         dev_dbg(hdev->dev, "Boot status = %d\n", status);
2213         return 0;
2214 }
2215
2216 /**
2217  * hl_fw_linux_update_state -   update internal data structures after Linux
2218  *                              is loaded.
2219  *                              Note: Linux initialization is comprised mainly
2220  *                              of two stages - loading kernel (SRAM_AVAIL)
2221  *                              & loading ARMCP.
2222  *                              Therefore reading boot device status in any of
2223  *                              these stages might result in different values.
2224  *
2225  * @hdev: pointer to the habanalabs device structure
2226  * @cpu_boot_dev_sts0_reg: register holding CPU boot dev status 0
2227  * @cpu_boot_dev_sts1_reg: register holding CPU boot dev status 1
2228  *
2229  * @return 0 on success, otherwise non-zero error code
2230  */
2231 static void hl_fw_linux_update_state(struct hl_device *hdev,
2232                                                 u32 cpu_boot_dev_sts0_reg,
2233                                                 u32 cpu_boot_dev_sts1_reg)
2234 {
2235         struct asic_fixed_properties *prop = &hdev->asic_prop;
2236
2237         hdev->fw_loader.fw_comp_loaded |= FW_TYPE_LINUX;
2238
2239         /* Read FW application security bits */
2240         if (prop->fw_cpu_boot_dev_sts0_valid) {
2241                 prop->fw_app_cpu_boot_dev_sts0 = RREG32(cpu_boot_dev_sts0_reg);
2242
2243                 prop->hard_reset_done_by_fw = !!(prop->fw_app_cpu_boot_dev_sts0 &
2244                                                         CPU_BOOT_DEV_STS0_FW_HARD_RST_EN);
2245
2246                 if (prop->fw_app_cpu_boot_dev_sts0 &
2247                                 CPU_BOOT_DEV_STS0_GIC_PRIVILEGED_EN)
2248                         prop->gic_interrupts_enable = false;
2249
2250                 dev_dbg(hdev->dev,
2251                         "Firmware application CPU status0 %#x\n",
2252                         prop->fw_app_cpu_boot_dev_sts0);
2253
2254                 dev_dbg(hdev->dev, "GIC controller is %s\n",
2255                                 prop->gic_interrupts_enable ?
2256                                                 "enabled" : "disabled");
2257         }
2258
2259         if (prop->fw_cpu_boot_dev_sts1_valid) {
2260                 prop->fw_app_cpu_boot_dev_sts1 = RREG32(cpu_boot_dev_sts1_reg);
2261
2262                 dev_dbg(hdev->dev,
2263                         "Firmware application CPU status1 %#x\n",
2264                         prop->fw_app_cpu_boot_dev_sts1);
2265         }
2266
2267         dev_dbg(hdev->dev, "Firmware application CPU hard-reset is %s\n",
2268                         prop->hard_reset_done_by_fw ? "enabled" : "disabled");
2269
2270         dev_info(hdev->dev, "Successfully loaded firmware to device\n");
2271 }
2272
2273 /**
2274  * hl_fw_dynamic_send_msg - send a COMMS message with attached data
2275  *
2276  * @hdev: pointer to the habanalabs device structure
2277  * @fw_loader: managing structure for loading device's FW
2278  * @msg_type: message type
2279  * @data: data to be sent
2280  *
2281  * @return 0 on success, otherwise non-zero error code
2282  */
2283 static int hl_fw_dynamic_send_msg(struct hl_device *hdev,
2284                 struct fw_load_mgr *fw_loader, u8 msg_type, void *data)
2285 {
2286         struct lkd_msg_comms msg;
2287         int rc;
2288
2289         memset(&msg, 0, sizeof(msg));
2290
2291         /* create message to be sent */
2292         msg.header.type = msg_type;
2293         msg.header.size = cpu_to_le16(sizeof(struct comms_msg_header));
2294         msg.header.magic = cpu_to_le32(HL_COMMS_MSG_MAGIC);
2295
2296         switch (msg_type) {
2297         case HL_COMMS_RESET_CAUSE_TYPE:
2298                 msg.reset_cause = *(__u8 *) data;
2299                 break;
2300         default:
2301                 dev_err(hdev->dev,
2302                         "Send COMMS message - invalid message type %u\n",
2303                         msg_type);
2304                 return -EINVAL;
2305         }
2306
2307         rc = hl_fw_dynamic_request_descriptor(hdev, fw_loader,
2308                         sizeof(struct lkd_msg_comms));
2309         if (rc)
2310                 return rc;
2311
2312         /* copy message to space allocated by FW */
2313         rc = hl_fw_dynamic_copy_msg(hdev, &msg, fw_loader);
2314         if (rc)
2315                 return rc;
2316
2317         rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_DATA_RDY,
2318                                                 0, true,
2319                                                 fw_loader->cpu_timeout);
2320         if (rc)
2321                 return rc;
2322
2323         rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_EXEC,
2324                                                 0, true,
2325                                                 fw_loader->cpu_timeout);
2326         if (rc)
2327                 return rc;
2328
2329         return 0;
2330 }
2331
2332 /**
2333  * hl_fw_dynamic_init_cpu - initialize the device CPU using dynamic protocol
2334  *
2335  * @hdev: pointer to the habanalabs device structure
2336  * @fw_loader: managing structure for loading device's FW
2337  *
2338  * @return 0 on success, otherwise non-zero error code
2339  *
2340  * brief: the dynamic protocol is master (LKD) slave (FW CPU) protocol.
2341  * the communication is done using registers:
2342  * - LKD command register
2343  * - FW status register
2344  * the protocol is race free. this goal is achieved by splitting the requests
2345  * and response to known synchronization points between the LKD and the FW.
2346  * each response to LKD request is known and bound to a predefined timeout.
2347  * in case of timeout expiration without the desired status from FW- the
2348  * protocol (and hence the boot) will fail.
2349  */
2350 static int hl_fw_dynamic_init_cpu(struct hl_device *hdev,
2351                                         struct fw_load_mgr *fw_loader)
2352 {
2353         struct cpu_dyn_regs *dyn_regs;
2354         int rc;
2355
2356         dev_info(hdev->dev,
2357                 "Loading firmware to device, may take some time...\n");
2358
2359         /* initialize FW descriptor as invalid */
2360         fw_loader->dynamic_loader.fw_desc_valid = false;
2361
2362         /*
2363          * In this stage, "cpu_dyn_regs" contains only LKD's hard coded values!
2364          * It will be updated from FW after hl_fw_dynamic_request_descriptor().
2365          */
2366         dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs;
2367
2368         rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_RST_STATE,
2369                                                 0, true,
2370                                                 fw_loader->cpu_timeout);
2371         if (rc)
2372                 goto protocol_err;
2373
2374         if (hdev->reset_info.curr_reset_cause) {
2375                 rc = hl_fw_dynamic_send_msg(hdev, fw_loader,
2376                                 HL_COMMS_RESET_CAUSE_TYPE, &hdev->reset_info.curr_reset_cause);
2377                 if (rc)
2378                         goto protocol_err;
2379
2380                 /* Clear current reset cause */
2381                 hdev->reset_info.curr_reset_cause = HL_RESET_CAUSE_UNKNOWN;
2382         }
2383
2384         if (!(hdev->fw_components & FW_TYPE_BOOT_CPU)) {
2385                 rc = hl_fw_dynamic_request_descriptor(hdev, fw_loader, 0);
2386                 if (rc)
2387                         goto protocol_err;
2388
2389                 /* read preboot version */
2390                 hl_fw_dynamic_read_device_fw_version(hdev, FW_COMP_PREBOOT,
2391                                 fw_loader->dynamic_loader.comm_desc.cur_fw_ver);
2392                 return 0;
2393         }
2394
2395         /* load boot fit to FW */
2396         rc = hl_fw_dynamic_load_image(hdev, fw_loader, FW_COMP_BOOT_FIT,
2397                                                 fw_loader->boot_fit_timeout);
2398         if (rc) {
2399                 dev_err(hdev->dev, "failed to load boot fit\n");
2400                 goto protocol_err;
2401         }
2402
2403         /*
2404          * when testing FW load (without Linux) on PLDM we don't want to
2405          * wait until boot fit is active as it may take several hours.
2406          * instead, we load the bootfit and let it do all initializations in
2407          * the background.
2408          */
2409         if (hdev->pldm && !(hdev->fw_components & FW_TYPE_LINUX))
2410                 return 0;
2411
2412         rc = hl_fw_dynamic_wait_for_boot_fit_active(hdev, fw_loader);
2413         if (rc)
2414                 goto protocol_err;
2415
2416         /* Enable DRAM scrambling before Linux boot and after successful
2417          *  UBoot
2418          */
2419         hdev->asic_funcs->init_cpu_scrambler_dram(hdev);
2420
2421         if (!(hdev->fw_components & FW_TYPE_LINUX)) {
2422                 dev_info(hdev->dev, "Skip loading Linux F/W\n");
2423                 return 0;
2424         }
2425
2426         if (fw_loader->skip_bmc) {
2427                 rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader,
2428                                                         COMMS_SKIP_BMC, 0,
2429                                                         true,
2430                                                         fw_loader->cpu_timeout);
2431                 if (rc) {
2432                         dev_err(hdev->dev, "failed to load boot fit\n");
2433                         goto protocol_err;
2434                 }
2435         }
2436
2437         /* load Linux image to FW */
2438         rc = hl_fw_dynamic_load_image(hdev, fw_loader, FW_COMP_LINUX,
2439                                                         fw_loader->cpu_timeout);
2440         if (rc) {
2441                 dev_err(hdev->dev, "failed to load Linux\n");
2442                 goto protocol_err;
2443         }
2444
2445         rc = hl_fw_dynamic_wait_for_linux_active(hdev, fw_loader);
2446         if (rc)
2447                 goto protocol_err;
2448
2449         hl_fw_linux_update_state(hdev, le32_to_cpu(dyn_regs->cpu_boot_dev_sts0),
2450                                 le32_to_cpu(dyn_regs->cpu_boot_dev_sts1));
2451
2452         hl_fw_dynamic_update_linux_interrupt_if(hdev);
2453
2454         return 0;
2455
2456 protocol_err:
2457         if (fw_loader->dynamic_loader.fw_desc_valid)
2458                 fw_read_errors(hdev, le32_to_cpu(dyn_regs->cpu_boot_err0),
2459                                 le32_to_cpu(dyn_regs->cpu_boot_err1),
2460                                 le32_to_cpu(dyn_regs->cpu_boot_dev_sts0),
2461                                 le32_to_cpu(dyn_regs->cpu_boot_dev_sts1));
2462         return rc;
2463 }
2464
2465 /**
2466  * hl_fw_static_init_cpu - initialize the device CPU using static protocol
2467  *
2468  * @hdev: pointer to the habanalabs device structure
2469  * @fw_loader: managing structure for loading device's FW
2470  *
2471  * @return 0 on success, otherwise non-zero error code
2472  */
2473 static int hl_fw_static_init_cpu(struct hl_device *hdev,
2474                                         struct fw_load_mgr *fw_loader)
2475 {
2476         u32 cpu_msg_status_reg, cpu_timeout, msg_to_cpu_reg, status;
2477         u32 cpu_boot_dev_status0_reg, cpu_boot_dev_status1_reg;
2478         struct static_fw_load_mgr *static_loader;
2479         u32 cpu_boot_status_reg;
2480         int rc;
2481
2482         if (!(hdev->fw_components & FW_TYPE_BOOT_CPU))
2483                 return 0;
2484
2485         /* init common loader parameters */
2486         cpu_timeout = fw_loader->cpu_timeout;
2487
2488         /* init static loader parameters */
2489         static_loader = &fw_loader->static_loader;
2490         cpu_msg_status_reg = static_loader->cpu_cmd_status_to_host_reg;
2491         msg_to_cpu_reg = static_loader->kmd_msg_to_cpu_reg;
2492         cpu_boot_dev_status0_reg = static_loader->cpu_boot_dev_status0_reg;
2493         cpu_boot_dev_status1_reg = static_loader->cpu_boot_dev_status1_reg;
2494         cpu_boot_status_reg = static_loader->cpu_boot_status_reg;
2495
2496         dev_info(hdev->dev, "Going to wait for device boot (up to %lds)\n",
2497                 cpu_timeout / USEC_PER_SEC);
2498
2499         /* Wait for boot FIT request */
2500         rc = hl_poll_timeout(
2501                 hdev,
2502                 cpu_boot_status_reg,
2503                 status,
2504                 status == CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT,
2505                 hdev->fw_poll_interval_usec,
2506                 fw_loader->boot_fit_timeout);
2507
2508         if (rc) {
2509                 dev_dbg(hdev->dev,
2510                         "No boot fit request received, resuming boot\n");
2511         } else {
2512                 rc = hdev->asic_funcs->load_boot_fit_to_device(hdev);
2513                 if (rc)
2514                         goto out;
2515
2516                 /* Clear device CPU message status */
2517                 WREG32(cpu_msg_status_reg, CPU_MSG_CLR);
2518
2519                 /* Signal device CPU that boot loader is ready */
2520                 WREG32(msg_to_cpu_reg, KMD_MSG_FIT_RDY);
2521
2522                 /* Poll for CPU device ack */
2523                 rc = hl_poll_timeout(
2524                         hdev,
2525                         cpu_msg_status_reg,
2526                         status,
2527                         status == CPU_MSG_OK,
2528                         hdev->fw_poll_interval_usec,
2529                         fw_loader->boot_fit_timeout);
2530
2531                 if (rc) {
2532                         dev_err(hdev->dev,
2533                                 "Timeout waiting for boot fit load ack\n");
2534                         goto out;
2535                 }
2536
2537                 /* Clear message */
2538                 WREG32(msg_to_cpu_reg, KMD_MSG_NA);
2539         }
2540
2541         /*
2542          * Make sure CPU boot-loader is running
2543          * Note that the CPU_BOOT_STATUS_SRAM_AVAIL is generally set by Linux
2544          * yet there is a debug scenario in which we loading uboot (without Linux)
2545          * which at later stage is relocated to DRAM. In this case we expect
2546          * uboot to set the CPU_BOOT_STATUS_SRAM_AVAIL and so we add it to the
2547          * poll flags
2548          */
2549         rc = hl_poll_timeout(
2550                 hdev,
2551                 cpu_boot_status_reg,
2552                 status,
2553                 (status == CPU_BOOT_STATUS_DRAM_RDY) ||
2554                 (status == CPU_BOOT_STATUS_NIC_FW_RDY) ||
2555                 (status == CPU_BOOT_STATUS_READY_TO_BOOT) ||
2556                 (status == CPU_BOOT_STATUS_SRAM_AVAIL),
2557                 hdev->fw_poll_interval_usec,
2558                 cpu_timeout);
2559
2560         dev_dbg(hdev->dev, "uboot status = %d\n", status);
2561
2562         /* Read U-Boot version now in case we will later fail */
2563         hl_fw_static_read_device_fw_version(hdev, FW_COMP_BOOT_FIT);
2564
2565         /* update state according to boot stage */
2566         hl_fw_boot_fit_update_state(hdev, cpu_boot_dev_status0_reg,
2567                                                 cpu_boot_dev_status1_reg);
2568
2569         if (rc) {
2570                 detect_cpu_boot_status(hdev, status);
2571                 rc = -EIO;
2572                 goto out;
2573         }
2574
2575         /* Enable DRAM scrambling before Linux boot and after successful
2576          *  UBoot
2577          */
2578         hdev->asic_funcs->init_cpu_scrambler_dram(hdev);
2579
2580         if (!(hdev->fw_components & FW_TYPE_LINUX)) {
2581                 dev_info(hdev->dev, "Skip loading Linux F/W\n");
2582                 rc = 0;
2583                 goto out;
2584         }
2585
2586         if (status == CPU_BOOT_STATUS_SRAM_AVAIL) {
2587                 rc = 0;
2588                 goto out;
2589         }
2590
2591         dev_info(hdev->dev,
2592                 "Loading firmware to device, may take some time...\n");
2593
2594         rc = hdev->asic_funcs->load_firmware_to_device(hdev);
2595         if (rc)
2596                 goto out;
2597
2598         if (fw_loader->skip_bmc) {
2599                 WREG32(msg_to_cpu_reg, KMD_MSG_SKIP_BMC);
2600
2601                 rc = hl_poll_timeout(
2602                         hdev,
2603                         cpu_boot_status_reg,
2604                         status,
2605                         (status == CPU_BOOT_STATUS_BMC_WAITING_SKIPPED),
2606                         hdev->fw_poll_interval_usec,
2607                         cpu_timeout);
2608
2609                 if (rc) {
2610                         dev_err(hdev->dev,
2611                                 "Failed to get ACK on skipping BMC, %d\n",
2612                                 status);
2613                         WREG32(msg_to_cpu_reg, KMD_MSG_NA);
2614                         rc = -EIO;
2615                         goto out;
2616                 }
2617         }
2618
2619         WREG32(msg_to_cpu_reg, KMD_MSG_FIT_RDY);
2620
2621         rc = hl_poll_timeout(
2622                 hdev,
2623                 cpu_boot_status_reg,
2624                 status,
2625                 (status == CPU_BOOT_STATUS_SRAM_AVAIL),
2626                 hdev->fw_poll_interval_usec,
2627                 cpu_timeout);
2628
2629         /* Clear message */
2630         WREG32(msg_to_cpu_reg, KMD_MSG_NA);
2631
2632         if (rc) {
2633                 if (status == CPU_BOOT_STATUS_FIT_CORRUPTED)
2634                         dev_err(hdev->dev,
2635                                 "Device reports FIT image is corrupted\n");
2636                 else
2637                         dev_err(hdev->dev,
2638                                 "Failed to load firmware to device, %d\n",
2639                                 status);
2640
2641                 rc = -EIO;
2642                 goto out;
2643         }
2644
2645         rc = fw_read_errors(hdev, fw_loader->static_loader.boot_err0_reg,
2646                                         fw_loader->static_loader.boot_err1_reg,
2647                                         cpu_boot_dev_status0_reg,
2648                                         cpu_boot_dev_status1_reg);
2649         if (rc)
2650                 return rc;
2651
2652         hl_fw_linux_update_state(hdev, cpu_boot_dev_status0_reg,
2653                                                 cpu_boot_dev_status1_reg);
2654
2655         return 0;
2656
2657 out:
2658         fw_read_errors(hdev, fw_loader->static_loader.boot_err0_reg,
2659                                         fw_loader->static_loader.boot_err1_reg,
2660                                         cpu_boot_dev_status0_reg,
2661                                         cpu_boot_dev_status1_reg);
2662
2663         return rc;
2664 }
2665
2666 /**
2667  * hl_fw_init_cpu - initialize the device CPU
2668  *
2669  * @hdev: pointer to the habanalabs device structure
2670  *
2671  * @return 0 on success, otherwise non-zero error code
2672  *
2673  * perform necessary initializations for device's CPU. takes into account if
2674  * init protocol is static or dynamic.
2675  */
2676 int hl_fw_init_cpu(struct hl_device *hdev)
2677 {
2678         struct asic_fixed_properties *prop = &hdev->asic_prop;
2679         struct fw_load_mgr *fw_loader = &hdev->fw_loader;
2680
2681         return  prop->dynamic_fw_load ?
2682                         hl_fw_dynamic_init_cpu(hdev, fw_loader) :
2683                         hl_fw_static_init_cpu(hdev, fw_loader);
2684 }
2685
2686 void hl_fw_set_pll_profile(struct hl_device *hdev)
2687 {
2688         hl_fw_set_frequency(hdev, hdev->asic_prop.clk_pll_index,
2689                                 hdev->asic_prop.max_freq_value);
2690 }
2691
2692 int hl_fw_get_clk_rate(struct hl_device *hdev, u32 *cur_clk, u32 *max_clk)
2693 {
2694         long value;
2695
2696         if (!hl_device_operational(hdev, NULL))
2697                 return -ENODEV;
2698
2699         if (!hdev->pdev) {
2700                 *cur_clk = 0;
2701                 *max_clk = 0;
2702                 return 0;
2703         }
2704
2705         value = hl_fw_get_frequency(hdev, hdev->asic_prop.clk_pll_index, false);
2706
2707         if (value < 0) {
2708                 dev_err(hdev->dev, "Failed to retrieve device max clock %ld\n", value);
2709                 return value;
2710         }
2711
2712         *max_clk = (value / 1000 / 1000);
2713
2714         value = hl_fw_get_frequency(hdev, hdev->asic_prop.clk_pll_index, true);
2715
2716         if (value < 0) {
2717                 dev_err(hdev->dev, "Failed to retrieve device current clock %ld\n", value);
2718                 return value;
2719         }
2720
2721         *cur_clk = (value / 1000 / 1000);
2722
2723         return 0;
2724 }
2725
2726 long hl_fw_get_frequency(struct hl_device *hdev, u32 pll_index, bool curr)
2727 {
2728         struct cpucp_packet pkt;
2729         u32 used_pll_idx;
2730         u64 result;
2731         int rc;
2732
2733         rc = get_used_pll_index(hdev, pll_index, &used_pll_idx);
2734         if (rc)
2735                 return rc;
2736
2737         memset(&pkt, 0, sizeof(pkt));
2738
2739         if (curr)
2740                 pkt.ctl = cpu_to_le32(CPUCP_PACKET_FREQUENCY_CURR_GET <<
2741                                                 CPUCP_PKT_CTL_OPCODE_SHIFT);
2742         else
2743                 pkt.ctl = cpu_to_le32(CPUCP_PACKET_FREQUENCY_GET << CPUCP_PKT_CTL_OPCODE_SHIFT);
2744
2745         pkt.pll_index = cpu_to_le32((u32)used_pll_idx);
2746
2747         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, &result);
2748
2749         if (rc) {
2750                 dev_err(hdev->dev, "Failed to get frequency of PLL %d, error %d\n",
2751                         used_pll_idx, rc);
2752                 return rc;
2753         }
2754
2755         return (long) result;
2756 }
2757
2758 void hl_fw_set_frequency(struct hl_device *hdev, u32 pll_index, u64 freq)
2759 {
2760         struct cpucp_packet pkt;
2761         u32 used_pll_idx;
2762         int rc;
2763
2764         rc = get_used_pll_index(hdev, pll_index, &used_pll_idx);
2765         if (rc)
2766                 return;
2767
2768         memset(&pkt, 0, sizeof(pkt));
2769
2770         pkt.ctl = cpu_to_le32(CPUCP_PACKET_FREQUENCY_SET << CPUCP_PKT_CTL_OPCODE_SHIFT);
2771         pkt.pll_index = cpu_to_le32((u32)used_pll_idx);
2772         pkt.value = cpu_to_le64(freq);
2773
2774         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, NULL);
2775
2776         if (rc)
2777                 dev_err(hdev->dev, "Failed to set frequency to PLL %d, error %d\n",
2778                         used_pll_idx, rc);
2779 }
2780
2781 u64 hl_fw_get_max_power(struct hl_device *hdev)
2782 {
2783         struct cpucp_packet pkt;
2784         u64 result;
2785         int rc;
2786
2787         memset(&pkt, 0, sizeof(pkt));
2788
2789         pkt.ctl = cpu_to_le32(CPUCP_PACKET_MAX_POWER_GET << CPUCP_PKT_CTL_OPCODE_SHIFT);
2790
2791         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, &result);
2792
2793         if (rc) {
2794                 dev_err(hdev->dev, "Failed to get max power, error %d\n", rc);
2795                 return (u64) rc;
2796         }
2797
2798         return result;
2799 }
2800
2801 void hl_fw_set_max_power(struct hl_device *hdev)
2802 {
2803         struct cpucp_packet pkt;
2804         int rc;
2805
2806         /* TODO: remove this after simulator supports this packet */
2807         if (!hdev->pdev)
2808                 return;
2809
2810         memset(&pkt, 0, sizeof(pkt));
2811
2812         pkt.ctl = cpu_to_le32(CPUCP_PACKET_MAX_POWER_SET << CPUCP_PKT_CTL_OPCODE_SHIFT);
2813         pkt.value = cpu_to_le64(hdev->max_power);
2814
2815         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, NULL);
2816
2817         if (rc)
2818                 dev_err(hdev->dev, "Failed to set max power, error %d\n", rc);
2819 }