drivers/misc/habanalabs/common/firmware_if.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 /*
   4  * Copyright 2016-2022 HabanaLabs, Ltd.
   5  * All Rights Reserved.
   6  */
   7
   8 #include "habanalabs.h"
   9 #include "../include/common/hl_boot_if.h"
  10
  11 #include <linux/firmware.h>
  12 #include <linux/crc32.h>
  13 #include <linux/slab.h>
  14 #include <linux/ctype.h>
  15
  16 #define FW_FILE_MAX_SIZE                0x1400000 /* maximum size of 20MB */
  17
  18 static char *extract_fw_ver_from_str(const char *fw_str)
  19 {
  20         char *str, *fw_ver, *whitespace;
  21
  22         fw_ver = kmalloc(16, GFP_KERNEL);
  23         if (!fw_ver)
  24                 return NULL;
  25
  26         str = strnstr(fw_str, "fw-", VERSION_MAX_LEN);
  27         if (!str)
  28                 goto free_fw_ver;
  29
  30         /* Skip the fw- part */
  31         str += 3;
  32
  33         /* Copy until the next whitespace */
  34         whitespace =  strnstr(str, " ", 15);
  35         if (!whitespace)
  36                 goto free_fw_ver;
  37
  38         strscpy(fw_ver, str, whitespace - str + 1);
  39
  40         return fw_ver;
  41
  42 free_fw_ver:
  43         kfree(fw_ver);
  44         return NULL;
  45 }
  46
  47 static int hl_request_fw(struct hl_device *hdev,
  48                                 const struct firmware **firmware_p,
  49                                 const char *fw_name)
  50 {
  51         size_t fw_size;
  52         int rc;
  53
  54         rc = request_firmware(firmware_p, fw_name, hdev->dev);
  55         if (rc) {
  56                 dev_err(hdev->dev, "Firmware file %s is not found! (error %d)\n",
  57                                 fw_name, rc);
  58                 goto out;
  59         }
  60
  61         fw_size = (*firmware_p)->size;
  62         if ((fw_size % 4) != 0) {
  63                 dev_err(hdev->dev, "Illegal %s firmware size %zu\n",
  64                                 fw_name, fw_size);
  65                 rc = -EINVAL;
  66                 goto release_fw;
  67         }
  68
  69         dev_dbg(hdev->dev, "%s firmware size == %zu\n", fw_name, fw_size);
  70
  71         if (fw_size > FW_FILE_MAX_SIZE) {
  72                 dev_err(hdev->dev,
  73                         "FW file size %zu exceeds maximum of %u bytes\n",
  74                         fw_size, FW_FILE_MAX_SIZE);
  75                 rc = -EINVAL;
  76                 goto release_fw;
  77         }
  78
  79         return 0;
  80
  81 release_fw:
  82         release_firmware(*firmware_p);
  83 out:
  84         return rc;
  85 }
  86
  87 /**
  88  * hl_release_firmware() - release FW
  89  *
  90  * @fw: fw descriptor
  91  *
  92  * note: this inline function added to serve as a comprehensive mirror for the
  93  *       hl_request_fw function.
  94  */
  95 static inline void hl_release_firmware(const struct firmware *fw)
  96 {
  97         release_firmware(fw);
  98 }
  99
 100 /**
 101  * hl_fw_copy_fw_to_device() - copy FW to device
 102  *
 103  * @hdev: pointer to hl_device structure.
 104  * @fw: fw descriptor
 105  * @dst: IO memory mapped address space to copy firmware to
 106  * @src_offset: offset in src FW to copy from
 107  * @size: amount of bytes to copy (0 to copy the whole binary)
 108  *
 109  * actual copy of FW binary data to device, shared by static and dynamic loaders
 110  */
 111 static int hl_fw_copy_fw_to_device(struct hl_device *hdev,
 112                                 const struct firmware *fw, void __iomem *dst,
 113                                 u32 src_offset, u32 size)
 114 {
 115         const void *fw_data;
 116
 117         /* size 0 indicates to copy the whole file */
 118         if (!size)
 119                 size = fw->size;
 120
 121         if (src_offset + size > fw->size) {
 122                 dev_err(hdev->dev,
 123                         "size to copy(%u) and offset(%u) are invalid\n",
 124                         size, src_offset);
 125                 return -EINVAL;
 126         }
 127
 128         fw_data = (const void *) fw->data;
 129
 130         memcpy_toio(dst, fw_data + src_offset, size);
 131         return 0;
 132 }
 133
 134 /**
 135  * hl_fw_copy_msg_to_device() - copy message to device
 136  *
 137  * @hdev: pointer to hl_device structure.
 138  * @msg: message
 139  * @dst: IO memory mapped address space to copy firmware to
 140  * @src_offset: offset in src message to copy from
 141  * @size: amount of bytes to copy (0 to copy the whole binary)
 142  *
 143  * actual copy of message data to device.
 144  */
 145 static int hl_fw_copy_msg_to_device(struct hl_device *hdev,
 146                 struct lkd_msg_comms *msg, void __iomem *dst,
 147                 u32 src_offset, u32 size)
 148 {
 149         void *msg_data;
 150
 151         /* size 0 indicates to copy the whole file */
 152         if (!size)
 153                 size = sizeof(struct lkd_msg_comms);
 154
 155         if (src_offset + size > sizeof(struct lkd_msg_comms)) {
 156                 dev_err(hdev->dev,
 157                         "size to copy(%u) and offset(%u) are invalid\n",
 158                         size, src_offset);
 159                 return -EINVAL;
 160         }
 161
 162         msg_data = (void *) msg;
 163
 164         memcpy_toio(dst, msg_data + src_offset, size);
 165
 166         return 0;
 167 }
 168
 169 /**
 170  * hl_fw_load_fw_to_device() - Load F/W code to device's memory.
 171  *
 172  * @hdev: pointer to hl_device structure.
 173  * @fw_name: the firmware image name
 174  * @dst: IO memory mapped address space to copy firmware to
 175  * @src_offset: offset in src FW to copy from
 176  * @size: amount of bytes to copy (0 to copy the whole binary)
 177  *
 178  * Copy fw code from firmware file to device memory.
 179  *
 180  * Return: 0 on success, non-zero for failure.
 181  */
 182 int hl_fw_load_fw_to_device(struct hl_device *hdev, const char *fw_name,
 183                                 void __iomem *dst, u32 src_offset, u32 size)
 184 {
 185         const struct firmware *fw;
 186         int rc;
 187
 188         rc = hl_request_fw(hdev, &fw, fw_name);
 189         if (rc)
 190                 return rc;
 191
 192         rc = hl_fw_copy_fw_to_device(hdev, fw, dst, src_offset, size);
 193
 194         hl_release_firmware(fw);
 195         return rc;
 196 }
 197
 198 int hl_fw_send_pci_access_msg(struct hl_device *hdev, u32 opcode)
 199 {
 200         struct cpucp_packet pkt = {};
 201
 202         pkt.ctl = cpu_to_le32(opcode << CPUCP_PKT_CTL_OPCODE_SHIFT);
 203
 204         return hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt,
 205                                                 sizeof(pkt), 0, NULL);
 206 }
 207
 208 int hl_fw_send_cpu_message(struct hl_device *hdev, u32 hw_queue_id, u32 *msg,
 209                                 u16 len, u32 timeout, u64 *result)
 210 {
 211         struct hl_hw_queue *queue = &hdev->kernel_queues[hw_queue_id];
 212         struct asic_fixed_properties *prop = &hdev->asic_prop;
 213         struct cpucp_packet *pkt;
 214         dma_addr_t pkt_dma_addr;
 215         struct hl_bd *sent_bd;
 216         u32 tmp, expected_ack_val, pi;
 217         int rc;
 218
 219         pkt = hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev, len,
 220                                                                 &pkt_dma_addr);
 221         if (!pkt) {
 222                 dev_err(hdev->dev,
 223                         "Failed to allocate DMA memory for packet to CPU\n");
 224                 return -ENOMEM;
 225         }
 226
 227         memcpy(pkt, msg, len);
 228
 229         mutex_lock(&hdev->send_cpu_message_lock);
 230
 231         /* CPU-CP messages can be sent during soft-reset */
 232         if (hdev->disabled && !hdev->reset_info.is_in_soft_reset) {
 233                 rc = 0;
 234                 goto out;
 235         }
 236
 237         if (hdev->device_cpu_disabled) {
 238                 rc = -EIO;
 239                 goto out;
 240         }
 241
 242         /* set fence to a non valid value */
 243         pkt->fence = cpu_to_le32(UINT_MAX);
 244         pi = queue->pi;
 245
 246         /*
 247          * The CPU queue is a synchronous queue with an effective depth of
 248          * a single entry (although it is allocated with room for multiple
 249          * entries). We lock on it using 'send_cpu_message_lock' which
 250          * serializes accesses to the CPU queue.
 251          * Which means that we don't need to lock the access to the entire H/W
 252          * queues module when submitting a JOB to the CPU queue.
 253          */
 254         hl_hw_queue_submit_bd(hdev, queue, hl_queue_inc_ptr(queue->pi), len, pkt_dma_addr);
 255
 256         if (prop->fw_app_cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_PKT_PI_ACK_EN)
 257                 expected_ack_val = queue->pi;
 258         else
 259                 expected_ack_val = CPUCP_PACKET_FENCE_VAL;
 260
 261         rc = hl_poll_timeout_memory(hdev, &pkt->fence, tmp,
 262                                 (tmp == expected_ack_val), 1000,
 263                                 timeout, true);
 264
 265         hl_hw_queue_inc_ci_kernel(hdev, hw_queue_id);
 266
 267         if (rc == -ETIMEDOUT) {
 268                 dev_err(hdev->dev, "Device CPU packet timeout (0x%x)\n", tmp);
 269                 hdev->device_cpu_disabled = true;
 270                 goto out;
 271         }
 272
 273         tmp = le32_to_cpu(pkt->ctl);
 274
 275         rc = (tmp & CPUCP_PKT_CTL_RC_MASK) >> CPUCP_PKT_CTL_RC_SHIFT;
 276         if (rc) {
 277                 dev_err(hdev->dev, "F/W ERROR %d for CPU packet %d\n",
 278                         rc,
 279                         (tmp & CPUCP_PKT_CTL_OPCODE_MASK)
 280                                                 >> CPUCP_PKT_CTL_OPCODE_SHIFT);
 281                 rc = -EIO;
 282         } else if (result) {
 283                 *result = le64_to_cpu(pkt->result);
 284         }
 285
 286         /* Scrub previous buffer descriptor 'ctl' field which contains the
 287          * previous PI value written during packet submission.
 288          * We must do this or else F/W can read an old value upon queue wraparound.
 289          */
 290         sent_bd = queue->kernel_address;
 291         sent_bd += hl_pi_2_offset(pi);
 292         sent_bd->ctl = cpu_to_le32(UINT_MAX);
 293
 294 out:
 295         mutex_unlock(&hdev->send_cpu_message_lock);
 296
 297         hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev, len, pkt);
 298
 299         return rc;
 300 }
 301
 302 int hl_fw_unmask_irq(struct hl_device *hdev, u16 event_type)
 303 {
 304         struct cpucp_packet pkt;
 305         u64 result;
 306         int rc;
 307
 308         memset(&pkt, 0, sizeof(pkt));
 309
 310         pkt.ctl = cpu_to_le32(CPUCP_PACKET_UNMASK_RAZWI_IRQ <<
 311                                 CPUCP_PKT_CTL_OPCODE_SHIFT);
 312         pkt.value = cpu_to_le64(event_type);
 313
 314         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
 315                                                 0, &result);
 316
 317         if (rc)
 318                 dev_err(hdev->dev, "failed to unmask RAZWI IRQ %d", event_type);
 319
 320         return rc;
 321 }
 322
 323 int hl_fw_unmask_irq_arr(struct hl_device *hdev, const u32 *irq_arr,
 324                 size_t irq_arr_size)
 325 {
 326         struct cpucp_unmask_irq_arr_packet *pkt;
 327         size_t total_pkt_size;
 328         u64 result;
 329         int rc;
 330
 331         total_pkt_size = sizeof(struct cpucp_unmask_irq_arr_packet) +
 332                         irq_arr_size;
 333
 334         /* data should be aligned to 8 bytes in order to CPU-CP to copy it */
 335         total_pkt_size = (total_pkt_size + 0x7) & ~0x7;
 336
 337         /* total_pkt_size is casted to u16 later on */
 338         if (total_pkt_size > USHRT_MAX) {
 339                 dev_err(hdev->dev, "too many elements in IRQ array\n");
 340                 return -EINVAL;
 341         }
 342
 343         pkt = kzalloc(total_pkt_size, GFP_KERNEL);
 344         if (!pkt)
 345                 return -ENOMEM;
 346
 347         pkt->length = cpu_to_le32(irq_arr_size / sizeof(irq_arr[0]));
 348         memcpy(&pkt->irqs, irq_arr, irq_arr_size);
 349
 350         pkt->cpucp_pkt.ctl = cpu_to_le32(CPUCP_PACKET_UNMASK_RAZWI_IRQ_ARRAY <<
 351                                                 CPUCP_PKT_CTL_OPCODE_SHIFT);
 352
 353         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) pkt,
 354                                                 total_pkt_size, 0, &result);
 355
 356         if (rc)
 357                 dev_err(hdev->dev, "failed to unmask IRQ array\n");
 358
 359         kfree(pkt);
 360
 361         return rc;
 362 }
 363
 364 int hl_fw_test_cpu_queue(struct hl_device *hdev)
 365 {
 366         struct cpucp_packet test_pkt = {};
 367         u64 result;
 368         int rc;
 369
 370         test_pkt.ctl = cpu_to_le32(CPUCP_PACKET_TEST <<
 371                                         CPUCP_PKT_CTL_OPCODE_SHIFT);
 372         test_pkt.value = cpu_to_le64(CPUCP_PACKET_FENCE_VAL);
 373
 374         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &test_pkt,
 375                                                 sizeof(test_pkt), 0, &result);
 376
 377         if (!rc) {
 378                 if (result != CPUCP_PACKET_FENCE_VAL)
 379                         dev_err(hdev->dev,
 380                                 "CPU queue test failed (%#08llx)\n", result);
 381         } else {
 382                 dev_err(hdev->dev, "CPU queue test failed, error %d\n", rc);
 383         }
 384
 385         return rc;
 386 }
 387
 388 void *hl_fw_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size,
 389                                                 dma_addr_t *dma_handle)
 390 {
 391         u64 kernel_addr;
 392
 393         kernel_addr = gen_pool_alloc(hdev->cpu_accessible_dma_pool, size);
 394
 395         *dma_handle = hdev->cpu_accessible_dma_address +
 396                 (kernel_addr - (u64) (uintptr_t) hdev->cpu_accessible_dma_mem);
 397
 398         return (void *) (uintptr_t) kernel_addr;
 399 }
 400
 401 void hl_fw_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size,
 402                                         void *vaddr)
 403 {
 404         gen_pool_free(hdev->cpu_accessible_dma_pool, (u64) (uintptr_t) vaddr,
 405                         size);
 406 }
 407
 408 int hl_fw_send_heartbeat(struct hl_device *hdev)
 409 {
 410         struct cpucp_packet hb_pkt;
 411         u64 result;
 412         int rc;
 413
 414         memset(&hb_pkt, 0, sizeof(hb_pkt));
 415         hb_pkt.ctl = cpu_to_le32(CPUCP_PACKET_TEST <<
 416                                         CPUCP_PKT_CTL_OPCODE_SHIFT);
 417         hb_pkt.value = cpu_to_le64(CPUCP_PACKET_FENCE_VAL);
 418
 419         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &hb_pkt,
 420                                                 sizeof(hb_pkt), 0, &result);
 421
 422         if ((rc) || (result != CPUCP_PACKET_FENCE_VAL))
 423                 return -EIO;
 424
 425         if (le32_to_cpu(hb_pkt.status_mask) &
 426                                         CPUCP_PKT_HB_STATUS_EQ_FAULT_MASK) {
 427                 dev_warn(hdev->dev, "FW reported EQ fault during heartbeat\n");
 428                 rc = -EIO;
 429         }
 430
 431         return rc;
 432 }
 433
 434 static bool fw_report_boot_dev0(struct hl_device *hdev, u32 err_val,
 435                                                                 u32 sts_val)
 436 {
 437         bool err_exists = false;
 438
 439         if (!(err_val & CPU_BOOT_ERR0_ENABLED))
 440                 return false;
 441
 442         if (err_val & CPU_BOOT_ERR0_DRAM_INIT_FAIL) {
 443                 dev_err(hdev->dev,
 444                         "Device boot error - DRAM initialization failed\n");
 445                 err_exists = true;
 446         }
 447
 448         if (err_val & CPU_BOOT_ERR0_FIT_CORRUPTED) {
 449                 dev_err(hdev->dev, "Device boot error - FIT image corrupted\n");
 450                 err_exists = true;
 451         }
 452
 453         if (err_val & CPU_BOOT_ERR0_TS_INIT_FAIL) {
 454                 dev_err(hdev->dev,
 455                         "Device boot error - Thermal Sensor initialization failed\n");
 456                 err_exists = true;
 457         }
 458
 459         if (err_val & CPU_BOOT_ERR0_BMC_WAIT_SKIPPED) {
 460                 if (hdev->bmc_enable) {
 461                         dev_err(hdev->dev,
 462                                 "Device boot error - Skipped waiting for BMC\n");
 463                         err_exists = true;
 464                 } else {
 465                         dev_info(hdev->dev,
 466                                 "Device boot message - Skipped waiting for BMC\n");
 467                         /* This is an info so we don't want it to disable the
 468                          * device
 469                          */
 470                         err_val &= ~CPU_BOOT_ERR0_BMC_WAIT_SKIPPED;
 471                 }
 472         }
 473
 474         if (err_val & CPU_BOOT_ERR0_NIC_DATA_NOT_RDY) {
 475                 dev_err(hdev->dev,
 476                         "Device boot error - Serdes data from BMC not available\n");
 477                 err_exists = true;
 478         }
 479
 480         if (err_val & CPU_BOOT_ERR0_NIC_FW_FAIL) {
 481                 dev_err(hdev->dev,
 482                         "Device boot error - NIC F/W initialization failed\n");
 483                 err_exists = true;
 484         }
 485
 486         if (err_val & CPU_BOOT_ERR0_SECURITY_NOT_RDY) {
 487                 dev_err(hdev->dev,
 488                         "Device boot warning - security not ready\n");
 489                 err_exists = true;
 490         }
 491
 492         if (err_val & CPU_BOOT_ERR0_SECURITY_FAIL) {
 493                 dev_err(hdev->dev, "Device boot error - security failure\n");
 494                 err_exists = true;
 495         }
 496
 497         if (err_val & CPU_BOOT_ERR0_EFUSE_FAIL) {
 498                 dev_err(hdev->dev, "Device boot error - eFuse failure\n");
 499                 err_exists = true;
 500         }
 501
 502         if (err_val & CPU_BOOT_ERR0_SEC_IMG_VER_FAIL) {
 503                 dev_err(hdev->dev, "Device boot error - Failed to load preboot secondary image\n");
 504                 err_exists = true;
 505         }
 506
 507         if (err_val & CPU_BOOT_ERR0_PLL_FAIL) {
 508                 dev_err(hdev->dev, "Device boot error - PLL failure\n");
 509                 err_exists = true;
 510         }
 511
 512         if (err_val & CPU_BOOT_ERR0_DEVICE_UNUSABLE_FAIL) {
 513                 /* Ignore this bit, don't prevent driver loading */
 514                 dev_dbg(hdev->dev, "device unusable status is set\n");
 515                 err_val &= ~CPU_BOOT_ERR0_DEVICE_UNUSABLE_FAIL;
 516         }
 517
 518         if (sts_val & CPU_BOOT_DEV_STS0_ENABLED)
 519                 dev_dbg(hdev->dev, "Device status0 %#x\n", sts_val);
 520
 521         /* All warnings should go here in order not to reach the unknown error validation */
 522         if (err_val & CPU_BOOT_ERR0_DRAM_SKIPPED) {
 523                 dev_warn(hdev->dev,
 524                         "Device boot warning - Skipped DRAM initialization\n");
 525                 /* This is a warning so we don't want it to disable the
 526                  * device
 527                  */
 528                 err_val &= ~CPU_BOOT_ERR0_DRAM_SKIPPED;
 529         }
 530
 531         if (err_val & CPU_BOOT_ERR0_PRI_IMG_VER_FAIL) {
 532                 dev_warn(hdev->dev,
 533                         "Device boot warning - Failed to load preboot primary image\n");
 534                 /* This is a warning so we don't want it to disable the
 535                  * device as we have a secondary preboot image
 536                  */
 537                 err_val &= ~CPU_BOOT_ERR0_PRI_IMG_VER_FAIL;
 538         }
 539
 540         if (err_val & CPU_BOOT_ERR0_TPM_FAIL) {
 541                 dev_warn(hdev->dev,
 542                         "Device boot warning - TPM failure\n");
 543                 /* This is a warning so we don't want it to disable the
 544                  * device
 545                  */
 546                 err_val &= ~CPU_BOOT_ERR0_TPM_FAIL;
 547         }
 548
 549         if (!err_exists && (err_val & ~CPU_BOOT_ERR0_ENABLED)) {
 550                 dev_err(hdev->dev,
 551                         "Device boot error - unknown ERR0 error 0x%08x\n", err_val);
 552                 err_exists = true;
 553         }
 554
 555         /* return error only if it's in the predefined mask */
 556         if (err_exists && ((err_val & ~CPU_BOOT_ERR0_ENABLED) &
 557                                 lower_32_bits(hdev->boot_error_status_mask)))
 558                 return true;
 559
 560         return false;
 561 }
 562
 563 /* placeholder for ERR1 as no errors defined there yet */
 564 static bool fw_report_boot_dev1(struct hl_device *hdev, u32 err_val,
 565                                                                 u32 sts_val)
 566 {
 567         /*
 568          * keep this variable to preserve the logic of the function.
 569          * this way it would require less modifications when error will be
 570          * added to DEV_ERR1
 571          */
 572         bool err_exists = false;
 573
 574         if (!(err_val & CPU_BOOT_ERR1_ENABLED))
 575                 return false;
 576
 577         if (sts_val & CPU_BOOT_DEV_STS1_ENABLED)
 578                 dev_dbg(hdev->dev, "Device status1 %#x\n", sts_val);
 579
 580         if (!err_exists && (err_val & ~CPU_BOOT_ERR1_ENABLED)) {
 581                 dev_err(hdev->dev,
 582                         "Device boot error - unknown ERR1 error 0x%08x\n",
 583                                                                 err_val);
 584                 err_exists = true;
 585         }
 586
 587         /* return error only if it's in the predefined mask */
 588         if (err_exists && ((err_val & ~CPU_BOOT_ERR1_ENABLED) &
 589                                 upper_32_bits(hdev->boot_error_status_mask)))
 590                 return true;
 591
 592         return false;
 593 }
 594
 595 static int fw_read_errors(struct hl_device *hdev, u32 boot_err0_reg,
 596                                 u32 boot_err1_reg, u32 cpu_boot_dev_status0_reg,
 597                                 u32 cpu_boot_dev_status1_reg)
 598 {
 599         u32 err_val, status_val;
 600         bool err_exists = false;
 601
 602         /* Some of the firmware status codes are deprecated in newer f/w
 603          * versions. In those versions, the errors are reported
 604          * in different registers. Therefore, we need to check those
 605          * registers and print the exact errors. Moreover, there
 606          * may be multiple errors, so we need to report on each error
 607          * separately. Some of the error codes might indicate a state
 608          * that is not an error per-se, but it is an error in production
 609          * environment
 610          */
 611         err_val = RREG32(boot_err0_reg);
 612         status_val = RREG32(cpu_boot_dev_status0_reg);
 613         err_exists = fw_report_boot_dev0(hdev, err_val, status_val);
 614
 615         err_val = RREG32(boot_err1_reg);
 616         status_val = RREG32(cpu_boot_dev_status1_reg);
 617         err_exists |= fw_report_boot_dev1(hdev, err_val, status_val);
 618
 619         if (err_exists)
 620                 return -EIO;
 621
 622         return 0;
 623 }
 624
 625 int hl_fw_cpucp_info_get(struct hl_device *hdev,
 626                                 u32 sts_boot_dev_sts0_reg,
 627                                 u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg,
 628                                 u32 boot_err1_reg)
 629 {
 630         struct asic_fixed_properties *prop = &hdev->asic_prop;
 631         struct cpucp_packet pkt = {};
 632         dma_addr_t cpucp_info_dma_addr;
 633         void *cpucp_info_cpu_addr;
 634         char *kernel_ver;
 635         u64 result;
 636         int rc;
 637
 638         cpucp_info_cpu_addr =
 639                         hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev,
 640                                         sizeof(struct cpucp_info),
 641                                         &cpucp_info_dma_addr);
 642         if (!cpucp_info_cpu_addr) {
 643                 dev_err(hdev->dev,
 644                         "Failed to allocate DMA memory for CPU-CP info packet\n");
 645                 return -ENOMEM;
 646         }
 647
 648         memset(cpucp_info_cpu_addr, 0, sizeof(struct cpucp_info));
 649
 650         pkt.ctl = cpu_to_le32(CPUCP_PACKET_INFO_GET <<
 651                                 CPUCP_PKT_CTL_OPCODE_SHIFT);
 652         pkt.addr = cpu_to_le64(cpucp_info_dma_addr);
 653         pkt.data_max_size = cpu_to_le32(sizeof(struct cpucp_info));
 654
 655         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
 656                                         HL_CPUCP_INFO_TIMEOUT_USEC, &result);
 657         if (rc) {
 658                 dev_err(hdev->dev,
 659                         "Failed to handle CPU-CP info pkt, error %d\n", rc);
 660                 goto out;
 661         }
 662
 663         rc = fw_read_errors(hdev, boot_err0_reg, boot_err1_reg,
 664                                 sts_boot_dev_sts0_reg, sts_boot_dev_sts1_reg);
 665         if (rc) {
 666                 dev_err(hdev->dev, "Errors in device boot\n");
 667                 goto out;
 668         }
 669
 670         memcpy(&prop->cpucp_info, cpucp_info_cpu_addr,
 671                         sizeof(prop->cpucp_info));
 672
 673         rc = hl_build_hwmon_channel_info(hdev, prop->cpucp_info.sensors);
 674         if (rc) {
 675                 dev_err(hdev->dev,
 676                         "Failed to build hwmon channel info, error %d\n", rc);
 677                 rc = -EFAULT;
 678                 goto out;
 679         }
 680
 681         kernel_ver = extract_fw_ver_from_str(prop->cpucp_info.kernel_version);
 682         if (kernel_ver) {
 683                 dev_info(hdev->dev, "Linux version %s", kernel_ver);
 684                 kfree(kernel_ver);
 685         }
 686
 687         /* assume EQ code doesn't need to check eqe index */
 688         hdev->event_queue.check_eqe_index = false;
 689
 690         /* Read FW application security bits again */
 691         if (prop->fw_cpu_boot_dev_sts0_valid) {
 692                 prop->fw_app_cpu_boot_dev_sts0 = RREG32(sts_boot_dev_sts0_reg);
 693                 if (prop->fw_app_cpu_boot_dev_sts0 &
 694                                 CPU_BOOT_DEV_STS0_EQ_INDEX_EN)
 695                         hdev->event_queue.check_eqe_index = true;
 696         }
 697
 698         if (prop->fw_cpu_boot_dev_sts1_valid)
 699                 prop->fw_app_cpu_boot_dev_sts1 = RREG32(sts_boot_dev_sts1_reg);
 700
 701 out:
 702         hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev,
 703                         sizeof(struct cpucp_info), cpucp_info_cpu_addr);
 704
 705         return rc;
 706 }
 707
 708 static int hl_fw_send_msi_info_msg(struct hl_device *hdev)
 709 {
 710         struct cpucp_array_data_packet *pkt;
 711         size_t total_pkt_size, data_size;
 712         u64 result;
 713         int rc;
 714
 715         /* skip sending this info for unsupported ASICs */
 716         if (!hdev->asic_funcs->get_msi_info)
 717                 return 0;
 718
 719         data_size = CPUCP_NUM_OF_MSI_TYPES * sizeof(u32);
 720         total_pkt_size = sizeof(struct cpucp_array_data_packet) + data_size;
 721
 722         /* data should be aligned to 8 bytes in order to CPU-CP to copy it */
 723         total_pkt_size = (total_pkt_size + 0x7) & ~0x7;
 724
 725         /* total_pkt_size is casted to u16 later on */
 726         if (total_pkt_size > USHRT_MAX) {
 727                 dev_err(hdev->dev, "CPUCP array data is too big\n");
 728                 return -EINVAL;
 729         }
 730
 731         pkt = kzalloc(total_pkt_size, GFP_KERNEL);
 732         if (!pkt)
 733                 return -ENOMEM;
 734
 735         pkt->length = cpu_to_le32(CPUCP_NUM_OF_MSI_TYPES);
 736
 737         memset((void *) &pkt->data, 0xFF, data_size);
 738         hdev->asic_funcs->get_msi_info(pkt->data);
 739
 740         pkt->cpucp_pkt.ctl = cpu_to_le32(CPUCP_PACKET_MSI_INFO_SET <<
 741                                                 CPUCP_PKT_CTL_OPCODE_SHIFT);
 742
 743         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *)pkt,
 744                                                 total_pkt_size, 0, &result);
 745
 746         /*
 747          * in case packet result is invalid it means that FW does not support
 748          * this feature and will use default/hard coded MSI values. no reason
 749          * to stop the boot
 750          */
 751         if (rc && result == cpucp_packet_invalid)
 752                 rc = 0;
 753
 754         if (rc)
 755                 dev_err(hdev->dev, "failed to send CPUCP array data\n");
 756
 757         kfree(pkt);
 758
 759         return rc;
 760 }
 761
 762 int hl_fw_cpucp_handshake(struct hl_device *hdev,
 763                                 u32 sts_boot_dev_sts0_reg,
 764                                 u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg,
 765                                 u32 boot_err1_reg)
 766 {
 767         int rc;
 768
 769         rc = hl_fw_cpucp_info_get(hdev, sts_boot_dev_sts0_reg,
 770                                         sts_boot_dev_sts1_reg, boot_err0_reg,
 771                                         boot_err1_reg);
 772         if (rc)
 773                 return rc;
 774
 775         return hl_fw_send_msi_info_msg(hdev);
 776 }
 777
 778 int hl_fw_get_eeprom_data(struct hl_device *hdev, void *data, size_t max_size)
 779 {
 780         struct cpucp_packet pkt = {};
 781         void *eeprom_info_cpu_addr;
 782         dma_addr_t eeprom_info_dma_addr;
 783         u64 result;
 784         int rc;
 785
 786         eeprom_info_cpu_addr =
 787                         hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev,
 788                                         max_size, &eeprom_info_dma_addr);
 789         if (!eeprom_info_cpu_addr) {
 790                 dev_err(hdev->dev,
 791                         "Failed to allocate DMA memory for CPU-CP EEPROM packet\n");
 792                 return -ENOMEM;
 793         }
 794
 795         memset(eeprom_info_cpu_addr, 0, max_size);
 796
 797         pkt.ctl = cpu_to_le32(CPUCP_PACKET_EEPROM_DATA_GET <<
 798                                 CPUCP_PKT_CTL_OPCODE_SHIFT);
 799         pkt.addr = cpu_to_le64(eeprom_info_dma_addr);
 800         pkt.data_max_size = cpu_to_le32(max_size);
 801
 802         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
 803                         HL_CPUCP_EEPROM_TIMEOUT_USEC, &result);
 804
 805         if (rc) {
 806                 dev_err(hdev->dev,
 807                         "Failed to handle CPU-CP EEPROM packet, error %d\n",
 808                         rc);
 809                 goto out;
 810         }
 811
 812         /* result contains the actual size */
 813         memcpy(data, eeprom_info_cpu_addr, min((size_t)result, max_size));
 814
 815 out:
 816         hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev, max_size,
 817                         eeprom_info_cpu_addr);
 818
 819         return rc;
 820 }
 821
 822 int hl_fw_cpucp_pci_counters_get(struct hl_device *hdev,
 823                 struct hl_info_pci_counters *counters)
 824 {
 825         struct cpucp_packet pkt = {};
 826         u64 result;
 827         int rc;
 828
 829         pkt.ctl = cpu_to_le32(CPUCP_PACKET_PCIE_THROUGHPUT_GET <<
 830                         CPUCP_PKT_CTL_OPCODE_SHIFT);
 831
 832         /* Fetch PCI rx counter */
 833         pkt.index = cpu_to_le32(cpucp_pcie_throughput_rx);
 834         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
 835                                         HL_CPUCP_INFO_TIMEOUT_USEC, &result);
 836         if (rc) {
 837                 dev_err(hdev->dev,
 838                         "Failed to handle CPU-CP PCI info pkt, error %d\n", rc);
 839                 return rc;
 840         }
 841         counters->rx_throughput = result;
 842
 843         memset(&pkt, 0, sizeof(pkt));
 844         pkt.ctl = cpu_to_le32(CPUCP_PACKET_PCIE_THROUGHPUT_GET <<
 845                         CPUCP_PKT_CTL_OPCODE_SHIFT);
 846
 847         /* Fetch PCI tx counter */
 848         pkt.index = cpu_to_le32(cpucp_pcie_throughput_tx);
 849         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
 850                                         HL_CPUCP_INFO_TIMEOUT_USEC, &result);
 851         if (rc) {
 852                 dev_err(hdev->dev,
 853                         "Failed to handle CPU-CP PCI info pkt, error %d\n", rc);
 854                 return rc;
 855         }
 856         counters->tx_throughput = result;
 857
 858         /* Fetch PCI replay counter */
 859         memset(&pkt, 0, sizeof(pkt));
 860         pkt.ctl = cpu_to_le32(CPUCP_PACKET_PCIE_REPLAY_CNT_GET <<
 861                         CPUCP_PKT_CTL_OPCODE_SHIFT);
 862
 863         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
 864                         HL_CPUCP_INFO_TIMEOUT_USEC, &result);
 865         if (rc) {
 866                 dev_err(hdev->dev,
 867                         "Failed to handle CPU-CP PCI info pkt, error %d\n", rc);
 868                 return rc;
 869         }
 870         counters->replay_cnt = (u32) result;
 871
 872         return rc;
 873 }
 874
 875 int hl_fw_cpucp_total_energy_get(struct hl_device *hdev, u64 *total_energy)
 876 {
 877         struct cpucp_packet pkt = {};
 878         u64 result;
 879         int rc;
 880
 881         pkt.ctl = cpu_to_le32(CPUCP_PACKET_TOTAL_ENERGY_GET <<
 882                                 CPUCP_PKT_CTL_OPCODE_SHIFT);
 883
 884         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
 885                                         HL_CPUCP_INFO_TIMEOUT_USEC, &result);
 886         if (rc) {
 887                 dev_err(hdev->dev,
 888                         "Failed to handle CpuCP total energy pkt, error %d\n",
 889                                 rc);
 890                 return rc;
 891         }
 892
 893         *total_energy = result;
 894
 895         return rc;
 896 }
 897
 898 int get_used_pll_index(struct hl_device *hdev, u32 input_pll_index,
 899                                                 enum pll_index *pll_index)
 900 {
 901         struct asic_fixed_properties *prop = &hdev->asic_prop;
 902         u8 pll_byte, pll_bit_off;
 903         bool dynamic_pll;
 904         int fw_pll_idx;
 905
 906         dynamic_pll = !!(prop->fw_app_cpu_boot_dev_sts0 &
 907                                                 CPU_BOOT_DEV_STS0_DYN_PLL_EN);
 908
 909         if (!dynamic_pll) {
 910                 /*
 911                  * in case we are working with legacy FW (each asic has unique
 912                  * PLL numbering) use the driver based index as they are
 913                  * aligned with fw legacy numbering
 914                  */
 915                 *pll_index = input_pll_index;
 916                 return 0;
 917         }
 918
 919         /* retrieve a FW compatible PLL index based on
 920          * ASIC specific user request
 921          */
 922         fw_pll_idx = hdev->asic_funcs->map_pll_idx_to_fw_idx(input_pll_index);
 923         if (fw_pll_idx < 0) {
 924                 dev_err(hdev->dev, "Invalid PLL index (%u) error %d\n",
 925                         input_pll_index, fw_pll_idx);
 926                 return -EINVAL;
 927         }
 928
 929         /* PLL map is a u8 array */
 930         pll_byte = prop->cpucp_info.pll_map[fw_pll_idx >> 3];
 931         pll_bit_off = fw_pll_idx & 0x7;
 932
 933         if (!(pll_byte & BIT(pll_bit_off))) {
 934                 dev_err(hdev->dev, "PLL index %d is not supported\n",
 935                         fw_pll_idx);
 936                 return -EINVAL;
 937         }
 938
 939         *pll_index = fw_pll_idx;
 940
 941         return 0;
 942 }
 943
 944 int hl_fw_cpucp_pll_info_get(struct hl_device *hdev, u32 pll_index,
 945                 u16 *pll_freq_arr)
 946 {
 947         struct cpucp_packet pkt;
 948         enum pll_index used_pll_idx;
 949         u64 result;
 950         int rc;
 951
 952         rc = get_used_pll_index(hdev, pll_index, &used_pll_idx);
 953         if (rc)
 954                 return rc;
 955
 956         memset(&pkt, 0, sizeof(pkt));
 957
 958         pkt.ctl = cpu_to_le32(CPUCP_PACKET_PLL_INFO_GET <<
 959                                 CPUCP_PKT_CTL_OPCODE_SHIFT);
 960         pkt.pll_type = __cpu_to_le16((u16)used_pll_idx);
 961
 962         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
 963                         HL_CPUCP_INFO_TIMEOUT_USEC, &result);
 964         if (rc) {
 965                 dev_err(hdev->dev, "Failed to read PLL info, error %d\n", rc);
 966                 return rc;
 967         }
 968
 969         pll_freq_arr[0] = FIELD_GET(CPUCP_PKT_RES_PLL_OUT0_MASK, result);
 970         pll_freq_arr[1] = FIELD_GET(CPUCP_PKT_RES_PLL_OUT1_MASK, result);
 971         pll_freq_arr[2] = FIELD_GET(CPUCP_PKT_RES_PLL_OUT2_MASK, result);
 972         pll_freq_arr[3] = FIELD_GET(CPUCP_PKT_RES_PLL_OUT3_MASK, result);
 973
 974         return 0;
 975 }
 976
 977 int hl_fw_cpucp_power_get(struct hl_device *hdev, u64 *power)
 978 {
 979         struct cpucp_packet pkt;
 980         u64 result;
 981         int rc;
 982
 983         memset(&pkt, 0, sizeof(pkt));
 984
 985         pkt.ctl = cpu_to_le32(CPUCP_PACKET_POWER_GET <<
 986                                 CPUCP_PKT_CTL_OPCODE_SHIFT);
 987         pkt.type = cpu_to_le16(CPUCP_POWER_INPUT);
 988
 989         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
 990                         HL_CPUCP_INFO_TIMEOUT_USEC, &result);
 991         if (rc) {
 992                 dev_err(hdev->dev, "Failed to read power, error %d\n", rc);
 993                 return rc;
 994         }
 995
 996         *power = result;
 997
 998         return rc;
 999 }
1000
1001 int hl_fw_dram_replaced_row_get(struct hl_device *hdev,
1002                                 struct cpucp_hbm_row_info *info)
1003 {
1004         struct cpucp_hbm_row_info *cpucp_repl_rows_info_cpu_addr;
1005         dma_addr_t cpucp_repl_rows_info_dma_addr;
1006         struct cpucp_packet pkt = {};
1007         u64 result;
1008         int rc;
1009
1010         cpucp_repl_rows_info_cpu_addr =
1011                         hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev,
1012                                         sizeof(struct cpucp_hbm_row_info),
1013                                         &cpucp_repl_rows_info_dma_addr);
1014         if (!cpucp_repl_rows_info_cpu_addr) {
1015                 dev_err(hdev->dev,
1016                         "Failed to allocate DMA memory for CPU-CP replaced rows info packet\n");
1017                 return -ENOMEM;
1018         }
1019
1020         memset(cpucp_repl_rows_info_cpu_addr, 0, sizeof(struct cpucp_hbm_row_info));
1021
1022         pkt.ctl = cpu_to_le32(CPUCP_PACKET_HBM_REPLACED_ROWS_INFO_GET <<
1023                                         CPUCP_PKT_CTL_OPCODE_SHIFT);
1024         pkt.addr = cpu_to_le64(cpucp_repl_rows_info_dma_addr);
1025         pkt.data_max_size = cpu_to_le32(sizeof(struct cpucp_hbm_row_info));
1026
1027         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
1028                                         HL_CPUCP_INFO_TIMEOUT_USEC, &result);
1029         if (rc) {
1030                 dev_err(hdev->dev,
1031                         "Failed to handle CPU-CP replaced rows info pkt, error %d\n", rc);
1032                 goto out;
1033         }
1034
1035         memcpy(info, cpucp_repl_rows_info_cpu_addr, sizeof(*info));
1036
1037 out:
1038         hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev,
1039                                         sizeof(struct cpucp_hbm_row_info),
1040                                         cpucp_repl_rows_info_cpu_addr);
1041
1042         return rc;
1043 }
1044
1045 int hl_fw_dram_pending_row_get(struct hl_device *hdev, u32 *pend_rows_num)
1046 {
1047         struct cpucp_packet pkt;
1048         u64 result;
1049         int rc;
1050
1051         memset(&pkt, 0, sizeof(pkt));
1052
1053         pkt.ctl = cpu_to_le32(CPUCP_PACKET_HBM_PENDING_ROWS_STATUS << CPUCP_PKT_CTL_OPCODE_SHIFT);
1054
1055         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, &result);
1056         if (rc) {
1057                 dev_err(hdev->dev,
1058                                 "Failed to handle CPU-CP pending rows info pkt, error %d\n", rc);
1059                 goto out;
1060         }
1061
1062         *pend_rows_num = (u32) result;
1063 out:
1064         return rc;
1065 }
1066
1067 int hl_fw_cpucp_engine_core_asid_set(struct hl_device *hdev, u32 asid)
1068 {
1069         struct cpucp_packet pkt;
1070         int rc;
1071
1072         memset(&pkt, 0, sizeof(pkt));
1073
1074         pkt.ctl = cpu_to_le32(CPUCP_PACKET_ENGINE_CORE_ASID_SET << CPUCP_PKT_CTL_OPCODE_SHIFT);
1075         pkt.value = cpu_to_le64(asid);
1076
1077         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
1078                                                 HL_CPUCP_INFO_TIMEOUT_USEC, NULL);
1079         if (rc)
1080                 dev_err(hdev->dev,
1081                         "Failed on ASID configuration request for engine core, error %d\n",
1082                         rc);
1083
1084         return rc;
1085 }
1086
1087 void hl_fw_ask_hard_reset_without_linux(struct hl_device *hdev)
1088 {
1089         struct static_fw_load_mgr *static_loader =
1090                         &hdev->fw_loader.static_loader;
1091         int rc;
1092
1093         if (hdev->asic_prop.dynamic_fw_load) {
1094                 rc = hl_fw_dynamic_send_protocol_cmd(hdev, &hdev->fw_loader,
1095                                 COMMS_RST_DEV, 0, false,
1096                                 hdev->fw_loader.cpu_timeout);
1097                 if (rc)
1098                         dev_warn(hdev->dev, "Failed sending COMMS_RST_DEV\n");
1099         } else {
1100                 WREG32(static_loader->kmd_msg_to_cpu_reg, KMD_MSG_RST_DEV);
1101         }
1102 }
1103
1104 void hl_fw_ask_halt_machine_without_linux(struct hl_device *hdev)
1105 {
1106         struct static_fw_load_mgr *static_loader =
1107                         &hdev->fw_loader.static_loader;
1108         int rc;
1109
1110         if (hdev->device_cpu_is_halted)
1111                 return;
1112
1113         /* Stop device CPU to make sure nothing bad happens */
1114         if (hdev->asic_prop.dynamic_fw_load) {
1115                 rc = hl_fw_dynamic_send_protocol_cmd(hdev, &hdev->fw_loader,
1116                                 COMMS_GOTO_WFE, 0, true,
1117                                 hdev->fw_loader.cpu_timeout);
1118                 if (rc)
1119                         dev_warn(hdev->dev, "Failed sending COMMS_GOTO_WFE\n");
1120         } else {
1121                 WREG32(static_loader->kmd_msg_to_cpu_reg, KMD_MSG_GOTO_WFE);
1122                 msleep(static_loader->cpu_reset_wait_msec);
1123
1124                 /* Must clear this register in order to prevent preboot
1125                  * from reading WFE after reboot
1126                  */
1127                 WREG32(static_loader->kmd_msg_to_cpu_reg, KMD_MSG_NA);
1128         }
1129
1130         hdev->device_cpu_is_halted = true;
1131 }
1132
1133 static void detect_cpu_boot_status(struct hl_device *hdev, u32 status)
1134 {
1135         /* Some of the status codes below are deprecated in newer f/w
1136          * versions but we keep them here for backward compatibility
1137          */
1138         switch (status) {
1139         case CPU_BOOT_STATUS_NA:
1140                 dev_err(hdev->dev,
1141                         "Device boot progress - BTL/ROM did NOT run\n");
1142                 break;
1143         case CPU_BOOT_STATUS_IN_WFE:
1144                 dev_err(hdev->dev,
1145                         "Device boot progress - Stuck inside WFE loop\n");
1146                 break;
1147         case CPU_BOOT_STATUS_IN_BTL:
1148                 dev_err(hdev->dev,
1149                         "Device boot progress - Stuck in BTL\n");
1150                 break;
1151         case CPU_BOOT_STATUS_IN_PREBOOT:
1152                 dev_err(hdev->dev,
1153                         "Device boot progress - Stuck in Preboot\n");
1154                 break;
1155         case CPU_BOOT_STATUS_IN_SPL:
1156                 dev_err(hdev->dev,
1157                         "Device boot progress - Stuck in SPL\n");
1158                 break;
1159         case CPU_BOOT_STATUS_IN_UBOOT:
1160                 dev_err(hdev->dev,
1161                         "Device boot progress - Stuck in u-boot\n");
1162                 break;
1163         case CPU_BOOT_STATUS_DRAM_INIT_FAIL:
1164                 dev_err(hdev->dev,
1165                         "Device boot progress - DRAM initialization failed\n");
1166                 break;
1167         case CPU_BOOT_STATUS_UBOOT_NOT_READY:
1168                 dev_err(hdev->dev,
1169                         "Device boot progress - Cannot boot\n");
1170                 break;
1171         case CPU_BOOT_STATUS_TS_INIT_FAIL:
1172                 dev_err(hdev->dev,
1173                         "Device boot progress - Thermal Sensor initialization failed\n");
1174                 break;
1175         case CPU_BOOT_STATUS_SECURITY_READY:
1176                 dev_err(hdev->dev,
1177                         "Device boot progress - Stuck in preboot after security initialization\n");
1178                 break;
1179         default:
1180                 dev_err(hdev->dev,
1181                         "Device boot progress - Invalid status code %d\n",
1182                         status);
1183                 break;
1184         }
1185 }
1186
1187 static int hl_fw_read_preboot_caps(struct hl_device *hdev,
1188                                         u32 cpu_boot_status_reg,
1189                                         u32 sts_boot_dev_sts0_reg,
1190                                         u32 sts_boot_dev_sts1_reg,
1191                                         u32 boot_err0_reg, u32 boot_err1_reg,
1192                                         u32 timeout)
1193 {
1194         struct asic_fixed_properties *prop = &hdev->asic_prop;
1195         u32 status, reg_val;
1196         int rc;
1197
1198         /* Need to check two possible scenarios:
1199          *
1200          * CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT - for newer firmwares where
1201          * the preboot is waiting for the boot fit
1202          *
1203          * All other status values - for older firmwares where the uboot was
1204          * loaded from the FLASH
1205          */
1206         rc = hl_poll_timeout(
1207                 hdev,
1208                 cpu_boot_status_reg,
1209                 status,
1210                 (status == CPU_BOOT_STATUS_NIC_FW_RDY) ||
1211                 (status == CPU_BOOT_STATUS_READY_TO_BOOT) ||
1212                 (status == CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT),
1213                 hdev->fw_poll_interval_usec,
1214                 timeout);
1215
1216         if (rc) {
1217                 dev_err(hdev->dev, "CPU boot ready status timeout\n");
1218                 detect_cpu_boot_status(hdev, status);
1219
1220                 /* If we read all FF, then something is totally wrong, no point
1221                  * of reading specific errors
1222                  */
1223                 if (status != -1)
1224                         fw_read_errors(hdev, boot_err0_reg, boot_err1_reg,
1225                                                         sts_boot_dev_sts0_reg,
1226                                                         sts_boot_dev_sts1_reg);
1227                 return -EIO;
1228         }
1229
1230         /*
1231          * the registers DEV_STS* contain FW capabilities/features.
1232          * We can rely on this registers only if bit CPU_BOOT_DEV_STS*_ENABLED
1233          * is set.
1234          * In the first read of this register we store the value of this
1235          * register ONLY if the register is enabled (which will be propagated
1236          * to next stages) and also mark the register as valid.
1237          * In case it is not enabled the stored value will be left 0- all
1238          * caps/features are off
1239          */
1240         reg_val = RREG32(sts_boot_dev_sts0_reg);
1241         if (reg_val & CPU_BOOT_DEV_STS0_ENABLED) {
1242                 prop->fw_cpu_boot_dev_sts0_valid = true;
1243                 prop->fw_preboot_cpu_boot_dev_sts0 = reg_val;
1244         }
1245
1246         reg_val = RREG32(sts_boot_dev_sts1_reg);
1247         if (reg_val & CPU_BOOT_DEV_STS1_ENABLED) {
1248                 prop->fw_cpu_boot_dev_sts1_valid = true;
1249                 prop->fw_preboot_cpu_boot_dev_sts1 = reg_val;
1250         }
1251
1252         prop->dynamic_fw_load = !!(prop->fw_preboot_cpu_boot_dev_sts0 &
1253                                                 CPU_BOOT_DEV_STS0_FW_LD_COM_EN);
1254
1255         /* initialize FW loader once we know what load protocol is used */
1256         hdev->asic_funcs->init_firmware_loader(hdev);
1257
1258         dev_dbg(hdev->dev, "Attempting %s FW load\n",
1259                         prop->dynamic_fw_load ? "dynamic" : "legacy");
1260         return 0;
1261 }
1262
1263 static int hl_fw_static_read_device_fw_version(struct hl_device *hdev,
1264                                         enum hl_fw_component fwc)
1265 {
1266         struct asic_fixed_properties *prop = &hdev->asic_prop;
1267         struct fw_load_mgr *fw_loader = &hdev->fw_loader;
1268         struct static_fw_load_mgr *static_loader;
1269         char *dest, *boot_ver, *preboot_ver;
1270         u32 ver_off, limit;
1271         const char *name;
1272         char btl_ver[32];
1273
1274         static_loader = &hdev->fw_loader.static_loader;
1275
1276         switch (fwc) {
1277         case FW_COMP_BOOT_FIT:
1278                 ver_off = RREG32(static_loader->boot_fit_version_offset_reg);
1279                 dest = prop->uboot_ver;
1280                 name = "Boot-fit";
1281                 limit = static_loader->boot_fit_version_max_off;
1282                 break;
1283         case FW_COMP_PREBOOT:
1284                 ver_off = RREG32(static_loader->preboot_version_offset_reg);
1285                 dest = prop->preboot_ver;
1286                 name = "Preboot";
1287                 limit = static_loader->preboot_version_max_off;
1288                 break;
1289         default:
1290                 dev_warn(hdev->dev, "Undefined FW component: %d\n", fwc);
1291                 return -EIO;
1292         }
1293
1294         ver_off &= static_loader->sram_offset_mask;
1295
1296         if (ver_off < limit) {
1297                 memcpy_fromio(dest,
1298                         hdev->pcie_bar[fw_loader->sram_bar_id] + ver_off,
1299                         VERSION_MAX_LEN);
1300         } else {
1301                 dev_err(hdev->dev, "%s version offset (0x%x) is above SRAM\n",
1302                                                                 name, ver_off);
1303                 strscpy(dest, "unavailable", VERSION_MAX_LEN);
1304                 return -EIO;
1305         }
1306
1307         if (fwc == FW_COMP_BOOT_FIT) {
1308                 boot_ver = extract_fw_ver_from_str(prop->uboot_ver);
1309                 if (boot_ver) {
1310                         dev_info(hdev->dev, "boot-fit version %s\n", boot_ver);
1311                         kfree(boot_ver);
1312                 }
1313         } else if (fwc == FW_COMP_PREBOOT) {
1314                 preboot_ver = strnstr(prop->preboot_ver, "Preboot",
1315                                                 VERSION_MAX_LEN);
1316                 if (preboot_ver && preboot_ver != prop->preboot_ver) {
1317                         strscpy(btl_ver, prop->preboot_ver,
1318                                 min((int) (preboot_ver - prop->preboot_ver),
1319                                                                         31));
1320                         dev_info(hdev->dev, "%s\n", btl_ver);
1321                 }
1322
1323                 preboot_ver = extract_fw_ver_from_str(prop->preboot_ver);
1324                 if (preboot_ver) {
1325                         dev_info(hdev->dev, "preboot version %s\n",
1326                                                                 preboot_ver);
1327                         kfree(preboot_ver);
1328                 }
1329         }
1330
1331         return 0;
1332 }
1333
1334 /**
1335  * hl_fw_preboot_update_state - update internal data structures during
1336  *                              handshake with preboot
1337  *
1338  *
1339  * @hdev: pointer to the habanalabs device structure
1340  *
1341  * @return 0 on success, otherwise non-zero error code
1342  */
1343 static void hl_fw_preboot_update_state(struct hl_device *hdev)
1344 {
1345         struct asic_fixed_properties *prop = &hdev->asic_prop;
1346         u32 cpu_boot_dev_sts0, cpu_boot_dev_sts1;
1347
1348         cpu_boot_dev_sts0 = prop->fw_preboot_cpu_boot_dev_sts0;
1349         cpu_boot_dev_sts1 = prop->fw_preboot_cpu_boot_dev_sts1;
1350
1351         /* We read boot_dev_sts registers multiple times during boot:
1352          * 1. preboot - a. Check whether the security status bits are valid
1353          *              b. Check whether fw security is enabled
1354          *              c. Check whether hard reset is done by preboot
1355          * 2. boot cpu - a. Fetch boot cpu security status
1356          *               b. Check whether hard reset is done by boot cpu
1357          * 3. FW application - a. Fetch fw application security status
1358          *                     b. Check whether hard reset is done by fw app
1359          */
1360         prop->hard_reset_done_by_fw = !!(cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_FW_HARD_RST_EN);
1361
1362         dev_dbg(hdev->dev, "Firmware preboot boot device status0 %#x\n",
1363                                                         cpu_boot_dev_sts0);
1364
1365         dev_dbg(hdev->dev, "Firmware preboot boot device status1 %#x\n",
1366                                                         cpu_boot_dev_sts1);
1367
1368         dev_dbg(hdev->dev, "Firmware preboot hard-reset is %s\n",
1369                         prop->hard_reset_done_by_fw ? "enabled" : "disabled");
1370
1371         dev_dbg(hdev->dev, "firmware-level security is %s\n",
1372                         prop->fw_security_enabled ? "enabled" : "disabled");
1373
1374         dev_dbg(hdev->dev, "GIC controller is %s\n",
1375                         prop->gic_interrupts_enable ? "enabled" : "disabled");
1376 }
1377
1378 static int hl_fw_static_read_preboot_status(struct hl_device *hdev)
1379 {
1380         int rc;
1381
1382         rc = hl_fw_static_read_device_fw_version(hdev, FW_COMP_PREBOOT);
1383         if (rc)
1384                 return rc;
1385
1386         return 0;
1387 }
1388
1389 int hl_fw_read_preboot_status(struct hl_device *hdev, u32 cpu_boot_status_reg,
1390                                 u32 sts_boot_dev_sts0_reg,
1391                                 u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg,
1392                                 u32 boot_err1_reg, u32 timeout)
1393 {
1394         int rc;
1395
1396         if (!(hdev->fw_components & FW_TYPE_PREBOOT_CPU))
1397                 return 0;
1398
1399         /*
1400          * In order to determine boot method (static VS dymanic) we need to
1401          * read the boot caps register
1402          */
1403         rc = hl_fw_read_preboot_caps(hdev, cpu_boot_status_reg,
1404                                         sts_boot_dev_sts0_reg,
1405                                         sts_boot_dev_sts1_reg, boot_err0_reg,
1406                                         boot_err1_reg, timeout);
1407         if (rc)
1408                 return rc;
1409
1410         hl_fw_preboot_update_state(hdev);
1411
1412         /* no need to read preboot status in dynamic load */
1413         if (hdev->asic_prop.dynamic_fw_load)
1414                 return 0;
1415
1416         return hl_fw_static_read_preboot_status(hdev);
1417 }
1418
1419 /* associate string with COMM status */
1420 static char *hl_dynamic_fw_status_str[COMMS_STS_INVLD_LAST] = {
1421         [COMMS_STS_NOOP] = "NOOP",
1422         [COMMS_STS_ACK] = "ACK",
1423         [COMMS_STS_OK] = "OK",
1424         [COMMS_STS_ERR] = "ERR",
1425         [COMMS_STS_VALID_ERR] = "VALID_ERR",
1426         [COMMS_STS_TIMEOUT_ERR] = "TIMEOUT_ERR",
1427 };
1428
1429 /**
1430  * hl_fw_dynamic_report_error_status - report error status
1431  *
1432  * @hdev: pointer to the habanalabs device structure
1433  * @status: value of FW status register
1434  * @expected_status: the expected status
1435  */
1436 static void hl_fw_dynamic_report_error_status(struct hl_device *hdev,
1437                                                 u32 status,
1438                                                 enum comms_sts expected_status)
1439 {
1440         enum comms_sts comm_status =
1441                                 FIELD_GET(COMMS_STATUS_STATUS_MASK, status);
1442
1443         if (comm_status < COMMS_STS_INVLD_LAST)
1444                 dev_err(hdev->dev, "Device status %s, expected status: %s\n",
1445                                 hl_dynamic_fw_status_str[comm_status],
1446                                 hl_dynamic_fw_status_str[expected_status]);
1447         else
1448                 dev_err(hdev->dev, "Device status unknown %d, expected status: %s\n",
1449                                 comm_status,
1450                                 hl_dynamic_fw_status_str[expected_status]);
1451 }
1452
1453 /**
1454  * hl_fw_dynamic_send_cmd - send LKD to FW cmd
1455  *
1456  * @hdev: pointer to the habanalabs device structure
1457  * @fw_loader: managing structure for loading device's FW
1458  * @cmd: LKD to FW cmd code
1459  * @size: size of next FW component to be loaded (0 if not necessary)
1460  *
1461  * LDK to FW exact command layout is defined at struct comms_command.
1462  * note: the size argument is used only when the next FW component should be
1463  *       loaded, otherwise it shall be 0. the size is used by the FW in later
1464  *       protocol stages and when sending only indicating the amount of memory
1465  *       to be allocated by the FW to receive the next boot component.
1466  */
1467 static void hl_fw_dynamic_send_cmd(struct hl_device *hdev,
1468                                 struct fw_load_mgr *fw_loader,
1469                                 enum comms_cmd cmd, unsigned int size)
1470 {
1471         struct cpu_dyn_regs *dyn_regs;
1472         u32 val;
1473
1474         dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs;
1475
1476         val = FIELD_PREP(COMMS_COMMAND_CMD_MASK, cmd);
1477         val |= FIELD_PREP(COMMS_COMMAND_SIZE_MASK, size);
1478
1479         WREG32(le32_to_cpu(dyn_regs->kmd_msg_to_cpu), val);
1480 }
1481
1482 /**
1483  * hl_fw_dynamic_extract_fw_response - update the FW response
1484  *
1485  * @hdev: pointer to the habanalabs device structure
1486  * @fw_loader: managing structure for loading device's FW
1487  * @response: FW response
1488  * @status: the status read from CPU status register
1489  *
1490  * @return 0 on success, otherwise non-zero error code
1491  */
1492 static int hl_fw_dynamic_extract_fw_response(struct hl_device *hdev,
1493                                                 struct fw_load_mgr *fw_loader,
1494                                                 struct fw_response *response,
1495                                                 u32 status)
1496 {
1497         response->status = FIELD_GET(COMMS_STATUS_STATUS_MASK, status);
1498         response->ram_offset = FIELD_GET(COMMS_STATUS_OFFSET_MASK, status) <<
1499                                                 COMMS_STATUS_OFFSET_ALIGN_SHIFT;
1500         response->ram_type = FIELD_GET(COMMS_STATUS_RAM_TYPE_MASK, status);
1501
1502         if ((response->ram_type != COMMS_SRAM) &&
1503                                         (response->ram_type != COMMS_DRAM)) {
1504                 dev_err(hdev->dev, "FW status: invalid RAM type %u\n",
1505                                                         response->ram_type);
1506                 return -EIO;
1507         }
1508
1509         return 0;
1510 }
1511
1512 /**
1513  * hl_fw_dynamic_wait_for_status - wait for status in dynamic FW load
1514  *
1515  * @hdev: pointer to the habanalabs device structure
1516  * @fw_loader: managing structure for loading device's FW
1517  * @expected_status: expected status to wait for
1518  * @timeout: timeout for status wait
1519  *
1520  * @return 0 on success, otherwise non-zero error code
1521  *
1522  * waiting for status from FW include polling the FW status register until
1523  * expected status is received or timeout occurs (whatever occurs first).
1524  */
1525 static int hl_fw_dynamic_wait_for_status(struct hl_device *hdev,
1526                                                 struct fw_load_mgr *fw_loader,
1527                                                 enum comms_sts expected_status,
1528                                                 u32 timeout)
1529 {
1530         struct cpu_dyn_regs *dyn_regs;
1531         u32 status;
1532         int rc;
1533
1534         dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs;
1535
1536         /* Wait for expected status */
1537         rc = hl_poll_timeout(
1538                 hdev,
1539                 le32_to_cpu(dyn_regs->cpu_cmd_status_to_host),
1540                 status,
1541                 FIELD_GET(COMMS_STATUS_STATUS_MASK, status) == expected_status,
1542                 hdev->fw_poll_interval_usec,
1543                 timeout);
1544
1545         if (rc) {
1546                 hl_fw_dynamic_report_error_status(hdev, status,
1547                                                         expected_status);
1548                 return -EIO;
1549         }
1550
1551         /*
1552          * skip storing FW response for NOOP to preserve the actual desired
1553          * FW status
1554          */
1555         if (expected_status == COMMS_STS_NOOP)
1556                 return 0;
1557
1558         rc = hl_fw_dynamic_extract_fw_response(hdev, fw_loader,
1559                                         &fw_loader->dynamic_loader.response,
1560                                         status);
1561         return rc;
1562 }
1563
1564 /**
1565  * hl_fw_dynamic_send_clear_cmd - send clear command to FW
1566  *
1567  * @hdev: pointer to the habanalabs device structure
1568  * @fw_loader: managing structure for loading device's FW
1569  *
1570  * @return 0 on success, otherwise non-zero error code
1571  *
1572  * after command cycle between LKD to FW CPU (i.e. LKD got an expected status
1573  * from FW) we need to clear the CPU status register in order to avoid garbage
1574  * between command cycles.
1575  * This is done by sending clear command and polling the CPU to LKD status
1576  * register to hold the status NOOP
1577  */
1578 static int hl_fw_dynamic_send_clear_cmd(struct hl_device *hdev,
1579                                                 struct fw_load_mgr *fw_loader)
1580 {
1581         hl_fw_dynamic_send_cmd(hdev, fw_loader, COMMS_CLR_STS, 0);
1582
1583         return hl_fw_dynamic_wait_for_status(hdev, fw_loader, COMMS_STS_NOOP,
1584                                                         fw_loader->cpu_timeout);
1585 }
1586
1587 /**
1588  * hl_fw_dynamic_send_protocol_cmd - send LKD to FW cmd and wait for ACK
1589  *
1590  * @hdev: pointer to the habanalabs device structure
1591  * @fw_loader: managing structure for loading device's FW
1592  * @cmd: LKD to FW cmd code
1593  * @size: size of next FW component to be loaded (0 if not necessary)
1594  * @wait_ok: if true also wait for OK response from FW
1595  * @timeout: timeout for status wait
1596  *
1597  * @return 0 on success, otherwise non-zero error code
1598  *
1599  * brief:
1600  * when sending protocol command we have the following steps:
1601  * - send clear (clear command and verify clear status register)
1602  * - send the actual protocol command
1603  * - wait for ACK on the protocol command
1604  * - send clear
1605  * - send NOOP
1606  * if, in addition, the specific protocol command should wait for OK then:
1607  * - wait for OK
1608  * - send clear
1609  * - send NOOP
1610  *
1611  * NOTES:
1612  * send clear: this is necessary in order to clear the status register to avoid
1613  *             leftovers between command
1614  * NOOP command: necessary to avoid loop on the clear command by the FW
1615  */
1616 int hl_fw_dynamic_send_protocol_cmd(struct hl_device *hdev,
1617                                 struct fw_load_mgr *fw_loader,
1618                                 enum comms_cmd cmd, unsigned int size,
1619                                 bool wait_ok, u32 timeout)
1620 {
1621         int rc;
1622
1623         /* first send clear command to clean former commands */
1624         rc = hl_fw_dynamic_send_clear_cmd(hdev, fw_loader);
1625
1626         /* send the actual command */
1627         hl_fw_dynamic_send_cmd(hdev, fw_loader, cmd, size);
1628
1629         /* wait for ACK for the command */
1630         rc = hl_fw_dynamic_wait_for_status(hdev, fw_loader, COMMS_STS_ACK,
1631                                                                 timeout);
1632         if (rc)
1633                 return rc;
1634
1635         /* clear command to prepare for NOOP command */
1636         rc = hl_fw_dynamic_send_clear_cmd(hdev, fw_loader);
1637         if (rc)
1638                 return rc;
1639
1640         /* send the actual NOOP command */
1641         hl_fw_dynamic_send_cmd(hdev, fw_loader, COMMS_NOOP, 0);
1642
1643         if (!wait_ok)
1644                 return 0;
1645
1646         rc = hl_fw_dynamic_wait_for_status(hdev, fw_loader, COMMS_STS_OK,
1647                                                                 timeout);
1648         if (rc)
1649                 return rc;
1650
1651         /* clear command to prepare for NOOP command */
1652         rc = hl_fw_dynamic_send_clear_cmd(hdev, fw_loader);
1653         if (rc)
1654                 return rc;
1655
1656         /* send the actual NOOP command */
1657         hl_fw_dynamic_send_cmd(hdev, fw_loader, COMMS_NOOP, 0);
1658
1659         return 0;
1660 }
1661
1662 /**
1663  * hl_fw_compat_crc32 - CRC compatible with FW
1664  *
1665  * @data: pointer to the data
1666  * @size: size of the data
1667  *
1668  * @return the CRC32 result
1669  *
1670  * NOTE: kernel's CRC32 differ's from standard CRC32 calculation.
1671  *       in order to be aligned we need to flip the bits of both the input
1672  *       initial CRC and kernel's CRC32 result.
1673  *       in addition both sides use initial CRC of 0,
1674  */
1675 static u32 hl_fw_compat_crc32(u8 *data, size_t size)
1676 {
1677         return ~crc32_le(~((u32)0), data, size);
1678 }
1679
1680 /**
1681  * hl_fw_dynamic_validate_memory_bound - validate memory bounds for memory
1682  *                                        transfer (image or descriptor) between
1683  *                                        host and FW
1684  *
1685  * @hdev: pointer to the habanalabs device structure
1686  * @addr: device address of memory transfer
1687  * @size: memory transter size
1688  * @region: PCI memory region
1689  *
1690  * @return 0 on success, otherwise non-zero error code
1691  */
1692 static int hl_fw_dynamic_validate_memory_bound(struct hl_device *hdev,
1693                                                 u64 addr, size_t size,
1694                                                 struct pci_mem_region *region)
1695 {
1696         u64 end_addr;
1697
1698         /* now make sure that the memory transfer is within region's bounds */
1699         end_addr = addr + size;
1700         if (end_addr >= region->region_base + region->region_size) {
1701                 dev_err(hdev->dev,
1702                         "dynamic FW load: memory transfer end address out of memory region bounds. addr: %llx\n",
1703                                                         end_addr);
1704                 return -EIO;
1705         }
1706
1707         /*
1708          * now make sure memory transfer is within predefined BAR bounds.
1709          * this is to make sure we do not need to set the bar (e.g. for DRAM
1710          * memory transfers)
1711          */
1712         if (end_addr >= region->region_base - region->offset_in_bar +
1713                                                         region->bar_size) {
1714                 dev_err(hdev->dev,
1715                         "FW image beyond PCI BAR bounds\n");
1716                 return -EIO;
1717         }
1718
1719         return 0;
1720 }
1721
1722 /**
1723  * hl_fw_dynamic_validate_descriptor - validate FW descriptor
1724  *
1725  * @hdev: pointer to the habanalabs device structure
1726  * @fw_loader: managing structure for loading device's FW
1727  * @fw_desc: the descriptor form FW
1728  *
1729  * @return 0 on success, otherwise non-zero error code
1730  */
1731 static int hl_fw_dynamic_validate_descriptor(struct hl_device *hdev,
1732                                         struct fw_load_mgr *fw_loader,
1733                                         struct lkd_fw_comms_desc *fw_desc)
1734 {
1735         struct pci_mem_region *region;
1736         enum pci_region region_id;
1737         size_t data_size;
1738         u32 data_crc32;
1739         u8 *data_ptr;
1740         u64 addr;
1741         int rc;
1742
1743         if (le32_to_cpu(fw_desc->header.magic) != HL_COMMS_DESC_MAGIC) {
1744                 dev_err(hdev->dev, "Invalid magic for dynamic FW descriptor (%x)\n",
1745                                 fw_desc->header.magic);
1746                 return -EIO;
1747         }
1748
1749         if (fw_desc->header.version != HL_COMMS_DESC_VER) {
1750                 dev_err(hdev->dev, "Invalid version for dynamic FW descriptor (%x)\n",
1751                                 fw_desc->header.version);
1752                 return -EIO;
1753         }
1754
1755         /*
1756          * calc CRC32 of data without header.
1757          * note that no alignment/stride address issues here as all structures
1758          * are 64 bit padded
1759          */
1760         data_size = sizeof(struct lkd_fw_comms_desc) -
1761                                         sizeof(struct comms_desc_header);
1762         data_ptr = (u8 *)fw_desc + sizeof(struct comms_desc_header);
1763
1764         if (le16_to_cpu(fw_desc->header.size) != data_size) {
1765                 dev_err(hdev->dev,
1766                         "Invalid descriptor size 0x%x, expected size 0x%zx\n",
1767                                 le16_to_cpu(fw_desc->header.size), data_size);
1768                 return -EIO;
1769         }
1770
1771         data_crc32 = hl_fw_compat_crc32(data_ptr, data_size);
1772
1773         if (data_crc32 != le32_to_cpu(fw_desc->header.crc32)) {
1774                 dev_err(hdev->dev,
1775                         "CRC32 mismatch for dynamic FW descriptor (%x:%x)\n",
1776                                         data_crc32, fw_desc->header.crc32);
1777                 return -EIO;
1778         }
1779
1780         /* find memory region to which to copy the image */
1781         addr = le64_to_cpu(fw_desc->img_addr);
1782         region_id = hl_get_pci_memory_region(hdev, addr);
1783         if ((region_id != PCI_REGION_SRAM) &&
1784                         ((region_id != PCI_REGION_DRAM))) {
1785                 dev_err(hdev->dev,
1786                         "Invalid region to copy FW image address=%llx\n", addr);
1787                 return -EIO;
1788         }
1789
1790         region = &hdev->pci_mem_region[region_id];
1791
1792         /* store the region for the copy stage */
1793         fw_loader->dynamic_loader.image_region = region;
1794
1795         /*
1796          * here we know that the start address is valid, now make sure that the
1797          * image is within region's bounds
1798          */
1799         rc = hl_fw_dynamic_validate_memory_bound(hdev, addr,
1800                                         fw_loader->dynamic_loader.fw_image_size,
1801                                         region);
1802         if (rc) {
1803                 dev_err(hdev->dev,
1804                         "invalid mem transfer request for FW image\n");
1805                 return rc;
1806         }
1807
1808         /* here we can mark the descriptor as valid as the content has been validated */
1809         fw_loader->dynamic_loader.fw_desc_valid = true;
1810
1811         return 0;
1812 }
1813
1814 static int hl_fw_dynamic_validate_response(struct hl_device *hdev,
1815                                                 struct fw_response *response,
1816                                                 struct pci_mem_region *region)
1817 {
1818         u64 device_addr;
1819         int rc;
1820
1821         device_addr = region->region_base + response->ram_offset;
1822
1823         /*
1824          * validate that the descriptor is within region's bounds
1825          * Note that as the start address was supplied according to the RAM
1826          * type- testing only the end address is enough
1827          */
1828         rc = hl_fw_dynamic_validate_memory_bound(hdev, device_addr,
1829                                         sizeof(struct lkd_fw_comms_desc),
1830                                         region);
1831         return rc;
1832 }
1833
1834 /**
1835  * hl_fw_dynamic_read_and_validate_descriptor - read and validate FW descriptor
1836  *
1837  * @hdev: pointer to the habanalabs device structure
1838  * @fw_loader: managing structure for loading device's FW
1839  *
1840  * @return 0 on success, otherwise non-zero error code
1841  */
1842 static int hl_fw_dynamic_read_and_validate_descriptor(struct hl_device *hdev,
1843                                                 struct fw_load_mgr *fw_loader)
1844 {
1845         struct lkd_fw_comms_desc *fw_desc;
1846         struct pci_mem_region *region;
1847         struct fw_response *response;
1848         enum pci_region region_id;
1849         void __iomem *src;
1850         int rc;
1851
1852         fw_desc = &fw_loader->dynamic_loader.comm_desc;
1853         response = &fw_loader->dynamic_loader.response;
1854
1855         region_id = (response->ram_type == COMMS_SRAM) ?
1856                                         PCI_REGION_SRAM : PCI_REGION_DRAM;
1857
1858         region = &hdev->pci_mem_region[region_id];
1859
1860         rc = hl_fw_dynamic_validate_response(hdev, response, region);
1861         if (rc) {
1862                 dev_err(hdev->dev,
1863                         "invalid mem transfer request for FW descriptor\n");
1864                 return rc;
1865         }
1866
1867         /*
1868          * extract address to copy the descriptor from
1869          * in addition, as the descriptor value is going to be over-ridden by new data- we mark it
1870          * as invalid.
1871          * it will be marked again as valid once validated
1872          */
1873         fw_loader->dynamic_loader.fw_desc_valid = false;
1874         src = hdev->pcie_bar[region->bar_id] + region->offset_in_bar +
1875                                                         response->ram_offset;
1876         memcpy_fromio(fw_desc, src, sizeof(struct lkd_fw_comms_desc));
1877
1878         return hl_fw_dynamic_validate_descriptor(hdev, fw_loader, fw_desc);
1879 }
1880
1881 /**
1882  * hl_fw_dynamic_request_descriptor - handshake with CPU to get FW descriptor
1883  *
1884  * @hdev: pointer to the habanalabs device structure
1885  * @fw_loader: managing structure for loading device's FW
1886  * @next_image_size: size to allocate for next FW component
1887  *
1888  * @return 0 on success, otherwise non-zero error code
1889  */
1890 static int hl_fw_dynamic_request_descriptor(struct hl_device *hdev,
1891                                                 struct fw_load_mgr *fw_loader,
1892                                                 size_t next_image_size)
1893 {
1894         int rc;
1895
1896         rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_PREP_DESC,
1897                                                 next_image_size, true,
1898                                                 fw_loader->cpu_timeout);
1899         if (rc)
1900                 return rc;
1901
1902         return hl_fw_dynamic_read_and_validate_descriptor(hdev, fw_loader);
1903 }
1904
1905 /**
1906  * hl_fw_dynamic_read_device_fw_version - read FW version to exposed properties
1907  *
1908  * @hdev: pointer to the habanalabs device structure
1909  * @fwc: the firmware component
1910  * @fw_version: fw component's version string
1911  */
1912 static void hl_fw_dynamic_read_device_fw_version(struct hl_device *hdev,
1913                                         enum hl_fw_component fwc,
1914                                         const char *fw_version)
1915 {
1916         struct asic_fixed_properties *prop = &hdev->asic_prop;
1917         char *preboot_ver, *boot_ver;
1918         char btl_ver[32];
1919
1920         switch (fwc) {
1921         case FW_COMP_BOOT_FIT:
1922                 strscpy(prop->uboot_ver, fw_version, VERSION_MAX_LEN);
1923                 boot_ver = extract_fw_ver_from_str(prop->uboot_ver);
1924                 if (boot_ver) {
1925                         dev_info(hdev->dev, "boot-fit version %s\n", boot_ver);
1926                         kfree(boot_ver);
1927                 }
1928
1929                 break;
1930         case FW_COMP_PREBOOT:
1931                 strscpy(prop->preboot_ver, fw_version, VERSION_MAX_LEN);
1932                 preboot_ver = strnstr(prop->preboot_ver, "Preboot",
1933                                                 VERSION_MAX_LEN);
1934                 if (preboot_ver && preboot_ver != prop->preboot_ver) {
1935                         strscpy(btl_ver, prop->preboot_ver,
1936                                 min((int) (preboot_ver - prop->preboot_ver),
1937                                                                         31));
1938                         dev_info(hdev->dev, "%s\n", btl_ver);
1939                 }
1940
1941                 preboot_ver = extract_fw_ver_from_str(prop->preboot_ver);
1942                 if (preboot_ver) {
1943                         dev_info(hdev->dev, "preboot version %s\n",
1944                                                                 preboot_ver);
1945                         kfree(preboot_ver);
1946                 }
1947
1948                 break;
1949         default:
1950                 dev_warn(hdev->dev, "Undefined FW component: %d\n", fwc);
1951                 return;
1952         }
1953 }
1954
1955 /**
1956  * hl_fw_dynamic_copy_image - copy image to memory allocated by the FW
1957  *
1958  * @hdev: pointer to the habanalabs device structure
1959  * @fw: fw descriptor
1960  * @fw_loader: managing structure for loading device's FW
1961  */
1962 static int hl_fw_dynamic_copy_image(struct hl_device *hdev,
1963                                                 const struct firmware *fw,
1964                                                 struct fw_load_mgr *fw_loader)
1965 {
1966         struct lkd_fw_comms_desc *fw_desc;
1967         struct pci_mem_region *region;
1968         void __iomem *dest;
1969         u64 addr;
1970         int rc;
1971
1972         fw_desc = &fw_loader->dynamic_loader.comm_desc;
1973         addr = le64_to_cpu(fw_desc->img_addr);
1974
1975         /* find memory region to which to copy the image */
1976         region = fw_loader->dynamic_loader.image_region;
1977
1978         dest = hdev->pcie_bar[region->bar_id] + region->offset_in_bar +
1979                                         (addr - region->region_base);
1980
1981         rc = hl_fw_copy_fw_to_device(hdev, fw, dest,
1982                                         fw_loader->boot_fit_img.src_off,
1983                                         fw_loader->boot_fit_img.copy_size);
1984
1985         return rc;
1986 }
1987
1988 /**
1989  * hl_fw_dynamic_copy_msg - copy msg to memory allocated by the FW
1990  *
1991  * @hdev: pointer to the habanalabs device structure
1992  * @msg: message
1993  * @fw_loader: managing structure for loading device's FW
1994  */
1995 static int hl_fw_dynamic_copy_msg(struct hl_device *hdev,
1996                 struct lkd_msg_comms *msg, struct fw_load_mgr *fw_loader)
1997 {
1998         struct lkd_fw_comms_desc *fw_desc;
1999         struct pci_mem_region *region;
2000         void __iomem *dest;
2001         u64 addr;
2002         int rc;
2003
2004         fw_desc = &fw_loader->dynamic_loader.comm_desc;
2005         addr = le64_to_cpu(fw_desc->img_addr);
2006
2007         /* find memory region to which to copy the image */
2008         region = fw_loader->dynamic_loader.image_region;
2009
2010         dest = hdev->pcie_bar[region->bar_id] + region->offset_in_bar +
2011                                         (addr - region->region_base);
2012
2013         rc = hl_fw_copy_msg_to_device(hdev, msg, dest, 0, 0);
2014
2015         return rc;
2016 }
2017
2018 /**
2019  * hl_fw_boot_fit_update_state - update internal data structures after boot-fit
2020  *                               is loaded
2021  *
2022  * @hdev: pointer to the habanalabs device structure
2023  * @cpu_boot_dev_sts0_reg: register holding CPU boot dev status 0
2024  * @cpu_boot_dev_sts1_reg: register holding CPU boot dev status 1
2025  *
2026  * @return 0 on success, otherwise non-zero error code
2027  */
2028 static void hl_fw_boot_fit_update_state(struct hl_device *hdev,
2029                                                 u32 cpu_boot_dev_sts0_reg,
2030                                                 u32 cpu_boot_dev_sts1_reg)
2031 {
2032         struct asic_fixed_properties *prop = &hdev->asic_prop;
2033
2034         hdev->fw_loader.fw_comp_loaded |= FW_TYPE_BOOT_CPU;
2035
2036         /* Read boot_cpu status bits */
2037         if (prop->fw_preboot_cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_ENABLED) {
2038                 prop->fw_bootfit_cpu_boot_dev_sts0 =
2039                                 RREG32(cpu_boot_dev_sts0_reg);
2040
2041                 prop->hard_reset_done_by_fw = !!(prop->fw_bootfit_cpu_boot_dev_sts0 &
2042                                                         CPU_BOOT_DEV_STS0_FW_HARD_RST_EN);
2043
2044                 dev_dbg(hdev->dev, "Firmware boot CPU status0 %#x\n",
2045                                         prop->fw_bootfit_cpu_boot_dev_sts0);
2046         }
2047
2048         if (prop->fw_cpu_boot_dev_sts1_valid) {
2049                 prop->fw_bootfit_cpu_boot_dev_sts1 =
2050                                 RREG32(cpu_boot_dev_sts1_reg);
2051
2052                 dev_dbg(hdev->dev, "Firmware boot CPU status1 %#x\n",
2053                                         prop->fw_bootfit_cpu_boot_dev_sts1);
2054         }
2055
2056         dev_dbg(hdev->dev, "Firmware boot CPU hard-reset is %s\n",
2057                         prop->hard_reset_done_by_fw ? "enabled" : "disabled");
2058 }
2059
2060 static void hl_fw_dynamic_update_linux_interrupt_if(struct hl_device *hdev)
2061 {
2062         struct cpu_dyn_regs *dyn_regs =
2063                         &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
2064
2065         /* Check whether all 3 interrupt interfaces are set, if not use a
2066          * single interface
2067          */
2068         if (!hdev->asic_prop.gic_interrupts_enable &&
2069                         !(hdev->asic_prop.fw_app_cpu_boot_dev_sts0 &
2070                                 CPU_BOOT_DEV_STS0_MULTI_IRQ_POLL_EN)) {
2071                 dyn_regs->gic_host_halt_irq = dyn_regs->gic_host_pi_upd_irq;
2072                 dyn_regs->gic_host_ints_irq = dyn_regs->gic_host_pi_upd_irq;
2073
2074                 dev_warn(hdev->dev,
2075                         "Using a single interrupt interface towards cpucp");
2076         }
2077 }
2078 /**
2079  * hl_fw_dynamic_load_image - load FW image using dynamic protocol
2080  *
2081  * @hdev: pointer to the habanalabs device structure
2082  * @fw_loader: managing structure for loading device's FW
2083  * @load_fwc: the FW component to be loaded
2084  * @img_ld_timeout: image load timeout
2085  *
2086  * @return 0 on success, otherwise non-zero error code
2087  */
2088 static int hl_fw_dynamic_load_image(struct hl_device *hdev,
2089                                                 struct fw_load_mgr *fw_loader,
2090                                                 enum hl_fw_component load_fwc,
2091                                                 u32 img_ld_timeout)
2092 {
2093         enum hl_fw_component cur_fwc;
2094         const struct firmware *fw;
2095         char *fw_name;
2096         int rc = 0;
2097
2098         /*
2099          * when loading image we have one of 2 scenarios:
2100          * 1. current FW component is preboot and we want to load boot-fit
2101          * 2. current FW component is boot-fit and we want to load linux
2102          */
2103         if (load_fwc == FW_COMP_BOOT_FIT) {
2104                 cur_fwc = FW_COMP_PREBOOT;
2105                 fw_name = fw_loader->boot_fit_img.image_name;
2106         } else {
2107                 cur_fwc = FW_COMP_BOOT_FIT;
2108                 fw_name = fw_loader->linux_img.image_name;
2109         }
2110
2111         /* request FW in order to communicate to FW the size to be allocated */
2112         rc = hl_request_fw(hdev, &fw, fw_name);
2113         if (rc)
2114                 return rc;
2115
2116         /* store the image size for future validation */
2117         fw_loader->dynamic_loader.fw_image_size = fw->size;
2118
2119         rc = hl_fw_dynamic_request_descriptor(hdev, fw_loader, fw->size);
2120         if (rc)
2121                 goto release_fw;
2122
2123         /* read preboot version */
2124         hl_fw_dynamic_read_device_fw_version(hdev, cur_fwc,
2125                                 fw_loader->dynamic_loader.comm_desc.cur_fw_ver);
2126
2127
2128         /* update state according to boot stage */
2129         if (cur_fwc == FW_COMP_BOOT_FIT) {
2130                 struct cpu_dyn_regs *dyn_regs;
2131
2132                 dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs;
2133                 hl_fw_boot_fit_update_state(hdev,
2134                                 le32_to_cpu(dyn_regs->cpu_boot_dev_sts0),
2135                                 le32_to_cpu(dyn_regs->cpu_boot_dev_sts1));
2136         }
2137
2138         /* copy boot fit to space allocated by FW */
2139         rc = hl_fw_dynamic_copy_image(hdev, fw, fw_loader);
2140         if (rc)
2141                 goto release_fw;
2142
2143         rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_DATA_RDY,
2144                                                 0, true,
2145                                                 fw_loader->cpu_timeout);
2146         if (rc)
2147                 goto release_fw;
2148
2149         rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_EXEC,
2150                                                 0, false,
2151                                                 img_ld_timeout);
2152
2153 release_fw:
2154         hl_release_firmware(fw);
2155         return rc;
2156 }
2157
2158 static int hl_fw_dynamic_wait_for_boot_fit_active(struct hl_device *hdev,
2159                                         struct fw_load_mgr *fw_loader)
2160 {
2161         struct dynamic_fw_load_mgr *dyn_loader;
2162         u32 status;
2163         int rc;
2164
2165         dyn_loader = &fw_loader->dynamic_loader;
2166
2167         /*
2168          * Make sure CPU boot-loader is running
2169          * Note that the CPU_BOOT_STATUS_SRAM_AVAIL is generally set by Linux
2170          * yet there is a debug scenario in which we loading uboot (without Linux)
2171          * which at later stage is relocated to DRAM. In this case we expect
2172          * uboot to set the CPU_BOOT_STATUS_SRAM_AVAIL and so we add it to the
2173          * poll flags
2174          */
2175         rc = hl_poll_timeout(
2176                 hdev,
2177                 le32_to_cpu(dyn_loader->comm_desc.cpu_dyn_regs.cpu_boot_status),
2178                 status,
2179                 (status == CPU_BOOT_STATUS_READY_TO_BOOT) ||
2180                 (status == CPU_BOOT_STATUS_SRAM_AVAIL),
2181                 hdev->fw_poll_interval_usec,
2182                 dyn_loader->wait_for_bl_timeout);
2183         if (rc) {
2184                 dev_err(hdev->dev, "failed to wait for boot\n");
2185                 return rc;
2186         }
2187
2188         dev_dbg(hdev->dev, "uboot status = %d\n", status);
2189         return 0;
2190 }
2191
2192 static int hl_fw_dynamic_wait_for_linux_active(struct hl_device *hdev,
2193                                                 struct fw_load_mgr *fw_loader)
2194 {
2195         struct dynamic_fw_load_mgr *dyn_loader;
2196         u32 status;
2197         int rc;
2198
2199         dyn_loader = &fw_loader->dynamic_loader;
2200
2201         /* Make sure CPU linux is running */
2202
2203         rc = hl_poll_timeout(
2204                 hdev,
2205                 le32_to_cpu(dyn_loader->comm_desc.cpu_dyn_regs.cpu_boot_status),
2206                 status,
2207                 (status == CPU_BOOT_STATUS_SRAM_AVAIL),
2208                 hdev->fw_poll_interval_usec,
2209                 fw_loader->cpu_timeout);
2210         if (rc) {
2211                 dev_err(hdev->dev, "failed to wait for Linux\n");
2212                 return rc;
2213         }
2214
2215         dev_dbg(hdev->dev, "Boot status = %d\n", status);
2216         return 0;
2217 }
2218
2219 /**
2220  * hl_fw_linux_update_state -   update internal data structures after Linux
2221  *                              is loaded.
2222  *                              Note: Linux initialization is comprised mainly
2223  *                              of two stages - loading kernel (SRAM_AVAIL)
2224  *                              & loading ARMCP.
2225  *                              Therefore reading boot device status in any of
2226  *                              these stages might result in different values.
2227  *
2228  * @hdev: pointer to the habanalabs device structure
2229  * @cpu_boot_dev_sts0_reg: register holding CPU boot dev status 0
2230  * @cpu_boot_dev_sts1_reg: register holding CPU boot dev status 1
2231  *
2232  * @return 0 on success, otherwise non-zero error code
2233  */
2234 static void hl_fw_linux_update_state(struct hl_device *hdev,
2235                                                 u32 cpu_boot_dev_sts0_reg,
2236                                                 u32 cpu_boot_dev_sts1_reg)
2237 {
2238         struct asic_fixed_properties *prop = &hdev->asic_prop;
2239
2240         hdev->fw_loader.fw_comp_loaded |= FW_TYPE_LINUX;
2241
2242         /* Read FW application security bits */
2243         if (prop->fw_cpu_boot_dev_sts0_valid) {
2244                 prop->fw_app_cpu_boot_dev_sts0 = RREG32(cpu_boot_dev_sts0_reg);
2245
2246                 prop->hard_reset_done_by_fw = !!(prop->fw_app_cpu_boot_dev_sts0 &
2247                                                         CPU_BOOT_DEV_STS0_FW_HARD_RST_EN);
2248
2249                 if (prop->fw_app_cpu_boot_dev_sts0 &
2250                                 CPU_BOOT_DEV_STS0_GIC_PRIVILEGED_EN)
2251                         prop->gic_interrupts_enable = false;
2252
2253                 dev_dbg(hdev->dev,
2254                         "Firmware application CPU status0 %#x\n",
2255                         prop->fw_app_cpu_boot_dev_sts0);
2256
2257                 dev_dbg(hdev->dev, "GIC controller is %s\n",
2258                                 prop->gic_interrupts_enable ?
2259                                                 "enabled" : "disabled");
2260         }
2261
2262         if (prop->fw_cpu_boot_dev_sts1_valid) {
2263                 prop->fw_app_cpu_boot_dev_sts1 = RREG32(cpu_boot_dev_sts1_reg);
2264
2265                 dev_dbg(hdev->dev,
2266                         "Firmware application CPU status1 %#x\n",
2267                         prop->fw_app_cpu_boot_dev_sts1);
2268         }
2269
2270         dev_dbg(hdev->dev, "Firmware application CPU hard-reset is %s\n",
2271                         prop->hard_reset_done_by_fw ? "enabled" : "disabled");
2272
2273         dev_info(hdev->dev, "Successfully loaded firmware to device\n");
2274 }
2275
2276 /**
2277  * hl_fw_dynamic_send_msg - send a COMMS message with attached data
2278  *
2279  * @hdev: pointer to the habanalabs device structure
2280  * @fw_loader: managing structure for loading device's FW
2281  * @msg_type: message type
2282  * @data: data to be sent
2283  *
2284  * @return 0 on success, otherwise non-zero error code
2285  */
2286 static int hl_fw_dynamic_send_msg(struct hl_device *hdev,
2287                 struct fw_load_mgr *fw_loader, u8 msg_type, void *data)
2288 {
2289         struct lkd_msg_comms msg;
2290         int rc;
2291
2292         memset(&msg, 0, sizeof(msg));
2293
2294         /* create message to be sent */
2295         msg.header.type = msg_type;
2296         msg.header.size = cpu_to_le16(sizeof(struct comms_msg_header));
2297         msg.header.magic = cpu_to_le32(HL_COMMS_MSG_MAGIC);
2298
2299         switch (msg_type) {
2300         case HL_COMMS_RESET_CAUSE_TYPE:
2301                 msg.reset_cause = *(__u8 *) data;
2302                 break;
2303         default:
2304                 dev_err(hdev->dev,
2305                         "Send COMMS message - invalid message type %u\n",
2306                         msg_type);
2307                 return -EINVAL;
2308         }
2309
2310         rc = hl_fw_dynamic_request_descriptor(hdev, fw_loader,
2311                         sizeof(struct lkd_msg_comms));
2312         if (rc)
2313                 return rc;
2314
2315         /* copy message to space allocated by FW */
2316         rc = hl_fw_dynamic_copy_msg(hdev, &msg, fw_loader);
2317         if (rc)
2318                 return rc;
2319
2320         rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_DATA_RDY,
2321                                                 0, true,
2322                                                 fw_loader->cpu_timeout);
2323         if (rc)
2324                 return rc;
2325
2326         rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_EXEC,
2327                                                 0, true,
2328                                                 fw_loader->cpu_timeout);
2329         if (rc)
2330                 return rc;
2331
2332         return 0;
2333 }
2334
2335 /**
2336  * hl_fw_dynamic_init_cpu - initialize the device CPU using dynamic protocol
2337  *
2338  * @hdev: pointer to the habanalabs device structure
2339  * @fw_loader: managing structure for loading device's FW
2340  *
2341  * @return 0 on success, otherwise non-zero error code
2342  *
2343  * brief: the dynamic protocol is master (LKD) slave (FW CPU) protocol.
2344  * the communication is done using registers:
2345  * - LKD command register
2346  * - FW status register
2347  * the protocol is race free. this goal is achieved by splitting the requests
2348  * and response to known synchronization points between the LKD and the FW.
2349  * each response to LKD request is known and bound to a predefined timeout.
2350  * in case of timeout expiration without the desired status from FW- the
2351  * protocol (and hence the boot) will fail.
2352  */
2353 static int hl_fw_dynamic_init_cpu(struct hl_device *hdev,
2354                                         struct fw_load_mgr *fw_loader)
2355 {
2356         struct cpu_dyn_regs *dyn_regs;
2357         int rc;
2358
2359         dev_info(hdev->dev,
2360                 "Loading firmware to device, may take some time...\n");
2361
2362         /* initialize FW descriptor as invalid */
2363         fw_loader->dynamic_loader.fw_desc_valid = false;
2364
2365         /*
2366          * In this stage, "cpu_dyn_regs" contains only LKD's hard coded values!
2367          * It will be updated from FW after hl_fw_dynamic_request_descriptor().
2368          */
2369         dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs;
2370
2371         rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_RST_STATE,
2372                                                 0, true,
2373                                                 fw_loader->cpu_timeout);
2374         if (rc)
2375                 goto protocol_err;
2376
2377         if (hdev->reset_info.curr_reset_cause) {
2378                 rc = hl_fw_dynamic_send_msg(hdev, fw_loader,
2379                                 HL_COMMS_RESET_CAUSE_TYPE, &hdev->reset_info.curr_reset_cause);
2380                 if (rc)
2381                         goto protocol_err;
2382
2383                 /* Clear current reset cause */
2384                 hdev->reset_info.curr_reset_cause = HL_RESET_CAUSE_UNKNOWN;
2385         }
2386
2387         if (!(hdev->fw_components & FW_TYPE_BOOT_CPU)) {
2388                 rc = hl_fw_dynamic_request_descriptor(hdev, fw_loader, 0);
2389                 if (rc)
2390                         goto protocol_err;
2391
2392                 /* read preboot version */
2393                 hl_fw_dynamic_read_device_fw_version(hdev, FW_COMP_PREBOOT,
2394                                 fw_loader->dynamic_loader.comm_desc.cur_fw_ver);
2395                 return 0;
2396         }
2397
2398         /* load boot fit to FW */
2399         rc = hl_fw_dynamic_load_image(hdev, fw_loader, FW_COMP_BOOT_FIT,
2400                                                 fw_loader->boot_fit_timeout);
2401         if (rc) {
2402                 dev_err(hdev->dev, "failed to load boot fit\n");
2403                 goto protocol_err;
2404         }
2405
2406         /*
2407          * when testing FW load (without Linux) on PLDM we don't want to
2408          * wait until boot fit is active as it may take several hours.
2409          * instead, we load the bootfit and let it do all initializations in
2410          * the background.
2411          */
2412         if (hdev->pldm && !(hdev->fw_components & FW_TYPE_LINUX))
2413                 return 0;
2414
2415         rc = hl_fw_dynamic_wait_for_boot_fit_active(hdev, fw_loader);
2416         if (rc)
2417                 goto protocol_err;
2418
2419         /* Enable DRAM scrambling before Linux boot and after successful
2420          *  UBoot
2421          */
2422         hdev->asic_funcs->init_cpu_scrambler_dram(hdev);
2423
2424         if (!(hdev->fw_components & FW_TYPE_LINUX)) {
2425                 dev_info(hdev->dev, "Skip loading Linux F/W\n");
2426                 return 0;
2427         }
2428
2429         if (fw_loader->skip_bmc) {
2430                 rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader,
2431                                                         COMMS_SKIP_BMC, 0,
2432                                                         true,
2433                                                         fw_loader->cpu_timeout);
2434                 if (rc) {
2435                         dev_err(hdev->dev, "failed to load boot fit\n");
2436                         goto protocol_err;
2437                 }
2438         }
2439
2440         /* load Linux image to FW */
2441         rc = hl_fw_dynamic_load_image(hdev, fw_loader, FW_COMP_LINUX,
2442                                                         fw_loader->cpu_timeout);
2443         if (rc) {
2444                 dev_err(hdev->dev, "failed to load Linux\n");
2445                 goto protocol_err;
2446         }
2447
2448         rc = hl_fw_dynamic_wait_for_linux_active(hdev, fw_loader);
2449         if (rc)
2450                 goto protocol_err;
2451
2452         hl_fw_linux_update_state(hdev, le32_to_cpu(dyn_regs->cpu_boot_dev_sts0),
2453                                 le32_to_cpu(dyn_regs->cpu_boot_dev_sts1));
2454
2455         hl_fw_dynamic_update_linux_interrupt_if(hdev);
2456
2457         return 0;
2458
2459 protocol_err:
2460         if (fw_loader->dynamic_loader.fw_desc_valid)
2461                 fw_read_errors(hdev, le32_to_cpu(dyn_regs->cpu_boot_err0),
2462                                 le32_to_cpu(dyn_regs->cpu_boot_err1),
2463                                 le32_to_cpu(dyn_regs->cpu_boot_dev_sts0),
2464                                 le32_to_cpu(dyn_regs->cpu_boot_dev_sts1));
2465         return rc;
2466 }
2467
2468 /**
2469  * hl_fw_static_init_cpu - initialize the device CPU using static protocol
2470  *
2471  * @hdev: pointer to the habanalabs device structure
2472  * @fw_loader: managing structure for loading device's FW
2473  *
2474  * @return 0 on success, otherwise non-zero error code
2475  */
2476 static int hl_fw_static_init_cpu(struct hl_device *hdev,
2477                                         struct fw_load_mgr *fw_loader)
2478 {
2479         u32 cpu_msg_status_reg, cpu_timeout, msg_to_cpu_reg, status;
2480         u32 cpu_boot_dev_status0_reg, cpu_boot_dev_status1_reg;
2481         struct static_fw_load_mgr *static_loader;
2482         u32 cpu_boot_status_reg;
2483         int rc;
2484
2485         if (!(hdev->fw_components & FW_TYPE_BOOT_CPU))
2486                 return 0;
2487
2488         /* init common loader parameters */
2489         cpu_timeout = fw_loader->cpu_timeout;
2490
2491         /* init static loader parameters */
2492         static_loader = &fw_loader->static_loader;
2493         cpu_msg_status_reg = static_loader->cpu_cmd_status_to_host_reg;
2494         msg_to_cpu_reg = static_loader->kmd_msg_to_cpu_reg;
2495         cpu_boot_dev_status0_reg = static_loader->cpu_boot_dev_status0_reg;
2496         cpu_boot_dev_status1_reg = static_loader->cpu_boot_dev_status1_reg;
2497         cpu_boot_status_reg = static_loader->cpu_boot_status_reg;
2498
2499         dev_info(hdev->dev, "Going to wait for device boot (up to %lds)\n",
2500                 cpu_timeout / USEC_PER_SEC);
2501
2502         /* Wait for boot FIT request */
2503         rc = hl_poll_timeout(
2504                 hdev,
2505                 cpu_boot_status_reg,
2506                 status,
2507                 status == CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT,
2508                 hdev->fw_poll_interval_usec,
2509                 fw_loader->boot_fit_timeout);
2510
2511         if (rc) {
2512                 dev_dbg(hdev->dev,
2513                         "No boot fit request received, resuming boot\n");
2514         } else {
2515                 rc = hdev->asic_funcs->load_boot_fit_to_device(hdev);
2516                 if (rc)
2517                         goto out;
2518
2519                 /* Clear device CPU message status */
2520                 WREG32(cpu_msg_status_reg, CPU_MSG_CLR);
2521
2522                 /* Signal device CPU that boot loader is ready */
2523                 WREG32(msg_to_cpu_reg, KMD_MSG_FIT_RDY);
2524
2525                 /* Poll for CPU device ack */
2526                 rc = hl_poll_timeout(
2527                         hdev,
2528                         cpu_msg_status_reg,
2529                         status,
2530                         status == CPU_MSG_OK,
2531                         hdev->fw_poll_interval_usec,
2532                         fw_loader->boot_fit_timeout);
2533
2534                 if (rc) {
2535                         dev_err(hdev->dev,
2536                                 "Timeout waiting for boot fit load ack\n");
2537                         goto out;
2538                 }
2539
2540                 /* Clear message */
2541                 WREG32(msg_to_cpu_reg, KMD_MSG_NA);
2542         }
2543
2544         /*
2545          * Make sure CPU boot-loader is running
2546          * Note that the CPU_BOOT_STATUS_SRAM_AVAIL is generally set by Linux
2547          * yet there is a debug scenario in which we loading uboot (without Linux)
2548          * which at later stage is relocated to DRAM. In this case we expect
2549          * uboot to set the CPU_BOOT_STATUS_SRAM_AVAIL and so we add it to the
2550          * poll flags
2551          */
2552         rc = hl_poll_timeout(
2553                 hdev,
2554                 cpu_boot_status_reg,
2555                 status,
2556                 (status == CPU_BOOT_STATUS_DRAM_RDY) ||
2557                 (status == CPU_BOOT_STATUS_NIC_FW_RDY) ||
2558                 (status == CPU_BOOT_STATUS_READY_TO_BOOT) ||
2559                 (status == CPU_BOOT_STATUS_SRAM_AVAIL),
2560                 hdev->fw_poll_interval_usec,
2561                 cpu_timeout);
2562
2563         dev_dbg(hdev->dev, "uboot status = %d\n", status);
2564
2565         /* Read U-Boot version now in case we will later fail */
2566         hl_fw_static_read_device_fw_version(hdev, FW_COMP_BOOT_FIT);
2567
2568         /* update state according to boot stage */
2569         hl_fw_boot_fit_update_state(hdev, cpu_boot_dev_status0_reg,
2570                                                 cpu_boot_dev_status1_reg);
2571
2572         if (rc) {
2573                 detect_cpu_boot_status(hdev, status);
2574                 rc = -EIO;
2575                 goto out;
2576         }
2577
2578         /* Enable DRAM scrambling before Linux boot and after successful
2579          *  UBoot
2580          */
2581         hdev->asic_funcs->init_cpu_scrambler_dram(hdev);
2582
2583         if (!(hdev->fw_components & FW_TYPE_LINUX)) {
2584                 dev_info(hdev->dev, "Skip loading Linux F/W\n");
2585                 rc = 0;
2586                 goto out;
2587         }
2588
2589         if (status == CPU_BOOT_STATUS_SRAM_AVAIL) {
2590                 rc = 0;
2591                 goto out;
2592         }
2593
2594         dev_info(hdev->dev,
2595                 "Loading firmware to device, may take some time...\n");
2596
2597         rc = hdev->asic_funcs->load_firmware_to_device(hdev);
2598         if (rc)
2599                 goto out;
2600
2601         if (fw_loader->skip_bmc) {
2602                 WREG32(msg_to_cpu_reg, KMD_MSG_SKIP_BMC);
2603
2604                 rc = hl_poll_timeout(
2605                         hdev,
2606                         cpu_boot_status_reg,
2607                         status,
2608                         (status == CPU_BOOT_STATUS_BMC_WAITING_SKIPPED),
2609                         hdev->fw_poll_interval_usec,
2610                         cpu_timeout);
2611
2612                 if (rc) {
2613                         dev_err(hdev->dev,
2614                                 "Failed to get ACK on skipping BMC, %d\n",
2615                                 status);
2616                         WREG32(msg_to_cpu_reg, KMD_MSG_NA);
2617                         rc = -EIO;
2618                         goto out;
2619                 }
2620         }
2621
2622         WREG32(msg_to_cpu_reg, KMD_MSG_FIT_RDY);
2623
2624         rc = hl_poll_timeout(
2625                 hdev,
2626                 cpu_boot_status_reg,
2627                 status,
2628                 (status == CPU_BOOT_STATUS_SRAM_AVAIL),
2629                 hdev->fw_poll_interval_usec,
2630                 cpu_timeout);
2631
2632         /* Clear message */
2633         WREG32(msg_to_cpu_reg, KMD_MSG_NA);
2634
2635         if (rc) {
2636                 if (status == CPU_BOOT_STATUS_FIT_CORRUPTED)
2637                         dev_err(hdev->dev,
2638                                 "Device reports FIT image is corrupted\n");
2639                 else
2640                         dev_err(hdev->dev,
2641                                 "Failed to load firmware to device, %d\n",
2642                                 status);
2643
2644                 rc = -EIO;
2645                 goto out;
2646         }
2647
2648         rc = fw_read_errors(hdev, fw_loader->static_loader.boot_err0_reg,
2649                                         fw_loader->static_loader.boot_err1_reg,
2650                                         cpu_boot_dev_status0_reg,
2651                                         cpu_boot_dev_status1_reg);
2652         if (rc)
2653                 return rc;
2654
2655         hl_fw_linux_update_state(hdev, cpu_boot_dev_status0_reg,
2656                                                 cpu_boot_dev_status1_reg);
2657
2658         return 0;
2659
2660 out:
2661         fw_read_errors(hdev, fw_loader->static_loader.boot_err0_reg,
2662                                         fw_loader->static_loader.boot_err1_reg,
2663                                         cpu_boot_dev_status0_reg,
2664                                         cpu_boot_dev_status1_reg);
2665
2666         return rc;
2667 }
2668
2669 /**
2670  * hl_fw_init_cpu - initialize the device CPU
2671  *
2672  * @hdev: pointer to the habanalabs device structure
2673  *
2674  * @return 0 on success, otherwise non-zero error code
2675  *
2676  * perform necessary initializations for device's CPU. takes into account if
2677  * init protocol is static or dynamic.
2678  */
2679 int hl_fw_init_cpu(struct hl_device *hdev)
2680 {
2681         struct asic_fixed_properties *prop = &hdev->asic_prop;
2682         struct fw_load_mgr *fw_loader = &hdev->fw_loader;
2683
2684         return  prop->dynamic_fw_load ?
2685                         hl_fw_dynamic_init_cpu(hdev, fw_loader) :
2686                         hl_fw_static_init_cpu(hdev, fw_loader);
2687 }
2688
2689 void hl_fw_set_pll_profile(struct hl_device *hdev)
2690 {
2691         hl_fw_set_frequency(hdev, hdev->asic_prop.clk_pll_index,
2692                                 hdev->asic_prop.max_freq_value);
2693 }
2694
2695 int hl_fw_get_clk_rate(struct hl_device *hdev, u32 *cur_clk, u32 *max_clk)
2696 {
2697         long value;
2698
2699         if (!hl_device_operational(hdev, NULL))
2700                 return -ENODEV;
2701
2702         if (!hdev->pdev) {
2703                 *cur_clk = 0;
2704                 *max_clk = 0;
2705                 return 0;
2706         }
2707
2708         value = hl_fw_get_frequency(hdev, hdev->asic_prop.clk_pll_index, false);
2709
2710         if (value < 0) {
2711                 dev_err(hdev->dev, "Failed to retrieve device max clock %ld\n", value);
2712                 return value;
2713         }
2714
2715         *max_clk = (value / 1000 / 1000);
2716
2717         value = hl_fw_get_frequency(hdev, hdev->asic_prop.clk_pll_index, true);
2718
2719         if (value < 0) {
2720                 dev_err(hdev->dev, "Failed to retrieve device current clock %ld\n", value);
2721                 return value;
2722         }
2723
2724         *cur_clk = (value / 1000 / 1000);
2725
2726         return 0;
2727 }
2728
2729 long hl_fw_get_frequency(struct hl_device *hdev, u32 pll_index, bool curr)
2730 {
2731         struct cpucp_packet pkt;
2732         u32 used_pll_idx;
2733         u64 result;
2734         int rc;
2735
2736         rc = get_used_pll_index(hdev, pll_index, &used_pll_idx);
2737         if (rc)
2738                 return rc;
2739
2740         memset(&pkt, 0, sizeof(pkt));
2741
2742         if (curr)
2743                 pkt.ctl = cpu_to_le32(CPUCP_PACKET_FREQUENCY_CURR_GET <<
2744                                                 CPUCP_PKT_CTL_OPCODE_SHIFT);
2745         else
2746                 pkt.ctl = cpu_to_le32(CPUCP_PACKET_FREQUENCY_GET << CPUCP_PKT_CTL_OPCODE_SHIFT);
2747
2748         pkt.pll_index = cpu_to_le32((u32)used_pll_idx);
2749
2750         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, &result);
2751
2752         if (rc) {
2753                 dev_err(hdev->dev, "Failed to get frequency of PLL %d, error %d\n",
2754                         used_pll_idx, rc);
2755                 return rc;
2756         }
2757
2758         return (long) result;
2759 }
2760
2761 void hl_fw_set_frequency(struct hl_device *hdev, u32 pll_index, u64 freq)
2762 {
2763         struct cpucp_packet pkt;
2764         u32 used_pll_idx;
2765         int rc;
2766
2767         rc = get_used_pll_index(hdev, pll_index, &used_pll_idx);
2768         if (rc)
2769                 return;
2770
2771         memset(&pkt, 0, sizeof(pkt));
2772
2773         pkt.ctl = cpu_to_le32(CPUCP_PACKET_FREQUENCY_SET << CPUCP_PKT_CTL_OPCODE_SHIFT);
2774         pkt.pll_index = cpu_to_le32((u32)used_pll_idx);
2775         pkt.value = cpu_to_le64(freq);
2776
2777         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, NULL);
2778
2779         if (rc)
2780                 dev_err(hdev->dev, "Failed to set frequency to PLL %d, error %d\n",
2781                         used_pll_idx, rc);
2782 }
2783
2784 long hl_fw_get_max_power(struct hl_device *hdev)
2785 {
2786         struct cpucp_packet pkt;
2787         u64 result;
2788         int rc;
2789
2790         memset(&pkt, 0, sizeof(pkt));
2791
2792         pkt.ctl = cpu_to_le32(CPUCP_PACKET_MAX_POWER_GET << CPUCP_PKT_CTL_OPCODE_SHIFT);
2793
2794         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, &result);
2795
2796         if (rc) {
2797                 dev_err(hdev->dev, "Failed to get max power, error %d\n", rc);
2798                 return rc;
2799         }
2800
2801         return result;
2802 }
2803
2804 void hl_fw_set_max_power(struct hl_device *hdev)
2805 {
2806         struct cpucp_packet pkt;
2807         int rc;
2808
2809         /* TODO: remove this after simulator supports this packet */
2810         if (!hdev->pdev)
2811                 return;
2812
2813         memset(&pkt, 0, sizeof(pkt));
2814
2815         pkt.ctl = cpu_to_le32(CPUCP_PACKET_MAX_POWER_SET << CPUCP_PKT_CTL_OPCODE_SHIFT);
2816         pkt.value = cpu_to_le64(hdev->max_power);
2817
2818         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, NULL);
2819
2820         if (rc)
2821                 dev_err(hdev->dev, "Failed to set max power, error %d\n", rc);
2822 }