drivers/misc/habanalabs/goya/goya.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 /*
   4  * Copyright 2016-2019 HabanaLabs, Ltd.
   5  * All Rights Reserved.
   6  */
   7
   8 #include "goyaP.h"
   9 #include "include/hw_ip/mmu/mmu_general.h"
  10 #include "include/hw_ip/mmu/mmu_v1_0.h"
  11 #include "include/goya/asic_reg/goya_masks.h"
  12
  13 #include <linux/pci.h>
  14 #include <linux/genalloc.h>
  15 #include <linux/hwmon.h>
  16 #include <linux/io-64-nonatomic-lo-hi.h>
  17
  18 /*
  19  * GOYA security scheme:
  20  *
  21  * 1. Host is protected by:
  22  *        - Range registers (When MMU is enabled, DMA RR does NOT protect host)
  23  *        - MMU
  24  *
  25  * 2. DRAM is protected by:
  26  *        - Range registers (protect the first 512MB)
  27  *        - MMU (isolation between users)
  28  *
  29  * 3. Configuration is protected by:
  30  *        - Range registers
  31  *        - Protection bits
  32  *
  33  * When MMU is disabled:
  34  *
  35  * QMAN DMA: PQ, CQ, CP, DMA are secured.
  36  * PQ, CB and the data are on the host.
  37  *
  38  * QMAN TPC/MME:
  39  * PQ, CQ and CP are not secured.
  40  * PQ, CB and the data are on the SRAM/DRAM.
  41  *
  42  * Since QMAN DMA is secured, KMD is parsing the DMA CB:
  43  *     - KMD checks DMA pointer
  44  *     - WREG, MSG_PROT are not allowed.
  45  *     - MSG_LONG/SHORT are allowed.
  46  *
  47  * A read/write transaction by the QMAN to a protected area will succeed if
  48  * and only if the QMAN's CP is secured and MSG_PROT is used
  49  *
  50  *
  51  * When MMU is enabled:
  52  *
  53  * QMAN DMA: PQ, CQ and CP are secured.
  54  * MMU is set to bypass on the Secure props register of the QMAN.
  55  * The reasons we don't enable MMU for PQ, CQ and CP are:
  56  *     - PQ entry is in kernel address space and KMD doesn't map it.
  57  *     - CP writes to MSIX register and to kernel address space (completion
  58  *       queue).
  59  *
  60  * DMA is not secured but because CP is secured, KMD still needs to parse the
  61  * CB, but doesn't need to check the DMA addresses.
  62  *
  63  * For QMAN DMA 0, DMA is also secured because only KMD uses this DMA and KMD
  64  * doesn't map memory in MMU.
  65  *
  66  * QMAN TPC/MME: PQ, CQ and CP aren't secured (no change from MMU disabled mode)
  67  *
  68  * DMA RR does NOT protect host because DMA is not secured
  69  *
  70  */
  71
  72 #define GOYA_MMU_REGS_NUM               63
  73
  74 #define GOYA_DMA_POOL_BLK_SIZE          0x100           /* 256 bytes */
  75
  76 #define GOYA_RESET_TIMEOUT_MSEC         500             /* 500ms */
  77 #define GOYA_PLDM_RESET_TIMEOUT_MSEC    20000           /* 20s */
  78 #define GOYA_RESET_WAIT_MSEC            1               /* 1ms */
  79 #define GOYA_CPU_RESET_WAIT_MSEC        100             /* 100ms */
  80 #define GOYA_PLDM_RESET_WAIT_MSEC       1000            /* 1s */
  81 #define GOYA_TEST_QUEUE_WAIT_USEC       100000          /* 100ms */
  82 #define GOYA_PLDM_MMU_TIMEOUT_USEC      (MMU_CONFIG_TIMEOUT_USEC * 100)
  83 #define GOYA_PLDM_QMAN0_TIMEOUT_USEC    (HL_DEVICE_TIMEOUT_USEC * 30)
  84
  85 #define GOYA_QMAN0_FENCE_VAL            0xD169B243
  86
  87 #define GOYA_MAX_STRING_LEN             20
  88
  89 #define GOYA_CB_POOL_CB_CNT             512
  90 #define GOYA_CB_POOL_CB_SIZE            0x20000         /* 128KB */
  91
  92 static const char goya_irq_name[GOYA_MSIX_ENTRIES][GOYA_MAX_STRING_LEN] = {
  93                 "goya cq 0", "goya cq 1", "goya cq 2", "goya cq 3",
  94                 "goya cq 4", "goya cpu eq"
  95 };
  96
  97 static u16 goya_packet_sizes[MAX_PACKET_ID] = {
  98         [PACKET_WREG_32]        = sizeof(struct packet_wreg32),
  99         [PACKET_WREG_BULK]      = sizeof(struct packet_wreg_bulk),
 100         [PACKET_MSG_LONG]       = sizeof(struct packet_msg_long),
 101         [PACKET_MSG_SHORT]      = sizeof(struct packet_msg_short),
 102         [PACKET_CP_DMA]         = sizeof(struct packet_cp_dma),
 103         [PACKET_MSG_PROT]       = sizeof(struct packet_msg_prot),
 104         [PACKET_FENCE]          = sizeof(struct packet_fence),
 105         [PACKET_LIN_DMA]        = sizeof(struct packet_lin_dma),
 106         [PACKET_NOP]            = sizeof(struct packet_nop),
 107         [PACKET_STOP]           = sizeof(struct packet_stop)
 108 };
 109
 110 static u64 goya_mmu_regs[GOYA_MMU_REGS_NUM] = {
 111         mmDMA_QM_0_GLBL_NON_SECURE_PROPS,
 112         mmDMA_QM_1_GLBL_NON_SECURE_PROPS,
 113         mmDMA_QM_2_GLBL_NON_SECURE_PROPS,
 114         mmDMA_QM_3_GLBL_NON_SECURE_PROPS,
 115         mmDMA_QM_4_GLBL_NON_SECURE_PROPS,
 116         mmTPC0_QM_GLBL_SECURE_PROPS,
 117         mmTPC0_QM_GLBL_NON_SECURE_PROPS,
 118         mmTPC0_CMDQ_GLBL_SECURE_PROPS,
 119         mmTPC0_CMDQ_GLBL_NON_SECURE_PROPS,
 120         mmTPC0_CFG_ARUSER,
 121         mmTPC0_CFG_AWUSER,
 122         mmTPC1_QM_GLBL_SECURE_PROPS,
 123         mmTPC1_QM_GLBL_NON_SECURE_PROPS,
 124         mmTPC1_CMDQ_GLBL_SECURE_PROPS,
 125         mmTPC1_CMDQ_GLBL_NON_SECURE_PROPS,
 126         mmTPC1_CFG_ARUSER,
 127         mmTPC1_CFG_AWUSER,
 128         mmTPC2_QM_GLBL_SECURE_PROPS,
 129         mmTPC2_QM_GLBL_NON_SECURE_PROPS,
 130         mmTPC2_CMDQ_GLBL_SECURE_PROPS,
 131         mmTPC2_CMDQ_GLBL_NON_SECURE_PROPS,
 132         mmTPC2_CFG_ARUSER,
 133         mmTPC2_CFG_AWUSER,
 134         mmTPC3_QM_GLBL_SECURE_PROPS,
 135         mmTPC3_QM_GLBL_NON_SECURE_PROPS,
 136         mmTPC3_CMDQ_GLBL_SECURE_PROPS,
 137         mmTPC3_CMDQ_GLBL_NON_SECURE_PROPS,
 138         mmTPC3_CFG_ARUSER,
 139         mmTPC3_CFG_AWUSER,
 140         mmTPC4_QM_GLBL_SECURE_PROPS,
 141         mmTPC4_QM_GLBL_NON_SECURE_PROPS,
 142         mmTPC4_CMDQ_GLBL_SECURE_PROPS,
 143         mmTPC4_CMDQ_GLBL_NON_SECURE_PROPS,
 144         mmTPC4_CFG_ARUSER,
 145         mmTPC4_CFG_AWUSER,
 146         mmTPC5_QM_GLBL_SECURE_PROPS,
 147         mmTPC5_QM_GLBL_NON_SECURE_PROPS,
 148         mmTPC5_CMDQ_GLBL_SECURE_PROPS,
 149         mmTPC5_CMDQ_GLBL_NON_SECURE_PROPS,
 150         mmTPC5_CFG_ARUSER,
 151         mmTPC5_CFG_AWUSER,
 152         mmTPC6_QM_GLBL_SECURE_PROPS,
 153         mmTPC6_QM_GLBL_NON_SECURE_PROPS,
 154         mmTPC6_CMDQ_GLBL_SECURE_PROPS,
 155         mmTPC6_CMDQ_GLBL_NON_SECURE_PROPS,
 156         mmTPC6_CFG_ARUSER,
 157         mmTPC6_CFG_AWUSER,
 158         mmTPC7_QM_GLBL_SECURE_PROPS,
 159         mmTPC7_QM_GLBL_NON_SECURE_PROPS,
 160         mmTPC7_CMDQ_GLBL_SECURE_PROPS,
 161         mmTPC7_CMDQ_GLBL_NON_SECURE_PROPS,
 162         mmTPC7_CFG_ARUSER,
 163         mmTPC7_CFG_AWUSER,
 164         mmMME_QM_GLBL_SECURE_PROPS,
 165         mmMME_QM_GLBL_NON_SECURE_PROPS,
 166         mmMME_CMDQ_GLBL_SECURE_PROPS,
 167         mmMME_CMDQ_GLBL_NON_SECURE_PROPS,
 168         mmMME_SBA_CONTROL_DATA,
 169         mmMME_SBB_CONTROL_DATA,
 170         mmMME_SBC_CONTROL_DATA,
 171         mmMME_WBC_CONTROL_DATA,
 172         mmPCIE_WRAP_PSOC_ARUSER,
 173         mmPCIE_WRAP_PSOC_AWUSER
 174 };
 175
 176 static u32 goya_all_events[] = {
 177         GOYA_ASYNC_EVENT_ID_PCIE_IF,
 178         GOYA_ASYNC_EVENT_ID_TPC0_ECC,
 179         GOYA_ASYNC_EVENT_ID_TPC1_ECC,
 180         GOYA_ASYNC_EVENT_ID_TPC2_ECC,
 181         GOYA_ASYNC_EVENT_ID_TPC3_ECC,
 182         GOYA_ASYNC_EVENT_ID_TPC4_ECC,
 183         GOYA_ASYNC_EVENT_ID_TPC5_ECC,
 184         GOYA_ASYNC_EVENT_ID_TPC6_ECC,
 185         GOYA_ASYNC_EVENT_ID_TPC7_ECC,
 186         GOYA_ASYNC_EVENT_ID_MME_ECC,
 187         GOYA_ASYNC_EVENT_ID_MME_ECC_EXT,
 188         GOYA_ASYNC_EVENT_ID_MMU_ECC,
 189         GOYA_ASYNC_EVENT_ID_DMA_MACRO,
 190         GOYA_ASYNC_EVENT_ID_DMA_ECC,
 191         GOYA_ASYNC_EVENT_ID_CPU_IF_ECC,
 192         GOYA_ASYNC_EVENT_ID_PSOC_MEM,
 193         GOYA_ASYNC_EVENT_ID_PSOC_CORESIGHT,
 194         GOYA_ASYNC_EVENT_ID_SRAM0,
 195         GOYA_ASYNC_EVENT_ID_SRAM1,
 196         GOYA_ASYNC_EVENT_ID_SRAM2,
 197         GOYA_ASYNC_EVENT_ID_SRAM3,
 198         GOYA_ASYNC_EVENT_ID_SRAM4,
 199         GOYA_ASYNC_EVENT_ID_SRAM5,
 200         GOYA_ASYNC_EVENT_ID_SRAM6,
 201         GOYA_ASYNC_EVENT_ID_SRAM7,
 202         GOYA_ASYNC_EVENT_ID_SRAM8,
 203         GOYA_ASYNC_EVENT_ID_SRAM9,
 204         GOYA_ASYNC_EVENT_ID_SRAM10,
 205         GOYA_ASYNC_EVENT_ID_SRAM11,
 206         GOYA_ASYNC_EVENT_ID_SRAM12,
 207         GOYA_ASYNC_EVENT_ID_SRAM13,
 208         GOYA_ASYNC_EVENT_ID_SRAM14,
 209         GOYA_ASYNC_EVENT_ID_SRAM15,
 210         GOYA_ASYNC_EVENT_ID_SRAM16,
 211         GOYA_ASYNC_EVENT_ID_SRAM17,
 212         GOYA_ASYNC_EVENT_ID_SRAM18,
 213         GOYA_ASYNC_EVENT_ID_SRAM19,
 214         GOYA_ASYNC_EVENT_ID_SRAM20,
 215         GOYA_ASYNC_EVENT_ID_SRAM21,
 216         GOYA_ASYNC_EVENT_ID_SRAM22,
 217         GOYA_ASYNC_EVENT_ID_SRAM23,
 218         GOYA_ASYNC_EVENT_ID_SRAM24,
 219         GOYA_ASYNC_EVENT_ID_SRAM25,
 220         GOYA_ASYNC_EVENT_ID_SRAM26,
 221         GOYA_ASYNC_EVENT_ID_SRAM27,
 222         GOYA_ASYNC_EVENT_ID_SRAM28,
 223         GOYA_ASYNC_EVENT_ID_SRAM29,
 224         GOYA_ASYNC_EVENT_ID_GIC500,
 225         GOYA_ASYNC_EVENT_ID_PLL0,
 226         GOYA_ASYNC_EVENT_ID_PLL1,
 227         GOYA_ASYNC_EVENT_ID_PLL3,
 228         GOYA_ASYNC_EVENT_ID_PLL4,
 229         GOYA_ASYNC_EVENT_ID_PLL5,
 230         GOYA_ASYNC_EVENT_ID_PLL6,
 231         GOYA_ASYNC_EVENT_ID_AXI_ECC,
 232         GOYA_ASYNC_EVENT_ID_L2_RAM_ECC,
 233         GOYA_ASYNC_EVENT_ID_PSOC_GPIO_05_SW_RESET,
 234         GOYA_ASYNC_EVENT_ID_PSOC_GPIO_10_VRHOT_ICRIT,
 235         GOYA_ASYNC_EVENT_ID_PCIE_DEC,
 236         GOYA_ASYNC_EVENT_ID_TPC0_DEC,
 237         GOYA_ASYNC_EVENT_ID_TPC1_DEC,
 238         GOYA_ASYNC_EVENT_ID_TPC2_DEC,
 239         GOYA_ASYNC_EVENT_ID_TPC3_DEC,
 240         GOYA_ASYNC_EVENT_ID_TPC4_DEC,
 241         GOYA_ASYNC_EVENT_ID_TPC5_DEC,
 242         GOYA_ASYNC_EVENT_ID_TPC6_DEC,
 243         GOYA_ASYNC_EVENT_ID_TPC7_DEC,
 244         GOYA_ASYNC_EVENT_ID_MME_WACS,
 245         GOYA_ASYNC_EVENT_ID_MME_WACSD,
 246         GOYA_ASYNC_EVENT_ID_CPU_AXI_SPLITTER,
 247         GOYA_ASYNC_EVENT_ID_PSOC_AXI_DEC,
 248         GOYA_ASYNC_EVENT_ID_PSOC,
 249         GOYA_ASYNC_EVENT_ID_TPC0_KRN_ERR,
 250         GOYA_ASYNC_EVENT_ID_TPC1_KRN_ERR,
 251         GOYA_ASYNC_EVENT_ID_TPC2_KRN_ERR,
 252         GOYA_ASYNC_EVENT_ID_TPC3_KRN_ERR,
 253         GOYA_ASYNC_EVENT_ID_TPC4_KRN_ERR,
 254         GOYA_ASYNC_EVENT_ID_TPC5_KRN_ERR,
 255         GOYA_ASYNC_EVENT_ID_TPC6_KRN_ERR,
 256         GOYA_ASYNC_EVENT_ID_TPC7_KRN_ERR,
 257         GOYA_ASYNC_EVENT_ID_TPC0_CMDQ,
 258         GOYA_ASYNC_EVENT_ID_TPC1_CMDQ,
 259         GOYA_ASYNC_EVENT_ID_TPC2_CMDQ,
 260         GOYA_ASYNC_EVENT_ID_TPC3_CMDQ,
 261         GOYA_ASYNC_EVENT_ID_TPC4_CMDQ,
 262         GOYA_ASYNC_EVENT_ID_TPC5_CMDQ,
 263         GOYA_ASYNC_EVENT_ID_TPC6_CMDQ,
 264         GOYA_ASYNC_EVENT_ID_TPC7_CMDQ,
 265         GOYA_ASYNC_EVENT_ID_TPC0_QM,
 266         GOYA_ASYNC_EVENT_ID_TPC1_QM,
 267         GOYA_ASYNC_EVENT_ID_TPC2_QM,
 268         GOYA_ASYNC_EVENT_ID_TPC3_QM,
 269         GOYA_ASYNC_EVENT_ID_TPC4_QM,
 270         GOYA_ASYNC_EVENT_ID_TPC5_QM,
 271         GOYA_ASYNC_EVENT_ID_TPC6_QM,
 272         GOYA_ASYNC_EVENT_ID_TPC7_QM,
 273         GOYA_ASYNC_EVENT_ID_MME_QM,
 274         GOYA_ASYNC_EVENT_ID_MME_CMDQ,
 275         GOYA_ASYNC_EVENT_ID_DMA0_QM,
 276         GOYA_ASYNC_EVENT_ID_DMA1_QM,
 277         GOYA_ASYNC_EVENT_ID_DMA2_QM,
 278         GOYA_ASYNC_EVENT_ID_DMA3_QM,
 279         GOYA_ASYNC_EVENT_ID_DMA4_QM,
 280         GOYA_ASYNC_EVENT_ID_DMA0_CH,
 281         GOYA_ASYNC_EVENT_ID_DMA1_CH,
 282         GOYA_ASYNC_EVENT_ID_DMA2_CH,
 283         GOYA_ASYNC_EVENT_ID_DMA3_CH,
 284         GOYA_ASYNC_EVENT_ID_DMA4_CH,
 285         GOYA_ASYNC_EVENT_ID_TPC0_BMON_SPMU,
 286         GOYA_ASYNC_EVENT_ID_TPC1_BMON_SPMU,
 287         GOYA_ASYNC_EVENT_ID_TPC2_BMON_SPMU,
 288         GOYA_ASYNC_EVENT_ID_TPC3_BMON_SPMU,
 289         GOYA_ASYNC_EVENT_ID_TPC4_BMON_SPMU,
 290         GOYA_ASYNC_EVENT_ID_TPC5_BMON_SPMU,
 291         GOYA_ASYNC_EVENT_ID_TPC6_BMON_SPMU,
 292         GOYA_ASYNC_EVENT_ID_TPC7_BMON_SPMU,
 293         GOYA_ASYNC_EVENT_ID_DMA_BM_CH0,
 294         GOYA_ASYNC_EVENT_ID_DMA_BM_CH1,
 295         GOYA_ASYNC_EVENT_ID_DMA_BM_CH2,
 296         GOYA_ASYNC_EVENT_ID_DMA_BM_CH3,
 297         GOYA_ASYNC_EVENT_ID_DMA_BM_CH4
 298 };
 299
 300 static int goya_mmu_clear_pgt_range(struct hl_device *hdev);
 301 static int goya_mmu_set_dram_default_page(struct hl_device *hdev);
 302 static int goya_mmu_add_mappings_for_device_cpu(struct hl_device *hdev);
 303 static void goya_mmu_prepare(struct hl_device *hdev, u32 asid);
 304
 305 void goya_get_fixed_properties(struct hl_device *hdev)
 306 {
 307         struct asic_fixed_properties *prop = &hdev->asic_prop;
 308         int i;
 309
 310         for (i = 0 ; i < NUMBER_OF_EXT_HW_QUEUES ; i++) {
 311                 prop->hw_queues_props[i].type = QUEUE_TYPE_EXT;
 312                 prop->hw_queues_props[i].kmd_only = 0;
 313         }
 314
 315         for (; i < NUMBER_OF_EXT_HW_QUEUES + NUMBER_OF_CPU_HW_QUEUES ; i++) {
 316                 prop->hw_queues_props[i].type = QUEUE_TYPE_CPU;
 317                 prop->hw_queues_props[i].kmd_only = 1;
 318         }
 319
 320         for (; i < NUMBER_OF_EXT_HW_QUEUES + NUMBER_OF_CPU_HW_QUEUES +
 321                         NUMBER_OF_INT_HW_QUEUES; i++) {
 322                 prop->hw_queues_props[i].type = QUEUE_TYPE_INT;
 323                 prop->hw_queues_props[i].kmd_only = 0;
 324         }
 325
 326         for (; i < HL_MAX_QUEUES; i++)
 327                 prop->hw_queues_props[i].type = QUEUE_TYPE_NA;
 328
 329         prop->completion_queues_count = NUMBER_OF_CMPLT_QUEUES;
 330
 331         prop->dram_base_address = DRAM_PHYS_BASE;
 332         prop->dram_size = DRAM_PHYS_DEFAULT_SIZE;
 333         prop->dram_end_address = prop->dram_base_address + prop->dram_size;
 334         prop->dram_user_base_address = DRAM_BASE_ADDR_USER;
 335
 336         prop->sram_base_address = SRAM_BASE_ADDR;
 337         prop->sram_size = SRAM_SIZE;
 338         prop->sram_end_address = prop->sram_base_address + prop->sram_size;
 339         prop->sram_user_base_address = prop->sram_base_address +
 340                                                 SRAM_USER_BASE_OFFSET;
 341
 342         prop->mmu_pgt_addr = MMU_PAGE_TABLES_ADDR;
 343         prop->mmu_dram_default_page_addr = MMU_DRAM_DEFAULT_PAGE_ADDR;
 344         if (hdev->pldm)
 345                 prop->mmu_pgt_size = 0x800000; /* 8MB */
 346         else
 347                 prop->mmu_pgt_size = MMU_PAGE_TABLES_SIZE;
 348         prop->mmu_pte_size = HL_PTE_SIZE;
 349         prop->mmu_hop_table_size = HOP_TABLE_SIZE;
 350         prop->mmu_hop0_tables_total_size = HOP0_TABLES_TOTAL_SIZE;
 351         prop->dram_page_size = PAGE_SIZE_2MB;
 352
 353         prop->va_space_host_start_address = VA_HOST_SPACE_START;
 354         prop->va_space_host_end_address = VA_HOST_SPACE_END;
 355         prop->va_space_dram_start_address = VA_DDR_SPACE_START;
 356         prop->va_space_dram_end_address = VA_DDR_SPACE_END;
 357         prop->dram_size_for_default_page_mapping =
 358                         prop->va_space_dram_end_address;
 359         prop->cfg_size = CFG_SIZE;
 360         prop->max_asid = MAX_ASID;
 361         prop->num_of_events = GOYA_ASYNC_EVENT_ID_SIZE;
 362         prop->high_pll = PLL_HIGH_DEFAULT;
 363         prop->cb_pool_cb_cnt = GOYA_CB_POOL_CB_CNT;
 364         prop->cb_pool_cb_size = GOYA_CB_POOL_CB_SIZE;
 365         prop->max_power_default = MAX_POWER_DEFAULT;
 366         prop->tpc_enabled_mask = TPC_ENABLED_MASK;
 367         prop->pcie_dbi_base_address = mmPCIE_DBI_BASE;
 368         prop->pcie_aux_dbi_reg_addr = CFG_BASE + mmPCIE_AUX_DBI;
 369 }
 370
 371 /*
 372  * goya_pci_bars_map - Map PCI BARS of Goya device
 373  *
 374  * @hdev: pointer to hl_device structure
 375  *
 376  * Request PCI regions and map them to kernel virtual addresses.
 377  * Returns 0 on success
 378  *
 379  */
 380 static int goya_pci_bars_map(struct hl_device *hdev)
 381 {
 382         static const char * const name[] = {"SRAM_CFG", "MSIX", "DDR"};
 383         bool is_wc[3] = {false, false, true};
 384         int rc;
 385
 386         rc = hl_pci_bars_map(hdev, name, is_wc);
 387         if (rc)
 388                 return rc;
 389
 390         hdev->rmmio = hdev->pcie_bar[SRAM_CFG_BAR_ID] +
 391                         (CFG_BASE - SRAM_BASE_ADDR);
 392
 393         return 0;
 394 }
 395
 396 static u64 goya_set_ddr_bar_base(struct hl_device *hdev, u64 addr)
 397 {
 398         struct goya_device *goya = hdev->asic_specific;
 399         u64 old_addr = addr;
 400         int rc;
 401
 402         if ((goya) && (goya->ddr_bar_cur_addr == addr))
 403                 return old_addr;
 404
 405         /* Inbound Region 1 - Bar 4 - Point to DDR */
 406         rc = hl_pci_set_dram_bar_base(hdev, 1, 4, addr);
 407         if (rc)
 408                 return U64_MAX;
 409
 410         if (goya) {
 411                 old_addr = goya->ddr_bar_cur_addr;
 412                 goya->ddr_bar_cur_addr = addr;
 413         }
 414
 415         return old_addr;
 416 }
 417
 418 /*
 419  * goya_init_iatu - Initialize the iATU unit inside the PCI controller
 420  *
 421  * @hdev: pointer to hl_device structure
 422  *
 423  * This is needed in case the firmware doesn't initialize the iATU
 424  *
 425  */
 426 static int goya_init_iatu(struct hl_device *hdev)
 427 {
 428         return hl_pci_init_iatu(hdev, SRAM_BASE_ADDR, DRAM_PHYS_BASE,
 429                                 HOST_PHYS_BASE, HOST_PHYS_SIZE);
 430 }
 431
 432 /*
 433  * goya_early_init - GOYA early initialization code
 434  *
 435  * @hdev: pointer to hl_device structure
 436  *
 437  * Verify PCI bars
 438  * Set DMA masks
 439  * PCI controller initialization
 440  * Map PCI bars
 441  *
 442  */
 443 static int goya_early_init(struct hl_device *hdev)
 444 {
 445         struct asic_fixed_properties *prop = &hdev->asic_prop;
 446         struct pci_dev *pdev = hdev->pdev;
 447         u32 val;
 448         int rc;
 449
 450         goya_get_fixed_properties(hdev);
 451
 452         /* Check BAR sizes */
 453         if (pci_resource_len(pdev, SRAM_CFG_BAR_ID) != CFG_BAR_SIZE) {
 454                 dev_err(hdev->dev,
 455                         "Not " HL_NAME "? BAR %d size %llu, expecting %llu\n",
 456                         SRAM_CFG_BAR_ID,
 457                         (unsigned long long) pci_resource_len(pdev,
 458                                                         SRAM_CFG_BAR_ID),
 459                         CFG_BAR_SIZE);
 460                 return -ENODEV;
 461         }
 462
 463         if (pci_resource_len(pdev, MSIX_BAR_ID) != MSIX_BAR_SIZE) {
 464                 dev_err(hdev->dev,
 465                         "Not " HL_NAME "? BAR %d size %llu, expecting %llu\n",
 466                         MSIX_BAR_ID,
 467                         (unsigned long long) pci_resource_len(pdev,
 468                                                                 MSIX_BAR_ID),
 469                         MSIX_BAR_SIZE);
 470                 return -ENODEV;
 471         }
 472
 473         prop->dram_pci_bar_size = pci_resource_len(pdev, DDR_BAR_ID);
 474
 475         rc = hl_pci_init(hdev, 48);
 476         if (rc)
 477                 return rc;
 478
 479         if (!hdev->pldm) {
 480                 val = RREG32(mmPSOC_GLOBAL_CONF_BOOT_STRAP_PINS);
 481                 if (val & PSOC_GLOBAL_CONF_BOOT_STRAP_PINS_SRIOV_EN_MASK)
 482                         dev_warn(hdev->dev,
 483                                 "PCI strap is not configured correctly, PCI bus errors may occur\n");
 484         }
 485
 486         return 0;
 487 }
 488
 489 /*
 490  * goya_early_fini - GOYA early finalization code
 491  *
 492  * @hdev: pointer to hl_device structure
 493  *
 494  * Unmap PCI bars
 495  *
 496  */
 497 static int goya_early_fini(struct hl_device *hdev)
 498 {
 499         hl_pci_fini(hdev);
 500
 501         return 0;
 502 }
 503
 504 static void goya_mmu_prepare_reg(struct hl_device *hdev, u64 reg, u32 asid)
 505 {
 506         /* mask to zero the MMBP and ASID bits */
 507         WREG32_AND(reg, ~0x7FF);
 508         WREG32_OR(reg, asid);
 509 }
 510
 511 static void goya_qman0_set_security(struct hl_device *hdev, bool secure)
 512 {
 513         struct goya_device *goya = hdev->asic_specific;
 514
 515         if (!(goya->hw_cap_initialized & HW_CAP_MMU))
 516                 return;
 517
 518         if (secure)
 519                 WREG32(mmDMA_QM_0_GLBL_PROT, QMAN_DMA_FULLY_TRUSTED);
 520         else
 521                 WREG32(mmDMA_QM_0_GLBL_PROT, QMAN_DMA_PARTLY_TRUSTED);
 522
 523         RREG32(mmDMA_QM_0_GLBL_PROT);
 524 }
 525
 526 /*
 527  * goya_fetch_psoc_frequency - Fetch PSOC frequency values
 528  *
 529  * @hdev: pointer to hl_device structure
 530  *
 531  */
 532 static void goya_fetch_psoc_frequency(struct hl_device *hdev)
 533 {
 534         struct asic_fixed_properties *prop = &hdev->asic_prop;
 535
 536         prop->psoc_pci_pll_nr = RREG32(mmPSOC_PCI_PLL_NR);
 537         prop->psoc_pci_pll_nf = RREG32(mmPSOC_PCI_PLL_NF);
 538         prop->psoc_pci_pll_od = RREG32(mmPSOC_PCI_PLL_OD);
 539         prop->psoc_pci_pll_div_factor = RREG32(mmPSOC_PCI_PLL_DIV_FACTOR_1);
 540 }
 541
 542 int goya_late_init(struct hl_device *hdev)
 543 {
 544         struct asic_fixed_properties *prop = &hdev->asic_prop;
 545         int rc;
 546
 547         goya_fetch_psoc_frequency(hdev);
 548
 549         rc = goya_mmu_clear_pgt_range(hdev);
 550         if (rc) {
 551                 dev_err(hdev->dev,
 552                         "Failed to clear MMU page tables range %d\n", rc);
 553                 return rc;
 554         }
 555
 556         rc = goya_mmu_set_dram_default_page(hdev);
 557         if (rc) {
 558                 dev_err(hdev->dev, "Failed to set DRAM default page %d\n", rc);
 559                 return rc;
 560         }
 561
 562         rc = goya_mmu_add_mappings_for_device_cpu(hdev);
 563         if (rc)
 564                 return rc;
 565
 566         rc = goya_init_cpu_queues(hdev);
 567         if (rc)
 568                 return rc;
 569
 570         rc = goya_test_cpu_queue(hdev);
 571         if (rc)
 572                 return rc;
 573
 574         rc = goya_armcp_info_get(hdev);
 575         if (rc) {
 576                 dev_err(hdev->dev, "Failed to get armcp info %d\n", rc);
 577                 return rc;
 578         }
 579
 580         /* Now that we have the DRAM size in ASIC prop, we need to check
 581          * its size and configure the DMA_IF DDR wrap protection (which is in
 582          * the MMU block) accordingly. The value is the log2 of the DRAM size
 583          */
 584         WREG32(mmMMU_LOG2_DDR_SIZE, ilog2(prop->dram_size));
 585
 586         rc = hl_fw_send_pci_access_msg(hdev, ARMCP_PACKET_ENABLE_PCI_ACCESS);
 587         if (rc) {
 588                 dev_err(hdev->dev,
 589                         "Failed to enable PCI access from CPU %d\n", rc);
 590                 return rc;
 591         }
 592
 593         WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR,
 594                         GOYA_ASYNC_EVENT_ID_INTS_REGISTER);
 595
 596         return 0;
 597 }
 598
 599 /*
 600  * goya_late_fini - GOYA late tear-down code
 601  *
 602  * @hdev: pointer to hl_device structure
 603  *
 604  * Free sensors allocated structures
 605  */
 606 void goya_late_fini(struct hl_device *hdev)
 607 {
 608         const struct hwmon_channel_info **channel_info_arr;
 609         int i = 0;
 610
 611         if (!hdev->hl_chip_info->info)
 612                 return;
 613
 614         channel_info_arr = hdev->hl_chip_info->info;
 615
 616         while (channel_info_arr[i]) {
 617                 kfree(channel_info_arr[i]->config);
 618                 kfree(channel_info_arr[i]);
 619                 i++;
 620         }
 621
 622         kfree(channel_info_arr);
 623
 624         hdev->hl_chip_info->info = NULL;
 625 }
 626
 627 /*
 628  * goya_sw_init - Goya software initialization code
 629  *
 630  * @hdev: pointer to hl_device structure
 631  *
 632  */
 633 static int goya_sw_init(struct hl_device *hdev)
 634 {
 635         struct goya_device *goya;
 636         int rc;
 637
 638         /* Allocate device structure */
 639         goya = kzalloc(sizeof(*goya), GFP_KERNEL);
 640         if (!goya)
 641                 return -ENOMEM;
 642
 643         /* according to goya_init_iatu */
 644         goya->ddr_bar_cur_addr = DRAM_PHYS_BASE;
 645
 646         goya->mme_clk = GOYA_PLL_FREQ_LOW;
 647         goya->tpc_clk = GOYA_PLL_FREQ_LOW;
 648         goya->ic_clk = GOYA_PLL_FREQ_LOW;
 649
 650         hdev->asic_specific = goya;
 651
 652         /* Create DMA pool for small allocations */
 653         hdev->dma_pool = dma_pool_create(dev_name(hdev->dev),
 654                         &hdev->pdev->dev, GOYA_DMA_POOL_BLK_SIZE, 8, 0);
 655         if (!hdev->dma_pool) {
 656                 dev_err(hdev->dev, "failed to create DMA pool\n");
 657                 rc = -ENOMEM;
 658                 goto free_goya_device;
 659         }
 660
 661         hdev->cpu_accessible_dma_mem =
 662                         hdev->asic_funcs->asic_dma_alloc_coherent(hdev,
 663                                         HL_CPU_ACCESSIBLE_MEM_SIZE,
 664                                         &hdev->cpu_accessible_dma_address,
 665                                         GFP_KERNEL | __GFP_ZERO);
 666
 667         if (!hdev->cpu_accessible_dma_mem) {
 668                 rc = -ENOMEM;
 669                 goto free_dma_pool;
 670         }
 671
 672         dev_dbg(hdev->dev, "cpu accessible memory at bus address 0x%llx\n",
 673                 hdev->cpu_accessible_dma_address);
 674
 675         hdev->cpu_accessible_dma_pool = gen_pool_create(ilog2(32), -1);
 676         if (!hdev->cpu_accessible_dma_pool) {
 677                 dev_err(hdev->dev,
 678                         "Failed to create CPU accessible DMA pool\n");
 679                 rc = -ENOMEM;
 680                 goto free_cpu_dma_mem;
 681         }
 682
 683         rc = gen_pool_add(hdev->cpu_accessible_dma_pool,
 684                                 (uintptr_t) hdev->cpu_accessible_dma_mem,
 685                                 HL_CPU_ACCESSIBLE_MEM_SIZE, -1);
 686         if (rc) {
 687                 dev_err(hdev->dev,
 688                         "Failed to add memory to CPU accessible DMA pool\n");
 689                 rc = -EFAULT;
 690                 goto free_cpu_accessible_dma_pool;
 691         }
 692
 693         spin_lock_init(&goya->hw_queues_lock);
 694
 695         return 0;
 696
 697 free_cpu_accessible_dma_pool:
 698         gen_pool_destroy(hdev->cpu_accessible_dma_pool);
 699 free_cpu_dma_mem:
 700         hdev->asic_funcs->asic_dma_free_coherent(hdev,
 701                         HL_CPU_ACCESSIBLE_MEM_SIZE,
 702                         hdev->cpu_accessible_dma_mem,
 703                         hdev->cpu_accessible_dma_address);
 704 free_dma_pool:
 705         dma_pool_destroy(hdev->dma_pool);
 706 free_goya_device:
 707         kfree(goya);
 708
 709         return rc;
 710 }
 711
 712 /*
 713  * goya_sw_fini - Goya software tear-down code
 714  *
 715  * @hdev: pointer to hl_device structure
 716  *
 717  */
 718 static int goya_sw_fini(struct hl_device *hdev)
 719 {
 720         struct goya_device *goya = hdev->asic_specific;
 721
 722         gen_pool_destroy(hdev->cpu_accessible_dma_pool);
 723
 724         hdev->asic_funcs->asic_dma_free_coherent(hdev,
 725                         HL_CPU_ACCESSIBLE_MEM_SIZE,
 726                         hdev->cpu_accessible_dma_mem,
 727                         hdev->cpu_accessible_dma_address);
 728
 729         dma_pool_destroy(hdev->dma_pool);
 730
 731         kfree(goya);
 732
 733         return 0;
 734 }
 735
 736 static void goya_init_dma_qman(struct hl_device *hdev, int dma_id,
 737                 dma_addr_t bus_address)
 738 {
 739         struct goya_device *goya = hdev->asic_specific;
 740         u32 mtr_base_lo, mtr_base_hi;
 741         u32 so_base_lo, so_base_hi;
 742         u32 gic_base_lo, gic_base_hi;
 743         u32 reg_off = dma_id * (mmDMA_QM_1_PQ_PI - mmDMA_QM_0_PQ_PI);
 744
 745         mtr_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
 746         mtr_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
 747         so_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
 748         so_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
 749
 750         gic_base_lo =
 751                 lower_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
 752         gic_base_hi =
 753                 upper_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
 754
 755         WREG32(mmDMA_QM_0_PQ_BASE_LO + reg_off, lower_32_bits(bus_address));
 756         WREG32(mmDMA_QM_0_PQ_BASE_HI + reg_off, upper_32_bits(bus_address));
 757
 758         WREG32(mmDMA_QM_0_PQ_SIZE + reg_off, ilog2(HL_QUEUE_LENGTH));
 759         WREG32(mmDMA_QM_0_PQ_PI + reg_off, 0);
 760         WREG32(mmDMA_QM_0_PQ_CI + reg_off, 0);
 761
 762         WREG32(mmDMA_QM_0_CP_MSG_BASE0_ADDR_LO + reg_off, mtr_base_lo);
 763         WREG32(mmDMA_QM_0_CP_MSG_BASE0_ADDR_HI + reg_off, mtr_base_hi);
 764         WREG32(mmDMA_QM_0_CP_MSG_BASE1_ADDR_LO + reg_off, so_base_lo);
 765         WREG32(mmDMA_QM_0_CP_MSG_BASE1_ADDR_HI + reg_off, so_base_hi);
 766         WREG32(mmDMA_QM_0_GLBL_ERR_ADDR_LO + reg_off, gic_base_lo);
 767         WREG32(mmDMA_QM_0_GLBL_ERR_ADDR_HI + reg_off, gic_base_hi);
 768         WREG32(mmDMA_QM_0_GLBL_ERR_WDATA + reg_off,
 769                         GOYA_ASYNC_EVENT_ID_DMA0_QM + dma_id);
 770
 771         /* PQ has buffer of 2 cache lines, while CQ has 8 lines */
 772         WREG32(mmDMA_QM_0_PQ_CFG1 + reg_off, 0x00020002);
 773         WREG32(mmDMA_QM_0_CQ_CFG1 + reg_off, 0x00080008);
 774
 775         if (goya->hw_cap_initialized & HW_CAP_MMU)
 776                 WREG32(mmDMA_QM_0_GLBL_PROT + reg_off, QMAN_DMA_PARTLY_TRUSTED);
 777         else
 778                 WREG32(mmDMA_QM_0_GLBL_PROT + reg_off, QMAN_DMA_FULLY_TRUSTED);
 779
 780         WREG32(mmDMA_QM_0_GLBL_ERR_CFG + reg_off, QMAN_DMA_ERR_MSG_EN);
 781         WREG32(mmDMA_QM_0_GLBL_CFG0 + reg_off, QMAN_DMA_ENABLE);
 782 }
 783
 784 static void goya_init_dma_ch(struct hl_device *hdev, int dma_id)
 785 {
 786         u32 gic_base_lo, gic_base_hi;
 787         u64 sob_addr;
 788         u32 reg_off = dma_id * (mmDMA_CH_1_CFG1 - mmDMA_CH_0_CFG1);
 789
 790         gic_base_lo =
 791                 lower_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
 792         gic_base_hi =
 793                 upper_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
 794
 795         WREG32(mmDMA_CH_0_ERRMSG_ADDR_LO + reg_off, gic_base_lo);
 796         WREG32(mmDMA_CH_0_ERRMSG_ADDR_HI + reg_off, gic_base_hi);
 797         WREG32(mmDMA_CH_0_ERRMSG_WDATA + reg_off,
 798                         GOYA_ASYNC_EVENT_ID_DMA0_CH + dma_id);
 799
 800         if (dma_id)
 801                 sob_addr = CFG_BASE + mmSYNC_MNGR_SOB_OBJ_1000 +
 802                                 (dma_id - 1) * 4;
 803         else
 804                 sob_addr = CFG_BASE + mmSYNC_MNGR_SOB_OBJ_1007;
 805
 806         WREG32(mmDMA_CH_0_WR_COMP_ADDR_HI + reg_off, upper_32_bits(sob_addr));
 807         WREG32(mmDMA_CH_0_WR_COMP_WDATA + reg_off, 0x80000001);
 808 }
 809
 810 /*
 811  * goya_init_dma_qmans - Initialize QMAN DMA registers
 812  *
 813  * @hdev: pointer to hl_device structure
 814  *
 815  * Initialize the H/W registers of the QMAN DMA channels
 816  *
 817  */
 818 void goya_init_dma_qmans(struct hl_device *hdev)
 819 {
 820         struct goya_device *goya = hdev->asic_specific;
 821         struct hl_hw_queue *q;
 822         int i;
 823
 824         if (goya->hw_cap_initialized & HW_CAP_DMA)
 825                 return;
 826
 827         q = &hdev->kernel_queues[0];
 828
 829         for (i = 0 ; i < NUMBER_OF_EXT_HW_QUEUES ; i++, q++) {
 830                 goya_init_dma_qman(hdev, i, q->bus_address);
 831                 goya_init_dma_ch(hdev, i);
 832         }
 833
 834         goya->hw_cap_initialized |= HW_CAP_DMA;
 835 }
 836
 837 /*
 838  * goya_disable_external_queues - Disable external queues
 839  *
 840  * @hdev: pointer to hl_device structure
 841  *
 842  */
 843 static void goya_disable_external_queues(struct hl_device *hdev)
 844 {
 845         WREG32(mmDMA_QM_0_GLBL_CFG0, 0);
 846         WREG32(mmDMA_QM_1_GLBL_CFG0, 0);
 847         WREG32(mmDMA_QM_2_GLBL_CFG0, 0);
 848         WREG32(mmDMA_QM_3_GLBL_CFG0, 0);
 849         WREG32(mmDMA_QM_4_GLBL_CFG0, 0);
 850 }
 851
 852 static int goya_stop_queue(struct hl_device *hdev, u32 cfg_reg,
 853                                 u32 cp_sts_reg, u32 glbl_sts0_reg)
 854 {
 855         int rc;
 856         u32 status;
 857
 858         /* use the values of TPC0 as they are all the same*/
 859
 860         WREG32(cfg_reg, 1 << TPC0_QM_GLBL_CFG1_CP_STOP_SHIFT);
 861
 862         status = RREG32(cp_sts_reg);
 863         if (status & TPC0_QM_CP_STS_FENCE_IN_PROGRESS_MASK) {
 864                 rc = hl_poll_timeout(
 865                         hdev,
 866                         cp_sts_reg,
 867                         status,
 868                         !(status & TPC0_QM_CP_STS_FENCE_IN_PROGRESS_MASK),
 869                         1000,
 870                         QMAN_FENCE_TIMEOUT_USEC);
 871
 872                 /* if QMAN is stuck in fence no need to check for stop */
 873                 if (rc)
 874                         return 0;
 875         }
 876
 877         rc = hl_poll_timeout(
 878                 hdev,
 879                 glbl_sts0_reg,
 880                 status,
 881                 (status & TPC0_QM_GLBL_STS0_CP_IS_STOP_MASK),
 882                 1000,
 883                 QMAN_STOP_TIMEOUT_USEC);
 884
 885         if (rc) {
 886                 dev_err(hdev->dev,
 887                         "Timeout while waiting for QMAN to stop\n");
 888                 return -EINVAL;
 889         }
 890
 891         return 0;
 892 }
 893
 894 /*
 895  * goya_stop_external_queues - Stop external queues
 896  *
 897  * @hdev: pointer to hl_device structure
 898  *
 899  * Returns 0 on success
 900  *
 901  */
 902 static int goya_stop_external_queues(struct hl_device *hdev)
 903 {
 904         int rc, retval = 0;
 905
 906         rc = goya_stop_queue(hdev,
 907                         mmDMA_QM_0_GLBL_CFG1,
 908                         mmDMA_QM_0_CP_STS,
 909                         mmDMA_QM_0_GLBL_STS0);
 910
 911         if (rc) {
 912                 dev_err(hdev->dev, "failed to stop DMA QMAN 0\n");
 913                 retval = -EIO;
 914         }
 915
 916         rc = goya_stop_queue(hdev,
 917                         mmDMA_QM_1_GLBL_CFG1,
 918                         mmDMA_QM_1_CP_STS,
 919                         mmDMA_QM_1_GLBL_STS0);
 920
 921         if (rc) {
 922                 dev_err(hdev->dev, "failed to stop DMA QMAN 1\n");
 923                 retval = -EIO;
 924         }
 925
 926         rc = goya_stop_queue(hdev,
 927                         mmDMA_QM_2_GLBL_CFG1,
 928                         mmDMA_QM_2_CP_STS,
 929                         mmDMA_QM_2_GLBL_STS0);
 930
 931         if (rc) {
 932                 dev_err(hdev->dev, "failed to stop DMA QMAN 2\n");
 933                 retval = -EIO;
 934         }
 935
 936         rc = goya_stop_queue(hdev,
 937                         mmDMA_QM_3_GLBL_CFG1,
 938                         mmDMA_QM_3_CP_STS,
 939                         mmDMA_QM_3_GLBL_STS0);
 940
 941         if (rc) {
 942                 dev_err(hdev->dev, "failed to stop DMA QMAN 3\n");
 943                 retval = -EIO;
 944         }
 945
 946         rc = goya_stop_queue(hdev,
 947                         mmDMA_QM_4_GLBL_CFG1,
 948                         mmDMA_QM_4_CP_STS,
 949                         mmDMA_QM_4_GLBL_STS0);
 950
 951         if (rc) {
 952                 dev_err(hdev->dev, "failed to stop DMA QMAN 4\n");
 953                 retval = -EIO;
 954         }
 955
 956         return retval;
 957 }
 958
 959 /*
 960  * goya_init_cpu_queues - Initialize PQ/CQ/EQ of CPU
 961  *
 962  * @hdev: pointer to hl_device structure
 963  *
 964  * Returns 0 on success
 965  *
 966  */
 967 int goya_init_cpu_queues(struct hl_device *hdev)
 968 {
 969         struct goya_device *goya = hdev->asic_specific;
 970         struct hl_eq *eq;
 971         u32 status;
 972         struct hl_hw_queue *cpu_pq = &hdev->kernel_queues[GOYA_QUEUE_ID_CPU_PQ];
 973         int err;
 974
 975         if (!hdev->cpu_queues_enable)
 976                 return 0;
 977
 978         if (goya->hw_cap_initialized & HW_CAP_CPU_Q)
 979                 return 0;
 980
 981         eq = &hdev->event_queue;
 982
 983         WREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_0,
 984                         lower_32_bits(cpu_pq->bus_address));
 985         WREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_1,
 986                         upper_32_bits(cpu_pq->bus_address));
 987
 988         WREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_2, lower_32_bits(eq->bus_address));
 989         WREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_3, upper_32_bits(eq->bus_address));
 990
 991         WREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_8,
 992                         lower_32_bits(VA_CPU_ACCESSIBLE_MEM_ADDR));
 993         WREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_9,
 994                         upper_32_bits(VA_CPU_ACCESSIBLE_MEM_ADDR));
 995
 996         WREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_5, HL_QUEUE_SIZE_IN_BYTES);
 997         WREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_4, HL_EQ_SIZE_IN_BYTES);
 998         WREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_10, HL_CPU_ACCESSIBLE_MEM_SIZE);
 999
1000         /* Used for EQ CI */
1001         WREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_6, 0);
1002
1003         WREG32(mmCPU_IF_PF_PQ_PI, 0);
1004
1005         WREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_7, PQ_INIT_STATUS_READY_FOR_CP);
1006
1007         WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR,
1008                         GOYA_ASYNC_EVENT_ID_PI_UPDATE);
1009
1010         err = hl_poll_timeout(
1011                 hdev,
1012                 mmPSOC_GLOBAL_CONF_SCRATCHPAD_7,
1013                 status,
1014                 (status == PQ_INIT_STATUS_READY_FOR_HOST),
1015                 1000,
1016                 GOYA_CPU_TIMEOUT_USEC);
1017
1018         if (err) {
1019                 dev_err(hdev->dev,
1020                         "Failed to setup communication with device CPU\n");
1021                 return -EIO;
1022         }
1023
1024         goya->hw_cap_initialized |= HW_CAP_CPU_Q;
1025         return 0;
1026 }
1027
1028 static void goya_set_pll_refclk(struct hl_device *hdev)
1029 {
1030         WREG32(mmCPU_PLL_DIV_SEL_0, 0x0);
1031         WREG32(mmCPU_PLL_DIV_SEL_1, 0x0);
1032         WREG32(mmCPU_PLL_DIV_SEL_2, 0x0);
1033         WREG32(mmCPU_PLL_DIV_SEL_3, 0x0);
1034
1035         WREG32(mmIC_PLL_DIV_SEL_0, 0x0);
1036         WREG32(mmIC_PLL_DIV_SEL_1, 0x0);
1037         WREG32(mmIC_PLL_DIV_SEL_2, 0x0);
1038         WREG32(mmIC_PLL_DIV_SEL_3, 0x0);
1039
1040         WREG32(mmMC_PLL_DIV_SEL_0, 0x0);
1041         WREG32(mmMC_PLL_DIV_SEL_1, 0x0);
1042         WREG32(mmMC_PLL_DIV_SEL_2, 0x0);
1043         WREG32(mmMC_PLL_DIV_SEL_3, 0x0);
1044
1045         WREG32(mmPSOC_MME_PLL_DIV_SEL_0, 0x0);
1046         WREG32(mmPSOC_MME_PLL_DIV_SEL_1, 0x0);
1047         WREG32(mmPSOC_MME_PLL_DIV_SEL_2, 0x0);
1048         WREG32(mmPSOC_MME_PLL_DIV_SEL_3, 0x0);
1049
1050         WREG32(mmPSOC_PCI_PLL_DIV_SEL_0, 0x0);
1051         WREG32(mmPSOC_PCI_PLL_DIV_SEL_1, 0x0);
1052         WREG32(mmPSOC_PCI_PLL_DIV_SEL_2, 0x0);
1053         WREG32(mmPSOC_PCI_PLL_DIV_SEL_3, 0x0);
1054
1055         WREG32(mmPSOC_EMMC_PLL_DIV_SEL_0, 0x0);
1056         WREG32(mmPSOC_EMMC_PLL_DIV_SEL_1, 0x0);
1057         WREG32(mmPSOC_EMMC_PLL_DIV_SEL_2, 0x0);
1058         WREG32(mmPSOC_EMMC_PLL_DIV_SEL_3, 0x0);
1059
1060         WREG32(mmTPC_PLL_DIV_SEL_0, 0x0);
1061         WREG32(mmTPC_PLL_DIV_SEL_1, 0x0);
1062         WREG32(mmTPC_PLL_DIV_SEL_2, 0x0);
1063         WREG32(mmTPC_PLL_DIV_SEL_3, 0x0);
1064 }
1065
1066 static void goya_disable_clk_rlx(struct hl_device *hdev)
1067 {
1068         WREG32(mmPSOC_MME_PLL_CLK_RLX_0, 0x100010);
1069         WREG32(mmIC_PLL_CLK_RLX_0, 0x100010);
1070 }
1071
1072 static void _goya_tpc_mbist_workaround(struct hl_device *hdev, u8 tpc_id)
1073 {
1074         u64 tpc_eml_address;
1075         u32 val, tpc_offset, tpc_eml_offset, tpc_slm_offset;
1076         int err, slm_index;
1077
1078         tpc_offset = tpc_id * 0x40000;
1079         tpc_eml_offset = tpc_id * 0x200000;
1080         tpc_eml_address = (mmTPC0_EML_CFG_BASE + tpc_eml_offset - CFG_BASE);
1081         tpc_slm_offset = tpc_eml_address + 0x100000;
1082
1083         /*
1084          * Workaround for Bug H2 #2443 :
1085          * "TPC SB is not initialized on chip reset"
1086          */
1087
1088         val = RREG32(mmTPC0_CFG_FUNC_MBIST_CNTRL + tpc_offset);
1089         if (val & TPC0_CFG_FUNC_MBIST_CNTRL_MBIST_ACTIVE_MASK)
1090                 dev_warn(hdev->dev, "TPC%d MBIST ACTIVE is not cleared\n",
1091                         tpc_id);
1092
1093         WREG32(mmTPC0_CFG_FUNC_MBIST_PAT + tpc_offset, val & 0xFFFFF000);
1094
1095         WREG32(mmTPC0_CFG_FUNC_MBIST_MEM_0 + tpc_offset, 0x37FF);
1096         WREG32(mmTPC0_CFG_FUNC_MBIST_MEM_1 + tpc_offset, 0x303F);
1097         WREG32(mmTPC0_CFG_FUNC_MBIST_MEM_2 + tpc_offset, 0x71FF);
1098         WREG32(mmTPC0_CFG_FUNC_MBIST_MEM_3 + tpc_offset, 0x71FF);
1099         WREG32(mmTPC0_CFG_FUNC_MBIST_MEM_4 + tpc_offset, 0x70FF);
1100         WREG32(mmTPC0_CFG_FUNC_MBIST_MEM_5 + tpc_offset, 0x70FF);
1101         WREG32(mmTPC0_CFG_FUNC_MBIST_MEM_6 + tpc_offset, 0x70FF);
1102         WREG32(mmTPC0_CFG_FUNC_MBIST_MEM_7 + tpc_offset, 0x70FF);
1103         WREG32(mmTPC0_CFG_FUNC_MBIST_MEM_8 + tpc_offset, 0x70FF);
1104         WREG32(mmTPC0_CFG_FUNC_MBIST_MEM_9 + tpc_offset, 0x70FF);
1105
1106         WREG32_OR(mmTPC0_CFG_FUNC_MBIST_CNTRL + tpc_offset,
1107                 1 << TPC0_CFG_FUNC_MBIST_CNTRL_MBIST_START_SHIFT);
1108
1109         err = hl_poll_timeout(
1110                 hdev,
1111                 mmTPC0_CFG_FUNC_MBIST_CNTRL + tpc_offset,
1112                 val,
1113                 (val & TPC0_CFG_FUNC_MBIST_CNTRL_MBIST_DONE_MASK),
1114                 1000,
1115                 HL_DEVICE_TIMEOUT_USEC);
1116
1117         if (err)
1118                 dev_err(hdev->dev,
1119                         "Timeout while waiting for TPC%d MBIST DONE\n", tpc_id);
1120
1121         WREG32_OR(mmTPC0_EML_CFG_DBG_CNT + tpc_eml_offset,
1122                 1 << TPC0_EML_CFG_DBG_CNT_CORE_RST_SHIFT);
1123
1124         msleep(GOYA_RESET_WAIT_MSEC);
1125
1126         WREG32_AND(mmTPC0_EML_CFG_DBG_CNT + tpc_eml_offset,
1127                 ~(1 << TPC0_EML_CFG_DBG_CNT_CORE_RST_SHIFT));
1128
1129         msleep(GOYA_RESET_WAIT_MSEC);
1130
1131         for (slm_index = 0 ; slm_index < 256 ; slm_index++)
1132                 WREG32(tpc_slm_offset + (slm_index << 2), 0);
1133
1134         val = RREG32(tpc_slm_offset);
1135 }
1136
1137 static void goya_tpc_mbist_workaround(struct hl_device *hdev)
1138 {
1139         struct goya_device *goya = hdev->asic_specific;
1140         int i;
1141
1142         if (hdev->pldm)
1143                 return;
1144
1145         if (goya->hw_cap_initialized & HW_CAP_TPC_MBIST)
1146                 return;
1147
1148         /* Workaround for H2 #2443 */
1149
1150         for (i = 0 ; i < TPC_MAX_NUM ; i++)
1151                 _goya_tpc_mbist_workaround(hdev, i);
1152
1153         goya->hw_cap_initialized |= HW_CAP_TPC_MBIST;
1154 }
1155
1156 /*
1157  * goya_init_golden_registers - Initialize golden registers
1158  *
1159  * @hdev: pointer to hl_device structure
1160  *
1161  * Initialize the H/W registers of the device
1162  *
1163  */
1164 static void goya_init_golden_registers(struct hl_device *hdev)
1165 {
1166         struct goya_device *goya = hdev->asic_specific;
1167         u32 polynom[10], tpc_intr_mask, offset;
1168         int i;
1169
1170         if (goya->hw_cap_initialized & HW_CAP_GOLDEN)
1171                 return;
1172
1173         polynom[0] = 0x00020080;
1174         polynom[1] = 0x00401000;
1175         polynom[2] = 0x00200800;
1176         polynom[3] = 0x00002000;
1177         polynom[4] = 0x00080200;
1178         polynom[5] = 0x00040100;
1179         polynom[6] = 0x00100400;
1180         polynom[7] = 0x00004000;
1181         polynom[8] = 0x00010000;
1182         polynom[9] = 0x00008000;
1183
1184         /* Mask all arithmetic interrupts from TPC */
1185         tpc_intr_mask = 0x7FFF;
1186
1187         for (i = 0, offset = 0 ; i < 6 ; i++, offset += 0x20000) {
1188                 WREG32(mmSRAM_Y0_X0_RTR_HBW_RD_RQ_L_ARB + offset, 0x302);
1189                 WREG32(mmSRAM_Y0_X1_RTR_HBW_RD_RQ_L_ARB + offset, 0x302);
1190                 WREG32(mmSRAM_Y0_X2_RTR_HBW_RD_RQ_L_ARB + offset, 0x302);
1191                 WREG32(mmSRAM_Y0_X3_RTR_HBW_RD_RQ_L_ARB + offset, 0x302);
1192                 WREG32(mmSRAM_Y0_X4_RTR_HBW_RD_RQ_L_ARB + offset, 0x302);
1193
1194                 WREG32(mmSRAM_Y0_X0_RTR_HBW_DATA_L_ARB + offset, 0x204);
1195                 WREG32(mmSRAM_Y0_X1_RTR_HBW_DATA_L_ARB + offset, 0x204);
1196                 WREG32(mmSRAM_Y0_X2_RTR_HBW_DATA_L_ARB + offset, 0x204);
1197                 WREG32(mmSRAM_Y0_X3_RTR_HBW_DATA_L_ARB + offset, 0x204);
1198                 WREG32(mmSRAM_Y0_X4_RTR_HBW_DATA_L_ARB + offset, 0x204);
1199
1200
1201                 WREG32(mmSRAM_Y0_X0_RTR_HBW_DATA_E_ARB + offset, 0x206);
1202                 WREG32(mmSRAM_Y0_X1_RTR_HBW_DATA_E_ARB + offset, 0x206);
1203                 WREG32(mmSRAM_Y0_X2_RTR_HBW_DATA_E_ARB + offset, 0x206);
1204                 WREG32(mmSRAM_Y0_X3_RTR_HBW_DATA_E_ARB + offset, 0x207);
1205                 WREG32(mmSRAM_Y0_X4_RTR_HBW_DATA_E_ARB + offset, 0x207);
1206
1207                 WREG32(mmSRAM_Y0_X0_RTR_HBW_DATA_W_ARB + offset, 0x207);
1208                 WREG32(mmSRAM_Y0_X1_RTR_HBW_DATA_W_ARB + offset, 0x207);
1209                 WREG32(mmSRAM_Y0_X2_RTR_HBW_DATA_W_ARB + offset, 0x206);
1210                 WREG32(mmSRAM_Y0_X3_RTR_HBW_DATA_W_ARB + offset, 0x206);
1211                 WREG32(mmSRAM_Y0_X4_RTR_HBW_DATA_W_ARB + offset, 0x206);
1212
1213                 WREG32(mmSRAM_Y0_X0_RTR_HBW_WR_RS_E_ARB + offset, 0x101);
1214                 WREG32(mmSRAM_Y0_X1_RTR_HBW_WR_RS_E_ARB + offset, 0x102);
1215                 WREG32(mmSRAM_Y0_X2_RTR_HBW_WR_RS_E_ARB + offset, 0x103);
1216                 WREG32(mmSRAM_Y0_X3_RTR_HBW_WR_RS_E_ARB + offset, 0x104);
1217                 WREG32(mmSRAM_Y0_X4_RTR_HBW_WR_RS_E_ARB + offset, 0x105);
1218
1219                 WREG32(mmSRAM_Y0_X0_RTR_HBW_WR_RS_W_ARB + offset, 0x105);
1220                 WREG32(mmSRAM_Y0_X1_RTR_HBW_WR_RS_W_ARB + offset, 0x104);
1221                 WREG32(mmSRAM_Y0_X2_RTR_HBW_WR_RS_W_ARB + offset, 0x103);
1222                 WREG32(mmSRAM_Y0_X3_RTR_HBW_WR_RS_W_ARB + offset, 0x102);
1223                 WREG32(mmSRAM_Y0_X4_RTR_HBW_WR_RS_W_ARB + offset, 0x101);
1224         }
1225
1226         WREG32(mmMME_STORE_MAX_CREDIT, 0x21);
1227         WREG32(mmMME_AGU, 0x0f0f0f10);
1228         WREG32(mmMME_SEI_MASK, ~0x0);
1229
1230         WREG32(mmMME6_RTR_HBW_RD_RQ_N_ARB, 0x01010101);
1231         WREG32(mmMME5_RTR_HBW_RD_RQ_N_ARB, 0x01040101);
1232         WREG32(mmMME4_RTR_HBW_RD_RQ_N_ARB, 0x01030101);
1233         WREG32(mmMME3_RTR_HBW_RD_RQ_N_ARB, 0x01020101);
1234         WREG32(mmMME2_RTR_HBW_RD_RQ_N_ARB, 0x01010101);
1235         WREG32(mmMME1_RTR_HBW_RD_RQ_N_ARB, 0x07010701);
1236         WREG32(mmMME6_RTR_HBW_RD_RQ_S_ARB, 0x04010401);
1237         WREG32(mmMME5_RTR_HBW_RD_RQ_S_ARB, 0x04050401);
1238         WREG32(mmMME4_RTR_HBW_RD_RQ_S_ARB, 0x03070301);
1239         WREG32(mmMME3_RTR_HBW_RD_RQ_S_ARB, 0x01030101);
1240         WREG32(mmMME2_RTR_HBW_RD_RQ_S_ARB, 0x01040101);
1241         WREG32(mmMME1_RTR_HBW_RD_RQ_S_ARB, 0x01050105);
1242         WREG32(mmMME6_RTR_HBW_RD_RQ_W_ARB, 0x01010501);
1243         WREG32(mmMME5_RTR_HBW_RD_RQ_W_ARB, 0x01010501);
1244         WREG32(mmMME4_RTR_HBW_RD_RQ_W_ARB, 0x01040301);
1245         WREG32(mmMME3_RTR_HBW_RD_RQ_W_ARB, 0x01030401);
1246         WREG32(mmMME2_RTR_HBW_RD_RQ_W_ARB, 0x01040101);
1247         WREG32(mmMME1_RTR_HBW_RD_RQ_W_ARB, 0x01050101);
1248         WREG32(mmMME6_RTR_HBW_WR_RQ_N_ARB, 0x02020202);
1249         WREG32(mmMME5_RTR_HBW_WR_RQ_N_ARB, 0x01070101);
1250         WREG32(mmMME4_RTR_HBW_WR_RQ_N_ARB, 0x02020201);
1251         WREG32(mmMME3_RTR_HBW_WR_RQ_N_ARB, 0x07020701);
1252         WREG32(mmMME2_RTR_HBW_WR_RQ_N_ARB, 0x01020101);
1253         WREG32(mmMME1_RTR_HBW_WR_RQ_S_ARB, 0x01010101);
1254         WREG32(mmMME6_RTR_HBW_WR_RQ_S_ARB, 0x01070101);
1255         WREG32(mmMME5_RTR_HBW_WR_RQ_S_ARB, 0x01070101);
1256         WREG32(mmMME4_RTR_HBW_WR_RQ_S_ARB, 0x07020701);
1257         WREG32(mmMME3_RTR_HBW_WR_RQ_S_ARB, 0x02020201);
1258         WREG32(mmMME2_RTR_HBW_WR_RQ_S_ARB, 0x01070101);
1259         WREG32(mmMME1_RTR_HBW_WR_RQ_S_ARB, 0x01020102);
1260         WREG32(mmMME6_RTR_HBW_WR_RQ_W_ARB, 0x01020701);
1261         WREG32(mmMME5_RTR_HBW_WR_RQ_W_ARB, 0x01020701);
1262         WREG32(mmMME4_RTR_HBW_WR_RQ_W_ARB, 0x07020707);
1263         WREG32(mmMME3_RTR_HBW_WR_RQ_W_ARB, 0x01020201);
1264         WREG32(mmMME2_RTR_HBW_WR_RQ_W_ARB, 0x01070201);
1265         WREG32(mmMME1_RTR_HBW_WR_RQ_W_ARB, 0x01070201);
1266         WREG32(mmMME6_RTR_HBW_RD_RS_N_ARB, 0x01070102);
1267         WREG32(mmMME5_RTR_HBW_RD_RS_N_ARB, 0x01070102);
1268         WREG32(mmMME4_RTR_HBW_RD_RS_N_ARB, 0x01060102);
1269         WREG32(mmMME3_RTR_HBW_RD_RS_N_ARB, 0x01040102);
1270         WREG32(mmMME2_RTR_HBW_RD_RS_N_ARB, 0x01020102);
1271         WREG32(mmMME1_RTR_HBW_RD_RS_N_ARB, 0x01020107);
1272         WREG32(mmMME6_RTR_HBW_RD_RS_S_ARB, 0x01020106);
1273         WREG32(mmMME5_RTR_HBW_RD_RS_S_ARB, 0x01020102);
1274         WREG32(mmMME4_RTR_HBW_RD_RS_S_ARB, 0x01040102);
1275         WREG32(mmMME3_RTR_HBW_RD_RS_S_ARB, 0x01060102);
1276         WREG32(mmMME2_RTR_HBW_RD_RS_S_ARB, 0x01070102);
1277         WREG32(mmMME1_RTR_HBW_RD_RS_S_ARB, 0x01070102);
1278         WREG32(mmMME6_RTR_HBW_RD_RS_E_ARB, 0x01020702);
1279         WREG32(mmMME5_RTR_HBW_RD_RS_E_ARB, 0x01020702);
1280         WREG32(mmMME4_RTR_HBW_RD_RS_E_ARB, 0x01040602);
1281         WREG32(mmMME3_RTR_HBW_RD_RS_E_ARB, 0x01060402);
1282         WREG32(mmMME2_RTR_HBW_RD_RS_E_ARB, 0x01070202);
1283         WREG32(mmMME1_RTR_HBW_RD_RS_E_ARB, 0x01070102);
1284         WREG32(mmMME6_RTR_HBW_RD_RS_W_ARB, 0x01060401);
1285         WREG32(mmMME5_RTR_HBW_RD_RS_W_ARB, 0x01060401);
1286         WREG32(mmMME4_RTR_HBW_RD_RS_W_ARB, 0x01060401);
1287         WREG32(mmMME3_RTR_HBW_RD_RS_W_ARB, 0x01060401);
1288         WREG32(mmMME2_RTR_HBW_RD_RS_W_ARB, 0x01060401);
1289         WREG32(mmMME1_RTR_HBW_RD_RS_W_ARB, 0x01060401);
1290         WREG32(mmMME6_RTR_HBW_WR_RS_N_ARB, 0x01050101);
1291         WREG32(mmMME5_RTR_HBW_WR_RS_N_ARB, 0x01040101);
1292         WREG32(mmMME4_RTR_HBW_WR_RS_N_ARB, 0x01030101);
1293         WREG32(mmMME3_RTR_HBW_WR_RS_N_ARB, 0x01020101);
1294         WREG32(mmMME2_RTR_HBW_WR_RS_N_ARB, 0x01010101);
1295         WREG32(mmMME1_RTR_HBW_WR_RS_N_ARB, 0x01010107);
1296         WREG32(mmMME6_RTR_HBW_WR_RS_S_ARB, 0x01010107);
1297         WREG32(mmMME5_RTR_HBW_WR_RS_S_ARB, 0x01010101);
1298         WREG32(mmMME4_RTR_HBW_WR_RS_S_ARB, 0x01020101);
1299         WREG32(mmMME3_RTR_HBW_WR_RS_S_ARB, 0x01030101);
1300         WREG32(mmMME2_RTR_HBW_WR_RS_S_ARB, 0x01040101);
1301         WREG32(mmMME1_RTR_HBW_WR_RS_S_ARB, 0x01050101);
1302         WREG32(mmMME6_RTR_HBW_WR_RS_E_ARB, 0x01010501);
1303         WREG32(mmMME5_RTR_HBW_WR_RS_E_ARB, 0x01010501);
1304         WREG32(mmMME4_RTR_HBW_WR_RS_E_ARB, 0x01040301);
1305         WREG32(mmMME3_RTR_HBW_WR_RS_E_ARB, 0x01030401);
1306         WREG32(mmMME2_RTR_HBW_WR_RS_E_ARB, 0x01040101);
1307         WREG32(mmMME1_RTR_HBW_WR_RS_E_ARB, 0x01050101);
1308         WREG32(mmMME6_RTR_HBW_WR_RS_W_ARB, 0x01010101);
1309         WREG32(mmMME5_RTR_HBW_WR_RS_W_ARB, 0x01010101);
1310         WREG32(mmMME4_RTR_HBW_WR_RS_W_ARB, 0x01010101);
1311         WREG32(mmMME3_RTR_HBW_WR_RS_W_ARB, 0x01010101);
1312         WREG32(mmMME2_RTR_HBW_WR_RS_W_ARB, 0x01010101);
1313         WREG32(mmMME1_RTR_HBW_WR_RS_W_ARB, 0x01010101);
1314
1315         WREG32(mmTPC1_RTR_HBW_RD_RQ_N_ARB, 0x01010101);
1316         WREG32(mmTPC1_RTR_HBW_RD_RQ_S_ARB, 0x01010101);
1317         WREG32(mmTPC1_RTR_HBW_RD_RQ_E_ARB, 0x01060101);
1318         WREG32(mmTPC1_RTR_HBW_WR_RQ_N_ARB, 0x02020102);
1319         WREG32(mmTPC1_RTR_HBW_WR_RQ_S_ARB, 0x01010101);
1320         WREG32(mmTPC1_RTR_HBW_WR_RQ_E_ARB, 0x02070202);
1321         WREG32(mmTPC1_RTR_HBW_RD_RS_N_ARB, 0x01020201);
1322         WREG32(mmTPC1_RTR_HBW_RD_RS_S_ARB, 0x01070201);
1323         WREG32(mmTPC1_RTR_HBW_RD_RS_W_ARB, 0x01070202);
1324         WREG32(mmTPC1_RTR_HBW_WR_RS_N_ARB, 0x01010101);
1325         WREG32(mmTPC1_RTR_HBW_WR_RS_S_ARB, 0x01050101);
1326         WREG32(mmTPC1_RTR_HBW_WR_RS_W_ARB, 0x01050101);
1327
1328         WREG32(mmTPC2_RTR_HBW_RD_RQ_N_ARB, 0x01020101);
1329         WREG32(mmTPC2_RTR_HBW_RD_RQ_S_ARB, 0x01050101);
1330         WREG32(mmTPC2_RTR_HBW_RD_RQ_E_ARB, 0x01010201);
1331         WREG32(mmTPC2_RTR_HBW_WR_RQ_N_ARB, 0x02040102);
1332         WREG32(mmTPC2_RTR_HBW_WR_RQ_S_ARB, 0x01050101);
1333         WREG32(mmTPC2_RTR_HBW_WR_RQ_E_ARB, 0x02060202);
1334         WREG32(mmTPC2_RTR_HBW_RD_RS_N_ARB, 0x01020201);
1335         WREG32(mmTPC2_RTR_HBW_RD_RS_S_ARB, 0x01070201);
1336         WREG32(mmTPC2_RTR_HBW_RD_RS_W_ARB, 0x01070202);
1337         WREG32(mmTPC2_RTR_HBW_WR_RS_N_ARB, 0x01010101);
1338         WREG32(mmTPC2_RTR_HBW_WR_RS_S_ARB, 0x01040101);
1339         WREG32(mmTPC2_RTR_HBW_WR_RS_W_ARB, 0x01040101);
1340
1341         WREG32(mmTPC3_RTR_HBW_RD_RQ_N_ARB, 0x01030101);
1342         WREG32(mmTPC3_RTR_HBW_RD_RQ_S_ARB, 0x01040101);
1343         WREG32(mmTPC3_RTR_HBW_RD_RQ_E_ARB, 0x01040301);
1344         WREG32(mmTPC3_RTR_HBW_WR_RQ_N_ARB, 0x02060102);
1345         WREG32(mmTPC3_RTR_HBW_WR_RQ_S_ARB, 0x01040101);
1346         WREG32(mmTPC3_RTR_HBW_WR_RQ_E_ARB, 0x01040301);
1347         WREG32(mmTPC3_RTR_HBW_RD_RS_N_ARB, 0x01040201);
1348         WREG32(mmTPC3_RTR_HBW_RD_RS_S_ARB, 0x01060201);
1349         WREG32(mmTPC3_RTR_HBW_RD_RS_W_ARB, 0x01060402);
1350         WREG32(mmTPC3_RTR_HBW_WR_RS_N_ARB, 0x01020101);
1351         WREG32(mmTPC3_RTR_HBW_WR_RS_S_ARB, 0x01030101);
1352         WREG32(mmTPC3_RTR_HBW_WR_RS_W_ARB, 0x01030401);
1353
1354         WREG32(mmTPC4_RTR_HBW_RD_RQ_N_ARB, 0x01040101);
1355         WREG32(mmTPC4_RTR_HBW_RD_RQ_S_ARB, 0x01030101);
1356         WREG32(mmTPC4_RTR_HBW_RD_RQ_E_ARB, 0x01030401);
1357         WREG32(mmTPC4_RTR_HBW_WR_RQ_N_ARB, 0x02070102);
1358         WREG32(mmTPC4_RTR_HBW_WR_RQ_S_ARB, 0x01030101);
1359         WREG32(mmTPC4_RTR_HBW_WR_RQ_E_ARB, 0x02060702);
1360         WREG32(mmTPC4_RTR_HBW_RD_RS_N_ARB, 0x01060201);
1361         WREG32(mmTPC4_RTR_HBW_RD_RS_S_ARB, 0x01040201);
1362         WREG32(mmTPC4_RTR_HBW_RD_RS_W_ARB, 0x01040602);
1363         WREG32(mmTPC4_RTR_HBW_WR_RS_N_ARB, 0x01030101);
1364         WREG32(mmTPC4_RTR_HBW_WR_RS_S_ARB, 0x01020101);
1365         WREG32(mmTPC4_RTR_HBW_WR_RS_W_ARB, 0x01040301);
1366
1367         WREG32(mmTPC5_RTR_HBW_RD_RQ_N_ARB, 0x01050101);
1368         WREG32(mmTPC5_RTR_HBW_RD_RQ_S_ARB, 0x01020101);
1369         WREG32(mmTPC5_RTR_HBW_RD_RQ_E_ARB, 0x01200501);
1370         WREG32(mmTPC5_RTR_HBW_WR_RQ_N_ARB, 0x02070102);
1371         WREG32(mmTPC5_RTR_HBW_WR_RQ_S_ARB, 0x01020101);
1372         WREG32(mmTPC5_RTR_HBW_WR_RQ_E_ARB, 0x02020602);
1373         WREG32(mmTPC5_RTR_HBW_RD_RS_N_ARB, 0x01070201);
1374         WREG32(mmTPC5_RTR_HBW_RD_RS_S_ARB, 0x01020201);
1375         WREG32(mmTPC5_RTR_HBW_RD_RS_W_ARB, 0x01020702);
1376         WREG32(mmTPC5_RTR_HBW_WR_RS_N_ARB, 0x01040101);
1377         WREG32(mmTPC5_RTR_HBW_WR_RS_S_ARB, 0x01010101);
1378         WREG32(mmTPC5_RTR_HBW_WR_RS_W_ARB, 0x01010501);
1379
1380         WREG32(mmTPC6_RTR_HBW_RD_RQ_N_ARB, 0x01010101);
1381         WREG32(mmTPC6_RTR_HBW_RD_RQ_S_ARB, 0x01010101);
1382         WREG32(mmTPC6_RTR_HBW_RD_RQ_E_ARB, 0x01010601);
1383         WREG32(mmTPC6_RTR_HBW_WR_RQ_N_ARB, 0x01010101);
1384         WREG32(mmTPC6_RTR_HBW_WR_RQ_S_ARB, 0x01010101);
1385         WREG32(mmTPC6_RTR_HBW_WR_RQ_E_ARB, 0x02020702);
1386         WREG32(mmTPC6_RTR_HBW_RD_RS_N_ARB, 0x01010101);
1387         WREG32(mmTPC6_RTR_HBW_RD_RS_S_ARB, 0x01010101);
1388         WREG32(mmTPC6_RTR_HBW_RD_RS_W_ARB, 0x01020702);
1389         WREG32(mmTPC6_RTR_HBW_WR_RS_N_ARB, 0x01050101);
1390         WREG32(mmTPC6_RTR_HBW_WR_RS_S_ARB, 0x01010101);
1391         WREG32(mmTPC6_RTR_HBW_WR_RS_W_ARB, 0x01010501);
1392
1393         for (i = 0, offset = 0 ; i < 10 ; i++, offset += 4) {
1394                 WREG32(mmMME1_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
1395                 WREG32(mmMME2_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
1396                 WREG32(mmMME3_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
1397                 WREG32(mmMME4_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
1398                 WREG32(mmMME5_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
1399                 WREG32(mmMME6_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
1400
1401                 WREG32(mmTPC0_NRTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
1402                 WREG32(mmTPC1_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
1403                 WREG32(mmTPC2_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
1404                 WREG32(mmTPC3_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
1405                 WREG32(mmTPC4_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
1406                 WREG32(mmTPC5_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
1407                 WREG32(mmTPC6_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
1408                 WREG32(mmTPC7_NRTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
1409
1410                 WREG32(mmPCI_NRTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
1411                 WREG32(mmDMA_NRTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
1412         }
1413
1414         for (i = 0, offset = 0 ; i < 6 ; i++, offset += 0x40000) {
1415                 WREG32(mmMME1_RTR_SCRAMB_EN + offset,
1416                                 1 << MME1_RTR_SCRAMB_EN_VAL_SHIFT);
1417                 WREG32(mmMME1_RTR_NON_LIN_SCRAMB + offset,
1418                                 1 << MME1_RTR_NON_LIN_SCRAMB_EN_SHIFT);
1419         }
1420
1421         for (i = 0, offset = 0 ; i < 8 ; i++, offset += 0x40000) {
1422                 /*
1423                  * Workaround for Bug H2 #2441 :
1424                  * "ST.NOP set trace event illegal opcode"
1425                  */
1426                 WREG32(mmTPC0_CFG_TPC_INTR_MASK + offset, tpc_intr_mask);
1427
1428                 WREG32(mmTPC0_NRTR_SCRAMB_EN + offset,
1429                                 1 << TPC0_NRTR_SCRAMB_EN_VAL_SHIFT);
1430                 WREG32(mmTPC0_NRTR_NON_LIN_SCRAMB + offset,
1431                                 1 << TPC0_NRTR_NON_LIN_SCRAMB_EN_SHIFT);
1432         }
1433
1434         WREG32(mmDMA_NRTR_SCRAMB_EN, 1 << DMA_NRTR_SCRAMB_EN_VAL_SHIFT);
1435         WREG32(mmDMA_NRTR_NON_LIN_SCRAMB,
1436                         1 << DMA_NRTR_NON_LIN_SCRAMB_EN_SHIFT);
1437
1438         WREG32(mmPCI_NRTR_SCRAMB_EN, 1 << PCI_NRTR_SCRAMB_EN_VAL_SHIFT);
1439         WREG32(mmPCI_NRTR_NON_LIN_SCRAMB,
1440                         1 << PCI_NRTR_NON_LIN_SCRAMB_EN_SHIFT);
1441
1442         /*
1443          * Workaround for H2 #HW-23 bug
1444          * Set DMA max outstanding read requests to 240 on DMA CH 1.
1445          * This limitation is still large enough to not affect Gen4 bandwidth.
1446          * We need to only limit that DMA channel because the user can only read
1447          * from Host using DMA CH 1
1448          */
1449         WREG32(mmDMA_CH_1_CFG0, 0x0fff00F0);
1450
1451         WREG32(mmTPC_PLL_CLK_RLX_0, 0x200020);
1452
1453         goya->hw_cap_initialized |= HW_CAP_GOLDEN;
1454 }
1455
1456 static void goya_init_mme_qman(struct hl_device *hdev)
1457 {
1458         u32 mtr_base_lo, mtr_base_hi;
1459         u32 so_base_lo, so_base_hi;
1460         u32 gic_base_lo, gic_base_hi;
1461         u64 qman_base_addr;
1462
1463         mtr_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
1464         mtr_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
1465         so_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
1466         so_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
1467
1468         gic_base_lo =
1469                 lower_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
1470         gic_base_hi =
1471                 upper_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
1472
1473         qman_base_addr = hdev->asic_prop.sram_base_address +
1474                                 MME_QMAN_BASE_OFFSET;
1475
1476         WREG32(mmMME_QM_PQ_BASE_LO, lower_32_bits(qman_base_addr));
1477         WREG32(mmMME_QM_PQ_BASE_HI, upper_32_bits(qman_base_addr));
1478         WREG32(mmMME_QM_PQ_SIZE, ilog2(MME_QMAN_LENGTH));
1479         WREG32(mmMME_QM_PQ_PI, 0);
1480         WREG32(mmMME_QM_PQ_CI, 0);
1481         WREG32(mmMME_QM_CP_LDMA_SRC_BASE_LO_OFFSET, 0x10C0);
1482         WREG32(mmMME_QM_CP_LDMA_SRC_BASE_HI_OFFSET, 0x10C4);
1483         WREG32(mmMME_QM_CP_LDMA_TSIZE_OFFSET, 0x10C8);
1484         WREG32(mmMME_QM_CP_LDMA_COMMIT_OFFSET, 0x10CC);
1485
1486         WREG32(mmMME_QM_CP_MSG_BASE0_ADDR_LO, mtr_base_lo);
1487         WREG32(mmMME_QM_CP_MSG_BASE0_ADDR_HI, mtr_base_hi);
1488         WREG32(mmMME_QM_CP_MSG_BASE1_ADDR_LO, so_base_lo);
1489         WREG32(mmMME_QM_CP_MSG_BASE1_ADDR_HI, so_base_hi);
1490
1491         /* QMAN CQ has 8 cache lines */
1492         WREG32(mmMME_QM_CQ_CFG1, 0x00080008);
1493
1494         WREG32(mmMME_QM_GLBL_ERR_ADDR_LO, gic_base_lo);
1495         WREG32(mmMME_QM_GLBL_ERR_ADDR_HI, gic_base_hi);
1496
1497         WREG32(mmMME_QM_GLBL_ERR_WDATA, GOYA_ASYNC_EVENT_ID_MME_QM);
1498
1499         WREG32(mmMME_QM_GLBL_ERR_CFG, QMAN_MME_ERR_MSG_EN);
1500
1501         WREG32(mmMME_QM_GLBL_PROT, QMAN_MME_ERR_PROT);
1502
1503         WREG32(mmMME_QM_GLBL_CFG0, QMAN_MME_ENABLE);
1504 }
1505
1506 static void goya_init_mme_cmdq(struct hl_device *hdev)
1507 {
1508         u32 mtr_base_lo, mtr_base_hi;
1509         u32 so_base_lo, so_base_hi;
1510         u32 gic_base_lo, gic_base_hi;
1511         u64 qman_base_addr;
1512
1513         mtr_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
1514         mtr_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
1515         so_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
1516         so_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
1517
1518         gic_base_lo =
1519                 lower_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
1520         gic_base_hi =
1521                 upper_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
1522
1523         qman_base_addr = hdev->asic_prop.sram_base_address +
1524                                 MME_QMAN_BASE_OFFSET;
1525
1526         WREG32(mmMME_CMDQ_CP_MSG_BASE0_ADDR_LO, mtr_base_lo);
1527         WREG32(mmMME_CMDQ_CP_MSG_BASE0_ADDR_HI, mtr_base_hi);
1528         WREG32(mmMME_CMDQ_CP_MSG_BASE1_ADDR_LO, so_base_lo);
1529         WREG32(mmMME_CMDQ_CP_MSG_BASE1_ADDR_HI, so_base_hi);
1530
1531         /* CMDQ CQ has 20 cache lines */
1532         WREG32(mmMME_CMDQ_CQ_CFG1, 0x00140014);
1533
1534         WREG32(mmMME_CMDQ_GLBL_ERR_ADDR_LO, gic_base_lo);
1535         WREG32(mmMME_CMDQ_GLBL_ERR_ADDR_HI, gic_base_hi);
1536
1537         WREG32(mmMME_CMDQ_GLBL_ERR_WDATA, GOYA_ASYNC_EVENT_ID_MME_CMDQ);
1538
1539         WREG32(mmMME_CMDQ_GLBL_ERR_CFG, CMDQ_MME_ERR_MSG_EN);
1540
1541         WREG32(mmMME_CMDQ_GLBL_PROT, CMDQ_MME_ERR_PROT);
1542
1543         WREG32(mmMME_CMDQ_GLBL_CFG0, CMDQ_MME_ENABLE);
1544 }
1545
1546 void goya_init_mme_qmans(struct hl_device *hdev)
1547 {
1548         struct goya_device *goya = hdev->asic_specific;
1549         u32 so_base_lo, so_base_hi;
1550
1551         if (goya->hw_cap_initialized & HW_CAP_MME)
1552                 return;
1553
1554         so_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
1555         so_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
1556
1557         WREG32(mmMME_SM_BASE_ADDRESS_LOW, so_base_lo);
1558         WREG32(mmMME_SM_BASE_ADDRESS_HIGH, so_base_hi);
1559
1560         goya_init_mme_qman(hdev);
1561         goya_init_mme_cmdq(hdev);
1562
1563         goya->hw_cap_initialized |= HW_CAP_MME;
1564 }
1565
1566 static void goya_init_tpc_qman(struct hl_device *hdev, u32 base_off, int tpc_id)
1567 {
1568         u32 mtr_base_lo, mtr_base_hi;
1569         u32 so_base_lo, so_base_hi;
1570         u32 gic_base_lo, gic_base_hi;
1571         u64 qman_base_addr;
1572         u32 reg_off = tpc_id * (mmTPC1_QM_PQ_PI - mmTPC0_QM_PQ_PI);
1573
1574         mtr_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
1575         mtr_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
1576         so_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
1577         so_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
1578
1579         gic_base_lo =
1580                 lower_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
1581         gic_base_hi =
1582                 upper_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
1583
1584         qman_base_addr = hdev->asic_prop.sram_base_address + base_off;
1585
1586         WREG32(mmTPC0_QM_PQ_BASE_LO + reg_off, lower_32_bits(qman_base_addr));
1587         WREG32(mmTPC0_QM_PQ_BASE_HI + reg_off, upper_32_bits(qman_base_addr));
1588         WREG32(mmTPC0_QM_PQ_SIZE + reg_off, ilog2(TPC_QMAN_LENGTH));
1589         WREG32(mmTPC0_QM_PQ_PI + reg_off, 0);
1590         WREG32(mmTPC0_QM_PQ_CI + reg_off, 0);
1591         WREG32(mmTPC0_QM_CP_LDMA_SRC_BASE_LO_OFFSET + reg_off, 0x10C0);
1592         WREG32(mmTPC0_QM_CP_LDMA_SRC_BASE_HI_OFFSET + reg_off, 0x10C4);
1593         WREG32(mmTPC0_QM_CP_LDMA_TSIZE_OFFSET + reg_off, 0x10C8);
1594         WREG32(mmTPC0_QM_CP_LDMA_COMMIT_OFFSET + reg_off, 0x10CC);
1595
1596         WREG32(mmTPC0_QM_CP_MSG_BASE0_ADDR_LO + reg_off, mtr_base_lo);
1597         WREG32(mmTPC0_QM_CP_MSG_BASE0_ADDR_HI + reg_off, mtr_base_hi);
1598         WREG32(mmTPC0_QM_CP_MSG_BASE1_ADDR_LO + reg_off, so_base_lo);
1599         WREG32(mmTPC0_QM_CP_MSG_BASE1_ADDR_HI + reg_off, so_base_hi);
1600
1601         WREG32(mmTPC0_QM_CQ_CFG1 + reg_off, 0x00080008);
1602
1603         WREG32(mmTPC0_QM_GLBL_ERR_ADDR_LO + reg_off, gic_base_lo);
1604         WREG32(mmTPC0_QM_GLBL_ERR_ADDR_HI + reg_off, gic_base_hi);
1605
1606         WREG32(mmTPC0_QM_GLBL_ERR_WDATA + reg_off,
1607                         GOYA_ASYNC_EVENT_ID_TPC0_QM + tpc_id);
1608
1609         WREG32(mmTPC0_QM_GLBL_ERR_CFG + reg_off, QMAN_TPC_ERR_MSG_EN);
1610
1611         WREG32(mmTPC0_QM_GLBL_PROT + reg_off, QMAN_TPC_ERR_PROT);
1612
1613         WREG32(mmTPC0_QM_GLBL_CFG0 + reg_off, QMAN_TPC_ENABLE);
1614 }
1615
1616 static void goya_init_tpc_cmdq(struct hl_device *hdev, int tpc_id)
1617 {
1618         u32 mtr_base_lo, mtr_base_hi;
1619         u32 so_base_lo, so_base_hi;
1620         u32 gic_base_lo, gic_base_hi;
1621         u32 reg_off = tpc_id * (mmTPC1_CMDQ_CQ_CFG1 - mmTPC0_CMDQ_CQ_CFG1);
1622
1623         mtr_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
1624         mtr_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
1625         so_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
1626         so_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
1627
1628         gic_base_lo =
1629                 lower_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
1630         gic_base_hi =
1631                 upper_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
1632
1633         WREG32(mmTPC0_CMDQ_CP_MSG_BASE0_ADDR_LO + reg_off, mtr_base_lo);
1634         WREG32(mmTPC0_CMDQ_CP_MSG_BASE0_ADDR_HI + reg_off, mtr_base_hi);
1635         WREG32(mmTPC0_CMDQ_CP_MSG_BASE1_ADDR_LO + reg_off, so_base_lo);
1636         WREG32(mmTPC0_CMDQ_CP_MSG_BASE1_ADDR_HI + reg_off, so_base_hi);
1637
1638         WREG32(mmTPC0_CMDQ_CQ_CFG1 + reg_off, 0x00140014);
1639
1640         WREG32(mmTPC0_CMDQ_GLBL_ERR_ADDR_LO + reg_off, gic_base_lo);
1641         WREG32(mmTPC0_CMDQ_GLBL_ERR_ADDR_HI + reg_off, gic_base_hi);
1642
1643         WREG32(mmTPC0_CMDQ_GLBL_ERR_WDATA + reg_off,
1644                         GOYA_ASYNC_EVENT_ID_TPC0_CMDQ + tpc_id);
1645
1646         WREG32(mmTPC0_CMDQ_GLBL_ERR_CFG + reg_off, CMDQ_TPC_ERR_MSG_EN);
1647
1648         WREG32(mmTPC0_CMDQ_GLBL_PROT + reg_off, CMDQ_TPC_ERR_PROT);
1649
1650         WREG32(mmTPC0_CMDQ_GLBL_CFG0 + reg_off, CMDQ_TPC_ENABLE);
1651 }
1652
1653 void goya_init_tpc_qmans(struct hl_device *hdev)
1654 {
1655         struct goya_device *goya = hdev->asic_specific;
1656         u32 so_base_lo, so_base_hi;
1657         u32 cfg_off = mmTPC1_CFG_SM_BASE_ADDRESS_LOW -
1658                         mmTPC0_CFG_SM_BASE_ADDRESS_LOW;
1659         int i;
1660
1661         if (goya->hw_cap_initialized & HW_CAP_TPC)
1662                 return;
1663
1664         so_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
1665         so_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
1666
1667         for (i = 0 ; i < TPC_MAX_NUM ; i++) {
1668                 WREG32(mmTPC0_CFG_SM_BASE_ADDRESS_LOW + i * cfg_off,
1669                                 so_base_lo);
1670                 WREG32(mmTPC0_CFG_SM_BASE_ADDRESS_HIGH + i * cfg_off,
1671                                 so_base_hi);
1672         }
1673
1674         goya_init_tpc_qman(hdev, TPC0_QMAN_BASE_OFFSET, 0);
1675         goya_init_tpc_qman(hdev, TPC1_QMAN_BASE_OFFSET, 1);
1676         goya_init_tpc_qman(hdev, TPC2_QMAN_BASE_OFFSET, 2);
1677         goya_init_tpc_qman(hdev, TPC3_QMAN_BASE_OFFSET, 3);
1678         goya_init_tpc_qman(hdev, TPC4_QMAN_BASE_OFFSET, 4);
1679         goya_init_tpc_qman(hdev, TPC5_QMAN_BASE_OFFSET, 5);
1680         goya_init_tpc_qman(hdev, TPC6_QMAN_BASE_OFFSET, 6);
1681         goya_init_tpc_qman(hdev, TPC7_QMAN_BASE_OFFSET, 7);
1682
1683         for (i = 0 ; i < TPC_MAX_NUM ; i++)
1684                 goya_init_tpc_cmdq(hdev, i);
1685
1686         goya->hw_cap_initialized |= HW_CAP_TPC;
1687 }
1688
1689 /*
1690  * goya_disable_internal_queues - Disable internal queues
1691  *
1692  * @hdev: pointer to hl_device structure
1693  *
1694  */
1695 static void goya_disable_internal_queues(struct hl_device *hdev)
1696 {
1697         WREG32(mmMME_QM_GLBL_CFG0, 0);
1698         WREG32(mmMME_CMDQ_GLBL_CFG0, 0);
1699
1700         WREG32(mmTPC0_QM_GLBL_CFG0, 0);
1701         WREG32(mmTPC0_CMDQ_GLBL_CFG0, 0);
1702
1703         WREG32(mmTPC1_QM_GLBL_CFG0, 0);
1704         WREG32(mmTPC1_CMDQ_GLBL_CFG0, 0);
1705
1706         WREG32(mmTPC2_QM_GLBL_CFG0, 0);
1707         WREG32(mmTPC2_CMDQ_GLBL_CFG0, 0);
1708
1709         WREG32(mmTPC3_QM_GLBL_CFG0, 0);
1710         WREG32(mmTPC3_CMDQ_GLBL_CFG0, 0);
1711
1712         WREG32(mmTPC4_QM_GLBL_CFG0, 0);
1713         WREG32(mmTPC4_CMDQ_GLBL_CFG0, 0);
1714
1715         WREG32(mmTPC5_QM_GLBL_CFG0, 0);
1716         WREG32(mmTPC5_CMDQ_GLBL_CFG0, 0);
1717
1718         WREG32(mmTPC6_QM_GLBL_CFG0, 0);
1719         WREG32(mmTPC6_CMDQ_GLBL_CFG0, 0);
1720
1721         WREG32(mmTPC7_QM_GLBL_CFG0, 0);
1722         WREG32(mmTPC7_CMDQ_GLBL_CFG0, 0);
1723 }
1724
1725 /*
1726  * goya_stop_internal_queues - Stop internal queues
1727  *
1728  * @hdev: pointer to hl_device structure
1729  *
1730  * Returns 0 on success
1731  *
1732  */
1733 static int goya_stop_internal_queues(struct hl_device *hdev)
1734 {
1735         int rc, retval = 0;
1736
1737         /*
1738          * Each queue (QMAN) is a separate H/W logic. That means that each
1739          * QMAN can be stopped independently and failure to stop one does NOT
1740          * mandate we should not try to stop other QMANs
1741          */
1742
1743         rc = goya_stop_queue(hdev,
1744                         mmMME_QM_GLBL_CFG1,
1745                         mmMME_QM_CP_STS,
1746                         mmMME_QM_GLBL_STS0);
1747
1748         if (rc) {
1749                 dev_err(hdev->dev, "failed to stop MME QMAN\n");
1750                 retval = -EIO;
1751         }
1752
1753         rc = goya_stop_queue(hdev,
1754                         mmMME_CMDQ_GLBL_CFG1,
1755                         mmMME_CMDQ_CP_STS,
1756                         mmMME_CMDQ_GLBL_STS0);
1757
1758         if (rc) {
1759                 dev_err(hdev->dev, "failed to stop MME CMDQ\n");
1760                 retval = -EIO;
1761         }
1762
1763         rc = goya_stop_queue(hdev,
1764                         mmTPC0_QM_GLBL_CFG1,
1765                         mmTPC0_QM_CP_STS,
1766                         mmTPC0_QM_GLBL_STS0);
1767
1768         if (rc) {
1769                 dev_err(hdev->dev, "failed to stop TPC 0 QMAN\n");
1770                 retval = -EIO;
1771         }
1772
1773         rc = goya_stop_queue(hdev,
1774                         mmTPC0_CMDQ_GLBL_CFG1,
1775                         mmTPC0_CMDQ_CP_STS,
1776                         mmTPC0_CMDQ_GLBL_STS0);
1777
1778         if (rc) {
1779                 dev_err(hdev->dev, "failed to stop TPC 0 CMDQ\n");
1780                 retval = -EIO;
1781         }
1782
1783         rc = goya_stop_queue(hdev,
1784                         mmTPC1_QM_GLBL_CFG1,
1785                         mmTPC1_QM_CP_STS,
1786                         mmTPC1_QM_GLBL_STS0);
1787
1788         if (rc) {
1789                 dev_err(hdev->dev, "failed to stop TPC 1 QMAN\n");
1790                 retval = -EIO;
1791         }
1792
1793         rc = goya_stop_queue(hdev,
1794                         mmTPC1_CMDQ_GLBL_CFG1,
1795                         mmTPC1_CMDQ_CP_STS,
1796                         mmTPC1_CMDQ_GLBL_STS0);
1797
1798         if (rc) {
1799                 dev_err(hdev->dev, "failed to stop TPC 1 CMDQ\n");
1800                 retval = -EIO;
1801         }
1802
1803         rc = goya_stop_queue(hdev,
1804                         mmTPC2_QM_GLBL_CFG1,
1805                         mmTPC2_QM_CP_STS,
1806                         mmTPC2_QM_GLBL_STS0);
1807
1808         if (rc) {
1809                 dev_err(hdev->dev, "failed to stop TPC 2 QMAN\n");
1810                 retval = -EIO;
1811         }
1812
1813         rc = goya_stop_queue(hdev,
1814                         mmTPC2_CMDQ_GLBL_CFG1,
1815                         mmTPC2_CMDQ_CP_STS,
1816                         mmTPC2_CMDQ_GLBL_STS0);
1817
1818         if (rc) {
1819                 dev_err(hdev->dev, "failed to stop TPC 2 CMDQ\n");
1820                 retval = -EIO;
1821         }
1822
1823         rc = goya_stop_queue(hdev,
1824                         mmTPC3_QM_GLBL_CFG1,
1825                         mmTPC3_QM_CP_STS,
1826                         mmTPC3_QM_GLBL_STS0);
1827
1828         if (rc) {
1829                 dev_err(hdev->dev, "failed to stop TPC 3 QMAN\n");
1830                 retval = -EIO;
1831         }
1832
1833         rc = goya_stop_queue(hdev,
1834                         mmTPC3_CMDQ_GLBL_CFG1,
1835                         mmTPC3_CMDQ_CP_STS,
1836                         mmTPC3_CMDQ_GLBL_STS0);
1837
1838         if (rc) {
1839                 dev_err(hdev->dev, "failed to stop TPC 3 CMDQ\n");
1840                 retval = -EIO;
1841         }
1842
1843         rc = goya_stop_queue(hdev,
1844                         mmTPC4_QM_GLBL_CFG1,
1845                         mmTPC4_QM_CP_STS,
1846                         mmTPC4_QM_GLBL_STS0);
1847
1848         if (rc) {
1849                 dev_err(hdev->dev, "failed to stop TPC 4 QMAN\n");
1850                 retval = -EIO;
1851         }
1852
1853         rc = goya_stop_queue(hdev,
1854                         mmTPC4_CMDQ_GLBL_CFG1,
1855                         mmTPC4_CMDQ_CP_STS,
1856                         mmTPC4_CMDQ_GLBL_STS0);
1857
1858         if (rc) {
1859                 dev_err(hdev->dev, "failed to stop TPC 4 CMDQ\n");
1860                 retval = -EIO;
1861         }
1862
1863         rc = goya_stop_queue(hdev,
1864                         mmTPC5_QM_GLBL_CFG1,
1865                         mmTPC5_QM_CP_STS,
1866                         mmTPC5_QM_GLBL_STS0);
1867
1868         if (rc) {
1869                 dev_err(hdev->dev, "failed to stop TPC 5 QMAN\n");
1870                 retval = -EIO;
1871         }
1872
1873         rc = goya_stop_queue(hdev,
1874                         mmTPC5_CMDQ_GLBL_CFG1,
1875                         mmTPC5_CMDQ_CP_STS,
1876                         mmTPC5_CMDQ_GLBL_STS0);
1877
1878         if (rc) {
1879                 dev_err(hdev->dev, "failed to stop TPC 5 CMDQ\n");
1880                 retval = -EIO;
1881         }
1882
1883         rc = goya_stop_queue(hdev,
1884                         mmTPC6_QM_GLBL_CFG1,
1885                         mmTPC6_QM_CP_STS,
1886                         mmTPC6_QM_GLBL_STS0);
1887
1888         if (rc) {
1889                 dev_err(hdev->dev, "failed to stop TPC 6 QMAN\n");
1890                 retval = -EIO;
1891         }
1892
1893         rc = goya_stop_queue(hdev,
1894                         mmTPC6_CMDQ_GLBL_CFG1,
1895                         mmTPC6_CMDQ_CP_STS,
1896                         mmTPC6_CMDQ_GLBL_STS0);
1897
1898         if (rc) {
1899                 dev_err(hdev->dev, "failed to stop TPC 6 CMDQ\n");
1900                 retval = -EIO;
1901         }
1902
1903         rc = goya_stop_queue(hdev,
1904                         mmTPC7_QM_GLBL_CFG1,
1905                         mmTPC7_QM_CP_STS,
1906                         mmTPC7_QM_GLBL_STS0);
1907
1908         if (rc) {
1909                 dev_err(hdev->dev, "failed to stop TPC 7 QMAN\n");
1910                 retval = -EIO;
1911         }
1912
1913         rc = goya_stop_queue(hdev,
1914                         mmTPC7_CMDQ_GLBL_CFG1,
1915                         mmTPC7_CMDQ_CP_STS,
1916                         mmTPC7_CMDQ_GLBL_STS0);
1917
1918         if (rc) {
1919                 dev_err(hdev->dev, "failed to stop TPC 7 CMDQ\n");
1920                 retval = -EIO;
1921         }
1922
1923         return retval;
1924 }
1925
1926 static void goya_dma_stall(struct hl_device *hdev)
1927 {
1928         WREG32(mmDMA_QM_0_GLBL_CFG1, 1 << DMA_QM_0_GLBL_CFG1_DMA_STOP_SHIFT);
1929         WREG32(mmDMA_QM_1_GLBL_CFG1, 1 << DMA_QM_1_GLBL_CFG1_DMA_STOP_SHIFT);
1930         WREG32(mmDMA_QM_2_GLBL_CFG1, 1 << DMA_QM_2_GLBL_CFG1_DMA_STOP_SHIFT);
1931         WREG32(mmDMA_QM_3_GLBL_CFG1, 1 << DMA_QM_3_GLBL_CFG1_DMA_STOP_SHIFT);
1932         WREG32(mmDMA_QM_4_GLBL_CFG1, 1 << DMA_QM_4_GLBL_CFG1_DMA_STOP_SHIFT);
1933 }
1934
1935 static void goya_tpc_stall(struct hl_device *hdev)
1936 {
1937         WREG32(mmTPC0_CFG_TPC_STALL, 1 << TPC0_CFG_TPC_STALL_V_SHIFT);
1938         WREG32(mmTPC1_CFG_TPC_STALL, 1 << TPC1_CFG_TPC_STALL_V_SHIFT);
1939         WREG32(mmTPC2_CFG_TPC_STALL, 1 << TPC2_CFG_TPC_STALL_V_SHIFT);
1940         WREG32(mmTPC3_CFG_TPC_STALL, 1 << TPC3_CFG_TPC_STALL_V_SHIFT);
1941         WREG32(mmTPC4_CFG_TPC_STALL, 1 << TPC4_CFG_TPC_STALL_V_SHIFT);
1942         WREG32(mmTPC5_CFG_TPC_STALL, 1 << TPC5_CFG_TPC_STALL_V_SHIFT);
1943         WREG32(mmTPC6_CFG_TPC_STALL, 1 << TPC6_CFG_TPC_STALL_V_SHIFT);
1944         WREG32(mmTPC7_CFG_TPC_STALL, 1 << TPC7_CFG_TPC_STALL_V_SHIFT);
1945 }
1946
1947 static void goya_mme_stall(struct hl_device *hdev)
1948 {
1949         WREG32(mmMME_STALL, 0xFFFFFFFF);
1950 }
1951
1952 static int goya_enable_msix(struct hl_device *hdev)
1953 {
1954         struct goya_device *goya = hdev->asic_specific;
1955         int cq_cnt = hdev->asic_prop.completion_queues_count;
1956         int rc, i, irq_cnt_init, irq;
1957
1958         if (goya->hw_cap_initialized & HW_CAP_MSIX)
1959                 return 0;
1960
1961         rc = pci_alloc_irq_vectors(hdev->pdev, GOYA_MSIX_ENTRIES,
1962                                 GOYA_MSIX_ENTRIES, PCI_IRQ_MSIX);
1963         if (rc < 0) {
1964                 dev_err(hdev->dev,
1965                         "MSI-X: Failed to enable support -- %d/%d\n",
1966                         GOYA_MSIX_ENTRIES, rc);
1967                 return rc;
1968         }
1969
1970         for (i = 0, irq_cnt_init = 0 ; i < cq_cnt ; i++, irq_cnt_init++) {
1971                 irq = pci_irq_vector(hdev->pdev, i);
1972                 rc = request_irq(irq, hl_irq_handler_cq, 0, goya_irq_name[i],
1973                                 &hdev->completion_queue[i]);
1974                 if (rc) {
1975                         dev_err(hdev->dev, "Failed to request IRQ %d", irq);
1976                         goto free_irqs;
1977                 }
1978         }
1979
1980         irq = pci_irq_vector(hdev->pdev, GOYA_EVENT_QUEUE_MSIX_IDX);
1981
1982         rc = request_irq(irq, hl_irq_handler_eq, 0,
1983                         goya_irq_name[GOYA_EVENT_QUEUE_MSIX_IDX],
1984                         &hdev->event_queue);
1985         if (rc) {
1986                 dev_err(hdev->dev, "Failed to request IRQ %d", irq);
1987                 goto free_irqs;
1988         }
1989
1990         goya->hw_cap_initialized |= HW_CAP_MSIX;
1991         return 0;
1992
1993 free_irqs:
1994         for (i = 0 ; i < irq_cnt_init ; i++)
1995                 free_irq(pci_irq_vector(hdev->pdev, i),
1996                         &hdev->completion_queue[i]);
1997
1998         pci_free_irq_vectors(hdev->pdev);
1999         return rc;
2000 }
2001
2002 static void goya_sync_irqs(struct hl_device *hdev)
2003 {
2004         struct goya_device *goya = hdev->asic_specific;
2005         int i;
2006
2007         if (!(goya->hw_cap_initialized & HW_CAP_MSIX))
2008                 return;
2009
2010         /* Wait for all pending IRQs to be finished */
2011         for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
2012                 synchronize_irq(pci_irq_vector(hdev->pdev, i));
2013
2014         synchronize_irq(pci_irq_vector(hdev->pdev, GOYA_EVENT_QUEUE_MSIX_IDX));
2015 }
2016
2017 static void goya_disable_msix(struct hl_device *hdev)
2018 {
2019         struct goya_device *goya = hdev->asic_specific;
2020         int i, irq;
2021
2022         if (!(goya->hw_cap_initialized & HW_CAP_MSIX))
2023                 return;
2024
2025         goya_sync_irqs(hdev);
2026
2027         irq = pci_irq_vector(hdev->pdev, GOYA_EVENT_QUEUE_MSIX_IDX);
2028         free_irq(irq, &hdev->event_queue);
2029
2030         for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) {
2031                 irq = pci_irq_vector(hdev->pdev, i);
2032                 free_irq(irq, &hdev->completion_queue[i]);
2033         }
2034
2035         pci_free_irq_vectors(hdev->pdev);
2036
2037         goya->hw_cap_initialized &= ~HW_CAP_MSIX;
2038 }
2039
2040 static void goya_halt_engines(struct hl_device *hdev, bool hard_reset)
2041 {
2042         u32 wait_timeout_ms, cpu_timeout_ms;
2043
2044         dev_info(hdev->dev,
2045                 "Halting compute engines and disabling interrupts\n");
2046
2047         if (hdev->pldm) {
2048                 wait_timeout_ms = GOYA_PLDM_RESET_WAIT_MSEC;
2049                 cpu_timeout_ms = GOYA_PLDM_RESET_WAIT_MSEC;
2050         } else {
2051                 wait_timeout_ms = GOYA_RESET_WAIT_MSEC;
2052                 cpu_timeout_ms = GOYA_CPU_RESET_WAIT_MSEC;
2053         }
2054
2055         if (hard_reset) {
2056                 /*
2057                  * I don't know what is the state of the CPU so make sure it is
2058                  * stopped in any means necessary
2059                  */
2060                 WREG32(mmPSOC_GLOBAL_CONF_UBOOT_MAGIC, KMD_MSG_GOTO_WFE);
2061                 WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR,
2062                         GOYA_ASYNC_EVENT_ID_HALT_MACHINE);
2063                 msleep(cpu_timeout_ms);
2064         }
2065
2066         goya_stop_external_queues(hdev);
2067         goya_stop_internal_queues(hdev);
2068
2069         msleep(wait_timeout_ms);
2070
2071         goya_dma_stall(hdev);
2072         goya_tpc_stall(hdev);
2073         goya_mme_stall(hdev);
2074
2075         msleep(wait_timeout_ms);
2076
2077         goya_disable_external_queues(hdev);
2078         goya_disable_internal_queues(hdev);
2079
2080         if (hard_reset) {
2081                 goya_disable_msix(hdev);
2082                 goya_mmu_remove_device_cpu_mappings(hdev);
2083         } else {
2084                 goya_sync_irqs(hdev);
2085         }
2086 }
2087
2088 /*
2089  * goya_push_uboot_to_device() - Push u-boot FW code to device.
2090  * @hdev: Pointer to hl_device structure.
2091  *
2092  * Copy u-boot fw code from firmware file to SRAM BAR.
2093  *
2094  * Return: 0 on success, non-zero for failure.
2095  */
2096 static int goya_push_uboot_to_device(struct hl_device *hdev)
2097 {
2098         char fw_name[200];
2099         void __iomem *dst;
2100
2101         snprintf(fw_name, sizeof(fw_name), "habanalabs/goya/goya-u-boot.bin");
2102         dst = hdev->pcie_bar[SRAM_CFG_BAR_ID] + UBOOT_FW_OFFSET;
2103
2104         return hl_fw_push_fw_to_device(hdev, fw_name, dst);
2105 }
2106
2107 /*
2108  * goya_push_linux_to_device() - Push LINUX FW code to device.
2109  * @hdev: Pointer to hl_device structure.
2110  *
2111  * Copy LINUX fw code from firmware file to HBM BAR.
2112  *
2113  * Return: 0 on success, non-zero for failure.
2114  */
2115 static int goya_push_linux_to_device(struct hl_device *hdev)
2116 {
2117         char fw_name[200];
2118         void __iomem *dst;
2119
2120         snprintf(fw_name, sizeof(fw_name), "habanalabs/goya/goya-fit.itb");
2121         dst = hdev->pcie_bar[DDR_BAR_ID] + LINUX_FW_OFFSET;
2122
2123         return hl_fw_push_fw_to_device(hdev, fw_name, dst);
2124 }
2125
2126 static int goya_pldm_init_cpu(struct hl_device *hdev)
2127 {
2128         u32 val, unit_rst_val;
2129         int rc;
2130
2131         /* Must initialize SRAM scrambler before pushing u-boot to SRAM */
2132         goya_init_golden_registers(hdev);
2133
2134         /* Put ARM cores into reset */
2135         WREG32(mmCPU_CA53_CFG_ARM_RST_CONTROL, CPU_RESET_ASSERT);
2136         val = RREG32(mmCPU_CA53_CFG_ARM_RST_CONTROL);
2137
2138         /* Reset the CA53 MACRO */
2139         unit_rst_val = RREG32(mmPSOC_GLOBAL_CONF_UNIT_RST_N);
2140         WREG32(mmPSOC_GLOBAL_CONF_UNIT_RST_N, CA53_RESET);
2141         val = RREG32(mmPSOC_GLOBAL_CONF_UNIT_RST_N);
2142         WREG32(mmPSOC_GLOBAL_CONF_UNIT_RST_N, unit_rst_val);
2143         val = RREG32(mmPSOC_GLOBAL_CONF_UNIT_RST_N);
2144
2145         rc = goya_push_uboot_to_device(hdev);
2146         if (rc)
2147                 return rc;
2148
2149         rc = goya_push_linux_to_device(hdev);
2150         if (rc)
2151                 return rc;
2152
2153         WREG32(mmPSOC_GLOBAL_CONF_UBOOT_MAGIC, KMD_MSG_FIT_RDY);
2154         WREG32(mmPSOC_GLOBAL_CONF_WARM_REBOOT, CPU_BOOT_STATUS_NA);
2155
2156         WREG32(mmCPU_CA53_CFG_RST_ADDR_LSB_0,
2157                 lower_32_bits(SRAM_BASE_ADDR + UBOOT_FW_OFFSET));
2158         WREG32(mmCPU_CA53_CFG_RST_ADDR_MSB_0,
2159                 upper_32_bits(SRAM_BASE_ADDR + UBOOT_FW_OFFSET));
2160
2161         /* Release ARM core 0 from reset */
2162         WREG32(mmCPU_CA53_CFG_ARM_RST_CONTROL,
2163                                         CPU_RESET_CORE0_DEASSERT);
2164         val = RREG32(mmCPU_CA53_CFG_ARM_RST_CONTROL);
2165
2166         return 0;
2167 }
2168
2169 /*
2170  * FW component passes an offset from SRAM_BASE_ADDR in SCRATCHPAD_xx.
2171  * The version string should be located by that offset.
2172  */
2173 static void goya_read_device_fw_version(struct hl_device *hdev,
2174                                         enum goya_fw_component fwc)
2175 {
2176         const char *name;
2177         u32 ver_off;
2178         char *dest;
2179
2180         switch (fwc) {
2181         case FW_COMP_UBOOT:
2182                 ver_off = RREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_29);
2183                 dest = hdev->asic_prop.uboot_ver;
2184                 name = "U-Boot";
2185                 break;
2186         case FW_COMP_PREBOOT:
2187                 ver_off = RREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_28);
2188                 dest = hdev->asic_prop.preboot_ver;
2189                 name = "Preboot";
2190                 break;
2191         default:
2192                 dev_warn(hdev->dev, "Undefined FW component: %d\n", fwc);
2193                 return;
2194         }
2195
2196         ver_off &= ~((u32)SRAM_BASE_ADDR);
2197
2198         if (ver_off < SRAM_SIZE - VERSION_MAX_LEN) {
2199                 memcpy_fromio(dest, hdev->pcie_bar[SRAM_CFG_BAR_ID] + ver_off,
2200                                                         VERSION_MAX_LEN);
2201         } else {
2202                 dev_err(hdev->dev, "%s version offset (0x%x) is above SRAM\n",
2203                                                                 name, ver_off);
2204                 strcpy(dest, "unavailable");
2205         }
2206 }
2207
2208 static int goya_init_cpu(struct hl_device *hdev, u32 cpu_timeout)
2209 {
2210         struct goya_device *goya = hdev->asic_specific;
2211         u32 status;
2212         int rc;
2213
2214         if (!hdev->cpu_enable)
2215                 return 0;
2216
2217         if (goya->hw_cap_initialized & HW_CAP_CPU)
2218                 return 0;
2219
2220         /*
2221          * Before pushing u-boot/linux to device, need to set the ddr bar to
2222          * base address of dram
2223          */
2224         if (goya_set_ddr_bar_base(hdev, DRAM_PHYS_BASE) == U64_MAX) {
2225                 dev_err(hdev->dev,
2226                         "failed to map DDR bar to DRAM base address\n");
2227                 return -EIO;
2228         }
2229
2230         if (hdev->pldm) {
2231                 rc = goya_pldm_init_cpu(hdev);
2232                 if (rc)
2233                         return rc;
2234
2235                 goto out;
2236         }
2237
2238         /* Make sure CPU boot-loader is running */
2239         rc = hl_poll_timeout(
2240                 hdev,
2241                 mmPSOC_GLOBAL_CONF_WARM_REBOOT,
2242                 status,
2243                 (status == CPU_BOOT_STATUS_DRAM_RDY) ||
2244                 (status == CPU_BOOT_STATUS_SRAM_AVAIL),
2245                 10000,
2246                 cpu_timeout);
2247
2248         if (rc) {
2249                 dev_err(hdev->dev, "Error in ARM u-boot!");
2250                 switch (status) {
2251                 case CPU_BOOT_STATUS_NA:
2252                         dev_err(hdev->dev,
2253                                 "ARM status %d - BTL did NOT run\n", status);
2254                         break;
2255                 case CPU_BOOT_STATUS_IN_WFE:
2256                         dev_err(hdev->dev,
2257                                 "ARM status %d - Inside WFE loop\n", status);
2258                         break;
2259                 case CPU_BOOT_STATUS_IN_BTL:
2260                         dev_err(hdev->dev,
2261                                 "ARM status %d - Stuck in BTL\n", status);
2262                         break;
2263                 case CPU_BOOT_STATUS_IN_PREBOOT:
2264                         dev_err(hdev->dev,
2265                                 "ARM status %d - Stuck in Preboot\n", status);
2266                         break;
2267                 case CPU_BOOT_STATUS_IN_SPL:
2268                         dev_err(hdev->dev,
2269                                 "ARM status %d - Stuck in SPL\n", status);
2270                         break;
2271                 case CPU_BOOT_STATUS_IN_UBOOT:
2272                         dev_err(hdev->dev,
2273                                 "ARM status %d - Stuck in u-boot\n", status);
2274                         break;
2275                 case CPU_BOOT_STATUS_DRAM_INIT_FAIL:
2276                         dev_err(hdev->dev,
2277                                 "ARM status %d - DDR initialization failed\n",
2278                                 status);
2279                         break;
2280                 case CPU_BOOT_STATUS_UBOOT_NOT_READY:
2281                         dev_err(hdev->dev,
2282                                 "ARM status %d - u-boot stopped by user\n",
2283                                 status);
2284                         break;
2285                 default:
2286                         dev_err(hdev->dev,
2287                                 "ARM status %d - Invalid status code\n",
2288                                 status);
2289                         break;
2290                 }
2291                 return -EIO;
2292         }
2293
2294         /* Read U-Boot version now in case we will later fail */
2295         goya_read_device_fw_version(hdev, FW_COMP_UBOOT);
2296         goya_read_device_fw_version(hdev, FW_COMP_PREBOOT);
2297
2298         if (!hdev->fw_loading) {
2299                 dev_info(hdev->dev, "Skip loading FW\n");
2300                 goto out;
2301         }
2302
2303         if (status == CPU_BOOT_STATUS_SRAM_AVAIL)
2304                 goto out;
2305
2306         rc = goya_push_linux_to_device(hdev);
2307         if (rc)
2308                 return rc;
2309
2310         WREG32(mmPSOC_GLOBAL_CONF_UBOOT_MAGIC, KMD_MSG_FIT_RDY);
2311
2312         rc = hl_poll_timeout(
2313                 hdev,
2314                 mmPSOC_GLOBAL_CONF_WARM_REBOOT,
2315                 status,
2316                 (status == CPU_BOOT_STATUS_SRAM_AVAIL),
2317                 10000,
2318                 cpu_timeout);
2319
2320         if (rc) {
2321                 if (status == CPU_BOOT_STATUS_FIT_CORRUPTED)
2322                         dev_err(hdev->dev,
2323                                 "ARM u-boot reports FIT image is corrupted\n");
2324                 else
2325                         dev_err(hdev->dev,
2326                                 "ARM Linux failed to load, %d\n", status);
2327                 WREG32(mmPSOC_GLOBAL_CONF_UBOOT_MAGIC, KMD_MSG_NA);
2328                 return -EIO;
2329         }
2330
2331         dev_info(hdev->dev, "Successfully loaded firmware to device\n");
2332
2333 out:
2334         goya->hw_cap_initialized |= HW_CAP_CPU;
2335
2336         return 0;
2337 }
2338
2339 static int goya_mmu_update_asid_hop0_addr(struct hl_device *hdev, u32 asid,
2340                                                 u64 phys_addr)
2341 {
2342         u32 status, timeout_usec;
2343         int rc;
2344
2345         if (hdev->pldm)
2346                 timeout_usec = GOYA_PLDM_MMU_TIMEOUT_USEC;
2347         else
2348                 timeout_usec = MMU_CONFIG_TIMEOUT_USEC;
2349
2350         WREG32(MMU_HOP0_PA43_12, phys_addr >> MMU_HOP0_PA43_12_SHIFT);
2351         WREG32(MMU_HOP0_PA49_44, phys_addr >> MMU_HOP0_PA49_44_SHIFT);
2352         WREG32(MMU_ASID_BUSY, 0x80000000 | asid);
2353
2354         rc = hl_poll_timeout(
2355                 hdev,
2356                 MMU_ASID_BUSY,
2357                 status,
2358                 !(status & 0x80000000),
2359                 1000,
2360                 timeout_usec);
2361
2362         if (rc) {
2363                 dev_err(hdev->dev,
2364                         "Timeout during MMU hop0 config of asid %d\n", asid);
2365                 return rc;
2366         }
2367
2368         return 0;
2369 }
2370
2371 int goya_mmu_init(struct hl_device *hdev)
2372 {
2373         struct asic_fixed_properties *prop = &hdev->asic_prop;
2374         struct goya_device *goya = hdev->asic_specific;
2375         u64 hop0_addr;
2376         int rc, i;
2377
2378         if (!hdev->mmu_enable)
2379                 return 0;
2380
2381         if (goya->hw_cap_initialized & HW_CAP_MMU)
2382                 return 0;
2383
2384         hdev->dram_supports_virtual_memory = true;
2385         hdev->dram_default_page_mapping = true;
2386
2387         for (i = 0 ; i < prop->max_asid ; i++) {
2388                 hop0_addr = prop->mmu_pgt_addr +
2389                                 (i * prop->mmu_hop_table_size);
2390
2391                 rc = goya_mmu_update_asid_hop0_addr(hdev, i, hop0_addr);
2392                 if (rc) {
2393                         dev_err(hdev->dev,
2394                                 "failed to set hop0 addr for asid %d\n", i);
2395                         goto err;
2396                 }
2397         }
2398
2399         goya->hw_cap_initialized |= HW_CAP_MMU;
2400
2401         /* init MMU cache manage page */
2402         WREG32(mmSTLB_CACHE_INV_BASE_39_8,
2403                                 lower_32_bits(MMU_CACHE_MNG_ADDR >> 8));
2404         WREG32(mmSTLB_CACHE_INV_BASE_49_40, MMU_CACHE_MNG_ADDR >> 40);
2405
2406         /* Remove follower feature due to performance bug */
2407         WREG32_AND(mmSTLB_STLB_FEATURE_EN,
2408                         (~STLB_STLB_FEATURE_EN_FOLLOWER_EN_MASK));
2409
2410         hdev->asic_funcs->mmu_invalidate_cache(hdev, true);
2411
2412         WREG32(mmMMU_MMU_ENABLE, 1);
2413         WREG32(mmMMU_SPI_MASK, 0xF);
2414
2415         return 0;
2416
2417 err:
2418         return rc;
2419 }
2420
2421 /*
2422  * goya_hw_init - Goya hardware initialization code
2423  *
2424  * @hdev: pointer to hl_device structure
2425  *
2426  * Returns 0 on success
2427  *
2428  */
2429 static int goya_hw_init(struct hl_device *hdev)
2430 {
2431         struct asic_fixed_properties *prop = &hdev->asic_prop;
2432         u32 val;
2433         int rc;
2434
2435         dev_info(hdev->dev, "Starting initialization of H/W\n");
2436
2437         /* Perform read from the device to make sure device is up */
2438         val = RREG32(mmPCIE_DBI_DEVICE_ID_VENDOR_ID_REG);
2439
2440         /*
2441          * Let's mark in the H/W that we have reached this point. We check
2442          * this value in the reset_before_init function to understand whether
2443          * we need to reset the chip before doing H/W init. This register is
2444          * cleared by the H/W upon H/W reset
2445          */
2446         WREG32(mmPSOC_GLOBAL_CONF_APP_STATUS, HL_DEVICE_HW_STATE_DIRTY);
2447
2448         rc = goya_init_cpu(hdev, GOYA_CPU_TIMEOUT_USEC);
2449         if (rc) {
2450                 dev_err(hdev->dev, "failed to initialize CPU\n");
2451                 return rc;
2452         }
2453
2454         goya_tpc_mbist_workaround(hdev);
2455
2456         goya_init_golden_registers(hdev);
2457
2458         /*
2459          * After CPU initialization is finished, change DDR bar mapping inside
2460          * iATU to point to the start address of the MMU page tables
2461          */
2462         if (goya_set_ddr_bar_base(hdev, DRAM_PHYS_BASE +
2463                         (MMU_PAGE_TABLES_ADDR &
2464                         ~(prop->dram_pci_bar_size - 0x1ull))) == U64_MAX) {
2465                 dev_err(hdev->dev,
2466                         "failed to map DDR bar to MMU page tables\n");
2467                 return -EIO;
2468         }
2469
2470         rc = goya_mmu_init(hdev);
2471         if (rc)
2472                 return rc;
2473
2474         goya_init_security(hdev);
2475
2476         goya_init_dma_qmans(hdev);
2477
2478         goya_init_mme_qmans(hdev);
2479
2480         goya_init_tpc_qmans(hdev);
2481
2482         /* MSI-X must be enabled before CPU queues are initialized */
2483         rc = goya_enable_msix(hdev);
2484         if (rc)
2485                 goto disable_queues;
2486
2487         /* Perform read from the device to flush all MSI-X configuration */
2488         val = RREG32(mmPCIE_DBI_DEVICE_ID_VENDOR_ID_REG);
2489
2490         return 0;
2491
2492 disable_queues:
2493         goya_disable_internal_queues(hdev);
2494         goya_disable_external_queues(hdev);
2495
2496         return rc;
2497 }
2498
2499 /*
2500  * goya_hw_fini - Goya hardware tear-down code
2501  *
2502  * @hdev: pointer to hl_device structure
2503  * @hard_reset: should we do hard reset to all engines or just reset the
2504  *              compute/dma engines
2505  */
2506 static void goya_hw_fini(struct hl_device *hdev, bool hard_reset)
2507 {
2508         struct goya_device *goya = hdev->asic_specific;
2509         u32 reset_timeout_ms, status;
2510
2511         if (hdev->pldm)
2512                 reset_timeout_ms = GOYA_PLDM_RESET_TIMEOUT_MSEC;
2513         else
2514                 reset_timeout_ms = GOYA_RESET_TIMEOUT_MSEC;
2515
2516         if (hard_reset) {
2517                 goya_set_ddr_bar_base(hdev, DRAM_PHYS_BASE);
2518                 goya_disable_clk_rlx(hdev);
2519                 goya_set_pll_refclk(hdev);
2520
2521                 WREG32(mmPSOC_GLOBAL_CONF_SW_ALL_RST_CFG, RESET_ALL);
2522                 dev_info(hdev->dev,
2523                         "Issued HARD reset command, going to wait %dms\n",
2524                         reset_timeout_ms);
2525         } else {
2526                 WREG32(mmPSOC_GLOBAL_CONF_SW_ALL_RST_CFG, DMA_MME_TPC_RESET);
2527                 dev_info(hdev->dev,
2528                         "Issued SOFT reset command, going to wait %dms\n",
2529                         reset_timeout_ms);
2530         }
2531
2532         /*
2533          * After hard reset, we can't poll the BTM_FSM register because the PSOC
2534          * itself is in reset. In either reset we need to wait until the reset
2535          * is deasserted
2536          */
2537         msleep(reset_timeout_ms);
2538
2539         status = RREG32(mmPSOC_GLOBAL_CONF_BTM_FSM);
2540         if (status & PSOC_GLOBAL_CONF_BTM_FSM_STATE_MASK)
2541                 dev_err(hdev->dev,
2542                         "Timeout while waiting for device to reset 0x%x\n",
2543                         status);
2544
2545         if (!hard_reset) {
2546                 goya->hw_cap_initialized &= ~(HW_CAP_DMA | HW_CAP_MME |
2547                                                 HW_CAP_GOLDEN | HW_CAP_TPC);
2548                 WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR,
2549                                 GOYA_ASYNC_EVENT_ID_SOFT_RESET);
2550                 return;
2551         }
2552
2553         /* Chicken bit to re-initiate boot sequencer flow */
2554         WREG32(mmPSOC_GLOBAL_CONF_BOOT_SEQ_RE_START,
2555                 1 << PSOC_GLOBAL_CONF_BOOT_SEQ_RE_START_IND_SHIFT);
2556         /* Move boot manager FSM to pre boot sequencer init state */
2557         WREG32(mmPSOC_GLOBAL_CONF_SW_BTM_FSM,
2558                         0xA << PSOC_GLOBAL_CONF_SW_BTM_FSM_CTRL_SHIFT);
2559
2560         goya->hw_cap_initialized &= ~(HW_CAP_CPU | HW_CAP_CPU_Q |
2561                                         HW_CAP_DDR_0 | HW_CAP_DDR_1 |
2562                                         HW_CAP_DMA | HW_CAP_MME |
2563                                         HW_CAP_MMU | HW_CAP_TPC_MBIST |
2564                                         HW_CAP_GOLDEN | HW_CAP_TPC);
2565         memset(goya->events_stat, 0, sizeof(goya->events_stat));
2566
2567         if (!hdev->pldm) {
2568                 int rc;
2569                 /* In case we are running inside VM and the VM is
2570                  * shutting down, we need to make sure CPU boot-loader
2571                  * is running before we can continue the VM shutdown.
2572                  * That is because the VM will send an FLR signal that
2573                  * we must answer
2574                  */
2575                 dev_info(hdev->dev,
2576                         "Going to wait up to %ds for CPU boot loader\n",
2577                         GOYA_CPU_TIMEOUT_USEC / 1000 / 1000);
2578
2579                 rc = hl_poll_timeout(
2580                         hdev,
2581                         mmPSOC_GLOBAL_CONF_WARM_REBOOT,
2582                         status,
2583                         (status == CPU_BOOT_STATUS_DRAM_RDY),
2584                         10000,
2585                         GOYA_CPU_TIMEOUT_USEC);
2586                 if (rc)
2587                         dev_err(hdev->dev,
2588                                 "failed to wait for CPU boot loader\n");
2589         }
2590 }
2591
2592 int goya_suspend(struct hl_device *hdev)
2593 {
2594         int rc;
2595
2596         rc = hl_fw_send_pci_access_msg(hdev, ARMCP_PACKET_DISABLE_PCI_ACCESS);
2597         if (rc)
2598                 dev_err(hdev->dev, "Failed to disable PCI access from CPU\n");
2599
2600         return rc;
2601 }
2602
2603 int goya_resume(struct hl_device *hdev)
2604 {
2605         return goya_init_iatu(hdev);
2606 }
2607
2608 static int goya_cb_mmap(struct hl_device *hdev, struct vm_area_struct *vma,
2609                 u64 kaddress, phys_addr_t paddress, u32 size)
2610 {
2611         int rc;
2612
2613         vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP |
2614                         VM_DONTCOPY | VM_NORESERVE;
2615
2616         rc = remap_pfn_range(vma, vma->vm_start, paddress >> PAGE_SHIFT,
2617                                 size, vma->vm_page_prot);
2618         if (rc)
2619                 dev_err(hdev->dev, "remap_pfn_range error %d", rc);
2620
2621         return rc;
2622 }
2623
2624 void goya_ring_doorbell(struct hl_device *hdev, u32 hw_queue_id, u32 pi)
2625 {
2626         u32 db_reg_offset, db_value;
2627
2628         switch (hw_queue_id) {
2629         case GOYA_QUEUE_ID_DMA_0:
2630                 db_reg_offset = mmDMA_QM_0_PQ_PI;
2631                 break;
2632
2633         case GOYA_QUEUE_ID_DMA_1:
2634                 db_reg_offset = mmDMA_QM_1_PQ_PI;
2635                 break;
2636
2637         case GOYA_QUEUE_ID_DMA_2:
2638                 db_reg_offset = mmDMA_QM_2_PQ_PI;
2639                 break;
2640
2641         case GOYA_QUEUE_ID_DMA_3:
2642                 db_reg_offset = mmDMA_QM_3_PQ_PI;
2643                 break;
2644
2645         case GOYA_QUEUE_ID_DMA_4:
2646                 db_reg_offset = mmDMA_QM_4_PQ_PI;
2647                 break;
2648
2649         case GOYA_QUEUE_ID_CPU_PQ:
2650                 db_reg_offset = mmCPU_IF_PF_PQ_PI;
2651                 break;
2652
2653         case GOYA_QUEUE_ID_MME:
2654                 db_reg_offset = mmMME_QM_PQ_PI;
2655                 break;
2656
2657         case GOYA_QUEUE_ID_TPC0:
2658                 db_reg_offset = mmTPC0_QM_PQ_PI;
2659                 break;
2660
2661         case GOYA_QUEUE_ID_TPC1:
2662                 db_reg_offset = mmTPC1_QM_PQ_PI;
2663                 break;
2664
2665         case GOYA_QUEUE_ID_TPC2:
2666                 db_reg_offset = mmTPC2_QM_PQ_PI;
2667                 break;
2668
2669         case GOYA_QUEUE_ID_TPC3:
2670                 db_reg_offset = mmTPC3_QM_PQ_PI;
2671                 break;
2672
2673         case GOYA_QUEUE_ID_TPC4:
2674                 db_reg_offset = mmTPC4_QM_PQ_PI;
2675                 break;
2676
2677         case GOYA_QUEUE_ID_TPC5:
2678                 db_reg_offset = mmTPC5_QM_PQ_PI;
2679                 break;
2680
2681         case GOYA_QUEUE_ID_TPC6:
2682                 db_reg_offset = mmTPC6_QM_PQ_PI;
2683                 break;
2684
2685         case GOYA_QUEUE_ID_TPC7:
2686                 db_reg_offset = mmTPC7_QM_PQ_PI;
2687                 break;
2688
2689         default:
2690                 /* Should never get here */
2691                 dev_err(hdev->dev, "H/W queue %d is invalid. Can't set pi\n",
2692                         hw_queue_id);
2693                 return;
2694         }
2695
2696         db_value = pi;
2697
2698         /* ring the doorbell */
2699         WREG32(db_reg_offset, db_value);
2700
2701         if (hw_queue_id == GOYA_QUEUE_ID_CPU_PQ)
2702                 WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR,
2703                                 GOYA_ASYNC_EVENT_ID_PI_UPDATE);
2704 }
2705
2706 void goya_flush_pq_write(struct hl_device *hdev, u64 *pq, u64 exp_val)
2707 {
2708         /* Not needed in Goya */
2709 }
2710
2711 static void *goya_dma_alloc_coherent(struct hl_device *hdev, size_t size,
2712                                         dma_addr_t *dma_handle, gfp_t flags)
2713 {
2714         void *kernel_addr = dma_alloc_coherent(&hdev->pdev->dev, size,
2715                                                 dma_handle, flags);
2716
2717         /* Shift to the device's base physical address of host memory */
2718         if (kernel_addr)
2719                 *dma_handle += HOST_PHYS_BASE;
2720
2721         return kernel_addr;
2722 }
2723
2724 static void goya_dma_free_coherent(struct hl_device *hdev, size_t size,
2725                                         void *cpu_addr, dma_addr_t dma_handle)
2726 {
2727         /* Cancel the device's base physical address of host memory */
2728         dma_addr_t fixed_dma_handle = dma_handle - HOST_PHYS_BASE;
2729
2730         dma_free_coherent(&hdev->pdev->dev, size, cpu_addr, fixed_dma_handle);
2731 }
2732
2733 void *goya_get_int_queue_base(struct hl_device *hdev, u32 queue_id,
2734                                 dma_addr_t *dma_handle, u16 *queue_len)
2735 {
2736         void *base;
2737         u32 offset;
2738
2739         *dma_handle = hdev->asic_prop.sram_base_address;
2740
2741         base = (void *) hdev->pcie_bar[SRAM_CFG_BAR_ID];
2742
2743         switch (queue_id) {
2744         case GOYA_QUEUE_ID_MME:
2745                 offset = MME_QMAN_BASE_OFFSET;
2746                 *queue_len = MME_QMAN_LENGTH;
2747                 break;
2748         case GOYA_QUEUE_ID_TPC0:
2749                 offset = TPC0_QMAN_BASE_OFFSET;
2750                 *queue_len = TPC_QMAN_LENGTH;
2751                 break;
2752         case GOYA_QUEUE_ID_TPC1:
2753                 offset = TPC1_QMAN_BASE_OFFSET;
2754                 *queue_len = TPC_QMAN_LENGTH;
2755                 break;
2756         case GOYA_QUEUE_ID_TPC2:
2757                 offset = TPC2_QMAN_BASE_OFFSET;
2758                 *queue_len = TPC_QMAN_LENGTH;
2759                 break;
2760         case GOYA_QUEUE_ID_TPC3:
2761                 offset = TPC3_QMAN_BASE_OFFSET;
2762                 *queue_len = TPC_QMAN_LENGTH;
2763                 break;
2764         case GOYA_QUEUE_ID_TPC4:
2765                 offset = TPC4_QMAN_BASE_OFFSET;
2766                 *queue_len = TPC_QMAN_LENGTH;
2767                 break;
2768         case GOYA_QUEUE_ID_TPC5:
2769                 offset = TPC5_QMAN_BASE_OFFSET;
2770                 *queue_len = TPC_QMAN_LENGTH;
2771                 break;
2772         case GOYA_QUEUE_ID_TPC6:
2773                 offset = TPC6_QMAN_BASE_OFFSET;
2774                 *queue_len = TPC_QMAN_LENGTH;
2775                 break;
2776         case GOYA_QUEUE_ID_TPC7:
2777                 offset = TPC7_QMAN_BASE_OFFSET;
2778                 *queue_len = TPC_QMAN_LENGTH;
2779                 break;
2780         default:
2781                 dev_err(hdev->dev, "Got invalid queue id %d\n", queue_id);
2782                 return NULL;
2783         }
2784
2785         base += offset;
2786         *dma_handle += offset;
2787
2788         return base;
2789 }
2790
2791 static int goya_send_job_on_qman0(struct hl_device *hdev, struct hl_cs_job *job)
2792 {
2793         struct packet_msg_prot *fence_pkt;
2794         u32 *fence_ptr;
2795         dma_addr_t fence_dma_addr;
2796         struct hl_cb *cb;
2797         u32 tmp, timeout;
2798         char buf[16] = {};
2799         int rc;
2800
2801         if (hdev->pldm)
2802                 timeout = GOYA_PLDM_QMAN0_TIMEOUT_USEC;
2803         else
2804                 timeout = HL_DEVICE_TIMEOUT_USEC;
2805
2806         if (!hdev->asic_funcs->is_device_idle(hdev, buf, sizeof(buf))) {
2807                 dev_err_ratelimited(hdev->dev,
2808                         "Can't send KMD job on QMAN0 because %s is busy\n",
2809                         buf);
2810                 return -EBUSY;
2811         }
2812
2813         fence_ptr = hdev->asic_funcs->asic_dma_pool_zalloc(hdev, 4, GFP_KERNEL,
2814                                                         &fence_dma_addr);
2815         if (!fence_ptr) {
2816                 dev_err(hdev->dev,
2817                         "Failed to allocate fence memory for QMAN0\n");
2818                 return -ENOMEM;
2819         }
2820
2821         goya_qman0_set_security(hdev, true);
2822
2823         cb = job->patched_cb;
2824
2825         fence_pkt = (struct packet_msg_prot *) (uintptr_t) (cb->kernel_address +
2826                         job->job_cb_size - sizeof(struct packet_msg_prot));
2827
2828         tmp = (PACKET_MSG_PROT << GOYA_PKT_CTL_OPCODE_SHIFT) |
2829                         (1 << GOYA_PKT_CTL_EB_SHIFT) |
2830                         (1 << GOYA_PKT_CTL_MB_SHIFT);
2831         fence_pkt->ctl = cpu_to_le32(tmp);
2832         fence_pkt->value = cpu_to_le32(GOYA_QMAN0_FENCE_VAL);
2833         fence_pkt->addr = cpu_to_le64(fence_dma_addr);
2834
2835         rc = hl_hw_queue_send_cb_no_cmpl(hdev, GOYA_QUEUE_ID_DMA_0,
2836                                         job->job_cb_size, cb->bus_address);
2837         if (rc) {
2838                 dev_err(hdev->dev, "Failed to send CB on QMAN0, %d\n", rc);
2839                 goto free_fence_ptr;
2840         }
2841
2842         rc = hl_poll_timeout_memory(hdev, fence_ptr, tmp,
2843                                 (tmp == GOYA_QMAN0_FENCE_VAL), 1000, timeout);
2844
2845         hl_hw_queue_inc_ci_kernel(hdev, GOYA_QUEUE_ID_DMA_0);
2846
2847         if (rc == -ETIMEDOUT) {
2848                 dev_err(hdev->dev, "QMAN0 Job timeout (0x%x)\n", tmp);
2849                 goto free_fence_ptr;
2850         }
2851
2852 free_fence_ptr:
2853         hdev->asic_funcs->asic_dma_pool_free(hdev, (void *) fence_ptr,
2854                                         fence_dma_addr);
2855
2856         goya_qman0_set_security(hdev, false);
2857
2858         return rc;
2859 }
2860
2861 int goya_send_cpu_message(struct hl_device *hdev, u32 *msg, u16 len,
2862                                 u32 timeout, long *result)
2863 {
2864         struct goya_device *goya = hdev->asic_specific;
2865
2866         if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q)) {
2867                 if (result)
2868                         *result = 0;
2869                 return 0;
2870         }
2871
2872         return hl_fw_send_cpu_message(hdev, GOYA_QUEUE_ID_CPU_PQ, msg, len,
2873                                         timeout, result);
2874 }
2875
2876 int goya_test_queue(struct hl_device *hdev, u32 hw_queue_id)
2877 {
2878         struct packet_msg_prot *fence_pkt;
2879         dma_addr_t pkt_dma_addr;
2880         u32 fence_val, tmp;
2881         dma_addr_t fence_dma_addr;
2882         u32 *fence_ptr;
2883         int rc;
2884
2885         fence_val = GOYA_QMAN0_FENCE_VAL;
2886
2887         fence_ptr = hdev->asic_funcs->asic_dma_pool_zalloc(hdev, 4, GFP_KERNEL,
2888                                                         &fence_dma_addr);
2889         if (!fence_ptr) {
2890                 dev_err(hdev->dev,
2891                         "Failed to allocate memory for queue testing\n");
2892                 return -ENOMEM;
2893         }
2894
2895         *fence_ptr = 0;
2896
2897         fence_pkt = hdev->asic_funcs->asic_dma_pool_zalloc(hdev,
2898                                         sizeof(struct packet_msg_prot),
2899                                         GFP_KERNEL, &pkt_dma_addr);
2900         if (!fence_pkt) {
2901                 dev_err(hdev->dev,
2902                         "Failed to allocate packet for queue testing\n");
2903                 rc = -ENOMEM;
2904                 goto free_fence_ptr;
2905         }
2906
2907         tmp = (PACKET_MSG_PROT << GOYA_PKT_CTL_OPCODE_SHIFT) |
2908                         (1 << GOYA_PKT_CTL_EB_SHIFT) |
2909                         (1 << GOYA_PKT_CTL_MB_SHIFT);
2910         fence_pkt->ctl = cpu_to_le32(tmp);
2911         fence_pkt->value = cpu_to_le32(fence_val);
2912         fence_pkt->addr = cpu_to_le64(fence_dma_addr);
2913
2914         rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id,
2915                                         sizeof(struct packet_msg_prot),
2916                                         pkt_dma_addr);
2917         if (rc) {
2918                 dev_err(hdev->dev,
2919                         "Failed to send fence packet\n");
2920                 goto free_pkt;
2921         }
2922
2923         rc = hl_poll_timeout_memory(hdev, fence_ptr, tmp, (tmp == fence_val),
2924                                         1000, GOYA_TEST_QUEUE_WAIT_USEC);
2925
2926         hl_hw_queue_inc_ci_kernel(hdev, hw_queue_id);
2927
2928         if (rc == -ETIMEDOUT) {
2929                 dev_err(hdev->dev,
2930                         "H/W queue %d test failed (scratch(0x%08llX) == 0x%08X)\n",
2931                         hw_queue_id, (unsigned long long) fence_dma_addr, tmp);
2932                 rc = -EIO;
2933         } else {
2934                 dev_info(hdev->dev, "queue test on H/W queue %d succeeded\n",
2935                         hw_queue_id);
2936         }
2937
2938 free_pkt:
2939         hdev->asic_funcs->asic_dma_pool_free(hdev, (void *) fence_pkt,
2940                                         pkt_dma_addr);
2941 free_fence_ptr:
2942         hdev->asic_funcs->asic_dma_pool_free(hdev, (void *) fence_ptr,
2943                                         fence_dma_addr);
2944         return rc;
2945 }
2946
2947 int goya_test_cpu_queue(struct hl_device *hdev)
2948 {
2949         struct goya_device *goya = hdev->asic_specific;
2950
2951         /*
2952          * check capability here as send_cpu_message() won't update the result
2953          * value if no capability
2954          */
2955         if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q))
2956                 return 0;
2957
2958         return hl_fw_test_cpu_queue(hdev);
2959 }
2960
2961 int goya_test_queues(struct hl_device *hdev)
2962 {
2963         int i, rc, ret_val = 0;
2964
2965         for (i = 0 ; i < NUMBER_OF_EXT_HW_QUEUES ; i++) {
2966                 rc = goya_test_queue(hdev, i);
2967                 if (rc)
2968                         ret_val = -EINVAL;
2969         }
2970
2971         return ret_val;
2972 }
2973
2974 static void *goya_dma_pool_zalloc(struct hl_device *hdev, size_t size,
2975                                         gfp_t mem_flags, dma_addr_t *dma_handle)
2976 {
2977         void *kernel_addr;
2978
2979         if (size > GOYA_DMA_POOL_BLK_SIZE)
2980                 return NULL;
2981
2982         kernel_addr =  dma_pool_zalloc(hdev->dma_pool, mem_flags, dma_handle);
2983
2984         /* Shift to the device's base physical address of host memory */
2985         if (kernel_addr)
2986                 *dma_handle += HOST_PHYS_BASE;
2987
2988         return kernel_addr;
2989 }
2990
2991 static void goya_dma_pool_free(struct hl_device *hdev, void *vaddr,
2992                                 dma_addr_t dma_addr)
2993 {
2994         /* Cancel the device's base physical address of host memory */
2995         dma_addr_t fixed_dma_addr = dma_addr - HOST_PHYS_BASE;
2996
2997         dma_pool_free(hdev->dma_pool, vaddr, fixed_dma_addr);
2998 }
2999
3000 void *goya_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size,
3001                                         dma_addr_t *dma_handle)
3002 {
3003         void *vaddr;
3004
3005         vaddr = hl_fw_cpu_accessible_dma_pool_alloc(hdev, size, dma_handle);
3006         *dma_handle = (*dma_handle) - hdev->cpu_accessible_dma_address +
3007                         VA_CPU_ACCESSIBLE_MEM_ADDR;
3008
3009         return vaddr;
3010 }
3011
3012 void goya_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size,
3013                                         void *vaddr)
3014 {
3015         hl_fw_cpu_accessible_dma_pool_free(hdev, size, vaddr);
3016 }
3017
3018 static int goya_dma_map_sg(struct hl_device *hdev, struct scatterlist *sgl,
3019                                 int nents, enum dma_data_direction dir)
3020 {
3021         struct scatterlist *sg;
3022         int i;
3023
3024         if (!dma_map_sg(&hdev->pdev->dev, sgl, nents, dir))
3025                 return -ENOMEM;
3026
3027         /* Shift to the device's base physical address of host memory */
3028         for_each_sg(sgl, sg, nents, i)
3029                 sg->dma_address += HOST_PHYS_BASE;
3030
3031         return 0;
3032 }
3033
3034 static void goya_dma_unmap_sg(struct hl_device *hdev, struct scatterlist *sgl,
3035                                 int nents, enum dma_data_direction dir)
3036 {
3037         struct scatterlist *sg;
3038         int i;
3039
3040         /* Cancel the device's base physical address of host memory */
3041         for_each_sg(sgl, sg, nents, i)
3042                 sg->dma_address -= HOST_PHYS_BASE;
3043
3044         dma_unmap_sg(&hdev->pdev->dev, sgl, nents, dir);
3045 }
3046
3047 u32 goya_get_dma_desc_list_size(struct hl_device *hdev, struct sg_table *sgt)
3048 {
3049         struct scatterlist *sg, *sg_next_iter;
3050         u32 count, dma_desc_cnt;
3051         u64 len, len_next;
3052         dma_addr_t addr, addr_next;
3053
3054         dma_desc_cnt = 0;
3055
3056         for_each_sg(sgt->sgl, sg, sgt->nents, count) {
3057
3058                 len = sg_dma_len(sg);
3059                 addr = sg_dma_address(sg);
3060
3061                 if (len == 0)
3062                         break;
3063
3064                 while ((count + 1) < sgt->nents) {
3065                         sg_next_iter = sg_next(sg);
3066                         len_next = sg_dma_len(sg_next_iter);
3067                         addr_next = sg_dma_address(sg_next_iter);
3068
3069                         if (len_next == 0)
3070                                 break;
3071
3072                         if ((addr + len == addr_next) &&
3073                                 (len + len_next <= DMA_MAX_TRANSFER_SIZE)) {
3074                                 len += len_next;
3075                                 count++;
3076                                 sg = sg_next_iter;
3077                         } else {
3078                                 break;
3079                         }
3080                 }
3081
3082                 dma_desc_cnt++;
3083         }
3084
3085         return dma_desc_cnt * sizeof(struct packet_lin_dma);
3086 }
3087
3088 static int goya_pin_memory_before_cs(struct hl_device *hdev,
3089                                 struct hl_cs_parser *parser,
3090                                 struct packet_lin_dma *user_dma_pkt,
3091                                 u64 addr, enum dma_data_direction dir)
3092 {
3093         struct hl_userptr *userptr;
3094         int rc;
3095
3096         if (hl_userptr_is_pinned(hdev, addr, le32_to_cpu(user_dma_pkt->tsize),
3097                         parser->job_userptr_list, &userptr))
3098                 goto already_pinned;
3099
3100         userptr = kzalloc(sizeof(*userptr), GFP_ATOMIC);
3101         if (!userptr)
3102                 return -ENOMEM;
3103
3104         rc = hl_pin_host_memory(hdev, addr, le32_to_cpu(user_dma_pkt->tsize),
3105                                 userptr);
3106         if (rc)
3107                 goto free_userptr;
3108
3109         list_add_tail(&userptr->job_node, parser->job_userptr_list);
3110
3111         rc = hdev->asic_funcs->asic_dma_map_sg(hdev, userptr->sgt->sgl,
3112                                         userptr->sgt->nents, dir);
3113         if (rc) {
3114                 dev_err(hdev->dev, "failed to map sgt with DMA region\n");
3115                 goto unpin_memory;
3116         }
3117
3118         userptr->dma_mapped = true;
3119         userptr->dir = dir;
3120
3121 already_pinned:
3122         parser->patched_cb_size +=
3123                         goya_get_dma_desc_list_size(hdev, userptr->sgt);
3124
3125         return 0;
3126
3127 unpin_memory:
3128         hl_unpin_host_memory(hdev, userptr);
3129 free_userptr:
3130         kfree(userptr);
3131         return rc;
3132 }
3133
3134 static int goya_validate_dma_pkt_host(struct hl_device *hdev,
3135                                 struct hl_cs_parser *parser,
3136                                 struct packet_lin_dma *user_dma_pkt)
3137 {
3138         u64 device_memory_addr, addr;
3139         enum dma_data_direction dir;
3140         enum goya_dma_direction user_dir;
3141         bool sram_addr = true;
3142         bool skip_host_mem_pin = false;
3143         bool user_memset;
3144         u32 ctl;
3145         int rc = 0;
3146
3147         ctl = le32_to_cpu(user_dma_pkt->ctl);
3148
3149         user_dir = (ctl & GOYA_PKT_LIN_DMA_CTL_DMA_DIR_MASK) >>
3150                         GOYA_PKT_LIN_DMA_CTL_DMA_DIR_SHIFT;
3151
3152         user_memset = (ctl & GOYA_PKT_LIN_DMA_CTL_MEMSET_MASK) >>
3153                         GOYA_PKT_LIN_DMA_CTL_MEMSET_SHIFT;
3154
3155         switch (user_dir) {
3156         case DMA_HOST_TO_DRAM:
3157                 dev_dbg(hdev->dev, "DMA direction is HOST --> DRAM\n");
3158                 dir = DMA_TO_DEVICE;
3159                 sram_addr = false;
3160                 addr = le64_to_cpu(user_dma_pkt->src_addr);
3161                 device_memory_addr = le64_to_cpu(user_dma_pkt->dst_addr);
3162                 if (user_memset)
3163                         skip_host_mem_pin = true;
3164                 break;
3165
3166         case DMA_DRAM_TO_HOST:
3167                 dev_dbg(hdev->dev, "DMA direction is DRAM --> HOST\n");
3168                 dir = DMA_FROM_DEVICE;
3169                 sram_addr = false;
3170                 addr = le64_to_cpu(user_dma_pkt->dst_addr);
3171                 device_memory_addr = le64_to_cpu(user_dma_pkt->src_addr);
3172                 break;
3173
3174         case DMA_HOST_TO_SRAM:
3175                 dev_dbg(hdev->dev, "DMA direction is HOST --> SRAM\n");
3176                 dir = DMA_TO_DEVICE;
3177                 addr = le64_to_cpu(user_dma_pkt->src_addr);
3178                 device_memory_addr = le64_to_cpu(user_dma_pkt->dst_addr);
3179                 if (user_memset)
3180                         skip_host_mem_pin = true;
3181                 break;
3182
3183         case DMA_SRAM_TO_HOST:
3184                 dev_dbg(hdev->dev, "DMA direction is SRAM --> HOST\n");
3185                 dir = DMA_FROM_DEVICE;
3186                 addr = le64_to_cpu(user_dma_pkt->dst_addr);
3187                 device_memory_addr = le64_to_cpu(user_dma_pkt->src_addr);
3188                 break;
3189         default:
3190                 dev_err(hdev->dev, "DMA direction is undefined\n");
3191                 return -EFAULT;
3192         }
3193
3194         if (sram_addr) {
3195                 if (!hl_mem_area_inside_range(device_memory_addr,
3196                                 le32_to_cpu(user_dma_pkt->tsize),
3197                                 hdev->asic_prop.sram_user_base_address,
3198                                 hdev->asic_prop.sram_end_address)) {
3199
3200                         dev_err(hdev->dev,
3201                                 "SRAM address 0x%llx + 0x%x is invalid\n",
3202                                 device_memory_addr,
3203                                 user_dma_pkt->tsize);
3204                         return -EFAULT;
3205                 }
3206         } else {
3207                 if (!hl_mem_area_inside_range(device_memory_addr,
3208                                 le32_to_cpu(user_dma_pkt->tsize),
3209                                 hdev->asic_prop.dram_user_base_address,
3210                                 hdev->asic_prop.dram_end_address)) {
3211
3212                         dev_err(hdev->dev,
3213                                 "DRAM address 0x%llx + 0x%x is invalid\n",
3214                                 device_memory_addr,
3215                                 user_dma_pkt->tsize);
3216                         return -EFAULT;
3217                 }
3218         }
3219
3220         if (skip_host_mem_pin)
3221                 parser->patched_cb_size += sizeof(*user_dma_pkt);
3222         else {
3223                 if ((dir == DMA_TO_DEVICE) &&
3224                                 (parser->hw_queue_id > GOYA_QUEUE_ID_DMA_1)) {
3225                         dev_err(hdev->dev,
3226                                 "Can't DMA from host on queue other then 1\n");
3227                         return -EFAULT;
3228                 }
3229
3230                 rc = goya_pin_memory_before_cs(hdev, parser, user_dma_pkt,
3231                                                 addr, dir);
3232         }
3233
3234         return rc;
3235 }
3236
3237 static int goya_validate_dma_pkt_no_host(struct hl_device *hdev,
3238                                 struct hl_cs_parser *parser,
3239                                 struct packet_lin_dma *user_dma_pkt)
3240 {
3241         u64 sram_memory_addr, dram_memory_addr;
3242         enum goya_dma_direction user_dir;
3243         u32 ctl;
3244
3245         ctl = le32_to_cpu(user_dma_pkt->ctl);
3246         user_dir = (ctl & GOYA_PKT_LIN_DMA_CTL_DMA_DIR_MASK) >>
3247                         GOYA_PKT_LIN_DMA_CTL_DMA_DIR_SHIFT;
3248
3249         if (user_dir == DMA_DRAM_TO_SRAM) {
3250                 dev_dbg(hdev->dev, "DMA direction is DRAM --> SRAM\n");
3251                 dram_memory_addr = le64_to_cpu(user_dma_pkt->src_addr);
3252                 sram_memory_addr = le64_to_cpu(user_dma_pkt->dst_addr);
3253         } else {
3254                 dev_dbg(hdev->dev, "DMA direction is SRAM --> DRAM\n");
3255                 sram_memory_addr = le64_to_cpu(user_dma_pkt->src_addr);
3256                 dram_memory_addr = le64_to_cpu(user_dma_pkt->dst_addr);
3257         }
3258
3259         if (!hl_mem_area_inside_range(sram_memory_addr,
3260                                 le32_to_cpu(user_dma_pkt->tsize),
3261                                 hdev->asic_prop.sram_user_base_address,
3262                                 hdev->asic_prop.sram_end_address)) {
3263                 dev_err(hdev->dev, "SRAM address 0x%llx + 0x%x is invalid\n",
3264                         sram_memory_addr, user_dma_pkt->tsize);
3265                 return -EFAULT;
3266         }
3267
3268         if (!hl_mem_area_inside_range(dram_memory_addr,
3269                                 le32_to_cpu(user_dma_pkt->tsize),
3270                                 hdev->asic_prop.dram_user_base_address,
3271                                 hdev->asic_prop.dram_end_address)) {
3272                 dev_err(hdev->dev, "DRAM address 0x%llx + 0x%x is invalid\n",
3273                         dram_memory_addr, user_dma_pkt->tsize);
3274                 return -EFAULT;
3275         }
3276
3277         parser->patched_cb_size += sizeof(*user_dma_pkt);
3278
3279         return 0;
3280 }
3281
3282 static int goya_validate_dma_pkt_no_mmu(struct hl_device *hdev,
3283                                 struct hl_cs_parser *parser,
3284                                 struct packet_lin_dma *user_dma_pkt)
3285 {
3286         enum goya_dma_direction user_dir;
3287         u32 ctl;
3288         int rc;
3289
3290         dev_dbg(hdev->dev, "DMA packet details:\n");
3291         dev_dbg(hdev->dev, "source == 0x%llx\n", user_dma_pkt->src_addr);
3292         dev_dbg(hdev->dev, "destination == 0x%llx\n", user_dma_pkt->dst_addr);
3293         dev_dbg(hdev->dev, "size == %u\n", user_dma_pkt->tsize);
3294
3295         ctl = le32_to_cpu(user_dma_pkt->ctl);
3296         user_dir = (ctl & GOYA_PKT_LIN_DMA_CTL_DMA_DIR_MASK) >>
3297                         GOYA_PKT_LIN_DMA_CTL_DMA_DIR_SHIFT;
3298
3299         /*
3300          * Special handling for DMA with size 0. The H/W has a bug where
3301          * this can cause the QMAN DMA to get stuck, so block it here.
3302          */
3303         if (user_dma_pkt->tsize == 0) {
3304                 dev_err(hdev->dev,
3305                         "Got DMA with size 0, might reset the device\n");
3306                 return -EINVAL;
3307         }
3308
3309         if ((user_dir == DMA_DRAM_TO_SRAM) || (user_dir == DMA_SRAM_TO_DRAM))
3310                 rc = goya_validate_dma_pkt_no_host(hdev, parser, user_dma_pkt);
3311         else
3312                 rc = goya_validate_dma_pkt_host(hdev, parser, user_dma_pkt);
3313
3314         return rc;
3315 }
3316
3317 static int goya_validate_dma_pkt_mmu(struct hl_device *hdev,
3318                                 struct hl_cs_parser *parser,
3319                                 struct packet_lin_dma *user_dma_pkt)
3320 {
3321         dev_dbg(hdev->dev, "DMA packet details:\n");
3322         dev_dbg(hdev->dev, "source == 0x%llx\n", user_dma_pkt->src_addr);
3323         dev_dbg(hdev->dev, "destination == 0x%llx\n", user_dma_pkt->dst_addr);
3324         dev_dbg(hdev->dev, "size == %u\n", user_dma_pkt->tsize);
3325
3326         /*
3327          * WA for HW-23.
3328          * We can't allow user to read from Host using QMANs other than 1.
3329          */
3330         if (parser->hw_queue_id != GOYA_QUEUE_ID_DMA_1 &&
3331                 hl_mem_area_inside_range(le64_to_cpu(user_dma_pkt->src_addr),
3332                                 le32_to_cpu(user_dma_pkt->tsize),
3333                                 hdev->asic_prop.va_space_host_start_address,
3334                                 hdev->asic_prop.va_space_host_end_address)) {
3335                 dev_err(hdev->dev,
3336                         "Can't DMA from host on queue other then 1\n");
3337                 return -EFAULT;
3338         }
3339
3340         if (user_dma_pkt->tsize == 0) {
3341                 dev_err(hdev->dev,
3342                         "Got DMA with size 0, might reset the device\n");
3343                 return -EINVAL;
3344         }
3345
3346         parser->patched_cb_size += sizeof(*user_dma_pkt);
3347
3348         return 0;
3349 }
3350
3351 static int goya_validate_wreg32(struct hl_device *hdev,
3352                                 struct hl_cs_parser *parser,
3353                                 struct packet_wreg32 *wreg_pkt)
3354 {
3355         struct goya_device *goya = hdev->asic_specific;
3356         u32 sob_start_addr, sob_end_addr;
3357         u16 reg_offset;
3358
3359         reg_offset = le32_to_cpu(wreg_pkt->ctl) &
3360                         GOYA_PKT_WREG32_CTL_REG_OFFSET_MASK;
3361
3362         dev_dbg(hdev->dev, "WREG32 packet details:\n");
3363         dev_dbg(hdev->dev, "reg_offset == 0x%x\n", reg_offset);
3364         dev_dbg(hdev->dev, "value      == 0x%x\n", wreg_pkt->value);
3365
3366         if (reg_offset != (mmDMA_CH_0_WR_COMP_ADDR_LO & 0x1FFF)) {
3367                 dev_err(hdev->dev, "WREG32 packet with illegal address 0x%x\n",
3368                         reg_offset);
3369                 return -EPERM;
3370         }
3371
3372         /*
3373          * With MMU, DMA channels are not secured, so it doesn't matter where
3374          * the WR COMP will be written to because it will go out with
3375          * non-secured property
3376          */
3377         if (goya->hw_cap_initialized & HW_CAP_MMU)
3378                 return 0;
3379
3380         sob_start_addr = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
3381         sob_end_addr = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_1023);
3382
3383         if ((le32_to_cpu(wreg_pkt->value) < sob_start_addr) ||
3384                         (le32_to_cpu(wreg_pkt->value) > sob_end_addr)) {
3385
3386                 dev_err(hdev->dev, "WREG32 packet with illegal value 0x%x\n",
3387                         wreg_pkt->value);
3388                 return -EPERM;
3389         }
3390
3391         return 0;
3392 }
3393
3394 static int goya_validate_cb(struct hl_device *hdev,
3395                         struct hl_cs_parser *parser, bool is_mmu)
3396 {
3397         u32 cb_parsed_length = 0;
3398         int rc = 0;
3399
3400         parser->patched_cb_size = 0;
3401
3402         /* cb_user_size is more than 0 so loop will always be executed */
3403         while (cb_parsed_length < parser->user_cb_size) {
3404                 enum packet_id pkt_id;
3405                 u16 pkt_size;
3406                 void *user_pkt;
3407
3408                 user_pkt = (void *) (uintptr_t)
3409                         (parser->user_cb->kernel_address + cb_parsed_length);
3410
3411                 pkt_id = (enum packet_id) (((*(u64 *) user_pkt) &
3412                                 PACKET_HEADER_PACKET_ID_MASK) >>
3413                                         PACKET_HEADER_PACKET_ID_SHIFT);
3414
3415                 pkt_size = goya_packet_sizes[pkt_id];
3416                 cb_parsed_length += pkt_size;
3417                 if (cb_parsed_length > parser->user_cb_size) {
3418                         dev_err(hdev->dev,
3419                                 "packet 0x%x is out of CB boundary\n", pkt_id);
3420                         rc = -EINVAL;
3421                         break;
3422                 }
3423
3424                 switch (pkt_id) {
3425                 case PACKET_WREG_32:
3426                         /*
3427                          * Although it is validated after copy in patch_cb(),
3428                          * need to validate here as well because patch_cb() is
3429                          * not called in MMU path while this function is called
3430                          */
3431                         rc = goya_validate_wreg32(hdev, parser, user_pkt);
3432                         break;
3433
3434                 case PACKET_WREG_BULK:
3435                         dev_err(hdev->dev,
3436                                 "User not allowed to use WREG_BULK\n");
3437                         rc = -EPERM;
3438                         break;
3439
3440                 case PACKET_MSG_PROT:
3441                         dev_err(hdev->dev,
3442                                 "User not allowed to use MSG_PROT\n");
3443                         rc = -EPERM;
3444                         break;
3445
3446                 case PACKET_CP_DMA:
3447                         dev_err(hdev->dev, "User not allowed to use CP_DMA\n");
3448                         rc = -EPERM;
3449                         break;
3450
3451                 case PACKET_STOP:
3452                         dev_err(hdev->dev, "User not allowed to use STOP\n");
3453                         rc = -EPERM;
3454                         break;
3455
3456                 case PACKET_LIN_DMA:
3457                         if (is_mmu)
3458                                 rc = goya_validate_dma_pkt_mmu(hdev, parser,
3459                                                 user_pkt);
3460                         else
3461                                 rc = goya_validate_dma_pkt_no_mmu(hdev, parser,
3462                                                 user_pkt);
3463                         break;
3464
3465                 case PACKET_MSG_LONG:
3466                 case PACKET_MSG_SHORT:
3467                 case PACKET_FENCE:
3468                 case PACKET_NOP:
3469                         parser->patched_cb_size += pkt_size;
3470                         break;
3471
3472                 default:
3473                         dev_err(hdev->dev, "Invalid packet header 0x%x\n",
3474                                 pkt_id);
3475                         rc = -EINVAL;
3476                         break;
3477                 }
3478
3479                 if (rc)
3480                         break;
3481         }
3482
3483         /*
3484          * The new CB should have space at the end for two MSG_PROT packets:
3485          * 1. A packet that will act as a completion packet
3486          * 2. A packet that will generate MSI-X interrupt
3487          */
3488         parser->patched_cb_size += sizeof(struct packet_msg_prot) * 2;
3489
3490         return rc;
3491 }
3492
3493 static int goya_patch_dma_packet(struct hl_device *hdev,
3494                                 struct hl_cs_parser *parser,
3495                                 struct packet_lin_dma *user_dma_pkt,
3496                                 struct packet_lin_dma *new_dma_pkt,
3497                                 u32 *new_dma_pkt_size)
3498 {
3499         struct hl_userptr *userptr;
3500         struct scatterlist *sg, *sg_next_iter;
3501         u32 count, dma_desc_cnt;
3502         u64 len, len_next;
3503         dma_addr_t dma_addr, dma_addr_next;
3504         enum goya_dma_direction user_dir;
3505         u64 device_memory_addr, addr;
3506         enum dma_data_direction dir;
3507         struct sg_table *sgt;
3508         bool skip_host_mem_pin = false;
3509         bool user_memset;
3510         u32 user_rdcomp_mask, user_wrcomp_mask, ctl;
3511
3512         ctl = le32_to_cpu(user_dma_pkt->ctl);
3513
3514         user_dir = (ctl & GOYA_PKT_LIN_DMA_CTL_DMA_DIR_MASK) >>
3515                         GOYA_PKT_LIN_DMA_CTL_DMA_DIR_SHIFT;
3516
3517         user_memset = (ctl & GOYA_PKT_LIN_DMA_CTL_MEMSET_MASK) >>
3518                         GOYA_PKT_LIN_DMA_CTL_MEMSET_SHIFT;
3519
3520         if ((user_dir == DMA_DRAM_TO_SRAM) || (user_dir == DMA_SRAM_TO_DRAM) ||
3521                         (user_dma_pkt->tsize == 0)) {
3522                 memcpy(new_dma_pkt, user_dma_pkt, sizeof(*new_dma_pkt));
3523                 *new_dma_pkt_size = sizeof(*new_dma_pkt);
3524                 return 0;
3525         }
3526
3527         if ((user_dir == DMA_HOST_TO_DRAM) || (user_dir == DMA_HOST_TO_SRAM)) {
3528                 addr = le64_to_cpu(user_dma_pkt->src_addr);
3529                 device_memory_addr = le64_to_cpu(user_dma_pkt->dst_addr);
3530                 dir = DMA_TO_DEVICE;
3531                 if (user_memset)
3532                         skip_host_mem_pin = true;
3533         } else {
3534                 addr = le64_to_cpu(user_dma_pkt->dst_addr);
3535                 device_memory_addr = le64_to_cpu(user_dma_pkt->src_addr);
3536                 dir = DMA_FROM_DEVICE;
3537         }
3538
3539         if ((!skip_host_mem_pin) &&
3540                 (hl_userptr_is_pinned(hdev, addr,
3541                         le32_to_cpu(user_dma_pkt->tsize),
3542                         parser->job_userptr_list, &userptr) == false)) {
3543                 dev_err(hdev->dev, "Userptr 0x%llx + 0x%x NOT mapped\n",
3544                                 addr, user_dma_pkt->tsize);
3545                 return -EFAULT;
3546         }
3547
3548         if ((user_memset) && (dir == DMA_TO_DEVICE)) {
3549                 memcpy(new_dma_pkt, user_dma_pkt, sizeof(*user_dma_pkt));
3550                 *new_dma_pkt_size = sizeof(*user_dma_pkt);
3551                 return 0;
3552         }
3553
3554         user_rdcomp_mask = ctl & GOYA_PKT_LIN_DMA_CTL_RDCOMP_MASK;
3555
3556         user_wrcomp_mask = ctl & GOYA_PKT_LIN_DMA_CTL_WRCOMP_MASK;
3557
3558         sgt = userptr->sgt;
3559         dma_desc_cnt = 0;
3560
3561         for_each_sg(sgt->sgl, sg, sgt->nents, count) {
3562                 len = sg_dma_len(sg);
3563                 dma_addr = sg_dma_address(sg);
3564
3565                 if (len == 0)
3566                         break;
3567
3568                 while ((count + 1) < sgt->nents) {
3569                         sg_next_iter = sg_next(sg);
3570                         len_next = sg_dma_len(sg_next_iter);
3571                         dma_addr_next = sg_dma_address(sg_next_iter);
3572
3573                         if (len_next == 0)
3574                                 break;
3575
3576                         if ((dma_addr + len == dma_addr_next) &&
3577                                 (len + len_next <= DMA_MAX_TRANSFER_SIZE)) {
3578                                 len += len_next;
3579                                 count++;
3580                                 sg = sg_next_iter;
3581                         } else {
3582                                 break;
3583                         }
3584                 }
3585
3586                 ctl = le32_to_cpu(user_dma_pkt->ctl);
3587                 if (likely(dma_desc_cnt))
3588                         ctl &= ~GOYA_PKT_CTL_EB_MASK;
3589                 ctl &= ~(GOYA_PKT_LIN_DMA_CTL_RDCOMP_MASK |
3590                                 GOYA_PKT_LIN_DMA_CTL_WRCOMP_MASK);
3591                 new_dma_pkt->ctl = cpu_to_le32(ctl);
3592                 new_dma_pkt->tsize = cpu_to_le32((u32) len);
3593
3594                 if (dir == DMA_TO_DEVICE) {
3595                         new_dma_pkt->src_addr = cpu_to_le64(dma_addr);
3596                         new_dma_pkt->dst_addr = cpu_to_le64(device_memory_addr);
3597                 } else {
3598                         new_dma_pkt->src_addr = cpu_to_le64(device_memory_addr);
3599                         new_dma_pkt->dst_addr = cpu_to_le64(dma_addr);
3600                 }
3601
3602                 if (!user_memset)
3603                         device_memory_addr += len;
3604                 dma_desc_cnt++;
3605                 new_dma_pkt++;
3606         }
3607
3608         if (!dma_desc_cnt) {
3609                 dev_err(hdev->dev,
3610                         "Error of 0 SG entries when patching DMA packet\n");
3611                 return -EFAULT;
3612         }
3613
3614         /* Fix the last dma packet - rdcomp/wrcomp must be as user set them */
3615         new_dma_pkt--;
3616         new_dma_pkt->ctl |= cpu_to_le32(user_rdcomp_mask | user_wrcomp_mask);
3617
3618         *new_dma_pkt_size = dma_desc_cnt * sizeof(struct packet_lin_dma);
3619
3620         return 0;
3621 }
3622
3623 static int goya_patch_cb(struct hl_device *hdev,
3624                                 struct hl_cs_parser *parser)
3625 {
3626         u32 cb_parsed_length = 0;
3627         u32 cb_patched_cur_length = 0;
3628         int rc = 0;
3629
3630         /* cb_user_size is more than 0 so loop will always be executed */
3631         while (cb_parsed_length < parser->user_cb_size) {
3632                 enum packet_id pkt_id;
3633                 u16 pkt_size;
3634                 u32 new_pkt_size = 0;
3635                 void *user_pkt, *kernel_pkt;
3636
3637                 user_pkt = (void *) (uintptr_t)
3638                         (parser->user_cb->kernel_address + cb_parsed_length);
3639                 kernel_pkt = (void *) (uintptr_t)
3640                         (parser->patched_cb->kernel_address +
3641                                         cb_patched_cur_length);
3642
3643                 pkt_id = (enum packet_id) (((*(u64 *) user_pkt) &
3644                                 PACKET_HEADER_PACKET_ID_MASK) >>
3645                                         PACKET_HEADER_PACKET_ID_SHIFT);
3646
3647                 pkt_size = goya_packet_sizes[pkt_id];
3648                 cb_parsed_length += pkt_size;
3649                 if (cb_parsed_length > parser->user_cb_size) {
3650                         dev_err(hdev->dev,
3651                                 "packet 0x%x is out of CB boundary\n", pkt_id);
3652                         rc = -EINVAL;
3653                         break;
3654                 }
3655
3656                 switch (pkt_id) {
3657                 case PACKET_LIN_DMA:
3658                         rc = goya_patch_dma_packet(hdev, parser, user_pkt,
3659                                                 kernel_pkt, &new_pkt_size);
3660                         cb_patched_cur_length += new_pkt_size;
3661                         break;
3662
3663                 case PACKET_WREG_32:
3664                         memcpy(kernel_pkt, user_pkt, pkt_size);
3665                         cb_patched_cur_length += pkt_size;
3666                         rc = goya_validate_wreg32(hdev, parser, kernel_pkt);
3667                         break;
3668
3669                 case PACKET_WREG_BULK:
3670                         dev_err(hdev->dev,
3671                                 "User not allowed to use WREG_BULK\n");
3672                         rc = -EPERM;
3673                         break;
3674
3675                 case PACKET_MSG_PROT:
3676                         dev_err(hdev->dev,
3677                                 "User not allowed to use MSG_PROT\n");
3678                         rc = -EPERM;
3679                         break;
3680
3681                 case PACKET_CP_DMA:
3682                         dev_err(hdev->dev, "User not allowed to use CP_DMA\n");
3683                         rc = -EPERM;
3684                         break;
3685
3686                 case PACKET_STOP:
3687                         dev_err(hdev->dev, "User not allowed to use STOP\n");
3688                         rc = -EPERM;
3689                         break;
3690
3691                 case PACKET_MSG_LONG:
3692                 case PACKET_MSG_SHORT:
3693                 case PACKET_FENCE:
3694                 case PACKET_NOP:
3695                         memcpy(kernel_pkt, user_pkt, pkt_size);
3696                         cb_patched_cur_length += pkt_size;
3697                         break;
3698
3699                 default:
3700                         dev_err(hdev->dev, "Invalid packet header 0x%x\n",
3701                                 pkt_id);
3702                         rc = -EINVAL;
3703                         break;
3704                 }
3705
3706                 if (rc)
3707                         break;
3708         }
3709
3710         return rc;
3711 }
3712
3713 static int goya_parse_cb_mmu(struct hl_device *hdev,
3714                 struct hl_cs_parser *parser)
3715 {
3716         u64 patched_cb_handle;
3717         u32 patched_cb_size;
3718         struct hl_cb *user_cb;
3719         int rc;
3720
3721         /*
3722          * The new CB should have space at the end for two MSG_PROT pkt:
3723          * 1. A packet that will act as a completion packet
3724          * 2. A packet that will generate MSI-X interrupt
3725          */
3726         parser->patched_cb_size = parser->user_cb_size +
3727                         sizeof(struct packet_msg_prot) * 2;
3728
3729         rc = hl_cb_create(hdev, &hdev->kernel_cb_mgr,
3730                                 parser->patched_cb_size,
3731                                 &patched_cb_handle, HL_KERNEL_ASID_ID);
3732
3733         if (rc) {
3734                 dev_err(hdev->dev,
3735                         "Failed to allocate patched CB for DMA CS %d\n",
3736                         rc);
3737                 return rc;
3738         }
3739
3740         patched_cb_handle >>= PAGE_SHIFT;
3741         parser->patched_cb = hl_cb_get(hdev, &hdev->kernel_cb_mgr,
3742                                 (u32) patched_cb_handle);
3743         /* hl_cb_get should never fail here so use kernel WARN */
3744         WARN(!parser->patched_cb, "DMA CB handle invalid 0x%x\n",
3745                         (u32) patched_cb_handle);
3746         if (!parser->patched_cb) {
3747                 rc = -EFAULT;
3748                 goto out;
3749         }
3750
3751         /*
3752          * The check that parser->user_cb_size <= parser->user_cb->size was done
3753          * in validate_queue_index().
3754          */
3755         memcpy((void *) (uintptr_t) parser->patched_cb->kernel_address,
3756                 (void *) (uintptr_t) parser->user_cb->kernel_address,
3757                 parser->user_cb_size);
3758
3759         patched_cb_size = parser->patched_cb_size;
3760
3761         /* validate patched CB instead of user CB */
3762         user_cb = parser->user_cb;
3763         parser->user_cb = parser->patched_cb;
3764         rc = goya_validate_cb(hdev, parser, true);
3765         parser->user_cb = user_cb;
3766
3767         if (rc) {
3768                 hl_cb_put(parser->patched_cb);
3769                 goto out;
3770         }
3771
3772         if (patched_cb_size != parser->patched_cb_size) {
3773                 dev_err(hdev->dev, "user CB size mismatch\n");
3774                 hl_cb_put(parser->patched_cb);
3775                 rc = -EINVAL;
3776                 goto out;
3777         }
3778
3779 out:
3780         /*
3781          * Always call cb destroy here because we still have 1 reference
3782          * to it by calling cb_get earlier. After the job will be completed,
3783          * cb_put will release it, but here we want to remove it from the
3784          * idr
3785          */
3786         hl_cb_destroy(hdev, &hdev->kernel_cb_mgr,
3787                                         patched_cb_handle << PAGE_SHIFT);
3788
3789         return rc;
3790 }
3791
3792 static int goya_parse_cb_no_mmu(struct hl_device *hdev,
3793                                 struct hl_cs_parser *parser)
3794 {
3795         u64 patched_cb_handle;
3796         int rc;
3797
3798         rc = goya_validate_cb(hdev, parser, false);
3799
3800         if (rc)
3801                 goto free_userptr;
3802
3803         rc = hl_cb_create(hdev, &hdev->kernel_cb_mgr,
3804                                 parser->patched_cb_size,
3805                                 &patched_cb_handle, HL_KERNEL_ASID_ID);
3806         if (rc) {
3807                 dev_err(hdev->dev,
3808                         "Failed to allocate patched CB for DMA CS %d\n", rc);
3809                 goto free_userptr;
3810         }
3811
3812         patched_cb_handle >>= PAGE_SHIFT;
3813         parser->patched_cb = hl_cb_get(hdev, &hdev->kernel_cb_mgr,
3814                                 (u32) patched_cb_handle);
3815         /* hl_cb_get should never fail here so use kernel WARN */
3816         WARN(!parser->patched_cb, "DMA CB handle invalid 0x%x\n",
3817                         (u32) patched_cb_handle);
3818         if (!parser->patched_cb) {
3819                 rc = -EFAULT;
3820                 goto out;
3821         }
3822
3823         rc = goya_patch_cb(hdev, parser);
3824
3825         if (rc)
3826                 hl_cb_put(parser->patched_cb);
3827
3828 out:
3829         /*
3830          * Always call cb destroy here because we still have 1 reference
3831          * to it by calling cb_get earlier. After the job will be completed,
3832          * cb_put will release it, but here we want to remove it from the
3833          * idr
3834          */
3835         hl_cb_destroy(hdev, &hdev->kernel_cb_mgr,
3836                                 patched_cb_handle << PAGE_SHIFT);
3837
3838 free_userptr:
3839         if (rc)
3840                 hl_userptr_delete_list(hdev, parser->job_userptr_list);
3841         return rc;
3842 }
3843
3844 static int goya_parse_cb_no_ext_queue(struct hl_device *hdev,
3845                                         struct hl_cs_parser *parser)
3846 {
3847         struct asic_fixed_properties *asic_prop = &hdev->asic_prop;
3848         struct goya_device *goya = hdev->asic_specific;
3849
3850         if (goya->hw_cap_initialized & HW_CAP_MMU)
3851                 return 0;
3852
3853         /* For internal queue jobs, just check if CB address is valid */
3854         if (hl_mem_area_inside_range(
3855                         (u64) (uintptr_t) parser->user_cb,
3856                         parser->user_cb_size,
3857                         asic_prop->sram_user_base_address,
3858                         asic_prop->sram_end_address))
3859                 return 0;
3860
3861         if (hl_mem_area_inside_range(
3862                         (u64) (uintptr_t) parser->user_cb,
3863                         parser->user_cb_size,
3864                         asic_prop->dram_user_base_address,
3865                         asic_prop->dram_end_address))
3866                 return 0;
3867
3868         dev_err(hdev->dev,
3869                 "Internal CB address %px + 0x%x is not in SRAM nor in DRAM\n",
3870                 parser->user_cb, parser->user_cb_size);
3871
3872         return -EFAULT;
3873 }
3874
3875 int goya_cs_parser(struct hl_device *hdev, struct hl_cs_parser *parser)
3876 {
3877         struct goya_device *goya = hdev->asic_specific;
3878
3879         if (!parser->ext_queue)
3880                 return goya_parse_cb_no_ext_queue(hdev, parser);
3881
3882         if (goya->hw_cap_initialized & HW_CAP_MMU)
3883                 return goya_parse_cb_mmu(hdev, parser);
3884         else
3885                 return goya_parse_cb_no_mmu(hdev, parser);
3886 }
3887
3888 void goya_add_end_of_cb_packets(struct hl_device *hdev, u64 kernel_address,
3889                                 u32 len, u64 cq_addr, u32 cq_val, u32 msix_vec)
3890 {
3891         struct packet_msg_prot *cq_pkt;
3892         u32 tmp;
3893
3894         cq_pkt = (struct packet_msg_prot *) (uintptr_t)
3895                 (kernel_address + len - (sizeof(struct packet_msg_prot) * 2));
3896
3897         tmp = (PACKET_MSG_PROT << GOYA_PKT_CTL_OPCODE_SHIFT) |
3898                         (1 << GOYA_PKT_CTL_EB_SHIFT) |
3899                         (1 << GOYA_PKT_CTL_MB_SHIFT);
3900         cq_pkt->ctl = cpu_to_le32(tmp);
3901         cq_pkt->value = cpu_to_le32(cq_val);
3902         cq_pkt->addr = cpu_to_le64(cq_addr);
3903
3904         cq_pkt++;
3905
3906         tmp = (PACKET_MSG_PROT << GOYA_PKT_CTL_OPCODE_SHIFT) |
3907                         (1 << GOYA_PKT_CTL_MB_SHIFT);
3908         cq_pkt->ctl = cpu_to_le32(tmp);
3909         cq_pkt->value = cpu_to_le32(msix_vec & 0x7FF);
3910         cq_pkt->addr = cpu_to_le64(CFG_BASE + mmPCIE_DBI_MSIX_DOORBELL_OFF);
3911 }
3912
3913 void goya_update_eq_ci(struct hl_device *hdev, u32 val)
3914 {
3915         WREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_6, val);
3916 }
3917
3918 void goya_restore_phase_topology(struct hl_device *hdev)
3919 {
3920
3921 }
3922
3923 static void goya_clear_sm_regs(struct hl_device *hdev)
3924 {
3925         int i, num_of_sob_in_longs, num_of_mon_in_longs;
3926
3927         num_of_sob_in_longs =
3928                 ((mmSYNC_MNGR_SOB_OBJ_1023 - mmSYNC_MNGR_SOB_OBJ_0) + 4);
3929
3930         num_of_mon_in_longs =
3931                 ((mmSYNC_MNGR_MON_STATUS_255 - mmSYNC_MNGR_MON_STATUS_0) + 4);
3932
3933         for (i = 0 ; i < num_of_sob_in_longs ; i += 4)
3934                 WREG32(mmSYNC_MNGR_SOB_OBJ_0 + i, 0);
3935
3936         for (i = 0 ; i < num_of_mon_in_longs ; i += 4)
3937                 WREG32(mmSYNC_MNGR_MON_STATUS_0 + i, 0);
3938
3939         /* Flush all WREG to prevent race */
3940         i = RREG32(mmSYNC_MNGR_SOB_OBJ_0);
3941 }
3942
3943 /*
3944  * goya_debugfs_read32 - read a 32bit value from a given device address
3945  *
3946  * @hdev:       pointer to hl_device structure
3947  * @addr:       address in device
3948  * @val:        returned value
3949  *
3950  * In case of DDR address that is not mapped into the default aperture that
3951  * the DDR bar exposes, the function will configure the iATU so that the DDR
3952  * bar will be positioned at a base address that allows reading from the
3953  * required address. Configuring the iATU during normal operation can
3954  * lead to undefined behavior and therefore, should be done with extreme care
3955  *
3956  */
3957 static int goya_debugfs_read32(struct hl_device *hdev, u64 addr, u32 *val)
3958 {
3959         struct asic_fixed_properties *prop = &hdev->asic_prop;
3960         u64 ddr_bar_addr;
3961         int rc = 0;
3962
3963         if ((addr >= CFG_BASE) && (addr < CFG_BASE + CFG_SIZE)) {
3964                 *val = RREG32(addr - CFG_BASE);
3965
3966         } else if ((addr >= SRAM_BASE_ADDR) &&
3967                         (addr < SRAM_BASE_ADDR + SRAM_SIZE)) {
3968
3969                 *val = readl(hdev->pcie_bar[SRAM_CFG_BAR_ID] +
3970                                 (addr - SRAM_BASE_ADDR));
3971
3972         } else if ((addr >= DRAM_PHYS_BASE) &&
3973                         (addr < DRAM_PHYS_BASE + hdev->asic_prop.dram_size)) {
3974
3975                 u64 bar_base_addr = DRAM_PHYS_BASE +
3976                                 (addr & ~(prop->dram_pci_bar_size - 0x1ull));
3977
3978                 ddr_bar_addr = goya_set_ddr_bar_base(hdev, bar_base_addr);
3979                 if (ddr_bar_addr != U64_MAX) {
3980                         *val = readl(hdev->pcie_bar[DDR_BAR_ID] +
3981                                                 (addr - bar_base_addr));
3982
3983                         ddr_bar_addr = goya_set_ddr_bar_base(hdev,
3984                                                         ddr_bar_addr);
3985                 }
3986                 if (ddr_bar_addr == U64_MAX)
3987                         rc = -EIO;
3988         } else {
3989                 rc = -EFAULT;
3990         }
3991
3992         return rc;
3993 }
3994
3995 /*
3996  * goya_debugfs_write32 - write a 32bit value to a given device address
3997  *
3998  * @hdev:       pointer to hl_device structure
3999  * @addr:       address in device
4000  * @val:        returned value
4001  *
4002  * In case of DDR address that is not mapped into the default aperture that
4003  * the DDR bar exposes, the function will configure the iATU so that the DDR
4004  * bar will be positioned at a base address that allows writing to the
4005  * required address. Configuring the iATU during normal operation can
4006  * lead to undefined behavior and therefore, should be done with extreme care
4007  *
4008  */
4009 static int goya_debugfs_write32(struct hl_device *hdev, u64 addr, u32 val)
4010 {
4011         struct asic_fixed_properties *prop = &hdev->asic_prop;
4012         u64 ddr_bar_addr;
4013         int rc = 0;
4014
4015         if ((addr >= CFG_BASE) && (addr < CFG_BASE + CFG_SIZE)) {
4016                 WREG32(addr - CFG_BASE, val);
4017
4018         } else if ((addr >= SRAM_BASE_ADDR) &&
4019                         (addr < SRAM_BASE_ADDR + SRAM_SIZE)) {
4020
4021                 writel(val, hdev->pcie_bar[SRAM_CFG_BAR_ID] +
4022                                         (addr - SRAM_BASE_ADDR));
4023
4024         } else if ((addr >= DRAM_PHYS_BASE) &&
4025                         (addr < DRAM_PHYS_BASE + hdev->asic_prop.dram_size)) {
4026
4027                 u64 bar_base_addr = DRAM_PHYS_BASE +
4028                                 (addr & ~(prop->dram_pci_bar_size - 0x1ull));
4029
4030                 ddr_bar_addr = goya_set_ddr_bar_base(hdev, bar_base_addr);
4031                 if (ddr_bar_addr != U64_MAX) {
4032                         writel(val, hdev->pcie_bar[DDR_BAR_ID] +
4033                                                 (addr - bar_base_addr));
4034
4035                         ddr_bar_addr = goya_set_ddr_bar_base(hdev,
4036                                                         ddr_bar_addr);
4037                 }
4038                 if (ddr_bar_addr == U64_MAX)
4039                         rc = -EIO;
4040         } else {
4041                 rc = -EFAULT;
4042         }
4043
4044         return rc;
4045 }
4046
4047 static u64 goya_read_pte(struct hl_device *hdev, u64 addr)
4048 {
4049         struct goya_device *goya = hdev->asic_specific;
4050
4051         if (hdev->hard_reset_pending)
4052                 return U64_MAX;
4053
4054         return readq(hdev->pcie_bar[DDR_BAR_ID] +
4055                         (addr - goya->ddr_bar_cur_addr));
4056 }
4057
4058 static void goya_write_pte(struct hl_device *hdev, u64 addr, u64 val)
4059 {
4060         struct goya_device *goya = hdev->asic_specific;
4061
4062         if (hdev->hard_reset_pending)
4063                 return;
4064
4065         writeq(val, hdev->pcie_bar[DDR_BAR_ID] +
4066                         (addr - goya->ddr_bar_cur_addr));
4067 }
4068
4069 static const char *_goya_get_event_desc(u16 event_type)
4070 {
4071         switch (event_type) {
4072         case GOYA_ASYNC_EVENT_ID_PCIE_IF:
4073                 return "PCIe_if";
4074         case GOYA_ASYNC_EVENT_ID_TPC0_ECC:
4075         case GOYA_ASYNC_EVENT_ID_TPC1_ECC:
4076         case GOYA_ASYNC_EVENT_ID_TPC2_ECC:
4077         case GOYA_ASYNC_EVENT_ID_TPC3_ECC:
4078         case GOYA_ASYNC_EVENT_ID_TPC4_ECC:
4079         case GOYA_ASYNC_EVENT_ID_TPC5_ECC:
4080         case GOYA_ASYNC_EVENT_ID_TPC6_ECC:
4081         case GOYA_ASYNC_EVENT_ID_TPC7_ECC:
4082                 return "TPC%d_ecc";
4083         case GOYA_ASYNC_EVENT_ID_MME_ECC:
4084                 return "MME_ecc";
4085         case GOYA_ASYNC_EVENT_ID_MME_ECC_EXT:
4086                 return "MME_ecc_ext";
4087         case GOYA_ASYNC_EVENT_ID_MMU_ECC:
4088                 return "MMU_ecc";
4089         case GOYA_ASYNC_EVENT_ID_DMA_MACRO:
4090                 return "DMA_macro";
4091         case GOYA_ASYNC_EVENT_ID_DMA_ECC:
4092                 return "DMA_ecc";
4093         case GOYA_ASYNC_EVENT_ID_CPU_IF_ECC:
4094                 return "CPU_if_ecc";
4095         case GOYA_ASYNC_EVENT_ID_PSOC_MEM:
4096                 return "PSOC_mem";
4097         case GOYA_ASYNC_EVENT_ID_PSOC_CORESIGHT:
4098                 return "PSOC_coresight";
4099         case GOYA_ASYNC_EVENT_ID_SRAM0 ... GOYA_ASYNC_EVENT_ID_SRAM29:
4100                 return "SRAM%d";
4101         case GOYA_ASYNC_EVENT_ID_GIC500:
4102                 return "GIC500";
4103         case GOYA_ASYNC_EVENT_ID_PLL0 ... GOYA_ASYNC_EVENT_ID_PLL6:
4104                 return "PLL%d";
4105         case GOYA_ASYNC_EVENT_ID_AXI_ECC:
4106                 return "AXI_ecc";
4107         case GOYA_ASYNC_EVENT_ID_L2_RAM_ECC:
4108                 return "L2_ram_ecc";
4109         case GOYA_ASYNC_EVENT_ID_PSOC_GPIO_05_SW_RESET:
4110                 return "PSOC_gpio_05_sw_reset";
4111         case GOYA_ASYNC_EVENT_ID_PSOC_GPIO_10_VRHOT_ICRIT:
4112                 return "PSOC_gpio_10_vrhot_icrit";
4113         case GOYA_ASYNC_EVENT_ID_PCIE_DEC:
4114                 return "PCIe_dec";
4115         case GOYA_ASYNC_EVENT_ID_TPC0_DEC:
4116         case GOYA_ASYNC_EVENT_ID_TPC1_DEC:
4117         case GOYA_ASYNC_EVENT_ID_TPC2_DEC:
4118         case GOYA_ASYNC_EVENT_ID_TPC3_DEC:
4119         case GOYA_ASYNC_EVENT_ID_TPC4_DEC:
4120         case GOYA_ASYNC_EVENT_ID_TPC5_DEC:
4121         case GOYA_ASYNC_EVENT_ID_TPC6_DEC:
4122         case GOYA_ASYNC_EVENT_ID_TPC7_DEC:
4123                 return "TPC%d_dec";
4124         case GOYA_ASYNC_EVENT_ID_MME_WACS:
4125                 return "MME_wacs";
4126         case GOYA_ASYNC_EVENT_ID_MME_WACSD:
4127                 return "MME_wacsd";
4128         case GOYA_ASYNC_EVENT_ID_CPU_AXI_SPLITTER:
4129                 return "CPU_axi_splitter";
4130         case GOYA_ASYNC_EVENT_ID_PSOC_AXI_DEC:
4131                 return "PSOC_axi_dec";
4132         case GOYA_ASYNC_EVENT_ID_PSOC:
4133                 return "PSOC";
4134         case GOYA_ASYNC_EVENT_ID_TPC0_KRN_ERR:
4135         case GOYA_ASYNC_EVENT_ID_TPC1_KRN_ERR:
4136         case GOYA_ASYNC_EVENT_ID_TPC2_KRN_ERR:
4137         case GOYA_ASYNC_EVENT_ID_TPC3_KRN_ERR:
4138         case GOYA_ASYNC_EVENT_ID_TPC4_KRN_ERR:
4139         case GOYA_ASYNC_EVENT_ID_TPC5_KRN_ERR:
4140         case GOYA_ASYNC_EVENT_ID_TPC6_KRN_ERR:
4141         case GOYA_ASYNC_EVENT_ID_TPC7_KRN_ERR:
4142                 return "TPC%d_krn_err";
4143         case GOYA_ASYNC_EVENT_ID_TPC0_CMDQ ... GOYA_ASYNC_EVENT_ID_TPC7_CMDQ:
4144                 return "TPC%d_cq";
4145         case GOYA_ASYNC_EVENT_ID_TPC0_QM ... GOYA_ASYNC_EVENT_ID_TPC7_QM:
4146                 return "TPC%d_qm";
4147         case GOYA_ASYNC_EVENT_ID_MME_QM:
4148                 return "MME_qm";
4149         case GOYA_ASYNC_EVENT_ID_MME_CMDQ:
4150                 return "MME_cq";
4151         case GOYA_ASYNC_EVENT_ID_DMA0_QM ... GOYA_ASYNC_EVENT_ID_DMA4_QM:
4152                 return "DMA%d_qm";
4153         case GOYA_ASYNC_EVENT_ID_DMA0_CH ... GOYA_ASYNC_EVENT_ID_DMA4_CH:
4154                 return "DMA%d_ch";
4155         case GOYA_ASYNC_EVENT_ID_TPC0_BMON_SPMU:
4156         case GOYA_ASYNC_EVENT_ID_TPC1_BMON_SPMU:
4157         case GOYA_ASYNC_EVENT_ID_TPC2_BMON_SPMU:
4158         case GOYA_ASYNC_EVENT_ID_TPC3_BMON_SPMU:
4159         case GOYA_ASYNC_EVENT_ID_TPC4_BMON_SPMU:
4160         case GOYA_ASYNC_EVENT_ID_TPC5_BMON_SPMU:
4161         case GOYA_ASYNC_EVENT_ID_TPC6_BMON_SPMU:
4162         case GOYA_ASYNC_EVENT_ID_TPC7_BMON_SPMU:
4163                 return "TPC%d_bmon_spmu";
4164         case GOYA_ASYNC_EVENT_ID_DMA_BM_CH0 ... GOYA_ASYNC_EVENT_ID_DMA_BM_CH4:
4165                 return "DMA_bm_ch%d";
4166         default:
4167                 return "N/A";
4168         }
4169 }
4170
4171 static void goya_get_event_desc(u16 event_type, char *desc, size_t size)
4172 {
4173         u8 index;
4174
4175         switch (event_type) {
4176         case GOYA_ASYNC_EVENT_ID_TPC0_ECC:
4177         case GOYA_ASYNC_EVENT_ID_TPC1_ECC:
4178         case GOYA_ASYNC_EVENT_ID_TPC2_ECC:
4179         case GOYA_ASYNC_EVENT_ID_TPC3_ECC:
4180         case GOYA_ASYNC_EVENT_ID_TPC4_ECC:
4181         case GOYA_ASYNC_EVENT_ID_TPC5_ECC:
4182         case GOYA_ASYNC_EVENT_ID_TPC6_ECC:
4183         case GOYA_ASYNC_EVENT_ID_TPC7_ECC:
4184                 index = (event_type - GOYA_ASYNC_EVENT_ID_TPC0_ECC) / 3;
4185                 snprintf(desc, size, _goya_get_event_desc(event_type), index);
4186                 break;
4187         case GOYA_ASYNC_EVENT_ID_SRAM0 ... GOYA_ASYNC_EVENT_ID_SRAM29:
4188                 index = event_type - GOYA_ASYNC_EVENT_ID_SRAM0;
4189                 snprintf(desc, size, _goya_get_event_desc(event_type), index);
4190                 break;
4191         case GOYA_ASYNC_EVENT_ID_PLL0 ... GOYA_ASYNC_EVENT_ID_PLL6:
4192                 index = event_type - GOYA_ASYNC_EVENT_ID_PLL0;
4193                 snprintf(desc, size, _goya_get_event_desc(event_type), index);
4194                 break;
4195         case GOYA_ASYNC_EVENT_ID_TPC0_DEC:
4196         case GOYA_ASYNC_EVENT_ID_TPC1_DEC:
4197         case GOYA_ASYNC_EVENT_ID_TPC2_DEC:
4198         case GOYA_ASYNC_EVENT_ID_TPC3_DEC:
4199         case GOYA_ASYNC_EVENT_ID_TPC4_DEC:
4200         case GOYA_ASYNC_EVENT_ID_TPC5_DEC:
4201         case GOYA_ASYNC_EVENT_ID_TPC6_DEC:
4202         case GOYA_ASYNC_EVENT_ID_TPC7_DEC:
4203                 index = (event_type - GOYA_ASYNC_EVENT_ID_TPC0_DEC) / 3;
4204                 snprintf(desc, size, _goya_get_event_desc(event_type), index);
4205                 break;
4206         case GOYA_ASYNC_EVENT_ID_TPC0_KRN_ERR:
4207         case GOYA_ASYNC_EVENT_ID_TPC1_KRN_ERR:
4208         case GOYA_ASYNC_EVENT_ID_TPC2_KRN_ERR:
4209         case GOYA_ASYNC_EVENT_ID_TPC3_KRN_ERR:
4210         case GOYA_ASYNC_EVENT_ID_TPC4_KRN_ERR:
4211         case GOYA_ASYNC_EVENT_ID_TPC5_KRN_ERR:
4212         case GOYA_ASYNC_EVENT_ID_TPC6_KRN_ERR:
4213         case GOYA_ASYNC_EVENT_ID_TPC7_KRN_ERR:
4214                 index = (event_type - GOYA_ASYNC_EVENT_ID_TPC0_KRN_ERR) / 10;
4215                 snprintf(desc, size, _goya_get_event_desc(event_type), index);
4216                 break;
4217         case GOYA_ASYNC_EVENT_ID_TPC0_CMDQ ... GOYA_ASYNC_EVENT_ID_TPC7_CMDQ:
4218                 index = event_type - GOYA_ASYNC_EVENT_ID_TPC0_CMDQ;
4219                 snprintf(desc, size, _goya_get_event_desc(event_type), index);
4220                 break;
4221         case GOYA_ASYNC_EVENT_ID_TPC0_QM ... GOYA_ASYNC_EVENT_ID_TPC7_QM:
4222                 index = event_type - GOYA_ASYNC_EVENT_ID_TPC0_QM;
4223                 snprintf(desc, size, _goya_get_event_desc(event_type), index);
4224                 break;
4225         case GOYA_ASYNC_EVENT_ID_DMA0_QM ... GOYA_ASYNC_EVENT_ID_DMA4_QM:
4226                 index = event_type - GOYA_ASYNC_EVENT_ID_DMA0_QM;
4227                 snprintf(desc, size, _goya_get_event_desc(event_type), index);
4228                 break;
4229         case GOYA_ASYNC_EVENT_ID_DMA0_CH ... GOYA_ASYNC_EVENT_ID_DMA4_CH:
4230                 index = event_type - GOYA_ASYNC_EVENT_ID_DMA0_CH;
4231                 snprintf(desc, size, _goya_get_event_desc(event_type), index);
4232                 break;
4233         case GOYA_ASYNC_EVENT_ID_TPC0_BMON_SPMU:
4234         case GOYA_ASYNC_EVENT_ID_TPC1_BMON_SPMU:
4235         case GOYA_ASYNC_EVENT_ID_TPC2_BMON_SPMU:
4236         case GOYA_ASYNC_EVENT_ID_TPC3_BMON_SPMU:
4237         case GOYA_ASYNC_EVENT_ID_TPC4_BMON_SPMU:
4238         case GOYA_ASYNC_EVENT_ID_TPC5_BMON_SPMU:
4239         case GOYA_ASYNC_EVENT_ID_TPC6_BMON_SPMU:
4240         case GOYA_ASYNC_EVENT_ID_TPC7_BMON_SPMU:
4241                 index = (event_type - GOYA_ASYNC_EVENT_ID_TPC0_BMON_SPMU) / 10;
4242                 snprintf(desc, size, _goya_get_event_desc(event_type), index);
4243                 break;
4244         case GOYA_ASYNC_EVENT_ID_DMA_BM_CH0 ... GOYA_ASYNC_EVENT_ID_DMA_BM_CH4:
4245                 index = event_type - GOYA_ASYNC_EVENT_ID_DMA_BM_CH0;
4246                 snprintf(desc, size, _goya_get_event_desc(event_type), index);
4247                 break;
4248         default:
4249                 snprintf(desc, size, _goya_get_event_desc(event_type));
4250                 break;
4251         }
4252 }
4253
4254 static void goya_print_razwi_info(struct hl_device *hdev)
4255 {
4256         if (RREG32(mmDMA_MACRO_RAZWI_LBW_WT_VLD)) {
4257                 dev_err(hdev->dev, "Illegal write to LBW\n");
4258                 WREG32(mmDMA_MACRO_RAZWI_LBW_WT_VLD, 0);
4259         }
4260
4261         if (RREG32(mmDMA_MACRO_RAZWI_LBW_RD_VLD)) {
4262                 dev_err(hdev->dev, "Illegal read from LBW\n");
4263                 WREG32(mmDMA_MACRO_RAZWI_LBW_RD_VLD, 0);
4264         }
4265
4266         if (RREG32(mmDMA_MACRO_RAZWI_HBW_WT_VLD)) {
4267                 dev_err(hdev->dev, "Illegal write to HBW\n");
4268                 WREG32(mmDMA_MACRO_RAZWI_HBW_WT_VLD, 0);
4269         }
4270
4271         if (RREG32(mmDMA_MACRO_RAZWI_HBW_RD_VLD)) {
4272                 dev_err(hdev->dev, "Illegal read from HBW\n");
4273                 WREG32(mmDMA_MACRO_RAZWI_HBW_RD_VLD, 0);
4274         }
4275 }
4276
4277 static void goya_print_mmu_error_info(struct hl_device *hdev)
4278 {
4279         struct goya_device *goya = hdev->asic_specific;
4280         u64 addr;
4281         u32 val;
4282
4283         if (!(goya->hw_cap_initialized & HW_CAP_MMU))
4284                 return;
4285
4286         val = RREG32(mmMMU_PAGE_ERROR_CAPTURE);
4287         if (val & MMU_PAGE_ERROR_CAPTURE_ENTRY_VALID_MASK) {
4288                 addr = val & MMU_PAGE_ERROR_CAPTURE_VA_49_32_MASK;
4289                 addr <<= 32;
4290                 addr |= RREG32(mmMMU_PAGE_ERROR_CAPTURE_VA);
4291
4292                 dev_err(hdev->dev, "MMU page fault on va 0x%llx\n", addr);
4293
4294                 WREG32(mmMMU_PAGE_ERROR_CAPTURE, 0);
4295         }
4296 }
4297
4298 static void goya_print_irq_info(struct hl_device *hdev, u16 event_type,
4299                                 bool razwi)
4300 {
4301         char desc[20] = "";
4302
4303         goya_get_event_desc(event_type, desc, sizeof(desc));
4304         dev_err(hdev->dev, "Received H/W interrupt %d [\"%s\"]\n",
4305                 event_type, desc);
4306
4307         if (razwi) {
4308                 goya_print_razwi_info(hdev);
4309                 goya_print_mmu_error_info(hdev);
4310         }
4311 }
4312
4313 static int goya_unmask_irq_arr(struct hl_device *hdev, u32 *irq_arr,
4314                 size_t irq_arr_size)
4315 {
4316         struct armcp_unmask_irq_arr_packet *pkt;
4317         size_t total_pkt_size;
4318         long result;
4319         int rc;
4320
4321         total_pkt_size = sizeof(struct armcp_unmask_irq_arr_packet) +
4322                         irq_arr_size;
4323
4324         /* data should be aligned to 8 bytes in order to ArmCP to copy it */
4325         total_pkt_size = (total_pkt_size + 0x7) & ~0x7;
4326
4327         /* total_pkt_size is casted to u16 later on */
4328         if (total_pkt_size > USHRT_MAX) {
4329                 dev_err(hdev->dev, "too many elements in IRQ array\n");
4330                 return -EINVAL;
4331         }
4332
4333         pkt = kzalloc(total_pkt_size, GFP_KERNEL);
4334         if (!pkt)
4335                 return -ENOMEM;
4336
4337         pkt->length = cpu_to_le32(irq_arr_size / sizeof(irq_arr[0]));
4338         memcpy(&pkt->irqs, irq_arr, irq_arr_size);
4339
4340         pkt->armcp_pkt.ctl = cpu_to_le32(ARMCP_PACKET_UNMASK_RAZWI_IRQ_ARRAY <<
4341                                                 ARMCP_PKT_CTL_OPCODE_SHIFT);
4342
4343         rc = goya_send_cpu_message(hdev, (u32 *) pkt, total_pkt_size,
4344                         HL_DEVICE_TIMEOUT_USEC, &result);
4345
4346         if (rc)
4347                 dev_err(hdev->dev, "failed to unmask IRQ array\n");
4348
4349         kfree(pkt);
4350
4351         return rc;
4352 }
4353
4354 static int goya_soft_reset_late_init(struct hl_device *hdev)
4355 {
4356         /*
4357          * Unmask all IRQs since some could have been received
4358          * during the soft reset
4359          */
4360         return goya_unmask_irq_arr(hdev, goya_all_events,
4361                                         sizeof(goya_all_events));
4362 }
4363
4364 static int goya_unmask_irq(struct hl_device *hdev, u16 event_type)
4365 {
4366         struct armcp_packet pkt;
4367         long result;
4368         int rc;
4369
4370         memset(&pkt, 0, sizeof(pkt));
4371
4372         pkt.ctl = cpu_to_le32(ARMCP_PACKET_UNMASK_RAZWI_IRQ <<
4373                                 ARMCP_PKT_CTL_OPCODE_SHIFT);
4374         pkt.value = cpu_to_le64(event_type);
4375
4376         rc = goya_send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
4377                         HL_DEVICE_TIMEOUT_USEC, &result);
4378
4379         if (rc)
4380                 dev_err(hdev->dev, "failed to unmask RAZWI IRQ %d", event_type);
4381
4382         return rc;
4383 }
4384
4385 void goya_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_entry)
4386 {
4387         u32 ctl = le32_to_cpu(eq_entry->hdr.ctl);
4388         u16 event_type = ((ctl & EQ_CTL_EVENT_TYPE_MASK)
4389                                 >> EQ_CTL_EVENT_TYPE_SHIFT);
4390         struct goya_device *goya = hdev->asic_specific;
4391
4392         goya->events_stat[event_type]++;
4393
4394         switch (event_type) {
4395         case GOYA_ASYNC_EVENT_ID_PCIE_IF:
4396         case GOYA_ASYNC_EVENT_ID_TPC0_ECC:
4397         case GOYA_ASYNC_EVENT_ID_TPC1_ECC:
4398         case GOYA_ASYNC_EVENT_ID_TPC2_ECC:
4399         case GOYA_ASYNC_EVENT_ID_TPC3_ECC:
4400         case GOYA_ASYNC_EVENT_ID_TPC4_ECC:
4401         case GOYA_ASYNC_EVENT_ID_TPC5_ECC:
4402         case GOYA_ASYNC_EVENT_ID_TPC6_ECC:
4403         case GOYA_ASYNC_EVENT_ID_TPC7_ECC:
4404         case GOYA_ASYNC_EVENT_ID_MME_ECC:
4405         case GOYA_ASYNC_EVENT_ID_MME_ECC_EXT:
4406         case GOYA_ASYNC_EVENT_ID_MMU_ECC:
4407         case GOYA_ASYNC_EVENT_ID_DMA_MACRO:
4408         case GOYA_ASYNC_EVENT_ID_DMA_ECC:
4409         case GOYA_ASYNC_EVENT_ID_CPU_IF_ECC:
4410         case GOYA_ASYNC_EVENT_ID_PSOC_MEM:
4411         case GOYA_ASYNC_EVENT_ID_PSOC_CORESIGHT:
4412         case GOYA_ASYNC_EVENT_ID_SRAM0 ... GOYA_ASYNC_EVENT_ID_SRAM29:
4413         case GOYA_ASYNC_EVENT_ID_GIC500:
4414         case GOYA_ASYNC_EVENT_ID_PLL0 ... GOYA_ASYNC_EVENT_ID_PLL6:
4415         case GOYA_ASYNC_EVENT_ID_AXI_ECC:
4416         case GOYA_ASYNC_EVENT_ID_L2_RAM_ECC:
4417         case GOYA_ASYNC_EVENT_ID_PSOC_GPIO_05_SW_RESET:
4418         case GOYA_ASYNC_EVENT_ID_PSOC_GPIO_10_VRHOT_ICRIT:
4419                 goya_print_irq_info(hdev, event_type, false);
4420                 hl_device_reset(hdev, true, false);
4421                 break;
4422
4423         case GOYA_ASYNC_EVENT_ID_PCIE_DEC:
4424         case GOYA_ASYNC_EVENT_ID_TPC0_DEC:
4425         case GOYA_ASYNC_EVENT_ID_TPC1_DEC:
4426         case GOYA_ASYNC_EVENT_ID_TPC2_DEC:
4427         case GOYA_ASYNC_EVENT_ID_TPC3_DEC:
4428         case GOYA_ASYNC_EVENT_ID_TPC4_DEC:
4429         case GOYA_ASYNC_EVENT_ID_TPC5_DEC:
4430         case GOYA_ASYNC_EVENT_ID_TPC6_DEC:
4431         case GOYA_ASYNC_EVENT_ID_TPC7_DEC:
4432         case GOYA_ASYNC_EVENT_ID_MME_WACS:
4433         case GOYA_ASYNC_EVENT_ID_MME_WACSD:
4434         case GOYA_ASYNC_EVENT_ID_CPU_AXI_SPLITTER:
4435         case GOYA_ASYNC_EVENT_ID_PSOC_AXI_DEC:
4436         case GOYA_ASYNC_EVENT_ID_PSOC:
4437         case GOYA_ASYNC_EVENT_ID_TPC0_KRN_ERR:
4438         case GOYA_ASYNC_EVENT_ID_TPC1_KRN_ERR:
4439         case GOYA_ASYNC_EVENT_ID_TPC2_KRN_ERR:
4440         case GOYA_ASYNC_EVENT_ID_TPC3_KRN_ERR:
4441         case GOYA_ASYNC_EVENT_ID_TPC4_KRN_ERR:
4442         case GOYA_ASYNC_EVENT_ID_TPC5_KRN_ERR:
4443         case GOYA_ASYNC_EVENT_ID_TPC6_KRN_ERR:
4444         case GOYA_ASYNC_EVENT_ID_TPC7_KRN_ERR:
4445         case GOYA_ASYNC_EVENT_ID_TPC0_CMDQ ... GOYA_ASYNC_EVENT_ID_TPC7_QM:
4446         case GOYA_ASYNC_EVENT_ID_MME_QM:
4447         case GOYA_ASYNC_EVENT_ID_MME_CMDQ:
4448         case GOYA_ASYNC_EVENT_ID_DMA0_QM ... GOYA_ASYNC_EVENT_ID_DMA4_QM:
4449         case GOYA_ASYNC_EVENT_ID_DMA0_CH ... GOYA_ASYNC_EVENT_ID_DMA4_CH:
4450                 goya_print_irq_info(hdev, event_type, true);
4451                 goya_unmask_irq(hdev, event_type);
4452                 break;
4453
4454         case GOYA_ASYNC_EVENT_ID_TPC0_BMON_SPMU:
4455         case GOYA_ASYNC_EVENT_ID_TPC1_BMON_SPMU:
4456         case GOYA_ASYNC_EVENT_ID_TPC2_BMON_SPMU:
4457         case GOYA_ASYNC_EVENT_ID_TPC3_BMON_SPMU:
4458         case GOYA_ASYNC_EVENT_ID_TPC4_BMON_SPMU:
4459         case GOYA_ASYNC_EVENT_ID_TPC5_BMON_SPMU:
4460         case GOYA_ASYNC_EVENT_ID_TPC6_BMON_SPMU:
4461         case GOYA_ASYNC_EVENT_ID_TPC7_BMON_SPMU:
4462         case GOYA_ASYNC_EVENT_ID_DMA_BM_CH0 ... GOYA_ASYNC_EVENT_ID_DMA_BM_CH4:
4463                 goya_print_irq_info(hdev, event_type, false);
4464                 goya_unmask_irq(hdev, event_type);
4465                 break;
4466
4467         default:
4468                 dev_err(hdev->dev, "Received invalid H/W interrupt %d\n",
4469                                 event_type);
4470                 break;
4471         }
4472 }
4473
4474 void *goya_get_events_stat(struct hl_device *hdev, u32 *size)
4475 {
4476         struct goya_device *goya = hdev->asic_specific;
4477
4478         *size = (u32) sizeof(goya->events_stat);
4479
4480         return goya->events_stat;
4481 }
4482
4483 static int goya_memset_device_memory(struct hl_device *hdev, u64 addr, u64 size,
4484                                 u64 val, bool is_dram)
4485 {
4486         struct packet_lin_dma *lin_dma_pkt;
4487         struct hl_cs_job *job;
4488         u32 cb_size, ctl;
4489         struct hl_cb *cb;
4490         int rc, lin_dma_pkts_cnt;
4491
4492         lin_dma_pkts_cnt = DIV_ROUND_UP_ULL(size, SZ_2G);
4493         cb_size = lin_dma_pkts_cnt * sizeof(struct packet_lin_dma) +
4494                                                 sizeof(struct packet_msg_prot);
4495         cb = hl_cb_kernel_create(hdev, cb_size);
4496         if (!cb)
4497                 return -ENOMEM;
4498
4499         lin_dma_pkt = (struct packet_lin_dma *) (uintptr_t) cb->kernel_address;
4500
4501         do {
4502                 memset(lin_dma_pkt, 0, sizeof(*lin_dma_pkt));
4503
4504                 ctl = ((PACKET_LIN_DMA << GOYA_PKT_CTL_OPCODE_SHIFT) |
4505                                 (1 << GOYA_PKT_LIN_DMA_CTL_MEMSET_SHIFT) |
4506                                 (1 << GOYA_PKT_LIN_DMA_CTL_WO_SHIFT) |
4507                                 (1 << GOYA_PKT_CTL_RB_SHIFT) |
4508                                 (1 << GOYA_PKT_CTL_MB_SHIFT));
4509                 ctl |= (is_dram ? DMA_HOST_TO_DRAM : DMA_HOST_TO_SRAM) <<
4510                                 GOYA_PKT_LIN_DMA_CTL_DMA_DIR_SHIFT;
4511                 lin_dma_pkt->ctl = cpu_to_le32(ctl);
4512
4513                 lin_dma_pkt->src_addr = cpu_to_le64(val);
4514                 lin_dma_pkt->dst_addr = cpu_to_le64(addr);
4515                 if (lin_dma_pkts_cnt > 1)
4516                         lin_dma_pkt->tsize = cpu_to_le32(SZ_2G);
4517                 else
4518                         lin_dma_pkt->tsize = cpu_to_le32(size);
4519
4520                 size -= SZ_2G;
4521                 addr += SZ_2G;
4522                 lin_dma_pkt++;
4523         } while (--lin_dma_pkts_cnt);
4524
4525         job = hl_cs_allocate_job(hdev, true);
4526         if (!job) {
4527                 dev_err(hdev->dev, "Failed to allocate a new job\n");
4528                 rc = -ENOMEM;
4529                 goto release_cb;
4530         }
4531
4532         job->id = 0;
4533         job->user_cb = cb;
4534         job->user_cb->cs_cnt++;
4535         job->user_cb_size = cb_size;
4536         job->hw_queue_id = GOYA_QUEUE_ID_DMA_0;
4537         job->patched_cb = job->user_cb;
4538         job->job_cb_size = job->user_cb_size;
4539
4540         hl_debugfs_add_job(hdev, job);
4541
4542         rc = goya_send_job_on_qman0(hdev, job);
4543
4544         hl_cb_put(job->patched_cb);
4545
4546         hl_debugfs_remove_job(hdev, job);
4547         kfree(job);
4548         cb->cs_cnt--;
4549
4550 release_cb:
4551         hl_cb_put(cb);
4552         hl_cb_destroy(hdev, &hdev->kernel_cb_mgr, cb->id << PAGE_SHIFT);
4553
4554         return rc;
4555 }
4556
4557 int goya_context_switch(struct hl_device *hdev, u32 asid)
4558 {
4559         struct asic_fixed_properties *prop = &hdev->asic_prop;
4560         u64 addr = prop->sram_base_address, sob_addr;
4561         u32 size = hdev->pldm ? 0x10000 : prop->sram_size;
4562         u64 val = 0x7777777777777777ull;
4563         int rc, dma_id;
4564         u32 channel_off = mmDMA_CH_1_WR_COMP_ADDR_LO -
4565                                         mmDMA_CH_0_WR_COMP_ADDR_LO;
4566
4567         rc = goya_memset_device_memory(hdev, addr, size, val, false);
4568         if (rc) {
4569                 dev_err(hdev->dev, "Failed to clear SRAM in context switch\n");
4570                 return rc;
4571         }
4572
4573         /* we need to reset registers that the user is allowed to change */
4574         sob_addr = CFG_BASE + mmSYNC_MNGR_SOB_OBJ_1007;
4575         WREG32(mmDMA_CH_0_WR_COMP_ADDR_LO, lower_32_bits(sob_addr));
4576
4577         for (dma_id = 1 ; dma_id < NUMBER_OF_EXT_HW_QUEUES ; dma_id++) {
4578                 sob_addr = CFG_BASE + mmSYNC_MNGR_SOB_OBJ_1000 +
4579                                                         (dma_id - 1) * 4;
4580                 WREG32(mmDMA_CH_0_WR_COMP_ADDR_LO + channel_off * dma_id,
4581                                                 lower_32_bits(sob_addr));
4582         }
4583
4584         WREG32(mmTPC_PLL_CLK_RLX_0, 0x200020);
4585
4586         goya_mmu_prepare(hdev, asid);
4587
4588         goya_clear_sm_regs(hdev);
4589
4590         return 0;
4591 }
4592
4593 static int goya_mmu_clear_pgt_range(struct hl_device *hdev)
4594 {
4595         struct asic_fixed_properties *prop = &hdev->asic_prop;
4596         struct goya_device *goya = hdev->asic_specific;
4597         u64 addr = prop->mmu_pgt_addr;
4598         u32 size = prop->mmu_pgt_size + MMU_DRAM_DEFAULT_PAGE_SIZE +
4599                         MMU_CACHE_MNG_SIZE;
4600
4601         if (!(goya->hw_cap_initialized & HW_CAP_MMU))
4602                 return 0;
4603
4604         return goya_memset_device_memory(hdev, addr, size, 0, true);
4605 }
4606
4607 static int goya_mmu_set_dram_default_page(struct hl_device *hdev)
4608 {
4609         struct goya_device *goya = hdev->asic_specific;
4610         u64 addr = hdev->asic_prop.mmu_dram_default_page_addr;
4611         u32 size = MMU_DRAM_DEFAULT_PAGE_SIZE;
4612         u64 val = 0x9999999999999999ull;
4613
4614         if (!(goya->hw_cap_initialized & HW_CAP_MMU))
4615                 return 0;
4616
4617         return goya_memset_device_memory(hdev, addr, size, val, true);
4618 }
4619
4620 static int goya_mmu_add_mappings_for_device_cpu(struct hl_device *hdev)
4621 {
4622         struct asic_fixed_properties *prop = &hdev->asic_prop;
4623         struct goya_device *goya = hdev->asic_specific;
4624         s64 off, cpu_off;
4625         int rc;
4626
4627         if (!(goya->hw_cap_initialized & HW_CAP_MMU))
4628                 return 0;
4629
4630         for (off = 0 ; off < CPU_FW_IMAGE_SIZE ; off += PAGE_SIZE_2MB) {
4631                 rc = hl_mmu_map(hdev->kernel_ctx, prop->dram_base_address + off,
4632                                 prop->dram_base_address + off, PAGE_SIZE_2MB);
4633                 if (rc) {
4634                         dev_err(hdev->dev, "Map failed for address 0x%llx\n",
4635                                 prop->dram_base_address + off);
4636                         goto unmap;
4637                 }
4638         }
4639
4640         if (!(hdev->cpu_accessible_dma_address & (PAGE_SIZE_2MB - 1))) {
4641                 rc = hl_mmu_map(hdev->kernel_ctx, VA_CPU_ACCESSIBLE_MEM_ADDR,
4642                         hdev->cpu_accessible_dma_address, PAGE_SIZE_2MB);
4643
4644                 if (rc) {
4645                         dev_err(hdev->dev,
4646                                 "Map failed for CPU accessible memory\n");
4647                         off -= PAGE_SIZE_2MB;
4648                         goto unmap;
4649                 }
4650         } else {
4651                 for (cpu_off = 0 ; cpu_off < SZ_2M ; cpu_off += PAGE_SIZE_4KB) {
4652                         rc = hl_mmu_map(hdev->kernel_ctx,
4653                                 VA_CPU_ACCESSIBLE_MEM_ADDR + cpu_off,
4654                                 hdev->cpu_accessible_dma_address + cpu_off,
4655                                 PAGE_SIZE_4KB);
4656                         if (rc) {
4657                                 dev_err(hdev->dev,
4658                                         "Map failed for CPU accessible memory\n");
4659                                 cpu_off -= PAGE_SIZE_4KB;
4660                                 goto unmap_cpu;
4661                         }
4662                 }
4663         }
4664
4665         goya_mmu_prepare_reg(hdev, mmCPU_IF_ARUSER_OVR, HL_KERNEL_ASID_ID);
4666         goya_mmu_prepare_reg(hdev, mmCPU_IF_AWUSER_OVR, HL_KERNEL_ASID_ID);
4667         WREG32(mmCPU_IF_ARUSER_OVR_EN, 0x7FF);
4668         WREG32(mmCPU_IF_AWUSER_OVR_EN, 0x7FF);
4669
4670         /* Make sure configuration is flushed to device */
4671         RREG32(mmCPU_IF_AWUSER_OVR_EN);
4672
4673         goya->device_cpu_mmu_mappings_done = true;
4674
4675         return 0;
4676
4677 unmap_cpu:
4678         for (; cpu_off >= 0 ; cpu_off -= PAGE_SIZE_4KB)
4679                 if (hl_mmu_unmap(hdev->kernel_ctx,
4680                                 VA_CPU_ACCESSIBLE_MEM_ADDR + cpu_off,
4681                                 PAGE_SIZE_4KB))
4682                         dev_warn_ratelimited(hdev->dev,
4683                                 "failed to unmap address 0x%llx\n",
4684                                 VA_CPU_ACCESSIBLE_MEM_ADDR + cpu_off);
4685 unmap:
4686         for (; off >= 0 ; off -= PAGE_SIZE_2MB)
4687                 if (hl_mmu_unmap(hdev->kernel_ctx,
4688                                 prop->dram_base_address + off, PAGE_SIZE_2MB))
4689                         dev_warn_ratelimited(hdev->dev,
4690                                 "failed to unmap address 0x%llx\n",
4691                                 prop->dram_base_address + off);
4692
4693         return rc;
4694 }
4695
4696 void goya_mmu_remove_device_cpu_mappings(struct hl_device *hdev)
4697 {
4698         struct asic_fixed_properties *prop = &hdev->asic_prop;
4699         struct goya_device *goya = hdev->asic_specific;
4700         u32 off, cpu_off;
4701
4702         if (!(goya->hw_cap_initialized & HW_CAP_MMU))
4703                 return;
4704
4705         if (!goya->device_cpu_mmu_mappings_done)
4706                 return;
4707
4708         WREG32(mmCPU_IF_ARUSER_OVR_EN, 0);
4709         WREG32(mmCPU_IF_AWUSER_OVR_EN, 0);
4710
4711         if (!(hdev->cpu_accessible_dma_address & (PAGE_SIZE_2MB - 1))) {
4712                 if (hl_mmu_unmap(hdev->kernel_ctx, VA_CPU_ACCESSIBLE_MEM_ADDR,
4713                                 PAGE_SIZE_2MB))
4714                         dev_warn(hdev->dev,
4715                                 "Failed to unmap CPU accessible memory\n");
4716         } else {
4717                 for (cpu_off = 0 ; cpu_off < SZ_2M ; cpu_off += PAGE_SIZE_4KB)
4718                         if (hl_mmu_unmap(hdev->kernel_ctx,
4719                                         VA_CPU_ACCESSIBLE_MEM_ADDR + cpu_off,
4720                                         PAGE_SIZE_4KB))
4721                                 dev_warn_ratelimited(hdev->dev,
4722                                         "failed to unmap address 0x%llx\n",
4723                                         VA_CPU_ACCESSIBLE_MEM_ADDR + cpu_off);
4724         }
4725
4726         for (off = 0 ; off < CPU_FW_IMAGE_SIZE ; off += PAGE_SIZE_2MB)
4727                 if (hl_mmu_unmap(hdev->kernel_ctx,
4728                                 prop->dram_base_address + off, PAGE_SIZE_2MB))
4729                         dev_warn_ratelimited(hdev->dev,
4730                                         "Failed to unmap address 0x%llx\n",
4731                                         prop->dram_base_address + off);
4732
4733         goya->device_cpu_mmu_mappings_done = false;
4734 }
4735
4736 static void goya_mmu_prepare(struct hl_device *hdev, u32 asid)
4737 {
4738         struct goya_device *goya = hdev->asic_specific;
4739         int i;
4740
4741         if (!(goya->hw_cap_initialized & HW_CAP_MMU))
4742                 return;
4743
4744         if (asid & ~MME_QM_GLBL_SECURE_PROPS_ASID_MASK) {
4745                 WARN(1, "asid %u is too big\n", asid);
4746                 return;
4747         }
4748
4749         /* zero the MMBP and ASID bits and then set the ASID */
4750         for (i = 0 ; i < GOYA_MMU_REGS_NUM ; i++)
4751                 goya_mmu_prepare_reg(hdev, goya_mmu_regs[i], asid);
4752 }
4753
4754 static void goya_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard)
4755 {
4756         struct goya_device *goya = hdev->asic_specific;
4757         u32 status, timeout_usec;
4758         int rc;
4759
4760         if (!(goya->hw_cap_initialized & HW_CAP_MMU))
4761                 return;
4762
4763         /* no need in L1 only invalidation in Goya */
4764         if (!is_hard)
4765                 return;
4766
4767         if (hdev->pldm)
4768                 timeout_usec = GOYA_PLDM_MMU_TIMEOUT_USEC;
4769         else
4770                 timeout_usec = MMU_CONFIG_TIMEOUT_USEC;
4771
4772         mutex_lock(&hdev->mmu_cache_lock);
4773
4774         /* L0 & L1 invalidation */
4775         WREG32(mmSTLB_INV_ALL_START, 1);
4776
4777         rc = hl_poll_timeout(
4778                 hdev,
4779                 mmSTLB_INV_ALL_START,
4780                 status,
4781                 !status,
4782                 1000,
4783                 timeout_usec);
4784
4785         mutex_unlock(&hdev->mmu_cache_lock);
4786
4787         if (rc)
4788                 dev_notice_ratelimited(hdev->dev,
4789                         "Timeout when waiting for MMU cache invalidation\n");
4790 }
4791
4792 static void goya_mmu_invalidate_cache_range(struct hl_device *hdev,
4793                 bool is_hard, u32 asid, u64 va, u64 size)
4794 {
4795         struct goya_device *goya = hdev->asic_specific;
4796         u32 status, timeout_usec, inv_data, pi;
4797         int rc;
4798
4799         if (!(goya->hw_cap_initialized & HW_CAP_MMU))
4800                 return;
4801
4802         /* no need in L1 only invalidation in Goya */
4803         if (!is_hard)
4804                 return;
4805
4806         if (hdev->pldm)
4807                 timeout_usec = GOYA_PLDM_MMU_TIMEOUT_USEC;
4808         else
4809                 timeout_usec = MMU_CONFIG_TIMEOUT_USEC;
4810
4811         mutex_lock(&hdev->mmu_cache_lock);
4812
4813         /*
4814          * TODO: currently invalidate entire L0 & L1 as in regular hard
4815          * invalidation. Need to apply invalidation of specific cache lines with
4816          * mask of ASID & VA & size.
4817          * Note that L1 with be flushed entirely in any case.
4818          */
4819
4820         /* L0 & L1 invalidation */
4821         inv_data = RREG32(mmSTLB_CACHE_INV);
4822         /* PI is 8 bit */
4823         pi = ((inv_data & STLB_CACHE_INV_PRODUCER_INDEX_MASK) + 1) & 0xFF;
4824         WREG32(mmSTLB_CACHE_INV,
4825                         (inv_data & STLB_CACHE_INV_INDEX_MASK_MASK) | pi);
4826
4827         rc = hl_poll_timeout(
4828                 hdev,
4829                 mmSTLB_INV_CONSUMER_INDEX,
4830                 status,
4831                 status == pi,
4832                 1000,
4833                 timeout_usec);
4834
4835         mutex_unlock(&hdev->mmu_cache_lock);
4836
4837         if (rc)
4838                 dev_notice_ratelimited(hdev->dev,
4839                         "Timeout when waiting for MMU cache invalidation\n");
4840 }
4841
4842 int goya_send_heartbeat(struct hl_device *hdev)
4843 {
4844         struct goya_device *goya = hdev->asic_specific;
4845
4846         if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q))
4847                 return 0;
4848
4849         return hl_fw_send_heartbeat(hdev);
4850 }
4851
4852 int goya_armcp_info_get(struct hl_device *hdev)
4853 {
4854         struct goya_device *goya = hdev->asic_specific;
4855         struct asic_fixed_properties *prop = &hdev->asic_prop;
4856         u64 dram_size;
4857         int rc;
4858
4859         if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q))
4860                 return 0;
4861
4862         rc = hl_fw_armcp_info_get(hdev);
4863         if (rc)
4864                 return rc;
4865
4866         dram_size = le64_to_cpu(prop->armcp_info.dram_size);
4867         if (dram_size) {
4868                 if ((!is_power_of_2(dram_size)) ||
4869                                 (dram_size < DRAM_PHYS_DEFAULT_SIZE)) {
4870                         dev_err(hdev->dev,
4871                                 "F/W reported invalid DRAM size %llu. Trying to use default size\n",
4872                                 dram_size);
4873                         dram_size = DRAM_PHYS_DEFAULT_SIZE;
4874                 }
4875
4876                 prop->dram_size = dram_size;
4877                 prop->dram_end_address = prop->dram_base_address + dram_size;
4878         }
4879
4880         return 0;
4881 }
4882
4883 static bool goya_is_device_idle(struct hl_device *hdev, char *buf, size_t size)
4884 {
4885         u64 offset, dma_qm_reg, tpc_qm_reg, tpc_cmdq_reg, tpc_cfg_reg;
4886         int i;
4887
4888         offset = mmDMA_QM_1_GLBL_STS0 - mmDMA_QM_0_GLBL_STS0;
4889
4890         for (i = 0 ; i < DMA_MAX_NUM ; i++) {
4891                 dma_qm_reg = mmDMA_QM_0_GLBL_STS0 + i * offset;
4892
4893                 if ((RREG32(dma_qm_reg) & DMA_QM_IDLE_MASK) !=
4894                                 DMA_QM_IDLE_MASK)
4895                         return HL_ENG_BUSY(buf, size, "DMA%d_QM", i);
4896         }
4897
4898         offset = mmTPC1_QM_GLBL_STS0 - mmTPC0_QM_GLBL_STS0;
4899
4900         for (i = 0 ; i < TPC_MAX_NUM ; i++) {
4901                 tpc_qm_reg = mmTPC0_QM_GLBL_STS0 + i * offset;
4902                 tpc_cmdq_reg = mmTPC0_CMDQ_GLBL_STS0 + i * offset;
4903                 tpc_cfg_reg = mmTPC0_CFG_STATUS + i * offset;
4904
4905                 if ((RREG32(tpc_qm_reg) & TPC_QM_IDLE_MASK) !=
4906                                 TPC_QM_IDLE_MASK)
4907                         return HL_ENG_BUSY(buf, size, "TPC%d_QM", i);
4908
4909                 if ((RREG32(tpc_cmdq_reg) & TPC_CMDQ_IDLE_MASK) !=
4910                                 TPC_CMDQ_IDLE_MASK)
4911                         return HL_ENG_BUSY(buf, size, "TPC%d_CMDQ", i);
4912
4913                 if ((RREG32(tpc_cfg_reg) & TPC_CFG_IDLE_MASK) !=
4914                                 TPC_CFG_IDLE_MASK)
4915                         return HL_ENG_BUSY(buf, size, "TPC%d_CFG", i);
4916         }
4917
4918         if ((RREG32(mmMME_QM_GLBL_STS0) & MME_QM_IDLE_MASK) !=
4919                         MME_QM_IDLE_MASK)
4920                 return HL_ENG_BUSY(buf, size, "MME_QM");
4921
4922         if ((RREG32(mmMME_CMDQ_GLBL_STS0) & MME_CMDQ_IDLE_MASK) !=
4923                         MME_CMDQ_IDLE_MASK)
4924                 return HL_ENG_BUSY(buf, size, "MME_CMDQ");
4925
4926         if ((RREG32(mmMME_ARCH_STATUS) & MME_ARCH_IDLE_MASK) !=
4927                         MME_ARCH_IDLE_MASK)
4928                 return HL_ENG_BUSY(buf, size, "MME_ARCH");
4929
4930         if (RREG32(mmMME_SHADOW_0_STATUS) & MME_SHADOW_IDLE_MASK)
4931                 return HL_ENG_BUSY(buf, size, "MME");
4932
4933         return true;
4934 }
4935
4936 static void goya_hw_queues_lock(struct hl_device *hdev)
4937 {
4938         struct goya_device *goya = hdev->asic_specific;
4939
4940         spin_lock(&goya->hw_queues_lock);
4941 }
4942
4943 static void goya_hw_queues_unlock(struct hl_device *hdev)
4944 {
4945         struct goya_device *goya = hdev->asic_specific;
4946
4947         spin_unlock(&goya->hw_queues_lock);
4948 }
4949
4950 static u32 goya_get_pci_id(struct hl_device *hdev)
4951 {
4952         return hdev->pdev->device;
4953 }
4954
4955 static int goya_get_eeprom_data(struct hl_device *hdev, void *data,
4956                                 size_t max_size)
4957 {
4958         struct goya_device *goya = hdev->asic_specific;
4959
4960         if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q))
4961                 return 0;
4962
4963         return hl_fw_get_eeprom_data(hdev, data, max_size);
4964 }
4965
4966 static enum hl_device_hw_state goya_get_hw_state(struct hl_device *hdev)
4967 {
4968         return RREG32(mmPSOC_GLOBAL_CONF_APP_STATUS);
4969 }
4970
4971 static const struct hl_asic_funcs goya_funcs = {
4972         .early_init = goya_early_init,
4973         .early_fini = goya_early_fini,
4974         .late_init = goya_late_init,
4975         .late_fini = goya_late_fini,
4976         .sw_init = goya_sw_init,
4977         .sw_fini = goya_sw_fini,
4978         .hw_init = goya_hw_init,
4979         .hw_fini = goya_hw_fini,
4980         .halt_engines = goya_halt_engines,
4981         .suspend = goya_suspend,
4982         .resume = goya_resume,
4983         .cb_mmap = goya_cb_mmap,
4984         .ring_doorbell = goya_ring_doorbell,
4985         .flush_pq_write = goya_flush_pq_write,
4986         .asic_dma_alloc_coherent = goya_dma_alloc_coherent,
4987         .asic_dma_free_coherent = goya_dma_free_coherent,
4988         .get_int_queue_base = goya_get_int_queue_base,
4989         .test_queues = goya_test_queues,
4990         .asic_dma_pool_zalloc = goya_dma_pool_zalloc,
4991         .asic_dma_pool_free = goya_dma_pool_free,
4992         .cpu_accessible_dma_pool_alloc = goya_cpu_accessible_dma_pool_alloc,
4993         .cpu_accessible_dma_pool_free = goya_cpu_accessible_dma_pool_free,
4994         .hl_dma_unmap_sg = goya_dma_unmap_sg,
4995         .cs_parser = goya_cs_parser,
4996         .asic_dma_map_sg = goya_dma_map_sg,
4997         .get_dma_desc_list_size = goya_get_dma_desc_list_size,
4998         .add_end_of_cb_packets = goya_add_end_of_cb_packets,
4999         .update_eq_ci = goya_update_eq_ci,
5000         .context_switch = goya_context_switch,
5001         .restore_phase_topology = goya_restore_phase_topology,
5002         .debugfs_read32 = goya_debugfs_read32,
5003         .debugfs_write32 = goya_debugfs_write32,
5004         .add_device_attr = goya_add_device_attr,
5005         .handle_eqe = goya_handle_eqe,
5006         .set_pll_profile = goya_set_pll_profile,
5007         .get_events_stat = goya_get_events_stat,
5008         .read_pte = goya_read_pte,
5009         .write_pte = goya_write_pte,
5010         .mmu_invalidate_cache = goya_mmu_invalidate_cache,
5011         .mmu_invalidate_cache_range = goya_mmu_invalidate_cache_range,
5012         .send_heartbeat = goya_send_heartbeat,
5013         .debug_coresight = goya_debug_coresight,
5014         .is_device_idle = goya_is_device_idle,
5015         .soft_reset_late_init = goya_soft_reset_late_init,
5016         .hw_queues_lock = goya_hw_queues_lock,
5017         .hw_queues_unlock = goya_hw_queues_unlock,
5018         .get_pci_id = goya_get_pci_id,
5019         .get_eeprom_data = goya_get_eeprom_data,
5020         .send_cpu_message = goya_send_cpu_message,
5021         .get_hw_state = goya_get_hw_state,
5022         .pci_bars_map = goya_pci_bars_map,
5023         .set_dram_bar_base = goya_set_ddr_bar_base,
5024         .init_iatu = goya_init_iatu,
5025         .rreg = hl_rreg,
5026         .wreg = hl_wreg,
5027         .halt_coresight = goya_halt_coresight
5028 };
5029
5030 /*
5031  * goya_set_asic_funcs - set Goya function pointers
5032  *
5033  * @*hdev: pointer to hl_device structure
5034  *
5035  */
5036 void goya_set_asic_funcs(struct hl_device *hdev)
5037 {
5038         hdev->asic_funcs = &goya_funcs;
5039 }