drivers/net/wireless/quantenna/qtnfmac/pcie/pearl_pcie.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /* Copyright (c) 2018 Quantenna Communications */
   3
   4 #include <linux/kernel.h>
   5 #include <linux/firmware.h>
   6 #include <linux/pci.h>
   7 #include <linux/vmalloc.h>
   8 #include <linux/delay.h>
   9 #include <linux/interrupt.h>
  10 #include <linux/sched.h>
  11 #include <linux/completion.h>
  12 #include <linux/crc32.h>
  13 #include <linux/spinlock.h>
  14 #include <linux/circ_buf.h>
  15 #include <linux/log2.h>
  16
  17 #include "pcie_priv.h"
  18 #include "pearl_pcie_regs.h"
  19 #include "pearl_pcie_ipc.h"
  20 #include "qtn_hw_ids.h"
  21 #include "core.h"
  22 #include "bus.h"
  23 #include "shm_ipc.h"
  24 #include "debug.h"
  25
  26 #define PEARL_TX_BD_SIZE_DEFAULT        32
  27
  28 struct qtnf_pearl_bda {
  29         __le16 bda_len;
  30         __le16 bda_version;
  31         __le32 bda_pci_endian;
  32         __le32 bda_ep_state;
  33         __le32 bda_rc_state;
  34         __le32 bda_dma_mask;
  35         __le32 bda_msi_addr;
  36         __le32 bda_flashsz;
  37         u8 bda_boardname[PCIE_BDA_NAMELEN];
  38         __le32 bda_rc_msi_enabled;
  39         u8 bda_hhbm_list[PCIE_HHBM_MAX_SIZE];
  40         __le32 bda_dsbw_start_index;
  41         __le32 bda_dsbw_end_index;
  42         __le32 bda_dsbw_total_bytes;
  43         __le32 bda_rc_tx_bd_base;
  44         __le32 bda_rc_tx_bd_num;
  45         u8 bda_pcie_mac[QTN_ENET_ADDR_LENGTH];
  46         struct qtnf_shm_ipc_region bda_shm_reg1 __aligned(4096); /* host TX */
  47         struct qtnf_shm_ipc_region bda_shm_reg2 __aligned(4096); /* host RX */
  48 } __packed;
  49
  50 struct qtnf_pearl_tx_bd {
  51         __le32 addr;
  52         __le32 addr_h;
  53         __le32 info;
  54         __le32 info_h;
  55 } __packed;
  56
  57 struct qtnf_pearl_rx_bd {
  58         __le32 addr;
  59         __le32 addr_h;
  60         __le32 info;
  61         __le32 info_h;
  62         __le32 next_ptr;
  63         __le32 next_ptr_h;
  64 } __packed;
  65
  66 struct qtnf_pearl_fw_hdr {
  67         u8 boardflg[8];
  68         __le32 fwsize;
  69         __le32 seqnum;
  70         __le32 type;
  71         __le32 pktlen;
  72         __le32 crc;
  73 } __packed;
  74
  75 struct qtnf_pcie_pearl_state {
  76         struct qtnf_pcie_bus_priv base;
  77
  78         /* lock for irq configuration changes */
  79         spinlock_t irq_lock;
  80
  81         struct qtnf_pearl_bda __iomem *bda;
  82         void __iomem *pcie_reg_base;
  83
  84         struct qtnf_pearl_tx_bd *tx_bd_vbase;
  85         dma_addr_t tx_bd_pbase;
  86
  87         struct qtnf_pearl_rx_bd *rx_bd_vbase;
  88         dma_addr_t rx_bd_pbase;
  89
  90         dma_addr_t bd_table_paddr;
  91         void *bd_table_vaddr;
  92         u32 bd_table_len;
  93         u32 pcie_irq_mask;
  94         u32 pcie_irq_rx_count;
  95         u32 pcie_irq_tx_count;
  96         u32 pcie_irq_uf_count;
  97 };
  98
  99 static inline void qtnf_init_hdp_irqs(struct qtnf_pcie_pearl_state *ps)
 100 {
 101         unsigned long flags;
 102
 103         spin_lock_irqsave(&ps->irq_lock, flags);
 104         ps->pcie_irq_mask = (PCIE_HDP_INT_RX_BITS | PCIE_HDP_INT_TX_BITS);
 105         spin_unlock_irqrestore(&ps->irq_lock, flags);
 106 }
 107
 108 static inline void qtnf_enable_hdp_irqs(struct qtnf_pcie_pearl_state *ps)
 109 {
 110         unsigned long flags;
 111
 112         spin_lock_irqsave(&ps->irq_lock, flags);
 113         writel(ps->pcie_irq_mask, PCIE_HDP_INT_EN(ps->pcie_reg_base));
 114         spin_unlock_irqrestore(&ps->irq_lock, flags);
 115 }
 116
 117 static inline void qtnf_disable_hdp_irqs(struct qtnf_pcie_pearl_state *ps)
 118 {
 119         unsigned long flags;
 120
 121         spin_lock_irqsave(&ps->irq_lock, flags);
 122         writel(0x0, PCIE_HDP_INT_EN(ps->pcie_reg_base));
 123         spin_unlock_irqrestore(&ps->irq_lock, flags);
 124 }
 125
 126 static inline void qtnf_en_rxdone_irq(struct qtnf_pcie_pearl_state *ps)
 127 {
 128         unsigned long flags;
 129
 130         spin_lock_irqsave(&ps->irq_lock, flags);
 131         ps->pcie_irq_mask |= PCIE_HDP_INT_RX_BITS;
 132         writel(ps->pcie_irq_mask, PCIE_HDP_INT_EN(ps->pcie_reg_base));
 133         spin_unlock_irqrestore(&ps->irq_lock, flags);
 134 }
 135
 136 static inline void qtnf_dis_rxdone_irq(struct qtnf_pcie_pearl_state *ps)
 137 {
 138         unsigned long flags;
 139
 140         spin_lock_irqsave(&ps->irq_lock, flags);
 141         ps->pcie_irq_mask &= ~PCIE_HDP_INT_RX_BITS;
 142         writel(ps->pcie_irq_mask, PCIE_HDP_INT_EN(ps->pcie_reg_base));
 143         spin_unlock_irqrestore(&ps->irq_lock, flags);
 144 }
 145
 146 static inline void qtnf_en_txdone_irq(struct qtnf_pcie_pearl_state *ps)
 147 {
 148         unsigned long flags;
 149
 150         spin_lock_irqsave(&ps->irq_lock, flags);
 151         ps->pcie_irq_mask |= PCIE_HDP_INT_TX_BITS;
 152         writel(ps->pcie_irq_mask, PCIE_HDP_INT_EN(ps->pcie_reg_base));
 153         spin_unlock_irqrestore(&ps->irq_lock, flags);
 154 }
 155
 156 static inline void qtnf_dis_txdone_irq(struct qtnf_pcie_pearl_state *ps)
 157 {
 158         unsigned long flags;
 159
 160         spin_lock_irqsave(&ps->irq_lock, flags);
 161         ps->pcie_irq_mask &= ~PCIE_HDP_INT_TX_BITS;
 162         writel(ps->pcie_irq_mask, PCIE_HDP_INT_EN(ps->pcie_reg_base));
 163         spin_unlock_irqrestore(&ps->irq_lock, flags);
 164 }
 165
 166 static void qtnf_deassert_intx(struct qtnf_pcie_pearl_state *ps)
 167 {
 168         void __iomem *reg = ps->base.sysctl_bar + PEARL_PCIE_CFG0_OFFSET;
 169         u32 cfg;
 170
 171         cfg = readl(reg);
 172         cfg &= ~PEARL_ASSERT_INTX;
 173         qtnf_non_posted_write(cfg, reg);
 174 }
 175
 176 static void qtnf_pearl_reset_ep(struct qtnf_pcie_pearl_state *ps)
 177 {
 178         const u32 data = QTN_PEARL_IPC_IRQ_WORD(QTN_PEARL_LHOST_EP_RESET);
 179         void __iomem *reg = ps->base.sysctl_bar +
 180                             QTN_PEARL_SYSCTL_LHOST_IRQ_OFFSET;
 181
 182         qtnf_non_posted_write(data, reg);
 183         msleep(QTN_EP_RESET_WAIT_MS);
 184         pci_restore_state(ps->base.pdev);
 185 }
 186
 187 static void qtnf_pcie_pearl_ipc_gen_ep_int(void *arg)
 188 {
 189         const struct qtnf_pcie_pearl_state *ps = arg;
 190         const u32 data = QTN_PEARL_IPC_IRQ_WORD(QTN_PEARL_LHOST_IPC_IRQ);
 191         void __iomem *reg = ps->base.sysctl_bar +
 192                             QTN_PEARL_SYSCTL_LHOST_IRQ_OFFSET;
 193
 194         qtnf_non_posted_write(data, reg);
 195 }
 196
 197 static int qtnf_is_state(__le32 __iomem *reg, u32 state)
 198 {
 199         u32 s = readl(reg);
 200
 201         return s & state;
 202 }
 203
 204 static void qtnf_set_state(__le32 __iomem *reg, u32 state)
 205 {
 206         u32 s = readl(reg);
 207
 208         qtnf_non_posted_write(state | s, reg);
 209 }
 210
 211 static void qtnf_clear_state(__le32 __iomem *reg, u32 state)
 212 {
 213         u32 s = readl(reg);
 214
 215         qtnf_non_posted_write(s & ~state, reg);
 216 }
 217
 218 static int qtnf_poll_state(__le32 __iomem *reg, u32 state, u32 delay_in_ms)
 219 {
 220         u32 timeout = 0;
 221
 222         while ((qtnf_is_state(reg, state) == 0)) {
 223                 usleep_range(1000, 1200);
 224                 if (++timeout > delay_in_ms)
 225                         return -1;
 226         }
 227
 228         return 0;
 229 }
 230
 231 static int pearl_alloc_bd_table(struct qtnf_pcie_pearl_state *ps)
 232 {
 233         struct qtnf_pcie_bus_priv *priv = &ps->base;
 234         dma_addr_t paddr;
 235         void *vaddr;
 236         int len;
 237
 238         len = priv->tx_bd_num * sizeof(struct qtnf_pearl_tx_bd) +
 239                 priv->rx_bd_num * sizeof(struct qtnf_pearl_rx_bd);
 240
 241         vaddr = dmam_alloc_coherent(&priv->pdev->dev, len, &paddr, GFP_KERNEL);
 242         if (!vaddr)
 243                 return -ENOMEM;
 244
 245         /* tx bd */
 246
 247         memset(vaddr, 0, len);
 248
 249         ps->bd_table_vaddr = vaddr;
 250         ps->bd_table_paddr = paddr;
 251         ps->bd_table_len = len;
 252
 253         ps->tx_bd_vbase = vaddr;
 254         ps->tx_bd_pbase = paddr;
 255
 256         pr_debug("TX descriptor table: vaddr=0x%p paddr=%pad\n", vaddr, &paddr);
 257
 258         priv->tx_bd_r_index = 0;
 259         priv->tx_bd_w_index = 0;
 260
 261         /* rx bd */
 262
 263         vaddr = ((struct qtnf_pearl_tx_bd *)vaddr) + priv->tx_bd_num;
 264         paddr += priv->tx_bd_num * sizeof(struct qtnf_pearl_tx_bd);
 265
 266         ps->rx_bd_vbase = vaddr;
 267         ps->rx_bd_pbase = paddr;
 268
 269 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
 270         writel(QTN_HOST_HI32(paddr),
 271                PCIE_HDP_TX_HOST_Q_BASE_H(ps->pcie_reg_base));
 272 #endif
 273         writel(QTN_HOST_LO32(paddr),
 274                PCIE_HDP_TX_HOST_Q_BASE_L(ps->pcie_reg_base));
 275         writel(priv->rx_bd_num | (sizeof(struct qtnf_pearl_rx_bd)) << 16,
 276                PCIE_HDP_TX_HOST_Q_SZ_CTRL(ps->pcie_reg_base));
 277
 278         pr_debug("RX descriptor table: vaddr=0x%p paddr=%pad\n", vaddr, &paddr);
 279
 280         return 0;
 281 }
 282
 283 static int pearl_skb2rbd_attach(struct qtnf_pcie_pearl_state *ps, u16 index)
 284 {
 285         struct qtnf_pcie_bus_priv *priv = &ps->base;
 286         struct qtnf_pearl_rx_bd *rxbd;
 287         struct sk_buff *skb;
 288         dma_addr_t paddr;
 289
 290         skb = __netdev_alloc_skb_ip_align(NULL, SKB_BUF_SIZE, GFP_ATOMIC);
 291         if (!skb) {
 292                 priv->rx_skb[index] = NULL;
 293                 return -ENOMEM;
 294         }
 295
 296         priv->rx_skb[index] = skb;
 297         rxbd = &ps->rx_bd_vbase[index];
 298
 299         paddr = pci_map_single(priv->pdev, skb->data,
 300                                SKB_BUF_SIZE, PCI_DMA_FROMDEVICE);
 301         if (pci_dma_mapping_error(priv->pdev, paddr)) {
 302                 pr_err("skb DMA mapping error: %pad\n", &paddr);
 303                 return -ENOMEM;
 304         }
 305
 306         /* keep rx skb paddrs in rx buffer descriptors for cleanup purposes */
 307         rxbd->addr = cpu_to_le32(QTN_HOST_LO32(paddr));
 308         rxbd->addr_h = cpu_to_le32(QTN_HOST_HI32(paddr));
 309         rxbd->info = 0x0;
 310
 311         priv->rx_bd_w_index = index;
 312
 313         /* sync up all descriptor updates */
 314         wmb();
 315
 316 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
 317         writel(QTN_HOST_HI32(paddr),
 318                PCIE_HDP_HHBM_BUF_PTR_H(ps->pcie_reg_base));
 319 #endif
 320         writel(QTN_HOST_LO32(paddr),
 321                PCIE_HDP_HHBM_BUF_PTR(ps->pcie_reg_base));
 322
 323         writel(index, PCIE_HDP_TX_HOST_Q_WR_PTR(ps->pcie_reg_base));
 324         return 0;
 325 }
 326
 327 static int pearl_alloc_rx_buffers(struct qtnf_pcie_pearl_state *ps)
 328 {
 329         u16 i;
 330         int ret = 0;
 331
 332         memset(ps->rx_bd_vbase, 0x0,
 333                ps->base.rx_bd_num * sizeof(struct qtnf_pearl_rx_bd));
 334
 335         for (i = 0; i < ps->base.rx_bd_num; i++) {
 336                 ret = pearl_skb2rbd_attach(ps, i);
 337                 if (ret)
 338                         break;
 339         }
 340
 341         return ret;
 342 }
 343
 344 /* all rx/tx activity should have ceased before calling this function */
 345 static void qtnf_pearl_free_xfer_buffers(struct qtnf_pcie_pearl_state *ps)
 346 {
 347         struct qtnf_pcie_bus_priv *priv = &ps->base;
 348         struct qtnf_pearl_tx_bd *txbd;
 349         struct qtnf_pearl_rx_bd *rxbd;
 350         struct sk_buff *skb;
 351         dma_addr_t paddr;
 352         int i;
 353
 354         /* free rx buffers */
 355         for (i = 0; i < priv->rx_bd_num; i++) {
 356                 if (priv->rx_skb && priv->rx_skb[i]) {
 357                         rxbd = &ps->rx_bd_vbase[i];
 358                         skb = priv->rx_skb[i];
 359                         paddr = QTN_HOST_ADDR(le32_to_cpu(rxbd->addr_h),
 360                                               le32_to_cpu(rxbd->addr));
 361                         pci_unmap_single(priv->pdev, paddr, SKB_BUF_SIZE,
 362                                          PCI_DMA_FROMDEVICE);
 363                         dev_kfree_skb_any(skb);
 364                         priv->rx_skb[i] = NULL;
 365                 }
 366         }
 367
 368         /* free tx buffers */
 369         for (i = 0; i < priv->tx_bd_num; i++) {
 370                 if (priv->tx_skb && priv->tx_skb[i]) {
 371                         txbd = &ps->tx_bd_vbase[i];
 372                         skb = priv->tx_skb[i];
 373                         paddr = QTN_HOST_ADDR(le32_to_cpu(txbd->addr_h),
 374                                               le32_to_cpu(txbd->addr));
 375                         pci_unmap_single(priv->pdev, paddr, skb->len,
 376                                          PCI_DMA_TODEVICE);
 377                         dev_kfree_skb_any(skb);
 378                         priv->tx_skb[i] = NULL;
 379                 }
 380         }
 381 }
 382
 383 static int pearl_hhbm_init(struct qtnf_pcie_pearl_state *ps)
 384 {
 385         u32 val;
 386
 387         val = readl(PCIE_HHBM_CONFIG(ps->pcie_reg_base));
 388         val |= HHBM_CONFIG_SOFT_RESET;
 389         writel(val, PCIE_HHBM_CONFIG(ps->pcie_reg_base));
 390         usleep_range(50, 100);
 391         val &= ~HHBM_CONFIG_SOFT_RESET;
 392 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
 393         val |= HHBM_64BIT;
 394 #endif
 395         writel(val, PCIE_HHBM_CONFIG(ps->pcie_reg_base));
 396         writel(ps->base.rx_bd_num, PCIE_HHBM_Q_LIMIT_REG(ps->pcie_reg_base));
 397
 398         return 0;
 399 }
 400
 401 static int qtnf_pcie_pearl_init_xfer(struct qtnf_pcie_pearl_state *ps,
 402                                      unsigned int tx_bd_size)
 403 {
 404         struct qtnf_pcie_bus_priv *priv = &ps->base;
 405         int ret;
 406         u32 val;
 407
 408         if (tx_bd_size == 0)
 409                 tx_bd_size = PEARL_TX_BD_SIZE_DEFAULT;
 410
 411         val = tx_bd_size * sizeof(struct qtnf_pearl_tx_bd);
 412
 413         if (!is_power_of_2(tx_bd_size) || val > PCIE_HHBM_MAX_SIZE) {
 414                 pr_warn("bad tx_bd_size value %u\n", tx_bd_size);
 415                 priv->tx_bd_num = PEARL_TX_BD_SIZE_DEFAULT;
 416         } else {
 417                 priv->tx_bd_num = tx_bd_size;
 418         }
 419
 420         priv->rx_bd_w_index = 0;
 421         priv->rx_bd_r_index = 0;
 422
 423         if (!priv->rx_bd_num || !is_power_of_2(priv->rx_bd_num)) {
 424                 pr_err("rx_bd_size_param %u is not power of two\n",
 425                        priv->rx_bd_num);
 426                 return -EINVAL;
 427         }
 428
 429         val = priv->rx_bd_num * sizeof(dma_addr_t);
 430         if (val > PCIE_HHBM_MAX_SIZE) {
 431                 pr_err("rx_bd_size_param %u is too large\n",
 432                        priv->rx_bd_num);
 433                 return -EINVAL;
 434         }
 435
 436         ret = pearl_hhbm_init(ps);
 437         if (ret) {
 438                 pr_err("failed to init h/w queues\n");
 439                 return ret;
 440         }
 441
 442         ret = qtnf_pcie_alloc_skb_array(priv);
 443         if (ret) {
 444                 pr_err("failed to allocate skb array\n");
 445                 return ret;
 446         }
 447
 448         ret = pearl_alloc_bd_table(ps);
 449         if (ret) {
 450                 pr_err("failed to allocate bd table\n");
 451                 return ret;
 452         }
 453
 454         ret = pearl_alloc_rx_buffers(ps);
 455         if (ret) {
 456                 pr_err("failed to allocate rx buffers\n");
 457                 return ret;
 458         }
 459
 460         return ret;
 461 }
 462
 463 static void qtnf_pearl_data_tx_reclaim(struct qtnf_pcie_pearl_state *ps)
 464 {
 465         struct qtnf_pcie_bus_priv *priv = &ps->base;
 466         struct qtnf_pearl_tx_bd *txbd;
 467         struct sk_buff *skb;
 468         unsigned long flags;
 469         dma_addr_t paddr;
 470         u32 tx_done_index;
 471         int count = 0;
 472         int i;
 473
 474         spin_lock_irqsave(&priv->tx_reclaim_lock, flags);
 475
 476         tx_done_index = readl(PCIE_HDP_RX0DMA_CNT(ps->pcie_reg_base))
 477                         & (priv->tx_bd_num - 1);
 478
 479         i = priv->tx_bd_r_index;
 480
 481         while (CIRC_CNT(tx_done_index, i, priv->tx_bd_num)) {
 482                 skb = priv->tx_skb[i];
 483                 if (likely(skb)) {
 484                         txbd = &ps->tx_bd_vbase[i];
 485                         paddr = QTN_HOST_ADDR(le32_to_cpu(txbd->addr_h),
 486                                               le32_to_cpu(txbd->addr));
 487                         pci_unmap_single(priv->pdev, paddr, skb->len,
 488                                          PCI_DMA_TODEVICE);
 489
 490                         if (skb->dev) {
 491                                 qtnf_update_tx_stats(skb->dev, skb);
 492                                 if (unlikely(priv->tx_stopped)) {
 493                                         qtnf_wake_all_queues(skb->dev);
 494                                         priv->tx_stopped = 0;
 495                                 }
 496                         }
 497
 498                         dev_kfree_skb_any(skb);
 499                 }
 500
 501                 priv->tx_skb[i] = NULL;
 502                 count++;
 503
 504                 if (++i >= priv->tx_bd_num)
 505                         i = 0;
 506         }
 507
 508         priv->tx_reclaim_done += count;
 509         priv->tx_reclaim_req++;
 510         priv->tx_bd_r_index = i;
 511
 512         spin_unlock_irqrestore(&priv->tx_reclaim_lock, flags);
 513 }
 514
 515 static int qtnf_tx_queue_ready(struct qtnf_pcie_pearl_state *ps)
 516 {
 517         struct qtnf_pcie_bus_priv *priv = &ps->base;
 518
 519         if (!CIRC_SPACE(priv->tx_bd_w_index, priv->tx_bd_r_index,
 520                         priv->tx_bd_num)) {
 521                 qtnf_pearl_data_tx_reclaim(ps);
 522
 523                 if (!CIRC_SPACE(priv->tx_bd_w_index, priv->tx_bd_r_index,
 524                                 priv->tx_bd_num)) {
 525                         pr_warn_ratelimited("reclaim full Tx queue\n");
 526                         priv->tx_full_count++;
 527                         return 0;
 528                 }
 529         }
 530
 531         return 1;
 532 }
 533
 534 static int qtnf_pcie_data_tx(struct qtnf_bus *bus, struct sk_buff *skb)
 535 {
 536         struct qtnf_pcie_pearl_state *ps = get_bus_priv(bus);
 537         struct qtnf_pcie_bus_priv *priv = &ps->base;
 538         dma_addr_t txbd_paddr, skb_paddr;
 539         struct qtnf_pearl_tx_bd *txbd;
 540         unsigned long flags;
 541         int len, i;
 542         u32 info;
 543         int ret = 0;
 544
 545         spin_lock_irqsave(&priv->tx_lock, flags);
 546
 547         if (!qtnf_tx_queue_ready(ps)) {
 548                 if (skb->dev) {
 549                         netif_tx_stop_all_queues(skb->dev);
 550                         priv->tx_stopped = 1;
 551                 }
 552
 553                 spin_unlock_irqrestore(&priv->tx_lock, flags);
 554                 return NETDEV_TX_BUSY;
 555         }
 556
 557         i = priv->tx_bd_w_index;
 558         priv->tx_skb[i] = skb;
 559         len = skb->len;
 560
 561         skb_paddr = pci_map_single(priv->pdev, skb->data,
 562                                    skb->len, PCI_DMA_TODEVICE);
 563         if (pci_dma_mapping_error(priv->pdev, skb_paddr)) {
 564                 pr_err("skb DMA mapping error: %pad\n", &skb_paddr);
 565                 ret = -ENOMEM;
 566                 goto tx_done;
 567         }
 568
 569         txbd = &ps->tx_bd_vbase[i];
 570         txbd->addr = cpu_to_le32(QTN_HOST_LO32(skb_paddr));
 571         txbd->addr_h = cpu_to_le32(QTN_HOST_HI32(skb_paddr));
 572
 573         info = (len & QTN_PCIE_TX_DESC_LEN_MASK) << QTN_PCIE_TX_DESC_LEN_SHIFT;
 574         txbd->info = cpu_to_le32(info);
 575
 576         /* sync up all descriptor updates before passing them to EP */
 577         dma_wmb();
 578
 579         /* write new TX descriptor to PCIE_RX_FIFO on EP */
 580         txbd_paddr = ps->tx_bd_pbase + i * sizeof(struct qtnf_pearl_tx_bd);
 581
 582 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
 583         writel(QTN_HOST_HI32(txbd_paddr),
 584                PCIE_HDP_HOST_WR_DESC0_H(ps->pcie_reg_base));
 585 #endif
 586         writel(QTN_HOST_LO32(txbd_paddr),
 587                PCIE_HDP_HOST_WR_DESC0(ps->pcie_reg_base));
 588
 589         if (++i >= priv->tx_bd_num)
 590                 i = 0;
 591
 592         priv->tx_bd_w_index = i;
 593
 594 tx_done:
 595         if (ret && skb) {
 596                 pr_err_ratelimited("drop skb\n");
 597                 if (skb->dev)
 598                         skb->dev->stats.tx_dropped++;
 599                 dev_kfree_skb_any(skb);
 600         }
 601
 602         priv->tx_done_count++;
 603         spin_unlock_irqrestore(&priv->tx_lock, flags);
 604
 605         qtnf_pearl_data_tx_reclaim(ps);
 606
 607         return NETDEV_TX_OK;
 608 }
 609
 610 static irqreturn_t qtnf_pcie_pearl_interrupt(int irq, void *data)
 611 {
 612         struct qtnf_bus *bus = (struct qtnf_bus *)data;
 613         struct qtnf_pcie_pearl_state *ps = get_bus_priv(bus);
 614         struct qtnf_pcie_bus_priv *priv = &ps->base;
 615         u32 status;
 616
 617         priv->pcie_irq_count++;
 618         status = readl(PCIE_HDP_INT_STATUS(ps->pcie_reg_base));
 619
 620         qtnf_shm_ipc_irq_handler(&priv->shm_ipc_ep_in);
 621         qtnf_shm_ipc_irq_handler(&priv->shm_ipc_ep_out);
 622
 623         if (!(status & ps->pcie_irq_mask))
 624                 goto irq_done;
 625
 626         if (status & PCIE_HDP_INT_RX_BITS)
 627                 ps->pcie_irq_rx_count++;
 628
 629         if (status & PCIE_HDP_INT_TX_BITS)
 630                 ps->pcie_irq_tx_count++;
 631
 632         if (status & PCIE_HDP_INT_HHBM_UF)
 633                 ps->pcie_irq_uf_count++;
 634
 635         if (status & PCIE_HDP_INT_RX_BITS) {
 636                 qtnf_dis_rxdone_irq(ps);
 637                 napi_schedule(&bus->mux_napi);
 638         }
 639
 640         if (status & PCIE_HDP_INT_TX_BITS) {
 641                 qtnf_dis_txdone_irq(ps);
 642                 tasklet_hi_schedule(&priv->reclaim_tq);
 643         }
 644
 645 irq_done:
 646         /* H/W workaround: clean all bits, not only enabled */
 647         qtnf_non_posted_write(~0U, PCIE_HDP_INT_STATUS(ps->pcie_reg_base));
 648
 649         if (!priv->msi_enabled)
 650                 qtnf_deassert_intx(ps);
 651
 652         return IRQ_HANDLED;
 653 }
 654
 655 static int qtnf_rx_data_ready(struct qtnf_pcie_pearl_state *ps)
 656 {
 657         u16 index = ps->base.rx_bd_r_index;
 658         struct qtnf_pearl_rx_bd *rxbd;
 659         u32 descw;
 660
 661         rxbd = &ps->rx_bd_vbase[index];
 662         descw = le32_to_cpu(rxbd->info);
 663
 664         if (descw & QTN_TXDONE_MASK)
 665                 return 1;
 666
 667         return 0;
 668 }
 669
 670 static int qtnf_pcie_pearl_rx_poll(struct napi_struct *napi, int budget)
 671 {
 672         struct qtnf_bus *bus = container_of(napi, struct qtnf_bus, mux_napi);
 673         struct qtnf_pcie_pearl_state *ps = get_bus_priv(bus);
 674         struct qtnf_pcie_bus_priv *priv = &ps->base;
 675         struct net_device *ndev = NULL;
 676         struct sk_buff *skb = NULL;
 677         int processed = 0;
 678         struct qtnf_pearl_rx_bd *rxbd;
 679         dma_addr_t skb_paddr;
 680         int consume;
 681         u32 descw;
 682         u32 psize;
 683         u16 r_idx;
 684         u16 w_idx;
 685         int ret;
 686
 687         while (processed < budget) {
 688                 if (!qtnf_rx_data_ready(ps))
 689                         goto rx_out;
 690
 691                 r_idx = priv->rx_bd_r_index;
 692                 rxbd = &ps->rx_bd_vbase[r_idx];
 693                 descw = le32_to_cpu(rxbd->info);
 694
 695                 skb = priv->rx_skb[r_idx];
 696                 psize = QTN_GET_LEN(descw);
 697                 consume = 1;
 698
 699                 if (!(descw & QTN_TXDONE_MASK)) {
 700                         pr_warn("skip invalid rxbd[%d]\n", r_idx);
 701                         consume = 0;
 702                 }
 703
 704                 if (!skb) {
 705                         pr_warn("skip missing rx_skb[%d]\n", r_idx);
 706                         consume = 0;
 707                 }
 708
 709                 if (skb && (skb_tailroom(skb) <  psize)) {
 710                         pr_err("skip packet with invalid length: %u > %u\n",
 711                                psize, skb_tailroom(skb));
 712                         consume = 0;
 713                 }
 714
 715                 if (skb) {
 716                         skb_paddr = QTN_HOST_ADDR(le32_to_cpu(rxbd->addr_h),
 717                                                   le32_to_cpu(rxbd->addr));
 718                         pci_unmap_single(priv->pdev, skb_paddr, SKB_BUF_SIZE,
 719                                          PCI_DMA_FROMDEVICE);
 720                 }
 721
 722                 if (consume) {
 723                         skb_put(skb, psize);
 724                         ndev = qtnf_classify_skb(bus, skb);
 725                         if (likely(ndev)) {
 726                                 qtnf_update_rx_stats(ndev, skb);
 727                                 skb->protocol = eth_type_trans(skb, ndev);
 728                                 napi_gro_receive(napi, skb);
 729                         } else {
 730                                 pr_debug("drop untagged skb\n");
 731                                 bus->mux_dev.stats.rx_dropped++;
 732                                 dev_kfree_skb_any(skb);
 733                         }
 734                 } else {
 735                         if (skb) {
 736                                 bus->mux_dev.stats.rx_dropped++;
 737                                 dev_kfree_skb_any(skb);
 738                         }
 739                 }
 740
 741                 priv->rx_skb[r_idx] = NULL;
 742                 if (++r_idx >= priv->rx_bd_num)
 743                         r_idx = 0;
 744
 745                 priv->rx_bd_r_index = r_idx;
 746
 747                 /* repalce processed buffer by a new one */
 748                 w_idx = priv->rx_bd_w_index;
 749                 while (CIRC_SPACE(priv->rx_bd_w_index, priv->rx_bd_r_index,
 750                                   priv->rx_bd_num) > 0) {
 751                         if (++w_idx >= priv->rx_bd_num)
 752                                 w_idx = 0;
 753
 754                         ret = pearl_skb2rbd_attach(ps, w_idx);
 755                         if (ret) {
 756                                 pr_err("failed to allocate new rx_skb[%d]\n",
 757                                        w_idx);
 758                                 break;
 759                         }
 760                 }
 761
 762                 processed++;
 763         }
 764
 765 rx_out:
 766         if (processed < budget) {
 767                 napi_complete(napi);
 768                 qtnf_en_rxdone_irq(ps);
 769         }
 770
 771         return processed;
 772 }
 773
 774 static void
 775 qtnf_pcie_data_tx_timeout(struct qtnf_bus *bus, struct net_device *ndev)
 776 {
 777         struct qtnf_pcie_pearl_state *ps = (void *)get_bus_priv(bus);
 778
 779         tasklet_hi_schedule(&ps->base.reclaim_tq);
 780 }
 781
 782 static void qtnf_pcie_data_rx_start(struct qtnf_bus *bus)
 783 {
 784         struct qtnf_pcie_pearl_state *ps = (void *)get_bus_priv(bus);
 785
 786         qtnf_enable_hdp_irqs(ps);
 787         napi_enable(&bus->mux_napi);
 788 }
 789
 790 static void qtnf_pcie_data_rx_stop(struct qtnf_bus *bus)
 791 {
 792         struct qtnf_pcie_pearl_state *ps = (void *)get_bus_priv(bus);
 793
 794         napi_disable(&bus->mux_napi);
 795         qtnf_disable_hdp_irqs(ps);
 796 }
 797
 798 static const struct qtnf_bus_ops qtnf_pcie_pearl_bus_ops = {
 799         /* control path methods */
 800         .control_tx     = qtnf_pcie_control_tx,
 801
 802         /* data path methods */
 803         .data_tx                = qtnf_pcie_data_tx,
 804         .data_tx_timeout        = qtnf_pcie_data_tx_timeout,
 805         .data_rx_start          = qtnf_pcie_data_rx_start,
 806         .data_rx_stop           = qtnf_pcie_data_rx_stop,
 807 };
 808
 809 static int qtnf_dbg_irq_stats(struct seq_file *s, void *data)
 810 {
 811         struct qtnf_bus *bus = dev_get_drvdata(s->private);
 812         struct qtnf_pcie_pearl_state *ps = get_bus_priv(bus);
 813         u32 reg = readl(PCIE_HDP_INT_EN(ps->pcie_reg_base));
 814         u32 status;
 815
 816         seq_printf(s, "pcie_irq_count(%u)\n", ps->base.pcie_irq_count);
 817         seq_printf(s, "pcie_irq_tx_count(%u)\n", ps->pcie_irq_tx_count);
 818         status = reg &  PCIE_HDP_INT_TX_BITS;
 819         seq_printf(s, "pcie_irq_tx_status(%s)\n",
 820                    (status == PCIE_HDP_INT_TX_BITS) ? "EN" : "DIS");
 821         seq_printf(s, "pcie_irq_rx_count(%u)\n", ps->pcie_irq_rx_count);
 822         status = reg &  PCIE_HDP_INT_RX_BITS;
 823         seq_printf(s, "pcie_irq_rx_status(%s)\n",
 824                    (status == PCIE_HDP_INT_RX_BITS) ? "EN" : "DIS");
 825         seq_printf(s, "pcie_irq_uf_count(%u)\n", ps->pcie_irq_uf_count);
 826         status = reg &  PCIE_HDP_INT_HHBM_UF;
 827         seq_printf(s, "pcie_irq_hhbm_uf_status(%s)\n",
 828                    (status == PCIE_HDP_INT_HHBM_UF) ? "EN" : "DIS");
 829
 830         return 0;
 831 }
 832
 833 static int qtnf_dbg_hdp_stats(struct seq_file *s, void *data)
 834 {
 835         struct qtnf_bus *bus = dev_get_drvdata(s->private);
 836         struct qtnf_pcie_pearl_state *ps = get_bus_priv(bus);
 837         struct qtnf_pcie_bus_priv *priv = &ps->base;
 838
 839         seq_printf(s, "tx_full_count(%u)\n", priv->tx_full_count);
 840         seq_printf(s, "tx_done_count(%u)\n", priv->tx_done_count);
 841         seq_printf(s, "tx_reclaim_done(%u)\n", priv->tx_reclaim_done);
 842         seq_printf(s, "tx_reclaim_req(%u)\n", priv->tx_reclaim_req);
 843
 844         seq_printf(s, "tx_bd_r_index(%u)\n", priv->tx_bd_r_index);
 845         seq_printf(s, "tx_bd_p_index(%u)\n",
 846                    readl(PCIE_HDP_RX0DMA_CNT(ps->pcie_reg_base))
 847                         & (priv->tx_bd_num - 1));
 848         seq_printf(s, "tx_bd_w_index(%u)\n", priv->tx_bd_w_index);
 849         seq_printf(s, "tx queue len(%u)\n",
 850                    CIRC_CNT(priv->tx_bd_w_index, priv->tx_bd_r_index,
 851                             priv->tx_bd_num));
 852
 853         seq_printf(s, "rx_bd_r_index(%u)\n", priv->rx_bd_r_index);
 854         seq_printf(s, "rx_bd_p_index(%u)\n",
 855                    readl(PCIE_HDP_TX0DMA_CNT(ps->pcie_reg_base))
 856                         & (priv->rx_bd_num - 1));
 857         seq_printf(s, "rx_bd_w_index(%u)\n", priv->rx_bd_w_index);
 858         seq_printf(s, "rx alloc queue len(%u)\n",
 859                    CIRC_SPACE(priv->rx_bd_w_index, priv->rx_bd_r_index,
 860                               priv->rx_bd_num));
 861
 862         return 0;
 863 }
 864
 865 static int qtnf_ep_fw_send(struct pci_dev *pdev, uint32_t size,
 866                            int blk, const u8 *pblk, const u8 *fw)
 867 {
 868         struct qtnf_bus *bus = pci_get_drvdata(pdev);
 869
 870         struct qtnf_pearl_fw_hdr *hdr;
 871         u8 *pdata;
 872
 873         int hds = sizeof(*hdr);
 874         struct sk_buff *skb = NULL;
 875         int len = 0;
 876         int ret;
 877
 878         skb = __dev_alloc_skb(QTN_PCIE_FW_BUFSZ, GFP_KERNEL);
 879         if (!skb)
 880                 return -ENOMEM;
 881
 882         skb->len = QTN_PCIE_FW_BUFSZ;
 883         skb->dev = NULL;
 884
 885         hdr = (struct qtnf_pearl_fw_hdr *)skb->data;
 886         memcpy(hdr->boardflg, QTN_PCIE_BOARDFLG, strlen(QTN_PCIE_BOARDFLG));
 887         hdr->fwsize = cpu_to_le32(size);
 888         hdr->seqnum = cpu_to_le32(blk);
 889
 890         if (blk)
 891                 hdr->type = cpu_to_le32(QTN_FW_DSUB);
 892         else
 893                 hdr->type = cpu_to_le32(QTN_FW_DBEGIN);
 894
 895         pdata = skb->data + hds;
 896
 897         len = QTN_PCIE_FW_BUFSZ - hds;
 898         if (pblk >= (fw + size - len)) {
 899                 len = fw + size - pblk;
 900                 hdr->type = cpu_to_le32(QTN_FW_DEND);
 901         }
 902
 903         hdr->pktlen = cpu_to_le32(len);
 904         memcpy(pdata, pblk, len);
 905         hdr->crc = cpu_to_le32(~crc32(0, pdata, len));
 906
 907         ret = qtnf_pcie_data_tx(bus, skb);
 908
 909         return (ret == NETDEV_TX_OK) ? len : 0;
 910 }
 911
 912 static int
 913 qtnf_ep_fw_load(struct qtnf_pcie_pearl_state *ps, const u8 *fw, u32 fw_size)
 914 {
 915         int blk_size = QTN_PCIE_FW_BUFSZ - sizeof(struct qtnf_pearl_fw_hdr);
 916         int blk_count = fw_size / blk_size + ((fw_size % blk_size) ? 1 : 0);
 917         const u8 *pblk = fw;
 918         int threshold = 0;
 919         int blk = 0;
 920         int len;
 921
 922         pr_debug("FW upload started: fw_addr=0x%p size=%d\n", fw, fw_size);
 923
 924         while (blk < blk_count) {
 925                 if (++threshold > 10000) {
 926                         pr_err("FW upload failed: too many retries\n");
 927                         return -ETIMEDOUT;
 928                 }
 929
 930                 len = qtnf_ep_fw_send(ps->base.pdev, fw_size, blk, pblk, fw);
 931                 if (len <= 0)
 932                         continue;
 933
 934                 if (!((blk + 1) & QTN_PCIE_FW_DLMASK) ||
 935                     (blk == (blk_count - 1))) {
 936                         qtnf_set_state(&ps->bda->bda_rc_state,
 937                                        QTN_RC_FW_SYNC);
 938                         if (qtnf_poll_state(&ps->bda->bda_ep_state,
 939                                             QTN_EP_FW_SYNC,
 940                                             QTN_FW_DL_TIMEOUT_MS)) {
 941                                 pr_err("FW upload failed: SYNC timed out\n");
 942                                 return -ETIMEDOUT;
 943                         }
 944
 945                         qtnf_clear_state(&ps->bda->bda_ep_state,
 946                                          QTN_EP_FW_SYNC);
 947
 948                         if (qtnf_is_state(&ps->bda->bda_ep_state,
 949                                           QTN_EP_FW_RETRY)) {
 950                                 if (blk == (blk_count - 1)) {
 951                                         int last_round =
 952                                                 blk_count & QTN_PCIE_FW_DLMASK;
 953                                         blk -= last_round;
 954                                         pblk -= ((last_round - 1) *
 955                                                 blk_size + len);
 956                                 } else {
 957                                         blk -= QTN_PCIE_FW_DLMASK;
 958                                         pblk -= QTN_PCIE_FW_DLMASK * blk_size;
 959                                 }
 960
 961                                 qtnf_clear_state(&ps->bda->bda_ep_state,
 962                                                  QTN_EP_FW_RETRY);
 963
 964                                 pr_warn("FW upload retry: block #%d\n", blk);
 965                                 continue;
 966                         }
 967
 968                         qtnf_pearl_data_tx_reclaim(ps);
 969                 }
 970
 971                 pblk += len;
 972                 blk++;
 973         }
 974
 975         pr_debug("FW upload completed: totally sent %d blocks\n", blk);
 976         return 0;
 977 }
 978
 979 static void qtnf_pearl_fw_work_handler(struct work_struct *work)
 980 {
 981         struct qtnf_bus *bus = container_of(work, struct qtnf_bus, fw_work);
 982         struct qtnf_pcie_pearl_state *ps = (void *)get_bus_priv(bus);
 983         struct pci_dev *pdev = ps->base.pdev;
 984         const struct firmware *fw;
 985         int ret;
 986         u32 state = QTN_RC_FW_LOADRDY | QTN_RC_FW_QLINK;
 987         const char *fwname = QTN_PCI_PEARL_FW_NAME;
 988         bool fw_boot_success = false;
 989
 990         if (ps->base.flashboot) {
 991                 state |= QTN_RC_FW_FLASHBOOT;
 992         } else {
 993                 ret = request_firmware(&fw, fwname, &pdev->dev);
 994                 if (ret < 0) {
 995                         pr_err("failed to get firmware %s\n", fwname);
 996                         goto fw_load_exit;
 997                 }
 998         }
 999
1000         qtnf_set_state(&ps->bda->bda_rc_state, state);
1001
1002         if (qtnf_poll_state(&ps->bda->bda_ep_state, QTN_EP_FW_LOADRDY,
1003                             QTN_FW_DL_TIMEOUT_MS)) {
1004                 pr_err("card is not ready\n");
1005
1006                 if (!ps->base.flashboot)
1007                         release_firmware(fw);
1008
1009                 goto fw_load_exit;
1010         }
1011
1012         qtnf_clear_state(&ps->bda->bda_ep_state, QTN_EP_FW_LOADRDY);
1013
1014         if (ps->base.flashboot) {
1015                 pr_info("booting firmware from flash\n");
1016
1017         } else {
1018                 pr_info("starting firmware upload: %s\n", fwname);
1019
1020                 ret = qtnf_ep_fw_load(ps, fw->data, fw->size);
1021                 release_firmware(fw);
1022                 if (ret) {
1023                         pr_err("firmware upload error\n");
1024                         goto fw_load_exit;
1025                 }
1026         }
1027
1028         if (qtnf_poll_state(&ps->bda->bda_ep_state, QTN_EP_FW_DONE,
1029                             QTN_FW_DL_TIMEOUT_MS)) {
1030                 pr_err("firmware bringup timed out\n");
1031                 goto fw_load_exit;
1032         }
1033
1034         pr_info("firmware is up and running\n");
1035
1036         if (qtnf_poll_state(&ps->bda->bda_ep_state,
1037                             QTN_EP_FW_QLINK_DONE, QTN_FW_QLINK_TIMEOUT_MS)) {
1038                 pr_err("firmware runtime failure\n");
1039                 goto fw_load_exit;
1040         }
1041
1042         fw_boot_success = true;
1043
1044 fw_load_exit:
1045         qtnf_pcie_fw_boot_done(bus, fw_boot_success);
1046
1047         if (fw_boot_success) {
1048                 qtnf_debugfs_add_entry(bus, "hdp_stats", qtnf_dbg_hdp_stats);
1049                 qtnf_debugfs_add_entry(bus, "irq_stats", qtnf_dbg_irq_stats);
1050         }
1051 }
1052
1053 static void qtnf_pearl_reclaim_tasklet_fn(unsigned long data)
1054 {
1055         struct qtnf_pcie_pearl_state *ps = (void *)data;
1056
1057         qtnf_pearl_data_tx_reclaim(ps);
1058         qtnf_en_txdone_irq(ps);
1059 }
1060
1061 static u64 qtnf_pearl_dma_mask_get(void)
1062 {
1063 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
1064         return DMA_BIT_MASK(64);
1065 #else
1066         return DMA_BIT_MASK(32);
1067 #endif
1068 }
1069
1070 static int qtnf_pcie_pearl_probe(struct qtnf_bus *bus, unsigned int tx_bd_size)
1071 {
1072         struct qtnf_shm_ipc_int ipc_int;
1073         struct qtnf_pcie_pearl_state *ps = get_bus_priv(bus);
1074         struct pci_dev *pdev = ps->base.pdev;
1075         int ret;
1076
1077         bus->bus_ops = &qtnf_pcie_pearl_bus_ops;
1078         spin_lock_init(&ps->irq_lock);
1079         INIT_WORK(&bus->fw_work, qtnf_pearl_fw_work_handler);
1080
1081         ps->pcie_reg_base = ps->base.dmareg_bar;
1082         ps->bda = ps->base.epmem_bar;
1083         writel(ps->base.msi_enabled, &ps->bda->bda_rc_msi_enabled);
1084
1085         ret = qtnf_pcie_pearl_init_xfer(ps, tx_bd_size);
1086         if (ret) {
1087                 pr_err("PCIE xfer init failed\n");
1088                 return ret;
1089         }
1090
1091         /* init default irq settings */
1092         qtnf_init_hdp_irqs(ps);
1093
1094         /* start with disabled irqs */
1095         qtnf_disable_hdp_irqs(ps);
1096
1097         ret = devm_request_irq(&pdev->dev, pdev->irq,
1098                                &qtnf_pcie_pearl_interrupt, 0,
1099                                "qtnf_pearl_irq", (void *)bus);
1100         if (ret) {
1101                 pr_err("failed to request pcie irq %d\n", pdev->irq);
1102                 qtnf_pearl_free_xfer_buffers(ps);
1103                 return ret;
1104         }
1105
1106         tasklet_init(&ps->base.reclaim_tq, qtnf_pearl_reclaim_tasklet_fn,
1107                      (unsigned long)ps);
1108         netif_napi_add(&bus->mux_dev, &bus->mux_napi,
1109                        qtnf_pcie_pearl_rx_poll, 10);
1110
1111         ipc_int.fn = qtnf_pcie_pearl_ipc_gen_ep_int;
1112         ipc_int.arg = ps;
1113         qtnf_pcie_init_shm_ipc(&ps->base, &ps->bda->bda_shm_reg1,
1114                                &ps->bda->bda_shm_reg2, &ipc_int);
1115
1116         return 0;
1117 }
1118
1119 static void qtnf_pcie_pearl_remove(struct qtnf_bus *bus)
1120 {
1121         struct qtnf_pcie_pearl_state *ps = get_bus_priv(bus);
1122
1123         qtnf_pearl_reset_ep(ps);
1124         qtnf_pearl_free_xfer_buffers(ps);
1125 }
1126
1127 #ifdef CONFIG_PM_SLEEP
1128 static int qtnf_pcie_pearl_suspend(struct qtnf_bus *bus)
1129 {
1130         return -EOPNOTSUPP;
1131 }
1132
1133 static int qtnf_pcie_pearl_resume(struct qtnf_bus *bus)
1134 {
1135         return 0;
1136 }
1137 #endif
1138
1139 struct qtnf_bus *qtnf_pcie_pearl_alloc(struct pci_dev *pdev)
1140 {
1141         struct qtnf_bus *bus;
1142         struct qtnf_pcie_pearl_state *ps;
1143
1144         bus = devm_kzalloc(&pdev->dev, sizeof(*bus) + sizeof(*ps), GFP_KERNEL);
1145         if (!bus)
1146                 return NULL;
1147
1148         ps = get_bus_priv(bus);
1149         ps->base.probe_cb = qtnf_pcie_pearl_probe;
1150         ps->base.remove_cb = qtnf_pcie_pearl_remove;
1151         ps->base.dma_mask_get_cb = qtnf_pearl_dma_mask_get;
1152 #ifdef CONFIG_PM_SLEEP
1153         ps->base.resume_cb = qtnf_pcie_pearl_resume;
1154         ps->base.suspend_cb = qtnf_pcie_pearl_suspend;
1155 #endif
1156
1157         return bus;
1158 }