drivers/crypto/cavium/nitrox/nitrox_isr.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #include <linux/pci.h>
   3 #include <linux/printk.h>
   4 #include <linux/slab.h>
   5
   6 #include "nitrox_dev.h"
   7 #include "nitrox_csr.h"
   8 #include "nitrox_common.h"
   9 #include "nitrox_hal.h"
  10 #include "nitrox_isr.h"
  11 #include "nitrox_mbx.h"
  12
  13 /*
  14  * One vector for each type of ring
  15  *  - NPS packet ring, AQMQ ring and ZQMQ ring
  16  */
  17 #define NR_RING_VECTORS 3
  18 #define NR_NON_RING_VECTORS 1
  19 /* base entry for packet ring/port */
  20 #define PKT_RING_MSIX_BASE 0
  21 #define NON_RING_MSIX_BASE 192
  22
  23 /**
  24  * nps_pkt_slc_isr - IRQ handler for NPS solicit port
  25  * @irq: irq number
  26  * @data: argument
  27  */
  28 static irqreturn_t nps_pkt_slc_isr(int irq, void *data)
  29 {
  30         struct nitrox_q_vector *qvec = data;
  31         union nps_pkt_slc_cnts slc_cnts;
  32         struct nitrox_cmdq *cmdq = qvec->cmdq;
  33
  34         slc_cnts.value = readq(cmdq->compl_cnt_csr_addr);
  35         /* New packet on SLC output port */
  36         if (slc_cnts.s.slc_int)
  37                 tasklet_hi_schedule(&qvec->resp_tasklet);
  38
  39         return IRQ_HANDLED;
  40 }
  41
  42 static void clear_nps_core_err_intr(struct nitrox_device *ndev)
  43 {
  44         u64 value;
  45
  46         /* Write 1 to clear */
  47         value = nitrox_read_csr(ndev, NPS_CORE_INT);
  48         nitrox_write_csr(ndev, NPS_CORE_INT, value);
  49
  50         dev_err_ratelimited(DEV(ndev), "NSP_CORE_INT  0x%016llx\n", value);
  51 }
  52
  53 static void clear_nps_pkt_err_intr(struct nitrox_device *ndev)
  54 {
  55         union nps_pkt_int pkt_int;
  56         unsigned long value, offset;
  57         int i;
  58
  59         pkt_int.value = nitrox_read_csr(ndev, NPS_PKT_INT);
  60         dev_err_ratelimited(DEV(ndev), "NPS_PKT_INT  0x%016llx\n",
  61                             pkt_int.value);
  62
  63         if (pkt_int.s.slc_err) {
  64                 offset = NPS_PKT_SLC_ERR_TYPE;
  65                 value = nitrox_read_csr(ndev, offset);
  66                 nitrox_write_csr(ndev, offset, value);
  67                 dev_err_ratelimited(DEV(ndev),
  68                                     "NPS_PKT_SLC_ERR_TYPE  0x%016lx\n", value);
  69
  70                 offset = NPS_PKT_SLC_RERR_LO;
  71                 value = nitrox_read_csr(ndev, offset);
  72                 nitrox_write_csr(ndev, offset, value);
  73                 /* enable the solicit ports */
  74                 for_each_set_bit(i, &value, BITS_PER_LONG)
  75                         enable_pkt_solicit_port(ndev, i);
  76
  77                 dev_err_ratelimited(DEV(ndev),
  78                                     "NPS_PKT_SLC_RERR_LO  0x%016lx\n", value);
  79
  80                 offset = NPS_PKT_SLC_RERR_HI;
  81                 value = nitrox_read_csr(ndev, offset);
  82                 nitrox_write_csr(ndev, offset, value);
  83                 dev_err_ratelimited(DEV(ndev),
  84                                     "NPS_PKT_SLC_RERR_HI  0x%016lx\n", value);
  85         }
  86
  87         if (pkt_int.s.in_err) {
  88                 offset = NPS_PKT_IN_ERR_TYPE;
  89                 value = nitrox_read_csr(ndev, offset);
  90                 nitrox_write_csr(ndev, offset, value);
  91                 dev_err_ratelimited(DEV(ndev),
  92                                     "NPS_PKT_IN_ERR_TYPE  0x%016lx\n", value);
  93                 offset = NPS_PKT_IN_RERR_LO;
  94                 value = nitrox_read_csr(ndev, offset);
  95                 nitrox_write_csr(ndev, offset, value);
  96                 /* enable the input ring */
  97                 for_each_set_bit(i, &value, BITS_PER_LONG)
  98                         enable_pkt_input_ring(ndev, i);
  99
 100                 dev_err_ratelimited(DEV(ndev),
 101                                     "NPS_PKT_IN_RERR_LO  0x%016lx\n", value);
 102
 103                 offset = NPS_PKT_IN_RERR_HI;
 104                 value = nitrox_read_csr(ndev, offset);
 105                 nitrox_write_csr(ndev, offset, value);
 106                 dev_err_ratelimited(DEV(ndev),
 107                                     "NPS_PKT_IN_RERR_HI  0x%016lx\n", value);
 108         }
 109 }
 110
 111 static void clear_pom_err_intr(struct nitrox_device *ndev)
 112 {
 113         u64 value;
 114
 115         value = nitrox_read_csr(ndev, POM_INT);
 116         nitrox_write_csr(ndev, POM_INT, value);
 117         dev_err_ratelimited(DEV(ndev), "POM_INT  0x%016llx\n", value);
 118 }
 119
 120 static void clear_pem_err_intr(struct nitrox_device *ndev)
 121 {
 122         u64 value;
 123
 124         value = nitrox_read_csr(ndev, PEM0_INT);
 125         nitrox_write_csr(ndev, PEM0_INT, value);
 126         dev_err_ratelimited(DEV(ndev), "PEM(0)_INT  0x%016llx\n", value);
 127 }
 128
 129 static void clear_lbc_err_intr(struct nitrox_device *ndev)
 130 {
 131         union lbc_int lbc_int;
 132         u64 value, offset;
 133         int i;
 134
 135         lbc_int.value = nitrox_read_csr(ndev, LBC_INT);
 136         dev_err_ratelimited(DEV(ndev), "LBC_INT  0x%016llx\n", lbc_int.value);
 137
 138         if (lbc_int.s.dma_rd_err) {
 139                 for (i = 0; i < NR_CLUSTERS; i++) {
 140                         offset = EFL_CORE_VF_ERR_INT0X(i);
 141                         value = nitrox_read_csr(ndev, offset);
 142                         nitrox_write_csr(ndev, offset, value);
 143                         offset = EFL_CORE_VF_ERR_INT1X(i);
 144                         value = nitrox_read_csr(ndev, offset);
 145                         nitrox_write_csr(ndev, offset, value);
 146                 }
 147         }
 148
 149         if (lbc_int.s.cam_soft_err) {
 150                 dev_err_ratelimited(DEV(ndev), "CAM_SOFT_ERR, invalidating LBC\n");
 151                 invalidate_lbc(ndev);
 152         }
 153
 154         if (lbc_int.s.pref_dat_len_mismatch_err) {
 155                 offset = LBC_PLM_VF1_64_INT;
 156                 value = nitrox_read_csr(ndev, offset);
 157                 nitrox_write_csr(ndev, offset, value);
 158                 offset = LBC_PLM_VF65_128_INT;
 159                 value = nitrox_read_csr(ndev, offset);
 160                 nitrox_write_csr(ndev, offset, value);
 161         }
 162
 163         if (lbc_int.s.rd_dat_len_mismatch_err) {
 164                 offset = LBC_ELM_VF1_64_INT;
 165                 value = nitrox_read_csr(ndev, offset);
 166                 nitrox_write_csr(ndev, offset, value);
 167                 offset = LBC_ELM_VF65_128_INT;
 168                 value = nitrox_read_csr(ndev, offset);
 169                 nitrox_write_csr(ndev, offset, value);
 170         }
 171         nitrox_write_csr(ndev, LBC_INT, lbc_int.value);
 172 }
 173
 174 static void clear_efl_err_intr(struct nitrox_device *ndev)
 175 {
 176         int i;
 177
 178         for (i = 0; i < NR_CLUSTERS; i++) {
 179                 union efl_core_int core_int;
 180                 u64 value, offset;
 181
 182                 offset = EFL_CORE_INTX(i);
 183                 core_int.value = nitrox_read_csr(ndev, offset);
 184                 nitrox_write_csr(ndev, offset, core_int.value);
 185                 dev_err_ratelimited(DEV(ndev), "ELF_CORE(%d)_INT  0x%016llx\n",
 186                                     i, core_int.value);
 187                 if (core_int.s.se_err) {
 188                         offset = EFL_CORE_SE_ERR_INTX(i);
 189                         value = nitrox_read_csr(ndev, offset);
 190                         nitrox_write_csr(ndev, offset, value);
 191                 }
 192         }
 193 }
 194
 195 static void clear_bmi_err_intr(struct nitrox_device *ndev)
 196 {
 197         u64 value;
 198
 199         value = nitrox_read_csr(ndev, BMI_INT);
 200         nitrox_write_csr(ndev, BMI_INT, value);
 201         dev_err_ratelimited(DEV(ndev), "BMI_INT  0x%016llx\n", value);
 202 }
 203
 204 static void nps_core_int_tasklet(unsigned long data)
 205 {
 206         struct nitrox_q_vector *qvec = (void *)(uintptr_t)(data);
 207         struct nitrox_device *ndev = qvec->ndev;
 208
 209         /* if pf mode do queue recovery */
 210         if (ndev->mode == __NDEV_MODE_PF) {
 211         } else {
 212                 /**
 213                  * if VF(s) enabled communicate the error information
 214                  * to VF(s)
 215                  */
 216         }
 217 }
 218
 219 /*
 220  * nps_core_int_isr - interrupt handler for NITROX errors and
 221  *   mailbox communication
 222  */
 223 static irqreturn_t nps_core_int_isr(int irq, void *data)
 224 {
 225         struct nitrox_q_vector *qvec = data;
 226         struct nitrox_device *ndev = qvec->ndev;
 227         union nps_core_int_active core_int;
 228
 229         core_int.value = nitrox_read_csr(ndev, NPS_CORE_INT_ACTIVE);
 230
 231         if (core_int.s.nps_core)
 232                 clear_nps_core_err_intr(ndev);
 233
 234         if (core_int.s.nps_pkt)
 235                 clear_nps_pkt_err_intr(ndev);
 236
 237         if (core_int.s.pom)
 238                 clear_pom_err_intr(ndev);
 239
 240         if (core_int.s.pem)
 241                 clear_pem_err_intr(ndev);
 242
 243         if (core_int.s.lbc)
 244                 clear_lbc_err_intr(ndev);
 245
 246         if (core_int.s.efl)
 247                 clear_efl_err_intr(ndev);
 248
 249         if (core_int.s.bmi)
 250                 clear_bmi_err_intr(ndev);
 251
 252         /* Mailbox interrupt */
 253         if (core_int.s.mbox)
 254                 nitrox_pf2vf_mbox_handler(ndev);
 255
 256         /* If more work callback the ISR, set resend */
 257         core_int.s.resend = 1;
 258         nitrox_write_csr(ndev, NPS_CORE_INT_ACTIVE, core_int.value);
 259
 260         return IRQ_HANDLED;
 261 }
 262
 263 void nitrox_unregister_interrupts(struct nitrox_device *ndev)
 264 {
 265         struct pci_dev *pdev = ndev->pdev;
 266         int i;
 267
 268         for (i = 0; i < ndev->num_vecs; i++) {
 269                 struct nitrox_q_vector *qvec;
 270                 int vec;
 271
 272                 qvec = ndev->qvec + i;
 273                 if (!qvec->valid)
 274                         continue;
 275
 276                 /* get the vector number */
 277                 vec = pci_irq_vector(pdev, i);
 278                 irq_set_affinity_hint(vec, NULL);
 279                 free_irq(vec, qvec);
 280
 281                 tasklet_disable(&qvec->resp_tasklet);
 282                 tasklet_kill(&qvec->resp_tasklet);
 283                 qvec->valid = false;
 284         }
 285         kfree(ndev->qvec);
 286         ndev->qvec = NULL;
 287         pci_free_irq_vectors(pdev);
 288 }
 289
 290 int nitrox_register_interrupts(struct nitrox_device *ndev)
 291 {
 292         struct pci_dev *pdev = ndev->pdev;
 293         struct nitrox_q_vector *qvec;
 294         int nr_vecs, vec, cpu;
 295         int ret, i;
 296
 297         /*
 298          * PF MSI-X vectors
 299          *
 300          * Entry 0: NPS PKT ring 0
 301          * Entry 1: AQMQ ring 0
 302          * Entry 2: ZQM ring 0
 303          * Entry 3: NPS PKT ring 1
 304          * Entry 4: AQMQ ring 1
 305          * Entry 5: ZQM ring 1
 306          * ....
 307          * Entry 192: NPS_CORE_INT_ACTIVE
 308          */
 309         nr_vecs = pci_msix_vec_count(pdev);
 310
 311         /* Enable MSI-X */
 312         ret = pci_alloc_irq_vectors(pdev, nr_vecs, nr_vecs, PCI_IRQ_MSIX);
 313         if (ret < 0) {
 314                 dev_err(DEV(ndev), "msix vectors %d alloc failed\n", nr_vecs);
 315                 return ret;
 316         }
 317         ndev->num_vecs = nr_vecs;
 318
 319         ndev->qvec = kcalloc(nr_vecs, sizeof(*qvec), GFP_KERNEL);
 320         if (!ndev->qvec) {
 321                 pci_free_irq_vectors(pdev);
 322                 return -ENOMEM;
 323         }
 324
 325         /* request irqs for packet rings/ports */
 326         for (i = PKT_RING_MSIX_BASE; i < (nr_vecs - 1); i += NR_RING_VECTORS) {
 327                 qvec = &ndev->qvec[i];
 328
 329                 qvec->ring = i / NR_RING_VECTORS;
 330                 if (qvec->ring >= ndev->nr_queues)
 331                         break;
 332
 333                 qvec->cmdq = &ndev->pkt_inq[qvec->ring];
 334                 snprintf(qvec->name, IRQ_NAMESZ, "nitrox-pkt%d", qvec->ring);
 335                 /* get the vector number */
 336                 vec = pci_irq_vector(pdev, i);
 337                 ret = request_irq(vec, nps_pkt_slc_isr, 0, qvec->name, qvec);
 338                 if (ret) {
 339                         dev_err(DEV(ndev), "irq failed for pkt ring/port%d\n",
 340                                 qvec->ring);
 341                         goto irq_fail;
 342                 }
 343                 cpu = qvec->ring % num_online_cpus();
 344                 irq_set_affinity_hint(vec, get_cpu_mask(cpu));
 345
 346                 tasklet_init(&qvec->resp_tasklet, pkt_slc_resp_tasklet,
 347                              (unsigned long)qvec);
 348                 qvec->valid = true;
 349         }
 350
 351         /* request irqs for non ring vectors */
 352         i = NON_RING_MSIX_BASE;
 353         qvec = &ndev->qvec[i];
 354         qvec->ndev = ndev;
 355
 356         snprintf(qvec->name, IRQ_NAMESZ, "nitrox-core-int%d", i);
 357         /* get the vector number */
 358         vec = pci_irq_vector(pdev, i);
 359         ret = request_irq(vec, nps_core_int_isr, 0, qvec->name, qvec);
 360         if (ret) {
 361                 dev_err(DEV(ndev), "irq failed for nitrox-core-int%d\n", i);
 362                 goto irq_fail;
 363         }
 364         cpu = num_online_cpus();
 365         irq_set_affinity_hint(vec, get_cpu_mask(cpu));
 366
 367         tasklet_init(&qvec->resp_tasklet, nps_core_int_tasklet,
 368                      (unsigned long)qvec);
 369         qvec->valid = true;
 370
 371         return 0;
 372
 373 irq_fail:
 374         nitrox_unregister_interrupts(ndev);
 375         return ret;
 376 }
 377
 378 void nitrox_sriov_unregister_interrupts(struct nitrox_device *ndev)
 379 {
 380         struct pci_dev *pdev = ndev->pdev;
 381         int i;
 382
 383         for (i = 0; i < ndev->num_vecs; i++) {
 384                 struct nitrox_q_vector *qvec;
 385                 int vec;
 386
 387                 qvec = ndev->qvec + i;
 388                 if (!qvec->valid)
 389                         continue;
 390
 391                 vec = ndev->iov.msix.vector;
 392                 irq_set_affinity_hint(vec, NULL);
 393                 free_irq(vec, qvec);
 394
 395                 tasklet_disable(&qvec->resp_tasklet);
 396                 tasklet_kill(&qvec->resp_tasklet);
 397                 qvec->valid = false;
 398         }
 399         kfree(ndev->qvec);
 400         ndev->qvec = NULL;
 401         pci_disable_msix(pdev);
 402 }
 403
 404 int nitrox_sriov_register_interupts(struct nitrox_device *ndev)
 405 {
 406         struct pci_dev *pdev = ndev->pdev;
 407         struct nitrox_q_vector *qvec;
 408         int vec, cpu;
 409         int ret;
 410
 411         /**
 412          * only non ring vectors i.e Entry 192 is available
 413          * for PF in SR-IOV mode.
 414          */
 415         ndev->iov.msix.entry = NON_RING_MSIX_BASE;
 416         ret = pci_enable_msix_exact(pdev, &ndev->iov.msix, NR_NON_RING_VECTORS);
 417         if (ret) {
 418                 dev_err(DEV(ndev), "failed to allocate nps-core-int%d\n",
 419                         NON_RING_MSIX_BASE);
 420                 return ret;
 421         }
 422
 423         qvec = kcalloc(NR_NON_RING_VECTORS, sizeof(*qvec), GFP_KERNEL);
 424         if (!qvec) {
 425                 pci_disable_msix(pdev);
 426                 return -ENOMEM;
 427         }
 428         qvec->ndev = ndev;
 429
 430         ndev->qvec = qvec;
 431         ndev->num_vecs = NR_NON_RING_VECTORS;
 432         snprintf(qvec->name, IRQ_NAMESZ, "nitrox-core-int%d",
 433                  NON_RING_MSIX_BASE);
 434
 435         vec = ndev->iov.msix.vector;
 436         ret = request_irq(vec, nps_core_int_isr, 0, qvec->name, qvec);
 437         if (ret) {
 438                 dev_err(DEV(ndev), "irq failed for nitrox-core-int%d\n",
 439                         NON_RING_MSIX_BASE);
 440                 goto iov_irq_fail;
 441         }
 442         cpu = num_online_cpus();
 443         irq_set_affinity_hint(vec, get_cpu_mask(cpu));
 444
 445         tasklet_init(&qvec->resp_tasklet, nps_core_int_tasklet,
 446                      (unsigned long)qvec);
 447         qvec->valid = true;
 448
 449         return 0;
 450
 451 iov_irq_fail:
 452         nitrox_sriov_unregister_interrupts(ndev);
 453         return ret;
 454 }