drivers/soc/fsl/dpio/qbman-portal.c

   1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
   2 /*
   3  * Copyright (C) 2014-2016 Freescale Semiconductor, Inc.
   4  * Copyright 2016-2019 NXP
   5  *
   6  */
   7
   8 #include <asm/cacheflush.h>
   9 #include <linux/io.h>
  10 #include <linux/slab.h>
  11 #include <linux/spinlock.h>
  12 #include <soc/fsl/dpaa2-global.h>
  13
  14 #include "qbman-portal.h"
  15
  16 /* All QBMan command and result structures use this "valid bit" encoding */
  17 #define QB_VALID_BIT ((u32)0x80)
  18
  19 /* QBMan portal management command codes */
  20 #define QBMAN_MC_ACQUIRE       0x30
  21 #define QBMAN_WQCHAN_CONFIGURE 0x46
  22
  23 /* CINH register offsets */
  24 #define QBMAN_CINH_SWP_EQCR_PI      0x800
  25 #define QBMAN_CINH_SWP_EQCR_CI      0x840
  26 #define QBMAN_CINH_SWP_EQAR    0x8c0
  27 #define QBMAN_CINH_SWP_CR_RT        0x900
  28 #define QBMAN_CINH_SWP_VDQCR_RT     0x940
  29 #define QBMAN_CINH_SWP_EQCR_AM_RT   0x980
  30 #define QBMAN_CINH_SWP_RCR_AM_RT    0x9c0
  31 #define QBMAN_CINH_SWP_DQPI    0xa00
  32 #define QBMAN_CINH_SWP_DCAP    0xac0
  33 #define QBMAN_CINH_SWP_SDQCR   0xb00
  34 #define QBMAN_CINH_SWP_EQCR_AM_RT2  0xb40
  35 #define QBMAN_CINH_SWP_RCR_PI       0xc00
  36 #define QBMAN_CINH_SWP_RAR     0xcc0
  37 #define QBMAN_CINH_SWP_ISR     0xe00
  38 #define QBMAN_CINH_SWP_IER     0xe40
  39 #define QBMAN_CINH_SWP_ISDR    0xe80
  40 #define QBMAN_CINH_SWP_IIR     0xec0
  41
  42 /* CENA register offsets */
  43 #define QBMAN_CENA_SWP_EQCR(n) (0x000 + ((u32)(n) << 6))
  44 #define QBMAN_CENA_SWP_DQRR(n) (0x200 + ((u32)(n) << 6))
  45 #define QBMAN_CENA_SWP_RCR(n)  (0x400 + ((u32)(n) << 6))
  46 #define QBMAN_CENA_SWP_CR      0x600
  47 #define QBMAN_CENA_SWP_RR(vb)  (0x700 + ((u32)(vb) >> 1))
  48 #define QBMAN_CENA_SWP_VDQCR   0x780
  49 #define QBMAN_CENA_SWP_EQCR_CI 0x840
  50 #define QBMAN_CENA_SWP_EQCR_CI_MEMBACK 0x1840
  51
  52 /* CENA register offsets in memory-backed mode */
  53 #define QBMAN_CENA_SWP_DQRR_MEM(n)  (0x800 + ((u32)(n) << 6))
  54 #define QBMAN_CENA_SWP_RCR_MEM(n)   (0x1400 + ((u32)(n) << 6))
  55 #define QBMAN_CENA_SWP_CR_MEM       0x1600
  56 #define QBMAN_CENA_SWP_RR_MEM       0x1680
  57 #define QBMAN_CENA_SWP_VDQCR_MEM    0x1780
  58
  59 /* Reverse mapping of QBMAN_CENA_SWP_DQRR() */
  60 #define QBMAN_IDX_FROM_DQRR(p) (((unsigned long)(p) & 0x1ff) >> 6)
  61
  62 /* Define token used to determine if response written to memory is valid */
  63 #define QMAN_DQ_TOKEN_VALID 1
  64
  65 /* SDQCR attribute codes */
  66 #define QB_SDQCR_FC_SHIFT   29
  67 #define QB_SDQCR_FC_MASK    0x1
  68 #define QB_SDQCR_DCT_SHIFT  24
  69 #define QB_SDQCR_DCT_MASK   0x3
  70 #define QB_SDQCR_TOK_SHIFT  16
  71 #define QB_SDQCR_TOK_MASK   0xff
  72 #define QB_SDQCR_SRC_SHIFT  0
  73 #define QB_SDQCR_SRC_MASK   0xffff
  74
  75 /* opaque token for static dequeues */
  76 #define QMAN_SDQCR_TOKEN    0xbb
  77
  78 #define QBMAN_EQCR_DCA_IDXMASK          0x0f
  79 #define QBMAN_ENQUEUE_FLAG_DCA          (1ULL << 31)
  80
  81 #define EQ_DESC_SIZE_WITHOUT_FD 29
  82 #define EQ_DESC_SIZE_FD_START 32
  83
  84 enum qbman_sdqcr_dct {
  85         qbman_sdqcr_dct_null = 0,
  86         qbman_sdqcr_dct_prio_ics,
  87         qbman_sdqcr_dct_active_ics,
  88         qbman_sdqcr_dct_active
  89 };
  90
  91 enum qbman_sdqcr_fc {
  92         qbman_sdqcr_fc_one = 0,
  93         qbman_sdqcr_fc_up_to_3 = 1
  94 };
  95
  96 /* Internal Function declaration */
  97 static int qbman_swp_enqueue_direct(struct qbman_swp *s,
  98                                     const struct qbman_eq_desc *d,
  99                                     const struct dpaa2_fd *fd);
 100 static int qbman_swp_enqueue_mem_back(struct qbman_swp *s,
 101                                       const struct qbman_eq_desc *d,
 102                                       const struct dpaa2_fd *fd);
 103 static int qbman_swp_enqueue_multiple_direct(struct qbman_swp *s,
 104                                              const struct qbman_eq_desc *d,
 105                                              const struct dpaa2_fd *fd,
 106                                              uint32_t *flags,
 107                                              int num_frames);
 108 static int qbman_swp_enqueue_multiple_mem_back(struct qbman_swp *s,
 109                                                const struct qbman_eq_desc *d,
 110                                                const struct dpaa2_fd *fd,
 111                                                uint32_t *flags,
 112                                                int num_frames);
 113 static int
 114 qbman_swp_enqueue_multiple_desc_direct(struct qbman_swp *s,
 115                                        const struct qbman_eq_desc *d,
 116                                        const struct dpaa2_fd *fd,
 117                                        int num_frames);
 118 static
 119 int qbman_swp_enqueue_multiple_desc_mem_back(struct qbman_swp *s,
 120                                              const struct qbman_eq_desc *d,
 121                                              const struct dpaa2_fd *fd,
 122                                              int num_frames);
 123 static int qbman_swp_pull_direct(struct qbman_swp *s,
 124                                  struct qbman_pull_desc *d);
 125 static int qbman_swp_pull_mem_back(struct qbman_swp *s,
 126                                    struct qbman_pull_desc *d);
 127
 128 const struct dpaa2_dq *qbman_swp_dqrr_next_direct(struct qbman_swp *s);
 129 const struct dpaa2_dq *qbman_swp_dqrr_next_mem_back(struct qbman_swp *s);
 130
 131 static int qbman_swp_release_direct(struct qbman_swp *s,
 132                                     const struct qbman_release_desc *d,
 133                                     const u64 *buffers,
 134                                     unsigned int num_buffers);
 135 static int qbman_swp_release_mem_back(struct qbman_swp *s,
 136                                       const struct qbman_release_desc *d,
 137                                       const u64 *buffers,
 138                                       unsigned int num_buffers);
 139
 140 /* Function pointers */
 141 int (*qbman_swp_enqueue_ptr)(struct qbman_swp *s,
 142                              const struct qbman_eq_desc *d,
 143                              const struct dpaa2_fd *fd)
 144         = qbman_swp_enqueue_direct;
 145
 146 int (*qbman_swp_enqueue_multiple_ptr)(struct qbman_swp *s,
 147                                       const struct qbman_eq_desc *d,
 148                                       const struct dpaa2_fd *fd,
 149                                       uint32_t *flags,
 150                                              int num_frames)
 151         = qbman_swp_enqueue_multiple_direct;
 152
 153 int
 154 (*qbman_swp_enqueue_multiple_desc_ptr)(struct qbman_swp *s,
 155                                        const struct qbman_eq_desc *d,
 156                                        const struct dpaa2_fd *fd,
 157                                        int num_frames)
 158         = qbman_swp_enqueue_multiple_desc_direct;
 159
 160 int (*qbman_swp_pull_ptr)(struct qbman_swp *s, struct qbman_pull_desc *d)
 161                         = qbman_swp_pull_direct;
 162
 163 const struct dpaa2_dq *(*qbman_swp_dqrr_next_ptr)(struct qbman_swp *s)
 164                         = qbman_swp_dqrr_next_direct;
 165
 166 int (*qbman_swp_release_ptr)(struct qbman_swp *s,
 167                              const struct qbman_release_desc *d,
 168                              const u64 *buffers,
 169                              unsigned int num_buffers)
 170                         = qbman_swp_release_direct;
 171
 172 /* Portal Access */
 173
 174 static inline u32 qbman_read_register(struct qbman_swp *p, u32 offset)
 175 {
 176         return readl_relaxed(p->addr_cinh + offset);
 177 }
 178
 179 static inline void qbman_write_register(struct qbman_swp *p, u32 offset,
 180                                         u32 value)
 181 {
 182         writel_relaxed(value, p->addr_cinh + offset);
 183 }
 184
 185 static inline void *qbman_get_cmd(struct qbman_swp *p, u32 offset)
 186 {
 187         return p->addr_cena + offset;
 188 }
 189
 190 #define QBMAN_CINH_SWP_CFG   0xd00
 191
 192 #define SWP_CFG_DQRR_MF_SHIFT 20
 193 #define SWP_CFG_EST_SHIFT     16
 194 #define SWP_CFG_CPBS_SHIFT    15
 195 #define SWP_CFG_WN_SHIFT      14
 196 #define SWP_CFG_RPM_SHIFT     12
 197 #define SWP_CFG_DCM_SHIFT     10
 198 #define SWP_CFG_EPM_SHIFT     8
 199 #define SWP_CFG_VPM_SHIFT     7
 200 #define SWP_CFG_CPM_SHIFT     6
 201 #define SWP_CFG_SD_SHIFT      5
 202 #define SWP_CFG_SP_SHIFT      4
 203 #define SWP_CFG_SE_SHIFT      3
 204 #define SWP_CFG_DP_SHIFT      2
 205 #define SWP_CFG_DE_SHIFT      1
 206 #define SWP_CFG_EP_SHIFT      0
 207
 208 static inline u32 qbman_set_swp_cfg(u8 max_fill, u8 wn, u8 est, u8 rpm, u8 dcm,
 209                                     u8 epm, int sd, int sp, int se,
 210                                     int dp, int de, int ep)
 211 {
 212         return (max_fill << SWP_CFG_DQRR_MF_SHIFT |
 213                 est << SWP_CFG_EST_SHIFT |
 214                 wn << SWP_CFG_WN_SHIFT |
 215                 rpm << SWP_CFG_RPM_SHIFT |
 216                 dcm << SWP_CFG_DCM_SHIFT |
 217                 epm << SWP_CFG_EPM_SHIFT |
 218                 sd << SWP_CFG_SD_SHIFT |
 219                 sp << SWP_CFG_SP_SHIFT |
 220                 se << SWP_CFG_SE_SHIFT |
 221                 dp << SWP_CFG_DP_SHIFT |
 222                 de << SWP_CFG_DE_SHIFT |
 223                 ep << SWP_CFG_EP_SHIFT);
 224 }
 225
 226 #define QMAN_RT_MODE       0x00000100
 227
 228 static inline u8 qm_cyc_diff(u8 ringsize, u8 first, u8 last)
 229 {
 230         /* 'first' is included, 'last' is excluded */
 231         if (first <= last)
 232                 return last - first;
 233         else
 234                 return (2 * ringsize) - (first - last);
 235 }
 236
 237 /**
 238  * qbman_swp_init() - Create a functional object representing the given
 239  *                    QBMan portal descriptor.
 240  * @d: the given qbman swp descriptor
 241  *
 242  * Return qbman_swp portal for success, NULL if the object cannot
 243  * be created.
 244  */
 245 struct qbman_swp *qbman_swp_init(const struct qbman_swp_desc *d)
 246 {
 247         struct qbman_swp *p = kzalloc(sizeof(*p), GFP_KERNEL);
 248         u32 reg;
 249         u32 mask_size;
 250         u32 eqcr_pi;
 251
 252         if (!p)
 253                 return NULL;
 254
 255         spin_lock_init(&p->access_spinlock);
 256
 257         p->desc = d;
 258         p->mc.valid_bit = QB_VALID_BIT;
 259         p->sdq = 0;
 260         p->sdq |= qbman_sdqcr_dct_prio_ics << QB_SDQCR_DCT_SHIFT;
 261         p->sdq |= qbman_sdqcr_fc_up_to_3 << QB_SDQCR_FC_SHIFT;
 262         p->sdq |= QMAN_SDQCR_TOKEN << QB_SDQCR_TOK_SHIFT;
 263         if ((p->desc->qman_version & QMAN_REV_MASK) >= QMAN_REV_5000)
 264                 p->mr.valid_bit = QB_VALID_BIT;
 265
 266         atomic_set(&p->vdq.available, 1);
 267         p->vdq.valid_bit = QB_VALID_BIT;
 268         p->dqrr.next_idx = 0;
 269         p->dqrr.valid_bit = QB_VALID_BIT;
 270
 271         if ((p->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_4100) {
 272                 p->dqrr.dqrr_size = 4;
 273                 p->dqrr.reset_bug = 1;
 274         } else {
 275                 p->dqrr.dqrr_size = 8;
 276                 p->dqrr.reset_bug = 0;
 277         }
 278
 279         p->addr_cena = d->cena_bar;
 280         p->addr_cinh = d->cinh_bar;
 281
 282         if ((p->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_5000) {
 283
 284                 reg = qbman_set_swp_cfg(p->dqrr.dqrr_size,
 285                         1, /* Writes Non-cacheable */
 286                         0, /* EQCR_CI stashing threshold */
 287                         3, /* RPM: RCR in array mode */
 288                         2, /* DCM: Discrete consumption ack */
 289                         2, /* EPM: EQCR in ring mode */
 290                         1, /* mem stashing drop enable enable */
 291                         1, /* mem stashing priority enable */
 292                         1, /* mem stashing enable */
 293                         1, /* dequeue stashing priority enable */
 294                         0, /* dequeue stashing enable enable */
 295                         0); /* EQCR_CI stashing priority enable */
 296         } else {
 297                 memset(p->addr_cena, 0, 64 * 1024);
 298                 reg = qbman_set_swp_cfg(p->dqrr.dqrr_size,
 299                         1, /* Writes Non-cacheable */
 300                         1, /* EQCR_CI stashing threshold */
 301                         3, /* RPM: RCR in array mode */
 302                         2, /* DCM: Discrete consumption ack */
 303                         0, /* EPM: EQCR in ring mode */
 304                         1, /* mem stashing drop enable */
 305                         1, /* mem stashing priority enable */
 306                         1, /* mem stashing enable */
 307                         1, /* dequeue stashing priority enable */
 308                         0, /* dequeue stashing enable */
 309                         0); /* EQCR_CI stashing priority enable */
 310                 reg |= 1 << SWP_CFG_CPBS_SHIFT | /* memory-backed mode */
 311                        1 << SWP_CFG_VPM_SHIFT |  /* VDQCR read triggered mode */
 312                        1 << SWP_CFG_CPM_SHIFT;   /* CR read triggered mode */
 313         }
 314
 315         qbman_write_register(p, QBMAN_CINH_SWP_CFG, reg);
 316         reg = qbman_read_register(p, QBMAN_CINH_SWP_CFG);
 317         if (!reg) {
 318                 pr_err("qbman: the portal is not enabled!\n");
 319                 kfree(p);
 320                 return NULL;
 321         }
 322
 323         if ((p->desc->qman_version & QMAN_REV_MASK) >= QMAN_REV_5000) {
 324                 qbman_write_register(p, QBMAN_CINH_SWP_EQCR_PI, QMAN_RT_MODE);
 325                 qbman_write_register(p, QBMAN_CINH_SWP_RCR_PI, QMAN_RT_MODE);
 326         }
 327         /*
 328          * SDQCR needs to be initialized to 0 when no channels are
 329          * being dequeued from or else the QMan HW will indicate an
 330          * error.  The values that were calculated above will be
 331          * applied when dequeues from a specific channel are enabled.
 332          */
 333         qbman_write_register(p, QBMAN_CINH_SWP_SDQCR, 0);
 334
 335         p->eqcr.pi_ring_size = 8;
 336         if ((p->desc->qman_version & QMAN_REV_MASK) >= QMAN_REV_5000) {
 337                 p->eqcr.pi_ring_size = 32;
 338                 qbman_swp_enqueue_ptr =
 339                         qbman_swp_enqueue_mem_back;
 340                 qbman_swp_enqueue_multiple_ptr =
 341                         qbman_swp_enqueue_multiple_mem_back;
 342                 qbman_swp_enqueue_multiple_desc_ptr =
 343                         qbman_swp_enqueue_multiple_desc_mem_back;
 344                 qbman_swp_pull_ptr = qbman_swp_pull_mem_back;
 345                 qbman_swp_dqrr_next_ptr = qbman_swp_dqrr_next_mem_back;
 346                 qbman_swp_release_ptr = qbman_swp_release_mem_back;
 347         }
 348
 349         for (mask_size = p->eqcr.pi_ring_size; mask_size > 0; mask_size >>= 1)
 350                 p->eqcr.pi_ci_mask = (p->eqcr.pi_ci_mask << 1) + 1;
 351         eqcr_pi = qbman_read_register(p, QBMAN_CINH_SWP_EQCR_PI);
 352         p->eqcr.pi = eqcr_pi & p->eqcr.pi_ci_mask;
 353         p->eqcr.pi_vb = eqcr_pi & QB_VALID_BIT;
 354         p->eqcr.ci = qbman_read_register(p, QBMAN_CINH_SWP_EQCR_CI)
 355                         & p->eqcr.pi_ci_mask;
 356         p->eqcr.available = p->eqcr.pi_ring_size;
 357
 358         return p;
 359 }
 360
 361 /**
 362  * qbman_swp_finish() - Create and destroy a functional object representing
 363  *                      the given QBMan portal descriptor.
 364  * @p: the qbman_swp object to be destroyed
 365  */
 366 void qbman_swp_finish(struct qbman_swp *p)
 367 {
 368         kfree(p);
 369 }
 370
 371 /**
 372  * qbman_swp_interrupt_read_status()
 373  * @p: the given software portal
 374  *
 375  * Return the value in the SWP_ISR register.
 376  */
 377 u32 qbman_swp_interrupt_read_status(struct qbman_swp *p)
 378 {
 379         return qbman_read_register(p, QBMAN_CINH_SWP_ISR);
 380 }
 381
 382 /**
 383  * qbman_swp_interrupt_clear_status()
 384  * @p: the given software portal
 385  * @mask: The mask to clear in SWP_ISR register
 386  */
 387 void qbman_swp_interrupt_clear_status(struct qbman_swp *p, u32 mask)
 388 {
 389         qbman_write_register(p, QBMAN_CINH_SWP_ISR, mask);
 390 }
 391
 392 /**
 393  * qbman_swp_interrupt_get_trigger() - read interrupt enable register
 394  * @p: the given software portal
 395  *
 396  * Return the value in the SWP_IER register.
 397  */
 398 u32 qbman_swp_interrupt_get_trigger(struct qbman_swp *p)
 399 {
 400         return qbman_read_register(p, QBMAN_CINH_SWP_IER);
 401 }
 402
 403 /**
 404  * qbman_swp_interrupt_set_trigger() - enable interrupts for a swp
 405  * @p: the given software portal
 406  * @mask: The mask of bits to enable in SWP_IER
 407  */
 408 void qbman_swp_interrupt_set_trigger(struct qbman_swp *p, u32 mask)
 409 {
 410         qbman_write_register(p, QBMAN_CINH_SWP_IER, mask);
 411 }
 412
 413 /**
 414  * qbman_swp_interrupt_get_inhibit() - read interrupt mask register
 415  * @p: the given software portal object
 416  *
 417  * Return the value in the SWP_IIR register.
 418  */
 419 int qbman_swp_interrupt_get_inhibit(struct qbman_swp *p)
 420 {
 421         return qbman_read_register(p, QBMAN_CINH_SWP_IIR);
 422 }
 423
 424 /**
 425  * qbman_swp_interrupt_set_inhibit() - write interrupt mask register
 426  * @p: the given software portal object
 427  * @mask: The mask to set in SWP_IIR register
 428  */
 429 void qbman_swp_interrupt_set_inhibit(struct qbman_swp *p, int inhibit)
 430 {
 431         qbman_write_register(p, QBMAN_CINH_SWP_IIR, inhibit ? 0xffffffff : 0);
 432 }
 433
 434 /*
 435  * Different management commands all use this common base layer of code to issue
 436  * commands and poll for results.
 437  */
 438
 439 /*
 440  * Returns a pointer to where the caller should fill in their management command
 441  * (caller should ignore the verb byte)
 442  */
 443 void *qbman_swp_mc_start(struct qbman_swp *p)
 444 {
 445         if ((p->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_5000)
 446                 return qbman_get_cmd(p, QBMAN_CENA_SWP_CR);
 447         else
 448                 return qbman_get_cmd(p, QBMAN_CENA_SWP_CR_MEM);
 449 }
 450
 451 /*
 452  * Commits merges in the caller-supplied command verb (which should not include
 453  * the valid-bit) and submits the command to hardware
 454  */
 455 void qbman_swp_mc_submit(struct qbman_swp *p, void *cmd, u8 cmd_verb)
 456 {
 457         u8 *v = cmd;
 458
 459         if ((p->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_5000) {
 460                 dma_wmb();
 461                 *v = cmd_verb | p->mc.valid_bit;
 462         } else {
 463                 *v = cmd_verb | p->mc.valid_bit;
 464                 dma_wmb();
 465                 qbman_write_register(p, QBMAN_CINH_SWP_CR_RT, QMAN_RT_MODE);
 466         }
 467 }
 468
 469 /*
 470  * Checks for a completed response (returns non-NULL if only if the response
 471  * is complete).
 472  */
 473 void *qbman_swp_mc_result(struct qbman_swp *p)
 474 {
 475         u32 *ret, verb;
 476
 477         if ((p->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_5000) {
 478                 ret = qbman_get_cmd(p, QBMAN_CENA_SWP_RR(p->mc.valid_bit));
 479                 /* Remove the valid-bit - command completed if the rest
 480                  * is non-zero.
 481                  */
 482                 verb = ret[0] & ~QB_VALID_BIT;
 483                 if (!verb)
 484                         return NULL;
 485                 p->mc.valid_bit ^= QB_VALID_BIT;
 486         } else {
 487                 ret = qbman_get_cmd(p, QBMAN_CENA_SWP_RR_MEM);
 488                 /* Command completed if the valid bit is toggled */
 489                 if (p->mr.valid_bit != (ret[0] & QB_VALID_BIT))
 490                         return NULL;
 491                 /* Command completed if the rest is non-zero */
 492                 verb = ret[0] & ~QB_VALID_BIT;
 493                 if (!verb)
 494                         return NULL;
 495                 p->mr.valid_bit ^= QB_VALID_BIT;
 496         }
 497
 498         return ret;
 499 }
 500
 501 #define QB_ENQUEUE_CMD_OPTIONS_SHIFT    0
 502 enum qb_enqueue_commands {
 503         enqueue_empty = 0,
 504         enqueue_response_always = 1,
 505         enqueue_rejects_to_fq = 2
 506 };
 507
 508 #define QB_ENQUEUE_CMD_ORP_ENABLE_SHIFT      2
 509 #define QB_ENQUEUE_CMD_IRQ_ON_DISPATCH_SHIFT 3
 510 #define QB_ENQUEUE_CMD_TARGET_TYPE_SHIFT     4
 511 #define QB_ENQUEUE_CMD_DCA_EN_SHIFT          7
 512
 513 /**
 514  * qbman_eq_desc_clear() - Clear the contents of a descriptor to
 515  *                         default/starting state.
 516  */
 517 void qbman_eq_desc_clear(struct qbman_eq_desc *d)
 518 {
 519         memset(d, 0, sizeof(*d));
 520 }
 521
 522 /**
 523  * qbman_eq_desc_set_no_orp() - Set enqueue descriptor without orp
 524  * @d:                the enqueue descriptor.
 525  * @response_success: 1 = enqueue with response always; 0 = enqueue with
 526  *                    rejections returned on a FQ.
 527  */
 528 void qbman_eq_desc_set_no_orp(struct qbman_eq_desc *d, int respond_success)
 529 {
 530         d->verb &= ~(1 << QB_ENQUEUE_CMD_ORP_ENABLE_SHIFT);
 531         if (respond_success)
 532                 d->verb |= enqueue_response_always;
 533         else
 534                 d->verb |= enqueue_rejects_to_fq;
 535 }
 536
 537 /*
 538  * Exactly one of the following descriptor "targets" should be set. (Calling any
 539  * one of these will replace the effect of any prior call to one of these.)
 540  *   -enqueue to a frame queue
 541  *   -enqueue to a queuing destination
 542  */
 543
 544 /**
 545  * qbman_eq_desc_set_fq() - set the FQ for the enqueue command
 546  * @d:    the enqueue descriptor
 547  * @fqid: the id of the frame queue to be enqueued
 548  */
 549 void qbman_eq_desc_set_fq(struct qbman_eq_desc *d, u32 fqid)
 550 {
 551         d->verb &= ~(1 << QB_ENQUEUE_CMD_TARGET_TYPE_SHIFT);
 552         d->tgtid = cpu_to_le32(fqid);
 553 }
 554
 555 /**
 556  * qbman_eq_desc_set_qd() - Set Queuing Destination for the enqueue command
 557  * @d:       the enqueue descriptor
 558  * @qdid:    the id of the queuing destination to be enqueued
 559  * @qd_bin:  the queuing destination bin
 560  * @qd_prio: the queuing destination priority
 561  */
 562 void qbman_eq_desc_set_qd(struct qbman_eq_desc *d, u32 qdid,
 563                           u32 qd_bin, u32 qd_prio)
 564 {
 565         d->verb |= 1 << QB_ENQUEUE_CMD_TARGET_TYPE_SHIFT;
 566         d->tgtid = cpu_to_le32(qdid);
 567         d->qdbin = cpu_to_le16(qd_bin);
 568         d->qpri = qd_prio;
 569 }
 570
 571 #define EQAR_IDX(eqar)     ((eqar) & 0x7)
 572 #define EQAR_VB(eqar)      ((eqar) & 0x80)
 573 #define EQAR_SUCCESS(eqar) ((eqar) & 0x100)
 574
 575 static inline void qbman_write_eqcr_am_rt_register(struct qbman_swp *p,
 576                                                    u8 idx)
 577 {
 578         if (idx < 16)
 579                 qbman_write_register(p, QBMAN_CINH_SWP_EQCR_AM_RT + idx * 4,
 580                                      QMAN_RT_MODE);
 581         else
 582                 qbman_write_register(p, QBMAN_CINH_SWP_EQCR_AM_RT2 +
 583                                      (idx - 16) * 4,
 584                                      QMAN_RT_MODE);
 585 }
 586
 587 #define QB_RT_BIT ((u32)0x100)
 588 /**
 589  * qbman_swp_enqueue_direct() - Issue an enqueue command
 590  * @s:  the software portal used for enqueue
 591  * @d:  the enqueue descriptor
 592  * @fd: the frame descriptor to be enqueued
 593  *
 594  * Please note that 'fd' should only be NULL if the "action" of the
 595  * descriptor is "orp_hole" or "orp_nesn".
 596  *
 597  * Return 0 for successful enqueue, -EBUSY if the EQCR is not ready.
 598  */
 599 static
 600 int qbman_swp_enqueue_direct(struct qbman_swp *s,
 601                              const struct qbman_eq_desc *d,
 602                              const struct dpaa2_fd *fd)
 603 {
 604         int flags = 0;
 605         int ret = qbman_swp_enqueue_multiple_direct(s, d, fd, &flags, 1);
 606
 607         if (ret >= 0)
 608                 ret = 0;
 609         else
 610                 ret = -EBUSY;
 611         return  ret;
 612 }
 613
 614 /**
 615  * qbman_swp_enqueue_mem_back() - Issue an enqueue command
 616  * @s:  the software portal used for enqueue
 617  * @d:  the enqueue descriptor
 618  * @fd: the frame descriptor to be enqueued
 619  *
 620  * Please note that 'fd' should only be NULL if the "action" of the
 621  * descriptor is "orp_hole" or "orp_nesn".
 622  *
 623  * Return 0 for successful enqueue, -EBUSY if the EQCR is not ready.
 624  */
 625 static
 626 int qbman_swp_enqueue_mem_back(struct qbman_swp *s,
 627                                const struct qbman_eq_desc *d,
 628                                const struct dpaa2_fd *fd)
 629 {
 630         int flags = 0;
 631         int ret = qbman_swp_enqueue_multiple_mem_back(s, d, fd, &flags, 1);
 632
 633         if (ret >= 0)
 634                 ret = 0;
 635         else
 636                 ret = -EBUSY;
 637         return  ret;
 638 }
 639
 640 /**
 641  * qbman_swp_enqueue_multiple_direct() - Issue a multi enqueue command
 642  * using one enqueue descriptor
 643  * @s:  the software portal used for enqueue
 644  * @d:  the enqueue descriptor
 645  * @fd: table pointer of frame descriptor table to be enqueued
 646  * @flags: table pointer of QBMAN_ENQUEUE_FLAG_DCA flags, not used if NULL
 647  * @num_frames: number of fd to be enqueued
 648  *
 649  * Return the number of fd enqueued, or a negative error number.
 650  */
 651 static
 652 int qbman_swp_enqueue_multiple_direct(struct qbman_swp *s,
 653                                       const struct qbman_eq_desc *d,
 654                                       const struct dpaa2_fd *fd,
 655                                       uint32_t *flags,
 656                                       int num_frames)
 657 {
 658         uint32_t *p = NULL;
 659         const uint32_t *cl = (uint32_t *)d;
 660         uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask;
 661         int i, num_enqueued = 0;
 662         uint64_t addr_cena;
 663
 664         spin_lock(&s->access_spinlock);
 665         half_mask = (s->eqcr.pi_ci_mask>>1);
 666         full_mask = s->eqcr.pi_ci_mask;
 667
 668         if (!s->eqcr.available) {
 669                 eqcr_ci = s->eqcr.ci;
 670                 p = s->addr_cena + QBMAN_CENA_SWP_EQCR_CI;
 671                 s->eqcr.ci = qbman_read_register(s, QBMAN_CINH_SWP_EQCR_CI);
 672
 673                 s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size,
 674                                         eqcr_ci, s->eqcr.ci);
 675                 if (!s->eqcr.available) {
 676                         spin_unlock(&s->access_spinlock);
 677                         return 0;
 678                 }
 679         }
 680
 681         eqcr_pi = s->eqcr.pi;
 682         num_enqueued = (s->eqcr.available < num_frames) ?
 683                         s->eqcr.available : num_frames;
 684         s->eqcr.available -= num_enqueued;
 685         /* Fill in the EQCR ring */
 686         for (i = 0; i < num_enqueued; i++) {
 687                 p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
 688                 /* Skip copying the verb */
 689                 memcpy(&p[1], &cl[1], EQ_DESC_SIZE_WITHOUT_FD - 1);
 690                 memcpy(&p[EQ_DESC_SIZE_FD_START/sizeof(uint32_t)],
 691                        &fd[i], sizeof(*fd));
 692                 eqcr_pi++;
 693         }
 694
 695         dma_wmb();
 696
 697         /* Set the verb byte, have to substitute in the valid-bit */
 698         eqcr_pi = s->eqcr.pi;
 699         for (i = 0; i < num_enqueued; i++) {
 700                 p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
 701                 p[0] = cl[0] | s->eqcr.pi_vb;
 702                 if (flags && (flags[i] & QBMAN_ENQUEUE_FLAG_DCA)) {
 703                         struct qbman_eq_desc *d = (struct qbman_eq_desc *)p;
 704
 705                         d->dca = (1 << QB_ENQUEUE_CMD_DCA_EN_SHIFT) |
 706                                 ((flags[i]) & QBMAN_EQCR_DCA_IDXMASK);
 707                 }
 708                 eqcr_pi++;
 709                 if (!(eqcr_pi & half_mask))
 710                         s->eqcr.pi_vb ^= QB_VALID_BIT;
 711         }
 712
 713         /* Flush all the cacheline without load/store in between */
 714         eqcr_pi = s->eqcr.pi;
 715         addr_cena = (size_t)s->addr_cena;
 716         for (i = 0; i < num_enqueued; i++)
 717                 eqcr_pi++;
 718         s->eqcr.pi = eqcr_pi & full_mask;
 719         spin_unlock(&s->access_spinlock);
 720
 721         return num_enqueued;
 722 }
 723
 724 /**
 725  * qbman_swp_enqueue_multiple_mem_back() - Issue a multi enqueue command
 726  * using one enqueue descriptor
 727  * @s:  the software portal used for enqueue
 728  * @d:  the enqueue descriptor
 729  * @fd: table pointer of frame descriptor table to be enqueued
 730  * @flags: table pointer of QBMAN_ENQUEUE_FLAG_DCA flags, not used if NULL
 731  * @num_frames: number of fd to be enqueued
 732  *
 733  * Return the number of fd enqueued, or a negative error number.
 734  */
 735 static
 736 int qbman_swp_enqueue_multiple_mem_back(struct qbman_swp *s,
 737                                         const struct qbman_eq_desc *d,
 738                                         const struct dpaa2_fd *fd,
 739                                         uint32_t *flags,
 740                                         int num_frames)
 741 {
 742         uint32_t *p = NULL;
 743         const uint32_t *cl = (uint32_t *)(d);
 744         uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask;
 745         int i, num_enqueued = 0;
 746         unsigned long irq_flags;
 747
 748         spin_lock(&s->access_spinlock);
 749         local_irq_save(irq_flags);
 750
 751         half_mask = (s->eqcr.pi_ci_mask>>1);
 752         full_mask = s->eqcr.pi_ci_mask;
 753         if (!s->eqcr.available) {
 754                 eqcr_ci = s->eqcr.ci;
 755                 p = s->addr_cena + QBMAN_CENA_SWP_EQCR_CI_MEMBACK;
 756                 s->eqcr.ci = __raw_readl(p) & full_mask;
 757                 s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size,
 758                                         eqcr_ci, s->eqcr.ci);
 759                 if (!s->eqcr.available) {
 760                         local_irq_restore(irq_flags);
 761                         spin_unlock(&s->access_spinlock);
 762                         return 0;
 763                 }
 764         }
 765
 766         eqcr_pi = s->eqcr.pi;
 767         num_enqueued = (s->eqcr.available < num_frames) ?
 768                         s->eqcr.available : num_frames;
 769         s->eqcr.available -= num_enqueued;
 770         /* Fill in the EQCR ring */
 771         for (i = 0; i < num_enqueued; i++) {
 772                 p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
 773                 /* Skip copying the verb */
 774                 memcpy(&p[1], &cl[1], EQ_DESC_SIZE_WITHOUT_FD - 1);
 775                 memcpy(&p[EQ_DESC_SIZE_FD_START/sizeof(uint32_t)],
 776                        &fd[i], sizeof(*fd));
 777                 eqcr_pi++;
 778         }
 779
 780         /* Set the verb byte, have to substitute in the valid-bit */
 781         eqcr_pi = s->eqcr.pi;
 782         for (i = 0; i < num_enqueued; i++) {
 783                 p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
 784                 p[0] = cl[0] | s->eqcr.pi_vb;
 785                 if (flags && (flags[i] & QBMAN_ENQUEUE_FLAG_DCA)) {
 786                         struct qbman_eq_desc *d = (struct qbman_eq_desc *)p;
 787
 788                         d->dca = (1 << QB_ENQUEUE_CMD_DCA_EN_SHIFT) |
 789                                 ((flags[i]) & QBMAN_EQCR_DCA_IDXMASK);
 790                 }
 791                 eqcr_pi++;
 792                 if (!(eqcr_pi & half_mask))
 793                         s->eqcr.pi_vb ^= QB_VALID_BIT;
 794         }
 795         s->eqcr.pi = eqcr_pi & full_mask;
 796
 797         dma_wmb();
 798         qbman_write_register(s, QBMAN_CINH_SWP_EQCR_PI,
 799                                 (QB_RT_BIT)|(s->eqcr.pi)|s->eqcr.pi_vb);
 800         local_irq_restore(irq_flags);
 801         spin_unlock(&s->access_spinlock);
 802
 803         return num_enqueued;
 804 }
 805
 806 /**
 807  * qbman_swp_enqueue_multiple_desc_direct() - Issue a multi enqueue command
 808  * using multiple enqueue descriptor
 809  * @s:  the software portal used for enqueue
 810  * @d:  table of minimal enqueue descriptor
 811  * @fd: table pointer of frame descriptor table to be enqueued
 812  * @num_frames: number of fd to be enqueued
 813  *
 814  * Return the number of fd enqueued, or a negative error number.
 815  */
 816 static
 817 int qbman_swp_enqueue_multiple_desc_direct(struct qbman_swp *s,
 818                                            const struct qbman_eq_desc *d,
 819                                            const struct dpaa2_fd *fd,
 820                                            int num_frames)
 821 {
 822         uint32_t *p;
 823         const uint32_t *cl;
 824         uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask;
 825         int i, num_enqueued = 0;
 826         uint64_t addr_cena;
 827
 828         half_mask = (s->eqcr.pi_ci_mask>>1);
 829         full_mask = s->eqcr.pi_ci_mask;
 830         if (!s->eqcr.available) {
 831                 eqcr_ci = s->eqcr.ci;
 832                 p = s->addr_cena + QBMAN_CENA_SWP_EQCR_CI;
 833                 s->eqcr.ci = qbman_read_register(s, QBMAN_CINH_SWP_EQCR_CI);
 834                 s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size,
 835                                         eqcr_ci, s->eqcr.ci);
 836                 if (!s->eqcr.available)
 837                         return 0;
 838         }
 839
 840         eqcr_pi = s->eqcr.pi;
 841         num_enqueued = (s->eqcr.available < num_frames) ?
 842                         s->eqcr.available : num_frames;
 843         s->eqcr.available -= num_enqueued;
 844         /* Fill in the EQCR ring */
 845         for (i = 0; i < num_enqueued; i++) {
 846                 p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
 847                 cl = (uint32_t *)(&d[i]);
 848                 /* Skip copying the verb */
 849                 memcpy(&p[1], &cl[1], EQ_DESC_SIZE_WITHOUT_FD - 1);
 850                 memcpy(&p[EQ_DESC_SIZE_FD_START/sizeof(uint32_t)],
 851                        &fd[i], sizeof(*fd));
 852                 eqcr_pi++;
 853         }
 854
 855         dma_wmb();
 856
 857         /* Set the verb byte, have to substitute in the valid-bit */
 858         eqcr_pi = s->eqcr.pi;
 859         for (i = 0; i < num_enqueued; i++) {
 860                 p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
 861                 cl = (uint32_t *)(&d[i]);
 862                 p[0] = cl[0] | s->eqcr.pi_vb;
 863                 eqcr_pi++;
 864                 if (!(eqcr_pi & half_mask))
 865                         s->eqcr.pi_vb ^= QB_VALID_BIT;
 866         }
 867
 868         /* Flush all the cacheline without load/store in between */
 869         eqcr_pi = s->eqcr.pi;
 870         addr_cena = (uint64_t)s->addr_cena;
 871         for (i = 0; i < num_enqueued; i++)
 872                 eqcr_pi++;
 873         s->eqcr.pi = eqcr_pi & full_mask;
 874
 875         return num_enqueued;
 876 }
 877
 878 /**
 879  * qbman_swp_enqueue_multiple_desc_mem_back() - Issue a multi enqueue command
 880  * using multiple enqueue descriptor
 881  * @s:  the software portal used for enqueue
 882  * @d:  table of minimal enqueue descriptor
 883  * @fd: table pointer of frame descriptor table to be enqueued
 884  * @num_frames: number of fd to be enqueued
 885  *
 886  * Return the number of fd enqueued, or a negative error number.
 887  */
 888 static
 889 int qbman_swp_enqueue_multiple_desc_mem_back(struct qbman_swp *s,
 890                                            const struct qbman_eq_desc *d,
 891                                            const struct dpaa2_fd *fd,
 892                                            int num_frames)
 893 {
 894         uint32_t *p;
 895         const uint32_t *cl;
 896         uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask;
 897         int i, num_enqueued = 0;
 898
 899         half_mask = (s->eqcr.pi_ci_mask>>1);
 900         full_mask = s->eqcr.pi_ci_mask;
 901         if (!s->eqcr.available) {
 902                 eqcr_ci = s->eqcr.ci;
 903                 p = s->addr_cena + QBMAN_CENA_SWP_EQCR_CI_MEMBACK;
 904                 s->eqcr.ci = __raw_readl(p) & full_mask;
 905                 s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size,
 906                                         eqcr_ci, s->eqcr.ci);
 907                 if (!s->eqcr.available)
 908                         return 0;
 909         }
 910
 911         eqcr_pi = s->eqcr.pi;
 912         num_enqueued = (s->eqcr.available < num_frames) ?
 913                         s->eqcr.available : num_frames;
 914         s->eqcr.available -= num_enqueued;
 915         /* Fill in the EQCR ring */
 916         for (i = 0; i < num_enqueued; i++) {
 917                 p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
 918                 cl = (uint32_t *)(&d[i]);
 919                 /* Skip copying the verb */
 920                 memcpy(&p[1], &cl[1], EQ_DESC_SIZE_WITHOUT_FD - 1);
 921                 memcpy(&p[EQ_DESC_SIZE_FD_START/sizeof(uint32_t)],
 922                        &fd[i], sizeof(*fd));
 923                 eqcr_pi++;
 924         }
 925
 926         /* Set the verb byte, have to substitute in the valid-bit */
 927         eqcr_pi = s->eqcr.pi;
 928         for (i = 0; i < num_enqueued; i++) {
 929                 p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
 930                 cl = (uint32_t *)(&d[i]);
 931                 p[0] = cl[0] | s->eqcr.pi_vb;
 932                 eqcr_pi++;
 933                 if (!(eqcr_pi & half_mask))
 934                         s->eqcr.pi_vb ^= QB_VALID_BIT;
 935         }
 936
 937         s->eqcr.pi = eqcr_pi & full_mask;
 938
 939         dma_wmb();
 940         qbman_write_register(s, QBMAN_CINH_SWP_EQCR_PI,
 941                                 (QB_RT_BIT)|(s->eqcr.pi)|s->eqcr.pi_vb);
 942
 943         return num_enqueued;
 944 }
 945
 946 /* Static (push) dequeue */
 947
 948 /**
 949  * qbman_swp_push_get() - Get the push dequeue setup
 950  * @p:           the software portal object
 951  * @channel_idx: the channel index to query
 952  * @enabled:     returned boolean to show whether the push dequeue is enabled
 953  *               for the given channel
 954  */
 955 void qbman_swp_push_get(struct qbman_swp *s, u8 channel_idx, int *enabled)
 956 {
 957         u16 src = (s->sdq >> QB_SDQCR_SRC_SHIFT) & QB_SDQCR_SRC_MASK;
 958
 959         WARN_ON(channel_idx > 15);
 960         *enabled = src | (1 << channel_idx);
 961 }
 962
 963 /**
 964  * qbman_swp_push_set() - Enable or disable push dequeue
 965  * @p:           the software portal object
 966  * @channel_idx: the channel index (0 to 15)
 967  * @enable:      enable or disable push dequeue
 968  */
 969 void qbman_swp_push_set(struct qbman_swp *s, u8 channel_idx, int enable)
 970 {
 971         u16 dqsrc;
 972
 973         WARN_ON(channel_idx > 15);
 974         if (enable)
 975                 s->sdq |= 1 << channel_idx;
 976         else
 977                 s->sdq &= ~(1 << channel_idx);
 978
 979         /* Read make the complete src map.  If no channels are enabled
 980          * the SDQCR must be 0 or else QMan will assert errors
 981          */
 982         dqsrc = (s->sdq >> QB_SDQCR_SRC_SHIFT) & QB_SDQCR_SRC_MASK;
 983         if (dqsrc != 0)
 984                 qbman_write_register(s, QBMAN_CINH_SWP_SDQCR, s->sdq);
 985         else
 986                 qbman_write_register(s, QBMAN_CINH_SWP_SDQCR, 0);
 987 }
 988
 989 #define QB_VDQCR_VERB_DCT_SHIFT    0
 990 #define QB_VDQCR_VERB_DT_SHIFT     2
 991 #define QB_VDQCR_VERB_RLS_SHIFT    4
 992 #define QB_VDQCR_VERB_WAE_SHIFT    5
 993
 994 enum qb_pull_dt_e {
 995         qb_pull_dt_channel,
 996         qb_pull_dt_workqueue,
 997         qb_pull_dt_framequeue
 998 };
 999
1000 /**
1001  * qbman_pull_desc_clear() - Clear the contents of a descriptor to
1002  *                           default/starting state
1003  * @d: the pull dequeue descriptor to be cleared
1004  */
1005 void qbman_pull_desc_clear(struct qbman_pull_desc *d)
1006 {
1007         memset(d, 0, sizeof(*d));
1008 }
1009
1010 /**
1011  * qbman_pull_desc_set_storage()- Set the pull dequeue storage
1012  * @d:            the pull dequeue descriptor to be set
1013  * @storage:      the pointer of the memory to store the dequeue result
1014  * @storage_phys: the physical address of the storage memory
1015  * @stash:        to indicate whether write allocate is enabled
1016  *
1017  * If not called, or if called with 'storage' as NULL, the result pull dequeues
1018  * will produce results to DQRR. If 'storage' is non-NULL, then results are
1019  * produced to the given memory location (using the DMA address which
1020  * the caller provides in 'storage_phys'), and 'stash' controls whether or not
1021  * those writes to main-memory express a cache-warming attribute.
1022  */
1023 void qbman_pull_desc_set_storage(struct qbman_pull_desc *d,
1024                                  struct dpaa2_dq *storage,
1025                                  dma_addr_t storage_phys,
1026                                  int stash)
1027 {
1028         /* save the virtual address */
1029         d->rsp_addr_virt = (u64)(uintptr_t)storage;
1030
1031         if (!storage) {
1032                 d->verb &= ~(1 << QB_VDQCR_VERB_RLS_SHIFT);
1033                 return;
1034         }
1035         d->verb |= 1 << QB_VDQCR_VERB_RLS_SHIFT;
1036         if (stash)
1037                 d->verb |= 1 << QB_VDQCR_VERB_WAE_SHIFT;
1038         else
1039                 d->verb &= ~(1 << QB_VDQCR_VERB_WAE_SHIFT);
1040
1041         d->rsp_addr = cpu_to_le64(storage_phys);
1042 }
1043
1044 /**
1045  * qbman_pull_desc_set_numframes() - Set the number of frames to be dequeued
1046  * @d:         the pull dequeue descriptor to be set
1047  * @numframes: number of frames to be set, must be between 1 and 16, inclusive
1048  */
1049 void qbman_pull_desc_set_numframes(struct qbman_pull_desc *d, u8 numframes)
1050 {
1051         d->numf = numframes - 1;
1052 }
1053
1054 /*
1055  * Exactly one of the following descriptor "actions" should be set. (Calling any
1056  * one of these will replace the effect of any prior call to one of these.)
1057  * - pull dequeue from the given frame queue (FQ)
1058  * - pull dequeue from any FQ in the given work queue (WQ)
1059  * - pull dequeue from any FQ in any WQ in the given channel
1060  */
1061
1062 /**
1063  * qbman_pull_desc_set_fq() - Set fqid from which the dequeue command dequeues
1064  * @fqid: the frame queue index of the given FQ
1065  */
1066 void qbman_pull_desc_set_fq(struct qbman_pull_desc *d, u32 fqid)
1067 {
1068         d->verb |= 1 << QB_VDQCR_VERB_DCT_SHIFT;
1069         d->verb |= qb_pull_dt_framequeue << QB_VDQCR_VERB_DT_SHIFT;
1070         d->dq_src = cpu_to_le32(fqid);
1071 }
1072
1073 /**
1074  * qbman_pull_desc_set_wq() - Set wqid from which the dequeue command dequeues
1075  * @wqid: composed of channel id and wqid within the channel
1076  * @dct:  the dequeue command type
1077  */
1078 void qbman_pull_desc_set_wq(struct qbman_pull_desc *d, u32 wqid,
1079                             enum qbman_pull_type_e dct)
1080 {
1081         d->verb |= dct << QB_VDQCR_VERB_DCT_SHIFT;
1082         d->verb |= qb_pull_dt_workqueue << QB_VDQCR_VERB_DT_SHIFT;
1083         d->dq_src = cpu_to_le32(wqid);
1084 }
1085
1086 /**
1087  * qbman_pull_desc_set_channel() - Set channelid from which the dequeue command
1088  *                                 dequeues
1089  * @chid: the channel id to be dequeued
1090  * @dct:  the dequeue command type
1091  */
1092 void qbman_pull_desc_set_channel(struct qbman_pull_desc *d, u32 chid,
1093                                  enum qbman_pull_type_e dct)
1094 {
1095         d->verb |= dct << QB_VDQCR_VERB_DCT_SHIFT;
1096         d->verb |= qb_pull_dt_channel << QB_VDQCR_VERB_DT_SHIFT;
1097         d->dq_src = cpu_to_le32(chid);
1098 }
1099
1100 /**
1101  * qbman_swp_pull_direct() - Issue the pull dequeue command
1102  * @s: the software portal object
1103  * @d: the software portal descriptor which has been configured with
1104  *     the set of qbman_pull_desc_set_*() calls
1105  *
1106  * Return 0 for success, and -EBUSY if the software portal is not ready
1107  * to do pull dequeue.
1108  */
1109 static
1110 int qbman_swp_pull_direct(struct qbman_swp *s, struct qbman_pull_desc *d)
1111 {
1112         struct qbman_pull_desc *p;
1113
1114         if (!atomic_dec_and_test(&s->vdq.available)) {
1115                 atomic_inc(&s->vdq.available);
1116                 return -EBUSY;
1117         }
1118         s->vdq.storage = (void *)(uintptr_t)d->rsp_addr_virt;
1119         if ((s->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_5000)
1120                 p = qbman_get_cmd(s, QBMAN_CENA_SWP_VDQCR);
1121         else
1122                 p = qbman_get_cmd(s, QBMAN_CENA_SWP_VDQCR_MEM);
1123         p->numf = d->numf;
1124         p->tok = QMAN_DQ_TOKEN_VALID;
1125         p->dq_src = d->dq_src;
1126         p->rsp_addr = d->rsp_addr;
1127         p->rsp_addr_virt = d->rsp_addr_virt;
1128         dma_wmb();
1129         /* Set the verb byte, have to substitute in the valid-bit */
1130         p->verb = d->verb | s->vdq.valid_bit;
1131         s->vdq.valid_bit ^= QB_VALID_BIT;
1132
1133         return 0;
1134 }
1135
1136 /**
1137  * qbman_swp_pull_mem_back() - Issue the pull dequeue command
1138  * @s: the software portal object
1139  * @d: the software portal descriptor which has been configured with
1140  *     the set of qbman_pull_desc_set_*() calls
1141  *
1142  * Return 0 for success, and -EBUSY if the software portal is not ready
1143  * to do pull dequeue.
1144  */
1145 static
1146 int qbman_swp_pull_mem_back(struct qbman_swp *s, struct qbman_pull_desc *d)
1147 {
1148         struct qbman_pull_desc *p;
1149
1150         if (!atomic_dec_and_test(&s->vdq.available)) {
1151                 atomic_inc(&s->vdq.available);
1152                 return -EBUSY;
1153         }
1154         s->vdq.storage = (void *)(uintptr_t)d->rsp_addr_virt;
1155         if ((s->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_5000)
1156                 p = qbman_get_cmd(s, QBMAN_CENA_SWP_VDQCR);
1157         else
1158                 p = qbman_get_cmd(s, QBMAN_CENA_SWP_VDQCR_MEM);
1159         p->numf = d->numf;
1160         p->tok = QMAN_DQ_TOKEN_VALID;
1161         p->dq_src = d->dq_src;
1162         p->rsp_addr = d->rsp_addr;
1163         p->rsp_addr_virt = d->rsp_addr_virt;
1164
1165         /* Set the verb byte, have to substitute in the valid-bit */
1166         p->verb = d->verb | s->vdq.valid_bit;
1167         s->vdq.valid_bit ^= QB_VALID_BIT;
1168         dma_wmb();
1169         qbman_write_register(s, QBMAN_CINH_SWP_VDQCR_RT, QMAN_RT_MODE);
1170
1171         return 0;
1172 }
1173
1174 #define QMAN_DQRR_PI_MASK   0xf
1175
1176 /**
1177  * qbman_swp_dqrr_next_direct() - Get an valid DQRR entry
1178  * @s: the software portal object
1179  *
1180  * Return NULL if there are no unconsumed DQRR entries. Return a DQRR entry
1181  * only once, so repeated calls can return a sequence of DQRR entries, without
1182  * requiring they be consumed immediately or in any particular order.
1183  */
1184 const struct dpaa2_dq *qbman_swp_dqrr_next_direct(struct qbman_swp *s)
1185 {
1186         u32 verb;
1187         u32 response_verb;
1188         u32 flags;
1189         struct dpaa2_dq *p;
1190
1191         /* Before using valid-bit to detect if something is there, we have to
1192          * handle the case of the DQRR reset bug...
1193          */
1194         if (unlikely(s->dqrr.reset_bug)) {
1195                 /*
1196                  * We pick up new entries by cache-inhibited producer index,
1197                  * which means that a non-coherent mapping would require us to
1198                  * invalidate and read *only* once that PI has indicated that
1199                  * there's an entry here. The first trip around the DQRR ring
1200                  * will be much less efficient than all subsequent trips around
1201                  * it...
1202                  */
1203                 u8 pi = qbman_read_register(s, QBMAN_CINH_SWP_DQPI) &
1204                         QMAN_DQRR_PI_MASK;
1205
1206                 /* there are new entries if pi != next_idx */
1207                 if (pi == s->dqrr.next_idx)
1208                         return NULL;
1209
1210                 /*
1211                  * if next_idx is/was the last ring index, and 'pi' is
1212                  * different, we can disable the workaround as all the ring
1213                  * entries have now been DMA'd to so valid-bit checking is
1214                  * repaired. Note: this logic needs to be based on next_idx
1215                  * (which increments one at a time), rather than on pi (which
1216                  * can burst and wrap-around between our snapshots of it).
1217                  */
1218                 if (s->dqrr.next_idx == (s->dqrr.dqrr_size - 1)) {
1219                         pr_debug("next_idx=%d, pi=%d, clear reset bug\n",
1220                                  s->dqrr.next_idx, pi);
1221                         s->dqrr.reset_bug = 0;
1222                 }
1223                 prefetch(qbman_get_cmd(s,
1224                                        QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx)));
1225         }
1226
1227         p = qbman_get_cmd(s, QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx));
1228         verb = p->dq.verb;
1229
1230         /*
1231          * If the valid-bit isn't of the expected polarity, nothing there. Note,
1232          * in the DQRR reset bug workaround, we shouldn't need to skip these
1233          * check, because we've already determined that a new entry is available
1234          * and we've invalidated the cacheline before reading it, so the
1235          * valid-bit behaviour is repaired and should tell us what we already
1236          * knew from reading PI.
1237          */
1238         if ((verb & QB_VALID_BIT) != s->dqrr.valid_bit) {
1239                 prefetch(qbman_get_cmd(s,
1240                                        QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx)));
1241                 return NULL;
1242         }
1243         /*
1244          * There's something there. Move "next_idx" attention to the next ring
1245          * entry (and prefetch it) before returning what we found.
1246          */
1247         s->dqrr.next_idx++;
1248         s->dqrr.next_idx &= s->dqrr.dqrr_size - 1; /* Wrap around */
1249         if (!s->dqrr.next_idx)
1250                 s->dqrr.valid_bit ^= QB_VALID_BIT;
1251
1252         /*
1253          * If this is the final response to a volatile dequeue command
1254          * indicate that the vdq is available
1255          */
1256         flags = p->dq.stat;
1257         response_verb = verb & QBMAN_RESULT_MASK;
1258         if ((response_verb == QBMAN_RESULT_DQ) &&
1259             (flags & DPAA2_DQ_STAT_VOLATILE) &&
1260             (flags & DPAA2_DQ_STAT_EXPIRED))
1261                 atomic_inc(&s->vdq.available);
1262
1263         prefetch(qbman_get_cmd(s, QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx)));
1264
1265         return p;
1266 }
1267
1268 /**
1269  * qbman_swp_dqrr_next_mem_back() - Get an valid DQRR entry
1270  * @s: the software portal object
1271  *
1272  * Return NULL if there are no unconsumed DQRR entries. Return a DQRR entry
1273  * only once, so repeated calls can return a sequence of DQRR entries, without
1274  * requiring they be consumed immediately or in any particular order.
1275  */
1276 const struct dpaa2_dq *qbman_swp_dqrr_next_mem_back(struct qbman_swp *s)
1277 {
1278         u32 verb;
1279         u32 response_verb;
1280         u32 flags;
1281         struct dpaa2_dq *p;
1282
1283         /* Before using valid-bit to detect if something is there, we have to
1284          * handle the case of the DQRR reset bug...
1285          */
1286         if (unlikely(s->dqrr.reset_bug)) {
1287                 /*
1288                  * We pick up new entries by cache-inhibited producer index,
1289                  * which means that a non-coherent mapping would require us to
1290                  * invalidate and read *only* once that PI has indicated that
1291                  * there's an entry here. The first trip around the DQRR ring
1292                  * will be much less efficient than all subsequent trips around
1293                  * it...
1294                  */
1295                 u8 pi = qbman_read_register(s, QBMAN_CINH_SWP_DQPI) &
1296                         QMAN_DQRR_PI_MASK;
1297
1298                 /* there are new entries if pi != next_idx */
1299                 if (pi == s->dqrr.next_idx)
1300                         return NULL;
1301
1302                 /*
1303                  * if next_idx is/was the last ring index, and 'pi' is
1304                  * different, we can disable the workaround as all the ring
1305                  * entries have now been DMA'd to so valid-bit checking is
1306                  * repaired. Note: this logic needs to be based on next_idx
1307                  * (which increments one at a time), rather than on pi (which
1308                  * can burst and wrap-around between our snapshots of it).
1309                  */
1310                 if (s->dqrr.next_idx == (s->dqrr.dqrr_size - 1)) {
1311                         pr_debug("next_idx=%d, pi=%d, clear reset bug\n",
1312                                  s->dqrr.next_idx, pi);
1313                         s->dqrr.reset_bug = 0;
1314                 }
1315                 prefetch(qbman_get_cmd(s,
1316                                        QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx)));
1317         }
1318
1319         p = qbman_get_cmd(s, QBMAN_CENA_SWP_DQRR_MEM(s->dqrr.next_idx));
1320         verb = p->dq.verb;
1321
1322         /*
1323          * If the valid-bit isn't of the expected polarity, nothing there. Note,
1324          * in the DQRR reset bug workaround, we shouldn't need to skip these
1325          * check, because we've already determined that a new entry is available
1326          * and we've invalidated the cacheline before reading it, so the
1327          * valid-bit behaviour is repaired and should tell us what we already
1328          * knew from reading PI.
1329          */
1330         if ((verb & QB_VALID_BIT) != s->dqrr.valid_bit) {
1331                 prefetch(qbman_get_cmd(s,
1332                                        QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx)));
1333                 return NULL;
1334         }
1335         /*
1336          * There's something there. Move "next_idx" attention to the next ring
1337          * entry (and prefetch it) before returning what we found.
1338          */
1339         s->dqrr.next_idx++;
1340         s->dqrr.next_idx &= s->dqrr.dqrr_size - 1; /* Wrap around */
1341         if (!s->dqrr.next_idx)
1342                 s->dqrr.valid_bit ^= QB_VALID_BIT;
1343
1344         /*
1345          * If this is the final response to a volatile dequeue command
1346          * indicate that the vdq is available
1347          */
1348         flags = p->dq.stat;
1349         response_verb = verb & QBMAN_RESULT_MASK;
1350         if ((response_verb == QBMAN_RESULT_DQ) &&
1351             (flags & DPAA2_DQ_STAT_VOLATILE) &&
1352             (flags & DPAA2_DQ_STAT_EXPIRED))
1353                 atomic_inc(&s->vdq.available);
1354
1355         prefetch(qbman_get_cmd(s, QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx)));
1356
1357         return p;
1358 }
1359
1360 /**
1361  * qbman_swp_dqrr_consume() -  Consume DQRR entries previously returned from
1362  *                             qbman_swp_dqrr_next().
1363  * @s: the software portal object
1364  * @dq: the DQRR entry to be consumed
1365  */
1366 void qbman_swp_dqrr_consume(struct qbman_swp *s, const struct dpaa2_dq *dq)
1367 {
1368         qbman_write_register(s, QBMAN_CINH_SWP_DCAP, QBMAN_IDX_FROM_DQRR(dq));
1369 }
1370
1371 /**
1372  * qbman_result_has_new_result() - Check and get the dequeue response from the
1373  *                                 dq storage memory set in pull dequeue command
1374  * @s: the software portal object
1375  * @dq: the dequeue result read from the memory
1376  *
1377  * Return 1 for getting a valid dequeue result, or 0 for not getting a valid
1378  * dequeue result.
1379  *
1380  * Only used for user-provided storage of dequeue results, not DQRR. For
1381  * efficiency purposes, the driver will perform any required endianness
1382  * conversion to ensure that the user's dequeue result storage is in host-endian
1383  * format. As such, once the user has called qbman_result_has_new_result() and
1384  * been returned a valid dequeue result, they should not call it again on
1385  * the same memory location (except of course if another dequeue command has
1386  * been executed to produce a new result to that location).
1387  */
1388 int qbman_result_has_new_result(struct qbman_swp *s, const struct dpaa2_dq *dq)
1389 {
1390         if (dq->dq.tok != QMAN_DQ_TOKEN_VALID)
1391                 return 0;
1392
1393         /*
1394          * Set token to be 0 so we will detect change back to 1
1395          * next time the looping is traversed. Const is cast away here
1396          * as we want users to treat the dequeue responses as read only.
1397          */
1398         ((struct dpaa2_dq *)dq)->dq.tok = 0;
1399
1400         /*
1401          * Determine whether VDQCR is available based on whether the
1402          * current result is sitting in the first storage location of
1403          * the busy command.
1404          */
1405         if (s->vdq.storage == dq) {
1406                 s->vdq.storage = NULL;
1407                 atomic_inc(&s->vdq.available);
1408         }
1409
1410         return 1;
1411 }
1412
1413 /**
1414  * qbman_release_desc_clear() - Clear the contents of a descriptor to
1415  *                              default/starting state.
1416  */
1417 void qbman_release_desc_clear(struct qbman_release_desc *d)
1418 {
1419         memset(d, 0, sizeof(*d));
1420         d->verb = 1 << 5; /* Release Command Valid */
1421 }
1422
1423 /**
1424  * qbman_release_desc_set_bpid() - Set the ID of the buffer pool to release to
1425  */
1426 void qbman_release_desc_set_bpid(struct qbman_release_desc *d, u16 bpid)
1427 {
1428         d->bpid = cpu_to_le16(bpid);
1429 }
1430
1431 /**
1432  * qbman_release_desc_set_rcdi() - Determines whether or not the portal's RCDI
1433  * interrupt source should be asserted after the release command is completed.
1434  */
1435 void qbman_release_desc_set_rcdi(struct qbman_release_desc *d, int enable)
1436 {
1437         if (enable)
1438                 d->verb |= 1 << 6;
1439         else
1440                 d->verb &= ~(1 << 6);
1441 }
1442
1443 #define RAR_IDX(rar)     ((rar) & 0x7)
1444 #define RAR_VB(rar)      ((rar) & 0x80)
1445 #define RAR_SUCCESS(rar) ((rar) & 0x100)
1446
1447 /**
1448  * qbman_swp_release_direct() - Issue a buffer release command
1449  * @s:           the software portal object
1450  * @d:           the release descriptor
1451  * @buffers:     a pointer pointing to the buffer address to be released
1452  * @num_buffers: number of buffers to be released,  must be less than 8
1453  *
1454  * Return 0 for success, -EBUSY if the release command ring is not ready.
1455  */
1456 int qbman_swp_release_direct(struct qbman_swp *s,
1457                              const struct qbman_release_desc *d,
1458                              const u64 *buffers, unsigned int num_buffers)
1459 {
1460         int i;
1461         struct qbman_release_desc *p;
1462         u32 rar;
1463
1464         if (!num_buffers || (num_buffers > 7))
1465                 return -EINVAL;
1466
1467         rar = qbman_read_register(s, QBMAN_CINH_SWP_RAR);
1468         if (!RAR_SUCCESS(rar))
1469                 return -EBUSY;
1470
1471         /* Start the release command */
1472         p = qbman_get_cmd(s, QBMAN_CENA_SWP_RCR(RAR_IDX(rar)));
1473
1474         /* Copy the caller's buffer pointers to the command */
1475         for (i = 0; i < num_buffers; i++)
1476                 p->buf[i] = cpu_to_le64(buffers[i]);
1477         p->bpid = d->bpid;
1478
1479         /*
1480          * Set the verb byte, have to substitute in the valid-bit
1481          * and the number of buffers.
1482          */
1483         dma_wmb();
1484         p->verb = d->verb | RAR_VB(rar) | num_buffers;
1485
1486         return 0;
1487 }
1488
1489 /**
1490  * qbman_swp_release_mem_back() - Issue a buffer release command
1491  * @s:           the software portal object
1492  * @d:           the release descriptor
1493  * @buffers:     a pointer pointing to the buffer address to be released
1494  * @num_buffers: number of buffers to be released,  must be less than 8
1495  *
1496  * Return 0 for success, -EBUSY if the release command ring is not ready.
1497  */
1498 int qbman_swp_release_mem_back(struct qbman_swp *s,
1499                                const struct qbman_release_desc *d,
1500                                const u64 *buffers, unsigned int num_buffers)
1501 {
1502         int i;
1503         struct qbman_release_desc *p;
1504         u32 rar;
1505
1506         if (!num_buffers || (num_buffers > 7))
1507                 return -EINVAL;
1508
1509         rar = qbman_read_register(s, QBMAN_CINH_SWP_RAR);
1510         if (!RAR_SUCCESS(rar))
1511                 return -EBUSY;
1512
1513         /* Start the release command */
1514         p = qbman_get_cmd(s, QBMAN_CENA_SWP_RCR_MEM(RAR_IDX(rar)));
1515
1516         /* Copy the caller's buffer pointers to the command */
1517         for (i = 0; i < num_buffers; i++)
1518                 p->buf[i] = cpu_to_le64(buffers[i]);
1519         p->bpid = d->bpid;
1520
1521         p->verb = d->verb | RAR_VB(rar) | num_buffers;
1522         dma_wmb();
1523         qbman_write_register(s, QBMAN_CINH_SWP_RCR_AM_RT +
1524                              RAR_IDX(rar)  * 4, QMAN_RT_MODE);
1525
1526         return 0;
1527 }
1528
1529 struct qbman_acquire_desc {
1530         u8 verb;
1531         u8 reserved;
1532         __le16 bpid;
1533         u8 num;
1534         u8 reserved2[59];
1535 };
1536
1537 struct qbman_acquire_rslt {
1538         u8 verb;
1539         u8 rslt;
1540         __le16 reserved;
1541         u8 num;
1542         u8 reserved2[3];
1543         __le64 buf[7];
1544 };
1545
1546 /**
1547  * qbman_swp_acquire() - Issue a buffer acquire command
1548  * @s:           the software portal object
1549  * @bpid:        the buffer pool index
1550  * @buffers:     a pointer pointing to the acquired buffer addresses
1551  * @num_buffers: number of buffers to be acquired, must be less than 8
1552  *
1553  * Return 0 for success, or negative error code if the acquire command
1554  * fails.
1555  */
1556 int qbman_swp_acquire(struct qbman_swp *s, u16 bpid, u64 *buffers,
1557                       unsigned int num_buffers)
1558 {
1559         struct qbman_acquire_desc *p;
1560         struct qbman_acquire_rslt *r;
1561         int i;
1562
1563         if (!num_buffers || (num_buffers > 7))
1564                 return -EINVAL;
1565
1566         /* Start the management command */
1567         p = qbman_swp_mc_start(s);
1568
1569         if (!p)
1570                 return -EBUSY;
1571
1572         /* Encode the caller-provided attributes */
1573         p->bpid = cpu_to_le16(bpid);
1574         p->num = num_buffers;
1575
1576         /* Complete the management command */
1577         r = qbman_swp_mc_complete(s, p, QBMAN_MC_ACQUIRE);
1578         if (unlikely(!r)) {
1579                 pr_err("qbman: acquire from BPID %d failed, no response\n",
1580                        bpid);
1581                 return -EIO;
1582         }
1583
1584         /* Decode the outcome */
1585         WARN_ON((r->verb & 0x7f) != QBMAN_MC_ACQUIRE);
1586
1587         /* Determine success or failure */
1588         if (unlikely(r->rslt != QBMAN_MC_RSLT_OK)) {
1589                 pr_err("qbman: acquire from BPID 0x%x failed, code=0x%02x\n",
1590                        bpid, r->rslt);
1591                 return -EIO;
1592         }
1593
1594         WARN_ON(r->num > num_buffers);
1595
1596         /* Copy the acquired buffers to the caller's array */
1597         for (i = 0; i < r->num; i++)
1598                 buffers[i] = le64_to_cpu(r->buf[i]);
1599
1600         return (int)r->num;
1601 }
1602
1603 struct qbman_alt_fq_state_desc {
1604         u8 verb;
1605         u8 reserved[3];
1606         __le32 fqid;
1607         u8 reserved2[56];
1608 };
1609
1610 struct qbman_alt_fq_state_rslt {
1611         u8 verb;
1612         u8 rslt;
1613         u8 reserved[62];
1614 };
1615
1616 #define ALT_FQ_FQID_MASK 0x00FFFFFF
1617
1618 int qbman_swp_alt_fq_state(struct qbman_swp *s, u32 fqid,
1619                            u8 alt_fq_verb)
1620 {
1621         struct qbman_alt_fq_state_desc *p;
1622         struct qbman_alt_fq_state_rslt *r;
1623
1624         /* Start the management command */
1625         p = qbman_swp_mc_start(s);
1626         if (!p)
1627                 return -EBUSY;
1628
1629         p->fqid = cpu_to_le32(fqid & ALT_FQ_FQID_MASK);
1630
1631         /* Complete the management command */
1632         r = qbman_swp_mc_complete(s, p, alt_fq_verb);
1633         if (unlikely(!r)) {
1634                 pr_err("qbman: mgmt cmd failed, no response (verb=0x%x)\n",
1635                        alt_fq_verb);
1636                 return -EIO;
1637         }
1638
1639         /* Decode the outcome */
1640         WARN_ON((r->verb & QBMAN_RESULT_MASK) != alt_fq_verb);
1641
1642         /* Determine success or failure */
1643         if (unlikely(r->rslt != QBMAN_MC_RSLT_OK)) {
1644                 pr_err("qbman: ALT FQID %d failed: verb = 0x%08x code = 0x%02x\n",
1645                        fqid, r->verb, r->rslt);
1646                 return -EIO;
1647         }
1648
1649         return 0;
1650 }
1651
1652 struct qbman_cdan_ctrl_desc {
1653         u8 verb;
1654         u8 reserved;
1655         __le16 ch;
1656         u8 we;
1657         u8 ctrl;
1658         __le16 reserved2;
1659         __le64 cdan_ctx;
1660         u8 reserved3[48];
1661
1662 };
1663
1664 struct qbman_cdan_ctrl_rslt {
1665         u8 verb;
1666         u8 rslt;
1667         __le16 ch;
1668         u8 reserved[60];
1669 };
1670
1671 int qbman_swp_CDAN_set(struct qbman_swp *s, u16 channelid,
1672                        u8 we_mask, u8 cdan_en,
1673                        u64 ctx)
1674 {
1675         struct qbman_cdan_ctrl_desc *p = NULL;
1676         struct qbman_cdan_ctrl_rslt *r = NULL;
1677
1678         /* Start the management command */
1679         p = qbman_swp_mc_start(s);
1680         if (!p)
1681                 return -EBUSY;
1682
1683         /* Encode the caller-provided attributes */
1684         p->ch = cpu_to_le16(channelid);
1685         p->we = we_mask;
1686         if (cdan_en)
1687                 p->ctrl = 1;
1688         else
1689                 p->ctrl = 0;
1690         p->cdan_ctx = cpu_to_le64(ctx);
1691
1692         /* Complete the management command */
1693         r = qbman_swp_mc_complete(s, p, QBMAN_WQCHAN_CONFIGURE);
1694         if (unlikely(!r)) {
1695                 pr_err("qbman: wqchan config failed, no response\n");
1696                 return -EIO;
1697         }
1698
1699         WARN_ON((r->verb & 0x7f) != QBMAN_WQCHAN_CONFIGURE);
1700
1701         /* Determine success or failure */
1702         if (unlikely(r->rslt != QBMAN_MC_RSLT_OK)) {
1703                 pr_err("qbman: CDAN cQID %d failed: code = 0x%02x\n",
1704                        channelid, r->rslt);
1705                 return -EIO;
1706         }
1707
1708         return 0;
1709 }
1710
1711 #define QBMAN_RESPONSE_VERB_MASK        0x7f
1712 #define QBMAN_FQ_QUERY_NP               0x45
1713 #define QBMAN_BP_QUERY                  0x32
1714
1715 struct qbman_fq_query_desc {
1716         u8 verb;
1717         u8 reserved[3];
1718         __le32 fqid;
1719         u8 reserved2[56];
1720 };
1721
1722 int qbman_fq_query_state(struct qbman_swp *s, u32 fqid,
1723                          struct qbman_fq_query_np_rslt *r)
1724 {
1725         struct qbman_fq_query_desc *p;
1726         void *resp;
1727
1728         p = (struct qbman_fq_query_desc *)qbman_swp_mc_start(s);
1729         if (!p)
1730                 return -EBUSY;
1731
1732         /* FQID is a 24 bit value */
1733         p->fqid = cpu_to_le32(fqid & 0x00FFFFFF);
1734         resp = qbman_swp_mc_complete(s, p, QBMAN_FQ_QUERY_NP);
1735         if (!resp) {
1736                 pr_err("qbman: Query FQID %d NP fields failed, no response\n",
1737                        fqid);
1738                 return -EIO;
1739         }
1740         *r = *(struct qbman_fq_query_np_rslt *)resp;
1741         /* Decode the outcome */
1742         WARN_ON((r->verb & QBMAN_RESPONSE_VERB_MASK) != QBMAN_FQ_QUERY_NP);
1743
1744         /* Determine success or failure */
1745         if (r->rslt != QBMAN_MC_RSLT_OK) {
1746                 pr_err("Query NP fields of FQID 0x%x failed, code=0x%02x\n",
1747                        p->fqid, r->rslt);
1748                 return -EIO;
1749         }
1750
1751         return 0;
1752 }
1753
1754 u32 qbman_fq_state_frame_count(const struct qbman_fq_query_np_rslt *r)
1755 {
1756         return (le32_to_cpu(r->frm_cnt) & 0x00FFFFFF);
1757 }
1758
1759 u32 qbman_fq_state_byte_count(const struct qbman_fq_query_np_rslt *r)
1760 {
1761         return le32_to_cpu(r->byte_cnt);
1762 }
1763
1764 struct qbman_bp_query_desc {
1765         u8 verb;
1766         u8 reserved;
1767         __le16 bpid;
1768         u8 reserved2[60];
1769 };
1770
1771 int qbman_bp_query(struct qbman_swp *s, u16 bpid,
1772                    struct qbman_bp_query_rslt *r)
1773 {
1774         struct qbman_bp_query_desc *p;
1775         void *resp;
1776
1777         p = (struct qbman_bp_query_desc *)qbman_swp_mc_start(s);
1778         if (!p)
1779                 return -EBUSY;
1780
1781         p->bpid = cpu_to_le16(bpid);
1782         resp = qbman_swp_mc_complete(s, p, QBMAN_BP_QUERY);
1783         if (!resp) {
1784                 pr_err("qbman: Query BPID %d fields failed, no response\n",
1785                        bpid);
1786                 return -EIO;
1787         }
1788         *r = *(struct qbman_bp_query_rslt *)resp;
1789         /* Decode the outcome */
1790         WARN_ON((r->verb & QBMAN_RESPONSE_VERB_MASK) != QBMAN_BP_QUERY);
1791
1792         /* Determine success or failure */
1793         if (r->rslt != QBMAN_MC_RSLT_OK) {
1794                 pr_err("Query fields of BPID 0x%x failed, code=0x%02x\n",
1795                        bpid, r->rslt);
1796                 return -EIO;
1797         }
1798
1799         return 0;
1800 }
1801
1802 u32 qbman_bp_info_num_free_bufs(struct qbman_bp_query_rslt *a)
1803 {
1804         return le32_to_cpu(a->fill);
1805 }