include/linux/qed/qed_chain.h

   1 /* SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) */
   2 /* QLogic qed NIC Driver
   3  * Copyright (c) 2015-2017  QLogic Corporation
   4  * Copyright (c) 2019-2020 Marvell International Ltd.
   5  */
   6
   7 #ifndef _QED_CHAIN_H
   8 #define _QED_CHAIN_H
   9
  10 #include <linux/types.h>
  11 #include <asm/byteorder.h>
  12 #include <linux/kernel.h>
  13 #include <linux/list.h>
  14 #include <linux/slab.h>
  15 #include <linux/qed/common_hsi.h>
  16
  17 enum qed_chain_mode {
  18         /* Each Page contains a next pointer at its end */
  19         QED_CHAIN_MODE_NEXT_PTR,
  20
  21         /* Chain is a single page (next ptr) is unrequired */
  22         QED_CHAIN_MODE_SINGLE,
  23
  24         /* Page pointers are located in a side list */
  25         QED_CHAIN_MODE_PBL,
  26 };
  27
  28 enum qed_chain_use_mode {
  29         QED_CHAIN_USE_TO_PRODUCE,                       /* Chain starts empty */
  30         QED_CHAIN_USE_TO_CONSUME,                       /* Chain starts full */
  31         QED_CHAIN_USE_TO_CONSUME_PRODUCE,               /* Chain starts empty */
  32 };
  33
  34 enum qed_chain_cnt_type {
  35         /* The chain's size/prod/cons are kept in 16-bit variables */
  36         QED_CHAIN_CNT_TYPE_U16,
  37
  38         /* The chain's size/prod/cons are kept in 32-bit variables  */
  39         QED_CHAIN_CNT_TYPE_U32,
  40 };
  41
  42 struct qed_chain_next {
  43         struct regpair                                  next_phys;
  44         void                                            *next_virt;
  45 };
  46
  47 struct qed_chain_pbl_u16 {
  48         u16                                             prod_page_idx;
  49         u16                                             cons_page_idx;
  50 };
  51
  52 struct qed_chain_pbl_u32 {
  53         u32                                             prod_page_idx;
  54         u32                                             cons_page_idx;
  55 };
  56
  57 struct qed_chain_ext_pbl {
  58         dma_addr_t                                      p_pbl_phys;
  59         void                                            *p_pbl_virt;
  60 };
  61
  62 struct qed_chain_u16 {
  63         /* Cyclic index of next element to produce/consme */
  64         u16                                             prod_idx;
  65         u16                                             cons_idx;
  66 };
  67
  68 struct qed_chain_u32 {
  69         /* Cyclic index of next element to produce/consme */
  70         u32                                             prod_idx;
  71         u32                                             cons_idx;
  72 };
  73
  74 struct addr_tbl_entry {
  75         void                                            *virt_addr;
  76         dma_addr_t                                      dma_map;
  77 };
  78
  79 struct qed_chain {
  80         /* Fastpath portion of the chain - required for commands such
  81          * as produce / consume.
  82          */
  83
  84         /* Point to next element to produce/consume */
  85         void                                            *p_prod_elem;
  86         void                                            *p_cons_elem;
  87
  88         /* Fastpath portions of the PBL [if exists] */
  89
  90         struct {
  91                 /* Table for keeping the virtual and physical addresses of the
  92                  * chain pages, respectively to the physical addresses
  93                  * in the pbl table.
  94                  */
  95                 struct addr_tbl_entry                   *pp_addr_tbl;
  96
  97                 union {
  98                         struct qed_chain_pbl_u16        u16;
  99                         struct qed_chain_pbl_u32        u32;
 100                 }                                       c;
 101         }                                               pbl;
 102
 103         union {
 104                 struct qed_chain_u16                    chain16;
 105                 struct qed_chain_u32                    chain32;
 106         }                                               u;
 107
 108         /* Capacity counts only usable elements */
 109         u32                                             capacity;
 110         u32                                             page_cnt;
 111
 112         enum qed_chain_mode                             mode;
 113
 114         /* Elements information for fast calculations */
 115         u16                                             elem_per_page;
 116         u16                                             elem_per_page_mask;
 117         u16                                             elem_size;
 118         u16                                             next_page_mask;
 119         u16                                             usable_per_page;
 120         u8                                              elem_unusable;
 121
 122         u8                                              cnt_type;
 123
 124         /* Slowpath of the chain - required for initialization and destruction,
 125          * but isn't involved in regular functionality.
 126          */
 127
 128         /* Base address of a pre-allocated buffer for pbl */
 129         struct {
 130                 __le64                                  *table_virt;
 131                 dma_addr_t                              table_phys;
 132                 size_t                                  table_size;
 133         }                                               pbl_sp;
 134
 135         /* Address of first page of the chain - the address is required
 136          * for fastpath operation [consume/produce] but only for the SINGLE
 137          * flavour which isn't considered fastpath [== SPQ].
 138          */
 139         void                                            *p_virt_addr;
 140         dma_addr_t                                      p_phys_addr;
 141
 142         /* Total number of elements [for entire chain] */
 143         u32                                             size;
 144
 145         u8                                              intended_use;
 146
 147         bool                                            b_external_pbl;
 148 };
 149
 150 #define QED_CHAIN_PAGE_SIZE                             0x1000
 151
 152 #define ELEMS_PER_PAGE(elem_size)                                            \
 153         (QED_CHAIN_PAGE_SIZE / (elem_size))
 154
 155 #define UNUSABLE_ELEMS_PER_PAGE(elem_size, mode)                             \
 156         (((mode) == QED_CHAIN_MODE_NEXT_PTR) ?                               \
 157          (u8)(1 + ((sizeof(struct qed_chain_next) - 1) / (elem_size))) :     \
 158          0)
 159
 160 #define USABLE_ELEMS_PER_PAGE(elem_size, mode)                               \
 161         ((u32)(ELEMS_PER_PAGE(elem_size) -                                   \
 162                UNUSABLE_ELEMS_PER_PAGE((elem_size), (mode))))
 163
 164 #define QED_CHAIN_PAGE_CNT(elem_cnt, elem_size, mode)                        \
 165         DIV_ROUND_UP((elem_cnt), USABLE_ELEMS_PER_PAGE((elem_size), (mode)))
 166
 167 #define is_chain_u16(p)                                                      \
 168         ((p)->cnt_type == QED_CHAIN_CNT_TYPE_U16)
 169 #define is_chain_u32(p)                                                      \
 170         ((p)->cnt_type == QED_CHAIN_CNT_TYPE_U32)
 171
 172 /* Accessors */
 173 static inline u16 qed_chain_get_prod_idx(struct qed_chain *p_chain)
 174 {
 175         return p_chain->u.chain16.prod_idx;
 176 }
 177
 178 static inline u16 qed_chain_get_cons_idx(struct qed_chain *p_chain)
 179 {
 180         return p_chain->u.chain16.cons_idx;
 181 }
 182
 183 static inline u32 qed_chain_get_cons_idx_u32(struct qed_chain *p_chain)
 184 {
 185         return p_chain->u.chain32.cons_idx;
 186 }
 187
 188 static inline u16 qed_chain_get_elem_left(struct qed_chain *p_chain)
 189 {
 190         u16 elem_per_page = p_chain->elem_per_page;
 191         u32 prod = p_chain->u.chain16.prod_idx;
 192         u32 cons = p_chain->u.chain16.cons_idx;
 193         u16 used;
 194
 195         if (prod < cons)
 196                 prod += (u32)U16_MAX + 1;
 197
 198         used = (u16)(prod - cons);
 199         if (p_chain->mode == QED_CHAIN_MODE_NEXT_PTR)
 200                 used -= prod / elem_per_page - cons / elem_per_page;
 201
 202         return (u16)(p_chain->capacity - used);
 203 }
 204
 205 static inline u32 qed_chain_get_elem_left_u32(struct qed_chain *p_chain)
 206 {
 207         u16 elem_per_page = p_chain->elem_per_page;
 208         u64 prod = p_chain->u.chain32.prod_idx;
 209         u64 cons = p_chain->u.chain32.cons_idx;
 210         u32 used;
 211
 212         if (prod < cons)
 213                 prod += (u64)U32_MAX + 1;
 214
 215         used = (u32)(prod - cons);
 216         if (p_chain->mode == QED_CHAIN_MODE_NEXT_PTR)
 217                 used -= (u32)(prod / elem_per_page - cons / elem_per_page);
 218
 219         return p_chain->capacity - used;
 220 }
 221
 222 static inline u16 qed_chain_get_usable_per_page(struct qed_chain *p_chain)
 223 {
 224         return p_chain->usable_per_page;
 225 }
 226
 227 static inline u8 qed_chain_get_unusable_per_page(struct qed_chain *p_chain)
 228 {
 229         return p_chain->elem_unusable;
 230 }
 231
 232 static inline u32 qed_chain_get_page_cnt(struct qed_chain *p_chain)
 233 {
 234         return p_chain->page_cnt;
 235 }
 236
 237 static inline dma_addr_t qed_chain_get_pbl_phys(struct qed_chain *p_chain)
 238 {
 239         return p_chain->pbl_sp.table_phys;
 240 }
 241
 242 /**
 243  * @brief qed_chain_advance_page -
 244  *
 245  * Advance the next element accros pages for a linked chain
 246  *
 247  * @param p_chain
 248  * @param p_next_elem
 249  * @param idx_to_inc
 250  * @param page_to_inc
 251  */
 252 static inline void
 253 qed_chain_advance_page(struct qed_chain *p_chain,
 254                        void **p_next_elem, void *idx_to_inc, void *page_to_inc)
 255 {
 256         struct qed_chain_next *p_next = NULL;
 257         u32 page_index = 0;
 258
 259         switch (p_chain->mode) {
 260         case QED_CHAIN_MODE_NEXT_PTR:
 261                 p_next = *p_next_elem;
 262                 *p_next_elem = p_next->next_virt;
 263                 if (is_chain_u16(p_chain))
 264                         *(u16 *)idx_to_inc += p_chain->elem_unusable;
 265                 else
 266                         *(u32 *)idx_to_inc += p_chain->elem_unusable;
 267                 break;
 268         case QED_CHAIN_MODE_SINGLE:
 269                 *p_next_elem = p_chain->p_virt_addr;
 270                 break;
 271
 272         case QED_CHAIN_MODE_PBL:
 273                 if (is_chain_u16(p_chain)) {
 274                         if (++(*(u16 *)page_to_inc) == p_chain->page_cnt)
 275                                 *(u16 *)page_to_inc = 0;
 276                         page_index = *(u16 *)page_to_inc;
 277                 } else {
 278                         if (++(*(u32 *)page_to_inc) == p_chain->page_cnt)
 279                                 *(u32 *)page_to_inc = 0;
 280                         page_index = *(u32 *)page_to_inc;
 281                 }
 282                 *p_next_elem = p_chain->pbl.pp_addr_tbl[page_index].virt_addr;
 283         }
 284 }
 285
 286 #define is_unusable_idx(p, idx) \
 287         (((p)->u.chain16.idx & (p)->elem_per_page_mask) == (p)->usable_per_page)
 288
 289 #define is_unusable_idx_u32(p, idx) \
 290         (((p)->u.chain32.idx & (p)->elem_per_page_mask) == (p)->usable_per_page)
 291 #define is_unusable_next_idx(p, idx)                             \
 292         ((((p)->u.chain16.idx + 1) & (p)->elem_per_page_mask) == \
 293          (p)->usable_per_page)
 294
 295 #define is_unusable_next_idx_u32(p, idx)                         \
 296         ((((p)->u.chain32.idx + 1) & (p)->elem_per_page_mask) == \
 297          (p)->usable_per_page)
 298
 299 #define test_and_skip(p, idx)                                              \
 300         do {                                            \
 301                 if (is_chain_u16(p)) {                                     \
 302                         if (is_unusable_idx(p, idx))                       \
 303                                 (p)->u.chain16.idx += (p)->elem_unusable;  \
 304                 } else {                                                   \
 305                         if (is_unusable_idx_u32(p, idx))                   \
 306                                 (p)->u.chain32.idx += (p)->elem_unusable;  \
 307                 }                                       \
 308         } while (0)
 309
 310 /**
 311  * @brief qed_chain_return_produced -
 312  *
 313  * A chain in which the driver "Produces" elements should use this API
 314  * to indicate previous produced elements are now consumed.
 315  *
 316  * @param p_chain
 317  */
 318 static inline void qed_chain_return_produced(struct qed_chain *p_chain)
 319 {
 320         if (is_chain_u16(p_chain))
 321                 p_chain->u.chain16.cons_idx++;
 322         else
 323                 p_chain->u.chain32.cons_idx++;
 324         test_and_skip(p_chain, cons_idx);
 325 }
 326
 327 /**
 328  * @brief qed_chain_produce -
 329  *
 330  * A chain in which the driver "Produces" elements should use this to get
 331  * a pointer to the next element which can be "Produced". It's driver
 332  * responsibility to validate that the chain has room for new element.
 333  *
 334  * @param p_chain
 335  *
 336  * @return void*, a pointer to next element
 337  */
 338 static inline void *qed_chain_produce(struct qed_chain *p_chain)
 339 {
 340         void *p_ret = NULL, *p_prod_idx, *p_prod_page_idx;
 341
 342         if (is_chain_u16(p_chain)) {
 343                 if ((p_chain->u.chain16.prod_idx &
 344                      p_chain->elem_per_page_mask) == p_chain->next_page_mask) {
 345                         p_prod_idx = &p_chain->u.chain16.prod_idx;
 346                         p_prod_page_idx = &p_chain->pbl.c.u16.prod_page_idx;
 347                         qed_chain_advance_page(p_chain, &p_chain->p_prod_elem,
 348                                                p_prod_idx, p_prod_page_idx);
 349                 }
 350                 p_chain->u.chain16.prod_idx++;
 351         } else {
 352                 if ((p_chain->u.chain32.prod_idx &
 353                      p_chain->elem_per_page_mask) == p_chain->next_page_mask) {
 354                         p_prod_idx = &p_chain->u.chain32.prod_idx;
 355                         p_prod_page_idx = &p_chain->pbl.c.u32.prod_page_idx;
 356                         qed_chain_advance_page(p_chain, &p_chain->p_prod_elem,
 357                                                p_prod_idx, p_prod_page_idx);
 358                 }
 359                 p_chain->u.chain32.prod_idx++;
 360         }
 361
 362         p_ret = p_chain->p_prod_elem;
 363         p_chain->p_prod_elem = (void *)(((u8 *)p_chain->p_prod_elem) +
 364                                         p_chain->elem_size);
 365
 366         return p_ret;
 367 }
 368
 369 /**
 370  * @brief qed_chain_get_capacity -
 371  *
 372  * Get the maximum number of BDs in chain
 373  *
 374  * @param p_chain
 375  * @param num
 376  *
 377  * @return number of unusable BDs
 378  */
 379 static inline u32 qed_chain_get_capacity(struct qed_chain *p_chain)
 380 {
 381         return p_chain->capacity;
 382 }
 383
 384 /**
 385  * @brief qed_chain_recycle_consumed -
 386  *
 387  * Returns an element which was previously consumed;
 388  * Increments producers so they could be written to FW.
 389  *
 390  * @param p_chain
 391  */
 392 static inline void qed_chain_recycle_consumed(struct qed_chain *p_chain)
 393 {
 394         test_and_skip(p_chain, prod_idx);
 395         if (is_chain_u16(p_chain))
 396                 p_chain->u.chain16.prod_idx++;
 397         else
 398                 p_chain->u.chain32.prod_idx++;
 399 }
 400
 401 /**
 402  * @brief qed_chain_consume -
 403  *
 404  * A Chain in which the driver utilizes data written by a different source
 405  * (i.e., FW) should use this to access passed buffers.
 406  *
 407  * @param p_chain
 408  *
 409  * @return void*, a pointer to the next buffer written
 410  */
 411 static inline void *qed_chain_consume(struct qed_chain *p_chain)
 412 {
 413         void *p_ret = NULL, *p_cons_idx, *p_cons_page_idx;
 414
 415         if (is_chain_u16(p_chain)) {
 416                 if ((p_chain->u.chain16.cons_idx &
 417                      p_chain->elem_per_page_mask) == p_chain->next_page_mask) {
 418                         p_cons_idx = &p_chain->u.chain16.cons_idx;
 419                         p_cons_page_idx = &p_chain->pbl.c.u16.cons_page_idx;
 420                         qed_chain_advance_page(p_chain, &p_chain->p_cons_elem,
 421                                                p_cons_idx, p_cons_page_idx);
 422                 }
 423                 p_chain->u.chain16.cons_idx++;
 424         } else {
 425                 if ((p_chain->u.chain32.cons_idx &
 426                      p_chain->elem_per_page_mask) == p_chain->next_page_mask) {
 427                         p_cons_idx = &p_chain->u.chain32.cons_idx;
 428                         p_cons_page_idx = &p_chain->pbl.c.u32.cons_page_idx;
 429                         qed_chain_advance_page(p_chain, &p_chain->p_cons_elem,
 430                                                p_cons_idx, p_cons_page_idx);
 431                 }
 432                 p_chain->u.chain32.cons_idx++;
 433         }
 434
 435         p_ret = p_chain->p_cons_elem;
 436         p_chain->p_cons_elem = (void *)(((u8 *)p_chain->p_cons_elem) +
 437                                         p_chain->elem_size);
 438
 439         return p_ret;
 440 }
 441
 442 /**
 443  * @brief qed_chain_reset - Resets the chain to its start state
 444  *
 445  * @param p_chain pointer to a previously allocted chain
 446  */
 447 static inline void qed_chain_reset(struct qed_chain *p_chain)
 448 {
 449         u32 i;
 450
 451         if (is_chain_u16(p_chain)) {
 452                 p_chain->u.chain16.prod_idx = 0;
 453                 p_chain->u.chain16.cons_idx = 0;
 454         } else {
 455                 p_chain->u.chain32.prod_idx = 0;
 456                 p_chain->u.chain32.cons_idx = 0;
 457         }
 458         p_chain->p_cons_elem = p_chain->p_virt_addr;
 459         p_chain->p_prod_elem = p_chain->p_virt_addr;
 460
 461         if (p_chain->mode == QED_CHAIN_MODE_PBL) {
 462                 /* Use (page_cnt - 1) as a reset value for the prod/cons page's
 463                  * indices, to avoid unnecessary page advancing on the first
 464                  * call to qed_chain_produce/consume. Instead, the indices
 465                  * will be advanced to page_cnt and then will be wrapped to 0.
 466                  */
 467                 u32 reset_val = p_chain->page_cnt - 1;
 468
 469                 if (is_chain_u16(p_chain)) {
 470                         p_chain->pbl.c.u16.prod_page_idx = (u16)reset_val;
 471                         p_chain->pbl.c.u16.cons_page_idx = (u16)reset_val;
 472                 } else {
 473                         p_chain->pbl.c.u32.prod_page_idx = reset_val;
 474                         p_chain->pbl.c.u32.cons_page_idx = reset_val;
 475                 }
 476         }
 477
 478         switch (p_chain->intended_use) {
 479         case QED_CHAIN_USE_TO_CONSUME:
 480                 /* produce empty elements */
 481                 for (i = 0; i < p_chain->capacity; i++)
 482                         qed_chain_recycle_consumed(p_chain);
 483                 break;
 484
 485         case QED_CHAIN_USE_TO_CONSUME_PRODUCE:
 486         case QED_CHAIN_USE_TO_PRODUCE:
 487         default:
 488                 /* Do nothing */
 489                 break;
 490         }
 491 }
 492
 493 /**
 494  * @brief qed_chain_init - Initalizes a basic chain struct
 495  *
 496  * @param p_chain
 497  * @param p_virt_addr
 498  * @param p_phys_addr   physical address of allocated buffer's beginning
 499  * @param page_cnt      number of pages in the allocated buffer
 500  * @param elem_size     size of each element in the chain
 501  * @param intended_use
 502  * @param mode
 503  */
 504 static inline void qed_chain_init_params(struct qed_chain *p_chain,
 505                                          u32 page_cnt,
 506                                          u8 elem_size,
 507                                          enum qed_chain_use_mode intended_use,
 508                                          enum qed_chain_mode mode,
 509                                          enum qed_chain_cnt_type cnt_type)
 510 {
 511         /* chain fixed parameters */
 512         p_chain->p_virt_addr = NULL;
 513         p_chain->p_phys_addr = 0;
 514         p_chain->elem_size      = elem_size;
 515         p_chain->intended_use = (u8)intended_use;
 516         p_chain->mode           = mode;
 517         p_chain->cnt_type = (u8)cnt_type;
 518
 519         p_chain->elem_per_page = ELEMS_PER_PAGE(elem_size);
 520         p_chain->usable_per_page = USABLE_ELEMS_PER_PAGE(elem_size, mode);
 521         p_chain->elem_per_page_mask = p_chain->elem_per_page - 1;
 522         p_chain->elem_unusable = UNUSABLE_ELEMS_PER_PAGE(elem_size, mode);
 523         p_chain->next_page_mask = (p_chain->usable_per_page &
 524                                    p_chain->elem_per_page_mask);
 525
 526         p_chain->page_cnt = page_cnt;
 527         p_chain->capacity = p_chain->usable_per_page * page_cnt;
 528         p_chain->size = p_chain->elem_per_page * page_cnt;
 529
 530         p_chain->pbl_sp.table_phys = 0;
 531         p_chain->pbl_sp.table_virt = NULL;
 532         p_chain->pbl.pp_addr_tbl = NULL;
 533 }
 534
 535 /**
 536  * @brief qed_chain_init_mem -
 537  *
 538  * Initalizes a basic chain struct with its chain buffers
 539  *
 540  * @param p_chain
 541  * @param p_virt_addr   virtual address of allocated buffer's beginning
 542  * @param p_phys_addr   physical address of allocated buffer's beginning
 543  *
 544  */
 545 static inline void qed_chain_init_mem(struct qed_chain *p_chain,
 546                                       void *p_virt_addr, dma_addr_t p_phys_addr)
 547 {
 548         p_chain->p_virt_addr = p_virt_addr;
 549         p_chain->p_phys_addr = p_phys_addr;
 550 }
 551
 552 /**
 553  * @brief qed_chain_init_pbl_mem -
 554  *
 555  * Initalizes a basic chain struct with its pbl buffers
 556  *
 557  * @param p_chain
 558  * @param p_virt_pbl    pointer to a pre allocated side table which will hold
 559  *                      virtual page addresses.
 560  * @param p_phys_pbl    pointer to a pre-allocated side table which will hold
 561  *                      physical page addresses.
 562  * @param pp_virt_addr_tbl
 563  *                      pointer to a pre-allocated side table which will hold
 564  *                      the virtual addresses of the chain pages.
 565  *
 566  */
 567 static inline void qed_chain_init_pbl_mem(struct qed_chain *p_chain,
 568                                           void *p_virt_pbl,
 569                                           dma_addr_t p_phys_pbl,
 570                                           struct addr_tbl_entry *pp_addr_tbl)
 571 {
 572         p_chain->pbl_sp.table_phys = p_phys_pbl;
 573         p_chain->pbl_sp.table_virt = p_virt_pbl;
 574         p_chain->pbl.pp_addr_tbl = pp_addr_tbl;
 575 }
 576
 577 /**
 578  * @brief qed_chain_init_next_ptr_elem -
 579  *
 580  * Initalizes a next pointer element
 581  *
 582  * @param p_chain
 583  * @param p_virt_curr   virtual address of a chain page of which the next
 584  *                      pointer element is initialized
 585  * @param p_virt_next   virtual address of the next chain page
 586  * @param p_phys_next   physical address of the next chain page
 587  *
 588  */
 589 static inline void
 590 qed_chain_init_next_ptr_elem(struct qed_chain *p_chain,
 591                              void *p_virt_curr,
 592                              void *p_virt_next, dma_addr_t p_phys_next)
 593 {
 594         struct qed_chain_next *p_next;
 595         u32 size;
 596
 597         size = p_chain->elem_size * p_chain->usable_per_page;
 598         p_next = (struct qed_chain_next *)((u8 *)p_virt_curr + size);
 599
 600         DMA_REGPAIR_LE(p_next->next_phys, p_phys_next);
 601
 602         p_next->next_virt = p_virt_next;
 603 }
 604
 605 /**
 606  * @brief qed_chain_get_last_elem -
 607  *
 608  * Returns a pointer to the last element of the chain
 609  *
 610  * @param p_chain
 611  *
 612  * @return void*
 613  */
 614 static inline void *qed_chain_get_last_elem(struct qed_chain *p_chain)
 615 {
 616         struct qed_chain_next *p_next = NULL;
 617         void *p_virt_addr = NULL;
 618         u32 size, last_page_idx;
 619
 620         if (!p_chain->p_virt_addr)
 621                 goto out;
 622
 623         switch (p_chain->mode) {
 624         case QED_CHAIN_MODE_NEXT_PTR:
 625                 size = p_chain->elem_size * p_chain->usable_per_page;
 626                 p_virt_addr = p_chain->p_virt_addr;
 627                 p_next = (struct qed_chain_next *)((u8 *)p_virt_addr + size);
 628                 while (p_next->next_virt != p_chain->p_virt_addr) {
 629                         p_virt_addr = p_next->next_virt;
 630                         p_next = (struct qed_chain_next *)((u8 *)p_virt_addr +
 631                                                            size);
 632                 }
 633                 break;
 634         case QED_CHAIN_MODE_SINGLE:
 635                 p_virt_addr = p_chain->p_virt_addr;
 636                 break;
 637         case QED_CHAIN_MODE_PBL:
 638                 last_page_idx = p_chain->page_cnt - 1;
 639                 p_virt_addr = p_chain->pbl.pp_addr_tbl[last_page_idx].virt_addr;
 640                 break;
 641         }
 642         /* p_virt_addr points at this stage to the last page of the chain */
 643         size = p_chain->elem_size * (p_chain->usable_per_page - 1);
 644         p_virt_addr = (u8 *)p_virt_addr + size;
 645 out:
 646         return p_virt_addr;
 647 }
 648
 649 /**
 650  * @brief qed_chain_set_prod - sets the prod to the given value
 651  *
 652  * @param prod_idx
 653  * @param p_prod_elem
 654  */
 655 static inline void qed_chain_set_prod(struct qed_chain *p_chain,
 656                                       u32 prod_idx, void *p_prod_elem)
 657 {
 658         if (p_chain->mode == QED_CHAIN_MODE_PBL) {
 659                 u32 cur_prod, page_mask, page_cnt, page_diff;
 660
 661                 cur_prod = is_chain_u16(p_chain) ? p_chain->u.chain16.prod_idx :
 662                            p_chain->u.chain32.prod_idx;
 663
 664                 /* Assume that number of elements in a page is power of 2 */
 665                 page_mask = ~p_chain->elem_per_page_mask;
 666
 667                 /* Use "cur_prod - 1" and "prod_idx - 1" since producer index
 668                  * reaches the first element of next page before the page index
 669                  * is incremented. See qed_chain_produce().
 670                  * Index wrap around is not a problem because the difference
 671                  * between current and given producer indices is always
 672                  * positive and lower than the chain's capacity.
 673                  */
 674                 page_diff = (((cur_prod - 1) & page_mask) -
 675                              ((prod_idx - 1) & page_mask)) /
 676                             p_chain->elem_per_page;
 677
 678                 page_cnt = qed_chain_get_page_cnt(p_chain);
 679                 if (is_chain_u16(p_chain))
 680                         p_chain->pbl.c.u16.prod_page_idx =
 681                                 (p_chain->pbl.c.u16.prod_page_idx -
 682                                  page_diff + page_cnt) % page_cnt;
 683                 else
 684                         p_chain->pbl.c.u32.prod_page_idx =
 685                                 (p_chain->pbl.c.u32.prod_page_idx -
 686                                  page_diff + page_cnt) % page_cnt;
 687         }
 688
 689         if (is_chain_u16(p_chain))
 690                 p_chain->u.chain16.prod_idx = (u16) prod_idx;
 691         else
 692                 p_chain->u.chain32.prod_idx = prod_idx;
 693         p_chain->p_prod_elem = p_prod_elem;
 694 }
 695
 696 /**
 697  * @brief qed_chain_pbl_zero_mem - set chain memory to 0
 698  *
 699  * @param p_chain
 700  */
 701 static inline void qed_chain_pbl_zero_mem(struct qed_chain *p_chain)
 702 {
 703         u32 i, page_cnt;
 704
 705         if (p_chain->mode != QED_CHAIN_MODE_PBL)
 706                 return;
 707
 708         page_cnt = qed_chain_get_page_cnt(p_chain);
 709
 710         for (i = 0; i < page_cnt; i++)
 711                 memset(p_chain->pbl.pp_addr_tbl[i].virt_addr, 0,
 712                        QED_CHAIN_PAGE_SIZE);
 713 }
 714
 715 #endif