drivers/net/ethernet/cavium/liquidio/octeon_droq.c

   1 /**********************************************************************
   2  * Author: Cavium, Inc.
   3  *
   4  * Contact: support@cavium.com
   5  *          Please include "LiquidIO" in the subject.
   6  *
   7  * Copyright (c) 2003-2016 Cavium, Inc.
   8  *
   9  * This file is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License, Version 2, as
  11  * published by the Free Software Foundation.
  12  *
  13  * This file is distributed in the hope that it will be useful, but
  14  * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
  15  * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
  16  * NONINFRINGEMENT.  See the GNU General Public License for more details.
  17  ***********************************************************************/
  18 #include <linux/pci.h>
  19 #include <linux/netdevice.h>
  20 #include <linux/vmalloc.h>
  21 #include "liquidio_common.h"
  22 #include "octeon_droq.h"
  23 #include "octeon_iq.h"
  24 #include "response_manager.h"
  25 #include "octeon_device.h"
  26 #include "octeon_main.h"
  27 #include "octeon_network.h"
  28 #include "cn66xx_regs.h"
  29 #include "cn66xx_device.h"
  30 #include "cn23xx_pf_device.h"
  31 #include "cn23xx_vf_device.h"
  32
  33 struct niclist {
  34         struct list_head list;
  35         void *ptr;
  36 };
  37
  38 struct __dispatch {
  39         struct list_head list;
  40         struct octeon_recv_info *rinfo;
  41         octeon_dispatch_fn_t disp_fn;
  42 };
  43
  44 /** Get the argument that the user set when registering dispatch
  45  *  function for a given opcode/subcode.
  46  *  @param  octeon_dev - the octeon device pointer.
  47  *  @param  opcode     - the opcode for which the dispatch argument
  48  *                       is to be checked.
  49  *  @param  subcode    - the subcode for which the dispatch argument
  50  *                       is to be checked.
  51  *  @return  Success: void * (argument to the dispatch function)
  52  *  @return  Failure: NULL
  53  *
  54  */
  55 void *octeon_get_dispatch_arg(struct octeon_device *octeon_dev,
  56                               u16 opcode, u16 subcode)
  57 {
  58         int idx;
  59         struct list_head *dispatch;
  60         void *fn_arg = NULL;
  61         u16 combined_opcode = OPCODE_SUBCODE(opcode, subcode);
  62
  63         idx = combined_opcode & OCTEON_OPCODE_MASK;
  64
  65         spin_lock_bh(&octeon_dev->dispatch.lock);
  66
  67         if (octeon_dev->dispatch.count == 0) {
  68                 spin_unlock_bh(&octeon_dev->dispatch.lock);
  69                 return NULL;
  70         }
  71
  72         if (octeon_dev->dispatch.dlist[idx].opcode == combined_opcode) {
  73                 fn_arg = octeon_dev->dispatch.dlist[idx].arg;
  74         } else {
  75                 list_for_each(dispatch,
  76                               &octeon_dev->dispatch.dlist[idx].list) {
  77                         if (((struct octeon_dispatch *)dispatch)->opcode ==
  78                             combined_opcode) {
  79                                 fn_arg = ((struct octeon_dispatch *)
  80                                           dispatch)->arg;
  81                                 break;
  82                         }
  83                 }
  84         }
  85
  86         spin_unlock_bh(&octeon_dev->dispatch.lock);
  87         return fn_arg;
  88 }
  89
  90 /** Check for packets on Droq. This function should be called with lock held.
  91  *  @param  droq - Droq on which count is checked.
  92  *  @return Returns packet count.
  93  */
  94 u32 octeon_droq_check_hw_for_pkts(struct octeon_droq *droq)
  95 {
  96         u32 pkt_count = 0;
  97         u32 last_count;
  98
  99         pkt_count = readl(droq->pkts_sent_reg);
 100
 101         last_count = pkt_count - droq->pkt_count;
 102         droq->pkt_count = pkt_count;
 103
 104         /* we shall write to cnts  at napi irq enable or end of droq tasklet */
 105         if (last_count)
 106                 atomic_add(last_count, &droq->pkts_pending);
 107
 108         return last_count;
 109 }
 110
 111 static void octeon_droq_compute_max_packet_bufs(struct octeon_droq *droq)
 112 {
 113         u32 count = 0;
 114
 115         /* max_empty_descs is the max. no. of descs that can have no buffers.
 116          * If the empty desc count goes beyond this value, we cannot safely
 117          * read in a 64K packet sent by Octeon
 118          * (64K is max pkt size from Octeon)
 119          */
 120         droq->max_empty_descs = 0;
 121
 122         do {
 123                 droq->max_empty_descs++;
 124                 count += droq->buffer_size;
 125         } while (count < (64 * 1024));
 126
 127         droq->max_empty_descs = droq->max_count - droq->max_empty_descs;
 128 }
 129
 130 static void octeon_droq_reset_indices(struct octeon_droq *droq)
 131 {
 132         droq->read_idx = 0;
 133         droq->write_idx = 0;
 134         droq->refill_idx = 0;
 135         droq->refill_count = 0;
 136         atomic_set(&droq->pkts_pending, 0);
 137 }
 138
 139 static void
 140 octeon_droq_destroy_ring_buffers(struct octeon_device *oct,
 141                                  struct octeon_droq *droq)
 142 {
 143         u32 i;
 144         struct octeon_skb_page_info *pg_info;
 145
 146         for (i = 0; i < droq->max_count; i++) {
 147                 pg_info = &droq->recv_buf_list[i].pg_info;
 148                 if (!pg_info)
 149                         continue;
 150
 151                 if (pg_info->dma)
 152                         lio_unmap_ring(oct->pci_dev,
 153                                        (u64)pg_info->dma);
 154                 pg_info->dma = 0;
 155
 156                 if (pg_info->page)
 157                         recv_buffer_destroy(droq->recv_buf_list[i].buffer,
 158                                             pg_info);
 159
 160                 droq->recv_buf_list[i].buffer = NULL;
 161         }
 162
 163         octeon_droq_reset_indices(droq);
 164 }
 165
 166 static int
 167 octeon_droq_setup_ring_buffers(struct octeon_device *oct,
 168                                struct octeon_droq *droq)
 169 {
 170         u32 i;
 171         void *buf;
 172         struct octeon_droq_desc *desc_ring = droq->desc_ring;
 173
 174         for (i = 0; i < droq->max_count; i++) {
 175                 buf = recv_buffer_alloc(oct, &droq->recv_buf_list[i].pg_info);
 176
 177                 if (!buf) {
 178                         dev_err(&oct->pci_dev->dev, "%s buffer alloc failed\n",
 179                                 __func__);
 180                         droq->stats.rx_alloc_failure++;
 181                         return -ENOMEM;
 182                 }
 183
 184                 droq->recv_buf_list[i].buffer = buf;
 185                 droq->recv_buf_list[i].data = get_rbd(buf);
 186                 desc_ring[i].info_ptr = 0;
 187                 desc_ring[i].buffer_ptr =
 188                         lio_map_ring(droq->recv_buf_list[i].buffer);
 189         }
 190
 191         octeon_droq_reset_indices(droq);
 192
 193         octeon_droq_compute_max_packet_bufs(droq);
 194
 195         return 0;
 196 }
 197
 198 int octeon_delete_droq(struct octeon_device *oct, u32 q_no)
 199 {
 200         struct octeon_droq *droq = oct->droq[q_no];
 201
 202         dev_dbg(&oct->pci_dev->dev, "%s[%d]\n", __func__, q_no);
 203
 204         octeon_droq_destroy_ring_buffers(oct, droq);
 205         vfree(droq->recv_buf_list);
 206
 207         if (droq->desc_ring)
 208                 lio_dma_free(oct, (droq->max_count * OCT_DROQ_DESC_SIZE),
 209                              droq->desc_ring, droq->desc_ring_dma);
 210
 211         memset(droq, 0, OCT_DROQ_SIZE);
 212         oct->io_qmask.oq &= ~(1ULL << q_no);
 213         vfree(oct->droq[q_no]);
 214         oct->droq[q_no] = NULL;
 215         oct->num_oqs--;
 216
 217         return 0;
 218 }
 219
 220 int octeon_init_droq(struct octeon_device *oct,
 221                      u32 q_no,
 222                      u32 num_descs,
 223                      u32 desc_size,
 224                      void *app_ctx)
 225 {
 226         struct octeon_droq *droq;
 227         u32 desc_ring_size = 0, c_num_descs = 0, c_buf_size = 0;
 228         u32 c_pkts_per_intr = 0, c_refill_threshold = 0;
 229         int numa_node = dev_to_node(&oct->pci_dev->dev);
 230
 231         dev_dbg(&oct->pci_dev->dev, "%s[%d]\n", __func__, q_no);
 232
 233         droq = oct->droq[q_no];
 234         memset(droq, 0, OCT_DROQ_SIZE);
 235
 236         droq->oct_dev = oct;
 237         droq->q_no = q_no;
 238         if (app_ctx)
 239                 droq->app_ctx = app_ctx;
 240         else
 241                 droq->app_ctx = (void *)(size_t)q_no;
 242
 243         c_num_descs = num_descs;
 244         c_buf_size = desc_size;
 245         if (OCTEON_CN6XXX(oct)) {
 246                 struct octeon_config *conf6x = CHIP_CONF(oct, cn6xxx);
 247
 248                 c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf6x);
 249                 c_refill_threshold =
 250                         (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf6x);
 251         } else if (OCTEON_CN23XX_PF(oct)) {
 252                 struct octeon_config *conf23 = CHIP_CONF(oct, cn23xx_pf);
 253
 254                 c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf23);
 255                 c_refill_threshold = (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf23);
 256         } else if (OCTEON_CN23XX_VF(oct)) {
 257                 struct octeon_config *conf23 = CHIP_CONF(oct, cn23xx_vf);
 258
 259                 c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf23);
 260                 c_refill_threshold = (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf23);
 261         } else {
 262                 return 1;
 263         }
 264
 265         droq->max_count = c_num_descs;
 266         droq->buffer_size = c_buf_size;
 267
 268         desc_ring_size = droq->max_count * OCT_DROQ_DESC_SIZE;
 269         droq->desc_ring = lio_dma_alloc(oct, desc_ring_size,
 270                                         (dma_addr_t *)&droq->desc_ring_dma);
 271
 272         if (!droq->desc_ring) {
 273                 dev_err(&oct->pci_dev->dev,
 274                         "Output queue %d ring alloc failed\n", q_no);
 275                 return 1;
 276         }
 277
 278         dev_dbg(&oct->pci_dev->dev, "droq[%d]: desc_ring: virt: 0x%p, dma: %lx\n",
 279                 q_no, droq->desc_ring, droq->desc_ring_dma);
 280         dev_dbg(&oct->pci_dev->dev, "droq[%d]: num_desc: %d\n", q_no,
 281                 droq->max_count);
 282
 283         droq->recv_buf_list = (struct octeon_recv_buffer *)
 284                               vzalloc_node(droq->max_count *
 285                                                 OCT_DROQ_RECVBUF_SIZE,
 286                                                 numa_node);
 287         if (!droq->recv_buf_list)
 288                 droq->recv_buf_list = (struct octeon_recv_buffer *)
 289                                       vzalloc(droq->max_count *
 290                                                 OCT_DROQ_RECVBUF_SIZE);
 291         if (!droq->recv_buf_list) {
 292                 dev_err(&oct->pci_dev->dev, "Output queue recv buf list alloc failed\n");
 293                 goto init_droq_fail;
 294         }
 295
 296         if (octeon_droq_setup_ring_buffers(oct, droq))
 297                 goto init_droq_fail;
 298
 299         droq->pkts_per_intr = c_pkts_per_intr;
 300         droq->refill_threshold = c_refill_threshold;
 301
 302         dev_dbg(&oct->pci_dev->dev, "DROQ INIT: max_empty_descs: %d\n",
 303                 droq->max_empty_descs);
 304
 305         spin_lock_init(&droq->lock);
 306
 307         INIT_LIST_HEAD(&droq->dispatch_list);
 308
 309         /* For 56xx Pass1, this function won't be called, so no checks. */
 310         oct->fn_list.setup_oq_regs(oct, q_no);
 311
 312         oct->io_qmask.oq |= BIT_ULL(q_no);
 313
 314         return 0;
 315
 316 init_droq_fail:
 317         octeon_delete_droq(oct, q_no);
 318         return 1;
 319 }
 320
 321 /* octeon_create_recv_info
 322  * Parameters:
 323  *  octeon_dev - pointer to the octeon device structure
 324  *  droq       - droq in which the packet arrived.
 325  *  buf_cnt    - no. of buffers used by the packet.
 326  *  idx        - index in the descriptor for the first buffer in the packet.
 327  * Description:
 328  *  Allocates a recv_info_t and copies the buffer addresses for packet data
 329  *  into the recv_pkt space which starts at an 8B offset from recv_info_t.
 330  *  Flags the descriptors for refill later. If available descriptors go
 331  *  below the threshold to receive a 64K pkt, new buffers are first allocated
 332  *  before the recv_pkt_t is created.
 333  *  This routine will be called in interrupt context.
 334  * Returns:
 335  *  Success: Pointer to recv_info_t
 336  *  Failure: NULL.
 337  * Locks:
 338  *  The droq->lock is held when this routine is called.
 339  */
 340 static inline struct octeon_recv_info *octeon_create_recv_info(
 341                 struct octeon_device *octeon_dev,
 342                 struct octeon_droq *droq,
 343                 u32 buf_cnt,
 344                 u32 idx)
 345 {
 346         struct octeon_droq_info *info;
 347         struct octeon_recv_pkt *recv_pkt;
 348         struct octeon_recv_info *recv_info;
 349         u32 i, bytes_left;
 350         struct octeon_skb_page_info *pg_info;
 351
 352         info = (struct octeon_droq_info *)droq->recv_buf_list[idx].data;
 353
 354         recv_info = octeon_alloc_recv_info(sizeof(struct __dispatch));
 355         if (!recv_info)
 356                 return NULL;
 357
 358         recv_pkt = recv_info->recv_pkt;
 359         recv_pkt->rh = info->rh;
 360         recv_pkt->length = (u32)info->length;
 361         recv_pkt->buffer_count = (u16)buf_cnt;
 362         recv_pkt->octeon_id = (u16)octeon_dev->octeon_id;
 363
 364         i = 0;
 365         bytes_left = (u32)info->length;
 366
 367         while (buf_cnt) {
 368                 {
 369                         pg_info = &droq->recv_buf_list[idx].pg_info;
 370
 371                         lio_unmap_ring(octeon_dev->pci_dev,
 372                                        (u64)pg_info->dma);
 373                         pg_info->page = NULL;
 374                         pg_info->dma = 0;
 375                 }
 376
 377                 recv_pkt->buffer_size[i] =
 378                         (bytes_left >=
 379                          droq->buffer_size) ? droq->buffer_size : bytes_left;
 380
 381                 recv_pkt->buffer_ptr[i] = droq->recv_buf_list[idx].buffer;
 382                 droq->recv_buf_list[idx].buffer = NULL;
 383
 384                 idx = incr_index(idx, 1, droq->max_count);
 385                 bytes_left -= droq->buffer_size;
 386                 i++;
 387                 buf_cnt--;
 388         }
 389
 390         return recv_info;
 391 }
 392
 393 /* If we were not able to refill all buffers, try to move around
 394  * the buffers that were not dispatched.
 395  */
 396 static inline u32
 397 octeon_droq_refill_pullup_descs(struct octeon_droq *droq,
 398                                 struct octeon_droq_desc *desc_ring)
 399 {
 400         u32 desc_refilled = 0;
 401
 402         u32 refill_index = droq->refill_idx;
 403
 404         while (refill_index != droq->read_idx) {
 405                 if (droq->recv_buf_list[refill_index].buffer) {
 406                         droq->recv_buf_list[droq->refill_idx].buffer =
 407                                 droq->recv_buf_list[refill_index].buffer;
 408                         droq->recv_buf_list[droq->refill_idx].data =
 409                                 droq->recv_buf_list[refill_index].data;
 410                         desc_ring[droq->refill_idx].buffer_ptr =
 411                                 desc_ring[refill_index].buffer_ptr;
 412                         droq->recv_buf_list[refill_index].buffer = NULL;
 413                         desc_ring[refill_index].buffer_ptr = 0;
 414                         do {
 415                                 droq->refill_idx = incr_index(droq->refill_idx,
 416                                                               1,
 417                                                               droq->max_count);
 418                                 desc_refilled++;
 419                                 droq->refill_count--;
 420                         } while (droq->recv_buf_list[droq->refill_idx].buffer);
 421                 }
 422                 refill_index = incr_index(refill_index, 1, droq->max_count);
 423         }                       /* while */
 424         return desc_refilled;
 425 }
 426
 427 /* octeon_droq_refill
 428  * Parameters:
 429  *  droq       - droq in which descriptors require new buffers.
 430  * Description:
 431  *  Called during normal DROQ processing in interrupt mode or by the poll
 432  *  thread to refill the descriptors from which buffers were dispatched
 433  *  to upper layers. Attempts to allocate new buffers. If that fails, moves
 434  *  up buffers (that were not dispatched) to form a contiguous ring.
 435  * Returns:
 436  *  No of descriptors refilled.
 437  * Locks:
 438  *  This routine is called with droq->lock held.
 439  */
 440 static u32
 441 octeon_droq_refill(struct octeon_device *octeon_dev, struct octeon_droq *droq)
 442 {
 443         struct octeon_droq_desc *desc_ring;
 444         void *buf = NULL;
 445         u8 *data;
 446         u32 desc_refilled = 0;
 447         struct octeon_skb_page_info *pg_info;
 448
 449         desc_ring = droq->desc_ring;
 450
 451         while (droq->refill_count && (desc_refilled < droq->max_count)) {
 452                 /* If a valid buffer exists (happens if there is no dispatch),
 453                  * reuse
 454                  * the buffer, else allocate.
 455                  */
 456                 if (!droq->recv_buf_list[droq->refill_idx].buffer) {
 457                         pg_info =
 458                                 &droq->recv_buf_list[droq->refill_idx].pg_info;
 459                         /* Either recycle the existing pages or go for
 460                          * new page alloc
 461                          */
 462                         if (pg_info->page)
 463                                 buf = recv_buffer_reuse(octeon_dev, pg_info);
 464                         else
 465                                 buf = recv_buffer_alloc(octeon_dev, pg_info);
 466                         /* If a buffer could not be allocated, no point in
 467                          * continuing
 468                          */
 469                         if (!buf) {
 470                                 droq->stats.rx_alloc_failure++;
 471                                 break;
 472                         }
 473                         droq->recv_buf_list[droq->refill_idx].buffer =
 474                                 buf;
 475                         data = get_rbd(buf);
 476                 } else {
 477                         data = get_rbd(droq->recv_buf_list
 478                                        [droq->refill_idx].buffer);
 479                 }
 480
 481                 droq->recv_buf_list[droq->refill_idx].data = data;
 482
 483                 desc_ring[droq->refill_idx].buffer_ptr =
 484                         lio_map_ring(droq->recv_buf_list[
 485                                      droq->refill_idx].buffer);
 486
 487                 droq->refill_idx = incr_index(droq->refill_idx, 1,
 488                                               droq->max_count);
 489                 desc_refilled++;
 490                 droq->refill_count--;
 491         }
 492
 493         if (droq->refill_count)
 494                 desc_refilled +=
 495                         octeon_droq_refill_pullup_descs(droq, desc_ring);
 496
 497         /* if droq->refill_count
 498          * The refill count would not change in pass two. We only moved buffers
 499          * to close the gap in the ring, but we would still have the same no. of
 500          * buffers to refill.
 501          */
 502         return desc_refilled;
 503 }
 504
 505 /** check if we can allocate packets to get out of oom.
 506  *  @param  droq - Droq being checked.
 507  *  @return does not return anything
 508  */
 509 void octeon_droq_check_oom(struct octeon_droq *droq)
 510 {
 511         int desc_refilled;
 512         struct octeon_device *oct = droq->oct_dev;
 513
 514         if (readl(droq->pkts_credit_reg) <= CN23XX_SLI_DEF_BP) {
 515                 spin_lock_bh(&droq->lock);
 516                 desc_refilled = octeon_droq_refill(oct, droq);
 517                 if (desc_refilled) {
 518                         /* Flush the droq descriptor data to memory to be sure
 519                          * that when we update the credits the data in memory
 520                          * is accurate.
 521                          */
 522                         wmb();
 523                         writel(desc_refilled, droq->pkts_credit_reg);
 524                         /* make sure mmio write completes */
 525                         mmiowb();
 526                 }
 527                 spin_unlock_bh(&droq->lock);
 528         }
 529 }
 530
 531 static inline u32
 532 octeon_droq_get_bufcount(u32 buf_size, u32 total_len)
 533 {
 534         return ((total_len + buf_size - 1) / buf_size);
 535 }
 536
 537 static int
 538 octeon_droq_dispatch_pkt(struct octeon_device *oct,
 539                          struct octeon_droq *droq,
 540                          union octeon_rh *rh,
 541                          struct octeon_droq_info *info)
 542 {
 543         u32 cnt;
 544         octeon_dispatch_fn_t disp_fn;
 545         struct octeon_recv_info *rinfo;
 546
 547         cnt = octeon_droq_get_bufcount(droq->buffer_size, (u32)info->length);
 548
 549         disp_fn = octeon_get_dispatch(oct, (u16)rh->r.opcode,
 550                                       (u16)rh->r.subcode);
 551         if (disp_fn) {
 552                 rinfo = octeon_create_recv_info(oct, droq, cnt, droq->read_idx);
 553                 if (rinfo) {
 554                         struct __dispatch *rdisp = rinfo->rsvd;
 555
 556                         rdisp->rinfo = rinfo;
 557                         rdisp->disp_fn = disp_fn;
 558                         rinfo->recv_pkt->rh = *rh;
 559                         list_add_tail(&rdisp->list,
 560                                       &droq->dispatch_list);
 561                 } else {
 562                         droq->stats.dropped_nomem++;
 563                 }
 564         } else {
 565                 dev_err(&oct->pci_dev->dev, "DROQ: No dispatch function (opcode %u/%u)\n",
 566                         (unsigned int)rh->r.opcode,
 567                         (unsigned int)rh->r.subcode);
 568                 droq->stats.dropped_nodispatch++;
 569         }
 570
 571         return cnt;
 572 }
 573
 574 static inline void octeon_droq_drop_packets(struct octeon_device *oct,
 575                                             struct octeon_droq *droq,
 576                                             u32 cnt)
 577 {
 578         u32 i = 0, buf_cnt;
 579         struct octeon_droq_info *info;
 580
 581         for (i = 0; i < cnt; i++) {
 582                 info = (struct octeon_droq_info *)
 583                         droq->recv_buf_list[droq->read_idx].data;
 584                 octeon_swap_8B_data((u64 *)info, 2);
 585
 586                 if (info->length) {
 587                         info->length += OCTNET_FRM_LENGTH_SIZE;
 588                         droq->stats.bytes_received += info->length;
 589                         buf_cnt = octeon_droq_get_bufcount(droq->buffer_size,
 590                                                            (u32)info->length);
 591                 } else {
 592                         dev_err(&oct->pci_dev->dev, "DROQ: In drop: pkt with len 0\n");
 593                         buf_cnt = 1;
 594                 }
 595
 596                 droq->read_idx = incr_index(droq->read_idx, buf_cnt,
 597                                             droq->max_count);
 598                 droq->refill_count += buf_cnt;
 599         }
 600 }
 601
 602 static u32
 603 octeon_droq_fast_process_packets(struct octeon_device *oct,
 604                                  struct octeon_droq *droq,
 605                                  u32 pkts_to_process)
 606 {
 607         struct octeon_droq_info *info;
 608         union octeon_rh *rh;
 609         u32 pkt, total_len = 0, pkt_count;
 610
 611         pkt_count = pkts_to_process;
 612
 613         for (pkt = 0; pkt < pkt_count; pkt++) {
 614                 u32 pkt_len = 0;
 615                 struct sk_buff *nicbuf = NULL;
 616                 struct octeon_skb_page_info *pg_info;
 617                 void *buf;
 618
 619                 info = (struct octeon_droq_info *)
 620                         droq->recv_buf_list[droq->read_idx].data;
 621                 octeon_swap_8B_data((u64 *)info, 2);
 622
 623                 if (!info->length) {
 624                         dev_err(&oct->pci_dev->dev,
 625                                 "DROQ[%d] idx: %d len:0, pkt_cnt: %d\n",
 626                                 droq->q_no, droq->read_idx, pkt_count);
 627                         print_hex_dump_bytes("", DUMP_PREFIX_ADDRESS,
 628                                              (u8 *)info,
 629                                              OCT_DROQ_INFO_SIZE);
 630                         break;
 631                 }
 632
 633                 /* Len of resp hdr in included in the received data len. */
 634                 rh = &info->rh;
 635
 636                 info->length += OCTNET_FRM_LENGTH_SIZE;
 637                 rh->r_dh.len += (ROUNDUP8(OCT_DROQ_INFO_SIZE) / sizeof(u64));
 638                 total_len += (u32)info->length;
 639                 if (opcode_slow_path(rh)) {
 640                         u32 buf_cnt;
 641
 642                         buf_cnt = octeon_droq_dispatch_pkt(oct, droq, rh, info);
 643                         droq->read_idx = incr_index(droq->read_idx,
 644                                                     buf_cnt, droq->max_count);
 645                         droq->refill_count += buf_cnt;
 646                 } else {
 647                         if (info->length <= droq->buffer_size) {
 648                                 pkt_len = (u32)info->length;
 649                                 nicbuf = droq->recv_buf_list[
 650                                         droq->read_idx].buffer;
 651                                 pg_info = &droq->recv_buf_list[
 652                                         droq->read_idx].pg_info;
 653                                 if (recv_buffer_recycle(oct, pg_info))
 654                                         pg_info->page = NULL;
 655                                 droq->recv_buf_list[droq->read_idx].buffer =
 656                                         NULL;
 657
 658                                 droq->read_idx = incr_index(droq->read_idx, 1,
 659                                                             droq->max_count);
 660                                 droq->refill_count++;
 661                         } else {
 662                                 nicbuf = octeon_fast_packet_alloc((u32)
 663                                                                   info->length);
 664                                 pkt_len = 0;
 665                                 /* nicbuf allocation can fail. We'll handle it
 666                                  * inside the loop.
 667                                  */
 668                                 while (pkt_len < info->length) {
 669                                         int cpy_len, idx = droq->read_idx;
 670
 671                                         cpy_len = ((pkt_len + droq->buffer_size)
 672                                                    > info->length) ?
 673                                                 ((u32)info->length - pkt_len) :
 674                                                 droq->buffer_size;
 675
 676                                         if (nicbuf) {
 677                                                 octeon_fast_packet_next(droq,
 678                                                                         nicbuf,
 679                                                                         cpy_len,
 680                                                                         idx);
 681                                                 buf = droq->recv_buf_list[
 682                                                         idx].buffer;
 683                                                 recv_buffer_fast_free(buf);
 684                                                 droq->recv_buf_list[idx].buffer
 685                                                         = NULL;
 686                                         } else {
 687                                                 droq->stats.rx_alloc_failure++;
 688                                         }
 689
 690                                         pkt_len += cpy_len;
 691                                         droq->read_idx =
 692                                                 incr_index(droq->read_idx, 1,
 693                                                            droq->max_count);
 694                                         droq->refill_count++;
 695                                 }
 696                         }
 697
 698                         if (nicbuf) {
 699                                 if (droq->ops.fptr) {
 700                                         droq->ops.fptr(oct->octeon_id,
 701                                                        nicbuf, pkt_len,
 702                                                        rh, &droq->napi,
 703                                                        droq->ops.farg);
 704                                 } else {
 705                                         recv_buffer_free(nicbuf);
 706                                 }
 707                         }
 708                 }
 709
 710                 if (droq->refill_count >= droq->refill_threshold) {
 711                         int desc_refilled = octeon_droq_refill(oct, droq);
 712
 713                         /* Flush the droq descriptor data to memory to be sure
 714                          * that when we update the credits the data in memory
 715                          * is accurate.
 716                          */
 717                         wmb();
 718                         writel((desc_refilled), droq->pkts_credit_reg);
 719                         /* make sure mmio write completes */
 720                         mmiowb();
 721                 }
 722
 723         }                       /* for (each packet)... */
 724
 725         /* Increment refill_count by the number of buffers processed. */
 726         droq->stats.pkts_received += pkt;
 727         droq->stats.bytes_received += total_len;
 728
 729         if ((droq->ops.drop_on_max) && (pkts_to_process - pkt)) {
 730                 octeon_droq_drop_packets(oct, droq, (pkts_to_process - pkt));
 731
 732                 droq->stats.dropped_toomany += (pkts_to_process - pkt);
 733                 return pkts_to_process;
 734         }
 735
 736         return pkt;
 737 }
 738
 739 int
 740 octeon_droq_process_packets(struct octeon_device *oct,
 741                             struct octeon_droq *droq,
 742                             u32 budget)
 743 {
 744         u32 pkt_count = 0, pkts_processed = 0;
 745         struct list_head *tmp, *tmp2;
 746
 747         /* Grab the droq lock */
 748         spin_lock(&droq->lock);
 749
 750         octeon_droq_check_hw_for_pkts(droq);
 751         pkt_count = atomic_read(&droq->pkts_pending);
 752
 753         if (!pkt_count) {
 754                 spin_unlock(&droq->lock);
 755                 return 0;
 756         }
 757
 758         if (pkt_count > budget)
 759                 pkt_count = budget;
 760
 761         pkts_processed = octeon_droq_fast_process_packets(oct, droq, pkt_count);
 762
 763         atomic_sub(pkts_processed, &droq->pkts_pending);
 764
 765         /* Release the spin lock */
 766         spin_unlock(&droq->lock);
 767
 768         list_for_each_safe(tmp, tmp2, &droq->dispatch_list) {
 769                 struct __dispatch *rdisp = (struct __dispatch *)tmp;
 770
 771                 list_del(tmp);
 772                 rdisp->disp_fn(rdisp->rinfo,
 773                                octeon_get_dispatch_arg
 774                                (oct,
 775                                 (u16)rdisp->rinfo->recv_pkt->rh.r.opcode,
 776                                 (u16)rdisp->rinfo->recv_pkt->rh.r.subcode));
 777         }
 778
 779         /* If there are packets pending. schedule tasklet again */
 780         if (atomic_read(&droq->pkts_pending))
 781                 return 1;
 782
 783         return 0;
 784 }
 785
 786 /**
 787  * Utility function to poll for packets. check_hw_for_packets must be
 788  * called before calling this routine.
 789  */
 790
 791 int
 792 octeon_droq_process_poll_pkts(struct octeon_device *oct,
 793                               struct octeon_droq *droq, u32 budget)
 794 {
 795         struct list_head *tmp, *tmp2;
 796         u32 pkts_available = 0, pkts_processed = 0;
 797         u32 total_pkts_processed = 0;
 798
 799         if (budget > droq->max_count)
 800                 budget = droq->max_count;
 801
 802         spin_lock(&droq->lock);
 803
 804         while (total_pkts_processed < budget) {
 805                 octeon_droq_check_hw_for_pkts(droq);
 806
 807                 pkts_available = min((budget - total_pkts_processed),
 808                                      (u32)(atomic_read(&droq->pkts_pending)));
 809
 810                 if (pkts_available == 0)
 811                         break;
 812
 813                 pkts_processed =
 814                         octeon_droq_fast_process_packets(oct, droq,
 815                                                          pkts_available);
 816
 817                 atomic_sub(pkts_processed, &droq->pkts_pending);
 818
 819                 total_pkts_processed += pkts_processed;
 820         }
 821
 822         spin_unlock(&droq->lock);
 823
 824         list_for_each_safe(tmp, tmp2, &droq->dispatch_list) {
 825                 struct __dispatch *rdisp = (struct __dispatch *)tmp;
 826
 827                 list_del(tmp);
 828                 rdisp->disp_fn(rdisp->rinfo,
 829                                octeon_get_dispatch_arg
 830                                (oct,
 831                                 (u16)rdisp->rinfo->recv_pkt->rh.r.opcode,
 832                                 (u16)rdisp->rinfo->recv_pkt->rh.r.subcode));
 833         }
 834
 835         return total_pkts_processed;
 836 }
 837
 838 /* Enable Pkt Interrupt */
 839 int
 840 octeon_enable_irq(struct octeon_device *oct, u32 q_no)
 841 {
 842         switch (oct->chip_id) {
 843         case OCTEON_CN66XX:
 844         case OCTEON_CN68XX: {
 845                 struct octeon_cn6xxx *cn6xxx =
 846                         (struct octeon_cn6xxx *)oct->chip;
 847                 unsigned long flags;
 848                 u32 value;
 849
 850                 spin_lock_irqsave
 851                         (&cn6xxx->lock_for_droq_int_enb_reg, flags);
 852                 value = octeon_read_csr(oct, CN6XXX_SLI_PKT_TIME_INT_ENB);
 853                 value |= (1 << q_no);
 854                 octeon_write_csr(oct, CN6XXX_SLI_PKT_TIME_INT_ENB, value);
 855                 value = octeon_read_csr(oct, CN6XXX_SLI_PKT_CNT_INT_ENB);
 856                 value |= (1 << q_no);
 857                 octeon_write_csr(oct, CN6XXX_SLI_PKT_CNT_INT_ENB, value);
 858
 859                 /* don't bother flushing the enables */
 860
 861                 spin_unlock_irqrestore
 862                         (&cn6xxx->lock_for_droq_int_enb_reg, flags);
 863         }
 864                 break;
 865         case OCTEON_CN23XX_PF_VID:
 866                 lio_enable_irq(oct->droq[q_no], oct->instr_queue[q_no]);
 867                 break;
 868
 869         case OCTEON_CN23XX_VF_VID:
 870                 lio_enable_irq(oct->droq[q_no], oct->instr_queue[q_no]);
 871                 break;
 872         default:
 873                 dev_err(&oct->pci_dev->dev, "%s Unknown Chip\n", __func__);
 874                 return 1;
 875         }
 876
 877         return 0;
 878 }
 879
 880 int octeon_register_droq_ops(struct octeon_device *oct, u32 q_no,
 881                              struct octeon_droq_ops *ops)
 882 {
 883         struct octeon_droq *droq;
 884         unsigned long flags;
 885         struct octeon_config *oct_cfg = NULL;
 886
 887         oct_cfg = octeon_get_conf(oct);
 888
 889         if (!oct_cfg)
 890                 return -EINVAL;
 891
 892         if (!(ops)) {
 893                 dev_err(&oct->pci_dev->dev, "%s: droq_ops pointer is NULL\n",
 894                         __func__);
 895                 return -EINVAL;
 896         }
 897
 898         if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) {
 899                 dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n",
 900                         __func__, q_no, (oct->num_oqs - 1));
 901                 return -EINVAL;
 902         }
 903
 904         droq = oct->droq[q_no];
 905
 906         spin_lock_irqsave(&droq->lock, flags);
 907
 908         memcpy(&droq->ops, ops, sizeof(struct octeon_droq_ops));
 909
 910         spin_unlock_irqrestore(&droq->lock, flags);
 911
 912         return 0;
 913 }
 914
 915 int octeon_unregister_droq_ops(struct octeon_device *oct, u32 q_no)
 916 {
 917         unsigned long flags;
 918         struct octeon_droq *droq;
 919         struct octeon_config *oct_cfg = NULL;
 920
 921         oct_cfg = octeon_get_conf(oct);
 922
 923         if (!oct_cfg)
 924                 return -EINVAL;
 925
 926         if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) {
 927                 dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n",
 928                         __func__, q_no, oct->num_oqs - 1);
 929                 return -EINVAL;
 930         }
 931
 932         droq = oct->droq[q_no];
 933
 934         if (!droq) {
 935                 dev_info(&oct->pci_dev->dev,
 936                          "Droq id (%d) not available.\n", q_no);
 937                 return 0;
 938         }
 939
 940         spin_lock_irqsave(&droq->lock, flags);
 941
 942         droq->ops.fptr = NULL;
 943         droq->ops.farg = NULL;
 944         droq->ops.drop_on_max = 0;
 945
 946         spin_unlock_irqrestore(&droq->lock, flags);
 947
 948         return 0;
 949 }
 950
 951 int octeon_create_droq(struct octeon_device *oct,
 952                        u32 q_no, u32 num_descs,
 953                        u32 desc_size, void *app_ctx)
 954 {
 955         struct octeon_droq *droq;
 956         int numa_node = dev_to_node(&oct->pci_dev->dev);
 957
 958         if (oct->droq[q_no]) {
 959                 dev_dbg(&oct->pci_dev->dev, "Droq already in use. Cannot create droq %d again\n",
 960                         q_no);
 961                 return 1;
 962         }
 963
 964         /* Allocate the DS for the new droq. */
 965         droq = vmalloc_node(sizeof(*droq), numa_node);
 966         if (!droq)
 967                 droq = vmalloc(sizeof(*droq));
 968         if (!droq)
 969                 return -1;
 970
 971         memset(droq, 0, sizeof(struct octeon_droq));
 972
 973         /*Disable the pkt o/p for this Q  */
 974         octeon_set_droq_pkt_op(oct, q_no, 0);
 975         oct->droq[q_no] = droq;
 976
 977         /* Initialize the Droq */
 978         if (octeon_init_droq(oct, q_no, num_descs, desc_size, app_ctx)) {
 979                 vfree(oct->droq[q_no]);
 980                 oct->droq[q_no] = NULL;
 981                 return -1;
 982         }
 983
 984         oct->num_oqs++;
 985
 986         dev_dbg(&oct->pci_dev->dev, "%s: Total number of OQ: %d\n", __func__,
 987                 oct->num_oqs);
 988
 989         /* Global Droq register settings */
 990
 991         /* As of now not required, as setting are done for all 32 Droqs at
 992          * the same time.
 993          */
 994         return 0;
 995 }