drivers/net/ethernet/sfc/efx_channels.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /****************************************************************************
   3  * Driver for Solarflare network controllers and boards
   4  * Copyright 2018 Solarflare Communications Inc.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 as published
   8  * by the Free Software Foundation, incorporated herein by reference.
   9  */
  10
  11 #include "net_driver.h"
  12 #include <linux/module.h>
  13 #include <linux/filter.h>
  14 #include "efx_channels.h"
  15 #include "efx.h"
  16 #include "efx_common.h"
  17 #include "tx_common.h"
  18 #include "rx_common.h"
  19 #include "nic.h"
  20 #include "sriov.h"
  21 #include "workarounds.h"
  22
  23 /* This is the first interrupt mode to try out of:
  24  * 0 => MSI-X
  25  * 1 => MSI
  26  * 2 => legacy
  27  */
  28 unsigned int efx_interrupt_mode = EFX_INT_MODE_MSIX;
  29
  30 /* This is the requested number of CPUs to use for Receive-Side Scaling (RSS),
  31  * i.e. the number of CPUs among which we may distribute simultaneous
  32  * interrupt handling.
  33  *
  34  * Cards without MSI-X will only target one CPU via legacy or MSI interrupt.
  35  * The default (0) means to assign an interrupt to each core.
  36  */
  37 unsigned int rss_cpus;
  38
  39 static unsigned int irq_adapt_low_thresh = 8000;
  40 module_param(irq_adapt_low_thresh, uint, 0644);
  41 MODULE_PARM_DESC(irq_adapt_low_thresh,
  42                  "Threshold score for reducing IRQ moderation");
  43
  44 static unsigned int irq_adapt_high_thresh = 16000;
  45 module_param(irq_adapt_high_thresh, uint, 0644);
  46 MODULE_PARM_DESC(irq_adapt_high_thresh,
  47                  "Threshold score for increasing IRQ moderation");
  48
  49 /* This is the weight assigned to each of the (per-channel) virtual
  50  * NAPI devices.
  51  */
  52 static int napi_weight = 64;
  53
  54 /***************
  55  * Housekeeping
  56  ***************/
  57
  58 int efx_channel_dummy_op_int(struct efx_channel *channel)
  59 {
  60         return 0;
  61 }
  62
  63 void efx_channel_dummy_op_void(struct efx_channel *channel)
  64 {
  65 }
  66
  67 static const struct efx_channel_type efx_default_channel_type = {
  68         .pre_probe              = efx_channel_dummy_op_int,
  69         .post_remove            = efx_channel_dummy_op_void,
  70         .get_name               = efx_get_channel_name,
  71         .copy                   = efx_copy_channel,
  72         .want_txqs              = efx_default_channel_want_txqs,
  73         .keep_eventq            = false,
  74         .want_pio               = true,
  75 };
  76
  77 /*************
  78  * INTERRUPTS
  79  *************/
  80
  81 static unsigned int count_online_cores(struct efx_nic *efx, bool local_node)
  82 {
  83         cpumask_var_t filter_mask;
  84         unsigned int count;
  85         int cpu;
  86
  87         if (unlikely(!zalloc_cpumask_var(&filter_mask, GFP_KERNEL))) {
  88                 netif_warn(efx, probe, efx->net_dev,
  89                            "RSS disabled due to allocation failure\n");
  90                 return 1;
  91         }
  92
  93         cpumask_copy(filter_mask, cpu_online_mask);
  94         if (local_node)
  95                 cpumask_and(filter_mask, filter_mask,
  96                             cpumask_of_pcibus(efx->pci_dev->bus));
  97
  98         count = 0;
  99         for_each_cpu(cpu, filter_mask) {
 100                 ++count;
 101                 cpumask_andnot(filter_mask, filter_mask, topology_sibling_cpumask(cpu));
 102         }
 103
 104         free_cpumask_var(filter_mask);
 105
 106         return count;
 107 }
 108
 109 static unsigned int efx_wanted_parallelism(struct efx_nic *efx)
 110 {
 111         unsigned int count;
 112
 113         if (rss_cpus) {
 114                 count = rss_cpus;
 115         } else {
 116                 count = count_online_cores(efx, true);
 117
 118                 /* If no online CPUs in local node, fallback to any online CPUs */
 119                 if (count == 0)
 120                         count = count_online_cores(efx, false);
 121         }
 122
 123         if (count > EFX_MAX_RX_QUEUES) {
 124                 netif_cond_dbg(efx, probe, efx->net_dev, !rss_cpus, warn,
 125                                "Reducing number of rx queues from %u to %u.\n",
 126                                count, EFX_MAX_RX_QUEUES);
 127                 count = EFX_MAX_RX_QUEUES;
 128         }
 129
 130         /* If RSS is requested for the PF *and* VFs then we can't write RSS
 131          * table entries that are inaccessible to VFs
 132          */
 133 #ifdef CONFIG_SFC_SRIOV
 134         if (efx->type->sriov_wanted) {
 135                 if (efx->type->sriov_wanted(efx) && efx_vf_size(efx) > 1 &&
 136                     count > efx_vf_size(efx)) {
 137                         netif_warn(efx, probe, efx->net_dev,
 138                                    "Reducing number of RSS channels from %u to %u for "
 139                                    "VF support. Increase vf-msix-limit to use more "
 140                                    "channels on the PF.\n",
 141                                    count, efx_vf_size(efx));
 142                         count = efx_vf_size(efx);
 143                 }
 144         }
 145 #endif
 146
 147         return count;
 148 }
 149
 150 static int efx_allocate_msix_channels(struct efx_nic *efx,
 151                                       unsigned int max_channels,
 152                                       unsigned int extra_channels,
 153                                       unsigned int parallelism)
 154 {
 155         unsigned int n_channels = parallelism;
 156         int vec_count;
 157         int tx_per_ev;
 158         int n_xdp_tx;
 159         int n_xdp_ev;
 160
 161         if (efx_separate_tx_channels)
 162                 n_channels *= 2;
 163         n_channels += extra_channels;
 164
 165         /* To allow XDP transmit to happen from arbitrary NAPI contexts
 166          * we allocate a TX queue per CPU. We share event queues across
 167          * multiple tx queues, assuming tx and ev queues are both
 168          * maximum size.
 169          */
 170         tx_per_ev = EFX_MAX_EVQ_SIZE / EFX_TXQ_MAX_ENT(efx);
 171         tx_per_ev = min(tx_per_ev, EFX_MAX_TXQ_PER_CHANNEL);
 172         n_xdp_tx = num_possible_cpus();
 173         n_xdp_ev = DIV_ROUND_UP(n_xdp_tx, tx_per_ev);
 174
 175         vec_count = pci_msix_vec_count(efx->pci_dev);
 176         if (vec_count < 0)
 177                 return vec_count;
 178
 179         max_channels = min_t(unsigned int, vec_count, max_channels);
 180
 181         /* Check resources.
 182          * We need a channel per event queue, plus a VI per tx queue.
 183          * This may be more pessimistic than it needs to be.
 184          */
 185         if (n_channels >= max_channels) {
 186                 efx->xdp_txq_queues_mode = EFX_XDP_TX_QUEUES_BORROWED;
 187                 netif_warn(efx, drv, efx->net_dev,
 188                            "Insufficient resources for %d XDP event queues (%d other channels, max %d)\n",
 189                            n_xdp_ev, n_channels, max_channels);
 190                 netif_warn(efx, drv, efx->net_dev,
 191                            "XDP_TX and XDP_REDIRECT might decrease device's performance\n");
 192         } else if (n_channels + n_xdp_tx > efx->max_vis) {
 193                 efx->xdp_txq_queues_mode = EFX_XDP_TX_QUEUES_BORROWED;
 194                 netif_warn(efx, drv, efx->net_dev,
 195                            "Insufficient resources for %d XDP TX queues (%d other channels, max VIs %d)\n",
 196                            n_xdp_tx, n_channels, efx->max_vis);
 197                 netif_warn(efx, drv, efx->net_dev,
 198                            "XDP_TX and XDP_REDIRECT might decrease device's performance\n");
 199         } else if (n_channels + n_xdp_ev > max_channels) {
 200                 efx->xdp_txq_queues_mode = EFX_XDP_TX_QUEUES_SHARED;
 201                 netif_warn(efx, drv, efx->net_dev,
 202                            "Insufficient resources for %d XDP event queues (%d other channels, max %d)\n",
 203                            n_xdp_ev, n_channels, max_channels);
 204
 205                 n_xdp_ev = max_channels - n_channels;
 206                 netif_warn(efx, drv, efx->net_dev,
 207                            "XDP_TX and XDP_REDIRECT will work with reduced performance (%d cpus/tx_queue)\n",
 208                            DIV_ROUND_UP(n_xdp_tx, tx_per_ev * n_xdp_ev));
 209         } else {
 210                 efx->xdp_txq_queues_mode = EFX_XDP_TX_QUEUES_DEDICATED;
 211         }
 212
 213         if (efx->xdp_txq_queues_mode != EFX_XDP_TX_QUEUES_BORROWED) {
 214                 efx->n_xdp_channels = n_xdp_ev;
 215                 efx->xdp_tx_per_channel = tx_per_ev;
 216                 efx->xdp_tx_queue_count = n_xdp_tx;
 217                 n_channels += n_xdp_ev;
 218                 netif_dbg(efx, drv, efx->net_dev,
 219                           "Allocating %d TX and %d event queues for XDP\n",
 220                           n_xdp_ev * tx_per_ev, n_xdp_ev);
 221         } else {
 222                 efx->n_xdp_channels = 0;
 223                 efx->xdp_tx_per_channel = 0;
 224                 efx->xdp_tx_queue_count = n_xdp_tx;
 225         }
 226
 227         if (vec_count < n_channels) {
 228                 netif_err(efx, drv, efx->net_dev,
 229                           "WARNING: Insufficient MSI-X vectors available (%d < %u).\n",
 230                           vec_count, n_channels);
 231                 netif_err(efx, drv, efx->net_dev,
 232                           "WARNING: Performance may be reduced.\n");
 233                 n_channels = vec_count;
 234         }
 235
 236         n_channels = min(n_channels, max_channels);
 237
 238         efx->n_channels = n_channels;
 239
 240         /* Ignore XDP tx channels when creating rx channels. */
 241         n_channels -= efx->n_xdp_channels;
 242
 243         if (efx_separate_tx_channels) {
 244                 efx->n_tx_channels =
 245                         min(max(n_channels / 2, 1U),
 246                             efx->max_tx_channels);
 247                 efx->tx_channel_offset =
 248                         n_channels - efx->n_tx_channels;
 249                 efx->n_rx_channels =
 250                         max(n_channels -
 251                             efx->n_tx_channels, 1U);
 252         } else {
 253                 efx->n_tx_channels = min(n_channels, efx->max_tx_channels);
 254                 efx->tx_channel_offset = 0;
 255                 efx->n_rx_channels = n_channels;
 256         }
 257
 258         efx->n_rx_channels = min(efx->n_rx_channels, parallelism);
 259         efx->n_tx_channels = min(efx->n_tx_channels, parallelism);
 260
 261         efx->xdp_channel_offset = n_channels;
 262
 263         netif_dbg(efx, drv, efx->net_dev,
 264                   "Allocating %u RX channels\n",
 265                   efx->n_rx_channels);
 266
 267         return efx->n_channels;
 268 }
 269
 270 /* Probe the number and type of interrupts we are able to obtain, and
 271  * the resulting numbers of channels and RX queues.
 272  */
 273 int efx_probe_interrupts(struct efx_nic *efx)
 274 {
 275         unsigned int extra_channels = 0;
 276         unsigned int rss_spread;
 277         unsigned int i, j;
 278         int rc;
 279
 280         for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++)
 281                 if (efx->extra_channel_type[i])
 282                         ++extra_channels;
 283
 284         if (efx->interrupt_mode == EFX_INT_MODE_MSIX) {
 285                 unsigned int parallelism = efx_wanted_parallelism(efx);
 286                 struct msix_entry xentries[EFX_MAX_CHANNELS];
 287                 unsigned int n_channels;
 288
 289                 rc = efx_allocate_msix_channels(efx, efx->max_channels,
 290                                                 extra_channels, parallelism);
 291                 if (rc >= 0) {
 292                         n_channels = rc;
 293                         for (i = 0; i < n_channels; i++)
 294                                 xentries[i].entry = i;
 295                         rc = pci_enable_msix_range(efx->pci_dev, xentries, 1,
 296                                                    n_channels);
 297                 }
 298                 if (rc < 0) {
 299                         /* Fall back to single channel MSI */
 300                         netif_err(efx, drv, efx->net_dev,
 301                                   "could not enable MSI-X\n");
 302                         if (efx->type->min_interrupt_mode >= EFX_INT_MODE_MSI)
 303                                 efx->interrupt_mode = EFX_INT_MODE_MSI;
 304                         else
 305                                 return rc;
 306                 } else if (rc < n_channels) {
 307                         netif_err(efx, drv, efx->net_dev,
 308                                   "WARNING: Insufficient MSI-X vectors"
 309                                   " available (%d < %u).\n", rc, n_channels);
 310                         netif_err(efx, drv, efx->net_dev,
 311                                   "WARNING: Performance may be reduced.\n");
 312                         n_channels = rc;
 313                 }
 314
 315                 if (rc > 0) {
 316                         for (i = 0; i < efx->n_channels; i++)
 317                                 efx_get_channel(efx, i)->irq =
 318                                         xentries[i].vector;
 319                 }
 320         }
 321
 322         /* Try single interrupt MSI */
 323         if (efx->interrupt_mode == EFX_INT_MODE_MSI) {
 324                 efx->n_channels = 1;
 325                 efx->n_rx_channels = 1;
 326                 efx->n_tx_channels = 1;
 327                 efx->n_xdp_channels = 0;
 328                 efx->xdp_channel_offset = efx->n_channels;
 329                 rc = pci_enable_msi(efx->pci_dev);
 330                 if (rc == 0) {
 331                         efx_get_channel(efx, 0)->irq = efx->pci_dev->irq;
 332                 } else {
 333                         netif_err(efx, drv, efx->net_dev,
 334                                   "could not enable MSI\n");
 335                         if (efx->type->min_interrupt_mode >= EFX_INT_MODE_LEGACY)
 336                                 efx->interrupt_mode = EFX_INT_MODE_LEGACY;
 337                         else
 338                                 return rc;
 339                 }
 340         }
 341
 342         /* Assume legacy interrupts */
 343         if (efx->interrupt_mode == EFX_INT_MODE_LEGACY) {
 344                 efx->n_channels = 1 + (efx_separate_tx_channels ? 1 : 0);
 345                 efx->n_rx_channels = 1;
 346                 efx->n_tx_channels = 1;
 347                 efx->n_xdp_channels = 0;
 348                 efx->xdp_channel_offset = efx->n_channels;
 349                 efx->legacy_irq = efx->pci_dev->irq;
 350         }
 351
 352         /* Assign extra channels if possible, before XDP channels */
 353         efx->n_extra_tx_channels = 0;
 354         j = efx->xdp_channel_offset;
 355         for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++) {
 356                 if (!efx->extra_channel_type[i])
 357                         continue;
 358                 if (j <= efx->tx_channel_offset + efx->n_tx_channels) {
 359                         efx->extra_channel_type[i]->handle_no_channel(efx);
 360                 } else {
 361                         --j;
 362                         efx_get_channel(efx, j)->type =
 363                                 efx->extra_channel_type[i];
 364                         if (efx_channel_has_tx_queues(efx_get_channel(efx, j)))
 365                                 efx->n_extra_tx_channels++;
 366                 }
 367         }
 368
 369         rss_spread = efx->n_rx_channels;
 370         /* RSS might be usable on VFs even if it is disabled on the PF */
 371 #ifdef CONFIG_SFC_SRIOV
 372         if (efx->type->sriov_wanted) {
 373                 efx->rss_spread = ((rss_spread > 1 ||
 374                                     !efx->type->sriov_wanted(efx)) ?
 375                                    rss_spread : efx_vf_size(efx));
 376                 return 0;
 377         }
 378 #endif
 379         efx->rss_spread = rss_spread;
 380
 381         return 0;
 382 }
 383
 384 #if defined(CONFIG_SMP)
 385 void efx_set_interrupt_affinity(struct efx_nic *efx)
 386 {
 387         const struct cpumask *numa_mask = cpumask_of_pcibus(efx->pci_dev->bus);
 388         struct efx_channel *channel;
 389         unsigned int cpu;
 390
 391         /* If no online CPUs in local node, fallback to any online CPU */
 392         if (cpumask_first_and(cpu_online_mask, numa_mask) >= nr_cpu_ids)
 393                 numa_mask = cpu_online_mask;
 394
 395         cpu = -1;
 396         efx_for_each_channel(channel, efx) {
 397                 cpu = cpumask_next_and(cpu, cpu_online_mask, numa_mask);
 398                 if (cpu >= nr_cpu_ids)
 399                         cpu = cpumask_first_and(cpu_online_mask, numa_mask);
 400                 irq_set_affinity_hint(channel->irq, cpumask_of(cpu));
 401         }
 402 }
 403
 404 void efx_clear_interrupt_affinity(struct efx_nic *efx)
 405 {
 406         struct efx_channel *channel;
 407
 408         efx_for_each_channel(channel, efx)
 409                 irq_set_affinity_hint(channel->irq, NULL);
 410 }
 411 #else
 412 void
 413 efx_set_interrupt_affinity(struct efx_nic *efx __attribute__ ((unused)))
 414 {
 415 }
 416
 417 void
 418 efx_clear_interrupt_affinity(struct efx_nic *efx __attribute__ ((unused)))
 419 {
 420 }
 421 #endif /* CONFIG_SMP */
 422
 423 void efx_remove_interrupts(struct efx_nic *efx)
 424 {
 425         struct efx_channel *channel;
 426
 427         /* Remove MSI/MSI-X interrupts */
 428         efx_for_each_channel(channel, efx)
 429                 channel->irq = 0;
 430         pci_disable_msi(efx->pci_dev);
 431         pci_disable_msix(efx->pci_dev);
 432
 433         /* Remove legacy interrupt */
 434         efx->legacy_irq = 0;
 435 }
 436
 437 /***************
 438  * EVENT QUEUES
 439  ***************/
 440
 441 /* Create event queue
 442  * Event queue memory allocations are done only once.  If the channel
 443  * is reset, the memory buffer will be reused; this guards against
 444  * errors during channel reset and also simplifies interrupt handling.
 445  */
 446 int efx_probe_eventq(struct efx_channel *channel)
 447 {
 448         struct efx_nic *efx = channel->efx;
 449         unsigned long entries;
 450
 451         netif_dbg(efx, probe, efx->net_dev,
 452                   "chan %d create event queue\n", channel->channel);
 453
 454         /* Build an event queue with room for one event per tx and rx buffer,
 455          * plus some extra for link state events and MCDI completions.
 456          */
 457         entries = roundup_pow_of_two(efx->rxq_entries + efx->txq_entries + 128);
 458         EFX_WARN_ON_PARANOID(entries > EFX_MAX_EVQ_SIZE);
 459         channel->eventq_mask = max(entries, EFX_MIN_EVQ_SIZE) - 1;
 460
 461         return efx_nic_probe_eventq(channel);
 462 }
 463
 464 /* Prepare channel's event queue */
 465 int efx_init_eventq(struct efx_channel *channel)
 466 {
 467         struct efx_nic *efx = channel->efx;
 468         int rc;
 469
 470         EFX_WARN_ON_PARANOID(channel->eventq_init);
 471
 472         netif_dbg(efx, drv, efx->net_dev,
 473                   "chan %d init event queue\n", channel->channel);
 474
 475         rc = efx_nic_init_eventq(channel);
 476         if (rc == 0) {
 477                 efx->type->push_irq_moderation(channel);
 478                 channel->eventq_read_ptr = 0;
 479                 channel->eventq_init = true;
 480         }
 481         return rc;
 482 }
 483
 484 /* Enable event queue processing and NAPI */
 485 void efx_start_eventq(struct efx_channel *channel)
 486 {
 487         netif_dbg(channel->efx, ifup, channel->efx->net_dev,
 488                   "chan %d start event queue\n", channel->channel);
 489
 490         /* Make sure the NAPI handler sees the enabled flag set */
 491         channel->enabled = true;
 492         smp_wmb();
 493
 494         napi_enable(&channel->napi_str);
 495         efx_nic_eventq_read_ack(channel);
 496 }
 497
 498 /* Disable event queue processing and NAPI */
 499 void efx_stop_eventq(struct efx_channel *channel)
 500 {
 501         if (!channel->enabled)
 502                 return;
 503
 504         napi_disable(&channel->napi_str);
 505         channel->enabled = false;
 506 }
 507
 508 void efx_fini_eventq(struct efx_channel *channel)
 509 {
 510         if (!channel->eventq_init)
 511                 return;
 512
 513         netif_dbg(channel->efx, drv, channel->efx->net_dev,
 514                   "chan %d fini event queue\n", channel->channel);
 515
 516         efx_nic_fini_eventq(channel);
 517         channel->eventq_init = false;
 518 }
 519
 520 void efx_remove_eventq(struct efx_channel *channel)
 521 {
 522         netif_dbg(channel->efx, drv, channel->efx->net_dev,
 523                   "chan %d remove event queue\n", channel->channel);
 524
 525         efx_nic_remove_eventq(channel);
 526 }
 527
 528 /**************************************************************************
 529  *
 530  * Channel handling
 531  *
 532  *************************************************************************/
 533
 534 #ifdef CONFIG_RFS_ACCEL
 535 static void efx_filter_rfs_expire(struct work_struct *data)
 536 {
 537         struct delayed_work *dwork = to_delayed_work(data);
 538         struct efx_channel *channel;
 539         unsigned int time, quota;
 540
 541         channel = container_of(dwork, struct efx_channel, filter_work);
 542         time = jiffies - channel->rfs_last_expiry;
 543         quota = channel->rfs_filter_count * time / (30 * HZ);
 544         if (quota >= 20 && __efx_filter_rfs_expire(channel, min(channel->rfs_filter_count, quota)))
 545                 channel->rfs_last_expiry += time;
 546         /* Ensure we do more work eventually even if NAPI poll is not happening */
 547         schedule_delayed_work(dwork, 30 * HZ);
 548 }
 549 #endif
 550
 551 /* Allocate and initialise a channel structure. */
 552 static struct efx_channel *efx_alloc_channel(struct efx_nic *efx, int i)
 553 {
 554         struct efx_rx_queue *rx_queue;
 555         struct efx_tx_queue *tx_queue;
 556         struct efx_channel *channel;
 557         int j;
 558
 559         channel = kzalloc(sizeof(*channel), GFP_KERNEL);
 560         if (!channel)
 561                 return NULL;
 562
 563         channel->efx = efx;
 564         channel->channel = i;
 565         channel->type = &efx_default_channel_type;
 566
 567         for (j = 0; j < EFX_MAX_TXQ_PER_CHANNEL; j++) {
 568                 tx_queue = &channel->tx_queue[j];
 569                 tx_queue->efx = efx;
 570                 tx_queue->queue = -1;
 571                 tx_queue->label = j;
 572                 tx_queue->channel = channel;
 573         }
 574
 575 #ifdef CONFIG_RFS_ACCEL
 576         INIT_DELAYED_WORK(&channel->filter_work, efx_filter_rfs_expire);
 577 #endif
 578
 579         rx_queue = &channel->rx_queue;
 580         rx_queue->efx = efx;
 581         timer_setup(&rx_queue->slow_fill, efx_rx_slow_fill, 0);
 582
 583         return channel;
 584 }
 585
 586 int efx_init_channels(struct efx_nic *efx)
 587 {
 588         unsigned int i;
 589
 590         for (i = 0; i < EFX_MAX_CHANNELS; i++) {
 591                 efx->channel[i] = efx_alloc_channel(efx, i);
 592                 if (!efx->channel[i])
 593                         return -ENOMEM;
 594                 efx->msi_context[i].efx = efx;
 595                 efx->msi_context[i].index = i;
 596         }
 597
 598         /* Higher numbered interrupt modes are less capable! */
 599         efx->interrupt_mode = min(efx->type->min_interrupt_mode,
 600                                   efx_interrupt_mode);
 601
 602         efx->max_channels = EFX_MAX_CHANNELS;
 603         efx->max_tx_channels = EFX_MAX_CHANNELS;
 604
 605         return 0;
 606 }
 607
 608 void efx_fini_channels(struct efx_nic *efx)
 609 {
 610         unsigned int i;
 611
 612         for (i = 0; i < EFX_MAX_CHANNELS; i++)
 613                 if (efx->channel[i]) {
 614                         kfree(efx->channel[i]);
 615                         efx->channel[i] = NULL;
 616                 }
 617 }
 618
 619 /* Allocate and initialise a channel structure, copying parameters
 620  * (but not resources) from an old channel structure.
 621  */
 622 struct efx_channel *efx_copy_channel(const struct efx_channel *old_channel)
 623 {
 624         struct efx_rx_queue *rx_queue;
 625         struct efx_tx_queue *tx_queue;
 626         struct efx_channel *channel;
 627         int j;
 628
 629         channel = kmalloc(sizeof(*channel), GFP_KERNEL);
 630         if (!channel)
 631                 return NULL;
 632
 633         *channel = *old_channel;
 634
 635         channel->napi_dev = NULL;
 636         INIT_HLIST_NODE(&channel->napi_str.napi_hash_node);
 637         channel->napi_str.napi_id = 0;
 638         channel->napi_str.state = 0;
 639         memset(&channel->eventq, 0, sizeof(channel->eventq));
 640
 641         for (j = 0; j < EFX_MAX_TXQ_PER_CHANNEL; j++) {
 642                 tx_queue = &channel->tx_queue[j];
 643                 if (tx_queue->channel)
 644                         tx_queue->channel = channel;
 645                 tx_queue->buffer = NULL;
 646                 tx_queue->cb_page = NULL;
 647                 memset(&tx_queue->txd, 0, sizeof(tx_queue->txd));
 648         }
 649
 650         rx_queue = &channel->rx_queue;
 651         rx_queue->buffer = NULL;
 652         memset(&rx_queue->rxd, 0, sizeof(rx_queue->rxd));
 653         timer_setup(&rx_queue->slow_fill, efx_rx_slow_fill, 0);
 654 #ifdef CONFIG_RFS_ACCEL
 655         INIT_DELAYED_WORK(&channel->filter_work, efx_filter_rfs_expire);
 656 #endif
 657
 658         return channel;
 659 }
 660
 661 static int efx_probe_channel(struct efx_channel *channel)
 662 {
 663         struct efx_tx_queue *tx_queue;
 664         struct efx_rx_queue *rx_queue;
 665         int rc;
 666
 667         netif_dbg(channel->efx, probe, channel->efx->net_dev,
 668                   "creating channel %d\n", channel->channel);
 669
 670         rc = channel->type->pre_probe(channel);
 671         if (rc)
 672                 goto fail;
 673
 674         rc = efx_probe_eventq(channel);
 675         if (rc)
 676                 goto fail;
 677
 678         efx_for_each_channel_tx_queue(tx_queue, channel) {
 679                 rc = efx_probe_tx_queue(tx_queue);
 680                 if (rc)
 681                         goto fail;
 682         }
 683
 684         efx_for_each_channel_rx_queue(rx_queue, channel) {
 685                 rc = efx_probe_rx_queue(rx_queue);
 686                 if (rc)
 687                         goto fail;
 688         }
 689
 690         channel->rx_list = NULL;
 691
 692         return 0;
 693
 694 fail:
 695         efx_remove_channel(channel);
 696         return rc;
 697 }
 698
 699 void efx_get_channel_name(struct efx_channel *channel, char *buf, size_t len)
 700 {
 701         struct efx_nic *efx = channel->efx;
 702         const char *type;
 703         int number;
 704
 705         number = channel->channel;
 706
 707         if (number >= efx->xdp_channel_offset &&
 708             !WARN_ON_ONCE(!efx->n_xdp_channels)) {
 709                 type = "-xdp";
 710                 number -= efx->xdp_channel_offset;
 711         } else if (efx->tx_channel_offset == 0) {
 712                 type = "";
 713         } else if (number < efx->tx_channel_offset) {
 714                 type = "-rx";
 715         } else {
 716                 type = "-tx";
 717                 number -= efx->tx_channel_offset;
 718         }
 719         snprintf(buf, len, "%s%s-%d", efx->name, type, number);
 720 }
 721
 722 void efx_set_channel_names(struct efx_nic *efx)
 723 {
 724         struct efx_channel *channel;
 725
 726         efx_for_each_channel(channel, efx)
 727                 channel->type->get_name(channel,
 728                                         efx->msi_context[channel->channel].name,
 729                                         sizeof(efx->msi_context[0].name));
 730 }
 731
 732 int efx_probe_channels(struct efx_nic *efx)
 733 {
 734         struct efx_channel *channel;
 735         int rc;
 736
 737         /* Restart special buffer allocation */
 738         efx->next_buffer_table = 0;
 739
 740         /* Probe channels in reverse, so that any 'extra' channels
 741          * use the start of the buffer table. This allows the traffic
 742          * channels to be resized without moving them or wasting the
 743          * entries before them.
 744          */
 745         efx_for_each_channel_rev(channel, efx) {
 746                 rc = efx_probe_channel(channel);
 747                 if (rc) {
 748                         netif_err(efx, probe, efx->net_dev,
 749                                   "failed to create channel %d\n",
 750                                   channel->channel);
 751                         goto fail;
 752                 }
 753         }
 754         efx_set_channel_names(efx);
 755
 756         return 0;
 757
 758 fail:
 759         efx_remove_channels(efx);
 760         return rc;
 761 }
 762
 763 void efx_remove_channel(struct efx_channel *channel)
 764 {
 765         struct efx_tx_queue *tx_queue;
 766         struct efx_rx_queue *rx_queue;
 767
 768         netif_dbg(channel->efx, drv, channel->efx->net_dev,
 769                   "destroy chan %d\n", channel->channel);
 770
 771         efx_for_each_channel_rx_queue(rx_queue, channel)
 772                 efx_remove_rx_queue(rx_queue);
 773         efx_for_each_channel_tx_queue(tx_queue, channel)
 774                 efx_remove_tx_queue(tx_queue);
 775         efx_remove_eventq(channel);
 776         channel->type->post_remove(channel);
 777 }
 778
 779 void efx_remove_channels(struct efx_nic *efx)
 780 {
 781         struct efx_channel *channel;
 782
 783         efx_for_each_channel(channel, efx)
 784                 efx_remove_channel(channel);
 785
 786         kfree(efx->xdp_tx_queues);
 787 }
 788
 789 static int efx_set_xdp_tx_queue(struct efx_nic *efx, int xdp_queue_number,
 790                                 struct efx_tx_queue *tx_queue)
 791 {
 792         if (xdp_queue_number >= efx->xdp_tx_queue_count)
 793                 return -EINVAL;
 794
 795         netif_dbg(efx, drv, efx->net_dev,
 796                   "Channel %u TXQ %u is XDP %u, HW %u\n",
 797                   tx_queue->channel->channel, tx_queue->label,
 798                   xdp_queue_number, tx_queue->queue);
 799         efx->xdp_tx_queues[xdp_queue_number] = tx_queue;
 800         return 0;
 801 }
 802
 803 static void efx_set_xdp_channels(struct efx_nic *efx)
 804 {
 805         struct efx_tx_queue *tx_queue;
 806         struct efx_channel *channel;
 807         unsigned int next_queue = 0;
 808         int xdp_queue_number = 0;
 809         int rc;
 810
 811         /* We need to mark which channels really have RX and TX
 812          * queues, and adjust the TX queue numbers if we have separate
 813          * RX-only and TX-only channels.
 814          */
 815         efx_for_each_channel(channel, efx) {
 816                 if (channel->channel < efx->tx_channel_offset)
 817                         continue;
 818
 819                 if (efx_channel_is_xdp_tx(channel)) {
 820                         efx_for_each_channel_tx_queue(tx_queue, channel) {
 821                                 tx_queue->queue = next_queue++;
 822                                 rc = efx_set_xdp_tx_queue(efx, xdp_queue_number,
 823                                                           tx_queue);
 824                                 if (rc == 0)
 825                                         xdp_queue_number++;
 826                         }
 827                 } else {
 828                         efx_for_each_channel_tx_queue(tx_queue, channel) {
 829                                 tx_queue->queue = next_queue++;
 830                                 netif_dbg(efx, drv, efx->net_dev,
 831                                           "Channel %u TXQ %u is HW %u\n",
 832                                           channel->channel, tx_queue->label,
 833                                           tx_queue->queue);
 834                         }
 835
 836                         /* If XDP is borrowing queues from net stack, it must
 837                          * use the queue with no csum offload, which is the
 838                          * first one of the channel
 839                          * (note: tx_queue_by_type is not initialized yet)
 840                          */
 841                         if (efx->xdp_txq_queues_mode ==
 842                             EFX_XDP_TX_QUEUES_BORROWED) {
 843                                 tx_queue = &channel->tx_queue[0];
 844                                 rc = efx_set_xdp_tx_queue(efx, xdp_queue_number,
 845                                                           tx_queue);
 846                                 if (rc == 0)
 847                                         xdp_queue_number++;
 848                         }
 849                 }
 850         }
 851         WARN_ON(efx->xdp_txq_queues_mode == EFX_XDP_TX_QUEUES_DEDICATED &&
 852                 xdp_queue_number != efx->xdp_tx_queue_count);
 853         WARN_ON(efx->xdp_txq_queues_mode != EFX_XDP_TX_QUEUES_DEDICATED &&
 854                 xdp_queue_number > efx->xdp_tx_queue_count);
 855
 856         /* If we have more CPUs than assigned XDP TX queues, assign the already
 857          * existing queues to the exceeding CPUs
 858          */
 859         next_queue = 0;
 860         while (xdp_queue_number < efx->xdp_tx_queue_count) {
 861                 tx_queue = efx->xdp_tx_queues[next_queue++];
 862                 rc = efx_set_xdp_tx_queue(efx, xdp_queue_number, tx_queue);
 863                 if (rc == 0)
 864                         xdp_queue_number++;
 865         }
 866 }
 867
 868 int efx_realloc_channels(struct efx_nic *efx, u32 rxq_entries, u32 txq_entries)
 869 {
 870         struct efx_channel *other_channel[EFX_MAX_CHANNELS], *channel,
 871                            *ptp_channel = efx_ptp_channel(efx);
 872         struct efx_ptp_data *ptp_data = efx->ptp_data;
 873         unsigned int i, next_buffer_table = 0;
 874         u32 old_rxq_entries, old_txq_entries;
 875         int rc, rc2;
 876
 877         rc = efx_check_disabled(efx);
 878         if (rc)
 879                 return rc;
 880
 881         /* Not all channels should be reallocated. We must avoid
 882          * reallocating their buffer table entries.
 883          */
 884         efx_for_each_channel(channel, efx) {
 885                 struct efx_rx_queue *rx_queue;
 886                 struct efx_tx_queue *tx_queue;
 887
 888                 if (channel->type->copy)
 889                         continue;
 890                 next_buffer_table = max(next_buffer_table,
 891                                         channel->eventq.index +
 892                                         channel->eventq.entries);
 893                 efx_for_each_channel_rx_queue(rx_queue, channel)
 894                         next_buffer_table = max(next_buffer_table,
 895                                                 rx_queue->rxd.index +
 896                                                 rx_queue->rxd.entries);
 897                 efx_for_each_channel_tx_queue(tx_queue, channel)
 898                         next_buffer_table = max(next_buffer_table,
 899                                                 tx_queue->txd.index +
 900                                                 tx_queue->txd.entries);
 901         }
 902
 903         efx_device_detach_sync(efx);
 904         efx_stop_all(efx);
 905         efx_soft_disable_interrupts(efx);
 906
 907         /* Clone channels (where possible) */
 908         memset(other_channel, 0, sizeof(other_channel));
 909         for (i = 0; i < efx->n_channels; i++) {
 910                 channel = efx->channel[i];
 911                 if (channel->type->copy)
 912                         channel = channel->type->copy(channel);
 913                 if (!channel) {
 914                         rc = -ENOMEM;
 915                         goto out;
 916                 }
 917                 other_channel[i] = channel;
 918         }
 919
 920         /* Swap entry counts and channel pointers */
 921         old_rxq_entries = efx->rxq_entries;
 922         old_txq_entries = efx->txq_entries;
 923         efx->rxq_entries = rxq_entries;
 924         efx->txq_entries = txq_entries;
 925         for (i = 0; i < efx->n_channels; i++)
 926                 swap(efx->channel[i], other_channel[i]);
 927
 928         /* Restart buffer table allocation */
 929         efx->next_buffer_table = next_buffer_table;
 930
 931         for (i = 0; i < efx->n_channels; i++) {
 932                 channel = efx->channel[i];
 933                 if (!channel->type->copy)
 934                         continue;
 935                 rc = efx_probe_channel(channel);
 936                 if (rc)
 937                         goto rollback;
 938                 efx_init_napi_channel(efx->channel[i]);
 939         }
 940
 941         efx_set_xdp_channels(efx);
 942 out:
 943         efx->ptp_data = NULL;
 944         /* Destroy unused channel structures */
 945         for (i = 0; i < efx->n_channels; i++) {
 946                 channel = other_channel[i];
 947                 if (channel && channel->type->copy) {
 948                         efx_fini_napi_channel(channel);
 949                         efx_remove_channel(channel);
 950                         kfree(channel);
 951                 }
 952         }
 953
 954         efx->ptp_data = ptp_data;
 955         rc2 = efx_soft_enable_interrupts(efx);
 956         if (rc2) {
 957                 rc = rc ? rc : rc2;
 958                 netif_err(efx, drv, efx->net_dev,
 959                           "unable to restart interrupts on channel reallocation\n");
 960                 efx_schedule_reset(efx, RESET_TYPE_DISABLE);
 961         } else {
 962                 efx_start_all(efx);
 963                 efx_device_attach_if_not_resetting(efx);
 964         }
 965         return rc;
 966
 967 rollback:
 968         /* Swap back */
 969         efx->rxq_entries = old_rxq_entries;
 970         efx->txq_entries = old_txq_entries;
 971         for (i = 0; i < efx->n_channels; i++)
 972                 swap(efx->channel[i], other_channel[i]);
 973         efx_ptp_update_channel(efx, ptp_channel);
 974         goto out;
 975 }
 976
 977 int efx_set_channels(struct efx_nic *efx)
 978 {
 979         struct efx_channel *channel;
 980         int rc;
 981
 982         efx->tx_channel_offset =
 983                 efx_separate_tx_channels ?
 984                 efx->n_channels - efx->n_tx_channels : 0;
 985
 986         if (efx->xdp_tx_queue_count) {
 987                 EFX_WARN_ON_PARANOID(efx->xdp_tx_queues);
 988
 989                 /* Allocate array for XDP TX queue lookup. */
 990                 efx->xdp_tx_queues = kcalloc(efx->xdp_tx_queue_count,
 991                                              sizeof(*efx->xdp_tx_queues),
 992                                              GFP_KERNEL);
 993                 if (!efx->xdp_tx_queues)
 994                         return -ENOMEM;
 995         }
 996
 997         efx_for_each_channel(channel, efx) {
 998                 if (channel->channel < efx->n_rx_channels)
 999                         channel->rx_queue.core_index = channel->channel;
1000                 else
1001                         channel->rx_queue.core_index = -1;
1002         }
1003
1004         efx_set_xdp_channels(efx);
1005
1006         rc = netif_set_real_num_tx_queues(efx->net_dev, efx->n_tx_channels);
1007         if (rc)
1008                 return rc;
1009         return netif_set_real_num_rx_queues(efx->net_dev, efx->n_rx_channels);
1010 }
1011
1012 bool efx_default_channel_want_txqs(struct efx_channel *channel)
1013 {
1014         return channel->channel - channel->efx->tx_channel_offset <
1015                 channel->efx->n_tx_channels;
1016 }
1017
1018 /*************
1019  * START/STOP
1020  *************/
1021
1022 int efx_soft_enable_interrupts(struct efx_nic *efx)
1023 {
1024         struct efx_channel *channel, *end_channel;
1025         int rc;
1026
1027         BUG_ON(efx->state == STATE_DISABLED);
1028
1029         efx->irq_soft_enabled = true;
1030         smp_wmb();
1031
1032         efx_for_each_channel(channel, efx) {
1033                 if (!channel->type->keep_eventq) {
1034                         rc = efx_init_eventq(channel);
1035                         if (rc)
1036                                 goto fail;
1037                 }
1038                 efx_start_eventq(channel);
1039         }
1040
1041         efx_mcdi_mode_event(efx);
1042
1043         return 0;
1044 fail:
1045         end_channel = channel;
1046         efx_for_each_channel(channel, efx) {
1047                 if (channel == end_channel)
1048                         break;
1049                 efx_stop_eventq(channel);
1050                 if (!channel->type->keep_eventq)
1051                         efx_fini_eventq(channel);
1052         }
1053
1054         return rc;
1055 }
1056
1057 void efx_soft_disable_interrupts(struct efx_nic *efx)
1058 {
1059         struct efx_channel *channel;
1060
1061         if (efx->state == STATE_DISABLED)
1062                 return;
1063
1064         efx_mcdi_mode_poll(efx);
1065
1066         efx->irq_soft_enabled = false;
1067         smp_wmb();
1068
1069         if (efx->legacy_irq)
1070                 synchronize_irq(efx->legacy_irq);
1071
1072         efx_for_each_channel(channel, efx) {
1073                 if (channel->irq)
1074                         synchronize_irq(channel->irq);
1075
1076                 efx_stop_eventq(channel);
1077                 if (!channel->type->keep_eventq)
1078                         efx_fini_eventq(channel);
1079         }
1080
1081         /* Flush the asynchronous MCDI request queue */
1082         efx_mcdi_flush_async(efx);
1083 }
1084
1085 int efx_enable_interrupts(struct efx_nic *efx)
1086 {
1087         struct efx_channel *channel, *end_channel;
1088         int rc;
1089
1090         /* TODO: Is this really a bug? */
1091         BUG_ON(efx->state == STATE_DISABLED);
1092
1093         if (efx->eeh_disabled_legacy_irq) {
1094                 enable_irq(efx->legacy_irq);
1095                 efx->eeh_disabled_legacy_irq = false;
1096         }
1097
1098         efx->type->irq_enable_master(efx);
1099
1100         efx_for_each_channel(channel, efx) {
1101                 if (channel->type->keep_eventq) {
1102                         rc = efx_init_eventq(channel);
1103                         if (rc)
1104                                 goto fail;
1105                 }
1106         }
1107
1108         rc = efx_soft_enable_interrupts(efx);
1109         if (rc)
1110                 goto fail;
1111
1112         return 0;
1113
1114 fail:
1115         end_channel = channel;
1116         efx_for_each_channel(channel, efx) {
1117                 if (channel == end_channel)
1118                         break;
1119                 if (channel->type->keep_eventq)
1120                         efx_fini_eventq(channel);
1121         }
1122
1123         efx->type->irq_disable_non_ev(efx);
1124
1125         return rc;
1126 }
1127
1128 void efx_disable_interrupts(struct efx_nic *efx)
1129 {
1130         struct efx_channel *channel;
1131
1132         efx_soft_disable_interrupts(efx);
1133
1134         efx_for_each_channel(channel, efx) {
1135                 if (channel->type->keep_eventq)
1136                         efx_fini_eventq(channel);
1137         }
1138
1139         efx->type->irq_disable_non_ev(efx);
1140 }
1141
1142 void efx_start_channels(struct efx_nic *efx)
1143 {
1144         struct efx_tx_queue *tx_queue;
1145         struct efx_rx_queue *rx_queue;
1146         struct efx_channel *channel;
1147
1148         efx_for_each_channel_rev(channel, efx) {
1149                 efx_for_each_channel_tx_queue(tx_queue, channel) {
1150                         efx_init_tx_queue(tx_queue);
1151                         atomic_inc(&efx->active_queues);
1152                 }
1153
1154                 efx_for_each_channel_rx_queue(rx_queue, channel) {
1155                         efx_init_rx_queue(rx_queue);
1156                         atomic_inc(&efx->active_queues);
1157                         efx_stop_eventq(channel);
1158                         efx_fast_push_rx_descriptors(rx_queue, false);
1159                         efx_start_eventq(channel);
1160                 }
1161
1162                 WARN_ON(channel->rx_pkt_n_frags);
1163         }
1164 }
1165
1166 void efx_stop_channels(struct efx_nic *efx)
1167 {
1168         struct efx_tx_queue *tx_queue;
1169         struct efx_rx_queue *rx_queue;
1170         struct efx_channel *channel;
1171         int rc = 0;
1172
1173         /* Stop RX refill */
1174         efx_for_each_channel(channel, efx) {
1175                 efx_for_each_channel_rx_queue(rx_queue, channel)
1176                         rx_queue->refill_enabled = false;
1177         }
1178
1179         efx_for_each_channel(channel, efx) {
1180                 /* RX packet processing is pipelined, so wait for the
1181                  * NAPI handler to complete.  At least event queue 0
1182                  * might be kept active by non-data events, so don't
1183                  * use napi_synchronize() but actually disable NAPI
1184                  * temporarily.
1185                  */
1186                 if (efx_channel_has_rx_queue(channel)) {
1187                         efx_stop_eventq(channel);
1188                         efx_start_eventq(channel);
1189                 }
1190         }
1191
1192         if (efx->type->fini_dmaq)
1193                 rc = efx->type->fini_dmaq(efx);
1194
1195         if (rc) {
1196                 netif_err(efx, drv, efx->net_dev, "failed to flush queues\n");
1197         } else {
1198                 netif_dbg(efx, drv, efx->net_dev,
1199                           "successfully flushed all queues\n");
1200         }
1201
1202         efx_for_each_channel(channel, efx) {
1203                 efx_for_each_channel_rx_queue(rx_queue, channel)
1204                         efx_fini_rx_queue(rx_queue);
1205                 efx_for_each_channel_tx_queue(tx_queue, channel)
1206                         efx_fini_tx_queue(tx_queue);
1207         }
1208 }
1209
1210 /**************************************************************************
1211  *
1212  * NAPI interface
1213  *
1214  *************************************************************************/
1215
1216 /* Process channel's event queue
1217  *
1218  * This function is responsible for processing the event queue of a
1219  * single channel.  The caller must guarantee that this function will
1220  * never be concurrently called more than once on the same channel,
1221  * though different channels may be being processed concurrently.
1222  */
1223 static int efx_process_channel(struct efx_channel *channel, int budget)
1224 {
1225         struct efx_tx_queue *tx_queue;
1226         struct list_head rx_list;
1227         int spent;
1228
1229         if (unlikely(!channel->enabled))
1230                 return 0;
1231
1232         /* Prepare the batch receive list */
1233         EFX_WARN_ON_PARANOID(channel->rx_list != NULL);
1234         INIT_LIST_HEAD(&rx_list);
1235         channel->rx_list = &rx_list;
1236
1237         efx_for_each_channel_tx_queue(tx_queue, channel) {
1238                 tx_queue->pkts_compl = 0;
1239                 tx_queue->bytes_compl = 0;
1240         }
1241
1242         spent = efx_nic_process_eventq(channel, budget);
1243         if (spent && efx_channel_has_rx_queue(channel)) {
1244                 struct efx_rx_queue *rx_queue =
1245                         efx_channel_get_rx_queue(channel);
1246
1247                 efx_rx_flush_packet(channel);
1248                 efx_fast_push_rx_descriptors(rx_queue, true);
1249         }
1250
1251         /* Update BQL */
1252         efx_for_each_channel_tx_queue(tx_queue, channel) {
1253                 if (tx_queue->bytes_compl) {
1254                         netdev_tx_completed_queue(tx_queue->core_txq,
1255                                                   tx_queue->pkts_compl,
1256                                                   tx_queue->bytes_compl);
1257                 }
1258         }
1259
1260         /* Receive any packets we queued up */
1261         netif_receive_skb_list(channel->rx_list);
1262         channel->rx_list = NULL;
1263
1264         return spent;
1265 }
1266
1267 static void efx_update_irq_mod(struct efx_nic *efx, struct efx_channel *channel)
1268 {
1269         int step = efx->irq_mod_step_us;
1270
1271         if (channel->irq_mod_score < irq_adapt_low_thresh) {
1272                 if (channel->irq_moderation_us > step) {
1273                         channel->irq_moderation_us -= step;
1274                         efx->type->push_irq_moderation(channel);
1275                 }
1276         } else if (channel->irq_mod_score > irq_adapt_high_thresh) {
1277                 if (channel->irq_moderation_us <
1278                     efx->irq_rx_moderation_us) {
1279                         channel->irq_moderation_us += step;
1280                         efx->type->push_irq_moderation(channel);
1281                 }
1282         }
1283
1284         channel->irq_count = 0;
1285         channel->irq_mod_score = 0;
1286 }
1287
1288 /* NAPI poll handler
1289  *
1290  * NAPI guarantees serialisation of polls of the same device, which
1291  * provides the guarantee required by efx_process_channel().
1292  */
1293 static int efx_poll(struct napi_struct *napi, int budget)
1294 {
1295         struct efx_channel *channel =
1296                 container_of(napi, struct efx_channel, napi_str);
1297         struct efx_nic *efx = channel->efx;
1298 #ifdef CONFIG_RFS_ACCEL
1299         unsigned int time;
1300 #endif
1301         int spent;
1302
1303         netif_vdbg(efx, intr, efx->net_dev,
1304                    "channel %d NAPI poll executing on CPU %d\n",
1305                    channel->channel, raw_smp_processor_id());
1306
1307         spent = efx_process_channel(channel, budget);
1308
1309         xdp_do_flush_map();
1310
1311         if (spent < budget) {
1312                 if (efx_channel_has_rx_queue(channel) &&
1313                     efx->irq_rx_adaptive &&
1314                     unlikely(++channel->irq_count == 1000)) {
1315                         efx_update_irq_mod(efx, channel);
1316                 }
1317
1318 #ifdef CONFIG_RFS_ACCEL
1319                 /* Perhaps expire some ARFS filters */
1320                 time = jiffies - channel->rfs_last_expiry;
1321                 /* Would our quota be >= 20? */
1322                 if (channel->rfs_filter_count * time >= 600 * HZ)
1323                         mod_delayed_work(system_wq, &channel->filter_work, 0);
1324 #endif
1325
1326                 /* There is no race here; although napi_disable() will
1327                  * only wait for napi_complete(), this isn't a problem
1328                  * since efx_nic_eventq_read_ack() will have no effect if
1329                  * interrupts have already been disabled.
1330                  */
1331                 if (napi_complete_done(napi, spent))
1332                         efx_nic_eventq_read_ack(channel);
1333         }
1334
1335         return spent;
1336 }
1337
1338 void efx_init_napi_channel(struct efx_channel *channel)
1339 {
1340         struct efx_nic *efx = channel->efx;
1341
1342         channel->napi_dev = efx->net_dev;
1343         netif_napi_add(channel->napi_dev, &channel->napi_str,
1344                        efx_poll, napi_weight);
1345 }
1346
1347 void efx_init_napi(struct efx_nic *efx)
1348 {
1349         struct efx_channel *channel;
1350
1351         efx_for_each_channel(channel, efx)
1352                 efx_init_napi_channel(channel);
1353 }
1354
1355 void efx_fini_napi_channel(struct efx_channel *channel)
1356 {
1357         if (channel->napi_dev)
1358                 netif_napi_del(&channel->napi_str);
1359
1360         channel->napi_dev = NULL;
1361 }
1362
1363 void efx_fini_napi(struct efx_nic *efx)
1364 {
1365         struct efx_channel *channel;
1366
1367         efx_for_each_channel(channel, efx)
1368                 efx_fini_napi_channel(channel);
1369 }