drivers/net/ethernet/sfc/efx_channels.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /****************************************************************************
   3  * Driver for Solarflare network controllers and boards
   4  * Copyright 2018 Solarflare Communications Inc.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 as published
   8  * by the Free Software Foundation, incorporated herein by reference.
   9  */
  10
  11 #include "net_driver.h"
  12 #include <linux/module.h>
  13 #include "efx_channels.h"
  14 #include "efx.h"
  15 #include "efx_common.h"
  16 #include "tx_common.h"
  17 #include "rx_common.h"
  18 #include "nic.h"
  19 #include "sriov.h"
  20
  21 /* This is the first interrupt mode to try out of:
  22  * 0 => MSI-X
  23  * 1 => MSI
  24  * 2 => legacy
  25  */
  26 unsigned int efx_interrupt_mode = EFX_INT_MODE_MSIX;
  27
  28 /* This is the requested number of CPUs to use for Receive-Side Scaling (RSS),
  29  * i.e. the number of CPUs among which we may distribute simultaneous
  30  * interrupt handling.
  31  *
  32  * Cards without MSI-X will only target one CPU via legacy or MSI interrupt.
  33  * The default (0) means to assign an interrupt to each core.
  34  */
  35 unsigned int rss_cpus;
  36
  37 static unsigned int irq_adapt_low_thresh = 8000;
  38 module_param(irq_adapt_low_thresh, uint, 0644);
  39 MODULE_PARM_DESC(irq_adapt_low_thresh,
  40                  "Threshold score for reducing IRQ moderation");
  41
  42 static unsigned int irq_adapt_high_thresh = 16000;
  43 module_param(irq_adapt_high_thresh, uint, 0644);
  44 MODULE_PARM_DESC(irq_adapt_high_thresh,
  45                  "Threshold score for increasing IRQ moderation");
  46
  47 /* This is the weight assigned to each of the (per-channel) virtual
  48  * NAPI devices.
  49  */
  50 static int napi_weight = 64;
  51
  52 /***************
  53  * Housekeeping
  54  ***************/
  55
  56 int efx_channel_dummy_op_int(struct efx_channel *channel)
  57 {
  58         return 0;
  59 }
  60
  61 void efx_channel_dummy_op_void(struct efx_channel *channel)
  62 {
  63 }
  64
  65 static const struct efx_channel_type efx_default_channel_type = {
  66         .pre_probe              = efx_channel_dummy_op_int,
  67         .post_remove            = efx_channel_dummy_op_void,
  68         .get_name               = efx_get_channel_name,
  69         .copy                   = efx_copy_channel,
  70         .want_txqs              = efx_default_channel_want_txqs,
  71         .keep_eventq            = false,
  72         .want_pio               = true,
  73 };
  74
  75 /*************
  76  * INTERRUPTS
  77  *************/
  78
  79 static unsigned int efx_wanted_parallelism(struct efx_nic *efx)
  80 {
  81         cpumask_var_t thread_mask;
  82         unsigned int count;
  83         int cpu;
  84
  85         if (rss_cpus) {
  86                 count = rss_cpus;
  87         } else {
  88                 if (unlikely(!zalloc_cpumask_var(&thread_mask, GFP_KERNEL))) {
  89                         netif_warn(efx, probe, efx->net_dev,
  90                                    "RSS disabled due to allocation failure\n");
  91                         return 1;
  92                 }
  93
  94                 count = 0;
  95                 for_each_online_cpu(cpu) {
  96                         if (!cpumask_test_cpu(cpu, thread_mask)) {
  97                                 ++count;
  98                                 cpumask_or(thread_mask, thread_mask,
  99                                            topology_sibling_cpumask(cpu));
 100                         }
 101                 }
 102
 103                 free_cpumask_var(thread_mask);
 104         }
 105
 106         if (count > EFX_MAX_RX_QUEUES) {
 107                 netif_cond_dbg(efx, probe, efx->net_dev, !rss_cpus, warn,
 108                                "Reducing number of rx queues from %u to %u.\n",
 109                                count, EFX_MAX_RX_QUEUES);
 110                 count = EFX_MAX_RX_QUEUES;
 111         }
 112
 113         /* If RSS is requested for the PF *and* VFs then we can't write RSS
 114          * table entries that are inaccessible to VFs
 115          */
 116 #ifdef CONFIG_SFC_SRIOV
 117         if (efx->type->sriov_wanted) {
 118                 if (efx->type->sriov_wanted(efx) && efx_vf_size(efx) > 1 &&
 119                     count > efx_vf_size(efx)) {
 120                         netif_warn(efx, probe, efx->net_dev,
 121                                    "Reducing number of RSS channels from %u to %u for "
 122                                    "VF support. Increase vf-msix-limit to use more "
 123                                    "channels on the PF.\n",
 124                                    count, efx_vf_size(efx));
 125                         count = efx_vf_size(efx);
 126                 }
 127         }
 128 #endif
 129
 130         return count;
 131 }
 132
 133 static int efx_allocate_msix_channels(struct efx_nic *efx,
 134                                       unsigned int max_channels,
 135                                       unsigned int extra_channels,
 136                                       unsigned int parallelism)
 137 {
 138         unsigned int n_channels = parallelism;
 139         int vec_count;
 140         int n_xdp_tx;
 141         int n_xdp_ev;
 142
 143         if (efx_separate_tx_channels)
 144                 n_channels *= 2;
 145         n_channels += extra_channels;
 146
 147         /* To allow XDP transmit to happen from arbitrary NAPI contexts
 148          * we allocate a TX queue per CPU. We share event queues across
 149          * multiple tx queues, assuming tx and ev queues are both
 150          * maximum size.
 151          */
 152
 153         n_xdp_tx = num_possible_cpus();
 154         n_xdp_ev = DIV_ROUND_UP(n_xdp_tx, EFX_TXQ_TYPES);
 155
 156         vec_count = pci_msix_vec_count(efx->pci_dev);
 157         if (vec_count < 0)
 158                 return vec_count;
 159
 160         max_channels = min_t(unsigned int, vec_count, max_channels);
 161
 162         /* Check resources.
 163          * We need a channel per event queue, plus a VI per tx queue.
 164          * This may be more pessimistic than it needs to be.
 165          */
 166         if (n_channels + n_xdp_ev > max_channels) {
 167                 netif_err(efx, drv, efx->net_dev,
 168                           "Insufficient resources for %d XDP event queues (%d other channels, max %d)\n",
 169                           n_xdp_ev, n_channels, max_channels);
 170                 efx->n_xdp_channels = 0;
 171                 efx->xdp_tx_per_channel = 0;
 172                 efx->xdp_tx_queue_count = 0;
 173         } else if (n_channels + n_xdp_tx > efx->max_vis) {
 174                 netif_err(efx, drv, efx->net_dev,
 175                           "Insufficient resources for %d XDP TX queues (%d other channels, max VIs %d)\n",
 176                           n_xdp_tx, n_channels, efx->max_vis);
 177                 efx->n_xdp_channels = 0;
 178                 efx->xdp_tx_per_channel = 0;
 179                 efx->xdp_tx_queue_count = 0;
 180         } else {
 181                 efx->n_xdp_channels = n_xdp_ev;
 182                 efx->xdp_tx_per_channel = EFX_TXQ_TYPES;
 183                 efx->xdp_tx_queue_count = n_xdp_tx;
 184                 n_channels += n_xdp_ev;
 185                 netif_dbg(efx, drv, efx->net_dev,
 186                           "Allocating %d TX and %d event queues for XDP\n",
 187                           n_xdp_tx, n_xdp_ev);
 188         }
 189
 190         if (vec_count < n_channels) {
 191                 netif_err(efx, drv, efx->net_dev,
 192                           "WARNING: Insufficient MSI-X vectors available (%d < %u).\n",
 193                           vec_count, n_channels);
 194                 netif_err(efx, drv, efx->net_dev,
 195                           "WARNING: Performance may be reduced.\n");
 196                 n_channels = vec_count;
 197         }
 198
 199         n_channels = min(n_channels, max_channels);
 200
 201         efx->n_channels = n_channels;
 202
 203         /* Ignore XDP tx channels when creating rx channels. */
 204         n_channels -= efx->n_xdp_channels;
 205
 206         if (efx_separate_tx_channels) {
 207                 efx->n_tx_channels =
 208                         min(max(n_channels / 2, 1U),
 209                             efx->max_tx_channels);
 210                 efx->tx_channel_offset =
 211                         n_channels - efx->n_tx_channels;
 212                 efx->n_rx_channels =
 213                         max(n_channels -
 214                             efx->n_tx_channels, 1U);
 215         } else {
 216                 efx->n_tx_channels = min(n_channels, efx->max_tx_channels);
 217                 efx->tx_channel_offset = 0;
 218                 efx->n_rx_channels = n_channels;
 219         }
 220
 221         efx->n_rx_channels = min(efx->n_rx_channels, parallelism);
 222         efx->n_tx_channels = min(efx->n_tx_channels, parallelism);
 223
 224         efx->xdp_channel_offset = n_channels;
 225
 226         netif_dbg(efx, drv, efx->net_dev,
 227                   "Allocating %u RX channels\n",
 228                   efx->n_rx_channels);
 229
 230         return efx->n_channels;
 231 }
 232
 233 /* Probe the number and type of interrupts we are able to obtain, and
 234  * the resulting numbers of channels and RX queues.
 235  */
 236 int efx_probe_interrupts(struct efx_nic *efx)
 237 {
 238         unsigned int extra_channels = 0;
 239         unsigned int rss_spread;
 240         unsigned int i, j;
 241         int rc;
 242
 243         for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++)
 244                 if (efx->extra_channel_type[i])
 245                         ++extra_channels;
 246
 247         if (efx->interrupt_mode == EFX_INT_MODE_MSIX) {
 248                 unsigned int parallelism = efx_wanted_parallelism(efx);
 249                 struct msix_entry xentries[EFX_MAX_CHANNELS];
 250                 unsigned int n_channels;
 251
 252                 rc = efx_allocate_msix_channels(efx, efx->max_channels,
 253                                                 extra_channels, parallelism);
 254                 if (rc >= 0) {
 255                         n_channels = rc;
 256                         for (i = 0; i < n_channels; i++)
 257                                 xentries[i].entry = i;
 258                         rc = pci_enable_msix_range(efx->pci_dev, xentries, 1,
 259                                                    n_channels);
 260                 }
 261                 if (rc < 0) {
 262                         /* Fall back to single channel MSI */
 263                         netif_err(efx, drv, efx->net_dev,
 264                                   "could not enable MSI-X\n");
 265                         if (efx->type->min_interrupt_mode >= EFX_INT_MODE_MSI)
 266                                 efx->interrupt_mode = EFX_INT_MODE_MSI;
 267                         else
 268                                 return rc;
 269                 } else if (rc < n_channels) {
 270                         netif_err(efx, drv, efx->net_dev,
 271                                   "WARNING: Insufficient MSI-X vectors"
 272                                   " available (%d < %u).\n", rc, n_channels);
 273                         netif_err(efx, drv, efx->net_dev,
 274                                   "WARNING: Performance may be reduced.\n");
 275                         n_channels = rc;
 276                 }
 277
 278                 if (rc > 0) {
 279                         for (i = 0; i < efx->n_channels; i++)
 280                                 efx_get_channel(efx, i)->irq =
 281                                         xentries[i].vector;
 282                 }
 283         }
 284
 285         /* Try single interrupt MSI */
 286         if (efx->interrupt_mode == EFX_INT_MODE_MSI) {
 287                 efx->n_channels = 1;
 288                 efx->n_rx_channels = 1;
 289                 efx->n_tx_channels = 1;
 290                 efx->n_xdp_channels = 0;
 291                 efx->xdp_channel_offset = efx->n_channels;
 292                 rc = pci_enable_msi(efx->pci_dev);
 293                 if (rc == 0) {
 294                         efx_get_channel(efx, 0)->irq = efx->pci_dev->irq;
 295                 } else {
 296                         netif_err(efx, drv, efx->net_dev,
 297                                   "could not enable MSI\n");
 298                         if (efx->type->min_interrupt_mode >= EFX_INT_MODE_LEGACY)
 299                                 efx->interrupt_mode = EFX_INT_MODE_LEGACY;
 300                         else
 301                                 return rc;
 302                 }
 303         }
 304
 305         /* Assume legacy interrupts */
 306         if (efx->interrupt_mode == EFX_INT_MODE_LEGACY) {
 307                 efx->n_channels = 1 + (efx_separate_tx_channels ? 1 : 0);
 308                 efx->n_rx_channels = 1;
 309                 efx->n_tx_channels = 1;
 310                 efx->n_xdp_channels = 0;
 311                 efx->xdp_channel_offset = efx->n_channels;
 312                 efx->legacy_irq = efx->pci_dev->irq;
 313         }
 314
 315         /* Assign extra channels if possible, before XDP channels */
 316         efx->n_extra_tx_channels = 0;
 317         j = efx->xdp_channel_offset;
 318         for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++) {
 319                 if (!efx->extra_channel_type[i])
 320                         continue;
 321                 if (j <= efx->tx_channel_offset + efx->n_tx_channels) {
 322                         efx->extra_channel_type[i]->handle_no_channel(efx);
 323                 } else {
 324                         --j;
 325                         efx_get_channel(efx, j)->type =
 326                                 efx->extra_channel_type[i];
 327                         if (efx_channel_has_tx_queues(efx_get_channel(efx, j)))
 328                                 efx->n_extra_tx_channels++;
 329                 }
 330         }
 331
 332         rss_spread = efx->n_rx_channels;
 333         /* RSS might be usable on VFs even if it is disabled on the PF */
 334 #ifdef CONFIG_SFC_SRIOV
 335         if (efx->type->sriov_wanted) {
 336                 efx->rss_spread = ((rss_spread > 1 ||
 337                                     !efx->type->sriov_wanted(efx)) ?
 338                                    rss_spread : efx_vf_size(efx));
 339                 return 0;
 340         }
 341 #endif
 342         efx->rss_spread = rss_spread;
 343
 344         return 0;
 345 }
 346
 347 #if defined(CONFIG_SMP)
 348 void efx_set_interrupt_affinity(struct efx_nic *efx)
 349 {
 350         struct efx_channel *channel;
 351         unsigned int cpu;
 352
 353         efx_for_each_channel(channel, efx) {
 354                 cpu = cpumask_local_spread(channel->channel,
 355                                            pcibus_to_node(efx->pci_dev->bus));
 356                 irq_set_affinity_hint(channel->irq, cpumask_of(cpu));
 357         }
 358 }
 359
 360 void efx_clear_interrupt_affinity(struct efx_nic *efx)
 361 {
 362         struct efx_channel *channel;
 363
 364         efx_for_each_channel(channel, efx)
 365                 irq_set_affinity_hint(channel->irq, NULL);
 366 }
 367 #else
 368 void
 369 efx_set_interrupt_affinity(struct efx_nic *efx __attribute__ ((unused)))
 370 {
 371 }
 372
 373 void
 374 efx_clear_interrupt_affinity(struct efx_nic *efx __attribute__ ((unused)))
 375 {
 376 }
 377 #endif /* CONFIG_SMP */
 378
 379 void efx_remove_interrupts(struct efx_nic *efx)
 380 {
 381         struct efx_channel *channel;
 382
 383         /* Remove MSI/MSI-X interrupts */
 384         efx_for_each_channel(channel, efx)
 385                 channel->irq = 0;
 386         pci_disable_msi(efx->pci_dev);
 387         pci_disable_msix(efx->pci_dev);
 388
 389         /* Remove legacy interrupt */
 390         efx->legacy_irq = 0;
 391 }
 392
 393 /***************
 394  * EVENT QUEUES
 395  ***************/
 396
 397 /* Create event queue
 398  * Event queue memory allocations are done only once.  If the channel
 399  * is reset, the memory buffer will be reused; this guards against
 400  * errors during channel reset and also simplifies interrupt handling.
 401  */
 402 int efx_probe_eventq(struct efx_channel *channel)
 403 {
 404         struct efx_nic *efx = channel->efx;
 405         unsigned long entries;
 406
 407         netif_dbg(efx, probe, efx->net_dev,
 408                   "chan %d create event queue\n", channel->channel);
 409
 410         /* Build an event queue with room for one event per tx and rx buffer,
 411          * plus some extra for link state events and MCDI completions.
 412          */
 413         entries = roundup_pow_of_two(efx->rxq_entries + efx->txq_entries + 128);
 414         EFX_WARN_ON_PARANOID(entries > EFX_MAX_EVQ_SIZE);
 415         channel->eventq_mask = max(entries, EFX_MIN_EVQ_SIZE) - 1;
 416
 417         return efx_nic_probe_eventq(channel);
 418 }
 419
 420 /* Prepare channel's event queue */
 421 int efx_init_eventq(struct efx_channel *channel)
 422 {
 423         struct efx_nic *efx = channel->efx;
 424         int rc;
 425
 426         EFX_WARN_ON_PARANOID(channel->eventq_init);
 427
 428         netif_dbg(efx, drv, efx->net_dev,
 429                   "chan %d init event queue\n", channel->channel);
 430
 431         rc = efx_nic_init_eventq(channel);
 432         if (rc == 0) {
 433                 efx->type->push_irq_moderation(channel);
 434                 channel->eventq_read_ptr = 0;
 435                 channel->eventq_init = true;
 436         }
 437         return rc;
 438 }
 439
 440 /* Enable event queue processing and NAPI */
 441 void efx_start_eventq(struct efx_channel *channel)
 442 {
 443         netif_dbg(channel->efx, ifup, channel->efx->net_dev,
 444                   "chan %d start event queue\n", channel->channel);
 445
 446         /* Make sure the NAPI handler sees the enabled flag set */
 447         channel->enabled = true;
 448         smp_wmb();
 449
 450         napi_enable(&channel->napi_str);
 451         efx_nic_eventq_read_ack(channel);
 452 }
 453
 454 /* Disable event queue processing and NAPI */
 455 void efx_stop_eventq(struct efx_channel *channel)
 456 {
 457         if (!channel->enabled)
 458                 return;
 459
 460         napi_disable(&channel->napi_str);
 461         channel->enabled = false;
 462 }
 463
 464 void efx_fini_eventq(struct efx_channel *channel)
 465 {
 466         if (!channel->eventq_init)
 467                 return;
 468
 469         netif_dbg(channel->efx, drv, channel->efx->net_dev,
 470                   "chan %d fini event queue\n", channel->channel);
 471
 472         efx_nic_fini_eventq(channel);
 473         channel->eventq_init = false;
 474 }
 475
 476 void efx_remove_eventq(struct efx_channel *channel)
 477 {
 478         netif_dbg(channel->efx, drv, channel->efx->net_dev,
 479                   "chan %d remove event queue\n", channel->channel);
 480
 481         efx_nic_remove_eventq(channel);
 482 }
 483
 484 /**************************************************************************
 485  *
 486  * Channel handling
 487  *
 488  *************************************************************************/
 489
 490 #ifdef CONFIG_RFS_ACCEL
 491 static void efx_filter_rfs_expire(struct work_struct *data)
 492 {
 493         struct delayed_work *dwork = to_delayed_work(data);
 494         struct efx_channel *channel;
 495         unsigned int time, quota;
 496
 497         channel = container_of(dwork, struct efx_channel, filter_work);
 498         time = jiffies - channel->rfs_last_expiry;
 499         quota = channel->rfs_filter_count * time / (30 * HZ);
 500         if (quota >= 20 && __efx_filter_rfs_expire(channel, min(channel->rfs_filter_count, quota)))
 501                 channel->rfs_last_expiry += time;
 502         /* Ensure we do more work eventually even if NAPI poll is not happening */
 503         schedule_delayed_work(dwork, 30 * HZ);
 504 }
 505 #endif
 506
 507 /* Allocate and initialise a channel structure. */
 508 struct efx_channel *
 509 efx_alloc_channel(struct efx_nic *efx, int i, struct efx_channel *old_channel)
 510 {
 511         struct efx_rx_queue *rx_queue;
 512         struct efx_tx_queue *tx_queue;
 513         struct efx_channel *channel;
 514         int j;
 515
 516         channel = kzalloc(sizeof(*channel), GFP_KERNEL);
 517         if (!channel)
 518                 return NULL;
 519
 520         channel->efx = efx;
 521         channel->channel = i;
 522         channel->type = &efx_default_channel_type;
 523
 524         for (j = 0; j < EFX_TXQ_TYPES; j++) {
 525                 tx_queue = &channel->tx_queue[j];
 526                 tx_queue->efx = efx;
 527                 tx_queue->queue = -1;
 528                 tx_queue->label = j;
 529                 tx_queue->channel = channel;
 530         }
 531
 532 #ifdef CONFIG_RFS_ACCEL
 533         INIT_DELAYED_WORK(&channel->filter_work, efx_filter_rfs_expire);
 534 #endif
 535
 536         rx_queue = &channel->rx_queue;
 537         rx_queue->efx = efx;
 538         timer_setup(&rx_queue->slow_fill, efx_rx_slow_fill, 0);
 539
 540         return channel;
 541 }
 542
 543 int efx_init_channels(struct efx_nic *efx)
 544 {
 545         unsigned int i;
 546
 547         for (i = 0; i < EFX_MAX_CHANNELS; i++) {
 548                 efx->channel[i] = efx_alloc_channel(efx, i, NULL);
 549                 if (!efx->channel[i])
 550                         return -ENOMEM;
 551                 efx->msi_context[i].efx = efx;
 552                 efx->msi_context[i].index = i;
 553         }
 554
 555         /* Higher numbered interrupt modes are less capable! */
 556         efx->interrupt_mode = min(efx->type->min_interrupt_mode,
 557                                   efx_interrupt_mode);
 558
 559         efx->max_channels = EFX_MAX_CHANNELS;
 560         efx->max_tx_channels = EFX_MAX_CHANNELS;
 561
 562         return 0;
 563 }
 564
 565 void efx_fini_channels(struct efx_nic *efx)
 566 {
 567         unsigned int i;
 568
 569         for (i = 0; i < EFX_MAX_CHANNELS; i++)
 570                 if (efx->channel[i]) {
 571                         kfree(efx->channel[i]);
 572                         efx->channel[i] = NULL;
 573                 }
 574 }
 575
 576 /* Allocate and initialise a channel structure, copying parameters
 577  * (but not resources) from an old channel structure.
 578  */
 579 struct efx_channel *efx_copy_channel(const struct efx_channel *old_channel)
 580 {
 581         struct efx_rx_queue *rx_queue;
 582         struct efx_tx_queue *tx_queue;
 583         struct efx_channel *channel;
 584         int j;
 585
 586         channel = kmalloc(sizeof(*channel), GFP_KERNEL);
 587         if (!channel)
 588                 return NULL;
 589
 590         *channel = *old_channel;
 591
 592         channel->napi_dev = NULL;
 593         INIT_HLIST_NODE(&channel->napi_str.napi_hash_node);
 594         channel->napi_str.napi_id = 0;
 595         channel->napi_str.state = 0;
 596         memset(&channel->eventq, 0, sizeof(channel->eventq));
 597
 598         for (j = 0; j < EFX_TXQ_TYPES; j++) {
 599                 tx_queue = &channel->tx_queue[j];
 600                 if (tx_queue->channel)
 601                         tx_queue->channel = channel;
 602                 tx_queue->buffer = NULL;
 603                 tx_queue->cb_page = NULL;
 604                 memset(&tx_queue->txd, 0, sizeof(tx_queue->txd));
 605         }
 606
 607         rx_queue = &channel->rx_queue;
 608         rx_queue->buffer = NULL;
 609         memset(&rx_queue->rxd, 0, sizeof(rx_queue->rxd));
 610         timer_setup(&rx_queue->slow_fill, efx_rx_slow_fill, 0);
 611 #ifdef CONFIG_RFS_ACCEL
 612         INIT_DELAYED_WORK(&channel->filter_work, efx_filter_rfs_expire);
 613 #endif
 614
 615         return channel;
 616 }
 617
 618 static int efx_probe_channel(struct efx_channel *channel)
 619 {
 620         struct efx_tx_queue *tx_queue;
 621         struct efx_rx_queue *rx_queue;
 622         int rc;
 623
 624         netif_dbg(channel->efx, probe, channel->efx->net_dev,
 625                   "creating channel %d\n", channel->channel);
 626
 627         rc = channel->type->pre_probe(channel);
 628         if (rc)
 629                 goto fail;
 630
 631         rc = efx_probe_eventq(channel);
 632         if (rc)
 633                 goto fail;
 634
 635         efx_for_each_channel_tx_queue(tx_queue, channel) {
 636                 rc = efx_probe_tx_queue(tx_queue);
 637                 if (rc)
 638                         goto fail;
 639         }
 640
 641         efx_for_each_channel_rx_queue(rx_queue, channel) {
 642                 rc = efx_probe_rx_queue(rx_queue);
 643                 if (rc)
 644                         goto fail;
 645         }
 646
 647         channel->rx_list = NULL;
 648
 649         return 0;
 650
 651 fail:
 652         efx_remove_channel(channel);
 653         return rc;
 654 }
 655
 656 void efx_get_channel_name(struct efx_channel *channel, char *buf, size_t len)
 657 {
 658         struct efx_nic *efx = channel->efx;
 659         const char *type;
 660         int number;
 661
 662         number = channel->channel;
 663
 664         if (number >= efx->xdp_channel_offset &&
 665             !WARN_ON_ONCE(!efx->n_xdp_channels)) {
 666                 type = "-xdp";
 667                 number -= efx->xdp_channel_offset;
 668         } else if (efx->tx_channel_offset == 0) {
 669                 type = "";
 670         } else if (number < efx->tx_channel_offset) {
 671                 type = "-rx";
 672         } else {
 673                 type = "-tx";
 674                 number -= efx->tx_channel_offset;
 675         }
 676         snprintf(buf, len, "%s%s-%d", efx->name, type, number);
 677 }
 678
 679 void efx_set_channel_names(struct efx_nic *efx)
 680 {
 681         struct efx_channel *channel;
 682
 683         efx_for_each_channel(channel, efx)
 684                 channel->type->get_name(channel,
 685                                         efx->msi_context[channel->channel].name,
 686                                         sizeof(efx->msi_context[0].name));
 687 }
 688
 689 int efx_probe_channels(struct efx_nic *efx)
 690 {
 691         struct efx_channel *channel;
 692         int rc;
 693
 694         /* Restart special buffer allocation */
 695         efx->next_buffer_table = 0;
 696
 697         /* Probe channels in reverse, so that any 'extra' channels
 698          * use the start of the buffer table. This allows the traffic
 699          * channels to be resized without moving them or wasting the
 700          * entries before them.
 701          */
 702         efx_for_each_channel_rev(channel, efx) {
 703                 rc = efx_probe_channel(channel);
 704                 if (rc) {
 705                         netif_err(efx, probe, efx->net_dev,
 706                                   "failed to create channel %d\n",
 707                                   channel->channel);
 708                         goto fail;
 709                 }
 710         }
 711         efx_set_channel_names(efx);
 712
 713         return 0;
 714
 715 fail:
 716         efx_remove_channels(efx);
 717         return rc;
 718 }
 719
 720 void efx_remove_channel(struct efx_channel *channel)
 721 {
 722         struct efx_tx_queue *tx_queue;
 723         struct efx_rx_queue *rx_queue;
 724
 725         netif_dbg(channel->efx, drv, channel->efx->net_dev,
 726                   "destroy chan %d\n", channel->channel);
 727
 728         efx_for_each_channel_rx_queue(rx_queue, channel)
 729                 efx_remove_rx_queue(rx_queue);
 730         efx_for_each_channel_tx_queue(tx_queue, channel)
 731                 efx_remove_tx_queue(tx_queue);
 732         efx_remove_eventq(channel);
 733         channel->type->post_remove(channel);
 734 }
 735
 736 void efx_remove_channels(struct efx_nic *efx)
 737 {
 738         struct efx_channel *channel;
 739
 740         efx_for_each_channel(channel, efx)
 741                 efx_remove_channel(channel);
 742
 743         kfree(efx->xdp_tx_queues);
 744 }
 745
 746 int efx_realloc_channels(struct efx_nic *efx, u32 rxq_entries, u32 txq_entries)
 747 {
 748         struct efx_channel *other_channel[EFX_MAX_CHANNELS], *channel;
 749         unsigned int i, next_buffer_table = 0;
 750         u32 old_rxq_entries, old_txq_entries;
 751         int rc, rc2;
 752
 753         rc = efx_check_disabled(efx);
 754         if (rc)
 755                 return rc;
 756
 757         /* Not all channels should be reallocated. We must avoid
 758          * reallocating their buffer table entries.
 759          */
 760         efx_for_each_channel(channel, efx) {
 761                 struct efx_rx_queue *rx_queue;
 762                 struct efx_tx_queue *tx_queue;
 763
 764                 if (channel->type->copy)
 765                         continue;
 766                 next_buffer_table = max(next_buffer_table,
 767                                         channel->eventq.index +
 768                                         channel->eventq.entries);
 769                 efx_for_each_channel_rx_queue(rx_queue, channel)
 770                         next_buffer_table = max(next_buffer_table,
 771                                                 rx_queue->rxd.index +
 772                                                 rx_queue->rxd.entries);
 773                 efx_for_each_channel_tx_queue(tx_queue, channel)
 774                         next_buffer_table = max(next_buffer_table,
 775                                                 tx_queue->txd.index +
 776                                                 tx_queue->txd.entries);
 777         }
 778
 779         efx_device_detach_sync(efx);
 780         efx_stop_all(efx);
 781         efx_soft_disable_interrupts(efx);
 782
 783         /* Clone channels (where possible) */
 784         memset(other_channel, 0, sizeof(other_channel));
 785         for (i = 0; i < efx->n_channels; i++) {
 786                 channel = efx->channel[i];
 787                 if (channel->type->copy)
 788                         channel = channel->type->copy(channel);
 789                 if (!channel) {
 790                         rc = -ENOMEM;
 791                         goto out;
 792                 }
 793                 other_channel[i] = channel;
 794         }
 795
 796         /* Swap entry counts and channel pointers */
 797         old_rxq_entries = efx->rxq_entries;
 798         old_txq_entries = efx->txq_entries;
 799         efx->rxq_entries = rxq_entries;
 800         efx->txq_entries = txq_entries;
 801         for (i = 0; i < efx->n_channels; i++) {
 802                 channel = efx->channel[i];
 803                 efx->channel[i] = other_channel[i];
 804                 other_channel[i] = channel;
 805         }
 806
 807         /* Restart buffer table allocation */
 808         efx->next_buffer_table = next_buffer_table;
 809
 810         for (i = 0; i < efx->n_channels; i++) {
 811                 channel = efx->channel[i];
 812                 if (!channel->type->copy)
 813                         continue;
 814                 rc = efx_probe_channel(channel);
 815                 if (rc)
 816                         goto rollback;
 817                 efx_init_napi_channel(efx->channel[i]);
 818         }
 819
 820 out:
 821         /* Destroy unused channel structures */
 822         for (i = 0; i < efx->n_channels; i++) {
 823                 channel = other_channel[i];
 824                 if (channel && channel->type->copy) {
 825                         efx_fini_napi_channel(channel);
 826                         efx_remove_channel(channel);
 827                         kfree(channel);
 828                 }
 829         }
 830
 831         rc2 = efx_soft_enable_interrupts(efx);
 832         if (rc2) {
 833                 rc = rc ? rc : rc2;
 834                 netif_err(efx, drv, efx->net_dev,
 835                           "unable to restart interrupts on channel reallocation\n");
 836                 efx_schedule_reset(efx, RESET_TYPE_DISABLE);
 837         } else {
 838                 efx_start_all(efx);
 839                 efx_device_attach_if_not_resetting(efx);
 840         }
 841         return rc;
 842
 843 rollback:
 844         /* Swap back */
 845         efx->rxq_entries = old_rxq_entries;
 846         efx->txq_entries = old_txq_entries;
 847         for (i = 0; i < efx->n_channels; i++) {
 848                 channel = efx->channel[i];
 849                 efx->channel[i] = other_channel[i];
 850                 other_channel[i] = channel;
 851         }
 852         goto out;
 853 }
 854
 855 int efx_set_channels(struct efx_nic *efx)
 856 {
 857         struct efx_tx_queue *tx_queue;
 858         struct efx_channel *channel;
 859         unsigned int next_queue = 0;
 860         int xdp_queue_number;
 861         int rc;
 862
 863         efx->tx_channel_offset =
 864                 efx_separate_tx_channels ?
 865                 efx->n_channels - efx->n_tx_channels : 0;
 866
 867         if (efx->xdp_tx_queue_count) {
 868                 EFX_WARN_ON_PARANOID(efx->xdp_tx_queues);
 869
 870                 /* Allocate array for XDP TX queue lookup. */
 871                 efx->xdp_tx_queues = kcalloc(efx->xdp_tx_queue_count,
 872                                              sizeof(*efx->xdp_tx_queues),
 873                                              GFP_KERNEL);
 874                 if (!efx->xdp_tx_queues)
 875                         return -ENOMEM;
 876         }
 877
 878         /* We need to mark which channels really have RX and TX
 879          * queues, and adjust the TX queue numbers if we have separate
 880          * RX-only and TX-only channels.
 881          */
 882         xdp_queue_number = 0;
 883         efx_for_each_channel(channel, efx) {
 884                 if (channel->channel < efx->n_rx_channels)
 885                         channel->rx_queue.core_index = channel->channel;
 886                 else
 887                         channel->rx_queue.core_index = -1;
 888
 889                 if (channel->channel >= efx->tx_channel_offset) {
 890                         if (efx_channel_is_xdp_tx(channel)) {
 891                                 efx_for_each_channel_tx_queue(tx_queue, channel) {
 892                                         tx_queue->queue = next_queue++;
 893                                         netif_dbg(efx, drv, efx->net_dev, "Channel %u TXQ %u is XDP %u, HW %u\n",
 894                                                   channel->channel, tx_queue->label,
 895                                                   xdp_queue_number, tx_queue->queue);
 896                                         /* We may have a few left-over XDP TX
 897                                          * queues owing to xdp_tx_queue_count
 898                                          * not dividing evenly by EFX_TXQ_TYPES.
 899                                          * We still allocate and probe those
 900                                          * TXQs, but never use them.
 901                                          */
 902                                         if (xdp_queue_number < efx->xdp_tx_queue_count)
 903                                                 efx->xdp_tx_queues[xdp_queue_number] = tx_queue;
 904                                         xdp_queue_number++;
 905                                 }
 906                         } else {
 907                                 efx_for_each_channel_tx_queue(tx_queue, channel) {
 908                                         tx_queue->queue = next_queue++;
 909                                         netif_dbg(efx, drv, efx->net_dev, "Channel %u TXQ %u is HW %u\n",
 910                                                   channel->channel, tx_queue->label,
 911                                                   tx_queue->queue);
 912                                 }
 913                         }
 914                 }
 915         }
 916
 917         rc = netif_set_real_num_tx_queues(efx->net_dev, efx->n_tx_channels);
 918         if (rc)
 919                 return rc;
 920         return netif_set_real_num_rx_queues(efx->net_dev, efx->n_rx_channels);
 921 }
 922
 923 bool efx_default_channel_want_txqs(struct efx_channel *channel)
 924 {
 925         return channel->channel - channel->efx->tx_channel_offset <
 926                 channel->efx->n_tx_channels;
 927 }
 928
 929 /*************
 930  * START/STOP
 931  *************/
 932
 933 int efx_soft_enable_interrupts(struct efx_nic *efx)
 934 {
 935         struct efx_channel *channel, *end_channel;
 936         int rc;
 937
 938         BUG_ON(efx->state == STATE_DISABLED);
 939
 940         efx->irq_soft_enabled = true;
 941         smp_wmb();
 942
 943         efx_for_each_channel(channel, efx) {
 944                 if (!channel->type->keep_eventq) {
 945                         rc = efx_init_eventq(channel);
 946                         if (rc)
 947                                 goto fail;
 948                 }
 949                 efx_start_eventq(channel);
 950         }
 951
 952         efx_mcdi_mode_event(efx);
 953
 954         return 0;
 955 fail:
 956         end_channel = channel;
 957         efx_for_each_channel(channel, efx) {
 958                 if (channel == end_channel)
 959                         break;
 960                 efx_stop_eventq(channel);
 961                 if (!channel->type->keep_eventq)
 962                         efx_fini_eventq(channel);
 963         }
 964
 965         return rc;
 966 }
 967
 968 void efx_soft_disable_interrupts(struct efx_nic *efx)
 969 {
 970         struct efx_channel *channel;
 971
 972         if (efx->state == STATE_DISABLED)
 973                 return;
 974
 975         efx_mcdi_mode_poll(efx);
 976
 977         efx->irq_soft_enabled = false;
 978         smp_wmb();
 979
 980         if (efx->legacy_irq)
 981                 synchronize_irq(efx->legacy_irq);
 982
 983         efx_for_each_channel(channel, efx) {
 984                 if (channel->irq)
 985                         synchronize_irq(channel->irq);
 986
 987                 efx_stop_eventq(channel);
 988                 if (!channel->type->keep_eventq)
 989                         efx_fini_eventq(channel);
 990         }
 991
 992         /* Flush the asynchronous MCDI request queue */
 993         efx_mcdi_flush_async(efx);
 994 }
 995
 996 int efx_enable_interrupts(struct efx_nic *efx)
 997 {
 998         struct efx_channel *channel, *end_channel;
 999         int rc;
1000
1001         /* TODO: Is this really a bug? */
1002         BUG_ON(efx->state == STATE_DISABLED);
1003
1004         if (efx->eeh_disabled_legacy_irq) {
1005                 enable_irq(efx->legacy_irq);
1006                 efx->eeh_disabled_legacy_irq = false;
1007         }
1008
1009         efx->type->irq_enable_master(efx);
1010
1011         efx_for_each_channel(channel, efx) {
1012                 if (channel->type->keep_eventq) {
1013                         rc = efx_init_eventq(channel);
1014                         if (rc)
1015                                 goto fail;
1016                 }
1017         }
1018
1019         rc = efx_soft_enable_interrupts(efx);
1020         if (rc)
1021                 goto fail;
1022
1023         return 0;
1024
1025 fail:
1026         end_channel = channel;
1027         efx_for_each_channel(channel, efx) {
1028                 if (channel == end_channel)
1029                         break;
1030                 if (channel->type->keep_eventq)
1031                         efx_fini_eventq(channel);
1032         }
1033
1034         efx->type->irq_disable_non_ev(efx);
1035
1036         return rc;
1037 }
1038
1039 void efx_disable_interrupts(struct efx_nic *efx)
1040 {
1041         struct efx_channel *channel;
1042
1043         efx_soft_disable_interrupts(efx);
1044
1045         efx_for_each_channel(channel, efx) {
1046                 if (channel->type->keep_eventq)
1047                         efx_fini_eventq(channel);
1048         }
1049
1050         efx->type->irq_disable_non_ev(efx);
1051 }
1052
1053 void efx_start_channels(struct efx_nic *efx)
1054 {
1055         struct efx_tx_queue *tx_queue;
1056         struct efx_rx_queue *rx_queue;
1057         struct efx_channel *channel;
1058
1059         efx_for_each_channel(channel, efx) {
1060                 efx_for_each_channel_tx_queue(tx_queue, channel) {
1061                         efx_init_tx_queue(tx_queue);
1062                         atomic_inc(&efx->active_queues);
1063                 }
1064
1065                 efx_for_each_channel_rx_queue(rx_queue, channel) {
1066                         efx_init_rx_queue(rx_queue);
1067                         atomic_inc(&efx->active_queues);
1068                         efx_stop_eventq(channel);
1069                         efx_fast_push_rx_descriptors(rx_queue, false);
1070                         efx_start_eventq(channel);
1071                 }
1072
1073                 WARN_ON(channel->rx_pkt_n_frags);
1074         }
1075 }
1076
1077 void efx_stop_channels(struct efx_nic *efx)
1078 {
1079         struct efx_tx_queue *tx_queue;
1080         struct efx_rx_queue *rx_queue;
1081         struct efx_channel *channel;
1082         int rc = 0;
1083
1084         /* Stop RX refill */
1085         efx_for_each_channel(channel, efx) {
1086                 efx_for_each_channel_rx_queue(rx_queue, channel)
1087                         rx_queue->refill_enabled = false;
1088         }
1089
1090         efx_for_each_channel(channel, efx) {
1091                 /* RX packet processing is pipelined, so wait for the
1092                  * NAPI handler to complete.  At least event queue 0
1093                  * might be kept active by non-data events, so don't
1094                  * use napi_synchronize() but actually disable NAPI
1095                  * temporarily.
1096                  */
1097                 if (efx_channel_has_rx_queue(channel)) {
1098                         efx_stop_eventq(channel);
1099                         efx_start_eventq(channel);
1100                 }
1101         }
1102
1103         if (efx->type->fini_dmaq)
1104                 rc = efx->type->fini_dmaq(efx);
1105
1106         if (rc) {
1107                 netif_err(efx, drv, efx->net_dev, "failed to flush queues\n");
1108         } else {
1109                 netif_dbg(efx, drv, efx->net_dev,
1110                           "successfully flushed all queues\n");
1111         }
1112
1113         efx_for_each_channel(channel, efx) {
1114                 efx_for_each_channel_rx_queue(rx_queue, channel)
1115                         efx_fini_rx_queue(rx_queue);
1116                 efx_for_each_channel_tx_queue(tx_queue, channel)
1117                         efx_fini_tx_queue(tx_queue);
1118         }
1119 }
1120
1121 /**************************************************************************
1122  *
1123  * NAPI interface
1124  *
1125  *************************************************************************/
1126
1127 /* Process channel's event queue
1128  *
1129  * This function is responsible for processing the event queue of a
1130  * single channel.  The caller must guarantee that this function will
1131  * never be concurrently called more than once on the same channel,
1132  * though different channels may be being processed concurrently.
1133  */
1134 static int efx_process_channel(struct efx_channel *channel, int budget)
1135 {
1136         struct efx_tx_queue *tx_queue;
1137         struct list_head rx_list;
1138         int spent;
1139
1140         if (unlikely(!channel->enabled))
1141                 return 0;
1142
1143         /* Prepare the batch receive list */
1144         EFX_WARN_ON_PARANOID(channel->rx_list != NULL);
1145         INIT_LIST_HEAD(&rx_list);
1146         channel->rx_list = &rx_list;
1147
1148         efx_for_each_channel_tx_queue(tx_queue, channel) {
1149                 tx_queue->pkts_compl = 0;
1150                 tx_queue->bytes_compl = 0;
1151         }
1152
1153         spent = efx_nic_process_eventq(channel, budget);
1154         if (spent && efx_channel_has_rx_queue(channel)) {
1155                 struct efx_rx_queue *rx_queue =
1156                         efx_channel_get_rx_queue(channel);
1157
1158                 efx_rx_flush_packet(channel);
1159                 efx_fast_push_rx_descriptors(rx_queue, true);
1160         }
1161
1162         /* Update BQL */
1163         efx_for_each_channel_tx_queue(tx_queue, channel) {
1164                 if (tx_queue->bytes_compl) {
1165                         netdev_tx_completed_queue(tx_queue->core_txq,
1166                                                   tx_queue->pkts_compl,
1167                                                   tx_queue->bytes_compl);
1168                 }
1169         }
1170
1171         /* Receive any packets we queued up */
1172         netif_receive_skb_list(channel->rx_list);
1173         channel->rx_list = NULL;
1174
1175         return spent;
1176 }
1177
1178 static void efx_update_irq_mod(struct efx_nic *efx, struct efx_channel *channel)
1179 {
1180         int step = efx->irq_mod_step_us;
1181
1182         if (channel->irq_mod_score < irq_adapt_low_thresh) {
1183                 if (channel->irq_moderation_us > step) {
1184                         channel->irq_moderation_us -= step;
1185                         efx->type->push_irq_moderation(channel);
1186                 }
1187         } else if (channel->irq_mod_score > irq_adapt_high_thresh) {
1188                 if (channel->irq_moderation_us <
1189                     efx->irq_rx_moderation_us) {
1190                         channel->irq_moderation_us += step;
1191                         efx->type->push_irq_moderation(channel);
1192                 }
1193         }
1194
1195         channel->irq_count = 0;
1196         channel->irq_mod_score = 0;
1197 }
1198
1199 /* NAPI poll handler
1200  *
1201  * NAPI guarantees serialisation of polls of the same device, which
1202  * provides the guarantee required by efx_process_channel().
1203  */
1204 static int efx_poll(struct napi_struct *napi, int budget)
1205 {
1206         struct efx_channel *channel =
1207                 container_of(napi, struct efx_channel, napi_str);
1208         struct efx_nic *efx = channel->efx;
1209 #ifdef CONFIG_RFS_ACCEL
1210         unsigned int time;
1211 #endif
1212         int spent;
1213
1214         netif_vdbg(efx, intr, efx->net_dev,
1215                    "channel %d NAPI poll executing on CPU %d\n",
1216                    channel->channel, raw_smp_processor_id());
1217
1218         spent = efx_process_channel(channel, budget);
1219
1220         xdp_do_flush_map();
1221
1222         if (spent < budget) {
1223                 if (efx_channel_has_rx_queue(channel) &&
1224                     efx->irq_rx_adaptive &&
1225                     unlikely(++channel->irq_count == 1000)) {
1226                         efx_update_irq_mod(efx, channel);
1227                 }
1228
1229 #ifdef CONFIG_RFS_ACCEL
1230                 /* Perhaps expire some ARFS filters */
1231                 time = jiffies - channel->rfs_last_expiry;
1232                 /* Would our quota be >= 20? */
1233                 if (channel->rfs_filter_count * time >= 600 * HZ)
1234                         mod_delayed_work(system_wq, &channel->filter_work, 0);
1235 #endif
1236
1237                 /* There is no race here; although napi_disable() will
1238                  * only wait for napi_complete(), this isn't a problem
1239                  * since efx_nic_eventq_read_ack() will have no effect if
1240                  * interrupts have already been disabled.
1241                  */
1242                 if (napi_complete_done(napi, spent))
1243                         efx_nic_eventq_read_ack(channel);
1244         }
1245
1246         return spent;
1247 }
1248
1249 void efx_init_napi_channel(struct efx_channel *channel)
1250 {
1251         struct efx_nic *efx = channel->efx;
1252
1253         channel->napi_dev = efx->net_dev;
1254         netif_napi_add(channel->napi_dev, &channel->napi_str,
1255                        efx_poll, napi_weight);
1256 }
1257
1258 void efx_init_napi(struct efx_nic *efx)
1259 {
1260         struct efx_channel *channel;
1261
1262         efx_for_each_channel(channel, efx)
1263                 efx_init_napi_channel(channel);
1264 }
1265
1266 void efx_fini_napi_channel(struct efx_channel *channel)
1267 {
1268         if (channel->napi_dev)
1269                 netif_napi_del(&channel->napi_str);
1270
1271         channel->napi_dev = NULL;
1272 }
1273
1274 void efx_fini_napi(struct efx_nic *efx)
1275 {
1276         struct efx_channel *channel;
1277
1278         efx_for_each_channel(channel, efx)
1279                 efx_fini_napi_channel(channel);
1280 }