drivers/net/virtio_net.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /* A network driver using virtio.
   3  *
   4  * Copyright 2007 Rusty Russell <rusty@rustcorp.com.au> IBM Corporation
   5  */
   6 //#define DEBUG
   7 #include <linux/netdevice.h>
   8 #include <linux/etherdevice.h>
   9 #include <linux/ethtool.h>
  10 #include <linux/module.h>
  11 #include <linux/virtio.h>
  12 #include <linux/virtio_net.h>
  13 #include <linux/bpf.h>
  14 #include <linux/bpf_trace.h>
  15 #include <linux/scatterlist.h>
  16 #include <linux/if_vlan.h>
  17 #include <linux/slab.h>
  18 #include <linux/cpu.h>
  19 #include <linux/average.h>
  20 #include <linux/filter.h>
  21 #include <linux/kernel.h>
  22 #include <net/route.h>
  23 #include <net/xdp.h>
  24 #include <net/net_failover.h>
  25
  26 static int napi_weight = NAPI_POLL_WEIGHT;
  27 module_param(napi_weight, int, 0444);
  28
  29 static bool csum = true, gso = true, napi_tx = true;
  30 module_param(csum, bool, 0444);
  31 module_param(gso, bool, 0444);
  32 module_param(napi_tx, bool, 0644);
  33
  34 /* FIXME: MTU in config. */
  35 #define GOOD_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN)
  36 #define GOOD_COPY_LEN   128
  37
  38 #define VIRTNET_RX_PAD (NET_IP_ALIGN + NET_SKB_PAD)
  39
  40 /* Amount of XDP headroom to prepend to packets for use by xdp_adjust_head */
  41 #define VIRTIO_XDP_HEADROOM 256
  42
  43 /* Separating two types of XDP xmit */
  44 #define VIRTIO_XDP_TX           BIT(0)
  45 #define VIRTIO_XDP_REDIR        BIT(1)
  46
  47 #define VIRTIO_XDP_FLAG BIT(0)
  48
  49 /* RX packet size EWMA. The average packet size is used to determine the packet
  50  * buffer size when refilling RX rings. As the entire RX ring may be refilled
  51  * at once, the weight is chosen so that the EWMA will be insensitive to short-
  52  * term, transient changes in packet size.
  53  */
  54 DECLARE_EWMA(pkt_len, 0, 64)
  55
  56 #define VIRTNET_DRIVER_VERSION "1.0.0"
  57
  58 static const unsigned long guest_offloads[] = {
  59         VIRTIO_NET_F_GUEST_TSO4,
  60         VIRTIO_NET_F_GUEST_TSO6,
  61         VIRTIO_NET_F_GUEST_ECN,
  62         VIRTIO_NET_F_GUEST_UFO,
  63         VIRTIO_NET_F_GUEST_CSUM
  64 };
  65
  66 #define GUEST_OFFLOAD_GRO_HW_MASK ((1ULL << VIRTIO_NET_F_GUEST_TSO4) | \
  67                                 (1ULL << VIRTIO_NET_F_GUEST_TSO6) | \
  68                                 (1ULL << VIRTIO_NET_F_GUEST_ECN)  | \
  69                                 (1ULL << VIRTIO_NET_F_GUEST_UFO))
  70
  71 struct virtnet_stat_desc {
  72         char desc[ETH_GSTRING_LEN];
  73         size_t offset;
  74 };
  75
  76 struct virtnet_sq_stats {
  77         struct u64_stats_sync syncp;
  78         u64 packets;
  79         u64 bytes;
  80         u64 xdp_tx;
  81         u64 xdp_tx_drops;
  82         u64 kicks;
  83         u64 tx_timeouts;
  84 };
  85
  86 struct virtnet_rq_stats {
  87         struct u64_stats_sync syncp;
  88         u64 packets;
  89         u64 bytes;
  90         u64 drops;
  91         u64 xdp_packets;
  92         u64 xdp_tx;
  93         u64 xdp_redirects;
  94         u64 xdp_drops;
  95         u64 kicks;
  96 };
  97
  98 #define VIRTNET_SQ_STAT(m)      offsetof(struct virtnet_sq_stats, m)
  99 #define VIRTNET_RQ_STAT(m)      offsetof(struct virtnet_rq_stats, m)
 100
 101 static const struct virtnet_stat_desc virtnet_sq_stats_desc[] = {
 102         { "packets",            VIRTNET_SQ_STAT(packets) },
 103         { "bytes",              VIRTNET_SQ_STAT(bytes) },
 104         { "xdp_tx",             VIRTNET_SQ_STAT(xdp_tx) },
 105         { "xdp_tx_drops",       VIRTNET_SQ_STAT(xdp_tx_drops) },
 106         { "kicks",              VIRTNET_SQ_STAT(kicks) },
 107         { "tx_timeouts",        VIRTNET_SQ_STAT(tx_timeouts) },
 108 };
 109
 110 static const struct virtnet_stat_desc virtnet_rq_stats_desc[] = {
 111         { "packets",            VIRTNET_RQ_STAT(packets) },
 112         { "bytes",              VIRTNET_RQ_STAT(bytes) },
 113         { "drops",              VIRTNET_RQ_STAT(drops) },
 114         { "xdp_packets",        VIRTNET_RQ_STAT(xdp_packets) },
 115         { "xdp_tx",             VIRTNET_RQ_STAT(xdp_tx) },
 116         { "xdp_redirects",      VIRTNET_RQ_STAT(xdp_redirects) },
 117         { "xdp_drops",          VIRTNET_RQ_STAT(xdp_drops) },
 118         { "kicks",              VIRTNET_RQ_STAT(kicks) },
 119 };
 120
 121 #define VIRTNET_SQ_STATS_LEN    ARRAY_SIZE(virtnet_sq_stats_desc)
 122 #define VIRTNET_RQ_STATS_LEN    ARRAY_SIZE(virtnet_rq_stats_desc)
 123
 124 /* Internal representation of a send virtqueue */
 125 struct send_queue {
 126         /* Virtqueue associated with this send _queue */
 127         struct virtqueue *vq;
 128
 129         /* TX: fragments + linear part + virtio header */
 130         struct scatterlist sg[MAX_SKB_FRAGS + 2];
 131
 132         /* Name of the send queue: output.$index */
 133         char name[40];
 134
 135         struct virtnet_sq_stats stats;
 136
 137         struct napi_struct napi;
 138 };
 139
 140 /* Internal representation of a receive virtqueue */
 141 struct receive_queue {
 142         /* Virtqueue associated with this receive_queue */
 143         struct virtqueue *vq;
 144
 145         struct napi_struct napi;
 146
 147         struct bpf_prog __rcu *xdp_prog;
 148
 149         struct virtnet_rq_stats stats;
 150
 151         /* Chain pages by the private ptr. */
 152         struct page *pages;
 153
 154         /* Average packet length for mergeable receive buffers. */
 155         struct ewma_pkt_len mrg_avg_pkt_len;
 156
 157         /* Page frag for packet buffer allocation. */
 158         struct page_frag alloc_frag;
 159
 160         /* RX: fragments + linear part + virtio header */
 161         struct scatterlist sg[MAX_SKB_FRAGS + 2];
 162
 163         /* Min single buffer size for mergeable buffers case. */
 164         unsigned int min_buf_len;
 165
 166         /* Name of this receive queue: input.$index */
 167         char name[40];
 168
 169         struct xdp_rxq_info xdp_rxq;
 170 };
 171
 172 /* This structure can contain rss message with maximum settings for indirection table and keysize
 173  * Note, that default structure that describes RSS configuration virtio_net_rss_config
 174  * contains same info but can't handle table values.
 175  * In any case, structure would be passed to virtio hw through sg_buf split by parts
 176  * because table sizes may be differ according to the device configuration.
 177  */
 178 #define VIRTIO_NET_RSS_MAX_KEY_SIZE     40
 179 #define VIRTIO_NET_RSS_MAX_TABLE_LEN    128
 180 struct virtio_net_ctrl_rss {
 181         u32 hash_types;
 182         u16 indirection_table_mask;
 183         u16 unclassified_queue;
 184         u16 indirection_table[VIRTIO_NET_RSS_MAX_TABLE_LEN];
 185         u16 max_tx_vq;
 186         u8 hash_key_length;
 187         u8 key[VIRTIO_NET_RSS_MAX_KEY_SIZE];
 188 };
 189
 190 /* Control VQ buffers: protected by the rtnl lock */
 191 struct control_buf {
 192         struct virtio_net_ctrl_hdr hdr;
 193         virtio_net_ctrl_ack status;
 194         struct virtio_net_ctrl_mq mq;
 195         u8 promisc;
 196         u8 allmulti;
 197         __virtio16 vid;
 198         __virtio64 offloads;
 199         struct virtio_net_ctrl_rss rss;
 200 };
 201
 202 struct virtnet_info {
 203         struct virtio_device *vdev;
 204         struct virtqueue *cvq;
 205         struct net_device *dev;
 206         struct send_queue *sq;
 207         struct receive_queue *rq;
 208         unsigned int status;
 209
 210         /* Max # of queue pairs supported by the device */
 211         u16 max_queue_pairs;
 212
 213         /* # of queue pairs currently used by the driver */
 214         u16 curr_queue_pairs;
 215
 216         /* # of XDP queue pairs currently used by the driver */
 217         u16 xdp_queue_pairs;
 218
 219         /* xdp_queue_pairs may be 0, when xdp is already loaded. So add this. */
 220         bool xdp_enabled;
 221
 222         /* I like... big packets and I cannot lie! */
 223         bool big_packets;
 224
 225         /* Host will merge rx buffers for big packets (shake it! shake it!) */
 226         bool mergeable_rx_bufs;
 227
 228         /* Host supports rss and/or hash report */
 229         bool has_rss;
 230         bool has_rss_hash_report;
 231         u8 rss_key_size;
 232         u16 rss_indir_table_size;
 233         u32 rss_hash_types_supported;
 234         u32 rss_hash_types_saved;
 235
 236         /* Has control virtqueue */
 237         bool has_cvq;
 238
 239         /* Host can handle any s/g split between our header and packet data */
 240         bool any_header_sg;
 241
 242         /* Packet virtio header size */
 243         u8 hdr_len;
 244
 245         /* Work struct for refilling if we run low on memory. */
 246         struct delayed_work refill;
 247
 248         /* Work struct for config space updates */
 249         struct work_struct config_work;
 250
 251         /* Does the affinity hint is set for virtqueues? */
 252         bool affinity_hint_set;
 253
 254         /* CPU hotplug instances for online & dead */
 255         struct hlist_node node;
 256         struct hlist_node node_dead;
 257
 258         struct control_buf *ctrl;
 259
 260         /* Ethtool settings */
 261         u8 duplex;
 262         u32 speed;
 263
 264         unsigned long guest_offloads;
 265         unsigned long guest_offloads_capable;
 266
 267         /* failover when STANDBY feature enabled */
 268         struct failover *failover;
 269 };
 270
 271 struct padded_vnet_hdr {
 272         struct virtio_net_hdr_v1_hash hdr;
 273         /*
 274          * hdr is in a separate sg buffer, and data sg buffer shares same page
 275          * with this header sg. This padding makes next sg 16 byte aligned
 276          * after the header.
 277          */
 278         char padding[12];
 279 };
 280
 281 static bool is_xdp_frame(void *ptr)
 282 {
 283         return (unsigned long)ptr & VIRTIO_XDP_FLAG;
 284 }
 285
 286 static void *xdp_to_ptr(struct xdp_frame *ptr)
 287 {
 288         return (void *)((unsigned long)ptr | VIRTIO_XDP_FLAG);
 289 }
 290
 291 static struct xdp_frame *ptr_to_xdp(void *ptr)
 292 {
 293         return (struct xdp_frame *)((unsigned long)ptr & ~VIRTIO_XDP_FLAG);
 294 }
 295
 296 /* Converting between virtqueue no. and kernel tx/rx queue no.
 297  * 0:rx0 1:tx0 2:rx1 3:tx1 ... 2N:rxN 2N+1:txN 2N+2:cvq
 298  */
 299 static int vq2txq(struct virtqueue *vq)
 300 {
 301         return (vq->index - 1) / 2;
 302 }
 303
 304 static int txq2vq(int txq)
 305 {
 306         return txq * 2 + 1;
 307 }
 308
 309 static int vq2rxq(struct virtqueue *vq)
 310 {
 311         return vq->index / 2;
 312 }
 313
 314 static int rxq2vq(int rxq)
 315 {
 316         return rxq * 2;
 317 }
 318
 319 static inline struct virtio_net_hdr_mrg_rxbuf *skb_vnet_hdr(struct sk_buff *skb)
 320 {
 321         return (struct virtio_net_hdr_mrg_rxbuf *)skb->cb;
 322 }
 323
 324 /*
 325  * private is used to chain pages for big packets, put the whole
 326  * most recent used list in the beginning for reuse
 327  */
 328 static void give_pages(struct receive_queue *rq, struct page *page)
 329 {
 330         struct page *end;
 331
 332         /* Find end of list, sew whole thing into vi->rq.pages. */
 333         for (end = page; end->private; end = (struct page *)end->private);
 334         end->private = (unsigned long)rq->pages;
 335         rq->pages = page;
 336 }
 337
 338 static struct page *get_a_page(struct receive_queue *rq, gfp_t gfp_mask)
 339 {
 340         struct page *p = rq->pages;
 341
 342         if (p) {
 343                 rq->pages = (struct page *)p->private;
 344                 /* clear private here, it is used to chain pages */
 345                 p->private = 0;
 346         } else
 347                 p = alloc_page(gfp_mask);
 348         return p;
 349 }
 350
 351 static void virtqueue_napi_schedule(struct napi_struct *napi,
 352                                     struct virtqueue *vq)
 353 {
 354         if (napi_schedule_prep(napi)) {
 355                 virtqueue_disable_cb(vq);
 356                 __napi_schedule(napi);
 357         }
 358 }
 359
 360 static void virtqueue_napi_complete(struct napi_struct *napi,
 361                                     struct virtqueue *vq, int processed)
 362 {
 363         int opaque;
 364
 365         opaque = virtqueue_enable_cb_prepare(vq);
 366         if (napi_complete_done(napi, processed)) {
 367                 if (unlikely(virtqueue_poll(vq, opaque)))
 368                         virtqueue_napi_schedule(napi, vq);
 369         } else {
 370                 virtqueue_disable_cb(vq);
 371         }
 372 }
 373
 374 static void skb_xmit_done(struct virtqueue *vq)
 375 {
 376         struct virtnet_info *vi = vq->vdev->priv;
 377         struct napi_struct *napi = &vi->sq[vq2txq(vq)].napi;
 378
 379         /* Suppress further interrupts. */
 380         virtqueue_disable_cb(vq);
 381
 382         if (napi->weight)
 383                 virtqueue_napi_schedule(napi, vq);
 384         else
 385                 /* We were probably waiting for more output buffers. */
 386                 netif_wake_subqueue(vi->dev, vq2txq(vq));
 387 }
 388
 389 #define MRG_CTX_HEADER_SHIFT 22
 390 static void *mergeable_len_to_ctx(unsigned int truesize,
 391                                   unsigned int headroom)
 392 {
 393         return (void *)(unsigned long)((headroom << MRG_CTX_HEADER_SHIFT) | truesize);
 394 }
 395
 396 static unsigned int mergeable_ctx_to_headroom(void *mrg_ctx)
 397 {
 398         return (unsigned long)mrg_ctx >> MRG_CTX_HEADER_SHIFT;
 399 }
 400
 401 static unsigned int mergeable_ctx_to_truesize(void *mrg_ctx)
 402 {
 403         return (unsigned long)mrg_ctx & ((1 << MRG_CTX_HEADER_SHIFT) - 1);
 404 }
 405
 406 /* Called from bottom half context */
 407 static struct sk_buff *page_to_skb(struct virtnet_info *vi,
 408                                    struct receive_queue *rq,
 409                                    struct page *page, unsigned int offset,
 410                                    unsigned int len, unsigned int truesize,
 411                                    bool hdr_valid, unsigned int metasize,
 412                                    unsigned int headroom)
 413 {
 414         struct sk_buff *skb;
 415         struct virtio_net_hdr_mrg_rxbuf *hdr;
 416         unsigned int copy, hdr_len, hdr_padded_len;
 417         struct page *page_to_free = NULL;
 418         int tailroom, shinfo_size;
 419         char *p, *hdr_p, *buf;
 420
 421         p = page_address(page) + offset;
 422         hdr_p = p;
 423
 424         hdr_len = vi->hdr_len;
 425         if (vi->mergeable_rx_bufs)
 426                 hdr_padded_len = hdr_len;
 427         else
 428                 hdr_padded_len = sizeof(struct padded_vnet_hdr);
 429
 430         /* If headroom is not 0, there is an offset between the beginning of the
 431          * data and the allocated space, otherwise the data and the allocated
 432          * space are aligned.
 433          *
 434          * Buffers with headroom use PAGE_SIZE as alloc size, see
 435          * add_recvbuf_mergeable() + get_mergeable_buf_len()
 436          */
 437         truesize = headroom ? PAGE_SIZE : truesize;
 438         tailroom = truesize - headroom;
 439         buf = p - headroom;
 440
 441         len -= hdr_len;
 442         offset += hdr_padded_len;
 443         p += hdr_padded_len;
 444         tailroom -= hdr_padded_len + len;
 445
 446         shinfo_size = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
 447
 448         /* copy small packet so we can reuse these pages */
 449         if (!NET_IP_ALIGN && len > GOOD_COPY_LEN && tailroom >= shinfo_size) {
 450                 skb = build_skb(buf, truesize);
 451                 if (unlikely(!skb))
 452                         return NULL;
 453
 454                 skb_reserve(skb, p - buf);
 455                 skb_put(skb, len);
 456
 457                 page = (struct page *)page->private;
 458                 if (page)
 459                         give_pages(rq, page);
 460                 goto ok;
 461         }
 462
 463         /* copy small packet so we can reuse these pages for small data */
 464         skb = napi_alloc_skb(&rq->napi, GOOD_COPY_LEN);
 465         if (unlikely(!skb))
 466                 return NULL;
 467
 468         /* Copy all frame if it fits skb->head, otherwise
 469          * we let virtio_net_hdr_to_skb() and GRO pull headers as needed.
 470          */
 471         if (len <= skb_tailroom(skb))
 472                 copy = len;
 473         else
 474                 copy = ETH_HLEN + metasize;
 475         skb_put_data(skb, p, copy);
 476
 477         len -= copy;
 478         offset += copy;
 479
 480         if (vi->mergeable_rx_bufs) {
 481                 if (len)
 482                         skb_add_rx_frag(skb, 0, page, offset, len, truesize);
 483                 else
 484                         page_to_free = page;
 485                 goto ok;
 486         }
 487
 488         /*
 489          * Verify that we can indeed put this data into a skb.
 490          * This is here to handle cases when the device erroneously
 491          * tries to receive more than is possible. This is usually
 492          * the case of a broken device.
 493          */
 494         if (unlikely(len > MAX_SKB_FRAGS * PAGE_SIZE)) {
 495                 net_dbg_ratelimited("%s: too much data\n", skb->dev->name);
 496                 dev_kfree_skb(skb);
 497                 return NULL;
 498         }
 499         BUG_ON(offset >= PAGE_SIZE);
 500         while (len) {
 501                 unsigned int frag_size = min((unsigned)PAGE_SIZE - offset, len);
 502                 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, offset,
 503                                 frag_size, truesize);
 504                 len -= frag_size;
 505                 page = (struct page *)page->private;
 506                 offset = 0;
 507         }
 508
 509         if (page)
 510                 give_pages(rq, page);
 511
 512 ok:
 513         /* hdr_valid means no XDP, so we can copy the vnet header */
 514         if (hdr_valid) {
 515                 hdr = skb_vnet_hdr(skb);
 516                 memcpy(hdr, hdr_p, hdr_len);
 517         }
 518         if (page_to_free)
 519                 put_page(page_to_free);
 520
 521         if (metasize) {
 522                 __skb_pull(skb, metasize);
 523                 skb_metadata_set(skb, metasize);
 524         }
 525
 526         return skb;
 527 }
 528
 529 static int __virtnet_xdp_xmit_one(struct virtnet_info *vi,
 530                                    struct send_queue *sq,
 531                                    struct xdp_frame *xdpf)
 532 {
 533         struct virtio_net_hdr_mrg_rxbuf *hdr;
 534         int err;
 535
 536         if (unlikely(xdpf->headroom < vi->hdr_len))
 537                 return -EOVERFLOW;
 538
 539         /* Make room for virtqueue hdr (also change xdpf->headroom?) */
 540         xdpf->data -= vi->hdr_len;
 541         /* Zero header and leave csum up to XDP layers */
 542         hdr = xdpf->data;
 543         memset(hdr, 0, vi->hdr_len);
 544         xdpf->len   += vi->hdr_len;
 545
 546         sg_init_one(sq->sg, xdpf->data, xdpf->len);
 547
 548         err = virtqueue_add_outbuf(sq->vq, sq->sg, 1, xdp_to_ptr(xdpf),
 549                                    GFP_ATOMIC);
 550         if (unlikely(err))
 551                 return -ENOSPC; /* Caller handle free/refcnt */
 552
 553         return 0;
 554 }
 555
 556 /* when vi->curr_queue_pairs > nr_cpu_ids, the txq/sq is only used for xdp tx on
 557  * the current cpu, so it does not need to be locked.
 558  *
 559  * Here we use marco instead of inline functions because we have to deal with
 560  * three issues at the same time: 1. the choice of sq. 2. judge and execute the
 561  * lock/unlock of txq 3. make sparse happy. It is difficult for two inline
 562  * functions to perfectly solve these three problems at the same time.
 563  */
 564 #define virtnet_xdp_get_sq(vi) ({                                       \
 565         int cpu = smp_processor_id();                                   \
 566         struct netdev_queue *txq;                                       \
 567         typeof(vi) v = (vi);                                            \
 568         unsigned int qp;                                                \
 569                                                                         \
 570         if (v->curr_queue_pairs > nr_cpu_ids) {                         \
 571                 qp = v->curr_queue_pairs - v->xdp_queue_pairs;          \
 572                 qp += cpu;                                              \
 573                 txq = netdev_get_tx_queue(v->dev, qp);                  \
 574                 __netif_tx_acquire(txq);                                \
 575         } else {                                                        \
 576                 qp = cpu % v->curr_queue_pairs;                         \
 577                 txq = netdev_get_tx_queue(v->dev, qp);                  \
 578                 __netif_tx_lock(txq, cpu);                              \
 579         }                                                               \
 580         v->sq + qp;                                                     \
 581 })
 582
 583 #define virtnet_xdp_put_sq(vi, q) {                                     \
 584         struct netdev_queue *txq;                                       \
 585         typeof(vi) v = (vi);                                            \
 586                                                                         \
 587         txq = netdev_get_tx_queue(v->dev, (q) - v->sq);                 \
 588         if (v->curr_queue_pairs > nr_cpu_ids)                           \
 589                 __netif_tx_release(txq);                                \
 590         else                                                            \
 591                 __netif_tx_unlock(txq);                                 \
 592 }
 593
 594 static int virtnet_xdp_xmit(struct net_device *dev,
 595                             int n, struct xdp_frame **frames, u32 flags)
 596 {
 597         struct virtnet_info *vi = netdev_priv(dev);
 598         struct receive_queue *rq = vi->rq;
 599         struct bpf_prog *xdp_prog;
 600         struct send_queue *sq;
 601         unsigned int len;
 602         int packets = 0;
 603         int bytes = 0;
 604         int nxmit = 0;
 605         int kicks = 0;
 606         void *ptr;
 607         int ret;
 608         int i;
 609
 610         /* Only allow ndo_xdp_xmit if XDP is loaded on dev, as this
 611          * indicate XDP resources have been successfully allocated.
 612          */
 613         xdp_prog = rcu_access_pointer(rq->xdp_prog);
 614         if (!xdp_prog)
 615                 return -ENXIO;
 616
 617         sq = virtnet_xdp_get_sq(vi);
 618
 619         if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) {
 620                 ret = -EINVAL;
 621                 goto out;
 622         }
 623
 624         /* Free up any pending old buffers before queueing new ones. */
 625         while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) {
 626                 if (likely(is_xdp_frame(ptr))) {
 627                         struct xdp_frame *frame = ptr_to_xdp(ptr);
 628
 629                         bytes += frame->len;
 630                         xdp_return_frame(frame);
 631                 } else {
 632                         struct sk_buff *skb = ptr;
 633
 634                         bytes += skb->len;
 635                         napi_consume_skb(skb, false);
 636                 }
 637                 packets++;
 638         }
 639
 640         for (i = 0; i < n; i++) {
 641                 struct xdp_frame *xdpf = frames[i];
 642
 643                 if (__virtnet_xdp_xmit_one(vi, sq, xdpf))
 644                         break;
 645                 nxmit++;
 646         }
 647         ret = nxmit;
 648
 649         if (flags & XDP_XMIT_FLUSH) {
 650                 if (virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq))
 651                         kicks = 1;
 652         }
 653 out:
 654         u64_stats_update_begin(&sq->stats.syncp);
 655         sq->stats.bytes += bytes;
 656         sq->stats.packets += packets;
 657         sq->stats.xdp_tx += n;
 658         sq->stats.xdp_tx_drops += n - nxmit;
 659         sq->stats.kicks += kicks;
 660         u64_stats_update_end(&sq->stats.syncp);
 661
 662         virtnet_xdp_put_sq(vi, sq);
 663         return ret;
 664 }
 665
 666 static unsigned int virtnet_get_headroom(struct virtnet_info *vi)
 667 {
 668         return vi->xdp_enabled ? VIRTIO_XDP_HEADROOM : 0;
 669 }
 670
 671 /* We copy the packet for XDP in the following cases:
 672  *
 673  * 1) Packet is scattered across multiple rx buffers.
 674  * 2) Headroom space is insufficient.
 675  *
 676  * This is inefficient but it's a temporary condition that
 677  * we hit right after XDP is enabled and until queue is refilled
 678  * with large buffers with sufficient headroom - so it should affect
 679  * at most queue size packets.
 680  * Afterwards, the conditions to enable
 681  * XDP should preclude the underlying device from sending packets
 682  * across multiple buffers (num_buf > 1), and we make sure buffers
 683  * have enough headroom.
 684  */
 685 static struct page *xdp_linearize_page(struct receive_queue *rq,
 686                                        u16 *num_buf,
 687                                        struct page *p,
 688                                        int offset,
 689                                        int page_off,
 690                                        unsigned int *len)
 691 {
 692         struct page *page = alloc_page(GFP_ATOMIC);
 693
 694         if (!page)
 695                 return NULL;
 696
 697         memcpy(page_address(page) + page_off, page_address(p) + offset, *len);
 698         page_off += *len;
 699
 700         while (--*num_buf) {
 701                 int tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
 702                 unsigned int buflen;
 703                 void *buf;
 704                 int off;
 705
 706                 buf = virtqueue_get_buf(rq->vq, &buflen);
 707                 if (unlikely(!buf))
 708                         goto err_buf;
 709
 710                 p = virt_to_head_page(buf);
 711                 off = buf - page_address(p);
 712
 713                 /* guard against a misconfigured or uncooperative backend that
 714                  * is sending packet larger than the MTU.
 715                  */
 716                 if ((page_off + buflen + tailroom) > PAGE_SIZE) {
 717                         put_page(p);
 718                         goto err_buf;
 719                 }
 720
 721                 memcpy(page_address(page) + page_off,
 722                        page_address(p) + off, buflen);
 723                 page_off += buflen;
 724                 put_page(p);
 725         }
 726
 727         /* Headroom does not contribute to packet length */
 728         *len = page_off - VIRTIO_XDP_HEADROOM;
 729         return page;
 730 err_buf:
 731         __free_pages(page, 0);
 732         return NULL;
 733 }
 734
 735 static struct sk_buff *receive_small(struct net_device *dev,
 736                                      struct virtnet_info *vi,
 737                                      struct receive_queue *rq,
 738                                      void *buf, void *ctx,
 739                                      unsigned int len,
 740                                      unsigned int *xdp_xmit,
 741                                      struct virtnet_rq_stats *stats)
 742 {
 743         struct sk_buff *skb;
 744         struct bpf_prog *xdp_prog;
 745         unsigned int xdp_headroom = (unsigned long)ctx;
 746         unsigned int header_offset = VIRTNET_RX_PAD + xdp_headroom;
 747         unsigned int headroom = vi->hdr_len + header_offset;
 748         unsigned int buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) +
 749                               SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
 750         struct page *page = virt_to_head_page(buf);
 751         unsigned int delta = 0;
 752         struct page *xdp_page;
 753         int err;
 754         unsigned int metasize = 0;
 755
 756         len -= vi->hdr_len;
 757         stats->bytes += len;
 758
 759         if (unlikely(len > GOOD_PACKET_LEN)) {
 760                 pr_debug("%s: rx error: len %u exceeds max size %d\n",
 761                          dev->name, len, GOOD_PACKET_LEN);
 762                 dev->stats.rx_length_errors++;
 763                 goto err;
 764         }
 765
 766         if (likely(!vi->xdp_enabled)) {
 767                 xdp_prog = NULL;
 768                 goto skip_xdp;
 769         }
 770
 771         rcu_read_lock();
 772         xdp_prog = rcu_dereference(rq->xdp_prog);
 773         if (xdp_prog) {
 774                 struct virtio_net_hdr_mrg_rxbuf *hdr = buf + header_offset;
 775                 struct xdp_frame *xdpf;
 776                 struct xdp_buff xdp;
 777                 void *orig_data;
 778                 u32 act;
 779
 780                 if (unlikely(hdr->hdr.gso_type))
 781                         goto err_xdp;
 782
 783                 if (unlikely(xdp_headroom < virtnet_get_headroom(vi))) {
 784                         int offset = buf - page_address(page) + header_offset;
 785                         unsigned int tlen = len + vi->hdr_len;
 786                         u16 num_buf = 1;
 787
 788                         xdp_headroom = virtnet_get_headroom(vi);
 789                         header_offset = VIRTNET_RX_PAD + xdp_headroom;
 790                         headroom = vi->hdr_len + header_offset;
 791                         buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) +
 792                                  SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
 793                         xdp_page = xdp_linearize_page(rq, &num_buf, page,
 794                                                       offset, header_offset,
 795                                                       &tlen);
 796                         if (!xdp_page)
 797                                 goto err_xdp;
 798
 799                         buf = page_address(xdp_page);
 800                         put_page(page);
 801                         page = xdp_page;
 802                 }
 803
 804                 xdp_init_buff(&xdp, buflen, &rq->xdp_rxq);
 805                 xdp_prepare_buff(&xdp, buf + VIRTNET_RX_PAD + vi->hdr_len,
 806                                  xdp_headroom, len, true);
 807                 orig_data = xdp.data;
 808                 act = bpf_prog_run_xdp(xdp_prog, &xdp);
 809                 stats->xdp_packets++;
 810
 811                 switch (act) {
 812                 case XDP_PASS:
 813                         /* Recalculate length in case bpf program changed it */
 814                         delta = orig_data - xdp.data;
 815                         len = xdp.data_end - xdp.data;
 816                         metasize = xdp.data - xdp.data_meta;
 817                         break;
 818                 case XDP_TX:
 819                         stats->xdp_tx++;
 820                         xdpf = xdp_convert_buff_to_frame(&xdp);
 821                         if (unlikely(!xdpf))
 822                                 goto err_xdp;
 823                         err = virtnet_xdp_xmit(dev, 1, &xdpf, 0);
 824                         if (unlikely(!err)) {
 825                                 xdp_return_frame_rx_napi(xdpf);
 826                         } else if (unlikely(err < 0)) {
 827                                 trace_xdp_exception(vi->dev, xdp_prog, act);
 828                                 goto err_xdp;
 829                         }
 830                         *xdp_xmit |= VIRTIO_XDP_TX;
 831                         rcu_read_unlock();
 832                         goto xdp_xmit;
 833                 case XDP_REDIRECT:
 834                         stats->xdp_redirects++;
 835                         err = xdp_do_redirect(dev, &xdp, xdp_prog);
 836                         if (err)
 837                                 goto err_xdp;
 838                         *xdp_xmit |= VIRTIO_XDP_REDIR;
 839                         rcu_read_unlock();
 840                         goto xdp_xmit;
 841                 default:
 842                         bpf_warn_invalid_xdp_action(vi->dev, xdp_prog, act);
 843                         fallthrough;
 844                 case XDP_ABORTED:
 845                         trace_xdp_exception(vi->dev, xdp_prog, act);
 846                         goto err_xdp;
 847                 case XDP_DROP:
 848                         goto err_xdp;
 849                 }
 850         }
 851         rcu_read_unlock();
 852
 853 skip_xdp:
 854         skb = build_skb(buf, buflen);
 855         if (!skb)
 856                 goto err;
 857         skb_reserve(skb, headroom - delta);
 858         skb_put(skb, len);
 859         if (!xdp_prog) {
 860                 buf += header_offset;
 861                 memcpy(skb_vnet_hdr(skb), buf, vi->hdr_len);
 862         } /* keep zeroed vnet hdr since XDP is loaded */
 863
 864         if (metasize)
 865                 skb_metadata_set(skb, metasize);
 866
 867         return skb;
 868
 869 err_xdp:
 870         rcu_read_unlock();
 871         stats->xdp_drops++;
 872 err:
 873         stats->drops++;
 874         put_page(page);
 875 xdp_xmit:
 876         return NULL;
 877 }
 878
 879 static struct sk_buff *receive_big(struct net_device *dev,
 880                                    struct virtnet_info *vi,
 881                                    struct receive_queue *rq,
 882                                    void *buf,
 883                                    unsigned int len,
 884                                    struct virtnet_rq_stats *stats)
 885 {
 886         struct page *page = buf;
 887         struct sk_buff *skb =
 888                 page_to_skb(vi, rq, page, 0, len, PAGE_SIZE, true, 0, 0);
 889
 890         stats->bytes += len - vi->hdr_len;
 891         if (unlikely(!skb))
 892                 goto err;
 893
 894         return skb;
 895
 896 err:
 897         stats->drops++;
 898         give_pages(rq, page);
 899         return NULL;
 900 }
 901
 902 static struct sk_buff *receive_mergeable(struct net_device *dev,
 903                                          struct virtnet_info *vi,
 904                                          struct receive_queue *rq,
 905                                          void *buf,
 906                                          void *ctx,
 907                                          unsigned int len,
 908                                          unsigned int *xdp_xmit,
 909                                          struct virtnet_rq_stats *stats)
 910 {
 911         struct virtio_net_hdr_mrg_rxbuf *hdr = buf;
 912         u16 num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers);
 913         struct page *page = virt_to_head_page(buf);
 914         int offset = buf - page_address(page);
 915         struct sk_buff *head_skb, *curr_skb;
 916         struct bpf_prog *xdp_prog;
 917         unsigned int truesize = mergeable_ctx_to_truesize(ctx);
 918         unsigned int headroom = mergeable_ctx_to_headroom(ctx);
 919         unsigned int metasize = 0;
 920         unsigned int frame_sz;
 921         int err;
 922
 923         head_skb = NULL;
 924         stats->bytes += len - vi->hdr_len;
 925
 926         if (unlikely(len > truesize)) {
 927                 pr_debug("%s: rx error: len %u exceeds truesize %lu\n",
 928                          dev->name, len, (unsigned long)ctx);
 929                 dev->stats.rx_length_errors++;
 930                 goto err_skb;
 931         }
 932
 933         if (likely(!vi->xdp_enabled)) {
 934                 xdp_prog = NULL;
 935                 goto skip_xdp;
 936         }
 937
 938         rcu_read_lock();
 939         xdp_prog = rcu_dereference(rq->xdp_prog);
 940         if (xdp_prog) {
 941                 struct xdp_frame *xdpf;
 942                 struct page *xdp_page;
 943                 struct xdp_buff xdp;
 944                 void *data;
 945                 u32 act;
 946
 947                 /* Transient failure which in theory could occur if
 948                  * in-flight packets from before XDP was enabled reach
 949                  * the receive path after XDP is loaded.
 950                  */
 951                 if (unlikely(hdr->hdr.gso_type))
 952                         goto err_xdp;
 953
 954                 /* Buffers with headroom use PAGE_SIZE as alloc size,
 955                  * see add_recvbuf_mergeable() + get_mergeable_buf_len()
 956                  */
 957                 frame_sz = headroom ? PAGE_SIZE : truesize;
 958
 959                 /* This happens when rx buffer size is underestimated
 960                  * or headroom is not enough because of the buffer
 961                  * was refilled before XDP is set. This should only
 962                  * happen for the first several packets, so we don't
 963                  * care much about its performance.
 964                  */
 965                 if (unlikely(num_buf > 1 ||
 966                              headroom < virtnet_get_headroom(vi))) {
 967                         /* linearize data for XDP */
 968                         xdp_page = xdp_linearize_page(rq, &num_buf,
 969                                                       page, offset,
 970                                                       VIRTIO_XDP_HEADROOM,
 971                                                       &len);
 972                         frame_sz = PAGE_SIZE;
 973
 974                         if (!xdp_page)
 975                                 goto err_xdp;
 976                         offset = VIRTIO_XDP_HEADROOM;
 977                 } else {
 978                         xdp_page = page;
 979                 }
 980
 981                 /* Allow consuming headroom but reserve enough space to push
 982                  * the descriptor on if we get an XDP_TX return code.
 983                  */
 984                 data = page_address(xdp_page) + offset;
 985                 xdp_init_buff(&xdp, frame_sz - vi->hdr_len, &rq->xdp_rxq);
 986                 xdp_prepare_buff(&xdp, data - VIRTIO_XDP_HEADROOM + vi->hdr_len,
 987                                  VIRTIO_XDP_HEADROOM, len - vi->hdr_len, true);
 988
 989                 act = bpf_prog_run_xdp(xdp_prog, &xdp);
 990                 stats->xdp_packets++;
 991
 992                 switch (act) {
 993                 case XDP_PASS:
 994                         metasize = xdp.data - xdp.data_meta;
 995
 996                         /* recalculate offset to account for any header
 997                          * adjustments and minus the metasize to copy the
 998                          * metadata in page_to_skb(). Note other cases do not
 999                          * build an skb and avoid using offset
1000                          */
1001                         offset = xdp.data - page_address(xdp_page) -
1002                                  vi->hdr_len - metasize;
1003
1004                         /* recalculate len if xdp.data, xdp.data_end or
1005                          * xdp.data_meta were adjusted
1006                          */
1007                         len = xdp.data_end - xdp.data + vi->hdr_len + metasize;
1008                         /* We can only create skb based on xdp_page. */
1009                         if (unlikely(xdp_page != page)) {
1010                                 rcu_read_unlock();
1011                                 put_page(page);
1012                                 head_skb = page_to_skb(vi, rq, xdp_page, offset,
1013                                                        len, PAGE_SIZE, false,
1014                                                        metasize,
1015                                                        VIRTIO_XDP_HEADROOM);
1016                                 return head_skb;
1017                         }
1018                         break;
1019                 case XDP_TX:
1020                         stats->xdp_tx++;
1021                         xdpf = xdp_convert_buff_to_frame(&xdp);
1022                         if (unlikely(!xdpf))
1023                                 goto err_xdp;
1024                         err = virtnet_xdp_xmit(dev, 1, &xdpf, 0);
1025                         if (unlikely(!err)) {
1026                                 xdp_return_frame_rx_napi(xdpf);
1027                         } else if (unlikely(err < 0)) {
1028                                 trace_xdp_exception(vi->dev, xdp_prog, act);
1029                                 if (unlikely(xdp_page != page))
1030                                         put_page(xdp_page);
1031                                 goto err_xdp;
1032                         }
1033                         *xdp_xmit |= VIRTIO_XDP_TX;
1034                         if (unlikely(xdp_page != page))
1035                                 put_page(page);
1036                         rcu_read_unlock();
1037                         goto xdp_xmit;
1038                 case XDP_REDIRECT:
1039                         stats->xdp_redirects++;
1040                         err = xdp_do_redirect(dev, &xdp, xdp_prog);
1041                         if (err) {
1042                                 if (unlikely(xdp_page != page))
1043                                         put_page(xdp_page);
1044                                 goto err_xdp;
1045                         }
1046                         *xdp_xmit |= VIRTIO_XDP_REDIR;
1047                         if (unlikely(xdp_page != page))
1048                                 put_page(page);
1049                         rcu_read_unlock();
1050                         goto xdp_xmit;
1051                 default:
1052                         bpf_warn_invalid_xdp_action(vi->dev, xdp_prog, act);
1053                         fallthrough;
1054                 case XDP_ABORTED:
1055                         trace_xdp_exception(vi->dev, xdp_prog, act);
1056                         fallthrough;
1057                 case XDP_DROP:
1058                         if (unlikely(xdp_page != page))
1059                                 __free_pages(xdp_page, 0);
1060                         goto err_xdp;
1061                 }
1062         }
1063         rcu_read_unlock();
1064
1065 skip_xdp:
1066         head_skb = page_to_skb(vi, rq, page, offset, len, truesize, !xdp_prog,
1067                                metasize, headroom);
1068         curr_skb = head_skb;
1069
1070         if (unlikely(!curr_skb))
1071                 goto err_skb;
1072         while (--num_buf) {
1073                 int num_skb_frags;
1074
1075                 buf = virtqueue_get_buf_ctx(rq->vq, &len, &ctx);
1076                 if (unlikely(!buf)) {
1077                         pr_debug("%s: rx error: %d buffers out of %d missing\n",
1078                                  dev->name, num_buf,
1079                                  virtio16_to_cpu(vi->vdev,
1080                                                  hdr->num_buffers));
1081                         dev->stats.rx_length_errors++;
1082                         goto err_buf;
1083                 }
1084
1085                 stats->bytes += len;
1086                 page = virt_to_head_page(buf);
1087
1088                 truesize = mergeable_ctx_to_truesize(ctx);
1089                 if (unlikely(len > truesize)) {
1090                         pr_debug("%s: rx error: len %u exceeds truesize %lu\n",
1091                                  dev->name, len, (unsigned long)ctx);
1092                         dev->stats.rx_length_errors++;
1093                         goto err_skb;
1094                 }
1095
1096                 num_skb_frags = skb_shinfo(curr_skb)->nr_frags;
1097                 if (unlikely(num_skb_frags == MAX_SKB_FRAGS)) {
1098                         struct sk_buff *nskb = alloc_skb(0, GFP_ATOMIC);
1099
1100                         if (unlikely(!nskb))
1101                                 goto err_skb;
1102                         if (curr_skb == head_skb)
1103                                 skb_shinfo(curr_skb)->frag_list = nskb;
1104                         else
1105                                 curr_skb->next = nskb;
1106                         curr_skb = nskb;
1107                         head_skb->truesize += nskb->truesize;
1108                         num_skb_frags = 0;
1109                 }
1110                 if (curr_skb != head_skb) {
1111                         head_skb->data_len += len;
1112                         head_skb->len += len;
1113                         head_skb->truesize += truesize;
1114                 }
1115                 offset = buf - page_address(page);
1116                 if (skb_can_coalesce(curr_skb, num_skb_frags, page, offset)) {
1117                         put_page(page);
1118                         skb_coalesce_rx_frag(curr_skb, num_skb_frags - 1,
1119                                              len, truesize);
1120                 } else {
1121                         skb_add_rx_frag(curr_skb, num_skb_frags, page,
1122                                         offset, len, truesize);
1123                 }
1124         }
1125
1126         ewma_pkt_len_add(&rq->mrg_avg_pkt_len, head_skb->len);
1127         return head_skb;
1128
1129 err_xdp:
1130         rcu_read_unlock();
1131         stats->xdp_drops++;
1132 err_skb:
1133         put_page(page);
1134         while (num_buf-- > 1) {
1135                 buf = virtqueue_get_buf(rq->vq, &len);
1136                 if (unlikely(!buf)) {
1137                         pr_debug("%s: rx error: %d buffers missing\n",
1138                                  dev->name, num_buf);
1139                         dev->stats.rx_length_errors++;
1140                         break;
1141                 }
1142                 stats->bytes += len;
1143                 page = virt_to_head_page(buf);
1144                 put_page(page);
1145         }
1146 err_buf:
1147         stats->drops++;
1148         dev_kfree_skb(head_skb);
1149 xdp_xmit:
1150         return NULL;
1151 }
1152
1153 static void virtio_skb_set_hash(const struct virtio_net_hdr_v1_hash *hdr_hash,
1154                                 struct sk_buff *skb)
1155 {
1156         enum pkt_hash_types rss_hash_type;
1157
1158         if (!hdr_hash || !skb)
1159                 return;
1160
1161         switch ((int)hdr_hash->hash_report) {
1162         case VIRTIO_NET_HASH_REPORT_TCPv4:
1163         case VIRTIO_NET_HASH_REPORT_UDPv4:
1164         case VIRTIO_NET_HASH_REPORT_TCPv6:
1165         case VIRTIO_NET_HASH_REPORT_UDPv6:
1166         case VIRTIO_NET_HASH_REPORT_TCPv6_EX:
1167         case VIRTIO_NET_HASH_REPORT_UDPv6_EX:
1168                 rss_hash_type = PKT_HASH_TYPE_L4;
1169                 break;
1170         case VIRTIO_NET_HASH_REPORT_IPv4:
1171         case VIRTIO_NET_HASH_REPORT_IPv6:
1172         case VIRTIO_NET_HASH_REPORT_IPv6_EX:
1173                 rss_hash_type = PKT_HASH_TYPE_L3;
1174                 break;
1175         case VIRTIO_NET_HASH_REPORT_NONE:
1176         default:
1177                 rss_hash_type = PKT_HASH_TYPE_NONE;
1178         }
1179         skb_set_hash(skb, (unsigned int)hdr_hash->hash_value, rss_hash_type);
1180 }
1181
1182 static void receive_buf(struct virtnet_info *vi, struct receive_queue *rq,
1183                         void *buf, unsigned int len, void **ctx,
1184                         unsigned int *xdp_xmit,
1185                         struct virtnet_rq_stats *stats)
1186 {
1187         struct net_device *dev = vi->dev;
1188         struct sk_buff *skb;
1189         struct virtio_net_hdr_mrg_rxbuf *hdr;
1190
1191         if (unlikely(len < vi->hdr_len + ETH_HLEN)) {
1192                 pr_debug("%s: short packet %i\n", dev->name, len);
1193                 dev->stats.rx_length_errors++;
1194                 if (vi->mergeable_rx_bufs) {
1195                         put_page(virt_to_head_page(buf));
1196                 } else if (vi->big_packets) {
1197                         give_pages(rq, buf);
1198                 } else {
1199                         put_page(virt_to_head_page(buf));
1200                 }
1201                 return;
1202         }
1203
1204         if (vi->mergeable_rx_bufs)
1205                 skb = receive_mergeable(dev, vi, rq, buf, ctx, len, xdp_xmit,
1206                                         stats);
1207         else if (vi->big_packets)
1208                 skb = receive_big(dev, vi, rq, buf, len, stats);
1209         else
1210                 skb = receive_small(dev, vi, rq, buf, ctx, len, xdp_xmit, stats);
1211
1212         if (unlikely(!skb))
1213                 return;
1214
1215         hdr = skb_vnet_hdr(skb);
1216         if (dev->features & NETIF_F_RXHASH && vi->has_rss_hash_report)
1217                 virtio_skb_set_hash((const struct virtio_net_hdr_v1_hash *)hdr, skb);
1218
1219         if (hdr->hdr.flags & VIRTIO_NET_HDR_F_DATA_VALID)
1220                 skb->ip_summed = CHECKSUM_UNNECESSARY;
1221
1222         if (virtio_net_hdr_to_skb(skb, &hdr->hdr,
1223                                   virtio_is_little_endian(vi->vdev))) {
1224                 net_warn_ratelimited("%s: bad gso: type: %u, size: %u\n",
1225                                      dev->name, hdr->hdr.gso_type,
1226                                      hdr->hdr.gso_size);
1227                 goto frame_err;
1228         }
1229
1230         skb_record_rx_queue(skb, vq2rxq(rq->vq));
1231         skb->protocol = eth_type_trans(skb, dev);
1232         pr_debug("Receiving skb proto 0x%04x len %i type %i\n",
1233                  ntohs(skb->protocol), skb->len, skb->pkt_type);
1234
1235         napi_gro_receive(&rq->napi, skb);
1236         return;
1237
1238 frame_err:
1239         dev->stats.rx_frame_errors++;
1240         dev_kfree_skb(skb);
1241 }
1242
1243 /* Unlike mergeable buffers, all buffers are allocated to the
1244  * same size, except for the headroom. For this reason we do
1245  * not need to use  mergeable_len_to_ctx here - it is enough
1246  * to store the headroom as the context ignoring the truesize.
1247  */
1248 static int add_recvbuf_small(struct virtnet_info *vi, struct receive_queue *rq,
1249                              gfp_t gfp)
1250 {
1251         struct page_frag *alloc_frag = &rq->alloc_frag;
1252         char *buf;
1253         unsigned int xdp_headroom = virtnet_get_headroom(vi);
1254         void *ctx = (void *)(unsigned long)xdp_headroom;
1255         int len = vi->hdr_len + VIRTNET_RX_PAD + GOOD_PACKET_LEN + xdp_headroom;
1256         int err;
1257
1258         len = SKB_DATA_ALIGN(len) +
1259               SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
1260         if (unlikely(!skb_page_frag_refill(len, alloc_frag, gfp)))
1261                 return -ENOMEM;
1262
1263         buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
1264         get_page(alloc_frag->page);
1265         alloc_frag->offset += len;
1266         sg_init_one(rq->sg, buf + VIRTNET_RX_PAD + xdp_headroom,
1267                     vi->hdr_len + GOOD_PACKET_LEN);
1268         err = virtqueue_add_inbuf_ctx(rq->vq, rq->sg, 1, buf, ctx, gfp);
1269         if (err < 0)
1270                 put_page(virt_to_head_page(buf));
1271         return err;
1272 }
1273
1274 static int add_recvbuf_big(struct virtnet_info *vi, struct receive_queue *rq,
1275                            gfp_t gfp)
1276 {
1277         struct page *first, *list = NULL;
1278         char *p;
1279         int i, err, offset;
1280
1281         sg_init_table(rq->sg, MAX_SKB_FRAGS + 2);
1282
1283         /* page in rq->sg[MAX_SKB_FRAGS + 1] is list tail */
1284         for (i = MAX_SKB_FRAGS + 1; i > 1; --i) {
1285                 first = get_a_page(rq, gfp);
1286                 if (!first) {
1287                         if (list)
1288                                 give_pages(rq, list);
1289                         return -ENOMEM;
1290                 }
1291                 sg_set_buf(&rq->sg[i], page_address(first), PAGE_SIZE);
1292
1293                 /* chain new page in list head to match sg */
1294                 first->private = (unsigned long)list;
1295                 list = first;
1296         }
1297
1298         first = get_a_page(rq, gfp);
1299         if (!first) {
1300                 give_pages(rq, list);
1301                 return -ENOMEM;
1302         }
1303         p = page_address(first);
1304
1305         /* rq->sg[0], rq->sg[1] share the same page */
1306         /* a separated rq->sg[0] for header - required in case !any_header_sg */
1307         sg_set_buf(&rq->sg[0], p, vi->hdr_len);
1308
1309         /* rq->sg[1] for data packet, from offset */
1310         offset = sizeof(struct padded_vnet_hdr);
1311         sg_set_buf(&rq->sg[1], p + offset, PAGE_SIZE - offset);
1312
1313         /* chain first in list head */
1314         first->private = (unsigned long)list;
1315         err = virtqueue_add_inbuf(rq->vq, rq->sg, MAX_SKB_FRAGS + 2,
1316                                   first, gfp);
1317         if (err < 0)
1318                 give_pages(rq, first);
1319
1320         return err;
1321 }
1322
1323 static unsigned int get_mergeable_buf_len(struct receive_queue *rq,
1324                                           struct ewma_pkt_len *avg_pkt_len,
1325                                           unsigned int room)
1326 {
1327         struct virtnet_info *vi = rq->vq->vdev->priv;
1328         const size_t hdr_len = vi->hdr_len;
1329         unsigned int len;
1330
1331         if (room)
1332                 return PAGE_SIZE - room;
1333
1334         len = hdr_len + clamp_t(unsigned int, ewma_pkt_len_read(avg_pkt_len),
1335                                 rq->min_buf_len, PAGE_SIZE - hdr_len);
1336
1337         return ALIGN(len, L1_CACHE_BYTES);
1338 }
1339
1340 static int add_recvbuf_mergeable(struct virtnet_info *vi,
1341                                  struct receive_queue *rq, gfp_t gfp)
1342 {
1343         struct page_frag *alloc_frag = &rq->alloc_frag;
1344         unsigned int headroom = virtnet_get_headroom(vi);
1345         unsigned int tailroom = headroom ? sizeof(struct skb_shared_info) : 0;
1346         unsigned int room = SKB_DATA_ALIGN(headroom + tailroom);
1347         char *buf;
1348         void *ctx;
1349         int err;
1350         unsigned int len, hole;
1351
1352         /* Extra tailroom is needed to satisfy XDP's assumption. This
1353          * means rx frags coalescing won't work, but consider we've
1354          * disabled GSO for XDP, it won't be a big issue.
1355          */
1356         len = get_mergeable_buf_len(rq, &rq->mrg_avg_pkt_len, room);
1357         if (unlikely(!skb_page_frag_refill(len + room, alloc_frag, gfp)))
1358                 return -ENOMEM;
1359
1360         buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
1361         buf += headroom; /* advance address leaving hole at front of pkt */
1362         get_page(alloc_frag->page);
1363         alloc_frag->offset += len + room;
1364         hole = alloc_frag->size - alloc_frag->offset;
1365         if (hole < len + room) {
1366                 /* To avoid internal fragmentation, if there is very likely not
1367                  * enough space for another buffer, add the remaining space to
1368                  * the current buffer.
1369                  */
1370                 len += hole;
1371                 alloc_frag->offset += hole;
1372         }
1373
1374         sg_init_one(rq->sg, buf, len);
1375         ctx = mergeable_len_to_ctx(len, headroom);
1376         err = virtqueue_add_inbuf_ctx(rq->vq, rq->sg, 1, buf, ctx, gfp);
1377         if (err < 0)
1378                 put_page(virt_to_head_page(buf));
1379
1380         return err;
1381 }
1382
1383 /*
1384  * Returns false if we couldn't fill entirely (OOM).
1385  *
1386  * Normally run in the receive path, but can also be run from ndo_open
1387  * before we're receiving packets, or from refill_work which is
1388  * careful to disable receiving (using napi_disable).
1389  */
1390 static bool try_fill_recv(struct virtnet_info *vi, struct receive_queue *rq,
1391                           gfp_t gfp)
1392 {
1393         int err;
1394         bool oom;
1395
1396         do {
1397                 if (vi->mergeable_rx_bufs)
1398                         err = add_recvbuf_mergeable(vi, rq, gfp);
1399                 else if (vi->big_packets)
1400                         err = add_recvbuf_big(vi, rq, gfp);
1401                 else
1402                         err = add_recvbuf_small(vi, rq, gfp);
1403
1404                 oom = err == -ENOMEM;
1405                 if (err)
1406                         break;
1407         } while (rq->vq->num_free);
1408         if (virtqueue_kick_prepare(rq->vq) && virtqueue_notify(rq->vq)) {
1409                 unsigned long flags;
1410
1411                 flags = u64_stats_update_begin_irqsave(&rq->stats.syncp);
1412                 rq->stats.kicks++;
1413                 u64_stats_update_end_irqrestore(&rq->stats.syncp, flags);
1414         }
1415
1416         return !oom;
1417 }
1418
1419 static void skb_recv_done(struct virtqueue *rvq)
1420 {
1421         struct virtnet_info *vi = rvq->vdev->priv;
1422         struct receive_queue *rq = &vi->rq[vq2rxq(rvq)];
1423
1424         virtqueue_napi_schedule(&rq->napi, rvq);
1425 }
1426
1427 static void virtnet_napi_enable(struct virtqueue *vq, struct napi_struct *napi)
1428 {
1429         napi_enable(napi);
1430
1431         /* If all buffers were filled by other side before we napi_enabled, we
1432          * won't get another interrupt, so process any outstanding packets now.
1433          * Call local_bh_enable after to trigger softIRQ processing.
1434          */
1435         local_bh_disable();
1436         virtqueue_napi_schedule(napi, vq);
1437         local_bh_enable();
1438 }
1439
1440 static void virtnet_napi_tx_enable(struct virtnet_info *vi,
1441                                    struct virtqueue *vq,
1442                                    struct napi_struct *napi)
1443 {
1444         if (!napi->weight)
1445                 return;
1446
1447         /* Tx napi touches cachelines on the cpu handling tx interrupts. Only
1448          * enable the feature if this is likely affine with the transmit path.
1449          */
1450         if (!vi->affinity_hint_set) {
1451                 napi->weight = 0;
1452                 return;
1453         }
1454
1455         return virtnet_napi_enable(vq, napi);
1456 }
1457
1458 static void virtnet_napi_tx_disable(struct napi_struct *napi)
1459 {
1460         if (napi->weight)
1461                 napi_disable(napi);
1462 }
1463
1464 static void refill_work(struct work_struct *work)
1465 {
1466         struct virtnet_info *vi =
1467                 container_of(work, struct virtnet_info, refill.work);
1468         bool still_empty;
1469         int i;
1470
1471         for (i = 0; i < vi->curr_queue_pairs; i++) {
1472                 struct receive_queue *rq = &vi->rq[i];
1473
1474                 napi_disable(&rq->napi);
1475                 still_empty = !try_fill_recv(vi, rq, GFP_KERNEL);
1476                 virtnet_napi_enable(rq->vq, &rq->napi);
1477
1478                 /* In theory, this can happen: if we don't get any buffers in
1479                  * we will *never* try to fill again.
1480                  */
1481                 if (still_empty)
1482                         schedule_delayed_work(&vi->refill, HZ/2);
1483         }
1484 }
1485
1486 static int virtnet_receive(struct receive_queue *rq, int budget,
1487                            unsigned int *xdp_xmit)
1488 {
1489         struct virtnet_info *vi = rq->vq->vdev->priv;
1490         struct virtnet_rq_stats stats = {};
1491         unsigned int len;
1492         void *buf;
1493         int i;
1494
1495         if (!vi->big_packets || vi->mergeable_rx_bufs) {
1496                 void *ctx;
1497
1498                 while (stats.packets < budget &&
1499                        (buf = virtqueue_get_buf_ctx(rq->vq, &len, &ctx))) {
1500                         receive_buf(vi, rq, buf, len, ctx, xdp_xmit, &stats);
1501                         stats.packets++;
1502                 }
1503         } else {
1504                 while (stats.packets < budget &&
1505                        (buf = virtqueue_get_buf(rq->vq, &len)) != NULL) {
1506                         receive_buf(vi, rq, buf, len, NULL, xdp_xmit, &stats);
1507                         stats.packets++;
1508                 }
1509         }
1510
1511         if (rq->vq->num_free > min((unsigned int)budget, virtqueue_get_vring_size(rq->vq)) / 2) {
1512                 if (!try_fill_recv(vi, rq, GFP_ATOMIC))
1513                         schedule_delayed_work(&vi->refill, 0);
1514         }
1515
1516         u64_stats_update_begin(&rq->stats.syncp);
1517         for (i = 0; i < VIRTNET_RQ_STATS_LEN; i++) {
1518                 size_t offset = virtnet_rq_stats_desc[i].offset;
1519                 u64 *item;
1520
1521                 item = (u64 *)((u8 *)&rq->stats + offset);
1522                 *item += *(u64 *)((u8 *)&stats + offset);
1523         }
1524         u64_stats_update_end(&rq->stats.syncp);
1525
1526         return stats.packets;
1527 }
1528
1529 static void free_old_xmit_skbs(struct send_queue *sq, bool in_napi)
1530 {
1531         unsigned int len;
1532         unsigned int packets = 0;
1533         unsigned int bytes = 0;
1534         void *ptr;
1535
1536         while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) {
1537                 if (likely(!is_xdp_frame(ptr))) {
1538                         struct sk_buff *skb = ptr;
1539
1540                         pr_debug("Sent skb %p\n", skb);
1541
1542                         bytes += skb->len;
1543                         napi_consume_skb(skb, in_napi);
1544                 } else {
1545                         struct xdp_frame *frame = ptr_to_xdp(ptr);
1546
1547                         bytes += frame->len;
1548                         xdp_return_frame(frame);
1549                 }
1550                 packets++;
1551         }
1552
1553         /* Avoid overhead when no packets have been processed
1554          * happens when called speculatively from start_xmit.
1555          */
1556         if (!packets)
1557                 return;
1558
1559         u64_stats_update_begin(&sq->stats.syncp);
1560         sq->stats.bytes += bytes;
1561         sq->stats.packets += packets;
1562         u64_stats_update_end(&sq->stats.syncp);
1563 }
1564
1565 static bool is_xdp_raw_buffer_queue(struct virtnet_info *vi, int q)
1566 {
1567         if (q < (vi->curr_queue_pairs - vi->xdp_queue_pairs))
1568                 return false;
1569         else if (q < vi->curr_queue_pairs)
1570                 return true;
1571         else
1572                 return false;
1573 }
1574
1575 static void virtnet_poll_cleantx(struct receive_queue *rq)
1576 {
1577         struct virtnet_info *vi = rq->vq->vdev->priv;
1578         unsigned int index = vq2rxq(rq->vq);
1579         struct send_queue *sq = &vi->sq[index];
1580         struct netdev_queue *txq = netdev_get_tx_queue(vi->dev, index);
1581
1582         if (!sq->napi.weight || is_xdp_raw_buffer_queue(vi, index))
1583                 return;
1584
1585         if (__netif_tx_trylock(txq)) {
1586                 do {
1587                         virtqueue_disable_cb(sq->vq);
1588                         free_old_xmit_skbs(sq, true);
1589                 } while (unlikely(!virtqueue_enable_cb_delayed(sq->vq)));
1590
1591                 if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS)
1592                         netif_tx_wake_queue(txq);
1593
1594                 __netif_tx_unlock(txq);
1595         }
1596 }
1597
1598 static int virtnet_poll(struct napi_struct *napi, int budget)
1599 {
1600         struct receive_queue *rq =
1601                 container_of(napi, struct receive_queue, napi);
1602         struct virtnet_info *vi = rq->vq->vdev->priv;
1603         struct send_queue *sq;
1604         unsigned int received;
1605         unsigned int xdp_xmit = 0;
1606
1607         virtnet_poll_cleantx(rq);
1608
1609         received = virtnet_receive(rq, budget, &xdp_xmit);
1610
1611         /* Out of packets? */
1612         if (received < budget)
1613                 virtqueue_napi_complete(napi, rq->vq, received);
1614
1615         if (xdp_xmit & VIRTIO_XDP_REDIR)
1616                 xdp_do_flush();
1617
1618         if (xdp_xmit & VIRTIO_XDP_TX) {
1619                 sq = virtnet_xdp_get_sq(vi);
1620                 if (virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq)) {
1621                         u64_stats_update_begin(&sq->stats.syncp);
1622                         sq->stats.kicks++;
1623                         u64_stats_update_end(&sq->stats.syncp);
1624                 }
1625                 virtnet_xdp_put_sq(vi, sq);
1626         }
1627
1628         return received;
1629 }
1630
1631 static int virtnet_open(struct net_device *dev)
1632 {
1633         struct virtnet_info *vi = netdev_priv(dev);
1634         int i, err;
1635
1636         for (i = 0; i < vi->max_queue_pairs; i++) {
1637                 if (i < vi->curr_queue_pairs)
1638                         /* Make sure we have some buffers: if oom use wq. */
1639                         if (!try_fill_recv(vi, &vi->rq[i], GFP_KERNEL))
1640                                 schedule_delayed_work(&vi->refill, 0);
1641
1642                 err = xdp_rxq_info_reg(&vi->rq[i].xdp_rxq, dev, i, vi->rq[i].napi.napi_id);
1643                 if (err < 0)
1644                         return err;
1645
1646                 err = xdp_rxq_info_reg_mem_model(&vi->rq[i].xdp_rxq,
1647                                                  MEM_TYPE_PAGE_SHARED, NULL);
1648                 if (err < 0) {
1649                         xdp_rxq_info_unreg(&vi->rq[i].xdp_rxq);
1650                         return err;
1651                 }
1652
1653                 virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi);
1654                 virtnet_napi_tx_enable(vi, vi->sq[i].vq, &vi->sq[i].napi);
1655         }
1656
1657         return 0;
1658 }
1659
1660 static int virtnet_poll_tx(struct napi_struct *napi, int budget)
1661 {
1662         struct send_queue *sq = container_of(napi, struct send_queue, napi);
1663         struct virtnet_info *vi = sq->vq->vdev->priv;
1664         unsigned int index = vq2txq(sq->vq);
1665         struct netdev_queue *txq;
1666         int opaque;
1667         bool done;
1668
1669         if (unlikely(is_xdp_raw_buffer_queue(vi, index))) {
1670                 /* We don't need to enable cb for XDP */
1671                 napi_complete_done(napi, 0);
1672                 return 0;
1673         }
1674
1675         txq = netdev_get_tx_queue(vi->dev, index);
1676         __netif_tx_lock(txq, raw_smp_processor_id());
1677         virtqueue_disable_cb(sq->vq);
1678         free_old_xmit_skbs(sq, true);
1679
1680         if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS)
1681                 netif_tx_wake_queue(txq);
1682
1683         opaque = virtqueue_enable_cb_prepare(sq->vq);
1684
1685         done = napi_complete_done(napi, 0);
1686
1687         if (!done)
1688                 virtqueue_disable_cb(sq->vq);
1689
1690         __netif_tx_unlock(txq);
1691
1692         if (done) {
1693                 if (unlikely(virtqueue_poll(sq->vq, opaque))) {
1694                         if (napi_schedule_prep(napi)) {
1695                                 __netif_tx_lock(txq, raw_smp_processor_id());
1696                                 virtqueue_disable_cb(sq->vq);
1697                                 __netif_tx_unlock(txq);
1698                                 __napi_schedule(napi);
1699                         }
1700                 }
1701         }
1702
1703         return 0;
1704 }
1705
1706 static int xmit_skb(struct send_queue *sq, struct sk_buff *skb)
1707 {
1708         struct virtio_net_hdr_mrg_rxbuf *hdr;
1709         const unsigned char *dest = ((struct ethhdr *)skb->data)->h_dest;
1710         struct virtnet_info *vi = sq->vq->vdev->priv;
1711         int num_sg;
1712         unsigned hdr_len = vi->hdr_len;
1713         bool can_push;
1714
1715         pr_debug("%s: xmit %p %pM\n", vi->dev->name, skb, dest);
1716
1717         can_push = vi->any_header_sg &&
1718                 !((unsigned long)skb->data & (__alignof__(*hdr) - 1)) &&
1719                 !skb_header_cloned(skb) && skb_headroom(skb) >= hdr_len;
1720         /* Even if we can, don't push here yet as this would skew
1721          * csum_start offset below. */
1722         if (can_push)
1723                 hdr = (struct virtio_net_hdr_mrg_rxbuf *)(skb->data - hdr_len);
1724         else
1725                 hdr = skb_vnet_hdr(skb);
1726
1727         if (virtio_net_hdr_from_skb(skb, &hdr->hdr,
1728                                     virtio_is_little_endian(vi->vdev), false,
1729                                     0))
1730                 return -EPROTO;
1731
1732         if (vi->mergeable_rx_bufs)
1733                 hdr->num_buffers = 0;
1734
1735         sg_init_table(sq->sg, skb_shinfo(skb)->nr_frags + (can_push ? 1 : 2));
1736         if (can_push) {
1737                 __skb_push(skb, hdr_len);
1738                 num_sg = skb_to_sgvec(skb, sq->sg, 0, skb->len);
1739                 if (unlikely(num_sg < 0))
1740                         return num_sg;
1741                 /* Pull header back to avoid skew in tx bytes calculations. */
1742                 __skb_pull(skb, hdr_len);
1743         } else {
1744                 sg_set_buf(sq->sg, hdr, hdr_len);
1745                 num_sg = skb_to_sgvec(skb, sq->sg + 1, 0, skb->len);
1746                 if (unlikely(num_sg < 0))
1747                         return num_sg;
1748                 num_sg++;
1749         }
1750         return virtqueue_add_outbuf(sq->vq, sq->sg, num_sg, skb, GFP_ATOMIC);
1751 }
1752
1753 static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev)
1754 {
1755         struct virtnet_info *vi = netdev_priv(dev);
1756         int qnum = skb_get_queue_mapping(skb);
1757         struct send_queue *sq = &vi->sq[qnum];
1758         int err;
1759         struct netdev_queue *txq = netdev_get_tx_queue(dev, qnum);
1760         bool kick = !netdev_xmit_more();
1761         bool use_napi = sq->napi.weight;
1762
1763         /* Free up any pending old buffers before queueing new ones. */
1764         do {
1765                 if (use_napi)
1766                         virtqueue_disable_cb(sq->vq);
1767
1768                 free_old_xmit_skbs(sq, false);
1769
1770         } while (use_napi && kick &&
1771                unlikely(!virtqueue_enable_cb_delayed(sq->vq)));
1772
1773         /* timestamp packet in software */
1774         skb_tx_timestamp(skb);
1775
1776         /* Try to transmit */
1777         err = xmit_skb(sq, skb);
1778
1779         /* This should not happen! */
1780         if (unlikely(err)) {
1781                 dev->stats.tx_fifo_errors++;
1782                 if (net_ratelimit())
1783                         dev_warn(&dev->dev,
1784                                  "Unexpected TXQ (%d) queue failure: %d\n",
1785                                  qnum, err);
1786                 dev->stats.tx_dropped++;
1787                 dev_kfree_skb_any(skb);
1788                 return NETDEV_TX_OK;
1789         }
1790
1791         /* Don't wait up for transmitted skbs to be freed. */
1792         if (!use_napi) {
1793                 skb_orphan(skb);
1794                 nf_reset_ct(skb);
1795         }
1796
1797         /* If running out of space, stop queue to avoid getting packets that we
1798          * are then unable to transmit.
1799          * An alternative would be to force queuing layer to requeue the skb by
1800          * returning NETDEV_TX_BUSY. However, NETDEV_TX_BUSY should not be
1801          * returned in a normal path of operation: it means that driver is not
1802          * maintaining the TX queue stop/start state properly, and causes
1803          * the stack to do a non-trivial amount of useless work.
1804          * Since most packets only take 1 or 2 ring slots, stopping the queue
1805          * early means 16 slots are typically wasted.
1806          */
1807         if (sq->vq->num_free < 2+MAX_SKB_FRAGS) {
1808                 netif_stop_subqueue(dev, qnum);
1809                 if (!use_napi &&
1810                     unlikely(!virtqueue_enable_cb_delayed(sq->vq))) {
1811                         /* More just got used, free them then recheck. */
1812                         free_old_xmit_skbs(sq, false);
1813                         if (sq->vq->num_free >= 2+MAX_SKB_FRAGS) {
1814                                 netif_start_subqueue(dev, qnum);
1815                                 virtqueue_disable_cb(sq->vq);
1816                         }
1817                 }
1818         }
1819
1820         if (kick || netif_xmit_stopped(txq)) {
1821                 if (virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq)) {
1822                         u64_stats_update_begin(&sq->stats.syncp);
1823                         sq->stats.kicks++;
1824                         u64_stats_update_end(&sq->stats.syncp);
1825                 }
1826         }
1827
1828         return NETDEV_TX_OK;
1829 }
1830
1831 /*
1832  * Send command via the control virtqueue and check status.  Commands
1833  * supported by the hypervisor, as indicated by feature bits, should
1834  * never fail unless improperly formatted.
1835  */
1836 static bool virtnet_send_command(struct virtnet_info *vi, u8 class, u8 cmd,
1837                                  struct scatterlist *out)
1838 {
1839         struct scatterlist *sgs[4], hdr, stat;
1840         unsigned out_num = 0, tmp;
1841         int ret;
1842
1843         /* Caller should know better */
1844         BUG_ON(!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ));
1845
1846         vi->ctrl->status = ~0;
1847         vi->ctrl->hdr.class = class;
1848         vi->ctrl->hdr.cmd = cmd;
1849         /* Add header */
1850         sg_init_one(&hdr, &vi->ctrl->hdr, sizeof(vi->ctrl->hdr));
1851         sgs[out_num++] = &hdr;
1852
1853         if (out)
1854                 sgs[out_num++] = out;
1855
1856         /* Add return status. */
1857         sg_init_one(&stat, &vi->ctrl->status, sizeof(vi->ctrl->status));
1858         sgs[out_num] = &stat;
1859
1860         BUG_ON(out_num + 1 > ARRAY_SIZE(sgs));
1861         ret = virtqueue_add_sgs(vi->cvq, sgs, out_num, 1, vi, GFP_ATOMIC);
1862         if (ret < 0) {
1863                 dev_warn(&vi->vdev->dev,
1864                          "Failed to add sgs for command vq: %d\n.", ret);
1865                 return false;
1866         }
1867
1868         if (unlikely(!virtqueue_kick(vi->cvq)))
1869                 return vi->ctrl->status == VIRTIO_NET_OK;
1870
1871         /* Spin for a response, the kick causes an ioport write, trapping
1872          * into the hypervisor, so the request should be handled immediately.
1873          */
1874         while (!virtqueue_get_buf(vi->cvq, &tmp) &&
1875                !virtqueue_is_broken(vi->cvq))
1876                 cpu_relax();
1877
1878         return vi->ctrl->status == VIRTIO_NET_OK;
1879 }
1880
1881 static int virtnet_set_mac_address(struct net_device *dev, void *p)
1882 {
1883         struct virtnet_info *vi = netdev_priv(dev);
1884         struct virtio_device *vdev = vi->vdev;
1885         int ret;
1886         struct sockaddr *addr;
1887         struct scatterlist sg;
1888
1889         if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STANDBY))
1890                 return -EOPNOTSUPP;
1891
1892         addr = kmemdup(p, sizeof(*addr), GFP_KERNEL);
1893         if (!addr)
1894                 return -ENOMEM;
1895
1896         ret = eth_prepare_mac_addr_change(dev, addr);
1897         if (ret)
1898                 goto out;
1899
1900         if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR)) {
1901                 sg_init_one(&sg, addr->sa_data, dev->addr_len);
1902                 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC,
1903                                           VIRTIO_NET_CTRL_MAC_ADDR_SET, &sg)) {
1904                         dev_warn(&vdev->dev,
1905                                  "Failed to set mac address by vq command.\n");
1906                         ret = -EINVAL;
1907                         goto out;
1908                 }
1909         } else if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC) &&
1910                    !virtio_has_feature(vdev, VIRTIO_F_VERSION_1)) {
1911                 unsigned int i;
1912
1913                 /* Naturally, this has an atomicity problem. */
1914                 for (i = 0; i < dev->addr_len; i++)
1915                         virtio_cwrite8(vdev,
1916                                        offsetof(struct virtio_net_config, mac) +
1917                                        i, addr->sa_data[i]);
1918         }
1919
1920         eth_commit_mac_addr_change(dev, p);
1921         ret = 0;
1922
1923 out:
1924         kfree(addr);
1925         return ret;
1926 }
1927
1928 static void virtnet_stats(struct net_device *dev,
1929                           struct rtnl_link_stats64 *tot)
1930 {
1931         struct virtnet_info *vi = netdev_priv(dev);
1932         unsigned int start;
1933         int i;
1934
1935         for (i = 0; i < vi->max_queue_pairs; i++) {
1936                 u64 tpackets, tbytes, terrors, rpackets, rbytes, rdrops;
1937                 struct receive_queue *rq = &vi->rq[i];
1938                 struct send_queue *sq = &vi->sq[i];
1939
1940                 do {
1941                         start = u64_stats_fetch_begin_irq(&sq->stats.syncp);
1942                         tpackets = sq->stats.packets;
1943                         tbytes   = sq->stats.bytes;
1944                         terrors  = sq->stats.tx_timeouts;
1945                 } while (u64_stats_fetch_retry_irq(&sq->stats.syncp, start));
1946
1947                 do {
1948                         start = u64_stats_fetch_begin_irq(&rq->stats.syncp);
1949                         rpackets = rq->stats.packets;
1950                         rbytes   = rq->stats.bytes;
1951                         rdrops   = rq->stats.drops;
1952                 } while (u64_stats_fetch_retry_irq(&rq->stats.syncp, start));
1953
1954                 tot->rx_packets += rpackets;
1955                 tot->tx_packets += tpackets;
1956                 tot->rx_bytes   += rbytes;
1957                 tot->tx_bytes   += tbytes;
1958                 tot->rx_dropped += rdrops;
1959                 tot->tx_errors  += terrors;
1960         }
1961
1962         tot->tx_dropped = dev->stats.tx_dropped;
1963         tot->tx_fifo_errors = dev->stats.tx_fifo_errors;
1964         tot->rx_length_errors = dev->stats.rx_length_errors;
1965         tot->rx_frame_errors = dev->stats.rx_frame_errors;
1966 }
1967
1968 static void virtnet_ack_link_announce(struct virtnet_info *vi)
1969 {
1970         rtnl_lock();
1971         if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_ANNOUNCE,
1972                                   VIRTIO_NET_CTRL_ANNOUNCE_ACK, NULL))
1973                 dev_warn(&vi->dev->dev, "Failed to ack link announce.\n");
1974         rtnl_unlock();
1975 }
1976
1977 static int _virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs)
1978 {
1979         struct scatterlist sg;
1980         struct net_device *dev = vi->dev;
1981
1982         if (!vi->has_cvq || !virtio_has_feature(vi->vdev, VIRTIO_NET_F_MQ))
1983                 return 0;
1984
1985         vi->ctrl->mq.virtqueue_pairs = cpu_to_virtio16(vi->vdev, queue_pairs);
1986         sg_init_one(&sg, &vi->ctrl->mq, sizeof(vi->ctrl->mq));
1987
1988         if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MQ,
1989                                   VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET, &sg)) {
1990                 dev_warn(&dev->dev, "Fail to set num of queue pairs to %d\n",
1991                          queue_pairs);
1992                 return -EINVAL;
1993         } else {
1994                 vi->curr_queue_pairs = queue_pairs;
1995                 /* virtnet_open() will refill when device is going to up. */
1996                 if (dev->flags & IFF_UP)
1997                         schedule_delayed_work(&vi->refill, 0);
1998         }
1999
2000         return 0;
2001 }
2002
2003 static int virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs)
2004 {
2005         int err;
2006
2007         rtnl_lock();
2008         err = _virtnet_set_queues(vi, queue_pairs);
2009         rtnl_unlock();
2010         return err;
2011 }
2012
2013 static int virtnet_close(struct net_device *dev)
2014 {
2015         struct virtnet_info *vi = netdev_priv(dev);
2016         int i;
2017
2018         /* Make sure refill_work doesn't re-enable napi! */
2019         cancel_delayed_work_sync(&vi->refill);
2020
2021         for (i = 0; i < vi->max_queue_pairs; i++) {
2022                 xdp_rxq_info_unreg(&vi->rq[i].xdp_rxq);
2023                 napi_disable(&vi->rq[i].napi);
2024                 virtnet_napi_tx_disable(&vi->sq[i].napi);
2025         }
2026
2027         return 0;
2028 }
2029
2030 static void virtnet_set_rx_mode(struct net_device *dev)
2031 {
2032         struct virtnet_info *vi = netdev_priv(dev);
2033         struct scatterlist sg[2];
2034         struct virtio_net_ctrl_mac *mac_data;
2035         struct netdev_hw_addr *ha;
2036         int uc_count;
2037         int mc_count;
2038         void *buf;
2039         int i;
2040
2041         /* We can't dynamically set ndo_set_rx_mode, so return gracefully */
2042         if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_RX))
2043                 return;
2044
2045         vi->ctrl->promisc = ((dev->flags & IFF_PROMISC) != 0);
2046         vi->ctrl->allmulti = ((dev->flags & IFF_ALLMULTI) != 0);
2047
2048         sg_init_one(sg, &vi->ctrl->promisc, sizeof(vi->ctrl->promisc));
2049
2050         if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,
2051                                   VIRTIO_NET_CTRL_RX_PROMISC, sg))
2052                 dev_warn(&dev->dev, "Failed to %sable promisc mode.\n",
2053                          vi->ctrl->promisc ? "en" : "dis");
2054
2055         sg_init_one(sg, &vi->ctrl->allmulti, sizeof(vi->ctrl->allmulti));
2056
2057         if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,
2058                                   VIRTIO_NET_CTRL_RX_ALLMULTI, sg))
2059                 dev_warn(&dev->dev, "Failed to %sable allmulti mode.\n",
2060                          vi->ctrl->allmulti ? "en" : "dis");
2061
2062         uc_count = netdev_uc_count(dev);
2063         mc_count = netdev_mc_count(dev);
2064         /* MAC filter - use one buffer for both lists */
2065         buf = kzalloc(((uc_count + mc_count) * ETH_ALEN) +
2066                       (2 * sizeof(mac_data->entries)), GFP_ATOMIC);
2067         mac_data = buf;
2068         if (!buf)
2069                 return;
2070
2071         sg_init_table(sg, 2);
2072
2073         /* Store the unicast list and count in the front of the buffer */
2074         mac_data->entries = cpu_to_virtio32(vi->vdev, uc_count);
2075         i = 0;
2076         netdev_for_each_uc_addr(ha, dev)
2077                 memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN);
2078
2079         sg_set_buf(&sg[0], mac_data,
2080                    sizeof(mac_data->entries) + (uc_count * ETH_ALEN));
2081
2082         /* multicast list and count fill the end */
2083         mac_data = (void *)&mac_data->macs[uc_count][0];
2084
2085         mac_data->entries = cpu_to_virtio32(vi->vdev, mc_count);
2086         i = 0;
2087         netdev_for_each_mc_addr(ha, dev)
2088                 memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN);
2089
2090         sg_set_buf(&sg[1], mac_data,
2091                    sizeof(mac_data->entries) + (mc_count * ETH_ALEN));
2092
2093         if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC,
2094                                   VIRTIO_NET_CTRL_MAC_TABLE_SET, sg))
2095                 dev_warn(&dev->dev, "Failed to set MAC filter table.\n");
2096
2097         kfree(buf);
2098 }
2099
2100 static int virtnet_vlan_rx_add_vid(struct net_device *dev,
2101                                    __be16 proto, u16 vid)
2102 {
2103         struct virtnet_info *vi = netdev_priv(dev);
2104         struct scatterlist sg;
2105
2106         vi->ctrl->vid = cpu_to_virtio16(vi->vdev, vid);
2107         sg_init_one(&sg, &vi->ctrl->vid, sizeof(vi->ctrl->vid));
2108
2109         if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,
2110                                   VIRTIO_NET_CTRL_VLAN_ADD, &sg))
2111                 dev_warn(&dev->dev, "Failed to add VLAN ID %d.\n", vid);
2112         return 0;
2113 }
2114
2115 static int virtnet_vlan_rx_kill_vid(struct net_device *dev,
2116                                     __be16 proto, u16 vid)
2117 {
2118         struct virtnet_info *vi = netdev_priv(dev);
2119         struct scatterlist sg;
2120
2121         vi->ctrl->vid = cpu_to_virtio16(vi->vdev, vid);
2122         sg_init_one(&sg, &vi->ctrl->vid, sizeof(vi->ctrl->vid));
2123
2124         if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,
2125                                   VIRTIO_NET_CTRL_VLAN_DEL, &sg))
2126                 dev_warn(&dev->dev, "Failed to kill VLAN ID %d.\n", vid);
2127         return 0;
2128 }
2129
2130 static void virtnet_clean_affinity(struct virtnet_info *vi)
2131 {
2132         int i;
2133
2134         if (vi->affinity_hint_set) {
2135                 for (i = 0; i < vi->max_queue_pairs; i++) {
2136                         virtqueue_set_affinity(vi->rq[i].vq, NULL);
2137                         virtqueue_set_affinity(vi->sq[i].vq, NULL);
2138                 }
2139
2140                 vi->affinity_hint_set = false;
2141         }
2142 }
2143
2144 static void virtnet_set_affinity(struct virtnet_info *vi)
2145 {
2146         cpumask_var_t mask;
2147         int stragglers;
2148         int group_size;
2149         int i, j, cpu;
2150         int num_cpu;
2151         int stride;
2152
2153         if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) {
2154                 virtnet_clean_affinity(vi);
2155                 return;
2156         }
2157
2158         num_cpu = num_online_cpus();
2159         stride = max_t(int, num_cpu / vi->curr_queue_pairs, 1);
2160         stragglers = num_cpu >= vi->curr_queue_pairs ?
2161                         num_cpu % vi->curr_queue_pairs :
2162                         0;
2163         cpu = cpumask_first(cpu_online_mask);
2164
2165         for (i = 0; i < vi->curr_queue_pairs; i++) {
2166                 group_size = stride + (i < stragglers ? 1 : 0);
2167
2168                 for (j = 0; j < group_size; j++) {
2169                         cpumask_set_cpu(cpu, mask);
2170                         cpu = cpumask_next_wrap(cpu, cpu_online_mask,
2171                                                 nr_cpu_ids, false);
2172                 }
2173                 virtqueue_set_affinity(vi->rq[i].vq, mask);
2174                 virtqueue_set_affinity(vi->sq[i].vq, mask);
2175                 __netif_set_xps_queue(vi->dev, cpumask_bits(mask), i, XPS_CPUS);
2176                 cpumask_clear(mask);
2177         }
2178
2179         vi->affinity_hint_set = true;
2180         free_cpumask_var(mask);
2181 }
2182
2183 static int virtnet_cpu_online(unsigned int cpu, struct hlist_node *node)
2184 {
2185         struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info,
2186                                                    node);
2187         virtnet_set_affinity(vi);
2188         return 0;
2189 }
2190
2191 static int virtnet_cpu_dead(unsigned int cpu, struct hlist_node *node)
2192 {
2193         struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info,
2194                                                    node_dead);
2195         virtnet_set_affinity(vi);
2196         return 0;
2197 }
2198
2199 static int virtnet_cpu_down_prep(unsigned int cpu, struct hlist_node *node)
2200 {
2201         struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info,
2202                                                    node);
2203
2204         virtnet_clean_affinity(vi);
2205         return 0;
2206 }
2207
2208 static enum cpuhp_state virtionet_online;
2209
2210 static int virtnet_cpu_notif_add(struct virtnet_info *vi)
2211 {
2212         int ret;
2213
2214         ret = cpuhp_state_add_instance_nocalls(virtionet_online, &vi->node);
2215         if (ret)
2216                 return ret;
2217         ret = cpuhp_state_add_instance_nocalls(CPUHP_VIRT_NET_DEAD,
2218                                                &vi->node_dead);
2219         if (!ret)
2220                 return ret;
2221         cpuhp_state_remove_instance_nocalls(virtionet_online, &vi->node);
2222         return ret;
2223 }
2224
2225 static void virtnet_cpu_notif_remove(struct virtnet_info *vi)
2226 {
2227         cpuhp_state_remove_instance_nocalls(virtionet_online, &vi->node);
2228         cpuhp_state_remove_instance_nocalls(CPUHP_VIRT_NET_DEAD,
2229                                             &vi->node_dead);
2230 }
2231
2232 static void virtnet_get_ringparam(struct net_device *dev,
2233                                   struct ethtool_ringparam *ring,
2234                                   struct kernel_ethtool_ringparam *kernel_ring,
2235                                   struct netlink_ext_ack *extack)
2236 {
2237         struct virtnet_info *vi = netdev_priv(dev);
2238
2239         ring->rx_max_pending = virtqueue_get_vring_size(vi->rq[0].vq);
2240         ring->tx_max_pending = virtqueue_get_vring_size(vi->sq[0].vq);
2241         ring->rx_pending = ring->rx_max_pending;
2242         ring->tx_pending = ring->tx_max_pending;
2243 }
2244
2245 static bool virtnet_commit_rss_command(struct virtnet_info *vi)
2246 {
2247         struct net_device *dev = vi->dev;
2248         struct scatterlist sgs[4];
2249         unsigned int sg_buf_size;
2250
2251         /* prepare sgs */
2252         sg_init_table(sgs, 4);
2253
2254         sg_buf_size = offsetof(struct virtio_net_ctrl_rss, indirection_table);
2255         sg_set_buf(&sgs[0], &vi->ctrl->rss, sg_buf_size);
2256
2257         sg_buf_size = sizeof(uint16_t) * (vi->ctrl->rss.indirection_table_mask + 1);
2258         sg_set_buf(&sgs[1], vi->ctrl->rss.indirection_table, sg_buf_size);
2259
2260         sg_buf_size = offsetof(struct virtio_net_ctrl_rss, key)
2261                         - offsetof(struct virtio_net_ctrl_rss, max_tx_vq);
2262         sg_set_buf(&sgs[2], &vi->ctrl->rss.max_tx_vq, sg_buf_size);
2263
2264         sg_buf_size = vi->rss_key_size;
2265         sg_set_buf(&sgs[3], vi->ctrl->rss.key, sg_buf_size);
2266
2267         if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MQ,
2268                                   vi->has_rss ? VIRTIO_NET_CTRL_MQ_RSS_CONFIG
2269                                   : VIRTIO_NET_CTRL_MQ_HASH_CONFIG, sgs)) {
2270                 dev_warn(&dev->dev, "VIRTIONET issue with committing RSS sgs\n");
2271                 return false;
2272         }
2273         return true;
2274 }
2275
2276 static void virtnet_init_default_rss(struct virtnet_info *vi)
2277 {
2278         u32 indir_val = 0;
2279         int i = 0;
2280
2281         vi->ctrl->rss.hash_types = vi->rss_hash_types_supported;
2282         vi->rss_hash_types_saved = vi->rss_hash_types_supported;
2283         vi->ctrl->rss.indirection_table_mask = vi->rss_indir_table_size
2284                                                 ? vi->rss_indir_table_size - 1 : 0;
2285         vi->ctrl->rss.unclassified_queue = 0;
2286
2287         for (; i < vi->rss_indir_table_size; ++i) {
2288                 indir_val = ethtool_rxfh_indir_default(i, vi->curr_queue_pairs);
2289                 vi->ctrl->rss.indirection_table[i] = indir_val;
2290         }
2291
2292         vi->ctrl->rss.max_tx_vq = vi->curr_queue_pairs;
2293         vi->ctrl->rss.hash_key_length = vi->rss_key_size;
2294
2295         netdev_rss_key_fill(vi->ctrl->rss.key, vi->rss_key_size);
2296 }
2297
2298 static void virtnet_get_hashflow(const struct virtnet_info *vi, struct ethtool_rxnfc *info)
2299 {
2300         info->data = 0;
2301         switch (info->flow_type) {
2302         case TCP_V4_FLOW:
2303                 if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_TCPv4) {
2304                         info->data = RXH_IP_SRC | RXH_IP_DST |
2305                                                  RXH_L4_B_0_1 | RXH_L4_B_2_3;
2306                 } else if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv4) {
2307                         info->data = RXH_IP_SRC | RXH_IP_DST;
2308                 }
2309                 break;
2310         case TCP_V6_FLOW:
2311                 if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_TCPv6) {
2312                         info->data = RXH_IP_SRC | RXH_IP_DST |
2313                                                  RXH_L4_B_0_1 | RXH_L4_B_2_3;
2314                 } else if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv6) {
2315                         info->data = RXH_IP_SRC | RXH_IP_DST;
2316                 }
2317                 break;
2318         case UDP_V4_FLOW:
2319                 if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_UDPv4) {
2320                         info->data = RXH_IP_SRC | RXH_IP_DST |
2321                                                  RXH_L4_B_0_1 | RXH_L4_B_2_3;
2322                 } else if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv4) {
2323                         info->data = RXH_IP_SRC | RXH_IP_DST;
2324                 }
2325                 break;
2326         case UDP_V6_FLOW:
2327                 if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_UDPv6) {
2328                         info->data = RXH_IP_SRC | RXH_IP_DST |
2329                                                  RXH_L4_B_0_1 | RXH_L4_B_2_3;
2330                 } else if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv6) {
2331                         info->data = RXH_IP_SRC | RXH_IP_DST;
2332                 }
2333                 break;
2334         case IPV4_FLOW:
2335                 if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv4)
2336                         info->data = RXH_IP_SRC | RXH_IP_DST;
2337
2338                 break;
2339         case IPV6_FLOW:
2340                 if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv6)
2341                         info->data = RXH_IP_SRC | RXH_IP_DST;
2342
2343                 break;
2344         default:
2345                 info->data = 0;
2346                 break;
2347         }
2348 }
2349
2350 static bool virtnet_set_hashflow(struct virtnet_info *vi, struct ethtool_rxnfc *info)
2351 {
2352         u32 new_hashtypes = vi->rss_hash_types_saved;
2353         bool is_disable = info->data & RXH_DISCARD;
2354         bool is_l4 = info->data == (RXH_IP_SRC | RXH_IP_DST | RXH_L4_B_0_1 | RXH_L4_B_2_3);
2355
2356         /* supports only 'sd', 'sdfn' and 'r' */
2357         if (!((info->data == (RXH_IP_SRC | RXH_IP_DST)) | is_l4 | is_disable))
2358                 return false;
2359
2360         switch (info->flow_type) {
2361         case TCP_V4_FLOW:
2362                 new_hashtypes &= ~(VIRTIO_NET_RSS_HASH_TYPE_IPv4 | VIRTIO_NET_RSS_HASH_TYPE_TCPv4);
2363                 if (!is_disable)
2364                         new_hashtypes |= VIRTIO_NET_RSS_HASH_TYPE_IPv4
2365                                 | (is_l4 ? VIRTIO_NET_RSS_HASH_TYPE_TCPv4 : 0);
2366                 break;
2367         case UDP_V4_FLOW:
2368                 new_hashtypes &= ~(VIRTIO_NET_RSS_HASH_TYPE_IPv4 | VIRTIO_NET_RSS_HASH_TYPE_UDPv4);
2369                 if (!is_disable)
2370                         new_hashtypes |= VIRTIO_NET_RSS_HASH_TYPE_IPv4
2371                                 | (is_l4 ? VIRTIO_NET_RSS_HASH_TYPE_UDPv4 : 0);
2372                 break;
2373         case IPV4_FLOW:
2374                 new_hashtypes &= ~VIRTIO_NET_RSS_HASH_TYPE_IPv4;
2375                 if (!is_disable)
2376                         new_hashtypes = VIRTIO_NET_RSS_HASH_TYPE_IPv4;
2377                 break;
2378         case TCP_V6_FLOW:
2379                 new_hashtypes &= ~(VIRTIO_NET_RSS_HASH_TYPE_IPv6 | VIRTIO_NET_RSS_HASH_TYPE_TCPv6);
2380                 if (!is_disable)
2381                         new_hashtypes |= VIRTIO_NET_RSS_HASH_TYPE_IPv6
2382                                 | (is_l4 ? VIRTIO_NET_RSS_HASH_TYPE_TCPv6 : 0);
2383                 break;
2384         case UDP_V6_FLOW:
2385                 new_hashtypes &= ~(VIRTIO_NET_RSS_HASH_TYPE_IPv6 | VIRTIO_NET_RSS_HASH_TYPE_UDPv6);
2386                 if (!is_disable)
2387                         new_hashtypes |= VIRTIO_NET_RSS_HASH_TYPE_IPv6
2388                                 | (is_l4 ? VIRTIO_NET_RSS_HASH_TYPE_UDPv6 : 0);
2389                 break;
2390         case IPV6_FLOW:
2391                 new_hashtypes &= ~VIRTIO_NET_RSS_HASH_TYPE_IPv6;
2392                 if (!is_disable)
2393                         new_hashtypes = VIRTIO_NET_RSS_HASH_TYPE_IPv6;
2394                 break;
2395         default:
2396                 /* unsupported flow */
2397                 return false;
2398         }
2399
2400         /* if unsupported hashtype was set */
2401         if (new_hashtypes != (new_hashtypes & vi->rss_hash_types_supported))
2402                 return false;
2403
2404         if (new_hashtypes != vi->rss_hash_types_saved) {
2405                 vi->rss_hash_types_saved = new_hashtypes;
2406                 vi->ctrl->rss.hash_types = vi->rss_hash_types_saved;
2407                 if (vi->dev->features & NETIF_F_RXHASH)
2408                         return virtnet_commit_rss_command(vi);
2409         }
2410
2411         return true;
2412 }
2413
2414 static void virtnet_get_drvinfo(struct net_device *dev,
2415                                 struct ethtool_drvinfo *info)
2416 {
2417         struct virtnet_info *vi = netdev_priv(dev);
2418         struct virtio_device *vdev = vi->vdev;
2419
2420         strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
2421         strlcpy(info->version, VIRTNET_DRIVER_VERSION, sizeof(info->version));
2422         strlcpy(info->bus_info, virtio_bus_name(vdev), sizeof(info->bus_info));
2423
2424 }
2425
2426 /* TODO: Eliminate OOO packets during switching */
2427 static int virtnet_set_channels(struct net_device *dev,
2428                                 struct ethtool_channels *channels)
2429 {
2430         struct virtnet_info *vi = netdev_priv(dev);
2431         u16 queue_pairs = channels->combined_count;
2432         int err;
2433
2434         /* We don't support separate rx/tx channels.
2435          * We don't allow setting 'other' channels.
2436          */
2437         if (channels->rx_count || channels->tx_count || channels->other_count)
2438                 return -EINVAL;
2439
2440         if (queue_pairs > vi->max_queue_pairs || queue_pairs == 0)
2441                 return -EINVAL;
2442
2443         /* For now we don't support modifying channels while XDP is loaded
2444          * also when XDP is loaded all RX queues have XDP programs so we only
2445          * need to check a single RX queue.
2446          */
2447         if (vi->rq[0].xdp_prog)
2448                 return -EINVAL;
2449
2450         cpus_read_lock();
2451         err = _virtnet_set_queues(vi, queue_pairs);
2452         if (err) {
2453                 cpus_read_unlock();
2454                 goto err;
2455         }
2456         virtnet_set_affinity(vi);
2457         cpus_read_unlock();
2458
2459         netif_set_real_num_tx_queues(dev, queue_pairs);
2460         netif_set_real_num_rx_queues(dev, queue_pairs);
2461  err:
2462         return err;
2463 }
2464
2465 static void virtnet_get_strings(struct net_device *dev, u32 stringset, u8 *data)
2466 {
2467         struct virtnet_info *vi = netdev_priv(dev);
2468         unsigned int i, j;
2469         u8 *p = data;
2470
2471         switch (stringset) {
2472         case ETH_SS_STATS:
2473                 for (i = 0; i < vi->curr_queue_pairs; i++) {
2474                         for (j = 0; j < VIRTNET_RQ_STATS_LEN; j++)
2475                                 ethtool_sprintf(&p, "rx_queue_%u_%s", i,
2476                                                 virtnet_rq_stats_desc[j].desc);
2477                 }
2478
2479                 for (i = 0; i < vi->curr_queue_pairs; i++) {
2480                         for (j = 0; j < VIRTNET_SQ_STATS_LEN; j++)
2481                                 ethtool_sprintf(&p, "tx_queue_%u_%s", i,
2482                                                 virtnet_sq_stats_desc[j].desc);
2483                 }
2484                 break;
2485         }
2486 }
2487
2488 static int virtnet_get_sset_count(struct net_device *dev, int sset)
2489 {
2490         struct virtnet_info *vi = netdev_priv(dev);
2491
2492         switch (sset) {
2493         case ETH_SS_STATS:
2494                 return vi->curr_queue_pairs * (VIRTNET_RQ_STATS_LEN +
2495                                                VIRTNET_SQ_STATS_LEN);
2496         default:
2497                 return -EOPNOTSUPP;
2498         }
2499 }
2500
2501 static void virtnet_get_ethtool_stats(struct net_device *dev,
2502                                       struct ethtool_stats *stats, u64 *data)
2503 {
2504         struct virtnet_info *vi = netdev_priv(dev);
2505         unsigned int idx = 0, start, i, j;
2506         const u8 *stats_base;
2507         size_t offset;
2508
2509         for (i = 0; i < vi->curr_queue_pairs; i++) {
2510                 struct receive_queue *rq = &vi->rq[i];
2511
2512                 stats_base = (u8 *)&rq->stats;
2513                 do {
2514                         start = u64_stats_fetch_begin_irq(&rq->stats.syncp);
2515                         for (j = 0; j < VIRTNET_RQ_STATS_LEN; j++) {
2516                                 offset = virtnet_rq_stats_desc[j].offset;
2517                                 data[idx + j] = *(u64 *)(stats_base + offset);
2518                         }
2519                 } while (u64_stats_fetch_retry_irq(&rq->stats.syncp, start));
2520                 idx += VIRTNET_RQ_STATS_LEN;
2521         }
2522
2523         for (i = 0; i < vi->curr_queue_pairs; i++) {
2524                 struct send_queue *sq = &vi->sq[i];
2525
2526                 stats_base = (u8 *)&sq->stats;
2527                 do {
2528                         start = u64_stats_fetch_begin_irq(&sq->stats.syncp);
2529                         for (j = 0; j < VIRTNET_SQ_STATS_LEN; j++) {
2530                                 offset = virtnet_sq_stats_desc[j].offset;
2531                                 data[idx + j] = *(u64 *)(stats_base + offset);
2532                         }
2533                 } while (u64_stats_fetch_retry_irq(&sq->stats.syncp, start));
2534                 idx += VIRTNET_SQ_STATS_LEN;
2535         }
2536 }
2537
2538 static void virtnet_get_channels(struct net_device *dev,
2539                                  struct ethtool_channels *channels)
2540 {
2541         struct virtnet_info *vi = netdev_priv(dev);
2542
2543         channels->combined_count = vi->curr_queue_pairs;
2544         channels->max_combined = vi->max_queue_pairs;
2545         channels->max_other = 0;
2546         channels->rx_count = 0;
2547         channels->tx_count = 0;
2548         channels->other_count = 0;
2549 }
2550
2551 static int virtnet_set_link_ksettings(struct net_device *dev,
2552                                       const struct ethtool_link_ksettings *cmd)
2553 {
2554         struct virtnet_info *vi = netdev_priv(dev);
2555
2556         return ethtool_virtdev_set_link_ksettings(dev, cmd,
2557                                                   &vi->speed, &vi->duplex);
2558 }
2559
2560 static int virtnet_get_link_ksettings(struct net_device *dev,
2561                                       struct ethtool_link_ksettings *cmd)
2562 {
2563         struct virtnet_info *vi = netdev_priv(dev);
2564
2565         cmd->base.speed = vi->speed;
2566         cmd->base.duplex = vi->duplex;
2567         cmd->base.port = PORT_OTHER;
2568
2569         return 0;
2570 }
2571
2572 static int virtnet_set_coalesce(struct net_device *dev,
2573                                 struct ethtool_coalesce *ec,
2574                                 struct kernel_ethtool_coalesce *kernel_coal,
2575                                 struct netlink_ext_ack *extack)
2576 {
2577         struct virtnet_info *vi = netdev_priv(dev);
2578         int i, napi_weight;
2579
2580         if (ec->tx_max_coalesced_frames > 1 ||
2581             ec->rx_max_coalesced_frames != 1)
2582                 return -EINVAL;
2583
2584         napi_weight = ec->tx_max_coalesced_frames ? NAPI_POLL_WEIGHT : 0;
2585         if (napi_weight ^ vi->sq[0].napi.weight) {
2586                 if (dev->flags & IFF_UP)
2587                         return -EBUSY;
2588                 for (i = 0; i < vi->max_queue_pairs; i++)
2589                         vi->sq[i].napi.weight = napi_weight;
2590         }
2591
2592         return 0;
2593 }
2594
2595 static int virtnet_get_coalesce(struct net_device *dev,
2596                                 struct ethtool_coalesce *ec,
2597                                 struct kernel_ethtool_coalesce *kernel_coal,
2598                                 struct netlink_ext_ack *extack)
2599 {
2600         struct ethtool_coalesce ec_default = {
2601                 .cmd = ETHTOOL_GCOALESCE,
2602                 .rx_max_coalesced_frames = 1,
2603         };
2604         struct virtnet_info *vi = netdev_priv(dev);
2605
2606         memcpy(ec, &ec_default, sizeof(ec_default));
2607
2608         if (vi->sq[0].napi.weight)
2609                 ec->tx_max_coalesced_frames = 1;
2610
2611         return 0;
2612 }
2613
2614 static void virtnet_init_settings(struct net_device *dev)
2615 {
2616         struct virtnet_info *vi = netdev_priv(dev);
2617
2618         vi->speed = SPEED_UNKNOWN;
2619         vi->duplex = DUPLEX_UNKNOWN;
2620 }
2621
2622 static void virtnet_update_settings(struct virtnet_info *vi)
2623 {
2624         u32 speed;
2625         u8 duplex;
2626
2627         if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_SPEED_DUPLEX))
2628                 return;
2629
2630         virtio_cread_le(vi->vdev, struct virtio_net_config, speed, &speed);
2631
2632         if (ethtool_validate_speed(speed))
2633                 vi->speed = speed;
2634
2635         virtio_cread_le(vi->vdev, struct virtio_net_config, duplex, &duplex);
2636
2637         if (ethtool_validate_duplex(duplex))
2638                 vi->duplex = duplex;
2639 }
2640
2641 static u32 virtnet_get_rxfh_key_size(struct net_device *dev)
2642 {
2643         return ((struct virtnet_info *)netdev_priv(dev))->rss_key_size;
2644 }
2645
2646 static u32 virtnet_get_rxfh_indir_size(struct net_device *dev)
2647 {
2648         return ((struct virtnet_info *)netdev_priv(dev))->rss_indir_table_size;
2649 }
2650
2651 static int virtnet_get_rxfh(struct net_device *dev, u32 *indir, u8 *key, u8 *hfunc)
2652 {
2653         struct virtnet_info *vi = netdev_priv(dev);
2654         int i;
2655
2656         if (indir) {
2657                 for (i = 0; i < vi->rss_indir_table_size; ++i)
2658                         indir[i] = vi->ctrl->rss.indirection_table[i];
2659         }
2660
2661         if (key)
2662                 memcpy(key, vi->ctrl->rss.key, vi->rss_key_size);
2663
2664         if (hfunc)
2665                 *hfunc = ETH_RSS_HASH_TOP;
2666
2667         return 0;
2668 }
2669
2670 static int virtnet_set_rxfh(struct net_device *dev, const u32 *indir, const u8 *key, const u8 hfunc)
2671 {
2672         struct virtnet_info *vi = netdev_priv(dev);
2673         int i;
2674
2675         if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP)
2676                 return -EOPNOTSUPP;
2677
2678         if (indir) {
2679                 for (i = 0; i < vi->rss_indir_table_size; ++i)
2680                         vi->ctrl->rss.indirection_table[i] = indir[i];
2681         }
2682         if (key)
2683                 memcpy(vi->ctrl->rss.key, key, vi->rss_key_size);
2684
2685         virtnet_commit_rss_command(vi);
2686
2687         return 0;
2688 }
2689
2690 static int virtnet_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info, u32 *rule_locs)
2691 {
2692         struct virtnet_info *vi = netdev_priv(dev);
2693         int rc = 0;
2694
2695         switch (info->cmd) {
2696         case ETHTOOL_GRXRINGS:
2697                 info->data = vi->curr_queue_pairs;
2698                 break;
2699         case ETHTOOL_GRXFH:
2700                 virtnet_get_hashflow(vi, info);
2701                 break;
2702         default:
2703                 rc = -EOPNOTSUPP;
2704         }
2705
2706         return rc;
2707 }
2708
2709 static int virtnet_set_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info)
2710 {
2711         struct virtnet_info *vi = netdev_priv(dev);
2712         int rc = 0;
2713
2714         switch (info->cmd) {
2715         case ETHTOOL_SRXFH:
2716                 if (!virtnet_set_hashflow(vi, info))
2717                         rc = -EINVAL;
2718
2719                 break;
2720         default:
2721                 rc = -EOPNOTSUPP;
2722         }
2723
2724         return rc;
2725 }
2726
2727 static const struct ethtool_ops virtnet_ethtool_ops = {
2728         .supported_coalesce_params = ETHTOOL_COALESCE_MAX_FRAMES,
2729         .get_drvinfo = virtnet_get_drvinfo,
2730         .get_link = ethtool_op_get_link,
2731         .get_ringparam = virtnet_get_ringparam,
2732         .get_strings = virtnet_get_strings,
2733         .get_sset_count = virtnet_get_sset_count,
2734         .get_ethtool_stats = virtnet_get_ethtool_stats,
2735         .set_channels = virtnet_set_channels,
2736         .get_channels = virtnet_get_channels,
2737         .get_ts_info = ethtool_op_get_ts_info,
2738         .get_link_ksettings = virtnet_get_link_ksettings,
2739         .set_link_ksettings = virtnet_set_link_ksettings,
2740         .set_coalesce = virtnet_set_coalesce,
2741         .get_coalesce = virtnet_get_coalesce,
2742         .get_rxfh_key_size = virtnet_get_rxfh_key_size,
2743         .get_rxfh_indir_size = virtnet_get_rxfh_indir_size,
2744         .get_rxfh = virtnet_get_rxfh,
2745         .set_rxfh = virtnet_set_rxfh,
2746         .get_rxnfc = virtnet_get_rxnfc,
2747         .set_rxnfc = virtnet_set_rxnfc,
2748 };
2749
2750 static void virtnet_freeze_down(struct virtio_device *vdev)
2751 {
2752         struct virtnet_info *vi = vdev->priv;
2753         int i;
2754
2755         /* Make sure no work handler is accessing the device */
2756         flush_work(&vi->config_work);
2757
2758         netif_tx_lock_bh(vi->dev);
2759         netif_device_detach(vi->dev);
2760         netif_tx_unlock_bh(vi->dev);
2761         cancel_delayed_work_sync(&vi->refill);
2762
2763         if (netif_running(vi->dev)) {
2764                 for (i = 0; i < vi->max_queue_pairs; i++) {
2765                         napi_disable(&vi->rq[i].napi);
2766                         virtnet_napi_tx_disable(&vi->sq[i].napi);
2767                 }
2768         }
2769 }
2770
2771 static int init_vqs(struct virtnet_info *vi);
2772
2773 static int virtnet_restore_up(struct virtio_device *vdev)
2774 {
2775         struct virtnet_info *vi = vdev->priv;
2776         int err, i;
2777
2778         err = init_vqs(vi);
2779         if (err)
2780                 return err;
2781
2782         virtio_device_ready(vdev);
2783
2784         if (netif_running(vi->dev)) {
2785                 for (i = 0; i < vi->curr_queue_pairs; i++)
2786                         if (!try_fill_recv(vi, &vi->rq[i], GFP_KERNEL))
2787                                 schedule_delayed_work(&vi->refill, 0);
2788
2789                 for (i = 0; i < vi->max_queue_pairs; i++) {
2790                         virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi);
2791                         virtnet_napi_tx_enable(vi, vi->sq[i].vq,
2792                                                &vi->sq[i].napi);
2793                 }
2794         }
2795
2796         netif_tx_lock_bh(vi->dev);
2797         netif_device_attach(vi->dev);
2798         netif_tx_unlock_bh(vi->dev);
2799         return err;
2800 }
2801
2802 static int virtnet_set_guest_offloads(struct virtnet_info *vi, u64 offloads)
2803 {
2804         struct scatterlist sg;
2805         vi->ctrl->offloads = cpu_to_virtio64(vi->vdev, offloads);
2806
2807         sg_init_one(&sg, &vi->ctrl->offloads, sizeof(vi->ctrl->offloads));
2808
2809         if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_GUEST_OFFLOADS,
2810                                   VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET, &sg)) {
2811                 dev_warn(&vi->dev->dev, "Fail to set guest offload.\n");
2812                 return -EINVAL;
2813         }
2814
2815         return 0;
2816 }
2817
2818 static int virtnet_clear_guest_offloads(struct virtnet_info *vi)
2819 {
2820         u64 offloads = 0;
2821
2822         if (!vi->guest_offloads)
2823                 return 0;
2824
2825         return virtnet_set_guest_offloads(vi, offloads);
2826 }
2827
2828 static int virtnet_restore_guest_offloads(struct virtnet_info *vi)
2829 {
2830         u64 offloads = vi->guest_offloads;
2831
2832         if (!vi->guest_offloads)
2833                 return 0;
2834
2835         return virtnet_set_guest_offloads(vi, offloads);
2836 }
2837
2838 static int virtnet_xdp_set(struct net_device *dev, struct bpf_prog *prog,
2839                            struct netlink_ext_ack *extack)
2840 {
2841         unsigned long int max_sz = PAGE_SIZE - sizeof(struct padded_vnet_hdr);
2842         struct virtnet_info *vi = netdev_priv(dev);
2843         struct bpf_prog *old_prog;
2844         u16 xdp_qp = 0, curr_qp;
2845         int i, err;
2846
2847         if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)
2848             && (virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO4) ||
2849                 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO6) ||
2850                 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_ECN) ||
2851                 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_UFO) ||
2852                 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_CSUM))) {
2853                 NL_SET_ERR_MSG_MOD(extack, "Can't set XDP while host is implementing GRO_HW/CSUM, disable GRO_HW/CSUM first");
2854                 return -EOPNOTSUPP;
2855         }
2856
2857         if (vi->mergeable_rx_bufs && !vi->any_header_sg) {
2858                 NL_SET_ERR_MSG_MOD(extack, "XDP expects header/data in single page, any_header_sg required");
2859                 return -EINVAL;
2860         }
2861
2862         if (dev->mtu > max_sz) {
2863                 NL_SET_ERR_MSG_MOD(extack, "MTU too large to enable XDP");
2864                 netdev_warn(dev, "XDP requires MTU less than %lu\n", max_sz);
2865                 return -EINVAL;
2866         }
2867
2868         curr_qp = vi->curr_queue_pairs - vi->xdp_queue_pairs;
2869         if (prog)
2870                 xdp_qp = nr_cpu_ids;
2871
2872         /* XDP requires extra queues for XDP_TX */
2873         if (curr_qp + xdp_qp > vi->max_queue_pairs) {
2874                 netdev_warn_once(dev, "XDP request %i queues but max is %i. XDP_TX and XDP_REDIRECT will operate in a slower locked tx mode.\n",
2875                                  curr_qp + xdp_qp, vi->max_queue_pairs);
2876                 xdp_qp = 0;
2877         }
2878
2879         old_prog = rtnl_dereference(vi->rq[0].xdp_prog);
2880         if (!prog && !old_prog)
2881                 return 0;
2882
2883         if (prog)
2884                 bpf_prog_add(prog, vi->max_queue_pairs - 1);
2885
2886         /* Make sure NAPI is not using any XDP TX queues for RX. */
2887         if (netif_running(dev)) {
2888                 for (i = 0; i < vi->max_queue_pairs; i++) {
2889                         napi_disable(&vi->rq[i].napi);
2890                         virtnet_napi_tx_disable(&vi->sq[i].napi);
2891                 }
2892         }
2893
2894         if (!prog) {
2895                 for (i = 0; i < vi->max_queue_pairs; i++) {
2896                         rcu_assign_pointer(vi->rq[i].xdp_prog, prog);
2897                         if (i == 0)
2898                                 virtnet_restore_guest_offloads(vi);
2899                 }
2900                 synchronize_net();
2901         }
2902
2903         err = _virtnet_set_queues(vi, curr_qp + xdp_qp);
2904         if (err)
2905                 goto err;
2906         netif_set_real_num_rx_queues(dev, curr_qp + xdp_qp);
2907         vi->xdp_queue_pairs = xdp_qp;
2908
2909         if (prog) {
2910                 vi->xdp_enabled = true;
2911                 for (i = 0; i < vi->max_queue_pairs; i++) {
2912                         rcu_assign_pointer(vi->rq[i].xdp_prog, prog);
2913                         if (i == 0 && !old_prog)
2914                                 virtnet_clear_guest_offloads(vi);
2915                 }
2916         } else {
2917                 vi->xdp_enabled = false;
2918         }
2919
2920         for (i = 0; i < vi->max_queue_pairs; i++) {
2921                 if (old_prog)
2922                         bpf_prog_put(old_prog);
2923                 if (netif_running(dev)) {
2924                         virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi);
2925                         virtnet_napi_tx_enable(vi, vi->sq[i].vq,
2926                                                &vi->sq[i].napi);
2927                 }
2928         }
2929
2930         return 0;
2931
2932 err:
2933         if (!prog) {
2934                 virtnet_clear_guest_offloads(vi);
2935                 for (i = 0; i < vi->max_queue_pairs; i++)
2936                         rcu_assign_pointer(vi->rq[i].xdp_prog, old_prog);
2937         }
2938
2939         if (netif_running(dev)) {
2940                 for (i = 0; i < vi->max_queue_pairs; i++) {
2941                         virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi);
2942                         virtnet_napi_tx_enable(vi, vi->sq[i].vq,
2943                                                &vi->sq[i].napi);
2944                 }
2945         }
2946         if (prog)
2947                 bpf_prog_sub(prog, vi->max_queue_pairs - 1);
2948         return err;
2949 }
2950
2951 static int virtnet_xdp(struct net_device *dev, struct netdev_bpf *xdp)
2952 {
2953         switch (xdp->command) {
2954         case XDP_SETUP_PROG:
2955                 return virtnet_xdp_set(dev, xdp->prog, xdp->extack);
2956         default:
2957                 return -EINVAL;
2958         }
2959 }
2960
2961 static int virtnet_get_phys_port_name(struct net_device *dev, char *buf,
2962                                       size_t len)
2963 {
2964         struct virtnet_info *vi = netdev_priv(dev);
2965         int ret;
2966
2967         if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_STANDBY))
2968                 return -EOPNOTSUPP;
2969
2970         ret = snprintf(buf, len, "sby");
2971         if (ret >= len)
2972                 return -EOPNOTSUPP;
2973
2974         return 0;
2975 }
2976
2977 static int virtnet_set_features(struct net_device *dev,
2978                                 netdev_features_t features)
2979 {
2980         struct virtnet_info *vi = netdev_priv(dev);
2981         u64 offloads;
2982         int err;
2983
2984         if ((dev->features ^ features) & NETIF_F_GRO_HW) {
2985                 if (vi->xdp_enabled)
2986                         return -EBUSY;
2987
2988                 if (features & NETIF_F_GRO_HW)
2989                         offloads = vi->guest_offloads_capable;
2990                 else
2991                         offloads = vi->guest_offloads_capable &
2992                                    ~GUEST_OFFLOAD_GRO_HW_MASK;
2993
2994                 err = virtnet_set_guest_offloads(vi, offloads);
2995                 if (err)
2996                         return err;
2997                 vi->guest_offloads = offloads;
2998         }
2999
3000         if ((dev->features ^ features) & NETIF_F_RXHASH) {
3001                 if (features & NETIF_F_RXHASH)
3002                         vi->ctrl->rss.hash_types = vi->rss_hash_types_saved;
3003                 else
3004                         vi->ctrl->rss.hash_types = VIRTIO_NET_HASH_REPORT_NONE;
3005
3006                 if (!virtnet_commit_rss_command(vi))
3007                         return -EINVAL;
3008         }
3009
3010         return 0;
3011 }
3012
3013 static void virtnet_tx_timeout(struct net_device *dev, unsigned int txqueue)
3014 {
3015         struct virtnet_info *priv = netdev_priv(dev);
3016         struct send_queue *sq = &priv->sq[txqueue];
3017         struct netdev_queue *txq = netdev_get_tx_queue(dev, txqueue);
3018
3019         u64_stats_update_begin(&sq->stats.syncp);
3020         sq->stats.tx_timeouts++;
3021         u64_stats_update_end(&sq->stats.syncp);
3022
3023         netdev_err(dev, "TX timeout on queue: %u, sq: %s, vq: 0x%x, name: %s, %u usecs ago\n",
3024                    txqueue, sq->name, sq->vq->index, sq->vq->name,
3025                    jiffies_to_usecs(jiffies - READ_ONCE(txq->trans_start)));
3026 }
3027
3028 static const struct net_device_ops virtnet_netdev = {
3029         .ndo_open            = virtnet_open,
3030         .ndo_stop            = virtnet_close,
3031         .ndo_start_xmit      = start_xmit,
3032         .ndo_validate_addr   = eth_validate_addr,
3033         .ndo_set_mac_address = virtnet_set_mac_address,
3034         .ndo_set_rx_mode     = virtnet_set_rx_mode,
3035         .ndo_get_stats64     = virtnet_stats,
3036         .ndo_vlan_rx_add_vid = virtnet_vlan_rx_add_vid,
3037         .ndo_vlan_rx_kill_vid = virtnet_vlan_rx_kill_vid,
3038         .ndo_bpf                = virtnet_xdp,
3039         .ndo_xdp_xmit           = virtnet_xdp_xmit,
3040         .ndo_features_check     = passthru_features_check,
3041         .ndo_get_phys_port_name = virtnet_get_phys_port_name,
3042         .ndo_set_features       = virtnet_set_features,
3043         .ndo_tx_timeout         = virtnet_tx_timeout,
3044 };
3045
3046 static void virtnet_config_changed_work(struct work_struct *work)
3047 {
3048         struct virtnet_info *vi =
3049                 container_of(work, struct virtnet_info, config_work);
3050         u16 v;
3051
3052         if (virtio_cread_feature(vi->vdev, VIRTIO_NET_F_STATUS,
3053                                  struct virtio_net_config, status, &v) < 0)
3054                 return;
3055
3056         if (v & VIRTIO_NET_S_ANNOUNCE) {
3057                 netdev_notify_peers(vi->dev);
3058                 virtnet_ack_link_announce(vi);
3059         }
3060
3061         /* Ignore unknown (future) status bits */
3062         v &= VIRTIO_NET_S_LINK_UP;
3063
3064         if (vi->status == v)
3065                 return;
3066
3067         vi->status = v;
3068
3069         if (vi->status & VIRTIO_NET_S_LINK_UP) {
3070                 virtnet_update_settings(vi);
3071                 netif_carrier_on(vi->dev);
3072                 netif_tx_wake_all_queues(vi->dev);
3073         } else {
3074                 netif_carrier_off(vi->dev);
3075                 netif_tx_stop_all_queues(vi->dev);
3076         }
3077 }
3078
3079 static void virtnet_config_changed(struct virtio_device *vdev)
3080 {
3081         struct virtnet_info *vi = vdev->priv;
3082
3083         schedule_work(&vi->config_work);
3084 }
3085
3086 static void virtnet_free_queues(struct virtnet_info *vi)
3087 {
3088         int i;
3089
3090         for (i = 0; i < vi->max_queue_pairs; i++) {
3091                 __netif_napi_del(&vi->rq[i].napi);
3092                 __netif_napi_del(&vi->sq[i].napi);
3093         }
3094
3095         /* We called __netif_napi_del(),
3096          * we need to respect an RCU grace period before freeing vi->rq
3097          */
3098         synchronize_net();
3099
3100         kfree(vi->rq);
3101         kfree(vi->sq);
3102         kfree(vi->ctrl);
3103 }
3104
3105 static void _free_receive_bufs(struct virtnet_info *vi)
3106 {
3107         struct bpf_prog *old_prog;
3108         int i;
3109
3110         for (i = 0; i < vi->max_queue_pairs; i++) {
3111                 while (vi->rq[i].pages)
3112                         __free_pages(get_a_page(&vi->rq[i], GFP_KERNEL), 0);
3113
3114                 old_prog = rtnl_dereference(vi->rq[i].xdp_prog);
3115                 RCU_INIT_POINTER(vi->rq[i].xdp_prog, NULL);
3116                 if (old_prog)
3117                         bpf_prog_put(old_prog);
3118         }
3119 }
3120
3121 static void free_receive_bufs(struct virtnet_info *vi)
3122 {
3123         rtnl_lock();
3124         _free_receive_bufs(vi);
3125         rtnl_unlock();
3126 }
3127
3128 static void free_receive_page_frags(struct virtnet_info *vi)
3129 {
3130         int i;
3131         for (i = 0; i < vi->max_queue_pairs; i++)
3132                 if (vi->rq[i].alloc_frag.page)
3133                         put_page(vi->rq[i].alloc_frag.page);
3134 }
3135
3136 static void free_unused_bufs(struct virtnet_info *vi)
3137 {
3138         void *buf;
3139         int i;
3140
3141         for (i = 0; i < vi->max_queue_pairs; i++) {
3142                 struct virtqueue *vq = vi->sq[i].vq;
3143                 while ((buf = virtqueue_detach_unused_buf(vq)) != NULL) {
3144                         if (!is_xdp_frame(buf))
3145                                 dev_kfree_skb(buf);
3146                         else
3147                                 xdp_return_frame(ptr_to_xdp(buf));
3148                 }
3149         }
3150
3151         for (i = 0; i < vi->max_queue_pairs; i++) {
3152                 struct virtqueue *vq = vi->rq[i].vq;
3153
3154                 while ((buf = virtqueue_detach_unused_buf(vq)) != NULL) {
3155                         if (vi->mergeable_rx_bufs) {
3156                                 put_page(virt_to_head_page(buf));
3157                         } else if (vi->big_packets) {
3158                                 give_pages(&vi->rq[i], buf);
3159                         } else {
3160                                 put_page(virt_to_head_page(buf));
3161                         }
3162                 }
3163         }
3164 }
3165
3166 static void virtnet_del_vqs(struct virtnet_info *vi)
3167 {
3168         struct virtio_device *vdev = vi->vdev;
3169
3170         virtnet_clean_affinity(vi);
3171
3172         vdev->config->del_vqs(vdev);
3173
3174         virtnet_free_queues(vi);
3175 }
3176
3177 /* How large should a single buffer be so a queue full of these can fit at
3178  * least one full packet?
3179  * Logic below assumes the mergeable buffer header is used.
3180  */
3181 static unsigned int mergeable_min_buf_len(struct virtnet_info *vi, struct virtqueue *vq)
3182 {
3183         const unsigned int hdr_len = vi->hdr_len;
3184         unsigned int rq_size = virtqueue_get_vring_size(vq);
3185         unsigned int packet_len = vi->big_packets ? IP_MAX_MTU : vi->dev->max_mtu;
3186         unsigned int buf_len = hdr_len + ETH_HLEN + VLAN_HLEN + packet_len;
3187         unsigned int min_buf_len = DIV_ROUND_UP(buf_len, rq_size);
3188
3189         return max(max(min_buf_len, hdr_len) - hdr_len,
3190                    (unsigned int)GOOD_PACKET_LEN);
3191 }
3192
3193 static int virtnet_find_vqs(struct virtnet_info *vi)
3194 {
3195         vq_callback_t **callbacks;
3196         struct virtqueue **vqs;
3197         int ret = -ENOMEM;
3198         int i, total_vqs;
3199         const char **names;
3200         bool *ctx;
3201
3202         /* We expect 1 RX virtqueue followed by 1 TX virtqueue, followed by
3203          * possible N-1 RX/TX queue pairs used in multiqueue mode, followed by
3204          * possible control vq.
3205          */
3206         total_vqs = vi->max_queue_pairs * 2 +
3207                     virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ);
3208
3209         /* Allocate space for find_vqs parameters */
3210         vqs = kcalloc(total_vqs, sizeof(*vqs), GFP_KERNEL);
3211         if (!vqs)
3212                 goto err_vq;
3213         callbacks = kmalloc_array(total_vqs, sizeof(*callbacks), GFP_KERNEL);
3214         if (!callbacks)
3215                 goto err_callback;
3216         names = kmalloc_array(total_vqs, sizeof(*names), GFP_KERNEL);
3217         if (!names)
3218                 goto err_names;
3219         if (!vi->big_packets || vi->mergeable_rx_bufs) {
3220                 ctx = kcalloc(total_vqs, sizeof(*ctx), GFP_KERNEL);
3221                 if (!ctx)
3222                         goto err_ctx;
3223         } else {
3224                 ctx = NULL;
3225         }
3226
3227         /* Parameters for control virtqueue, if any */
3228         if (vi->has_cvq) {
3229                 callbacks[total_vqs - 1] = NULL;
3230                 names[total_vqs - 1] = "control";
3231         }
3232
3233         /* Allocate/initialize parameters for send/receive virtqueues */
3234         for (i = 0; i < vi->max_queue_pairs; i++) {
3235                 callbacks[rxq2vq(i)] = skb_recv_done;
3236                 callbacks[txq2vq(i)] = skb_xmit_done;
3237                 sprintf(vi->rq[i].name, "input.%d", i);
3238                 sprintf(vi->sq[i].name, "output.%d", i);
3239                 names[rxq2vq(i)] = vi->rq[i].name;
3240                 names[txq2vq(i)] = vi->sq[i].name;
3241                 if (ctx)
3242                         ctx[rxq2vq(i)] = true;
3243         }
3244
3245         ret = virtio_find_vqs_ctx(vi->vdev, total_vqs, vqs, callbacks,
3246                                   names, ctx, NULL);
3247         if (ret)
3248                 goto err_find;
3249
3250         if (vi->has_cvq) {
3251                 vi->cvq = vqs[total_vqs - 1];
3252                 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VLAN))
3253                         vi->dev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;
3254         }
3255
3256         for (i = 0; i < vi->max_queue_pairs; i++) {
3257                 vi->rq[i].vq = vqs[rxq2vq(i)];
3258                 vi->rq[i].min_buf_len = mergeable_min_buf_len(vi, vi->rq[i].vq);
3259                 vi->sq[i].vq = vqs[txq2vq(i)];
3260         }
3261
3262         /* run here: ret == 0. */
3263
3264
3265 err_find:
3266         kfree(ctx);
3267 err_ctx:
3268         kfree(names);
3269 err_names:
3270         kfree(callbacks);
3271 err_callback:
3272         kfree(vqs);
3273 err_vq:
3274         return ret;
3275 }
3276
3277 static int virtnet_alloc_queues(struct virtnet_info *vi)
3278 {
3279         int i;
3280
3281         if (vi->has_cvq) {
3282                 vi->ctrl = kzalloc(sizeof(*vi->ctrl), GFP_KERNEL);
3283                 if (!vi->ctrl)
3284                         goto err_ctrl;
3285         } else {
3286                 vi->ctrl = NULL;
3287         }
3288         vi->sq = kcalloc(vi->max_queue_pairs, sizeof(*vi->sq), GFP_KERNEL);
3289         if (!vi->sq)
3290                 goto err_sq;
3291         vi->rq = kcalloc(vi->max_queue_pairs, sizeof(*vi->rq), GFP_KERNEL);
3292         if (!vi->rq)
3293                 goto err_rq;
3294
3295         INIT_DELAYED_WORK(&vi->refill, refill_work);
3296         for (i = 0; i < vi->max_queue_pairs; i++) {
3297                 vi->rq[i].pages = NULL;
3298                 netif_napi_add(vi->dev, &vi->rq[i].napi, virtnet_poll,
3299                                napi_weight);
3300                 netif_tx_napi_add(vi->dev, &vi->sq[i].napi, virtnet_poll_tx,
3301                                   napi_tx ? napi_weight : 0);
3302
3303                 sg_init_table(vi->rq[i].sg, ARRAY_SIZE(vi->rq[i].sg));
3304                 ewma_pkt_len_init(&vi->rq[i].mrg_avg_pkt_len);
3305                 sg_init_table(vi->sq[i].sg, ARRAY_SIZE(vi->sq[i].sg));
3306
3307                 u64_stats_init(&vi->rq[i].stats.syncp);
3308                 u64_stats_init(&vi->sq[i].stats.syncp);
3309         }
3310
3311         return 0;
3312
3313 err_rq:
3314         kfree(vi->sq);
3315 err_sq:
3316         kfree(vi->ctrl);
3317 err_ctrl:
3318         return -ENOMEM;
3319 }
3320
3321 static int init_vqs(struct virtnet_info *vi)
3322 {
3323         int ret;
3324
3325         /* Allocate send & receive queues */
3326         ret = virtnet_alloc_queues(vi);
3327         if (ret)
3328                 goto err;
3329
3330         ret = virtnet_find_vqs(vi);
3331         if (ret)
3332                 goto err_free;
3333
3334         cpus_read_lock();
3335         virtnet_set_affinity(vi);
3336         cpus_read_unlock();
3337
3338         return 0;
3339
3340 err_free:
3341         virtnet_free_queues(vi);
3342 err:
3343         return ret;
3344 }
3345
3346 #ifdef CONFIG_SYSFS
3347 static ssize_t mergeable_rx_buffer_size_show(struct netdev_rx_queue *queue,
3348                 char *buf)
3349 {
3350         struct virtnet_info *vi = netdev_priv(queue->dev);
3351         unsigned int queue_index = get_netdev_rx_queue_index(queue);
3352         unsigned int headroom = virtnet_get_headroom(vi);
3353         unsigned int tailroom = headroom ? sizeof(struct skb_shared_info) : 0;
3354         struct ewma_pkt_len *avg;
3355
3356         BUG_ON(queue_index >= vi->max_queue_pairs);
3357         avg = &vi->rq[queue_index].mrg_avg_pkt_len;
3358         return sprintf(buf, "%u\n",
3359                        get_mergeable_buf_len(&vi->rq[queue_index], avg,
3360                                        SKB_DATA_ALIGN(headroom + tailroom)));
3361 }
3362
3363 static struct rx_queue_attribute mergeable_rx_buffer_size_attribute =
3364         __ATTR_RO(mergeable_rx_buffer_size);
3365
3366 static struct attribute *virtio_net_mrg_rx_attrs[] = {
3367         &mergeable_rx_buffer_size_attribute.attr,
3368         NULL
3369 };
3370
3371 static const struct attribute_group virtio_net_mrg_rx_group = {
3372         .name = "virtio_net",
3373         .attrs = virtio_net_mrg_rx_attrs
3374 };
3375 #endif
3376
3377 static bool virtnet_fail_on_feature(struct virtio_device *vdev,
3378                                     unsigned int fbit,
3379                                     const char *fname, const char *dname)
3380 {
3381         if (!virtio_has_feature(vdev, fbit))
3382                 return false;
3383
3384         dev_err(&vdev->dev, "device advertises feature %s but not %s",
3385                 fname, dname);
3386
3387         return true;
3388 }
3389
3390 #define VIRTNET_FAIL_ON(vdev, fbit, dbit)                       \
3391         virtnet_fail_on_feature(vdev, fbit, #fbit, dbit)
3392
3393 static bool virtnet_validate_features(struct virtio_device *vdev)
3394 {
3395         if (!virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ) &&
3396             (VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_RX,
3397                              "VIRTIO_NET_F_CTRL_VQ") ||
3398              VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_VLAN,
3399                              "VIRTIO_NET_F_CTRL_VQ") ||
3400              VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_GUEST_ANNOUNCE,
3401                              "VIRTIO_NET_F_CTRL_VQ") ||
3402              VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_MQ, "VIRTIO_NET_F_CTRL_VQ") ||
3403              VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR,
3404                              "VIRTIO_NET_F_CTRL_VQ") ||
3405              VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_RSS,
3406                              "VIRTIO_NET_F_CTRL_VQ") ||
3407              VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_HASH_REPORT,
3408                              "VIRTIO_NET_F_CTRL_VQ"))) {
3409                 return false;
3410         }
3411
3412         return true;
3413 }
3414
3415 #define MIN_MTU ETH_MIN_MTU
3416 #define MAX_MTU ETH_MAX_MTU
3417
3418 static int virtnet_validate(struct virtio_device *vdev)
3419 {
3420         if (!vdev->config->get) {
3421                 dev_err(&vdev->dev, "%s failure: config access disabled\n",
3422                         __func__);
3423                 return -EINVAL;
3424         }
3425
3426         if (!virtnet_validate_features(vdev))
3427                 return -EINVAL;
3428
3429         if (virtio_has_feature(vdev, VIRTIO_NET_F_MTU)) {
3430                 int mtu = virtio_cread16(vdev,
3431                                          offsetof(struct virtio_net_config,
3432                                                   mtu));
3433                 if (mtu < MIN_MTU)
3434                         __virtio_clear_bit(vdev, VIRTIO_NET_F_MTU);
3435         }
3436
3437         return 0;
3438 }
3439
3440 static int virtnet_probe(struct virtio_device *vdev)
3441 {
3442         int i, err = -ENOMEM;
3443         struct net_device *dev;
3444         struct virtnet_info *vi;
3445         u16 max_queue_pairs;
3446         int mtu;
3447
3448         /* Find if host supports multiqueue/rss virtio_net device */
3449         max_queue_pairs = 1;
3450         if (virtio_has_feature(vdev, VIRTIO_NET_F_MQ) || virtio_has_feature(vdev, VIRTIO_NET_F_RSS))
3451                 max_queue_pairs =
3452                      virtio_cread16(vdev, offsetof(struct virtio_net_config, max_virtqueue_pairs));
3453
3454         /* We need at least 2 queue's */
3455         if (max_queue_pairs < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN ||
3456             max_queue_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX ||
3457             !virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ))
3458                 max_queue_pairs = 1;
3459
3460         /* Allocate ourselves a network device with room for our info */
3461         dev = alloc_etherdev_mq(sizeof(struct virtnet_info), max_queue_pairs);
3462         if (!dev)
3463                 return -ENOMEM;
3464
3465         /* Set up network device as normal. */
3466         dev->priv_flags |= IFF_UNICAST_FLT | IFF_LIVE_ADDR_CHANGE |
3467                            IFF_TX_SKB_NO_LINEAR;
3468         dev->netdev_ops = &virtnet_netdev;
3469         dev->features = NETIF_F_HIGHDMA;
3470
3471         dev->ethtool_ops = &virtnet_ethtool_ops;
3472         SET_NETDEV_DEV(dev, &vdev->dev);
3473
3474         /* Do we support "hardware" checksums? */
3475         if (virtio_has_feature(vdev, VIRTIO_NET_F_CSUM)) {
3476                 /* This opens up the world of extra features. */
3477                 dev->hw_features |= NETIF_F_HW_CSUM | NETIF_F_SG;
3478                 if (csum)
3479                         dev->features |= NETIF_F_HW_CSUM | NETIF_F_SG;
3480
3481                 if (virtio_has_feature(vdev, VIRTIO_NET_F_GSO)) {
3482                         dev->hw_features |= NETIF_F_TSO
3483                                 | NETIF_F_TSO_ECN | NETIF_F_TSO6;
3484                 }
3485                 /* Individual feature bits: what can host handle? */
3486                 if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO4))
3487                         dev->hw_features |= NETIF_F_TSO;
3488                 if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO6))
3489                         dev->hw_features |= NETIF_F_TSO6;
3490                 if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_ECN))
3491                         dev->hw_features |= NETIF_F_TSO_ECN;
3492
3493                 dev->features |= NETIF_F_GSO_ROBUST;
3494
3495                 if (gso)
3496                         dev->features |= dev->hw_features & NETIF_F_ALL_TSO;
3497                 /* (!csum && gso) case will be fixed by register_netdev() */
3498         }
3499         if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_CSUM))
3500                 dev->features |= NETIF_F_RXCSUM;
3501         if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO4) ||
3502             virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO6))
3503                 dev->features |= NETIF_F_GRO_HW;
3504         if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS))
3505                 dev->hw_features |= NETIF_F_GRO_HW;
3506
3507         dev->vlan_features = dev->features;
3508
3509         /* MTU range: 68 - 65535 */
3510         dev->min_mtu = MIN_MTU;
3511         dev->max_mtu = MAX_MTU;
3512
3513         /* Configuration may specify what MAC to use.  Otherwise random. */
3514         if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC)) {
3515                 u8 addr[ETH_ALEN];
3516
3517                 virtio_cread_bytes(vdev,
3518                                    offsetof(struct virtio_net_config, mac),
3519                                    addr, ETH_ALEN);
3520                 eth_hw_addr_set(dev, addr);
3521         } else {
3522                 eth_hw_addr_random(dev);
3523         }
3524
3525         /* Set up our device-specific information */
3526         vi = netdev_priv(dev);
3527         vi->dev = dev;
3528         vi->vdev = vdev;
3529         vdev->priv = vi;
3530
3531         INIT_WORK(&vi->config_work, virtnet_config_changed_work);
3532
3533         /* If we can receive ANY GSO packets, we must allocate large ones. */
3534         if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO4) ||
3535             virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO6) ||
3536             virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_ECN) ||
3537             virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_UFO))
3538                 vi->big_packets = true;
3539
3540         if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF))
3541                 vi->mergeable_rx_bufs = true;
3542
3543         if (virtio_has_feature(vdev, VIRTIO_NET_F_HASH_REPORT))
3544                 vi->has_rss_hash_report = true;
3545
3546         if (virtio_has_feature(vdev, VIRTIO_NET_F_RSS))
3547                 vi->has_rss = true;
3548
3549         if (vi->has_rss || vi->has_rss_hash_report) {
3550                 vi->rss_indir_table_size =
3551                         virtio_cread16(vdev, offsetof(struct virtio_net_config,
3552                                 rss_max_indirection_table_length));
3553                 vi->rss_key_size =
3554                         virtio_cread8(vdev, offsetof(struct virtio_net_config, rss_max_key_size));
3555
3556                 vi->rss_hash_types_supported =
3557                     virtio_cread32(vdev, offsetof(struct virtio_net_config, supported_hash_types));
3558                 vi->rss_hash_types_supported &=
3559                                 ~(VIRTIO_NET_RSS_HASH_TYPE_IP_EX |
3560                                   VIRTIO_NET_RSS_HASH_TYPE_TCP_EX |
3561                                   VIRTIO_NET_RSS_HASH_TYPE_UDP_EX);
3562
3563                 dev->hw_features |= NETIF_F_RXHASH;
3564         }
3565
3566         if (vi->has_rss_hash_report)
3567                 vi->hdr_len = sizeof(struct virtio_net_hdr_v1_hash);
3568         else if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF) ||
3569                  virtio_has_feature(vdev, VIRTIO_F_VERSION_1))
3570                 vi->hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);
3571         else
3572                 vi->hdr_len = sizeof(struct virtio_net_hdr);
3573
3574         if (virtio_has_feature(vdev, VIRTIO_F_ANY_LAYOUT) ||
3575             virtio_has_feature(vdev, VIRTIO_F_VERSION_1))
3576                 vi->any_header_sg = true;
3577
3578         if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ))
3579                 vi->has_cvq = true;
3580
3581         if (virtio_has_feature(vdev, VIRTIO_NET_F_MTU)) {
3582                 mtu = virtio_cread16(vdev,
3583                                      offsetof(struct virtio_net_config,
3584                                               mtu));
3585                 if (mtu < dev->min_mtu) {
3586                         /* Should never trigger: MTU was previously validated
3587                          * in virtnet_validate.
3588                          */
3589                         dev_err(&vdev->dev,
3590                                 "device MTU appears to have changed it is now %d < %d",
3591                                 mtu, dev->min_mtu);
3592                         err = -EINVAL;
3593                         goto free;
3594                 }
3595
3596                 dev->mtu = mtu;
3597                 dev->max_mtu = mtu;
3598
3599                 /* TODO: size buffers correctly in this case. */
3600                 if (dev->mtu > ETH_DATA_LEN)
3601                         vi->big_packets = true;
3602         }
3603
3604         if (vi->any_header_sg)
3605                 dev->needed_headroom = vi->hdr_len;
3606
3607         /* Enable multiqueue by default */
3608         if (num_online_cpus() >= max_queue_pairs)
3609                 vi->curr_queue_pairs = max_queue_pairs;
3610         else
3611                 vi->curr_queue_pairs = num_online_cpus();
3612         vi->max_queue_pairs = max_queue_pairs;
3613
3614         /* Allocate/initialize the rx/tx queues, and invoke find_vqs */
3615         err = init_vqs(vi);
3616         if (err)
3617                 goto free;
3618
3619 #ifdef CONFIG_SYSFS
3620         if (vi->mergeable_rx_bufs)
3621                 dev->sysfs_rx_queue_group = &virtio_net_mrg_rx_group;
3622 #endif
3623         netif_set_real_num_tx_queues(dev, vi->curr_queue_pairs);
3624         netif_set_real_num_rx_queues(dev, vi->curr_queue_pairs);
3625
3626         virtnet_init_settings(dev);
3627
3628         if (virtio_has_feature(vdev, VIRTIO_NET_F_STANDBY)) {
3629                 vi->failover = net_failover_create(vi->dev);
3630                 if (IS_ERR(vi->failover)) {
3631                         err = PTR_ERR(vi->failover);
3632                         goto free_vqs;
3633                 }
3634         }
3635
3636         if (vi->has_rss || vi->has_rss_hash_report)
3637                 virtnet_init_default_rss(vi);
3638
3639         err = register_netdev(dev);
3640         if (err) {
3641                 pr_debug("virtio_net: registering device failed\n");
3642                 goto free_failover;
3643         }
3644
3645         virtio_device_ready(vdev);
3646
3647         err = virtnet_cpu_notif_add(vi);
3648         if (err) {
3649                 pr_debug("virtio_net: registering cpu notifier failed\n");
3650                 goto free_unregister_netdev;
3651         }
3652
3653         virtnet_set_queues(vi, vi->curr_queue_pairs);
3654
3655         /* Assume link up if device can't report link status,
3656            otherwise get link status from config. */
3657         netif_carrier_off(dev);
3658         if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STATUS)) {
3659                 schedule_work(&vi->config_work);
3660         } else {
3661                 vi->status = VIRTIO_NET_S_LINK_UP;
3662                 virtnet_update_settings(vi);
3663                 netif_carrier_on(dev);
3664         }
3665
3666         for (i = 0; i < ARRAY_SIZE(guest_offloads); i++)
3667                 if (virtio_has_feature(vi->vdev, guest_offloads[i]))
3668                         set_bit(guest_offloads[i], &vi->guest_offloads);
3669         vi->guest_offloads_capable = vi->guest_offloads;
3670
3671         pr_debug("virtnet: registered device %s with %d RX and TX vq's\n",
3672                  dev->name, max_queue_pairs);
3673
3674         return 0;
3675
3676 free_unregister_netdev:
3677         virtio_reset_device(vdev);
3678
3679         unregister_netdev(dev);
3680 free_failover:
3681         net_failover_destroy(vi->failover);
3682 free_vqs:
3683         cancel_delayed_work_sync(&vi->refill);
3684         free_receive_page_frags(vi);
3685         virtnet_del_vqs(vi);
3686 free:
3687         free_netdev(dev);
3688         return err;
3689 }
3690
3691 static void remove_vq_common(struct virtnet_info *vi)
3692 {
3693         virtio_reset_device(vi->vdev);
3694
3695         /* Free unused buffers in both send and recv, if any. */
3696         free_unused_bufs(vi);
3697
3698         free_receive_bufs(vi);
3699
3700         free_receive_page_frags(vi);
3701
3702         virtnet_del_vqs(vi);
3703 }
3704
3705 static void virtnet_remove(struct virtio_device *vdev)
3706 {
3707         struct virtnet_info *vi = vdev->priv;
3708
3709         virtnet_cpu_notif_remove(vi);
3710
3711         /* Make sure no work handler is accessing the device. */
3712         flush_work(&vi->config_work);
3713
3714         unregister_netdev(vi->dev);
3715
3716         net_failover_destroy(vi->failover);
3717
3718         remove_vq_common(vi);
3719
3720         free_netdev(vi->dev);
3721 }
3722
3723 static __maybe_unused int virtnet_freeze(struct virtio_device *vdev)
3724 {
3725         struct virtnet_info *vi = vdev->priv;
3726
3727         virtnet_cpu_notif_remove(vi);
3728         virtnet_freeze_down(vdev);
3729         remove_vq_common(vi);
3730
3731         return 0;
3732 }
3733
3734 static __maybe_unused int virtnet_restore(struct virtio_device *vdev)
3735 {
3736         struct virtnet_info *vi = vdev->priv;
3737         int err;
3738
3739         err = virtnet_restore_up(vdev);
3740         if (err)
3741                 return err;
3742         virtnet_set_queues(vi, vi->curr_queue_pairs);
3743
3744         err = virtnet_cpu_notif_add(vi);
3745         if (err) {
3746                 virtnet_freeze_down(vdev);
3747                 remove_vq_common(vi);
3748                 return err;
3749         }
3750
3751         return 0;
3752 }
3753
3754 static struct virtio_device_id id_table[] = {
3755         { VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID },
3756         { 0 },
3757 };
3758
3759 #define VIRTNET_FEATURES \
3760         VIRTIO_NET_F_CSUM, VIRTIO_NET_F_GUEST_CSUM, \
3761         VIRTIO_NET_F_MAC, \
3762         VIRTIO_NET_F_HOST_TSO4, VIRTIO_NET_F_HOST_UFO, VIRTIO_NET_F_HOST_TSO6, \
3763         VIRTIO_NET_F_HOST_ECN, VIRTIO_NET_F_GUEST_TSO4, VIRTIO_NET_F_GUEST_TSO6, \
3764         VIRTIO_NET_F_GUEST_ECN, VIRTIO_NET_F_GUEST_UFO, \
3765         VIRTIO_NET_F_MRG_RXBUF, VIRTIO_NET_F_STATUS, VIRTIO_NET_F_CTRL_VQ, \
3766         VIRTIO_NET_F_CTRL_RX, VIRTIO_NET_F_CTRL_VLAN, \
3767         VIRTIO_NET_F_GUEST_ANNOUNCE, VIRTIO_NET_F_MQ, \
3768         VIRTIO_NET_F_CTRL_MAC_ADDR, \
3769         VIRTIO_NET_F_MTU, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, \
3770         VIRTIO_NET_F_SPEED_DUPLEX, VIRTIO_NET_F_STANDBY, \
3771         VIRTIO_NET_F_RSS, VIRTIO_NET_F_HASH_REPORT
3772
3773 static unsigned int features[] = {
3774         VIRTNET_FEATURES,
3775 };
3776
3777 static unsigned int features_legacy[] = {
3778         VIRTNET_FEATURES,
3779         VIRTIO_NET_F_GSO,
3780         VIRTIO_F_ANY_LAYOUT,
3781 };
3782
3783 static struct virtio_driver virtio_net_driver = {
3784         .feature_table = features,
3785         .feature_table_size = ARRAY_SIZE(features),
3786         .feature_table_legacy = features_legacy,
3787         .feature_table_size_legacy = ARRAY_SIZE(features_legacy),
3788         .driver.name =  KBUILD_MODNAME,
3789         .driver.owner = THIS_MODULE,
3790         .id_table =     id_table,
3791         .validate =     virtnet_validate,
3792         .probe =        virtnet_probe,
3793         .remove =       virtnet_remove,
3794         .config_changed = virtnet_config_changed,
3795 #ifdef CONFIG_PM_SLEEP
3796         .freeze =       virtnet_freeze,
3797         .restore =      virtnet_restore,
3798 #endif
3799 };
3800
3801 static __init int virtio_net_driver_init(void)
3802 {
3803         int ret;
3804
3805         ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "virtio/net:online",
3806                                       virtnet_cpu_online,
3807                                       virtnet_cpu_down_prep);
3808         if (ret < 0)
3809                 goto out;
3810         virtionet_online = ret;
3811         ret = cpuhp_setup_state_multi(CPUHP_VIRT_NET_DEAD, "virtio/net:dead",
3812                                       NULL, virtnet_cpu_dead);
3813         if (ret)
3814                 goto err_dead;
3815         ret = register_virtio_driver(&virtio_net_driver);
3816         if (ret)
3817                 goto err_virtio;
3818         return 0;
3819 err_virtio:
3820         cpuhp_remove_multi_state(CPUHP_VIRT_NET_DEAD);
3821 err_dead:
3822         cpuhp_remove_multi_state(virtionet_online);
3823 out:
3824         return ret;
3825 }
3826 module_init(virtio_net_driver_init);
3827
3828 static __exit void virtio_net_driver_exit(void)
3829 {
3830         unregister_virtio_driver(&virtio_net_driver);
3831         cpuhp_remove_multi_state(CPUHP_VIRT_NET_DEAD);
3832         cpuhp_remove_multi_state(virtionet_online);
3833 }
3834 module_exit(virtio_net_driver_exit);
3835
3836 MODULE_DEVICE_TABLE(virtio, id_table);
3837 MODULE_DESCRIPTION("Virtio network driver");
3838 MODULE_LICENSE("GPL");