drivers/net/virtio_net.c

   1 /* A network driver using virtio.
   2  *
   3  * Copyright 2007 Rusty Russell <rusty@rustcorp.com.au> IBM Corporation
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  17  */
  18 //#define DEBUG
  19 #include <linux/netdevice.h>
  20 #include <linux/etherdevice.h>
  21 #include <linux/ethtool.h>
  22 #include <linux/module.h>
  23 #include <linux/virtio.h>
  24 #include <linux/virtio_net.h>
  25 #include <linux/bpf.h>
  26 #include <linux/bpf_trace.h>
  27 #include <linux/scatterlist.h>
  28 #include <linux/if_vlan.h>
  29 #include <linux/slab.h>
  30 #include <linux/cpu.h>
  31 #include <linux/average.h>
  32 #include <net/route.h>
  33
  34 static int napi_weight = NAPI_POLL_WEIGHT;
  35 module_param(napi_weight, int, 0444);
  36
  37 static bool csum = true, gso = true;
  38 module_param(csum, bool, 0444);
  39 module_param(gso, bool, 0444);
  40
  41 /* FIXME: MTU in config. */
  42 #define GOOD_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN)
  43 #define GOOD_COPY_LEN   128
  44
  45 #define VIRTNET_RX_PAD (NET_IP_ALIGN + NET_SKB_PAD)
  46
  47 /* Amount of XDP headroom to prepend to packets for use by xdp_adjust_head */
  48 #define VIRTIO_XDP_HEADROOM 256
  49
  50 /* RX packet size EWMA. The average packet size is used to determine the packet
  51  * buffer size when refilling RX rings. As the entire RX ring may be refilled
  52  * at once, the weight is chosen so that the EWMA will be insensitive to short-
  53  * term, transient changes in packet size.
  54  */
  55 DECLARE_EWMA(pkt_len, 0, 64)
  56
  57 #define VIRTNET_DRIVER_VERSION "1.0.0"
  58
  59 struct virtnet_stats {
  60         struct u64_stats_sync tx_syncp;
  61         struct u64_stats_sync rx_syncp;
  62         u64 tx_bytes;
  63         u64 tx_packets;
  64
  65         u64 rx_bytes;
  66         u64 rx_packets;
  67 };
  68
  69 /* Internal representation of a send virtqueue */
  70 struct send_queue {
  71         /* Virtqueue associated with this send _queue */
  72         struct virtqueue *vq;
  73
  74         /* TX: fragments + linear part + virtio header */
  75         struct scatterlist sg[MAX_SKB_FRAGS + 2];
  76
  77         /* Name of the send queue: output.$index */
  78         char name[40];
  79 };
  80
  81 /* Internal representation of a receive virtqueue */
  82 struct receive_queue {
  83         /* Virtqueue associated with this receive_queue */
  84         struct virtqueue *vq;
  85
  86         struct napi_struct napi;
  87
  88         struct bpf_prog __rcu *xdp_prog;
  89
  90         /* Chain pages by the private ptr. */
  91         struct page *pages;
  92
  93         /* Average packet length for mergeable receive buffers. */
  94         struct ewma_pkt_len mrg_avg_pkt_len;
  95
  96         /* Page frag for packet buffer allocation. */
  97         struct page_frag alloc_frag;
  98
  99         /* RX: fragments + linear part + virtio header */
 100         struct scatterlist sg[MAX_SKB_FRAGS + 2];
 101
 102         /* Min single buffer size for mergeable buffers case. */
 103         unsigned int min_buf_len;
 104
 105         /* Name of this receive queue: input.$index */
 106         char name[40];
 107 };
 108
 109 struct virtnet_info {
 110         struct virtio_device *vdev;
 111         struct virtqueue *cvq;
 112         struct net_device *dev;
 113         struct send_queue *sq;
 114         struct receive_queue *rq;
 115         unsigned int status;
 116
 117         /* Max # of queue pairs supported by the device */
 118         u16 max_queue_pairs;
 119
 120         /* # of queue pairs currently used by the driver */
 121         u16 curr_queue_pairs;
 122
 123         /* # of XDP queue pairs currently used by the driver */
 124         u16 xdp_queue_pairs;
 125
 126         /* I like... big packets and I cannot lie! */
 127         bool big_packets;
 128
 129         /* Host will merge rx buffers for big packets (shake it! shake it!) */
 130         bool mergeable_rx_bufs;
 131
 132         /* Has control virtqueue */
 133         bool has_cvq;
 134
 135         /* Host can handle any s/g split between our header and packet data */
 136         bool any_header_sg;
 137
 138         /* Packet virtio header size */
 139         u8 hdr_len;
 140
 141         /* Active statistics */
 142         struct virtnet_stats __percpu *stats;
 143
 144         /* Work struct for refilling if we run low on memory. */
 145         struct delayed_work refill;
 146
 147         /* Work struct for config space updates */
 148         struct work_struct config_work;
 149
 150         /* Does the affinity hint is set for virtqueues? */
 151         bool affinity_hint_set;
 152
 153         /* CPU hotplug instances for online & dead */
 154         struct hlist_node node;
 155         struct hlist_node node_dead;
 156
 157         /* Control VQ buffers: protected by the rtnl lock */
 158         struct virtio_net_ctrl_hdr ctrl_hdr;
 159         virtio_net_ctrl_ack ctrl_status;
 160         struct virtio_net_ctrl_mq ctrl_mq;
 161         u8 ctrl_promisc;
 162         u8 ctrl_allmulti;
 163         u16 ctrl_vid;
 164
 165         /* Ethtool settings */
 166         u8 duplex;
 167         u32 speed;
 168 };
 169
 170 struct padded_vnet_hdr {
 171         struct virtio_net_hdr_mrg_rxbuf hdr;
 172         /*
 173          * hdr is in a separate sg buffer, and data sg buffer shares same page
 174          * with this header sg. This padding makes next sg 16 byte aligned
 175          * after the header.
 176          */
 177         char padding[4];
 178 };
 179
 180 /* Converting between virtqueue no. and kernel tx/rx queue no.
 181  * 0:rx0 1:tx0 2:rx1 3:tx1 ... 2N:rxN 2N+1:txN 2N+2:cvq
 182  */
 183 static int vq2txq(struct virtqueue *vq)
 184 {
 185         return (vq->index - 1) / 2;
 186 }
 187
 188 static int txq2vq(int txq)
 189 {
 190         return txq * 2 + 1;
 191 }
 192
 193 static int vq2rxq(struct virtqueue *vq)
 194 {
 195         return vq->index / 2;
 196 }
 197
 198 static int rxq2vq(int rxq)
 199 {
 200         return rxq * 2;
 201 }
 202
 203 static inline struct virtio_net_hdr_mrg_rxbuf *skb_vnet_hdr(struct sk_buff *skb)
 204 {
 205         return (struct virtio_net_hdr_mrg_rxbuf *)skb->cb;
 206 }
 207
 208 /*
 209  * private is used to chain pages for big packets, put the whole
 210  * most recent used list in the beginning for reuse
 211  */
 212 static void give_pages(struct receive_queue *rq, struct page *page)
 213 {
 214         struct page *end;
 215
 216         /* Find end of list, sew whole thing into vi->rq.pages. */
 217         for (end = page; end->private; end = (struct page *)end->private);
 218         end->private = (unsigned long)rq->pages;
 219         rq->pages = page;
 220 }
 221
 222 static struct page *get_a_page(struct receive_queue *rq, gfp_t gfp_mask)
 223 {
 224         struct page *p = rq->pages;
 225
 226         if (p) {
 227                 rq->pages = (struct page *)p->private;
 228                 /* clear private here, it is used to chain pages */
 229                 p->private = 0;
 230         } else
 231                 p = alloc_page(gfp_mask);
 232         return p;
 233 }
 234
 235 static void skb_xmit_done(struct virtqueue *vq)
 236 {
 237         struct virtnet_info *vi = vq->vdev->priv;
 238
 239         /* Suppress further interrupts. */
 240         virtqueue_disable_cb(vq);
 241
 242         /* We were probably waiting for more output buffers. */
 243         netif_wake_subqueue(vi->dev, vq2txq(vq));
 244 }
 245
 246 /* Called from bottom half context */
 247 static struct sk_buff *page_to_skb(struct virtnet_info *vi,
 248                                    struct receive_queue *rq,
 249                                    struct page *page, unsigned int offset,
 250                                    unsigned int len, unsigned int truesize)
 251 {
 252         struct sk_buff *skb;
 253         struct virtio_net_hdr_mrg_rxbuf *hdr;
 254         unsigned int copy, hdr_len, hdr_padded_len;
 255         char *p;
 256
 257         p = page_address(page) + offset;
 258
 259         /* copy small packet so we can reuse these pages for small data */
 260         skb = napi_alloc_skb(&rq->napi, GOOD_COPY_LEN);
 261         if (unlikely(!skb))
 262                 return NULL;
 263
 264         hdr = skb_vnet_hdr(skb);
 265
 266         hdr_len = vi->hdr_len;
 267         if (vi->mergeable_rx_bufs)
 268                 hdr_padded_len = sizeof *hdr;
 269         else
 270                 hdr_padded_len = sizeof(struct padded_vnet_hdr);
 271
 272         memcpy(hdr, p, hdr_len);
 273
 274         len -= hdr_len;
 275         offset += hdr_padded_len;
 276         p += hdr_padded_len;
 277
 278         copy = len;
 279         if (copy > skb_tailroom(skb))
 280                 copy = skb_tailroom(skb);
 281         memcpy(skb_put(skb, copy), p, copy);
 282
 283         len -= copy;
 284         offset += copy;
 285
 286         if (vi->mergeable_rx_bufs) {
 287                 if (len)
 288                         skb_add_rx_frag(skb, 0, page, offset, len, truesize);
 289                 else
 290                         put_page(page);
 291                 return skb;
 292         }
 293
 294         /*
 295          * Verify that we can indeed put this data into a skb.
 296          * This is here to handle cases when the device erroneously
 297          * tries to receive more than is possible. This is usually
 298          * the case of a broken device.
 299          */
 300         if (unlikely(len > MAX_SKB_FRAGS * PAGE_SIZE)) {
 301                 net_dbg_ratelimited("%s: too much data\n", skb->dev->name);
 302                 dev_kfree_skb(skb);
 303                 return NULL;
 304         }
 305         BUG_ON(offset >= PAGE_SIZE);
 306         while (len) {
 307                 unsigned int frag_size = min((unsigned)PAGE_SIZE - offset, len);
 308                 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, offset,
 309                                 frag_size, truesize);
 310                 len -= frag_size;
 311                 page = (struct page *)page->private;
 312                 offset = 0;
 313         }
 314
 315         if (page)
 316                 give_pages(rq, page);
 317
 318         return skb;
 319 }
 320
 321 static bool virtnet_xdp_xmit(struct virtnet_info *vi,
 322                              struct receive_queue *rq,
 323                              struct xdp_buff *xdp)
 324 {
 325         struct virtio_net_hdr_mrg_rxbuf *hdr;
 326         unsigned int len;
 327         struct send_queue *sq;
 328         unsigned int qp;
 329         void *xdp_sent;
 330         int err;
 331
 332         qp = vi->curr_queue_pairs - vi->xdp_queue_pairs + smp_processor_id();
 333         sq = &vi->sq[qp];
 334
 335         /* Free up any pending old buffers before queueing new ones. */
 336         while ((xdp_sent = virtqueue_get_buf(sq->vq, &len)) != NULL) {
 337                 struct page *sent_page = virt_to_head_page(xdp_sent);
 338
 339                 put_page(sent_page);
 340         }
 341
 342         xdp->data -= vi->hdr_len;
 343         /* Zero header and leave csum up to XDP layers */
 344         hdr = xdp->data;
 345         memset(hdr, 0, vi->hdr_len);
 346
 347         sg_init_one(sq->sg, xdp->data, xdp->data_end - xdp->data);
 348
 349         err = virtqueue_add_outbuf(sq->vq, sq->sg, 1, xdp->data, GFP_ATOMIC);
 350         if (unlikely(err)) {
 351                 struct page *page = virt_to_head_page(xdp->data);
 352
 353                 put_page(page);
 354                 return false;
 355         }
 356
 357         virtqueue_kick(sq->vq);
 358         return true;
 359 }
 360
 361 static unsigned int virtnet_get_headroom(struct virtnet_info *vi)
 362 {
 363         return vi->xdp_queue_pairs ? VIRTIO_XDP_HEADROOM : 0;
 364 }
 365
 366 static struct sk_buff *receive_small(struct net_device *dev,
 367                                      struct virtnet_info *vi,
 368                                      struct receive_queue *rq,
 369                                      void *buf, unsigned int len)
 370 {
 371         struct sk_buff *skb;
 372         struct bpf_prog *xdp_prog;
 373         unsigned int xdp_headroom = virtnet_get_headroom(vi);
 374         unsigned int header_offset = VIRTNET_RX_PAD + xdp_headroom;
 375         unsigned int headroom = vi->hdr_len + header_offset;
 376         unsigned int buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) +
 377                               SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
 378         unsigned int delta = 0;
 379         len -= vi->hdr_len;
 380
 381         rcu_read_lock();
 382         xdp_prog = rcu_dereference(rq->xdp_prog);
 383         if (xdp_prog) {
 384                 struct virtio_net_hdr_mrg_rxbuf *hdr = buf + header_offset;
 385                 struct xdp_buff xdp;
 386                 void *orig_data;
 387                 u32 act;
 388
 389                 if (unlikely(hdr->hdr.gso_type || hdr->hdr.flags))
 390                         goto err_xdp;
 391
 392                 xdp.data_hard_start = buf + VIRTNET_RX_PAD + vi->hdr_len;
 393                 xdp.data = xdp.data_hard_start + xdp_headroom;
 394                 xdp.data_end = xdp.data + len;
 395                 orig_data = xdp.data;
 396                 act = bpf_prog_run_xdp(xdp_prog, &xdp);
 397
 398                 switch (act) {
 399                 case XDP_PASS:
 400                         /* Recalculate length in case bpf program changed it */
 401                         delta = orig_data - xdp.data;
 402                         break;
 403                 case XDP_TX:
 404                         if (unlikely(!virtnet_xdp_xmit(vi, rq, &xdp)))
 405                                 trace_xdp_exception(vi->dev, xdp_prog, act);
 406                         rcu_read_unlock();
 407                         goto xdp_xmit;
 408                 default:
 409                         bpf_warn_invalid_xdp_action(act);
 410                 case XDP_ABORTED:
 411                         trace_xdp_exception(vi->dev, xdp_prog, act);
 412                 case XDP_DROP:
 413                         goto err_xdp;
 414                 }
 415         }
 416         rcu_read_unlock();
 417
 418         skb = build_skb(buf, buflen);
 419         if (!skb) {
 420                 put_page(virt_to_head_page(buf));
 421                 goto err;
 422         }
 423         skb_reserve(skb, headroom - delta);
 424         skb_put(skb, len + delta);
 425         if (!delta) {
 426                 buf += header_offset;
 427                 memcpy(skb_vnet_hdr(skb), buf, vi->hdr_len);
 428         } /* keep zeroed vnet hdr since packet was changed by bpf */
 429
 430 err:
 431         return skb;
 432
 433 err_xdp:
 434         rcu_read_unlock();
 435         dev->stats.rx_dropped++;
 436         put_page(virt_to_head_page(buf));
 437 xdp_xmit:
 438         return NULL;
 439 }
 440
 441 static struct sk_buff *receive_big(struct net_device *dev,
 442                                    struct virtnet_info *vi,
 443                                    struct receive_queue *rq,
 444                                    void *buf,
 445                                    unsigned int len)
 446 {
 447         struct page *page = buf;
 448         struct sk_buff *skb = page_to_skb(vi, rq, page, 0, len, PAGE_SIZE);
 449
 450         if (unlikely(!skb))
 451                 goto err;
 452
 453         return skb;
 454
 455 err:
 456         dev->stats.rx_dropped++;
 457         give_pages(rq, page);
 458         return NULL;
 459 }
 460
 461 /* The conditions to enable XDP should preclude the underlying device from
 462  * sending packets across multiple buffers (num_buf > 1). However per spec
 463  * it does not appear to be illegal to do so but rather just against convention.
 464  * So in order to avoid making a system unresponsive the packets are pushed
 465  * into a page and the XDP program is run. This will be extremely slow and we
 466  * push a warning to the user to fix this as soon as possible. Fixing this may
 467  * require resolving the underlying hardware to determine why multiple buffers
 468  * are being received or simply loading the XDP program in the ingress stack
 469  * after the skb is built because there is no advantage to running it here
 470  * anymore.
 471  */
 472 static struct page *xdp_linearize_page(struct receive_queue *rq,
 473                                        u16 *num_buf,
 474                                        struct page *p,
 475                                        int offset,
 476                                        unsigned int *len)
 477 {
 478         struct page *page = alloc_page(GFP_ATOMIC);
 479         unsigned int page_off = VIRTIO_XDP_HEADROOM;
 480
 481         if (!page)
 482                 return NULL;
 483
 484         memcpy(page_address(page) + page_off, page_address(p) + offset, *len);
 485         page_off += *len;
 486
 487         while (--*num_buf) {
 488                 unsigned int buflen;
 489                 void *buf;
 490                 int off;
 491
 492                 buf = virtqueue_get_buf(rq->vq, &buflen);
 493                 if (unlikely(!buf))
 494                         goto err_buf;
 495
 496                 p = virt_to_head_page(buf);
 497                 off = buf - page_address(p);
 498
 499                 /* guard against a misconfigured or uncooperative backend that
 500                  * is sending packet larger than the MTU.
 501                  */
 502                 if ((page_off + buflen) > PAGE_SIZE) {
 503                         put_page(p);
 504                         goto err_buf;
 505                 }
 506
 507                 memcpy(page_address(page) + page_off,
 508                        page_address(p) + off, buflen);
 509                 page_off += buflen;
 510                 put_page(p);
 511         }
 512
 513         /* Headroom does not contribute to packet length */
 514         *len = page_off - VIRTIO_XDP_HEADROOM;
 515         return page;
 516 err_buf:
 517         __free_pages(page, 0);
 518         return NULL;
 519 }
 520
 521 static struct sk_buff *receive_mergeable(struct net_device *dev,
 522                                          struct virtnet_info *vi,
 523                                          struct receive_queue *rq,
 524                                          void *buf,
 525                                          void *ctx,
 526                                          unsigned int len)
 527 {
 528         struct virtio_net_hdr_mrg_rxbuf *hdr = buf;
 529         u16 num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers);
 530         struct page *page = virt_to_head_page(buf);
 531         int offset = buf - page_address(page);
 532         struct sk_buff *head_skb, *curr_skb;
 533         struct bpf_prog *xdp_prog;
 534         unsigned int truesize;
 535
 536         head_skb = NULL;
 537
 538         rcu_read_lock();
 539         xdp_prog = rcu_dereference(rq->xdp_prog);
 540         if (xdp_prog) {
 541                 struct page *xdp_page;
 542                 struct xdp_buff xdp;
 543                 void *data;
 544                 u32 act;
 545
 546                 /* This happens when rx buffer size is underestimated */
 547                 if (unlikely(num_buf > 1)) {
 548                         /* linearize data for XDP */
 549                         xdp_page = xdp_linearize_page(rq, &num_buf,
 550                                                       page, offset, &len);
 551                         if (!xdp_page)
 552                                 goto err_xdp;
 553                         offset = VIRTIO_XDP_HEADROOM;
 554                 } else {
 555                         xdp_page = page;
 556                 }
 557
 558                 /* Transient failure which in theory could occur if
 559                  * in-flight packets from before XDP was enabled reach
 560                  * the receive path after XDP is loaded. In practice I
 561                  * was not able to create this condition.
 562                  */
 563                 if (unlikely(hdr->hdr.gso_type))
 564                         goto err_xdp;
 565
 566                 /* Allow consuming headroom but reserve enough space to push
 567                  * the descriptor on if we get an XDP_TX return code.
 568                  */
 569                 data = page_address(xdp_page) + offset;
 570                 xdp.data_hard_start = data - VIRTIO_XDP_HEADROOM + vi->hdr_len;
 571                 xdp.data = data + vi->hdr_len;
 572                 xdp.data_end = xdp.data + (len - vi->hdr_len);
 573                 act = bpf_prog_run_xdp(xdp_prog, &xdp);
 574
 575                 switch (act) {
 576                 case XDP_PASS:
 577                         /* recalculate offset to account for any header
 578                          * adjustments. Note other cases do not build an
 579                          * skb and avoid using offset
 580                          */
 581                         offset = xdp.data -
 582                                         page_address(xdp_page) - vi->hdr_len;
 583
 584                         /* We can only create skb based on xdp_page. */
 585                         if (unlikely(xdp_page != page)) {
 586                                 rcu_read_unlock();
 587                                 put_page(page);
 588                                 head_skb = page_to_skb(vi, rq, xdp_page,
 589                                                        offset, len, PAGE_SIZE);
 590                                 ewma_pkt_len_add(&rq->mrg_avg_pkt_len, len);
 591                                 return head_skb;
 592                         }
 593                         break;
 594                 case XDP_TX:
 595                         if (unlikely(!virtnet_xdp_xmit(vi, rq, &xdp)))
 596                                 trace_xdp_exception(vi->dev, xdp_prog, act);
 597                         ewma_pkt_len_add(&rq->mrg_avg_pkt_len, len);
 598                         if (unlikely(xdp_page != page))
 599                                 goto err_xdp;
 600                         rcu_read_unlock();
 601                         goto xdp_xmit;
 602                 default:
 603                         bpf_warn_invalid_xdp_action(act);
 604                 case XDP_ABORTED:
 605                         trace_xdp_exception(vi->dev, xdp_prog, act);
 606                 case XDP_DROP:
 607                         if (unlikely(xdp_page != page))
 608                                 __free_pages(xdp_page, 0);
 609                         ewma_pkt_len_add(&rq->mrg_avg_pkt_len, len);
 610                         goto err_xdp;
 611                 }
 612         }
 613         rcu_read_unlock();
 614
 615         if (unlikely(len > (unsigned long)ctx)) {
 616                 pr_debug("%s: rx error: len %u exceeds truesize 0x%lu\n",
 617                          dev->name, len, (unsigned long)ctx);
 618                 dev->stats.rx_length_errors++;
 619                 goto err_skb;
 620         }
 621         truesize = (unsigned long)ctx;
 622         head_skb = page_to_skb(vi, rq, page, offset, len, truesize);
 623         curr_skb = head_skb;
 624
 625         if (unlikely(!curr_skb))
 626                 goto err_skb;
 627         while (--num_buf) {
 628                 int num_skb_frags;
 629
 630                 buf = virtqueue_get_buf_ctx(rq->vq, &len, &ctx);
 631                 if (unlikely(!ctx)) {
 632                         pr_debug("%s: rx error: %d buffers out of %d missing\n",
 633                                  dev->name, num_buf,
 634                                  virtio16_to_cpu(vi->vdev,
 635                                                  hdr->num_buffers));
 636                         dev->stats.rx_length_errors++;
 637                         goto err_buf;
 638                 }
 639
 640                 page = virt_to_head_page(buf);
 641                 if (unlikely(len > (unsigned long)ctx)) {
 642                         pr_debug("%s: rx error: len %u exceeds truesize 0x%lu\n",
 643                                  dev->name, len, (unsigned long)ctx);
 644                         dev->stats.rx_length_errors++;
 645                         goto err_skb;
 646                 }
 647                 truesize = (unsigned long)ctx;
 648
 649                 num_skb_frags = skb_shinfo(curr_skb)->nr_frags;
 650                 if (unlikely(num_skb_frags == MAX_SKB_FRAGS)) {
 651                         struct sk_buff *nskb = alloc_skb(0, GFP_ATOMIC);
 652
 653                         if (unlikely(!nskb))
 654                                 goto err_skb;
 655                         if (curr_skb == head_skb)
 656                                 skb_shinfo(curr_skb)->frag_list = nskb;
 657                         else
 658                                 curr_skb->next = nskb;
 659                         curr_skb = nskb;
 660                         head_skb->truesize += nskb->truesize;
 661                         num_skb_frags = 0;
 662                 }
 663                 if (curr_skb != head_skb) {
 664                         head_skb->data_len += len;
 665                         head_skb->len += len;
 666                         head_skb->truesize += truesize;
 667                 }
 668                 offset = buf - page_address(page);
 669                 if (skb_can_coalesce(curr_skb, num_skb_frags, page, offset)) {
 670                         put_page(page);
 671                         skb_coalesce_rx_frag(curr_skb, num_skb_frags - 1,
 672                                              len, truesize);
 673                 } else {
 674                         skb_add_rx_frag(curr_skb, num_skb_frags, page,
 675                                         offset, len, truesize);
 676                 }
 677         }
 678
 679         ewma_pkt_len_add(&rq->mrg_avg_pkt_len, head_skb->len);
 680         return head_skb;
 681
 682 err_xdp:
 683         rcu_read_unlock();
 684 err_skb:
 685         put_page(page);
 686         while (--num_buf) {
 687                 buf = virtqueue_get_buf(rq->vq, &len);
 688                 if (unlikely(!buf)) {
 689                         pr_debug("%s: rx error: %d buffers missing\n",
 690                                  dev->name, num_buf);
 691                         dev->stats.rx_length_errors++;
 692                         break;
 693                 }
 694                 page = virt_to_head_page(buf);
 695                 put_page(page);
 696         }
 697 err_buf:
 698         dev->stats.rx_dropped++;
 699         dev_kfree_skb(head_skb);
 700 xdp_xmit:
 701         return NULL;
 702 }
 703
 704 static int receive_buf(struct virtnet_info *vi, struct receive_queue *rq,
 705                        void *buf, unsigned int len, void **ctx)
 706 {
 707         struct net_device *dev = vi->dev;
 708         struct sk_buff *skb;
 709         struct virtio_net_hdr_mrg_rxbuf *hdr;
 710         int ret;
 711
 712         if (unlikely(len < vi->hdr_len + ETH_HLEN)) {
 713                 pr_debug("%s: short packet %i\n", dev->name, len);
 714                 dev->stats.rx_length_errors++;
 715                 if (vi->mergeable_rx_bufs) {
 716                         put_page(virt_to_head_page(buf));
 717                 } else if (vi->big_packets) {
 718                         give_pages(rq, buf);
 719                 } else {
 720                         put_page(virt_to_head_page(buf));
 721                 }
 722                 return 0;
 723         }
 724
 725         if (vi->mergeable_rx_bufs)
 726                 skb = receive_mergeable(dev, vi, rq, buf, ctx, len);
 727         else if (vi->big_packets)
 728                 skb = receive_big(dev, vi, rq, buf, len);
 729         else
 730                 skb = receive_small(dev, vi, rq, buf, len);
 731
 732         if (unlikely(!skb))
 733                 return 0;
 734
 735         hdr = skb_vnet_hdr(skb);
 736
 737         ret = skb->len;
 738
 739         if (hdr->hdr.flags & VIRTIO_NET_HDR_F_DATA_VALID)
 740                 skb->ip_summed = CHECKSUM_UNNECESSARY;
 741
 742         if (virtio_net_hdr_to_skb(skb, &hdr->hdr,
 743                                   virtio_is_little_endian(vi->vdev))) {
 744                 net_warn_ratelimited("%s: bad gso: type: %u, size: %u\n",
 745                                      dev->name, hdr->hdr.gso_type,
 746                                      hdr->hdr.gso_size);
 747                 goto frame_err;
 748         }
 749
 750         skb->protocol = eth_type_trans(skb, dev);
 751         pr_debug("Receiving skb proto 0x%04x len %i type %i\n",
 752                  ntohs(skb->protocol), skb->len, skb->pkt_type);
 753
 754         napi_gro_receive(&rq->napi, skb);
 755         return ret;
 756
 757 frame_err:
 758         dev->stats.rx_frame_errors++;
 759         dev_kfree_skb(skb);
 760         return 0;
 761 }
 762
 763 static int add_recvbuf_small(struct virtnet_info *vi, struct receive_queue *rq,
 764                              gfp_t gfp)
 765 {
 766         struct page_frag *alloc_frag = &rq->alloc_frag;
 767         char *buf;
 768         unsigned int xdp_headroom = virtnet_get_headroom(vi);
 769         int len = vi->hdr_len + VIRTNET_RX_PAD + GOOD_PACKET_LEN + xdp_headroom;
 770         int err;
 771
 772         len = SKB_DATA_ALIGN(len) +
 773               SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
 774         if (unlikely(!skb_page_frag_refill(len, alloc_frag, gfp)))
 775                 return -ENOMEM;
 776
 777         buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
 778         get_page(alloc_frag->page);
 779         alloc_frag->offset += len;
 780         sg_init_one(rq->sg, buf + VIRTNET_RX_PAD + xdp_headroom,
 781                     vi->hdr_len + GOOD_PACKET_LEN);
 782         err = virtqueue_add_inbuf(rq->vq, rq->sg, 1, buf, gfp);
 783         if (err < 0)
 784                 put_page(virt_to_head_page(buf));
 785
 786         return err;
 787 }
 788
 789 static int add_recvbuf_big(struct virtnet_info *vi, struct receive_queue *rq,
 790                            gfp_t gfp)
 791 {
 792         struct page *first, *list = NULL;
 793         char *p;
 794         int i, err, offset;
 795
 796         sg_init_table(rq->sg, MAX_SKB_FRAGS + 2);
 797
 798         /* page in rq->sg[MAX_SKB_FRAGS + 1] is list tail */
 799         for (i = MAX_SKB_FRAGS + 1; i > 1; --i) {
 800                 first = get_a_page(rq, gfp);
 801                 if (!first) {
 802                         if (list)
 803                                 give_pages(rq, list);
 804                         return -ENOMEM;
 805                 }
 806                 sg_set_buf(&rq->sg[i], page_address(first), PAGE_SIZE);
 807
 808                 /* chain new page in list head to match sg */
 809                 first->private = (unsigned long)list;
 810                 list = first;
 811         }
 812
 813         first = get_a_page(rq, gfp);
 814         if (!first) {
 815                 give_pages(rq, list);
 816                 return -ENOMEM;
 817         }
 818         p = page_address(first);
 819
 820         /* rq->sg[0], rq->sg[1] share the same page */
 821         /* a separated rq->sg[0] for header - required in case !any_header_sg */
 822         sg_set_buf(&rq->sg[0], p, vi->hdr_len);
 823
 824         /* rq->sg[1] for data packet, from offset */
 825         offset = sizeof(struct padded_vnet_hdr);
 826         sg_set_buf(&rq->sg[1], p + offset, PAGE_SIZE - offset);
 827
 828         /* chain first in list head */
 829         first->private = (unsigned long)list;
 830         err = virtqueue_add_inbuf(rq->vq, rq->sg, MAX_SKB_FRAGS + 2,
 831                                   first, gfp);
 832         if (err < 0)
 833                 give_pages(rq, first);
 834
 835         return err;
 836 }
 837
 838 static unsigned int get_mergeable_buf_len(struct receive_queue *rq,
 839                                           struct ewma_pkt_len *avg_pkt_len)
 840 {
 841         const size_t hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);
 842         unsigned int len;
 843
 844         len = hdr_len + clamp_t(unsigned int, ewma_pkt_len_read(avg_pkt_len),
 845                                 rq->min_buf_len - hdr_len, PAGE_SIZE - hdr_len);
 846         return ALIGN(len, L1_CACHE_BYTES);
 847 }
 848
 849 static int add_recvbuf_mergeable(struct virtnet_info *vi,
 850                                  struct receive_queue *rq, gfp_t gfp)
 851 {
 852         struct page_frag *alloc_frag = &rq->alloc_frag;
 853         unsigned int headroom = virtnet_get_headroom(vi);
 854         char *buf;
 855         void *ctx;
 856         int err;
 857         unsigned int len, hole;
 858
 859         len = get_mergeable_buf_len(rq, &rq->mrg_avg_pkt_len);
 860         if (unlikely(!skb_page_frag_refill(len + headroom, alloc_frag, gfp)))
 861                 return -ENOMEM;
 862
 863         buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
 864         buf += headroom; /* advance address leaving hole at front of pkt */
 865         ctx = (void *)(unsigned long)len;
 866         get_page(alloc_frag->page);
 867         alloc_frag->offset += len + headroom;
 868         hole = alloc_frag->size - alloc_frag->offset;
 869         if (hole < len + headroom) {
 870                 /* To avoid internal fragmentation, if there is very likely not
 871                  * enough space for another buffer, add the remaining space to
 872                  * the current buffer. This extra space is not included in
 873                  * the truesize stored in ctx.
 874                  */
 875                 len += hole;
 876                 alloc_frag->offset += hole;
 877         }
 878
 879         sg_init_one(rq->sg, buf, len);
 880         err = virtqueue_add_inbuf_ctx(rq->vq, rq->sg, 1, buf, ctx, gfp);
 881         if (err < 0)
 882                 put_page(virt_to_head_page(buf));
 883
 884         return err;
 885 }
 886
 887 /*
 888  * Returns false if we couldn't fill entirely (OOM).
 889  *
 890  * Normally run in the receive path, but can also be run from ndo_open
 891  * before we're receiving packets, or from refill_work which is
 892  * careful to disable receiving (using napi_disable).
 893  */
 894 static bool try_fill_recv(struct virtnet_info *vi, struct receive_queue *rq,
 895                           gfp_t gfp)
 896 {
 897         int err;
 898         bool oom;
 899
 900         gfp |= __GFP_COLD;
 901         do {
 902                 if (vi->mergeable_rx_bufs)
 903                         err = add_recvbuf_mergeable(vi, rq, gfp);
 904                 else if (vi->big_packets)
 905                         err = add_recvbuf_big(vi, rq, gfp);
 906                 else
 907                         err = add_recvbuf_small(vi, rq, gfp);
 908
 909                 oom = err == -ENOMEM;
 910                 if (err)
 911                         break;
 912         } while (rq->vq->num_free);
 913         virtqueue_kick(rq->vq);
 914         return !oom;
 915 }
 916
 917 static void skb_recv_done(struct virtqueue *rvq)
 918 {
 919         struct virtnet_info *vi = rvq->vdev->priv;
 920         struct receive_queue *rq = &vi->rq[vq2rxq(rvq)];
 921
 922         /* Schedule NAPI, Suppress further interrupts if successful. */
 923         if (napi_schedule_prep(&rq->napi)) {
 924                 virtqueue_disable_cb(rvq);
 925                 __napi_schedule(&rq->napi);
 926         }
 927 }
 928
 929 static void virtnet_napi_enable(struct receive_queue *rq)
 930 {
 931         napi_enable(&rq->napi);
 932
 933         /* If all buffers were filled by other side before we napi_enabled, we
 934          * won't get another interrupt, so process any outstanding packets
 935          * now.  virtnet_poll wants re-enable the queue, so we disable here.
 936          * We synchronize against interrupts via NAPI_STATE_SCHED */
 937         if (napi_schedule_prep(&rq->napi)) {
 938                 virtqueue_disable_cb(rq->vq);
 939                 local_bh_disable();
 940                 __napi_schedule(&rq->napi);
 941                 local_bh_enable();
 942         }
 943 }
 944
 945 static void refill_work(struct work_struct *work)
 946 {
 947         struct virtnet_info *vi =
 948                 container_of(work, struct virtnet_info, refill.work);
 949         bool still_empty;
 950         int i;
 951
 952         for (i = 0; i < vi->curr_queue_pairs; i++) {
 953                 struct receive_queue *rq = &vi->rq[i];
 954
 955                 napi_disable(&rq->napi);
 956                 still_empty = !try_fill_recv(vi, rq, GFP_KERNEL);
 957                 virtnet_napi_enable(rq);
 958
 959                 /* In theory, this can happen: if we don't get any buffers in
 960                  * we will *never* try to fill again.
 961                  */
 962                 if (still_empty)
 963                         schedule_delayed_work(&vi->refill, HZ/2);
 964         }
 965 }
 966
 967 static int virtnet_receive(struct receive_queue *rq, int budget)
 968 {
 969         struct virtnet_info *vi = rq->vq->vdev->priv;
 970         unsigned int len, received = 0, bytes = 0;
 971         void *buf;
 972         struct virtnet_stats *stats = this_cpu_ptr(vi->stats);
 973
 974         if (vi->mergeable_rx_bufs) {
 975                 void *ctx;
 976
 977                 while (received < budget &&
 978                        (buf = virtqueue_get_buf_ctx(rq->vq, &len, &ctx))) {
 979                         bytes += receive_buf(vi, rq, buf, len, ctx);
 980                         received++;
 981                 }
 982         } else {
 983                 while (received < budget &&
 984                        (buf = virtqueue_get_buf(rq->vq, &len)) != NULL) {
 985                         bytes += receive_buf(vi, rq, buf, len, NULL);
 986                         received++;
 987                 }
 988         }
 989
 990         if (rq->vq->num_free > virtqueue_get_vring_size(rq->vq) / 2) {
 991                 if (!try_fill_recv(vi, rq, GFP_ATOMIC))
 992                         schedule_delayed_work(&vi->refill, 0);
 993         }
 994
 995         u64_stats_update_begin(&stats->rx_syncp);
 996         stats->rx_bytes += bytes;
 997         stats->rx_packets += received;
 998         u64_stats_update_end(&stats->rx_syncp);
 999
1000         return received;
1001 }
1002
1003 static int virtnet_poll(struct napi_struct *napi, int budget)
1004 {
1005         struct receive_queue *rq =
1006                 container_of(napi, struct receive_queue, napi);
1007         unsigned int r, received;
1008
1009         received = virtnet_receive(rq, budget);
1010
1011         /* Out of packets? */
1012         if (received < budget) {
1013                 r = virtqueue_enable_cb_prepare(rq->vq);
1014                 if (napi_complete_done(napi, received)) {
1015                         if (unlikely(virtqueue_poll(rq->vq, r)) &&
1016                             napi_schedule_prep(napi)) {
1017                                 virtqueue_disable_cb(rq->vq);
1018                                 __napi_schedule(napi);
1019                         }
1020                 }
1021         }
1022
1023         return received;
1024 }
1025
1026 static int virtnet_open(struct net_device *dev)
1027 {
1028         struct virtnet_info *vi = netdev_priv(dev);
1029         int i;
1030
1031         for (i = 0; i < vi->max_queue_pairs; i++) {
1032                 if (i < vi->curr_queue_pairs)
1033                         /* Make sure we have some buffers: if oom use wq. */
1034                         if (!try_fill_recv(vi, &vi->rq[i], GFP_KERNEL))
1035                                 schedule_delayed_work(&vi->refill, 0);
1036                 virtnet_napi_enable(&vi->rq[i]);
1037         }
1038
1039         return 0;
1040 }
1041
1042 static void free_old_xmit_skbs(struct send_queue *sq)
1043 {
1044         struct sk_buff *skb;
1045         unsigned int len;
1046         struct virtnet_info *vi = sq->vq->vdev->priv;
1047         struct virtnet_stats *stats = this_cpu_ptr(vi->stats);
1048         unsigned int packets = 0;
1049         unsigned int bytes = 0;
1050
1051         while ((skb = virtqueue_get_buf(sq->vq, &len)) != NULL) {
1052                 pr_debug("Sent skb %p\n", skb);
1053
1054                 bytes += skb->len;
1055                 packets++;
1056
1057                 dev_kfree_skb_any(skb);
1058         }
1059
1060         /* Avoid overhead when no packets have been processed
1061          * happens when called speculatively from start_xmit.
1062          */
1063         if (!packets)
1064                 return;
1065
1066         u64_stats_update_begin(&stats->tx_syncp);
1067         stats->tx_bytes += bytes;
1068         stats->tx_packets += packets;
1069         u64_stats_update_end(&stats->tx_syncp);
1070 }
1071
1072 static int xmit_skb(struct send_queue *sq, struct sk_buff *skb)
1073 {
1074         struct virtio_net_hdr_mrg_rxbuf *hdr;
1075         const unsigned char *dest = ((struct ethhdr *)skb->data)->h_dest;
1076         struct virtnet_info *vi = sq->vq->vdev->priv;
1077         unsigned num_sg;
1078         unsigned hdr_len = vi->hdr_len;
1079         bool can_push;
1080
1081         pr_debug("%s: xmit %p %pM\n", vi->dev->name, skb, dest);
1082
1083         can_push = vi->any_header_sg &&
1084                 !((unsigned long)skb->data & (__alignof__(*hdr) - 1)) &&
1085                 !skb_header_cloned(skb) && skb_headroom(skb) >= hdr_len;
1086         /* Even if we can, don't push here yet as this would skew
1087          * csum_start offset below. */
1088         if (can_push)
1089                 hdr = (struct virtio_net_hdr_mrg_rxbuf *)(skb->data - hdr_len);
1090         else
1091                 hdr = skb_vnet_hdr(skb);
1092
1093         if (virtio_net_hdr_from_skb(skb, &hdr->hdr,
1094                                     virtio_is_little_endian(vi->vdev), false))
1095                 BUG();
1096
1097         if (vi->mergeable_rx_bufs)
1098                 hdr->num_buffers = 0;
1099
1100         sg_init_table(sq->sg, skb_shinfo(skb)->nr_frags + (can_push ? 1 : 2));
1101         if (can_push) {
1102                 __skb_push(skb, hdr_len);
1103                 num_sg = skb_to_sgvec(skb, sq->sg, 0, skb->len);
1104                 /* Pull header back to avoid skew in tx bytes calculations. */
1105                 __skb_pull(skb, hdr_len);
1106         } else {
1107                 sg_set_buf(sq->sg, hdr, hdr_len);
1108                 num_sg = skb_to_sgvec(skb, sq->sg + 1, 0, skb->len) + 1;
1109         }
1110         return virtqueue_add_outbuf(sq->vq, sq->sg, num_sg, skb, GFP_ATOMIC);
1111 }
1112
1113 static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev)
1114 {
1115         struct virtnet_info *vi = netdev_priv(dev);
1116         int qnum = skb_get_queue_mapping(skb);
1117         struct send_queue *sq = &vi->sq[qnum];
1118         int err;
1119         struct netdev_queue *txq = netdev_get_tx_queue(dev, qnum);
1120         bool kick = !skb->xmit_more;
1121
1122         /* Free up any pending old buffers before queueing new ones. */
1123         free_old_xmit_skbs(sq);
1124
1125         /* timestamp packet in software */
1126         skb_tx_timestamp(skb);
1127
1128         /* Try to transmit */
1129         err = xmit_skb(sq, skb);
1130
1131         /* This should not happen! */
1132         if (unlikely(err)) {
1133                 dev->stats.tx_fifo_errors++;
1134                 if (net_ratelimit())
1135                         dev_warn(&dev->dev,
1136                                  "Unexpected TXQ (%d) queue failure: %d\n", qnum, err);
1137                 dev->stats.tx_dropped++;
1138                 dev_kfree_skb_any(skb);
1139                 return NETDEV_TX_OK;
1140         }
1141
1142         /* Don't wait up for transmitted skbs to be freed. */
1143         skb_orphan(skb);
1144         nf_reset(skb);
1145
1146         /* If running out of space, stop queue to avoid getting packets that we
1147          * are then unable to transmit.
1148          * An alternative would be to force queuing layer to requeue the skb by
1149          * returning NETDEV_TX_BUSY. However, NETDEV_TX_BUSY should not be
1150          * returned in a normal path of operation: it means that driver is not
1151          * maintaining the TX queue stop/start state properly, and causes
1152          * the stack to do a non-trivial amount of useless work.
1153          * Since most packets only take 1 or 2 ring slots, stopping the queue
1154          * early means 16 slots are typically wasted.
1155          */
1156         if (sq->vq->num_free < 2+MAX_SKB_FRAGS) {
1157                 netif_stop_subqueue(dev, qnum);
1158                 if (unlikely(!virtqueue_enable_cb_delayed(sq->vq))) {
1159                         /* More just got used, free them then recheck. */
1160                         free_old_xmit_skbs(sq);
1161                         if (sq->vq->num_free >= 2+MAX_SKB_FRAGS) {
1162                                 netif_start_subqueue(dev, qnum);
1163                                 virtqueue_disable_cb(sq->vq);
1164                         }
1165                 }
1166         }
1167
1168         if (kick || netif_xmit_stopped(txq))
1169                 virtqueue_kick(sq->vq);
1170
1171         return NETDEV_TX_OK;
1172 }
1173
1174 /*
1175  * Send command via the control virtqueue and check status.  Commands
1176  * supported by the hypervisor, as indicated by feature bits, should
1177  * never fail unless improperly formatted.
1178  */
1179 static bool virtnet_send_command(struct virtnet_info *vi, u8 class, u8 cmd,
1180                                  struct scatterlist *out)
1181 {
1182         struct scatterlist *sgs[4], hdr, stat;
1183         unsigned out_num = 0, tmp;
1184
1185         /* Caller should know better */
1186         BUG_ON(!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ));
1187
1188         vi->ctrl_status = ~0;
1189         vi->ctrl_hdr.class = class;
1190         vi->ctrl_hdr.cmd = cmd;
1191         /* Add header */
1192         sg_init_one(&hdr, &vi->ctrl_hdr, sizeof(vi->ctrl_hdr));
1193         sgs[out_num++] = &hdr;
1194
1195         if (out)
1196                 sgs[out_num++] = out;
1197
1198         /* Add return status. */
1199         sg_init_one(&stat, &vi->ctrl_status, sizeof(vi->ctrl_status));
1200         sgs[out_num] = &stat;
1201
1202         BUG_ON(out_num + 1 > ARRAY_SIZE(sgs));
1203         virtqueue_add_sgs(vi->cvq, sgs, out_num, 1, vi, GFP_ATOMIC);
1204
1205         if (unlikely(!virtqueue_kick(vi->cvq)))
1206                 return vi->ctrl_status == VIRTIO_NET_OK;
1207
1208         /* Spin for a response, the kick causes an ioport write, trapping
1209          * into the hypervisor, so the request should be handled immediately.
1210          */
1211         while (!virtqueue_get_buf(vi->cvq, &tmp) &&
1212                !virtqueue_is_broken(vi->cvq))
1213                 cpu_relax();
1214
1215         return vi->ctrl_status == VIRTIO_NET_OK;
1216 }
1217
1218 static int virtnet_set_mac_address(struct net_device *dev, void *p)
1219 {
1220         struct virtnet_info *vi = netdev_priv(dev);
1221         struct virtio_device *vdev = vi->vdev;
1222         int ret;
1223         struct sockaddr *addr;
1224         struct scatterlist sg;
1225
1226         addr = kmemdup(p, sizeof(*addr), GFP_KERNEL);
1227         if (!addr)
1228                 return -ENOMEM;
1229
1230         ret = eth_prepare_mac_addr_change(dev, addr);
1231         if (ret)
1232                 goto out;
1233
1234         if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR)) {
1235                 sg_init_one(&sg, addr->sa_data, dev->addr_len);
1236                 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC,
1237                                           VIRTIO_NET_CTRL_MAC_ADDR_SET, &sg)) {
1238                         dev_warn(&vdev->dev,
1239                                  "Failed to set mac address by vq command.\n");
1240                         ret = -EINVAL;
1241                         goto out;
1242                 }
1243         } else if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC) &&
1244                    !virtio_has_feature(vdev, VIRTIO_F_VERSION_1)) {
1245                 unsigned int i;
1246
1247                 /* Naturally, this has an atomicity problem. */
1248                 for (i = 0; i < dev->addr_len; i++)
1249                         virtio_cwrite8(vdev,
1250                                        offsetof(struct virtio_net_config, mac) +
1251                                        i, addr->sa_data[i]);
1252         }
1253
1254         eth_commit_mac_addr_change(dev, p);
1255         ret = 0;
1256
1257 out:
1258         kfree(addr);
1259         return ret;
1260 }
1261
1262 static void virtnet_stats(struct net_device *dev,
1263                           struct rtnl_link_stats64 *tot)
1264 {
1265         struct virtnet_info *vi = netdev_priv(dev);
1266         int cpu;
1267         unsigned int start;
1268
1269         for_each_possible_cpu(cpu) {
1270                 struct virtnet_stats *stats = per_cpu_ptr(vi->stats, cpu);
1271                 u64 tpackets, tbytes, rpackets, rbytes;
1272
1273                 do {
1274                         start = u64_stats_fetch_begin_irq(&stats->tx_syncp);
1275                         tpackets = stats->tx_packets;
1276                         tbytes   = stats->tx_bytes;
1277                 } while (u64_stats_fetch_retry_irq(&stats->tx_syncp, start));
1278
1279                 do {
1280                         start = u64_stats_fetch_begin_irq(&stats->rx_syncp);
1281                         rpackets = stats->rx_packets;
1282                         rbytes   = stats->rx_bytes;
1283                 } while (u64_stats_fetch_retry_irq(&stats->rx_syncp, start));
1284
1285                 tot->rx_packets += rpackets;
1286                 tot->tx_packets += tpackets;
1287                 tot->rx_bytes   += rbytes;
1288                 tot->tx_bytes   += tbytes;
1289         }
1290
1291         tot->tx_dropped = dev->stats.tx_dropped;
1292         tot->tx_fifo_errors = dev->stats.tx_fifo_errors;
1293         tot->rx_dropped = dev->stats.rx_dropped;
1294         tot->rx_length_errors = dev->stats.rx_length_errors;
1295         tot->rx_frame_errors = dev->stats.rx_frame_errors;
1296 }
1297
1298 #ifdef CONFIG_NET_POLL_CONTROLLER
1299 static void virtnet_netpoll(struct net_device *dev)
1300 {
1301         struct virtnet_info *vi = netdev_priv(dev);
1302         int i;
1303
1304         for (i = 0; i < vi->curr_queue_pairs; i++)
1305                 napi_schedule(&vi->rq[i].napi);
1306 }
1307 #endif
1308
1309 static void virtnet_ack_link_announce(struct virtnet_info *vi)
1310 {
1311         rtnl_lock();
1312         if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_ANNOUNCE,
1313                                   VIRTIO_NET_CTRL_ANNOUNCE_ACK, NULL))
1314                 dev_warn(&vi->dev->dev, "Failed to ack link announce.\n");
1315         rtnl_unlock();
1316 }
1317
1318 static int _virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs)
1319 {
1320         struct scatterlist sg;
1321         struct net_device *dev = vi->dev;
1322
1323         if (!vi->has_cvq || !virtio_has_feature(vi->vdev, VIRTIO_NET_F_MQ))
1324                 return 0;
1325
1326         vi->ctrl_mq.virtqueue_pairs = cpu_to_virtio16(vi->vdev, queue_pairs);
1327         sg_init_one(&sg, &vi->ctrl_mq, sizeof(vi->ctrl_mq));
1328
1329         if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MQ,
1330                                   VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET, &sg)) {
1331                 dev_warn(&dev->dev, "Fail to set num of queue pairs to %d\n",
1332                          queue_pairs);
1333                 return -EINVAL;
1334         } else {
1335                 vi->curr_queue_pairs = queue_pairs;
1336                 /* virtnet_open() will refill when device is going to up. */
1337                 if (dev->flags & IFF_UP)
1338                         schedule_delayed_work(&vi->refill, 0);
1339         }
1340
1341         return 0;
1342 }
1343
1344 static int virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs)
1345 {
1346         int err;
1347
1348         rtnl_lock();
1349         err = _virtnet_set_queues(vi, queue_pairs);
1350         rtnl_unlock();
1351         return err;
1352 }
1353
1354 static int virtnet_close(struct net_device *dev)
1355 {
1356         struct virtnet_info *vi = netdev_priv(dev);
1357         int i;
1358
1359         /* Make sure refill_work doesn't re-enable napi! */
1360         cancel_delayed_work_sync(&vi->refill);
1361
1362         for (i = 0; i < vi->max_queue_pairs; i++)
1363                 napi_disable(&vi->rq[i].napi);
1364
1365         return 0;
1366 }
1367
1368 static void virtnet_set_rx_mode(struct net_device *dev)
1369 {
1370         struct virtnet_info *vi = netdev_priv(dev);
1371         struct scatterlist sg[2];
1372         struct virtio_net_ctrl_mac *mac_data;
1373         struct netdev_hw_addr *ha;
1374         int uc_count;
1375         int mc_count;
1376         void *buf;
1377         int i;
1378
1379         /* We can't dynamically set ndo_set_rx_mode, so return gracefully */
1380         if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_RX))
1381                 return;
1382
1383         vi->ctrl_promisc = ((dev->flags & IFF_PROMISC) != 0);
1384         vi->ctrl_allmulti = ((dev->flags & IFF_ALLMULTI) != 0);
1385
1386         sg_init_one(sg, &vi->ctrl_promisc, sizeof(vi->ctrl_promisc));
1387
1388         if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,
1389                                   VIRTIO_NET_CTRL_RX_PROMISC, sg))
1390                 dev_warn(&dev->dev, "Failed to %sable promisc mode.\n",
1391                          vi->ctrl_promisc ? "en" : "dis");
1392
1393         sg_init_one(sg, &vi->ctrl_allmulti, sizeof(vi->ctrl_allmulti));
1394
1395         if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,
1396                                   VIRTIO_NET_CTRL_RX_ALLMULTI, sg))
1397                 dev_warn(&dev->dev, "Failed to %sable allmulti mode.\n",
1398                          vi->ctrl_allmulti ? "en" : "dis");
1399
1400         uc_count = netdev_uc_count(dev);
1401         mc_count = netdev_mc_count(dev);
1402         /* MAC filter - use one buffer for both lists */
1403         buf = kzalloc(((uc_count + mc_count) * ETH_ALEN) +
1404                       (2 * sizeof(mac_data->entries)), GFP_ATOMIC);
1405         mac_data = buf;
1406         if (!buf)
1407                 return;
1408
1409         sg_init_table(sg, 2);
1410
1411         /* Store the unicast list and count in the front of the buffer */
1412         mac_data->entries = cpu_to_virtio32(vi->vdev, uc_count);
1413         i = 0;
1414         netdev_for_each_uc_addr(ha, dev)
1415                 memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN);
1416
1417         sg_set_buf(&sg[0], mac_data,
1418                    sizeof(mac_data->entries) + (uc_count * ETH_ALEN));
1419
1420         /* multicast list and count fill the end */
1421         mac_data = (void *)&mac_data->macs[uc_count][0];
1422
1423         mac_data->entries = cpu_to_virtio32(vi->vdev, mc_count);
1424         i = 0;
1425         netdev_for_each_mc_addr(ha, dev)
1426                 memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN);
1427
1428         sg_set_buf(&sg[1], mac_data,
1429                    sizeof(mac_data->entries) + (mc_count * ETH_ALEN));
1430
1431         if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC,
1432                                   VIRTIO_NET_CTRL_MAC_TABLE_SET, sg))
1433                 dev_warn(&dev->dev, "Failed to set MAC filter table.\n");
1434
1435         kfree(buf);
1436 }
1437
1438 static int virtnet_vlan_rx_add_vid(struct net_device *dev,
1439                                    __be16 proto, u16 vid)
1440 {
1441         struct virtnet_info *vi = netdev_priv(dev);
1442         struct scatterlist sg;
1443
1444         vi->ctrl_vid = vid;
1445         sg_init_one(&sg, &vi->ctrl_vid, sizeof(vi->ctrl_vid));
1446
1447         if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,
1448                                   VIRTIO_NET_CTRL_VLAN_ADD, &sg))
1449                 dev_warn(&dev->dev, "Failed to add VLAN ID %d.\n", vid);
1450         return 0;
1451 }
1452
1453 static int virtnet_vlan_rx_kill_vid(struct net_device *dev,
1454                                     __be16 proto, u16 vid)
1455 {
1456         struct virtnet_info *vi = netdev_priv(dev);
1457         struct scatterlist sg;
1458
1459         vi->ctrl_vid = vid;
1460         sg_init_one(&sg, &vi->ctrl_vid, sizeof(vi->ctrl_vid));
1461
1462         if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,
1463                                   VIRTIO_NET_CTRL_VLAN_DEL, &sg))
1464                 dev_warn(&dev->dev, "Failed to kill VLAN ID %d.\n", vid);
1465         return 0;
1466 }
1467
1468 static void virtnet_clean_affinity(struct virtnet_info *vi, long hcpu)
1469 {
1470         int i;
1471
1472         if (vi->affinity_hint_set) {
1473                 for (i = 0; i < vi->max_queue_pairs; i++) {
1474                         virtqueue_set_affinity(vi->rq[i].vq, -1);
1475                         virtqueue_set_affinity(vi->sq[i].vq, -1);
1476                 }
1477
1478                 vi->affinity_hint_set = false;
1479         }
1480 }
1481
1482 static void virtnet_set_affinity(struct virtnet_info *vi)
1483 {
1484         int i;
1485         int cpu;
1486
1487         /* In multiqueue mode, when the number of cpu is equal to the number of
1488          * queue pairs, we let the queue pairs to be private to one cpu by
1489          * setting the affinity hint to eliminate the contention.
1490          */
1491         if (vi->curr_queue_pairs == 1 ||
1492             vi->max_queue_pairs != num_online_cpus()) {
1493                 virtnet_clean_affinity(vi, -1);
1494                 return;
1495         }
1496
1497         i = 0;
1498         for_each_online_cpu(cpu) {
1499                 virtqueue_set_affinity(vi->rq[i].vq, cpu);
1500                 virtqueue_set_affinity(vi->sq[i].vq, cpu);
1501                 netif_set_xps_queue(vi->dev, cpumask_of(cpu), i);
1502                 i++;
1503         }
1504
1505         vi->affinity_hint_set = true;
1506 }
1507
1508 static int virtnet_cpu_online(unsigned int cpu, struct hlist_node *node)
1509 {
1510         struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info,
1511                                                    node);
1512         virtnet_set_affinity(vi);
1513         return 0;
1514 }
1515
1516 static int virtnet_cpu_dead(unsigned int cpu, struct hlist_node *node)
1517 {
1518         struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info,
1519                                                    node_dead);
1520         virtnet_set_affinity(vi);
1521         return 0;
1522 }
1523
1524 static int virtnet_cpu_down_prep(unsigned int cpu, struct hlist_node *node)
1525 {
1526         struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info,
1527                                                    node);
1528
1529         virtnet_clean_affinity(vi, cpu);
1530         return 0;
1531 }
1532
1533 static enum cpuhp_state virtionet_online;
1534
1535 static int virtnet_cpu_notif_add(struct virtnet_info *vi)
1536 {
1537         int ret;
1538
1539         ret = cpuhp_state_add_instance_nocalls(virtionet_online, &vi->node);
1540         if (ret)
1541                 return ret;
1542         ret = cpuhp_state_add_instance_nocalls(CPUHP_VIRT_NET_DEAD,
1543                                                &vi->node_dead);
1544         if (!ret)
1545                 return ret;
1546         cpuhp_state_remove_instance_nocalls(virtionet_online, &vi->node);
1547         return ret;
1548 }
1549
1550 static void virtnet_cpu_notif_remove(struct virtnet_info *vi)
1551 {
1552         cpuhp_state_remove_instance_nocalls(virtionet_online, &vi->node);
1553         cpuhp_state_remove_instance_nocalls(CPUHP_VIRT_NET_DEAD,
1554                                             &vi->node_dead);
1555 }
1556
1557 static void virtnet_get_ringparam(struct net_device *dev,
1558                                 struct ethtool_ringparam *ring)
1559 {
1560         struct virtnet_info *vi = netdev_priv(dev);
1561
1562         ring->rx_max_pending = virtqueue_get_vring_size(vi->rq[0].vq);
1563         ring->tx_max_pending = virtqueue_get_vring_size(vi->sq[0].vq);
1564         ring->rx_pending = ring->rx_max_pending;
1565         ring->tx_pending = ring->tx_max_pending;
1566 }
1567
1568
1569 static void virtnet_get_drvinfo(struct net_device *dev,
1570                                 struct ethtool_drvinfo *info)
1571 {
1572         struct virtnet_info *vi = netdev_priv(dev);
1573         struct virtio_device *vdev = vi->vdev;
1574
1575         strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
1576         strlcpy(info->version, VIRTNET_DRIVER_VERSION, sizeof(info->version));
1577         strlcpy(info->bus_info, virtio_bus_name(vdev), sizeof(info->bus_info));
1578
1579 }
1580
1581 /* TODO: Eliminate OOO packets during switching */
1582 static int virtnet_set_channels(struct net_device *dev,
1583                                 struct ethtool_channels *channels)
1584 {
1585         struct virtnet_info *vi = netdev_priv(dev);
1586         u16 queue_pairs = channels->combined_count;
1587         int err;
1588
1589         /* We don't support separate rx/tx channels.
1590          * We don't allow setting 'other' channels.
1591          */
1592         if (channels->rx_count || channels->tx_count || channels->other_count)
1593                 return -EINVAL;
1594
1595         if (queue_pairs > vi->max_queue_pairs || queue_pairs == 0)
1596                 return -EINVAL;
1597
1598         /* For now we don't support modifying channels while XDP is loaded
1599          * also when XDP is loaded all RX queues have XDP programs so we only
1600          * need to check a single RX queue.
1601          */
1602         if (vi->rq[0].xdp_prog)
1603                 return -EINVAL;
1604
1605         get_online_cpus();
1606         err = _virtnet_set_queues(vi, queue_pairs);
1607         if (!err) {
1608                 netif_set_real_num_tx_queues(dev, queue_pairs);
1609                 netif_set_real_num_rx_queues(dev, queue_pairs);
1610
1611                 virtnet_set_affinity(vi);
1612         }
1613         put_online_cpus();
1614
1615         return err;
1616 }
1617
1618 static void virtnet_get_channels(struct net_device *dev,
1619                                  struct ethtool_channels *channels)
1620 {
1621         struct virtnet_info *vi = netdev_priv(dev);
1622
1623         channels->combined_count = vi->curr_queue_pairs;
1624         channels->max_combined = vi->max_queue_pairs;
1625         channels->max_other = 0;
1626         channels->rx_count = 0;
1627         channels->tx_count = 0;
1628         channels->other_count = 0;
1629 }
1630
1631 /* Check if the user is trying to change anything besides speed/duplex */
1632 static bool virtnet_validate_ethtool_cmd(const struct ethtool_cmd *cmd)
1633 {
1634         struct ethtool_cmd diff1 = *cmd;
1635         struct ethtool_cmd diff2 = {};
1636
1637         /* cmd is always set so we need to clear it, validate the port type
1638          * and also without autonegotiation we can ignore advertising
1639          */
1640         ethtool_cmd_speed_set(&diff1, 0);
1641         diff2.port = PORT_OTHER;
1642         diff1.advertising = 0;
1643         diff1.duplex = 0;
1644         diff1.cmd = 0;
1645
1646         return !memcmp(&diff1, &diff2, sizeof(diff1));
1647 }
1648
1649 static int virtnet_set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1650 {
1651         struct virtnet_info *vi = netdev_priv(dev);
1652         u32 speed;
1653
1654         speed = ethtool_cmd_speed(cmd);
1655         /* don't allow custom speed and duplex */
1656         if (!ethtool_validate_speed(speed) ||
1657             !ethtool_validate_duplex(cmd->duplex) ||
1658             !virtnet_validate_ethtool_cmd(cmd))
1659                 return -EINVAL;
1660         vi->speed = speed;
1661         vi->duplex = cmd->duplex;
1662
1663         return 0;
1664 }
1665
1666 static int virtnet_get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1667 {
1668         struct virtnet_info *vi = netdev_priv(dev);
1669
1670         ethtool_cmd_speed_set(cmd, vi->speed);
1671         cmd->duplex = vi->duplex;
1672         cmd->port = PORT_OTHER;
1673
1674         return 0;
1675 }
1676
1677 static void virtnet_init_settings(struct net_device *dev)
1678 {
1679         struct virtnet_info *vi = netdev_priv(dev);
1680
1681         vi->speed = SPEED_UNKNOWN;
1682         vi->duplex = DUPLEX_UNKNOWN;
1683 }
1684
1685 static const struct ethtool_ops virtnet_ethtool_ops = {
1686         .get_drvinfo = virtnet_get_drvinfo,
1687         .get_link = ethtool_op_get_link,
1688         .get_ringparam = virtnet_get_ringparam,
1689         .set_channels = virtnet_set_channels,
1690         .get_channels = virtnet_get_channels,
1691         .get_ts_info = ethtool_op_get_ts_info,
1692         .get_settings = virtnet_get_settings,
1693         .set_settings = virtnet_set_settings,
1694 };
1695
1696 static void virtnet_freeze_down(struct virtio_device *vdev)
1697 {
1698         struct virtnet_info *vi = vdev->priv;
1699         int i;
1700
1701         /* Make sure no work handler is accessing the device */
1702         flush_work(&vi->config_work);
1703
1704         netif_device_detach(vi->dev);
1705         cancel_delayed_work_sync(&vi->refill);
1706
1707         if (netif_running(vi->dev)) {
1708                 for (i = 0; i < vi->max_queue_pairs; i++)
1709                         napi_disable(&vi->rq[i].napi);
1710         }
1711 }
1712
1713 static int init_vqs(struct virtnet_info *vi);
1714 static void _remove_vq_common(struct virtnet_info *vi);
1715
1716 static int virtnet_restore_up(struct virtio_device *vdev)
1717 {
1718         struct virtnet_info *vi = vdev->priv;
1719         int err, i;
1720
1721         err = init_vqs(vi);
1722         if (err)
1723                 return err;
1724
1725         virtio_device_ready(vdev);
1726
1727         if (netif_running(vi->dev)) {
1728                 for (i = 0; i < vi->curr_queue_pairs; i++)
1729                         if (!try_fill_recv(vi, &vi->rq[i], GFP_KERNEL))
1730                                 schedule_delayed_work(&vi->refill, 0);
1731
1732                 for (i = 0; i < vi->max_queue_pairs; i++)
1733                         virtnet_napi_enable(&vi->rq[i]);
1734         }
1735
1736         netif_device_attach(vi->dev);
1737         return err;
1738 }
1739
1740 static int virtnet_reset(struct virtnet_info *vi, int curr_qp, int xdp_qp)
1741 {
1742         struct virtio_device *dev = vi->vdev;
1743         int ret;
1744
1745         virtio_config_disable(dev);
1746         dev->failed = dev->config->get_status(dev) & VIRTIO_CONFIG_S_FAILED;
1747         virtnet_freeze_down(dev);
1748         _remove_vq_common(vi);
1749
1750         virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
1751         virtio_add_status(dev, VIRTIO_CONFIG_S_DRIVER);
1752
1753         ret = virtio_finalize_features(dev);
1754         if (ret)
1755                 goto err;
1756
1757         vi->xdp_queue_pairs = xdp_qp;
1758         ret = virtnet_restore_up(dev);
1759         if (ret)
1760                 goto err;
1761         ret = _virtnet_set_queues(vi, curr_qp);
1762         if (ret)
1763                 goto err;
1764
1765         virtio_add_status(dev, VIRTIO_CONFIG_S_DRIVER_OK);
1766         virtio_config_enable(dev);
1767         return 0;
1768 err:
1769         virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
1770         return ret;
1771 }
1772
1773 static int virtnet_xdp_set(struct net_device *dev, struct bpf_prog *prog)
1774 {
1775         unsigned long int max_sz = PAGE_SIZE - sizeof(struct padded_vnet_hdr);
1776         struct virtnet_info *vi = netdev_priv(dev);
1777         struct bpf_prog *old_prog;
1778         u16 xdp_qp = 0, curr_qp;
1779         int i, err;
1780
1781         if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO4) ||
1782             virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO6) ||
1783             virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_ECN) ||
1784             virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_UFO)) {
1785                 netdev_warn(dev, "can't set XDP while host is implementing LRO, disable LRO first\n");
1786                 return -EOPNOTSUPP;
1787         }
1788
1789         if (vi->mergeable_rx_bufs && !vi->any_header_sg) {
1790                 netdev_warn(dev, "XDP expects header/data in single page, any_header_sg required\n");
1791                 return -EINVAL;
1792         }
1793
1794         if (dev->mtu > max_sz) {
1795                 netdev_warn(dev, "XDP requires MTU less than %lu\n", max_sz);
1796                 return -EINVAL;
1797         }
1798
1799         curr_qp = vi->curr_queue_pairs - vi->xdp_queue_pairs;
1800         if (prog)
1801                 xdp_qp = nr_cpu_ids;
1802
1803         /* XDP requires extra queues for XDP_TX */
1804         if (curr_qp + xdp_qp > vi->max_queue_pairs) {
1805                 netdev_warn(dev, "request %i queues but max is %i\n",
1806                             curr_qp + xdp_qp, vi->max_queue_pairs);
1807                 return -ENOMEM;
1808         }
1809
1810         if (prog) {
1811                 prog = bpf_prog_add(prog, vi->max_queue_pairs - 1);
1812                 if (IS_ERR(prog))
1813                         return PTR_ERR(prog);
1814         }
1815
1816         /* Changing the headroom in buffers is a disruptive operation because
1817          * existing buffers must be flushed and reallocated. This will happen
1818          * when a xdp program is initially added or xdp is disabled by removing
1819          * the xdp program resulting in number of XDP queues changing.
1820          */
1821         if (vi->xdp_queue_pairs != xdp_qp) {
1822                 err = virtnet_reset(vi, curr_qp + xdp_qp, xdp_qp);
1823                 if (err) {
1824                         dev_warn(&dev->dev, "XDP reset failure.\n");
1825                         goto virtio_reset_err;
1826                 }
1827         }
1828
1829         netif_set_real_num_rx_queues(dev, curr_qp + xdp_qp);
1830
1831         for (i = 0; i < vi->max_queue_pairs; i++) {
1832                 old_prog = rtnl_dereference(vi->rq[i].xdp_prog);
1833                 rcu_assign_pointer(vi->rq[i].xdp_prog, prog);
1834                 if (old_prog)
1835                         bpf_prog_put(old_prog);
1836         }
1837
1838         return 0;
1839
1840 virtio_reset_err:
1841         /* On reset error do our best to unwind XDP changes inflight and return
1842          * error up to user space for resolution. The underlying reset hung on
1843          * us so not much we can do here.
1844          */
1845         if (prog)
1846                 bpf_prog_sub(prog, vi->max_queue_pairs - 1);
1847         return err;
1848 }
1849
1850 static bool virtnet_xdp_query(struct net_device *dev)
1851 {
1852         struct virtnet_info *vi = netdev_priv(dev);
1853         int i;
1854
1855         for (i = 0; i < vi->max_queue_pairs; i++) {
1856                 if (vi->rq[i].xdp_prog)
1857                         return true;
1858         }
1859         return false;
1860 }
1861
1862 static int virtnet_xdp(struct net_device *dev, struct netdev_xdp *xdp)
1863 {
1864         switch (xdp->command) {
1865         case XDP_SETUP_PROG:
1866                 return virtnet_xdp_set(dev, xdp->prog);
1867         case XDP_QUERY_PROG:
1868                 xdp->prog_attached = virtnet_xdp_query(dev);
1869                 return 0;
1870         default:
1871                 return -EINVAL;
1872         }
1873 }
1874
1875 static const struct net_device_ops virtnet_netdev = {
1876         .ndo_open            = virtnet_open,
1877         .ndo_stop            = virtnet_close,
1878         .ndo_start_xmit      = start_xmit,
1879         .ndo_validate_addr   = eth_validate_addr,
1880         .ndo_set_mac_address = virtnet_set_mac_address,
1881         .ndo_set_rx_mode     = virtnet_set_rx_mode,
1882         .ndo_get_stats64     = virtnet_stats,
1883         .ndo_vlan_rx_add_vid = virtnet_vlan_rx_add_vid,
1884         .ndo_vlan_rx_kill_vid = virtnet_vlan_rx_kill_vid,
1885 #ifdef CONFIG_NET_POLL_CONTROLLER
1886         .ndo_poll_controller = virtnet_netpoll,
1887 #endif
1888         .ndo_xdp                = virtnet_xdp,
1889 };
1890
1891 static void virtnet_config_changed_work(struct work_struct *work)
1892 {
1893         struct virtnet_info *vi =
1894                 container_of(work, struct virtnet_info, config_work);
1895         u16 v;
1896
1897         if (virtio_cread_feature(vi->vdev, VIRTIO_NET_F_STATUS,
1898                                  struct virtio_net_config, status, &v) < 0)
1899                 return;
1900
1901         if (v & VIRTIO_NET_S_ANNOUNCE) {
1902                 netdev_notify_peers(vi->dev);
1903                 virtnet_ack_link_announce(vi);
1904         }
1905
1906         /* Ignore unknown (future) status bits */
1907         v &= VIRTIO_NET_S_LINK_UP;
1908
1909         if (vi->status == v)
1910                 return;
1911
1912         vi->status = v;
1913
1914         if (vi->status & VIRTIO_NET_S_LINK_UP) {
1915                 netif_carrier_on(vi->dev);
1916                 netif_tx_wake_all_queues(vi->dev);
1917         } else {
1918                 netif_carrier_off(vi->dev);
1919                 netif_tx_stop_all_queues(vi->dev);
1920         }
1921 }
1922
1923 static void virtnet_config_changed(struct virtio_device *vdev)
1924 {
1925         struct virtnet_info *vi = vdev->priv;
1926
1927         schedule_work(&vi->config_work);
1928 }
1929
1930 static void virtnet_free_queues(struct virtnet_info *vi)
1931 {
1932         int i;
1933
1934         for (i = 0; i < vi->max_queue_pairs; i++) {
1935                 napi_hash_del(&vi->rq[i].napi);
1936                 netif_napi_del(&vi->rq[i].napi);
1937         }
1938
1939         /* We called napi_hash_del() before netif_napi_del(),
1940          * we need to respect an RCU grace period before freeing vi->rq
1941          */
1942         synchronize_net();
1943
1944         kfree(vi->rq);
1945         kfree(vi->sq);
1946 }
1947
1948 static void _free_receive_bufs(struct virtnet_info *vi)
1949 {
1950         struct bpf_prog *old_prog;
1951         int i;
1952
1953         for (i = 0; i < vi->max_queue_pairs; i++) {
1954                 while (vi->rq[i].pages)
1955                         __free_pages(get_a_page(&vi->rq[i], GFP_KERNEL), 0);
1956
1957                 old_prog = rtnl_dereference(vi->rq[i].xdp_prog);
1958                 RCU_INIT_POINTER(vi->rq[i].xdp_prog, NULL);
1959                 if (old_prog)
1960                         bpf_prog_put(old_prog);
1961         }
1962 }
1963
1964 static void free_receive_bufs(struct virtnet_info *vi)
1965 {
1966         rtnl_lock();
1967         _free_receive_bufs(vi);
1968         rtnl_unlock();
1969 }
1970
1971 static void free_receive_page_frags(struct virtnet_info *vi)
1972 {
1973         int i;
1974         for (i = 0; i < vi->max_queue_pairs; i++)
1975                 if (vi->rq[i].alloc_frag.page)
1976                         put_page(vi->rq[i].alloc_frag.page);
1977 }
1978
1979 static bool is_xdp_raw_buffer_queue(struct virtnet_info *vi, int q)
1980 {
1981         if (q < (vi->curr_queue_pairs - vi->xdp_queue_pairs))
1982                 return false;
1983         else if (q < vi->curr_queue_pairs)
1984                 return true;
1985         else
1986                 return false;
1987 }
1988
1989 static void free_unused_bufs(struct virtnet_info *vi)
1990 {
1991         void *buf;
1992         int i;
1993
1994         for (i = 0; i < vi->max_queue_pairs; i++) {
1995                 struct virtqueue *vq = vi->sq[i].vq;
1996                 while ((buf = virtqueue_detach_unused_buf(vq)) != NULL) {
1997                         if (!is_xdp_raw_buffer_queue(vi, i))
1998                                 dev_kfree_skb(buf);
1999                         else
2000                                 put_page(virt_to_head_page(buf));
2001                 }
2002         }
2003
2004         for (i = 0; i < vi->max_queue_pairs; i++) {
2005                 struct virtqueue *vq = vi->rq[i].vq;
2006
2007                 while ((buf = virtqueue_detach_unused_buf(vq)) != NULL) {
2008                         if (vi->mergeable_rx_bufs) {
2009                                 put_page(virt_to_head_page(buf));
2010                         } else if (vi->big_packets) {
2011                                 give_pages(&vi->rq[i], buf);
2012                         } else {
2013                                 put_page(virt_to_head_page(buf));
2014                         }
2015                 }
2016         }
2017 }
2018
2019 static void virtnet_del_vqs(struct virtnet_info *vi)
2020 {
2021         struct virtio_device *vdev = vi->vdev;
2022
2023         virtnet_clean_affinity(vi, -1);
2024
2025         vdev->config->del_vqs(vdev);
2026
2027         virtnet_free_queues(vi);
2028 }
2029
2030 /* How large should a single buffer be so a queue full of these can fit at
2031  * least one full packet?
2032  * Logic below assumes the mergeable buffer header is used.
2033  */
2034 static unsigned int mergeable_min_buf_len(struct virtnet_info *vi, struct virtqueue *vq)
2035 {
2036         const unsigned int hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);
2037         unsigned int rq_size = virtqueue_get_vring_size(vq);
2038         unsigned int packet_len = vi->big_packets ? IP_MAX_MTU : vi->dev->max_mtu;
2039         unsigned int buf_len = hdr_len + ETH_HLEN + VLAN_HLEN + packet_len;
2040         unsigned int min_buf_len = DIV_ROUND_UP(buf_len, rq_size);
2041
2042         return max(min_buf_len, hdr_len);
2043 }
2044
2045 static int virtnet_find_vqs(struct virtnet_info *vi)
2046 {
2047         vq_callback_t **callbacks;
2048         struct virtqueue **vqs;
2049         int ret = -ENOMEM;
2050         int i, total_vqs;
2051         const char **names;
2052         bool *ctx;
2053
2054         /* We expect 1 RX virtqueue followed by 1 TX virtqueue, followed by
2055          * possible N-1 RX/TX queue pairs used in multiqueue mode, followed by
2056          * possible control vq.
2057          */
2058         total_vqs = vi->max_queue_pairs * 2 +
2059                     virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ);
2060
2061         /* Allocate space for find_vqs parameters */
2062         vqs = kzalloc(total_vqs * sizeof(*vqs), GFP_KERNEL);
2063         if (!vqs)
2064                 goto err_vq;
2065         callbacks = kmalloc(total_vqs * sizeof(*callbacks), GFP_KERNEL);
2066         if (!callbacks)
2067                 goto err_callback;
2068         names = kmalloc(total_vqs * sizeof(*names), GFP_KERNEL);
2069         if (!names)
2070                 goto err_names;
2071         if (vi->mergeable_rx_bufs) {
2072                 ctx = kzalloc(total_vqs * sizeof(*ctx), GFP_KERNEL);
2073                 if (!ctx)
2074                         goto err_ctx;
2075         } else {
2076                 ctx = NULL;
2077         }
2078
2079         /* Parameters for control virtqueue, if any */
2080         if (vi->has_cvq) {
2081                 callbacks[total_vqs - 1] = NULL;
2082                 names[total_vqs - 1] = "control";
2083         }
2084
2085         /* Allocate/initialize parameters for send/receive virtqueues */
2086         for (i = 0; i < vi->max_queue_pairs; i++) {
2087                 callbacks[rxq2vq(i)] = skb_recv_done;
2088                 callbacks[txq2vq(i)] = skb_xmit_done;
2089                 sprintf(vi->rq[i].name, "input.%d", i);
2090                 sprintf(vi->sq[i].name, "output.%d", i);
2091                 names[rxq2vq(i)] = vi->rq[i].name;
2092                 names[txq2vq(i)] = vi->sq[i].name;
2093                 if (ctx)
2094                         ctx[rxq2vq(i)] = true;
2095         }
2096
2097         ret = vi->vdev->config->find_vqs(vi->vdev, total_vqs, vqs, callbacks,
2098                                          names, ctx, NULL);
2099         if (ret)
2100                 goto err_find;
2101
2102         if (vi->has_cvq) {
2103                 vi->cvq = vqs[total_vqs - 1];
2104                 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VLAN))
2105                         vi->dev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;
2106         }
2107
2108         for (i = 0; i < vi->max_queue_pairs; i++) {
2109                 vi->rq[i].vq = vqs[rxq2vq(i)];
2110                 vi->rq[i].min_buf_len = mergeable_min_buf_len(vi, vi->rq[i].vq);
2111                 vi->sq[i].vq = vqs[txq2vq(i)];
2112         }
2113
2114         kfree(names);
2115         kfree(callbacks);
2116         kfree(vqs);
2117
2118         return 0;
2119
2120 err_find:
2121         kfree(ctx);
2122 err_ctx:
2123         kfree(names);
2124 err_names:
2125         kfree(callbacks);
2126 err_callback:
2127         kfree(vqs);
2128 err_vq:
2129         return ret;
2130 }
2131
2132 static int virtnet_alloc_queues(struct virtnet_info *vi)
2133 {
2134         int i;
2135
2136         vi->sq = kzalloc(sizeof(*vi->sq) * vi->max_queue_pairs, GFP_KERNEL);
2137         if (!vi->sq)
2138                 goto err_sq;
2139         vi->rq = kzalloc(sizeof(*vi->rq) * vi->max_queue_pairs, GFP_KERNEL);
2140         if (!vi->rq)
2141                 goto err_rq;
2142
2143         INIT_DELAYED_WORK(&vi->refill, refill_work);
2144         for (i = 0; i < vi->max_queue_pairs; i++) {
2145                 vi->rq[i].pages = NULL;
2146                 netif_napi_add(vi->dev, &vi->rq[i].napi, virtnet_poll,
2147                                napi_weight);
2148
2149                 sg_init_table(vi->rq[i].sg, ARRAY_SIZE(vi->rq[i].sg));
2150                 ewma_pkt_len_init(&vi->rq[i].mrg_avg_pkt_len);
2151                 sg_init_table(vi->sq[i].sg, ARRAY_SIZE(vi->sq[i].sg));
2152         }
2153
2154         return 0;
2155
2156 err_rq:
2157         kfree(vi->sq);
2158 err_sq:
2159         return -ENOMEM;
2160 }
2161
2162 static int init_vqs(struct virtnet_info *vi)
2163 {
2164         int ret;
2165
2166         /* Allocate send & receive queues */
2167         ret = virtnet_alloc_queues(vi);
2168         if (ret)
2169                 goto err;
2170
2171         ret = virtnet_find_vqs(vi);
2172         if (ret)
2173                 goto err_free;
2174
2175         get_online_cpus();
2176         virtnet_set_affinity(vi);
2177         put_online_cpus();
2178
2179         return 0;
2180
2181 err_free:
2182         virtnet_free_queues(vi);
2183 err:
2184         return ret;
2185 }
2186
2187 #ifdef CONFIG_SYSFS
2188 static ssize_t mergeable_rx_buffer_size_show(struct netdev_rx_queue *queue,
2189                 struct rx_queue_attribute *attribute, char *buf)
2190 {
2191         struct virtnet_info *vi = netdev_priv(queue->dev);
2192         unsigned int queue_index = get_netdev_rx_queue_index(queue);
2193         struct ewma_pkt_len *avg;
2194
2195         BUG_ON(queue_index >= vi->max_queue_pairs);
2196         avg = &vi->rq[queue_index].mrg_avg_pkt_len;
2197         return sprintf(buf, "%u\n",
2198                        get_mergeable_buf_len(&vi->rq[queue_index], avg));
2199 }
2200
2201 static struct rx_queue_attribute mergeable_rx_buffer_size_attribute =
2202         __ATTR_RO(mergeable_rx_buffer_size);
2203
2204 static struct attribute *virtio_net_mrg_rx_attrs[] = {
2205         &mergeable_rx_buffer_size_attribute.attr,
2206         NULL
2207 };
2208
2209 static const struct attribute_group virtio_net_mrg_rx_group = {
2210         .name = "virtio_net",
2211         .attrs = virtio_net_mrg_rx_attrs
2212 };
2213 #endif
2214
2215 static bool virtnet_fail_on_feature(struct virtio_device *vdev,
2216                                     unsigned int fbit,
2217                                     const char *fname, const char *dname)
2218 {
2219         if (!virtio_has_feature(vdev, fbit))
2220                 return false;
2221
2222         dev_err(&vdev->dev, "device advertises feature %s but not %s",
2223                 fname, dname);
2224
2225         return true;
2226 }
2227
2228 #define VIRTNET_FAIL_ON(vdev, fbit, dbit)                       \
2229         virtnet_fail_on_feature(vdev, fbit, #fbit, dbit)
2230
2231 static bool virtnet_validate_features(struct virtio_device *vdev)
2232 {
2233         if (!virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ) &&
2234             (VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_RX,
2235                              "VIRTIO_NET_F_CTRL_VQ") ||
2236              VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_VLAN,
2237                              "VIRTIO_NET_F_CTRL_VQ") ||
2238              VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_GUEST_ANNOUNCE,
2239                              "VIRTIO_NET_F_CTRL_VQ") ||
2240              VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_MQ, "VIRTIO_NET_F_CTRL_VQ") ||
2241              VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR,
2242                              "VIRTIO_NET_F_CTRL_VQ"))) {
2243                 return false;
2244         }
2245
2246         return true;
2247 }
2248
2249 #define MIN_MTU ETH_MIN_MTU
2250 #define MAX_MTU ETH_MAX_MTU
2251
2252 static int virtnet_validate(struct virtio_device *vdev)
2253 {
2254         if (!vdev->config->get) {
2255                 dev_err(&vdev->dev, "%s failure: config access disabled\n",
2256                         __func__);
2257                 return -EINVAL;
2258         }
2259
2260         if (!virtnet_validate_features(vdev))
2261                 return -EINVAL;
2262
2263         if (virtio_has_feature(vdev, VIRTIO_NET_F_MTU)) {
2264                 int mtu = virtio_cread16(vdev,
2265                                          offsetof(struct virtio_net_config,
2266                                                   mtu));
2267                 if (mtu < MIN_MTU)
2268                         __virtio_clear_bit(vdev, VIRTIO_NET_F_MTU);
2269         }
2270
2271         return 0;
2272 }
2273
2274 static int virtnet_probe(struct virtio_device *vdev)
2275 {
2276         int i, err;
2277         struct net_device *dev;
2278         struct virtnet_info *vi;
2279         u16 max_queue_pairs;
2280         int mtu;
2281
2282         /* Find if host supports multiqueue virtio_net device */
2283         err = virtio_cread_feature(vdev, VIRTIO_NET_F_MQ,
2284                                    struct virtio_net_config,
2285                                    max_virtqueue_pairs, &max_queue_pairs);
2286
2287         /* We need at least 2 queue's */
2288         if (err || max_queue_pairs < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN ||
2289             max_queue_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX ||
2290             !virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ))
2291                 max_queue_pairs = 1;
2292
2293         /* Allocate ourselves a network device with room for our info */
2294         dev = alloc_etherdev_mq(sizeof(struct virtnet_info), max_queue_pairs);
2295         if (!dev)
2296                 return -ENOMEM;
2297
2298         /* Set up network device as normal. */
2299         dev->priv_flags |= IFF_UNICAST_FLT | IFF_LIVE_ADDR_CHANGE;
2300         dev->netdev_ops = &virtnet_netdev;
2301         dev->features = NETIF_F_HIGHDMA;
2302
2303         dev->ethtool_ops = &virtnet_ethtool_ops;
2304         SET_NETDEV_DEV(dev, &vdev->dev);
2305
2306         /* Do we support "hardware" checksums? */
2307         if (virtio_has_feature(vdev, VIRTIO_NET_F_CSUM)) {
2308                 /* This opens up the world of extra features. */
2309                 dev->hw_features |= NETIF_F_HW_CSUM | NETIF_F_SG;
2310                 if (csum)
2311                         dev->features |= NETIF_F_HW_CSUM | NETIF_F_SG;
2312
2313                 if (virtio_has_feature(vdev, VIRTIO_NET_F_GSO)) {
2314                         dev->hw_features |= NETIF_F_TSO | NETIF_F_UFO
2315                                 | NETIF_F_TSO_ECN | NETIF_F_TSO6;
2316                 }
2317                 /* Individual feature bits: what can host handle? */
2318                 if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO4))
2319                         dev->hw_features |= NETIF_F_TSO;
2320                 if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO6))
2321                         dev->hw_features |= NETIF_F_TSO6;
2322                 if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_ECN))
2323                         dev->hw_features |= NETIF_F_TSO_ECN;
2324                 if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_UFO))
2325                         dev->hw_features |= NETIF_F_UFO;
2326
2327                 dev->features |= NETIF_F_GSO_ROBUST;
2328
2329                 if (gso)
2330                         dev->features |= dev->hw_features & (NETIF_F_ALL_TSO|NETIF_F_UFO);
2331                 /* (!csum && gso) case will be fixed by register_netdev() */
2332         }
2333         if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_CSUM))
2334                 dev->features |= NETIF_F_RXCSUM;
2335
2336         dev->vlan_features = dev->features;
2337
2338         /* MTU range: 68 - 65535 */
2339         dev->min_mtu = MIN_MTU;
2340         dev->max_mtu = MAX_MTU;
2341
2342         /* Configuration may specify what MAC to use.  Otherwise random. */
2343         if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC))
2344                 virtio_cread_bytes(vdev,
2345                                    offsetof(struct virtio_net_config, mac),
2346                                    dev->dev_addr, dev->addr_len);
2347         else
2348                 eth_hw_addr_random(dev);
2349
2350         /* Set up our device-specific information */
2351         vi = netdev_priv(dev);
2352         vi->dev = dev;
2353         vi->vdev = vdev;
2354         vdev->priv = vi;
2355         vi->stats = alloc_percpu(struct virtnet_stats);
2356         err = -ENOMEM;
2357         if (vi->stats == NULL)
2358                 goto free;
2359
2360         for_each_possible_cpu(i) {
2361                 struct virtnet_stats *virtnet_stats;
2362                 virtnet_stats = per_cpu_ptr(vi->stats, i);
2363                 u64_stats_init(&virtnet_stats->tx_syncp);
2364                 u64_stats_init(&virtnet_stats->rx_syncp);
2365         }
2366
2367         INIT_WORK(&vi->config_work, virtnet_config_changed_work);
2368
2369         /* If we can receive ANY GSO packets, we must allocate large ones. */
2370         if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO4) ||
2371             virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO6) ||
2372             virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_ECN) ||
2373             virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_UFO))
2374                 vi->big_packets = true;
2375
2376         if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF))
2377                 vi->mergeable_rx_bufs = true;
2378
2379         if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF) ||
2380             virtio_has_feature(vdev, VIRTIO_F_VERSION_1))
2381                 vi->hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);
2382         else
2383                 vi->hdr_len = sizeof(struct virtio_net_hdr);
2384
2385         if (virtio_has_feature(vdev, VIRTIO_F_ANY_LAYOUT) ||
2386             virtio_has_feature(vdev, VIRTIO_F_VERSION_1))
2387                 vi->any_header_sg = true;
2388
2389         if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ))
2390                 vi->has_cvq = true;
2391
2392         if (virtio_has_feature(vdev, VIRTIO_NET_F_MTU)) {
2393                 mtu = virtio_cread16(vdev,
2394                                      offsetof(struct virtio_net_config,
2395                                               mtu));
2396                 if (mtu < dev->min_mtu) {
2397                         /* Should never trigger: MTU was previously validated
2398                          * in virtnet_validate.
2399                          */
2400                         dev_err(&vdev->dev, "device MTU appears to have changed "
2401                                 "it is now %d < %d", mtu, dev->min_mtu);
2402                         goto free_stats;
2403                 }
2404
2405                 dev->mtu = mtu;
2406                 dev->max_mtu = mtu;
2407
2408                 /* TODO: size buffers correctly in this case. */
2409                 if (dev->mtu > ETH_DATA_LEN)
2410                         vi->big_packets = true;
2411         }
2412
2413         if (vi->any_header_sg)
2414                 dev->needed_headroom = vi->hdr_len;
2415
2416         /* Enable multiqueue by default */
2417         if (num_online_cpus() >= max_queue_pairs)
2418                 vi->curr_queue_pairs = max_queue_pairs;
2419         else
2420                 vi->curr_queue_pairs = num_online_cpus();
2421         vi->max_queue_pairs = max_queue_pairs;
2422
2423         /* Allocate/initialize the rx/tx queues, and invoke find_vqs */
2424         err = init_vqs(vi);
2425         if (err)
2426                 goto free_stats;
2427
2428 #ifdef CONFIG_SYSFS
2429         if (vi->mergeable_rx_bufs)
2430                 dev->sysfs_rx_queue_group = &virtio_net_mrg_rx_group;
2431 #endif
2432         netif_set_real_num_tx_queues(dev, vi->curr_queue_pairs);
2433         netif_set_real_num_rx_queues(dev, vi->curr_queue_pairs);
2434
2435         virtnet_init_settings(dev);
2436
2437         err = register_netdev(dev);
2438         if (err) {
2439                 pr_debug("virtio_net: registering device failed\n");
2440                 goto free_vqs;
2441         }
2442
2443         virtio_device_ready(vdev);
2444
2445         err = virtnet_cpu_notif_add(vi);
2446         if (err) {
2447                 pr_debug("virtio_net: registering cpu notifier failed\n");
2448                 goto free_unregister_netdev;
2449         }
2450
2451         virtnet_set_queues(vi, vi->curr_queue_pairs);
2452
2453         /* Assume link up if device can't report link status,
2454            otherwise get link status from config. */
2455         if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STATUS)) {
2456                 netif_carrier_off(dev);
2457                 schedule_work(&vi->config_work);
2458         } else {
2459                 vi->status = VIRTIO_NET_S_LINK_UP;
2460                 netif_carrier_on(dev);
2461         }
2462
2463         pr_debug("virtnet: registered device %s with %d RX and TX vq's\n",
2464                  dev->name, max_queue_pairs);
2465
2466         return 0;
2467
2468 free_unregister_netdev:
2469         vi->vdev->config->reset(vdev);
2470
2471         unregister_netdev(dev);
2472 free_vqs:
2473         cancel_delayed_work_sync(&vi->refill);
2474         free_receive_page_frags(vi);
2475         virtnet_del_vqs(vi);
2476 free_stats:
2477         free_percpu(vi->stats);
2478 free:
2479         free_netdev(dev);
2480         return err;
2481 }
2482
2483 static void _remove_vq_common(struct virtnet_info *vi)
2484 {
2485         vi->vdev->config->reset(vi->vdev);
2486         free_unused_bufs(vi);
2487         _free_receive_bufs(vi);
2488         free_receive_page_frags(vi);
2489         virtnet_del_vqs(vi);
2490 }
2491
2492 static void remove_vq_common(struct virtnet_info *vi)
2493 {
2494         vi->vdev->config->reset(vi->vdev);
2495
2496         /* Free unused buffers in both send and recv, if any. */
2497         free_unused_bufs(vi);
2498
2499         free_receive_bufs(vi);
2500
2501         free_receive_page_frags(vi);
2502
2503         virtnet_del_vqs(vi);
2504 }
2505
2506 static void virtnet_remove(struct virtio_device *vdev)
2507 {
2508         struct virtnet_info *vi = vdev->priv;
2509
2510         virtnet_cpu_notif_remove(vi);
2511
2512         /* Make sure no work handler is accessing the device. */
2513         flush_work(&vi->config_work);
2514
2515         unregister_netdev(vi->dev);
2516
2517         remove_vq_common(vi);
2518
2519         free_percpu(vi->stats);
2520         free_netdev(vi->dev);
2521 }
2522
2523 #ifdef CONFIG_PM_SLEEP
2524 static int virtnet_freeze(struct virtio_device *vdev)
2525 {
2526         struct virtnet_info *vi = vdev->priv;
2527
2528         virtnet_cpu_notif_remove(vi);
2529         virtnet_freeze_down(vdev);
2530         remove_vq_common(vi);
2531
2532         return 0;
2533 }
2534
2535 static int virtnet_restore(struct virtio_device *vdev)
2536 {
2537         struct virtnet_info *vi = vdev->priv;
2538         int err;
2539
2540         err = virtnet_restore_up(vdev);
2541         if (err)
2542                 return err;
2543         virtnet_set_queues(vi, vi->curr_queue_pairs);
2544
2545         err = virtnet_cpu_notif_add(vi);
2546         if (err)
2547                 return err;
2548
2549         return 0;
2550 }
2551 #endif
2552
2553 static struct virtio_device_id id_table[] = {
2554         { VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID },
2555         { 0 },
2556 };
2557
2558 #define VIRTNET_FEATURES \
2559         VIRTIO_NET_F_CSUM, VIRTIO_NET_F_GUEST_CSUM, \
2560         VIRTIO_NET_F_MAC, \
2561         VIRTIO_NET_F_HOST_TSO4, VIRTIO_NET_F_HOST_UFO, VIRTIO_NET_F_HOST_TSO6, \
2562         VIRTIO_NET_F_HOST_ECN, VIRTIO_NET_F_GUEST_TSO4, VIRTIO_NET_F_GUEST_TSO6, \
2563         VIRTIO_NET_F_GUEST_ECN, VIRTIO_NET_F_GUEST_UFO, \
2564         VIRTIO_NET_F_MRG_RXBUF, VIRTIO_NET_F_STATUS, VIRTIO_NET_F_CTRL_VQ, \
2565         VIRTIO_NET_F_CTRL_RX, VIRTIO_NET_F_CTRL_VLAN, \
2566         VIRTIO_NET_F_GUEST_ANNOUNCE, VIRTIO_NET_F_MQ, \
2567         VIRTIO_NET_F_CTRL_MAC_ADDR, \
2568         VIRTIO_NET_F_MTU
2569
2570 static unsigned int features[] = {
2571         VIRTNET_FEATURES,
2572 };
2573
2574 static unsigned int features_legacy[] = {
2575         VIRTNET_FEATURES,
2576         VIRTIO_NET_F_GSO,
2577         VIRTIO_F_ANY_LAYOUT,
2578 };
2579
2580 static struct virtio_driver virtio_net_driver = {
2581         .feature_table = features,
2582         .feature_table_size = ARRAY_SIZE(features),
2583         .feature_table_legacy = features_legacy,
2584         .feature_table_size_legacy = ARRAY_SIZE(features_legacy),
2585         .driver.name =  KBUILD_MODNAME,
2586         .driver.owner = THIS_MODULE,
2587         .id_table =     id_table,
2588         .validate =     virtnet_validate,
2589         .probe =        virtnet_probe,
2590         .remove =       virtnet_remove,
2591         .config_changed = virtnet_config_changed,
2592 #ifdef CONFIG_PM_SLEEP
2593         .freeze =       virtnet_freeze,
2594         .restore =      virtnet_restore,
2595 #endif
2596 };
2597
2598 static __init int virtio_net_driver_init(void)
2599 {
2600         int ret;
2601
2602         ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "virtio/net:online",
2603                                       virtnet_cpu_online,
2604                                       virtnet_cpu_down_prep);
2605         if (ret < 0)
2606                 goto out;
2607         virtionet_online = ret;
2608         ret = cpuhp_setup_state_multi(CPUHP_VIRT_NET_DEAD, "virtio/net:dead",
2609                                       NULL, virtnet_cpu_dead);
2610         if (ret)
2611                 goto err_dead;
2612
2613         ret = register_virtio_driver(&virtio_net_driver);
2614         if (ret)
2615                 goto err_virtio;
2616         return 0;
2617 err_virtio:
2618         cpuhp_remove_multi_state(CPUHP_VIRT_NET_DEAD);
2619 err_dead:
2620         cpuhp_remove_multi_state(virtionet_online);
2621 out:
2622         return ret;
2623 }
2624 module_init(virtio_net_driver_init);
2625
2626 static __exit void virtio_net_driver_exit(void)
2627 {
2628         cpuhp_remove_multi_state(CPUHP_VIRT_NET_DEAD);
2629         cpuhp_remove_multi_state(virtionet_online);
2630         unregister_virtio_driver(&virtio_net_driver);
2631 }
2632 module_exit(virtio_net_driver_exit);
2633
2634 MODULE_DEVICE_TABLE(virtio, id_table);
2635 MODULE_DESCRIPTION("Virtio network driver");
2636 MODULE_LICENSE("GPL");