1 /* A network driver using virtio.
3 * Copyright 2007 Rusty Russell <rusty@rustcorp.com.au> IBM Corporation
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, see <http://www.gnu.org/licenses/>.
19 #include <linux/netdevice.h>
20 #include <linux/etherdevice.h>
21 #include <linux/ethtool.h>
22 #include <linux/module.h>
23 #include <linux/virtio.h>
24 #include <linux/virtio_net.h>
25 #include <linux/bpf.h>
26 #include <linux/bpf_trace.h>
27 #include <linux/scatterlist.h>
28 #include <linux/if_vlan.h>
29 #include <linux/slab.h>
30 #include <linux/cpu.h>
31 #include <linux/average.h>
33 static int napi_weight = NAPI_POLL_WEIGHT;
34 module_param(napi_weight, int, 0444);
36 static bool csum = true, gso = true, napi_tx;
37 module_param(csum, bool, 0444);
38 module_param(gso, bool, 0444);
39 module_param(napi_tx, bool, 0644);
41 /* FIXME: MTU in config. */
42 #define GOOD_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN)
43 #define GOOD_COPY_LEN 128
45 #define VIRTNET_RX_PAD (NET_IP_ALIGN + NET_SKB_PAD)
47 /* Amount of XDP headroom to prepend to packets for use by xdp_adjust_head */
48 #define VIRTIO_XDP_HEADROOM 256
50 /* RX packet size EWMA. The average packet size is used to determine the packet
51 * buffer size when refilling RX rings. As the entire RX ring may be refilled
52 * at once, the weight is chosen so that the EWMA will be insensitive to short-
53 * term, transient changes in packet size.
55 DECLARE_EWMA(pkt_len, 0, 64)
57 /* With mergeable buffers we align buffer address and use the low bits to
58 * encode its true size. Buffer size is up to 1 page so we need to align to
59 * square root of page size to ensure we reserve enough bits to encode the true
62 #define MERGEABLE_BUFFER_MIN_ALIGN_SHIFT ((PAGE_SHIFT + 1) / 2)
64 /* Minimum alignment for mergeable packet buffers. */
65 #define MERGEABLE_BUFFER_ALIGN max(L1_CACHE_BYTES, \
66 1 << MERGEABLE_BUFFER_MIN_ALIGN_SHIFT)
68 #define VIRTNET_DRIVER_VERSION "1.0.0"
70 struct virtnet_stats {
71 struct u64_stats_sync tx_syncp;
72 struct u64_stats_sync rx_syncp;
80 /* Internal representation of a send virtqueue */
82 /* Virtqueue associated with this send _queue */
85 /* TX: fragments + linear part + virtio header */
86 struct scatterlist sg[MAX_SKB_FRAGS + 2];
88 /* Name of the send queue: output.$index */
91 struct napi_struct napi;
94 /* Internal representation of a receive virtqueue */
95 struct receive_queue {
96 /* Virtqueue associated with this receive_queue */
99 struct napi_struct napi;
101 struct bpf_prog __rcu *xdp_prog;
103 /* Chain pages by the private ptr. */
106 /* Average packet length for mergeable receive buffers. */
107 struct ewma_pkt_len mrg_avg_pkt_len;
109 /* Page frag for packet buffer allocation. */
110 struct page_frag alloc_frag;
112 /* RX: fragments + linear part + virtio header */
113 struct scatterlist sg[MAX_SKB_FRAGS + 2];
115 /* Name of this receive queue: input.$index */
119 struct virtnet_info {
120 struct virtio_device *vdev;
121 struct virtqueue *cvq;
122 struct net_device *dev;
123 struct send_queue *sq;
124 struct receive_queue *rq;
127 /* Max # of queue pairs supported by the device */
130 /* # of queue pairs currently used by the driver */
131 u16 curr_queue_pairs;
133 /* # of XDP queue pairs currently used by the driver */
136 /* I like... big packets and I cannot lie! */
139 /* Host will merge rx buffers for big packets (shake it! shake it!) */
140 bool mergeable_rx_bufs;
142 /* Has control virtqueue */
145 /* Host can handle any s/g split between our header and packet data */
148 /* Packet virtio header size */
151 /* Active statistics */
152 struct virtnet_stats __percpu *stats;
154 /* Work struct for refilling if we run low on memory. */
155 struct delayed_work refill;
157 /* Work struct for config space updates */
158 struct work_struct config_work;
160 /* Does the affinity hint is set for virtqueues? */
161 bool affinity_hint_set;
163 /* CPU hotplug instances for online & dead */
164 struct hlist_node node;
165 struct hlist_node node_dead;
167 /* Control VQ buffers: protected by the rtnl lock */
168 struct virtio_net_ctrl_hdr ctrl_hdr;
169 virtio_net_ctrl_ack ctrl_status;
170 struct virtio_net_ctrl_mq ctrl_mq;
175 /* Ethtool settings */
180 struct padded_vnet_hdr {
181 struct virtio_net_hdr_mrg_rxbuf hdr;
183 * hdr is in a separate sg buffer, and data sg buffer shares same page
184 * with this header sg. This padding makes next sg 16 byte aligned
190 /* Converting between virtqueue no. and kernel tx/rx queue no.
191 * 0:rx0 1:tx0 2:rx1 3:tx1 ... 2N:rxN 2N+1:txN 2N+2:cvq
193 static int vq2txq(struct virtqueue *vq)
195 return (vq->index - 1) / 2;
198 static int txq2vq(int txq)
203 static int vq2rxq(struct virtqueue *vq)
205 return vq->index / 2;
208 static int rxq2vq(int rxq)
213 static inline struct virtio_net_hdr_mrg_rxbuf *skb_vnet_hdr(struct sk_buff *skb)
215 return (struct virtio_net_hdr_mrg_rxbuf *)skb->cb;
219 * private is used to chain pages for big packets, put the whole
220 * most recent used list in the beginning for reuse
222 static void give_pages(struct receive_queue *rq, struct page *page)
226 /* Find end of list, sew whole thing into vi->rq.pages. */
227 for (end = page; end->private; end = (struct page *)end->private);
228 end->private = (unsigned long)rq->pages;
232 static struct page *get_a_page(struct receive_queue *rq, gfp_t gfp_mask)
234 struct page *p = rq->pages;
237 rq->pages = (struct page *)p->private;
238 /* clear private here, it is used to chain pages */
241 p = alloc_page(gfp_mask);
245 static void virtqueue_napi_schedule(struct napi_struct *napi,
246 struct virtqueue *vq)
248 if (napi_schedule_prep(napi)) {
249 virtqueue_disable_cb(vq);
250 __napi_schedule(napi);
254 static void virtqueue_napi_complete(struct napi_struct *napi,
255 struct virtqueue *vq, int processed)
259 opaque = virtqueue_enable_cb_prepare(vq);
260 if (napi_complete_done(napi, processed) &&
261 unlikely(virtqueue_poll(vq, opaque)))
262 virtqueue_napi_schedule(napi, vq);
265 static void skb_xmit_done(struct virtqueue *vq)
267 struct virtnet_info *vi = vq->vdev->priv;
268 struct napi_struct *napi = &vi->sq[vq2txq(vq)].napi;
270 /* Suppress further interrupts. */
271 virtqueue_disable_cb(vq);
274 virtqueue_napi_schedule(napi, vq);
276 /* We were probably waiting for more output buffers. */
277 netif_wake_subqueue(vi->dev, vq2txq(vq));
280 static unsigned int mergeable_ctx_to_buf_truesize(unsigned long mrg_ctx)
282 unsigned int truesize = mrg_ctx & (MERGEABLE_BUFFER_ALIGN - 1);
283 return (truesize + 1) * MERGEABLE_BUFFER_ALIGN;
286 static void *mergeable_ctx_to_buf_address(unsigned long mrg_ctx)
288 return (void *)(mrg_ctx & -MERGEABLE_BUFFER_ALIGN);
292 static unsigned long mergeable_buf_to_ctx(void *buf, unsigned int truesize)
294 unsigned int size = truesize / MERGEABLE_BUFFER_ALIGN;
295 return (unsigned long)buf | (size - 1);
298 /* Called from bottom half context */
299 static struct sk_buff *page_to_skb(struct virtnet_info *vi,
300 struct receive_queue *rq,
301 struct page *page, unsigned int offset,
302 unsigned int len, unsigned int truesize)
305 struct virtio_net_hdr_mrg_rxbuf *hdr;
306 unsigned int copy, hdr_len, hdr_padded_len;
309 p = page_address(page) + offset;
311 /* copy small packet so we can reuse these pages for small data */
312 skb = napi_alloc_skb(&rq->napi, GOOD_COPY_LEN);
316 hdr = skb_vnet_hdr(skb);
318 hdr_len = vi->hdr_len;
319 if (vi->mergeable_rx_bufs)
320 hdr_padded_len = sizeof *hdr;
322 hdr_padded_len = sizeof(struct padded_vnet_hdr);
324 memcpy(hdr, p, hdr_len);
327 offset += hdr_padded_len;
331 if (copy > skb_tailroom(skb))
332 copy = skb_tailroom(skb);
333 memcpy(skb_put(skb, copy), p, copy);
338 if (vi->mergeable_rx_bufs) {
340 skb_add_rx_frag(skb, 0, page, offset, len, truesize);
347 * Verify that we can indeed put this data into a skb.
348 * This is here to handle cases when the device erroneously
349 * tries to receive more than is possible. This is usually
350 * the case of a broken device.
352 if (unlikely(len > MAX_SKB_FRAGS * PAGE_SIZE)) {
353 net_dbg_ratelimited("%s: too much data\n", skb->dev->name);
357 BUG_ON(offset >= PAGE_SIZE);
359 unsigned int frag_size = min((unsigned)PAGE_SIZE - offset, len);
360 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, offset,
361 frag_size, truesize);
363 page = (struct page *)page->private;
368 give_pages(rq, page);
373 static bool virtnet_xdp_xmit(struct virtnet_info *vi,
374 struct receive_queue *rq,
375 struct xdp_buff *xdp)
377 struct virtio_net_hdr_mrg_rxbuf *hdr;
379 struct send_queue *sq;
384 qp = vi->curr_queue_pairs - vi->xdp_queue_pairs + smp_processor_id();
387 /* Free up any pending old buffers before queueing new ones. */
388 while ((xdp_sent = virtqueue_get_buf(sq->vq, &len)) != NULL) {
389 struct page *sent_page = virt_to_head_page(xdp_sent);
394 xdp->data -= vi->hdr_len;
395 /* Zero header and leave csum up to XDP layers */
397 memset(hdr, 0, vi->hdr_len);
399 sg_init_one(sq->sg, xdp->data, xdp->data_end - xdp->data);
401 err = virtqueue_add_outbuf(sq->vq, sq->sg, 1, xdp->data, GFP_ATOMIC);
403 struct page *page = virt_to_head_page(xdp->data);
409 virtqueue_kick(sq->vq);
413 static unsigned int virtnet_get_headroom(struct virtnet_info *vi)
415 return vi->xdp_queue_pairs ? VIRTIO_XDP_HEADROOM : 0;
418 static struct sk_buff *receive_small(struct net_device *dev,
419 struct virtnet_info *vi,
420 struct receive_queue *rq,
421 void *buf, unsigned int len)
424 struct bpf_prog *xdp_prog;
425 unsigned int xdp_headroom = virtnet_get_headroom(vi);
426 unsigned int header_offset = VIRTNET_RX_PAD + xdp_headroom;
427 unsigned int headroom = vi->hdr_len + header_offset;
428 unsigned int buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) +
429 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
430 unsigned int delta = 0;
434 xdp_prog = rcu_dereference(rq->xdp_prog);
436 struct virtio_net_hdr_mrg_rxbuf *hdr = buf + header_offset;
441 if (unlikely(hdr->hdr.gso_type || hdr->hdr.flags))
444 xdp.data_hard_start = buf + VIRTNET_RX_PAD + vi->hdr_len;
445 xdp.data = xdp.data_hard_start + xdp_headroom;
446 xdp.data_end = xdp.data + len;
447 orig_data = xdp.data;
448 act = bpf_prog_run_xdp(xdp_prog, &xdp);
452 /* Recalculate length in case bpf program changed it */
453 delta = orig_data - xdp.data;
456 if (unlikely(!virtnet_xdp_xmit(vi, rq, &xdp)))
457 trace_xdp_exception(vi->dev, xdp_prog, act);
461 bpf_warn_invalid_xdp_action(act);
463 trace_xdp_exception(vi->dev, xdp_prog, act);
470 skb = build_skb(buf, buflen);
472 put_page(virt_to_head_page(buf));
475 skb_reserve(skb, headroom - delta);
476 skb_put(skb, len + delta);
478 buf += header_offset;
479 memcpy(skb_vnet_hdr(skb), buf, vi->hdr_len);
480 } /* keep zeroed vnet hdr since packet was changed by bpf */
487 dev->stats.rx_dropped++;
488 put_page(virt_to_head_page(buf));
493 static struct sk_buff *receive_big(struct net_device *dev,
494 struct virtnet_info *vi,
495 struct receive_queue *rq,
499 struct page *page = buf;
500 struct sk_buff *skb = page_to_skb(vi, rq, page, 0, len, PAGE_SIZE);
508 dev->stats.rx_dropped++;
509 give_pages(rq, page);
513 /* The conditions to enable XDP should preclude the underlying device from
514 * sending packets across multiple buffers (num_buf > 1). However per spec
515 * it does not appear to be illegal to do so but rather just against convention.
516 * So in order to avoid making a system unresponsive the packets are pushed
517 * into a page and the XDP program is run. This will be extremely slow and we
518 * push a warning to the user to fix this as soon as possible. Fixing this may
519 * require resolving the underlying hardware to determine why multiple buffers
520 * are being received or simply loading the XDP program in the ingress stack
521 * after the skb is built because there is no advantage to running it here
524 static struct page *xdp_linearize_page(struct receive_queue *rq,
530 struct page *page = alloc_page(GFP_ATOMIC);
531 unsigned int page_off = VIRTIO_XDP_HEADROOM;
536 memcpy(page_address(page) + page_off, page_address(p) + offset, *len);
545 ctx = (unsigned long)virtqueue_get_buf(rq->vq, &buflen);
549 buf = mergeable_ctx_to_buf_address(ctx);
550 p = virt_to_head_page(buf);
551 off = buf - page_address(p);
553 /* guard against a misconfigured or uncooperative backend that
554 * is sending packet larger than the MTU.
556 if ((page_off + buflen) > PAGE_SIZE) {
561 memcpy(page_address(page) + page_off,
562 page_address(p) + off, buflen);
567 /* Headroom does not contribute to packet length */
568 *len = page_off - VIRTIO_XDP_HEADROOM;
571 __free_pages(page, 0);
575 static struct sk_buff *receive_mergeable(struct net_device *dev,
576 struct virtnet_info *vi,
577 struct receive_queue *rq,
581 void *buf = mergeable_ctx_to_buf_address(ctx);
582 struct virtio_net_hdr_mrg_rxbuf *hdr = buf;
583 u16 num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers);
584 struct page *page = virt_to_head_page(buf);
585 int offset = buf - page_address(page);
586 struct sk_buff *head_skb, *curr_skb;
587 struct bpf_prog *xdp_prog;
588 unsigned int truesize;
593 xdp_prog = rcu_dereference(rq->xdp_prog);
595 struct page *xdp_page;
600 /* This happens when rx buffer size is underestimated */
601 if (unlikely(num_buf > 1)) {
602 /* linearize data for XDP */
603 xdp_page = xdp_linearize_page(rq, &num_buf,
607 offset = VIRTIO_XDP_HEADROOM;
612 /* Transient failure which in theory could occur if
613 * in-flight packets from before XDP was enabled reach
614 * the receive path after XDP is loaded. In practice I
615 * was not able to create this condition.
617 if (unlikely(hdr->hdr.gso_type))
620 /* Allow consuming headroom but reserve enough space to push
621 * the descriptor on if we get an XDP_TX return code.
623 data = page_address(xdp_page) + offset;
624 xdp.data_hard_start = data - VIRTIO_XDP_HEADROOM + vi->hdr_len;
625 xdp.data = data + vi->hdr_len;
626 xdp.data_end = xdp.data + (len - vi->hdr_len);
627 act = bpf_prog_run_xdp(xdp_prog, &xdp);
631 /* recalculate offset to account for any header
632 * adjustments. Note other cases do not build an
633 * skb and avoid using offset
636 page_address(xdp_page) - vi->hdr_len;
638 /* We can only create skb based on xdp_page. */
639 if (unlikely(xdp_page != page)) {
642 head_skb = page_to_skb(vi, rq, xdp_page,
643 offset, len, PAGE_SIZE);
644 ewma_pkt_len_add(&rq->mrg_avg_pkt_len, len);
649 if (unlikely(!virtnet_xdp_xmit(vi, rq, &xdp)))
650 trace_xdp_exception(vi->dev, xdp_prog, act);
651 ewma_pkt_len_add(&rq->mrg_avg_pkt_len, len);
652 if (unlikely(xdp_page != page))
657 bpf_warn_invalid_xdp_action(act);
659 trace_xdp_exception(vi->dev, xdp_prog, act);
661 if (unlikely(xdp_page != page))
662 __free_pages(xdp_page, 0);
663 ewma_pkt_len_add(&rq->mrg_avg_pkt_len, len);
669 truesize = max(len, mergeable_ctx_to_buf_truesize(ctx));
670 head_skb = page_to_skb(vi, rq, page, offset, len, truesize);
673 if (unlikely(!curr_skb))
678 ctx = (unsigned long)virtqueue_get_buf(rq->vq, &len);
679 if (unlikely(!ctx)) {
680 pr_debug("%s: rx error: %d buffers out of %d missing\n",
682 virtio16_to_cpu(vi->vdev,
684 dev->stats.rx_length_errors++;
688 buf = mergeable_ctx_to_buf_address(ctx);
689 page = virt_to_head_page(buf);
691 num_skb_frags = skb_shinfo(curr_skb)->nr_frags;
692 if (unlikely(num_skb_frags == MAX_SKB_FRAGS)) {
693 struct sk_buff *nskb = alloc_skb(0, GFP_ATOMIC);
697 if (curr_skb == head_skb)
698 skb_shinfo(curr_skb)->frag_list = nskb;
700 curr_skb->next = nskb;
702 head_skb->truesize += nskb->truesize;
705 truesize = max(len, mergeable_ctx_to_buf_truesize(ctx));
706 if (curr_skb != head_skb) {
707 head_skb->data_len += len;
708 head_skb->len += len;
709 head_skb->truesize += truesize;
711 offset = buf - page_address(page);
712 if (skb_can_coalesce(curr_skb, num_skb_frags, page, offset)) {
714 skb_coalesce_rx_frag(curr_skb, num_skb_frags - 1,
717 skb_add_rx_frag(curr_skb, num_skb_frags, page,
718 offset, len, truesize);
722 ewma_pkt_len_add(&rq->mrg_avg_pkt_len, head_skb->len);
730 ctx = (unsigned long)virtqueue_get_buf(rq->vq, &len);
731 if (unlikely(!ctx)) {
732 pr_debug("%s: rx error: %d buffers missing\n",
734 dev->stats.rx_length_errors++;
737 page = virt_to_head_page(mergeable_ctx_to_buf_address(ctx));
741 dev->stats.rx_dropped++;
742 dev_kfree_skb(head_skb);
747 static int receive_buf(struct virtnet_info *vi, struct receive_queue *rq,
748 void *buf, unsigned int len)
750 struct net_device *dev = vi->dev;
752 struct virtio_net_hdr_mrg_rxbuf *hdr;
755 if (unlikely(len < vi->hdr_len + ETH_HLEN)) {
756 pr_debug("%s: short packet %i\n", dev->name, len);
757 dev->stats.rx_length_errors++;
758 if (vi->mergeable_rx_bufs) {
759 unsigned long ctx = (unsigned long)buf;
760 void *base = mergeable_ctx_to_buf_address(ctx);
761 put_page(virt_to_head_page(base));
762 } else if (vi->big_packets) {
765 put_page(virt_to_head_page(buf));
770 if (vi->mergeable_rx_bufs)
771 skb = receive_mergeable(dev, vi, rq, (unsigned long)buf, len);
772 else if (vi->big_packets)
773 skb = receive_big(dev, vi, rq, buf, len);
775 skb = receive_small(dev, vi, rq, buf, len);
780 hdr = skb_vnet_hdr(skb);
784 if (hdr->hdr.flags & VIRTIO_NET_HDR_F_DATA_VALID)
785 skb->ip_summed = CHECKSUM_UNNECESSARY;
787 if (virtio_net_hdr_to_skb(skb, &hdr->hdr,
788 virtio_is_little_endian(vi->vdev))) {
789 net_warn_ratelimited("%s: bad gso: type: %u, size: %u\n",
790 dev->name, hdr->hdr.gso_type,
795 skb->protocol = eth_type_trans(skb, dev);
796 pr_debug("Receiving skb proto 0x%04x len %i type %i\n",
797 ntohs(skb->protocol), skb->len, skb->pkt_type);
799 napi_gro_receive(&rq->napi, skb);
803 dev->stats.rx_frame_errors++;
808 static int add_recvbuf_small(struct virtnet_info *vi, struct receive_queue *rq,
811 struct page_frag *alloc_frag = &rq->alloc_frag;
813 unsigned int xdp_headroom = virtnet_get_headroom(vi);
814 int len = vi->hdr_len + VIRTNET_RX_PAD + GOOD_PACKET_LEN + xdp_headroom;
817 len = SKB_DATA_ALIGN(len) +
818 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
819 if (unlikely(!skb_page_frag_refill(len, alloc_frag, gfp)))
822 buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
823 get_page(alloc_frag->page);
824 alloc_frag->offset += len;
825 sg_init_one(rq->sg, buf + VIRTNET_RX_PAD + xdp_headroom,
826 vi->hdr_len + GOOD_PACKET_LEN);
827 err = virtqueue_add_inbuf(rq->vq, rq->sg, 1, buf, gfp);
829 put_page(virt_to_head_page(buf));
834 static int add_recvbuf_big(struct virtnet_info *vi, struct receive_queue *rq,
837 struct page *first, *list = NULL;
841 sg_init_table(rq->sg, MAX_SKB_FRAGS + 2);
843 /* page in rq->sg[MAX_SKB_FRAGS + 1] is list tail */
844 for (i = MAX_SKB_FRAGS + 1; i > 1; --i) {
845 first = get_a_page(rq, gfp);
848 give_pages(rq, list);
851 sg_set_buf(&rq->sg[i], page_address(first), PAGE_SIZE);
853 /* chain new page in list head to match sg */
854 first->private = (unsigned long)list;
858 first = get_a_page(rq, gfp);
860 give_pages(rq, list);
863 p = page_address(first);
865 /* rq->sg[0], rq->sg[1] share the same page */
866 /* a separated rq->sg[0] for header - required in case !any_header_sg */
867 sg_set_buf(&rq->sg[0], p, vi->hdr_len);
869 /* rq->sg[1] for data packet, from offset */
870 offset = sizeof(struct padded_vnet_hdr);
871 sg_set_buf(&rq->sg[1], p + offset, PAGE_SIZE - offset);
873 /* chain first in list head */
874 first->private = (unsigned long)list;
875 err = virtqueue_add_inbuf(rq->vq, rq->sg, MAX_SKB_FRAGS + 2,
878 give_pages(rq, first);
883 static unsigned int get_mergeable_buf_len(struct ewma_pkt_len *avg_pkt_len)
885 const size_t hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);
888 len = hdr_len + clamp_t(unsigned int, ewma_pkt_len_read(avg_pkt_len),
889 GOOD_PACKET_LEN, PAGE_SIZE - hdr_len);
890 return ALIGN(len, MERGEABLE_BUFFER_ALIGN);
893 static int add_recvbuf_mergeable(struct virtnet_info *vi,
894 struct receive_queue *rq, gfp_t gfp)
896 struct page_frag *alloc_frag = &rq->alloc_frag;
897 unsigned int headroom = virtnet_get_headroom(vi);
901 unsigned int len, hole;
903 len = get_mergeable_buf_len(&rq->mrg_avg_pkt_len);
904 if (unlikely(!skb_page_frag_refill(len + headroom, alloc_frag, gfp)))
907 buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
908 buf += headroom; /* advance address leaving hole at front of pkt */
909 ctx = mergeable_buf_to_ctx(buf, len);
910 get_page(alloc_frag->page);
911 alloc_frag->offset += len + headroom;
912 hole = alloc_frag->size - alloc_frag->offset;
913 if (hole < len + headroom) {
914 /* To avoid internal fragmentation, if there is very likely not
915 * enough space for another buffer, add the remaining space to
916 * the current buffer. This extra space is not included in
917 * the truesize stored in ctx.
920 alloc_frag->offset += hole;
923 sg_init_one(rq->sg, buf, len);
924 err = virtqueue_add_inbuf(rq->vq, rq->sg, 1, (void *)ctx, gfp);
926 put_page(virt_to_head_page(buf));
932 * Returns false if we couldn't fill entirely (OOM).
934 * Normally run in the receive path, but can also be run from ndo_open
935 * before we're receiving packets, or from refill_work which is
936 * careful to disable receiving (using napi_disable).
938 static bool try_fill_recv(struct virtnet_info *vi, struct receive_queue *rq,
946 if (vi->mergeable_rx_bufs)
947 err = add_recvbuf_mergeable(vi, rq, gfp);
948 else if (vi->big_packets)
949 err = add_recvbuf_big(vi, rq, gfp);
951 err = add_recvbuf_small(vi, rq, gfp);
953 oom = err == -ENOMEM;
956 } while (rq->vq->num_free);
957 virtqueue_kick(rq->vq);
961 static void skb_recv_done(struct virtqueue *rvq)
963 struct virtnet_info *vi = rvq->vdev->priv;
964 struct receive_queue *rq = &vi->rq[vq2rxq(rvq)];
966 virtqueue_napi_schedule(&rq->napi, rvq);
969 static void virtnet_napi_enable(struct virtqueue *vq, struct napi_struct *napi)
973 /* If all buffers were filled by other side before we napi_enabled, we
974 * won't get another interrupt, so process any outstanding packets now.
975 * Call local_bh_enable after to trigger softIRQ processing.
978 virtqueue_napi_schedule(napi, vq);
982 static void virtnet_napi_tx_enable(struct virtnet_info *vi,
983 struct virtqueue *vq,
984 struct napi_struct *napi)
989 /* Tx napi touches cachelines on the cpu handling tx interrupts. Only
990 * enable the feature if this is likely affine with the transmit path.
992 if (!vi->affinity_hint_set) {
997 return virtnet_napi_enable(vq, napi);
1000 static void virtnet_napi_tx_disable(struct napi_struct *napi)
1006 static void refill_work(struct work_struct *work)
1008 struct virtnet_info *vi =
1009 container_of(work, struct virtnet_info, refill.work);
1013 for (i = 0; i < vi->curr_queue_pairs; i++) {
1014 struct receive_queue *rq = &vi->rq[i];
1016 napi_disable(&rq->napi);
1017 still_empty = !try_fill_recv(vi, rq, GFP_KERNEL);
1018 virtnet_napi_enable(rq->vq, &rq->napi);
1020 /* In theory, this can happen: if we don't get any buffers in
1021 * we will *never* try to fill again.
1024 schedule_delayed_work(&vi->refill, HZ/2);
1028 static int virtnet_receive(struct receive_queue *rq, int budget)
1030 struct virtnet_info *vi = rq->vq->vdev->priv;
1031 unsigned int len, received = 0, bytes = 0;
1033 struct virtnet_stats *stats = this_cpu_ptr(vi->stats);
1035 while (received < budget &&
1036 (buf = virtqueue_get_buf(rq->vq, &len)) != NULL) {
1037 bytes += receive_buf(vi, rq, buf, len);
1041 if (rq->vq->num_free > virtqueue_get_vring_size(rq->vq) / 2) {
1042 if (!try_fill_recv(vi, rq, GFP_ATOMIC))
1043 schedule_delayed_work(&vi->refill, 0);
1046 u64_stats_update_begin(&stats->rx_syncp);
1047 stats->rx_bytes += bytes;
1048 stats->rx_packets += received;
1049 u64_stats_update_end(&stats->rx_syncp);
1054 static void free_old_xmit_skbs(struct send_queue *sq)
1056 struct sk_buff *skb;
1058 struct virtnet_info *vi = sq->vq->vdev->priv;
1059 struct virtnet_stats *stats = this_cpu_ptr(vi->stats);
1060 unsigned int packets = 0;
1061 unsigned int bytes = 0;
1063 while ((skb = virtqueue_get_buf(sq->vq, &len)) != NULL) {
1064 pr_debug("Sent skb %p\n", skb);
1069 dev_kfree_skb_any(skb);
1072 /* Avoid overhead when no packets have been processed
1073 * happens when called speculatively from start_xmit.
1078 u64_stats_update_begin(&stats->tx_syncp);
1079 stats->tx_bytes += bytes;
1080 stats->tx_packets += packets;
1081 u64_stats_update_end(&stats->tx_syncp);
1084 static void virtnet_poll_cleantx(struct receive_queue *rq)
1086 struct virtnet_info *vi = rq->vq->vdev->priv;
1087 unsigned int index = vq2rxq(rq->vq);
1088 struct send_queue *sq = &vi->sq[index];
1089 struct netdev_queue *txq = netdev_get_tx_queue(vi->dev, index);
1091 if (!sq->napi.weight)
1094 if (__netif_tx_trylock(txq)) {
1095 free_old_xmit_skbs(sq);
1096 __netif_tx_unlock(txq);
1099 if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS)
1100 netif_tx_wake_queue(txq);
1103 static int virtnet_poll(struct napi_struct *napi, int budget)
1105 struct receive_queue *rq =
1106 container_of(napi, struct receive_queue, napi);
1107 unsigned int received;
1109 virtnet_poll_cleantx(rq);
1111 received = virtnet_receive(rq, budget);
1113 /* Out of packets? */
1114 if (received < budget)
1115 virtqueue_napi_complete(napi, rq->vq, received);
1120 static int virtnet_open(struct net_device *dev)
1122 struct virtnet_info *vi = netdev_priv(dev);
1125 for (i = 0; i < vi->max_queue_pairs; i++) {
1126 if (i < vi->curr_queue_pairs)
1127 /* Make sure we have some buffers: if oom use wq. */
1128 if (!try_fill_recv(vi, &vi->rq[i], GFP_KERNEL))
1129 schedule_delayed_work(&vi->refill, 0);
1130 virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi);
1131 virtnet_napi_tx_enable(vi, vi->sq[i].vq, &vi->sq[i].napi);
1137 static int virtnet_poll_tx(struct napi_struct *napi, int budget)
1139 struct send_queue *sq = container_of(napi, struct send_queue, napi);
1140 struct virtnet_info *vi = sq->vq->vdev->priv;
1141 struct netdev_queue *txq = netdev_get_tx_queue(vi->dev, vq2txq(sq->vq));
1143 __netif_tx_lock(txq, raw_smp_processor_id());
1144 free_old_xmit_skbs(sq);
1145 __netif_tx_unlock(txq);
1147 virtqueue_napi_complete(napi, sq->vq, 0);
1149 if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS)
1150 netif_tx_wake_queue(txq);
1155 static int xmit_skb(struct send_queue *sq, struct sk_buff *skb)
1157 struct virtio_net_hdr_mrg_rxbuf *hdr;
1158 const unsigned char *dest = ((struct ethhdr *)skb->data)->h_dest;
1159 struct virtnet_info *vi = sq->vq->vdev->priv;
1161 unsigned hdr_len = vi->hdr_len;
1164 pr_debug("%s: xmit %p %pM\n", vi->dev->name, skb, dest);
1166 can_push = vi->any_header_sg &&
1167 !((unsigned long)skb->data & (__alignof__(*hdr) - 1)) &&
1168 !skb_header_cloned(skb) && skb_headroom(skb) >= hdr_len;
1169 /* Even if we can, don't push here yet as this would skew
1170 * csum_start offset below. */
1172 hdr = (struct virtio_net_hdr_mrg_rxbuf *)(skb->data - hdr_len);
1174 hdr = skb_vnet_hdr(skb);
1176 if (virtio_net_hdr_from_skb(skb, &hdr->hdr,
1177 virtio_is_little_endian(vi->vdev), false))
1180 if (vi->mergeable_rx_bufs)
1181 hdr->num_buffers = 0;
1183 sg_init_table(sq->sg, skb_shinfo(skb)->nr_frags + (can_push ? 1 : 2));
1185 __skb_push(skb, hdr_len);
1186 num_sg = skb_to_sgvec(skb, sq->sg, 0, skb->len);
1187 /* Pull header back to avoid skew in tx bytes calculations. */
1188 __skb_pull(skb, hdr_len);
1190 sg_set_buf(sq->sg, hdr, hdr_len);
1191 num_sg = skb_to_sgvec(skb, sq->sg + 1, 0, skb->len) + 1;
1193 return virtqueue_add_outbuf(sq->vq, sq->sg, num_sg, skb, GFP_ATOMIC);
1196 static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev)
1198 struct virtnet_info *vi = netdev_priv(dev);
1199 int qnum = skb_get_queue_mapping(skb);
1200 struct send_queue *sq = &vi->sq[qnum];
1202 struct netdev_queue *txq = netdev_get_tx_queue(dev, qnum);
1203 bool kick = !skb->xmit_more;
1204 bool use_napi = sq->napi.weight;
1206 /* Free up any pending old buffers before queueing new ones. */
1207 free_old_xmit_skbs(sq);
1209 if (use_napi && kick)
1210 virtqueue_enable_cb_delayed(sq->vq);
1212 /* timestamp packet in software */
1213 skb_tx_timestamp(skb);
1215 /* Try to transmit */
1216 err = xmit_skb(sq, skb);
1218 /* This should not happen! */
1219 if (unlikely(err)) {
1220 dev->stats.tx_fifo_errors++;
1221 if (net_ratelimit())
1223 "Unexpected TXQ (%d) queue failure: %d\n", qnum, err);
1224 dev->stats.tx_dropped++;
1225 dev_kfree_skb_any(skb);
1226 return NETDEV_TX_OK;
1229 /* Don't wait up for transmitted skbs to be freed. */
1235 /* If running out of space, stop queue to avoid getting packets that we
1236 * are then unable to transmit.
1237 * An alternative would be to force queuing layer to requeue the skb by
1238 * returning NETDEV_TX_BUSY. However, NETDEV_TX_BUSY should not be
1239 * returned in a normal path of operation: it means that driver is not
1240 * maintaining the TX queue stop/start state properly, and causes
1241 * the stack to do a non-trivial amount of useless work.
1242 * Since most packets only take 1 or 2 ring slots, stopping the queue
1243 * early means 16 slots are typically wasted.
1245 if (sq->vq->num_free < 2+MAX_SKB_FRAGS) {
1246 netif_stop_subqueue(dev, qnum);
1248 unlikely(!virtqueue_enable_cb_delayed(sq->vq))) {
1249 /* More just got used, free them then recheck. */
1250 free_old_xmit_skbs(sq);
1251 if (sq->vq->num_free >= 2+MAX_SKB_FRAGS) {
1252 netif_start_subqueue(dev, qnum);
1253 virtqueue_disable_cb(sq->vq);
1258 if (kick || netif_xmit_stopped(txq))
1259 virtqueue_kick(sq->vq);
1261 return NETDEV_TX_OK;
1265 * Send command via the control virtqueue and check status. Commands
1266 * supported by the hypervisor, as indicated by feature bits, should
1267 * never fail unless improperly formatted.
1269 static bool virtnet_send_command(struct virtnet_info *vi, u8 class, u8 cmd,
1270 struct scatterlist *out)
1272 struct scatterlist *sgs[4], hdr, stat;
1273 unsigned out_num = 0, tmp;
1275 /* Caller should know better */
1276 BUG_ON(!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ));
1278 vi->ctrl_status = ~0;
1279 vi->ctrl_hdr.class = class;
1280 vi->ctrl_hdr.cmd = cmd;
1282 sg_init_one(&hdr, &vi->ctrl_hdr, sizeof(vi->ctrl_hdr));
1283 sgs[out_num++] = &hdr;
1286 sgs[out_num++] = out;
1288 /* Add return status. */
1289 sg_init_one(&stat, &vi->ctrl_status, sizeof(vi->ctrl_status));
1290 sgs[out_num] = &stat;
1292 BUG_ON(out_num + 1 > ARRAY_SIZE(sgs));
1293 virtqueue_add_sgs(vi->cvq, sgs, out_num, 1, vi, GFP_ATOMIC);
1295 if (unlikely(!virtqueue_kick(vi->cvq)))
1296 return vi->ctrl_status == VIRTIO_NET_OK;
1298 /* Spin for a response, the kick causes an ioport write, trapping
1299 * into the hypervisor, so the request should be handled immediately.
1301 while (!virtqueue_get_buf(vi->cvq, &tmp) &&
1302 !virtqueue_is_broken(vi->cvq))
1305 return vi->ctrl_status == VIRTIO_NET_OK;
1308 static int virtnet_set_mac_address(struct net_device *dev, void *p)
1310 struct virtnet_info *vi = netdev_priv(dev);
1311 struct virtio_device *vdev = vi->vdev;
1313 struct sockaddr *addr;
1314 struct scatterlist sg;
1316 addr = kmemdup(p, sizeof(*addr), GFP_KERNEL);
1320 ret = eth_prepare_mac_addr_change(dev, addr);
1324 if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR)) {
1325 sg_init_one(&sg, addr->sa_data, dev->addr_len);
1326 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC,
1327 VIRTIO_NET_CTRL_MAC_ADDR_SET, &sg)) {
1328 dev_warn(&vdev->dev,
1329 "Failed to set mac address by vq command.\n");
1333 } else if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC) &&
1334 !virtio_has_feature(vdev, VIRTIO_F_VERSION_1)) {
1337 /* Naturally, this has an atomicity problem. */
1338 for (i = 0; i < dev->addr_len; i++)
1339 virtio_cwrite8(vdev,
1340 offsetof(struct virtio_net_config, mac) +
1341 i, addr->sa_data[i]);
1344 eth_commit_mac_addr_change(dev, p);
1352 static void virtnet_stats(struct net_device *dev,
1353 struct rtnl_link_stats64 *tot)
1355 struct virtnet_info *vi = netdev_priv(dev);
1359 for_each_possible_cpu(cpu) {
1360 struct virtnet_stats *stats = per_cpu_ptr(vi->stats, cpu);
1361 u64 tpackets, tbytes, rpackets, rbytes;
1364 start = u64_stats_fetch_begin_irq(&stats->tx_syncp);
1365 tpackets = stats->tx_packets;
1366 tbytes = stats->tx_bytes;
1367 } while (u64_stats_fetch_retry_irq(&stats->tx_syncp, start));
1370 start = u64_stats_fetch_begin_irq(&stats->rx_syncp);
1371 rpackets = stats->rx_packets;
1372 rbytes = stats->rx_bytes;
1373 } while (u64_stats_fetch_retry_irq(&stats->rx_syncp, start));
1375 tot->rx_packets += rpackets;
1376 tot->tx_packets += tpackets;
1377 tot->rx_bytes += rbytes;
1378 tot->tx_bytes += tbytes;
1381 tot->tx_dropped = dev->stats.tx_dropped;
1382 tot->tx_fifo_errors = dev->stats.tx_fifo_errors;
1383 tot->rx_dropped = dev->stats.rx_dropped;
1384 tot->rx_length_errors = dev->stats.rx_length_errors;
1385 tot->rx_frame_errors = dev->stats.rx_frame_errors;
1388 #ifdef CONFIG_NET_POLL_CONTROLLER
1389 static void virtnet_netpoll(struct net_device *dev)
1391 struct virtnet_info *vi = netdev_priv(dev);
1394 for (i = 0; i < vi->curr_queue_pairs; i++)
1395 napi_schedule(&vi->rq[i].napi);
1399 static void virtnet_ack_link_announce(struct virtnet_info *vi)
1402 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_ANNOUNCE,
1403 VIRTIO_NET_CTRL_ANNOUNCE_ACK, NULL))
1404 dev_warn(&vi->dev->dev, "Failed to ack link announce.\n");
1408 static int _virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs)
1410 struct scatterlist sg;
1411 struct net_device *dev = vi->dev;
1413 if (!vi->has_cvq || !virtio_has_feature(vi->vdev, VIRTIO_NET_F_MQ))
1416 vi->ctrl_mq.virtqueue_pairs = cpu_to_virtio16(vi->vdev, queue_pairs);
1417 sg_init_one(&sg, &vi->ctrl_mq, sizeof(vi->ctrl_mq));
1419 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MQ,
1420 VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET, &sg)) {
1421 dev_warn(&dev->dev, "Fail to set num of queue pairs to %d\n",
1425 vi->curr_queue_pairs = queue_pairs;
1426 /* virtnet_open() will refill when device is going to up. */
1427 if (dev->flags & IFF_UP)
1428 schedule_delayed_work(&vi->refill, 0);
1434 static int virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs)
1439 err = _virtnet_set_queues(vi, queue_pairs);
1444 static int virtnet_close(struct net_device *dev)
1446 struct virtnet_info *vi = netdev_priv(dev);
1449 /* Make sure refill_work doesn't re-enable napi! */
1450 cancel_delayed_work_sync(&vi->refill);
1452 for (i = 0; i < vi->max_queue_pairs; i++) {
1453 napi_disable(&vi->rq[i].napi);
1454 virtnet_napi_tx_disable(&vi->sq[i].napi);
1460 static void virtnet_set_rx_mode(struct net_device *dev)
1462 struct virtnet_info *vi = netdev_priv(dev);
1463 struct scatterlist sg[2];
1464 struct virtio_net_ctrl_mac *mac_data;
1465 struct netdev_hw_addr *ha;
1471 /* We can't dynamically set ndo_set_rx_mode, so return gracefully */
1472 if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_RX))
1475 vi->ctrl_promisc = ((dev->flags & IFF_PROMISC) != 0);
1476 vi->ctrl_allmulti = ((dev->flags & IFF_ALLMULTI) != 0);
1478 sg_init_one(sg, &vi->ctrl_promisc, sizeof(vi->ctrl_promisc));
1480 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,
1481 VIRTIO_NET_CTRL_RX_PROMISC, sg))
1482 dev_warn(&dev->dev, "Failed to %sable promisc mode.\n",
1483 vi->ctrl_promisc ? "en" : "dis");
1485 sg_init_one(sg, &vi->ctrl_allmulti, sizeof(vi->ctrl_allmulti));
1487 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,
1488 VIRTIO_NET_CTRL_RX_ALLMULTI, sg))
1489 dev_warn(&dev->dev, "Failed to %sable allmulti mode.\n",
1490 vi->ctrl_allmulti ? "en" : "dis");
1492 uc_count = netdev_uc_count(dev);
1493 mc_count = netdev_mc_count(dev);
1494 /* MAC filter - use one buffer for both lists */
1495 buf = kzalloc(((uc_count + mc_count) * ETH_ALEN) +
1496 (2 * sizeof(mac_data->entries)), GFP_ATOMIC);
1501 sg_init_table(sg, 2);
1503 /* Store the unicast list and count in the front of the buffer */
1504 mac_data->entries = cpu_to_virtio32(vi->vdev, uc_count);
1506 netdev_for_each_uc_addr(ha, dev)
1507 memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN);
1509 sg_set_buf(&sg[0], mac_data,
1510 sizeof(mac_data->entries) + (uc_count * ETH_ALEN));
1512 /* multicast list and count fill the end */
1513 mac_data = (void *)&mac_data->macs[uc_count][0];
1515 mac_data->entries = cpu_to_virtio32(vi->vdev, mc_count);
1517 netdev_for_each_mc_addr(ha, dev)
1518 memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN);
1520 sg_set_buf(&sg[1], mac_data,
1521 sizeof(mac_data->entries) + (mc_count * ETH_ALEN));
1523 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC,
1524 VIRTIO_NET_CTRL_MAC_TABLE_SET, sg))
1525 dev_warn(&dev->dev, "Failed to set MAC filter table.\n");
1530 static int virtnet_vlan_rx_add_vid(struct net_device *dev,
1531 __be16 proto, u16 vid)
1533 struct virtnet_info *vi = netdev_priv(dev);
1534 struct scatterlist sg;
1537 sg_init_one(&sg, &vi->ctrl_vid, sizeof(vi->ctrl_vid));
1539 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,
1540 VIRTIO_NET_CTRL_VLAN_ADD, &sg))
1541 dev_warn(&dev->dev, "Failed to add VLAN ID %d.\n", vid);
1545 static int virtnet_vlan_rx_kill_vid(struct net_device *dev,
1546 __be16 proto, u16 vid)
1548 struct virtnet_info *vi = netdev_priv(dev);
1549 struct scatterlist sg;
1552 sg_init_one(&sg, &vi->ctrl_vid, sizeof(vi->ctrl_vid));
1554 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,
1555 VIRTIO_NET_CTRL_VLAN_DEL, &sg))
1556 dev_warn(&dev->dev, "Failed to kill VLAN ID %d.\n", vid);
1560 static void virtnet_clean_affinity(struct virtnet_info *vi, long hcpu)
1564 if (vi->affinity_hint_set) {
1565 for (i = 0; i < vi->max_queue_pairs; i++) {
1566 virtqueue_set_affinity(vi->rq[i].vq, -1);
1567 virtqueue_set_affinity(vi->sq[i].vq, -1);
1570 vi->affinity_hint_set = false;
1574 static void virtnet_set_affinity(struct virtnet_info *vi)
1579 /* In multiqueue mode, when the number of cpu is equal to the number of
1580 * queue pairs, we let the queue pairs to be private to one cpu by
1581 * setting the affinity hint to eliminate the contention.
1583 if (vi->curr_queue_pairs == 1 ||
1584 vi->max_queue_pairs != num_online_cpus()) {
1585 virtnet_clean_affinity(vi, -1);
1590 for_each_online_cpu(cpu) {
1591 virtqueue_set_affinity(vi->rq[i].vq, cpu);
1592 virtqueue_set_affinity(vi->sq[i].vq, cpu);
1593 netif_set_xps_queue(vi->dev, cpumask_of(cpu), i);
1597 vi->affinity_hint_set = true;
1600 static int virtnet_cpu_online(unsigned int cpu, struct hlist_node *node)
1602 struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info,
1604 virtnet_set_affinity(vi);
1608 static int virtnet_cpu_dead(unsigned int cpu, struct hlist_node *node)
1610 struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info,
1612 virtnet_set_affinity(vi);
1616 static int virtnet_cpu_down_prep(unsigned int cpu, struct hlist_node *node)
1618 struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info,
1621 virtnet_clean_affinity(vi, cpu);
1625 static enum cpuhp_state virtionet_online;
1627 static int virtnet_cpu_notif_add(struct virtnet_info *vi)
1631 ret = cpuhp_state_add_instance_nocalls(virtionet_online, &vi->node);
1634 ret = cpuhp_state_add_instance_nocalls(CPUHP_VIRT_NET_DEAD,
1638 cpuhp_state_remove_instance_nocalls(virtionet_online, &vi->node);
1642 static void virtnet_cpu_notif_remove(struct virtnet_info *vi)
1644 cpuhp_state_remove_instance_nocalls(virtionet_online, &vi->node);
1645 cpuhp_state_remove_instance_nocalls(CPUHP_VIRT_NET_DEAD,
1649 static void virtnet_get_ringparam(struct net_device *dev,
1650 struct ethtool_ringparam *ring)
1652 struct virtnet_info *vi = netdev_priv(dev);
1654 ring->rx_max_pending = virtqueue_get_vring_size(vi->rq[0].vq);
1655 ring->tx_max_pending = virtqueue_get_vring_size(vi->sq[0].vq);
1656 ring->rx_pending = ring->rx_max_pending;
1657 ring->tx_pending = ring->tx_max_pending;
1661 static void virtnet_get_drvinfo(struct net_device *dev,
1662 struct ethtool_drvinfo *info)
1664 struct virtnet_info *vi = netdev_priv(dev);
1665 struct virtio_device *vdev = vi->vdev;
1667 strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
1668 strlcpy(info->version, VIRTNET_DRIVER_VERSION, sizeof(info->version));
1669 strlcpy(info->bus_info, virtio_bus_name(vdev), sizeof(info->bus_info));
1673 /* TODO: Eliminate OOO packets during switching */
1674 static int virtnet_set_channels(struct net_device *dev,
1675 struct ethtool_channels *channels)
1677 struct virtnet_info *vi = netdev_priv(dev);
1678 u16 queue_pairs = channels->combined_count;
1681 /* We don't support separate rx/tx channels.
1682 * We don't allow setting 'other' channels.
1684 if (channels->rx_count || channels->tx_count || channels->other_count)
1687 if (queue_pairs > vi->max_queue_pairs || queue_pairs == 0)
1690 /* For now we don't support modifying channels while XDP is loaded
1691 * also when XDP is loaded all RX queues have XDP programs so we only
1692 * need to check a single RX queue.
1694 if (vi->rq[0].xdp_prog)
1698 err = _virtnet_set_queues(vi, queue_pairs);
1700 netif_set_real_num_tx_queues(dev, queue_pairs);
1701 netif_set_real_num_rx_queues(dev, queue_pairs);
1703 virtnet_set_affinity(vi);
1710 static void virtnet_get_channels(struct net_device *dev,
1711 struct ethtool_channels *channels)
1713 struct virtnet_info *vi = netdev_priv(dev);
1715 channels->combined_count = vi->curr_queue_pairs;
1716 channels->max_combined = vi->max_queue_pairs;
1717 channels->max_other = 0;
1718 channels->rx_count = 0;
1719 channels->tx_count = 0;
1720 channels->other_count = 0;
1723 /* Check if the user is trying to change anything besides speed/duplex */
1725 virtnet_validate_ethtool_cmd(const struct ethtool_link_ksettings *cmd)
1727 struct ethtool_link_ksettings diff1 = *cmd;
1728 struct ethtool_link_ksettings diff2 = {};
1730 /* cmd is always set so we need to clear it, validate the port type
1731 * and also without autonegotiation we can ignore advertising
1733 diff1.base.speed = 0;
1734 diff2.base.port = PORT_OTHER;
1735 ethtool_link_ksettings_zero_link_mode(&diff1, advertising);
1736 diff1.base.duplex = 0;
1738 diff1.base.link_mode_masks_nwords = 0;
1740 return !memcmp(&diff1.base, &diff2.base, sizeof(diff1.base)) &&
1741 bitmap_empty(diff1.link_modes.supported,
1742 __ETHTOOL_LINK_MODE_MASK_NBITS) &&
1743 bitmap_empty(diff1.link_modes.advertising,
1744 __ETHTOOL_LINK_MODE_MASK_NBITS) &&
1745 bitmap_empty(diff1.link_modes.lp_advertising,
1746 __ETHTOOL_LINK_MODE_MASK_NBITS);
1749 static int virtnet_set_link_ksettings(struct net_device *dev,
1750 const struct ethtool_link_ksettings *cmd)
1752 struct virtnet_info *vi = netdev_priv(dev);
1755 speed = cmd->base.speed;
1756 /* don't allow custom speed and duplex */
1757 if (!ethtool_validate_speed(speed) ||
1758 !ethtool_validate_duplex(cmd->base.duplex) ||
1759 !virtnet_validate_ethtool_cmd(cmd))
1762 vi->duplex = cmd->base.duplex;
1767 static int virtnet_get_link_ksettings(struct net_device *dev,
1768 struct ethtool_link_ksettings *cmd)
1770 struct virtnet_info *vi = netdev_priv(dev);
1772 cmd->base.speed = vi->speed;
1773 cmd->base.duplex = vi->duplex;
1774 cmd->base.port = PORT_OTHER;
1779 static void virtnet_init_settings(struct net_device *dev)
1781 struct virtnet_info *vi = netdev_priv(dev);
1783 vi->speed = SPEED_UNKNOWN;
1784 vi->duplex = DUPLEX_UNKNOWN;
1787 static const struct ethtool_ops virtnet_ethtool_ops = {
1788 .get_drvinfo = virtnet_get_drvinfo,
1789 .get_link = ethtool_op_get_link,
1790 .get_ringparam = virtnet_get_ringparam,
1791 .set_channels = virtnet_set_channels,
1792 .get_channels = virtnet_get_channels,
1793 .get_ts_info = ethtool_op_get_ts_info,
1794 .get_link_ksettings = virtnet_get_link_ksettings,
1795 .set_link_ksettings = virtnet_set_link_ksettings,
1798 static void virtnet_freeze_down(struct virtio_device *vdev)
1800 struct virtnet_info *vi = vdev->priv;
1803 /* Make sure no work handler is accessing the device */
1804 flush_work(&vi->config_work);
1806 netif_device_detach(vi->dev);
1807 cancel_delayed_work_sync(&vi->refill);
1809 if (netif_running(vi->dev)) {
1810 for (i = 0; i < vi->max_queue_pairs; i++) {
1811 napi_disable(&vi->rq[i].napi);
1812 virtnet_napi_tx_disable(&vi->sq[i].napi);
1817 static int init_vqs(struct virtnet_info *vi);
1818 static void _remove_vq_common(struct virtnet_info *vi);
1820 static int virtnet_restore_up(struct virtio_device *vdev)
1822 struct virtnet_info *vi = vdev->priv;
1829 virtio_device_ready(vdev);
1831 if (netif_running(vi->dev)) {
1832 for (i = 0; i < vi->curr_queue_pairs; i++)
1833 if (!try_fill_recv(vi, &vi->rq[i], GFP_KERNEL))
1834 schedule_delayed_work(&vi->refill, 0);
1836 for (i = 0; i < vi->max_queue_pairs; i++) {
1837 virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi);
1838 virtnet_napi_tx_enable(vi, vi->sq[i].vq,
1843 netif_device_attach(vi->dev);
1847 static int virtnet_reset(struct virtnet_info *vi, int curr_qp, int xdp_qp)
1849 struct virtio_device *dev = vi->vdev;
1852 virtio_config_disable(dev);
1853 dev->failed = dev->config->get_status(dev) & VIRTIO_CONFIG_S_FAILED;
1854 virtnet_freeze_down(dev);
1855 _remove_vq_common(vi);
1857 dev->config->reset(dev);
1858 virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
1859 virtio_add_status(dev, VIRTIO_CONFIG_S_DRIVER);
1861 ret = virtio_finalize_features(dev);
1865 vi->xdp_queue_pairs = xdp_qp;
1866 ret = virtnet_restore_up(dev);
1869 ret = _virtnet_set_queues(vi, curr_qp);
1873 virtio_add_status(dev, VIRTIO_CONFIG_S_DRIVER_OK);
1874 virtio_config_enable(dev);
1877 virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
1881 static int virtnet_xdp_set(struct net_device *dev, struct bpf_prog *prog,
1882 struct netlink_ext_ack *extack)
1884 unsigned long int max_sz = PAGE_SIZE - sizeof(struct padded_vnet_hdr);
1885 struct virtnet_info *vi = netdev_priv(dev);
1886 struct bpf_prog *old_prog;
1887 u16 xdp_qp = 0, curr_qp;
1890 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO4) ||
1891 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO6) ||
1892 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_ECN) ||
1893 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_UFO)) {
1894 NL_SET_ERR_MSG_MOD(extack, "Can't set XDP while host is implementing LRO, disable LRO first");
1898 if (vi->mergeable_rx_bufs && !vi->any_header_sg) {
1899 NL_SET_ERR_MSG_MOD(extack, "XDP expects header/data in single page, any_header_sg required");
1903 if (dev->mtu > max_sz) {
1904 NL_SET_ERR_MSG_MOD(extack, "MTU too large to enable XDP");
1905 netdev_warn(dev, "XDP requires MTU less than %lu\n", max_sz);
1909 curr_qp = vi->curr_queue_pairs - vi->xdp_queue_pairs;
1911 xdp_qp = nr_cpu_ids;
1913 /* XDP requires extra queues for XDP_TX */
1914 if (curr_qp + xdp_qp > vi->max_queue_pairs) {
1915 NL_SET_ERR_MSG_MOD(extack, "Too few free TX rings available");
1916 netdev_warn(dev, "request %i queues but max is %i\n",
1917 curr_qp + xdp_qp, vi->max_queue_pairs);
1922 prog = bpf_prog_add(prog, vi->max_queue_pairs - 1);
1924 return PTR_ERR(prog);
1927 /* Changing the headroom in buffers is a disruptive operation because
1928 * existing buffers must be flushed and reallocated. This will happen
1929 * when a xdp program is initially added or xdp is disabled by removing
1930 * the xdp program resulting in number of XDP queues changing.
1932 if (vi->xdp_queue_pairs != xdp_qp) {
1933 err = virtnet_reset(vi, curr_qp + xdp_qp, xdp_qp);
1935 dev_warn(&dev->dev, "XDP reset failure.\n");
1936 goto virtio_reset_err;
1940 netif_set_real_num_rx_queues(dev, curr_qp + xdp_qp);
1942 for (i = 0; i < vi->max_queue_pairs; i++) {
1943 old_prog = rtnl_dereference(vi->rq[i].xdp_prog);
1944 rcu_assign_pointer(vi->rq[i].xdp_prog, prog);
1946 bpf_prog_put(old_prog);
1952 /* On reset error do our best to unwind XDP changes inflight and return
1953 * error up to user space for resolution. The underlying reset hung on
1954 * us so not much we can do here.
1957 bpf_prog_sub(prog, vi->max_queue_pairs - 1);
1961 static bool virtnet_xdp_query(struct net_device *dev)
1963 struct virtnet_info *vi = netdev_priv(dev);
1966 for (i = 0; i < vi->max_queue_pairs; i++) {
1967 if (vi->rq[i].xdp_prog)
1973 static int virtnet_xdp(struct net_device *dev, struct netdev_xdp *xdp)
1975 switch (xdp->command) {
1976 case XDP_SETUP_PROG:
1977 return virtnet_xdp_set(dev, xdp->prog, xdp->extack);
1978 case XDP_QUERY_PROG:
1979 xdp->prog_attached = virtnet_xdp_query(dev);
1986 static const struct net_device_ops virtnet_netdev = {
1987 .ndo_open = virtnet_open,
1988 .ndo_stop = virtnet_close,
1989 .ndo_start_xmit = start_xmit,
1990 .ndo_validate_addr = eth_validate_addr,
1991 .ndo_set_mac_address = virtnet_set_mac_address,
1992 .ndo_set_rx_mode = virtnet_set_rx_mode,
1993 .ndo_get_stats64 = virtnet_stats,
1994 .ndo_vlan_rx_add_vid = virtnet_vlan_rx_add_vid,
1995 .ndo_vlan_rx_kill_vid = virtnet_vlan_rx_kill_vid,
1996 #ifdef CONFIG_NET_POLL_CONTROLLER
1997 .ndo_poll_controller = virtnet_netpoll,
1999 .ndo_xdp = virtnet_xdp,
2002 static void virtnet_config_changed_work(struct work_struct *work)
2004 struct virtnet_info *vi =
2005 container_of(work, struct virtnet_info, config_work);
2008 if (virtio_cread_feature(vi->vdev, VIRTIO_NET_F_STATUS,
2009 struct virtio_net_config, status, &v) < 0)
2012 if (v & VIRTIO_NET_S_ANNOUNCE) {
2013 netdev_notify_peers(vi->dev);
2014 virtnet_ack_link_announce(vi);
2017 /* Ignore unknown (future) status bits */
2018 v &= VIRTIO_NET_S_LINK_UP;
2020 if (vi->status == v)
2025 if (vi->status & VIRTIO_NET_S_LINK_UP) {
2026 netif_carrier_on(vi->dev);
2027 netif_tx_wake_all_queues(vi->dev);
2029 netif_carrier_off(vi->dev);
2030 netif_tx_stop_all_queues(vi->dev);
2034 static void virtnet_config_changed(struct virtio_device *vdev)
2036 struct virtnet_info *vi = vdev->priv;
2038 schedule_work(&vi->config_work);
2041 static void virtnet_free_queues(struct virtnet_info *vi)
2045 for (i = 0; i < vi->max_queue_pairs; i++) {
2046 napi_hash_del(&vi->rq[i].napi);
2047 netif_napi_del(&vi->rq[i].napi);
2048 netif_napi_del(&vi->sq[i].napi);
2051 /* We called napi_hash_del() before netif_napi_del(),
2052 * we need to respect an RCU grace period before freeing vi->rq
2060 static void _free_receive_bufs(struct virtnet_info *vi)
2062 struct bpf_prog *old_prog;
2065 for (i = 0; i < vi->max_queue_pairs; i++) {
2066 while (vi->rq[i].pages)
2067 __free_pages(get_a_page(&vi->rq[i], GFP_KERNEL), 0);
2069 old_prog = rtnl_dereference(vi->rq[i].xdp_prog);
2070 RCU_INIT_POINTER(vi->rq[i].xdp_prog, NULL);
2072 bpf_prog_put(old_prog);
2076 static void free_receive_bufs(struct virtnet_info *vi)
2079 _free_receive_bufs(vi);
2083 static void free_receive_page_frags(struct virtnet_info *vi)
2086 for (i = 0; i < vi->max_queue_pairs; i++)
2087 if (vi->rq[i].alloc_frag.page)
2088 put_page(vi->rq[i].alloc_frag.page);
2091 static bool is_xdp_raw_buffer_queue(struct virtnet_info *vi, int q)
2093 if (q < (vi->curr_queue_pairs - vi->xdp_queue_pairs))
2095 else if (q < vi->curr_queue_pairs)
2101 static void free_unused_bufs(struct virtnet_info *vi)
2106 for (i = 0; i < vi->max_queue_pairs; i++) {
2107 struct virtqueue *vq = vi->sq[i].vq;
2108 while ((buf = virtqueue_detach_unused_buf(vq)) != NULL) {
2109 if (!is_xdp_raw_buffer_queue(vi, i))
2112 put_page(virt_to_head_page(buf));
2116 for (i = 0; i < vi->max_queue_pairs; i++) {
2117 struct virtqueue *vq = vi->rq[i].vq;
2119 while ((buf = virtqueue_detach_unused_buf(vq)) != NULL) {
2120 if (vi->mergeable_rx_bufs) {
2121 unsigned long ctx = (unsigned long)buf;
2122 void *base = mergeable_ctx_to_buf_address(ctx);
2123 put_page(virt_to_head_page(base));
2124 } else if (vi->big_packets) {
2125 give_pages(&vi->rq[i], buf);
2127 put_page(virt_to_head_page(buf));
2133 static void virtnet_del_vqs(struct virtnet_info *vi)
2135 struct virtio_device *vdev = vi->vdev;
2137 virtnet_clean_affinity(vi, -1);
2139 vdev->config->del_vqs(vdev);
2141 virtnet_free_queues(vi);
2144 static int virtnet_find_vqs(struct virtnet_info *vi)
2146 vq_callback_t **callbacks;
2147 struct virtqueue **vqs;
2152 /* We expect 1 RX virtqueue followed by 1 TX virtqueue, followed by
2153 * possible N-1 RX/TX queue pairs used in multiqueue mode, followed by
2154 * possible control vq.
2156 total_vqs = vi->max_queue_pairs * 2 +
2157 virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ);
2159 /* Allocate space for find_vqs parameters */
2160 vqs = kzalloc(total_vqs * sizeof(*vqs), GFP_KERNEL);
2163 callbacks = kmalloc(total_vqs * sizeof(*callbacks), GFP_KERNEL);
2166 names = kmalloc(total_vqs * sizeof(*names), GFP_KERNEL);
2170 /* Parameters for control virtqueue, if any */
2172 callbacks[total_vqs - 1] = NULL;
2173 names[total_vqs - 1] = "control";
2176 /* Allocate/initialize parameters for send/receive virtqueues */
2177 for (i = 0; i < vi->max_queue_pairs; i++) {
2178 callbacks[rxq2vq(i)] = skb_recv_done;
2179 callbacks[txq2vq(i)] = skb_xmit_done;
2180 sprintf(vi->rq[i].name, "input.%d", i);
2181 sprintf(vi->sq[i].name, "output.%d", i);
2182 names[rxq2vq(i)] = vi->rq[i].name;
2183 names[txq2vq(i)] = vi->sq[i].name;
2186 ret = vi->vdev->config->find_vqs(vi->vdev, total_vqs, vqs, callbacks,
2192 vi->cvq = vqs[total_vqs - 1];
2193 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VLAN))
2194 vi->dev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;
2197 for (i = 0; i < vi->max_queue_pairs; i++) {
2198 vi->rq[i].vq = vqs[rxq2vq(i)];
2199 vi->sq[i].vq = vqs[txq2vq(i)];
2218 static int virtnet_alloc_queues(struct virtnet_info *vi)
2222 vi->sq = kzalloc(sizeof(*vi->sq) * vi->max_queue_pairs, GFP_KERNEL);
2225 vi->rq = kzalloc(sizeof(*vi->rq) * vi->max_queue_pairs, GFP_KERNEL);
2229 INIT_DELAYED_WORK(&vi->refill, refill_work);
2230 for (i = 0; i < vi->max_queue_pairs; i++) {
2231 vi->rq[i].pages = NULL;
2232 netif_napi_add(vi->dev, &vi->rq[i].napi, virtnet_poll,
2234 netif_tx_napi_add(vi->dev, &vi->sq[i].napi, virtnet_poll_tx,
2235 napi_tx ? napi_weight : 0);
2237 sg_init_table(vi->rq[i].sg, ARRAY_SIZE(vi->rq[i].sg));
2238 ewma_pkt_len_init(&vi->rq[i].mrg_avg_pkt_len);
2239 sg_init_table(vi->sq[i].sg, ARRAY_SIZE(vi->sq[i].sg));
2250 static int init_vqs(struct virtnet_info *vi)
2254 /* Allocate send & receive queues */
2255 ret = virtnet_alloc_queues(vi);
2259 ret = virtnet_find_vqs(vi);
2264 virtnet_set_affinity(vi);
2270 virtnet_free_queues(vi);
2276 static ssize_t mergeable_rx_buffer_size_show(struct netdev_rx_queue *queue,
2277 struct rx_queue_attribute *attribute, char *buf)
2279 struct virtnet_info *vi = netdev_priv(queue->dev);
2280 unsigned int queue_index = get_netdev_rx_queue_index(queue);
2281 struct ewma_pkt_len *avg;
2283 BUG_ON(queue_index >= vi->max_queue_pairs);
2284 avg = &vi->rq[queue_index].mrg_avg_pkt_len;
2285 return sprintf(buf, "%u\n", get_mergeable_buf_len(avg));
2288 static struct rx_queue_attribute mergeable_rx_buffer_size_attribute =
2289 __ATTR_RO(mergeable_rx_buffer_size);
2291 static struct attribute *virtio_net_mrg_rx_attrs[] = {
2292 &mergeable_rx_buffer_size_attribute.attr,
2296 static const struct attribute_group virtio_net_mrg_rx_group = {
2297 .name = "virtio_net",
2298 .attrs = virtio_net_mrg_rx_attrs
2302 static bool virtnet_fail_on_feature(struct virtio_device *vdev,
2304 const char *fname, const char *dname)
2306 if (!virtio_has_feature(vdev, fbit))
2309 dev_err(&vdev->dev, "device advertises feature %s but not %s",
2315 #define VIRTNET_FAIL_ON(vdev, fbit, dbit) \
2316 virtnet_fail_on_feature(vdev, fbit, #fbit, dbit)
2318 static bool virtnet_validate_features(struct virtio_device *vdev)
2320 if (!virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ) &&
2321 (VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_RX,
2322 "VIRTIO_NET_F_CTRL_VQ") ||
2323 VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_VLAN,
2324 "VIRTIO_NET_F_CTRL_VQ") ||
2325 VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_GUEST_ANNOUNCE,
2326 "VIRTIO_NET_F_CTRL_VQ") ||
2327 VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_MQ, "VIRTIO_NET_F_CTRL_VQ") ||
2328 VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR,
2329 "VIRTIO_NET_F_CTRL_VQ"))) {
2336 #define MIN_MTU ETH_MIN_MTU
2337 #define MAX_MTU ETH_MAX_MTU
2339 static int virtnet_validate(struct virtio_device *vdev)
2341 if (!vdev->config->get) {
2342 dev_err(&vdev->dev, "%s failure: config access disabled\n",
2347 if (!virtnet_validate_features(vdev))
2350 if (virtio_has_feature(vdev, VIRTIO_NET_F_MTU)) {
2351 int mtu = virtio_cread16(vdev,
2352 offsetof(struct virtio_net_config,
2355 __virtio_clear_bit(vdev, VIRTIO_NET_F_MTU);
2361 static int virtnet_probe(struct virtio_device *vdev)
2364 struct net_device *dev;
2365 struct virtnet_info *vi;
2366 u16 max_queue_pairs;
2369 /* Find if host supports multiqueue virtio_net device */
2370 err = virtio_cread_feature(vdev, VIRTIO_NET_F_MQ,
2371 struct virtio_net_config,
2372 max_virtqueue_pairs, &max_queue_pairs);
2374 /* We need at least 2 queue's */
2375 if (err || max_queue_pairs < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN ||
2376 max_queue_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX ||
2377 !virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ))
2378 max_queue_pairs = 1;
2380 /* Allocate ourselves a network device with room for our info */
2381 dev = alloc_etherdev_mq(sizeof(struct virtnet_info), max_queue_pairs);
2385 /* Set up network device as normal. */
2386 dev->priv_flags |= IFF_UNICAST_FLT | IFF_LIVE_ADDR_CHANGE;
2387 dev->netdev_ops = &virtnet_netdev;
2388 dev->features = NETIF_F_HIGHDMA;
2390 dev->ethtool_ops = &virtnet_ethtool_ops;
2391 SET_NETDEV_DEV(dev, &vdev->dev);
2393 /* Do we support "hardware" checksums? */
2394 if (virtio_has_feature(vdev, VIRTIO_NET_F_CSUM)) {
2395 /* This opens up the world of extra features. */
2396 dev->hw_features |= NETIF_F_HW_CSUM | NETIF_F_SG;
2398 dev->features |= NETIF_F_HW_CSUM | NETIF_F_SG;
2400 if (virtio_has_feature(vdev, VIRTIO_NET_F_GSO)) {
2401 dev->hw_features |= NETIF_F_TSO | NETIF_F_UFO
2402 | NETIF_F_TSO_ECN | NETIF_F_TSO6;
2404 /* Individual feature bits: what can host handle? */
2405 if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO4))
2406 dev->hw_features |= NETIF_F_TSO;
2407 if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO6))
2408 dev->hw_features |= NETIF_F_TSO6;
2409 if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_ECN))
2410 dev->hw_features |= NETIF_F_TSO_ECN;
2411 if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_UFO))
2412 dev->hw_features |= NETIF_F_UFO;
2414 dev->features |= NETIF_F_GSO_ROBUST;
2417 dev->features |= dev->hw_features & (NETIF_F_ALL_TSO|NETIF_F_UFO);
2418 /* (!csum && gso) case will be fixed by register_netdev() */
2420 if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_CSUM))
2421 dev->features |= NETIF_F_RXCSUM;
2423 dev->vlan_features = dev->features;
2425 /* MTU range: 68 - 65535 */
2426 dev->min_mtu = MIN_MTU;
2427 dev->max_mtu = MAX_MTU;
2429 /* Configuration may specify what MAC to use. Otherwise random. */
2430 if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC))
2431 virtio_cread_bytes(vdev,
2432 offsetof(struct virtio_net_config, mac),
2433 dev->dev_addr, dev->addr_len);
2435 eth_hw_addr_random(dev);
2437 /* Set up our device-specific information */
2438 vi = netdev_priv(dev);
2442 vi->stats = alloc_percpu(struct virtnet_stats);
2444 if (vi->stats == NULL)
2447 for_each_possible_cpu(i) {
2448 struct virtnet_stats *virtnet_stats;
2449 virtnet_stats = per_cpu_ptr(vi->stats, i);
2450 u64_stats_init(&virtnet_stats->tx_syncp);
2451 u64_stats_init(&virtnet_stats->rx_syncp);
2454 INIT_WORK(&vi->config_work, virtnet_config_changed_work);
2456 /* If we can receive ANY GSO packets, we must allocate large ones. */
2457 if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO4) ||
2458 virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO6) ||
2459 virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_ECN) ||
2460 virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_UFO))
2461 vi->big_packets = true;
2463 if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF))
2464 vi->mergeable_rx_bufs = true;
2466 if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF) ||
2467 virtio_has_feature(vdev, VIRTIO_F_VERSION_1))
2468 vi->hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);
2470 vi->hdr_len = sizeof(struct virtio_net_hdr);
2472 if (virtio_has_feature(vdev, VIRTIO_F_ANY_LAYOUT) ||
2473 virtio_has_feature(vdev, VIRTIO_F_VERSION_1))
2474 vi->any_header_sg = true;
2476 if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ))
2479 if (virtio_has_feature(vdev, VIRTIO_NET_F_MTU)) {
2480 mtu = virtio_cread16(vdev,
2481 offsetof(struct virtio_net_config,
2483 if (mtu < dev->min_mtu) {
2484 /* Should never trigger: MTU was previously validated
2485 * in virtnet_validate.
2487 dev_err(&vdev->dev, "device MTU appears to have changed "
2488 "it is now %d < %d", mtu, dev->min_mtu);
2495 /* TODO: size buffers correctly in this case. */
2496 if (dev->mtu > ETH_DATA_LEN)
2497 vi->big_packets = true;
2500 if (vi->any_header_sg)
2501 dev->needed_headroom = vi->hdr_len;
2503 /* Enable multiqueue by default */
2504 if (num_online_cpus() >= max_queue_pairs)
2505 vi->curr_queue_pairs = max_queue_pairs;
2507 vi->curr_queue_pairs = num_online_cpus();
2508 vi->max_queue_pairs = max_queue_pairs;
2510 /* Allocate/initialize the rx/tx queues, and invoke find_vqs */
2516 if (vi->mergeable_rx_bufs)
2517 dev->sysfs_rx_queue_group = &virtio_net_mrg_rx_group;
2519 netif_set_real_num_tx_queues(dev, vi->curr_queue_pairs);
2520 netif_set_real_num_rx_queues(dev, vi->curr_queue_pairs);
2522 virtnet_init_settings(dev);
2524 err = register_netdev(dev);
2526 pr_debug("virtio_net: registering device failed\n");
2530 virtio_device_ready(vdev);
2532 err = virtnet_cpu_notif_add(vi);
2534 pr_debug("virtio_net: registering cpu notifier failed\n");
2535 goto free_unregister_netdev;
2538 virtnet_set_queues(vi, vi->curr_queue_pairs);
2540 /* Assume link up if device can't report link status,
2541 otherwise get link status from config. */
2542 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STATUS)) {
2543 netif_carrier_off(dev);
2544 schedule_work(&vi->config_work);
2546 vi->status = VIRTIO_NET_S_LINK_UP;
2547 netif_carrier_on(dev);
2550 pr_debug("virtnet: registered device %s with %d RX and TX vq's\n",
2551 dev->name, max_queue_pairs);
2555 free_unregister_netdev:
2556 vi->vdev->config->reset(vdev);
2558 unregister_netdev(dev);
2560 cancel_delayed_work_sync(&vi->refill);
2561 free_receive_page_frags(vi);
2562 virtnet_del_vqs(vi);
2564 free_percpu(vi->stats);
2570 static void _remove_vq_common(struct virtnet_info *vi)
2572 vi->vdev->config->reset(vi->vdev);
2573 free_unused_bufs(vi);
2574 _free_receive_bufs(vi);
2575 free_receive_page_frags(vi);
2576 virtnet_del_vqs(vi);
2579 static void remove_vq_common(struct virtnet_info *vi)
2581 vi->vdev->config->reset(vi->vdev);
2583 /* Free unused buffers in both send and recv, if any. */
2584 free_unused_bufs(vi);
2586 free_receive_bufs(vi);
2588 free_receive_page_frags(vi);
2590 virtnet_del_vqs(vi);
2593 static void virtnet_remove(struct virtio_device *vdev)
2595 struct virtnet_info *vi = vdev->priv;
2597 virtnet_cpu_notif_remove(vi);
2599 /* Make sure no work handler is accessing the device. */
2600 flush_work(&vi->config_work);
2602 unregister_netdev(vi->dev);
2604 remove_vq_common(vi);
2606 free_percpu(vi->stats);
2607 free_netdev(vi->dev);
2610 #ifdef CONFIG_PM_SLEEP
2611 static int virtnet_freeze(struct virtio_device *vdev)
2613 struct virtnet_info *vi = vdev->priv;
2615 virtnet_cpu_notif_remove(vi);
2616 virtnet_freeze_down(vdev);
2617 remove_vq_common(vi);
2622 static int virtnet_restore(struct virtio_device *vdev)
2624 struct virtnet_info *vi = vdev->priv;
2627 err = virtnet_restore_up(vdev);
2630 virtnet_set_queues(vi, vi->curr_queue_pairs);
2632 err = virtnet_cpu_notif_add(vi);
2640 static struct virtio_device_id id_table[] = {
2641 { VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID },
2645 #define VIRTNET_FEATURES \
2646 VIRTIO_NET_F_CSUM, VIRTIO_NET_F_GUEST_CSUM, \
2648 VIRTIO_NET_F_HOST_TSO4, VIRTIO_NET_F_HOST_UFO, VIRTIO_NET_F_HOST_TSO6, \
2649 VIRTIO_NET_F_HOST_ECN, VIRTIO_NET_F_GUEST_TSO4, VIRTIO_NET_F_GUEST_TSO6, \
2650 VIRTIO_NET_F_GUEST_ECN, VIRTIO_NET_F_GUEST_UFO, \
2651 VIRTIO_NET_F_MRG_RXBUF, VIRTIO_NET_F_STATUS, VIRTIO_NET_F_CTRL_VQ, \
2652 VIRTIO_NET_F_CTRL_RX, VIRTIO_NET_F_CTRL_VLAN, \
2653 VIRTIO_NET_F_GUEST_ANNOUNCE, VIRTIO_NET_F_MQ, \
2654 VIRTIO_NET_F_CTRL_MAC_ADDR, \
2657 static unsigned int features[] = {
2661 static unsigned int features_legacy[] = {
2664 VIRTIO_F_ANY_LAYOUT,
2667 static struct virtio_driver virtio_net_driver = {
2668 .feature_table = features,
2669 .feature_table_size = ARRAY_SIZE(features),
2670 .feature_table_legacy = features_legacy,
2671 .feature_table_size_legacy = ARRAY_SIZE(features_legacy),
2672 .driver.name = KBUILD_MODNAME,
2673 .driver.owner = THIS_MODULE,
2674 .id_table = id_table,
2675 .validate = virtnet_validate,
2676 .probe = virtnet_probe,
2677 .remove = virtnet_remove,
2678 .config_changed = virtnet_config_changed,
2679 #ifdef CONFIG_PM_SLEEP
2680 .freeze = virtnet_freeze,
2681 .restore = virtnet_restore,
2685 static __init int virtio_net_driver_init(void)
2689 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "virtio/net:online",
2691 virtnet_cpu_down_prep);
2694 virtionet_online = ret;
2695 ret = cpuhp_setup_state_multi(CPUHP_VIRT_NET_DEAD, "virtio/net:dead",
2696 NULL, virtnet_cpu_dead);
2700 ret = register_virtio_driver(&virtio_net_driver);
2705 cpuhp_remove_multi_state(CPUHP_VIRT_NET_DEAD);
2707 cpuhp_remove_multi_state(virtionet_online);
2711 module_init(virtio_net_driver_init);
2713 static __exit void virtio_net_driver_exit(void)
2715 cpuhp_remove_multi_state(CPUHP_VIRT_NET_DEAD);
2716 cpuhp_remove_multi_state(virtionet_online);
2717 unregister_virtio_driver(&virtio_net_driver);
2719 module_exit(virtio_net_driver_exit);
2721 MODULE_DEVICE_TABLE(virtio, id_table);
2722 MODULE_DESCRIPTION("Virtio network driver");
2723 MODULE_LICENSE("GPL");