drivers/net/hyperv/netvsc_drv.c

   1 /*
   2  * Copyright (c) 2009, Microsoft Corporation.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms and conditions of the GNU General Public License,
   6  * version 2, as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along with
  14  * this program; if not, see <http://www.gnu.org/licenses/>.
  15  *
  16  * Authors:
  17  *   Haiyang Zhang <haiyangz@microsoft.com>
  18  *   Hank Janssen  <hjanssen@microsoft.com>
  19  */
  20 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  21
  22 #include <linux/init.h>
  23 #include <linux/atomic.h>
  24 #include <linux/module.h>
  25 #include <linux/highmem.h>
  26 #include <linux/device.h>
  27 #include <linux/io.h>
  28 #include <linux/delay.h>
  29 #include <linux/netdevice.h>
  30 #include <linux/inetdevice.h>
  31 #include <linux/etherdevice.h>
  32 #include <linux/skbuff.h>
  33 #include <linux/if_vlan.h>
  34 #include <linux/in.h>
  35 #include <linux/slab.h>
  36 #include <linux/rtnetlink.h>
  37 #include <linux/netpoll.h>
  38 #include <linux/reciprocal_div.h>
  39
  40 #include <net/arp.h>
  41 #include <net/route.h>
  42 #include <net/sock.h>
  43 #include <net/pkt_sched.h>
  44 #include <net/checksum.h>
  45 #include <net/ip6_checksum.h>
  46
  47 #include "hyperv_net.h"
  48
  49 #define RING_SIZE_MIN           64
  50
  51 #define LINKCHANGE_INT (2 * HZ)
  52 #define VF_TAKEOVER_INT (HZ / 10)
  53
  54 static unsigned int ring_size __ro_after_init = 128;
  55 module_param(ring_size, uint, S_IRUGO);
  56 MODULE_PARM_DESC(ring_size, "Ring buffer size (# of pages)");
  57 unsigned int netvsc_ring_bytes __ro_after_init;
  58 struct reciprocal_value netvsc_ring_reciprocal __ro_after_init;
  59
  60 static const u32 default_msg = NETIF_MSG_DRV | NETIF_MSG_PROBE |
  61                                 NETIF_MSG_LINK | NETIF_MSG_IFUP |
  62                                 NETIF_MSG_IFDOWN | NETIF_MSG_RX_ERR |
  63                                 NETIF_MSG_TX_ERR;
  64
  65 static int debug = -1;
  66 module_param(debug, int, S_IRUGO);
  67 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
  68
  69 static void netvsc_set_multicast_list(struct net_device *net)
  70 {
  71         struct net_device_context *net_device_ctx = netdev_priv(net);
  72         struct netvsc_device *nvdev = rtnl_dereference(net_device_ctx->nvdev);
  73
  74         rndis_filter_update(nvdev);
  75 }
  76
  77 static int netvsc_open(struct net_device *net)
  78 {
  79         struct net_device_context *ndev_ctx = netdev_priv(net);
  80         struct net_device *vf_netdev = rtnl_dereference(ndev_ctx->vf_netdev);
  81         struct netvsc_device *nvdev = rtnl_dereference(ndev_ctx->nvdev);
  82         struct rndis_device *rdev;
  83         int ret = 0;
  84
  85         netif_carrier_off(net);
  86
  87         /* Open up the device */
  88         ret = rndis_filter_open(nvdev);
  89         if (ret != 0) {
  90                 netdev_err(net, "unable to open device (ret %d).\n", ret);
  91                 return ret;
  92         }
  93
  94         rdev = nvdev->extension;
  95         if (!rdev->link_state) {
  96                 netif_carrier_on(net);
  97                 netif_tx_wake_all_queues(net);
  98         }
  99
 100         if (vf_netdev) {
 101                 /* Setting synthetic device up transparently sets
 102                  * slave as up. If open fails, then slave will be
 103                  * still be offline (and not used).
 104                  */
 105                 ret = dev_open(vf_netdev);
 106                 if (ret)
 107                         netdev_warn(net,
 108                                     "unable to open slave: %s: %d\n",
 109                                     vf_netdev->name, ret);
 110         }
 111         return 0;
 112 }
 113
 114 static int netvsc_close(struct net_device *net)
 115 {
 116         struct net_device_context *net_device_ctx = netdev_priv(net);
 117         struct net_device *vf_netdev
 118                 = rtnl_dereference(net_device_ctx->vf_netdev);
 119         struct netvsc_device *nvdev = rtnl_dereference(net_device_ctx->nvdev);
 120         int ret = 0;
 121         u32 aread, i, msec = 10, retry = 0, retry_max = 20;
 122         struct vmbus_channel *chn;
 123
 124         netif_tx_disable(net);
 125
 126         /* No need to close rndis filter if it is removed already */
 127         if (!nvdev)
 128                 goto out;
 129
 130         ret = rndis_filter_close(nvdev);
 131         if (ret != 0) {
 132                 netdev_err(net, "unable to close device (ret %d).\n", ret);
 133                 return ret;
 134         }
 135
 136         /* Ensure pending bytes in ring are read */
 137         while (true) {
 138                 aread = 0;
 139                 for (i = 0; i < nvdev->num_chn; i++) {
 140                         chn = nvdev->chan_table[i].channel;
 141                         if (!chn)
 142                                 continue;
 143
 144                         aread = hv_get_bytes_to_read(&chn->inbound);
 145                         if (aread)
 146                                 break;
 147
 148                         aread = hv_get_bytes_to_read(&chn->outbound);
 149                         if (aread)
 150                                 break;
 151                 }
 152
 153                 retry++;
 154                 if (retry > retry_max || aread == 0)
 155                         break;
 156
 157                 msleep(msec);
 158
 159                 if (msec < 1000)
 160                         msec *= 2;
 161         }
 162
 163         if (aread) {
 164                 netdev_err(net, "Ring buffer not empty after closing rndis\n");
 165                 ret = -ETIMEDOUT;
 166         }
 167
 168 out:
 169         if (vf_netdev)
 170                 dev_close(vf_netdev);
 171
 172         return ret;
 173 }
 174
 175 static inline void *init_ppi_data(struct rndis_message *msg,
 176                                   u32 ppi_size, u32 pkt_type)
 177 {
 178         struct rndis_packet *rndis_pkt = &msg->msg.pkt;
 179         struct rndis_per_packet_info *ppi;
 180
 181         rndis_pkt->data_offset += ppi_size;
 182         ppi = (void *)rndis_pkt + rndis_pkt->per_pkt_info_offset
 183                 + rndis_pkt->per_pkt_info_len;
 184
 185         ppi->size = ppi_size;
 186         ppi->type = pkt_type;
 187         ppi->ppi_offset = sizeof(struct rndis_per_packet_info);
 188
 189         rndis_pkt->per_pkt_info_len += ppi_size;
 190
 191         return ppi + 1;
 192 }
 193
 194 /* Azure hosts don't support non-TCP port numbers in hashing for fragmented
 195  * packets. We can use ethtool to change UDP hash level when necessary.
 196  */
 197 static inline u32 netvsc_get_hash(
 198         struct sk_buff *skb,
 199         const struct net_device_context *ndc)
 200 {
 201         struct flow_keys flow;
 202         u32 hash, pkt_proto = 0;
 203         static u32 hashrnd __read_mostly;
 204
 205         net_get_random_once(&hashrnd, sizeof(hashrnd));
 206
 207         if (!skb_flow_dissect_flow_keys(skb, &flow, 0))
 208                 return 0;
 209
 210         switch (flow.basic.ip_proto) {
 211         case IPPROTO_TCP:
 212                 if (flow.basic.n_proto == htons(ETH_P_IP))
 213                         pkt_proto = HV_TCP4_L4HASH;
 214                 else if (flow.basic.n_proto == htons(ETH_P_IPV6))
 215                         pkt_proto = HV_TCP6_L4HASH;
 216
 217                 break;
 218
 219         case IPPROTO_UDP:
 220                 if (flow.basic.n_proto == htons(ETH_P_IP))
 221                         pkt_proto = HV_UDP4_L4HASH;
 222                 else if (flow.basic.n_proto == htons(ETH_P_IPV6))
 223                         pkt_proto = HV_UDP6_L4HASH;
 224
 225                 break;
 226         }
 227
 228         if (pkt_proto & ndc->l4_hash) {
 229                 return skb_get_hash(skb);
 230         } else {
 231                 if (flow.basic.n_proto == htons(ETH_P_IP))
 232                         hash = jhash2((u32 *)&flow.addrs.v4addrs, 2, hashrnd);
 233                 else if (flow.basic.n_proto == htons(ETH_P_IPV6))
 234                         hash = jhash2((u32 *)&flow.addrs.v6addrs, 8, hashrnd);
 235                 else
 236                         hash = 0;
 237
 238                 skb_set_hash(skb, hash, PKT_HASH_TYPE_L3);
 239         }
 240
 241         return hash;
 242 }
 243
 244 static inline int netvsc_get_tx_queue(struct net_device *ndev,
 245                                       struct sk_buff *skb, int old_idx)
 246 {
 247         const struct net_device_context *ndc = netdev_priv(ndev);
 248         struct sock *sk = skb->sk;
 249         int q_idx;
 250
 251         q_idx = ndc->tx_table[netvsc_get_hash(skb, ndc) &
 252                               (VRSS_SEND_TAB_SIZE - 1)];
 253
 254         /* If queue index changed record the new value */
 255         if (q_idx != old_idx &&
 256             sk && sk_fullsock(sk) && rcu_access_pointer(sk->sk_dst_cache))
 257                 sk_tx_queue_set(sk, q_idx);
 258
 259         return q_idx;
 260 }
 261
 262 /*
 263  * Select queue for transmit.
 264  *
 265  * If a valid queue has already been assigned, then use that.
 266  * Otherwise compute tx queue based on hash and the send table.
 267  *
 268  * This is basically similar to default (__netdev_pick_tx) with the added step
 269  * of using the host send_table when no other queue has been assigned.
 270  *
 271  * TODO support XPS - but get_xps_queue not exported
 272  */
 273 static u16 netvsc_pick_tx(struct net_device *ndev, struct sk_buff *skb)
 274 {
 275         int q_idx = sk_tx_queue_get(skb->sk);
 276
 277         if (q_idx < 0 || skb->ooo_okay || q_idx >= ndev->real_num_tx_queues) {
 278                 /* If forwarding a packet, we use the recorded queue when
 279                  * available for better cache locality.
 280                  */
 281                 if (skb_rx_queue_recorded(skb))
 282                         q_idx = skb_get_rx_queue(skb);
 283                 else
 284                         q_idx = netvsc_get_tx_queue(ndev, skb, q_idx);
 285         }
 286
 287         return q_idx;
 288 }
 289
 290 static u16 netvsc_select_queue(struct net_device *ndev, struct sk_buff *skb,
 291                                void *accel_priv,
 292                                select_queue_fallback_t fallback)
 293 {
 294         struct net_device_context *ndc = netdev_priv(ndev);
 295         struct net_device *vf_netdev;
 296         u16 txq;
 297
 298         rcu_read_lock();
 299         vf_netdev = rcu_dereference(ndc->vf_netdev);
 300         if (vf_netdev) {
 301                 const struct net_device_ops *vf_ops = vf_netdev->netdev_ops;
 302
 303                 if (vf_ops->ndo_select_queue)
 304                         txq = vf_ops->ndo_select_queue(vf_netdev, skb,
 305                                                        accel_priv, fallback);
 306                 else
 307                         txq = fallback(vf_netdev, skb);
 308
 309                 /* Record the queue selected by VF so that it can be
 310                  * used for common case where VF has more queues than
 311                  * the synthetic device.
 312                  */
 313                 qdisc_skb_cb(skb)->slave_dev_queue_mapping = txq;
 314         } else {
 315                 txq = netvsc_pick_tx(ndev, skb);
 316         }
 317         rcu_read_unlock();
 318
 319         while (unlikely(txq >= ndev->real_num_tx_queues))
 320                 txq -= ndev->real_num_tx_queues;
 321
 322         return txq;
 323 }
 324
 325 static u32 fill_pg_buf(struct page *page, u32 offset, u32 len,
 326                        struct hv_page_buffer *pb)
 327 {
 328         int j = 0;
 329
 330         /* Deal with compund pages by ignoring unused part
 331          * of the page.
 332          */
 333         page += (offset >> PAGE_SHIFT);
 334         offset &= ~PAGE_MASK;
 335
 336         while (len > 0) {
 337                 unsigned long bytes;
 338
 339                 bytes = PAGE_SIZE - offset;
 340                 if (bytes > len)
 341                         bytes = len;
 342                 pb[j].pfn = page_to_pfn(page);
 343                 pb[j].offset = offset;
 344                 pb[j].len = bytes;
 345
 346                 offset += bytes;
 347                 len -= bytes;
 348
 349                 if (offset == PAGE_SIZE && len) {
 350                         page++;
 351                         offset = 0;
 352                         j++;
 353                 }
 354         }
 355
 356         return j + 1;
 357 }
 358
 359 static u32 init_page_array(void *hdr, u32 len, struct sk_buff *skb,
 360                            struct hv_netvsc_packet *packet,
 361                            struct hv_page_buffer *pb)
 362 {
 363         u32 slots_used = 0;
 364         char *data = skb->data;
 365         int frags = skb_shinfo(skb)->nr_frags;
 366         int i;
 367
 368         /* The packet is laid out thus:
 369          * 1. hdr: RNDIS header and PPI
 370          * 2. skb linear data
 371          * 3. skb fragment data
 372          */
 373         slots_used += fill_pg_buf(virt_to_page(hdr),
 374                                   offset_in_page(hdr),
 375                                   len, &pb[slots_used]);
 376
 377         packet->rmsg_size = len;
 378         packet->rmsg_pgcnt = slots_used;
 379
 380         slots_used += fill_pg_buf(virt_to_page(data),
 381                                 offset_in_page(data),
 382                                 skb_headlen(skb), &pb[slots_used]);
 383
 384         for (i = 0; i < frags; i++) {
 385                 skb_frag_t *frag = skb_shinfo(skb)->frags + i;
 386
 387                 slots_used += fill_pg_buf(skb_frag_page(frag),
 388                                         frag->page_offset,
 389                                         skb_frag_size(frag), &pb[slots_used]);
 390         }
 391         return slots_used;
 392 }
 393
 394 static int count_skb_frag_slots(struct sk_buff *skb)
 395 {
 396         int i, frags = skb_shinfo(skb)->nr_frags;
 397         int pages = 0;
 398
 399         for (i = 0; i < frags; i++) {
 400                 skb_frag_t *frag = skb_shinfo(skb)->frags + i;
 401                 unsigned long size = skb_frag_size(frag);
 402                 unsigned long offset = frag->page_offset;
 403
 404                 /* Skip unused frames from start of page */
 405                 offset &= ~PAGE_MASK;
 406                 pages += PFN_UP(offset + size);
 407         }
 408         return pages;
 409 }
 410
 411 static int netvsc_get_slots(struct sk_buff *skb)
 412 {
 413         char *data = skb->data;
 414         unsigned int offset = offset_in_page(data);
 415         unsigned int len = skb_headlen(skb);
 416         int slots;
 417         int frag_slots;
 418
 419         slots = DIV_ROUND_UP(offset + len, PAGE_SIZE);
 420         frag_slots = count_skb_frag_slots(skb);
 421         return slots + frag_slots;
 422 }
 423
 424 static u32 net_checksum_info(struct sk_buff *skb)
 425 {
 426         if (skb->protocol == htons(ETH_P_IP)) {
 427                 struct iphdr *ip = ip_hdr(skb);
 428
 429                 if (ip->protocol == IPPROTO_TCP)
 430                         return TRANSPORT_INFO_IPV4_TCP;
 431                 else if (ip->protocol == IPPROTO_UDP)
 432                         return TRANSPORT_INFO_IPV4_UDP;
 433         } else {
 434                 struct ipv6hdr *ip6 = ipv6_hdr(skb);
 435
 436                 if (ip6->nexthdr == IPPROTO_TCP)
 437                         return TRANSPORT_INFO_IPV6_TCP;
 438                 else if (ip6->nexthdr == IPPROTO_UDP)
 439                         return TRANSPORT_INFO_IPV6_UDP;
 440         }
 441
 442         return TRANSPORT_INFO_NOT_IP;
 443 }
 444
 445 /* Send skb on the slave VF device. */
 446 static int netvsc_vf_xmit(struct net_device *net, struct net_device *vf_netdev,
 447                           struct sk_buff *skb)
 448 {
 449         struct net_device_context *ndev_ctx = netdev_priv(net);
 450         unsigned int len = skb->len;
 451         int rc;
 452
 453         skb->dev = vf_netdev;
 454         skb->queue_mapping = qdisc_skb_cb(skb)->slave_dev_queue_mapping;
 455
 456         rc = dev_queue_xmit(skb);
 457         if (likely(rc == NET_XMIT_SUCCESS || rc == NET_XMIT_CN)) {
 458                 struct netvsc_vf_pcpu_stats *pcpu_stats
 459                         = this_cpu_ptr(ndev_ctx->vf_stats);
 460
 461                 u64_stats_update_begin(&pcpu_stats->syncp);
 462                 pcpu_stats->tx_packets++;
 463                 pcpu_stats->tx_bytes += len;
 464                 u64_stats_update_end(&pcpu_stats->syncp);
 465         } else {
 466                 this_cpu_inc(ndev_ctx->vf_stats->tx_dropped);
 467         }
 468
 469         return rc;
 470 }
 471
 472 static int netvsc_start_xmit(struct sk_buff *skb, struct net_device *net)
 473 {
 474         struct net_device_context *net_device_ctx = netdev_priv(net);
 475         struct hv_netvsc_packet *packet = NULL;
 476         int ret;
 477         unsigned int num_data_pgs;
 478         struct rndis_message *rndis_msg;
 479         struct net_device *vf_netdev;
 480         u32 rndis_msg_size;
 481         u32 hash;
 482         struct hv_page_buffer pb[MAX_PAGE_BUFFER_COUNT];
 483
 484         /* if VF is present and up then redirect packets
 485          * already called with rcu_read_lock_bh
 486          */
 487         vf_netdev = rcu_dereference_bh(net_device_ctx->vf_netdev);
 488         if (vf_netdev && netif_running(vf_netdev) &&
 489             !netpoll_tx_running(net))
 490                 return netvsc_vf_xmit(net, vf_netdev, skb);
 491
 492         /* We will atmost need two pages to describe the rndis
 493          * header. We can only transmit MAX_PAGE_BUFFER_COUNT number
 494          * of pages in a single packet. If skb is scattered around
 495          * more pages we try linearizing it.
 496          */
 497
 498         num_data_pgs = netvsc_get_slots(skb) + 2;
 499
 500         if (unlikely(num_data_pgs > MAX_PAGE_BUFFER_COUNT)) {
 501                 ++net_device_ctx->eth_stats.tx_scattered;
 502
 503                 if (skb_linearize(skb))
 504                         goto no_memory;
 505
 506                 num_data_pgs = netvsc_get_slots(skb) + 2;
 507                 if (num_data_pgs > MAX_PAGE_BUFFER_COUNT) {
 508                         ++net_device_ctx->eth_stats.tx_too_big;
 509                         goto drop;
 510                 }
 511         }
 512
 513         /*
 514          * Place the rndis header in the skb head room and
 515          * the skb->cb will be used for hv_netvsc_packet
 516          * structure.
 517          */
 518         ret = skb_cow_head(skb, RNDIS_AND_PPI_SIZE);
 519         if (ret)
 520                 goto no_memory;
 521
 522         /* Use the skb control buffer for building up the packet */
 523         BUILD_BUG_ON(sizeof(struct hv_netvsc_packet) >
 524                         FIELD_SIZEOF(struct sk_buff, cb));
 525         packet = (struct hv_netvsc_packet *)skb->cb;
 526
 527         packet->q_idx = skb_get_queue_mapping(skb);
 528
 529         packet->total_data_buflen = skb->len;
 530         packet->total_bytes = skb->len;
 531         packet->total_packets = 1;
 532
 533         rndis_msg = (struct rndis_message *)skb->head;
 534
 535         /* Add the rndis header */
 536         rndis_msg->ndis_msg_type = RNDIS_MSG_PACKET;
 537         rndis_msg->msg_len = packet->total_data_buflen;
 538
 539         rndis_msg->msg.pkt = (struct rndis_packet) {
 540                 .data_offset = sizeof(struct rndis_packet),
 541                 .data_len = packet->total_data_buflen,
 542                 .per_pkt_info_offset = sizeof(struct rndis_packet),
 543         };
 544
 545         rndis_msg_size = RNDIS_MESSAGE_SIZE(struct rndis_packet);
 546
 547         hash = skb_get_hash_raw(skb);
 548         if (hash != 0 && net->real_num_tx_queues > 1) {
 549                 u32 *hash_info;
 550
 551                 rndis_msg_size += NDIS_HASH_PPI_SIZE;
 552                 hash_info = init_ppi_data(rndis_msg, NDIS_HASH_PPI_SIZE,
 553                                           NBL_HASH_VALUE);
 554                 *hash_info = hash;
 555         }
 556
 557         if (skb_vlan_tag_present(skb)) {
 558                 struct ndis_pkt_8021q_info *vlan;
 559
 560                 rndis_msg_size += NDIS_VLAN_PPI_SIZE;
 561                 vlan = init_ppi_data(rndis_msg, NDIS_VLAN_PPI_SIZE,
 562                                      IEEE_8021Q_INFO);
 563
 564                 vlan->value = 0;
 565                 vlan->vlanid = skb->vlan_tci & VLAN_VID_MASK;
 566                 vlan->pri = (skb->vlan_tci & VLAN_PRIO_MASK) >>
 567                                 VLAN_PRIO_SHIFT;
 568         }
 569
 570         if (skb_is_gso(skb)) {
 571                 struct ndis_tcp_lso_info *lso_info;
 572
 573                 rndis_msg_size += NDIS_LSO_PPI_SIZE;
 574                 lso_info = init_ppi_data(rndis_msg, NDIS_LSO_PPI_SIZE,
 575                                          TCP_LARGESEND_PKTINFO);
 576
 577                 lso_info->value = 0;
 578                 lso_info->lso_v2_transmit.type = NDIS_TCP_LARGE_SEND_OFFLOAD_V2_TYPE;
 579                 if (skb->protocol == htons(ETH_P_IP)) {
 580                         lso_info->lso_v2_transmit.ip_version =
 581                                 NDIS_TCP_LARGE_SEND_OFFLOAD_IPV4;
 582                         ip_hdr(skb)->tot_len = 0;
 583                         ip_hdr(skb)->check = 0;
 584                         tcp_hdr(skb)->check =
 585                                 ~csum_tcpudp_magic(ip_hdr(skb)->saddr,
 586                                                    ip_hdr(skb)->daddr, 0, IPPROTO_TCP, 0);
 587                 } else {
 588                         lso_info->lso_v2_transmit.ip_version =
 589                                 NDIS_TCP_LARGE_SEND_OFFLOAD_IPV6;
 590                         ipv6_hdr(skb)->payload_len = 0;
 591                         tcp_hdr(skb)->check =
 592                                 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
 593                                                  &ipv6_hdr(skb)->daddr, 0, IPPROTO_TCP, 0);
 594                 }
 595                 lso_info->lso_v2_transmit.tcp_header_offset = skb_transport_offset(skb);
 596                 lso_info->lso_v2_transmit.mss = skb_shinfo(skb)->gso_size;
 597         } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
 598                 if (net_checksum_info(skb) & net_device_ctx->tx_checksum_mask) {
 599                         struct ndis_tcp_ip_checksum_info *csum_info;
 600
 601                         rndis_msg_size += NDIS_CSUM_PPI_SIZE;
 602                         csum_info = init_ppi_data(rndis_msg, NDIS_CSUM_PPI_SIZE,
 603                                                   TCPIP_CHKSUM_PKTINFO);
 604
 605                         csum_info->value = 0;
 606                         csum_info->transmit.tcp_header_offset = skb_transport_offset(skb);
 607
 608                         if (skb->protocol == htons(ETH_P_IP)) {
 609                                 csum_info->transmit.is_ipv4 = 1;
 610
 611                                 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
 612                                         csum_info->transmit.tcp_checksum = 1;
 613                                 else
 614                                         csum_info->transmit.udp_checksum = 1;
 615                         } else {
 616                                 csum_info->transmit.is_ipv6 = 1;
 617
 618                                 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
 619                                         csum_info->transmit.tcp_checksum = 1;
 620                                 else
 621                                         csum_info->transmit.udp_checksum = 1;
 622                         }
 623                 } else {
 624                         /* Can't do offload of this type of checksum */
 625                         if (skb_checksum_help(skb))
 626                                 goto drop;
 627                 }
 628         }
 629
 630         /* Start filling in the page buffers with the rndis hdr */
 631         rndis_msg->msg_len += rndis_msg_size;
 632         packet->total_data_buflen = rndis_msg->msg_len;
 633         packet->page_buf_cnt = init_page_array(rndis_msg, rndis_msg_size,
 634                                                skb, packet, pb);
 635
 636         /* timestamp packet in software */
 637         skb_tx_timestamp(skb);
 638
 639         ret = netvsc_send(net, packet, rndis_msg, pb, skb);
 640         if (likely(ret == 0))
 641                 return NETDEV_TX_OK;
 642
 643         if (ret == -EAGAIN) {
 644                 ++net_device_ctx->eth_stats.tx_busy;
 645                 return NETDEV_TX_BUSY;
 646         }
 647
 648         if (ret == -ENOSPC)
 649                 ++net_device_ctx->eth_stats.tx_no_space;
 650
 651 drop:
 652         dev_kfree_skb_any(skb);
 653         net->stats.tx_dropped++;
 654
 655         return NETDEV_TX_OK;
 656
 657 no_memory:
 658         ++net_device_ctx->eth_stats.tx_no_memory;
 659         goto drop;
 660 }
 661
 662 /*
 663  * netvsc_linkstatus_callback - Link up/down notification
 664  */
 665 void netvsc_linkstatus_callback(struct net_device *net,
 666                                 struct rndis_message *resp)
 667 {
 668         struct rndis_indicate_status *indicate = &resp->msg.indicate_status;
 669         struct net_device_context *ndev_ctx = netdev_priv(net);
 670         struct netvsc_reconfig *event;
 671         unsigned long flags;
 672
 673         /* Update the physical link speed when changing to another vSwitch */
 674         if (indicate->status == RNDIS_STATUS_LINK_SPEED_CHANGE) {
 675                 u32 speed;
 676
 677                 speed = *(u32 *)((void *)indicate
 678                                  + indicate->status_buf_offset) / 10000;
 679                 ndev_ctx->speed = speed;
 680                 return;
 681         }
 682
 683         /* Handle these link change statuses below */
 684         if (indicate->status != RNDIS_STATUS_NETWORK_CHANGE &&
 685             indicate->status != RNDIS_STATUS_MEDIA_CONNECT &&
 686             indicate->status != RNDIS_STATUS_MEDIA_DISCONNECT)
 687                 return;
 688
 689         if (net->reg_state != NETREG_REGISTERED)
 690                 return;
 691
 692         event = kzalloc(sizeof(*event), GFP_ATOMIC);
 693         if (!event)
 694                 return;
 695         event->event = indicate->status;
 696
 697         spin_lock_irqsave(&ndev_ctx->lock, flags);
 698         list_add_tail(&event->list, &ndev_ctx->reconfig_events);
 699         spin_unlock_irqrestore(&ndev_ctx->lock, flags);
 700
 701         schedule_delayed_work(&ndev_ctx->dwork, 0);
 702 }
 703
 704 static struct sk_buff *netvsc_alloc_recv_skb(struct net_device *net,
 705                                              struct napi_struct *napi,
 706                                              const struct ndis_tcp_ip_checksum_info *csum_info,
 707                                              const struct ndis_pkt_8021q_info *vlan,
 708                                              void *data, u32 buflen)
 709 {
 710         struct sk_buff *skb;
 711
 712         skb = napi_alloc_skb(napi, buflen);
 713         if (!skb)
 714                 return skb;
 715
 716         /*
 717          * Copy to skb. This copy is needed here since the memory pointed by
 718          * hv_netvsc_packet cannot be deallocated
 719          */
 720         skb_put_data(skb, data, buflen);
 721
 722         skb->protocol = eth_type_trans(skb, net);
 723
 724         /* skb is already created with CHECKSUM_NONE */
 725         skb_checksum_none_assert(skb);
 726
 727         /*
 728          * In Linux, the IP checksum is always checked.
 729          * Do L4 checksum offload if enabled and present.
 730          */
 731         if (csum_info && (net->features & NETIF_F_RXCSUM)) {
 732                 if (csum_info->receive.tcp_checksum_succeeded ||
 733                     csum_info->receive.udp_checksum_succeeded)
 734                         skb->ip_summed = CHECKSUM_UNNECESSARY;
 735         }
 736
 737         if (vlan) {
 738                 u16 vlan_tci = vlan->vlanid | (vlan->pri << VLAN_PRIO_SHIFT);
 739
 740                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
 741                                        vlan_tci);
 742         }
 743
 744         return skb;
 745 }
 746
 747 /*
 748  * netvsc_recv_callback -  Callback when we receive a packet from the
 749  * "wire" on the specified device.
 750  */
 751 int netvsc_recv_callback(struct net_device *net,
 752                          struct netvsc_device *net_device,
 753                          struct vmbus_channel *channel,
 754                          void  *data, u32 len,
 755                          const struct ndis_tcp_ip_checksum_info *csum_info,
 756                          const struct ndis_pkt_8021q_info *vlan)
 757 {
 758         struct net_device_context *net_device_ctx = netdev_priv(net);
 759         u16 q_idx = channel->offermsg.offer.sub_channel_index;
 760         struct netvsc_channel *nvchan = &net_device->chan_table[q_idx];
 761         struct sk_buff *skb;
 762         struct netvsc_stats *rx_stats;
 763
 764         if (net->reg_state != NETREG_REGISTERED)
 765                 return NVSP_STAT_FAIL;
 766
 767         /* Allocate a skb - TODO direct I/O to pages? */
 768         skb = netvsc_alloc_recv_skb(net, &nvchan->napi,
 769                                     csum_info, vlan, data, len);
 770         if (unlikely(!skb)) {
 771                 ++net_device_ctx->eth_stats.rx_no_memory;
 772                 rcu_read_unlock();
 773                 return NVSP_STAT_FAIL;
 774         }
 775
 776         skb_record_rx_queue(skb, q_idx);
 777
 778         /*
 779          * Even if injecting the packet, record the statistics
 780          * on the synthetic device because modifying the VF device
 781          * statistics will not work correctly.
 782          */
 783         rx_stats = &nvchan->rx_stats;
 784         u64_stats_update_begin(&rx_stats->syncp);
 785         rx_stats->packets++;
 786         rx_stats->bytes += len;
 787
 788         if (skb->pkt_type == PACKET_BROADCAST)
 789                 ++rx_stats->broadcast;
 790         else if (skb->pkt_type == PACKET_MULTICAST)
 791                 ++rx_stats->multicast;
 792         u64_stats_update_end(&rx_stats->syncp);
 793
 794         napi_gro_receive(&nvchan->napi, skb);
 795         return 0;
 796 }
 797
 798 static void netvsc_get_drvinfo(struct net_device *net,
 799                                struct ethtool_drvinfo *info)
 800 {
 801         strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
 802         strlcpy(info->fw_version, "N/A", sizeof(info->fw_version));
 803 }
 804
 805 static void netvsc_get_channels(struct net_device *net,
 806                                 struct ethtool_channels *channel)
 807 {
 808         struct net_device_context *net_device_ctx = netdev_priv(net);
 809         struct netvsc_device *nvdev = rtnl_dereference(net_device_ctx->nvdev);
 810
 811         if (nvdev) {
 812                 channel->max_combined   = nvdev->max_chn;
 813                 channel->combined_count = nvdev->num_chn;
 814         }
 815 }
 816
 817 static int netvsc_set_channels(struct net_device *net,
 818                                struct ethtool_channels *channels)
 819 {
 820         struct net_device_context *net_device_ctx = netdev_priv(net);
 821         struct hv_device *dev = net_device_ctx->device_ctx;
 822         struct netvsc_device *nvdev = rtnl_dereference(net_device_ctx->nvdev);
 823         unsigned int orig, count = channels->combined_count;
 824         struct netvsc_device_info device_info;
 825         bool was_opened;
 826         int ret = 0;
 827
 828         /* We do not support separate count for rx, tx, or other */
 829         if (count == 0 ||
 830             channels->rx_count || channels->tx_count || channels->other_count)
 831                 return -EINVAL;
 832
 833         if (!nvdev || nvdev->destroy)
 834                 return -ENODEV;
 835
 836         if (nvdev->nvsp_version < NVSP_PROTOCOL_VERSION_5)
 837                 return -EINVAL;
 838
 839         if (count > nvdev->max_chn)
 840                 return -EINVAL;
 841
 842         orig = nvdev->num_chn;
 843         was_opened = rndis_filter_opened(nvdev);
 844         if (was_opened)
 845                 rndis_filter_close(nvdev);
 846
 847         memset(&device_info, 0, sizeof(device_info));
 848         device_info.num_chn = count;
 849         device_info.send_sections = nvdev->send_section_cnt;
 850         device_info.send_section_size = nvdev->send_section_size;
 851         device_info.recv_sections = nvdev->recv_section_cnt;
 852         device_info.recv_section_size = nvdev->recv_section_size;
 853
 854         rndis_filter_device_remove(dev, nvdev);
 855
 856         nvdev = rndis_filter_device_add(dev, &device_info);
 857         if (IS_ERR(nvdev)) {
 858                 ret = PTR_ERR(nvdev);
 859                 device_info.num_chn = orig;
 860                 nvdev = rndis_filter_device_add(dev, &device_info);
 861
 862                 if (IS_ERR(nvdev)) {
 863                         netdev_err(net, "restoring channel setting failed: %ld\n",
 864                                    PTR_ERR(nvdev));
 865                         return ret;
 866                 }
 867         }
 868
 869         if (was_opened)
 870                 rndis_filter_open(nvdev);
 871
 872         /* We may have missed link change notifications */
 873         net_device_ctx->last_reconfig = 0;
 874         schedule_delayed_work(&net_device_ctx->dwork, 0);
 875
 876         return ret;
 877 }
 878
 879 static bool
 880 netvsc_validate_ethtool_ss_cmd(const struct ethtool_link_ksettings *cmd)
 881 {
 882         struct ethtool_link_ksettings diff1 = *cmd;
 883         struct ethtool_link_ksettings diff2 = {};
 884
 885         diff1.base.speed = 0;
 886         diff1.base.duplex = 0;
 887         /* advertising and cmd are usually set */
 888         ethtool_link_ksettings_zero_link_mode(&diff1, advertising);
 889         diff1.base.cmd = 0;
 890         /* We set port to PORT_OTHER */
 891         diff2.base.port = PORT_OTHER;
 892
 893         return !memcmp(&diff1, &diff2, sizeof(diff1));
 894 }
 895
 896 static void netvsc_init_settings(struct net_device *dev)
 897 {
 898         struct net_device_context *ndc = netdev_priv(dev);
 899
 900         ndc->l4_hash = HV_DEFAULT_L4HASH;
 901
 902         ndc->speed = SPEED_UNKNOWN;
 903         ndc->duplex = DUPLEX_FULL;
 904 }
 905
 906 static int netvsc_get_link_ksettings(struct net_device *dev,
 907                                      struct ethtool_link_ksettings *cmd)
 908 {
 909         struct net_device_context *ndc = netdev_priv(dev);
 910
 911         cmd->base.speed = ndc->speed;
 912         cmd->base.duplex = ndc->duplex;
 913         cmd->base.port = PORT_OTHER;
 914
 915         return 0;
 916 }
 917
 918 static int netvsc_set_link_ksettings(struct net_device *dev,
 919                                      const struct ethtool_link_ksettings *cmd)
 920 {
 921         struct net_device_context *ndc = netdev_priv(dev);
 922         u32 speed;
 923
 924         speed = cmd->base.speed;
 925         if (!ethtool_validate_speed(speed) ||
 926             !ethtool_validate_duplex(cmd->base.duplex) ||
 927             !netvsc_validate_ethtool_ss_cmd(cmd))
 928                 return -EINVAL;
 929
 930         ndc->speed = speed;
 931         ndc->duplex = cmd->base.duplex;
 932
 933         return 0;
 934 }
 935
 936 static int netvsc_change_mtu(struct net_device *ndev, int mtu)
 937 {
 938         struct net_device_context *ndevctx = netdev_priv(ndev);
 939         struct net_device *vf_netdev = rtnl_dereference(ndevctx->vf_netdev);
 940         struct netvsc_device *nvdev = rtnl_dereference(ndevctx->nvdev);
 941         struct hv_device *hdev = ndevctx->device_ctx;
 942         int orig_mtu = ndev->mtu;
 943         struct netvsc_device_info device_info;
 944         bool was_opened;
 945         int ret = 0;
 946
 947         if (!nvdev || nvdev->destroy)
 948                 return -ENODEV;
 949
 950         /* Change MTU of underlying VF netdev first. */
 951         if (vf_netdev) {
 952                 ret = dev_set_mtu(vf_netdev, mtu);
 953                 if (ret)
 954                         return ret;
 955         }
 956
 957         netif_device_detach(ndev);
 958         was_opened = rndis_filter_opened(nvdev);
 959         if (was_opened)
 960                 rndis_filter_close(nvdev);
 961
 962         memset(&device_info, 0, sizeof(device_info));
 963         device_info.num_chn = nvdev->num_chn;
 964         device_info.send_sections = nvdev->send_section_cnt;
 965         device_info.send_section_size = nvdev->send_section_size;
 966         device_info.recv_sections = nvdev->recv_section_cnt;
 967         device_info.recv_section_size = nvdev->recv_section_size;
 968
 969         rndis_filter_device_remove(hdev, nvdev);
 970
 971         ndev->mtu = mtu;
 972
 973         nvdev = rndis_filter_device_add(hdev, &device_info);
 974         if (IS_ERR(nvdev)) {
 975                 ret = PTR_ERR(nvdev);
 976
 977                 /* Attempt rollback to original MTU */
 978                 ndev->mtu = orig_mtu;
 979                 nvdev = rndis_filter_device_add(hdev, &device_info);
 980
 981                 if (vf_netdev)
 982                         dev_set_mtu(vf_netdev, orig_mtu);
 983
 984                 if (IS_ERR(nvdev)) {
 985                         netdev_err(ndev, "restoring mtu failed: %ld\n",
 986                                    PTR_ERR(nvdev));
 987                         return ret;
 988                 }
 989         }
 990
 991         if (was_opened)
 992                 rndis_filter_open(nvdev);
 993
 994         netif_device_attach(ndev);
 995
 996         /* We may have missed link change notifications */
 997         schedule_delayed_work(&ndevctx->dwork, 0);
 998
 999         return ret;
1000 }
1001
1002 static void netvsc_get_vf_stats(struct net_device *net,
1003                                 struct netvsc_vf_pcpu_stats *tot)
1004 {
1005         struct net_device_context *ndev_ctx = netdev_priv(net);
1006         int i;
1007
1008         memset(tot, 0, sizeof(*tot));
1009
1010         for_each_possible_cpu(i) {
1011                 const struct netvsc_vf_pcpu_stats *stats
1012                         = per_cpu_ptr(ndev_ctx->vf_stats, i);
1013                 u64 rx_packets, rx_bytes, tx_packets, tx_bytes;
1014                 unsigned int start;
1015
1016                 do {
1017                         start = u64_stats_fetch_begin_irq(&stats->syncp);
1018                         rx_packets = stats->rx_packets;
1019                         tx_packets = stats->tx_packets;
1020                         rx_bytes = stats->rx_bytes;
1021                         tx_bytes = stats->tx_bytes;
1022                 } while (u64_stats_fetch_retry_irq(&stats->syncp, start));
1023
1024                 tot->rx_packets += rx_packets;
1025                 tot->tx_packets += tx_packets;
1026                 tot->rx_bytes   += rx_bytes;
1027                 tot->tx_bytes   += tx_bytes;
1028                 tot->tx_dropped += stats->tx_dropped;
1029         }
1030 }
1031
1032 static void netvsc_get_stats64(struct net_device *net,
1033                                struct rtnl_link_stats64 *t)
1034 {
1035         struct net_device_context *ndev_ctx = netdev_priv(net);
1036         struct netvsc_device *nvdev = rcu_dereference_rtnl(ndev_ctx->nvdev);
1037         struct netvsc_vf_pcpu_stats vf_tot;
1038         int i;
1039
1040         if (!nvdev)
1041                 return;
1042
1043         netdev_stats_to_stats64(t, &net->stats);
1044
1045         netvsc_get_vf_stats(net, &vf_tot);
1046         t->rx_packets += vf_tot.rx_packets;
1047         t->tx_packets += vf_tot.tx_packets;
1048         t->rx_bytes   += vf_tot.rx_bytes;
1049         t->tx_bytes   += vf_tot.tx_bytes;
1050         t->tx_dropped += vf_tot.tx_dropped;
1051
1052         for (i = 0; i < nvdev->num_chn; i++) {
1053                 const struct netvsc_channel *nvchan = &nvdev->chan_table[i];
1054                 const struct netvsc_stats *stats;
1055                 u64 packets, bytes, multicast;
1056                 unsigned int start;
1057
1058                 stats = &nvchan->tx_stats;
1059                 do {
1060                         start = u64_stats_fetch_begin_irq(&stats->syncp);
1061                         packets = stats->packets;
1062                         bytes = stats->bytes;
1063                 } while (u64_stats_fetch_retry_irq(&stats->syncp, start));
1064
1065                 t->tx_bytes     += bytes;
1066                 t->tx_packets   += packets;
1067
1068                 stats = &nvchan->rx_stats;
1069                 do {
1070                         start = u64_stats_fetch_begin_irq(&stats->syncp);
1071                         packets = stats->packets;
1072                         bytes = stats->bytes;
1073                         multicast = stats->multicast + stats->broadcast;
1074                 } while (u64_stats_fetch_retry_irq(&stats->syncp, start));
1075
1076                 t->rx_bytes     += bytes;
1077                 t->rx_packets   += packets;
1078                 t->multicast    += multicast;
1079         }
1080 }
1081
1082 static int netvsc_set_mac_addr(struct net_device *ndev, void *p)
1083 {
1084         struct net_device_context *ndc = netdev_priv(ndev);
1085         struct net_device *vf_netdev = rtnl_dereference(ndc->vf_netdev);
1086         struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev);
1087         struct sockaddr *addr = p;
1088         int err;
1089
1090         err = eth_prepare_mac_addr_change(ndev, p);
1091         if (err)
1092                 return err;
1093
1094         if (!nvdev)
1095                 return -ENODEV;
1096
1097         if (vf_netdev) {
1098                 err = dev_set_mac_address(vf_netdev, addr);
1099                 if (err)
1100                         return err;
1101         }
1102
1103         err = rndis_filter_set_device_mac(nvdev, addr->sa_data);
1104         if (!err) {
1105                 eth_commit_mac_addr_change(ndev, p);
1106         } else if (vf_netdev) {
1107                 /* rollback change on VF */
1108                 memcpy(addr->sa_data, ndev->dev_addr, ETH_ALEN);
1109                 dev_set_mac_address(vf_netdev, addr);
1110         }
1111
1112         return err;
1113 }
1114
1115 static const struct {
1116         char name[ETH_GSTRING_LEN];
1117         u16 offset;
1118 } netvsc_stats[] = {
1119         { "tx_scattered", offsetof(struct netvsc_ethtool_stats, tx_scattered) },
1120         { "tx_no_memory", offsetof(struct netvsc_ethtool_stats, tx_no_memory) },
1121         { "tx_no_space",  offsetof(struct netvsc_ethtool_stats, tx_no_space) },
1122         { "tx_too_big",   offsetof(struct netvsc_ethtool_stats, tx_too_big) },
1123         { "tx_busy",      offsetof(struct netvsc_ethtool_stats, tx_busy) },
1124         { "tx_send_full", offsetof(struct netvsc_ethtool_stats, tx_send_full) },
1125         { "rx_comp_busy", offsetof(struct netvsc_ethtool_stats, rx_comp_busy) },
1126         { "rx_no_memory", offsetof(struct netvsc_ethtool_stats, rx_no_memory) },
1127         { "stop_queue", offsetof(struct netvsc_ethtool_stats, stop_queue) },
1128         { "wake_queue", offsetof(struct netvsc_ethtool_stats, wake_queue) },
1129 }, vf_stats[] = {
1130         { "vf_rx_packets", offsetof(struct netvsc_vf_pcpu_stats, rx_packets) },
1131         { "vf_rx_bytes",   offsetof(struct netvsc_vf_pcpu_stats, rx_bytes) },
1132         { "vf_tx_packets", offsetof(struct netvsc_vf_pcpu_stats, tx_packets) },
1133         { "vf_tx_bytes",   offsetof(struct netvsc_vf_pcpu_stats, tx_bytes) },
1134         { "vf_tx_dropped", offsetof(struct netvsc_vf_pcpu_stats, tx_dropped) },
1135 };
1136
1137 #define NETVSC_GLOBAL_STATS_LEN ARRAY_SIZE(netvsc_stats)
1138 #define NETVSC_VF_STATS_LEN     ARRAY_SIZE(vf_stats)
1139
1140 /* 4 statistics per queue (rx/tx packets/bytes) */
1141 #define NETVSC_QUEUE_STATS_LEN(dev) ((dev)->num_chn * 4)
1142
1143 static int netvsc_get_sset_count(struct net_device *dev, int string_set)
1144 {
1145         struct net_device_context *ndc = netdev_priv(dev);
1146         struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev);
1147
1148         if (!nvdev)
1149                 return -ENODEV;
1150
1151         switch (string_set) {
1152         case ETH_SS_STATS:
1153                 return NETVSC_GLOBAL_STATS_LEN
1154                         + NETVSC_VF_STATS_LEN
1155                         + NETVSC_QUEUE_STATS_LEN(nvdev);
1156         default:
1157                 return -EINVAL;
1158         }
1159 }
1160
1161 static void netvsc_get_ethtool_stats(struct net_device *dev,
1162                                      struct ethtool_stats *stats, u64 *data)
1163 {
1164         struct net_device_context *ndc = netdev_priv(dev);
1165         struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev);
1166         const void *nds = &ndc->eth_stats;
1167         const struct netvsc_stats *qstats;
1168         struct netvsc_vf_pcpu_stats sum;
1169         unsigned int start;
1170         u64 packets, bytes;
1171         int i, j;
1172
1173         if (!nvdev)
1174                 return;
1175
1176         for (i = 0; i < NETVSC_GLOBAL_STATS_LEN; i++)
1177                 data[i] = *(unsigned long *)(nds + netvsc_stats[i].offset);
1178
1179         netvsc_get_vf_stats(dev, &sum);
1180         for (j = 0; j < NETVSC_VF_STATS_LEN; j++)
1181                 data[i++] = *(u64 *)((void *)&sum + vf_stats[j].offset);
1182
1183         for (j = 0; j < nvdev->num_chn; j++) {
1184                 qstats = &nvdev->chan_table[j].tx_stats;
1185
1186                 do {
1187                         start = u64_stats_fetch_begin_irq(&qstats->syncp);
1188                         packets = qstats->packets;
1189                         bytes = qstats->bytes;
1190                 } while (u64_stats_fetch_retry_irq(&qstats->syncp, start));
1191                 data[i++] = packets;
1192                 data[i++] = bytes;
1193
1194                 qstats = &nvdev->chan_table[j].rx_stats;
1195                 do {
1196                         start = u64_stats_fetch_begin_irq(&qstats->syncp);
1197                         packets = qstats->packets;
1198                         bytes = qstats->bytes;
1199                 } while (u64_stats_fetch_retry_irq(&qstats->syncp, start));
1200                 data[i++] = packets;
1201                 data[i++] = bytes;
1202         }
1203 }
1204
1205 static void netvsc_get_strings(struct net_device *dev, u32 stringset, u8 *data)
1206 {
1207         struct net_device_context *ndc = netdev_priv(dev);
1208         struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev);
1209         u8 *p = data;
1210         int i;
1211
1212         if (!nvdev)
1213                 return;
1214
1215         switch (stringset) {
1216         case ETH_SS_STATS:
1217                 for (i = 0; i < ARRAY_SIZE(netvsc_stats); i++) {
1218                         memcpy(p, netvsc_stats[i].name, ETH_GSTRING_LEN);
1219                         p += ETH_GSTRING_LEN;
1220                 }
1221
1222                 for (i = 0; i < ARRAY_SIZE(vf_stats); i++) {
1223                         memcpy(p, vf_stats[i].name, ETH_GSTRING_LEN);
1224                         p += ETH_GSTRING_LEN;
1225                 }
1226
1227                 for (i = 0; i < nvdev->num_chn; i++) {
1228                         sprintf(p, "tx_queue_%u_packets", i);
1229                         p += ETH_GSTRING_LEN;
1230                         sprintf(p, "tx_queue_%u_bytes", i);
1231                         p += ETH_GSTRING_LEN;
1232                         sprintf(p, "rx_queue_%u_packets", i);
1233                         p += ETH_GSTRING_LEN;
1234                         sprintf(p, "rx_queue_%u_bytes", i);
1235                         p += ETH_GSTRING_LEN;
1236                 }
1237
1238                 break;
1239         }
1240 }
1241
1242 static int
1243 netvsc_get_rss_hash_opts(struct net_device_context *ndc,
1244                          struct ethtool_rxnfc *info)
1245 {
1246         const u32 l4_flag = RXH_L4_B_0_1 | RXH_L4_B_2_3;
1247
1248         info->data = RXH_IP_SRC | RXH_IP_DST;
1249
1250         switch (info->flow_type) {
1251         case TCP_V4_FLOW:
1252                 if (ndc->l4_hash & HV_TCP4_L4HASH)
1253                         info->data |= l4_flag;
1254
1255                 break;
1256
1257         case TCP_V6_FLOW:
1258                 if (ndc->l4_hash & HV_TCP6_L4HASH)
1259                         info->data |= l4_flag;
1260
1261                 break;
1262
1263         case UDP_V4_FLOW:
1264                 if (ndc->l4_hash & HV_UDP4_L4HASH)
1265                         info->data |= l4_flag;
1266
1267                 break;
1268
1269         case UDP_V6_FLOW:
1270                 if (ndc->l4_hash & HV_UDP6_L4HASH)
1271                         info->data |= l4_flag;
1272
1273                 break;
1274
1275         case IPV4_FLOW:
1276         case IPV6_FLOW:
1277                 break;
1278         default:
1279                 info->data = 0;
1280                 break;
1281         }
1282
1283         return 0;
1284 }
1285
1286 static int
1287 netvsc_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info,
1288                  u32 *rules)
1289 {
1290         struct net_device_context *ndc = netdev_priv(dev);
1291         struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev);
1292
1293         if (!nvdev)
1294                 return -ENODEV;
1295
1296         switch (info->cmd) {
1297         case ETHTOOL_GRXRINGS:
1298                 info->data = nvdev->num_chn;
1299                 return 0;
1300
1301         case ETHTOOL_GRXFH:
1302                 return netvsc_get_rss_hash_opts(ndc, info);
1303         }
1304         return -EOPNOTSUPP;
1305 }
1306
1307 static int netvsc_set_rss_hash_opts(struct net_device_context *ndc,
1308                                     struct ethtool_rxnfc *info)
1309 {
1310         if (info->data == (RXH_IP_SRC | RXH_IP_DST |
1311                            RXH_L4_B_0_1 | RXH_L4_B_2_3)) {
1312                 switch (info->flow_type) {
1313                 case TCP_V4_FLOW:
1314                         ndc->l4_hash |= HV_TCP4_L4HASH;
1315                         break;
1316
1317                 case TCP_V6_FLOW:
1318                         ndc->l4_hash |= HV_TCP6_L4HASH;
1319                         break;
1320
1321                 case UDP_V4_FLOW:
1322                         ndc->l4_hash |= HV_UDP4_L4HASH;
1323                         break;
1324
1325                 case UDP_V6_FLOW:
1326                         ndc->l4_hash |= HV_UDP6_L4HASH;
1327                         break;
1328
1329                 default:
1330                         return -EOPNOTSUPP;
1331                 }
1332
1333                 return 0;
1334         }
1335
1336         if (info->data == (RXH_IP_SRC | RXH_IP_DST)) {
1337                 switch (info->flow_type) {
1338                 case TCP_V4_FLOW:
1339                         ndc->l4_hash &= ~HV_TCP4_L4HASH;
1340                         break;
1341
1342                 case TCP_V6_FLOW:
1343                         ndc->l4_hash &= ~HV_TCP6_L4HASH;
1344                         break;
1345
1346                 case UDP_V4_FLOW:
1347                         ndc->l4_hash &= ~HV_UDP4_L4HASH;
1348                         break;
1349
1350                 case UDP_V6_FLOW:
1351                         ndc->l4_hash &= ~HV_UDP6_L4HASH;
1352                         break;
1353
1354                 default:
1355                         return -EOPNOTSUPP;
1356                 }
1357
1358                 return 0;
1359         }
1360
1361         return -EOPNOTSUPP;
1362 }
1363
1364 static int
1365 netvsc_set_rxnfc(struct net_device *ndev, struct ethtool_rxnfc *info)
1366 {
1367         struct net_device_context *ndc = netdev_priv(ndev);
1368
1369         if (info->cmd == ETHTOOL_SRXFH)
1370                 return netvsc_set_rss_hash_opts(ndc, info);
1371
1372         return -EOPNOTSUPP;
1373 }
1374
1375 #ifdef CONFIG_NET_POLL_CONTROLLER
1376 static void netvsc_poll_controller(struct net_device *dev)
1377 {
1378         struct net_device_context *ndc = netdev_priv(dev);
1379         struct netvsc_device *ndev;
1380         int i;
1381
1382         rcu_read_lock();
1383         ndev = rcu_dereference(ndc->nvdev);
1384         if (ndev) {
1385                 for (i = 0; i < ndev->num_chn; i++) {
1386                         struct netvsc_channel *nvchan = &ndev->chan_table[i];
1387
1388                         napi_schedule(&nvchan->napi);
1389                 }
1390         }
1391         rcu_read_unlock();
1392 }
1393 #endif
1394
1395 static u32 netvsc_get_rxfh_key_size(struct net_device *dev)
1396 {
1397         return NETVSC_HASH_KEYLEN;
1398 }
1399
1400 static u32 netvsc_rss_indir_size(struct net_device *dev)
1401 {
1402         return ITAB_NUM;
1403 }
1404
1405 static int netvsc_get_rxfh(struct net_device *dev, u32 *indir, u8 *key,
1406                            u8 *hfunc)
1407 {
1408         struct net_device_context *ndc = netdev_priv(dev);
1409         struct netvsc_device *ndev = rtnl_dereference(ndc->nvdev);
1410         struct rndis_device *rndis_dev;
1411         int i;
1412
1413         if (!ndev)
1414                 return -ENODEV;
1415
1416         if (hfunc)
1417                 *hfunc = ETH_RSS_HASH_TOP;      /* Toeplitz */
1418
1419         rndis_dev = ndev->extension;
1420         if (indir) {
1421                 for (i = 0; i < ITAB_NUM; i++)
1422                         indir[i] = rndis_dev->rx_table[i];
1423         }
1424
1425         if (key)
1426                 memcpy(key, rndis_dev->rss_key, NETVSC_HASH_KEYLEN);
1427
1428         return 0;
1429 }
1430
1431 static int netvsc_set_rxfh(struct net_device *dev, const u32 *indir,
1432                            const u8 *key, const u8 hfunc)
1433 {
1434         struct net_device_context *ndc = netdev_priv(dev);
1435         struct netvsc_device *ndev = rtnl_dereference(ndc->nvdev);
1436         struct rndis_device *rndis_dev;
1437         int i;
1438
1439         if (!ndev)
1440                 return -ENODEV;
1441
1442         if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP)
1443                 return -EOPNOTSUPP;
1444
1445         rndis_dev = ndev->extension;
1446         if (indir) {
1447                 for (i = 0; i < ITAB_NUM; i++)
1448                         if (indir[i] >= ndev->num_chn)
1449                                 return -EINVAL;
1450
1451                 for (i = 0; i < ITAB_NUM; i++)
1452                         rndis_dev->rx_table[i] = indir[i];
1453         }
1454
1455         if (!key) {
1456                 if (!indir)
1457                         return 0;
1458
1459                 key = rndis_dev->rss_key;
1460         }
1461
1462         return rndis_filter_set_rss_param(rndis_dev, key);
1463 }
1464
1465 /* Hyper-V RNDIS protocol does not have ring in the HW sense.
1466  * It does have pre-allocated receive area which is divided into sections.
1467  */
1468 static void __netvsc_get_ringparam(struct netvsc_device *nvdev,
1469                                    struct ethtool_ringparam *ring)
1470 {
1471         u32 max_buf_size;
1472
1473         ring->rx_pending = nvdev->recv_section_cnt;
1474         ring->tx_pending = nvdev->send_section_cnt;
1475
1476         if (nvdev->nvsp_version <= NVSP_PROTOCOL_VERSION_2)
1477                 max_buf_size = NETVSC_RECEIVE_BUFFER_SIZE_LEGACY;
1478         else
1479                 max_buf_size = NETVSC_RECEIVE_BUFFER_SIZE;
1480
1481         ring->rx_max_pending = max_buf_size / nvdev->recv_section_size;
1482         ring->tx_max_pending = NETVSC_SEND_BUFFER_SIZE
1483                 / nvdev->send_section_size;
1484 }
1485
1486 static void netvsc_get_ringparam(struct net_device *ndev,
1487                                  struct ethtool_ringparam *ring)
1488 {
1489         struct net_device_context *ndevctx = netdev_priv(ndev);
1490         struct netvsc_device *nvdev = rtnl_dereference(ndevctx->nvdev);
1491
1492         if (!nvdev)
1493                 return;
1494
1495         __netvsc_get_ringparam(nvdev, ring);
1496 }
1497
1498 static int netvsc_set_ringparam(struct net_device *ndev,
1499                                 struct ethtool_ringparam *ring)
1500 {
1501         struct net_device_context *ndevctx = netdev_priv(ndev);
1502         struct netvsc_device *nvdev = rtnl_dereference(ndevctx->nvdev);
1503         struct hv_device *hdev = ndevctx->device_ctx;
1504         struct netvsc_device_info device_info;
1505         struct ethtool_ringparam orig;
1506         u32 new_tx, new_rx;
1507         bool was_opened;
1508         int ret = 0;
1509
1510         if (!nvdev || nvdev->destroy)
1511                 return -ENODEV;
1512
1513         memset(&orig, 0, sizeof(orig));
1514         __netvsc_get_ringparam(nvdev, &orig);
1515
1516         new_tx = clamp_t(u32, ring->tx_pending,
1517                          NETVSC_MIN_TX_SECTIONS, orig.tx_max_pending);
1518         new_rx = clamp_t(u32, ring->rx_pending,
1519                          NETVSC_MIN_RX_SECTIONS, orig.rx_max_pending);
1520
1521         if (new_tx == orig.tx_pending &&
1522             new_rx == orig.rx_pending)
1523                 return 0;        /* no change */
1524
1525         memset(&device_info, 0, sizeof(device_info));
1526         device_info.num_chn = nvdev->num_chn;
1527         device_info.send_sections = new_tx;
1528         device_info.send_section_size = nvdev->send_section_size;
1529         device_info.recv_sections = new_rx;
1530         device_info.recv_section_size = nvdev->recv_section_size;
1531
1532         netif_device_detach(ndev);
1533         was_opened = rndis_filter_opened(nvdev);
1534         if (was_opened)
1535                 rndis_filter_close(nvdev);
1536
1537         rndis_filter_device_remove(hdev, nvdev);
1538
1539         nvdev = rndis_filter_device_add(hdev, &device_info);
1540         if (IS_ERR(nvdev)) {
1541                 ret = PTR_ERR(nvdev);
1542
1543                 device_info.send_sections = orig.tx_pending;
1544                 device_info.recv_sections = orig.rx_pending;
1545                 nvdev = rndis_filter_device_add(hdev, &device_info);
1546                 if (IS_ERR(nvdev)) {
1547                         netdev_err(ndev, "restoring ringparam failed: %ld\n",
1548                                    PTR_ERR(nvdev));
1549                         return ret;
1550                 }
1551         }
1552
1553         if (was_opened)
1554                 rndis_filter_open(nvdev);
1555         netif_device_attach(ndev);
1556
1557         /* We may have missed link change notifications */
1558         ndevctx->last_reconfig = 0;
1559         schedule_delayed_work(&ndevctx->dwork, 0);
1560
1561         return ret;
1562 }
1563
1564 static const struct ethtool_ops ethtool_ops = {
1565         .get_drvinfo    = netvsc_get_drvinfo,
1566         .get_link       = ethtool_op_get_link,
1567         .get_ethtool_stats = netvsc_get_ethtool_stats,
1568         .get_sset_count = netvsc_get_sset_count,
1569         .get_strings    = netvsc_get_strings,
1570         .get_channels   = netvsc_get_channels,
1571         .set_channels   = netvsc_set_channels,
1572         .get_ts_info    = ethtool_op_get_ts_info,
1573         .get_rxnfc      = netvsc_get_rxnfc,
1574         .set_rxnfc      = netvsc_set_rxnfc,
1575         .get_rxfh_key_size = netvsc_get_rxfh_key_size,
1576         .get_rxfh_indir_size = netvsc_rss_indir_size,
1577         .get_rxfh       = netvsc_get_rxfh,
1578         .set_rxfh       = netvsc_set_rxfh,
1579         .get_link_ksettings = netvsc_get_link_ksettings,
1580         .set_link_ksettings = netvsc_set_link_ksettings,
1581         .get_ringparam  = netvsc_get_ringparam,
1582         .set_ringparam  = netvsc_set_ringparam,
1583 };
1584
1585 static const struct net_device_ops device_ops = {
1586         .ndo_open =                     netvsc_open,
1587         .ndo_stop =                     netvsc_close,
1588         .ndo_start_xmit =               netvsc_start_xmit,
1589         .ndo_set_rx_mode =              netvsc_set_multicast_list,
1590         .ndo_change_mtu =               netvsc_change_mtu,
1591         .ndo_validate_addr =            eth_validate_addr,
1592         .ndo_set_mac_address =          netvsc_set_mac_addr,
1593         .ndo_select_queue =             netvsc_select_queue,
1594         .ndo_get_stats64 =              netvsc_get_stats64,
1595 #ifdef CONFIG_NET_POLL_CONTROLLER
1596         .ndo_poll_controller =          netvsc_poll_controller,
1597 #endif
1598 };
1599
1600 /*
1601  * Handle link status changes. For RNDIS_STATUS_NETWORK_CHANGE emulate link
1602  * down/up sequence. In case of RNDIS_STATUS_MEDIA_CONNECT when carrier is
1603  * present send GARP packet to network peers with netif_notify_peers().
1604  */
1605 static void netvsc_link_change(struct work_struct *w)
1606 {
1607         struct net_device_context *ndev_ctx =
1608                 container_of(w, struct net_device_context, dwork.work);
1609         struct hv_device *device_obj = ndev_ctx->device_ctx;
1610         struct net_device *net = hv_get_drvdata(device_obj);
1611         struct netvsc_device *net_device;
1612         struct rndis_device *rdev;
1613         struct netvsc_reconfig *event = NULL;
1614         bool notify = false, reschedule = false;
1615         unsigned long flags, next_reconfig, delay;
1616
1617         /* if changes are happening, comeback later */
1618         if (!rtnl_trylock()) {
1619                 schedule_delayed_work(&ndev_ctx->dwork, LINKCHANGE_INT);
1620                 return;
1621         }
1622
1623         net_device = rtnl_dereference(ndev_ctx->nvdev);
1624         if (!net_device)
1625                 goto out_unlock;
1626
1627         rdev = net_device->extension;
1628
1629         next_reconfig = ndev_ctx->last_reconfig + LINKCHANGE_INT;
1630         if (time_is_after_jiffies(next_reconfig)) {
1631                 /* link_watch only sends one notification with current state
1632                  * per second, avoid doing reconfig more frequently. Handle
1633                  * wrap around.
1634                  */
1635                 delay = next_reconfig - jiffies;
1636                 delay = delay < LINKCHANGE_INT ? delay : LINKCHANGE_INT;
1637                 schedule_delayed_work(&ndev_ctx->dwork, delay);
1638                 goto out_unlock;
1639         }
1640         ndev_ctx->last_reconfig = jiffies;
1641
1642         spin_lock_irqsave(&ndev_ctx->lock, flags);
1643         if (!list_empty(&ndev_ctx->reconfig_events)) {
1644                 event = list_first_entry(&ndev_ctx->reconfig_events,
1645                                          struct netvsc_reconfig, list);
1646                 list_del(&event->list);
1647                 reschedule = !list_empty(&ndev_ctx->reconfig_events);
1648         }
1649         spin_unlock_irqrestore(&ndev_ctx->lock, flags);
1650
1651         if (!event)
1652                 goto out_unlock;
1653
1654         switch (event->event) {
1655                 /* Only the following events are possible due to the check in
1656                  * netvsc_linkstatus_callback()
1657                  */
1658         case RNDIS_STATUS_MEDIA_CONNECT:
1659                 if (rdev->link_state) {
1660                         rdev->link_state = false;
1661                         netif_carrier_on(net);
1662                         netif_tx_wake_all_queues(net);
1663                 } else {
1664                         notify = true;
1665                 }
1666                 kfree(event);
1667                 break;
1668         case RNDIS_STATUS_MEDIA_DISCONNECT:
1669                 if (!rdev->link_state) {
1670                         rdev->link_state = true;
1671                         netif_carrier_off(net);
1672                         netif_tx_stop_all_queues(net);
1673                 }
1674                 kfree(event);
1675                 break;
1676         case RNDIS_STATUS_NETWORK_CHANGE:
1677                 /* Only makes sense if carrier is present */
1678                 if (!rdev->link_state) {
1679                         rdev->link_state = true;
1680                         netif_carrier_off(net);
1681                         netif_tx_stop_all_queues(net);
1682                         event->event = RNDIS_STATUS_MEDIA_CONNECT;
1683                         spin_lock_irqsave(&ndev_ctx->lock, flags);
1684                         list_add(&event->list, &ndev_ctx->reconfig_events);
1685                         spin_unlock_irqrestore(&ndev_ctx->lock, flags);
1686                         reschedule = true;
1687                 }
1688                 break;
1689         }
1690
1691         rtnl_unlock();
1692
1693         if (notify)
1694                 netdev_notify_peers(net);
1695
1696         /* link_watch only sends one notification with current state per
1697          * second, handle next reconfig event in 2 seconds.
1698          */
1699         if (reschedule)
1700                 schedule_delayed_work(&ndev_ctx->dwork, LINKCHANGE_INT);
1701
1702         return;
1703
1704 out_unlock:
1705         rtnl_unlock();
1706 }
1707
1708 static struct net_device *get_netvsc_bymac(const u8 *mac)
1709 {
1710         struct net_device *dev;
1711
1712         ASSERT_RTNL();
1713
1714         for_each_netdev(&init_net, dev) {
1715                 if (dev->netdev_ops != &device_ops)
1716                         continue;       /* not a netvsc device */
1717
1718                 if (ether_addr_equal(mac, dev->perm_addr))
1719                         return dev;
1720         }
1721
1722         return NULL;
1723 }
1724
1725 static struct net_device *get_netvsc_byref(struct net_device *vf_netdev)
1726 {
1727         struct net_device *dev;
1728
1729         ASSERT_RTNL();
1730
1731         for_each_netdev(&init_net, dev) {
1732                 struct net_device_context *net_device_ctx;
1733
1734                 if (dev->netdev_ops != &device_ops)
1735                         continue;       /* not a netvsc device */
1736
1737                 net_device_ctx = netdev_priv(dev);
1738                 if (!rtnl_dereference(net_device_ctx->nvdev))
1739                         continue;       /* device is removed */
1740
1741                 if (rtnl_dereference(net_device_ctx->vf_netdev) == vf_netdev)
1742                         return dev;     /* a match */
1743         }
1744
1745         return NULL;
1746 }
1747
1748 /* Called when VF is injecting data into network stack.
1749  * Change the associated network device from VF to netvsc.
1750  * note: already called with rcu_read_lock
1751  */
1752 static rx_handler_result_t netvsc_vf_handle_frame(struct sk_buff **pskb)
1753 {
1754         struct sk_buff *skb = *pskb;
1755         struct net_device *ndev = rcu_dereference(skb->dev->rx_handler_data);
1756         struct net_device_context *ndev_ctx = netdev_priv(ndev);
1757         struct netvsc_vf_pcpu_stats *pcpu_stats
1758                  = this_cpu_ptr(ndev_ctx->vf_stats);
1759
1760         skb->dev = ndev;
1761
1762         u64_stats_update_begin(&pcpu_stats->syncp);
1763         pcpu_stats->rx_packets++;
1764         pcpu_stats->rx_bytes += skb->len;
1765         u64_stats_update_end(&pcpu_stats->syncp);
1766
1767         return RX_HANDLER_ANOTHER;
1768 }
1769
1770 static int netvsc_vf_join(struct net_device *vf_netdev,
1771                           struct net_device *ndev)
1772 {
1773         struct net_device_context *ndev_ctx = netdev_priv(ndev);
1774         int ret;
1775
1776         ret = netdev_rx_handler_register(vf_netdev,
1777                                          netvsc_vf_handle_frame, ndev);
1778         if (ret != 0) {
1779                 netdev_err(vf_netdev,
1780                            "can not register netvsc VF receive handler (err = %d)\n",
1781                            ret);
1782                 goto rx_handler_failed;
1783         }
1784
1785         ret = netdev_upper_dev_link(vf_netdev, ndev, NULL);
1786         if (ret != 0) {
1787                 netdev_err(vf_netdev,
1788                            "can not set master device %s (err = %d)\n",
1789                            ndev->name, ret);
1790                 goto upper_link_failed;
1791         }
1792
1793         /* set slave flag before open to prevent IPv6 addrconf */
1794         vf_netdev->flags |= IFF_SLAVE;
1795
1796         schedule_delayed_work(&ndev_ctx->vf_takeover, VF_TAKEOVER_INT);
1797
1798         call_netdevice_notifiers(NETDEV_JOIN, vf_netdev);
1799
1800         netdev_info(vf_netdev, "joined to %s\n", ndev->name);
1801         return 0;
1802
1803 upper_link_failed:
1804         netdev_rx_handler_unregister(vf_netdev);
1805 rx_handler_failed:
1806         return ret;
1807 }
1808
1809 static void __netvsc_vf_setup(struct net_device *ndev,
1810                               struct net_device *vf_netdev)
1811 {
1812         int ret;
1813
1814         /* Align MTU of VF with master */
1815         ret = dev_set_mtu(vf_netdev, ndev->mtu);
1816         if (ret)
1817                 netdev_warn(vf_netdev,
1818                             "unable to change mtu to %u\n", ndev->mtu);
1819
1820         if (netif_running(ndev)) {
1821                 ret = dev_open(vf_netdev);
1822                 if (ret)
1823                         netdev_warn(vf_netdev,
1824                                     "unable to open: %d\n", ret);
1825         }
1826 }
1827
1828 /* Setup VF as slave of the synthetic device.
1829  * Runs in workqueue to avoid recursion in netlink callbacks.
1830  */
1831 static void netvsc_vf_setup(struct work_struct *w)
1832 {
1833         struct net_device_context *ndev_ctx
1834                 = container_of(w, struct net_device_context, vf_takeover.work);
1835         struct net_device *ndev = hv_get_drvdata(ndev_ctx->device_ctx);
1836         struct net_device *vf_netdev;
1837
1838         if (!rtnl_trylock()) {
1839                 schedule_delayed_work(&ndev_ctx->vf_takeover, 0);
1840                 return;
1841         }
1842
1843         vf_netdev = rtnl_dereference(ndev_ctx->vf_netdev);
1844         if (vf_netdev)
1845                 __netvsc_vf_setup(ndev, vf_netdev);
1846
1847         rtnl_unlock();
1848 }
1849
1850 static int netvsc_register_vf(struct net_device *vf_netdev)
1851 {
1852         struct net_device *ndev;
1853         struct net_device_context *net_device_ctx;
1854         struct netvsc_device *netvsc_dev;
1855
1856         if (vf_netdev->addr_len != ETH_ALEN)
1857                 return NOTIFY_DONE;
1858
1859         /*
1860          * We will use the MAC address to locate the synthetic interface to
1861          * associate with the VF interface. If we don't find a matching
1862          * synthetic interface, move on.
1863          */
1864         ndev = get_netvsc_bymac(vf_netdev->perm_addr);
1865         if (!ndev)
1866                 return NOTIFY_DONE;
1867
1868         net_device_ctx = netdev_priv(ndev);
1869         netvsc_dev = rtnl_dereference(net_device_ctx->nvdev);
1870         if (!netvsc_dev || rtnl_dereference(net_device_ctx->vf_netdev))
1871                 return NOTIFY_DONE;
1872
1873         if (netvsc_vf_join(vf_netdev, ndev) != 0)
1874                 return NOTIFY_DONE;
1875
1876         netdev_info(ndev, "VF registering: %s\n", vf_netdev->name);
1877
1878         dev_hold(vf_netdev);
1879         rcu_assign_pointer(net_device_ctx->vf_netdev, vf_netdev);
1880         return NOTIFY_OK;
1881 }
1882
1883 /* VF up/down change detected, schedule to change data path */
1884 static int netvsc_vf_changed(struct net_device *vf_netdev)
1885 {
1886         struct net_device_context *net_device_ctx;
1887         struct netvsc_device *netvsc_dev;
1888         struct net_device *ndev;
1889         bool vf_is_up = netif_running(vf_netdev);
1890
1891         ndev = get_netvsc_byref(vf_netdev);
1892         if (!ndev)
1893                 return NOTIFY_DONE;
1894
1895         net_device_ctx = netdev_priv(ndev);
1896         netvsc_dev = rtnl_dereference(net_device_ctx->nvdev);
1897         if (!netvsc_dev)
1898                 return NOTIFY_DONE;
1899
1900         netvsc_switch_datapath(ndev, vf_is_up);
1901         netdev_info(ndev, "Data path switched %s VF: %s\n",
1902                     vf_is_up ? "to" : "from", vf_netdev->name);
1903
1904         return NOTIFY_OK;
1905 }
1906
1907 static int netvsc_unregister_vf(struct net_device *vf_netdev)
1908 {
1909         struct net_device *ndev;
1910         struct net_device_context *net_device_ctx;
1911
1912         ndev = get_netvsc_byref(vf_netdev);
1913         if (!ndev)
1914                 return NOTIFY_DONE;
1915
1916         net_device_ctx = netdev_priv(ndev);
1917         cancel_delayed_work_sync(&net_device_ctx->vf_takeover);
1918
1919         netdev_info(ndev, "VF unregistering: %s\n", vf_netdev->name);
1920
1921         netdev_rx_handler_unregister(vf_netdev);
1922         netdev_upper_dev_unlink(vf_netdev, ndev);
1923         RCU_INIT_POINTER(net_device_ctx->vf_netdev, NULL);
1924         dev_put(vf_netdev);
1925
1926         return NOTIFY_OK;
1927 }
1928
1929 static int netvsc_probe(struct hv_device *dev,
1930                         const struct hv_vmbus_device_id *dev_id)
1931 {
1932         struct net_device *net = NULL;
1933         struct net_device_context *net_device_ctx;
1934         struct netvsc_device_info device_info;
1935         struct netvsc_device *nvdev;
1936         int ret = -ENOMEM;
1937
1938         net = alloc_etherdev_mq(sizeof(struct net_device_context),
1939                                 VRSS_CHANNEL_MAX);
1940         if (!net)
1941                 goto no_net;
1942
1943         netif_carrier_off(net);
1944
1945         netvsc_init_settings(net);
1946
1947         net_device_ctx = netdev_priv(net);
1948         net_device_ctx->device_ctx = dev;
1949         net_device_ctx->msg_enable = netif_msg_init(debug, default_msg);
1950         if (netif_msg_probe(net_device_ctx))
1951                 netdev_dbg(net, "netvsc msg_enable: %d\n",
1952                            net_device_ctx->msg_enable);
1953
1954         hv_set_drvdata(dev, net);
1955
1956         INIT_DELAYED_WORK(&net_device_ctx->dwork, netvsc_link_change);
1957
1958         spin_lock_init(&net_device_ctx->lock);
1959         INIT_LIST_HEAD(&net_device_ctx->reconfig_events);
1960         INIT_DELAYED_WORK(&net_device_ctx->vf_takeover, netvsc_vf_setup);
1961
1962         net_device_ctx->vf_stats
1963                 = netdev_alloc_pcpu_stats(struct netvsc_vf_pcpu_stats);
1964         if (!net_device_ctx->vf_stats)
1965                 goto no_stats;
1966
1967         net->netdev_ops = &device_ops;
1968         net->ethtool_ops = &ethtool_ops;
1969         SET_NETDEV_DEV(net, &dev->device);
1970
1971         /* We always need headroom for rndis header */
1972         net->needed_headroom = RNDIS_AND_PPI_SIZE;
1973
1974         /* Initialize the number of queues to be 1, we may change it if more
1975          * channels are offered later.
1976          */
1977         netif_set_real_num_tx_queues(net, 1);
1978         netif_set_real_num_rx_queues(net, 1);
1979
1980         /* Notify the netvsc driver of the new device */
1981         memset(&device_info, 0, sizeof(device_info));
1982         device_info.num_chn = VRSS_CHANNEL_DEFAULT;
1983         device_info.send_sections = NETVSC_DEFAULT_TX;
1984         device_info.send_section_size = NETVSC_SEND_SECTION_SIZE;
1985         device_info.recv_sections = NETVSC_DEFAULT_RX;
1986         device_info.recv_section_size = NETVSC_RECV_SECTION_SIZE;
1987
1988         nvdev = rndis_filter_device_add(dev, &device_info);
1989         if (IS_ERR(nvdev)) {
1990                 ret = PTR_ERR(nvdev);
1991                 netdev_err(net, "unable to add netvsc device (ret %d)\n", ret);
1992                 goto rndis_failed;
1993         }
1994
1995         memcpy(net->dev_addr, device_info.mac_adr, ETH_ALEN);
1996
1997         /* hw_features computed in rndis_netdev_set_hwcaps() */
1998         net->features = net->hw_features |
1999                 NETIF_F_HIGHDMA | NETIF_F_SG |
2000                 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX;
2001         net->vlan_features = net->features;
2002
2003         netdev_lockdep_set_classes(net);
2004
2005         /* MTU range: 68 - 1500 or 65521 */
2006         net->min_mtu = NETVSC_MTU_MIN;
2007         if (nvdev->nvsp_version >= NVSP_PROTOCOL_VERSION_2)
2008                 net->max_mtu = NETVSC_MTU - ETH_HLEN;
2009         else
2010                 net->max_mtu = ETH_DATA_LEN;
2011
2012         ret = register_netdev(net);
2013         if (ret != 0) {
2014                 pr_err("Unable to register netdev.\n");
2015                 goto register_failed;
2016         }
2017
2018         return ret;
2019
2020 register_failed:
2021         rndis_filter_device_remove(dev, nvdev);
2022 rndis_failed:
2023         free_percpu(net_device_ctx->vf_stats);
2024 no_stats:
2025         hv_set_drvdata(dev, NULL);
2026         free_netdev(net);
2027 no_net:
2028         return ret;
2029 }
2030
2031 static int netvsc_remove(struct hv_device *dev)
2032 {
2033         struct net_device_context *ndev_ctx;
2034         struct net_device *vf_netdev;
2035         struct net_device *net;
2036
2037         net = hv_get_drvdata(dev);
2038         if (net == NULL) {
2039                 dev_err(&dev->device, "No net device to remove\n");
2040                 return 0;
2041         }
2042
2043         ndev_ctx = netdev_priv(net);
2044
2045         netif_device_detach(net);
2046
2047         cancel_delayed_work_sync(&ndev_ctx->dwork);
2048
2049         /*
2050          * Call to the vsc driver to let it know that the device is being
2051          * removed. Also blocks mtu and channel changes.
2052          */
2053         rtnl_lock();
2054         vf_netdev = rtnl_dereference(ndev_ctx->vf_netdev);
2055         if (vf_netdev)
2056                 netvsc_unregister_vf(vf_netdev);
2057
2058         unregister_netdevice(net);
2059
2060         rndis_filter_device_remove(dev,
2061                                    rtnl_dereference(ndev_ctx->nvdev));
2062         rtnl_unlock();
2063
2064         hv_set_drvdata(dev, NULL);
2065
2066         free_percpu(ndev_ctx->vf_stats);
2067         free_netdev(net);
2068         return 0;
2069 }
2070
2071 static const struct hv_vmbus_device_id id_table[] = {
2072         /* Network guid */
2073         { HV_NIC_GUID, },
2074         { },
2075 };
2076
2077 MODULE_DEVICE_TABLE(vmbus, id_table);
2078
2079 /* The one and only one */
2080 static struct  hv_driver netvsc_drv = {
2081         .name = KBUILD_MODNAME,
2082         .id_table = id_table,
2083         .probe = netvsc_probe,
2084         .remove = netvsc_remove,
2085 };
2086
2087 /*
2088  * On Hyper-V, every VF interface is matched with a corresponding
2089  * synthetic interface. The synthetic interface is presented first
2090  * to the guest. When the corresponding VF instance is registered,
2091  * we will take care of switching the data path.
2092  */
2093 static int netvsc_netdev_event(struct notifier_block *this,
2094                                unsigned long event, void *ptr)
2095 {
2096         struct net_device *event_dev = netdev_notifier_info_to_dev(ptr);
2097
2098         /* Skip our own events */
2099         if (event_dev->netdev_ops == &device_ops)
2100                 return NOTIFY_DONE;
2101
2102         /* Avoid non-Ethernet type devices */
2103         if (event_dev->type != ARPHRD_ETHER)
2104                 return NOTIFY_DONE;
2105
2106         /* Avoid Vlan dev with same MAC registering as VF */
2107         if (is_vlan_dev(event_dev))
2108                 return NOTIFY_DONE;
2109
2110         /* Avoid Bonding master dev with same MAC registering as VF */
2111         if ((event_dev->priv_flags & IFF_BONDING) &&
2112             (event_dev->flags & IFF_MASTER))
2113                 return NOTIFY_DONE;
2114
2115         switch (event) {
2116         case NETDEV_REGISTER:
2117                 return netvsc_register_vf(event_dev);
2118         case NETDEV_UNREGISTER:
2119                 return netvsc_unregister_vf(event_dev);
2120         case NETDEV_UP:
2121         case NETDEV_DOWN:
2122                 return netvsc_vf_changed(event_dev);
2123         default:
2124                 return NOTIFY_DONE;
2125         }
2126 }
2127
2128 static struct notifier_block netvsc_netdev_notifier = {
2129         .notifier_call = netvsc_netdev_event,
2130 };
2131
2132 static void __exit netvsc_drv_exit(void)
2133 {
2134         unregister_netdevice_notifier(&netvsc_netdev_notifier);
2135         vmbus_driver_unregister(&netvsc_drv);
2136 }
2137
2138 static int __init netvsc_drv_init(void)
2139 {
2140         int ret;
2141
2142         if (ring_size < RING_SIZE_MIN) {
2143                 ring_size = RING_SIZE_MIN;
2144                 pr_info("Increased ring_size to %u (min allowed)\n",
2145                         ring_size);
2146         }
2147         netvsc_ring_bytes = ring_size * PAGE_SIZE;
2148         netvsc_ring_reciprocal = reciprocal_value(netvsc_ring_bytes);
2149
2150         ret = vmbus_driver_register(&netvsc_drv);
2151         if (ret)
2152                 return ret;
2153
2154         register_netdevice_notifier(&netvsc_netdev_notifier);
2155         return 0;
2156 }
2157
2158 MODULE_LICENSE("GPL");
2159 MODULE_DESCRIPTION("Microsoft Hyper-V network driver");
2160
2161 module_init(netvsc_drv_init);
2162 module_exit(netvsc_drv_exit);