drivers/net/wireguard/device.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
   4  */
   5
   6 #include "queueing.h"
   7 #include "socket.h"
   8 #include "timers.h"
   9 #include "device.h"
  10 #include "ratelimiter.h"
  11 #include "peer.h"
  12 #include "messages.h"
  13
  14 #include <linux/module.h>
  15 #include <linux/rtnetlink.h>
  16 #include <linux/inet.h>
  17 #include <linux/netdevice.h>
  18 #include <linux/inetdevice.h>
  19 #include <linux/if_arp.h>
  20 #include <linux/icmp.h>
  21 #include <linux/suspend.h>
  22 #include <net/icmp.h>
  23 #include <net/rtnetlink.h>
  24 #include <net/ip_tunnels.h>
  25 #include <net/addrconf.h>
  26
  27 static LIST_HEAD(device_list);
  28
  29 static int wg_open(struct net_device *dev)
  30 {
  31         struct in_device *dev_v4 = __in_dev_get_rtnl(dev);
  32         struct inet6_dev *dev_v6 = __in6_dev_get(dev);
  33         struct wg_device *wg = netdev_priv(dev);
  34         struct wg_peer *peer;
  35         int ret;
  36
  37         if (dev_v4) {
  38                 /* At some point we might put this check near the ip_rt_send_
  39                  * redirect call of ip_forward in net/ipv4/ip_forward.c, similar
  40                  * to the current secpath check.
  41                  */
  42                 IN_DEV_CONF_SET(dev_v4, SEND_REDIRECTS, false);
  43                 IPV4_DEVCONF_ALL(dev_net(dev), SEND_REDIRECTS) = false;
  44         }
  45         if (dev_v6)
  46                 dev_v6->cnf.addr_gen_mode = IN6_ADDR_GEN_MODE_NONE;
  47
  48         mutex_lock(&wg->device_update_lock);
  49         ret = wg_socket_init(wg, wg->incoming_port);
  50         if (ret < 0)
  51                 goto out;
  52         list_for_each_entry(peer, &wg->peer_list, peer_list) {
  53                 wg_packet_send_staged_packets(peer);
  54                 if (peer->persistent_keepalive_interval)
  55                         wg_packet_send_keepalive(peer);
  56         }
  57 out:
  58         mutex_unlock(&wg->device_update_lock);
  59         return ret;
  60 }
  61
  62 #ifdef CONFIG_PM_SLEEP
  63 static int wg_pm_notification(struct notifier_block *nb, unsigned long action,
  64                               void *data)
  65 {
  66         struct wg_device *wg;
  67         struct wg_peer *peer;
  68
  69         /* If the machine is constantly suspending and resuming, as part of
  70          * its normal operation rather than as a somewhat rare event, then we
  71          * don't actually want to clear keys.
  72          */
  73         if (IS_ENABLED(CONFIG_PM_AUTOSLEEP) || IS_ENABLED(CONFIG_ANDROID))
  74                 return 0;
  75
  76         if (action != PM_HIBERNATION_PREPARE && action != PM_SUSPEND_PREPARE)
  77                 return 0;
  78
  79         rtnl_lock();
  80         list_for_each_entry(wg, &device_list, device_list) {
  81                 mutex_lock(&wg->device_update_lock);
  82                 list_for_each_entry(peer, &wg->peer_list, peer_list) {
  83                         del_timer(&peer->timer_zero_key_material);
  84                         wg_noise_handshake_clear(&peer->handshake);
  85                         wg_noise_keypairs_clear(&peer->keypairs);
  86                 }
  87                 mutex_unlock(&wg->device_update_lock);
  88         }
  89         rtnl_unlock();
  90         rcu_barrier();
  91         return 0;
  92 }
  93
  94 static struct notifier_block pm_notifier = { .notifier_call = wg_pm_notification };
  95 #endif
  96
  97 static int wg_stop(struct net_device *dev)
  98 {
  99         struct wg_device *wg = netdev_priv(dev);
 100         struct wg_peer *peer;
 101
 102         mutex_lock(&wg->device_update_lock);
 103         list_for_each_entry(peer, &wg->peer_list, peer_list) {
 104                 wg_packet_purge_staged_packets(peer);
 105                 wg_timers_stop(peer);
 106                 wg_noise_handshake_clear(&peer->handshake);
 107                 wg_noise_keypairs_clear(&peer->keypairs);
 108                 wg_noise_reset_last_sent_handshake(&peer->last_sent_handshake);
 109         }
 110         mutex_unlock(&wg->device_update_lock);
 111         skb_queue_purge(&wg->incoming_handshakes);
 112         wg_socket_reinit(wg, NULL, NULL);
 113         return 0;
 114 }
 115
 116 static netdev_tx_t wg_xmit(struct sk_buff *skb, struct net_device *dev)
 117 {
 118         struct wg_device *wg = netdev_priv(dev);
 119         struct sk_buff_head packets;
 120         struct wg_peer *peer;
 121         struct sk_buff *next;
 122         sa_family_t family;
 123         u32 mtu;
 124         int ret;
 125
 126         if (unlikely(!wg_check_packet_protocol(skb))) {
 127                 ret = -EPROTONOSUPPORT;
 128                 net_dbg_ratelimited("%s: Invalid IP packet\n", dev->name);
 129                 goto err;
 130         }
 131
 132         peer = wg_allowedips_lookup_dst(&wg->peer_allowedips, skb);
 133         if (unlikely(!peer)) {
 134                 ret = -ENOKEY;
 135                 if (skb->protocol == htons(ETH_P_IP))
 136                         net_dbg_ratelimited("%s: No peer has allowed IPs matching %pI4\n",
 137                                             dev->name, &ip_hdr(skb)->daddr);
 138                 else if (skb->protocol == htons(ETH_P_IPV6))
 139                         net_dbg_ratelimited("%s: No peer has allowed IPs matching %pI6\n",
 140                                             dev->name, &ipv6_hdr(skb)->daddr);
 141                 goto err_icmp;
 142         }
 143
 144         family = READ_ONCE(peer->endpoint.addr.sa_family);
 145         if (unlikely(family != AF_INET && family != AF_INET6)) {
 146                 ret = -EDESTADDRREQ;
 147                 net_dbg_ratelimited("%s: No valid endpoint has been configured or discovered for peer %llu\n",
 148                                     dev->name, peer->internal_id);
 149                 goto err_peer;
 150         }
 151
 152         mtu = skb_dst(skb) ? dst_mtu(skb_dst(skb)) : dev->mtu;
 153
 154         __skb_queue_head_init(&packets);
 155         if (!skb_is_gso(skb)) {
 156                 skb_mark_not_on_list(skb);
 157         } else {
 158                 struct sk_buff *segs = skb_gso_segment(skb, 0);
 159
 160                 if (IS_ERR(segs)) {
 161                         ret = PTR_ERR(segs);
 162                         goto err_peer;
 163                 }
 164                 dev_kfree_skb(skb);
 165                 skb = segs;
 166         }
 167
 168         skb_list_walk_safe(skb, skb, next) {
 169                 skb_mark_not_on_list(skb);
 170
 171                 skb = skb_share_check(skb, GFP_ATOMIC);
 172                 if (unlikely(!skb))
 173                         continue;
 174
 175                 /* We only need to keep the original dst around for icmp,
 176                  * so at this point we're in a position to drop it.
 177                  */
 178                 skb_dst_drop(skb);
 179
 180                 PACKET_CB(skb)->mtu = mtu;
 181
 182                 __skb_queue_tail(&packets, skb);
 183         }
 184
 185         spin_lock_bh(&peer->staged_packet_queue.lock);
 186         /* If the queue is getting too big, we start removing the oldest packets
 187          * until it's small again. We do this before adding the new packet, so
 188          * we don't remove GSO segments that are in excess.
 189          */
 190         while (skb_queue_len(&peer->staged_packet_queue) > MAX_STAGED_PACKETS) {
 191                 dev_kfree_skb(__skb_dequeue(&peer->staged_packet_queue));
 192                 ++dev->stats.tx_dropped;
 193         }
 194         skb_queue_splice_tail(&packets, &peer->staged_packet_queue);
 195         spin_unlock_bh(&peer->staged_packet_queue.lock);
 196
 197         wg_packet_send_staged_packets(peer);
 198
 199         wg_peer_put(peer);
 200         return NETDEV_TX_OK;
 201
 202 err_peer:
 203         wg_peer_put(peer);
 204 err_icmp:
 205         if (skb->protocol == htons(ETH_P_IP))
 206                 icmp_ndo_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0);
 207         else if (skb->protocol == htons(ETH_P_IPV6))
 208                 icmpv6_ndo_send(skb, ICMPV6_DEST_UNREACH, ICMPV6_ADDR_UNREACH, 0);
 209 err:
 210         ++dev->stats.tx_errors;
 211         kfree_skb(skb);
 212         return ret;
 213 }
 214
 215 static const struct net_device_ops netdev_ops = {
 216         .ndo_open               = wg_open,
 217         .ndo_stop               = wg_stop,
 218         .ndo_start_xmit         = wg_xmit,
 219         .ndo_get_stats64        = dev_get_tstats64
 220 };
 221
 222 static void wg_destruct(struct net_device *dev)
 223 {
 224         struct wg_device *wg = netdev_priv(dev);
 225
 226         rtnl_lock();
 227         list_del(&wg->device_list);
 228         rtnl_unlock();
 229         mutex_lock(&wg->device_update_lock);
 230         rcu_assign_pointer(wg->creating_net, NULL);
 231         wg->incoming_port = 0;
 232         wg_socket_reinit(wg, NULL, NULL);
 233         /* The final references are cleared in the below calls to destroy_workqueue. */
 234         wg_peer_remove_all(wg);
 235         destroy_workqueue(wg->handshake_receive_wq);
 236         destroy_workqueue(wg->handshake_send_wq);
 237         destroy_workqueue(wg->packet_crypt_wq);
 238         wg_packet_queue_free(&wg->decrypt_queue);
 239         wg_packet_queue_free(&wg->encrypt_queue);
 240         rcu_barrier(); /* Wait for all the peers to be actually freed. */
 241         wg_ratelimiter_uninit();
 242         memzero_explicit(&wg->static_identity, sizeof(wg->static_identity));
 243         skb_queue_purge(&wg->incoming_handshakes);
 244         free_percpu(dev->tstats);
 245         free_percpu(wg->incoming_handshakes_worker);
 246         kvfree(wg->index_hashtable);
 247         kvfree(wg->peer_hashtable);
 248         mutex_unlock(&wg->device_update_lock);
 249
 250         pr_debug("%s: Interface destroyed\n", dev->name);
 251         free_netdev(dev);
 252 }
 253
 254 static const struct device_type device_type = { .name = KBUILD_MODNAME };
 255
 256 static void wg_setup(struct net_device *dev)
 257 {
 258         struct wg_device *wg = netdev_priv(dev);
 259         enum { WG_NETDEV_FEATURES = NETIF_F_HW_CSUM | NETIF_F_RXCSUM |
 260                                     NETIF_F_SG | NETIF_F_GSO |
 261                                     NETIF_F_GSO_SOFTWARE | NETIF_F_HIGHDMA };
 262         const int overhead = MESSAGE_MINIMUM_LENGTH + sizeof(struct udphdr) +
 263                              max(sizeof(struct ipv6hdr), sizeof(struct iphdr));
 264
 265         dev->netdev_ops = &netdev_ops;
 266         dev->header_ops = &ip_tunnel_header_ops;
 267         dev->hard_header_len = 0;
 268         dev->addr_len = 0;
 269         dev->needed_headroom = DATA_PACKET_HEAD_ROOM;
 270         dev->needed_tailroom = noise_encrypted_len(MESSAGE_PADDING_MULTIPLE);
 271         dev->type = ARPHRD_NONE;
 272         dev->flags = IFF_POINTOPOINT | IFF_NOARP;
 273         dev->priv_flags |= IFF_NO_QUEUE;
 274         dev->features |= NETIF_F_LLTX;
 275         dev->features |= WG_NETDEV_FEATURES;
 276         dev->hw_features |= WG_NETDEV_FEATURES;
 277         dev->hw_enc_features |= WG_NETDEV_FEATURES;
 278         dev->mtu = ETH_DATA_LEN - overhead;
 279         dev->max_mtu = round_down(INT_MAX, MESSAGE_PADDING_MULTIPLE) - overhead;
 280
 281         SET_NETDEV_DEVTYPE(dev, &device_type);
 282
 283         /* We need to keep the dst around in case of icmp replies. */
 284         netif_keep_dst(dev);
 285
 286         memset(wg, 0, sizeof(*wg));
 287         wg->dev = dev;
 288 }
 289
 290 static int wg_newlink(struct net *src_net, struct net_device *dev,
 291                       struct nlattr *tb[], struct nlattr *data[],
 292                       struct netlink_ext_ack *extack)
 293 {
 294         struct wg_device *wg = netdev_priv(dev);
 295         int ret = -ENOMEM;
 296
 297         rcu_assign_pointer(wg->creating_net, src_net);
 298         init_rwsem(&wg->static_identity.lock);
 299         mutex_init(&wg->socket_update_lock);
 300         mutex_init(&wg->device_update_lock);
 301         skb_queue_head_init(&wg->incoming_handshakes);
 302         wg_allowedips_init(&wg->peer_allowedips);
 303         wg_cookie_checker_init(&wg->cookie_checker, wg);
 304         INIT_LIST_HEAD(&wg->peer_list);
 305         wg->device_update_gen = 1;
 306
 307         wg->peer_hashtable = wg_pubkey_hashtable_alloc();
 308         if (!wg->peer_hashtable)
 309                 return ret;
 310
 311         wg->index_hashtable = wg_index_hashtable_alloc();
 312         if (!wg->index_hashtable)
 313                 goto err_free_peer_hashtable;
 314
 315         dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats);
 316         if (!dev->tstats)
 317                 goto err_free_index_hashtable;
 318
 319         wg->incoming_handshakes_worker =
 320                 wg_packet_percpu_multicore_worker_alloc(
 321                                 wg_packet_handshake_receive_worker, wg);
 322         if (!wg->incoming_handshakes_worker)
 323                 goto err_free_tstats;
 324
 325         wg->handshake_receive_wq = alloc_workqueue("wg-kex-%s",
 326                         WQ_CPU_INTENSIVE | WQ_FREEZABLE, 0, dev->name);
 327         if (!wg->handshake_receive_wq)
 328                 goto err_free_incoming_handshakes;
 329
 330         wg->handshake_send_wq = alloc_workqueue("wg-kex-%s",
 331                         WQ_UNBOUND | WQ_FREEZABLE, 0, dev->name);
 332         if (!wg->handshake_send_wq)
 333                 goto err_destroy_handshake_receive;
 334
 335         wg->packet_crypt_wq = alloc_workqueue("wg-crypt-%s",
 336                         WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM, 0, dev->name);
 337         if (!wg->packet_crypt_wq)
 338                 goto err_destroy_handshake_send;
 339
 340         ret = wg_packet_queue_init(&wg->encrypt_queue, wg_packet_encrypt_worker,
 341                                    MAX_QUEUED_PACKETS);
 342         if (ret < 0)
 343                 goto err_destroy_packet_crypt;
 344
 345         ret = wg_packet_queue_init(&wg->decrypt_queue, wg_packet_decrypt_worker,
 346                                    MAX_QUEUED_PACKETS);
 347         if (ret < 0)
 348                 goto err_free_encrypt_queue;
 349
 350         ret = wg_ratelimiter_init();
 351         if (ret < 0)
 352                 goto err_free_decrypt_queue;
 353
 354         ret = register_netdevice(dev);
 355         if (ret < 0)
 356                 goto err_uninit_ratelimiter;
 357
 358         list_add(&wg->device_list, &device_list);
 359
 360         /* We wait until the end to assign priv_destructor, so that
 361          * register_netdevice doesn't call it for us if it fails.
 362          */
 363         dev->priv_destructor = wg_destruct;
 364
 365         pr_debug("%s: Interface created\n", dev->name);
 366         return ret;
 367
 368 err_uninit_ratelimiter:
 369         wg_ratelimiter_uninit();
 370 err_free_decrypt_queue:
 371         wg_packet_queue_free(&wg->decrypt_queue);
 372 err_free_encrypt_queue:
 373         wg_packet_queue_free(&wg->encrypt_queue);
 374 err_destroy_packet_crypt:
 375         destroy_workqueue(wg->packet_crypt_wq);
 376 err_destroy_handshake_send:
 377         destroy_workqueue(wg->handshake_send_wq);
 378 err_destroy_handshake_receive:
 379         destroy_workqueue(wg->handshake_receive_wq);
 380 err_free_incoming_handshakes:
 381         free_percpu(wg->incoming_handshakes_worker);
 382 err_free_tstats:
 383         free_percpu(dev->tstats);
 384 err_free_index_hashtable:
 385         kvfree(wg->index_hashtable);
 386 err_free_peer_hashtable:
 387         kvfree(wg->peer_hashtable);
 388         return ret;
 389 }
 390
 391 static struct rtnl_link_ops link_ops __read_mostly = {
 392         .kind                   = KBUILD_MODNAME,
 393         .priv_size              = sizeof(struct wg_device),
 394         .setup                  = wg_setup,
 395         .newlink                = wg_newlink,
 396 };
 397
 398 static void wg_netns_pre_exit(struct net *net)
 399 {
 400         struct wg_device *wg;
 401
 402         rtnl_lock();
 403         list_for_each_entry(wg, &device_list, device_list) {
 404                 if (rcu_access_pointer(wg->creating_net) == net) {
 405                         pr_debug("%s: Creating namespace exiting\n", wg->dev->name);
 406                         netif_carrier_off(wg->dev);
 407                         mutex_lock(&wg->device_update_lock);
 408                         rcu_assign_pointer(wg->creating_net, NULL);
 409                         wg_socket_reinit(wg, NULL, NULL);
 410                         mutex_unlock(&wg->device_update_lock);
 411                 }
 412         }
 413         rtnl_unlock();
 414 }
 415
 416 static struct pernet_operations pernet_ops = {
 417         .pre_exit = wg_netns_pre_exit
 418 };
 419
 420 int __init wg_device_init(void)
 421 {
 422         int ret;
 423
 424 #ifdef CONFIG_PM_SLEEP
 425         ret = register_pm_notifier(&pm_notifier);
 426         if (ret)
 427                 return ret;
 428 #endif
 429
 430         ret = register_pernet_device(&pernet_ops);
 431         if (ret)
 432                 goto error_pm;
 433
 434         ret = rtnl_link_register(&link_ops);
 435         if (ret)
 436                 goto error_pernet;
 437
 438         return 0;
 439
 440 error_pernet:
 441         unregister_pernet_device(&pernet_ops);
 442 error_pm:
 443 #ifdef CONFIG_PM_SLEEP
 444         unregister_pm_notifier(&pm_notifier);
 445 #endif
 446         return ret;
 447 }
 448
 449 void wg_device_uninit(void)
 450 {
 451         rtnl_link_unregister(&link_ops);
 452         unregister_pernet_device(&pernet_ops);
 453 #ifdef CONFIG_PM_SLEEP
 454         unregister_pm_notifier(&pm_notifier);
 455 #endif
 456         rcu_barrier();
 457 }