kernel/bpf/offload.c

   1 /*
   2  * Copyright (C) 2017-2018 Netronome Systems, Inc.
   3  *
   4  * This software is licensed under the GNU General License Version 2,
   5  * June 1991 as shown in the file COPYING in the top-level directory of this
   6  * source tree.
   7  *
   8  * THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS"
   9  * WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING,
  10  * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  11  * FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE
  12  * OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME
  13  * THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
  14  */
  15
  16 #include <linux/bpf.h>
  17 #include <linux/bpf_verifier.h>
  18 #include <linux/bug.h>
  19 #include <linux/kdev_t.h>
  20 #include <linux/list.h>
  21 #include <linux/lockdep.h>
  22 #include <linux/netdevice.h>
  23 #include <linux/printk.h>
  24 #include <linux/proc_ns.h>
  25 #include <linux/rhashtable.h>
  26 #include <linux/rtnetlink.h>
  27 #include <linux/rwsem.h>
  28
  29 /* Protects offdevs, members of bpf_offload_netdev and offload members
  30  * of all progs.
  31  * RTNL lock cannot be taken when holding this lock.
  32  */
  33 static DECLARE_RWSEM(bpf_devs_lock);
  34
  35 struct bpf_offload_dev {
  36         const struct bpf_prog_offload_ops *ops;
  37         struct list_head netdevs;
  38 };
  39
  40 struct bpf_offload_netdev {
  41         struct rhash_head l;
  42         struct net_device *netdev;
  43         struct bpf_offload_dev *offdev;
  44         struct list_head progs;
  45         struct list_head maps;
  46         struct list_head offdev_netdevs;
  47 };
  48
  49 static const struct rhashtable_params offdevs_params = {
  50         .nelem_hint             = 4,
  51         .key_len                = sizeof(struct net_device *),
  52         .key_offset             = offsetof(struct bpf_offload_netdev, netdev),
  53         .head_offset            = offsetof(struct bpf_offload_netdev, l),
  54         .automatic_shrinking    = true,
  55 };
  56
  57 static struct rhashtable offdevs;
  58 static bool offdevs_inited;
  59
  60 static int bpf_dev_offload_check(struct net_device *netdev)
  61 {
  62         if (!netdev)
  63                 return -EINVAL;
  64         if (!netdev->netdev_ops->ndo_bpf)
  65                 return -EOPNOTSUPP;
  66         return 0;
  67 }
  68
  69 static struct bpf_offload_netdev *
  70 bpf_offload_find_netdev(struct net_device *netdev)
  71 {
  72         lockdep_assert_held(&bpf_devs_lock);
  73
  74         if (!offdevs_inited)
  75                 return NULL;
  76         return rhashtable_lookup_fast(&offdevs, &netdev, offdevs_params);
  77 }
  78
  79 int bpf_prog_offload_init(struct bpf_prog *prog, union bpf_attr *attr)
  80 {
  81         struct bpf_offload_netdev *ondev;
  82         struct bpf_prog_offload *offload;
  83         int err;
  84
  85         if (attr->prog_type != BPF_PROG_TYPE_SCHED_CLS &&
  86             attr->prog_type != BPF_PROG_TYPE_XDP)
  87                 return -EINVAL;
  88
  89         if (attr->prog_flags)
  90                 return -EINVAL;
  91
  92         offload = kzalloc(sizeof(*offload), GFP_USER);
  93         if (!offload)
  94                 return -ENOMEM;
  95
  96         offload->prog = prog;
  97
  98         offload->netdev = dev_get_by_index(current->nsproxy->net_ns,
  99                                            attr->prog_ifindex);
 100         err = bpf_dev_offload_check(offload->netdev);
 101         if (err)
 102                 goto err_maybe_put;
 103
 104         down_write(&bpf_devs_lock);
 105         ondev = bpf_offload_find_netdev(offload->netdev);
 106         if (!ondev) {
 107                 err = -EINVAL;
 108                 goto err_unlock;
 109         }
 110         offload->offdev = ondev->offdev;
 111         prog->aux->offload = offload;
 112         list_add_tail(&offload->offloads, &ondev->progs);
 113         dev_put(offload->netdev);
 114         up_write(&bpf_devs_lock);
 115
 116         return 0;
 117 err_unlock:
 118         up_write(&bpf_devs_lock);
 119 err_maybe_put:
 120         if (offload->netdev)
 121                 dev_put(offload->netdev);
 122         kfree(offload);
 123         return err;
 124 }
 125
 126 int bpf_prog_offload_verifier_prep(struct bpf_prog *prog)
 127 {
 128         struct bpf_prog_offload *offload;
 129         int ret = -ENODEV;
 130
 131         down_read(&bpf_devs_lock);
 132         offload = prog->aux->offload;
 133         if (offload) {
 134                 ret = offload->offdev->ops->prepare(prog);
 135                 offload->dev_state = !ret;
 136         }
 137         up_read(&bpf_devs_lock);
 138
 139         return ret;
 140 }
 141
 142 int bpf_prog_offload_verify_insn(struct bpf_verifier_env *env,
 143                                  int insn_idx, int prev_insn_idx)
 144 {
 145         struct bpf_prog_offload *offload;
 146         int ret = -ENODEV;
 147
 148         down_read(&bpf_devs_lock);
 149         offload = env->prog->aux->offload;
 150         if (offload)
 151                 ret = offload->offdev->ops->insn_hook(env, insn_idx,
 152                                                       prev_insn_idx);
 153         up_read(&bpf_devs_lock);
 154
 155         return ret;
 156 }
 157
 158 int bpf_prog_offload_finalize(struct bpf_verifier_env *env)
 159 {
 160         struct bpf_prog_offload *offload;
 161         int ret = -ENODEV;
 162
 163         down_read(&bpf_devs_lock);
 164         offload = env->prog->aux->offload;
 165         if (offload) {
 166                 if (offload->offdev->ops->finalize)
 167                         ret = offload->offdev->ops->finalize(env);
 168                 else
 169                         ret = 0;
 170         }
 171         up_read(&bpf_devs_lock);
 172
 173         return ret;
 174 }
 175
 176 static void __bpf_prog_offload_destroy(struct bpf_prog *prog)
 177 {
 178         struct bpf_prog_offload *offload = prog->aux->offload;
 179
 180         if (offload->dev_state)
 181                 offload->offdev->ops->destroy(prog);
 182
 183         /* Make sure BPF_PROG_GET_NEXT_ID can't find this dead program */
 184         bpf_prog_free_id(prog, true);
 185
 186         list_del_init(&offload->offloads);
 187         kfree(offload);
 188         prog->aux->offload = NULL;
 189 }
 190
 191 void bpf_prog_offload_destroy(struct bpf_prog *prog)
 192 {
 193         down_write(&bpf_devs_lock);
 194         if (prog->aux->offload)
 195                 __bpf_prog_offload_destroy(prog);
 196         up_write(&bpf_devs_lock);
 197 }
 198
 199 static int bpf_prog_offload_translate(struct bpf_prog *prog)
 200 {
 201         struct bpf_prog_offload *offload;
 202         int ret = -ENODEV;
 203
 204         down_read(&bpf_devs_lock);
 205         offload = prog->aux->offload;
 206         if (offload)
 207                 ret = offload->offdev->ops->translate(prog);
 208         up_read(&bpf_devs_lock);
 209
 210         return ret;
 211 }
 212
 213 static unsigned int bpf_prog_warn_on_exec(const void *ctx,
 214                                           const struct bpf_insn *insn)
 215 {
 216         WARN(1, "attempt to execute device eBPF program on the host!");
 217         return 0;
 218 }
 219
 220 int bpf_prog_offload_compile(struct bpf_prog *prog)
 221 {
 222         prog->bpf_func = bpf_prog_warn_on_exec;
 223
 224         return bpf_prog_offload_translate(prog);
 225 }
 226
 227 struct ns_get_path_bpf_prog_args {
 228         struct bpf_prog *prog;
 229         struct bpf_prog_info *info;
 230 };
 231
 232 static struct ns_common *bpf_prog_offload_info_fill_ns(void *private_data)
 233 {
 234         struct ns_get_path_bpf_prog_args *args = private_data;
 235         struct bpf_prog_aux *aux = args->prog->aux;
 236         struct ns_common *ns;
 237         struct net *net;
 238
 239         rtnl_lock();
 240         down_read(&bpf_devs_lock);
 241
 242         if (aux->offload) {
 243                 args->info->ifindex = aux->offload->netdev->ifindex;
 244                 net = dev_net(aux->offload->netdev);
 245                 get_net(net);
 246                 ns = &net->ns;
 247         } else {
 248                 args->info->ifindex = 0;
 249                 ns = NULL;
 250         }
 251
 252         up_read(&bpf_devs_lock);
 253         rtnl_unlock();
 254
 255         return ns;
 256 }
 257
 258 int bpf_prog_offload_info_fill(struct bpf_prog_info *info,
 259                                struct bpf_prog *prog)
 260 {
 261         struct ns_get_path_bpf_prog_args args = {
 262                 .prog   = prog,
 263                 .info   = info,
 264         };
 265         struct bpf_prog_aux *aux = prog->aux;
 266         struct inode *ns_inode;
 267         struct path ns_path;
 268         char __user *uinsns;
 269         void *res;
 270         u32 ulen;
 271
 272         res = ns_get_path_cb(&ns_path, bpf_prog_offload_info_fill_ns, &args);
 273         if (IS_ERR(res)) {
 274                 if (!info->ifindex)
 275                         return -ENODEV;
 276                 return PTR_ERR(res);
 277         }
 278
 279         down_read(&bpf_devs_lock);
 280
 281         if (!aux->offload) {
 282                 up_read(&bpf_devs_lock);
 283                 return -ENODEV;
 284         }
 285
 286         ulen = info->jited_prog_len;
 287         info->jited_prog_len = aux->offload->jited_len;
 288         if (info->jited_prog_len & ulen) {
 289                 uinsns = u64_to_user_ptr(info->jited_prog_insns);
 290                 ulen = min_t(u32, info->jited_prog_len, ulen);
 291                 if (copy_to_user(uinsns, aux->offload->jited_image, ulen)) {
 292                         up_read(&bpf_devs_lock);
 293                         return -EFAULT;
 294                 }
 295         }
 296
 297         up_read(&bpf_devs_lock);
 298
 299         ns_inode = ns_path.dentry->d_inode;
 300         info->netns_dev = new_encode_dev(ns_inode->i_sb->s_dev);
 301         info->netns_ino = ns_inode->i_ino;
 302         path_put(&ns_path);
 303
 304         return 0;
 305 }
 306
 307 const struct bpf_prog_ops bpf_offload_prog_ops = {
 308 };
 309
 310 static int bpf_map_offload_ndo(struct bpf_offloaded_map *offmap,
 311                                enum bpf_netdev_command cmd)
 312 {
 313         struct netdev_bpf data = {};
 314         struct net_device *netdev;
 315
 316         ASSERT_RTNL();
 317
 318         data.command = cmd;
 319         data.offmap = offmap;
 320         /* Caller must make sure netdev is valid */
 321         netdev = offmap->netdev;
 322
 323         return netdev->netdev_ops->ndo_bpf(netdev, &data);
 324 }
 325
 326 struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr)
 327 {
 328         struct net *net = current->nsproxy->net_ns;
 329         struct bpf_offload_netdev *ondev;
 330         struct bpf_offloaded_map *offmap;
 331         int err;
 332
 333         if (!capable(CAP_SYS_ADMIN))
 334                 return ERR_PTR(-EPERM);
 335         if (attr->map_type != BPF_MAP_TYPE_ARRAY &&
 336             attr->map_type != BPF_MAP_TYPE_HASH)
 337                 return ERR_PTR(-EINVAL);
 338
 339         offmap = kzalloc(sizeof(*offmap), GFP_USER);
 340         if (!offmap)
 341                 return ERR_PTR(-ENOMEM);
 342
 343         bpf_map_init_from_attr(&offmap->map, attr);
 344
 345         rtnl_lock();
 346         down_write(&bpf_devs_lock);
 347         offmap->netdev = __dev_get_by_index(net, attr->map_ifindex);
 348         err = bpf_dev_offload_check(offmap->netdev);
 349         if (err)
 350                 goto err_unlock;
 351
 352         ondev = bpf_offload_find_netdev(offmap->netdev);
 353         if (!ondev) {
 354                 err = -EINVAL;
 355                 goto err_unlock;
 356         }
 357
 358         err = bpf_map_offload_ndo(offmap, BPF_OFFLOAD_MAP_ALLOC);
 359         if (err)
 360                 goto err_unlock;
 361
 362         list_add_tail(&offmap->offloads, &ondev->maps);
 363         up_write(&bpf_devs_lock);
 364         rtnl_unlock();
 365
 366         return &offmap->map;
 367
 368 err_unlock:
 369         up_write(&bpf_devs_lock);
 370         rtnl_unlock();
 371         kfree(offmap);
 372         return ERR_PTR(err);
 373 }
 374
 375 static void __bpf_map_offload_destroy(struct bpf_offloaded_map *offmap)
 376 {
 377         WARN_ON(bpf_map_offload_ndo(offmap, BPF_OFFLOAD_MAP_FREE));
 378         /* Make sure BPF_MAP_GET_NEXT_ID can't find this dead map */
 379         bpf_map_free_id(&offmap->map, true);
 380         list_del_init(&offmap->offloads);
 381         offmap->netdev = NULL;
 382 }
 383
 384 void bpf_map_offload_map_free(struct bpf_map *map)
 385 {
 386         struct bpf_offloaded_map *offmap = map_to_offmap(map);
 387
 388         rtnl_lock();
 389         down_write(&bpf_devs_lock);
 390         if (offmap->netdev)
 391                 __bpf_map_offload_destroy(offmap);
 392         up_write(&bpf_devs_lock);
 393         rtnl_unlock();
 394
 395         kfree(offmap);
 396 }
 397
 398 int bpf_map_offload_lookup_elem(struct bpf_map *map, void *key, void *value)
 399 {
 400         struct bpf_offloaded_map *offmap = map_to_offmap(map);
 401         int ret = -ENODEV;
 402
 403         down_read(&bpf_devs_lock);
 404         if (offmap->netdev)
 405                 ret = offmap->dev_ops->map_lookup_elem(offmap, key, value);
 406         up_read(&bpf_devs_lock);
 407
 408         return ret;
 409 }
 410
 411 int bpf_map_offload_update_elem(struct bpf_map *map,
 412                                 void *key, void *value, u64 flags)
 413 {
 414         struct bpf_offloaded_map *offmap = map_to_offmap(map);
 415         int ret = -ENODEV;
 416
 417         if (unlikely(flags > BPF_EXIST))
 418                 return -EINVAL;
 419
 420         down_read(&bpf_devs_lock);
 421         if (offmap->netdev)
 422                 ret = offmap->dev_ops->map_update_elem(offmap, key, value,
 423                                                        flags);
 424         up_read(&bpf_devs_lock);
 425
 426         return ret;
 427 }
 428
 429 int bpf_map_offload_delete_elem(struct bpf_map *map, void *key)
 430 {
 431         struct bpf_offloaded_map *offmap = map_to_offmap(map);
 432         int ret = -ENODEV;
 433
 434         down_read(&bpf_devs_lock);
 435         if (offmap->netdev)
 436                 ret = offmap->dev_ops->map_delete_elem(offmap, key);
 437         up_read(&bpf_devs_lock);
 438
 439         return ret;
 440 }
 441
 442 int bpf_map_offload_get_next_key(struct bpf_map *map, void *key, void *next_key)
 443 {
 444         struct bpf_offloaded_map *offmap = map_to_offmap(map);
 445         int ret = -ENODEV;
 446
 447         down_read(&bpf_devs_lock);
 448         if (offmap->netdev)
 449                 ret = offmap->dev_ops->map_get_next_key(offmap, key, next_key);
 450         up_read(&bpf_devs_lock);
 451
 452         return ret;
 453 }
 454
 455 struct ns_get_path_bpf_map_args {
 456         struct bpf_offloaded_map *offmap;
 457         struct bpf_map_info *info;
 458 };
 459
 460 static struct ns_common *bpf_map_offload_info_fill_ns(void *private_data)
 461 {
 462         struct ns_get_path_bpf_map_args *args = private_data;
 463         struct ns_common *ns;
 464         struct net *net;
 465
 466         rtnl_lock();
 467         down_read(&bpf_devs_lock);
 468
 469         if (args->offmap->netdev) {
 470                 args->info->ifindex = args->offmap->netdev->ifindex;
 471                 net = dev_net(args->offmap->netdev);
 472                 get_net(net);
 473                 ns = &net->ns;
 474         } else {
 475                 args->info->ifindex = 0;
 476                 ns = NULL;
 477         }
 478
 479         up_read(&bpf_devs_lock);
 480         rtnl_unlock();
 481
 482         return ns;
 483 }
 484
 485 int bpf_map_offload_info_fill(struct bpf_map_info *info, struct bpf_map *map)
 486 {
 487         struct ns_get_path_bpf_map_args args = {
 488                 .offmap = map_to_offmap(map),
 489                 .info   = info,
 490         };
 491         struct inode *ns_inode;
 492         struct path ns_path;
 493         void *res;
 494
 495         res = ns_get_path_cb(&ns_path, bpf_map_offload_info_fill_ns, &args);
 496         if (IS_ERR(res)) {
 497                 if (!info->ifindex)
 498                         return -ENODEV;
 499                 return PTR_ERR(res);
 500         }
 501
 502         ns_inode = ns_path.dentry->d_inode;
 503         info->netns_dev = new_encode_dev(ns_inode->i_sb->s_dev);
 504         info->netns_ino = ns_inode->i_ino;
 505         path_put(&ns_path);
 506
 507         return 0;
 508 }
 509
 510 static bool __bpf_offload_dev_match(struct bpf_prog *prog,
 511                                     struct net_device *netdev)
 512 {
 513         struct bpf_offload_netdev *ondev1, *ondev2;
 514         struct bpf_prog_offload *offload;
 515
 516         if (!bpf_prog_is_dev_bound(prog->aux))
 517                 return false;
 518
 519         offload = prog->aux->offload;
 520         if (!offload)
 521                 return false;
 522         if (offload->netdev == netdev)
 523                 return true;
 524
 525         ondev1 = bpf_offload_find_netdev(offload->netdev);
 526         ondev2 = bpf_offload_find_netdev(netdev);
 527
 528         return ondev1 && ondev2 && ondev1->offdev == ondev2->offdev;
 529 }
 530
 531 bool bpf_offload_dev_match(struct bpf_prog *prog, struct net_device *netdev)
 532 {
 533         bool ret;
 534
 535         down_read(&bpf_devs_lock);
 536         ret = __bpf_offload_dev_match(prog, netdev);
 537         up_read(&bpf_devs_lock);
 538
 539         return ret;
 540 }
 541 EXPORT_SYMBOL_GPL(bpf_offload_dev_match);
 542
 543 bool bpf_offload_prog_map_match(struct bpf_prog *prog, struct bpf_map *map)
 544 {
 545         struct bpf_offloaded_map *offmap;
 546         bool ret;
 547
 548         if (!bpf_map_is_dev_bound(map))
 549                 return bpf_map_offload_neutral(map);
 550         offmap = map_to_offmap(map);
 551
 552         down_read(&bpf_devs_lock);
 553         ret = __bpf_offload_dev_match(prog, offmap->netdev);
 554         up_read(&bpf_devs_lock);
 555
 556         return ret;
 557 }
 558
 559 int bpf_offload_dev_netdev_register(struct bpf_offload_dev *offdev,
 560                                     struct net_device *netdev)
 561 {
 562         struct bpf_offload_netdev *ondev;
 563         int err;
 564
 565         ondev = kzalloc(sizeof(*ondev), GFP_KERNEL);
 566         if (!ondev)
 567                 return -ENOMEM;
 568
 569         ondev->netdev = netdev;
 570         ondev->offdev = offdev;
 571         INIT_LIST_HEAD(&ondev->progs);
 572         INIT_LIST_HEAD(&ondev->maps);
 573
 574         down_write(&bpf_devs_lock);
 575         err = rhashtable_insert_fast(&offdevs, &ondev->l, offdevs_params);
 576         if (err) {
 577                 netdev_warn(netdev, "failed to register for BPF offload\n");
 578                 goto err_unlock_free;
 579         }
 580
 581         list_add(&ondev->offdev_netdevs, &offdev->netdevs);
 582         up_write(&bpf_devs_lock);
 583         return 0;
 584
 585 err_unlock_free:
 586         up_write(&bpf_devs_lock);
 587         kfree(ondev);
 588         return err;
 589 }
 590 EXPORT_SYMBOL_GPL(bpf_offload_dev_netdev_register);
 591
 592 void bpf_offload_dev_netdev_unregister(struct bpf_offload_dev *offdev,
 593                                        struct net_device *netdev)
 594 {
 595         struct bpf_offload_netdev *ondev, *altdev;
 596         struct bpf_offloaded_map *offmap, *mtmp;
 597         struct bpf_prog_offload *offload, *ptmp;
 598
 599         ASSERT_RTNL();
 600
 601         down_write(&bpf_devs_lock);
 602         ondev = rhashtable_lookup_fast(&offdevs, &netdev, offdevs_params);
 603         if (WARN_ON(!ondev))
 604                 goto unlock;
 605
 606         WARN_ON(rhashtable_remove_fast(&offdevs, &ondev->l, offdevs_params));
 607         list_del(&ondev->offdev_netdevs);
 608
 609         /* Try to move the objects to another netdev of the device */
 610         altdev = list_first_entry_or_null(&offdev->netdevs,
 611                                           struct bpf_offload_netdev,
 612                                           offdev_netdevs);
 613         if (altdev) {
 614                 list_for_each_entry(offload, &ondev->progs, offloads)
 615                         offload->netdev = altdev->netdev;
 616                 list_splice_init(&ondev->progs, &altdev->progs);
 617
 618                 list_for_each_entry(offmap, &ondev->maps, offloads)
 619                         offmap->netdev = altdev->netdev;
 620                 list_splice_init(&ondev->maps, &altdev->maps);
 621         } else {
 622                 list_for_each_entry_safe(offload, ptmp, &ondev->progs, offloads)
 623                         __bpf_prog_offload_destroy(offload->prog);
 624                 list_for_each_entry_safe(offmap, mtmp, &ondev->maps, offloads)
 625                         __bpf_map_offload_destroy(offmap);
 626         }
 627
 628         WARN_ON(!list_empty(&ondev->progs));
 629         WARN_ON(!list_empty(&ondev->maps));
 630         kfree(ondev);
 631 unlock:
 632         up_write(&bpf_devs_lock);
 633 }
 634 EXPORT_SYMBOL_GPL(bpf_offload_dev_netdev_unregister);
 635
 636 struct bpf_offload_dev *
 637 bpf_offload_dev_create(const struct bpf_prog_offload_ops *ops)
 638 {
 639         struct bpf_offload_dev *offdev;
 640         int err;
 641
 642         down_write(&bpf_devs_lock);
 643         if (!offdevs_inited) {
 644                 err = rhashtable_init(&offdevs, &offdevs_params);
 645                 if (err)
 646                         return ERR_PTR(err);
 647                 offdevs_inited = true;
 648         }
 649         up_write(&bpf_devs_lock);
 650
 651         offdev = kzalloc(sizeof(*offdev), GFP_KERNEL);
 652         if (!offdev)
 653                 return ERR_PTR(-ENOMEM);
 654
 655         offdev->ops = ops;
 656         INIT_LIST_HEAD(&offdev->netdevs);
 657
 658         return offdev;
 659 }
 660 EXPORT_SYMBOL_GPL(bpf_offload_dev_create);
 661
 662 void bpf_offload_dev_destroy(struct bpf_offload_dev *offdev)
 663 {
 664         WARN_ON(!list_empty(&offdev->netdevs));
 665         kfree(offdev);
 666 }
 667 EXPORT_SYMBOL_GPL(bpf_offload_dev_destroy);