kernel/bpf/syscall.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
   3  */
   4 #include <linux/bpf.h>
   5 #include <linux/bpf_trace.h>
   6 #include <linux/bpf_lirc.h>
   7 #include <linux/bpf_verifier.h>
   8 #include <linux/btf.h>
   9 #include <linux/syscalls.h>
  10 #include <linux/slab.h>
  11 #include <linux/sched/signal.h>
  12 #include <linux/vmalloc.h>
  13 #include <linux/mmzone.h>
  14 #include <linux/anon_inodes.h>
  15 #include <linux/fdtable.h>
  16 #include <linux/file.h>
  17 #include <linux/fs.h>
  18 #include <linux/license.h>
  19 #include <linux/filter.h>
  20 #include <linux/version.h>
  21 #include <linux/kernel.h>
  22 #include <linux/idr.h>
  23 #include <linux/cred.h>
  24 #include <linux/timekeeping.h>
  25 #include <linux/ctype.h>
  26 #include <linux/nospec.h>
  27 #include <linux/audit.h>
  28 #include <uapi/linux/btf.h>
  29 #include <linux/pgtable.h>
  30 #include <linux/bpf_lsm.h>
  31 #include <linux/poll.h>
  32 #include <linux/bpf-netns.h>
  33 #include <linux/rcupdate_trace.h>
  34
  35 #define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \
  36                           (map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \
  37                           (map)->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS)
  38 #define IS_FD_PROG_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PROG_ARRAY)
  39 #define IS_FD_HASH(map) ((map)->map_type == BPF_MAP_TYPE_HASH_OF_MAPS)
  40 #define IS_FD_MAP(map) (IS_FD_ARRAY(map) || IS_FD_PROG_ARRAY(map) || \
  41                         IS_FD_HASH(map))
  42
  43 #define BPF_OBJ_FLAG_MASK   (BPF_F_RDONLY | BPF_F_WRONLY)
  44
  45 DEFINE_PER_CPU(int, bpf_prog_active);
  46 static DEFINE_IDR(prog_idr);
  47 static DEFINE_SPINLOCK(prog_idr_lock);
  48 static DEFINE_IDR(map_idr);
  49 static DEFINE_SPINLOCK(map_idr_lock);
  50 static DEFINE_IDR(link_idr);
  51 static DEFINE_SPINLOCK(link_idr_lock);
  52
  53 int sysctl_unprivileged_bpf_disabled __read_mostly;
  54
  55 static const struct bpf_map_ops * const bpf_map_types[] = {
  56 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type)
  57 #define BPF_MAP_TYPE(_id, _ops) \
  58         [_id] = &_ops,
  59 #define BPF_LINK_TYPE(_id, _name)
  60 #include <linux/bpf_types.h>
  61 #undef BPF_PROG_TYPE
  62 #undef BPF_MAP_TYPE
  63 #undef BPF_LINK_TYPE
  64 };
  65
  66 /*
  67  * If we're handed a bigger struct than we know of, ensure all the unknown bits
  68  * are 0 - i.e. new user-space does not rely on any kernel feature extensions
  69  * we don't know about yet.
  70  *
  71  * There is a ToCToU between this function call and the following
  72  * copy_from_user() call. However, this is not a concern since this function is
  73  * meant to be a future-proofing of bits.
  74  */
  75 int bpf_check_uarg_tail_zero(void __user *uaddr,
  76                              size_t expected_size,
  77                              size_t actual_size)
  78 {
  79         unsigned char __user *addr = uaddr + expected_size;
  80         int res;
  81
  82         if (unlikely(actual_size > PAGE_SIZE))  /* silly large */
  83                 return -E2BIG;
  84
  85         if (actual_size <= expected_size)
  86                 return 0;
  87
  88         res = check_zeroed_user(addr, actual_size - expected_size);
  89         if (res < 0)
  90                 return res;
  91         return res ? 0 : -E2BIG;
  92 }
  93
  94 const struct bpf_map_ops bpf_map_offload_ops = {
  95         .map_meta_equal = bpf_map_meta_equal,
  96         .map_alloc = bpf_map_offload_map_alloc,
  97         .map_free = bpf_map_offload_map_free,
  98         .map_check_btf = map_check_no_btf,
  99 };
 100
 101 static struct bpf_map *find_and_alloc_map(union bpf_attr *attr)
 102 {
 103         const struct bpf_map_ops *ops;
 104         u32 type = attr->map_type;
 105         struct bpf_map *map;
 106         int err;
 107
 108         if (type >= ARRAY_SIZE(bpf_map_types))
 109                 return ERR_PTR(-EINVAL);
 110         type = array_index_nospec(type, ARRAY_SIZE(bpf_map_types));
 111         ops = bpf_map_types[type];
 112         if (!ops)
 113                 return ERR_PTR(-EINVAL);
 114
 115         if (ops->map_alloc_check) {
 116                 err = ops->map_alloc_check(attr);
 117                 if (err)
 118                         return ERR_PTR(err);
 119         }
 120         if (attr->map_ifindex)
 121                 ops = &bpf_map_offload_ops;
 122         map = ops->map_alloc(attr);
 123         if (IS_ERR(map))
 124                 return map;
 125         map->ops = ops;
 126         map->map_type = type;
 127         return map;
 128 }
 129
 130 static u32 bpf_map_value_size(struct bpf_map *map)
 131 {
 132         if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
 133             map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||
 134             map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY ||
 135             map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE)
 136                 return round_up(map->value_size, 8) * num_possible_cpus();
 137         else if (IS_FD_MAP(map))
 138                 return sizeof(u32);
 139         else
 140                 return  map->value_size;
 141 }
 142
 143 static void maybe_wait_bpf_programs(struct bpf_map *map)
 144 {
 145         /* Wait for any running BPF programs to complete so that
 146          * userspace, when we return to it, knows that all programs
 147          * that could be running use the new map value.
 148          */
 149         if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS ||
 150             map->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS)
 151                 synchronize_rcu();
 152 }
 153
 154 static int bpf_map_update_value(struct bpf_map *map, struct fd f, void *key,
 155                                 void *value, __u64 flags)
 156 {
 157         int err;
 158
 159         /* Need to create a kthread, thus must support schedule */
 160         if (bpf_map_is_dev_bound(map)) {
 161                 return bpf_map_offload_update_elem(map, key, value, flags);
 162         } else if (map->map_type == BPF_MAP_TYPE_CPUMAP ||
 163                    map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
 164                 return map->ops->map_update_elem(map, key, value, flags);
 165         } else if (map->map_type == BPF_MAP_TYPE_SOCKHASH ||
 166                    map->map_type == BPF_MAP_TYPE_SOCKMAP) {
 167                 return sock_map_update_elem_sys(map, key, value, flags);
 168         } else if (IS_FD_PROG_ARRAY(map)) {
 169                 return bpf_fd_array_map_update_elem(map, f.file, key, value,
 170                                                     flags);
 171         }
 172
 173         bpf_disable_instrumentation();
 174         if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
 175             map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
 176                 err = bpf_percpu_hash_update(map, key, value, flags);
 177         } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
 178                 err = bpf_percpu_array_update(map, key, value, flags);
 179         } else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) {
 180                 err = bpf_percpu_cgroup_storage_update(map, key, value,
 181                                                        flags);
 182         } else if (IS_FD_ARRAY(map)) {
 183                 rcu_read_lock();
 184                 err = bpf_fd_array_map_update_elem(map, f.file, key, value,
 185                                                    flags);
 186                 rcu_read_unlock();
 187         } else if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) {
 188                 rcu_read_lock();
 189                 err = bpf_fd_htab_map_update_elem(map, f.file, key, value,
 190                                                   flags);
 191                 rcu_read_unlock();
 192         } else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) {
 193                 /* rcu_read_lock() is not needed */
 194                 err = bpf_fd_reuseport_array_update_elem(map, key, value,
 195                                                          flags);
 196         } else if (map->map_type == BPF_MAP_TYPE_QUEUE ||
 197                    map->map_type == BPF_MAP_TYPE_STACK) {
 198                 err = map->ops->map_push_elem(map, value, flags);
 199         } else {
 200                 rcu_read_lock();
 201                 err = map->ops->map_update_elem(map, key, value, flags);
 202                 rcu_read_unlock();
 203         }
 204         bpf_enable_instrumentation();
 205         maybe_wait_bpf_programs(map);
 206
 207         return err;
 208 }
 209
 210 static int bpf_map_copy_value(struct bpf_map *map, void *key, void *value,
 211                               __u64 flags)
 212 {
 213         void *ptr;
 214         int err;
 215
 216         if (bpf_map_is_dev_bound(map))
 217                 return bpf_map_offload_lookup_elem(map, key, value);
 218
 219         bpf_disable_instrumentation();
 220         if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
 221             map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
 222                 err = bpf_percpu_hash_copy(map, key, value);
 223         } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
 224                 err = bpf_percpu_array_copy(map, key, value);
 225         } else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) {
 226                 err = bpf_percpu_cgroup_storage_copy(map, key, value);
 227         } else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) {
 228                 err = bpf_stackmap_copy(map, key, value);
 229         } else if (IS_FD_ARRAY(map) || IS_FD_PROG_ARRAY(map)) {
 230                 err = bpf_fd_array_map_lookup_elem(map, key, value);
 231         } else if (IS_FD_HASH(map)) {
 232                 err = bpf_fd_htab_map_lookup_elem(map, key, value);
 233         } else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) {
 234                 err = bpf_fd_reuseport_array_lookup_elem(map, key, value);
 235         } else if (map->map_type == BPF_MAP_TYPE_QUEUE ||
 236                    map->map_type == BPF_MAP_TYPE_STACK) {
 237                 err = map->ops->map_peek_elem(map, value);
 238         } else if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
 239                 /* struct_ops map requires directly updating "value" */
 240                 err = bpf_struct_ops_map_sys_lookup_elem(map, key, value);
 241         } else {
 242                 rcu_read_lock();
 243                 if (map->ops->map_lookup_elem_sys_only)
 244                         ptr = map->ops->map_lookup_elem_sys_only(map, key);
 245                 else
 246                         ptr = map->ops->map_lookup_elem(map, key);
 247                 if (IS_ERR(ptr)) {
 248                         err = PTR_ERR(ptr);
 249                 } else if (!ptr) {
 250                         err = -ENOENT;
 251                 } else {
 252                         err = 0;
 253                         if (flags & BPF_F_LOCK)
 254                                 /* lock 'ptr' and copy everything but lock */
 255                                 copy_map_value_locked(map, value, ptr, true);
 256                         else
 257                                 copy_map_value(map, value, ptr);
 258                         /* mask lock, since value wasn't zero inited */
 259                         check_and_init_map_lock(map, value);
 260                 }
 261                 rcu_read_unlock();
 262         }
 263
 264         bpf_enable_instrumentation();
 265         maybe_wait_bpf_programs(map);
 266
 267         return err;
 268 }
 269
 270 static void *__bpf_map_area_alloc(u64 size, int numa_node, bool mmapable)
 271 {
 272         /* We really just want to fail instead of triggering OOM killer
 273          * under memory pressure, therefore we set __GFP_NORETRY to kmalloc,
 274          * which is used for lower order allocation requests.
 275          *
 276          * It has been observed that higher order allocation requests done by
 277          * vmalloc with __GFP_NORETRY being set might fail due to not trying
 278          * to reclaim memory from the page cache, thus we set
 279          * __GFP_RETRY_MAYFAIL to avoid such situations.
 280          */
 281
 282         const gfp_t gfp = __GFP_NOWARN | __GFP_ZERO;
 283         unsigned int flags = 0;
 284         unsigned long align = 1;
 285         void *area;
 286
 287         if (size >= SIZE_MAX)
 288                 return NULL;
 289
 290         /* kmalloc()'ed memory can't be mmap()'ed */
 291         if (mmapable) {
 292                 BUG_ON(!PAGE_ALIGNED(size));
 293                 align = SHMLBA;
 294                 flags = VM_USERMAP;
 295         } else if (size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) {
 296                 area = kmalloc_node(size, gfp | GFP_USER | __GFP_NORETRY,
 297                                     numa_node);
 298                 if (area != NULL)
 299                         return area;
 300         }
 301
 302         return __vmalloc_node_range(size, align, VMALLOC_START, VMALLOC_END,
 303                         gfp | GFP_KERNEL | __GFP_RETRY_MAYFAIL, PAGE_KERNEL,
 304                         flags, numa_node, __builtin_return_address(0));
 305 }
 306
 307 void *bpf_map_area_alloc(u64 size, int numa_node)
 308 {
 309         return __bpf_map_area_alloc(size, numa_node, false);
 310 }
 311
 312 void *bpf_map_area_mmapable_alloc(u64 size, int numa_node)
 313 {
 314         return __bpf_map_area_alloc(size, numa_node, true);
 315 }
 316
 317 void bpf_map_area_free(void *area)
 318 {
 319         kvfree(area);
 320 }
 321
 322 static u32 bpf_map_flags_retain_permanent(u32 flags)
 323 {
 324         /* Some map creation flags are not tied to the map object but
 325          * rather to the map fd instead, so they have no meaning upon
 326          * map object inspection since multiple file descriptors with
 327          * different (access) properties can exist here. Thus, given
 328          * this has zero meaning for the map itself, lets clear these
 329          * from here.
 330          */
 331         return flags & ~(BPF_F_RDONLY | BPF_F_WRONLY);
 332 }
 333
 334 void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr)
 335 {
 336         map->map_type = attr->map_type;
 337         map->key_size = attr->key_size;
 338         map->value_size = attr->value_size;
 339         map->max_entries = attr->max_entries;
 340         map->map_flags = bpf_map_flags_retain_permanent(attr->map_flags);
 341         map->numa_node = bpf_map_attr_numa_node(attr);
 342 }
 343
 344 static int bpf_charge_memlock(struct user_struct *user, u32 pages)
 345 {
 346         unsigned long memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
 347
 348         if (atomic_long_add_return(pages, &user->locked_vm) > memlock_limit) {
 349                 atomic_long_sub(pages, &user->locked_vm);
 350                 return -EPERM;
 351         }
 352         return 0;
 353 }
 354
 355 static void bpf_uncharge_memlock(struct user_struct *user, u32 pages)
 356 {
 357         if (user)
 358                 atomic_long_sub(pages, &user->locked_vm);
 359 }
 360
 361 int bpf_map_charge_init(struct bpf_map_memory *mem, u64 size)
 362 {
 363         u32 pages = round_up(size, PAGE_SIZE) >> PAGE_SHIFT;
 364         struct user_struct *user;
 365         int ret;
 366
 367         if (size >= U32_MAX - PAGE_SIZE)
 368                 return -E2BIG;
 369
 370         user = get_current_user();
 371         ret = bpf_charge_memlock(user, pages);
 372         if (ret) {
 373                 free_uid(user);
 374                 return ret;
 375         }
 376
 377         mem->pages = pages;
 378         mem->user = user;
 379
 380         return 0;
 381 }
 382
 383 void bpf_map_charge_finish(struct bpf_map_memory *mem)
 384 {
 385         bpf_uncharge_memlock(mem->user, mem->pages);
 386         free_uid(mem->user);
 387 }
 388
 389 void bpf_map_charge_move(struct bpf_map_memory *dst,
 390                          struct bpf_map_memory *src)
 391 {
 392         *dst = *src;
 393
 394         /* Make sure src will not be used for the redundant uncharging. */
 395         memset(src, 0, sizeof(struct bpf_map_memory));
 396 }
 397
 398 int bpf_map_charge_memlock(struct bpf_map *map, u32 pages)
 399 {
 400         int ret;
 401
 402         ret = bpf_charge_memlock(map->memory.user, pages);
 403         if (ret)
 404                 return ret;
 405         map->memory.pages += pages;
 406         return ret;
 407 }
 408
 409 void bpf_map_uncharge_memlock(struct bpf_map *map, u32 pages)
 410 {
 411         bpf_uncharge_memlock(map->memory.user, pages);
 412         map->memory.pages -= pages;
 413 }
 414
 415 static int bpf_map_alloc_id(struct bpf_map *map)
 416 {
 417         int id;
 418
 419         idr_preload(GFP_KERNEL);
 420         spin_lock_bh(&map_idr_lock);
 421         id = idr_alloc_cyclic(&map_idr, map, 1, INT_MAX, GFP_ATOMIC);
 422         if (id > 0)
 423                 map->id = id;
 424         spin_unlock_bh(&map_idr_lock);
 425         idr_preload_end();
 426
 427         if (WARN_ON_ONCE(!id))
 428                 return -ENOSPC;
 429
 430         return id > 0 ? 0 : id;
 431 }
 432
 433 void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock)
 434 {
 435         unsigned long flags;
 436
 437         /* Offloaded maps are removed from the IDR store when their device
 438          * disappears - even if someone holds an fd to them they are unusable,
 439          * the memory is gone, all ops will fail; they are simply waiting for
 440          * refcnt to drop to be freed.
 441          */
 442         if (!map->id)
 443                 return;
 444
 445         if (do_idr_lock)
 446                 spin_lock_irqsave(&map_idr_lock, flags);
 447         else
 448                 __acquire(&map_idr_lock);
 449
 450         idr_remove(&map_idr, map->id);
 451         map->id = 0;
 452
 453         if (do_idr_lock)
 454                 spin_unlock_irqrestore(&map_idr_lock, flags);
 455         else
 456                 __release(&map_idr_lock);
 457 }
 458
 459 /* called from workqueue */
 460 static void bpf_map_free_deferred(struct work_struct *work)
 461 {
 462         struct bpf_map *map = container_of(work, struct bpf_map, work);
 463         struct bpf_map_memory mem;
 464
 465         bpf_map_charge_move(&mem, &map->memory);
 466         security_bpf_map_free(map);
 467         /* implementation dependent freeing */
 468         map->ops->map_free(map);
 469         bpf_map_charge_finish(&mem);
 470 }
 471
 472 static void bpf_map_put_uref(struct bpf_map *map)
 473 {
 474         if (atomic64_dec_and_test(&map->usercnt)) {
 475                 if (map->ops->map_release_uref)
 476                         map->ops->map_release_uref(map);
 477         }
 478 }
 479
 480 /* decrement map refcnt and schedule it for freeing via workqueue
 481  * (unrelying map implementation ops->map_free() might sleep)
 482  */
 483 static void __bpf_map_put(struct bpf_map *map, bool do_idr_lock)
 484 {
 485         if (atomic64_dec_and_test(&map->refcnt)) {
 486                 /* bpf_map_free_id() must be called first */
 487                 bpf_map_free_id(map, do_idr_lock);
 488                 btf_put(map->btf);
 489                 INIT_WORK(&map->work, bpf_map_free_deferred);
 490                 schedule_work(&map->work);
 491         }
 492 }
 493
 494 void bpf_map_put(struct bpf_map *map)
 495 {
 496         __bpf_map_put(map, true);
 497 }
 498 EXPORT_SYMBOL_GPL(bpf_map_put);
 499
 500 void bpf_map_put_with_uref(struct bpf_map *map)
 501 {
 502         bpf_map_put_uref(map);
 503         bpf_map_put(map);
 504 }
 505
 506 static int bpf_map_release(struct inode *inode, struct file *filp)
 507 {
 508         struct bpf_map *map = filp->private_data;
 509
 510         if (map->ops->map_release)
 511                 map->ops->map_release(map, filp);
 512
 513         bpf_map_put_with_uref(map);
 514         return 0;
 515 }
 516
 517 static fmode_t map_get_sys_perms(struct bpf_map *map, struct fd f)
 518 {
 519         fmode_t mode = f.file->f_mode;
 520
 521         /* Our file permissions may have been overridden by global
 522          * map permissions facing syscall side.
 523          */
 524         if (READ_ONCE(map->frozen))
 525                 mode &= ~FMODE_CAN_WRITE;
 526         return mode;
 527 }
 528
 529 #ifdef CONFIG_PROC_FS
 530 static void bpf_map_show_fdinfo(struct seq_file *m, struct file *filp)
 531 {
 532         const struct bpf_map *map = filp->private_data;
 533         const struct bpf_array *array;
 534         u32 type = 0, jited = 0;
 535
 536         if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY) {
 537                 array = container_of(map, struct bpf_array, map);
 538                 type  = array->aux->type;
 539                 jited = array->aux->jited;
 540         }
 541
 542         seq_printf(m,
 543                    "map_type:\t%u\n"
 544                    "key_size:\t%u\n"
 545                    "value_size:\t%u\n"
 546                    "max_entries:\t%u\n"
 547                    "map_flags:\t%#x\n"
 548                    "memlock:\t%llu\n"
 549                    "map_id:\t%u\n"
 550                    "frozen:\t%u\n",
 551                    map->map_type,
 552                    map->key_size,
 553                    map->value_size,
 554                    map->max_entries,
 555                    map->map_flags,
 556                    map->memory.pages * 1ULL << PAGE_SHIFT,
 557                    map->id,
 558                    READ_ONCE(map->frozen));
 559         if (type) {
 560                 seq_printf(m, "owner_prog_type:\t%u\n", type);
 561                 seq_printf(m, "owner_jited:\t%u\n", jited);
 562         }
 563 }
 564 #endif
 565
 566 static ssize_t bpf_dummy_read(struct file *filp, char __user *buf, size_t siz,
 567                               loff_t *ppos)
 568 {
 569         /* We need this handler such that alloc_file() enables
 570          * f_mode with FMODE_CAN_READ.
 571          */
 572         return -EINVAL;
 573 }
 574
 575 static ssize_t bpf_dummy_write(struct file *filp, const char __user *buf,
 576                                size_t siz, loff_t *ppos)
 577 {
 578         /* We need this handler such that alloc_file() enables
 579          * f_mode with FMODE_CAN_WRITE.
 580          */
 581         return -EINVAL;
 582 }
 583
 584 /* called for any extra memory-mapped regions (except initial) */
 585 static void bpf_map_mmap_open(struct vm_area_struct *vma)
 586 {
 587         struct bpf_map *map = vma->vm_file->private_data;
 588
 589         if (vma->vm_flags & VM_MAYWRITE) {
 590                 mutex_lock(&map->freeze_mutex);
 591                 map->writecnt++;
 592                 mutex_unlock(&map->freeze_mutex);
 593         }
 594 }
 595
 596 /* called for all unmapped memory region (including initial) */
 597 static void bpf_map_mmap_close(struct vm_area_struct *vma)
 598 {
 599         struct bpf_map *map = vma->vm_file->private_data;
 600
 601         if (vma->vm_flags & VM_MAYWRITE) {
 602                 mutex_lock(&map->freeze_mutex);
 603                 map->writecnt--;
 604                 mutex_unlock(&map->freeze_mutex);
 605         }
 606 }
 607
 608 static const struct vm_operations_struct bpf_map_default_vmops = {
 609         .open           = bpf_map_mmap_open,
 610         .close          = bpf_map_mmap_close,
 611 };
 612
 613 static int bpf_map_mmap(struct file *filp, struct vm_area_struct *vma)
 614 {
 615         struct bpf_map *map = filp->private_data;
 616         int err;
 617
 618         if (!map->ops->map_mmap || map_value_has_spin_lock(map))
 619                 return -ENOTSUPP;
 620
 621         if (!(vma->vm_flags & VM_SHARED))
 622                 return -EINVAL;
 623
 624         mutex_lock(&map->freeze_mutex);
 625
 626         if (vma->vm_flags & VM_WRITE) {
 627                 if (map->frozen) {
 628                         err = -EPERM;
 629                         goto out;
 630                 }
 631                 /* map is meant to be read-only, so do not allow mapping as
 632                  * writable, because it's possible to leak a writable page
 633                  * reference and allows user-space to still modify it after
 634                  * freezing, while verifier will assume contents do not change
 635                  */
 636                 if (map->map_flags & BPF_F_RDONLY_PROG) {
 637                         err = -EACCES;
 638                         goto out;
 639                 }
 640         }
 641
 642         /* set default open/close callbacks */
 643         vma->vm_ops = &bpf_map_default_vmops;
 644         vma->vm_private_data = map;
 645         vma->vm_flags &= ~VM_MAYEXEC;
 646         if (!(vma->vm_flags & VM_WRITE))
 647                 /* disallow re-mapping with PROT_WRITE */
 648                 vma->vm_flags &= ~VM_MAYWRITE;
 649
 650         err = map->ops->map_mmap(map, vma);
 651         if (err)
 652                 goto out;
 653
 654         if (vma->vm_flags & VM_MAYWRITE)
 655                 map->writecnt++;
 656 out:
 657         mutex_unlock(&map->freeze_mutex);
 658         return err;
 659 }
 660
 661 static __poll_t bpf_map_poll(struct file *filp, struct poll_table_struct *pts)
 662 {
 663         struct bpf_map *map = filp->private_data;
 664
 665         if (map->ops->map_poll)
 666                 return map->ops->map_poll(map, filp, pts);
 667
 668         return EPOLLERR;
 669 }
 670
 671 const struct file_operations bpf_map_fops = {
 672 #ifdef CONFIG_PROC_FS
 673         .show_fdinfo    = bpf_map_show_fdinfo,
 674 #endif
 675         .release        = bpf_map_release,
 676         .read           = bpf_dummy_read,
 677         .write          = bpf_dummy_write,
 678         .mmap           = bpf_map_mmap,
 679         .poll           = bpf_map_poll,
 680 };
 681
 682 int bpf_map_new_fd(struct bpf_map *map, int flags)
 683 {
 684         int ret;
 685
 686         ret = security_bpf_map(map, OPEN_FMODE(flags));
 687         if (ret < 0)
 688                 return ret;
 689
 690         return anon_inode_getfd("bpf-map", &bpf_map_fops, map,
 691                                 flags | O_CLOEXEC);
 692 }
 693
 694 int bpf_get_file_flag(int flags)
 695 {
 696         if ((flags & BPF_F_RDONLY) && (flags & BPF_F_WRONLY))
 697                 return -EINVAL;
 698         if (flags & BPF_F_RDONLY)
 699                 return O_RDONLY;
 700         if (flags & BPF_F_WRONLY)
 701                 return O_WRONLY;
 702         return O_RDWR;
 703 }
 704
 705 /* helper macro to check that unused fields 'union bpf_attr' are zero */
 706 #define CHECK_ATTR(CMD) \
 707         memchr_inv((void *) &attr->CMD##_LAST_FIELD + \
 708                    sizeof(attr->CMD##_LAST_FIELD), 0, \
 709                    sizeof(*attr) - \
 710                    offsetof(union bpf_attr, CMD##_LAST_FIELD) - \
 711                    sizeof(attr->CMD##_LAST_FIELD)) != NULL
 712
 713 /* dst and src must have at least "size" number of bytes.
 714  * Return strlen on success and < 0 on error.
 715  */
 716 int bpf_obj_name_cpy(char *dst, const char *src, unsigned int size)
 717 {
 718         const char *end = src + size;
 719         const char *orig_src = src;
 720
 721         memset(dst, 0, size);
 722         /* Copy all isalnum(), '_' and '.' chars. */
 723         while (src < end && *src) {
 724                 if (!isalnum(*src) &&
 725                     *src != '_' && *src != '.')
 726                         return -EINVAL;
 727                 *dst++ = *src++;
 728         }
 729
 730         /* No '\0' found in "size" number of bytes */
 731         if (src == end)
 732                 return -EINVAL;
 733
 734         return src - orig_src;
 735 }
 736
 737 int map_check_no_btf(const struct bpf_map *map,
 738                      const struct btf *btf,
 739                      const struct btf_type *key_type,
 740                      const struct btf_type *value_type)
 741 {
 742         return -ENOTSUPP;
 743 }
 744
 745 static int map_check_btf(struct bpf_map *map, const struct btf *btf,
 746                          u32 btf_key_id, u32 btf_value_id)
 747 {
 748         const struct btf_type *key_type, *value_type;
 749         u32 key_size, value_size;
 750         int ret = 0;
 751
 752         /* Some maps allow key to be unspecified. */
 753         if (btf_key_id) {
 754                 key_type = btf_type_id_size(btf, &btf_key_id, &key_size);
 755                 if (!key_type || key_size != map->key_size)
 756                         return -EINVAL;
 757         } else {
 758                 key_type = btf_type_by_id(btf, 0);
 759                 if (!map->ops->map_check_btf)
 760                         return -EINVAL;
 761         }
 762
 763         value_type = btf_type_id_size(btf, &btf_value_id, &value_size);
 764         if (!value_type || value_size != map->value_size)
 765                 return -EINVAL;
 766
 767         map->spin_lock_off = btf_find_spin_lock(btf, value_type);
 768
 769         if (map_value_has_spin_lock(map)) {
 770                 if (map->map_flags & BPF_F_RDONLY_PROG)
 771                         return -EACCES;
 772                 if (map->map_type != BPF_MAP_TYPE_HASH &&
 773                     map->map_type != BPF_MAP_TYPE_ARRAY &&
 774                     map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE &&
 775                     map->map_type != BPF_MAP_TYPE_SK_STORAGE &&
 776                     map->map_type != BPF_MAP_TYPE_INODE_STORAGE)
 777                         return -ENOTSUPP;
 778                 if (map->spin_lock_off + sizeof(struct bpf_spin_lock) >
 779                     map->value_size) {
 780                         WARN_ONCE(1,
 781                                   "verifier bug spin_lock_off %d value_size %d\n",
 782                                   map->spin_lock_off, map->value_size);
 783                         return -EFAULT;
 784                 }
 785         }
 786
 787         if (map->ops->map_check_btf)
 788                 ret = map->ops->map_check_btf(map, btf, key_type, value_type);
 789
 790         return ret;
 791 }
 792
 793 #define BPF_MAP_CREATE_LAST_FIELD btf_vmlinux_value_type_id
 794 /* called via syscall */
 795 static int map_create(union bpf_attr *attr)
 796 {
 797         int numa_node = bpf_map_attr_numa_node(attr);
 798         struct bpf_map_memory mem;
 799         struct bpf_map *map;
 800         int f_flags;
 801         int err;
 802
 803         err = CHECK_ATTR(BPF_MAP_CREATE);
 804         if (err)
 805                 return -EINVAL;
 806
 807         if (attr->btf_vmlinux_value_type_id) {
 808                 if (attr->map_type != BPF_MAP_TYPE_STRUCT_OPS ||
 809                     attr->btf_key_type_id || attr->btf_value_type_id)
 810                         return -EINVAL;
 811         } else if (attr->btf_key_type_id && !attr->btf_value_type_id) {
 812                 return -EINVAL;
 813         }
 814
 815         f_flags = bpf_get_file_flag(attr->map_flags);
 816         if (f_flags < 0)
 817                 return f_flags;
 818
 819         if (numa_node != NUMA_NO_NODE &&
 820             ((unsigned int)numa_node >= nr_node_ids ||
 821              !node_online(numa_node)))
 822                 return -EINVAL;
 823
 824         /* find map type and init map: hashtable vs rbtree vs bloom vs ... */
 825         map = find_and_alloc_map(attr);
 826         if (IS_ERR(map))
 827                 return PTR_ERR(map);
 828
 829         err = bpf_obj_name_cpy(map->name, attr->map_name,
 830                                sizeof(attr->map_name));
 831         if (err < 0)
 832                 goto free_map;
 833
 834         atomic64_set(&map->refcnt, 1);
 835         atomic64_set(&map->usercnt, 1);
 836         mutex_init(&map->freeze_mutex);
 837
 838         map->spin_lock_off = -EINVAL;
 839         if (attr->btf_key_type_id || attr->btf_value_type_id ||
 840             /* Even the map's value is a kernel's struct,
 841              * the bpf_prog.o must have BTF to begin with
 842              * to figure out the corresponding kernel's
 843              * counter part.  Thus, attr->btf_fd has
 844              * to be valid also.
 845              */
 846             attr->btf_vmlinux_value_type_id) {
 847                 struct btf *btf;
 848
 849                 btf = btf_get_by_fd(attr->btf_fd);
 850                 if (IS_ERR(btf)) {
 851                         err = PTR_ERR(btf);
 852                         goto free_map;
 853                 }
 854                 map->btf = btf;
 855
 856                 if (attr->btf_value_type_id) {
 857                         err = map_check_btf(map, btf, attr->btf_key_type_id,
 858                                             attr->btf_value_type_id);
 859                         if (err)
 860                                 goto free_map;
 861                 }
 862
 863                 map->btf_key_type_id = attr->btf_key_type_id;
 864                 map->btf_value_type_id = attr->btf_value_type_id;
 865                 map->btf_vmlinux_value_type_id =
 866                         attr->btf_vmlinux_value_type_id;
 867         }
 868
 869         err = security_bpf_map_alloc(map);
 870         if (err)
 871                 goto free_map;
 872
 873         err = bpf_map_alloc_id(map);
 874         if (err)
 875                 goto free_map_sec;
 876
 877         err = bpf_map_new_fd(map, f_flags);
 878         if (err < 0) {
 879                 /* failed to allocate fd.
 880                  * bpf_map_put_with_uref() is needed because the above
 881                  * bpf_map_alloc_id() has published the map
 882                  * to the userspace and the userspace may
 883                  * have refcnt-ed it through BPF_MAP_GET_FD_BY_ID.
 884                  */
 885                 bpf_map_put_with_uref(map);
 886                 return err;
 887         }
 888
 889         return err;
 890
 891 free_map_sec:
 892         security_bpf_map_free(map);
 893 free_map:
 894         btf_put(map->btf);
 895         bpf_map_charge_move(&mem, &map->memory);
 896         map->ops->map_free(map);
 897         bpf_map_charge_finish(&mem);
 898         return err;
 899 }
 900
 901 /* if error is returned, fd is released.
 902  * On success caller should complete fd access with matching fdput()
 903  */
 904 struct bpf_map *__bpf_map_get(struct fd f)
 905 {
 906         if (!f.file)
 907                 return ERR_PTR(-EBADF);
 908         if (f.file->f_op != &bpf_map_fops) {
 909                 fdput(f);
 910                 return ERR_PTR(-EINVAL);
 911         }
 912
 913         return f.file->private_data;
 914 }
 915
 916 void bpf_map_inc(struct bpf_map *map)
 917 {
 918         atomic64_inc(&map->refcnt);
 919 }
 920 EXPORT_SYMBOL_GPL(bpf_map_inc);
 921
 922 void bpf_map_inc_with_uref(struct bpf_map *map)
 923 {
 924         atomic64_inc(&map->refcnt);
 925         atomic64_inc(&map->usercnt);
 926 }
 927 EXPORT_SYMBOL_GPL(bpf_map_inc_with_uref);
 928
 929 struct bpf_map *bpf_map_get(u32 ufd)
 930 {
 931         struct fd f = fdget(ufd);
 932         struct bpf_map *map;
 933
 934         map = __bpf_map_get(f);
 935         if (IS_ERR(map))
 936                 return map;
 937
 938         bpf_map_inc(map);
 939         fdput(f);
 940
 941         return map;
 942 }
 943
 944 struct bpf_map *bpf_map_get_with_uref(u32 ufd)
 945 {
 946         struct fd f = fdget(ufd);
 947         struct bpf_map *map;
 948
 949         map = __bpf_map_get(f);
 950         if (IS_ERR(map))
 951                 return map;
 952
 953         bpf_map_inc_with_uref(map);
 954         fdput(f);
 955
 956         return map;
 957 }
 958
 959 /* map_idr_lock should have been held */
 960 static struct bpf_map *__bpf_map_inc_not_zero(struct bpf_map *map, bool uref)
 961 {
 962         int refold;
 963
 964         refold = atomic64_fetch_add_unless(&map->refcnt, 1, 0);
 965         if (!refold)
 966                 return ERR_PTR(-ENOENT);
 967         if (uref)
 968                 atomic64_inc(&map->usercnt);
 969
 970         return map;
 971 }
 972
 973 struct bpf_map *bpf_map_inc_not_zero(struct bpf_map *map)
 974 {
 975         spin_lock_bh(&map_idr_lock);
 976         map = __bpf_map_inc_not_zero(map, false);
 977         spin_unlock_bh(&map_idr_lock);
 978
 979         return map;
 980 }
 981 EXPORT_SYMBOL_GPL(bpf_map_inc_not_zero);
 982
 983 int __weak bpf_stackmap_copy(struct bpf_map *map, void *key, void *value)
 984 {
 985         return -ENOTSUPP;
 986 }
 987
 988 static void *__bpf_copy_key(void __user *ukey, u64 key_size)
 989 {
 990         if (key_size)
 991                 return memdup_user(ukey, key_size);
 992
 993         if (ukey)
 994                 return ERR_PTR(-EINVAL);
 995
 996         return NULL;
 997 }
 998
 999 /* last field in 'union bpf_attr' used by this command */
1000 #define BPF_MAP_LOOKUP_ELEM_LAST_FIELD flags
1001
1002 static int map_lookup_elem(union bpf_attr *attr)
1003 {
1004         void __user *ukey = u64_to_user_ptr(attr->key);
1005         void __user *uvalue = u64_to_user_ptr(attr->value);
1006         int ufd = attr->map_fd;
1007         struct bpf_map *map;
1008         void *key, *value;
1009         u32 value_size;
1010         struct fd f;
1011         int err;
1012
1013         if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM))
1014                 return -EINVAL;
1015
1016         if (attr->flags & ~BPF_F_LOCK)
1017                 return -EINVAL;
1018
1019         f = fdget(ufd);
1020         map = __bpf_map_get(f);
1021         if (IS_ERR(map))
1022                 return PTR_ERR(map);
1023         if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ)) {
1024                 err = -EPERM;
1025                 goto err_put;
1026         }
1027
1028         if ((attr->flags & BPF_F_LOCK) &&
1029             !map_value_has_spin_lock(map)) {
1030                 err = -EINVAL;
1031                 goto err_put;
1032         }
1033
1034         key = __bpf_copy_key(ukey, map->key_size);
1035         if (IS_ERR(key)) {
1036                 err = PTR_ERR(key);
1037                 goto err_put;
1038         }
1039
1040         value_size = bpf_map_value_size(map);
1041
1042         err = -ENOMEM;
1043         value = kmalloc(value_size, GFP_USER | __GFP_NOWARN);
1044         if (!value)
1045                 goto free_key;
1046
1047         err = bpf_map_copy_value(map, key, value, attr->flags);
1048         if (err)
1049                 goto free_value;
1050
1051         err = -EFAULT;
1052         if (copy_to_user(uvalue, value, value_size) != 0)
1053                 goto free_value;
1054
1055         err = 0;
1056
1057 free_value:
1058         kfree(value);
1059 free_key:
1060         kfree(key);
1061 err_put:
1062         fdput(f);
1063         return err;
1064 }
1065
1066
1067 #define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags
1068
1069 static int map_update_elem(union bpf_attr *attr)
1070 {
1071         void __user *ukey = u64_to_user_ptr(attr->key);
1072         void __user *uvalue = u64_to_user_ptr(attr->value);
1073         int ufd = attr->map_fd;
1074         struct bpf_map *map;
1075         void *key, *value;
1076         u32 value_size;
1077         struct fd f;
1078         int err;
1079
1080         if (CHECK_ATTR(BPF_MAP_UPDATE_ELEM))
1081                 return -EINVAL;
1082
1083         f = fdget(ufd);
1084         map = __bpf_map_get(f);
1085         if (IS_ERR(map))
1086                 return PTR_ERR(map);
1087         if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
1088                 err = -EPERM;
1089                 goto err_put;
1090         }
1091
1092         if ((attr->flags & BPF_F_LOCK) &&
1093             !map_value_has_spin_lock(map)) {
1094                 err = -EINVAL;
1095                 goto err_put;
1096         }
1097
1098         key = __bpf_copy_key(ukey, map->key_size);
1099         if (IS_ERR(key)) {
1100                 err = PTR_ERR(key);
1101                 goto err_put;
1102         }
1103
1104         if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
1105             map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||
1106             map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY ||
1107             map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE)
1108                 value_size = round_up(map->value_size, 8) * num_possible_cpus();
1109         else
1110                 value_size = map->value_size;
1111
1112         err = -ENOMEM;
1113         value = kmalloc(value_size, GFP_USER | __GFP_NOWARN);
1114         if (!value)
1115                 goto free_key;
1116
1117         err = -EFAULT;
1118         if (copy_from_user(value, uvalue, value_size) != 0)
1119                 goto free_value;
1120
1121         err = bpf_map_update_value(map, f, key, value, attr->flags);
1122
1123 free_value:
1124         kfree(value);
1125 free_key:
1126         kfree(key);
1127 err_put:
1128         fdput(f);
1129         return err;
1130 }
1131
1132 #define BPF_MAP_DELETE_ELEM_LAST_FIELD key
1133
1134 static int map_delete_elem(union bpf_attr *attr)
1135 {
1136         void __user *ukey = u64_to_user_ptr(attr->key);
1137         int ufd = attr->map_fd;
1138         struct bpf_map *map;
1139         struct fd f;
1140         void *key;
1141         int err;
1142
1143         if (CHECK_ATTR(BPF_MAP_DELETE_ELEM))
1144                 return -EINVAL;
1145
1146         f = fdget(ufd);
1147         map = __bpf_map_get(f);
1148         if (IS_ERR(map))
1149                 return PTR_ERR(map);
1150         if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
1151                 err = -EPERM;
1152                 goto err_put;
1153         }
1154
1155         key = __bpf_copy_key(ukey, map->key_size);
1156         if (IS_ERR(key)) {
1157                 err = PTR_ERR(key);
1158                 goto err_put;
1159         }
1160
1161         if (bpf_map_is_dev_bound(map)) {
1162                 err = bpf_map_offload_delete_elem(map, key);
1163                 goto out;
1164         } else if (IS_FD_PROG_ARRAY(map) ||
1165                    map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
1166                 /* These maps require sleepable context */
1167                 err = map->ops->map_delete_elem(map, key);
1168                 goto out;
1169         }
1170
1171         bpf_disable_instrumentation();
1172         rcu_read_lock();
1173         err = map->ops->map_delete_elem(map, key);
1174         rcu_read_unlock();
1175         bpf_enable_instrumentation();
1176         maybe_wait_bpf_programs(map);
1177 out:
1178         kfree(key);
1179 err_put:
1180         fdput(f);
1181         return err;
1182 }
1183
1184 /* last field in 'union bpf_attr' used by this command */
1185 #define BPF_MAP_GET_NEXT_KEY_LAST_FIELD next_key
1186
1187 static int map_get_next_key(union bpf_attr *attr)
1188 {
1189         void __user *ukey = u64_to_user_ptr(attr->key);
1190         void __user *unext_key = u64_to_user_ptr(attr->next_key);
1191         int ufd = attr->map_fd;
1192         struct bpf_map *map;
1193         void *key, *next_key;
1194         struct fd f;
1195         int err;
1196
1197         if (CHECK_ATTR(BPF_MAP_GET_NEXT_KEY))
1198                 return -EINVAL;
1199
1200         f = fdget(ufd);
1201         map = __bpf_map_get(f);
1202         if (IS_ERR(map))
1203                 return PTR_ERR(map);
1204         if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ)) {
1205                 err = -EPERM;
1206                 goto err_put;
1207         }
1208
1209         if (ukey) {
1210                 key = __bpf_copy_key(ukey, map->key_size);
1211                 if (IS_ERR(key)) {
1212                         err = PTR_ERR(key);
1213                         goto err_put;
1214                 }
1215         } else {
1216                 key = NULL;
1217         }
1218
1219         err = -ENOMEM;
1220         next_key = kmalloc(map->key_size, GFP_USER);
1221         if (!next_key)
1222                 goto free_key;
1223
1224         if (bpf_map_is_dev_bound(map)) {
1225                 err = bpf_map_offload_get_next_key(map, key, next_key);
1226                 goto out;
1227         }
1228
1229         rcu_read_lock();
1230         err = map->ops->map_get_next_key(map, key, next_key);
1231         rcu_read_unlock();
1232 out:
1233         if (err)
1234                 goto free_next_key;
1235
1236         err = -EFAULT;
1237         if (copy_to_user(unext_key, next_key, map->key_size) != 0)
1238                 goto free_next_key;
1239
1240         err = 0;
1241
1242 free_next_key:
1243         kfree(next_key);
1244 free_key:
1245         kfree(key);
1246 err_put:
1247         fdput(f);
1248         return err;
1249 }
1250
1251 int generic_map_delete_batch(struct bpf_map *map,
1252                              const union bpf_attr *attr,
1253                              union bpf_attr __user *uattr)
1254 {
1255         void __user *keys = u64_to_user_ptr(attr->batch.keys);
1256         u32 cp, max_count;
1257         int err = 0;
1258         void *key;
1259
1260         if (attr->batch.elem_flags & ~BPF_F_LOCK)
1261                 return -EINVAL;
1262
1263         if ((attr->batch.elem_flags & BPF_F_LOCK) &&
1264             !map_value_has_spin_lock(map)) {
1265                 return -EINVAL;
1266         }
1267
1268         max_count = attr->batch.count;
1269         if (!max_count)
1270                 return 0;
1271
1272         key = kmalloc(map->key_size, GFP_USER | __GFP_NOWARN);
1273         if (!key)
1274                 return -ENOMEM;
1275
1276         for (cp = 0; cp < max_count; cp++) {
1277                 err = -EFAULT;
1278                 if (copy_from_user(key, keys + cp * map->key_size,
1279                                    map->key_size))
1280                         break;
1281
1282                 if (bpf_map_is_dev_bound(map)) {
1283                         err = bpf_map_offload_delete_elem(map, key);
1284                         break;
1285                 }
1286
1287                 bpf_disable_instrumentation();
1288                 rcu_read_lock();
1289                 err = map->ops->map_delete_elem(map, key);
1290                 rcu_read_unlock();
1291                 bpf_enable_instrumentation();
1292                 maybe_wait_bpf_programs(map);
1293                 if (err)
1294                         break;
1295         }
1296         if (copy_to_user(&uattr->batch.count, &cp, sizeof(cp)))
1297                 err = -EFAULT;
1298
1299         kfree(key);
1300         return err;
1301 }
1302
1303 int generic_map_update_batch(struct bpf_map *map,
1304                              const union bpf_attr *attr,
1305                              union bpf_attr __user *uattr)
1306 {
1307         void __user *values = u64_to_user_ptr(attr->batch.values);
1308         void __user *keys = u64_to_user_ptr(attr->batch.keys);
1309         u32 value_size, cp, max_count;
1310         int ufd = attr->map_fd;
1311         void *key, *value;
1312         struct fd f;
1313         int err = 0;
1314
1315         f = fdget(ufd);
1316         if (attr->batch.elem_flags & ~BPF_F_LOCK)
1317                 return -EINVAL;
1318
1319         if ((attr->batch.elem_flags & BPF_F_LOCK) &&
1320             !map_value_has_spin_lock(map)) {
1321                 return -EINVAL;
1322         }
1323
1324         value_size = bpf_map_value_size(map);
1325
1326         max_count = attr->batch.count;
1327         if (!max_count)
1328                 return 0;
1329
1330         key = kmalloc(map->key_size, GFP_USER | __GFP_NOWARN);
1331         if (!key)
1332                 return -ENOMEM;
1333
1334         value = kmalloc(value_size, GFP_USER | __GFP_NOWARN);
1335         if (!value) {
1336                 kfree(key);
1337                 return -ENOMEM;
1338         }
1339
1340         for (cp = 0; cp < max_count; cp++) {
1341                 err = -EFAULT;
1342                 if (copy_from_user(key, keys + cp * map->key_size,
1343                     map->key_size) ||
1344                     copy_from_user(value, values + cp * value_size, value_size))
1345                         break;
1346
1347                 err = bpf_map_update_value(map, f, key, value,
1348                                            attr->batch.elem_flags);
1349
1350                 if (err)
1351                         break;
1352         }
1353
1354         if (copy_to_user(&uattr->batch.count, &cp, sizeof(cp)))
1355                 err = -EFAULT;
1356
1357         kfree(value);
1358         kfree(key);
1359         return err;
1360 }
1361
1362 #define MAP_LOOKUP_RETRIES 3
1363
1364 int generic_map_lookup_batch(struct bpf_map *map,
1365                                     const union bpf_attr *attr,
1366                                     union bpf_attr __user *uattr)
1367 {
1368         void __user *uobatch = u64_to_user_ptr(attr->batch.out_batch);
1369         void __user *ubatch = u64_to_user_ptr(attr->batch.in_batch);
1370         void __user *values = u64_to_user_ptr(attr->batch.values);
1371         void __user *keys = u64_to_user_ptr(attr->batch.keys);
1372         void *buf, *buf_prevkey, *prev_key, *key, *value;
1373         int err, retry = MAP_LOOKUP_RETRIES;
1374         u32 value_size, cp, max_count;
1375
1376         if (attr->batch.elem_flags & ~BPF_F_LOCK)
1377                 return -EINVAL;
1378
1379         if ((attr->batch.elem_flags & BPF_F_LOCK) &&
1380             !map_value_has_spin_lock(map))
1381                 return -EINVAL;
1382
1383         value_size = bpf_map_value_size(map);
1384
1385         max_count = attr->batch.count;
1386         if (!max_count)
1387                 return 0;
1388
1389         if (put_user(0, &uattr->batch.count))
1390                 return -EFAULT;
1391
1392         buf_prevkey = kmalloc(map->key_size, GFP_USER | __GFP_NOWARN);
1393         if (!buf_prevkey)
1394                 return -ENOMEM;
1395
1396         buf = kmalloc(map->key_size + value_size, GFP_USER | __GFP_NOWARN);
1397         if (!buf) {
1398                 kfree(buf_prevkey);
1399                 return -ENOMEM;
1400         }
1401
1402         err = -EFAULT;
1403         prev_key = NULL;
1404         if (ubatch && copy_from_user(buf_prevkey, ubatch, map->key_size))
1405                 goto free_buf;
1406         key = buf;
1407         value = key + map->key_size;
1408         if (ubatch)
1409                 prev_key = buf_prevkey;
1410
1411         for (cp = 0; cp < max_count;) {
1412                 rcu_read_lock();
1413                 err = map->ops->map_get_next_key(map, prev_key, key);
1414                 rcu_read_unlock();
1415                 if (err)
1416                         break;
1417                 err = bpf_map_copy_value(map, key, value,
1418                                          attr->batch.elem_flags);
1419
1420                 if (err == -ENOENT) {
1421                         if (retry) {
1422                                 retry--;
1423                                 continue;
1424                         }
1425                         err = -EINTR;
1426                         break;
1427                 }
1428
1429                 if (err)
1430                         goto free_buf;
1431
1432                 if (copy_to_user(keys + cp * map->key_size, key,
1433                                  map->key_size)) {
1434                         err = -EFAULT;
1435                         goto free_buf;
1436                 }
1437                 if (copy_to_user(values + cp * value_size, value, value_size)) {
1438                         err = -EFAULT;
1439                         goto free_buf;
1440                 }
1441
1442                 if (!prev_key)
1443                         prev_key = buf_prevkey;
1444
1445                 swap(prev_key, key);
1446                 retry = MAP_LOOKUP_RETRIES;
1447                 cp++;
1448         }
1449
1450         if (err == -EFAULT)
1451                 goto free_buf;
1452
1453         if ((copy_to_user(&uattr->batch.count, &cp, sizeof(cp)) ||
1454                     (cp && copy_to_user(uobatch, prev_key, map->key_size))))
1455                 err = -EFAULT;
1456
1457 free_buf:
1458         kfree(buf_prevkey);
1459         kfree(buf);
1460         return err;
1461 }
1462
1463 #define BPF_MAP_LOOKUP_AND_DELETE_ELEM_LAST_FIELD value
1464
1465 static int map_lookup_and_delete_elem(union bpf_attr *attr)
1466 {
1467         void __user *ukey = u64_to_user_ptr(attr->key);
1468         void __user *uvalue = u64_to_user_ptr(attr->value);
1469         int ufd = attr->map_fd;
1470         struct bpf_map *map;
1471         void *key, *value;
1472         u32 value_size;
1473         struct fd f;
1474         int err;
1475
1476         if (CHECK_ATTR(BPF_MAP_LOOKUP_AND_DELETE_ELEM))
1477                 return -EINVAL;
1478
1479         f = fdget(ufd);
1480         map = __bpf_map_get(f);
1481         if (IS_ERR(map))
1482                 return PTR_ERR(map);
1483         if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ) ||
1484             !(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
1485                 err = -EPERM;
1486                 goto err_put;
1487         }
1488
1489         key = __bpf_copy_key(ukey, map->key_size);
1490         if (IS_ERR(key)) {
1491                 err = PTR_ERR(key);
1492                 goto err_put;
1493         }
1494
1495         value_size = map->value_size;
1496
1497         err = -ENOMEM;
1498         value = kmalloc(value_size, GFP_USER | __GFP_NOWARN);
1499         if (!value)
1500                 goto free_key;
1501
1502         if (map->map_type == BPF_MAP_TYPE_QUEUE ||
1503             map->map_type == BPF_MAP_TYPE_STACK) {
1504                 err = map->ops->map_pop_elem(map, value);
1505         } else {
1506                 err = -ENOTSUPP;
1507         }
1508
1509         if (err)
1510                 goto free_value;
1511
1512         if (copy_to_user(uvalue, value, value_size) != 0) {
1513                 err = -EFAULT;
1514                 goto free_value;
1515         }
1516
1517         err = 0;
1518
1519 free_value:
1520         kfree(value);
1521 free_key:
1522         kfree(key);
1523 err_put:
1524         fdput(f);
1525         return err;
1526 }
1527
1528 #define BPF_MAP_FREEZE_LAST_FIELD map_fd
1529
1530 static int map_freeze(const union bpf_attr *attr)
1531 {
1532         int err = 0, ufd = attr->map_fd;
1533         struct bpf_map *map;
1534         struct fd f;
1535
1536         if (CHECK_ATTR(BPF_MAP_FREEZE))
1537                 return -EINVAL;
1538
1539         f = fdget(ufd);
1540         map = __bpf_map_get(f);
1541         if (IS_ERR(map))
1542                 return PTR_ERR(map);
1543
1544         if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
1545                 fdput(f);
1546                 return -ENOTSUPP;
1547         }
1548
1549         mutex_lock(&map->freeze_mutex);
1550
1551         if (map->writecnt) {
1552                 err = -EBUSY;
1553                 goto err_put;
1554         }
1555         if (READ_ONCE(map->frozen)) {
1556                 err = -EBUSY;
1557                 goto err_put;
1558         }
1559         if (!bpf_capable()) {
1560                 err = -EPERM;
1561                 goto err_put;
1562         }
1563
1564         WRITE_ONCE(map->frozen, true);
1565 err_put:
1566         mutex_unlock(&map->freeze_mutex);
1567         fdput(f);
1568         return err;
1569 }
1570
1571 static const struct bpf_prog_ops * const bpf_prog_types[] = {
1572 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \
1573         [_id] = & _name ## _prog_ops,
1574 #define BPF_MAP_TYPE(_id, _ops)
1575 #define BPF_LINK_TYPE(_id, _name)
1576 #include <linux/bpf_types.h>
1577 #undef BPF_PROG_TYPE
1578 #undef BPF_MAP_TYPE
1579 #undef BPF_LINK_TYPE
1580 };
1581
1582 static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog)
1583 {
1584         const struct bpf_prog_ops *ops;
1585
1586         if (type >= ARRAY_SIZE(bpf_prog_types))
1587                 return -EINVAL;
1588         type = array_index_nospec(type, ARRAY_SIZE(bpf_prog_types));
1589         ops = bpf_prog_types[type];
1590         if (!ops)
1591                 return -EINVAL;
1592
1593         if (!bpf_prog_is_dev_bound(prog->aux))
1594                 prog->aux->ops = ops;
1595         else
1596                 prog->aux->ops = &bpf_offload_prog_ops;
1597         prog->type = type;
1598         return 0;
1599 }
1600
1601 enum bpf_audit {
1602         BPF_AUDIT_LOAD,
1603         BPF_AUDIT_UNLOAD,
1604         BPF_AUDIT_MAX,
1605 };
1606
1607 static const char * const bpf_audit_str[BPF_AUDIT_MAX] = {
1608         [BPF_AUDIT_LOAD]   = "LOAD",
1609         [BPF_AUDIT_UNLOAD] = "UNLOAD",
1610 };
1611
1612 static void bpf_audit_prog(const struct bpf_prog *prog, unsigned int op)
1613 {
1614         struct audit_context *ctx = NULL;
1615         struct audit_buffer *ab;
1616
1617         if (WARN_ON_ONCE(op >= BPF_AUDIT_MAX))
1618                 return;
1619         if (audit_enabled == AUDIT_OFF)
1620                 return;
1621         if (op == BPF_AUDIT_LOAD)
1622                 ctx = audit_context();
1623         ab = audit_log_start(ctx, GFP_ATOMIC, AUDIT_BPF);
1624         if (unlikely(!ab))
1625                 return;
1626         audit_log_format(ab, "prog-id=%u op=%s",
1627                          prog->aux->id, bpf_audit_str[op]);
1628         audit_log_end(ab);
1629 }
1630
1631 int __bpf_prog_charge(struct user_struct *user, u32 pages)
1632 {
1633         unsigned long memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
1634         unsigned long user_bufs;
1635
1636         if (user) {
1637                 user_bufs = atomic_long_add_return(pages, &user->locked_vm);
1638                 if (user_bufs > memlock_limit) {
1639                         atomic_long_sub(pages, &user->locked_vm);
1640                         return -EPERM;
1641                 }
1642         }
1643
1644         return 0;
1645 }
1646
1647 void __bpf_prog_uncharge(struct user_struct *user, u32 pages)
1648 {
1649         if (user)
1650                 atomic_long_sub(pages, &user->locked_vm);
1651 }
1652
1653 static int bpf_prog_charge_memlock(struct bpf_prog *prog)
1654 {
1655         struct user_struct *user = get_current_user();
1656         int ret;
1657
1658         ret = __bpf_prog_charge(user, prog->pages);
1659         if (ret) {
1660                 free_uid(user);
1661                 return ret;
1662         }
1663
1664         prog->aux->user = user;
1665         return 0;
1666 }
1667
1668 static void bpf_prog_uncharge_memlock(struct bpf_prog *prog)
1669 {
1670         struct user_struct *user = prog->aux->user;
1671
1672         __bpf_prog_uncharge(user, prog->pages);
1673         free_uid(user);
1674 }
1675
1676 static int bpf_prog_alloc_id(struct bpf_prog *prog)
1677 {
1678         int id;
1679
1680         idr_preload(GFP_KERNEL);
1681         spin_lock_bh(&prog_idr_lock);
1682         id = idr_alloc_cyclic(&prog_idr, prog, 1, INT_MAX, GFP_ATOMIC);
1683         if (id > 0)
1684                 prog->aux->id = id;
1685         spin_unlock_bh(&prog_idr_lock);
1686         idr_preload_end();
1687
1688         /* id is in [1, INT_MAX) */
1689         if (WARN_ON_ONCE(!id))
1690                 return -ENOSPC;
1691
1692         return id > 0 ? 0 : id;
1693 }
1694
1695 void bpf_prog_free_id(struct bpf_prog *prog, bool do_idr_lock)
1696 {
1697         /* cBPF to eBPF migrations are currently not in the idr store.
1698          * Offloaded programs are removed from the store when their device
1699          * disappears - even if someone grabs an fd to them they are unusable,
1700          * simply waiting for refcnt to drop to be freed.
1701          */
1702         if (!prog->aux->id)
1703                 return;
1704
1705         if (do_idr_lock)
1706                 spin_lock_bh(&prog_idr_lock);
1707         else
1708                 __acquire(&prog_idr_lock);
1709
1710         idr_remove(&prog_idr, prog->aux->id);
1711         prog->aux->id = 0;
1712
1713         if (do_idr_lock)
1714                 spin_unlock_bh(&prog_idr_lock);
1715         else
1716                 __release(&prog_idr_lock);
1717 }
1718
1719 static void __bpf_prog_put_rcu(struct rcu_head *rcu)
1720 {
1721         struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu);
1722
1723         kvfree(aux->func_info);
1724         kfree(aux->func_info_aux);
1725         bpf_prog_uncharge_memlock(aux->prog);
1726         security_bpf_prog_free(aux);
1727         bpf_prog_free(aux->prog);
1728 }
1729
1730 static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred)
1731 {
1732         bpf_prog_kallsyms_del_all(prog);
1733         btf_put(prog->aux->btf);
1734         bpf_prog_free_linfo(prog);
1735
1736         if (deferred) {
1737                 if (prog->aux->sleepable)
1738                         call_rcu_tasks_trace(&prog->aux->rcu, __bpf_prog_put_rcu);
1739                 else
1740                         call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu);
1741         } else {
1742                 __bpf_prog_put_rcu(&prog->aux->rcu);
1743         }
1744 }
1745
1746 static void __bpf_prog_put(struct bpf_prog *prog, bool do_idr_lock)
1747 {
1748         if (atomic64_dec_and_test(&prog->aux->refcnt)) {
1749                 perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_UNLOAD, 0);
1750                 bpf_audit_prog(prog, BPF_AUDIT_UNLOAD);
1751                 /* bpf_prog_free_id() must be called first */
1752                 bpf_prog_free_id(prog, do_idr_lock);
1753                 __bpf_prog_put_noref(prog, true);
1754         }
1755 }
1756
1757 void bpf_prog_put(struct bpf_prog *prog)
1758 {
1759         __bpf_prog_put(prog, true);
1760 }
1761 EXPORT_SYMBOL_GPL(bpf_prog_put);
1762
1763 static int bpf_prog_release(struct inode *inode, struct file *filp)
1764 {
1765         struct bpf_prog *prog = filp->private_data;
1766
1767         bpf_prog_put(prog);
1768         return 0;
1769 }
1770
1771 static void bpf_prog_get_stats(const struct bpf_prog *prog,
1772                                struct bpf_prog_stats *stats)
1773 {
1774         u64 nsecs = 0, cnt = 0;
1775         int cpu;
1776
1777         for_each_possible_cpu(cpu) {
1778                 const struct bpf_prog_stats *st;
1779                 unsigned int start;
1780                 u64 tnsecs, tcnt;
1781
1782                 st = per_cpu_ptr(prog->aux->stats, cpu);
1783                 do {
1784                         start = u64_stats_fetch_begin_irq(&st->syncp);
1785                         tnsecs = st->nsecs;
1786                         tcnt = st->cnt;
1787                 } while (u64_stats_fetch_retry_irq(&st->syncp, start));
1788                 nsecs += tnsecs;
1789                 cnt += tcnt;
1790         }
1791         stats->nsecs = nsecs;
1792         stats->cnt = cnt;
1793 }
1794
1795 #ifdef CONFIG_PROC_FS
1796 static void bpf_prog_show_fdinfo(struct seq_file *m, struct file *filp)
1797 {
1798         const struct bpf_prog *prog = filp->private_data;
1799         char prog_tag[sizeof(prog->tag) * 2 + 1] = { };
1800         struct bpf_prog_stats stats;
1801
1802         bpf_prog_get_stats(prog, &stats);
1803         bin2hex(prog_tag, prog->tag, sizeof(prog->tag));
1804         seq_printf(m,
1805                    "prog_type:\t%u\n"
1806                    "prog_jited:\t%u\n"
1807                    "prog_tag:\t%s\n"
1808                    "memlock:\t%llu\n"
1809                    "prog_id:\t%u\n"
1810                    "run_time_ns:\t%llu\n"
1811                    "run_cnt:\t%llu\n",
1812                    prog->type,
1813                    prog->jited,
1814                    prog_tag,
1815                    prog->pages * 1ULL << PAGE_SHIFT,
1816                    prog->aux->id,
1817                    stats.nsecs,
1818                    stats.cnt);
1819 }
1820 #endif
1821
1822 const struct file_operations bpf_prog_fops = {
1823 #ifdef CONFIG_PROC_FS
1824         .show_fdinfo    = bpf_prog_show_fdinfo,
1825 #endif
1826         .release        = bpf_prog_release,
1827         .read           = bpf_dummy_read,
1828         .write          = bpf_dummy_write,
1829 };
1830
1831 int bpf_prog_new_fd(struct bpf_prog *prog)
1832 {
1833         int ret;
1834
1835         ret = security_bpf_prog(prog);
1836         if (ret < 0)
1837                 return ret;
1838
1839         return anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog,
1840                                 O_RDWR | O_CLOEXEC);
1841 }
1842
1843 static struct bpf_prog *____bpf_prog_get(struct fd f)
1844 {
1845         if (!f.file)
1846                 return ERR_PTR(-EBADF);
1847         if (f.file->f_op != &bpf_prog_fops) {
1848                 fdput(f);
1849                 return ERR_PTR(-EINVAL);
1850         }
1851
1852         return f.file->private_data;
1853 }
1854
1855 void bpf_prog_add(struct bpf_prog *prog, int i)
1856 {
1857         atomic64_add(i, &prog->aux->refcnt);
1858 }
1859 EXPORT_SYMBOL_GPL(bpf_prog_add);
1860
1861 void bpf_prog_sub(struct bpf_prog *prog, int i)
1862 {
1863         /* Only to be used for undoing previous bpf_prog_add() in some
1864          * error path. We still know that another entity in our call
1865          * path holds a reference to the program, thus atomic_sub() can
1866          * be safely used in such cases!
1867          */
1868         WARN_ON(atomic64_sub_return(i, &prog->aux->refcnt) == 0);
1869 }
1870 EXPORT_SYMBOL_GPL(bpf_prog_sub);
1871
1872 void bpf_prog_inc(struct bpf_prog *prog)
1873 {
1874         atomic64_inc(&prog->aux->refcnt);
1875 }
1876 EXPORT_SYMBOL_GPL(bpf_prog_inc);
1877
1878 /* prog_idr_lock should have been held */
1879 struct bpf_prog *bpf_prog_inc_not_zero(struct bpf_prog *prog)
1880 {
1881         int refold;
1882
1883         refold = atomic64_fetch_add_unless(&prog->aux->refcnt, 1, 0);
1884
1885         if (!refold)
1886                 return ERR_PTR(-ENOENT);
1887
1888         return prog;
1889 }
1890 EXPORT_SYMBOL_GPL(bpf_prog_inc_not_zero);
1891
1892 bool bpf_prog_get_ok(struct bpf_prog *prog,
1893                             enum bpf_prog_type *attach_type, bool attach_drv)
1894 {
1895         /* not an attachment, just a refcount inc, always allow */
1896         if (!attach_type)
1897                 return true;
1898
1899         if (prog->type != *attach_type)
1900                 return false;
1901         if (bpf_prog_is_dev_bound(prog->aux) && !attach_drv)
1902                 return false;
1903
1904         return true;
1905 }
1906
1907 static struct bpf_prog *__bpf_prog_get(u32 ufd, enum bpf_prog_type *attach_type,
1908                                        bool attach_drv)
1909 {
1910         struct fd f = fdget(ufd);
1911         struct bpf_prog *prog;
1912
1913         prog = ____bpf_prog_get(f);
1914         if (IS_ERR(prog))
1915                 return prog;
1916         if (!bpf_prog_get_ok(prog, attach_type, attach_drv)) {
1917                 prog = ERR_PTR(-EINVAL);
1918                 goto out;
1919         }
1920
1921         bpf_prog_inc(prog);
1922 out:
1923         fdput(f);
1924         return prog;
1925 }
1926
1927 struct bpf_prog *bpf_prog_get(u32 ufd)
1928 {
1929         return __bpf_prog_get(ufd, NULL, false);
1930 }
1931
1932 struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type,
1933                                        bool attach_drv)
1934 {
1935         return __bpf_prog_get(ufd, &type, attach_drv);
1936 }
1937 EXPORT_SYMBOL_GPL(bpf_prog_get_type_dev);
1938
1939 /* Initially all BPF programs could be loaded w/o specifying
1940  * expected_attach_type. Later for some of them specifying expected_attach_type
1941  * at load time became required so that program could be validated properly.
1942  * Programs of types that are allowed to be loaded both w/ and w/o (for
1943  * backward compatibility) expected_attach_type, should have the default attach
1944  * type assigned to expected_attach_type for the latter case, so that it can be
1945  * validated later at attach time.
1946  *
1947  * bpf_prog_load_fixup_attach_type() sets expected_attach_type in @attr if
1948  * prog type requires it but has some attach types that have to be backward
1949  * compatible.
1950  */
1951 static void bpf_prog_load_fixup_attach_type(union bpf_attr *attr)
1952 {
1953         switch (attr->prog_type) {
1954         case BPF_PROG_TYPE_CGROUP_SOCK:
1955                 /* Unfortunately BPF_ATTACH_TYPE_UNSPEC enumeration doesn't
1956                  * exist so checking for non-zero is the way to go here.
1957                  */
1958                 if (!attr->expected_attach_type)
1959                         attr->expected_attach_type =
1960                                 BPF_CGROUP_INET_SOCK_CREATE;
1961                 break;
1962         }
1963 }
1964
1965 static int
1966 bpf_prog_load_check_attach(enum bpf_prog_type prog_type,
1967                            enum bpf_attach_type expected_attach_type,
1968                            u32 btf_id, u32 prog_fd)
1969 {
1970         if (btf_id) {
1971                 if (btf_id > BTF_MAX_TYPE)
1972                         return -EINVAL;
1973
1974                 switch (prog_type) {
1975                 case BPF_PROG_TYPE_TRACING:
1976                 case BPF_PROG_TYPE_LSM:
1977                 case BPF_PROG_TYPE_STRUCT_OPS:
1978                 case BPF_PROG_TYPE_EXT:
1979                         break;
1980                 default:
1981                         return -EINVAL;
1982                 }
1983         }
1984
1985         if (prog_fd && prog_type != BPF_PROG_TYPE_TRACING &&
1986             prog_type != BPF_PROG_TYPE_EXT)
1987                 return -EINVAL;
1988
1989         switch (prog_type) {
1990         case BPF_PROG_TYPE_CGROUP_SOCK:
1991                 switch (expected_attach_type) {
1992                 case BPF_CGROUP_INET_SOCK_CREATE:
1993                 case BPF_CGROUP_INET_SOCK_RELEASE:
1994                 case BPF_CGROUP_INET4_POST_BIND:
1995                 case BPF_CGROUP_INET6_POST_BIND:
1996                         return 0;
1997                 default:
1998                         return -EINVAL;
1999                 }
2000         case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
2001                 switch (expected_attach_type) {
2002                 case BPF_CGROUP_INET4_BIND:
2003                 case BPF_CGROUP_INET6_BIND:
2004                 case BPF_CGROUP_INET4_CONNECT:
2005                 case BPF_CGROUP_INET6_CONNECT:
2006                 case BPF_CGROUP_INET4_GETPEERNAME:
2007                 case BPF_CGROUP_INET6_GETPEERNAME:
2008                 case BPF_CGROUP_INET4_GETSOCKNAME:
2009                 case BPF_CGROUP_INET6_GETSOCKNAME:
2010                 case BPF_CGROUP_UDP4_SENDMSG:
2011                 case BPF_CGROUP_UDP6_SENDMSG:
2012                 case BPF_CGROUP_UDP4_RECVMSG:
2013                 case BPF_CGROUP_UDP6_RECVMSG:
2014                         return 0;
2015                 default:
2016                         return -EINVAL;
2017                 }
2018         case BPF_PROG_TYPE_CGROUP_SKB:
2019                 switch (expected_attach_type) {
2020                 case BPF_CGROUP_INET_INGRESS:
2021                 case BPF_CGROUP_INET_EGRESS:
2022                         return 0;
2023                 default:
2024                         return -EINVAL;
2025                 }
2026         case BPF_PROG_TYPE_CGROUP_SOCKOPT:
2027                 switch (expected_attach_type) {
2028                 case BPF_CGROUP_SETSOCKOPT:
2029                 case BPF_CGROUP_GETSOCKOPT:
2030                         return 0;
2031                 default:
2032                         return -EINVAL;
2033                 }
2034         case BPF_PROG_TYPE_SK_LOOKUP:
2035                 if (expected_attach_type == BPF_SK_LOOKUP)
2036                         return 0;
2037                 return -EINVAL;
2038         case BPF_PROG_TYPE_EXT:
2039                 if (expected_attach_type)
2040                         return -EINVAL;
2041                 fallthrough;
2042         default:
2043                 return 0;
2044         }
2045 }
2046
2047 static bool is_net_admin_prog_type(enum bpf_prog_type prog_type)
2048 {
2049         switch (prog_type) {
2050         case BPF_PROG_TYPE_SCHED_CLS:
2051         case BPF_PROG_TYPE_SCHED_ACT:
2052         case BPF_PROG_TYPE_XDP:
2053         case BPF_PROG_TYPE_LWT_IN:
2054         case BPF_PROG_TYPE_LWT_OUT:
2055         case BPF_PROG_TYPE_LWT_XMIT:
2056         case BPF_PROG_TYPE_LWT_SEG6LOCAL:
2057         case BPF_PROG_TYPE_SK_SKB:
2058         case BPF_PROG_TYPE_SK_MSG:
2059         case BPF_PROG_TYPE_LIRC_MODE2:
2060         case BPF_PROG_TYPE_FLOW_DISSECTOR:
2061         case BPF_PROG_TYPE_CGROUP_DEVICE:
2062         case BPF_PROG_TYPE_CGROUP_SOCK:
2063         case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
2064         case BPF_PROG_TYPE_CGROUP_SOCKOPT:
2065         case BPF_PROG_TYPE_CGROUP_SYSCTL:
2066         case BPF_PROG_TYPE_SOCK_OPS:
2067         case BPF_PROG_TYPE_EXT: /* extends any prog */
2068                 return true;
2069         case BPF_PROG_TYPE_CGROUP_SKB:
2070                 /* always unpriv */
2071         case BPF_PROG_TYPE_SK_REUSEPORT:
2072                 /* equivalent to SOCKET_FILTER. need CAP_BPF only */
2073         default:
2074                 return false;
2075         }
2076 }
2077
2078 static bool is_perfmon_prog_type(enum bpf_prog_type prog_type)
2079 {
2080         switch (prog_type) {
2081         case BPF_PROG_TYPE_KPROBE:
2082         case BPF_PROG_TYPE_TRACEPOINT:
2083         case BPF_PROG_TYPE_PERF_EVENT:
2084         case BPF_PROG_TYPE_RAW_TRACEPOINT:
2085         case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE:
2086         case BPF_PROG_TYPE_TRACING:
2087         case BPF_PROG_TYPE_LSM:
2088         case BPF_PROG_TYPE_STRUCT_OPS: /* has access to struct sock */
2089         case BPF_PROG_TYPE_EXT: /* extends any prog */
2090                 return true;
2091         default:
2092                 return false;
2093         }
2094 }
2095
2096 /* last field in 'union bpf_attr' used by this command */
2097 #define BPF_PROG_LOAD_LAST_FIELD attach_prog_fd
2098
2099 static int bpf_prog_load(union bpf_attr *attr, union bpf_attr __user *uattr)
2100 {
2101         enum bpf_prog_type type = attr->prog_type;
2102         struct bpf_prog *prog;
2103         int err;
2104         char license[128];
2105         bool is_gpl;
2106
2107         if (CHECK_ATTR(BPF_PROG_LOAD))
2108                 return -EINVAL;
2109
2110         if (attr->prog_flags & ~(BPF_F_STRICT_ALIGNMENT |
2111                                  BPF_F_ANY_ALIGNMENT |
2112                                  BPF_F_TEST_STATE_FREQ |
2113                                  BPF_F_SLEEPABLE |
2114                                  BPF_F_TEST_RND_HI32))
2115                 return -EINVAL;
2116
2117         if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) &&
2118             (attr->prog_flags & BPF_F_ANY_ALIGNMENT) &&
2119             !bpf_capable())
2120                 return -EPERM;
2121
2122         /* copy eBPF program license from user space */
2123         if (strncpy_from_user(license, u64_to_user_ptr(attr->license),
2124                               sizeof(license) - 1) < 0)
2125                 return -EFAULT;
2126         license[sizeof(license) - 1] = 0;
2127
2128         /* eBPF programs must be GPL compatible to use GPL-ed functions */
2129         is_gpl = license_is_gpl_compatible(license);
2130
2131         if (attr->insn_cnt == 0 ||
2132             attr->insn_cnt > (bpf_capable() ? BPF_COMPLEXITY_LIMIT_INSNS : BPF_MAXINSNS))
2133                 return -E2BIG;
2134         if (type != BPF_PROG_TYPE_SOCKET_FILTER &&
2135             type != BPF_PROG_TYPE_CGROUP_SKB &&
2136             !bpf_capable())
2137                 return -EPERM;
2138
2139         if (is_net_admin_prog_type(type) && !capable(CAP_NET_ADMIN) && !capable(CAP_SYS_ADMIN))
2140                 return -EPERM;
2141         if (is_perfmon_prog_type(type) && !perfmon_capable())
2142                 return -EPERM;
2143
2144         bpf_prog_load_fixup_attach_type(attr);
2145         if (bpf_prog_load_check_attach(type, attr->expected_attach_type,
2146                                        attr->attach_btf_id,
2147                                        attr->attach_prog_fd))
2148                 return -EINVAL;
2149
2150         /* plain bpf_prog allocation */
2151         prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER);
2152         if (!prog)
2153                 return -ENOMEM;
2154
2155         prog->expected_attach_type = attr->expected_attach_type;
2156         prog->aux->attach_btf_id = attr->attach_btf_id;
2157         if (attr->attach_prog_fd) {
2158                 struct bpf_prog *dst_prog;
2159
2160                 dst_prog = bpf_prog_get(attr->attach_prog_fd);
2161                 if (IS_ERR(dst_prog)) {
2162                         err = PTR_ERR(dst_prog);
2163                         goto free_prog_nouncharge;
2164                 }
2165                 prog->aux->dst_prog = dst_prog;
2166         }
2167
2168         prog->aux->offload_requested = !!attr->prog_ifindex;
2169         prog->aux->sleepable = attr->prog_flags & BPF_F_SLEEPABLE;
2170
2171         err = security_bpf_prog_alloc(prog->aux);
2172         if (err)
2173                 goto free_prog_nouncharge;
2174
2175         err = bpf_prog_charge_memlock(prog);
2176         if (err)
2177                 goto free_prog_sec;
2178
2179         prog->len = attr->insn_cnt;
2180
2181         err = -EFAULT;
2182         if (copy_from_user(prog->insns, u64_to_user_ptr(attr->insns),
2183                            bpf_prog_insn_size(prog)) != 0)
2184                 goto free_prog;
2185
2186         prog->orig_prog = NULL;
2187         prog->jited = 0;
2188
2189         atomic64_set(&prog->aux->refcnt, 1);
2190         prog->gpl_compatible = is_gpl ? 1 : 0;
2191
2192         if (bpf_prog_is_dev_bound(prog->aux)) {
2193                 err = bpf_prog_offload_init(prog, attr);
2194                 if (err)
2195                         goto free_prog;
2196         }
2197
2198         /* find program type: socket_filter vs tracing_filter */
2199         err = find_prog_type(type, prog);
2200         if (err < 0)
2201                 goto free_prog;
2202
2203         prog->aux->load_time = ktime_get_boottime_ns();
2204         err = bpf_obj_name_cpy(prog->aux->name, attr->prog_name,
2205                                sizeof(attr->prog_name));
2206         if (err < 0)
2207                 goto free_prog;
2208
2209         /* run eBPF verifier */
2210         err = bpf_check(&prog, attr, uattr);
2211         if (err < 0)
2212                 goto free_used_maps;
2213
2214         prog = bpf_prog_select_runtime(prog, &err);
2215         if (err < 0)
2216                 goto free_used_maps;
2217
2218         err = bpf_prog_alloc_id(prog);
2219         if (err)
2220                 goto free_used_maps;
2221
2222         /* Upon success of bpf_prog_alloc_id(), the BPF prog is
2223          * effectively publicly exposed. However, retrieving via
2224          * bpf_prog_get_fd_by_id() will take another reference,
2225          * therefore it cannot be gone underneath us.
2226          *
2227          * Only for the time /after/ successful bpf_prog_new_fd()
2228          * and before returning to userspace, we might just hold
2229          * one reference and any parallel close on that fd could
2230          * rip everything out. Hence, below notifications must
2231          * happen before bpf_prog_new_fd().
2232          *
2233          * Also, any failure handling from this point onwards must
2234          * be using bpf_prog_put() given the program is exposed.
2235          */
2236         bpf_prog_kallsyms_add(prog);
2237         perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_LOAD, 0);
2238         bpf_audit_prog(prog, BPF_AUDIT_LOAD);
2239
2240         err = bpf_prog_new_fd(prog);
2241         if (err < 0)
2242                 bpf_prog_put(prog);
2243         return err;
2244
2245 free_used_maps:
2246         /* In case we have subprogs, we need to wait for a grace
2247          * period before we can tear down JIT memory since symbols
2248          * are already exposed under kallsyms.
2249          */
2250         __bpf_prog_put_noref(prog, prog->aux->func_cnt);
2251         return err;
2252 free_prog:
2253         bpf_prog_uncharge_memlock(prog);
2254 free_prog_sec:
2255         security_bpf_prog_free(prog->aux);
2256 free_prog_nouncharge:
2257         bpf_prog_free(prog);
2258         return err;
2259 }
2260
2261 #define BPF_OBJ_LAST_FIELD file_flags
2262
2263 static int bpf_obj_pin(const union bpf_attr *attr)
2264 {
2265         if (CHECK_ATTR(BPF_OBJ) || attr->file_flags != 0)
2266                 return -EINVAL;
2267
2268         return bpf_obj_pin_user(attr->bpf_fd, u64_to_user_ptr(attr->pathname));
2269 }
2270
2271 static int bpf_obj_get(const union bpf_attr *attr)
2272 {
2273         if (CHECK_ATTR(BPF_OBJ) || attr->bpf_fd != 0 ||
2274             attr->file_flags & ~BPF_OBJ_FLAG_MASK)
2275                 return -EINVAL;
2276
2277         return bpf_obj_get_user(u64_to_user_ptr(attr->pathname),
2278                                 attr->file_flags);
2279 }
2280
2281 void bpf_link_init(struct bpf_link *link, enum bpf_link_type type,
2282                    const struct bpf_link_ops *ops, struct bpf_prog *prog)
2283 {
2284         atomic64_set(&link->refcnt, 1);
2285         link->type = type;
2286         link->id = 0;
2287         link->ops = ops;
2288         link->prog = prog;
2289 }
2290
2291 static void bpf_link_free_id(int id)
2292 {
2293         if (!id)
2294                 return;
2295
2296         spin_lock_bh(&link_idr_lock);
2297         idr_remove(&link_idr, id);
2298         spin_unlock_bh(&link_idr_lock);
2299 }
2300
2301 /* Clean up bpf_link and corresponding anon_inode file and FD. After
2302  * anon_inode is created, bpf_link can't be just kfree()'d due to deferred
2303  * anon_inode's release() call. This helper marksbpf_link as
2304  * defunct, releases anon_inode file and puts reserved FD. bpf_prog's refcnt
2305  * is not decremented, it's the responsibility of a calling code that failed
2306  * to complete bpf_link initialization.
2307  */
2308 void bpf_link_cleanup(struct bpf_link_primer *primer)
2309 {
2310         primer->link->prog = NULL;
2311         bpf_link_free_id(primer->id);
2312         fput(primer->file);
2313         put_unused_fd(primer->fd);
2314 }
2315
2316 void bpf_link_inc(struct bpf_link *link)
2317 {
2318         atomic64_inc(&link->refcnt);
2319 }
2320
2321 /* bpf_link_free is guaranteed to be called from process context */
2322 static void bpf_link_free(struct bpf_link *link)
2323 {
2324         bpf_link_free_id(link->id);
2325         if (link->prog) {
2326                 /* detach BPF program, clean up used resources */
2327                 link->ops->release(link);
2328                 bpf_prog_put(link->prog);
2329         }
2330         /* free bpf_link and its containing memory */
2331         link->ops->dealloc(link);
2332 }
2333
2334 static void bpf_link_put_deferred(struct work_struct *work)
2335 {
2336         struct bpf_link *link = container_of(work, struct bpf_link, work);
2337
2338         bpf_link_free(link);
2339 }
2340
2341 /* bpf_link_put can be called from atomic context, but ensures that resources
2342  * are freed from process context
2343  */
2344 void bpf_link_put(struct bpf_link *link)
2345 {
2346         if (!atomic64_dec_and_test(&link->refcnt))
2347                 return;
2348
2349         if (in_atomic()) {
2350                 INIT_WORK(&link->work, bpf_link_put_deferred);
2351                 schedule_work(&link->work);
2352         } else {
2353                 bpf_link_free(link);
2354         }
2355 }
2356
2357 static int bpf_link_release(struct inode *inode, struct file *filp)
2358 {
2359         struct bpf_link *link = filp->private_data;
2360
2361         bpf_link_put(link);
2362         return 0;
2363 }
2364
2365 #ifdef CONFIG_PROC_FS
2366 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type)
2367 #define BPF_MAP_TYPE(_id, _ops)
2368 #define BPF_LINK_TYPE(_id, _name) [_id] = #_name,
2369 static const char *bpf_link_type_strs[] = {
2370         [BPF_LINK_TYPE_UNSPEC] = "<invalid>",
2371 #include <linux/bpf_types.h>
2372 };
2373 #undef BPF_PROG_TYPE
2374 #undef BPF_MAP_TYPE
2375 #undef BPF_LINK_TYPE
2376
2377 static void bpf_link_show_fdinfo(struct seq_file *m, struct file *filp)
2378 {
2379         const struct bpf_link *link = filp->private_data;
2380         const struct bpf_prog *prog = link->prog;
2381         char prog_tag[sizeof(prog->tag) * 2 + 1] = { };
2382
2383         bin2hex(prog_tag, prog->tag, sizeof(prog->tag));
2384         seq_printf(m,
2385                    "link_type:\t%s\n"
2386                    "link_id:\t%u\n"
2387                    "prog_tag:\t%s\n"
2388                    "prog_id:\t%u\n",
2389                    bpf_link_type_strs[link->type],
2390                    link->id,
2391                    prog_tag,
2392                    prog->aux->id);
2393         if (link->ops->show_fdinfo)
2394                 link->ops->show_fdinfo(link, m);
2395 }
2396 #endif
2397
2398 static const struct file_operations bpf_link_fops = {
2399 #ifdef CONFIG_PROC_FS
2400         .show_fdinfo    = bpf_link_show_fdinfo,
2401 #endif
2402         .release        = bpf_link_release,
2403         .read           = bpf_dummy_read,
2404         .write          = bpf_dummy_write,
2405 };
2406
2407 static int bpf_link_alloc_id(struct bpf_link *link)
2408 {
2409         int id;
2410
2411         idr_preload(GFP_KERNEL);
2412         spin_lock_bh(&link_idr_lock);
2413         id = idr_alloc_cyclic(&link_idr, link, 1, INT_MAX, GFP_ATOMIC);
2414         spin_unlock_bh(&link_idr_lock);
2415         idr_preload_end();
2416
2417         return id;
2418 }
2419
2420 /* Prepare bpf_link to be exposed to user-space by allocating anon_inode file,
2421  * reserving unused FD and allocating ID from link_idr. This is to be paired
2422  * with bpf_link_settle() to install FD and ID and expose bpf_link to
2423  * user-space, if bpf_link is successfully attached. If not, bpf_link and
2424  * pre-allocated resources are to be freed with bpf_cleanup() call. All the
2425  * transient state is passed around in struct bpf_link_primer.
2426  * This is preferred way to create and initialize bpf_link, especially when
2427  * there are complicated and expensive operations inbetween creating bpf_link
2428  * itself and attaching it to BPF hook. By using bpf_link_prime() and
2429  * bpf_link_settle() kernel code using bpf_link doesn't have to perform
2430  * expensive (and potentially failing) roll back operations in a rare case
2431  * that file, FD, or ID can't be allocated.
2432  */
2433 int bpf_link_prime(struct bpf_link *link, struct bpf_link_primer *primer)
2434 {
2435         struct file *file;
2436         int fd, id;
2437
2438         fd = get_unused_fd_flags(O_CLOEXEC);
2439         if (fd < 0)
2440                 return fd;
2441
2442
2443         id = bpf_link_alloc_id(link);
2444         if (id < 0) {
2445                 put_unused_fd(fd);
2446                 return id;
2447         }
2448
2449         file = anon_inode_getfile("bpf_link", &bpf_link_fops, link, O_CLOEXEC);
2450         if (IS_ERR(file)) {
2451                 bpf_link_free_id(id);
2452                 put_unused_fd(fd);
2453                 return PTR_ERR(file);
2454         }
2455
2456         primer->link = link;
2457         primer->file = file;
2458         primer->fd = fd;
2459         primer->id = id;
2460         return 0;
2461 }
2462
2463 int bpf_link_settle(struct bpf_link_primer *primer)
2464 {
2465         /* make bpf_link fetchable by ID */
2466         spin_lock_bh(&link_idr_lock);
2467         primer->link->id = primer->id;
2468         spin_unlock_bh(&link_idr_lock);
2469         /* make bpf_link fetchable by FD */
2470         fd_install(primer->fd, primer->file);
2471         /* pass through installed FD */
2472         return primer->fd;
2473 }
2474
2475 int bpf_link_new_fd(struct bpf_link *link)
2476 {
2477         return anon_inode_getfd("bpf-link", &bpf_link_fops, link, O_CLOEXEC);
2478 }
2479
2480 struct bpf_link *bpf_link_get_from_fd(u32 ufd)
2481 {
2482         struct fd f = fdget(ufd);
2483         struct bpf_link *link;
2484
2485         if (!f.file)
2486                 return ERR_PTR(-EBADF);
2487         if (f.file->f_op != &bpf_link_fops) {
2488                 fdput(f);
2489                 return ERR_PTR(-EINVAL);
2490         }
2491
2492         link = f.file->private_data;
2493         bpf_link_inc(link);
2494         fdput(f);
2495
2496         return link;
2497 }
2498
2499 struct bpf_tracing_link {
2500         struct bpf_link link;
2501         enum bpf_attach_type attach_type;
2502         struct bpf_trampoline *trampoline;
2503         struct bpf_prog *tgt_prog;
2504 };
2505
2506 static void bpf_tracing_link_release(struct bpf_link *link)
2507 {
2508         struct bpf_tracing_link *tr_link =
2509                 container_of(link, struct bpf_tracing_link, link);
2510
2511         WARN_ON_ONCE(bpf_trampoline_unlink_prog(link->prog,
2512                                                 tr_link->trampoline));
2513
2514         bpf_trampoline_put(tr_link->trampoline);
2515
2516         /* tgt_prog is NULL if target is a kernel function */
2517         if (tr_link->tgt_prog)
2518                 bpf_prog_put(tr_link->tgt_prog);
2519 }
2520
2521 static void bpf_tracing_link_dealloc(struct bpf_link *link)
2522 {
2523         struct bpf_tracing_link *tr_link =
2524                 container_of(link, struct bpf_tracing_link, link);
2525
2526         kfree(tr_link);
2527 }
2528
2529 static void bpf_tracing_link_show_fdinfo(const struct bpf_link *link,
2530                                          struct seq_file *seq)
2531 {
2532         struct bpf_tracing_link *tr_link =
2533                 container_of(link, struct bpf_tracing_link, link);
2534
2535         seq_printf(seq,
2536                    "attach_type:\t%d\n",
2537                    tr_link->attach_type);
2538 }
2539
2540 static int bpf_tracing_link_fill_link_info(const struct bpf_link *link,
2541                                            struct bpf_link_info *info)
2542 {
2543         struct bpf_tracing_link *tr_link =
2544                 container_of(link, struct bpf_tracing_link, link);
2545
2546         info->tracing.attach_type = tr_link->attach_type;
2547
2548         return 0;
2549 }
2550
2551 static const struct bpf_link_ops bpf_tracing_link_lops = {
2552         .release = bpf_tracing_link_release,
2553         .dealloc = bpf_tracing_link_dealloc,
2554         .show_fdinfo = bpf_tracing_link_show_fdinfo,
2555         .fill_link_info = bpf_tracing_link_fill_link_info,
2556 };
2557
2558 static int bpf_tracing_prog_attach(struct bpf_prog *prog,
2559                                    int tgt_prog_fd,
2560                                    u32 btf_id)
2561 {
2562         struct bpf_link_primer link_primer;
2563         struct bpf_prog *tgt_prog = NULL;
2564         struct bpf_trampoline *tr = NULL;
2565         struct bpf_tracing_link *link;
2566         u64 key = 0;
2567         int err;
2568
2569         switch (prog->type) {
2570         case BPF_PROG_TYPE_TRACING:
2571                 if (prog->expected_attach_type != BPF_TRACE_FENTRY &&
2572                     prog->expected_attach_type != BPF_TRACE_FEXIT &&
2573                     prog->expected_attach_type != BPF_MODIFY_RETURN) {
2574                         err = -EINVAL;
2575                         goto out_put_prog;
2576                 }
2577                 break;
2578         case BPF_PROG_TYPE_EXT:
2579                 if (prog->expected_attach_type != 0) {
2580                         err = -EINVAL;
2581                         goto out_put_prog;
2582                 }
2583                 break;
2584         case BPF_PROG_TYPE_LSM:
2585                 if (prog->expected_attach_type != BPF_LSM_MAC) {
2586                         err = -EINVAL;
2587                         goto out_put_prog;
2588                 }
2589                 break;
2590         default:
2591                 err = -EINVAL;
2592                 goto out_put_prog;
2593         }
2594
2595         if (!!tgt_prog_fd != !!btf_id) {
2596                 err = -EINVAL;
2597                 goto out_put_prog;
2598         }
2599
2600         if (tgt_prog_fd) {
2601                 /* For now we only allow new targets for BPF_PROG_TYPE_EXT */
2602                 if (prog->type != BPF_PROG_TYPE_EXT) {
2603                         err = -EINVAL;
2604                         goto out_put_prog;
2605                 }
2606
2607                 tgt_prog = bpf_prog_get(tgt_prog_fd);
2608                 if (IS_ERR(tgt_prog)) {
2609                         err = PTR_ERR(tgt_prog);
2610                         tgt_prog = NULL;
2611                         goto out_put_prog;
2612                 }
2613
2614                 key = bpf_trampoline_compute_key(tgt_prog, btf_id);
2615         }
2616
2617         link = kzalloc(sizeof(*link), GFP_USER);
2618         if (!link) {
2619                 err = -ENOMEM;
2620                 goto out_put_prog;
2621         }
2622         bpf_link_init(&link->link, BPF_LINK_TYPE_TRACING,
2623                       &bpf_tracing_link_lops, prog);
2624         link->attach_type = prog->expected_attach_type;
2625
2626         mutex_lock(&prog->aux->dst_mutex);
2627
2628         /* There are a few possible cases here:
2629          *
2630          * - if prog->aux->dst_trampoline is set, the program was just loaded
2631          *   and not yet attached to anything, so we can use the values stored
2632          *   in prog->aux
2633          *
2634          * - if prog->aux->dst_trampoline is NULL, the program has already been
2635          *   attached to a target and its initial target was cleared (below)
2636          *
2637          * - if tgt_prog != NULL, the caller specified tgt_prog_fd +
2638          *   target_btf_id using the link_create API.
2639          *
2640          * - if tgt_prog == NULL when this function was called using the old
2641          *   raw_tracepoint_open API, and we need a target from prog->aux
2642          *
2643          * The combination of no saved target in prog->aux, and no target
2644          * specified on load is illegal, and we reject that here.
2645          */
2646         if (!prog->aux->dst_trampoline && !tgt_prog) {
2647                 err = -ENOENT;
2648                 goto out_unlock;
2649         }
2650
2651         if (!prog->aux->dst_trampoline ||
2652             (key && key != prog->aux->dst_trampoline->key)) {
2653                 /* If there is no saved target, or the specified target is
2654                  * different from the destination specified at load time, we
2655                  * need a new trampoline and a check for compatibility
2656                  */
2657                 struct bpf_attach_target_info tgt_info = {};
2658
2659                 err = bpf_check_attach_target(NULL, prog, tgt_prog, btf_id,
2660                                               &tgt_info);
2661                 if (err)
2662                         goto out_unlock;
2663
2664                 tr = bpf_trampoline_get(key, &tgt_info);
2665                 if (!tr) {
2666                         err = -ENOMEM;
2667                         goto out_unlock;
2668                 }
2669         } else {
2670                 /* The caller didn't specify a target, or the target was the
2671                  * same as the destination supplied during program load. This
2672                  * means we can reuse the trampoline and reference from program
2673                  * load time, and there is no need to allocate a new one. This
2674                  * can only happen once for any program, as the saved values in
2675                  * prog->aux are cleared below.
2676                  */
2677                 tr = prog->aux->dst_trampoline;
2678                 tgt_prog = prog->aux->dst_prog;
2679         }
2680
2681         err = bpf_link_prime(&link->link, &link_primer);
2682         if (err)
2683                 goto out_unlock;
2684
2685         err = bpf_trampoline_link_prog(prog, tr);
2686         if (err) {
2687                 bpf_link_cleanup(&link_primer);
2688                 link = NULL;
2689                 goto out_unlock;
2690         }
2691
2692         link->tgt_prog = tgt_prog;
2693         link->trampoline = tr;
2694
2695         /* Always clear the trampoline and target prog from prog->aux to make
2696          * sure the original attach destination is not kept alive after a
2697          * program is (re-)attached to another target.
2698          */
2699         if (prog->aux->dst_prog &&
2700             (tgt_prog_fd || tr != prog->aux->dst_trampoline))
2701                 /* got extra prog ref from syscall, or attaching to different prog */
2702                 bpf_prog_put(prog->aux->dst_prog);
2703         if (prog->aux->dst_trampoline && tr != prog->aux->dst_trampoline)
2704                 /* we allocated a new trampoline, so free the old one */
2705                 bpf_trampoline_put(prog->aux->dst_trampoline);
2706
2707         prog->aux->dst_prog = NULL;
2708         prog->aux->dst_trampoline = NULL;
2709         mutex_unlock(&prog->aux->dst_mutex);
2710
2711         return bpf_link_settle(&link_primer);
2712 out_unlock:
2713         if (tr && tr != prog->aux->dst_trampoline)
2714                 bpf_trampoline_put(tr);
2715         mutex_unlock(&prog->aux->dst_mutex);
2716         kfree(link);
2717 out_put_prog:
2718         if (tgt_prog_fd && tgt_prog)
2719                 bpf_prog_put(tgt_prog);
2720         bpf_prog_put(prog);
2721         return err;
2722 }
2723
2724 struct bpf_raw_tp_link {
2725         struct bpf_link link;
2726         struct bpf_raw_event_map *btp;
2727 };
2728
2729 static void bpf_raw_tp_link_release(struct bpf_link *link)
2730 {
2731         struct bpf_raw_tp_link *raw_tp =
2732                 container_of(link, struct bpf_raw_tp_link, link);
2733
2734         bpf_probe_unregister(raw_tp->btp, raw_tp->link.prog);
2735         bpf_put_raw_tracepoint(raw_tp->btp);
2736 }
2737
2738 static void bpf_raw_tp_link_dealloc(struct bpf_link *link)
2739 {
2740         struct bpf_raw_tp_link *raw_tp =
2741                 container_of(link, struct bpf_raw_tp_link, link);
2742
2743         kfree(raw_tp);
2744 }
2745
2746 static void bpf_raw_tp_link_show_fdinfo(const struct bpf_link *link,
2747                                         struct seq_file *seq)
2748 {
2749         struct bpf_raw_tp_link *raw_tp_link =
2750                 container_of(link, struct bpf_raw_tp_link, link);
2751
2752         seq_printf(seq,
2753                    "tp_name:\t%s\n",
2754                    raw_tp_link->btp->tp->name);
2755 }
2756
2757 static int bpf_raw_tp_link_fill_link_info(const struct bpf_link *link,
2758                                           struct bpf_link_info *info)
2759 {
2760         struct bpf_raw_tp_link *raw_tp_link =
2761                 container_of(link, struct bpf_raw_tp_link, link);
2762         char __user *ubuf = u64_to_user_ptr(info->raw_tracepoint.tp_name);
2763         const char *tp_name = raw_tp_link->btp->tp->name;
2764         u32 ulen = info->raw_tracepoint.tp_name_len;
2765         size_t tp_len = strlen(tp_name);
2766
2767         if (!ulen ^ !ubuf)
2768                 return -EINVAL;
2769
2770         info->raw_tracepoint.tp_name_len = tp_len + 1;
2771
2772         if (!ubuf)
2773                 return 0;
2774
2775         if (ulen >= tp_len + 1) {
2776                 if (copy_to_user(ubuf, tp_name, tp_len + 1))
2777                         return -EFAULT;
2778         } else {
2779                 char zero = '\0';
2780
2781                 if (copy_to_user(ubuf, tp_name, ulen - 1))
2782                         return -EFAULT;
2783                 if (put_user(zero, ubuf + ulen - 1))
2784                         return -EFAULT;
2785                 return -ENOSPC;
2786         }
2787
2788         return 0;
2789 }
2790
2791 static const struct bpf_link_ops bpf_raw_tp_link_lops = {
2792         .release = bpf_raw_tp_link_release,
2793         .dealloc = bpf_raw_tp_link_dealloc,
2794         .show_fdinfo = bpf_raw_tp_link_show_fdinfo,
2795         .fill_link_info = bpf_raw_tp_link_fill_link_info,
2796 };
2797
2798 #define BPF_RAW_TRACEPOINT_OPEN_LAST_FIELD raw_tracepoint.prog_fd
2799
2800 static int bpf_raw_tracepoint_open(const union bpf_attr *attr)
2801 {
2802         struct bpf_link_primer link_primer;
2803         struct bpf_raw_tp_link *link;
2804         struct bpf_raw_event_map *btp;
2805         struct bpf_prog *prog;
2806         const char *tp_name;
2807         char buf[128];
2808         int err;
2809
2810         if (CHECK_ATTR(BPF_RAW_TRACEPOINT_OPEN))
2811                 return -EINVAL;
2812
2813         prog = bpf_prog_get(attr->raw_tracepoint.prog_fd);
2814         if (IS_ERR(prog))
2815                 return PTR_ERR(prog);
2816
2817         switch (prog->type) {
2818         case BPF_PROG_TYPE_TRACING:
2819         case BPF_PROG_TYPE_EXT:
2820         case BPF_PROG_TYPE_LSM:
2821                 if (attr->raw_tracepoint.name) {
2822                         /* The attach point for this category of programs
2823                          * should be specified via btf_id during program load.
2824                          */
2825                         err = -EINVAL;
2826                         goto out_put_prog;
2827                 }
2828                 if (prog->type == BPF_PROG_TYPE_TRACING &&
2829                     prog->expected_attach_type == BPF_TRACE_RAW_TP) {
2830                         tp_name = prog->aux->attach_func_name;
2831                         break;
2832                 }
2833                 return bpf_tracing_prog_attach(prog, 0, 0);
2834         case BPF_PROG_TYPE_RAW_TRACEPOINT:
2835         case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE:
2836                 if (strncpy_from_user(buf,
2837                                       u64_to_user_ptr(attr->raw_tracepoint.name),
2838                                       sizeof(buf) - 1) < 0) {
2839                         err = -EFAULT;
2840                         goto out_put_prog;
2841                 }
2842                 buf[sizeof(buf) - 1] = 0;
2843                 tp_name = buf;
2844                 break;
2845         default:
2846                 err = -EINVAL;
2847                 goto out_put_prog;
2848         }
2849
2850         btp = bpf_get_raw_tracepoint(tp_name);
2851         if (!btp) {
2852                 err = -ENOENT;
2853                 goto out_put_prog;
2854         }
2855
2856         link = kzalloc(sizeof(*link), GFP_USER);
2857         if (!link) {
2858                 err = -ENOMEM;
2859                 goto out_put_btp;
2860         }
2861         bpf_link_init(&link->link, BPF_LINK_TYPE_RAW_TRACEPOINT,
2862                       &bpf_raw_tp_link_lops, prog);
2863         link->btp = btp;
2864
2865         err = bpf_link_prime(&link->link, &link_primer);
2866         if (err) {
2867                 kfree(link);
2868                 goto out_put_btp;
2869         }
2870
2871         err = bpf_probe_register(link->btp, prog);
2872         if (err) {
2873                 bpf_link_cleanup(&link_primer);
2874                 goto out_put_btp;
2875         }
2876
2877         return bpf_link_settle(&link_primer);
2878
2879 out_put_btp:
2880         bpf_put_raw_tracepoint(btp);
2881 out_put_prog:
2882         bpf_prog_put(prog);
2883         return err;
2884 }
2885
2886 static int bpf_prog_attach_check_attach_type(const struct bpf_prog *prog,
2887                                              enum bpf_attach_type attach_type)
2888 {
2889         switch (prog->type) {
2890         case BPF_PROG_TYPE_CGROUP_SOCK:
2891         case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
2892         case BPF_PROG_TYPE_CGROUP_SOCKOPT:
2893         case BPF_PROG_TYPE_SK_LOOKUP:
2894                 return attach_type == prog->expected_attach_type ? 0 : -EINVAL;
2895         case BPF_PROG_TYPE_CGROUP_SKB:
2896                 if (!capable(CAP_NET_ADMIN))
2897                         /* cg-skb progs can be loaded by unpriv user.
2898                          * check permissions at attach time.
2899                          */
2900                         return -EPERM;
2901                 return prog->enforce_expected_attach_type &&
2902                         prog->expected_attach_type != attach_type ?
2903                         -EINVAL : 0;
2904         default:
2905                 return 0;
2906         }
2907 }
2908
2909 static enum bpf_prog_type
2910 attach_type_to_prog_type(enum bpf_attach_type attach_type)
2911 {
2912         switch (attach_type) {
2913         case BPF_CGROUP_INET_INGRESS:
2914         case BPF_CGROUP_INET_EGRESS:
2915                 return BPF_PROG_TYPE_CGROUP_SKB;
2916         case BPF_CGROUP_INET_SOCK_CREATE:
2917         case BPF_CGROUP_INET_SOCK_RELEASE:
2918         case BPF_CGROUP_INET4_POST_BIND:
2919         case BPF_CGROUP_INET6_POST_BIND:
2920                 return BPF_PROG_TYPE_CGROUP_SOCK;
2921         case BPF_CGROUP_INET4_BIND:
2922         case BPF_CGROUP_INET6_BIND:
2923         case BPF_CGROUP_INET4_CONNECT:
2924         case BPF_CGROUP_INET6_CONNECT:
2925         case BPF_CGROUP_INET4_GETPEERNAME:
2926         case BPF_CGROUP_INET6_GETPEERNAME:
2927         case BPF_CGROUP_INET4_GETSOCKNAME:
2928         case BPF_CGROUP_INET6_GETSOCKNAME:
2929         case BPF_CGROUP_UDP4_SENDMSG:
2930         case BPF_CGROUP_UDP6_SENDMSG:
2931         case BPF_CGROUP_UDP4_RECVMSG:
2932         case BPF_CGROUP_UDP6_RECVMSG:
2933                 return BPF_PROG_TYPE_CGROUP_SOCK_ADDR;
2934         case BPF_CGROUP_SOCK_OPS:
2935                 return BPF_PROG_TYPE_SOCK_OPS;
2936         case BPF_CGROUP_DEVICE:
2937                 return BPF_PROG_TYPE_CGROUP_DEVICE;
2938         case BPF_SK_MSG_VERDICT:
2939                 return BPF_PROG_TYPE_SK_MSG;
2940         case BPF_SK_SKB_STREAM_PARSER:
2941         case BPF_SK_SKB_STREAM_VERDICT:
2942                 return BPF_PROG_TYPE_SK_SKB;
2943         case BPF_LIRC_MODE2:
2944                 return BPF_PROG_TYPE_LIRC_MODE2;
2945         case BPF_FLOW_DISSECTOR:
2946                 return BPF_PROG_TYPE_FLOW_DISSECTOR;
2947         case BPF_CGROUP_SYSCTL:
2948                 return BPF_PROG_TYPE_CGROUP_SYSCTL;
2949         case BPF_CGROUP_GETSOCKOPT:
2950         case BPF_CGROUP_SETSOCKOPT:
2951                 return BPF_PROG_TYPE_CGROUP_SOCKOPT;
2952         case BPF_TRACE_ITER:
2953                 return BPF_PROG_TYPE_TRACING;
2954         case BPF_SK_LOOKUP:
2955                 return BPF_PROG_TYPE_SK_LOOKUP;
2956         case BPF_XDP:
2957                 return BPF_PROG_TYPE_XDP;
2958         default:
2959                 return BPF_PROG_TYPE_UNSPEC;
2960         }
2961 }
2962
2963 #define BPF_PROG_ATTACH_LAST_FIELD replace_bpf_fd
2964
2965 #define BPF_F_ATTACH_MASK \
2966         (BPF_F_ALLOW_OVERRIDE | BPF_F_ALLOW_MULTI | BPF_F_REPLACE)
2967
2968 static int bpf_prog_attach(const union bpf_attr *attr)
2969 {
2970         enum bpf_prog_type ptype;
2971         struct bpf_prog *prog;
2972         int ret;
2973
2974         if (CHECK_ATTR(BPF_PROG_ATTACH))
2975                 return -EINVAL;
2976
2977         if (attr->attach_flags & ~BPF_F_ATTACH_MASK)
2978                 return -EINVAL;
2979
2980         ptype = attach_type_to_prog_type(attr->attach_type);
2981         if (ptype == BPF_PROG_TYPE_UNSPEC)
2982                 return -EINVAL;
2983
2984         prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype);
2985         if (IS_ERR(prog))
2986                 return PTR_ERR(prog);
2987
2988         if (bpf_prog_attach_check_attach_type(prog, attr->attach_type)) {
2989                 bpf_prog_put(prog);
2990                 return -EINVAL;
2991         }
2992
2993         switch (ptype) {
2994         case BPF_PROG_TYPE_SK_SKB:
2995         case BPF_PROG_TYPE_SK_MSG:
2996                 ret = sock_map_get_from_fd(attr, prog);
2997                 break;
2998         case BPF_PROG_TYPE_LIRC_MODE2:
2999                 ret = lirc_prog_attach(attr, prog);
3000                 break;
3001         case BPF_PROG_TYPE_FLOW_DISSECTOR:
3002                 ret = netns_bpf_prog_attach(attr, prog);
3003                 break;
3004         case BPF_PROG_TYPE_CGROUP_DEVICE:
3005         case BPF_PROG_TYPE_CGROUP_SKB:
3006         case BPF_PROG_TYPE_CGROUP_SOCK:
3007         case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
3008         case BPF_PROG_TYPE_CGROUP_SOCKOPT:
3009         case BPF_PROG_TYPE_CGROUP_SYSCTL:
3010         case BPF_PROG_TYPE_SOCK_OPS:
3011                 ret = cgroup_bpf_prog_attach(attr, ptype, prog);
3012                 break;
3013         default:
3014                 ret = -EINVAL;
3015         }
3016
3017         if (ret)
3018                 bpf_prog_put(prog);
3019         return ret;
3020 }
3021
3022 #define BPF_PROG_DETACH_LAST_FIELD attach_type
3023
3024 static int bpf_prog_detach(const union bpf_attr *attr)
3025 {
3026         enum bpf_prog_type ptype;
3027
3028         if (CHECK_ATTR(BPF_PROG_DETACH))
3029                 return -EINVAL;
3030
3031         ptype = attach_type_to_prog_type(attr->attach_type);
3032
3033         switch (ptype) {
3034         case BPF_PROG_TYPE_SK_MSG:
3035         case BPF_PROG_TYPE_SK_SKB:
3036                 return sock_map_prog_detach(attr, ptype);
3037         case BPF_PROG_TYPE_LIRC_MODE2:
3038                 return lirc_prog_detach(attr);
3039         case BPF_PROG_TYPE_FLOW_DISSECTOR:
3040                 return netns_bpf_prog_detach(attr, ptype);
3041         case BPF_PROG_TYPE_CGROUP_DEVICE:
3042         case BPF_PROG_TYPE_CGROUP_SKB:
3043         case BPF_PROG_TYPE_CGROUP_SOCK:
3044         case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
3045         case BPF_PROG_TYPE_CGROUP_SOCKOPT:
3046         case BPF_PROG_TYPE_CGROUP_SYSCTL:
3047         case BPF_PROG_TYPE_SOCK_OPS:
3048                 return cgroup_bpf_prog_detach(attr, ptype);
3049         default:
3050                 return -EINVAL;
3051         }
3052 }
3053
3054 #define BPF_PROG_QUERY_LAST_FIELD query.prog_cnt
3055
3056 static int bpf_prog_query(const union bpf_attr *attr,
3057                           union bpf_attr __user *uattr)
3058 {
3059         if (!capable(CAP_NET_ADMIN))
3060                 return -EPERM;
3061         if (CHECK_ATTR(BPF_PROG_QUERY))
3062                 return -EINVAL;
3063         if (attr->query.query_flags & ~BPF_F_QUERY_EFFECTIVE)
3064                 return -EINVAL;
3065
3066         switch (attr->query.attach_type) {
3067         case BPF_CGROUP_INET_INGRESS:
3068         case BPF_CGROUP_INET_EGRESS:
3069         case BPF_CGROUP_INET_SOCK_CREATE:
3070         case BPF_CGROUP_INET_SOCK_RELEASE:
3071         case BPF_CGROUP_INET4_BIND:
3072         case BPF_CGROUP_INET6_BIND:
3073         case BPF_CGROUP_INET4_POST_BIND:
3074         case BPF_CGROUP_INET6_POST_BIND:
3075         case BPF_CGROUP_INET4_CONNECT:
3076         case BPF_CGROUP_INET6_CONNECT:
3077         case BPF_CGROUP_INET4_GETPEERNAME:
3078         case BPF_CGROUP_INET6_GETPEERNAME:
3079         case BPF_CGROUP_INET4_GETSOCKNAME:
3080         case BPF_CGROUP_INET6_GETSOCKNAME:
3081         case BPF_CGROUP_UDP4_SENDMSG:
3082         case BPF_CGROUP_UDP6_SENDMSG:
3083         case BPF_CGROUP_UDP4_RECVMSG:
3084         case BPF_CGROUP_UDP6_RECVMSG:
3085         case BPF_CGROUP_SOCK_OPS:
3086         case BPF_CGROUP_DEVICE:
3087         case BPF_CGROUP_SYSCTL:
3088         case BPF_CGROUP_GETSOCKOPT:
3089         case BPF_CGROUP_SETSOCKOPT:
3090                 return cgroup_bpf_prog_query(attr, uattr);
3091         case BPF_LIRC_MODE2:
3092                 return lirc_prog_query(attr, uattr);
3093         case BPF_FLOW_DISSECTOR:
3094         case BPF_SK_LOOKUP:
3095                 return netns_bpf_prog_query(attr, uattr);
3096         default:
3097                 return -EINVAL;
3098         }
3099 }
3100
3101 #define BPF_PROG_TEST_RUN_LAST_FIELD test.cpu
3102
3103 static int bpf_prog_test_run(const union bpf_attr *attr,
3104                              union bpf_attr __user *uattr)
3105 {
3106         struct bpf_prog *prog;
3107         int ret = -ENOTSUPP;
3108
3109         if (CHECK_ATTR(BPF_PROG_TEST_RUN))
3110                 return -EINVAL;
3111
3112         if ((attr->test.ctx_size_in && !attr->test.ctx_in) ||
3113             (!attr->test.ctx_size_in && attr->test.ctx_in))
3114                 return -EINVAL;
3115
3116         if ((attr->test.ctx_size_out && !attr->test.ctx_out) ||
3117             (!attr->test.ctx_size_out && attr->test.ctx_out))
3118                 return -EINVAL;
3119
3120         prog = bpf_prog_get(attr->test.prog_fd);
3121         if (IS_ERR(prog))
3122                 return PTR_ERR(prog);
3123
3124         if (prog->aux->ops->test_run)
3125                 ret = prog->aux->ops->test_run(prog, attr, uattr);
3126
3127         bpf_prog_put(prog);
3128         return ret;
3129 }
3130
3131 #define BPF_OBJ_GET_NEXT_ID_LAST_FIELD next_id
3132
3133 static int bpf_obj_get_next_id(const union bpf_attr *attr,
3134                                union bpf_attr __user *uattr,
3135                                struct idr *idr,
3136                                spinlock_t *lock)
3137 {
3138         u32 next_id = attr->start_id;
3139         int err = 0;
3140
3141         if (CHECK_ATTR(BPF_OBJ_GET_NEXT_ID) || next_id >= INT_MAX)
3142                 return -EINVAL;
3143
3144         if (!capable(CAP_SYS_ADMIN))
3145                 return -EPERM;
3146
3147         next_id++;
3148         spin_lock_bh(lock);
3149         if (!idr_get_next(idr, &next_id))
3150                 err = -ENOENT;
3151         spin_unlock_bh(lock);
3152
3153         if (!err)
3154                 err = put_user(next_id, &uattr->next_id);
3155
3156         return err;
3157 }
3158
3159 struct bpf_map *bpf_map_get_curr_or_next(u32 *id)
3160 {
3161         struct bpf_map *map;
3162
3163         spin_lock_bh(&map_idr_lock);
3164 again:
3165         map = idr_get_next(&map_idr, id);
3166         if (map) {
3167                 map = __bpf_map_inc_not_zero(map, false);
3168                 if (IS_ERR(map)) {
3169                         (*id)++;
3170                         goto again;
3171                 }
3172         }
3173         spin_unlock_bh(&map_idr_lock);
3174
3175         return map;
3176 }
3177
3178 struct bpf_prog *bpf_prog_get_curr_or_next(u32 *id)
3179 {
3180         struct bpf_prog *prog;
3181
3182         spin_lock_bh(&prog_idr_lock);
3183 again:
3184         prog = idr_get_next(&prog_idr, id);
3185         if (prog) {
3186                 prog = bpf_prog_inc_not_zero(prog);
3187                 if (IS_ERR(prog)) {
3188                         (*id)++;
3189                         goto again;
3190                 }
3191         }
3192         spin_unlock_bh(&prog_idr_lock);
3193
3194         return prog;
3195 }
3196
3197 #define BPF_PROG_GET_FD_BY_ID_LAST_FIELD prog_id
3198
3199 struct bpf_prog *bpf_prog_by_id(u32 id)
3200 {
3201         struct bpf_prog *prog;
3202
3203         if (!id)
3204                 return ERR_PTR(-ENOENT);
3205
3206         spin_lock_bh(&prog_idr_lock);
3207         prog = idr_find(&prog_idr, id);
3208         if (prog)
3209                 prog = bpf_prog_inc_not_zero(prog);
3210         else
3211                 prog = ERR_PTR(-ENOENT);
3212         spin_unlock_bh(&prog_idr_lock);
3213         return prog;
3214 }
3215
3216 static int bpf_prog_get_fd_by_id(const union bpf_attr *attr)
3217 {
3218         struct bpf_prog *prog;
3219         u32 id = attr->prog_id;
3220         int fd;
3221
3222         if (CHECK_ATTR(BPF_PROG_GET_FD_BY_ID))
3223                 return -EINVAL;
3224
3225         if (!capable(CAP_SYS_ADMIN))
3226                 return -EPERM;
3227
3228         prog = bpf_prog_by_id(id);
3229         if (IS_ERR(prog))
3230                 return PTR_ERR(prog);
3231
3232         fd = bpf_prog_new_fd(prog);
3233         if (fd < 0)
3234                 bpf_prog_put(prog);
3235
3236         return fd;
3237 }
3238
3239 #define BPF_MAP_GET_FD_BY_ID_LAST_FIELD open_flags
3240
3241 static int bpf_map_get_fd_by_id(const union bpf_attr *attr)
3242 {
3243         struct bpf_map *map;
3244         u32 id = attr->map_id;
3245         int f_flags;
3246         int fd;
3247
3248         if (CHECK_ATTR(BPF_MAP_GET_FD_BY_ID) ||
3249             attr->open_flags & ~BPF_OBJ_FLAG_MASK)
3250                 return -EINVAL;
3251
3252         if (!capable(CAP_SYS_ADMIN))
3253                 return -EPERM;
3254
3255         f_flags = bpf_get_file_flag(attr->open_flags);
3256         if (f_flags < 0)
3257                 return f_flags;
3258
3259         spin_lock_bh(&map_idr_lock);
3260         map = idr_find(&map_idr, id);
3261         if (map)
3262                 map = __bpf_map_inc_not_zero(map, true);
3263         else
3264                 map = ERR_PTR(-ENOENT);
3265         spin_unlock_bh(&map_idr_lock);
3266
3267         if (IS_ERR(map))
3268                 return PTR_ERR(map);
3269
3270         fd = bpf_map_new_fd(map, f_flags);
3271         if (fd < 0)
3272                 bpf_map_put_with_uref(map);
3273
3274         return fd;
3275 }
3276
3277 static const struct bpf_map *bpf_map_from_imm(const struct bpf_prog *prog,
3278                                               unsigned long addr, u32 *off,
3279                                               u32 *type)
3280 {
3281         const struct bpf_map *map;
3282         int i;
3283
3284         mutex_lock(&prog->aux->used_maps_mutex);
3285         for (i = 0, *off = 0; i < prog->aux->used_map_cnt; i++) {
3286                 map = prog->aux->used_maps[i];
3287                 if (map == (void *)addr) {
3288                         *type = BPF_PSEUDO_MAP_FD;
3289                         goto out;
3290                 }
3291                 if (!map->ops->map_direct_value_meta)
3292                         continue;
3293                 if (!map->ops->map_direct_value_meta(map, addr, off)) {
3294                         *type = BPF_PSEUDO_MAP_VALUE;
3295                         goto out;
3296                 }
3297         }
3298         map = NULL;
3299
3300 out:
3301         mutex_unlock(&prog->aux->used_maps_mutex);
3302         return map;
3303 }
3304
3305 static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog,
3306                                               const struct cred *f_cred)
3307 {
3308         const struct bpf_map *map;
3309         struct bpf_insn *insns;
3310         u32 off, type;
3311         u64 imm;
3312         u8 code;
3313         int i;
3314
3315         insns = kmemdup(prog->insnsi, bpf_prog_insn_size(prog),
3316                         GFP_USER);
3317         if (!insns)
3318                 return insns;
3319
3320         for (i = 0; i < prog->len; i++) {
3321                 code = insns[i].code;
3322
3323                 if (code == (BPF_JMP | BPF_TAIL_CALL)) {
3324                         insns[i].code = BPF_JMP | BPF_CALL;
3325                         insns[i].imm = BPF_FUNC_tail_call;
3326                         /* fall-through */
3327                 }
3328                 if (code == (BPF_JMP | BPF_CALL) ||
3329                     code == (BPF_JMP | BPF_CALL_ARGS)) {
3330                         if (code == (BPF_JMP | BPF_CALL_ARGS))
3331                                 insns[i].code = BPF_JMP | BPF_CALL;
3332                         if (!bpf_dump_raw_ok(f_cred))
3333                                 insns[i].imm = 0;
3334                         continue;
3335                 }
3336                 if (BPF_CLASS(code) == BPF_LDX && BPF_MODE(code) == BPF_PROBE_MEM) {
3337                         insns[i].code = BPF_LDX | BPF_SIZE(code) | BPF_MEM;
3338                         continue;
3339                 }
3340
3341                 if (code != (BPF_LD | BPF_IMM | BPF_DW))
3342                         continue;
3343
3344                 imm = ((u64)insns[i + 1].imm << 32) | (u32)insns[i].imm;
3345                 map = bpf_map_from_imm(prog, imm, &off, &type);
3346                 if (map) {
3347                         insns[i].src_reg = type;
3348                         insns[i].imm = map->id;
3349                         insns[i + 1].imm = off;
3350                         continue;
3351                 }
3352         }
3353
3354         return insns;
3355 }
3356
3357 static int set_info_rec_size(struct bpf_prog_info *info)
3358 {
3359         /*
3360          * Ensure info.*_rec_size is the same as kernel expected size
3361          *
3362          * or
3363          *
3364          * Only allow zero *_rec_size if both _rec_size and _cnt are
3365          * zero.  In this case, the kernel will set the expected
3366          * _rec_size back to the info.
3367          */
3368
3369         if ((info->nr_func_info || info->func_info_rec_size) &&
3370             info->func_info_rec_size != sizeof(struct bpf_func_info))
3371                 return -EINVAL;
3372
3373         if ((info->nr_line_info || info->line_info_rec_size) &&
3374             info->line_info_rec_size != sizeof(struct bpf_line_info))
3375                 return -EINVAL;
3376
3377         if ((info->nr_jited_line_info || info->jited_line_info_rec_size) &&
3378             info->jited_line_info_rec_size != sizeof(__u64))
3379                 return -EINVAL;
3380
3381         info->func_info_rec_size = sizeof(struct bpf_func_info);
3382         info->line_info_rec_size = sizeof(struct bpf_line_info);
3383         info->jited_line_info_rec_size = sizeof(__u64);
3384
3385         return 0;
3386 }
3387
3388 static int bpf_prog_get_info_by_fd(struct file *file,
3389                                    struct bpf_prog *prog,
3390                                    const union bpf_attr *attr,
3391                                    union bpf_attr __user *uattr)
3392 {
3393         struct bpf_prog_info __user *uinfo = u64_to_user_ptr(attr->info.info);
3394         struct bpf_prog_info info;
3395         u32 info_len = attr->info.info_len;
3396         struct bpf_prog_stats stats;
3397         char __user *uinsns;
3398         u32 ulen;
3399         int err;
3400
3401         err = bpf_check_uarg_tail_zero(uinfo, sizeof(info), info_len);
3402         if (err)
3403                 return err;
3404         info_len = min_t(u32, sizeof(info), info_len);
3405
3406         memset(&info, 0, sizeof(info));
3407         if (copy_from_user(&info, uinfo, info_len))
3408                 return -EFAULT;
3409
3410         info.type = prog->type;
3411         info.id = prog->aux->id;
3412         info.load_time = prog->aux->load_time;
3413         info.created_by_uid = from_kuid_munged(current_user_ns(),
3414                                                prog->aux->user->uid);
3415         info.gpl_compatible = prog->gpl_compatible;
3416
3417         memcpy(info.tag, prog->tag, sizeof(prog->tag));
3418         memcpy(info.name, prog->aux->name, sizeof(prog->aux->name));
3419
3420         mutex_lock(&prog->aux->used_maps_mutex);
3421         ulen = info.nr_map_ids;
3422         info.nr_map_ids = prog->aux->used_map_cnt;
3423         ulen = min_t(u32, info.nr_map_ids, ulen);
3424         if (ulen) {
3425                 u32 __user *user_map_ids = u64_to_user_ptr(info.map_ids);
3426                 u32 i;
3427
3428                 for (i = 0; i < ulen; i++)
3429                         if (put_user(prog->aux->used_maps[i]->id,
3430                                      &user_map_ids[i])) {
3431                                 mutex_unlock(&prog->aux->used_maps_mutex);
3432                                 return -EFAULT;
3433                         }
3434         }
3435         mutex_unlock(&prog->aux->used_maps_mutex);
3436
3437         err = set_info_rec_size(&info);
3438         if (err)
3439                 return err;
3440
3441         bpf_prog_get_stats(prog, &stats);
3442         info.run_time_ns = stats.nsecs;
3443         info.run_cnt = stats.cnt;
3444
3445         if (!bpf_capable()) {
3446                 info.jited_prog_len = 0;
3447                 info.xlated_prog_len = 0;
3448                 info.nr_jited_ksyms = 0;
3449                 info.nr_jited_func_lens = 0;
3450                 info.nr_func_info = 0;
3451                 info.nr_line_info = 0;
3452                 info.nr_jited_line_info = 0;
3453                 goto done;
3454         }
3455
3456         ulen = info.xlated_prog_len;
3457         info.xlated_prog_len = bpf_prog_insn_size(prog);
3458         if (info.xlated_prog_len && ulen) {
3459                 struct bpf_insn *insns_sanitized;
3460                 bool fault;
3461
3462                 if (prog->blinded && !bpf_dump_raw_ok(file->f_cred)) {
3463                         info.xlated_prog_insns = 0;
3464                         goto done;
3465                 }
3466                 insns_sanitized = bpf_insn_prepare_dump(prog, file->f_cred);
3467                 if (!insns_sanitized)
3468                         return -ENOMEM;
3469                 uinsns = u64_to_user_ptr(info.xlated_prog_insns);
3470                 ulen = min_t(u32, info.xlated_prog_len, ulen);
3471                 fault = copy_to_user(uinsns, insns_sanitized, ulen);
3472                 kfree(insns_sanitized);
3473                 if (fault)
3474                         return -EFAULT;
3475         }
3476
3477         if (bpf_prog_is_dev_bound(prog->aux)) {
3478                 err = bpf_prog_offload_info_fill(&info, prog);
3479                 if (err)
3480                         return err;
3481                 goto done;
3482         }
3483
3484         /* NOTE: the following code is supposed to be skipped for offload.
3485          * bpf_prog_offload_info_fill() is the place to fill similar fields
3486          * for offload.
3487          */
3488         ulen = info.jited_prog_len;
3489         if (prog->aux->func_cnt) {
3490                 u32 i;
3491
3492                 info.jited_prog_len = 0;
3493                 for (i = 0; i < prog->aux->func_cnt; i++)
3494                         info.jited_prog_len += prog->aux->func[i]->jited_len;
3495         } else {
3496                 info.jited_prog_len = prog->jited_len;
3497         }
3498
3499         if (info.jited_prog_len && ulen) {
3500                 if (bpf_dump_raw_ok(file->f_cred)) {
3501                         uinsns = u64_to_user_ptr(info.jited_prog_insns);
3502                         ulen = min_t(u32, info.jited_prog_len, ulen);
3503
3504                         /* for multi-function programs, copy the JITed
3505                          * instructions for all the functions
3506                          */
3507                         if (prog->aux->func_cnt) {
3508                                 u32 len, free, i;
3509                                 u8 *img;
3510
3511                                 free = ulen;
3512                                 for (i = 0; i < prog->aux->func_cnt; i++) {
3513                                         len = prog->aux->func[i]->jited_len;
3514                                         len = min_t(u32, len, free);
3515                                         img = (u8 *) prog->aux->func[i]->bpf_func;
3516                                         if (copy_to_user(uinsns, img, len))
3517                                                 return -EFAULT;
3518                                         uinsns += len;
3519                                         free -= len;
3520                                         if (!free)
3521                                                 break;
3522                                 }
3523                         } else {
3524                                 if (copy_to_user(uinsns, prog->bpf_func, ulen))
3525                                         return -EFAULT;
3526                         }
3527                 } else {
3528                         info.jited_prog_insns = 0;
3529                 }
3530         }
3531
3532         ulen = info.nr_jited_ksyms;
3533         info.nr_jited_ksyms = prog->aux->func_cnt ? : 1;
3534         if (ulen) {
3535                 if (bpf_dump_raw_ok(file->f_cred)) {
3536                         unsigned long ksym_addr;
3537                         u64 __user *user_ksyms;
3538                         u32 i;
3539
3540                         /* copy the address of the kernel symbol
3541                          * corresponding to each function
3542                          */
3543                         ulen = min_t(u32, info.nr_jited_ksyms, ulen);
3544                         user_ksyms = u64_to_user_ptr(info.jited_ksyms);
3545                         if (prog->aux->func_cnt) {
3546                                 for (i = 0; i < ulen; i++) {
3547                                         ksym_addr = (unsigned long)
3548                                                 prog->aux->func[i]->bpf_func;
3549                                         if (put_user((u64) ksym_addr,
3550                                                      &user_ksyms[i]))
3551                                                 return -EFAULT;
3552                                 }
3553                         } else {
3554                                 ksym_addr = (unsigned long) prog->bpf_func;
3555                                 if (put_user((u64) ksym_addr, &user_ksyms[0]))
3556                                         return -EFAULT;
3557                         }
3558                 } else {
3559                         info.jited_ksyms = 0;
3560                 }
3561         }
3562
3563         ulen = info.nr_jited_func_lens;
3564         info.nr_jited_func_lens = prog->aux->func_cnt ? : 1;
3565         if (ulen) {
3566                 if (bpf_dump_raw_ok(file->f_cred)) {
3567                         u32 __user *user_lens;
3568                         u32 func_len, i;
3569
3570                         /* copy the JITed image lengths for each function */
3571                         ulen = min_t(u32, info.nr_jited_func_lens, ulen);
3572                         user_lens = u64_to_user_ptr(info.jited_func_lens);
3573                         if (prog->aux->func_cnt) {
3574                                 for (i = 0; i < ulen; i++) {
3575                                         func_len =
3576                                                 prog->aux->func[i]->jited_len;
3577                                         if (put_user(func_len, &user_lens[i]))
3578                                                 return -EFAULT;
3579                                 }
3580                         } else {
3581                                 func_len = prog->jited_len;
3582                                 if (put_user(func_len, &user_lens[0]))
3583                                         return -EFAULT;
3584                         }
3585                 } else {
3586                         info.jited_func_lens = 0;
3587                 }
3588         }
3589
3590         if (prog->aux->btf)
3591                 info.btf_id = btf_id(prog->aux->btf);
3592
3593         ulen = info.nr_func_info;
3594         info.nr_func_info = prog->aux->func_info_cnt;
3595         if (info.nr_func_info && ulen) {
3596                 char __user *user_finfo;
3597
3598                 user_finfo = u64_to_user_ptr(info.func_info);
3599                 ulen = min_t(u32, info.nr_func_info, ulen);
3600                 if (copy_to_user(user_finfo, prog->aux->func_info,
3601                                  info.func_info_rec_size * ulen))
3602                         return -EFAULT;
3603         }
3604
3605         ulen = info.nr_line_info;
3606         info.nr_line_info = prog->aux->nr_linfo;
3607         if (info.nr_line_info && ulen) {
3608                 __u8 __user *user_linfo;
3609
3610                 user_linfo = u64_to_user_ptr(info.line_info);
3611                 ulen = min_t(u32, info.nr_line_info, ulen);
3612                 if (copy_to_user(user_linfo, prog->aux->linfo,
3613                                  info.line_info_rec_size * ulen))
3614                         return -EFAULT;
3615         }
3616
3617         ulen = info.nr_jited_line_info;
3618         if (prog->aux->jited_linfo)
3619                 info.nr_jited_line_info = prog->aux->nr_linfo;
3620         else
3621                 info.nr_jited_line_info = 0;
3622         if (info.nr_jited_line_info && ulen) {
3623                 if (bpf_dump_raw_ok(file->f_cred)) {
3624                         __u64 __user *user_linfo;
3625                         u32 i;
3626
3627                         user_linfo = u64_to_user_ptr(info.jited_line_info);
3628                         ulen = min_t(u32, info.nr_jited_line_info, ulen);
3629                         for (i = 0; i < ulen; i++) {
3630                                 if (put_user((__u64)(long)prog->aux->jited_linfo[i],
3631                                              &user_linfo[i]))
3632                                         return -EFAULT;
3633                         }
3634                 } else {
3635                         info.jited_line_info = 0;
3636                 }
3637         }
3638
3639         ulen = info.nr_prog_tags;
3640         info.nr_prog_tags = prog->aux->func_cnt ? : 1;
3641         if (ulen) {
3642                 __u8 __user (*user_prog_tags)[BPF_TAG_SIZE];
3643                 u32 i;
3644
3645                 user_prog_tags = u64_to_user_ptr(info.prog_tags);
3646                 ulen = min_t(u32, info.nr_prog_tags, ulen);
3647                 if (prog->aux->func_cnt) {
3648                         for (i = 0; i < ulen; i++) {
3649                                 if (copy_to_user(user_prog_tags[i],
3650                                                  prog->aux->func[i]->tag,
3651                                                  BPF_TAG_SIZE))
3652                                         return -EFAULT;
3653                         }
3654                 } else {
3655                         if (copy_to_user(user_prog_tags[0],
3656                                          prog->tag, BPF_TAG_SIZE))
3657                                 return -EFAULT;
3658                 }
3659         }
3660
3661 done:
3662         if (copy_to_user(uinfo, &info, info_len) ||
3663             put_user(info_len, &uattr->info.info_len))
3664                 return -EFAULT;
3665
3666         return 0;
3667 }
3668
3669 static int bpf_map_get_info_by_fd(struct file *file,
3670                                   struct bpf_map *map,
3671                                   const union bpf_attr *attr,
3672                                   union bpf_attr __user *uattr)
3673 {
3674         struct bpf_map_info __user *uinfo = u64_to_user_ptr(attr->info.info);
3675         struct bpf_map_info info;
3676         u32 info_len = attr->info.info_len;
3677         int err;
3678
3679         err = bpf_check_uarg_tail_zero(uinfo, sizeof(info), info_len);
3680         if (err)
3681                 return err;
3682         info_len = min_t(u32, sizeof(info), info_len);
3683
3684         memset(&info, 0, sizeof(info));
3685         info.type = map->map_type;
3686         info.id = map->id;
3687         info.key_size = map->key_size;
3688         info.value_size = map->value_size;
3689         info.max_entries = map->max_entries;
3690         info.map_flags = map->map_flags;
3691         memcpy(info.name, map->name, sizeof(map->name));
3692
3693         if (map->btf) {
3694                 info.btf_id = btf_id(map->btf);
3695                 info.btf_key_type_id = map->btf_key_type_id;
3696                 info.btf_value_type_id = map->btf_value_type_id;
3697         }
3698         info.btf_vmlinux_value_type_id = map->btf_vmlinux_value_type_id;
3699
3700         if (bpf_map_is_dev_bound(map)) {
3701                 err = bpf_map_offload_info_fill(&info, map);
3702                 if (err)
3703                         return err;
3704         }
3705
3706         if (copy_to_user(uinfo, &info, info_len) ||
3707             put_user(info_len, &uattr->info.info_len))
3708                 return -EFAULT;
3709
3710         return 0;
3711 }
3712
3713 static int bpf_btf_get_info_by_fd(struct file *file,
3714                                   struct btf *btf,
3715                                   const union bpf_attr *attr,
3716                                   union bpf_attr __user *uattr)
3717 {
3718         struct bpf_btf_info __user *uinfo = u64_to_user_ptr(attr->info.info);
3719         u32 info_len = attr->info.info_len;
3720         int err;
3721
3722         err = bpf_check_uarg_tail_zero(uinfo, sizeof(*uinfo), info_len);
3723         if (err)
3724                 return err;
3725
3726         return btf_get_info_by_fd(btf, attr, uattr);
3727 }
3728
3729 static int bpf_link_get_info_by_fd(struct file *file,
3730                                   struct bpf_link *link,
3731                                   const union bpf_attr *attr,
3732                                   union bpf_attr __user *uattr)
3733 {
3734         struct bpf_link_info __user *uinfo = u64_to_user_ptr(attr->info.info);
3735         struct bpf_link_info info;
3736         u32 info_len = attr->info.info_len;
3737         int err;
3738
3739         err = bpf_check_uarg_tail_zero(uinfo, sizeof(info), info_len);
3740         if (err)
3741                 return err;
3742         info_len = min_t(u32, sizeof(info), info_len);
3743
3744         memset(&info, 0, sizeof(info));
3745         if (copy_from_user(&info, uinfo, info_len))
3746                 return -EFAULT;
3747
3748         info.type = link->type;
3749         info.id = link->id;
3750         info.prog_id = link->prog->aux->id;
3751
3752         if (link->ops->fill_link_info) {
3753                 err = link->ops->fill_link_info(link, &info);
3754                 if (err)
3755                         return err;
3756         }
3757
3758         if (copy_to_user(uinfo, &info, info_len) ||
3759             put_user(info_len, &uattr->info.info_len))
3760                 return -EFAULT;
3761
3762         return 0;
3763 }
3764
3765
3766 #define BPF_OBJ_GET_INFO_BY_FD_LAST_FIELD info.info
3767
3768 static int bpf_obj_get_info_by_fd(const union bpf_attr *attr,
3769                                   union bpf_attr __user *uattr)
3770 {
3771         int ufd = attr->info.bpf_fd;
3772         struct fd f;
3773         int err;
3774
3775         if (CHECK_ATTR(BPF_OBJ_GET_INFO_BY_FD))
3776                 return -EINVAL;
3777
3778         f = fdget(ufd);
3779         if (!f.file)
3780                 return -EBADFD;
3781
3782         if (f.file->f_op == &bpf_prog_fops)
3783                 err = bpf_prog_get_info_by_fd(f.file, f.file->private_data, attr,
3784                                               uattr);
3785         else if (f.file->f_op == &bpf_map_fops)
3786                 err = bpf_map_get_info_by_fd(f.file, f.file->private_data, attr,
3787                                              uattr);
3788         else if (f.file->f_op == &btf_fops)
3789                 err = bpf_btf_get_info_by_fd(f.file, f.file->private_data, attr, uattr);
3790         else if (f.file->f_op == &bpf_link_fops)
3791                 err = bpf_link_get_info_by_fd(f.file, f.file->private_data,
3792                                               attr, uattr);
3793         else
3794                 err = -EINVAL;
3795
3796         fdput(f);
3797         return err;
3798 }
3799
3800 #define BPF_BTF_LOAD_LAST_FIELD btf_log_level
3801
3802 static int bpf_btf_load(const union bpf_attr *attr)
3803 {
3804         if (CHECK_ATTR(BPF_BTF_LOAD))
3805                 return -EINVAL;
3806
3807         if (!bpf_capable())
3808                 return -EPERM;
3809
3810         return btf_new_fd(attr);
3811 }
3812
3813 #define BPF_BTF_GET_FD_BY_ID_LAST_FIELD btf_id
3814
3815 static int bpf_btf_get_fd_by_id(const union bpf_attr *attr)
3816 {
3817         if (CHECK_ATTR(BPF_BTF_GET_FD_BY_ID))
3818                 return -EINVAL;
3819
3820         if (!capable(CAP_SYS_ADMIN))
3821                 return -EPERM;
3822
3823         return btf_get_fd_by_id(attr->btf_id);
3824 }
3825
3826 static int bpf_task_fd_query_copy(const union bpf_attr *attr,
3827                                     union bpf_attr __user *uattr,
3828                                     u32 prog_id, u32 fd_type,
3829                                     const char *buf, u64 probe_offset,
3830                                     u64 probe_addr)
3831 {
3832         char __user *ubuf = u64_to_user_ptr(attr->task_fd_query.buf);
3833         u32 len = buf ? strlen(buf) : 0, input_len;
3834         int err = 0;
3835
3836         if (put_user(len, &uattr->task_fd_query.buf_len))
3837                 return -EFAULT;
3838         input_len = attr->task_fd_query.buf_len;
3839         if (input_len && ubuf) {
3840                 if (!len) {
3841                         /* nothing to copy, just make ubuf NULL terminated */
3842                         char zero = '\0';
3843
3844                         if (put_user(zero, ubuf))
3845                                 return -EFAULT;
3846                 } else if (input_len >= len + 1) {
3847                         /* ubuf can hold the string with NULL terminator */
3848                         if (copy_to_user(ubuf, buf, len + 1))
3849                                 return -EFAULT;
3850                 } else {
3851                         /* ubuf cannot hold the string with NULL terminator,
3852                          * do a partial copy with NULL terminator.
3853                          */
3854                         char zero = '\0';
3855
3856                         err = -ENOSPC;
3857                         if (copy_to_user(ubuf, buf, input_len - 1))
3858                                 return -EFAULT;
3859                         if (put_user(zero, ubuf + input_len - 1))
3860                                 return -EFAULT;
3861                 }
3862         }
3863
3864         if (put_user(prog_id, &uattr->task_fd_query.prog_id) ||
3865             put_user(fd_type, &uattr->task_fd_query.fd_type) ||
3866             put_user(probe_offset, &uattr->task_fd_query.probe_offset) ||
3867             put_user(probe_addr, &uattr->task_fd_query.probe_addr))
3868                 return -EFAULT;
3869
3870         return err;
3871 }
3872
3873 #define BPF_TASK_FD_QUERY_LAST_FIELD task_fd_query.probe_addr
3874
3875 static int bpf_task_fd_query(const union bpf_attr *attr,
3876                              union bpf_attr __user *uattr)
3877 {
3878         pid_t pid = attr->task_fd_query.pid;
3879         u32 fd = attr->task_fd_query.fd;
3880         const struct perf_event *event;
3881         struct files_struct *files;
3882         struct task_struct *task;
3883         struct file *file;
3884         int err;
3885
3886         if (CHECK_ATTR(BPF_TASK_FD_QUERY))
3887                 return -EINVAL;
3888
3889         if (!capable(CAP_SYS_ADMIN))
3890                 return -EPERM;
3891
3892         if (attr->task_fd_query.flags != 0)
3893                 return -EINVAL;
3894
3895         task = get_pid_task(find_vpid(pid), PIDTYPE_PID);
3896         if (!task)
3897                 return -ENOENT;
3898
3899         files = get_files_struct(task);
3900         put_task_struct(task);
3901         if (!files)
3902                 return -ENOENT;
3903
3904         err = 0;
3905         spin_lock(&files->file_lock);
3906         file = fcheck_files(files, fd);
3907         if (!file)
3908                 err = -EBADF;
3909         else
3910                 get_file(file);
3911         spin_unlock(&files->file_lock);
3912         put_files_struct(files);
3913
3914         if (err)
3915                 goto out;
3916
3917         if (file->f_op == &bpf_link_fops) {
3918                 struct bpf_link *link = file->private_data;
3919
3920                 if (link->ops == &bpf_raw_tp_link_lops) {
3921                         struct bpf_raw_tp_link *raw_tp =
3922                                 container_of(link, struct bpf_raw_tp_link, link);
3923                         struct bpf_raw_event_map *btp = raw_tp->btp;
3924
3925                         err = bpf_task_fd_query_copy(attr, uattr,
3926                                                      raw_tp->link.prog->aux->id,
3927                                                      BPF_FD_TYPE_RAW_TRACEPOINT,
3928                                                      btp->tp->name, 0, 0);
3929                         goto put_file;
3930                 }
3931                 goto out_not_supp;
3932         }
3933
3934         event = perf_get_event(file);
3935         if (!IS_ERR(event)) {
3936                 u64 probe_offset, probe_addr;
3937                 u32 prog_id, fd_type;
3938                 const char *buf;
3939
3940                 err = bpf_get_perf_event_info(event, &prog_id, &fd_type,
3941                                               &buf, &probe_offset,
3942                                               &probe_addr);
3943                 if (!err)
3944                         err = bpf_task_fd_query_copy(attr, uattr, prog_id,
3945                                                      fd_type, buf,
3946                                                      probe_offset,
3947                                                      probe_addr);
3948                 goto put_file;
3949         }
3950
3951 out_not_supp:
3952         err = -ENOTSUPP;
3953 put_file:
3954         fput(file);
3955 out:
3956         return err;
3957 }
3958
3959 #define BPF_MAP_BATCH_LAST_FIELD batch.flags
3960
3961 #define BPF_DO_BATCH(fn)                        \
3962         do {                                    \
3963                 if (!fn) {                      \
3964                         err = -ENOTSUPP;        \
3965                         goto err_put;           \
3966                 }                               \
3967                 err = fn(map, attr, uattr);     \
3968         } while (0)
3969
3970 static int bpf_map_do_batch(const union bpf_attr *attr,
3971                             union bpf_attr __user *uattr,
3972                             int cmd)
3973 {
3974         struct bpf_map *map;
3975         int err, ufd;
3976         struct fd f;
3977
3978         if (CHECK_ATTR(BPF_MAP_BATCH))
3979                 return -EINVAL;
3980
3981         ufd = attr->batch.map_fd;
3982         f = fdget(ufd);
3983         map = __bpf_map_get(f);
3984         if (IS_ERR(map))
3985                 return PTR_ERR(map);
3986
3987         if ((cmd == BPF_MAP_LOOKUP_BATCH ||
3988              cmd == BPF_MAP_LOOKUP_AND_DELETE_BATCH) &&
3989             !(map_get_sys_perms(map, f) & FMODE_CAN_READ)) {
3990                 err = -EPERM;
3991                 goto err_put;
3992         }
3993
3994         if (cmd != BPF_MAP_LOOKUP_BATCH &&
3995             !(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
3996                 err = -EPERM;
3997                 goto err_put;
3998         }
3999
4000         if (cmd == BPF_MAP_LOOKUP_BATCH)
4001                 BPF_DO_BATCH(map->ops->map_lookup_batch);
4002         else if (cmd == BPF_MAP_LOOKUP_AND_DELETE_BATCH)
4003                 BPF_DO_BATCH(map->ops->map_lookup_and_delete_batch);
4004         else if (cmd == BPF_MAP_UPDATE_BATCH)
4005                 BPF_DO_BATCH(map->ops->map_update_batch);
4006         else
4007                 BPF_DO_BATCH(map->ops->map_delete_batch);
4008
4009 err_put:
4010         fdput(f);
4011         return err;
4012 }
4013
4014 static int tracing_bpf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
4015 {
4016         if (attr->link_create.attach_type != prog->expected_attach_type)
4017                 return -EINVAL;
4018
4019         if (prog->expected_attach_type == BPF_TRACE_ITER)
4020                 return bpf_iter_link_attach(attr, prog);
4021         else if (prog->type == BPF_PROG_TYPE_EXT)
4022                 return bpf_tracing_prog_attach(prog,
4023                                                attr->link_create.target_fd,
4024                                                attr->link_create.target_btf_id);
4025         return -EINVAL;
4026 }
4027
4028 #define BPF_LINK_CREATE_LAST_FIELD link_create.iter_info_len
4029 static int link_create(union bpf_attr *attr)
4030 {
4031         enum bpf_prog_type ptype;
4032         struct bpf_prog *prog;
4033         int ret;
4034
4035         if (CHECK_ATTR(BPF_LINK_CREATE))
4036                 return -EINVAL;
4037
4038         prog = bpf_prog_get(attr->link_create.prog_fd);
4039         if (IS_ERR(prog))
4040                 return PTR_ERR(prog);
4041
4042         ret = bpf_prog_attach_check_attach_type(prog,
4043                                                 attr->link_create.attach_type);
4044         if (ret)
4045                 goto out;
4046
4047         if (prog->type == BPF_PROG_TYPE_EXT) {
4048                 ret = tracing_bpf_link_attach(attr, prog);
4049                 goto out;
4050         }
4051
4052         ptype = attach_type_to_prog_type(attr->link_create.attach_type);
4053         if (ptype == BPF_PROG_TYPE_UNSPEC || ptype != prog->type) {
4054                 ret = -EINVAL;
4055                 goto out;
4056         }
4057
4058         switch (ptype) {
4059         case BPF_PROG_TYPE_CGROUP_SKB:
4060         case BPF_PROG_TYPE_CGROUP_SOCK:
4061         case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
4062         case BPF_PROG_TYPE_SOCK_OPS:
4063         case BPF_PROG_TYPE_CGROUP_DEVICE:
4064         case BPF_PROG_TYPE_CGROUP_SYSCTL:
4065         case BPF_PROG_TYPE_CGROUP_SOCKOPT:
4066                 ret = cgroup_bpf_link_attach(attr, prog);
4067                 break;
4068         case BPF_PROG_TYPE_TRACING:
4069                 ret = tracing_bpf_link_attach(attr, prog);
4070                 break;
4071         case BPF_PROG_TYPE_FLOW_DISSECTOR:
4072         case BPF_PROG_TYPE_SK_LOOKUP:
4073                 ret = netns_bpf_link_create(attr, prog);
4074                 break;
4075 #ifdef CONFIG_NET
4076         case BPF_PROG_TYPE_XDP:
4077                 ret = bpf_xdp_link_attach(attr, prog);
4078                 break;
4079 #endif
4080         default:
4081                 ret = -EINVAL;
4082         }
4083
4084 out:
4085         if (ret < 0)
4086                 bpf_prog_put(prog);
4087         return ret;
4088 }
4089
4090 #define BPF_LINK_UPDATE_LAST_FIELD link_update.old_prog_fd
4091
4092 static int link_update(union bpf_attr *attr)
4093 {
4094         struct bpf_prog *old_prog = NULL, *new_prog;
4095         struct bpf_link *link;
4096         u32 flags;
4097         int ret;
4098
4099         if (CHECK_ATTR(BPF_LINK_UPDATE))
4100                 return -EINVAL;
4101
4102         flags = attr->link_update.flags;
4103         if (flags & ~BPF_F_REPLACE)
4104                 return -EINVAL;
4105
4106         link = bpf_link_get_from_fd(attr->link_update.link_fd);
4107         if (IS_ERR(link))
4108                 return PTR_ERR(link);
4109
4110         new_prog = bpf_prog_get(attr->link_update.new_prog_fd);
4111         if (IS_ERR(new_prog)) {
4112                 ret = PTR_ERR(new_prog);
4113                 goto out_put_link;
4114         }
4115
4116         if (flags & BPF_F_REPLACE) {
4117                 old_prog = bpf_prog_get(attr->link_update.old_prog_fd);
4118                 if (IS_ERR(old_prog)) {
4119                         ret = PTR_ERR(old_prog);
4120                         old_prog = NULL;
4121                         goto out_put_progs;
4122                 }
4123         } else if (attr->link_update.old_prog_fd) {
4124                 ret = -EINVAL;
4125                 goto out_put_progs;
4126         }
4127
4128         if (link->ops->update_prog)
4129                 ret = link->ops->update_prog(link, new_prog, old_prog);
4130         else
4131                 ret = -EINVAL;
4132
4133 out_put_progs:
4134         if (old_prog)
4135                 bpf_prog_put(old_prog);
4136         if (ret)
4137                 bpf_prog_put(new_prog);
4138 out_put_link:
4139         bpf_link_put(link);
4140         return ret;
4141 }
4142
4143 #define BPF_LINK_DETACH_LAST_FIELD link_detach.link_fd
4144
4145 static int link_detach(union bpf_attr *attr)
4146 {
4147         struct bpf_link *link;
4148         int ret;
4149
4150         if (CHECK_ATTR(BPF_LINK_DETACH))
4151                 return -EINVAL;
4152
4153         link = bpf_link_get_from_fd(attr->link_detach.link_fd);
4154         if (IS_ERR(link))
4155                 return PTR_ERR(link);
4156
4157         if (link->ops->detach)
4158                 ret = link->ops->detach(link);
4159         else
4160                 ret = -EOPNOTSUPP;
4161
4162         bpf_link_put(link);
4163         return ret;
4164 }
4165
4166 static struct bpf_link *bpf_link_inc_not_zero(struct bpf_link *link)
4167 {
4168         return atomic64_fetch_add_unless(&link->refcnt, 1, 0) ? link : ERR_PTR(-ENOENT);
4169 }
4170
4171 struct bpf_link *bpf_link_by_id(u32 id)
4172 {
4173         struct bpf_link *link;
4174
4175         if (!id)
4176                 return ERR_PTR(-ENOENT);
4177
4178         spin_lock_bh(&link_idr_lock);
4179         /* before link is "settled", ID is 0, pretend it doesn't exist yet */
4180         link = idr_find(&link_idr, id);
4181         if (link) {
4182                 if (link->id)
4183                         link = bpf_link_inc_not_zero(link);
4184                 else
4185                         link = ERR_PTR(-EAGAIN);
4186         } else {
4187                 link = ERR_PTR(-ENOENT);
4188         }
4189         spin_unlock_bh(&link_idr_lock);
4190         return link;
4191 }
4192
4193 #define BPF_LINK_GET_FD_BY_ID_LAST_FIELD link_id
4194
4195 static int bpf_link_get_fd_by_id(const union bpf_attr *attr)
4196 {
4197         struct bpf_link *link;
4198         u32 id = attr->link_id;
4199         int fd;
4200
4201         if (CHECK_ATTR(BPF_LINK_GET_FD_BY_ID))
4202                 return -EINVAL;
4203
4204         if (!capable(CAP_SYS_ADMIN))
4205                 return -EPERM;
4206
4207         link = bpf_link_by_id(id);
4208         if (IS_ERR(link))
4209                 return PTR_ERR(link);
4210
4211         fd = bpf_link_new_fd(link);
4212         if (fd < 0)
4213                 bpf_link_put(link);
4214
4215         return fd;
4216 }
4217
4218 DEFINE_MUTEX(bpf_stats_enabled_mutex);
4219
4220 static int bpf_stats_release(struct inode *inode, struct file *file)
4221 {
4222         mutex_lock(&bpf_stats_enabled_mutex);
4223         static_key_slow_dec(&bpf_stats_enabled_key.key);
4224         mutex_unlock(&bpf_stats_enabled_mutex);
4225         return 0;
4226 }
4227
4228 static const struct file_operations bpf_stats_fops = {
4229         .release = bpf_stats_release,
4230 };
4231
4232 static int bpf_enable_runtime_stats(void)
4233 {
4234         int fd;
4235
4236         mutex_lock(&bpf_stats_enabled_mutex);
4237
4238         /* Set a very high limit to avoid overflow */
4239         if (static_key_count(&bpf_stats_enabled_key.key) > INT_MAX / 2) {
4240                 mutex_unlock(&bpf_stats_enabled_mutex);
4241                 return -EBUSY;
4242         }
4243
4244         fd = anon_inode_getfd("bpf-stats", &bpf_stats_fops, NULL, O_CLOEXEC);
4245         if (fd >= 0)
4246                 static_key_slow_inc(&bpf_stats_enabled_key.key);
4247
4248         mutex_unlock(&bpf_stats_enabled_mutex);
4249         return fd;
4250 }
4251
4252 #define BPF_ENABLE_STATS_LAST_FIELD enable_stats.type
4253
4254 static int bpf_enable_stats(union bpf_attr *attr)
4255 {
4256
4257         if (CHECK_ATTR(BPF_ENABLE_STATS))
4258                 return -EINVAL;
4259
4260         if (!capable(CAP_SYS_ADMIN))
4261                 return -EPERM;
4262
4263         switch (attr->enable_stats.type) {
4264         case BPF_STATS_RUN_TIME:
4265                 return bpf_enable_runtime_stats();
4266         default:
4267                 break;
4268         }
4269         return -EINVAL;
4270 }
4271
4272 #define BPF_ITER_CREATE_LAST_FIELD iter_create.flags
4273
4274 static int bpf_iter_create(union bpf_attr *attr)
4275 {
4276         struct bpf_link *link;
4277         int err;
4278
4279         if (CHECK_ATTR(BPF_ITER_CREATE))
4280                 return -EINVAL;
4281
4282         if (attr->iter_create.flags)
4283                 return -EINVAL;
4284
4285         link = bpf_link_get_from_fd(attr->iter_create.link_fd);
4286         if (IS_ERR(link))
4287                 return PTR_ERR(link);
4288
4289         err = bpf_iter_new_fd(link);
4290         bpf_link_put(link);
4291
4292         return err;
4293 }
4294
4295 #define BPF_PROG_BIND_MAP_LAST_FIELD prog_bind_map.flags
4296
4297 static int bpf_prog_bind_map(union bpf_attr *attr)
4298 {
4299         struct bpf_prog *prog;
4300         struct bpf_map *map;
4301         struct bpf_map **used_maps_old, **used_maps_new;
4302         int i, ret = 0;
4303
4304         if (CHECK_ATTR(BPF_PROG_BIND_MAP))
4305                 return -EINVAL;
4306
4307         if (attr->prog_bind_map.flags)
4308                 return -EINVAL;
4309
4310         prog = bpf_prog_get(attr->prog_bind_map.prog_fd);
4311         if (IS_ERR(prog))
4312                 return PTR_ERR(prog);
4313
4314         map = bpf_map_get(attr->prog_bind_map.map_fd);
4315         if (IS_ERR(map)) {
4316                 ret = PTR_ERR(map);
4317                 goto out_prog_put;
4318         }
4319
4320         mutex_lock(&prog->aux->used_maps_mutex);
4321
4322         used_maps_old = prog->aux->used_maps;
4323
4324         for (i = 0; i < prog->aux->used_map_cnt; i++)
4325                 if (used_maps_old[i] == map) {
4326                         bpf_map_put(map);
4327                         goto out_unlock;
4328                 }
4329
4330         used_maps_new = kmalloc_array(prog->aux->used_map_cnt + 1,
4331                                       sizeof(used_maps_new[0]),
4332                                       GFP_KERNEL);
4333         if (!used_maps_new) {
4334                 ret = -ENOMEM;
4335                 goto out_unlock;
4336         }
4337
4338         memcpy(used_maps_new, used_maps_old,
4339                sizeof(used_maps_old[0]) * prog->aux->used_map_cnt);
4340         used_maps_new[prog->aux->used_map_cnt] = map;
4341
4342         prog->aux->used_map_cnt++;
4343         prog->aux->used_maps = used_maps_new;
4344
4345         kfree(used_maps_old);
4346
4347 out_unlock:
4348         mutex_unlock(&prog->aux->used_maps_mutex);
4349
4350         if (ret)
4351                 bpf_map_put(map);
4352 out_prog_put:
4353         bpf_prog_put(prog);
4354         return ret;
4355 }
4356
4357 SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size)
4358 {
4359         union bpf_attr attr;
4360         int err;
4361
4362         if (sysctl_unprivileged_bpf_disabled && !bpf_capable())
4363                 return -EPERM;
4364
4365         err = bpf_check_uarg_tail_zero(uattr, sizeof(attr), size);
4366         if (err)
4367                 return err;
4368         size = min_t(u32, size, sizeof(attr));
4369
4370         /* copy attributes from user space, may be less than sizeof(bpf_attr) */
4371         memset(&attr, 0, sizeof(attr));
4372         if (copy_from_user(&attr, uattr, size) != 0)
4373                 return -EFAULT;
4374
4375         err = security_bpf(cmd, &attr, size);
4376         if (err < 0)
4377                 return err;
4378
4379         switch (cmd) {
4380         case BPF_MAP_CREATE:
4381                 err = map_create(&attr);
4382                 break;
4383         case BPF_MAP_LOOKUP_ELEM:
4384                 err = map_lookup_elem(&attr);
4385                 break;
4386         case BPF_MAP_UPDATE_ELEM:
4387                 err = map_update_elem(&attr);
4388                 break;
4389         case BPF_MAP_DELETE_ELEM:
4390                 err = map_delete_elem(&attr);
4391                 break;
4392         case BPF_MAP_GET_NEXT_KEY:
4393                 err = map_get_next_key(&attr);
4394                 break;
4395         case BPF_MAP_FREEZE:
4396                 err = map_freeze(&attr);
4397                 break;
4398         case BPF_PROG_LOAD:
4399                 err = bpf_prog_load(&attr, uattr);
4400                 break;
4401         case BPF_OBJ_PIN:
4402                 err = bpf_obj_pin(&attr);
4403                 break;
4404         case BPF_OBJ_GET:
4405                 err = bpf_obj_get(&attr);
4406                 break;
4407         case BPF_PROG_ATTACH:
4408                 err = bpf_prog_attach(&attr);
4409                 break;
4410         case BPF_PROG_DETACH:
4411                 err = bpf_prog_detach(&attr);
4412                 break;
4413         case BPF_PROG_QUERY:
4414                 err = bpf_prog_query(&attr, uattr);
4415                 break;
4416         case BPF_PROG_TEST_RUN:
4417                 err = bpf_prog_test_run(&attr, uattr);
4418                 break;
4419         case BPF_PROG_GET_NEXT_ID:
4420                 err = bpf_obj_get_next_id(&attr, uattr,
4421                                           &prog_idr, &prog_idr_lock);
4422                 break;
4423         case BPF_MAP_GET_NEXT_ID:
4424                 err = bpf_obj_get_next_id(&attr, uattr,
4425                                           &map_idr, &map_idr_lock);
4426                 break;
4427         case BPF_BTF_GET_NEXT_ID:
4428                 err = bpf_obj_get_next_id(&attr, uattr,
4429                                           &btf_idr, &btf_idr_lock);
4430                 break;
4431         case BPF_PROG_GET_FD_BY_ID:
4432                 err = bpf_prog_get_fd_by_id(&attr);
4433                 break;
4434         case BPF_MAP_GET_FD_BY_ID:
4435                 err = bpf_map_get_fd_by_id(&attr);
4436                 break;
4437         case BPF_OBJ_GET_INFO_BY_FD:
4438                 err = bpf_obj_get_info_by_fd(&attr, uattr);
4439                 break;
4440         case BPF_RAW_TRACEPOINT_OPEN:
4441                 err = bpf_raw_tracepoint_open(&attr);
4442                 break;
4443         case BPF_BTF_LOAD:
4444                 err = bpf_btf_load(&attr);
4445                 break;
4446         case BPF_BTF_GET_FD_BY_ID:
4447                 err = bpf_btf_get_fd_by_id(&attr);
4448                 break;
4449         case BPF_TASK_FD_QUERY:
4450                 err = bpf_task_fd_query(&attr, uattr);
4451                 break;
4452         case BPF_MAP_LOOKUP_AND_DELETE_ELEM:
4453                 err = map_lookup_and_delete_elem(&attr);
4454                 break;
4455         case BPF_MAP_LOOKUP_BATCH:
4456                 err = bpf_map_do_batch(&attr, uattr, BPF_MAP_LOOKUP_BATCH);
4457                 break;
4458         case BPF_MAP_LOOKUP_AND_DELETE_BATCH:
4459                 err = bpf_map_do_batch(&attr, uattr,
4460                                        BPF_MAP_LOOKUP_AND_DELETE_BATCH);
4461                 break;
4462         case BPF_MAP_UPDATE_BATCH:
4463                 err = bpf_map_do_batch(&attr, uattr, BPF_MAP_UPDATE_BATCH);
4464                 break;
4465         case BPF_MAP_DELETE_BATCH:
4466                 err = bpf_map_do_batch(&attr, uattr, BPF_MAP_DELETE_BATCH);
4467                 break;
4468         case BPF_LINK_CREATE:
4469                 err = link_create(&attr);
4470                 break;
4471         case BPF_LINK_UPDATE:
4472                 err = link_update(&attr);
4473                 break;
4474         case BPF_LINK_GET_FD_BY_ID:
4475                 err = bpf_link_get_fd_by_id(&attr);
4476                 break;
4477         case BPF_LINK_GET_NEXT_ID:
4478                 err = bpf_obj_get_next_id(&attr, uattr,
4479                                           &link_idr, &link_idr_lock);
4480                 break;
4481         case BPF_ENABLE_STATS:
4482                 err = bpf_enable_stats(&attr);
4483                 break;
4484         case BPF_ITER_CREATE:
4485                 err = bpf_iter_create(&attr);
4486                 break;
4487         case BPF_LINK_DETACH:
4488                 err = link_detach(&attr);
4489                 break;
4490         case BPF_PROG_BIND_MAP:
4491                 err = bpf_prog_bind_map(&attr);
4492                 break;
4493         default:
4494                 err = -EINVAL;
4495                 break;
4496         }
4497
4498         return err;
4499 }