kernel/bpf/syscall.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
   3  */
   4 #include <linux/bpf.h>
   5 #include <linux/bpf_trace.h>
   6 #include <linux/bpf_lirc.h>
   7 #include <linux/bpf_verifier.h>
   8 #include <linux/btf.h>
   9 #include <linux/syscalls.h>
  10 #include <linux/slab.h>
  11 #include <linux/sched/signal.h>
  12 #include <linux/vmalloc.h>
  13 #include <linux/mmzone.h>
  14 #include <linux/anon_inodes.h>
  15 #include <linux/fdtable.h>
  16 #include <linux/file.h>
  17 #include <linux/fs.h>
  18 #include <linux/license.h>
  19 #include <linux/filter.h>
  20 #include <linux/kernel.h>
  21 #include <linux/idr.h>
  22 #include <linux/cred.h>
  23 #include <linux/timekeeping.h>
  24 #include <linux/ctype.h>
  25 #include <linux/nospec.h>
  26 #include <linux/audit.h>
  27 #include <uapi/linux/btf.h>
  28 #include <linux/pgtable.h>
  29 #include <linux/bpf_lsm.h>
  30 #include <linux/poll.h>
  31 #include <linux/bpf-netns.h>
  32 #include <linux/rcupdate_trace.h>
  33 #include <linux/memcontrol.h>
  34
  35 #define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \
  36                           (map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \
  37                           (map)->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS)
  38 #define IS_FD_PROG_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PROG_ARRAY)
  39 #define IS_FD_HASH(map) ((map)->map_type == BPF_MAP_TYPE_HASH_OF_MAPS)
  40 #define IS_FD_MAP(map) (IS_FD_ARRAY(map) || IS_FD_PROG_ARRAY(map) || \
  41                         IS_FD_HASH(map))
  42
  43 #define BPF_OBJ_FLAG_MASK   (BPF_F_RDONLY | BPF_F_WRONLY)
  44
  45 DEFINE_PER_CPU(int, bpf_prog_active);
  46 static DEFINE_IDR(prog_idr);
  47 static DEFINE_SPINLOCK(prog_idr_lock);
  48 static DEFINE_IDR(map_idr);
  49 static DEFINE_SPINLOCK(map_idr_lock);
  50 static DEFINE_IDR(link_idr);
  51 static DEFINE_SPINLOCK(link_idr_lock);
  52
  53 int sysctl_unprivileged_bpf_disabled __read_mostly =
  54         IS_BUILTIN(CONFIG_BPF_UNPRIV_DEFAULT_OFF) ? 2 : 0;
  55
  56 static const struct bpf_map_ops * const bpf_map_types[] = {
  57 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type)
  58 #define BPF_MAP_TYPE(_id, _ops) \
  59         [_id] = &_ops,
  60 #define BPF_LINK_TYPE(_id, _name)
  61 #include <linux/bpf_types.h>
  62 #undef BPF_PROG_TYPE
  63 #undef BPF_MAP_TYPE
  64 #undef BPF_LINK_TYPE
  65 };
  66
  67 /*
  68  * If we're handed a bigger struct than we know of, ensure all the unknown bits
  69  * are 0 - i.e. new user-space does not rely on any kernel feature extensions
  70  * we don't know about yet.
  71  *
  72  * There is a ToCToU between this function call and the following
  73  * copy_from_user() call. However, this is not a concern since this function is
  74  * meant to be a future-proofing of bits.
  75  */
  76 int bpf_check_uarg_tail_zero(bpfptr_t uaddr,
  77                              size_t expected_size,
  78                              size_t actual_size)
  79 {
  80         int res;
  81
  82         if (unlikely(actual_size > PAGE_SIZE))  /* silly large */
  83                 return -E2BIG;
  84
  85         if (actual_size <= expected_size)
  86                 return 0;
  87
  88         if (uaddr.is_kernel)
  89                 res = memchr_inv(uaddr.kernel + expected_size, 0,
  90                                  actual_size - expected_size) == NULL;
  91         else
  92                 res = check_zeroed_user(uaddr.user + expected_size,
  93                                         actual_size - expected_size);
  94         if (res < 0)
  95                 return res;
  96         return res ? 0 : -E2BIG;
  97 }
  98
  99 const struct bpf_map_ops bpf_map_offload_ops = {
 100         .map_meta_equal = bpf_map_meta_equal,
 101         .map_alloc = bpf_map_offload_map_alloc,
 102         .map_free = bpf_map_offload_map_free,
 103         .map_check_btf = map_check_no_btf,
 104 };
 105
 106 static struct bpf_map *find_and_alloc_map(union bpf_attr *attr)
 107 {
 108         const struct bpf_map_ops *ops;
 109         u32 type = attr->map_type;
 110         struct bpf_map *map;
 111         int err;
 112
 113         if (type >= ARRAY_SIZE(bpf_map_types))
 114                 return ERR_PTR(-EINVAL);
 115         type = array_index_nospec(type, ARRAY_SIZE(bpf_map_types));
 116         ops = bpf_map_types[type];
 117         if (!ops)
 118                 return ERR_PTR(-EINVAL);
 119
 120         if (ops->map_alloc_check) {
 121                 err = ops->map_alloc_check(attr);
 122                 if (err)
 123                         return ERR_PTR(err);
 124         }
 125         if (attr->map_ifindex)
 126                 ops = &bpf_map_offload_ops;
 127         map = ops->map_alloc(attr);
 128         if (IS_ERR(map))
 129                 return map;
 130         map->ops = ops;
 131         map->map_type = type;
 132         return map;
 133 }
 134
 135 static u32 bpf_map_value_size(const struct bpf_map *map)
 136 {
 137         if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
 138             map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||
 139             map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY ||
 140             map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE)
 141                 return round_up(map->value_size, 8) * num_possible_cpus();
 142         else if (IS_FD_MAP(map))
 143                 return sizeof(u32);
 144         else
 145                 return  map->value_size;
 146 }
 147
 148 static void maybe_wait_bpf_programs(struct bpf_map *map)
 149 {
 150         /* Wait for any running BPF programs to complete so that
 151          * userspace, when we return to it, knows that all programs
 152          * that could be running use the new map value.
 153          */
 154         if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS ||
 155             map->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS)
 156                 synchronize_rcu();
 157 }
 158
 159 static int bpf_map_update_value(struct bpf_map *map, struct fd f, void *key,
 160                                 void *value, __u64 flags)
 161 {
 162         int err;
 163
 164         /* Need to create a kthread, thus must support schedule */
 165         if (bpf_map_is_dev_bound(map)) {
 166                 return bpf_map_offload_update_elem(map, key, value, flags);
 167         } else if (map->map_type == BPF_MAP_TYPE_CPUMAP ||
 168                    map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
 169                 return map->ops->map_update_elem(map, key, value, flags);
 170         } else if (map->map_type == BPF_MAP_TYPE_SOCKHASH ||
 171                    map->map_type == BPF_MAP_TYPE_SOCKMAP) {
 172                 return sock_map_update_elem_sys(map, key, value, flags);
 173         } else if (IS_FD_PROG_ARRAY(map)) {
 174                 return bpf_fd_array_map_update_elem(map, f.file, key, value,
 175                                                     flags);
 176         }
 177
 178         bpf_disable_instrumentation();
 179         if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
 180             map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
 181                 err = bpf_percpu_hash_update(map, key, value, flags);
 182         } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
 183                 err = bpf_percpu_array_update(map, key, value, flags);
 184         } else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) {
 185                 err = bpf_percpu_cgroup_storage_update(map, key, value,
 186                                                        flags);
 187         } else if (IS_FD_ARRAY(map)) {
 188                 rcu_read_lock();
 189                 err = bpf_fd_array_map_update_elem(map, f.file, key, value,
 190                                                    flags);
 191                 rcu_read_unlock();
 192         } else if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) {
 193                 rcu_read_lock();
 194                 err = bpf_fd_htab_map_update_elem(map, f.file, key, value,
 195                                                   flags);
 196                 rcu_read_unlock();
 197         } else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) {
 198                 /* rcu_read_lock() is not needed */
 199                 err = bpf_fd_reuseport_array_update_elem(map, key, value,
 200                                                          flags);
 201         } else if (map->map_type == BPF_MAP_TYPE_QUEUE ||
 202                    map->map_type == BPF_MAP_TYPE_STACK) {
 203                 err = map->ops->map_push_elem(map, value, flags);
 204         } else {
 205                 rcu_read_lock();
 206                 err = map->ops->map_update_elem(map, key, value, flags);
 207                 rcu_read_unlock();
 208         }
 209         bpf_enable_instrumentation();
 210         maybe_wait_bpf_programs(map);
 211
 212         return err;
 213 }
 214
 215 static int bpf_map_copy_value(struct bpf_map *map, void *key, void *value,
 216                               __u64 flags)
 217 {
 218         void *ptr;
 219         int err;
 220
 221         if (bpf_map_is_dev_bound(map))
 222                 return bpf_map_offload_lookup_elem(map, key, value);
 223
 224         bpf_disable_instrumentation();
 225         if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
 226             map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
 227                 err = bpf_percpu_hash_copy(map, key, value);
 228         } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
 229                 err = bpf_percpu_array_copy(map, key, value);
 230         } else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) {
 231                 err = bpf_percpu_cgroup_storage_copy(map, key, value);
 232         } else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) {
 233                 err = bpf_stackmap_copy(map, key, value);
 234         } else if (IS_FD_ARRAY(map) || IS_FD_PROG_ARRAY(map)) {
 235                 err = bpf_fd_array_map_lookup_elem(map, key, value);
 236         } else if (IS_FD_HASH(map)) {
 237                 err = bpf_fd_htab_map_lookup_elem(map, key, value);
 238         } else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) {
 239                 err = bpf_fd_reuseport_array_lookup_elem(map, key, value);
 240         } else if (map->map_type == BPF_MAP_TYPE_QUEUE ||
 241                    map->map_type == BPF_MAP_TYPE_STACK) {
 242                 err = map->ops->map_peek_elem(map, value);
 243         } else if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
 244                 /* struct_ops map requires directly updating "value" */
 245                 err = bpf_struct_ops_map_sys_lookup_elem(map, key, value);
 246         } else {
 247                 rcu_read_lock();
 248                 if (map->ops->map_lookup_elem_sys_only)
 249                         ptr = map->ops->map_lookup_elem_sys_only(map, key);
 250                 else
 251                         ptr = map->ops->map_lookup_elem(map, key);
 252                 if (IS_ERR(ptr)) {
 253                         err = PTR_ERR(ptr);
 254                 } else if (!ptr) {
 255                         err = -ENOENT;
 256                 } else {
 257                         err = 0;
 258                         if (flags & BPF_F_LOCK)
 259                                 /* lock 'ptr' and copy everything but lock */
 260                                 copy_map_value_locked(map, value, ptr, true);
 261                         else
 262                                 copy_map_value(map, value, ptr);
 263                         /* mask lock and timer, since value wasn't zero inited */
 264                         check_and_init_map_value(map, value);
 265                 }
 266                 rcu_read_unlock();
 267         }
 268
 269         bpf_enable_instrumentation();
 270         maybe_wait_bpf_programs(map);
 271
 272         return err;
 273 }
 274
 275 /* Please, do not use this function outside from the map creation path
 276  * (e.g. in map update path) without taking care of setting the active
 277  * memory cgroup (see at bpf_map_kmalloc_node() for example).
 278  */
 279 static void *__bpf_map_area_alloc(u64 size, int numa_node, bool mmapable)
 280 {
 281         /* We really just want to fail instead of triggering OOM killer
 282          * under memory pressure, therefore we set __GFP_NORETRY to kmalloc,
 283          * which is used for lower order allocation requests.
 284          *
 285          * It has been observed that higher order allocation requests done by
 286          * vmalloc with __GFP_NORETRY being set might fail due to not trying
 287          * to reclaim memory from the page cache, thus we set
 288          * __GFP_RETRY_MAYFAIL to avoid such situations.
 289          */
 290
 291         const gfp_t gfp = __GFP_NOWARN | __GFP_ZERO | __GFP_ACCOUNT;
 292         unsigned int flags = 0;
 293         unsigned long align = 1;
 294         void *area;
 295
 296         if (size >= SIZE_MAX)
 297                 return NULL;
 298
 299         /* kmalloc()'ed memory can't be mmap()'ed */
 300         if (mmapable) {
 301                 BUG_ON(!PAGE_ALIGNED(size));
 302                 align = SHMLBA;
 303                 flags = VM_USERMAP;
 304         } else if (size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) {
 305                 area = kmalloc_node(size, gfp | GFP_USER | __GFP_NORETRY,
 306                                     numa_node);
 307                 if (area != NULL)
 308                         return area;
 309         }
 310
 311         return __vmalloc_node_range(size, align, VMALLOC_START, VMALLOC_END,
 312                         gfp | GFP_KERNEL | __GFP_RETRY_MAYFAIL, PAGE_KERNEL,
 313                         flags, numa_node, __builtin_return_address(0));
 314 }
 315
 316 void *bpf_map_area_alloc(u64 size, int numa_node)
 317 {
 318         return __bpf_map_area_alloc(size, numa_node, false);
 319 }
 320
 321 void *bpf_map_area_mmapable_alloc(u64 size, int numa_node)
 322 {
 323         return __bpf_map_area_alloc(size, numa_node, true);
 324 }
 325
 326 void bpf_map_area_free(void *area)
 327 {
 328         kvfree(area);
 329 }
 330
 331 static u32 bpf_map_flags_retain_permanent(u32 flags)
 332 {
 333         /* Some map creation flags are not tied to the map object but
 334          * rather to the map fd instead, so they have no meaning upon
 335          * map object inspection since multiple file descriptors with
 336          * different (access) properties can exist here. Thus, given
 337          * this has zero meaning for the map itself, lets clear these
 338          * from here.
 339          */
 340         return flags & ~(BPF_F_RDONLY | BPF_F_WRONLY);
 341 }
 342
 343 void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr)
 344 {
 345         map->map_type = attr->map_type;
 346         map->key_size = attr->key_size;
 347         map->value_size = attr->value_size;
 348         map->max_entries = attr->max_entries;
 349         map->map_flags = bpf_map_flags_retain_permanent(attr->map_flags);
 350         map->numa_node = bpf_map_attr_numa_node(attr);
 351 }
 352
 353 static int bpf_map_alloc_id(struct bpf_map *map)
 354 {
 355         int id;
 356
 357         idr_preload(GFP_KERNEL);
 358         spin_lock_bh(&map_idr_lock);
 359         id = idr_alloc_cyclic(&map_idr, map, 1, INT_MAX, GFP_ATOMIC);
 360         if (id > 0)
 361                 map->id = id;
 362         spin_unlock_bh(&map_idr_lock);
 363         idr_preload_end();
 364
 365         if (WARN_ON_ONCE(!id))
 366                 return -ENOSPC;
 367
 368         return id > 0 ? 0 : id;
 369 }
 370
 371 void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock)
 372 {
 373         unsigned long flags;
 374
 375         /* Offloaded maps are removed from the IDR store when their device
 376          * disappears - even if someone holds an fd to them they are unusable,
 377          * the memory is gone, all ops will fail; they are simply waiting for
 378          * refcnt to drop to be freed.
 379          */
 380         if (!map->id)
 381                 return;
 382
 383         if (do_idr_lock)
 384                 spin_lock_irqsave(&map_idr_lock, flags);
 385         else
 386                 __acquire(&map_idr_lock);
 387
 388         idr_remove(&map_idr, map->id);
 389         map->id = 0;
 390
 391         if (do_idr_lock)
 392                 spin_unlock_irqrestore(&map_idr_lock, flags);
 393         else
 394                 __release(&map_idr_lock);
 395 }
 396
 397 #ifdef CONFIG_MEMCG_KMEM
 398 static void bpf_map_save_memcg(struct bpf_map *map)
 399 {
 400         map->memcg = get_mem_cgroup_from_mm(current->mm);
 401 }
 402
 403 static void bpf_map_release_memcg(struct bpf_map *map)
 404 {
 405         mem_cgroup_put(map->memcg);
 406 }
 407
 408 void *bpf_map_kmalloc_node(const struct bpf_map *map, size_t size, gfp_t flags,
 409                            int node)
 410 {
 411         struct mem_cgroup *old_memcg;
 412         void *ptr;
 413
 414         old_memcg = set_active_memcg(map->memcg);
 415         ptr = kmalloc_node(size, flags | __GFP_ACCOUNT, node);
 416         set_active_memcg(old_memcg);
 417
 418         return ptr;
 419 }
 420
 421 void *bpf_map_kzalloc(const struct bpf_map *map, size_t size, gfp_t flags)
 422 {
 423         struct mem_cgroup *old_memcg;
 424         void *ptr;
 425
 426         old_memcg = set_active_memcg(map->memcg);
 427         ptr = kzalloc(size, flags | __GFP_ACCOUNT);
 428         set_active_memcg(old_memcg);
 429
 430         return ptr;
 431 }
 432
 433 void __percpu *bpf_map_alloc_percpu(const struct bpf_map *map, size_t size,
 434                                     size_t align, gfp_t flags)
 435 {
 436         struct mem_cgroup *old_memcg;
 437         void __percpu *ptr;
 438
 439         old_memcg = set_active_memcg(map->memcg);
 440         ptr = __alloc_percpu_gfp(size, align, flags | __GFP_ACCOUNT);
 441         set_active_memcg(old_memcg);
 442
 443         return ptr;
 444 }
 445
 446 #else
 447 static void bpf_map_save_memcg(struct bpf_map *map)
 448 {
 449 }
 450
 451 static void bpf_map_release_memcg(struct bpf_map *map)
 452 {
 453 }
 454 #endif
 455
 456 /* called from workqueue */
 457 static void bpf_map_free_deferred(struct work_struct *work)
 458 {
 459         struct bpf_map *map = container_of(work, struct bpf_map, work);
 460
 461         security_bpf_map_free(map);
 462         bpf_map_release_memcg(map);
 463         /* implementation dependent freeing */
 464         map->ops->map_free(map);
 465 }
 466
 467 static void bpf_map_put_uref(struct bpf_map *map)
 468 {
 469         if (atomic64_dec_and_test(&map->usercnt)) {
 470                 if (map->ops->map_release_uref)
 471                         map->ops->map_release_uref(map);
 472         }
 473 }
 474
 475 /* decrement map refcnt and schedule it for freeing via workqueue
 476  * (unrelying map implementation ops->map_free() might sleep)
 477  */
 478 static void __bpf_map_put(struct bpf_map *map, bool do_idr_lock)
 479 {
 480         if (atomic64_dec_and_test(&map->refcnt)) {
 481                 /* bpf_map_free_id() must be called first */
 482                 bpf_map_free_id(map, do_idr_lock);
 483                 btf_put(map->btf);
 484                 INIT_WORK(&map->work, bpf_map_free_deferred);
 485                 schedule_work(&map->work);
 486         }
 487 }
 488
 489 void bpf_map_put(struct bpf_map *map)
 490 {
 491         __bpf_map_put(map, true);
 492 }
 493 EXPORT_SYMBOL_GPL(bpf_map_put);
 494
 495 void bpf_map_put_with_uref(struct bpf_map *map)
 496 {
 497         bpf_map_put_uref(map);
 498         bpf_map_put(map);
 499 }
 500
 501 static int bpf_map_release(struct inode *inode, struct file *filp)
 502 {
 503         struct bpf_map *map = filp->private_data;
 504
 505         if (map->ops->map_release)
 506                 map->ops->map_release(map, filp);
 507
 508         bpf_map_put_with_uref(map);
 509         return 0;
 510 }
 511
 512 static fmode_t map_get_sys_perms(struct bpf_map *map, struct fd f)
 513 {
 514         fmode_t mode = f.file->f_mode;
 515
 516         /* Our file permissions may have been overridden by global
 517          * map permissions facing syscall side.
 518          */
 519         if (READ_ONCE(map->frozen))
 520                 mode &= ~FMODE_CAN_WRITE;
 521         return mode;
 522 }
 523
 524 #ifdef CONFIG_PROC_FS
 525 /* Provides an approximation of the map's memory footprint.
 526  * Used only to provide a backward compatibility and display
 527  * a reasonable "memlock" info.
 528  */
 529 static unsigned long bpf_map_memory_footprint(const struct bpf_map *map)
 530 {
 531         unsigned long size;
 532
 533         size = round_up(map->key_size + bpf_map_value_size(map), 8);
 534
 535         return round_up(map->max_entries * size, PAGE_SIZE);
 536 }
 537
 538 static void bpf_map_show_fdinfo(struct seq_file *m, struct file *filp)
 539 {
 540         const struct bpf_map *map = filp->private_data;
 541         const struct bpf_array *array;
 542         u32 type = 0, jited = 0;
 543
 544         if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY) {
 545                 array = container_of(map, struct bpf_array, map);
 546                 type  = array->aux->type;
 547                 jited = array->aux->jited;
 548         }
 549
 550         seq_printf(m,
 551                    "map_type:\t%u\n"
 552                    "key_size:\t%u\n"
 553                    "value_size:\t%u\n"
 554                    "max_entries:\t%u\n"
 555                    "map_flags:\t%#x\n"
 556                    "memlock:\t%lu\n"
 557                    "map_id:\t%u\n"
 558                    "frozen:\t%u\n",
 559                    map->map_type,
 560                    map->key_size,
 561                    map->value_size,
 562                    map->max_entries,
 563                    map->map_flags,
 564                    bpf_map_memory_footprint(map),
 565                    map->id,
 566                    READ_ONCE(map->frozen));
 567         if (type) {
 568                 seq_printf(m, "owner_prog_type:\t%u\n", type);
 569                 seq_printf(m, "owner_jited:\t%u\n", jited);
 570         }
 571 }
 572 #endif
 573
 574 static ssize_t bpf_dummy_read(struct file *filp, char __user *buf, size_t siz,
 575                               loff_t *ppos)
 576 {
 577         /* We need this handler such that alloc_file() enables
 578          * f_mode with FMODE_CAN_READ.
 579          */
 580         return -EINVAL;
 581 }
 582
 583 static ssize_t bpf_dummy_write(struct file *filp, const char __user *buf,
 584                                size_t siz, loff_t *ppos)
 585 {
 586         /* We need this handler such that alloc_file() enables
 587          * f_mode with FMODE_CAN_WRITE.
 588          */
 589         return -EINVAL;
 590 }
 591
 592 /* called for any extra memory-mapped regions (except initial) */
 593 static void bpf_map_mmap_open(struct vm_area_struct *vma)
 594 {
 595         struct bpf_map *map = vma->vm_file->private_data;
 596
 597         if (vma->vm_flags & VM_MAYWRITE) {
 598                 mutex_lock(&map->freeze_mutex);
 599                 map->writecnt++;
 600                 mutex_unlock(&map->freeze_mutex);
 601         }
 602 }
 603
 604 /* called for all unmapped memory region (including initial) */
 605 static void bpf_map_mmap_close(struct vm_area_struct *vma)
 606 {
 607         struct bpf_map *map = vma->vm_file->private_data;
 608
 609         if (vma->vm_flags & VM_MAYWRITE) {
 610                 mutex_lock(&map->freeze_mutex);
 611                 map->writecnt--;
 612                 mutex_unlock(&map->freeze_mutex);
 613         }
 614 }
 615
 616 static const struct vm_operations_struct bpf_map_default_vmops = {
 617         .open           = bpf_map_mmap_open,
 618         .close          = bpf_map_mmap_close,
 619 };
 620
 621 static int bpf_map_mmap(struct file *filp, struct vm_area_struct *vma)
 622 {
 623         struct bpf_map *map = filp->private_data;
 624         int err;
 625
 626         if (!map->ops->map_mmap || map_value_has_spin_lock(map) ||
 627             map_value_has_timer(map))
 628                 return -ENOTSUPP;
 629
 630         if (!(vma->vm_flags & VM_SHARED))
 631                 return -EINVAL;
 632
 633         mutex_lock(&map->freeze_mutex);
 634
 635         if (vma->vm_flags & VM_WRITE) {
 636                 if (map->frozen) {
 637                         err = -EPERM;
 638                         goto out;
 639                 }
 640                 /* map is meant to be read-only, so do not allow mapping as
 641                  * writable, because it's possible to leak a writable page
 642                  * reference and allows user-space to still modify it after
 643                  * freezing, while verifier will assume contents do not change
 644                  */
 645                 if (map->map_flags & BPF_F_RDONLY_PROG) {
 646                         err = -EACCES;
 647                         goto out;
 648                 }
 649         }
 650
 651         /* set default open/close callbacks */
 652         vma->vm_ops = &bpf_map_default_vmops;
 653         vma->vm_private_data = map;
 654         vma->vm_flags &= ~VM_MAYEXEC;
 655         if (!(vma->vm_flags & VM_WRITE))
 656                 /* disallow re-mapping with PROT_WRITE */
 657                 vma->vm_flags &= ~VM_MAYWRITE;
 658
 659         err = map->ops->map_mmap(map, vma);
 660         if (err)
 661                 goto out;
 662
 663         if (vma->vm_flags & VM_MAYWRITE)
 664                 map->writecnt++;
 665 out:
 666         mutex_unlock(&map->freeze_mutex);
 667         return err;
 668 }
 669
 670 static __poll_t bpf_map_poll(struct file *filp, struct poll_table_struct *pts)
 671 {
 672         struct bpf_map *map = filp->private_data;
 673
 674         if (map->ops->map_poll)
 675                 return map->ops->map_poll(map, filp, pts);
 676
 677         return EPOLLERR;
 678 }
 679
 680 const struct file_operations bpf_map_fops = {
 681 #ifdef CONFIG_PROC_FS
 682         .show_fdinfo    = bpf_map_show_fdinfo,
 683 #endif
 684         .release        = bpf_map_release,
 685         .read           = bpf_dummy_read,
 686         .write          = bpf_dummy_write,
 687         .mmap           = bpf_map_mmap,
 688         .poll           = bpf_map_poll,
 689 };
 690
 691 int bpf_map_new_fd(struct bpf_map *map, int flags)
 692 {
 693         int ret;
 694
 695         ret = security_bpf_map(map, OPEN_FMODE(flags));
 696         if (ret < 0)
 697                 return ret;
 698
 699         return anon_inode_getfd("bpf-map", &bpf_map_fops, map,
 700                                 flags | O_CLOEXEC);
 701 }
 702
 703 int bpf_get_file_flag(int flags)
 704 {
 705         if ((flags & BPF_F_RDONLY) && (flags & BPF_F_WRONLY))
 706                 return -EINVAL;
 707         if (flags & BPF_F_RDONLY)
 708                 return O_RDONLY;
 709         if (flags & BPF_F_WRONLY)
 710                 return O_WRONLY;
 711         return O_RDWR;
 712 }
 713
 714 /* helper macro to check that unused fields 'union bpf_attr' are zero */
 715 #define CHECK_ATTR(CMD) \
 716         memchr_inv((void *) &attr->CMD##_LAST_FIELD + \
 717                    sizeof(attr->CMD##_LAST_FIELD), 0, \
 718                    sizeof(*attr) - \
 719                    offsetof(union bpf_attr, CMD##_LAST_FIELD) - \
 720                    sizeof(attr->CMD##_LAST_FIELD)) != NULL
 721
 722 /* dst and src must have at least "size" number of bytes.
 723  * Return strlen on success and < 0 on error.
 724  */
 725 int bpf_obj_name_cpy(char *dst, const char *src, unsigned int size)
 726 {
 727         const char *end = src + size;
 728         const char *orig_src = src;
 729
 730         memset(dst, 0, size);
 731         /* Copy all isalnum(), '_' and '.' chars. */
 732         while (src < end && *src) {
 733                 if (!isalnum(*src) &&
 734                     *src != '_' && *src != '.')
 735                         return -EINVAL;
 736                 *dst++ = *src++;
 737         }
 738
 739         /* No '\0' found in "size" number of bytes */
 740         if (src == end)
 741                 return -EINVAL;
 742
 743         return src - orig_src;
 744 }
 745
 746 int map_check_no_btf(const struct bpf_map *map,
 747                      const struct btf *btf,
 748                      const struct btf_type *key_type,
 749                      const struct btf_type *value_type)
 750 {
 751         return -ENOTSUPP;
 752 }
 753
 754 static int map_check_btf(struct bpf_map *map, const struct btf *btf,
 755                          u32 btf_key_id, u32 btf_value_id)
 756 {
 757         const struct btf_type *key_type, *value_type;
 758         u32 key_size, value_size;
 759         int ret = 0;
 760
 761         /* Some maps allow key to be unspecified. */
 762         if (btf_key_id) {
 763                 key_type = btf_type_id_size(btf, &btf_key_id, &key_size);
 764                 if (!key_type || key_size != map->key_size)
 765                         return -EINVAL;
 766         } else {
 767                 key_type = btf_type_by_id(btf, 0);
 768                 if (!map->ops->map_check_btf)
 769                         return -EINVAL;
 770         }
 771
 772         value_type = btf_type_id_size(btf, &btf_value_id, &value_size);
 773         if (!value_type || value_size != map->value_size)
 774                 return -EINVAL;
 775
 776         map->spin_lock_off = btf_find_spin_lock(btf, value_type);
 777
 778         if (map_value_has_spin_lock(map)) {
 779                 if (map->map_flags & BPF_F_RDONLY_PROG)
 780                         return -EACCES;
 781                 if (map->map_type != BPF_MAP_TYPE_HASH &&
 782                     map->map_type != BPF_MAP_TYPE_ARRAY &&
 783                     map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE &&
 784                     map->map_type != BPF_MAP_TYPE_SK_STORAGE &&
 785                     map->map_type != BPF_MAP_TYPE_INODE_STORAGE &&
 786                     map->map_type != BPF_MAP_TYPE_TASK_STORAGE)
 787                         return -ENOTSUPP;
 788                 if (map->spin_lock_off + sizeof(struct bpf_spin_lock) >
 789                     map->value_size) {
 790                         WARN_ONCE(1,
 791                                   "verifier bug spin_lock_off %d value_size %d\n",
 792                                   map->spin_lock_off, map->value_size);
 793                         return -EFAULT;
 794                 }
 795         }
 796
 797         map->timer_off = btf_find_timer(btf, value_type);
 798         if (map_value_has_timer(map)) {
 799                 if (map->map_flags & BPF_F_RDONLY_PROG)
 800                         return -EACCES;
 801                 if (map->map_type != BPF_MAP_TYPE_HASH &&
 802                     map->map_type != BPF_MAP_TYPE_LRU_HASH &&
 803                     map->map_type != BPF_MAP_TYPE_ARRAY)
 804                         return -EOPNOTSUPP;
 805         }
 806
 807         if (map->ops->map_check_btf)
 808                 ret = map->ops->map_check_btf(map, btf, key_type, value_type);
 809
 810         return ret;
 811 }
 812
 813 #define BPF_MAP_CREATE_LAST_FIELD btf_vmlinux_value_type_id
 814 /* called via syscall */
 815 static int map_create(union bpf_attr *attr)
 816 {
 817         int numa_node = bpf_map_attr_numa_node(attr);
 818         struct bpf_map *map;
 819         int f_flags;
 820         int err;
 821
 822         err = CHECK_ATTR(BPF_MAP_CREATE);
 823         if (err)
 824                 return -EINVAL;
 825
 826         if (attr->btf_vmlinux_value_type_id) {
 827                 if (attr->map_type != BPF_MAP_TYPE_STRUCT_OPS ||
 828                     attr->btf_key_type_id || attr->btf_value_type_id)
 829                         return -EINVAL;
 830         } else if (attr->btf_key_type_id && !attr->btf_value_type_id) {
 831                 return -EINVAL;
 832         }
 833
 834         f_flags = bpf_get_file_flag(attr->map_flags);
 835         if (f_flags < 0)
 836                 return f_flags;
 837
 838         if (numa_node != NUMA_NO_NODE &&
 839             ((unsigned int)numa_node >= nr_node_ids ||
 840              !node_online(numa_node)))
 841                 return -EINVAL;
 842
 843         /* find map type and init map: hashtable vs rbtree vs bloom vs ... */
 844         map = find_and_alloc_map(attr);
 845         if (IS_ERR(map))
 846                 return PTR_ERR(map);
 847
 848         err = bpf_obj_name_cpy(map->name, attr->map_name,
 849                                sizeof(attr->map_name));
 850         if (err < 0)
 851                 goto free_map;
 852
 853         atomic64_set(&map->refcnt, 1);
 854         atomic64_set(&map->usercnt, 1);
 855         mutex_init(&map->freeze_mutex);
 856
 857         map->spin_lock_off = -EINVAL;
 858         map->timer_off = -EINVAL;
 859         if (attr->btf_key_type_id || attr->btf_value_type_id ||
 860             /* Even the map's value is a kernel's struct,
 861              * the bpf_prog.o must have BTF to begin with
 862              * to figure out the corresponding kernel's
 863              * counter part.  Thus, attr->btf_fd has
 864              * to be valid also.
 865              */
 866             attr->btf_vmlinux_value_type_id) {
 867                 struct btf *btf;
 868
 869                 btf = btf_get_by_fd(attr->btf_fd);
 870                 if (IS_ERR(btf)) {
 871                         err = PTR_ERR(btf);
 872                         goto free_map;
 873                 }
 874                 if (btf_is_kernel(btf)) {
 875                         btf_put(btf);
 876                         err = -EACCES;
 877                         goto free_map;
 878                 }
 879                 map->btf = btf;
 880
 881                 if (attr->btf_value_type_id) {
 882                         err = map_check_btf(map, btf, attr->btf_key_type_id,
 883                                             attr->btf_value_type_id);
 884                         if (err)
 885                                 goto free_map;
 886                 }
 887
 888                 map->btf_key_type_id = attr->btf_key_type_id;
 889                 map->btf_value_type_id = attr->btf_value_type_id;
 890                 map->btf_vmlinux_value_type_id =
 891                         attr->btf_vmlinux_value_type_id;
 892         }
 893
 894         err = security_bpf_map_alloc(map);
 895         if (err)
 896                 goto free_map;
 897
 898         err = bpf_map_alloc_id(map);
 899         if (err)
 900                 goto free_map_sec;
 901
 902         bpf_map_save_memcg(map);
 903
 904         err = bpf_map_new_fd(map, f_flags);
 905         if (err < 0) {
 906                 /* failed to allocate fd.
 907                  * bpf_map_put_with_uref() is needed because the above
 908                  * bpf_map_alloc_id() has published the map
 909                  * to the userspace and the userspace may
 910                  * have refcnt-ed it through BPF_MAP_GET_FD_BY_ID.
 911                  */
 912                 bpf_map_put_with_uref(map);
 913                 return err;
 914         }
 915
 916         return err;
 917
 918 free_map_sec:
 919         security_bpf_map_free(map);
 920 free_map:
 921         btf_put(map->btf);
 922         map->ops->map_free(map);
 923         return err;
 924 }
 925
 926 /* if error is returned, fd is released.
 927  * On success caller should complete fd access with matching fdput()
 928  */
 929 struct bpf_map *__bpf_map_get(struct fd f)
 930 {
 931         if (!f.file)
 932                 return ERR_PTR(-EBADF);
 933         if (f.file->f_op != &bpf_map_fops) {
 934                 fdput(f);
 935                 return ERR_PTR(-EINVAL);
 936         }
 937
 938         return f.file->private_data;
 939 }
 940
 941 void bpf_map_inc(struct bpf_map *map)
 942 {
 943         atomic64_inc(&map->refcnt);
 944 }
 945 EXPORT_SYMBOL_GPL(bpf_map_inc);
 946
 947 void bpf_map_inc_with_uref(struct bpf_map *map)
 948 {
 949         atomic64_inc(&map->refcnt);
 950         atomic64_inc(&map->usercnt);
 951 }
 952 EXPORT_SYMBOL_GPL(bpf_map_inc_with_uref);
 953
 954 struct bpf_map *bpf_map_get(u32 ufd)
 955 {
 956         struct fd f = fdget(ufd);
 957         struct bpf_map *map;
 958
 959         map = __bpf_map_get(f);
 960         if (IS_ERR(map))
 961                 return map;
 962
 963         bpf_map_inc(map);
 964         fdput(f);
 965
 966         return map;
 967 }
 968
 969 struct bpf_map *bpf_map_get_with_uref(u32 ufd)
 970 {
 971         struct fd f = fdget(ufd);
 972         struct bpf_map *map;
 973
 974         map = __bpf_map_get(f);
 975         if (IS_ERR(map))
 976                 return map;
 977
 978         bpf_map_inc_with_uref(map);
 979         fdput(f);
 980
 981         return map;
 982 }
 983
 984 /* map_idr_lock should have been held */
 985 static struct bpf_map *__bpf_map_inc_not_zero(struct bpf_map *map, bool uref)
 986 {
 987         int refold;
 988
 989         refold = atomic64_fetch_add_unless(&map->refcnt, 1, 0);
 990         if (!refold)
 991                 return ERR_PTR(-ENOENT);
 992         if (uref)
 993                 atomic64_inc(&map->usercnt);
 994
 995         return map;
 996 }
 997
 998 struct bpf_map *bpf_map_inc_not_zero(struct bpf_map *map)
 999 {
1000         spin_lock_bh(&map_idr_lock);
1001         map = __bpf_map_inc_not_zero(map, false);
1002         spin_unlock_bh(&map_idr_lock);
1003
1004         return map;
1005 }
1006 EXPORT_SYMBOL_GPL(bpf_map_inc_not_zero);
1007
1008 int __weak bpf_stackmap_copy(struct bpf_map *map, void *key, void *value)
1009 {
1010         return -ENOTSUPP;
1011 }
1012
1013 static void *__bpf_copy_key(void __user *ukey, u64 key_size)
1014 {
1015         if (key_size)
1016                 return memdup_user(ukey, key_size);
1017
1018         if (ukey)
1019                 return ERR_PTR(-EINVAL);
1020
1021         return NULL;
1022 }
1023
1024 static void *___bpf_copy_key(bpfptr_t ukey, u64 key_size)
1025 {
1026         if (key_size)
1027                 return memdup_bpfptr(ukey, key_size);
1028
1029         if (!bpfptr_is_null(ukey))
1030                 return ERR_PTR(-EINVAL);
1031
1032         return NULL;
1033 }
1034
1035 /* last field in 'union bpf_attr' used by this command */
1036 #define BPF_MAP_LOOKUP_ELEM_LAST_FIELD flags
1037
1038 static int map_lookup_elem(union bpf_attr *attr)
1039 {
1040         void __user *ukey = u64_to_user_ptr(attr->key);
1041         void __user *uvalue = u64_to_user_ptr(attr->value);
1042         int ufd = attr->map_fd;
1043         struct bpf_map *map;
1044         void *key, *value;
1045         u32 value_size;
1046         struct fd f;
1047         int err;
1048
1049         if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM))
1050                 return -EINVAL;
1051
1052         if (attr->flags & ~BPF_F_LOCK)
1053                 return -EINVAL;
1054
1055         f = fdget(ufd);
1056         map = __bpf_map_get(f);
1057         if (IS_ERR(map))
1058                 return PTR_ERR(map);
1059         if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ)) {
1060                 err = -EPERM;
1061                 goto err_put;
1062         }
1063
1064         if ((attr->flags & BPF_F_LOCK) &&
1065             !map_value_has_spin_lock(map)) {
1066                 err = -EINVAL;
1067                 goto err_put;
1068         }
1069
1070         key = __bpf_copy_key(ukey, map->key_size);
1071         if (IS_ERR(key)) {
1072                 err = PTR_ERR(key);
1073                 goto err_put;
1074         }
1075
1076         value_size = bpf_map_value_size(map);
1077
1078         err = -ENOMEM;
1079         value = kmalloc(value_size, GFP_USER | __GFP_NOWARN);
1080         if (!value)
1081                 goto free_key;
1082
1083         err = bpf_map_copy_value(map, key, value, attr->flags);
1084         if (err)
1085                 goto free_value;
1086
1087         err = -EFAULT;
1088         if (copy_to_user(uvalue, value, value_size) != 0)
1089                 goto free_value;
1090
1091         err = 0;
1092
1093 free_value:
1094         kfree(value);
1095 free_key:
1096         kfree(key);
1097 err_put:
1098         fdput(f);
1099         return err;
1100 }
1101
1102
1103 #define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags
1104
1105 static int map_update_elem(union bpf_attr *attr, bpfptr_t uattr)
1106 {
1107         bpfptr_t ukey = make_bpfptr(attr->key, uattr.is_kernel);
1108         bpfptr_t uvalue = make_bpfptr(attr->value, uattr.is_kernel);
1109         int ufd = attr->map_fd;
1110         struct bpf_map *map;
1111         void *key, *value;
1112         u32 value_size;
1113         struct fd f;
1114         int err;
1115
1116         if (CHECK_ATTR(BPF_MAP_UPDATE_ELEM))
1117                 return -EINVAL;
1118
1119         f = fdget(ufd);
1120         map = __bpf_map_get(f);
1121         if (IS_ERR(map))
1122                 return PTR_ERR(map);
1123         if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
1124                 err = -EPERM;
1125                 goto err_put;
1126         }
1127
1128         if ((attr->flags & BPF_F_LOCK) &&
1129             !map_value_has_spin_lock(map)) {
1130                 err = -EINVAL;
1131                 goto err_put;
1132         }
1133
1134         key = ___bpf_copy_key(ukey, map->key_size);
1135         if (IS_ERR(key)) {
1136                 err = PTR_ERR(key);
1137                 goto err_put;
1138         }
1139
1140         if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
1141             map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||
1142             map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY ||
1143             map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE)
1144                 value_size = round_up(map->value_size, 8) * num_possible_cpus();
1145         else
1146                 value_size = map->value_size;
1147
1148         err = -ENOMEM;
1149         value = kmalloc(value_size, GFP_USER | __GFP_NOWARN);
1150         if (!value)
1151                 goto free_key;
1152
1153         err = -EFAULT;
1154         if (copy_from_bpfptr(value, uvalue, value_size) != 0)
1155                 goto free_value;
1156
1157         err = bpf_map_update_value(map, f, key, value, attr->flags);
1158
1159 free_value:
1160         kfree(value);
1161 free_key:
1162         kfree(key);
1163 err_put:
1164         fdput(f);
1165         return err;
1166 }
1167
1168 #define BPF_MAP_DELETE_ELEM_LAST_FIELD key
1169
1170 static int map_delete_elem(union bpf_attr *attr)
1171 {
1172         void __user *ukey = u64_to_user_ptr(attr->key);
1173         int ufd = attr->map_fd;
1174         struct bpf_map *map;
1175         struct fd f;
1176         void *key;
1177         int err;
1178
1179         if (CHECK_ATTR(BPF_MAP_DELETE_ELEM))
1180                 return -EINVAL;
1181
1182         f = fdget(ufd);
1183         map = __bpf_map_get(f);
1184         if (IS_ERR(map))
1185                 return PTR_ERR(map);
1186         if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
1187                 err = -EPERM;
1188                 goto err_put;
1189         }
1190
1191         key = __bpf_copy_key(ukey, map->key_size);
1192         if (IS_ERR(key)) {
1193                 err = PTR_ERR(key);
1194                 goto err_put;
1195         }
1196
1197         if (bpf_map_is_dev_bound(map)) {
1198                 err = bpf_map_offload_delete_elem(map, key);
1199                 goto out;
1200         } else if (IS_FD_PROG_ARRAY(map) ||
1201                    map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
1202                 /* These maps require sleepable context */
1203                 err = map->ops->map_delete_elem(map, key);
1204                 goto out;
1205         }
1206
1207         bpf_disable_instrumentation();
1208         rcu_read_lock();
1209         err = map->ops->map_delete_elem(map, key);
1210         rcu_read_unlock();
1211         bpf_enable_instrumentation();
1212         maybe_wait_bpf_programs(map);
1213 out:
1214         kfree(key);
1215 err_put:
1216         fdput(f);
1217         return err;
1218 }
1219
1220 /* last field in 'union bpf_attr' used by this command */
1221 #define BPF_MAP_GET_NEXT_KEY_LAST_FIELD next_key
1222
1223 static int map_get_next_key(union bpf_attr *attr)
1224 {
1225         void __user *ukey = u64_to_user_ptr(attr->key);
1226         void __user *unext_key = u64_to_user_ptr(attr->next_key);
1227         int ufd = attr->map_fd;
1228         struct bpf_map *map;
1229         void *key, *next_key;
1230         struct fd f;
1231         int err;
1232
1233         if (CHECK_ATTR(BPF_MAP_GET_NEXT_KEY))
1234                 return -EINVAL;
1235
1236         f = fdget(ufd);
1237         map = __bpf_map_get(f);
1238         if (IS_ERR(map))
1239                 return PTR_ERR(map);
1240         if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ)) {
1241                 err = -EPERM;
1242                 goto err_put;
1243         }
1244
1245         if (ukey) {
1246                 key = __bpf_copy_key(ukey, map->key_size);
1247                 if (IS_ERR(key)) {
1248                         err = PTR_ERR(key);
1249                         goto err_put;
1250                 }
1251         } else {
1252                 key = NULL;
1253         }
1254
1255         err = -ENOMEM;
1256         next_key = kmalloc(map->key_size, GFP_USER);
1257         if (!next_key)
1258                 goto free_key;
1259
1260         if (bpf_map_is_dev_bound(map)) {
1261                 err = bpf_map_offload_get_next_key(map, key, next_key);
1262                 goto out;
1263         }
1264
1265         rcu_read_lock();
1266         err = map->ops->map_get_next_key(map, key, next_key);
1267         rcu_read_unlock();
1268 out:
1269         if (err)
1270                 goto free_next_key;
1271
1272         err = -EFAULT;
1273         if (copy_to_user(unext_key, next_key, map->key_size) != 0)
1274                 goto free_next_key;
1275
1276         err = 0;
1277
1278 free_next_key:
1279         kfree(next_key);
1280 free_key:
1281         kfree(key);
1282 err_put:
1283         fdput(f);
1284         return err;
1285 }
1286
1287 int generic_map_delete_batch(struct bpf_map *map,
1288                              const union bpf_attr *attr,
1289                              union bpf_attr __user *uattr)
1290 {
1291         void __user *keys = u64_to_user_ptr(attr->batch.keys);
1292         u32 cp, max_count;
1293         int err = 0;
1294         void *key;
1295
1296         if (attr->batch.elem_flags & ~BPF_F_LOCK)
1297                 return -EINVAL;
1298
1299         if ((attr->batch.elem_flags & BPF_F_LOCK) &&
1300             !map_value_has_spin_lock(map)) {
1301                 return -EINVAL;
1302         }
1303
1304         max_count = attr->batch.count;
1305         if (!max_count)
1306                 return 0;
1307
1308         key = kmalloc(map->key_size, GFP_USER | __GFP_NOWARN);
1309         if (!key)
1310                 return -ENOMEM;
1311
1312         for (cp = 0; cp < max_count; cp++) {
1313                 err = -EFAULT;
1314                 if (copy_from_user(key, keys + cp * map->key_size,
1315                                    map->key_size))
1316                         break;
1317
1318                 if (bpf_map_is_dev_bound(map)) {
1319                         err = bpf_map_offload_delete_elem(map, key);
1320                         break;
1321                 }
1322
1323                 bpf_disable_instrumentation();
1324                 rcu_read_lock();
1325                 err = map->ops->map_delete_elem(map, key);
1326                 rcu_read_unlock();
1327                 bpf_enable_instrumentation();
1328                 maybe_wait_bpf_programs(map);
1329                 if (err)
1330                         break;
1331         }
1332         if (copy_to_user(&uattr->batch.count, &cp, sizeof(cp)))
1333                 err = -EFAULT;
1334
1335         kfree(key);
1336         return err;
1337 }
1338
1339 int generic_map_update_batch(struct bpf_map *map,
1340                              const union bpf_attr *attr,
1341                              union bpf_attr __user *uattr)
1342 {
1343         void __user *values = u64_to_user_ptr(attr->batch.values);
1344         void __user *keys = u64_to_user_ptr(attr->batch.keys);
1345         u32 value_size, cp, max_count;
1346         int ufd = attr->map_fd;
1347         void *key, *value;
1348         struct fd f;
1349         int err = 0;
1350
1351         f = fdget(ufd);
1352         if (attr->batch.elem_flags & ~BPF_F_LOCK)
1353                 return -EINVAL;
1354
1355         if ((attr->batch.elem_flags & BPF_F_LOCK) &&
1356             !map_value_has_spin_lock(map)) {
1357                 return -EINVAL;
1358         }
1359
1360         value_size = bpf_map_value_size(map);
1361
1362         max_count = attr->batch.count;
1363         if (!max_count)
1364                 return 0;
1365
1366         key = kmalloc(map->key_size, GFP_USER | __GFP_NOWARN);
1367         if (!key)
1368                 return -ENOMEM;
1369
1370         value = kmalloc(value_size, GFP_USER | __GFP_NOWARN);
1371         if (!value) {
1372                 kfree(key);
1373                 return -ENOMEM;
1374         }
1375
1376         for (cp = 0; cp < max_count; cp++) {
1377                 err = -EFAULT;
1378                 if (copy_from_user(key, keys + cp * map->key_size,
1379                     map->key_size) ||
1380                     copy_from_user(value, values + cp * value_size, value_size))
1381                         break;
1382
1383                 err = bpf_map_update_value(map, f, key, value,
1384                                            attr->batch.elem_flags);
1385
1386                 if (err)
1387                         break;
1388         }
1389
1390         if (copy_to_user(&uattr->batch.count, &cp, sizeof(cp)))
1391                 err = -EFAULT;
1392
1393         kfree(value);
1394         kfree(key);
1395         return err;
1396 }
1397
1398 #define MAP_LOOKUP_RETRIES 3
1399
1400 int generic_map_lookup_batch(struct bpf_map *map,
1401                                     const union bpf_attr *attr,
1402                                     union bpf_attr __user *uattr)
1403 {
1404         void __user *uobatch = u64_to_user_ptr(attr->batch.out_batch);
1405         void __user *ubatch = u64_to_user_ptr(attr->batch.in_batch);
1406         void __user *values = u64_to_user_ptr(attr->batch.values);
1407         void __user *keys = u64_to_user_ptr(attr->batch.keys);
1408         void *buf, *buf_prevkey, *prev_key, *key, *value;
1409         int err, retry = MAP_LOOKUP_RETRIES;
1410         u32 value_size, cp, max_count;
1411
1412         if (attr->batch.elem_flags & ~BPF_F_LOCK)
1413                 return -EINVAL;
1414
1415         if ((attr->batch.elem_flags & BPF_F_LOCK) &&
1416             !map_value_has_spin_lock(map))
1417                 return -EINVAL;
1418
1419         value_size = bpf_map_value_size(map);
1420
1421         max_count = attr->batch.count;
1422         if (!max_count)
1423                 return 0;
1424
1425         if (put_user(0, &uattr->batch.count))
1426                 return -EFAULT;
1427
1428         buf_prevkey = kmalloc(map->key_size, GFP_USER | __GFP_NOWARN);
1429         if (!buf_prevkey)
1430                 return -ENOMEM;
1431
1432         buf = kmalloc(map->key_size + value_size, GFP_USER | __GFP_NOWARN);
1433         if (!buf) {
1434                 kfree(buf_prevkey);
1435                 return -ENOMEM;
1436         }
1437
1438         err = -EFAULT;
1439         prev_key = NULL;
1440         if (ubatch && copy_from_user(buf_prevkey, ubatch, map->key_size))
1441                 goto free_buf;
1442         key = buf;
1443         value = key + map->key_size;
1444         if (ubatch)
1445                 prev_key = buf_prevkey;
1446
1447         for (cp = 0; cp < max_count;) {
1448                 rcu_read_lock();
1449                 err = map->ops->map_get_next_key(map, prev_key, key);
1450                 rcu_read_unlock();
1451                 if (err)
1452                         break;
1453                 err = bpf_map_copy_value(map, key, value,
1454                                          attr->batch.elem_flags);
1455
1456                 if (err == -ENOENT) {
1457                         if (retry) {
1458                                 retry--;
1459                                 continue;
1460                         }
1461                         err = -EINTR;
1462                         break;
1463                 }
1464
1465                 if (err)
1466                         goto free_buf;
1467
1468                 if (copy_to_user(keys + cp * map->key_size, key,
1469                                  map->key_size)) {
1470                         err = -EFAULT;
1471                         goto free_buf;
1472                 }
1473                 if (copy_to_user(values + cp * value_size, value, value_size)) {
1474                         err = -EFAULT;
1475                         goto free_buf;
1476                 }
1477
1478                 if (!prev_key)
1479                         prev_key = buf_prevkey;
1480
1481                 swap(prev_key, key);
1482                 retry = MAP_LOOKUP_RETRIES;
1483                 cp++;
1484         }
1485
1486         if (err == -EFAULT)
1487                 goto free_buf;
1488
1489         if ((copy_to_user(&uattr->batch.count, &cp, sizeof(cp)) ||
1490                     (cp && copy_to_user(uobatch, prev_key, map->key_size))))
1491                 err = -EFAULT;
1492
1493 free_buf:
1494         kfree(buf_prevkey);
1495         kfree(buf);
1496         return err;
1497 }
1498
1499 #define BPF_MAP_LOOKUP_AND_DELETE_ELEM_LAST_FIELD flags
1500
1501 static int map_lookup_and_delete_elem(union bpf_attr *attr)
1502 {
1503         void __user *ukey = u64_to_user_ptr(attr->key);
1504         void __user *uvalue = u64_to_user_ptr(attr->value);
1505         int ufd = attr->map_fd;
1506         struct bpf_map *map;
1507         void *key, *value;
1508         u32 value_size;
1509         struct fd f;
1510         int err;
1511
1512         if (CHECK_ATTR(BPF_MAP_LOOKUP_AND_DELETE_ELEM))
1513                 return -EINVAL;
1514
1515         if (attr->flags & ~BPF_F_LOCK)
1516                 return -EINVAL;
1517
1518         f = fdget(ufd);
1519         map = __bpf_map_get(f);
1520         if (IS_ERR(map))
1521                 return PTR_ERR(map);
1522         if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ) ||
1523             !(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
1524                 err = -EPERM;
1525                 goto err_put;
1526         }
1527
1528         if (attr->flags &&
1529             (map->map_type == BPF_MAP_TYPE_QUEUE ||
1530              map->map_type == BPF_MAP_TYPE_STACK)) {
1531                 err = -EINVAL;
1532                 goto err_put;
1533         }
1534
1535         if ((attr->flags & BPF_F_LOCK) &&
1536             !map_value_has_spin_lock(map)) {
1537                 err = -EINVAL;
1538                 goto err_put;
1539         }
1540
1541         key = __bpf_copy_key(ukey, map->key_size);
1542         if (IS_ERR(key)) {
1543                 err = PTR_ERR(key);
1544                 goto err_put;
1545         }
1546
1547         value_size = bpf_map_value_size(map);
1548
1549         err = -ENOMEM;
1550         value = kmalloc(value_size, GFP_USER | __GFP_NOWARN);
1551         if (!value)
1552                 goto free_key;
1553
1554         err = -ENOTSUPP;
1555         if (map->map_type == BPF_MAP_TYPE_QUEUE ||
1556             map->map_type == BPF_MAP_TYPE_STACK) {
1557                 err = map->ops->map_pop_elem(map, value);
1558         } else if (map->map_type == BPF_MAP_TYPE_HASH ||
1559                    map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
1560                    map->map_type == BPF_MAP_TYPE_LRU_HASH ||
1561                    map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
1562                 if (!bpf_map_is_dev_bound(map)) {
1563                         bpf_disable_instrumentation();
1564                         rcu_read_lock();
1565                         err = map->ops->map_lookup_and_delete_elem(map, key, value, attr->flags);
1566                         rcu_read_unlock();
1567                         bpf_enable_instrumentation();
1568                 }
1569         }
1570
1571         if (err)
1572                 goto free_value;
1573
1574         if (copy_to_user(uvalue, value, value_size) != 0) {
1575                 err = -EFAULT;
1576                 goto free_value;
1577         }
1578
1579         err = 0;
1580
1581 free_value:
1582         kfree(value);
1583 free_key:
1584         kfree(key);
1585 err_put:
1586         fdput(f);
1587         return err;
1588 }
1589
1590 #define BPF_MAP_FREEZE_LAST_FIELD map_fd
1591
1592 static int map_freeze(const union bpf_attr *attr)
1593 {
1594         int err = 0, ufd = attr->map_fd;
1595         struct bpf_map *map;
1596         struct fd f;
1597
1598         if (CHECK_ATTR(BPF_MAP_FREEZE))
1599                 return -EINVAL;
1600
1601         f = fdget(ufd);
1602         map = __bpf_map_get(f);
1603         if (IS_ERR(map))
1604                 return PTR_ERR(map);
1605
1606         if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS ||
1607             map_value_has_timer(map)) {
1608                 fdput(f);
1609                 return -ENOTSUPP;
1610         }
1611
1612         mutex_lock(&map->freeze_mutex);
1613
1614         if (map->writecnt) {
1615                 err = -EBUSY;
1616                 goto err_put;
1617         }
1618         if (READ_ONCE(map->frozen)) {
1619                 err = -EBUSY;
1620                 goto err_put;
1621         }
1622         if (!bpf_capable()) {
1623                 err = -EPERM;
1624                 goto err_put;
1625         }
1626
1627         WRITE_ONCE(map->frozen, true);
1628 err_put:
1629         mutex_unlock(&map->freeze_mutex);
1630         fdput(f);
1631         return err;
1632 }
1633
1634 static const struct bpf_prog_ops * const bpf_prog_types[] = {
1635 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \
1636         [_id] = & _name ## _prog_ops,
1637 #define BPF_MAP_TYPE(_id, _ops)
1638 #define BPF_LINK_TYPE(_id, _name)
1639 #include <linux/bpf_types.h>
1640 #undef BPF_PROG_TYPE
1641 #undef BPF_MAP_TYPE
1642 #undef BPF_LINK_TYPE
1643 };
1644
1645 static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog)
1646 {
1647         const struct bpf_prog_ops *ops;
1648
1649         if (type >= ARRAY_SIZE(bpf_prog_types))
1650                 return -EINVAL;
1651         type = array_index_nospec(type, ARRAY_SIZE(bpf_prog_types));
1652         ops = bpf_prog_types[type];
1653         if (!ops)
1654                 return -EINVAL;
1655
1656         if (!bpf_prog_is_dev_bound(prog->aux))
1657                 prog->aux->ops = ops;
1658         else
1659                 prog->aux->ops = &bpf_offload_prog_ops;
1660         prog->type = type;
1661         return 0;
1662 }
1663
1664 enum bpf_audit {
1665         BPF_AUDIT_LOAD,
1666         BPF_AUDIT_UNLOAD,
1667         BPF_AUDIT_MAX,
1668 };
1669
1670 static const char * const bpf_audit_str[BPF_AUDIT_MAX] = {
1671         [BPF_AUDIT_LOAD]   = "LOAD",
1672         [BPF_AUDIT_UNLOAD] = "UNLOAD",
1673 };
1674
1675 static void bpf_audit_prog(const struct bpf_prog *prog, unsigned int op)
1676 {
1677         struct audit_context *ctx = NULL;
1678         struct audit_buffer *ab;
1679
1680         if (WARN_ON_ONCE(op >= BPF_AUDIT_MAX))
1681                 return;
1682         if (audit_enabled == AUDIT_OFF)
1683                 return;
1684         if (op == BPF_AUDIT_LOAD)
1685                 ctx = audit_context();
1686         ab = audit_log_start(ctx, GFP_ATOMIC, AUDIT_BPF);
1687         if (unlikely(!ab))
1688                 return;
1689         audit_log_format(ab, "prog-id=%u op=%s",
1690                          prog->aux->id, bpf_audit_str[op]);
1691         audit_log_end(ab);
1692 }
1693
1694 static int bpf_prog_alloc_id(struct bpf_prog *prog)
1695 {
1696         int id;
1697
1698         idr_preload(GFP_KERNEL);
1699         spin_lock_bh(&prog_idr_lock);
1700         id = idr_alloc_cyclic(&prog_idr, prog, 1, INT_MAX, GFP_ATOMIC);
1701         if (id > 0)
1702                 prog->aux->id = id;
1703         spin_unlock_bh(&prog_idr_lock);
1704         idr_preload_end();
1705
1706         /* id is in [1, INT_MAX) */
1707         if (WARN_ON_ONCE(!id))
1708                 return -ENOSPC;
1709
1710         return id > 0 ? 0 : id;
1711 }
1712
1713 void bpf_prog_free_id(struct bpf_prog *prog, bool do_idr_lock)
1714 {
1715         unsigned long flags;
1716
1717         /* cBPF to eBPF migrations are currently not in the idr store.
1718          * Offloaded programs are removed from the store when their device
1719          * disappears - even if someone grabs an fd to them they are unusable,
1720          * simply waiting for refcnt to drop to be freed.
1721          */
1722         if (!prog->aux->id)
1723                 return;
1724
1725         if (do_idr_lock)
1726                 spin_lock_irqsave(&prog_idr_lock, flags);
1727         else
1728                 __acquire(&prog_idr_lock);
1729
1730         idr_remove(&prog_idr, prog->aux->id);
1731         prog->aux->id = 0;
1732
1733         if (do_idr_lock)
1734                 spin_unlock_irqrestore(&prog_idr_lock, flags);
1735         else
1736                 __release(&prog_idr_lock);
1737 }
1738
1739 static void __bpf_prog_put_rcu(struct rcu_head *rcu)
1740 {
1741         struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu);
1742
1743         kvfree(aux->func_info);
1744         kfree(aux->func_info_aux);
1745         free_uid(aux->user);
1746         security_bpf_prog_free(aux);
1747         bpf_prog_free(aux->prog);
1748 }
1749
1750 static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred)
1751 {
1752         bpf_prog_kallsyms_del_all(prog);
1753         btf_put(prog->aux->btf);
1754         kvfree(prog->aux->jited_linfo);
1755         kvfree(prog->aux->linfo);
1756         kfree(prog->aux->kfunc_tab);
1757         if (prog->aux->attach_btf)
1758                 btf_put(prog->aux->attach_btf);
1759
1760         if (deferred) {
1761                 if (prog->aux->sleepable)
1762                         call_rcu_tasks_trace(&prog->aux->rcu, __bpf_prog_put_rcu);
1763                 else
1764                         call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu);
1765         } else {
1766                 __bpf_prog_put_rcu(&prog->aux->rcu);
1767         }
1768 }
1769
1770 static void bpf_prog_put_deferred(struct work_struct *work)
1771 {
1772         struct bpf_prog_aux *aux;
1773         struct bpf_prog *prog;
1774
1775         aux = container_of(work, struct bpf_prog_aux, work);
1776         prog = aux->prog;
1777         perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_UNLOAD, 0);
1778         bpf_audit_prog(prog, BPF_AUDIT_UNLOAD);
1779         __bpf_prog_put_noref(prog, true);
1780 }
1781
1782 static void __bpf_prog_put(struct bpf_prog *prog, bool do_idr_lock)
1783 {
1784         struct bpf_prog_aux *aux = prog->aux;
1785
1786         if (atomic64_dec_and_test(&aux->refcnt)) {
1787                 /* bpf_prog_free_id() must be called first */
1788                 bpf_prog_free_id(prog, do_idr_lock);
1789
1790                 if (in_irq() || irqs_disabled()) {
1791                         INIT_WORK(&aux->work, bpf_prog_put_deferred);
1792                         schedule_work(&aux->work);
1793                 } else {
1794                         bpf_prog_put_deferred(&aux->work);
1795                 }
1796         }
1797 }
1798
1799 void bpf_prog_put(struct bpf_prog *prog)
1800 {
1801         __bpf_prog_put(prog, true);
1802 }
1803 EXPORT_SYMBOL_GPL(bpf_prog_put);
1804
1805 static int bpf_prog_release(struct inode *inode, struct file *filp)
1806 {
1807         struct bpf_prog *prog = filp->private_data;
1808
1809         bpf_prog_put(prog);
1810         return 0;
1811 }
1812
1813 static void bpf_prog_get_stats(const struct bpf_prog *prog,
1814                                struct bpf_prog_stats *stats)
1815 {
1816         u64 nsecs = 0, cnt = 0, misses = 0;
1817         int cpu;
1818
1819         for_each_possible_cpu(cpu) {
1820                 const struct bpf_prog_stats *st;
1821                 unsigned int start;
1822                 u64 tnsecs, tcnt, tmisses;
1823
1824                 st = per_cpu_ptr(prog->stats, cpu);
1825                 do {
1826                         start = u64_stats_fetch_begin_irq(&st->syncp);
1827                         tnsecs = st->nsecs;
1828                         tcnt = st->cnt;
1829                         tmisses = st->misses;
1830                 } while (u64_stats_fetch_retry_irq(&st->syncp, start));
1831                 nsecs += tnsecs;
1832                 cnt += tcnt;
1833                 misses += tmisses;
1834         }
1835         stats->nsecs = nsecs;
1836         stats->cnt = cnt;
1837         stats->misses = misses;
1838 }
1839
1840 #ifdef CONFIG_PROC_FS
1841 static void bpf_prog_show_fdinfo(struct seq_file *m, struct file *filp)
1842 {
1843         const struct bpf_prog *prog = filp->private_data;
1844         char prog_tag[sizeof(prog->tag) * 2 + 1] = { };
1845         struct bpf_prog_stats stats;
1846
1847         bpf_prog_get_stats(prog, &stats);
1848         bin2hex(prog_tag, prog->tag, sizeof(prog->tag));
1849         seq_printf(m,
1850                    "prog_type:\t%u\n"
1851                    "prog_jited:\t%u\n"
1852                    "prog_tag:\t%s\n"
1853                    "memlock:\t%llu\n"
1854                    "prog_id:\t%u\n"
1855                    "run_time_ns:\t%llu\n"
1856                    "run_cnt:\t%llu\n"
1857                    "recursion_misses:\t%llu\n",
1858                    prog->type,
1859                    prog->jited,
1860                    prog_tag,
1861                    prog->pages * 1ULL << PAGE_SHIFT,
1862                    prog->aux->id,
1863                    stats.nsecs,
1864                    stats.cnt,
1865                    stats.misses);
1866 }
1867 #endif
1868
1869 const struct file_operations bpf_prog_fops = {
1870 #ifdef CONFIG_PROC_FS
1871         .show_fdinfo    = bpf_prog_show_fdinfo,
1872 #endif
1873         .release        = bpf_prog_release,
1874         .read           = bpf_dummy_read,
1875         .write          = bpf_dummy_write,
1876 };
1877
1878 int bpf_prog_new_fd(struct bpf_prog *prog)
1879 {
1880         int ret;
1881
1882         ret = security_bpf_prog(prog);
1883         if (ret < 0)
1884                 return ret;
1885
1886         return anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog,
1887                                 O_RDWR | O_CLOEXEC);
1888 }
1889
1890 static struct bpf_prog *____bpf_prog_get(struct fd f)
1891 {
1892         if (!f.file)
1893                 return ERR_PTR(-EBADF);
1894         if (f.file->f_op != &bpf_prog_fops) {
1895                 fdput(f);
1896                 return ERR_PTR(-EINVAL);
1897         }
1898
1899         return f.file->private_data;
1900 }
1901
1902 void bpf_prog_add(struct bpf_prog *prog, int i)
1903 {
1904         atomic64_add(i, &prog->aux->refcnt);
1905 }
1906 EXPORT_SYMBOL_GPL(bpf_prog_add);
1907
1908 void bpf_prog_sub(struct bpf_prog *prog, int i)
1909 {
1910         /* Only to be used for undoing previous bpf_prog_add() in some
1911          * error path. We still know that another entity in our call
1912          * path holds a reference to the program, thus atomic_sub() can
1913          * be safely used in such cases!
1914          */
1915         WARN_ON(atomic64_sub_return(i, &prog->aux->refcnt) == 0);
1916 }
1917 EXPORT_SYMBOL_GPL(bpf_prog_sub);
1918
1919 void bpf_prog_inc(struct bpf_prog *prog)
1920 {
1921         atomic64_inc(&prog->aux->refcnt);
1922 }
1923 EXPORT_SYMBOL_GPL(bpf_prog_inc);
1924
1925 /* prog_idr_lock should have been held */
1926 struct bpf_prog *bpf_prog_inc_not_zero(struct bpf_prog *prog)
1927 {
1928         int refold;
1929
1930         refold = atomic64_fetch_add_unless(&prog->aux->refcnt, 1, 0);
1931
1932         if (!refold)
1933                 return ERR_PTR(-ENOENT);
1934
1935         return prog;
1936 }
1937 EXPORT_SYMBOL_GPL(bpf_prog_inc_not_zero);
1938
1939 bool bpf_prog_get_ok(struct bpf_prog *prog,
1940                             enum bpf_prog_type *attach_type, bool attach_drv)
1941 {
1942         /* not an attachment, just a refcount inc, always allow */
1943         if (!attach_type)
1944                 return true;
1945
1946         if (prog->type != *attach_type)
1947                 return false;
1948         if (bpf_prog_is_dev_bound(prog->aux) && !attach_drv)
1949                 return false;
1950
1951         return true;
1952 }
1953
1954 static struct bpf_prog *__bpf_prog_get(u32 ufd, enum bpf_prog_type *attach_type,
1955                                        bool attach_drv)
1956 {
1957         struct fd f = fdget(ufd);
1958         struct bpf_prog *prog;
1959
1960         prog = ____bpf_prog_get(f);
1961         if (IS_ERR(prog))
1962                 return prog;
1963         if (!bpf_prog_get_ok(prog, attach_type, attach_drv)) {
1964                 prog = ERR_PTR(-EINVAL);
1965                 goto out;
1966         }
1967
1968         bpf_prog_inc(prog);
1969 out:
1970         fdput(f);
1971         return prog;
1972 }
1973
1974 struct bpf_prog *bpf_prog_get(u32 ufd)
1975 {
1976         return __bpf_prog_get(ufd, NULL, false);
1977 }
1978
1979 struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type,
1980                                        bool attach_drv)
1981 {
1982         return __bpf_prog_get(ufd, &type, attach_drv);
1983 }
1984 EXPORT_SYMBOL_GPL(bpf_prog_get_type_dev);
1985
1986 /* Initially all BPF programs could be loaded w/o specifying
1987  * expected_attach_type. Later for some of them specifying expected_attach_type
1988  * at load time became required so that program could be validated properly.
1989  * Programs of types that are allowed to be loaded both w/ and w/o (for
1990  * backward compatibility) expected_attach_type, should have the default attach
1991  * type assigned to expected_attach_type for the latter case, so that it can be
1992  * validated later at attach time.
1993  *
1994  * bpf_prog_load_fixup_attach_type() sets expected_attach_type in @attr if
1995  * prog type requires it but has some attach types that have to be backward
1996  * compatible.
1997  */
1998 static void bpf_prog_load_fixup_attach_type(union bpf_attr *attr)
1999 {
2000         switch (attr->prog_type) {
2001         case BPF_PROG_TYPE_CGROUP_SOCK:
2002                 /* Unfortunately BPF_ATTACH_TYPE_UNSPEC enumeration doesn't
2003                  * exist so checking for non-zero is the way to go here.
2004                  */
2005                 if (!attr->expected_attach_type)
2006                         attr->expected_attach_type =
2007                                 BPF_CGROUP_INET_SOCK_CREATE;
2008                 break;
2009         case BPF_PROG_TYPE_SK_REUSEPORT:
2010                 if (!attr->expected_attach_type)
2011                         attr->expected_attach_type =
2012                                 BPF_SK_REUSEPORT_SELECT;
2013                 break;
2014         }
2015 }
2016
2017 static int
2018 bpf_prog_load_check_attach(enum bpf_prog_type prog_type,
2019                            enum bpf_attach_type expected_attach_type,
2020                            struct btf *attach_btf, u32 btf_id,
2021                            struct bpf_prog *dst_prog)
2022 {
2023         if (btf_id) {
2024                 if (btf_id > BTF_MAX_TYPE)
2025                         return -EINVAL;
2026
2027                 if (!attach_btf && !dst_prog)
2028                         return -EINVAL;
2029
2030                 switch (prog_type) {
2031                 case BPF_PROG_TYPE_TRACING:
2032                 case BPF_PROG_TYPE_LSM:
2033                 case BPF_PROG_TYPE_STRUCT_OPS:
2034                 case BPF_PROG_TYPE_EXT:
2035                         break;
2036                 default:
2037                         return -EINVAL;
2038                 }
2039         }
2040
2041         if (attach_btf && (!btf_id || dst_prog))
2042                 return -EINVAL;
2043
2044         if (dst_prog && prog_type != BPF_PROG_TYPE_TRACING &&
2045             prog_type != BPF_PROG_TYPE_EXT)
2046                 return -EINVAL;
2047
2048         switch (prog_type) {
2049         case BPF_PROG_TYPE_CGROUP_SOCK:
2050                 switch (expected_attach_type) {
2051                 case BPF_CGROUP_INET_SOCK_CREATE:
2052                 case BPF_CGROUP_INET_SOCK_RELEASE:
2053                 case BPF_CGROUP_INET4_POST_BIND:
2054                 case BPF_CGROUP_INET6_POST_BIND:
2055                         return 0;
2056                 default:
2057                         return -EINVAL;
2058                 }
2059         case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
2060                 switch (expected_attach_type) {
2061                 case BPF_CGROUP_INET4_BIND:
2062                 case BPF_CGROUP_INET6_BIND:
2063                 case BPF_CGROUP_INET4_CONNECT:
2064                 case BPF_CGROUP_INET6_CONNECT:
2065                 case BPF_CGROUP_INET4_GETPEERNAME:
2066                 case BPF_CGROUP_INET6_GETPEERNAME:
2067                 case BPF_CGROUP_INET4_GETSOCKNAME:
2068                 case BPF_CGROUP_INET6_GETSOCKNAME:
2069                 case BPF_CGROUP_UDP4_SENDMSG:
2070                 case BPF_CGROUP_UDP6_SENDMSG:
2071                 case BPF_CGROUP_UDP4_RECVMSG:
2072                 case BPF_CGROUP_UDP6_RECVMSG:
2073                         return 0;
2074                 default:
2075                         return -EINVAL;
2076                 }
2077         case BPF_PROG_TYPE_CGROUP_SKB:
2078                 switch (expected_attach_type) {
2079                 case BPF_CGROUP_INET_INGRESS:
2080                 case BPF_CGROUP_INET_EGRESS:
2081                         return 0;
2082                 default:
2083                         return -EINVAL;
2084                 }
2085         case BPF_PROG_TYPE_CGROUP_SOCKOPT:
2086                 switch (expected_attach_type) {
2087                 case BPF_CGROUP_SETSOCKOPT:
2088                 case BPF_CGROUP_GETSOCKOPT:
2089                         return 0;
2090                 default:
2091                         return -EINVAL;
2092                 }
2093         case BPF_PROG_TYPE_SK_LOOKUP:
2094                 if (expected_attach_type == BPF_SK_LOOKUP)
2095                         return 0;
2096                 return -EINVAL;
2097         case BPF_PROG_TYPE_SK_REUSEPORT:
2098                 switch (expected_attach_type) {
2099                 case BPF_SK_REUSEPORT_SELECT:
2100                 case BPF_SK_REUSEPORT_SELECT_OR_MIGRATE:
2101                         return 0;
2102                 default:
2103                         return -EINVAL;
2104                 }
2105         case BPF_PROG_TYPE_SYSCALL:
2106         case BPF_PROG_TYPE_EXT:
2107                 if (expected_attach_type)
2108                         return -EINVAL;
2109                 fallthrough;
2110         default:
2111                 return 0;
2112         }
2113 }
2114
2115 static bool is_net_admin_prog_type(enum bpf_prog_type prog_type)
2116 {
2117         switch (prog_type) {
2118         case BPF_PROG_TYPE_SCHED_CLS:
2119         case BPF_PROG_TYPE_SCHED_ACT:
2120         case BPF_PROG_TYPE_XDP:
2121         case BPF_PROG_TYPE_LWT_IN:
2122         case BPF_PROG_TYPE_LWT_OUT:
2123         case BPF_PROG_TYPE_LWT_XMIT:
2124         case BPF_PROG_TYPE_LWT_SEG6LOCAL:
2125         case BPF_PROG_TYPE_SK_SKB:
2126         case BPF_PROG_TYPE_SK_MSG:
2127         case BPF_PROG_TYPE_LIRC_MODE2:
2128         case BPF_PROG_TYPE_FLOW_DISSECTOR:
2129         case BPF_PROG_TYPE_CGROUP_DEVICE:
2130         case BPF_PROG_TYPE_CGROUP_SOCK:
2131         case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
2132         case BPF_PROG_TYPE_CGROUP_SOCKOPT:
2133         case BPF_PROG_TYPE_CGROUP_SYSCTL:
2134         case BPF_PROG_TYPE_SOCK_OPS:
2135         case BPF_PROG_TYPE_EXT: /* extends any prog */
2136                 return true;
2137         case BPF_PROG_TYPE_CGROUP_SKB:
2138                 /* always unpriv */
2139         case BPF_PROG_TYPE_SK_REUSEPORT:
2140                 /* equivalent to SOCKET_FILTER. need CAP_BPF only */
2141         default:
2142                 return false;
2143         }
2144 }
2145
2146 static bool is_perfmon_prog_type(enum bpf_prog_type prog_type)
2147 {
2148         switch (prog_type) {
2149         case BPF_PROG_TYPE_KPROBE:
2150         case BPF_PROG_TYPE_TRACEPOINT:
2151         case BPF_PROG_TYPE_PERF_EVENT:
2152         case BPF_PROG_TYPE_RAW_TRACEPOINT:
2153         case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE:
2154         case BPF_PROG_TYPE_TRACING:
2155         case BPF_PROG_TYPE_LSM:
2156         case BPF_PROG_TYPE_STRUCT_OPS: /* has access to struct sock */
2157         case BPF_PROG_TYPE_EXT: /* extends any prog */
2158                 return true;
2159         default:
2160                 return false;
2161         }
2162 }
2163
2164 /* last field in 'union bpf_attr' used by this command */
2165 #define BPF_PROG_LOAD_LAST_FIELD fd_array
2166
2167 static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr)
2168 {
2169         enum bpf_prog_type type = attr->prog_type;
2170         struct bpf_prog *prog, *dst_prog = NULL;
2171         struct btf *attach_btf = NULL;
2172         int err;
2173         char license[128];
2174         bool is_gpl;
2175
2176         if (CHECK_ATTR(BPF_PROG_LOAD))
2177                 return -EINVAL;
2178
2179         if (attr->prog_flags & ~(BPF_F_STRICT_ALIGNMENT |
2180                                  BPF_F_ANY_ALIGNMENT |
2181                                  BPF_F_TEST_STATE_FREQ |
2182                                  BPF_F_SLEEPABLE |
2183                                  BPF_F_TEST_RND_HI32))
2184                 return -EINVAL;
2185
2186         if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) &&
2187             (attr->prog_flags & BPF_F_ANY_ALIGNMENT) &&
2188             !bpf_capable())
2189                 return -EPERM;
2190
2191         /* copy eBPF program license from user space */
2192         if (strncpy_from_bpfptr(license,
2193                                 make_bpfptr(attr->license, uattr.is_kernel),
2194                                 sizeof(license) - 1) < 0)
2195                 return -EFAULT;
2196         license[sizeof(license) - 1] = 0;
2197
2198         /* eBPF programs must be GPL compatible to use GPL-ed functions */
2199         is_gpl = license_is_gpl_compatible(license);
2200
2201         if (attr->insn_cnt == 0 ||
2202             attr->insn_cnt > (bpf_capable() ? BPF_COMPLEXITY_LIMIT_INSNS : BPF_MAXINSNS))
2203                 return -E2BIG;
2204         if (type != BPF_PROG_TYPE_SOCKET_FILTER &&
2205             type != BPF_PROG_TYPE_CGROUP_SKB &&
2206             !bpf_capable())
2207                 return -EPERM;
2208
2209         if (is_net_admin_prog_type(type) && !capable(CAP_NET_ADMIN) && !capable(CAP_SYS_ADMIN))
2210                 return -EPERM;
2211         if (is_perfmon_prog_type(type) && !perfmon_capable())
2212                 return -EPERM;
2213
2214         /* attach_prog_fd/attach_btf_obj_fd can specify fd of either bpf_prog
2215          * or btf, we need to check which one it is
2216          */
2217         if (attr->attach_prog_fd) {
2218                 dst_prog = bpf_prog_get(attr->attach_prog_fd);
2219                 if (IS_ERR(dst_prog)) {
2220                         dst_prog = NULL;
2221                         attach_btf = btf_get_by_fd(attr->attach_btf_obj_fd);
2222                         if (IS_ERR(attach_btf))
2223                                 return -EINVAL;
2224                         if (!btf_is_kernel(attach_btf)) {
2225                                 /* attaching through specifying bpf_prog's BTF
2226                                  * objects directly might be supported eventually
2227                                  */
2228                                 btf_put(attach_btf);
2229                                 return -ENOTSUPP;
2230                         }
2231                 }
2232         } else if (attr->attach_btf_id) {
2233                 /* fall back to vmlinux BTF, if BTF type ID is specified */
2234                 attach_btf = bpf_get_btf_vmlinux();
2235                 if (IS_ERR(attach_btf))
2236                         return PTR_ERR(attach_btf);
2237                 if (!attach_btf)
2238                         return -EINVAL;
2239                 btf_get(attach_btf);
2240         }
2241
2242         bpf_prog_load_fixup_attach_type(attr);
2243         if (bpf_prog_load_check_attach(type, attr->expected_attach_type,
2244                                        attach_btf, attr->attach_btf_id,
2245                                        dst_prog)) {
2246                 if (dst_prog)
2247                         bpf_prog_put(dst_prog);
2248                 if (attach_btf)
2249                         btf_put(attach_btf);
2250                 return -EINVAL;
2251         }
2252
2253         /* plain bpf_prog allocation */
2254         prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER);
2255         if (!prog) {
2256                 if (dst_prog)
2257                         bpf_prog_put(dst_prog);
2258                 if (attach_btf)
2259                         btf_put(attach_btf);
2260                 return -ENOMEM;
2261         }
2262
2263         prog->expected_attach_type = attr->expected_attach_type;
2264         prog->aux->attach_btf = attach_btf;
2265         prog->aux->attach_btf_id = attr->attach_btf_id;
2266         prog->aux->dst_prog = dst_prog;
2267         prog->aux->offload_requested = !!attr->prog_ifindex;
2268         prog->aux->sleepable = attr->prog_flags & BPF_F_SLEEPABLE;
2269
2270         err = security_bpf_prog_alloc(prog->aux);
2271         if (err)
2272                 goto free_prog;
2273
2274         prog->aux->user = get_current_user();
2275         prog->len = attr->insn_cnt;
2276
2277         err = -EFAULT;
2278         if (copy_from_bpfptr(prog->insns,
2279                              make_bpfptr(attr->insns, uattr.is_kernel),
2280                              bpf_prog_insn_size(prog)) != 0)
2281                 goto free_prog_sec;
2282
2283         prog->orig_prog = NULL;
2284         prog->jited = 0;
2285
2286         atomic64_set(&prog->aux->refcnt, 1);
2287         prog->gpl_compatible = is_gpl ? 1 : 0;
2288
2289         if (bpf_prog_is_dev_bound(prog->aux)) {
2290                 err = bpf_prog_offload_init(prog, attr);
2291                 if (err)
2292                         goto free_prog_sec;
2293         }
2294
2295         /* find program type: socket_filter vs tracing_filter */
2296         err = find_prog_type(type, prog);
2297         if (err < 0)
2298                 goto free_prog_sec;
2299
2300         prog->aux->load_time = ktime_get_boottime_ns();
2301         err = bpf_obj_name_cpy(prog->aux->name, attr->prog_name,
2302                                sizeof(attr->prog_name));
2303         if (err < 0)
2304                 goto free_prog_sec;
2305
2306         /* run eBPF verifier */
2307         err = bpf_check(&prog, attr, uattr);
2308         if (err < 0)
2309                 goto free_used_maps;
2310
2311         prog = bpf_prog_select_runtime(prog, &err);
2312         if (err < 0)
2313                 goto free_used_maps;
2314
2315         err = bpf_prog_alloc_id(prog);
2316         if (err)
2317                 goto free_used_maps;
2318
2319         /* Upon success of bpf_prog_alloc_id(), the BPF prog is
2320          * effectively publicly exposed. However, retrieving via
2321          * bpf_prog_get_fd_by_id() will take another reference,
2322          * therefore it cannot be gone underneath us.
2323          *
2324          * Only for the time /after/ successful bpf_prog_new_fd()
2325          * and before returning to userspace, we might just hold
2326          * one reference and any parallel close on that fd could
2327          * rip everything out. Hence, below notifications must
2328          * happen before bpf_prog_new_fd().
2329          *
2330          * Also, any failure handling from this point onwards must
2331          * be using bpf_prog_put() given the program is exposed.
2332          */
2333         bpf_prog_kallsyms_add(prog);
2334         perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_LOAD, 0);
2335         bpf_audit_prog(prog, BPF_AUDIT_LOAD);
2336
2337         err = bpf_prog_new_fd(prog);
2338         if (err < 0)
2339                 bpf_prog_put(prog);
2340         return err;
2341
2342 free_used_maps:
2343         /* In case we have subprogs, we need to wait for a grace
2344          * period before we can tear down JIT memory since symbols
2345          * are already exposed under kallsyms.
2346          */
2347         __bpf_prog_put_noref(prog, prog->aux->func_cnt);
2348         return err;
2349 free_prog_sec:
2350         free_uid(prog->aux->user);
2351         security_bpf_prog_free(prog->aux);
2352 free_prog:
2353         if (prog->aux->attach_btf)
2354                 btf_put(prog->aux->attach_btf);
2355         bpf_prog_free(prog);
2356         return err;
2357 }
2358
2359 #define BPF_OBJ_LAST_FIELD file_flags
2360
2361 static int bpf_obj_pin(const union bpf_attr *attr)
2362 {
2363         if (CHECK_ATTR(BPF_OBJ) || attr->file_flags != 0)
2364                 return -EINVAL;
2365
2366         return bpf_obj_pin_user(attr->bpf_fd, u64_to_user_ptr(attr->pathname));
2367 }
2368
2369 static int bpf_obj_get(const union bpf_attr *attr)
2370 {
2371         if (CHECK_ATTR(BPF_OBJ) || attr->bpf_fd != 0 ||
2372             attr->file_flags & ~BPF_OBJ_FLAG_MASK)
2373                 return -EINVAL;
2374
2375         return bpf_obj_get_user(u64_to_user_ptr(attr->pathname),
2376                                 attr->file_flags);
2377 }
2378
2379 void bpf_link_init(struct bpf_link *link, enum bpf_link_type type,
2380                    const struct bpf_link_ops *ops, struct bpf_prog *prog)
2381 {
2382         atomic64_set(&link->refcnt, 1);
2383         link->type = type;
2384         link->id = 0;
2385         link->ops = ops;
2386         link->prog = prog;
2387 }
2388
2389 static void bpf_link_free_id(int id)
2390 {
2391         if (!id)
2392                 return;
2393
2394         spin_lock_bh(&link_idr_lock);
2395         idr_remove(&link_idr, id);
2396         spin_unlock_bh(&link_idr_lock);
2397 }
2398
2399 /* Clean up bpf_link and corresponding anon_inode file and FD. After
2400  * anon_inode is created, bpf_link can't be just kfree()'d due to deferred
2401  * anon_inode's release() call. This helper marksbpf_link as
2402  * defunct, releases anon_inode file and puts reserved FD. bpf_prog's refcnt
2403  * is not decremented, it's the responsibility of a calling code that failed
2404  * to complete bpf_link initialization.
2405  */
2406 void bpf_link_cleanup(struct bpf_link_primer *primer)
2407 {
2408         primer->link->prog = NULL;
2409         bpf_link_free_id(primer->id);
2410         fput(primer->file);
2411         put_unused_fd(primer->fd);
2412 }
2413
2414 void bpf_link_inc(struct bpf_link *link)
2415 {
2416         atomic64_inc(&link->refcnt);
2417 }
2418
2419 /* bpf_link_free is guaranteed to be called from process context */
2420 static void bpf_link_free(struct bpf_link *link)
2421 {
2422         bpf_link_free_id(link->id);
2423         if (link->prog) {
2424                 /* detach BPF program, clean up used resources */
2425                 link->ops->release(link);
2426                 bpf_prog_put(link->prog);
2427         }
2428         /* free bpf_link and its containing memory */
2429         link->ops->dealloc(link);
2430 }
2431
2432 static void bpf_link_put_deferred(struct work_struct *work)
2433 {
2434         struct bpf_link *link = container_of(work, struct bpf_link, work);
2435
2436         bpf_link_free(link);
2437 }
2438
2439 /* bpf_link_put can be called from atomic context, but ensures that resources
2440  * are freed from process context
2441  */
2442 void bpf_link_put(struct bpf_link *link)
2443 {
2444         if (!atomic64_dec_and_test(&link->refcnt))
2445                 return;
2446
2447         if (in_atomic()) {
2448                 INIT_WORK(&link->work, bpf_link_put_deferred);
2449                 schedule_work(&link->work);
2450         } else {
2451                 bpf_link_free(link);
2452         }
2453 }
2454
2455 static int bpf_link_release(struct inode *inode, struct file *filp)
2456 {
2457         struct bpf_link *link = filp->private_data;
2458
2459         bpf_link_put(link);
2460         return 0;
2461 }
2462
2463 #ifdef CONFIG_PROC_FS
2464 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type)
2465 #define BPF_MAP_TYPE(_id, _ops)
2466 #define BPF_LINK_TYPE(_id, _name) [_id] = #_name,
2467 static const char *bpf_link_type_strs[] = {
2468         [BPF_LINK_TYPE_UNSPEC] = "<invalid>",
2469 #include <linux/bpf_types.h>
2470 };
2471 #undef BPF_PROG_TYPE
2472 #undef BPF_MAP_TYPE
2473 #undef BPF_LINK_TYPE
2474
2475 static void bpf_link_show_fdinfo(struct seq_file *m, struct file *filp)
2476 {
2477         const struct bpf_link *link = filp->private_data;
2478         const struct bpf_prog *prog = link->prog;
2479         char prog_tag[sizeof(prog->tag) * 2 + 1] = { };
2480
2481         bin2hex(prog_tag, prog->tag, sizeof(prog->tag));
2482         seq_printf(m,
2483                    "link_type:\t%s\n"
2484                    "link_id:\t%u\n"
2485                    "prog_tag:\t%s\n"
2486                    "prog_id:\t%u\n",
2487                    bpf_link_type_strs[link->type],
2488                    link->id,
2489                    prog_tag,
2490                    prog->aux->id);
2491         if (link->ops->show_fdinfo)
2492                 link->ops->show_fdinfo(link, m);
2493 }
2494 #endif
2495
2496 static const struct file_operations bpf_link_fops = {
2497 #ifdef CONFIG_PROC_FS
2498         .show_fdinfo    = bpf_link_show_fdinfo,
2499 #endif
2500         .release        = bpf_link_release,
2501         .read           = bpf_dummy_read,
2502         .write          = bpf_dummy_write,
2503 };
2504
2505 static int bpf_link_alloc_id(struct bpf_link *link)
2506 {
2507         int id;
2508
2509         idr_preload(GFP_KERNEL);
2510         spin_lock_bh(&link_idr_lock);
2511         id = idr_alloc_cyclic(&link_idr, link, 1, INT_MAX, GFP_ATOMIC);
2512         spin_unlock_bh(&link_idr_lock);
2513         idr_preload_end();
2514
2515         return id;
2516 }
2517
2518 /* Prepare bpf_link to be exposed to user-space by allocating anon_inode file,
2519  * reserving unused FD and allocating ID from link_idr. This is to be paired
2520  * with bpf_link_settle() to install FD and ID and expose bpf_link to
2521  * user-space, if bpf_link is successfully attached. If not, bpf_link and
2522  * pre-allocated resources are to be freed with bpf_cleanup() call. All the
2523  * transient state is passed around in struct bpf_link_primer.
2524  * This is preferred way to create and initialize bpf_link, especially when
2525  * there are complicated and expensive operations inbetween creating bpf_link
2526  * itself and attaching it to BPF hook. By using bpf_link_prime() and
2527  * bpf_link_settle() kernel code using bpf_link doesn't have to perform
2528  * expensive (and potentially failing) roll back operations in a rare case
2529  * that file, FD, or ID can't be allocated.
2530  */
2531 int bpf_link_prime(struct bpf_link *link, struct bpf_link_primer *primer)
2532 {
2533         struct file *file;
2534         int fd, id;
2535
2536         fd = get_unused_fd_flags(O_CLOEXEC);
2537         if (fd < 0)
2538                 return fd;
2539
2540
2541         id = bpf_link_alloc_id(link);
2542         if (id < 0) {
2543                 put_unused_fd(fd);
2544                 return id;
2545         }
2546
2547         file = anon_inode_getfile("bpf_link", &bpf_link_fops, link, O_CLOEXEC);
2548         if (IS_ERR(file)) {
2549                 bpf_link_free_id(id);
2550                 put_unused_fd(fd);
2551                 return PTR_ERR(file);
2552         }
2553
2554         primer->link = link;
2555         primer->file = file;
2556         primer->fd = fd;
2557         primer->id = id;
2558         return 0;
2559 }
2560
2561 int bpf_link_settle(struct bpf_link_primer *primer)
2562 {
2563         /* make bpf_link fetchable by ID */
2564         spin_lock_bh(&link_idr_lock);
2565         primer->link->id = primer->id;
2566         spin_unlock_bh(&link_idr_lock);
2567         /* make bpf_link fetchable by FD */
2568         fd_install(primer->fd, primer->file);
2569         /* pass through installed FD */
2570         return primer->fd;
2571 }
2572
2573 int bpf_link_new_fd(struct bpf_link *link)
2574 {
2575         return anon_inode_getfd("bpf-link", &bpf_link_fops, link, O_CLOEXEC);
2576 }
2577
2578 struct bpf_link *bpf_link_get_from_fd(u32 ufd)
2579 {
2580         struct fd f = fdget(ufd);
2581         struct bpf_link *link;
2582
2583         if (!f.file)
2584                 return ERR_PTR(-EBADF);
2585         if (f.file->f_op != &bpf_link_fops) {
2586                 fdput(f);
2587                 return ERR_PTR(-EINVAL);
2588         }
2589
2590         link = f.file->private_data;
2591         bpf_link_inc(link);
2592         fdput(f);
2593
2594         return link;
2595 }
2596
2597 struct bpf_tracing_link {
2598         struct bpf_link link;
2599         enum bpf_attach_type attach_type;
2600         struct bpf_trampoline *trampoline;
2601         struct bpf_prog *tgt_prog;
2602 };
2603
2604 static void bpf_tracing_link_release(struct bpf_link *link)
2605 {
2606         struct bpf_tracing_link *tr_link =
2607                 container_of(link, struct bpf_tracing_link, link);
2608
2609         WARN_ON_ONCE(bpf_trampoline_unlink_prog(link->prog,
2610                                                 tr_link->trampoline));
2611
2612         bpf_trampoline_put(tr_link->trampoline);
2613
2614         /* tgt_prog is NULL if target is a kernel function */
2615         if (tr_link->tgt_prog)
2616                 bpf_prog_put(tr_link->tgt_prog);
2617 }
2618
2619 static void bpf_tracing_link_dealloc(struct bpf_link *link)
2620 {
2621         struct bpf_tracing_link *tr_link =
2622                 container_of(link, struct bpf_tracing_link, link);
2623
2624         kfree(tr_link);
2625 }
2626
2627 static void bpf_tracing_link_show_fdinfo(const struct bpf_link *link,
2628                                          struct seq_file *seq)
2629 {
2630         struct bpf_tracing_link *tr_link =
2631                 container_of(link, struct bpf_tracing_link, link);
2632
2633         seq_printf(seq,
2634                    "attach_type:\t%d\n",
2635                    tr_link->attach_type);
2636 }
2637
2638 static int bpf_tracing_link_fill_link_info(const struct bpf_link *link,
2639                                            struct bpf_link_info *info)
2640 {
2641         struct bpf_tracing_link *tr_link =
2642                 container_of(link, struct bpf_tracing_link, link);
2643
2644         info->tracing.attach_type = tr_link->attach_type;
2645         bpf_trampoline_unpack_key(tr_link->trampoline->key,
2646                                   &info->tracing.target_obj_id,
2647                                   &info->tracing.target_btf_id);
2648
2649         return 0;
2650 }
2651
2652 static const struct bpf_link_ops bpf_tracing_link_lops = {
2653         .release = bpf_tracing_link_release,
2654         .dealloc = bpf_tracing_link_dealloc,
2655         .show_fdinfo = bpf_tracing_link_show_fdinfo,
2656         .fill_link_info = bpf_tracing_link_fill_link_info,
2657 };
2658
2659 static int bpf_tracing_prog_attach(struct bpf_prog *prog,
2660                                    int tgt_prog_fd,
2661                                    u32 btf_id)
2662 {
2663         struct bpf_link_primer link_primer;
2664         struct bpf_prog *tgt_prog = NULL;
2665         struct bpf_trampoline *tr = NULL;
2666         struct bpf_tracing_link *link;
2667         u64 key = 0;
2668         int err;
2669
2670         switch (prog->type) {
2671         case BPF_PROG_TYPE_TRACING:
2672                 if (prog->expected_attach_type != BPF_TRACE_FENTRY &&
2673                     prog->expected_attach_type != BPF_TRACE_FEXIT &&
2674                     prog->expected_attach_type != BPF_MODIFY_RETURN) {
2675                         err = -EINVAL;
2676                         goto out_put_prog;
2677                 }
2678                 break;
2679         case BPF_PROG_TYPE_EXT:
2680                 if (prog->expected_attach_type != 0) {
2681                         err = -EINVAL;
2682                         goto out_put_prog;
2683                 }
2684                 break;
2685         case BPF_PROG_TYPE_LSM:
2686                 if (prog->expected_attach_type != BPF_LSM_MAC) {
2687                         err = -EINVAL;
2688                         goto out_put_prog;
2689                 }
2690                 break;
2691         default:
2692                 err = -EINVAL;
2693                 goto out_put_prog;
2694         }
2695
2696         if (!!tgt_prog_fd != !!btf_id) {
2697                 err = -EINVAL;
2698                 goto out_put_prog;
2699         }
2700
2701         if (tgt_prog_fd) {
2702                 /* For now we only allow new targets for BPF_PROG_TYPE_EXT */
2703                 if (prog->type != BPF_PROG_TYPE_EXT) {
2704                         err = -EINVAL;
2705                         goto out_put_prog;
2706                 }
2707
2708                 tgt_prog = bpf_prog_get(tgt_prog_fd);
2709                 if (IS_ERR(tgt_prog)) {
2710                         err = PTR_ERR(tgt_prog);
2711                         tgt_prog = NULL;
2712                         goto out_put_prog;
2713                 }
2714
2715                 key = bpf_trampoline_compute_key(tgt_prog, NULL, btf_id);
2716         }
2717
2718         link = kzalloc(sizeof(*link), GFP_USER);
2719         if (!link) {
2720                 err = -ENOMEM;
2721                 goto out_put_prog;
2722         }
2723         bpf_link_init(&link->link, BPF_LINK_TYPE_TRACING,
2724                       &bpf_tracing_link_lops, prog);
2725         link->attach_type = prog->expected_attach_type;
2726
2727         mutex_lock(&prog->aux->dst_mutex);
2728
2729         /* There are a few possible cases here:
2730          *
2731          * - if prog->aux->dst_trampoline is set, the program was just loaded
2732          *   and not yet attached to anything, so we can use the values stored
2733          *   in prog->aux
2734          *
2735          * - if prog->aux->dst_trampoline is NULL, the program has already been
2736          *   attached to a target and its initial target was cleared (below)
2737          *
2738          * - if tgt_prog != NULL, the caller specified tgt_prog_fd +
2739          *   target_btf_id using the link_create API.
2740          *
2741          * - if tgt_prog == NULL when this function was called using the old
2742          *   raw_tracepoint_open API, and we need a target from prog->aux
2743          *
2744          * - if prog->aux->dst_trampoline and tgt_prog is NULL, the program
2745          *   was detached and is going for re-attachment.
2746          */
2747         if (!prog->aux->dst_trampoline && !tgt_prog) {
2748                 /*
2749                  * Allow re-attach for TRACING and LSM programs. If it's
2750                  * currently linked, bpf_trampoline_link_prog will fail.
2751                  * EXT programs need to specify tgt_prog_fd, so they
2752                  * re-attach in separate code path.
2753                  */
2754                 if (prog->type != BPF_PROG_TYPE_TRACING &&
2755                     prog->type != BPF_PROG_TYPE_LSM) {
2756                         err = -EINVAL;
2757                         goto out_unlock;
2758                 }
2759                 btf_id = prog->aux->attach_btf_id;
2760                 key = bpf_trampoline_compute_key(NULL, prog->aux->attach_btf, btf_id);
2761         }
2762
2763         if (!prog->aux->dst_trampoline ||
2764             (key && key != prog->aux->dst_trampoline->key)) {
2765                 /* If there is no saved target, or the specified target is
2766                  * different from the destination specified at load time, we
2767                  * need a new trampoline and a check for compatibility
2768                  */
2769                 struct bpf_attach_target_info tgt_info = {};
2770
2771                 err = bpf_check_attach_target(NULL, prog, tgt_prog, btf_id,
2772                                               &tgt_info);
2773                 if (err)
2774                         goto out_unlock;
2775
2776                 tr = bpf_trampoline_get(key, &tgt_info);
2777                 if (!tr) {
2778                         err = -ENOMEM;
2779                         goto out_unlock;
2780                 }
2781         } else {
2782                 /* The caller didn't specify a target, or the target was the
2783                  * same as the destination supplied during program load. This
2784                  * means we can reuse the trampoline and reference from program
2785                  * load time, and there is no need to allocate a new one. This
2786                  * can only happen once for any program, as the saved values in
2787                  * prog->aux are cleared below.
2788                  */
2789                 tr = prog->aux->dst_trampoline;
2790                 tgt_prog = prog->aux->dst_prog;
2791         }
2792
2793         err = bpf_link_prime(&link->link, &link_primer);
2794         if (err)
2795                 goto out_unlock;
2796
2797         err = bpf_trampoline_link_prog(prog, tr);
2798         if (err) {
2799                 bpf_link_cleanup(&link_primer);
2800                 link = NULL;
2801                 goto out_unlock;
2802         }
2803
2804         link->tgt_prog = tgt_prog;
2805         link->trampoline = tr;
2806
2807         /* Always clear the trampoline and target prog from prog->aux to make
2808          * sure the original attach destination is not kept alive after a
2809          * program is (re-)attached to another target.
2810          */
2811         if (prog->aux->dst_prog &&
2812             (tgt_prog_fd || tr != prog->aux->dst_trampoline))
2813                 /* got extra prog ref from syscall, or attaching to different prog */
2814                 bpf_prog_put(prog->aux->dst_prog);
2815         if (prog->aux->dst_trampoline && tr != prog->aux->dst_trampoline)
2816                 /* we allocated a new trampoline, so free the old one */
2817                 bpf_trampoline_put(prog->aux->dst_trampoline);
2818
2819         prog->aux->dst_prog = NULL;
2820         prog->aux->dst_trampoline = NULL;
2821         mutex_unlock(&prog->aux->dst_mutex);
2822
2823         return bpf_link_settle(&link_primer);
2824 out_unlock:
2825         if (tr && tr != prog->aux->dst_trampoline)
2826                 bpf_trampoline_put(tr);
2827         mutex_unlock(&prog->aux->dst_mutex);
2828         kfree(link);
2829 out_put_prog:
2830         if (tgt_prog_fd && tgt_prog)
2831                 bpf_prog_put(tgt_prog);
2832         return err;
2833 }
2834
2835 struct bpf_raw_tp_link {
2836         struct bpf_link link;
2837         struct bpf_raw_event_map *btp;
2838 };
2839
2840 static void bpf_raw_tp_link_release(struct bpf_link *link)
2841 {
2842         struct bpf_raw_tp_link *raw_tp =
2843                 container_of(link, struct bpf_raw_tp_link, link);
2844
2845         bpf_probe_unregister(raw_tp->btp, raw_tp->link.prog);
2846         bpf_put_raw_tracepoint(raw_tp->btp);
2847 }
2848
2849 static void bpf_raw_tp_link_dealloc(struct bpf_link *link)
2850 {
2851         struct bpf_raw_tp_link *raw_tp =
2852                 container_of(link, struct bpf_raw_tp_link, link);
2853
2854         kfree(raw_tp);
2855 }
2856
2857 static void bpf_raw_tp_link_show_fdinfo(const struct bpf_link *link,
2858                                         struct seq_file *seq)
2859 {
2860         struct bpf_raw_tp_link *raw_tp_link =
2861                 container_of(link, struct bpf_raw_tp_link, link);
2862
2863         seq_printf(seq,
2864                    "tp_name:\t%s\n",
2865                    raw_tp_link->btp->tp->name);
2866 }
2867
2868 static int bpf_raw_tp_link_fill_link_info(const struct bpf_link *link,
2869                                           struct bpf_link_info *info)
2870 {
2871         struct bpf_raw_tp_link *raw_tp_link =
2872                 container_of(link, struct bpf_raw_tp_link, link);
2873         char __user *ubuf = u64_to_user_ptr(info->raw_tracepoint.tp_name);
2874         const char *tp_name = raw_tp_link->btp->tp->name;
2875         u32 ulen = info->raw_tracepoint.tp_name_len;
2876         size_t tp_len = strlen(tp_name);
2877
2878         if (!ulen ^ !ubuf)
2879                 return -EINVAL;
2880
2881         info->raw_tracepoint.tp_name_len = tp_len + 1;
2882
2883         if (!ubuf)
2884                 return 0;
2885
2886         if (ulen >= tp_len + 1) {
2887                 if (copy_to_user(ubuf, tp_name, tp_len + 1))
2888                         return -EFAULT;
2889         } else {
2890                 char zero = '\0';
2891
2892                 if (copy_to_user(ubuf, tp_name, ulen - 1))
2893                         return -EFAULT;
2894                 if (put_user(zero, ubuf + ulen - 1))
2895                         return -EFAULT;
2896                 return -ENOSPC;
2897         }
2898
2899         return 0;
2900 }
2901
2902 static const struct bpf_link_ops bpf_raw_tp_link_lops = {
2903         .release = bpf_raw_tp_link_release,
2904         .dealloc = bpf_raw_tp_link_dealloc,
2905         .show_fdinfo = bpf_raw_tp_link_show_fdinfo,
2906         .fill_link_info = bpf_raw_tp_link_fill_link_info,
2907 };
2908
2909 #define BPF_RAW_TRACEPOINT_OPEN_LAST_FIELD raw_tracepoint.prog_fd
2910
2911 static int bpf_raw_tracepoint_open(const union bpf_attr *attr)
2912 {
2913         struct bpf_link_primer link_primer;
2914         struct bpf_raw_tp_link *link;
2915         struct bpf_raw_event_map *btp;
2916         struct bpf_prog *prog;
2917         const char *tp_name;
2918         char buf[128];
2919         int err;
2920
2921         if (CHECK_ATTR(BPF_RAW_TRACEPOINT_OPEN))
2922                 return -EINVAL;
2923
2924         prog = bpf_prog_get(attr->raw_tracepoint.prog_fd);
2925         if (IS_ERR(prog))
2926                 return PTR_ERR(prog);
2927
2928         switch (prog->type) {
2929         case BPF_PROG_TYPE_TRACING:
2930         case BPF_PROG_TYPE_EXT:
2931         case BPF_PROG_TYPE_LSM:
2932                 if (attr->raw_tracepoint.name) {
2933                         /* The attach point for this category of programs
2934                          * should be specified via btf_id during program load.
2935                          */
2936                         err = -EINVAL;
2937                         goto out_put_prog;
2938                 }
2939                 if (prog->type == BPF_PROG_TYPE_TRACING &&
2940                     prog->expected_attach_type == BPF_TRACE_RAW_TP) {
2941                         tp_name = prog->aux->attach_func_name;
2942                         break;
2943                 }
2944                 err = bpf_tracing_prog_attach(prog, 0, 0);
2945                 if (err >= 0)
2946                         return err;
2947                 goto out_put_prog;
2948         case BPF_PROG_TYPE_RAW_TRACEPOINT:
2949         case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE:
2950                 if (strncpy_from_user(buf,
2951                                       u64_to_user_ptr(attr->raw_tracepoint.name),
2952                                       sizeof(buf) - 1) < 0) {
2953                         err = -EFAULT;
2954                         goto out_put_prog;
2955                 }
2956                 buf[sizeof(buf) - 1] = 0;
2957                 tp_name = buf;
2958                 break;
2959         default:
2960                 err = -EINVAL;
2961                 goto out_put_prog;
2962         }
2963
2964         btp = bpf_get_raw_tracepoint(tp_name);
2965         if (!btp) {
2966                 err = -ENOENT;
2967                 goto out_put_prog;
2968         }
2969
2970         link = kzalloc(sizeof(*link), GFP_USER);
2971         if (!link) {
2972                 err = -ENOMEM;
2973                 goto out_put_btp;
2974         }
2975         bpf_link_init(&link->link, BPF_LINK_TYPE_RAW_TRACEPOINT,
2976                       &bpf_raw_tp_link_lops, prog);
2977         link->btp = btp;
2978
2979         err = bpf_link_prime(&link->link, &link_primer);
2980         if (err) {
2981                 kfree(link);
2982                 goto out_put_btp;
2983         }
2984
2985         err = bpf_probe_register(link->btp, prog);
2986         if (err) {
2987                 bpf_link_cleanup(&link_primer);
2988                 goto out_put_btp;
2989         }
2990
2991         return bpf_link_settle(&link_primer);
2992
2993 out_put_btp:
2994         bpf_put_raw_tracepoint(btp);
2995 out_put_prog:
2996         bpf_prog_put(prog);
2997         return err;
2998 }
2999
3000 static int bpf_prog_attach_check_attach_type(const struct bpf_prog *prog,
3001                                              enum bpf_attach_type attach_type)
3002 {
3003         switch (prog->type) {
3004         case BPF_PROG_TYPE_CGROUP_SOCK:
3005         case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
3006         case BPF_PROG_TYPE_CGROUP_SOCKOPT:
3007         case BPF_PROG_TYPE_SK_LOOKUP:
3008                 return attach_type == prog->expected_attach_type ? 0 : -EINVAL;
3009         case BPF_PROG_TYPE_CGROUP_SKB:
3010                 if (!capable(CAP_NET_ADMIN))
3011                         /* cg-skb progs can be loaded by unpriv user.
3012                          * check permissions at attach time.
3013                          */
3014                         return -EPERM;
3015                 return prog->enforce_expected_attach_type &&
3016                         prog->expected_attach_type != attach_type ?
3017                         -EINVAL : 0;
3018         default:
3019                 return 0;
3020         }
3021 }
3022
3023 static enum bpf_prog_type
3024 attach_type_to_prog_type(enum bpf_attach_type attach_type)
3025 {
3026         switch (attach_type) {
3027         case BPF_CGROUP_INET_INGRESS:
3028         case BPF_CGROUP_INET_EGRESS:
3029                 return BPF_PROG_TYPE_CGROUP_SKB;
3030         case BPF_CGROUP_INET_SOCK_CREATE:
3031         case BPF_CGROUP_INET_SOCK_RELEASE:
3032         case BPF_CGROUP_INET4_POST_BIND:
3033         case BPF_CGROUP_INET6_POST_BIND:
3034                 return BPF_PROG_TYPE_CGROUP_SOCK;
3035         case BPF_CGROUP_INET4_BIND:
3036         case BPF_CGROUP_INET6_BIND:
3037         case BPF_CGROUP_INET4_CONNECT:
3038         case BPF_CGROUP_INET6_CONNECT:
3039         case BPF_CGROUP_INET4_GETPEERNAME:
3040         case BPF_CGROUP_INET6_GETPEERNAME:
3041         case BPF_CGROUP_INET4_GETSOCKNAME:
3042         case BPF_CGROUP_INET6_GETSOCKNAME:
3043         case BPF_CGROUP_UDP4_SENDMSG:
3044         case BPF_CGROUP_UDP6_SENDMSG:
3045         case BPF_CGROUP_UDP4_RECVMSG:
3046         case BPF_CGROUP_UDP6_RECVMSG:
3047                 return BPF_PROG_TYPE_CGROUP_SOCK_ADDR;
3048         case BPF_CGROUP_SOCK_OPS:
3049                 return BPF_PROG_TYPE_SOCK_OPS;
3050         case BPF_CGROUP_DEVICE:
3051                 return BPF_PROG_TYPE_CGROUP_DEVICE;
3052         case BPF_SK_MSG_VERDICT:
3053                 return BPF_PROG_TYPE_SK_MSG;
3054         case BPF_SK_SKB_STREAM_PARSER:
3055         case BPF_SK_SKB_STREAM_VERDICT:
3056         case BPF_SK_SKB_VERDICT:
3057                 return BPF_PROG_TYPE_SK_SKB;
3058         case BPF_LIRC_MODE2:
3059                 return BPF_PROG_TYPE_LIRC_MODE2;
3060         case BPF_FLOW_DISSECTOR:
3061                 return BPF_PROG_TYPE_FLOW_DISSECTOR;
3062         case BPF_CGROUP_SYSCTL:
3063                 return BPF_PROG_TYPE_CGROUP_SYSCTL;
3064         case BPF_CGROUP_GETSOCKOPT:
3065         case BPF_CGROUP_SETSOCKOPT:
3066                 return BPF_PROG_TYPE_CGROUP_SOCKOPT;
3067         case BPF_TRACE_ITER:
3068                 return BPF_PROG_TYPE_TRACING;
3069         case BPF_SK_LOOKUP:
3070                 return BPF_PROG_TYPE_SK_LOOKUP;
3071         case BPF_XDP:
3072                 return BPF_PROG_TYPE_XDP;
3073         default:
3074                 return BPF_PROG_TYPE_UNSPEC;
3075         }
3076 }
3077
3078 #define BPF_PROG_ATTACH_LAST_FIELD replace_bpf_fd
3079
3080 #define BPF_F_ATTACH_MASK \
3081         (BPF_F_ALLOW_OVERRIDE | BPF_F_ALLOW_MULTI | BPF_F_REPLACE)
3082
3083 static int bpf_prog_attach(const union bpf_attr *attr)
3084 {
3085         enum bpf_prog_type ptype;
3086         struct bpf_prog *prog;
3087         int ret;
3088
3089         if (CHECK_ATTR(BPF_PROG_ATTACH))
3090                 return -EINVAL;
3091
3092         if (attr->attach_flags & ~BPF_F_ATTACH_MASK)
3093                 return -EINVAL;
3094
3095         ptype = attach_type_to_prog_type(attr->attach_type);
3096         if (ptype == BPF_PROG_TYPE_UNSPEC)
3097                 return -EINVAL;
3098
3099         prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype);
3100         if (IS_ERR(prog))
3101                 return PTR_ERR(prog);
3102
3103         if (bpf_prog_attach_check_attach_type(prog, attr->attach_type)) {
3104                 bpf_prog_put(prog);
3105                 return -EINVAL;
3106         }
3107
3108         switch (ptype) {
3109         case BPF_PROG_TYPE_SK_SKB:
3110         case BPF_PROG_TYPE_SK_MSG:
3111                 ret = sock_map_get_from_fd(attr, prog);
3112                 break;
3113         case BPF_PROG_TYPE_LIRC_MODE2:
3114                 ret = lirc_prog_attach(attr, prog);
3115                 break;
3116         case BPF_PROG_TYPE_FLOW_DISSECTOR:
3117                 ret = netns_bpf_prog_attach(attr, prog);
3118                 break;
3119         case BPF_PROG_TYPE_CGROUP_DEVICE:
3120         case BPF_PROG_TYPE_CGROUP_SKB:
3121         case BPF_PROG_TYPE_CGROUP_SOCK:
3122         case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
3123         case BPF_PROG_TYPE_CGROUP_SOCKOPT:
3124         case BPF_PROG_TYPE_CGROUP_SYSCTL:
3125         case BPF_PROG_TYPE_SOCK_OPS:
3126                 ret = cgroup_bpf_prog_attach(attr, ptype, prog);
3127                 break;
3128         default:
3129                 ret = -EINVAL;
3130         }
3131
3132         if (ret)
3133                 bpf_prog_put(prog);
3134         return ret;
3135 }
3136
3137 #define BPF_PROG_DETACH_LAST_FIELD attach_type
3138
3139 static int bpf_prog_detach(const union bpf_attr *attr)
3140 {
3141         enum bpf_prog_type ptype;
3142
3143         if (CHECK_ATTR(BPF_PROG_DETACH))
3144                 return -EINVAL;
3145
3146         ptype = attach_type_to_prog_type(attr->attach_type);
3147
3148         switch (ptype) {
3149         case BPF_PROG_TYPE_SK_MSG:
3150         case BPF_PROG_TYPE_SK_SKB:
3151                 return sock_map_prog_detach(attr, ptype);
3152         case BPF_PROG_TYPE_LIRC_MODE2:
3153                 return lirc_prog_detach(attr);
3154         case BPF_PROG_TYPE_FLOW_DISSECTOR:
3155                 return netns_bpf_prog_detach(attr, ptype);
3156         case BPF_PROG_TYPE_CGROUP_DEVICE:
3157         case BPF_PROG_TYPE_CGROUP_SKB:
3158         case BPF_PROG_TYPE_CGROUP_SOCK:
3159         case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
3160         case BPF_PROG_TYPE_CGROUP_SOCKOPT:
3161         case BPF_PROG_TYPE_CGROUP_SYSCTL:
3162         case BPF_PROG_TYPE_SOCK_OPS:
3163                 return cgroup_bpf_prog_detach(attr, ptype);
3164         default:
3165                 return -EINVAL;
3166         }
3167 }
3168
3169 #define BPF_PROG_QUERY_LAST_FIELD query.prog_cnt
3170
3171 static int bpf_prog_query(const union bpf_attr *attr,
3172                           union bpf_attr __user *uattr)
3173 {
3174         if (!capable(CAP_NET_ADMIN))
3175                 return -EPERM;
3176         if (CHECK_ATTR(BPF_PROG_QUERY))
3177                 return -EINVAL;
3178         if (attr->query.query_flags & ~BPF_F_QUERY_EFFECTIVE)
3179                 return -EINVAL;
3180
3181         switch (attr->query.attach_type) {
3182         case BPF_CGROUP_INET_INGRESS:
3183         case BPF_CGROUP_INET_EGRESS:
3184         case BPF_CGROUP_INET_SOCK_CREATE:
3185         case BPF_CGROUP_INET_SOCK_RELEASE:
3186         case BPF_CGROUP_INET4_BIND:
3187         case BPF_CGROUP_INET6_BIND:
3188         case BPF_CGROUP_INET4_POST_BIND:
3189         case BPF_CGROUP_INET6_POST_BIND:
3190         case BPF_CGROUP_INET4_CONNECT:
3191         case BPF_CGROUP_INET6_CONNECT:
3192         case BPF_CGROUP_INET4_GETPEERNAME:
3193         case BPF_CGROUP_INET6_GETPEERNAME:
3194         case BPF_CGROUP_INET4_GETSOCKNAME:
3195         case BPF_CGROUP_INET6_GETSOCKNAME:
3196         case BPF_CGROUP_UDP4_SENDMSG:
3197         case BPF_CGROUP_UDP6_SENDMSG:
3198         case BPF_CGROUP_UDP4_RECVMSG:
3199         case BPF_CGROUP_UDP6_RECVMSG:
3200         case BPF_CGROUP_SOCK_OPS:
3201         case BPF_CGROUP_DEVICE:
3202         case BPF_CGROUP_SYSCTL:
3203         case BPF_CGROUP_GETSOCKOPT:
3204         case BPF_CGROUP_SETSOCKOPT:
3205                 return cgroup_bpf_prog_query(attr, uattr);
3206         case BPF_LIRC_MODE2:
3207                 return lirc_prog_query(attr, uattr);
3208         case BPF_FLOW_DISSECTOR:
3209         case BPF_SK_LOOKUP:
3210                 return netns_bpf_prog_query(attr, uattr);
3211         default:
3212                 return -EINVAL;
3213         }
3214 }
3215
3216 #define BPF_PROG_TEST_RUN_LAST_FIELD test.cpu
3217
3218 static int bpf_prog_test_run(const union bpf_attr *attr,
3219                              union bpf_attr __user *uattr)
3220 {
3221         struct bpf_prog *prog;
3222         int ret = -ENOTSUPP;
3223
3224         if (CHECK_ATTR(BPF_PROG_TEST_RUN))
3225                 return -EINVAL;
3226
3227         if ((attr->test.ctx_size_in && !attr->test.ctx_in) ||
3228             (!attr->test.ctx_size_in && attr->test.ctx_in))
3229                 return -EINVAL;
3230
3231         if ((attr->test.ctx_size_out && !attr->test.ctx_out) ||
3232             (!attr->test.ctx_size_out && attr->test.ctx_out))
3233                 return -EINVAL;
3234
3235         prog = bpf_prog_get(attr->test.prog_fd);
3236         if (IS_ERR(prog))
3237                 return PTR_ERR(prog);
3238
3239         if (prog->aux->ops->test_run)
3240                 ret = prog->aux->ops->test_run(prog, attr, uattr);
3241
3242         bpf_prog_put(prog);
3243         return ret;
3244 }
3245
3246 #define BPF_OBJ_GET_NEXT_ID_LAST_FIELD next_id
3247
3248 static int bpf_obj_get_next_id(const union bpf_attr *attr,
3249                                union bpf_attr __user *uattr,
3250                                struct idr *idr,
3251                                spinlock_t *lock)
3252 {
3253         u32 next_id = attr->start_id;
3254         int err = 0;
3255
3256         if (CHECK_ATTR(BPF_OBJ_GET_NEXT_ID) || next_id >= INT_MAX)
3257                 return -EINVAL;
3258
3259         if (!capable(CAP_SYS_ADMIN))
3260                 return -EPERM;
3261
3262         next_id++;
3263         spin_lock_bh(lock);
3264         if (!idr_get_next(idr, &next_id))
3265                 err = -ENOENT;
3266         spin_unlock_bh(lock);
3267
3268         if (!err)
3269                 err = put_user(next_id, &uattr->next_id);
3270
3271         return err;
3272 }
3273
3274 struct bpf_map *bpf_map_get_curr_or_next(u32 *id)
3275 {
3276         struct bpf_map *map;
3277
3278         spin_lock_bh(&map_idr_lock);
3279 again:
3280         map = idr_get_next(&map_idr, id);
3281         if (map) {
3282                 map = __bpf_map_inc_not_zero(map, false);
3283                 if (IS_ERR(map)) {
3284                         (*id)++;
3285                         goto again;
3286                 }
3287         }
3288         spin_unlock_bh(&map_idr_lock);
3289
3290         return map;
3291 }
3292
3293 struct bpf_prog *bpf_prog_get_curr_or_next(u32 *id)
3294 {
3295         struct bpf_prog *prog;
3296
3297         spin_lock_bh(&prog_idr_lock);
3298 again:
3299         prog = idr_get_next(&prog_idr, id);
3300         if (prog) {
3301                 prog = bpf_prog_inc_not_zero(prog);
3302                 if (IS_ERR(prog)) {
3303                         (*id)++;
3304                         goto again;
3305                 }
3306         }
3307         spin_unlock_bh(&prog_idr_lock);
3308
3309         return prog;
3310 }
3311
3312 #define BPF_PROG_GET_FD_BY_ID_LAST_FIELD prog_id
3313
3314 struct bpf_prog *bpf_prog_by_id(u32 id)
3315 {
3316         struct bpf_prog *prog;
3317
3318         if (!id)
3319                 return ERR_PTR(-ENOENT);
3320
3321         spin_lock_bh(&prog_idr_lock);
3322         prog = idr_find(&prog_idr, id);
3323         if (prog)
3324                 prog = bpf_prog_inc_not_zero(prog);
3325         else
3326                 prog = ERR_PTR(-ENOENT);
3327         spin_unlock_bh(&prog_idr_lock);
3328         return prog;
3329 }
3330
3331 static int bpf_prog_get_fd_by_id(const union bpf_attr *attr)
3332 {
3333         struct bpf_prog *prog;
3334         u32 id = attr->prog_id;
3335         int fd;
3336
3337         if (CHECK_ATTR(BPF_PROG_GET_FD_BY_ID))
3338                 return -EINVAL;
3339
3340         if (!capable(CAP_SYS_ADMIN))
3341                 return -EPERM;
3342
3343         prog = bpf_prog_by_id(id);
3344         if (IS_ERR(prog))
3345                 return PTR_ERR(prog);
3346
3347         fd = bpf_prog_new_fd(prog);
3348         if (fd < 0)
3349                 bpf_prog_put(prog);
3350
3351         return fd;
3352 }
3353
3354 #define BPF_MAP_GET_FD_BY_ID_LAST_FIELD open_flags
3355
3356 static int bpf_map_get_fd_by_id(const union bpf_attr *attr)
3357 {
3358         struct bpf_map *map;
3359         u32 id = attr->map_id;
3360         int f_flags;
3361         int fd;
3362
3363         if (CHECK_ATTR(BPF_MAP_GET_FD_BY_ID) ||
3364             attr->open_flags & ~BPF_OBJ_FLAG_MASK)
3365                 return -EINVAL;
3366
3367         if (!capable(CAP_SYS_ADMIN))
3368                 return -EPERM;
3369
3370         f_flags = bpf_get_file_flag(attr->open_flags);
3371         if (f_flags < 0)
3372                 return f_flags;
3373
3374         spin_lock_bh(&map_idr_lock);
3375         map = idr_find(&map_idr, id);
3376         if (map)
3377                 map = __bpf_map_inc_not_zero(map, true);
3378         else
3379                 map = ERR_PTR(-ENOENT);
3380         spin_unlock_bh(&map_idr_lock);
3381
3382         if (IS_ERR(map))
3383                 return PTR_ERR(map);
3384
3385         fd = bpf_map_new_fd(map, f_flags);
3386         if (fd < 0)
3387                 bpf_map_put_with_uref(map);
3388
3389         return fd;
3390 }
3391
3392 static const struct bpf_map *bpf_map_from_imm(const struct bpf_prog *prog,
3393                                               unsigned long addr, u32 *off,
3394                                               u32 *type)
3395 {
3396         const struct bpf_map *map;
3397         int i;
3398
3399         mutex_lock(&prog->aux->used_maps_mutex);
3400         for (i = 0, *off = 0; i < prog->aux->used_map_cnt; i++) {
3401                 map = prog->aux->used_maps[i];
3402                 if (map == (void *)addr) {
3403                         *type = BPF_PSEUDO_MAP_FD;
3404                         goto out;
3405                 }
3406                 if (!map->ops->map_direct_value_meta)
3407                         continue;
3408                 if (!map->ops->map_direct_value_meta(map, addr, off)) {
3409                         *type = BPF_PSEUDO_MAP_VALUE;
3410                         goto out;
3411                 }
3412         }
3413         map = NULL;
3414
3415 out:
3416         mutex_unlock(&prog->aux->used_maps_mutex);
3417         return map;
3418 }
3419
3420 static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog,
3421                                               const struct cred *f_cred)
3422 {
3423         const struct bpf_map *map;
3424         struct bpf_insn *insns;
3425         u32 off, type;
3426         u64 imm;
3427         u8 code;
3428         int i;
3429
3430         insns = kmemdup(prog->insnsi, bpf_prog_insn_size(prog),
3431                         GFP_USER);
3432         if (!insns)
3433                 return insns;
3434
3435         for (i = 0; i < prog->len; i++) {
3436                 code = insns[i].code;
3437
3438                 if (code == (BPF_JMP | BPF_TAIL_CALL)) {
3439                         insns[i].code = BPF_JMP | BPF_CALL;
3440                         insns[i].imm = BPF_FUNC_tail_call;
3441                         /* fall-through */
3442                 }
3443                 if (code == (BPF_JMP | BPF_CALL) ||
3444                     code == (BPF_JMP | BPF_CALL_ARGS)) {
3445                         if (code == (BPF_JMP | BPF_CALL_ARGS))
3446                                 insns[i].code = BPF_JMP | BPF_CALL;
3447                         if (!bpf_dump_raw_ok(f_cred))
3448                                 insns[i].imm = 0;
3449                         continue;
3450                 }
3451                 if (BPF_CLASS(code) == BPF_LDX && BPF_MODE(code) == BPF_PROBE_MEM) {
3452                         insns[i].code = BPF_LDX | BPF_SIZE(code) | BPF_MEM;
3453                         continue;
3454                 }
3455
3456                 if (code != (BPF_LD | BPF_IMM | BPF_DW))
3457                         continue;
3458
3459                 imm = ((u64)insns[i + 1].imm << 32) | (u32)insns[i].imm;
3460                 map = bpf_map_from_imm(prog, imm, &off, &type);
3461                 if (map) {
3462                         insns[i].src_reg = type;
3463                         insns[i].imm = map->id;
3464                         insns[i + 1].imm = off;
3465                         continue;
3466                 }
3467         }
3468
3469         return insns;
3470 }
3471
3472 static int set_info_rec_size(struct bpf_prog_info *info)
3473 {
3474         /*
3475          * Ensure info.*_rec_size is the same as kernel expected size
3476          *
3477          * or
3478          *
3479          * Only allow zero *_rec_size if both _rec_size and _cnt are
3480          * zero.  In this case, the kernel will set the expected
3481          * _rec_size back to the info.
3482          */
3483
3484         if ((info->nr_func_info || info->func_info_rec_size) &&
3485             info->func_info_rec_size != sizeof(struct bpf_func_info))
3486                 return -EINVAL;
3487
3488         if ((info->nr_line_info || info->line_info_rec_size) &&
3489             info->line_info_rec_size != sizeof(struct bpf_line_info))
3490                 return -EINVAL;
3491
3492         if ((info->nr_jited_line_info || info->jited_line_info_rec_size) &&
3493             info->jited_line_info_rec_size != sizeof(__u64))
3494                 return -EINVAL;
3495
3496         info->func_info_rec_size = sizeof(struct bpf_func_info);
3497         info->line_info_rec_size = sizeof(struct bpf_line_info);
3498         info->jited_line_info_rec_size = sizeof(__u64);
3499
3500         return 0;
3501 }
3502
3503 static int bpf_prog_get_info_by_fd(struct file *file,
3504                                    struct bpf_prog *prog,
3505                                    const union bpf_attr *attr,
3506                                    union bpf_attr __user *uattr)
3507 {
3508         struct bpf_prog_info __user *uinfo = u64_to_user_ptr(attr->info.info);
3509         struct bpf_prog_info info;
3510         u32 info_len = attr->info.info_len;
3511         struct bpf_prog_stats stats;
3512         char __user *uinsns;
3513         u32 ulen;
3514         int err;
3515
3516         err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(info), info_len);
3517         if (err)
3518                 return err;
3519         info_len = min_t(u32, sizeof(info), info_len);
3520
3521         memset(&info, 0, sizeof(info));
3522         if (copy_from_user(&info, uinfo, info_len))
3523                 return -EFAULT;
3524
3525         info.type = prog->type;
3526         info.id = prog->aux->id;
3527         info.load_time = prog->aux->load_time;
3528         info.created_by_uid = from_kuid_munged(current_user_ns(),
3529                                                prog->aux->user->uid);
3530         info.gpl_compatible = prog->gpl_compatible;
3531
3532         memcpy(info.tag, prog->tag, sizeof(prog->tag));
3533         memcpy(info.name, prog->aux->name, sizeof(prog->aux->name));
3534
3535         mutex_lock(&prog->aux->used_maps_mutex);
3536         ulen = info.nr_map_ids;
3537         info.nr_map_ids = prog->aux->used_map_cnt;
3538         ulen = min_t(u32, info.nr_map_ids, ulen);
3539         if (ulen) {
3540                 u32 __user *user_map_ids = u64_to_user_ptr(info.map_ids);
3541                 u32 i;
3542
3543                 for (i = 0; i < ulen; i++)
3544                         if (put_user(prog->aux->used_maps[i]->id,
3545                                      &user_map_ids[i])) {
3546                                 mutex_unlock(&prog->aux->used_maps_mutex);
3547                                 return -EFAULT;
3548                         }
3549         }
3550         mutex_unlock(&prog->aux->used_maps_mutex);
3551
3552         err = set_info_rec_size(&info);
3553         if (err)
3554                 return err;
3555
3556         bpf_prog_get_stats(prog, &stats);
3557         info.run_time_ns = stats.nsecs;
3558         info.run_cnt = stats.cnt;
3559         info.recursion_misses = stats.misses;
3560
3561         if (!bpf_capable()) {
3562                 info.jited_prog_len = 0;
3563                 info.xlated_prog_len = 0;
3564                 info.nr_jited_ksyms = 0;
3565                 info.nr_jited_func_lens = 0;
3566                 info.nr_func_info = 0;
3567                 info.nr_line_info = 0;
3568                 info.nr_jited_line_info = 0;
3569                 goto done;
3570         }
3571
3572         ulen = info.xlated_prog_len;
3573         info.xlated_prog_len = bpf_prog_insn_size(prog);
3574         if (info.xlated_prog_len && ulen) {
3575                 struct bpf_insn *insns_sanitized;
3576                 bool fault;
3577
3578                 if (prog->blinded && !bpf_dump_raw_ok(file->f_cred)) {
3579                         info.xlated_prog_insns = 0;
3580                         goto done;
3581                 }
3582                 insns_sanitized = bpf_insn_prepare_dump(prog, file->f_cred);
3583                 if (!insns_sanitized)
3584                         return -ENOMEM;
3585                 uinsns = u64_to_user_ptr(info.xlated_prog_insns);
3586                 ulen = min_t(u32, info.xlated_prog_len, ulen);
3587                 fault = copy_to_user(uinsns, insns_sanitized, ulen);
3588                 kfree(insns_sanitized);
3589                 if (fault)
3590                         return -EFAULT;
3591         }
3592
3593         if (bpf_prog_is_dev_bound(prog->aux)) {
3594                 err = bpf_prog_offload_info_fill(&info, prog);
3595                 if (err)
3596                         return err;
3597                 goto done;
3598         }
3599
3600         /* NOTE: the following code is supposed to be skipped for offload.
3601          * bpf_prog_offload_info_fill() is the place to fill similar fields
3602          * for offload.
3603          */
3604         ulen = info.jited_prog_len;
3605         if (prog->aux->func_cnt) {
3606                 u32 i;
3607
3608                 info.jited_prog_len = 0;
3609                 for (i = 0; i < prog->aux->func_cnt; i++)
3610                         info.jited_prog_len += prog->aux->func[i]->jited_len;
3611         } else {
3612                 info.jited_prog_len = prog->jited_len;
3613         }
3614
3615         if (info.jited_prog_len && ulen) {
3616                 if (bpf_dump_raw_ok(file->f_cred)) {
3617                         uinsns = u64_to_user_ptr(info.jited_prog_insns);
3618                         ulen = min_t(u32, info.jited_prog_len, ulen);
3619
3620                         /* for multi-function programs, copy the JITed
3621                          * instructions for all the functions
3622                          */
3623                         if (prog->aux->func_cnt) {
3624                                 u32 len, free, i;
3625                                 u8 *img;
3626
3627                                 free = ulen;
3628                                 for (i = 0; i < prog->aux->func_cnt; i++) {
3629                                         len = prog->aux->func[i]->jited_len;
3630                                         len = min_t(u32, len, free);
3631                                         img = (u8 *) prog->aux->func[i]->bpf_func;
3632                                         if (copy_to_user(uinsns, img, len))
3633                                                 return -EFAULT;
3634                                         uinsns += len;
3635                                         free -= len;
3636                                         if (!free)
3637                                                 break;
3638                                 }
3639                         } else {
3640                                 if (copy_to_user(uinsns, prog->bpf_func, ulen))
3641                                         return -EFAULT;
3642                         }
3643                 } else {
3644                         info.jited_prog_insns = 0;
3645                 }
3646         }
3647
3648         ulen = info.nr_jited_ksyms;
3649         info.nr_jited_ksyms = prog->aux->func_cnt ? : 1;
3650         if (ulen) {
3651                 if (bpf_dump_raw_ok(file->f_cred)) {
3652                         unsigned long ksym_addr;
3653                         u64 __user *user_ksyms;
3654                         u32 i;
3655
3656                         /* copy the address of the kernel symbol
3657                          * corresponding to each function
3658                          */
3659                         ulen = min_t(u32, info.nr_jited_ksyms, ulen);
3660                         user_ksyms = u64_to_user_ptr(info.jited_ksyms);
3661                         if (prog->aux->func_cnt) {
3662                                 for (i = 0; i < ulen; i++) {
3663                                         ksym_addr = (unsigned long)
3664                                                 prog->aux->func[i]->bpf_func;
3665                                         if (put_user((u64) ksym_addr,
3666                                                      &user_ksyms[i]))
3667                                                 return -EFAULT;
3668                                 }
3669                         } else {
3670                                 ksym_addr = (unsigned long) prog->bpf_func;
3671                                 if (put_user((u64) ksym_addr, &user_ksyms[0]))
3672                                         return -EFAULT;
3673                         }
3674                 } else {
3675                         info.jited_ksyms = 0;
3676                 }
3677         }
3678
3679         ulen = info.nr_jited_func_lens;
3680         info.nr_jited_func_lens = prog->aux->func_cnt ? : 1;
3681         if (ulen) {
3682                 if (bpf_dump_raw_ok(file->f_cred)) {
3683                         u32 __user *user_lens;
3684                         u32 func_len, i;
3685
3686                         /* copy the JITed image lengths for each function */
3687                         ulen = min_t(u32, info.nr_jited_func_lens, ulen);
3688                         user_lens = u64_to_user_ptr(info.jited_func_lens);
3689                         if (prog->aux->func_cnt) {
3690                                 for (i = 0; i < ulen; i++) {
3691                                         func_len =
3692                                                 prog->aux->func[i]->jited_len;
3693                                         if (put_user(func_len, &user_lens[i]))
3694                                                 return -EFAULT;
3695                                 }
3696                         } else {
3697                                 func_len = prog->jited_len;
3698                                 if (put_user(func_len, &user_lens[0]))
3699                                         return -EFAULT;
3700                         }
3701                 } else {
3702                         info.jited_func_lens = 0;
3703                 }
3704         }
3705
3706         if (prog->aux->btf)
3707                 info.btf_id = btf_obj_id(prog->aux->btf);
3708
3709         ulen = info.nr_func_info;
3710         info.nr_func_info = prog->aux->func_info_cnt;
3711         if (info.nr_func_info && ulen) {
3712                 char __user *user_finfo;
3713
3714                 user_finfo = u64_to_user_ptr(info.func_info);
3715                 ulen = min_t(u32, info.nr_func_info, ulen);
3716                 if (copy_to_user(user_finfo, prog->aux->func_info,
3717                                  info.func_info_rec_size * ulen))
3718                         return -EFAULT;
3719         }
3720
3721         ulen = info.nr_line_info;
3722         info.nr_line_info = prog->aux->nr_linfo;
3723         if (info.nr_line_info && ulen) {
3724                 __u8 __user *user_linfo;
3725
3726                 user_linfo = u64_to_user_ptr(info.line_info);
3727                 ulen = min_t(u32, info.nr_line_info, ulen);
3728                 if (copy_to_user(user_linfo, prog->aux->linfo,
3729                                  info.line_info_rec_size * ulen))
3730                         return -EFAULT;
3731         }
3732
3733         ulen = info.nr_jited_line_info;
3734         if (prog->aux->jited_linfo)
3735                 info.nr_jited_line_info = prog->aux->nr_linfo;
3736         else
3737                 info.nr_jited_line_info = 0;
3738         if (info.nr_jited_line_info && ulen) {
3739                 if (bpf_dump_raw_ok(file->f_cred)) {
3740                         __u64 __user *user_linfo;
3741                         u32 i;
3742
3743                         user_linfo = u64_to_user_ptr(info.jited_line_info);
3744                         ulen = min_t(u32, info.nr_jited_line_info, ulen);
3745                         for (i = 0; i < ulen; i++) {
3746                                 if (put_user((__u64)(long)prog->aux->jited_linfo[i],
3747                                              &user_linfo[i]))
3748                                         return -EFAULT;
3749                         }
3750                 } else {
3751                         info.jited_line_info = 0;
3752                 }
3753         }
3754
3755         ulen = info.nr_prog_tags;
3756         info.nr_prog_tags = prog->aux->func_cnt ? : 1;
3757         if (ulen) {
3758                 __u8 __user (*user_prog_tags)[BPF_TAG_SIZE];
3759                 u32 i;
3760
3761                 user_prog_tags = u64_to_user_ptr(info.prog_tags);
3762                 ulen = min_t(u32, info.nr_prog_tags, ulen);
3763                 if (prog->aux->func_cnt) {
3764                         for (i = 0; i < ulen; i++) {
3765                                 if (copy_to_user(user_prog_tags[i],
3766                                                  prog->aux->func[i]->tag,
3767                                                  BPF_TAG_SIZE))
3768                                         return -EFAULT;
3769                         }
3770                 } else {
3771                         if (copy_to_user(user_prog_tags[0],
3772                                          prog->tag, BPF_TAG_SIZE))
3773                                 return -EFAULT;
3774                 }
3775         }
3776
3777 done:
3778         if (copy_to_user(uinfo, &info, info_len) ||
3779             put_user(info_len, &uattr->info.info_len))
3780                 return -EFAULT;
3781
3782         return 0;
3783 }
3784
3785 static int bpf_map_get_info_by_fd(struct file *file,
3786                                   struct bpf_map *map,
3787                                   const union bpf_attr *attr,
3788                                   union bpf_attr __user *uattr)
3789 {
3790         struct bpf_map_info __user *uinfo = u64_to_user_ptr(attr->info.info);
3791         struct bpf_map_info info;
3792         u32 info_len = attr->info.info_len;
3793         int err;
3794
3795         err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(info), info_len);
3796         if (err)
3797                 return err;
3798         info_len = min_t(u32, sizeof(info), info_len);
3799
3800         memset(&info, 0, sizeof(info));
3801         info.type = map->map_type;
3802         info.id = map->id;
3803         info.key_size = map->key_size;
3804         info.value_size = map->value_size;
3805         info.max_entries = map->max_entries;
3806         info.map_flags = map->map_flags;
3807         memcpy(info.name, map->name, sizeof(map->name));
3808
3809         if (map->btf) {
3810                 info.btf_id = btf_obj_id(map->btf);
3811                 info.btf_key_type_id = map->btf_key_type_id;
3812                 info.btf_value_type_id = map->btf_value_type_id;
3813         }
3814         info.btf_vmlinux_value_type_id = map->btf_vmlinux_value_type_id;
3815
3816         if (bpf_map_is_dev_bound(map)) {
3817                 err = bpf_map_offload_info_fill(&info, map);
3818                 if (err)
3819                         return err;
3820         }
3821
3822         if (copy_to_user(uinfo, &info, info_len) ||
3823             put_user(info_len, &uattr->info.info_len))
3824                 return -EFAULT;
3825
3826         return 0;
3827 }
3828
3829 static int bpf_btf_get_info_by_fd(struct file *file,
3830                                   struct btf *btf,
3831                                   const union bpf_attr *attr,
3832                                   union bpf_attr __user *uattr)
3833 {
3834         struct bpf_btf_info __user *uinfo = u64_to_user_ptr(attr->info.info);
3835         u32 info_len = attr->info.info_len;
3836         int err;
3837
3838         err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(*uinfo), info_len);
3839         if (err)
3840                 return err;
3841
3842         return btf_get_info_by_fd(btf, attr, uattr);
3843 }
3844
3845 static int bpf_link_get_info_by_fd(struct file *file,
3846                                   struct bpf_link *link,
3847                                   const union bpf_attr *attr,
3848                                   union bpf_attr __user *uattr)
3849 {
3850         struct bpf_link_info __user *uinfo = u64_to_user_ptr(attr->info.info);
3851         struct bpf_link_info info;
3852         u32 info_len = attr->info.info_len;
3853         int err;
3854
3855         err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(info), info_len);
3856         if (err)
3857                 return err;
3858         info_len = min_t(u32, sizeof(info), info_len);
3859
3860         memset(&info, 0, sizeof(info));
3861         if (copy_from_user(&info, uinfo, info_len))
3862                 return -EFAULT;
3863
3864         info.type = link->type;
3865         info.id = link->id;
3866         info.prog_id = link->prog->aux->id;
3867
3868         if (link->ops->fill_link_info) {
3869                 err = link->ops->fill_link_info(link, &info);
3870                 if (err)
3871                         return err;
3872         }
3873
3874         if (copy_to_user(uinfo, &info, info_len) ||
3875             put_user(info_len, &uattr->info.info_len))
3876                 return -EFAULT;
3877
3878         return 0;
3879 }
3880
3881
3882 #define BPF_OBJ_GET_INFO_BY_FD_LAST_FIELD info.info
3883
3884 static int bpf_obj_get_info_by_fd(const union bpf_attr *attr,
3885                                   union bpf_attr __user *uattr)
3886 {
3887         int ufd = attr->info.bpf_fd;
3888         struct fd f;
3889         int err;
3890
3891         if (CHECK_ATTR(BPF_OBJ_GET_INFO_BY_FD))
3892                 return -EINVAL;
3893
3894         f = fdget(ufd);
3895         if (!f.file)
3896                 return -EBADFD;
3897
3898         if (f.file->f_op == &bpf_prog_fops)
3899                 err = bpf_prog_get_info_by_fd(f.file, f.file->private_data, attr,
3900                                               uattr);
3901         else if (f.file->f_op == &bpf_map_fops)
3902                 err = bpf_map_get_info_by_fd(f.file, f.file->private_data, attr,
3903                                              uattr);
3904         else if (f.file->f_op == &btf_fops)
3905                 err = bpf_btf_get_info_by_fd(f.file, f.file->private_data, attr, uattr);
3906         else if (f.file->f_op == &bpf_link_fops)
3907                 err = bpf_link_get_info_by_fd(f.file, f.file->private_data,
3908                                               attr, uattr);
3909         else
3910                 err = -EINVAL;
3911
3912         fdput(f);
3913         return err;
3914 }
3915
3916 #define BPF_BTF_LOAD_LAST_FIELD btf_log_level
3917
3918 static int bpf_btf_load(const union bpf_attr *attr, bpfptr_t uattr)
3919 {
3920         if (CHECK_ATTR(BPF_BTF_LOAD))
3921                 return -EINVAL;
3922
3923         if (!bpf_capable())
3924                 return -EPERM;
3925
3926         return btf_new_fd(attr, uattr);
3927 }
3928
3929 #define BPF_BTF_GET_FD_BY_ID_LAST_FIELD btf_id
3930
3931 static int bpf_btf_get_fd_by_id(const union bpf_attr *attr)
3932 {
3933         if (CHECK_ATTR(BPF_BTF_GET_FD_BY_ID))
3934                 return -EINVAL;
3935
3936         if (!capable(CAP_SYS_ADMIN))
3937                 return -EPERM;
3938
3939         return btf_get_fd_by_id(attr->btf_id);
3940 }
3941
3942 static int bpf_task_fd_query_copy(const union bpf_attr *attr,
3943                                     union bpf_attr __user *uattr,
3944                                     u32 prog_id, u32 fd_type,
3945                                     const char *buf, u64 probe_offset,
3946                                     u64 probe_addr)
3947 {
3948         char __user *ubuf = u64_to_user_ptr(attr->task_fd_query.buf);
3949         u32 len = buf ? strlen(buf) : 0, input_len;
3950         int err = 0;
3951
3952         if (put_user(len, &uattr->task_fd_query.buf_len))
3953                 return -EFAULT;
3954         input_len = attr->task_fd_query.buf_len;
3955         if (input_len && ubuf) {
3956                 if (!len) {
3957                         /* nothing to copy, just make ubuf NULL terminated */
3958                         char zero = '\0';
3959
3960                         if (put_user(zero, ubuf))
3961                                 return -EFAULT;
3962                 } else if (input_len >= len + 1) {
3963                         /* ubuf can hold the string with NULL terminator */
3964                         if (copy_to_user(ubuf, buf, len + 1))
3965                                 return -EFAULT;
3966                 } else {
3967                         /* ubuf cannot hold the string with NULL terminator,
3968                          * do a partial copy with NULL terminator.
3969                          */
3970                         char zero = '\0';
3971
3972                         err = -ENOSPC;
3973                         if (copy_to_user(ubuf, buf, input_len - 1))
3974                                 return -EFAULT;
3975                         if (put_user(zero, ubuf + input_len - 1))
3976                                 return -EFAULT;
3977                 }
3978         }
3979
3980         if (put_user(prog_id, &uattr->task_fd_query.prog_id) ||
3981             put_user(fd_type, &uattr->task_fd_query.fd_type) ||
3982             put_user(probe_offset, &uattr->task_fd_query.probe_offset) ||
3983             put_user(probe_addr, &uattr->task_fd_query.probe_addr))
3984                 return -EFAULT;
3985
3986         return err;
3987 }
3988
3989 #define BPF_TASK_FD_QUERY_LAST_FIELD task_fd_query.probe_addr
3990
3991 static int bpf_task_fd_query(const union bpf_attr *attr,
3992                              union bpf_attr __user *uattr)
3993 {
3994         pid_t pid = attr->task_fd_query.pid;
3995         u32 fd = attr->task_fd_query.fd;
3996         const struct perf_event *event;
3997         struct task_struct *task;
3998         struct file *file;
3999         int err;
4000
4001         if (CHECK_ATTR(BPF_TASK_FD_QUERY))
4002                 return -EINVAL;
4003
4004         if (!capable(CAP_SYS_ADMIN))
4005                 return -EPERM;
4006
4007         if (attr->task_fd_query.flags != 0)
4008                 return -EINVAL;
4009
4010         task = get_pid_task(find_vpid(pid), PIDTYPE_PID);
4011         if (!task)
4012                 return -ENOENT;
4013
4014         err = 0;
4015         file = fget_task(task, fd);
4016         put_task_struct(task);
4017         if (!file)
4018                 return -EBADF;
4019
4020         if (file->f_op == &bpf_link_fops) {
4021                 struct bpf_link *link = file->private_data;
4022
4023                 if (link->ops == &bpf_raw_tp_link_lops) {
4024                         struct bpf_raw_tp_link *raw_tp =
4025                                 container_of(link, struct bpf_raw_tp_link, link);
4026                         struct bpf_raw_event_map *btp = raw_tp->btp;
4027
4028                         err = bpf_task_fd_query_copy(attr, uattr,
4029                                                      raw_tp->link.prog->aux->id,
4030                                                      BPF_FD_TYPE_RAW_TRACEPOINT,
4031                                                      btp->tp->name, 0, 0);
4032                         goto put_file;
4033                 }
4034                 goto out_not_supp;
4035         }
4036
4037         event = perf_get_event(file);
4038         if (!IS_ERR(event)) {
4039                 u64 probe_offset, probe_addr;
4040                 u32 prog_id, fd_type;
4041                 const char *buf;
4042
4043                 err = bpf_get_perf_event_info(event, &prog_id, &fd_type,
4044                                               &buf, &probe_offset,
4045                                               &probe_addr);
4046                 if (!err)
4047                         err = bpf_task_fd_query_copy(attr, uattr, prog_id,
4048                                                      fd_type, buf,
4049                                                      probe_offset,
4050                                                      probe_addr);
4051                 goto put_file;
4052         }
4053
4054 out_not_supp:
4055         err = -ENOTSUPP;
4056 put_file:
4057         fput(file);
4058         return err;
4059 }
4060
4061 #define BPF_MAP_BATCH_LAST_FIELD batch.flags
4062
4063 #define BPF_DO_BATCH(fn)                        \
4064         do {                                    \
4065                 if (!fn) {                      \
4066                         err = -ENOTSUPP;        \
4067                         goto err_put;           \
4068                 }                               \
4069                 err = fn(map, attr, uattr);     \
4070         } while (0)
4071
4072 static int bpf_map_do_batch(const union bpf_attr *attr,
4073                             union bpf_attr __user *uattr,
4074                             int cmd)
4075 {
4076         struct bpf_map *map;
4077         int err, ufd;
4078         struct fd f;
4079
4080         if (CHECK_ATTR(BPF_MAP_BATCH))
4081                 return -EINVAL;
4082
4083         ufd = attr->batch.map_fd;
4084         f = fdget(ufd);
4085         map = __bpf_map_get(f);
4086         if (IS_ERR(map))
4087                 return PTR_ERR(map);
4088
4089         if ((cmd == BPF_MAP_LOOKUP_BATCH ||
4090              cmd == BPF_MAP_LOOKUP_AND_DELETE_BATCH) &&
4091             !(map_get_sys_perms(map, f) & FMODE_CAN_READ)) {
4092                 err = -EPERM;
4093                 goto err_put;
4094         }
4095
4096         if (cmd != BPF_MAP_LOOKUP_BATCH &&
4097             !(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
4098                 err = -EPERM;
4099                 goto err_put;
4100         }
4101
4102         if (cmd == BPF_MAP_LOOKUP_BATCH)
4103                 BPF_DO_BATCH(map->ops->map_lookup_batch);
4104         else if (cmd == BPF_MAP_LOOKUP_AND_DELETE_BATCH)
4105                 BPF_DO_BATCH(map->ops->map_lookup_and_delete_batch);
4106         else if (cmd == BPF_MAP_UPDATE_BATCH)
4107                 BPF_DO_BATCH(map->ops->map_update_batch);
4108         else
4109                 BPF_DO_BATCH(map->ops->map_delete_batch);
4110
4111 err_put:
4112         fdput(f);
4113         return err;
4114 }
4115
4116 static int tracing_bpf_link_attach(const union bpf_attr *attr, bpfptr_t uattr,
4117                                    struct bpf_prog *prog)
4118 {
4119         if (attr->link_create.attach_type != prog->expected_attach_type)
4120                 return -EINVAL;
4121
4122         if (prog->expected_attach_type == BPF_TRACE_ITER)
4123                 return bpf_iter_link_attach(attr, uattr, prog);
4124         else if (prog->type == BPF_PROG_TYPE_EXT)
4125                 return bpf_tracing_prog_attach(prog,
4126                                                attr->link_create.target_fd,
4127                                                attr->link_create.target_btf_id);
4128         return -EINVAL;
4129 }
4130
4131 #define BPF_LINK_CREATE_LAST_FIELD link_create.iter_info_len
4132 static int link_create(union bpf_attr *attr, bpfptr_t uattr)
4133 {
4134         enum bpf_prog_type ptype;
4135         struct bpf_prog *prog;
4136         int ret;
4137
4138         if (CHECK_ATTR(BPF_LINK_CREATE))
4139                 return -EINVAL;
4140
4141         prog = bpf_prog_get(attr->link_create.prog_fd);
4142         if (IS_ERR(prog))
4143                 return PTR_ERR(prog);
4144
4145         ret = bpf_prog_attach_check_attach_type(prog,
4146                                                 attr->link_create.attach_type);
4147         if (ret)
4148                 goto out;
4149
4150         if (prog->type == BPF_PROG_TYPE_EXT) {
4151                 ret = tracing_bpf_link_attach(attr, uattr, prog);
4152                 goto out;
4153         }
4154
4155         ptype = attach_type_to_prog_type(attr->link_create.attach_type);
4156         if (ptype == BPF_PROG_TYPE_UNSPEC || ptype != prog->type) {
4157                 ret = -EINVAL;
4158                 goto out;
4159         }
4160
4161         switch (ptype) {
4162         case BPF_PROG_TYPE_CGROUP_SKB:
4163         case BPF_PROG_TYPE_CGROUP_SOCK:
4164         case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
4165         case BPF_PROG_TYPE_SOCK_OPS:
4166         case BPF_PROG_TYPE_CGROUP_DEVICE:
4167         case BPF_PROG_TYPE_CGROUP_SYSCTL:
4168         case BPF_PROG_TYPE_CGROUP_SOCKOPT:
4169                 ret = cgroup_bpf_link_attach(attr, prog);
4170                 break;
4171         case BPF_PROG_TYPE_TRACING:
4172                 ret = tracing_bpf_link_attach(attr, uattr, prog);
4173                 break;
4174         case BPF_PROG_TYPE_FLOW_DISSECTOR:
4175         case BPF_PROG_TYPE_SK_LOOKUP:
4176                 ret = netns_bpf_link_create(attr, prog);
4177                 break;
4178 #ifdef CONFIG_NET
4179         case BPF_PROG_TYPE_XDP:
4180                 ret = bpf_xdp_link_attach(attr, prog);
4181                 break;
4182 #endif
4183         default:
4184                 ret = -EINVAL;
4185         }
4186
4187 out:
4188         if (ret < 0)
4189                 bpf_prog_put(prog);
4190         return ret;
4191 }
4192
4193 #define BPF_LINK_UPDATE_LAST_FIELD link_update.old_prog_fd
4194
4195 static int link_update(union bpf_attr *attr)
4196 {
4197         struct bpf_prog *old_prog = NULL, *new_prog;
4198         struct bpf_link *link;
4199         u32 flags;
4200         int ret;
4201
4202         if (CHECK_ATTR(BPF_LINK_UPDATE))
4203                 return -EINVAL;
4204
4205         flags = attr->link_update.flags;
4206         if (flags & ~BPF_F_REPLACE)
4207                 return -EINVAL;
4208
4209         link = bpf_link_get_from_fd(attr->link_update.link_fd);
4210         if (IS_ERR(link))
4211                 return PTR_ERR(link);
4212
4213         new_prog = bpf_prog_get(attr->link_update.new_prog_fd);
4214         if (IS_ERR(new_prog)) {
4215                 ret = PTR_ERR(new_prog);
4216                 goto out_put_link;
4217         }
4218
4219         if (flags & BPF_F_REPLACE) {
4220                 old_prog = bpf_prog_get(attr->link_update.old_prog_fd);
4221                 if (IS_ERR(old_prog)) {
4222                         ret = PTR_ERR(old_prog);
4223                         old_prog = NULL;
4224                         goto out_put_progs;
4225                 }
4226         } else if (attr->link_update.old_prog_fd) {
4227                 ret = -EINVAL;
4228                 goto out_put_progs;
4229         }
4230
4231         if (link->ops->update_prog)
4232                 ret = link->ops->update_prog(link, new_prog, old_prog);
4233         else
4234                 ret = -EINVAL;
4235
4236 out_put_progs:
4237         if (old_prog)
4238                 bpf_prog_put(old_prog);
4239         if (ret)
4240                 bpf_prog_put(new_prog);
4241 out_put_link:
4242         bpf_link_put(link);
4243         return ret;
4244 }
4245
4246 #define BPF_LINK_DETACH_LAST_FIELD link_detach.link_fd
4247
4248 static int link_detach(union bpf_attr *attr)
4249 {
4250         struct bpf_link *link;
4251         int ret;
4252
4253         if (CHECK_ATTR(BPF_LINK_DETACH))
4254                 return -EINVAL;
4255
4256         link = bpf_link_get_from_fd(attr->link_detach.link_fd);
4257         if (IS_ERR(link))
4258                 return PTR_ERR(link);
4259
4260         if (link->ops->detach)
4261                 ret = link->ops->detach(link);
4262         else
4263                 ret = -EOPNOTSUPP;
4264
4265         bpf_link_put(link);
4266         return ret;
4267 }
4268
4269 static struct bpf_link *bpf_link_inc_not_zero(struct bpf_link *link)
4270 {
4271         return atomic64_fetch_add_unless(&link->refcnt, 1, 0) ? link : ERR_PTR(-ENOENT);
4272 }
4273
4274 struct bpf_link *bpf_link_by_id(u32 id)
4275 {
4276         struct bpf_link *link;
4277
4278         if (!id)
4279                 return ERR_PTR(-ENOENT);
4280
4281         spin_lock_bh(&link_idr_lock);
4282         /* before link is "settled", ID is 0, pretend it doesn't exist yet */
4283         link = idr_find(&link_idr, id);
4284         if (link) {
4285                 if (link->id)
4286                         link = bpf_link_inc_not_zero(link);
4287                 else
4288                         link = ERR_PTR(-EAGAIN);
4289         } else {
4290                 link = ERR_PTR(-ENOENT);
4291         }
4292         spin_unlock_bh(&link_idr_lock);
4293         return link;
4294 }
4295
4296 #define BPF_LINK_GET_FD_BY_ID_LAST_FIELD link_id
4297
4298 static int bpf_link_get_fd_by_id(const union bpf_attr *attr)
4299 {
4300         struct bpf_link *link;
4301         u32 id = attr->link_id;
4302         int fd;
4303
4304         if (CHECK_ATTR(BPF_LINK_GET_FD_BY_ID))
4305                 return -EINVAL;
4306
4307         if (!capable(CAP_SYS_ADMIN))
4308                 return -EPERM;
4309
4310         link = bpf_link_by_id(id);
4311         if (IS_ERR(link))
4312                 return PTR_ERR(link);
4313
4314         fd = bpf_link_new_fd(link);
4315         if (fd < 0)
4316                 bpf_link_put(link);
4317
4318         return fd;
4319 }
4320
4321 DEFINE_MUTEX(bpf_stats_enabled_mutex);
4322
4323 static int bpf_stats_release(struct inode *inode, struct file *file)
4324 {
4325         mutex_lock(&bpf_stats_enabled_mutex);
4326         static_key_slow_dec(&bpf_stats_enabled_key.key);
4327         mutex_unlock(&bpf_stats_enabled_mutex);
4328         return 0;
4329 }
4330
4331 static const struct file_operations bpf_stats_fops = {
4332         .release = bpf_stats_release,
4333 };
4334
4335 static int bpf_enable_runtime_stats(void)
4336 {
4337         int fd;
4338
4339         mutex_lock(&bpf_stats_enabled_mutex);
4340
4341         /* Set a very high limit to avoid overflow */
4342         if (static_key_count(&bpf_stats_enabled_key.key) > INT_MAX / 2) {
4343                 mutex_unlock(&bpf_stats_enabled_mutex);
4344                 return -EBUSY;
4345         }
4346
4347         fd = anon_inode_getfd("bpf-stats", &bpf_stats_fops, NULL, O_CLOEXEC);
4348         if (fd >= 0)
4349                 static_key_slow_inc(&bpf_stats_enabled_key.key);
4350
4351         mutex_unlock(&bpf_stats_enabled_mutex);
4352         return fd;
4353 }
4354
4355 #define BPF_ENABLE_STATS_LAST_FIELD enable_stats.type
4356
4357 static int bpf_enable_stats(union bpf_attr *attr)
4358 {
4359
4360         if (CHECK_ATTR(BPF_ENABLE_STATS))
4361                 return -EINVAL;
4362
4363         if (!capable(CAP_SYS_ADMIN))
4364                 return -EPERM;
4365
4366         switch (attr->enable_stats.type) {
4367         case BPF_STATS_RUN_TIME:
4368                 return bpf_enable_runtime_stats();
4369         default:
4370                 break;
4371         }
4372         return -EINVAL;
4373 }
4374
4375 #define BPF_ITER_CREATE_LAST_FIELD iter_create.flags
4376
4377 static int bpf_iter_create(union bpf_attr *attr)
4378 {
4379         struct bpf_link *link;
4380         int err;
4381
4382         if (CHECK_ATTR(BPF_ITER_CREATE))
4383                 return -EINVAL;
4384
4385         if (attr->iter_create.flags)
4386                 return -EINVAL;
4387
4388         link = bpf_link_get_from_fd(attr->iter_create.link_fd);
4389         if (IS_ERR(link))
4390                 return PTR_ERR(link);
4391
4392         err = bpf_iter_new_fd(link);
4393         bpf_link_put(link);
4394
4395         return err;
4396 }
4397
4398 #define BPF_PROG_BIND_MAP_LAST_FIELD prog_bind_map.flags
4399
4400 static int bpf_prog_bind_map(union bpf_attr *attr)
4401 {
4402         struct bpf_prog *prog;
4403         struct bpf_map *map;
4404         struct bpf_map **used_maps_old, **used_maps_new;
4405         int i, ret = 0;
4406
4407         if (CHECK_ATTR(BPF_PROG_BIND_MAP))
4408                 return -EINVAL;
4409
4410         if (attr->prog_bind_map.flags)
4411                 return -EINVAL;
4412
4413         prog = bpf_prog_get(attr->prog_bind_map.prog_fd);
4414         if (IS_ERR(prog))
4415                 return PTR_ERR(prog);
4416
4417         map = bpf_map_get(attr->prog_bind_map.map_fd);
4418         if (IS_ERR(map)) {
4419                 ret = PTR_ERR(map);
4420                 goto out_prog_put;
4421         }
4422
4423         mutex_lock(&prog->aux->used_maps_mutex);
4424
4425         used_maps_old = prog->aux->used_maps;
4426
4427         for (i = 0; i < prog->aux->used_map_cnt; i++)
4428                 if (used_maps_old[i] == map) {
4429                         bpf_map_put(map);
4430                         goto out_unlock;
4431                 }
4432
4433         used_maps_new = kmalloc_array(prog->aux->used_map_cnt + 1,
4434                                       sizeof(used_maps_new[0]),
4435                                       GFP_KERNEL);
4436         if (!used_maps_new) {
4437                 ret = -ENOMEM;
4438                 goto out_unlock;
4439         }
4440
4441         memcpy(used_maps_new, used_maps_old,
4442                sizeof(used_maps_old[0]) * prog->aux->used_map_cnt);
4443         used_maps_new[prog->aux->used_map_cnt] = map;
4444
4445         prog->aux->used_map_cnt++;
4446         prog->aux->used_maps = used_maps_new;
4447
4448         kfree(used_maps_old);
4449
4450 out_unlock:
4451         mutex_unlock(&prog->aux->used_maps_mutex);
4452
4453         if (ret)
4454                 bpf_map_put(map);
4455 out_prog_put:
4456         bpf_prog_put(prog);
4457         return ret;
4458 }
4459
4460 static int __sys_bpf(int cmd, bpfptr_t uattr, unsigned int size)
4461 {
4462         union bpf_attr attr;
4463         int err;
4464
4465         if (sysctl_unprivileged_bpf_disabled && !bpf_capable())
4466                 return -EPERM;
4467
4468         err = bpf_check_uarg_tail_zero(uattr, sizeof(attr), size);
4469         if (err)
4470                 return err;
4471         size = min_t(u32, size, sizeof(attr));
4472
4473         /* copy attributes from user space, may be less than sizeof(bpf_attr) */
4474         memset(&attr, 0, sizeof(attr));
4475         if (copy_from_bpfptr(&attr, uattr, size) != 0)
4476                 return -EFAULT;
4477
4478         err = security_bpf(cmd, &attr, size);
4479         if (err < 0)
4480                 return err;
4481
4482         switch (cmd) {
4483         case BPF_MAP_CREATE:
4484                 err = map_create(&attr);
4485                 break;
4486         case BPF_MAP_LOOKUP_ELEM:
4487                 err = map_lookup_elem(&attr);
4488                 break;
4489         case BPF_MAP_UPDATE_ELEM:
4490                 err = map_update_elem(&attr, uattr);
4491                 break;
4492         case BPF_MAP_DELETE_ELEM:
4493                 err = map_delete_elem(&attr);
4494                 break;
4495         case BPF_MAP_GET_NEXT_KEY:
4496                 err = map_get_next_key(&attr);
4497                 break;
4498         case BPF_MAP_FREEZE:
4499                 err = map_freeze(&attr);
4500                 break;
4501         case BPF_PROG_LOAD:
4502                 err = bpf_prog_load(&attr, uattr);
4503                 break;
4504         case BPF_OBJ_PIN:
4505                 err = bpf_obj_pin(&attr);
4506                 break;
4507         case BPF_OBJ_GET:
4508                 err = bpf_obj_get(&attr);
4509                 break;
4510         case BPF_PROG_ATTACH:
4511                 err = bpf_prog_attach(&attr);
4512                 break;
4513         case BPF_PROG_DETACH:
4514                 err = bpf_prog_detach(&attr);
4515                 break;
4516         case BPF_PROG_QUERY:
4517                 err = bpf_prog_query(&attr, uattr.user);
4518                 break;
4519         case BPF_PROG_TEST_RUN:
4520                 err = bpf_prog_test_run(&attr, uattr.user);
4521                 break;
4522         case BPF_PROG_GET_NEXT_ID:
4523                 err = bpf_obj_get_next_id(&attr, uattr.user,
4524                                           &prog_idr, &prog_idr_lock);
4525                 break;
4526         case BPF_MAP_GET_NEXT_ID:
4527                 err = bpf_obj_get_next_id(&attr, uattr.user,
4528                                           &map_idr, &map_idr_lock);
4529                 break;
4530         case BPF_BTF_GET_NEXT_ID:
4531                 err = bpf_obj_get_next_id(&attr, uattr.user,
4532                                           &btf_idr, &btf_idr_lock);
4533                 break;
4534         case BPF_PROG_GET_FD_BY_ID:
4535                 err = bpf_prog_get_fd_by_id(&attr);
4536                 break;
4537         case BPF_MAP_GET_FD_BY_ID:
4538                 err = bpf_map_get_fd_by_id(&attr);
4539                 break;
4540         case BPF_OBJ_GET_INFO_BY_FD:
4541                 err = bpf_obj_get_info_by_fd(&attr, uattr.user);
4542                 break;
4543         case BPF_RAW_TRACEPOINT_OPEN:
4544                 err = bpf_raw_tracepoint_open(&attr);
4545                 break;
4546         case BPF_BTF_LOAD:
4547                 err = bpf_btf_load(&attr, uattr);
4548                 break;
4549         case BPF_BTF_GET_FD_BY_ID:
4550                 err = bpf_btf_get_fd_by_id(&attr);
4551                 break;
4552         case BPF_TASK_FD_QUERY:
4553                 err = bpf_task_fd_query(&attr, uattr.user);
4554                 break;
4555         case BPF_MAP_LOOKUP_AND_DELETE_ELEM:
4556                 err = map_lookup_and_delete_elem(&attr);
4557                 break;
4558         case BPF_MAP_LOOKUP_BATCH:
4559                 err = bpf_map_do_batch(&attr, uattr.user, BPF_MAP_LOOKUP_BATCH);
4560                 break;
4561         case BPF_MAP_LOOKUP_AND_DELETE_BATCH:
4562                 err = bpf_map_do_batch(&attr, uattr.user,
4563                                        BPF_MAP_LOOKUP_AND_DELETE_BATCH);
4564                 break;
4565         case BPF_MAP_UPDATE_BATCH:
4566                 err = bpf_map_do_batch(&attr, uattr.user, BPF_MAP_UPDATE_BATCH);
4567                 break;
4568         case BPF_MAP_DELETE_BATCH:
4569                 err = bpf_map_do_batch(&attr, uattr.user, BPF_MAP_DELETE_BATCH);
4570                 break;
4571         case BPF_LINK_CREATE:
4572                 err = link_create(&attr, uattr);
4573                 break;
4574         case BPF_LINK_UPDATE:
4575                 err = link_update(&attr);
4576                 break;
4577         case BPF_LINK_GET_FD_BY_ID:
4578                 err = bpf_link_get_fd_by_id(&attr);
4579                 break;
4580         case BPF_LINK_GET_NEXT_ID:
4581                 err = bpf_obj_get_next_id(&attr, uattr.user,
4582                                           &link_idr, &link_idr_lock);
4583                 break;
4584         case BPF_ENABLE_STATS:
4585                 err = bpf_enable_stats(&attr);
4586                 break;
4587         case BPF_ITER_CREATE:
4588                 err = bpf_iter_create(&attr);
4589                 break;
4590         case BPF_LINK_DETACH:
4591                 err = link_detach(&attr);
4592                 break;
4593         case BPF_PROG_BIND_MAP:
4594                 err = bpf_prog_bind_map(&attr);
4595                 break;
4596         default:
4597                 err = -EINVAL;
4598                 break;
4599         }
4600
4601         return err;
4602 }
4603
4604 SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size)
4605 {
4606         return __sys_bpf(cmd, USER_BPFPTR(uattr), size);
4607 }
4608
4609 static bool syscall_prog_is_valid_access(int off, int size,
4610                                          enum bpf_access_type type,
4611                                          const struct bpf_prog *prog,
4612                                          struct bpf_insn_access_aux *info)
4613 {
4614         if (off < 0 || off >= U16_MAX)
4615                 return false;
4616         if (off % size != 0)
4617                 return false;
4618         return true;
4619 }
4620
4621 BPF_CALL_3(bpf_sys_bpf, int, cmd, void *, attr, u32, attr_size)
4622 {
4623         switch (cmd) {
4624         case BPF_MAP_CREATE:
4625         case BPF_MAP_UPDATE_ELEM:
4626         case BPF_MAP_FREEZE:
4627         case BPF_PROG_LOAD:
4628         case BPF_BTF_LOAD:
4629                 break;
4630         /* case BPF_PROG_TEST_RUN:
4631          * is not part of this list to prevent recursive test_run
4632          */
4633         default:
4634                 return -EINVAL;
4635         }
4636         return __sys_bpf(cmd, KERNEL_BPFPTR(attr), attr_size);
4637 }
4638
4639 static const struct bpf_func_proto bpf_sys_bpf_proto = {
4640         .func           = bpf_sys_bpf,
4641         .gpl_only       = false,
4642         .ret_type       = RET_INTEGER,
4643         .arg1_type      = ARG_ANYTHING,
4644         .arg2_type      = ARG_PTR_TO_MEM,
4645         .arg3_type      = ARG_CONST_SIZE,
4646 };
4647
4648 const struct bpf_func_proto * __weak
4649 tracing_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
4650 {
4651         return bpf_base_func_proto(func_id);
4652 }
4653
4654 BPF_CALL_1(bpf_sys_close, u32, fd)
4655 {
4656         /* When bpf program calls this helper there should not be
4657          * an fdget() without matching completed fdput().
4658          * This helper is allowed in the following callchain only:
4659          * sys_bpf->prog_test_run->bpf_prog->bpf_sys_close
4660          */
4661         return close_fd(fd);
4662 }
4663
4664 static const struct bpf_func_proto bpf_sys_close_proto = {
4665         .func           = bpf_sys_close,
4666         .gpl_only       = false,
4667         .ret_type       = RET_INTEGER,
4668         .arg1_type      = ARG_ANYTHING,
4669 };
4670
4671 static const struct bpf_func_proto *
4672 syscall_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
4673 {
4674         switch (func_id) {
4675         case BPF_FUNC_sys_bpf:
4676                 return &bpf_sys_bpf_proto;
4677         case BPF_FUNC_btf_find_by_name_kind:
4678                 return &bpf_btf_find_by_name_kind_proto;
4679         case BPF_FUNC_sys_close:
4680                 return &bpf_sys_close_proto;
4681         default:
4682                 return tracing_prog_func_proto(func_id, prog);
4683         }
4684 }
4685
4686 const struct bpf_verifier_ops bpf_syscall_verifier_ops = {
4687         .get_func_proto  = syscall_prog_func_proto,
4688         .is_valid_access = syscall_prog_is_valid_access,
4689 };
4690
4691 const struct bpf_prog_ops bpf_syscall_prog_ops = {
4692         .test_run = bpf_prog_test_run_syscall,
4693 };