kernel/bpf/syscall.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
   3  */
   4 #include <linux/bpf.h>
   5 #include <linux/bpf_trace.h>
   6 #include <linux/bpf_lirc.h>
   7 #include <linux/bpf_verifier.h>
   8 #include <linux/btf.h>
   9 #include <linux/syscalls.h>
  10 #include <linux/slab.h>
  11 #include <linux/sched/signal.h>
  12 #include <linux/vmalloc.h>
  13 #include <linux/mmzone.h>
  14 #include <linux/anon_inodes.h>
  15 #include <linux/fdtable.h>
  16 #include <linux/file.h>
  17 #include <linux/fs.h>
  18 #include <linux/license.h>
  19 #include <linux/filter.h>
  20 #include <linux/kernel.h>
  21 #include <linux/idr.h>
  22 #include <linux/cred.h>
  23 #include <linux/timekeeping.h>
  24 #include <linux/ctype.h>
  25 #include <linux/nospec.h>
  26 #include <linux/audit.h>
  27 #include <uapi/linux/btf.h>
  28 #include <linux/pgtable.h>
  29 #include <linux/bpf_lsm.h>
  30 #include <linux/poll.h>
  31 #include <linux/bpf-netns.h>
  32 #include <linux/rcupdate_trace.h>
  33 #include <linux/memcontrol.h>
  34
  35 #define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \
  36                           (map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \
  37                           (map)->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS)
  38 #define IS_FD_PROG_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PROG_ARRAY)
  39 #define IS_FD_HASH(map) ((map)->map_type == BPF_MAP_TYPE_HASH_OF_MAPS)
  40 #define IS_FD_MAP(map) (IS_FD_ARRAY(map) || IS_FD_PROG_ARRAY(map) || \
  41                         IS_FD_HASH(map))
  42
  43 #define BPF_OBJ_FLAG_MASK   (BPF_F_RDONLY | BPF_F_WRONLY)
  44
  45 DEFINE_PER_CPU(int, bpf_prog_active);
  46 static DEFINE_IDR(prog_idr);
  47 static DEFINE_SPINLOCK(prog_idr_lock);
  48 static DEFINE_IDR(map_idr);
  49 static DEFINE_SPINLOCK(map_idr_lock);
  50 static DEFINE_IDR(link_idr);
  51 static DEFINE_SPINLOCK(link_idr_lock);
  52
  53 int sysctl_unprivileged_bpf_disabled __read_mostly =
  54         IS_BUILTIN(CONFIG_BPF_UNPRIV_DEFAULT_OFF) ? 2 : 0;
  55
  56 static const struct bpf_map_ops * const bpf_map_types[] = {
  57 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type)
  58 #define BPF_MAP_TYPE(_id, _ops) \
  59         [_id] = &_ops,
  60 #define BPF_LINK_TYPE(_id, _name)
  61 #include <linux/bpf_types.h>
  62 #undef BPF_PROG_TYPE
  63 #undef BPF_MAP_TYPE
  64 #undef BPF_LINK_TYPE
  65 };
  66
  67 /*
  68  * If we're handed a bigger struct than we know of, ensure all the unknown bits
  69  * are 0 - i.e. new user-space does not rely on any kernel feature extensions
  70  * we don't know about yet.
  71  *
  72  * There is a ToCToU between this function call and the following
  73  * copy_from_user() call. However, this is not a concern since this function is
  74  * meant to be a future-proofing of bits.
  75  */
  76 int bpf_check_uarg_tail_zero(bpfptr_t uaddr,
  77                              size_t expected_size,
  78                              size_t actual_size)
  79 {
  80         int res;
  81
  82         if (unlikely(actual_size > PAGE_SIZE))  /* silly large */
  83                 return -E2BIG;
  84
  85         if (actual_size <= expected_size)
  86                 return 0;
  87
  88         if (uaddr.is_kernel)
  89                 res = memchr_inv(uaddr.kernel + expected_size, 0,
  90                                  actual_size - expected_size) == NULL;
  91         else
  92                 res = check_zeroed_user(uaddr.user + expected_size,
  93                                         actual_size - expected_size);
  94         if (res < 0)
  95                 return res;
  96         return res ? 0 : -E2BIG;
  97 }
  98
  99 const struct bpf_map_ops bpf_map_offload_ops = {
 100         .map_meta_equal = bpf_map_meta_equal,
 101         .map_alloc = bpf_map_offload_map_alloc,
 102         .map_free = bpf_map_offload_map_free,
 103         .map_check_btf = map_check_no_btf,
 104 };
 105
 106 static struct bpf_map *find_and_alloc_map(union bpf_attr *attr)
 107 {
 108         const struct bpf_map_ops *ops;
 109         u32 type = attr->map_type;
 110         struct bpf_map *map;
 111         int err;
 112
 113         if (type >= ARRAY_SIZE(bpf_map_types))
 114                 return ERR_PTR(-EINVAL);
 115         type = array_index_nospec(type, ARRAY_SIZE(bpf_map_types));
 116         ops = bpf_map_types[type];
 117         if (!ops)
 118                 return ERR_PTR(-EINVAL);
 119
 120         if (ops->map_alloc_check) {
 121                 err = ops->map_alloc_check(attr);
 122                 if (err)
 123                         return ERR_PTR(err);
 124         }
 125         if (attr->map_ifindex)
 126                 ops = &bpf_map_offload_ops;
 127         map = ops->map_alloc(attr);
 128         if (IS_ERR(map))
 129                 return map;
 130         map->ops = ops;
 131         map->map_type = type;
 132         return map;
 133 }
 134
 135 static u32 bpf_map_value_size(const struct bpf_map *map)
 136 {
 137         if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
 138             map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||
 139             map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY ||
 140             map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE)
 141                 return round_up(map->value_size, 8) * num_possible_cpus();
 142         else if (IS_FD_MAP(map))
 143                 return sizeof(u32);
 144         else
 145                 return  map->value_size;
 146 }
 147
 148 static void maybe_wait_bpf_programs(struct bpf_map *map)
 149 {
 150         /* Wait for any running BPF programs to complete so that
 151          * userspace, when we return to it, knows that all programs
 152          * that could be running use the new map value.
 153          */
 154         if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS ||
 155             map->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS)
 156                 synchronize_rcu();
 157 }
 158
 159 static int bpf_map_update_value(struct bpf_map *map, struct fd f, void *key,
 160                                 void *value, __u64 flags)
 161 {
 162         int err;
 163
 164         /* Need to create a kthread, thus must support schedule */
 165         if (bpf_map_is_dev_bound(map)) {
 166                 return bpf_map_offload_update_elem(map, key, value, flags);
 167         } else if (map->map_type == BPF_MAP_TYPE_CPUMAP ||
 168                    map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
 169                 return map->ops->map_update_elem(map, key, value, flags);
 170         } else if (map->map_type == BPF_MAP_TYPE_SOCKHASH ||
 171                    map->map_type == BPF_MAP_TYPE_SOCKMAP) {
 172                 return sock_map_update_elem_sys(map, key, value, flags);
 173         } else if (IS_FD_PROG_ARRAY(map)) {
 174                 return bpf_fd_array_map_update_elem(map, f.file, key, value,
 175                                                     flags);
 176         }
 177
 178         bpf_disable_instrumentation();
 179         if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
 180             map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
 181                 err = bpf_percpu_hash_update(map, key, value, flags);
 182         } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
 183                 err = bpf_percpu_array_update(map, key, value, flags);
 184         } else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) {
 185                 err = bpf_percpu_cgroup_storage_update(map, key, value,
 186                                                        flags);
 187         } else if (IS_FD_ARRAY(map)) {
 188                 rcu_read_lock();
 189                 err = bpf_fd_array_map_update_elem(map, f.file, key, value,
 190                                                    flags);
 191                 rcu_read_unlock();
 192         } else if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) {
 193                 rcu_read_lock();
 194                 err = bpf_fd_htab_map_update_elem(map, f.file, key, value,
 195                                                   flags);
 196                 rcu_read_unlock();
 197         } else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) {
 198                 /* rcu_read_lock() is not needed */
 199                 err = bpf_fd_reuseport_array_update_elem(map, key, value,
 200                                                          flags);
 201         } else if (map->map_type == BPF_MAP_TYPE_QUEUE ||
 202                    map->map_type == BPF_MAP_TYPE_STACK) {
 203                 err = map->ops->map_push_elem(map, value, flags);
 204         } else {
 205                 rcu_read_lock();
 206                 err = map->ops->map_update_elem(map, key, value, flags);
 207                 rcu_read_unlock();
 208         }
 209         bpf_enable_instrumentation();
 210         maybe_wait_bpf_programs(map);
 211
 212         return err;
 213 }
 214
 215 static int bpf_map_copy_value(struct bpf_map *map, void *key, void *value,
 216                               __u64 flags)
 217 {
 218         void *ptr;
 219         int err;
 220
 221         if (bpf_map_is_dev_bound(map))
 222                 return bpf_map_offload_lookup_elem(map, key, value);
 223
 224         bpf_disable_instrumentation();
 225         if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
 226             map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
 227                 err = bpf_percpu_hash_copy(map, key, value);
 228         } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
 229                 err = bpf_percpu_array_copy(map, key, value);
 230         } else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) {
 231                 err = bpf_percpu_cgroup_storage_copy(map, key, value);
 232         } else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) {
 233                 err = bpf_stackmap_copy(map, key, value);
 234         } else if (IS_FD_ARRAY(map) || IS_FD_PROG_ARRAY(map)) {
 235                 err = bpf_fd_array_map_lookup_elem(map, key, value);
 236         } else if (IS_FD_HASH(map)) {
 237                 err = bpf_fd_htab_map_lookup_elem(map, key, value);
 238         } else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) {
 239                 err = bpf_fd_reuseport_array_lookup_elem(map, key, value);
 240         } else if (map->map_type == BPF_MAP_TYPE_QUEUE ||
 241                    map->map_type == BPF_MAP_TYPE_STACK) {
 242                 err = map->ops->map_peek_elem(map, value);
 243         } else if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
 244                 /* struct_ops map requires directly updating "value" */
 245                 err = bpf_struct_ops_map_sys_lookup_elem(map, key, value);
 246         } else {
 247                 rcu_read_lock();
 248                 if (map->ops->map_lookup_elem_sys_only)
 249                         ptr = map->ops->map_lookup_elem_sys_only(map, key);
 250                 else
 251                         ptr = map->ops->map_lookup_elem(map, key);
 252                 if (IS_ERR(ptr)) {
 253                         err = PTR_ERR(ptr);
 254                 } else if (!ptr) {
 255                         err = -ENOENT;
 256                 } else {
 257                         err = 0;
 258                         if (flags & BPF_F_LOCK)
 259                                 /* lock 'ptr' and copy everything but lock */
 260                                 copy_map_value_locked(map, value, ptr, true);
 261                         else
 262                                 copy_map_value(map, value, ptr);
 263                         /* mask lock and timer, since value wasn't zero inited */
 264                         check_and_init_map_value(map, value);
 265                 }
 266                 rcu_read_unlock();
 267         }
 268
 269         bpf_enable_instrumentation();
 270         maybe_wait_bpf_programs(map);
 271
 272         return err;
 273 }
 274
 275 /* Please, do not use this function outside from the map creation path
 276  * (e.g. in map update path) without taking care of setting the active
 277  * memory cgroup (see at bpf_map_kmalloc_node() for example).
 278  */
 279 static void *__bpf_map_area_alloc(u64 size, int numa_node, bool mmapable)
 280 {
 281         /* We really just want to fail instead of triggering OOM killer
 282          * under memory pressure, therefore we set __GFP_NORETRY to kmalloc,
 283          * which is used for lower order allocation requests.
 284          *
 285          * It has been observed that higher order allocation requests done by
 286          * vmalloc with __GFP_NORETRY being set might fail due to not trying
 287          * to reclaim memory from the page cache, thus we set
 288          * __GFP_RETRY_MAYFAIL to avoid such situations.
 289          */
 290
 291         const gfp_t gfp = __GFP_NOWARN | __GFP_ZERO | __GFP_ACCOUNT;
 292         unsigned int flags = 0;
 293         unsigned long align = 1;
 294         void *area;
 295
 296         if (size >= SIZE_MAX)
 297                 return NULL;
 298
 299         /* kmalloc()'ed memory can't be mmap()'ed */
 300         if (mmapable) {
 301                 BUG_ON(!PAGE_ALIGNED(size));
 302                 align = SHMLBA;
 303                 flags = VM_USERMAP;
 304         } else if (size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) {
 305                 area = kmalloc_node(size, gfp | GFP_USER | __GFP_NORETRY,
 306                                     numa_node);
 307                 if (area != NULL)
 308                         return area;
 309         }
 310
 311         return __vmalloc_node_range(size, align, VMALLOC_START, VMALLOC_END,
 312                         gfp | GFP_KERNEL | __GFP_RETRY_MAYFAIL, PAGE_KERNEL,
 313                         flags, numa_node, __builtin_return_address(0));
 314 }
 315
 316 void *bpf_map_area_alloc(u64 size, int numa_node)
 317 {
 318         return __bpf_map_area_alloc(size, numa_node, false);
 319 }
 320
 321 void *bpf_map_area_mmapable_alloc(u64 size, int numa_node)
 322 {
 323         return __bpf_map_area_alloc(size, numa_node, true);
 324 }
 325
 326 void bpf_map_area_free(void *area)
 327 {
 328         kvfree(area);
 329 }
 330
 331 static u32 bpf_map_flags_retain_permanent(u32 flags)
 332 {
 333         /* Some map creation flags are not tied to the map object but
 334          * rather to the map fd instead, so they have no meaning upon
 335          * map object inspection since multiple file descriptors with
 336          * different (access) properties can exist here. Thus, given
 337          * this has zero meaning for the map itself, lets clear these
 338          * from here.
 339          */
 340         return flags & ~(BPF_F_RDONLY | BPF_F_WRONLY);
 341 }
 342
 343 void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr)
 344 {
 345         map->map_type = attr->map_type;
 346         map->key_size = attr->key_size;
 347         map->value_size = attr->value_size;
 348         map->max_entries = attr->max_entries;
 349         map->map_flags = bpf_map_flags_retain_permanent(attr->map_flags);
 350         map->numa_node = bpf_map_attr_numa_node(attr);
 351 }
 352
 353 static int bpf_map_alloc_id(struct bpf_map *map)
 354 {
 355         int id;
 356
 357         idr_preload(GFP_KERNEL);
 358         spin_lock_bh(&map_idr_lock);
 359         id = idr_alloc_cyclic(&map_idr, map, 1, INT_MAX, GFP_ATOMIC);
 360         if (id > 0)
 361                 map->id = id;
 362         spin_unlock_bh(&map_idr_lock);
 363         idr_preload_end();
 364
 365         if (WARN_ON_ONCE(!id))
 366                 return -ENOSPC;
 367
 368         return id > 0 ? 0 : id;
 369 }
 370
 371 void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock)
 372 {
 373         unsigned long flags;
 374
 375         /* Offloaded maps are removed from the IDR store when their device
 376          * disappears - even if someone holds an fd to them they are unusable,
 377          * the memory is gone, all ops will fail; they are simply waiting for
 378          * refcnt to drop to be freed.
 379          */
 380         if (!map->id)
 381                 return;
 382
 383         if (do_idr_lock)
 384                 spin_lock_irqsave(&map_idr_lock, flags);
 385         else
 386                 __acquire(&map_idr_lock);
 387
 388         idr_remove(&map_idr, map->id);
 389         map->id = 0;
 390
 391         if (do_idr_lock)
 392                 spin_unlock_irqrestore(&map_idr_lock, flags);
 393         else
 394                 __release(&map_idr_lock);
 395 }
 396
 397 #ifdef CONFIG_MEMCG_KMEM
 398 static void bpf_map_save_memcg(struct bpf_map *map)
 399 {
 400         map->memcg = get_mem_cgroup_from_mm(current->mm);
 401 }
 402
 403 static void bpf_map_release_memcg(struct bpf_map *map)
 404 {
 405         mem_cgroup_put(map->memcg);
 406 }
 407
 408 void *bpf_map_kmalloc_node(const struct bpf_map *map, size_t size, gfp_t flags,
 409                            int node)
 410 {
 411         struct mem_cgroup *old_memcg;
 412         void *ptr;
 413
 414         old_memcg = set_active_memcg(map->memcg);
 415         ptr = kmalloc_node(size, flags | __GFP_ACCOUNT, node);
 416         set_active_memcg(old_memcg);
 417
 418         return ptr;
 419 }
 420
 421 void *bpf_map_kzalloc(const struct bpf_map *map, size_t size, gfp_t flags)
 422 {
 423         struct mem_cgroup *old_memcg;
 424         void *ptr;
 425
 426         old_memcg = set_active_memcg(map->memcg);
 427         ptr = kzalloc(size, flags | __GFP_ACCOUNT);
 428         set_active_memcg(old_memcg);
 429
 430         return ptr;
 431 }
 432
 433 void __percpu *bpf_map_alloc_percpu(const struct bpf_map *map, size_t size,
 434                                     size_t align, gfp_t flags)
 435 {
 436         struct mem_cgroup *old_memcg;
 437         void __percpu *ptr;
 438
 439         old_memcg = set_active_memcg(map->memcg);
 440         ptr = __alloc_percpu_gfp(size, align, flags | __GFP_ACCOUNT);
 441         set_active_memcg(old_memcg);
 442
 443         return ptr;
 444 }
 445
 446 #else
 447 static void bpf_map_save_memcg(struct bpf_map *map)
 448 {
 449 }
 450
 451 static void bpf_map_release_memcg(struct bpf_map *map)
 452 {
 453 }
 454 #endif
 455
 456 /* called from workqueue */
 457 static void bpf_map_free_deferred(struct work_struct *work)
 458 {
 459         struct bpf_map *map = container_of(work, struct bpf_map, work);
 460
 461         security_bpf_map_free(map);
 462         bpf_map_release_memcg(map);
 463         /* implementation dependent freeing */
 464         map->ops->map_free(map);
 465 }
 466
 467 static void bpf_map_put_uref(struct bpf_map *map)
 468 {
 469         if (atomic64_dec_and_test(&map->usercnt)) {
 470                 if (map->ops->map_release_uref)
 471                         map->ops->map_release_uref(map);
 472         }
 473 }
 474
 475 /* decrement map refcnt and schedule it for freeing via workqueue
 476  * (unrelying map implementation ops->map_free() might sleep)
 477  */
 478 static void __bpf_map_put(struct bpf_map *map, bool do_idr_lock)
 479 {
 480         if (atomic64_dec_and_test(&map->refcnt)) {
 481                 /* bpf_map_free_id() must be called first */
 482                 bpf_map_free_id(map, do_idr_lock);
 483                 btf_put(map->btf);
 484                 INIT_WORK(&map->work, bpf_map_free_deferred);
 485                 schedule_work(&map->work);
 486         }
 487 }
 488
 489 void bpf_map_put(struct bpf_map *map)
 490 {
 491         __bpf_map_put(map, true);
 492 }
 493 EXPORT_SYMBOL_GPL(bpf_map_put);
 494
 495 void bpf_map_put_with_uref(struct bpf_map *map)
 496 {
 497         bpf_map_put_uref(map);
 498         bpf_map_put(map);
 499 }
 500
 501 static int bpf_map_release(struct inode *inode, struct file *filp)
 502 {
 503         struct bpf_map *map = filp->private_data;
 504
 505         if (map->ops->map_release)
 506                 map->ops->map_release(map, filp);
 507
 508         bpf_map_put_with_uref(map);
 509         return 0;
 510 }
 511
 512 static fmode_t map_get_sys_perms(struct bpf_map *map, struct fd f)
 513 {
 514         fmode_t mode = f.file->f_mode;
 515
 516         /* Our file permissions may have been overridden by global
 517          * map permissions facing syscall side.
 518          */
 519         if (READ_ONCE(map->frozen))
 520                 mode &= ~FMODE_CAN_WRITE;
 521         return mode;
 522 }
 523
 524 #ifdef CONFIG_PROC_FS
 525 /* Provides an approximation of the map's memory footprint.
 526  * Used only to provide a backward compatibility and display
 527  * a reasonable "memlock" info.
 528  */
 529 static unsigned long bpf_map_memory_footprint(const struct bpf_map *map)
 530 {
 531         unsigned long size;
 532
 533         size = round_up(map->key_size + bpf_map_value_size(map), 8);
 534
 535         return round_up(map->max_entries * size, PAGE_SIZE);
 536 }
 537
 538 static void bpf_map_show_fdinfo(struct seq_file *m, struct file *filp)
 539 {
 540         const struct bpf_map *map = filp->private_data;
 541         const struct bpf_array *array;
 542         u32 type = 0, jited = 0;
 543
 544         if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY) {
 545                 array = container_of(map, struct bpf_array, map);
 546                 type  = array->aux->type;
 547                 jited = array->aux->jited;
 548         }
 549
 550         seq_printf(m,
 551                    "map_type:\t%u\n"
 552                    "key_size:\t%u\n"
 553                    "value_size:\t%u\n"
 554                    "max_entries:\t%u\n"
 555                    "map_flags:\t%#x\n"
 556                    "memlock:\t%lu\n"
 557                    "map_id:\t%u\n"
 558                    "frozen:\t%u\n",
 559                    map->map_type,
 560                    map->key_size,
 561                    map->value_size,
 562                    map->max_entries,
 563                    map->map_flags,
 564                    bpf_map_memory_footprint(map),
 565                    map->id,
 566                    READ_ONCE(map->frozen));
 567         if (type) {
 568                 seq_printf(m, "owner_prog_type:\t%u\n", type);
 569                 seq_printf(m, "owner_jited:\t%u\n", jited);
 570         }
 571 }
 572 #endif
 573
 574 static ssize_t bpf_dummy_read(struct file *filp, char __user *buf, size_t siz,
 575                               loff_t *ppos)
 576 {
 577         /* We need this handler such that alloc_file() enables
 578          * f_mode with FMODE_CAN_READ.
 579          */
 580         return -EINVAL;
 581 }
 582
 583 static ssize_t bpf_dummy_write(struct file *filp, const char __user *buf,
 584                                size_t siz, loff_t *ppos)
 585 {
 586         /* We need this handler such that alloc_file() enables
 587          * f_mode with FMODE_CAN_WRITE.
 588          */
 589         return -EINVAL;
 590 }
 591
 592 /* called for any extra memory-mapped regions (except initial) */
 593 static void bpf_map_mmap_open(struct vm_area_struct *vma)
 594 {
 595         struct bpf_map *map = vma->vm_file->private_data;
 596
 597         if (vma->vm_flags & VM_MAYWRITE) {
 598                 mutex_lock(&map->freeze_mutex);
 599                 map->writecnt++;
 600                 mutex_unlock(&map->freeze_mutex);
 601         }
 602 }
 603
 604 /* called for all unmapped memory region (including initial) */
 605 static void bpf_map_mmap_close(struct vm_area_struct *vma)
 606 {
 607         struct bpf_map *map = vma->vm_file->private_data;
 608
 609         if (vma->vm_flags & VM_MAYWRITE) {
 610                 mutex_lock(&map->freeze_mutex);
 611                 map->writecnt--;
 612                 mutex_unlock(&map->freeze_mutex);
 613         }
 614 }
 615
 616 static const struct vm_operations_struct bpf_map_default_vmops = {
 617         .open           = bpf_map_mmap_open,
 618         .close          = bpf_map_mmap_close,
 619 };
 620
 621 static int bpf_map_mmap(struct file *filp, struct vm_area_struct *vma)
 622 {
 623         struct bpf_map *map = filp->private_data;
 624         int err;
 625
 626         if (!map->ops->map_mmap || map_value_has_spin_lock(map) ||
 627             map_value_has_timer(map))
 628                 return -ENOTSUPP;
 629
 630         if (!(vma->vm_flags & VM_SHARED))
 631                 return -EINVAL;
 632
 633         mutex_lock(&map->freeze_mutex);
 634
 635         if (vma->vm_flags & VM_WRITE) {
 636                 if (map->frozen) {
 637                         err = -EPERM;
 638                         goto out;
 639                 }
 640                 /* map is meant to be read-only, so do not allow mapping as
 641                  * writable, because it's possible to leak a writable page
 642                  * reference and allows user-space to still modify it after
 643                  * freezing, while verifier will assume contents do not change
 644                  */
 645                 if (map->map_flags & BPF_F_RDONLY_PROG) {
 646                         err = -EACCES;
 647                         goto out;
 648                 }
 649         }
 650
 651         /* set default open/close callbacks */
 652         vma->vm_ops = &bpf_map_default_vmops;
 653         vma->vm_private_data = map;
 654         vma->vm_flags &= ~VM_MAYEXEC;
 655         if (!(vma->vm_flags & VM_WRITE))
 656                 /* disallow re-mapping with PROT_WRITE */
 657                 vma->vm_flags &= ~VM_MAYWRITE;
 658
 659         err = map->ops->map_mmap(map, vma);
 660         if (err)
 661                 goto out;
 662
 663         if (vma->vm_flags & VM_MAYWRITE)
 664                 map->writecnt++;
 665 out:
 666         mutex_unlock(&map->freeze_mutex);
 667         return err;
 668 }
 669
 670 static __poll_t bpf_map_poll(struct file *filp, struct poll_table_struct *pts)
 671 {
 672         struct bpf_map *map = filp->private_data;
 673
 674         if (map->ops->map_poll)
 675                 return map->ops->map_poll(map, filp, pts);
 676
 677         return EPOLLERR;
 678 }
 679
 680 const struct file_operations bpf_map_fops = {
 681 #ifdef CONFIG_PROC_FS
 682         .show_fdinfo    = bpf_map_show_fdinfo,
 683 #endif
 684         .release        = bpf_map_release,
 685         .read           = bpf_dummy_read,
 686         .write          = bpf_dummy_write,
 687         .mmap           = bpf_map_mmap,
 688         .poll           = bpf_map_poll,
 689 };
 690
 691 int bpf_map_new_fd(struct bpf_map *map, int flags)
 692 {
 693         int ret;
 694
 695         ret = security_bpf_map(map, OPEN_FMODE(flags));
 696         if (ret < 0)
 697                 return ret;
 698
 699         return anon_inode_getfd("bpf-map", &bpf_map_fops, map,
 700                                 flags | O_CLOEXEC);
 701 }
 702
 703 int bpf_get_file_flag(int flags)
 704 {
 705         if ((flags & BPF_F_RDONLY) && (flags & BPF_F_WRONLY))
 706                 return -EINVAL;
 707         if (flags & BPF_F_RDONLY)
 708                 return O_RDONLY;
 709         if (flags & BPF_F_WRONLY)
 710                 return O_WRONLY;
 711         return O_RDWR;
 712 }
 713
 714 /* helper macro to check that unused fields 'union bpf_attr' are zero */
 715 #define CHECK_ATTR(CMD) \
 716         memchr_inv((void *) &attr->CMD##_LAST_FIELD + \
 717                    sizeof(attr->CMD##_LAST_FIELD), 0, \
 718                    sizeof(*attr) - \
 719                    offsetof(union bpf_attr, CMD##_LAST_FIELD) - \
 720                    sizeof(attr->CMD##_LAST_FIELD)) != NULL
 721
 722 /* dst and src must have at least "size" number of bytes.
 723  * Return strlen on success and < 0 on error.
 724  */
 725 int bpf_obj_name_cpy(char *dst, const char *src, unsigned int size)
 726 {
 727         const char *end = src + size;
 728         const char *orig_src = src;
 729
 730         memset(dst, 0, size);
 731         /* Copy all isalnum(), '_' and '.' chars. */
 732         while (src < end && *src) {
 733                 if (!isalnum(*src) &&
 734                     *src != '_' && *src != '.')
 735                         return -EINVAL;
 736                 *dst++ = *src++;
 737         }
 738
 739         /* No '\0' found in "size" number of bytes */
 740         if (src == end)
 741                 return -EINVAL;
 742
 743         return src - orig_src;
 744 }
 745
 746 int map_check_no_btf(const struct bpf_map *map,
 747                      const struct btf *btf,
 748                      const struct btf_type *key_type,
 749                      const struct btf_type *value_type)
 750 {
 751         return -ENOTSUPP;
 752 }
 753
 754 static int map_check_btf(struct bpf_map *map, const struct btf *btf,
 755                          u32 btf_key_id, u32 btf_value_id)
 756 {
 757         const struct btf_type *key_type, *value_type;
 758         u32 key_size, value_size;
 759         int ret = 0;
 760
 761         /* Some maps allow key to be unspecified. */
 762         if (btf_key_id) {
 763                 key_type = btf_type_id_size(btf, &btf_key_id, &key_size);
 764                 if (!key_type || key_size != map->key_size)
 765                         return -EINVAL;
 766         } else {
 767                 key_type = btf_type_by_id(btf, 0);
 768                 if (!map->ops->map_check_btf)
 769                         return -EINVAL;
 770         }
 771
 772         value_type = btf_type_id_size(btf, &btf_value_id, &value_size);
 773         if (!value_type || value_size != map->value_size)
 774                 return -EINVAL;
 775
 776         map->spin_lock_off = btf_find_spin_lock(btf, value_type);
 777
 778         if (map_value_has_spin_lock(map)) {
 779                 if (map->map_flags & BPF_F_RDONLY_PROG)
 780                         return -EACCES;
 781                 if (map->map_type != BPF_MAP_TYPE_HASH &&
 782                     map->map_type != BPF_MAP_TYPE_ARRAY &&
 783                     map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE &&
 784                     map->map_type != BPF_MAP_TYPE_SK_STORAGE &&
 785                     map->map_type != BPF_MAP_TYPE_INODE_STORAGE &&
 786                     map->map_type != BPF_MAP_TYPE_TASK_STORAGE)
 787                         return -ENOTSUPP;
 788                 if (map->spin_lock_off + sizeof(struct bpf_spin_lock) >
 789                     map->value_size) {
 790                         WARN_ONCE(1,
 791                                   "verifier bug spin_lock_off %d value_size %d\n",
 792                                   map->spin_lock_off, map->value_size);
 793                         return -EFAULT;
 794                 }
 795         }
 796
 797         map->timer_off = btf_find_timer(btf, value_type);
 798         if (map_value_has_timer(map)) {
 799                 if (map->map_flags & BPF_F_RDONLY_PROG)
 800                         return -EACCES;
 801                 if (map->map_type != BPF_MAP_TYPE_HASH &&
 802                     map->map_type != BPF_MAP_TYPE_LRU_HASH &&
 803                     map->map_type != BPF_MAP_TYPE_ARRAY)
 804                         return -EOPNOTSUPP;
 805         }
 806
 807         if (map->ops->map_check_btf)
 808                 ret = map->ops->map_check_btf(map, btf, key_type, value_type);
 809
 810         return ret;
 811 }
 812
 813 #define BPF_MAP_CREATE_LAST_FIELD btf_vmlinux_value_type_id
 814 /* called via syscall */
 815 static int map_create(union bpf_attr *attr)
 816 {
 817         int numa_node = bpf_map_attr_numa_node(attr);
 818         struct bpf_map *map;
 819         int f_flags;
 820         int err;
 821
 822         err = CHECK_ATTR(BPF_MAP_CREATE);
 823         if (err)
 824                 return -EINVAL;
 825
 826         if (attr->btf_vmlinux_value_type_id) {
 827                 if (attr->map_type != BPF_MAP_TYPE_STRUCT_OPS ||
 828                     attr->btf_key_type_id || attr->btf_value_type_id)
 829                         return -EINVAL;
 830         } else if (attr->btf_key_type_id && !attr->btf_value_type_id) {
 831                 return -EINVAL;
 832         }
 833
 834         f_flags = bpf_get_file_flag(attr->map_flags);
 835         if (f_flags < 0)
 836                 return f_flags;
 837
 838         if (numa_node != NUMA_NO_NODE &&
 839             ((unsigned int)numa_node >= nr_node_ids ||
 840              !node_online(numa_node)))
 841                 return -EINVAL;
 842
 843         /* find map type and init map: hashtable vs rbtree vs bloom vs ... */
 844         map = find_and_alloc_map(attr);
 845         if (IS_ERR(map))
 846                 return PTR_ERR(map);
 847
 848         err = bpf_obj_name_cpy(map->name, attr->map_name,
 849                                sizeof(attr->map_name));
 850         if (err < 0)
 851                 goto free_map;
 852
 853         atomic64_set(&map->refcnt, 1);
 854         atomic64_set(&map->usercnt, 1);
 855         mutex_init(&map->freeze_mutex);
 856
 857         map->spin_lock_off = -EINVAL;
 858         map->timer_off = -EINVAL;
 859         if (attr->btf_key_type_id || attr->btf_value_type_id ||
 860             /* Even the map's value is a kernel's struct,
 861              * the bpf_prog.o must have BTF to begin with
 862              * to figure out the corresponding kernel's
 863              * counter part.  Thus, attr->btf_fd has
 864              * to be valid also.
 865              */
 866             attr->btf_vmlinux_value_type_id) {
 867                 struct btf *btf;
 868
 869                 btf = btf_get_by_fd(attr->btf_fd);
 870                 if (IS_ERR(btf)) {
 871                         err = PTR_ERR(btf);
 872                         goto free_map;
 873                 }
 874                 if (btf_is_kernel(btf)) {
 875                         btf_put(btf);
 876                         err = -EACCES;
 877                         goto free_map;
 878                 }
 879                 map->btf = btf;
 880
 881                 if (attr->btf_value_type_id) {
 882                         err = map_check_btf(map, btf, attr->btf_key_type_id,
 883                                             attr->btf_value_type_id);
 884                         if (err)
 885                                 goto free_map;
 886                 }
 887
 888                 map->btf_key_type_id = attr->btf_key_type_id;
 889                 map->btf_value_type_id = attr->btf_value_type_id;
 890                 map->btf_vmlinux_value_type_id =
 891                         attr->btf_vmlinux_value_type_id;
 892         }
 893
 894         err = security_bpf_map_alloc(map);
 895         if (err)
 896                 goto free_map;
 897
 898         err = bpf_map_alloc_id(map);
 899         if (err)
 900                 goto free_map_sec;
 901
 902         bpf_map_save_memcg(map);
 903
 904         err = bpf_map_new_fd(map, f_flags);
 905         if (err < 0) {
 906                 /* failed to allocate fd.
 907                  * bpf_map_put_with_uref() is needed because the above
 908                  * bpf_map_alloc_id() has published the map
 909                  * to the userspace and the userspace may
 910                  * have refcnt-ed it through BPF_MAP_GET_FD_BY_ID.
 911                  */
 912                 bpf_map_put_with_uref(map);
 913                 return err;
 914         }
 915
 916         return err;
 917
 918 free_map_sec:
 919         security_bpf_map_free(map);
 920 free_map:
 921         btf_put(map->btf);
 922         map->ops->map_free(map);
 923         return err;
 924 }
 925
 926 /* if error is returned, fd is released.
 927  * On success caller should complete fd access with matching fdput()
 928  */
 929 struct bpf_map *__bpf_map_get(struct fd f)
 930 {
 931         if (!f.file)
 932                 return ERR_PTR(-EBADF);
 933         if (f.file->f_op != &bpf_map_fops) {
 934                 fdput(f);
 935                 return ERR_PTR(-EINVAL);
 936         }
 937
 938         return f.file->private_data;
 939 }
 940
 941 void bpf_map_inc(struct bpf_map *map)
 942 {
 943         atomic64_inc(&map->refcnt);
 944 }
 945 EXPORT_SYMBOL_GPL(bpf_map_inc);
 946
 947 void bpf_map_inc_with_uref(struct bpf_map *map)
 948 {
 949         atomic64_inc(&map->refcnt);
 950         atomic64_inc(&map->usercnt);
 951 }
 952 EXPORT_SYMBOL_GPL(bpf_map_inc_with_uref);
 953
 954 struct bpf_map *bpf_map_get(u32 ufd)
 955 {
 956         struct fd f = fdget(ufd);
 957         struct bpf_map *map;
 958
 959         map = __bpf_map_get(f);
 960         if (IS_ERR(map))
 961                 return map;
 962
 963         bpf_map_inc(map);
 964         fdput(f);
 965
 966         return map;
 967 }
 968
 969 struct bpf_map *bpf_map_get_with_uref(u32 ufd)
 970 {
 971         struct fd f = fdget(ufd);
 972         struct bpf_map *map;
 973
 974         map = __bpf_map_get(f);
 975         if (IS_ERR(map))
 976                 return map;
 977
 978         bpf_map_inc_with_uref(map);
 979         fdput(f);
 980
 981         return map;
 982 }
 983
 984 /* map_idr_lock should have been held */
 985 static struct bpf_map *__bpf_map_inc_not_zero(struct bpf_map *map, bool uref)
 986 {
 987         int refold;
 988
 989         refold = atomic64_fetch_add_unless(&map->refcnt, 1, 0);
 990         if (!refold)
 991                 return ERR_PTR(-ENOENT);
 992         if (uref)
 993                 atomic64_inc(&map->usercnt);
 994
 995         return map;
 996 }
 997
 998 struct bpf_map *bpf_map_inc_not_zero(struct bpf_map *map)
 999 {
1000         spin_lock_bh(&map_idr_lock);
1001         map = __bpf_map_inc_not_zero(map, false);
1002         spin_unlock_bh(&map_idr_lock);
1003
1004         return map;
1005 }
1006 EXPORT_SYMBOL_GPL(bpf_map_inc_not_zero);
1007
1008 int __weak bpf_stackmap_copy(struct bpf_map *map, void *key, void *value)
1009 {
1010         return -ENOTSUPP;
1011 }
1012
1013 static void *__bpf_copy_key(void __user *ukey, u64 key_size)
1014 {
1015         if (key_size)
1016                 return memdup_user(ukey, key_size);
1017
1018         if (ukey)
1019                 return ERR_PTR(-EINVAL);
1020
1021         return NULL;
1022 }
1023
1024 static void *___bpf_copy_key(bpfptr_t ukey, u64 key_size)
1025 {
1026         if (key_size)
1027                 return memdup_bpfptr(ukey, key_size);
1028
1029         if (!bpfptr_is_null(ukey))
1030                 return ERR_PTR(-EINVAL);
1031
1032         return NULL;
1033 }
1034
1035 /* last field in 'union bpf_attr' used by this command */
1036 #define BPF_MAP_LOOKUP_ELEM_LAST_FIELD flags
1037
1038 static int map_lookup_elem(union bpf_attr *attr)
1039 {
1040         void __user *ukey = u64_to_user_ptr(attr->key);
1041         void __user *uvalue = u64_to_user_ptr(attr->value);
1042         int ufd = attr->map_fd;
1043         struct bpf_map *map;
1044         void *key, *value;
1045         u32 value_size;
1046         struct fd f;
1047         int err;
1048
1049         if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM))
1050                 return -EINVAL;
1051
1052         if (attr->flags & ~BPF_F_LOCK)
1053                 return -EINVAL;
1054
1055         f = fdget(ufd);
1056         map = __bpf_map_get(f);
1057         if (IS_ERR(map))
1058                 return PTR_ERR(map);
1059         if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ)) {
1060                 err = -EPERM;
1061                 goto err_put;
1062         }
1063
1064         if ((attr->flags & BPF_F_LOCK) &&
1065             !map_value_has_spin_lock(map)) {
1066                 err = -EINVAL;
1067                 goto err_put;
1068         }
1069
1070         key = __bpf_copy_key(ukey, map->key_size);
1071         if (IS_ERR(key)) {
1072                 err = PTR_ERR(key);
1073                 goto err_put;
1074         }
1075
1076         value_size = bpf_map_value_size(map);
1077
1078         err = -ENOMEM;
1079         value = kmalloc(value_size, GFP_USER | __GFP_NOWARN);
1080         if (!value)
1081                 goto free_key;
1082
1083         err = bpf_map_copy_value(map, key, value, attr->flags);
1084         if (err)
1085                 goto free_value;
1086
1087         err = -EFAULT;
1088         if (copy_to_user(uvalue, value, value_size) != 0)
1089                 goto free_value;
1090
1091         err = 0;
1092
1093 free_value:
1094         kfree(value);
1095 free_key:
1096         kfree(key);
1097 err_put:
1098         fdput(f);
1099         return err;
1100 }
1101
1102
1103 #define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags
1104
1105 static int map_update_elem(union bpf_attr *attr, bpfptr_t uattr)
1106 {
1107         bpfptr_t ukey = make_bpfptr(attr->key, uattr.is_kernel);
1108         bpfptr_t uvalue = make_bpfptr(attr->value, uattr.is_kernel);
1109         int ufd = attr->map_fd;
1110         struct bpf_map *map;
1111         void *key, *value;
1112         u32 value_size;
1113         struct fd f;
1114         int err;
1115
1116         if (CHECK_ATTR(BPF_MAP_UPDATE_ELEM))
1117                 return -EINVAL;
1118
1119         f = fdget(ufd);
1120         map = __bpf_map_get(f);
1121         if (IS_ERR(map))
1122                 return PTR_ERR(map);
1123         if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
1124                 err = -EPERM;
1125                 goto err_put;
1126         }
1127
1128         if ((attr->flags & BPF_F_LOCK) &&
1129             !map_value_has_spin_lock(map)) {
1130                 err = -EINVAL;
1131                 goto err_put;
1132         }
1133
1134         key = ___bpf_copy_key(ukey, map->key_size);
1135         if (IS_ERR(key)) {
1136                 err = PTR_ERR(key);
1137                 goto err_put;
1138         }
1139
1140         if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
1141             map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||
1142             map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY ||
1143             map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE)
1144                 value_size = round_up(map->value_size, 8) * num_possible_cpus();
1145         else
1146                 value_size = map->value_size;
1147
1148         err = -ENOMEM;
1149         value = kmalloc(value_size, GFP_USER | __GFP_NOWARN);
1150         if (!value)
1151                 goto free_key;
1152
1153         err = -EFAULT;
1154         if (copy_from_bpfptr(value, uvalue, value_size) != 0)
1155                 goto free_value;
1156
1157         err = bpf_map_update_value(map, f, key, value, attr->flags);
1158
1159 free_value:
1160         kfree(value);
1161 free_key:
1162         kfree(key);
1163 err_put:
1164         fdput(f);
1165         return err;
1166 }
1167
1168 #define BPF_MAP_DELETE_ELEM_LAST_FIELD key
1169
1170 static int map_delete_elem(union bpf_attr *attr)
1171 {
1172         void __user *ukey = u64_to_user_ptr(attr->key);
1173         int ufd = attr->map_fd;
1174         struct bpf_map *map;
1175         struct fd f;
1176         void *key;
1177         int err;
1178
1179         if (CHECK_ATTR(BPF_MAP_DELETE_ELEM))
1180                 return -EINVAL;
1181
1182         f = fdget(ufd);
1183         map = __bpf_map_get(f);
1184         if (IS_ERR(map))
1185                 return PTR_ERR(map);
1186         if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
1187                 err = -EPERM;
1188                 goto err_put;
1189         }
1190
1191         key = __bpf_copy_key(ukey, map->key_size);
1192         if (IS_ERR(key)) {
1193                 err = PTR_ERR(key);
1194                 goto err_put;
1195         }
1196
1197         if (bpf_map_is_dev_bound(map)) {
1198                 err = bpf_map_offload_delete_elem(map, key);
1199                 goto out;
1200         } else if (IS_FD_PROG_ARRAY(map) ||
1201                    map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
1202                 /* These maps require sleepable context */
1203                 err = map->ops->map_delete_elem(map, key);
1204                 goto out;
1205         }
1206
1207         bpf_disable_instrumentation();
1208         rcu_read_lock();
1209         err = map->ops->map_delete_elem(map, key);
1210         rcu_read_unlock();
1211         bpf_enable_instrumentation();
1212         maybe_wait_bpf_programs(map);
1213 out:
1214         kfree(key);
1215 err_put:
1216         fdput(f);
1217         return err;
1218 }
1219
1220 /* last field in 'union bpf_attr' used by this command */
1221 #define BPF_MAP_GET_NEXT_KEY_LAST_FIELD next_key
1222
1223 static int map_get_next_key(union bpf_attr *attr)
1224 {
1225         void __user *ukey = u64_to_user_ptr(attr->key);
1226         void __user *unext_key = u64_to_user_ptr(attr->next_key);
1227         int ufd = attr->map_fd;
1228         struct bpf_map *map;
1229         void *key, *next_key;
1230         struct fd f;
1231         int err;
1232
1233         if (CHECK_ATTR(BPF_MAP_GET_NEXT_KEY))
1234                 return -EINVAL;
1235
1236         f = fdget(ufd);
1237         map = __bpf_map_get(f);
1238         if (IS_ERR(map))
1239                 return PTR_ERR(map);
1240         if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ)) {
1241                 err = -EPERM;
1242                 goto err_put;
1243         }
1244
1245         if (ukey) {
1246                 key = __bpf_copy_key(ukey, map->key_size);
1247                 if (IS_ERR(key)) {
1248                         err = PTR_ERR(key);
1249                         goto err_put;
1250                 }
1251         } else {
1252                 key = NULL;
1253         }
1254
1255         err = -ENOMEM;
1256         next_key = kmalloc(map->key_size, GFP_USER);
1257         if (!next_key)
1258                 goto free_key;
1259
1260         if (bpf_map_is_dev_bound(map)) {
1261                 err = bpf_map_offload_get_next_key(map, key, next_key);
1262                 goto out;
1263         }
1264
1265         rcu_read_lock();
1266         err = map->ops->map_get_next_key(map, key, next_key);
1267         rcu_read_unlock();
1268 out:
1269         if (err)
1270                 goto free_next_key;
1271
1272         err = -EFAULT;
1273         if (copy_to_user(unext_key, next_key, map->key_size) != 0)
1274                 goto free_next_key;
1275
1276         err = 0;
1277
1278 free_next_key:
1279         kfree(next_key);
1280 free_key:
1281         kfree(key);
1282 err_put:
1283         fdput(f);
1284         return err;
1285 }
1286
1287 int generic_map_delete_batch(struct bpf_map *map,
1288                              const union bpf_attr *attr,
1289                              union bpf_attr __user *uattr)
1290 {
1291         void __user *keys = u64_to_user_ptr(attr->batch.keys);
1292         u32 cp, max_count;
1293         int err = 0;
1294         void *key;
1295
1296         if (attr->batch.elem_flags & ~BPF_F_LOCK)
1297                 return -EINVAL;
1298
1299         if ((attr->batch.elem_flags & BPF_F_LOCK) &&
1300             !map_value_has_spin_lock(map)) {
1301                 return -EINVAL;
1302         }
1303
1304         max_count = attr->batch.count;
1305         if (!max_count)
1306                 return 0;
1307
1308         key = kmalloc(map->key_size, GFP_USER | __GFP_NOWARN);
1309         if (!key)
1310                 return -ENOMEM;
1311
1312         for (cp = 0; cp < max_count; cp++) {
1313                 err = -EFAULT;
1314                 if (copy_from_user(key, keys + cp * map->key_size,
1315                                    map->key_size))
1316                         break;
1317
1318                 if (bpf_map_is_dev_bound(map)) {
1319                         err = bpf_map_offload_delete_elem(map, key);
1320                         break;
1321                 }
1322
1323                 bpf_disable_instrumentation();
1324                 rcu_read_lock();
1325                 err = map->ops->map_delete_elem(map, key);
1326                 rcu_read_unlock();
1327                 bpf_enable_instrumentation();
1328                 maybe_wait_bpf_programs(map);
1329                 if (err)
1330                         break;
1331         }
1332         if (copy_to_user(&uattr->batch.count, &cp, sizeof(cp)))
1333                 err = -EFAULT;
1334
1335         kfree(key);
1336         return err;
1337 }
1338
1339 int generic_map_update_batch(struct bpf_map *map,
1340                              const union bpf_attr *attr,
1341                              union bpf_attr __user *uattr)
1342 {
1343         void __user *values = u64_to_user_ptr(attr->batch.values);
1344         void __user *keys = u64_to_user_ptr(attr->batch.keys);
1345         u32 value_size, cp, max_count;
1346         int ufd = attr->map_fd;
1347         void *key, *value;
1348         struct fd f;
1349         int err = 0;
1350
1351         f = fdget(ufd);
1352         if (attr->batch.elem_flags & ~BPF_F_LOCK)
1353                 return -EINVAL;
1354
1355         if ((attr->batch.elem_flags & BPF_F_LOCK) &&
1356             !map_value_has_spin_lock(map)) {
1357                 return -EINVAL;
1358         }
1359
1360         value_size = bpf_map_value_size(map);
1361
1362         max_count = attr->batch.count;
1363         if (!max_count)
1364                 return 0;
1365
1366         key = kmalloc(map->key_size, GFP_USER | __GFP_NOWARN);
1367         if (!key)
1368                 return -ENOMEM;
1369
1370         value = kmalloc(value_size, GFP_USER | __GFP_NOWARN);
1371         if (!value) {
1372                 kfree(key);
1373                 return -ENOMEM;
1374         }
1375
1376         for (cp = 0; cp < max_count; cp++) {
1377                 err = -EFAULT;
1378                 if (copy_from_user(key, keys + cp * map->key_size,
1379                     map->key_size) ||
1380                     copy_from_user(value, values + cp * value_size, value_size))
1381                         break;
1382
1383                 err = bpf_map_update_value(map, f, key, value,
1384                                            attr->batch.elem_flags);
1385
1386                 if (err)
1387                         break;
1388         }
1389
1390         if (copy_to_user(&uattr->batch.count, &cp, sizeof(cp)))
1391                 err = -EFAULT;
1392
1393         kfree(value);
1394         kfree(key);
1395         return err;
1396 }
1397
1398 #define MAP_LOOKUP_RETRIES 3
1399
1400 int generic_map_lookup_batch(struct bpf_map *map,
1401                                     const union bpf_attr *attr,
1402                                     union bpf_attr __user *uattr)
1403 {
1404         void __user *uobatch = u64_to_user_ptr(attr->batch.out_batch);
1405         void __user *ubatch = u64_to_user_ptr(attr->batch.in_batch);
1406         void __user *values = u64_to_user_ptr(attr->batch.values);
1407         void __user *keys = u64_to_user_ptr(attr->batch.keys);
1408         void *buf, *buf_prevkey, *prev_key, *key, *value;
1409         int err, retry = MAP_LOOKUP_RETRIES;
1410         u32 value_size, cp, max_count;
1411
1412         if (attr->batch.elem_flags & ~BPF_F_LOCK)
1413                 return -EINVAL;
1414
1415         if ((attr->batch.elem_flags & BPF_F_LOCK) &&
1416             !map_value_has_spin_lock(map))
1417                 return -EINVAL;
1418
1419         value_size = bpf_map_value_size(map);
1420
1421         max_count = attr->batch.count;
1422         if (!max_count)
1423                 return 0;
1424
1425         if (put_user(0, &uattr->batch.count))
1426                 return -EFAULT;
1427
1428         buf_prevkey = kmalloc(map->key_size, GFP_USER | __GFP_NOWARN);
1429         if (!buf_prevkey)
1430                 return -ENOMEM;
1431
1432         buf = kmalloc(map->key_size + value_size, GFP_USER | __GFP_NOWARN);
1433         if (!buf) {
1434                 kfree(buf_prevkey);
1435                 return -ENOMEM;
1436         }
1437
1438         err = -EFAULT;
1439         prev_key = NULL;
1440         if (ubatch && copy_from_user(buf_prevkey, ubatch, map->key_size))
1441                 goto free_buf;
1442         key = buf;
1443         value = key + map->key_size;
1444         if (ubatch)
1445                 prev_key = buf_prevkey;
1446
1447         for (cp = 0; cp < max_count;) {
1448                 rcu_read_lock();
1449                 err = map->ops->map_get_next_key(map, prev_key, key);
1450                 rcu_read_unlock();
1451                 if (err)
1452                         break;
1453                 err = bpf_map_copy_value(map, key, value,
1454                                          attr->batch.elem_flags);
1455
1456                 if (err == -ENOENT) {
1457                         if (retry) {
1458                                 retry--;
1459                                 continue;
1460                         }
1461                         err = -EINTR;
1462                         break;
1463                 }
1464
1465                 if (err)
1466                         goto free_buf;
1467
1468                 if (copy_to_user(keys + cp * map->key_size, key,
1469                                  map->key_size)) {
1470                         err = -EFAULT;
1471                         goto free_buf;
1472                 }
1473                 if (copy_to_user(values + cp * value_size, value, value_size)) {
1474                         err = -EFAULT;
1475                         goto free_buf;
1476                 }
1477
1478                 if (!prev_key)
1479                         prev_key = buf_prevkey;
1480
1481                 swap(prev_key, key);
1482                 retry = MAP_LOOKUP_RETRIES;
1483                 cp++;
1484         }
1485
1486         if (err == -EFAULT)
1487                 goto free_buf;
1488
1489         if ((copy_to_user(&uattr->batch.count, &cp, sizeof(cp)) ||
1490                     (cp && copy_to_user(uobatch, prev_key, map->key_size))))
1491                 err = -EFAULT;
1492
1493 free_buf:
1494         kfree(buf_prevkey);
1495         kfree(buf);
1496         return err;
1497 }
1498
1499 #define BPF_MAP_LOOKUP_AND_DELETE_ELEM_LAST_FIELD flags
1500
1501 static int map_lookup_and_delete_elem(union bpf_attr *attr)
1502 {
1503         void __user *ukey = u64_to_user_ptr(attr->key);
1504         void __user *uvalue = u64_to_user_ptr(attr->value);
1505         int ufd = attr->map_fd;
1506         struct bpf_map *map;
1507         void *key, *value;
1508         u32 value_size;
1509         struct fd f;
1510         int err;
1511
1512         if (CHECK_ATTR(BPF_MAP_LOOKUP_AND_DELETE_ELEM))
1513                 return -EINVAL;
1514
1515         if (attr->flags & ~BPF_F_LOCK)
1516                 return -EINVAL;
1517
1518         f = fdget(ufd);
1519         map = __bpf_map_get(f);
1520         if (IS_ERR(map))
1521                 return PTR_ERR(map);
1522         if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ) ||
1523             !(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
1524                 err = -EPERM;
1525                 goto err_put;
1526         }
1527
1528         if (attr->flags &&
1529             (map->map_type == BPF_MAP_TYPE_QUEUE ||
1530              map->map_type == BPF_MAP_TYPE_STACK)) {
1531                 err = -EINVAL;
1532                 goto err_put;
1533         }
1534
1535         if ((attr->flags & BPF_F_LOCK) &&
1536             !map_value_has_spin_lock(map)) {
1537                 err = -EINVAL;
1538                 goto err_put;
1539         }
1540
1541         key = __bpf_copy_key(ukey, map->key_size);
1542         if (IS_ERR(key)) {
1543                 err = PTR_ERR(key);
1544                 goto err_put;
1545         }
1546
1547         value_size = bpf_map_value_size(map);
1548
1549         err = -ENOMEM;
1550         value = kmalloc(value_size, GFP_USER | __GFP_NOWARN);
1551         if (!value)
1552                 goto free_key;
1553
1554         err = -ENOTSUPP;
1555         if (map->map_type == BPF_MAP_TYPE_QUEUE ||
1556             map->map_type == BPF_MAP_TYPE_STACK) {
1557                 err = map->ops->map_pop_elem(map, value);
1558         } else if (map->map_type == BPF_MAP_TYPE_HASH ||
1559                    map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
1560                    map->map_type == BPF_MAP_TYPE_LRU_HASH ||
1561                    map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
1562                 if (!bpf_map_is_dev_bound(map)) {
1563                         bpf_disable_instrumentation();
1564                         rcu_read_lock();
1565                         err = map->ops->map_lookup_and_delete_elem(map, key, value, attr->flags);
1566                         rcu_read_unlock();
1567                         bpf_enable_instrumentation();
1568                 }
1569         }
1570
1571         if (err)
1572                 goto free_value;
1573
1574         if (copy_to_user(uvalue, value, value_size) != 0) {
1575                 err = -EFAULT;
1576                 goto free_value;
1577         }
1578
1579         err = 0;
1580
1581 free_value:
1582         kfree(value);
1583 free_key:
1584         kfree(key);
1585 err_put:
1586         fdput(f);
1587         return err;
1588 }
1589
1590 #define BPF_MAP_FREEZE_LAST_FIELD map_fd
1591
1592 static int map_freeze(const union bpf_attr *attr)
1593 {
1594         int err = 0, ufd = attr->map_fd;
1595         struct bpf_map *map;
1596         struct fd f;
1597
1598         if (CHECK_ATTR(BPF_MAP_FREEZE))
1599                 return -EINVAL;
1600
1601         f = fdget(ufd);
1602         map = __bpf_map_get(f);
1603         if (IS_ERR(map))
1604                 return PTR_ERR(map);
1605
1606         if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS ||
1607             map_value_has_timer(map)) {
1608                 fdput(f);
1609                 return -ENOTSUPP;
1610         }
1611
1612         mutex_lock(&map->freeze_mutex);
1613
1614         if (map->writecnt) {
1615                 err = -EBUSY;
1616                 goto err_put;
1617         }
1618         if (READ_ONCE(map->frozen)) {
1619                 err = -EBUSY;
1620                 goto err_put;
1621         }
1622         if (!bpf_capable()) {
1623                 err = -EPERM;
1624                 goto err_put;
1625         }
1626
1627         WRITE_ONCE(map->frozen, true);
1628 err_put:
1629         mutex_unlock(&map->freeze_mutex);
1630         fdput(f);
1631         return err;
1632 }
1633
1634 static const struct bpf_prog_ops * const bpf_prog_types[] = {
1635 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \
1636         [_id] = & _name ## _prog_ops,
1637 #define BPF_MAP_TYPE(_id, _ops)
1638 #define BPF_LINK_TYPE(_id, _name)
1639 #include <linux/bpf_types.h>
1640 #undef BPF_PROG_TYPE
1641 #undef BPF_MAP_TYPE
1642 #undef BPF_LINK_TYPE
1643 };
1644
1645 static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog)
1646 {
1647         const struct bpf_prog_ops *ops;
1648
1649         if (type >= ARRAY_SIZE(bpf_prog_types))
1650                 return -EINVAL;
1651         type = array_index_nospec(type, ARRAY_SIZE(bpf_prog_types));
1652         ops = bpf_prog_types[type];
1653         if (!ops)
1654                 return -EINVAL;
1655
1656         if (!bpf_prog_is_dev_bound(prog->aux))
1657                 prog->aux->ops = ops;
1658         else
1659                 prog->aux->ops = &bpf_offload_prog_ops;
1660         prog->type = type;
1661         return 0;
1662 }
1663
1664 enum bpf_audit {
1665         BPF_AUDIT_LOAD,
1666         BPF_AUDIT_UNLOAD,
1667         BPF_AUDIT_MAX,
1668 };
1669
1670 static const char * const bpf_audit_str[BPF_AUDIT_MAX] = {
1671         [BPF_AUDIT_LOAD]   = "LOAD",
1672         [BPF_AUDIT_UNLOAD] = "UNLOAD",
1673 };
1674
1675 static void bpf_audit_prog(const struct bpf_prog *prog, unsigned int op)
1676 {
1677         struct audit_context *ctx = NULL;
1678         struct audit_buffer *ab;
1679
1680         if (WARN_ON_ONCE(op >= BPF_AUDIT_MAX))
1681                 return;
1682         if (audit_enabled == AUDIT_OFF)
1683                 return;
1684         if (op == BPF_AUDIT_LOAD)
1685                 ctx = audit_context();
1686         ab = audit_log_start(ctx, GFP_ATOMIC, AUDIT_BPF);
1687         if (unlikely(!ab))
1688                 return;
1689         audit_log_format(ab, "prog-id=%u op=%s",
1690                          prog->aux->id, bpf_audit_str[op]);
1691         audit_log_end(ab);
1692 }
1693
1694 static int bpf_prog_alloc_id(struct bpf_prog *prog)
1695 {
1696         int id;
1697
1698         idr_preload(GFP_KERNEL);
1699         spin_lock_bh(&prog_idr_lock);
1700         id = idr_alloc_cyclic(&prog_idr, prog, 1, INT_MAX, GFP_ATOMIC);
1701         if (id > 0)
1702                 prog->aux->id = id;
1703         spin_unlock_bh(&prog_idr_lock);
1704         idr_preload_end();
1705
1706         /* id is in [1, INT_MAX) */
1707         if (WARN_ON_ONCE(!id))
1708                 return -ENOSPC;
1709
1710         return id > 0 ? 0 : id;
1711 }
1712
1713 void bpf_prog_free_id(struct bpf_prog *prog, bool do_idr_lock)
1714 {
1715         unsigned long flags;
1716
1717         /* cBPF to eBPF migrations are currently not in the idr store.
1718          * Offloaded programs are removed from the store when their device
1719          * disappears - even if someone grabs an fd to them they are unusable,
1720          * simply waiting for refcnt to drop to be freed.
1721          */
1722         if (!prog->aux->id)
1723                 return;
1724
1725         if (do_idr_lock)
1726                 spin_lock_irqsave(&prog_idr_lock, flags);
1727         else
1728                 __acquire(&prog_idr_lock);
1729
1730         idr_remove(&prog_idr, prog->aux->id);
1731         prog->aux->id = 0;
1732
1733         if (do_idr_lock)
1734                 spin_unlock_irqrestore(&prog_idr_lock, flags);
1735         else
1736                 __release(&prog_idr_lock);
1737 }
1738
1739 static void __bpf_prog_put_rcu(struct rcu_head *rcu)
1740 {
1741         struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu);
1742
1743         kvfree(aux->func_info);
1744         kfree(aux->func_info_aux);
1745         free_uid(aux->user);
1746         security_bpf_prog_free(aux);
1747         bpf_prog_free(aux->prog);
1748 }
1749
1750 static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred)
1751 {
1752         bpf_prog_kallsyms_del_all(prog);
1753         btf_put(prog->aux->btf);
1754         kvfree(prog->aux->jited_linfo);
1755         kvfree(prog->aux->linfo);
1756         kfree(prog->aux->kfunc_tab);
1757         if (prog->aux->attach_btf)
1758                 btf_put(prog->aux->attach_btf);
1759
1760         if (deferred) {
1761                 if (prog->aux->sleepable)
1762                         call_rcu_tasks_trace(&prog->aux->rcu, __bpf_prog_put_rcu);
1763                 else
1764                         call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu);
1765         } else {
1766                 __bpf_prog_put_rcu(&prog->aux->rcu);
1767         }
1768 }
1769
1770 static void bpf_prog_put_deferred(struct work_struct *work)
1771 {
1772         struct bpf_prog_aux *aux;
1773         struct bpf_prog *prog;
1774
1775         aux = container_of(work, struct bpf_prog_aux, work);
1776         prog = aux->prog;
1777         perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_UNLOAD, 0);
1778         bpf_audit_prog(prog, BPF_AUDIT_UNLOAD);
1779         __bpf_prog_put_noref(prog, true);
1780 }
1781
1782 static void __bpf_prog_put(struct bpf_prog *prog, bool do_idr_lock)
1783 {
1784         struct bpf_prog_aux *aux = prog->aux;
1785
1786         if (atomic64_dec_and_test(&aux->refcnt)) {
1787                 /* bpf_prog_free_id() must be called first */
1788                 bpf_prog_free_id(prog, do_idr_lock);
1789
1790                 if (in_irq() || irqs_disabled()) {
1791                         INIT_WORK(&aux->work, bpf_prog_put_deferred);
1792                         schedule_work(&aux->work);
1793                 } else {
1794                         bpf_prog_put_deferred(&aux->work);
1795                 }
1796         }
1797 }
1798
1799 void bpf_prog_put(struct bpf_prog *prog)
1800 {
1801         __bpf_prog_put(prog, true);
1802 }
1803 EXPORT_SYMBOL_GPL(bpf_prog_put);
1804
1805 static int bpf_prog_release(struct inode *inode, struct file *filp)
1806 {
1807         struct bpf_prog *prog = filp->private_data;
1808
1809         bpf_prog_put(prog);
1810         return 0;
1811 }
1812
1813 static void bpf_prog_get_stats(const struct bpf_prog *prog,
1814                                struct bpf_prog_stats *stats)
1815 {
1816         u64 nsecs = 0, cnt = 0, misses = 0;
1817         int cpu;
1818
1819         for_each_possible_cpu(cpu) {
1820                 const struct bpf_prog_stats *st;
1821                 unsigned int start;
1822                 u64 tnsecs, tcnt, tmisses;
1823
1824                 st = per_cpu_ptr(prog->stats, cpu);
1825                 do {
1826                         start = u64_stats_fetch_begin_irq(&st->syncp);
1827                         tnsecs = st->nsecs;
1828                         tcnt = st->cnt;
1829                         tmisses = st->misses;
1830                 } while (u64_stats_fetch_retry_irq(&st->syncp, start));
1831                 nsecs += tnsecs;
1832                 cnt += tcnt;
1833                 misses += tmisses;
1834         }
1835         stats->nsecs = nsecs;
1836         stats->cnt = cnt;
1837         stats->misses = misses;
1838 }
1839
1840 #ifdef CONFIG_PROC_FS
1841 static void bpf_prog_show_fdinfo(struct seq_file *m, struct file *filp)
1842 {
1843         const struct bpf_prog *prog = filp->private_data;
1844         char prog_tag[sizeof(prog->tag) * 2 + 1] = { };
1845         struct bpf_prog_stats stats;
1846
1847         bpf_prog_get_stats(prog, &stats);
1848         bin2hex(prog_tag, prog->tag, sizeof(prog->tag));
1849         seq_printf(m,
1850                    "prog_type:\t%u\n"
1851                    "prog_jited:\t%u\n"
1852                    "prog_tag:\t%s\n"
1853                    "memlock:\t%llu\n"
1854                    "prog_id:\t%u\n"
1855                    "run_time_ns:\t%llu\n"
1856                    "run_cnt:\t%llu\n"
1857                    "recursion_misses:\t%llu\n",
1858                    prog->type,
1859                    prog->jited,
1860                    prog_tag,
1861                    prog->pages * 1ULL << PAGE_SHIFT,
1862                    prog->aux->id,
1863                    stats.nsecs,
1864                    stats.cnt,
1865                    stats.misses);
1866 }
1867 #endif
1868
1869 const struct file_operations bpf_prog_fops = {
1870 #ifdef CONFIG_PROC_FS
1871         .show_fdinfo    = bpf_prog_show_fdinfo,
1872 #endif
1873         .release        = bpf_prog_release,
1874         .read           = bpf_dummy_read,
1875         .write          = bpf_dummy_write,
1876 };
1877
1878 int bpf_prog_new_fd(struct bpf_prog *prog)
1879 {
1880         int ret;
1881
1882         ret = security_bpf_prog(prog);
1883         if (ret < 0)
1884                 return ret;
1885
1886         return anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog,
1887                                 O_RDWR | O_CLOEXEC);
1888 }
1889
1890 static struct bpf_prog *____bpf_prog_get(struct fd f)
1891 {
1892         if (!f.file)
1893                 return ERR_PTR(-EBADF);
1894         if (f.file->f_op != &bpf_prog_fops) {
1895                 fdput(f);
1896                 return ERR_PTR(-EINVAL);
1897         }
1898
1899         return f.file->private_data;
1900 }
1901
1902 void bpf_prog_add(struct bpf_prog *prog, int i)
1903 {
1904         atomic64_add(i, &prog->aux->refcnt);
1905 }
1906 EXPORT_SYMBOL_GPL(bpf_prog_add);
1907
1908 void bpf_prog_sub(struct bpf_prog *prog, int i)
1909 {
1910         /* Only to be used for undoing previous bpf_prog_add() in some
1911          * error path. We still know that another entity in our call
1912          * path holds a reference to the program, thus atomic_sub() can
1913          * be safely used in such cases!
1914          */
1915         WARN_ON(atomic64_sub_return(i, &prog->aux->refcnt) == 0);
1916 }
1917 EXPORT_SYMBOL_GPL(bpf_prog_sub);
1918
1919 void bpf_prog_inc(struct bpf_prog *prog)
1920 {
1921         atomic64_inc(&prog->aux->refcnt);
1922 }
1923 EXPORT_SYMBOL_GPL(bpf_prog_inc);
1924
1925 /* prog_idr_lock should have been held */
1926 struct bpf_prog *bpf_prog_inc_not_zero(struct bpf_prog *prog)
1927 {
1928         int refold;
1929
1930         refold = atomic64_fetch_add_unless(&prog->aux->refcnt, 1, 0);
1931
1932         if (!refold)
1933                 return ERR_PTR(-ENOENT);
1934
1935         return prog;
1936 }
1937 EXPORT_SYMBOL_GPL(bpf_prog_inc_not_zero);
1938
1939 bool bpf_prog_get_ok(struct bpf_prog *prog,
1940                             enum bpf_prog_type *attach_type, bool attach_drv)
1941 {
1942         /* not an attachment, just a refcount inc, always allow */
1943         if (!attach_type)
1944                 return true;
1945
1946         if (prog->type != *attach_type)
1947                 return false;
1948         if (bpf_prog_is_dev_bound(prog->aux) && !attach_drv)
1949                 return false;
1950
1951         return true;
1952 }
1953
1954 static struct bpf_prog *__bpf_prog_get(u32 ufd, enum bpf_prog_type *attach_type,
1955                                        bool attach_drv)
1956 {
1957         struct fd f = fdget(ufd);
1958         struct bpf_prog *prog;
1959
1960         prog = ____bpf_prog_get(f);
1961         if (IS_ERR(prog))
1962                 return prog;
1963         if (!bpf_prog_get_ok(prog, attach_type, attach_drv)) {
1964                 prog = ERR_PTR(-EINVAL);
1965                 goto out;
1966         }
1967
1968         bpf_prog_inc(prog);
1969 out:
1970         fdput(f);
1971         return prog;
1972 }
1973
1974 struct bpf_prog *bpf_prog_get(u32 ufd)
1975 {
1976         return __bpf_prog_get(ufd, NULL, false);
1977 }
1978
1979 struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type,
1980                                        bool attach_drv)
1981 {
1982         return __bpf_prog_get(ufd, &type, attach_drv);
1983 }
1984 EXPORT_SYMBOL_GPL(bpf_prog_get_type_dev);
1985
1986 /* Initially all BPF programs could be loaded w/o specifying
1987  * expected_attach_type. Later for some of them specifying expected_attach_type
1988  * at load time became required so that program could be validated properly.
1989  * Programs of types that are allowed to be loaded both w/ and w/o (for
1990  * backward compatibility) expected_attach_type, should have the default attach
1991  * type assigned to expected_attach_type for the latter case, so that it can be
1992  * validated later at attach time.
1993  *
1994  * bpf_prog_load_fixup_attach_type() sets expected_attach_type in @attr if
1995  * prog type requires it but has some attach types that have to be backward
1996  * compatible.
1997  */
1998 static void bpf_prog_load_fixup_attach_type(union bpf_attr *attr)
1999 {
2000         switch (attr->prog_type) {
2001         case BPF_PROG_TYPE_CGROUP_SOCK:
2002                 /* Unfortunately BPF_ATTACH_TYPE_UNSPEC enumeration doesn't
2003                  * exist so checking for non-zero is the way to go here.
2004                  */
2005                 if (!attr->expected_attach_type)
2006                         attr->expected_attach_type =
2007                                 BPF_CGROUP_INET_SOCK_CREATE;
2008                 break;
2009         case BPF_PROG_TYPE_SK_REUSEPORT:
2010                 if (!attr->expected_attach_type)
2011                         attr->expected_attach_type =
2012                                 BPF_SK_REUSEPORT_SELECT;
2013                 break;
2014         }
2015 }
2016
2017 static int
2018 bpf_prog_load_check_attach(enum bpf_prog_type prog_type,
2019                            enum bpf_attach_type expected_attach_type,
2020                            struct btf *attach_btf, u32 btf_id,
2021                            struct bpf_prog *dst_prog)
2022 {
2023         if (btf_id) {
2024                 if (btf_id > BTF_MAX_TYPE)
2025                         return -EINVAL;
2026
2027                 if (!attach_btf && !dst_prog)
2028                         return -EINVAL;
2029
2030                 switch (prog_type) {
2031                 case BPF_PROG_TYPE_TRACING:
2032                 case BPF_PROG_TYPE_LSM:
2033                 case BPF_PROG_TYPE_STRUCT_OPS:
2034                 case BPF_PROG_TYPE_EXT:
2035                         break;
2036                 default:
2037                         return -EINVAL;
2038                 }
2039         }
2040
2041         if (attach_btf && (!btf_id || dst_prog))
2042                 return -EINVAL;
2043
2044         if (dst_prog && prog_type != BPF_PROG_TYPE_TRACING &&
2045             prog_type != BPF_PROG_TYPE_EXT)
2046                 return -EINVAL;
2047
2048         switch (prog_type) {
2049         case BPF_PROG_TYPE_CGROUP_SOCK:
2050                 switch (expected_attach_type) {
2051                 case BPF_CGROUP_INET_SOCK_CREATE:
2052                 case BPF_CGROUP_INET_SOCK_RELEASE:
2053                 case BPF_CGROUP_INET4_POST_BIND:
2054                 case BPF_CGROUP_INET6_POST_BIND:
2055                         return 0;
2056                 default:
2057                         return -EINVAL;
2058                 }
2059         case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
2060                 switch (expected_attach_type) {
2061                 case BPF_CGROUP_INET4_BIND:
2062                 case BPF_CGROUP_INET6_BIND:
2063                 case BPF_CGROUP_INET4_CONNECT:
2064                 case BPF_CGROUP_INET6_CONNECT:
2065                 case BPF_CGROUP_INET4_GETPEERNAME:
2066                 case BPF_CGROUP_INET6_GETPEERNAME:
2067                 case BPF_CGROUP_INET4_GETSOCKNAME:
2068                 case BPF_CGROUP_INET6_GETSOCKNAME:
2069                 case BPF_CGROUP_UDP4_SENDMSG:
2070                 case BPF_CGROUP_UDP6_SENDMSG:
2071                 case BPF_CGROUP_UDP4_RECVMSG:
2072                 case BPF_CGROUP_UDP6_RECVMSG:
2073                         return 0;
2074                 default:
2075                         return -EINVAL;
2076                 }
2077         case BPF_PROG_TYPE_CGROUP_SKB:
2078                 switch (expected_attach_type) {
2079                 case BPF_CGROUP_INET_INGRESS:
2080                 case BPF_CGROUP_INET_EGRESS:
2081                         return 0;
2082                 default:
2083                         return -EINVAL;
2084                 }
2085         case BPF_PROG_TYPE_CGROUP_SOCKOPT:
2086                 switch (expected_attach_type) {
2087                 case BPF_CGROUP_SETSOCKOPT:
2088                 case BPF_CGROUP_GETSOCKOPT:
2089                         return 0;
2090                 default:
2091                         return -EINVAL;
2092                 }
2093         case BPF_PROG_TYPE_SK_LOOKUP:
2094                 if (expected_attach_type == BPF_SK_LOOKUP)
2095                         return 0;
2096                 return -EINVAL;
2097         case BPF_PROG_TYPE_SK_REUSEPORT:
2098                 switch (expected_attach_type) {
2099                 case BPF_SK_REUSEPORT_SELECT:
2100                 case BPF_SK_REUSEPORT_SELECT_OR_MIGRATE:
2101                         return 0;
2102                 default:
2103                         return -EINVAL;
2104                 }
2105         case BPF_PROG_TYPE_SYSCALL:
2106         case BPF_PROG_TYPE_EXT:
2107                 if (expected_attach_type)
2108                         return -EINVAL;
2109                 fallthrough;
2110         default:
2111                 return 0;
2112         }
2113 }
2114
2115 static bool is_net_admin_prog_type(enum bpf_prog_type prog_type)
2116 {
2117         switch (prog_type) {
2118         case BPF_PROG_TYPE_SCHED_CLS:
2119         case BPF_PROG_TYPE_SCHED_ACT:
2120         case BPF_PROG_TYPE_XDP:
2121         case BPF_PROG_TYPE_LWT_IN:
2122         case BPF_PROG_TYPE_LWT_OUT:
2123         case BPF_PROG_TYPE_LWT_XMIT:
2124         case BPF_PROG_TYPE_LWT_SEG6LOCAL:
2125         case BPF_PROG_TYPE_SK_SKB:
2126         case BPF_PROG_TYPE_SK_MSG:
2127         case BPF_PROG_TYPE_LIRC_MODE2:
2128         case BPF_PROG_TYPE_FLOW_DISSECTOR:
2129         case BPF_PROG_TYPE_CGROUP_DEVICE:
2130         case BPF_PROG_TYPE_CGROUP_SOCK:
2131         case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
2132         case BPF_PROG_TYPE_CGROUP_SOCKOPT:
2133         case BPF_PROG_TYPE_CGROUP_SYSCTL:
2134         case BPF_PROG_TYPE_SOCK_OPS:
2135         case BPF_PROG_TYPE_EXT: /* extends any prog */
2136                 return true;
2137         case BPF_PROG_TYPE_CGROUP_SKB:
2138                 /* always unpriv */
2139         case BPF_PROG_TYPE_SK_REUSEPORT:
2140                 /* equivalent to SOCKET_FILTER. need CAP_BPF only */
2141         default:
2142                 return false;
2143         }
2144 }
2145
2146 static bool is_perfmon_prog_type(enum bpf_prog_type prog_type)
2147 {
2148         switch (prog_type) {
2149         case BPF_PROG_TYPE_KPROBE:
2150         case BPF_PROG_TYPE_TRACEPOINT:
2151         case BPF_PROG_TYPE_PERF_EVENT:
2152         case BPF_PROG_TYPE_RAW_TRACEPOINT:
2153         case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE:
2154         case BPF_PROG_TYPE_TRACING:
2155         case BPF_PROG_TYPE_LSM:
2156         case BPF_PROG_TYPE_STRUCT_OPS: /* has access to struct sock */
2157         case BPF_PROG_TYPE_EXT: /* extends any prog */
2158                 return true;
2159         default:
2160                 return false;
2161         }
2162 }
2163
2164 /* last field in 'union bpf_attr' used by this command */
2165 #define BPF_PROG_LOAD_LAST_FIELD fd_array
2166
2167 static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr)
2168 {
2169         enum bpf_prog_type type = attr->prog_type;
2170         struct bpf_prog *prog, *dst_prog = NULL;
2171         struct btf *attach_btf = NULL;
2172         int err;
2173         char license[128];
2174         bool is_gpl;
2175
2176         if (CHECK_ATTR(BPF_PROG_LOAD))
2177                 return -EINVAL;
2178
2179         if (attr->prog_flags & ~(BPF_F_STRICT_ALIGNMENT |
2180                                  BPF_F_ANY_ALIGNMENT |
2181                                  BPF_F_TEST_STATE_FREQ |
2182                                  BPF_F_SLEEPABLE |
2183                                  BPF_F_TEST_RND_HI32))
2184                 return -EINVAL;
2185
2186         if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) &&
2187             (attr->prog_flags & BPF_F_ANY_ALIGNMENT) &&
2188             !bpf_capable())
2189                 return -EPERM;
2190
2191         /* copy eBPF program license from user space */
2192         if (strncpy_from_bpfptr(license,
2193                                 make_bpfptr(attr->license, uattr.is_kernel),
2194                                 sizeof(license) - 1) < 0)
2195                 return -EFAULT;
2196         license[sizeof(license) - 1] = 0;
2197
2198         /* eBPF programs must be GPL compatible to use GPL-ed functions */
2199         is_gpl = license_is_gpl_compatible(license);
2200
2201         if (attr->insn_cnt == 0 ||
2202             attr->insn_cnt > (bpf_capable() ? BPF_COMPLEXITY_LIMIT_INSNS : BPF_MAXINSNS))
2203                 return -E2BIG;
2204         if (type != BPF_PROG_TYPE_SOCKET_FILTER &&
2205             type != BPF_PROG_TYPE_CGROUP_SKB &&
2206             !bpf_capable())
2207                 return -EPERM;
2208
2209         if (is_net_admin_prog_type(type) && !capable(CAP_NET_ADMIN) && !capable(CAP_SYS_ADMIN))
2210                 return -EPERM;
2211         if (is_perfmon_prog_type(type) && !perfmon_capable())
2212                 return -EPERM;
2213
2214         /* attach_prog_fd/attach_btf_obj_fd can specify fd of either bpf_prog
2215          * or btf, we need to check which one it is
2216          */
2217         if (attr->attach_prog_fd) {
2218                 dst_prog = bpf_prog_get(attr->attach_prog_fd);
2219                 if (IS_ERR(dst_prog)) {
2220                         dst_prog = NULL;
2221                         attach_btf = btf_get_by_fd(attr->attach_btf_obj_fd);
2222                         if (IS_ERR(attach_btf))
2223                                 return -EINVAL;
2224                         if (!btf_is_kernel(attach_btf)) {
2225                                 /* attaching through specifying bpf_prog's BTF
2226                                  * objects directly might be supported eventually
2227                                  */
2228                                 btf_put(attach_btf);
2229                                 return -ENOTSUPP;
2230                         }
2231                 }
2232         } else if (attr->attach_btf_id) {
2233                 /* fall back to vmlinux BTF, if BTF type ID is specified */
2234                 attach_btf = bpf_get_btf_vmlinux();
2235                 if (IS_ERR(attach_btf))
2236                         return PTR_ERR(attach_btf);
2237                 if (!attach_btf)
2238                         return -EINVAL;
2239                 btf_get(attach_btf);
2240         }
2241
2242         bpf_prog_load_fixup_attach_type(attr);
2243         if (bpf_prog_load_check_attach(type, attr->expected_attach_type,
2244                                        attach_btf, attr->attach_btf_id,
2245                                        dst_prog)) {
2246                 if (dst_prog)
2247                         bpf_prog_put(dst_prog);
2248                 if (attach_btf)
2249                         btf_put(attach_btf);
2250                 return -EINVAL;
2251         }
2252
2253         /* plain bpf_prog allocation */
2254         prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER);
2255         if (!prog) {
2256                 if (dst_prog)
2257                         bpf_prog_put(dst_prog);
2258                 if (attach_btf)
2259                         btf_put(attach_btf);
2260                 return -ENOMEM;
2261         }
2262
2263         prog->expected_attach_type = attr->expected_attach_type;
2264         prog->aux->attach_btf = attach_btf;
2265         prog->aux->attach_btf_id = attr->attach_btf_id;
2266         prog->aux->dst_prog = dst_prog;
2267         prog->aux->offload_requested = !!attr->prog_ifindex;
2268         prog->aux->sleepable = attr->prog_flags & BPF_F_SLEEPABLE;
2269
2270         err = security_bpf_prog_alloc(prog->aux);
2271         if (err)
2272                 goto free_prog;
2273
2274         prog->aux->user = get_current_user();
2275         prog->len = attr->insn_cnt;
2276
2277         err = -EFAULT;
2278         if (copy_from_bpfptr(prog->insns,
2279                              make_bpfptr(attr->insns, uattr.is_kernel),
2280                              bpf_prog_insn_size(prog)) != 0)
2281                 goto free_prog_sec;
2282
2283         prog->orig_prog = NULL;
2284         prog->jited = 0;
2285
2286         atomic64_set(&prog->aux->refcnt, 1);
2287         prog->gpl_compatible = is_gpl ? 1 : 0;
2288
2289         if (bpf_prog_is_dev_bound(prog->aux)) {
2290                 err = bpf_prog_offload_init(prog, attr);
2291                 if (err)
2292                         goto free_prog_sec;
2293         }
2294
2295         /* find program type: socket_filter vs tracing_filter */
2296         err = find_prog_type(type, prog);
2297         if (err < 0)
2298                 goto free_prog_sec;
2299
2300         prog->aux->load_time = ktime_get_boottime_ns();
2301         err = bpf_obj_name_cpy(prog->aux->name, attr->prog_name,
2302                                sizeof(attr->prog_name));
2303         if (err < 0)
2304                 goto free_prog_sec;
2305
2306         /* run eBPF verifier */
2307         err = bpf_check(&prog, attr, uattr);
2308         if (err < 0)
2309                 goto free_used_maps;
2310
2311         prog = bpf_prog_select_runtime(prog, &err);
2312         if (err < 0)
2313                 goto free_used_maps;
2314
2315         err = bpf_prog_alloc_id(prog);
2316         if (err)
2317                 goto free_used_maps;
2318
2319         /* Upon success of bpf_prog_alloc_id(), the BPF prog is
2320          * effectively publicly exposed. However, retrieving via
2321          * bpf_prog_get_fd_by_id() will take another reference,
2322          * therefore it cannot be gone underneath us.
2323          *
2324          * Only for the time /after/ successful bpf_prog_new_fd()
2325          * and before returning to userspace, we might just hold
2326          * one reference and any parallel close on that fd could
2327          * rip everything out. Hence, below notifications must
2328          * happen before bpf_prog_new_fd().
2329          *
2330          * Also, any failure handling from this point onwards must
2331          * be using bpf_prog_put() given the program is exposed.
2332          */
2333         bpf_prog_kallsyms_add(prog);
2334         perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_LOAD, 0);
2335         bpf_audit_prog(prog, BPF_AUDIT_LOAD);
2336
2337         err = bpf_prog_new_fd(prog);
2338         if (err < 0)
2339                 bpf_prog_put(prog);
2340         return err;
2341
2342 free_used_maps:
2343         /* In case we have subprogs, we need to wait for a grace
2344          * period before we can tear down JIT memory since symbols
2345          * are already exposed under kallsyms.
2346          */
2347         __bpf_prog_put_noref(prog, prog->aux->func_cnt);
2348         return err;
2349 free_prog_sec:
2350         free_uid(prog->aux->user);
2351         security_bpf_prog_free(prog->aux);
2352 free_prog:
2353         if (prog->aux->attach_btf)
2354                 btf_put(prog->aux->attach_btf);
2355         bpf_prog_free(prog);
2356         return err;
2357 }
2358
2359 #define BPF_OBJ_LAST_FIELD file_flags
2360
2361 static int bpf_obj_pin(const union bpf_attr *attr)
2362 {
2363         if (CHECK_ATTR(BPF_OBJ) || attr->file_flags != 0)
2364                 return -EINVAL;
2365
2366         return bpf_obj_pin_user(attr->bpf_fd, u64_to_user_ptr(attr->pathname));
2367 }
2368
2369 static int bpf_obj_get(const union bpf_attr *attr)
2370 {
2371         if (CHECK_ATTR(BPF_OBJ) || attr->bpf_fd != 0 ||
2372             attr->file_flags & ~BPF_OBJ_FLAG_MASK)
2373                 return -EINVAL;
2374
2375         return bpf_obj_get_user(u64_to_user_ptr(attr->pathname),
2376                                 attr->file_flags);
2377 }
2378
2379 void bpf_link_init(struct bpf_link *link, enum bpf_link_type type,
2380                    const struct bpf_link_ops *ops, struct bpf_prog *prog)
2381 {
2382         atomic64_set(&link->refcnt, 1);
2383         link->type = type;
2384         link->id = 0;
2385         link->ops = ops;
2386         link->prog = prog;
2387 }
2388
2389 static void bpf_link_free_id(int id)
2390 {
2391         if (!id)
2392                 return;
2393
2394         spin_lock_bh(&link_idr_lock);
2395         idr_remove(&link_idr, id);
2396         spin_unlock_bh(&link_idr_lock);
2397 }
2398
2399 /* Clean up bpf_link and corresponding anon_inode file and FD. After
2400  * anon_inode is created, bpf_link can't be just kfree()'d due to deferred
2401  * anon_inode's release() call. This helper marksbpf_link as
2402  * defunct, releases anon_inode file and puts reserved FD. bpf_prog's refcnt
2403  * is not decremented, it's the responsibility of a calling code that failed
2404  * to complete bpf_link initialization.
2405  */
2406 void bpf_link_cleanup(struct bpf_link_primer *primer)
2407 {
2408         primer->link->prog = NULL;
2409         bpf_link_free_id(primer->id);
2410         fput(primer->file);
2411         put_unused_fd(primer->fd);
2412 }
2413
2414 void bpf_link_inc(struct bpf_link *link)
2415 {
2416         atomic64_inc(&link->refcnt);
2417 }
2418
2419 /* bpf_link_free is guaranteed to be called from process context */
2420 static void bpf_link_free(struct bpf_link *link)
2421 {
2422         bpf_link_free_id(link->id);
2423         if (link->prog) {
2424                 /* detach BPF program, clean up used resources */
2425                 link->ops->release(link);
2426                 bpf_prog_put(link->prog);
2427         }
2428         /* free bpf_link and its containing memory */
2429         link->ops->dealloc(link);
2430 }
2431
2432 static void bpf_link_put_deferred(struct work_struct *work)
2433 {
2434         struct bpf_link *link = container_of(work, struct bpf_link, work);
2435
2436         bpf_link_free(link);
2437 }
2438
2439 /* bpf_link_put can be called from atomic context, but ensures that resources
2440  * are freed from process context
2441  */
2442 void bpf_link_put(struct bpf_link *link)
2443 {
2444         if (!atomic64_dec_and_test(&link->refcnt))
2445                 return;
2446
2447         if (in_atomic()) {
2448                 INIT_WORK(&link->work, bpf_link_put_deferred);
2449                 schedule_work(&link->work);
2450         } else {
2451                 bpf_link_free(link);
2452         }
2453 }
2454
2455 static int bpf_link_release(struct inode *inode, struct file *filp)
2456 {
2457         struct bpf_link *link = filp->private_data;
2458
2459         bpf_link_put(link);
2460         return 0;
2461 }
2462
2463 #ifdef CONFIG_PROC_FS
2464 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type)
2465 #define BPF_MAP_TYPE(_id, _ops)
2466 #define BPF_LINK_TYPE(_id, _name) [_id] = #_name,
2467 static const char *bpf_link_type_strs[] = {
2468         [BPF_LINK_TYPE_UNSPEC] = "<invalid>",
2469 #include <linux/bpf_types.h>
2470 };
2471 #undef BPF_PROG_TYPE
2472 #undef BPF_MAP_TYPE
2473 #undef BPF_LINK_TYPE
2474
2475 static void bpf_link_show_fdinfo(struct seq_file *m, struct file *filp)
2476 {
2477         const struct bpf_link *link = filp->private_data;
2478         const struct bpf_prog *prog = link->prog;
2479         char prog_tag[sizeof(prog->tag) * 2 + 1] = { };
2480
2481         bin2hex(prog_tag, prog->tag, sizeof(prog->tag));
2482         seq_printf(m,
2483                    "link_type:\t%s\n"
2484                    "link_id:\t%u\n"
2485                    "prog_tag:\t%s\n"
2486                    "prog_id:\t%u\n",
2487                    bpf_link_type_strs[link->type],
2488                    link->id,
2489                    prog_tag,
2490                    prog->aux->id);
2491         if (link->ops->show_fdinfo)
2492                 link->ops->show_fdinfo(link, m);
2493 }
2494 #endif
2495
2496 static const struct file_operations bpf_link_fops = {
2497 #ifdef CONFIG_PROC_FS
2498         .show_fdinfo    = bpf_link_show_fdinfo,
2499 #endif
2500         .release        = bpf_link_release,
2501         .read           = bpf_dummy_read,
2502         .write          = bpf_dummy_write,
2503 };
2504
2505 static int bpf_link_alloc_id(struct bpf_link *link)
2506 {
2507         int id;
2508
2509         idr_preload(GFP_KERNEL);
2510         spin_lock_bh(&link_idr_lock);
2511         id = idr_alloc_cyclic(&link_idr, link, 1, INT_MAX, GFP_ATOMIC);
2512         spin_unlock_bh(&link_idr_lock);
2513         idr_preload_end();
2514
2515         return id;
2516 }
2517
2518 /* Prepare bpf_link to be exposed to user-space by allocating anon_inode file,
2519  * reserving unused FD and allocating ID from link_idr. This is to be paired
2520  * with bpf_link_settle() to install FD and ID and expose bpf_link to
2521  * user-space, if bpf_link is successfully attached. If not, bpf_link and
2522  * pre-allocated resources are to be freed with bpf_cleanup() call. All the
2523  * transient state is passed around in struct bpf_link_primer.
2524  * This is preferred way to create and initialize bpf_link, especially when
2525  * there are complicated and expensive operations inbetween creating bpf_link
2526  * itself and attaching it to BPF hook. By using bpf_link_prime() and
2527  * bpf_link_settle() kernel code using bpf_link doesn't have to perform
2528  * expensive (and potentially failing) roll back operations in a rare case
2529  * that file, FD, or ID can't be allocated.
2530  */
2531 int bpf_link_prime(struct bpf_link *link, struct bpf_link_primer *primer)
2532 {
2533         struct file *file;
2534         int fd, id;
2535
2536         fd = get_unused_fd_flags(O_CLOEXEC);
2537         if (fd < 0)
2538                 return fd;
2539
2540
2541         id = bpf_link_alloc_id(link);
2542         if (id < 0) {
2543                 put_unused_fd(fd);
2544                 return id;
2545         }
2546
2547         file = anon_inode_getfile("bpf_link", &bpf_link_fops, link, O_CLOEXEC);
2548         if (IS_ERR(file)) {
2549                 bpf_link_free_id(id);
2550                 put_unused_fd(fd);
2551                 return PTR_ERR(file);
2552         }
2553
2554         primer->link = link;
2555         primer->file = file;
2556         primer->fd = fd;
2557         primer->id = id;
2558         return 0;
2559 }
2560
2561 int bpf_link_settle(struct bpf_link_primer *primer)
2562 {
2563         /* make bpf_link fetchable by ID */
2564         spin_lock_bh(&link_idr_lock);
2565         primer->link->id = primer->id;
2566         spin_unlock_bh(&link_idr_lock);
2567         /* make bpf_link fetchable by FD */
2568         fd_install(primer->fd, primer->file);
2569         /* pass through installed FD */
2570         return primer->fd;
2571 }
2572
2573 int bpf_link_new_fd(struct bpf_link *link)
2574 {
2575         return anon_inode_getfd("bpf-link", &bpf_link_fops, link, O_CLOEXEC);
2576 }
2577
2578 struct bpf_link *bpf_link_get_from_fd(u32 ufd)
2579 {
2580         struct fd f = fdget(ufd);
2581         struct bpf_link *link;
2582
2583         if (!f.file)
2584                 return ERR_PTR(-EBADF);
2585         if (f.file->f_op != &bpf_link_fops) {
2586                 fdput(f);
2587                 return ERR_PTR(-EINVAL);
2588         }
2589
2590         link = f.file->private_data;
2591         bpf_link_inc(link);
2592         fdput(f);
2593
2594         return link;
2595 }
2596
2597 struct bpf_tracing_link {
2598         struct bpf_link link;
2599         enum bpf_attach_type attach_type;
2600         struct bpf_trampoline *trampoline;
2601         struct bpf_prog *tgt_prog;
2602 };
2603
2604 static void bpf_tracing_link_release(struct bpf_link *link)
2605 {
2606         struct bpf_tracing_link *tr_link =
2607                 container_of(link, struct bpf_tracing_link, link);
2608
2609         WARN_ON_ONCE(bpf_trampoline_unlink_prog(link->prog,
2610                                                 tr_link->trampoline));
2611
2612         bpf_trampoline_put(tr_link->trampoline);
2613
2614         /* tgt_prog is NULL if target is a kernel function */
2615         if (tr_link->tgt_prog)
2616                 bpf_prog_put(tr_link->tgt_prog);
2617 }
2618
2619 static void bpf_tracing_link_dealloc(struct bpf_link *link)
2620 {
2621         struct bpf_tracing_link *tr_link =
2622                 container_of(link, struct bpf_tracing_link, link);
2623
2624         kfree(tr_link);
2625 }
2626
2627 static void bpf_tracing_link_show_fdinfo(const struct bpf_link *link,
2628                                          struct seq_file *seq)
2629 {
2630         struct bpf_tracing_link *tr_link =
2631                 container_of(link, struct bpf_tracing_link, link);
2632
2633         seq_printf(seq,
2634                    "attach_type:\t%d\n",
2635                    tr_link->attach_type);
2636 }
2637
2638 static int bpf_tracing_link_fill_link_info(const struct bpf_link *link,
2639                                            struct bpf_link_info *info)
2640 {
2641         struct bpf_tracing_link *tr_link =
2642                 container_of(link, struct bpf_tracing_link, link);
2643
2644         info->tracing.attach_type = tr_link->attach_type;
2645         bpf_trampoline_unpack_key(tr_link->trampoline->key,
2646                                   &info->tracing.target_obj_id,
2647                                   &info->tracing.target_btf_id);
2648
2649         return 0;
2650 }
2651
2652 static const struct bpf_link_ops bpf_tracing_link_lops = {
2653         .release = bpf_tracing_link_release,
2654         .dealloc = bpf_tracing_link_dealloc,
2655         .show_fdinfo = bpf_tracing_link_show_fdinfo,
2656         .fill_link_info = bpf_tracing_link_fill_link_info,
2657 };
2658
2659 static int bpf_tracing_prog_attach(struct bpf_prog *prog,
2660                                    int tgt_prog_fd,
2661                                    u32 btf_id)
2662 {
2663         struct bpf_link_primer link_primer;
2664         struct bpf_prog *tgt_prog = NULL;
2665         struct bpf_trampoline *tr = NULL;
2666         struct bpf_tracing_link *link;
2667         u64 key = 0;
2668         int err;
2669
2670         switch (prog->type) {
2671         case BPF_PROG_TYPE_TRACING:
2672                 if (prog->expected_attach_type != BPF_TRACE_FENTRY &&
2673                     prog->expected_attach_type != BPF_TRACE_FEXIT &&
2674                     prog->expected_attach_type != BPF_MODIFY_RETURN) {
2675                         err = -EINVAL;
2676                         goto out_put_prog;
2677                 }
2678                 break;
2679         case BPF_PROG_TYPE_EXT:
2680                 if (prog->expected_attach_type != 0) {
2681                         err = -EINVAL;
2682                         goto out_put_prog;
2683                 }
2684                 break;
2685         case BPF_PROG_TYPE_LSM:
2686                 if (prog->expected_attach_type != BPF_LSM_MAC) {
2687                         err = -EINVAL;
2688                         goto out_put_prog;
2689                 }
2690                 break;
2691         default:
2692                 err = -EINVAL;
2693                 goto out_put_prog;
2694         }
2695
2696         if (!!tgt_prog_fd != !!btf_id) {
2697                 err = -EINVAL;
2698                 goto out_put_prog;
2699         }
2700
2701         if (tgt_prog_fd) {
2702                 /* For now we only allow new targets for BPF_PROG_TYPE_EXT */
2703                 if (prog->type != BPF_PROG_TYPE_EXT) {
2704                         err = -EINVAL;
2705                         goto out_put_prog;
2706                 }
2707
2708                 tgt_prog = bpf_prog_get(tgt_prog_fd);
2709                 if (IS_ERR(tgt_prog)) {
2710                         err = PTR_ERR(tgt_prog);
2711                         tgt_prog = NULL;
2712                         goto out_put_prog;
2713                 }
2714
2715                 key = bpf_trampoline_compute_key(tgt_prog, NULL, btf_id);
2716         }
2717
2718         link = kzalloc(sizeof(*link), GFP_USER);
2719         if (!link) {
2720                 err = -ENOMEM;
2721                 goto out_put_prog;
2722         }
2723         bpf_link_init(&link->link, BPF_LINK_TYPE_TRACING,
2724                       &bpf_tracing_link_lops, prog);
2725         link->attach_type = prog->expected_attach_type;
2726
2727         mutex_lock(&prog->aux->dst_mutex);
2728
2729         /* There are a few possible cases here:
2730          *
2731          * - if prog->aux->dst_trampoline is set, the program was just loaded
2732          *   and not yet attached to anything, so we can use the values stored
2733          *   in prog->aux
2734          *
2735          * - if prog->aux->dst_trampoline is NULL, the program has already been
2736          *   attached to a target and its initial target was cleared (below)
2737          *
2738          * - if tgt_prog != NULL, the caller specified tgt_prog_fd +
2739          *   target_btf_id using the link_create API.
2740          *
2741          * - if tgt_prog == NULL when this function was called using the old
2742          *   raw_tracepoint_open API, and we need a target from prog->aux
2743          *
2744          * - if prog->aux->dst_trampoline and tgt_prog is NULL, the program
2745          *   was detached and is going for re-attachment.
2746          */
2747         if (!prog->aux->dst_trampoline && !tgt_prog) {
2748                 /*
2749                  * Allow re-attach for TRACING and LSM programs. If it's
2750                  * currently linked, bpf_trampoline_link_prog will fail.
2751                  * EXT programs need to specify tgt_prog_fd, so they
2752                  * re-attach in separate code path.
2753                  */
2754                 if (prog->type != BPF_PROG_TYPE_TRACING &&
2755                     prog->type != BPF_PROG_TYPE_LSM) {
2756                         err = -EINVAL;
2757                         goto out_unlock;
2758                 }
2759                 btf_id = prog->aux->attach_btf_id;
2760                 key = bpf_trampoline_compute_key(NULL, prog->aux->attach_btf, btf_id);
2761         }
2762
2763         if (!prog->aux->dst_trampoline ||
2764             (key && key != prog->aux->dst_trampoline->key)) {
2765                 /* If there is no saved target, or the specified target is
2766                  * different from the destination specified at load time, we
2767                  * need a new trampoline and a check for compatibility
2768                  */
2769                 struct bpf_attach_target_info tgt_info = {};
2770
2771                 err = bpf_check_attach_target(NULL, prog, tgt_prog, btf_id,
2772                                               &tgt_info);
2773                 if (err)
2774                         goto out_unlock;
2775
2776                 tr = bpf_trampoline_get(key, &tgt_info);
2777                 if (!tr) {
2778                         err = -ENOMEM;
2779                         goto out_unlock;
2780                 }
2781         } else {
2782                 /* The caller didn't specify a target, or the target was the
2783                  * same as the destination supplied during program load. This
2784                  * means we can reuse the trampoline and reference from program
2785                  * load time, and there is no need to allocate a new one. This
2786                  * can only happen once for any program, as the saved values in
2787                  * prog->aux are cleared below.
2788                  */
2789                 tr = prog->aux->dst_trampoline;
2790                 tgt_prog = prog->aux->dst_prog;
2791         }
2792
2793         err = bpf_link_prime(&link->link, &link_primer);
2794         if (err)
2795                 goto out_unlock;
2796
2797         err = bpf_trampoline_link_prog(prog, tr);
2798         if (err) {
2799                 bpf_link_cleanup(&link_primer);
2800                 link = NULL;
2801                 goto out_unlock;
2802         }
2803
2804         link->tgt_prog = tgt_prog;
2805         link->trampoline = tr;
2806
2807         /* Always clear the trampoline and target prog from prog->aux to make
2808          * sure the original attach destination is not kept alive after a
2809          * program is (re-)attached to another target.
2810          */
2811         if (prog->aux->dst_prog &&
2812             (tgt_prog_fd || tr != prog->aux->dst_trampoline))
2813                 /* got extra prog ref from syscall, or attaching to different prog */
2814                 bpf_prog_put(prog->aux->dst_prog);
2815         if (prog->aux->dst_trampoline && tr != prog->aux->dst_trampoline)
2816                 /* we allocated a new trampoline, so free the old one */
2817                 bpf_trampoline_put(prog->aux->dst_trampoline);
2818
2819         prog->aux->dst_prog = NULL;
2820         prog->aux->dst_trampoline = NULL;
2821         mutex_unlock(&prog->aux->dst_mutex);
2822
2823         return bpf_link_settle(&link_primer);
2824 out_unlock:
2825         if (tr && tr != prog->aux->dst_trampoline)
2826                 bpf_trampoline_put(tr);
2827         mutex_unlock(&prog->aux->dst_mutex);
2828         kfree(link);
2829 out_put_prog:
2830         if (tgt_prog_fd && tgt_prog)
2831                 bpf_prog_put(tgt_prog);
2832         return err;
2833 }
2834
2835 struct bpf_raw_tp_link {
2836         struct bpf_link link;
2837         struct bpf_raw_event_map *btp;
2838 };
2839
2840 static void bpf_raw_tp_link_release(struct bpf_link *link)
2841 {
2842         struct bpf_raw_tp_link *raw_tp =
2843                 container_of(link, struct bpf_raw_tp_link, link);
2844
2845         bpf_probe_unregister(raw_tp->btp, raw_tp->link.prog);
2846         bpf_put_raw_tracepoint(raw_tp->btp);
2847 }
2848
2849 static void bpf_raw_tp_link_dealloc(struct bpf_link *link)
2850 {
2851         struct bpf_raw_tp_link *raw_tp =
2852                 container_of(link, struct bpf_raw_tp_link, link);
2853
2854         kfree(raw_tp);
2855 }
2856
2857 static void bpf_raw_tp_link_show_fdinfo(const struct bpf_link *link,
2858                                         struct seq_file *seq)
2859 {
2860         struct bpf_raw_tp_link *raw_tp_link =
2861                 container_of(link, struct bpf_raw_tp_link, link);
2862
2863         seq_printf(seq,
2864                    "tp_name:\t%s\n",
2865                    raw_tp_link->btp->tp->name);
2866 }
2867
2868 static int bpf_raw_tp_link_fill_link_info(const struct bpf_link *link,
2869                                           struct bpf_link_info *info)
2870 {
2871         struct bpf_raw_tp_link *raw_tp_link =
2872                 container_of(link, struct bpf_raw_tp_link, link);
2873         char __user *ubuf = u64_to_user_ptr(info->raw_tracepoint.tp_name);
2874         const char *tp_name = raw_tp_link->btp->tp->name;
2875         u32 ulen = info->raw_tracepoint.tp_name_len;
2876         size_t tp_len = strlen(tp_name);
2877
2878         if (!ulen ^ !ubuf)
2879                 return -EINVAL;
2880
2881         info->raw_tracepoint.tp_name_len = tp_len + 1;
2882
2883         if (!ubuf)
2884                 return 0;
2885
2886         if (ulen >= tp_len + 1) {
2887                 if (copy_to_user(ubuf, tp_name, tp_len + 1))
2888                         return -EFAULT;
2889         } else {
2890                 char zero = '\0';
2891
2892                 if (copy_to_user(ubuf, tp_name, ulen - 1))
2893                         return -EFAULT;
2894                 if (put_user(zero, ubuf + ulen - 1))
2895                         return -EFAULT;
2896                 return -ENOSPC;
2897         }
2898
2899         return 0;
2900 }
2901
2902 static const struct bpf_link_ops bpf_raw_tp_link_lops = {
2903         .release = bpf_raw_tp_link_release,
2904         .dealloc = bpf_raw_tp_link_dealloc,
2905         .show_fdinfo = bpf_raw_tp_link_show_fdinfo,
2906         .fill_link_info = bpf_raw_tp_link_fill_link_info,
2907 };
2908
2909 #ifdef CONFIG_PERF_EVENTS
2910 struct bpf_perf_link {
2911         struct bpf_link link;
2912         struct file *perf_file;
2913 };
2914
2915 static void bpf_perf_link_release(struct bpf_link *link)
2916 {
2917         struct bpf_perf_link *perf_link = container_of(link, struct bpf_perf_link, link);
2918         struct perf_event *event = perf_link->perf_file->private_data;
2919
2920         perf_event_free_bpf_prog(event);
2921         fput(perf_link->perf_file);
2922 }
2923
2924 static void bpf_perf_link_dealloc(struct bpf_link *link)
2925 {
2926         struct bpf_perf_link *perf_link = container_of(link, struct bpf_perf_link, link);
2927
2928         kfree(perf_link);
2929 }
2930
2931 static const struct bpf_link_ops bpf_perf_link_lops = {
2932         .release = bpf_perf_link_release,
2933         .dealloc = bpf_perf_link_dealloc,
2934 };
2935
2936 static int bpf_perf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
2937 {
2938         struct bpf_link_primer link_primer;
2939         struct bpf_perf_link *link;
2940         struct perf_event *event;
2941         struct file *perf_file;
2942         int err;
2943
2944         if (attr->link_create.flags)
2945                 return -EINVAL;
2946
2947         perf_file = perf_event_get(attr->link_create.target_fd);
2948         if (IS_ERR(perf_file))
2949                 return PTR_ERR(perf_file);
2950
2951         link = kzalloc(sizeof(*link), GFP_USER);
2952         if (!link) {
2953                 err = -ENOMEM;
2954                 goto out_put_file;
2955         }
2956         bpf_link_init(&link->link, BPF_LINK_TYPE_PERF_EVENT, &bpf_perf_link_lops, prog);
2957         link->perf_file = perf_file;
2958
2959         err = bpf_link_prime(&link->link, &link_primer);
2960         if (err) {
2961                 kfree(link);
2962                 goto out_put_file;
2963         }
2964
2965         event = perf_file->private_data;
2966         err = perf_event_set_bpf_prog(event, prog);
2967         if (err) {
2968                 bpf_link_cleanup(&link_primer);
2969                 goto out_put_file;
2970         }
2971         /* perf_event_set_bpf_prog() doesn't take its own refcnt on prog */
2972         bpf_prog_inc(prog);
2973
2974         return bpf_link_settle(&link_primer);
2975
2976 out_put_file:
2977         fput(perf_file);
2978         return err;
2979 }
2980 #endif /* CONFIG_PERF_EVENTS */
2981
2982 #define BPF_RAW_TRACEPOINT_OPEN_LAST_FIELD raw_tracepoint.prog_fd
2983
2984 static int bpf_raw_tracepoint_open(const union bpf_attr *attr)
2985 {
2986         struct bpf_link_primer link_primer;
2987         struct bpf_raw_tp_link *link;
2988         struct bpf_raw_event_map *btp;
2989         struct bpf_prog *prog;
2990         const char *tp_name;
2991         char buf[128];
2992         int err;
2993
2994         if (CHECK_ATTR(BPF_RAW_TRACEPOINT_OPEN))
2995                 return -EINVAL;
2996
2997         prog = bpf_prog_get(attr->raw_tracepoint.prog_fd);
2998         if (IS_ERR(prog))
2999                 return PTR_ERR(prog);
3000
3001         switch (prog->type) {
3002         case BPF_PROG_TYPE_TRACING:
3003         case BPF_PROG_TYPE_EXT:
3004         case BPF_PROG_TYPE_LSM:
3005                 if (attr->raw_tracepoint.name) {
3006                         /* The attach point for this category of programs
3007                          * should be specified via btf_id during program load.
3008                          */
3009                         err = -EINVAL;
3010                         goto out_put_prog;
3011                 }
3012                 if (prog->type == BPF_PROG_TYPE_TRACING &&
3013                     prog->expected_attach_type == BPF_TRACE_RAW_TP) {
3014                         tp_name = prog->aux->attach_func_name;
3015                         break;
3016                 }
3017                 err = bpf_tracing_prog_attach(prog, 0, 0);
3018                 if (err >= 0)
3019                         return err;
3020                 goto out_put_prog;
3021         case BPF_PROG_TYPE_RAW_TRACEPOINT:
3022         case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE:
3023                 if (strncpy_from_user(buf,
3024                                       u64_to_user_ptr(attr->raw_tracepoint.name),
3025                                       sizeof(buf) - 1) < 0) {
3026                         err = -EFAULT;
3027                         goto out_put_prog;
3028                 }
3029                 buf[sizeof(buf) - 1] = 0;
3030                 tp_name = buf;
3031                 break;
3032         default:
3033                 err = -EINVAL;
3034                 goto out_put_prog;
3035         }
3036
3037         btp = bpf_get_raw_tracepoint(tp_name);
3038         if (!btp) {
3039                 err = -ENOENT;
3040                 goto out_put_prog;
3041         }
3042
3043         link = kzalloc(sizeof(*link), GFP_USER);
3044         if (!link) {
3045                 err = -ENOMEM;
3046                 goto out_put_btp;
3047         }
3048         bpf_link_init(&link->link, BPF_LINK_TYPE_RAW_TRACEPOINT,
3049                       &bpf_raw_tp_link_lops, prog);
3050         link->btp = btp;
3051
3052         err = bpf_link_prime(&link->link, &link_primer);
3053         if (err) {
3054                 kfree(link);
3055                 goto out_put_btp;
3056         }
3057
3058         err = bpf_probe_register(link->btp, prog);
3059         if (err) {
3060                 bpf_link_cleanup(&link_primer);
3061                 goto out_put_btp;
3062         }
3063
3064         return bpf_link_settle(&link_primer);
3065
3066 out_put_btp:
3067         bpf_put_raw_tracepoint(btp);
3068 out_put_prog:
3069         bpf_prog_put(prog);
3070         return err;
3071 }
3072
3073 static int bpf_prog_attach_check_attach_type(const struct bpf_prog *prog,
3074                                              enum bpf_attach_type attach_type)
3075 {
3076         switch (prog->type) {
3077         case BPF_PROG_TYPE_CGROUP_SOCK:
3078         case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
3079         case BPF_PROG_TYPE_CGROUP_SOCKOPT:
3080         case BPF_PROG_TYPE_SK_LOOKUP:
3081                 return attach_type == prog->expected_attach_type ? 0 : -EINVAL;
3082         case BPF_PROG_TYPE_CGROUP_SKB:
3083                 if (!capable(CAP_NET_ADMIN))
3084                         /* cg-skb progs can be loaded by unpriv user.
3085                          * check permissions at attach time.
3086                          */
3087                         return -EPERM;
3088                 return prog->enforce_expected_attach_type &&
3089                         prog->expected_attach_type != attach_type ?
3090                         -EINVAL : 0;
3091         default:
3092                 return 0;
3093         }
3094 }
3095
3096 static enum bpf_prog_type
3097 attach_type_to_prog_type(enum bpf_attach_type attach_type)
3098 {
3099         switch (attach_type) {
3100         case BPF_CGROUP_INET_INGRESS:
3101         case BPF_CGROUP_INET_EGRESS:
3102                 return BPF_PROG_TYPE_CGROUP_SKB;
3103         case BPF_CGROUP_INET_SOCK_CREATE:
3104         case BPF_CGROUP_INET_SOCK_RELEASE:
3105         case BPF_CGROUP_INET4_POST_BIND:
3106         case BPF_CGROUP_INET6_POST_BIND:
3107                 return BPF_PROG_TYPE_CGROUP_SOCK;
3108         case BPF_CGROUP_INET4_BIND:
3109         case BPF_CGROUP_INET6_BIND:
3110         case BPF_CGROUP_INET4_CONNECT:
3111         case BPF_CGROUP_INET6_CONNECT:
3112         case BPF_CGROUP_INET4_GETPEERNAME:
3113         case BPF_CGROUP_INET6_GETPEERNAME:
3114         case BPF_CGROUP_INET4_GETSOCKNAME:
3115         case BPF_CGROUP_INET6_GETSOCKNAME:
3116         case BPF_CGROUP_UDP4_SENDMSG:
3117         case BPF_CGROUP_UDP6_SENDMSG:
3118         case BPF_CGROUP_UDP4_RECVMSG:
3119         case BPF_CGROUP_UDP6_RECVMSG:
3120                 return BPF_PROG_TYPE_CGROUP_SOCK_ADDR;
3121         case BPF_CGROUP_SOCK_OPS:
3122                 return BPF_PROG_TYPE_SOCK_OPS;
3123         case BPF_CGROUP_DEVICE:
3124                 return BPF_PROG_TYPE_CGROUP_DEVICE;
3125         case BPF_SK_MSG_VERDICT:
3126                 return BPF_PROG_TYPE_SK_MSG;
3127         case BPF_SK_SKB_STREAM_PARSER:
3128         case BPF_SK_SKB_STREAM_VERDICT:
3129         case BPF_SK_SKB_VERDICT:
3130                 return BPF_PROG_TYPE_SK_SKB;
3131         case BPF_LIRC_MODE2:
3132                 return BPF_PROG_TYPE_LIRC_MODE2;
3133         case BPF_FLOW_DISSECTOR:
3134                 return BPF_PROG_TYPE_FLOW_DISSECTOR;
3135         case BPF_CGROUP_SYSCTL:
3136                 return BPF_PROG_TYPE_CGROUP_SYSCTL;
3137         case BPF_CGROUP_GETSOCKOPT:
3138         case BPF_CGROUP_SETSOCKOPT:
3139                 return BPF_PROG_TYPE_CGROUP_SOCKOPT;
3140         case BPF_TRACE_ITER:
3141                 return BPF_PROG_TYPE_TRACING;
3142         case BPF_SK_LOOKUP:
3143                 return BPF_PROG_TYPE_SK_LOOKUP;
3144         case BPF_XDP:
3145                 return BPF_PROG_TYPE_XDP;
3146         default:
3147                 return BPF_PROG_TYPE_UNSPEC;
3148         }
3149 }
3150
3151 #define BPF_PROG_ATTACH_LAST_FIELD replace_bpf_fd
3152
3153 #define BPF_F_ATTACH_MASK \
3154         (BPF_F_ALLOW_OVERRIDE | BPF_F_ALLOW_MULTI | BPF_F_REPLACE)
3155
3156 static int bpf_prog_attach(const union bpf_attr *attr)
3157 {
3158         enum bpf_prog_type ptype;
3159         struct bpf_prog *prog;
3160         int ret;
3161
3162         if (CHECK_ATTR(BPF_PROG_ATTACH))
3163                 return -EINVAL;
3164
3165         if (attr->attach_flags & ~BPF_F_ATTACH_MASK)
3166                 return -EINVAL;
3167
3168         ptype = attach_type_to_prog_type(attr->attach_type);
3169         if (ptype == BPF_PROG_TYPE_UNSPEC)
3170                 return -EINVAL;
3171
3172         prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype);
3173         if (IS_ERR(prog))
3174                 return PTR_ERR(prog);
3175
3176         if (bpf_prog_attach_check_attach_type(prog, attr->attach_type)) {
3177                 bpf_prog_put(prog);
3178                 return -EINVAL;
3179         }
3180
3181         switch (ptype) {
3182         case BPF_PROG_TYPE_SK_SKB:
3183         case BPF_PROG_TYPE_SK_MSG:
3184                 ret = sock_map_get_from_fd(attr, prog);
3185                 break;
3186         case BPF_PROG_TYPE_LIRC_MODE2:
3187                 ret = lirc_prog_attach(attr, prog);
3188                 break;
3189         case BPF_PROG_TYPE_FLOW_DISSECTOR:
3190                 ret = netns_bpf_prog_attach(attr, prog);
3191                 break;
3192         case BPF_PROG_TYPE_CGROUP_DEVICE:
3193         case BPF_PROG_TYPE_CGROUP_SKB:
3194         case BPF_PROG_TYPE_CGROUP_SOCK:
3195         case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
3196         case BPF_PROG_TYPE_CGROUP_SOCKOPT:
3197         case BPF_PROG_TYPE_CGROUP_SYSCTL:
3198         case BPF_PROG_TYPE_SOCK_OPS:
3199                 ret = cgroup_bpf_prog_attach(attr, ptype, prog);
3200                 break;
3201         default:
3202                 ret = -EINVAL;
3203         }
3204
3205         if (ret)
3206                 bpf_prog_put(prog);
3207         return ret;
3208 }
3209
3210 #define BPF_PROG_DETACH_LAST_FIELD attach_type
3211
3212 static int bpf_prog_detach(const union bpf_attr *attr)
3213 {
3214         enum bpf_prog_type ptype;
3215
3216         if (CHECK_ATTR(BPF_PROG_DETACH))
3217                 return -EINVAL;
3218
3219         ptype = attach_type_to_prog_type(attr->attach_type);
3220
3221         switch (ptype) {
3222         case BPF_PROG_TYPE_SK_MSG:
3223         case BPF_PROG_TYPE_SK_SKB:
3224                 return sock_map_prog_detach(attr, ptype);
3225         case BPF_PROG_TYPE_LIRC_MODE2:
3226                 return lirc_prog_detach(attr);
3227         case BPF_PROG_TYPE_FLOW_DISSECTOR:
3228                 return netns_bpf_prog_detach(attr, ptype);
3229         case BPF_PROG_TYPE_CGROUP_DEVICE:
3230         case BPF_PROG_TYPE_CGROUP_SKB:
3231         case BPF_PROG_TYPE_CGROUP_SOCK:
3232         case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
3233         case BPF_PROG_TYPE_CGROUP_SOCKOPT:
3234         case BPF_PROG_TYPE_CGROUP_SYSCTL:
3235         case BPF_PROG_TYPE_SOCK_OPS:
3236                 return cgroup_bpf_prog_detach(attr, ptype);
3237         default:
3238                 return -EINVAL;
3239         }
3240 }
3241
3242 #define BPF_PROG_QUERY_LAST_FIELD query.prog_cnt
3243
3244 static int bpf_prog_query(const union bpf_attr *attr,
3245                           union bpf_attr __user *uattr)
3246 {
3247         if (!capable(CAP_NET_ADMIN))
3248                 return -EPERM;
3249         if (CHECK_ATTR(BPF_PROG_QUERY))
3250                 return -EINVAL;
3251         if (attr->query.query_flags & ~BPF_F_QUERY_EFFECTIVE)
3252                 return -EINVAL;
3253
3254         switch (attr->query.attach_type) {
3255         case BPF_CGROUP_INET_INGRESS:
3256         case BPF_CGROUP_INET_EGRESS:
3257         case BPF_CGROUP_INET_SOCK_CREATE:
3258         case BPF_CGROUP_INET_SOCK_RELEASE:
3259         case BPF_CGROUP_INET4_BIND:
3260         case BPF_CGROUP_INET6_BIND:
3261         case BPF_CGROUP_INET4_POST_BIND:
3262         case BPF_CGROUP_INET6_POST_BIND:
3263         case BPF_CGROUP_INET4_CONNECT:
3264         case BPF_CGROUP_INET6_CONNECT:
3265         case BPF_CGROUP_INET4_GETPEERNAME:
3266         case BPF_CGROUP_INET6_GETPEERNAME:
3267         case BPF_CGROUP_INET4_GETSOCKNAME:
3268         case BPF_CGROUP_INET6_GETSOCKNAME:
3269         case BPF_CGROUP_UDP4_SENDMSG:
3270         case BPF_CGROUP_UDP6_SENDMSG:
3271         case BPF_CGROUP_UDP4_RECVMSG:
3272         case BPF_CGROUP_UDP6_RECVMSG:
3273         case BPF_CGROUP_SOCK_OPS:
3274         case BPF_CGROUP_DEVICE:
3275         case BPF_CGROUP_SYSCTL:
3276         case BPF_CGROUP_GETSOCKOPT:
3277         case BPF_CGROUP_SETSOCKOPT:
3278                 return cgroup_bpf_prog_query(attr, uattr);
3279         case BPF_LIRC_MODE2:
3280                 return lirc_prog_query(attr, uattr);
3281         case BPF_FLOW_DISSECTOR:
3282         case BPF_SK_LOOKUP:
3283                 return netns_bpf_prog_query(attr, uattr);
3284         default:
3285                 return -EINVAL;
3286         }
3287 }
3288
3289 #define BPF_PROG_TEST_RUN_LAST_FIELD test.cpu
3290
3291 static int bpf_prog_test_run(const union bpf_attr *attr,
3292                              union bpf_attr __user *uattr)
3293 {
3294         struct bpf_prog *prog;
3295         int ret = -ENOTSUPP;
3296
3297         if (CHECK_ATTR(BPF_PROG_TEST_RUN))
3298                 return -EINVAL;
3299
3300         if ((attr->test.ctx_size_in && !attr->test.ctx_in) ||
3301             (!attr->test.ctx_size_in && attr->test.ctx_in))
3302                 return -EINVAL;
3303
3304         if ((attr->test.ctx_size_out && !attr->test.ctx_out) ||
3305             (!attr->test.ctx_size_out && attr->test.ctx_out))
3306                 return -EINVAL;
3307
3308         prog = bpf_prog_get(attr->test.prog_fd);
3309         if (IS_ERR(prog))
3310                 return PTR_ERR(prog);
3311
3312         if (prog->aux->ops->test_run)
3313                 ret = prog->aux->ops->test_run(prog, attr, uattr);
3314
3315         bpf_prog_put(prog);
3316         return ret;
3317 }
3318
3319 #define BPF_OBJ_GET_NEXT_ID_LAST_FIELD next_id
3320
3321 static int bpf_obj_get_next_id(const union bpf_attr *attr,
3322                                union bpf_attr __user *uattr,
3323                                struct idr *idr,
3324                                spinlock_t *lock)
3325 {
3326         u32 next_id = attr->start_id;
3327         int err = 0;
3328
3329         if (CHECK_ATTR(BPF_OBJ_GET_NEXT_ID) || next_id >= INT_MAX)
3330                 return -EINVAL;
3331
3332         if (!capable(CAP_SYS_ADMIN))
3333                 return -EPERM;
3334
3335         next_id++;
3336         spin_lock_bh(lock);
3337         if (!idr_get_next(idr, &next_id))
3338                 err = -ENOENT;
3339         spin_unlock_bh(lock);
3340
3341         if (!err)
3342                 err = put_user(next_id, &uattr->next_id);
3343
3344         return err;
3345 }
3346
3347 struct bpf_map *bpf_map_get_curr_or_next(u32 *id)
3348 {
3349         struct bpf_map *map;
3350
3351         spin_lock_bh(&map_idr_lock);
3352 again:
3353         map = idr_get_next(&map_idr, id);
3354         if (map) {
3355                 map = __bpf_map_inc_not_zero(map, false);
3356                 if (IS_ERR(map)) {
3357                         (*id)++;
3358                         goto again;
3359                 }
3360         }
3361         spin_unlock_bh(&map_idr_lock);
3362
3363         return map;
3364 }
3365
3366 struct bpf_prog *bpf_prog_get_curr_or_next(u32 *id)
3367 {
3368         struct bpf_prog *prog;
3369
3370         spin_lock_bh(&prog_idr_lock);
3371 again:
3372         prog = idr_get_next(&prog_idr, id);
3373         if (prog) {
3374                 prog = bpf_prog_inc_not_zero(prog);
3375                 if (IS_ERR(prog)) {
3376                         (*id)++;
3377                         goto again;
3378                 }
3379         }
3380         spin_unlock_bh(&prog_idr_lock);
3381
3382         return prog;
3383 }
3384
3385 #define BPF_PROG_GET_FD_BY_ID_LAST_FIELD prog_id
3386
3387 struct bpf_prog *bpf_prog_by_id(u32 id)
3388 {
3389         struct bpf_prog *prog;
3390
3391         if (!id)
3392                 return ERR_PTR(-ENOENT);
3393
3394         spin_lock_bh(&prog_idr_lock);
3395         prog = idr_find(&prog_idr, id);
3396         if (prog)
3397                 prog = bpf_prog_inc_not_zero(prog);
3398         else
3399                 prog = ERR_PTR(-ENOENT);
3400         spin_unlock_bh(&prog_idr_lock);
3401         return prog;
3402 }
3403
3404 static int bpf_prog_get_fd_by_id(const union bpf_attr *attr)
3405 {
3406         struct bpf_prog *prog;
3407         u32 id = attr->prog_id;
3408         int fd;
3409
3410         if (CHECK_ATTR(BPF_PROG_GET_FD_BY_ID))
3411                 return -EINVAL;
3412
3413         if (!capable(CAP_SYS_ADMIN))
3414                 return -EPERM;
3415
3416         prog = bpf_prog_by_id(id);
3417         if (IS_ERR(prog))
3418                 return PTR_ERR(prog);
3419
3420         fd = bpf_prog_new_fd(prog);
3421         if (fd < 0)
3422                 bpf_prog_put(prog);
3423
3424         return fd;
3425 }
3426
3427 #define BPF_MAP_GET_FD_BY_ID_LAST_FIELD open_flags
3428
3429 static int bpf_map_get_fd_by_id(const union bpf_attr *attr)
3430 {
3431         struct bpf_map *map;
3432         u32 id = attr->map_id;
3433         int f_flags;
3434         int fd;
3435
3436         if (CHECK_ATTR(BPF_MAP_GET_FD_BY_ID) ||
3437             attr->open_flags & ~BPF_OBJ_FLAG_MASK)
3438                 return -EINVAL;
3439
3440         if (!capable(CAP_SYS_ADMIN))
3441                 return -EPERM;
3442
3443         f_flags = bpf_get_file_flag(attr->open_flags);
3444         if (f_flags < 0)
3445                 return f_flags;
3446
3447         spin_lock_bh(&map_idr_lock);
3448         map = idr_find(&map_idr, id);
3449         if (map)
3450                 map = __bpf_map_inc_not_zero(map, true);
3451         else
3452                 map = ERR_PTR(-ENOENT);
3453         spin_unlock_bh(&map_idr_lock);
3454
3455         if (IS_ERR(map))
3456                 return PTR_ERR(map);
3457
3458         fd = bpf_map_new_fd(map, f_flags);
3459         if (fd < 0)
3460                 bpf_map_put_with_uref(map);
3461
3462         return fd;
3463 }
3464
3465 static const struct bpf_map *bpf_map_from_imm(const struct bpf_prog *prog,
3466                                               unsigned long addr, u32 *off,
3467                                               u32 *type)
3468 {
3469         const struct bpf_map *map;
3470         int i;
3471
3472         mutex_lock(&prog->aux->used_maps_mutex);
3473         for (i = 0, *off = 0; i < prog->aux->used_map_cnt; i++) {
3474                 map = prog->aux->used_maps[i];
3475                 if (map == (void *)addr) {
3476                         *type = BPF_PSEUDO_MAP_FD;
3477                         goto out;
3478                 }
3479                 if (!map->ops->map_direct_value_meta)
3480                         continue;
3481                 if (!map->ops->map_direct_value_meta(map, addr, off)) {
3482                         *type = BPF_PSEUDO_MAP_VALUE;
3483                         goto out;
3484                 }
3485         }
3486         map = NULL;
3487
3488 out:
3489         mutex_unlock(&prog->aux->used_maps_mutex);
3490         return map;
3491 }
3492
3493 static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog,
3494                                               const struct cred *f_cred)
3495 {
3496         const struct bpf_map *map;
3497         struct bpf_insn *insns;
3498         u32 off, type;
3499         u64 imm;
3500         u8 code;
3501         int i;
3502
3503         insns = kmemdup(prog->insnsi, bpf_prog_insn_size(prog),
3504                         GFP_USER);
3505         if (!insns)
3506                 return insns;
3507
3508         for (i = 0; i < prog->len; i++) {
3509                 code = insns[i].code;
3510
3511                 if (code == (BPF_JMP | BPF_TAIL_CALL)) {
3512                         insns[i].code = BPF_JMP | BPF_CALL;
3513                         insns[i].imm = BPF_FUNC_tail_call;
3514                         /* fall-through */
3515                 }
3516                 if (code == (BPF_JMP | BPF_CALL) ||
3517                     code == (BPF_JMP | BPF_CALL_ARGS)) {
3518                         if (code == (BPF_JMP | BPF_CALL_ARGS))
3519                                 insns[i].code = BPF_JMP | BPF_CALL;
3520                         if (!bpf_dump_raw_ok(f_cred))
3521                                 insns[i].imm = 0;
3522                         continue;
3523                 }
3524                 if (BPF_CLASS(code) == BPF_LDX && BPF_MODE(code) == BPF_PROBE_MEM) {
3525                         insns[i].code = BPF_LDX | BPF_SIZE(code) | BPF_MEM;
3526                         continue;
3527                 }
3528
3529                 if (code != (BPF_LD | BPF_IMM | BPF_DW))
3530                         continue;
3531
3532                 imm = ((u64)insns[i + 1].imm << 32) | (u32)insns[i].imm;
3533                 map = bpf_map_from_imm(prog, imm, &off, &type);
3534                 if (map) {
3535                         insns[i].src_reg = type;
3536                         insns[i].imm = map->id;
3537                         insns[i + 1].imm = off;
3538                         continue;
3539                 }
3540         }
3541
3542         return insns;
3543 }
3544
3545 static int set_info_rec_size(struct bpf_prog_info *info)
3546 {
3547         /*
3548          * Ensure info.*_rec_size is the same as kernel expected size
3549          *
3550          * or
3551          *
3552          * Only allow zero *_rec_size if both _rec_size and _cnt are
3553          * zero.  In this case, the kernel will set the expected
3554          * _rec_size back to the info.
3555          */
3556
3557         if ((info->nr_func_info || info->func_info_rec_size) &&
3558             info->func_info_rec_size != sizeof(struct bpf_func_info))
3559                 return -EINVAL;
3560
3561         if ((info->nr_line_info || info->line_info_rec_size) &&
3562             info->line_info_rec_size != sizeof(struct bpf_line_info))
3563                 return -EINVAL;
3564
3565         if ((info->nr_jited_line_info || info->jited_line_info_rec_size) &&
3566             info->jited_line_info_rec_size != sizeof(__u64))
3567                 return -EINVAL;
3568
3569         info->func_info_rec_size = sizeof(struct bpf_func_info);
3570         info->line_info_rec_size = sizeof(struct bpf_line_info);
3571         info->jited_line_info_rec_size = sizeof(__u64);
3572
3573         return 0;
3574 }
3575
3576 static int bpf_prog_get_info_by_fd(struct file *file,
3577                                    struct bpf_prog *prog,
3578                                    const union bpf_attr *attr,
3579                                    union bpf_attr __user *uattr)
3580 {
3581         struct bpf_prog_info __user *uinfo = u64_to_user_ptr(attr->info.info);
3582         struct bpf_prog_info info;
3583         u32 info_len = attr->info.info_len;
3584         struct bpf_prog_stats stats;
3585         char __user *uinsns;
3586         u32 ulen;
3587         int err;
3588
3589         err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(info), info_len);
3590         if (err)
3591                 return err;
3592         info_len = min_t(u32, sizeof(info), info_len);
3593
3594         memset(&info, 0, sizeof(info));
3595         if (copy_from_user(&info, uinfo, info_len))
3596                 return -EFAULT;
3597
3598         info.type = prog->type;
3599         info.id = prog->aux->id;
3600         info.load_time = prog->aux->load_time;
3601         info.created_by_uid = from_kuid_munged(current_user_ns(),
3602                                                prog->aux->user->uid);
3603         info.gpl_compatible = prog->gpl_compatible;
3604
3605         memcpy(info.tag, prog->tag, sizeof(prog->tag));
3606         memcpy(info.name, prog->aux->name, sizeof(prog->aux->name));
3607
3608         mutex_lock(&prog->aux->used_maps_mutex);
3609         ulen = info.nr_map_ids;
3610         info.nr_map_ids = prog->aux->used_map_cnt;
3611         ulen = min_t(u32, info.nr_map_ids, ulen);
3612         if (ulen) {
3613                 u32 __user *user_map_ids = u64_to_user_ptr(info.map_ids);
3614                 u32 i;
3615
3616                 for (i = 0; i < ulen; i++)
3617                         if (put_user(prog->aux->used_maps[i]->id,
3618                                      &user_map_ids[i])) {
3619                                 mutex_unlock(&prog->aux->used_maps_mutex);
3620                                 return -EFAULT;
3621                         }
3622         }
3623         mutex_unlock(&prog->aux->used_maps_mutex);
3624
3625         err = set_info_rec_size(&info);
3626         if (err)
3627                 return err;
3628
3629         bpf_prog_get_stats(prog, &stats);
3630         info.run_time_ns = stats.nsecs;
3631         info.run_cnt = stats.cnt;
3632         info.recursion_misses = stats.misses;
3633
3634         if (!bpf_capable()) {
3635                 info.jited_prog_len = 0;
3636                 info.xlated_prog_len = 0;
3637                 info.nr_jited_ksyms = 0;
3638                 info.nr_jited_func_lens = 0;
3639                 info.nr_func_info = 0;
3640                 info.nr_line_info = 0;
3641                 info.nr_jited_line_info = 0;
3642                 goto done;
3643         }
3644
3645         ulen = info.xlated_prog_len;
3646         info.xlated_prog_len = bpf_prog_insn_size(prog);
3647         if (info.xlated_prog_len && ulen) {
3648                 struct bpf_insn *insns_sanitized;
3649                 bool fault;
3650
3651                 if (prog->blinded && !bpf_dump_raw_ok(file->f_cred)) {
3652                         info.xlated_prog_insns = 0;
3653                         goto done;
3654                 }
3655                 insns_sanitized = bpf_insn_prepare_dump(prog, file->f_cred);
3656                 if (!insns_sanitized)
3657                         return -ENOMEM;
3658                 uinsns = u64_to_user_ptr(info.xlated_prog_insns);
3659                 ulen = min_t(u32, info.xlated_prog_len, ulen);
3660                 fault = copy_to_user(uinsns, insns_sanitized, ulen);
3661                 kfree(insns_sanitized);
3662                 if (fault)
3663                         return -EFAULT;
3664         }
3665
3666         if (bpf_prog_is_dev_bound(prog->aux)) {
3667                 err = bpf_prog_offload_info_fill(&info, prog);
3668                 if (err)
3669                         return err;
3670                 goto done;
3671         }
3672
3673         /* NOTE: the following code is supposed to be skipped for offload.
3674          * bpf_prog_offload_info_fill() is the place to fill similar fields
3675          * for offload.
3676          */
3677         ulen = info.jited_prog_len;
3678         if (prog->aux->func_cnt) {
3679                 u32 i;
3680
3681                 info.jited_prog_len = 0;
3682                 for (i = 0; i < prog->aux->func_cnt; i++)
3683                         info.jited_prog_len += prog->aux->func[i]->jited_len;
3684         } else {
3685                 info.jited_prog_len = prog->jited_len;
3686         }
3687
3688         if (info.jited_prog_len && ulen) {
3689                 if (bpf_dump_raw_ok(file->f_cred)) {
3690                         uinsns = u64_to_user_ptr(info.jited_prog_insns);
3691                         ulen = min_t(u32, info.jited_prog_len, ulen);
3692
3693                         /* for multi-function programs, copy the JITed
3694                          * instructions for all the functions
3695                          */
3696                         if (prog->aux->func_cnt) {
3697                                 u32 len, free, i;
3698                                 u8 *img;
3699
3700                                 free = ulen;
3701                                 for (i = 0; i < prog->aux->func_cnt; i++) {
3702                                         len = prog->aux->func[i]->jited_len;
3703                                         len = min_t(u32, len, free);
3704                                         img = (u8 *) prog->aux->func[i]->bpf_func;
3705                                         if (copy_to_user(uinsns, img, len))
3706                                                 return -EFAULT;
3707                                         uinsns += len;
3708                                         free -= len;
3709                                         if (!free)
3710                                                 break;
3711                                 }
3712                         } else {
3713                                 if (copy_to_user(uinsns, prog->bpf_func, ulen))
3714                                         return -EFAULT;
3715                         }
3716                 } else {
3717                         info.jited_prog_insns = 0;
3718                 }
3719         }
3720
3721         ulen = info.nr_jited_ksyms;
3722         info.nr_jited_ksyms = prog->aux->func_cnt ? : 1;
3723         if (ulen) {
3724                 if (bpf_dump_raw_ok(file->f_cred)) {
3725                         unsigned long ksym_addr;
3726                         u64 __user *user_ksyms;
3727                         u32 i;
3728
3729                         /* copy the address of the kernel symbol
3730                          * corresponding to each function
3731                          */
3732                         ulen = min_t(u32, info.nr_jited_ksyms, ulen);
3733                         user_ksyms = u64_to_user_ptr(info.jited_ksyms);
3734                         if (prog->aux->func_cnt) {
3735                                 for (i = 0; i < ulen; i++) {
3736                                         ksym_addr = (unsigned long)
3737                                                 prog->aux->func[i]->bpf_func;
3738                                         if (put_user((u64) ksym_addr,
3739                                                      &user_ksyms[i]))
3740                                                 return -EFAULT;
3741                                 }
3742                         } else {
3743                                 ksym_addr = (unsigned long) prog->bpf_func;
3744                                 if (put_user((u64) ksym_addr, &user_ksyms[0]))
3745                                         return -EFAULT;
3746                         }
3747                 } else {
3748                         info.jited_ksyms = 0;
3749                 }
3750         }
3751
3752         ulen = info.nr_jited_func_lens;
3753         info.nr_jited_func_lens = prog->aux->func_cnt ? : 1;
3754         if (ulen) {
3755                 if (bpf_dump_raw_ok(file->f_cred)) {
3756                         u32 __user *user_lens;
3757                         u32 func_len, i;
3758
3759                         /* copy the JITed image lengths for each function */
3760                         ulen = min_t(u32, info.nr_jited_func_lens, ulen);
3761                         user_lens = u64_to_user_ptr(info.jited_func_lens);
3762                         if (prog->aux->func_cnt) {
3763                                 for (i = 0; i < ulen; i++) {
3764                                         func_len =
3765                                                 prog->aux->func[i]->jited_len;
3766                                         if (put_user(func_len, &user_lens[i]))
3767                                                 return -EFAULT;
3768                                 }
3769                         } else {
3770                                 func_len = prog->jited_len;
3771                                 if (put_user(func_len, &user_lens[0]))
3772                                         return -EFAULT;
3773                         }
3774                 } else {
3775                         info.jited_func_lens = 0;
3776                 }
3777         }
3778
3779         if (prog->aux->btf)
3780                 info.btf_id = btf_obj_id(prog->aux->btf);
3781
3782         ulen = info.nr_func_info;
3783         info.nr_func_info = prog->aux->func_info_cnt;
3784         if (info.nr_func_info && ulen) {
3785                 char __user *user_finfo;
3786
3787                 user_finfo = u64_to_user_ptr(info.func_info);
3788                 ulen = min_t(u32, info.nr_func_info, ulen);
3789                 if (copy_to_user(user_finfo, prog->aux->func_info,
3790                                  info.func_info_rec_size * ulen))
3791                         return -EFAULT;
3792         }
3793
3794         ulen = info.nr_line_info;
3795         info.nr_line_info = prog->aux->nr_linfo;
3796         if (info.nr_line_info && ulen) {
3797                 __u8 __user *user_linfo;
3798
3799                 user_linfo = u64_to_user_ptr(info.line_info);
3800                 ulen = min_t(u32, info.nr_line_info, ulen);
3801                 if (copy_to_user(user_linfo, prog->aux->linfo,
3802                                  info.line_info_rec_size * ulen))
3803                         return -EFAULT;
3804         }
3805
3806         ulen = info.nr_jited_line_info;
3807         if (prog->aux->jited_linfo)
3808                 info.nr_jited_line_info = prog->aux->nr_linfo;
3809         else
3810                 info.nr_jited_line_info = 0;
3811         if (info.nr_jited_line_info && ulen) {
3812                 if (bpf_dump_raw_ok(file->f_cred)) {
3813                         __u64 __user *user_linfo;
3814                         u32 i;
3815
3816                         user_linfo = u64_to_user_ptr(info.jited_line_info);
3817                         ulen = min_t(u32, info.nr_jited_line_info, ulen);
3818                         for (i = 0; i < ulen; i++) {
3819                                 if (put_user((__u64)(long)prog->aux->jited_linfo[i],
3820                                              &user_linfo[i]))
3821                                         return -EFAULT;
3822                         }
3823                 } else {
3824                         info.jited_line_info = 0;
3825                 }
3826         }
3827
3828         ulen = info.nr_prog_tags;
3829         info.nr_prog_tags = prog->aux->func_cnt ? : 1;
3830         if (ulen) {
3831                 __u8 __user (*user_prog_tags)[BPF_TAG_SIZE];
3832                 u32 i;
3833
3834                 user_prog_tags = u64_to_user_ptr(info.prog_tags);
3835                 ulen = min_t(u32, info.nr_prog_tags, ulen);
3836                 if (prog->aux->func_cnt) {
3837                         for (i = 0; i < ulen; i++) {
3838                                 if (copy_to_user(user_prog_tags[i],
3839                                                  prog->aux->func[i]->tag,
3840                                                  BPF_TAG_SIZE))
3841                                         return -EFAULT;
3842                         }
3843                 } else {
3844                         if (copy_to_user(user_prog_tags[0],
3845                                          prog->tag, BPF_TAG_SIZE))
3846                                 return -EFAULT;
3847                 }
3848         }
3849
3850 done:
3851         if (copy_to_user(uinfo, &info, info_len) ||
3852             put_user(info_len, &uattr->info.info_len))
3853                 return -EFAULT;
3854
3855         return 0;
3856 }
3857
3858 static int bpf_map_get_info_by_fd(struct file *file,
3859                                   struct bpf_map *map,
3860                                   const union bpf_attr *attr,
3861                                   union bpf_attr __user *uattr)
3862 {
3863         struct bpf_map_info __user *uinfo = u64_to_user_ptr(attr->info.info);
3864         struct bpf_map_info info;
3865         u32 info_len = attr->info.info_len;
3866         int err;
3867
3868         err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(info), info_len);
3869         if (err)
3870                 return err;
3871         info_len = min_t(u32, sizeof(info), info_len);
3872
3873         memset(&info, 0, sizeof(info));
3874         info.type = map->map_type;
3875         info.id = map->id;
3876         info.key_size = map->key_size;
3877         info.value_size = map->value_size;
3878         info.max_entries = map->max_entries;
3879         info.map_flags = map->map_flags;
3880         memcpy(info.name, map->name, sizeof(map->name));
3881
3882         if (map->btf) {
3883                 info.btf_id = btf_obj_id(map->btf);
3884                 info.btf_key_type_id = map->btf_key_type_id;
3885                 info.btf_value_type_id = map->btf_value_type_id;
3886         }
3887         info.btf_vmlinux_value_type_id = map->btf_vmlinux_value_type_id;
3888
3889         if (bpf_map_is_dev_bound(map)) {
3890                 err = bpf_map_offload_info_fill(&info, map);
3891                 if (err)
3892                         return err;
3893         }
3894
3895         if (copy_to_user(uinfo, &info, info_len) ||
3896             put_user(info_len, &uattr->info.info_len))
3897                 return -EFAULT;
3898
3899         return 0;
3900 }
3901
3902 static int bpf_btf_get_info_by_fd(struct file *file,
3903                                   struct btf *btf,
3904                                   const union bpf_attr *attr,
3905                                   union bpf_attr __user *uattr)
3906 {
3907         struct bpf_btf_info __user *uinfo = u64_to_user_ptr(attr->info.info);
3908         u32 info_len = attr->info.info_len;
3909         int err;
3910
3911         err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(*uinfo), info_len);
3912         if (err)
3913                 return err;
3914
3915         return btf_get_info_by_fd(btf, attr, uattr);
3916 }
3917
3918 static int bpf_link_get_info_by_fd(struct file *file,
3919                                   struct bpf_link *link,
3920                                   const union bpf_attr *attr,
3921                                   union bpf_attr __user *uattr)
3922 {
3923         struct bpf_link_info __user *uinfo = u64_to_user_ptr(attr->info.info);
3924         struct bpf_link_info info;
3925         u32 info_len = attr->info.info_len;
3926         int err;
3927
3928         err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(info), info_len);
3929         if (err)
3930                 return err;
3931         info_len = min_t(u32, sizeof(info), info_len);
3932
3933         memset(&info, 0, sizeof(info));
3934         if (copy_from_user(&info, uinfo, info_len))
3935                 return -EFAULT;
3936
3937         info.type = link->type;
3938         info.id = link->id;
3939         info.prog_id = link->prog->aux->id;
3940
3941         if (link->ops->fill_link_info) {
3942                 err = link->ops->fill_link_info(link, &info);
3943                 if (err)
3944                         return err;
3945         }
3946
3947         if (copy_to_user(uinfo, &info, info_len) ||
3948             put_user(info_len, &uattr->info.info_len))
3949                 return -EFAULT;
3950
3951         return 0;
3952 }
3953
3954
3955 #define BPF_OBJ_GET_INFO_BY_FD_LAST_FIELD info.info
3956
3957 static int bpf_obj_get_info_by_fd(const union bpf_attr *attr,
3958                                   union bpf_attr __user *uattr)
3959 {
3960         int ufd = attr->info.bpf_fd;
3961         struct fd f;
3962         int err;
3963
3964         if (CHECK_ATTR(BPF_OBJ_GET_INFO_BY_FD))
3965                 return -EINVAL;
3966
3967         f = fdget(ufd);
3968         if (!f.file)
3969                 return -EBADFD;
3970
3971         if (f.file->f_op == &bpf_prog_fops)
3972                 err = bpf_prog_get_info_by_fd(f.file, f.file->private_data, attr,
3973                                               uattr);
3974         else if (f.file->f_op == &bpf_map_fops)
3975                 err = bpf_map_get_info_by_fd(f.file, f.file->private_data, attr,
3976                                              uattr);
3977         else if (f.file->f_op == &btf_fops)
3978                 err = bpf_btf_get_info_by_fd(f.file, f.file->private_data, attr, uattr);
3979         else if (f.file->f_op == &bpf_link_fops)
3980                 err = bpf_link_get_info_by_fd(f.file, f.file->private_data,
3981                                               attr, uattr);
3982         else
3983                 err = -EINVAL;
3984
3985         fdput(f);
3986         return err;
3987 }
3988
3989 #define BPF_BTF_LOAD_LAST_FIELD btf_log_level
3990
3991 static int bpf_btf_load(const union bpf_attr *attr, bpfptr_t uattr)
3992 {
3993         if (CHECK_ATTR(BPF_BTF_LOAD))
3994                 return -EINVAL;
3995
3996         if (!bpf_capable())
3997                 return -EPERM;
3998
3999         return btf_new_fd(attr, uattr);
4000 }
4001
4002 #define BPF_BTF_GET_FD_BY_ID_LAST_FIELD btf_id
4003
4004 static int bpf_btf_get_fd_by_id(const union bpf_attr *attr)
4005 {
4006         if (CHECK_ATTR(BPF_BTF_GET_FD_BY_ID))
4007                 return -EINVAL;
4008
4009         if (!capable(CAP_SYS_ADMIN))
4010                 return -EPERM;
4011
4012         return btf_get_fd_by_id(attr->btf_id);
4013 }
4014
4015 static int bpf_task_fd_query_copy(const union bpf_attr *attr,
4016                                     union bpf_attr __user *uattr,
4017                                     u32 prog_id, u32 fd_type,
4018                                     const char *buf, u64 probe_offset,
4019                                     u64 probe_addr)
4020 {
4021         char __user *ubuf = u64_to_user_ptr(attr->task_fd_query.buf);
4022         u32 len = buf ? strlen(buf) : 0, input_len;
4023         int err = 0;
4024
4025         if (put_user(len, &uattr->task_fd_query.buf_len))
4026                 return -EFAULT;
4027         input_len = attr->task_fd_query.buf_len;
4028         if (input_len && ubuf) {
4029                 if (!len) {
4030                         /* nothing to copy, just make ubuf NULL terminated */
4031                         char zero = '\0';
4032
4033                         if (put_user(zero, ubuf))
4034                                 return -EFAULT;
4035                 } else if (input_len >= len + 1) {
4036                         /* ubuf can hold the string with NULL terminator */
4037                         if (copy_to_user(ubuf, buf, len + 1))
4038                                 return -EFAULT;
4039                 } else {
4040                         /* ubuf cannot hold the string with NULL terminator,
4041                          * do a partial copy with NULL terminator.
4042                          */
4043                         char zero = '\0';
4044
4045                         err = -ENOSPC;
4046                         if (copy_to_user(ubuf, buf, input_len - 1))
4047                                 return -EFAULT;
4048                         if (put_user(zero, ubuf + input_len - 1))
4049                                 return -EFAULT;
4050                 }
4051         }
4052
4053         if (put_user(prog_id, &uattr->task_fd_query.prog_id) ||
4054             put_user(fd_type, &uattr->task_fd_query.fd_type) ||
4055             put_user(probe_offset, &uattr->task_fd_query.probe_offset) ||
4056             put_user(probe_addr, &uattr->task_fd_query.probe_addr))
4057                 return -EFAULT;
4058
4059         return err;
4060 }
4061
4062 #define BPF_TASK_FD_QUERY_LAST_FIELD task_fd_query.probe_addr
4063
4064 static int bpf_task_fd_query(const union bpf_attr *attr,
4065                              union bpf_attr __user *uattr)
4066 {
4067         pid_t pid = attr->task_fd_query.pid;
4068         u32 fd = attr->task_fd_query.fd;
4069         const struct perf_event *event;
4070         struct task_struct *task;
4071         struct file *file;
4072         int err;
4073
4074         if (CHECK_ATTR(BPF_TASK_FD_QUERY))
4075                 return -EINVAL;
4076
4077         if (!capable(CAP_SYS_ADMIN))
4078                 return -EPERM;
4079
4080         if (attr->task_fd_query.flags != 0)
4081                 return -EINVAL;
4082
4083         task = get_pid_task(find_vpid(pid), PIDTYPE_PID);
4084         if (!task)
4085                 return -ENOENT;
4086
4087         err = 0;
4088         file = fget_task(task, fd);
4089         put_task_struct(task);
4090         if (!file)
4091                 return -EBADF;
4092
4093         if (file->f_op == &bpf_link_fops) {
4094                 struct bpf_link *link = file->private_data;
4095
4096                 if (link->ops == &bpf_raw_tp_link_lops) {
4097                         struct bpf_raw_tp_link *raw_tp =
4098                                 container_of(link, struct bpf_raw_tp_link, link);
4099                         struct bpf_raw_event_map *btp = raw_tp->btp;
4100
4101                         err = bpf_task_fd_query_copy(attr, uattr,
4102                                                      raw_tp->link.prog->aux->id,
4103                                                      BPF_FD_TYPE_RAW_TRACEPOINT,
4104                                                      btp->tp->name, 0, 0);
4105                         goto put_file;
4106                 }
4107                 goto out_not_supp;
4108         }
4109
4110         event = perf_get_event(file);
4111         if (!IS_ERR(event)) {
4112                 u64 probe_offset, probe_addr;
4113                 u32 prog_id, fd_type;
4114                 const char *buf;
4115
4116                 err = bpf_get_perf_event_info(event, &prog_id, &fd_type,
4117                                               &buf, &probe_offset,
4118                                               &probe_addr);
4119                 if (!err)
4120                         err = bpf_task_fd_query_copy(attr, uattr, prog_id,
4121                                                      fd_type, buf,
4122                                                      probe_offset,
4123                                                      probe_addr);
4124                 goto put_file;
4125         }
4126
4127 out_not_supp:
4128         err = -ENOTSUPP;
4129 put_file:
4130         fput(file);
4131         return err;
4132 }
4133
4134 #define BPF_MAP_BATCH_LAST_FIELD batch.flags
4135
4136 #define BPF_DO_BATCH(fn)                        \
4137         do {                                    \
4138                 if (!fn) {                      \
4139                         err = -ENOTSUPP;        \
4140                         goto err_put;           \
4141                 }                               \
4142                 err = fn(map, attr, uattr);     \
4143         } while (0)
4144
4145 static int bpf_map_do_batch(const union bpf_attr *attr,
4146                             union bpf_attr __user *uattr,
4147                             int cmd)
4148 {
4149         struct bpf_map *map;
4150         int err, ufd;
4151         struct fd f;
4152
4153         if (CHECK_ATTR(BPF_MAP_BATCH))
4154                 return -EINVAL;
4155
4156         ufd = attr->batch.map_fd;
4157         f = fdget(ufd);
4158         map = __bpf_map_get(f);
4159         if (IS_ERR(map))
4160                 return PTR_ERR(map);
4161
4162         if ((cmd == BPF_MAP_LOOKUP_BATCH ||
4163              cmd == BPF_MAP_LOOKUP_AND_DELETE_BATCH) &&
4164             !(map_get_sys_perms(map, f) & FMODE_CAN_READ)) {
4165                 err = -EPERM;
4166                 goto err_put;
4167         }
4168
4169         if (cmd != BPF_MAP_LOOKUP_BATCH &&
4170             !(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
4171                 err = -EPERM;
4172                 goto err_put;
4173         }
4174
4175         if (cmd == BPF_MAP_LOOKUP_BATCH)
4176                 BPF_DO_BATCH(map->ops->map_lookup_batch);
4177         else if (cmd == BPF_MAP_LOOKUP_AND_DELETE_BATCH)
4178                 BPF_DO_BATCH(map->ops->map_lookup_and_delete_batch);
4179         else if (cmd == BPF_MAP_UPDATE_BATCH)
4180                 BPF_DO_BATCH(map->ops->map_update_batch);
4181         else
4182                 BPF_DO_BATCH(map->ops->map_delete_batch);
4183
4184 err_put:
4185         fdput(f);
4186         return err;
4187 }
4188
4189 static int tracing_bpf_link_attach(const union bpf_attr *attr, bpfptr_t uattr,
4190                                    struct bpf_prog *prog)
4191 {
4192         if (attr->link_create.attach_type != prog->expected_attach_type)
4193                 return -EINVAL;
4194
4195         if (prog->expected_attach_type == BPF_TRACE_ITER)
4196                 return bpf_iter_link_attach(attr, uattr, prog);
4197         else if (prog->type == BPF_PROG_TYPE_EXT)
4198                 return bpf_tracing_prog_attach(prog,
4199                                                attr->link_create.target_fd,
4200                                                attr->link_create.target_btf_id);
4201         return -EINVAL;
4202 }
4203
4204 #define BPF_LINK_CREATE_LAST_FIELD link_create.iter_info_len
4205 static int link_create(union bpf_attr *attr, bpfptr_t uattr)
4206 {
4207         enum bpf_prog_type ptype;
4208         struct bpf_prog *prog;
4209         int ret;
4210
4211         if (CHECK_ATTR(BPF_LINK_CREATE))
4212                 return -EINVAL;
4213
4214         prog = bpf_prog_get(attr->link_create.prog_fd);
4215         if (IS_ERR(prog))
4216                 return PTR_ERR(prog);
4217
4218         ret = bpf_prog_attach_check_attach_type(prog,
4219                                                 attr->link_create.attach_type);
4220         if (ret)
4221                 goto out;
4222
4223         switch (prog->type) {
4224         case BPF_PROG_TYPE_EXT:
4225                 ret = tracing_bpf_link_attach(attr, uattr, prog);
4226                 goto out;
4227         case BPF_PROG_TYPE_PERF_EVENT:
4228         case BPF_PROG_TYPE_KPROBE:
4229         case BPF_PROG_TYPE_TRACEPOINT:
4230                 if (attr->link_create.attach_type != BPF_PERF_EVENT) {
4231                         ret = -EINVAL;
4232                         goto out;
4233                 }
4234                 ptype = prog->type;
4235                 break;
4236         default:
4237                 ptype = attach_type_to_prog_type(attr->link_create.attach_type);
4238                 if (ptype == BPF_PROG_TYPE_UNSPEC || ptype != prog->type) {
4239                         ret = -EINVAL;
4240                         goto out;
4241                 }
4242                 break;
4243         }
4244
4245         switch (ptype) {
4246         case BPF_PROG_TYPE_CGROUP_SKB:
4247         case BPF_PROG_TYPE_CGROUP_SOCK:
4248         case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
4249         case BPF_PROG_TYPE_SOCK_OPS:
4250         case BPF_PROG_TYPE_CGROUP_DEVICE:
4251         case BPF_PROG_TYPE_CGROUP_SYSCTL:
4252         case BPF_PROG_TYPE_CGROUP_SOCKOPT:
4253                 ret = cgroup_bpf_link_attach(attr, prog);
4254                 break;
4255         case BPF_PROG_TYPE_TRACING:
4256                 ret = tracing_bpf_link_attach(attr, uattr, prog);
4257                 break;
4258         case BPF_PROG_TYPE_FLOW_DISSECTOR:
4259         case BPF_PROG_TYPE_SK_LOOKUP:
4260                 ret = netns_bpf_link_create(attr, prog);
4261                 break;
4262 #ifdef CONFIG_NET
4263         case BPF_PROG_TYPE_XDP:
4264                 ret = bpf_xdp_link_attach(attr, prog);
4265                 break;
4266 #endif
4267 #ifdef CONFIG_PERF_EVENTS
4268         case BPF_PROG_TYPE_PERF_EVENT:
4269         case BPF_PROG_TYPE_TRACEPOINT:
4270         case BPF_PROG_TYPE_KPROBE:
4271                 ret = bpf_perf_link_attach(attr, prog);
4272                 break;
4273 #endif
4274         default:
4275                 ret = -EINVAL;
4276         }
4277
4278 out:
4279         if (ret < 0)
4280                 bpf_prog_put(prog);
4281         return ret;
4282 }
4283
4284 #define BPF_LINK_UPDATE_LAST_FIELD link_update.old_prog_fd
4285
4286 static int link_update(union bpf_attr *attr)
4287 {
4288         struct bpf_prog *old_prog = NULL, *new_prog;
4289         struct bpf_link *link;
4290         u32 flags;
4291         int ret;
4292
4293         if (CHECK_ATTR(BPF_LINK_UPDATE))
4294                 return -EINVAL;
4295
4296         flags = attr->link_update.flags;
4297         if (flags & ~BPF_F_REPLACE)
4298                 return -EINVAL;
4299
4300         link = bpf_link_get_from_fd(attr->link_update.link_fd);
4301         if (IS_ERR(link))
4302                 return PTR_ERR(link);
4303
4304         new_prog = bpf_prog_get(attr->link_update.new_prog_fd);
4305         if (IS_ERR(new_prog)) {
4306                 ret = PTR_ERR(new_prog);
4307                 goto out_put_link;
4308         }
4309
4310         if (flags & BPF_F_REPLACE) {
4311                 old_prog = bpf_prog_get(attr->link_update.old_prog_fd);
4312                 if (IS_ERR(old_prog)) {
4313                         ret = PTR_ERR(old_prog);
4314                         old_prog = NULL;
4315                         goto out_put_progs;
4316                 }
4317         } else if (attr->link_update.old_prog_fd) {
4318                 ret = -EINVAL;
4319                 goto out_put_progs;
4320         }
4321
4322         if (link->ops->update_prog)
4323                 ret = link->ops->update_prog(link, new_prog, old_prog);
4324         else
4325                 ret = -EINVAL;
4326
4327 out_put_progs:
4328         if (old_prog)
4329                 bpf_prog_put(old_prog);
4330         if (ret)
4331                 bpf_prog_put(new_prog);
4332 out_put_link:
4333         bpf_link_put(link);
4334         return ret;
4335 }
4336
4337 #define BPF_LINK_DETACH_LAST_FIELD link_detach.link_fd
4338
4339 static int link_detach(union bpf_attr *attr)
4340 {
4341         struct bpf_link *link;
4342         int ret;
4343
4344         if (CHECK_ATTR(BPF_LINK_DETACH))
4345                 return -EINVAL;
4346
4347         link = bpf_link_get_from_fd(attr->link_detach.link_fd);
4348         if (IS_ERR(link))
4349                 return PTR_ERR(link);
4350
4351         if (link->ops->detach)
4352                 ret = link->ops->detach(link);
4353         else
4354                 ret = -EOPNOTSUPP;
4355
4356         bpf_link_put(link);
4357         return ret;
4358 }
4359
4360 static struct bpf_link *bpf_link_inc_not_zero(struct bpf_link *link)
4361 {
4362         return atomic64_fetch_add_unless(&link->refcnt, 1, 0) ? link : ERR_PTR(-ENOENT);
4363 }
4364
4365 struct bpf_link *bpf_link_by_id(u32 id)
4366 {
4367         struct bpf_link *link;
4368
4369         if (!id)
4370                 return ERR_PTR(-ENOENT);
4371
4372         spin_lock_bh(&link_idr_lock);
4373         /* before link is "settled", ID is 0, pretend it doesn't exist yet */
4374         link = idr_find(&link_idr, id);
4375         if (link) {
4376                 if (link->id)
4377                         link = bpf_link_inc_not_zero(link);
4378                 else
4379                         link = ERR_PTR(-EAGAIN);
4380         } else {
4381                 link = ERR_PTR(-ENOENT);
4382         }
4383         spin_unlock_bh(&link_idr_lock);
4384         return link;
4385 }
4386
4387 #define BPF_LINK_GET_FD_BY_ID_LAST_FIELD link_id
4388
4389 static int bpf_link_get_fd_by_id(const union bpf_attr *attr)
4390 {
4391         struct bpf_link *link;
4392         u32 id = attr->link_id;
4393         int fd;
4394
4395         if (CHECK_ATTR(BPF_LINK_GET_FD_BY_ID))
4396                 return -EINVAL;
4397
4398         if (!capable(CAP_SYS_ADMIN))
4399                 return -EPERM;
4400
4401         link = bpf_link_by_id(id);
4402         if (IS_ERR(link))
4403                 return PTR_ERR(link);
4404
4405         fd = bpf_link_new_fd(link);
4406         if (fd < 0)
4407                 bpf_link_put(link);
4408
4409         return fd;
4410 }
4411
4412 DEFINE_MUTEX(bpf_stats_enabled_mutex);
4413
4414 static int bpf_stats_release(struct inode *inode, struct file *file)
4415 {
4416         mutex_lock(&bpf_stats_enabled_mutex);
4417         static_key_slow_dec(&bpf_stats_enabled_key.key);
4418         mutex_unlock(&bpf_stats_enabled_mutex);
4419         return 0;
4420 }
4421
4422 static const struct file_operations bpf_stats_fops = {
4423         .release = bpf_stats_release,
4424 };
4425
4426 static int bpf_enable_runtime_stats(void)
4427 {
4428         int fd;
4429
4430         mutex_lock(&bpf_stats_enabled_mutex);
4431
4432         /* Set a very high limit to avoid overflow */
4433         if (static_key_count(&bpf_stats_enabled_key.key) > INT_MAX / 2) {
4434                 mutex_unlock(&bpf_stats_enabled_mutex);
4435                 return -EBUSY;
4436         }
4437
4438         fd = anon_inode_getfd("bpf-stats", &bpf_stats_fops, NULL, O_CLOEXEC);
4439         if (fd >= 0)
4440                 static_key_slow_inc(&bpf_stats_enabled_key.key);
4441
4442         mutex_unlock(&bpf_stats_enabled_mutex);
4443         return fd;
4444 }
4445
4446 #define BPF_ENABLE_STATS_LAST_FIELD enable_stats.type
4447
4448 static int bpf_enable_stats(union bpf_attr *attr)
4449 {
4450
4451         if (CHECK_ATTR(BPF_ENABLE_STATS))
4452                 return -EINVAL;
4453
4454         if (!capable(CAP_SYS_ADMIN))
4455                 return -EPERM;
4456
4457         switch (attr->enable_stats.type) {
4458         case BPF_STATS_RUN_TIME:
4459                 return bpf_enable_runtime_stats();
4460         default:
4461                 break;
4462         }
4463         return -EINVAL;
4464 }
4465
4466 #define BPF_ITER_CREATE_LAST_FIELD iter_create.flags
4467
4468 static int bpf_iter_create(union bpf_attr *attr)
4469 {
4470         struct bpf_link *link;
4471         int err;
4472
4473         if (CHECK_ATTR(BPF_ITER_CREATE))
4474                 return -EINVAL;
4475
4476         if (attr->iter_create.flags)
4477                 return -EINVAL;
4478
4479         link = bpf_link_get_from_fd(attr->iter_create.link_fd);
4480         if (IS_ERR(link))
4481                 return PTR_ERR(link);
4482
4483         err = bpf_iter_new_fd(link);
4484         bpf_link_put(link);
4485
4486         return err;
4487 }
4488
4489 #define BPF_PROG_BIND_MAP_LAST_FIELD prog_bind_map.flags
4490
4491 static int bpf_prog_bind_map(union bpf_attr *attr)
4492 {
4493         struct bpf_prog *prog;
4494         struct bpf_map *map;
4495         struct bpf_map **used_maps_old, **used_maps_new;
4496         int i, ret = 0;
4497
4498         if (CHECK_ATTR(BPF_PROG_BIND_MAP))
4499                 return -EINVAL;
4500
4501         if (attr->prog_bind_map.flags)
4502                 return -EINVAL;
4503
4504         prog = bpf_prog_get(attr->prog_bind_map.prog_fd);
4505         if (IS_ERR(prog))
4506                 return PTR_ERR(prog);
4507
4508         map = bpf_map_get(attr->prog_bind_map.map_fd);
4509         if (IS_ERR(map)) {
4510                 ret = PTR_ERR(map);
4511                 goto out_prog_put;
4512         }
4513
4514         mutex_lock(&prog->aux->used_maps_mutex);
4515
4516         used_maps_old = prog->aux->used_maps;
4517
4518         for (i = 0; i < prog->aux->used_map_cnt; i++)
4519                 if (used_maps_old[i] == map) {
4520                         bpf_map_put(map);
4521                         goto out_unlock;
4522                 }
4523
4524         used_maps_new = kmalloc_array(prog->aux->used_map_cnt + 1,
4525                                       sizeof(used_maps_new[0]),
4526                                       GFP_KERNEL);
4527         if (!used_maps_new) {
4528                 ret = -ENOMEM;
4529                 goto out_unlock;
4530         }
4531
4532         memcpy(used_maps_new, used_maps_old,
4533                sizeof(used_maps_old[0]) * prog->aux->used_map_cnt);
4534         used_maps_new[prog->aux->used_map_cnt] = map;
4535
4536         prog->aux->used_map_cnt++;
4537         prog->aux->used_maps = used_maps_new;
4538
4539         kfree(used_maps_old);
4540
4541 out_unlock:
4542         mutex_unlock(&prog->aux->used_maps_mutex);
4543
4544         if (ret)
4545                 bpf_map_put(map);
4546 out_prog_put:
4547         bpf_prog_put(prog);
4548         return ret;
4549 }
4550
4551 static int __sys_bpf(int cmd, bpfptr_t uattr, unsigned int size)
4552 {
4553         union bpf_attr attr;
4554         int err;
4555
4556         if (sysctl_unprivileged_bpf_disabled && !bpf_capable())
4557                 return -EPERM;
4558
4559         err = bpf_check_uarg_tail_zero(uattr, sizeof(attr), size);
4560         if (err)
4561                 return err;
4562         size = min_t(u32, size, sizeof(attr));
4563
4564         /* copy attributes from user space, may be less than sizeof(bpf_attr) */
4565         memset(&attr, 0, sizeof(attr));
4566         if (copy_from_bpfptr(&attr, uattr, size) != 0)
4567                 return -EFAULT;
4568
4569         err = security_bpf(cmd, &attr, size);
4570         if (err < 0)
4571                 return err;
4572
4573         switch (cmd) {
4574         case BPF_MAP_CREATE:
4575                 err = map_create(&attr);
4576                 break;
4577         case BPF_MAP_LOOKUP_ELEM:
4578                 err = map_lookup_elem(&attr);
4579                 break;
4580         case BPF_MAP_UPDATE_ELEM:
4581                 err = map_update_elem(&attr, uattr);
4582                 break;
4583         case BPF_MAP_DELETE_ELEM:
4584                 err = map_delete_elem(&attr);
4585                 break;
4586         case BPF_MAP_GET_NEXT_KEY:
4587                 err = map_get_next_key(&attr);
4588                 break;
4589         case BPF_MAP_FREEZE:
4590                 err = map_freeze(&attr);
4591                 break;
4592         case BPF_PROG_LOAD:
4593                 err = bpf_prog_load(&attr, uattr);
4594                 break;
4595         case BPF_OBJ_PIN:
4596                 err = bpf_obj_pin(&attr);
4597                 break;
4598         case BPF_OBJ_GET:
4599                 err = bpf_obj_get(&attr);
4600                 break;
4601         case BPF_PROG_ATTACH:
4602                 err = bpf_prog_attach(&attr);
4603                 break;
4604         case BPF_PROG_DETACH:
4605                 err = bpf_prog_detach(&attr);
4606                 break;
4607         case BPF_PROG_QUERY:
4608                 err = bpf_prog_query(&attr, uattr.user);
4609                 break;
4610         case BPF_PROG_TEST_RUN:
4611                 err = bpf_prog_test_run(&attr, uattr.user);
4612                 break;
4613         case BPF_PROG_GET_NEXT_ID:
4614                 err = bpf_obj_get_next_id(&attr, uattr.user,
4615                                           &prog_idr, &prog_idr_lock);
4616                 break;
4617         case BPF_MAP_GET_NEXT_ID:
4618                 err = bpf_obj_get_next_id(&attr, uattr.user,
4619                                           &map_idr, &map_idr_lock);
4620                 break;
4621         case BPF_BTF_GET_NEXT_ID:
4622                 err = bpf_obj_get_next_id(&attr, uattr.user,
4623                                           &btf_idr, &btf_idr_lock);
4624                 break;
4625         case BPF_PROG_GET_FD_BY_ID:
4626                 err = bpf_prog_get_fd_by_id(&attr);
4627                 break;
4628         case BPF_MAP_GET_FD_BY_ID:
4629                 err = bpf_map_get_fd_by_id(&attr);
4630                 break;
4631         case BPF_OBJ_GET_INFO_BY_FD:
4632                 err = bpf_obj_get_info_by_fd(&attr, uattr.user);
4633                 break;
4634         case BPF_RAW_TRACEPOINT_OPEN:
4635                 err = bpf_raw_tracepoint_open(&attr);
4636                 break;
4637         case BPF_BTF_LOAD:
4638                 err = bpf_btf_load(&attr, uattr);
4639                 break;
4640         case BPF_BTF_GET_FD_BY_ID:
4641                 err = bpf_btf_get_fd_by_id(&attr);
4642                 break;
4643         case BPF_TASK_FD_QUERY:
4644                 err = bpf_task_fd_query(&attr, uattr.user);
4645                 break;
4646         case BPF_MAP_LOOKUP_AND_DELETE_ELEM:
4647                 err = map_lookup_and_delete_elem(&attr);
4648                 break;
4649         case BPF_MAP_LOOKUP_BATCH:
4650                 err = bpf_map_do_batch(&attr, uattr.user, BPF_MAP_LOOKUP_BATCH);
4651                 break;
4652         case BPF_MAP_LOOKUP_AND_DELETE_BATCH:
4653                 err = bpf_map_do_batch(&attr, uattr.user,
4654                                        BPF_MAP_LOOKUP_AND_DELETE_BATCH);
4655                 break;
4656         case BPF_MAP_UPDATE_BATCH:
4657                 err = bpf_map_do_batch(&attr, uattr.user, BPF_MAP_UPDATE_BATCH);
4658                 break;
4659         case BPF_MAP_DELETE_BATCH:
4660                 err = bpf_map_do_batch(&attr, uattr.user, BPF_MAP_DELETE_BATCH);
4661                 break;
4662         case BPF_LINK_CREATE:
4663                 err = link_create(&attr, uattr);
4664                 break;
4665         case BPF_LINK_UPDATE:
4666                 err = link_update(&attr);
4667                 break;
4668         case BPF_LINK_GET_FD_BY_ID:
4669                 err = bpf_link_get_fd_by_id(&attr);
4670                 break;
4671         case BPF_LINK_GET_NEXT_ID:
4672                 err = bpf_obj_get_next_id(&attr, uattr.user,
4673                                           &link_idr, &link_idr_lock);
4674                 break;
4675         case BPF_ENABLE_STATS:
4676                 err = bpf_enable_stats(&attr);
4677                 break;
4678         case BPF_ITER_CREATE:
4679                 err = bpf_iter_create(&attr);
4680                 break;
4681         case BPF_LINK_DETACH:
4682                 err = link_detach(&attr);
4683                 break;
4684         case BPF_PROG_BIND_MAP:
4685                 err = bpf_prog_bind_map(&attr);
4686                 break;
4687         default:
4688                 err = -EINVAL;
4689                 break;
4690         }
4691
4692         return err;
4693 }
4694
4695 SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size)
4696 {
4697         return __sys_bpf(cmd, USER_BPFPTR(uattr), size);
4698 }
4699
4700 static bool syscall_prog_is_valid_access(int off, int size,
4701                                          enum bpf_access_type type,
4702                                          const struct bpf_prog *prog,
4703                                          struct bpf_insn_access_aux *info)
4704 {
4705         if (off < 0 || off >= U16_MAX)
4706                 return false;
4707         if (off % size != 0)
4708                 return false;
4709         return true;
4710 }
4711
4712 BPF_CALL_3(bpf_sys_bpf, int, cmd, void *, attr, u32, attr_size)
4713 {
4714         switch (cmd) {
4715         case BPF_MAP_CREATE:
4716         case BPF_MAP_UPDATE_ELEM:
4717         case BPF_MAP_FREEZE:
4718         case BPF_PROG_LOAD:
4719         case BPF_BTF_LOAD:
4720                 break;
4721         /* case BPF_PROG_TEST_RUN:
4722          * is not part of this list to prevent recursive test_run
4723          */
4724         default:
4725                 return -EINVAL;
4726         }
4727         return __sys_bpf(cmd, KERNEL_BPFPTR(attr), attr_size);
4728 }
4729
4730 static const struct bpf_func_proto bpf_sys_bpf_proto = {
4731         .func           = bpf_sys_bpf,
4732         .gpl_only       = false,
4733         .ret_type       = RET_INTEGER,
4734         .arg1_type      = ARG_ANYTHING,
4735         .arg2_type      = ARG_PTR_TO_MEM,
4736         .arg3_type      = ARG_CONST_SIZE,
4737 };
4738
4739 const struct bpf_func_proto * __weak
4740 tracing_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
4741 {
4742         return bpf_base_func_proto(func_id);
4743 }
4744
4745 BPF_CALL_1(bpf_sys_close, u32, fd)
4746 {
4747         /* When bpf program calls this helper there should not be
4748          * an fdget() without matching completed fdput().
4749          * This helper is allowed in the following callchain only:
4750          * sys_bpf->prog_test_run->bpf_prog->bpf_sys_close
4751          */
4752         return close_fd(fd);
4753 }
4754
4755 static const struct bpf_func_proto bpf_sys_close_proto = {
4756         .func           = bpf_sys_close,
4757         .gpl_only       = false,
4758         .ret_type       = RET_INTEGER,
4759         .arg1_type      = ARG_ANYTHING,
4760 };
4761
4762 static const struct bpf_func_proto *
4763 syscall_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
4764 {
4765         switch (func_id) {
4766         case BPF_FUNC_sys_bpf:
4767                 return &bpf_sys_bpf_proto;
4768         case BPF_FUNC_btf_find_by_name_kind:
4769                 return &bpf_btf_find_by_name_kind_proto;
4770         case BPF_FUNC_sys_close:
4771                 return &bpf_sys_close_proto;
4772         default:
4773                 return tracing_prog_func_proto(func_id, prog);
4774         }
4775 }
4776
4777 const struct bpf_verifier_ops bpf_syscall_verifier_ops = {
4778         .get_func_proto  = syscall_prog_func_proto,
4779         .is_valid_access = syscall_prog_is_valid_access,
4780 };
4781
4782 const struct bpf_prog_ops bpf_syscall_prog_ops = {
4783         .test_run = bpf_prog_test_run_syscall,
4784 };