1 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
3 * This program is free software; you can redistribute it and/or
4 * modify it under the terms of version 2 of the GNU General Public
5 * License as published by the Free Software Foundation.
7 * This program is distributed in the hope that it will be useful, but
8 * WITHOUT ANY WARRANTY; without even the implied warranty of
9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10 * General Public License for more details.
12 #include <linux/bpf.h>
13 #include <linux/bpf_trace.h>
14 #include <linux/syscalls.h>
15 #include <linux/slab.h>
16 #include <linux/vmalloc.h>
17 #include <linux/mmzone.h>
18 #include <linux/anon_inodes.h>
19 #include <linux/file.h>
20 #include <linux/license.h>
21 #include <linux/filter.h>
22 #include <linux/version.h>
23 #include <linux/kernel.h>
25 DEFINE_PER_CPU(int, bpf_prog_active);
27 int sysctl_unprivileged_bpf_disabled __read_mostly;
29 static LIST_HEAD(bpf_map_types);
31 static struct bpf_map *find_and_alloc_map(union bpf_attr *attr)
33 struct bpf_map_type_list *tl;
36 list_for_each_entry(tl, &bpf_map_types, list_node) {
37 if (tl->type == attr->map_type) {
38 map = tl->ops->map_alloc(attr);
42 map->map_type = attr->map_type;
46 return ERR_PTR(-EINVAL);
49 /* boot time registration of different map implementations */
50 void bpf_register_map_type(struct bpf_map_type_list *tl)
52 list_add(&tl->list_node, &bpf_map_types);
55 void *bpf_map_area_alloc(size_t size)
57 /* We definitely need __GFP_NORETRY, so OOM killer doesn't
58 * trigger under memory pressure as we really just want to
61 const gfp_t flags = __GFP_NOWARN | __GFP_NORETRY | __GFP_ZERO;
64 if (size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) {
65 area = kmalloc(size, GFP_USER | flags);
70 return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM | flags,
74 void bpf_map_area_free(void *area)
79 int bpf_map_precharge_memlock(u32 pages)
81 struct user_struct *user = get_current_user();
82 unsigned long memlock_limit, cur;
84 memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
85 cur = atomic_long_read(&user->locked_vm);
87 if (cur + pages > memlock_limit)
92 static int bpf_map_charge_memlock(struct bpf_map *map)
94 struct user_struct *user = get_current_user();
95 unsigned long memlock_limit;
97 memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
99 atomic_long_add(map->pages, &user->locked_vm);
101 if (atomic_long_read(&user->locked_vm) > memlock_limit) {
102 atomic_long_sub(map->pages, &user->locked_vm);
110 static void bpf_map_uncharge_memlock(struct bpf_map *map)
112 struct user_struct *user = map->user;
114 atomic_long_sub(map->pages, &user->locked_vm);
118 /* called from workqueue */
119 static void bpf_map_free_deferred(struct work_struct *work)
121 struct bpf_map *map = container_of(work, struct bpf_map, work);
123 bpf_map_uncharge_memlock(map);
124 /* implementation dependent freeing */
125 map->ops->map_free(map);
128 static void bpf_map_put_uref(struct bpf_map *map)
130 if (atomic_dec_and_test(&map->usercnt)) {
131 if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY)
132 bpf_fd_array_map_clear(map);
136 /* decrement map refcnt and schedule it for freeing via workqueue
137 * (unrelying map implementation ops->map_free() might sleep)
139 void bpf_map_put(struct bpf_map *map)
141 if (atomic_dec_and_test(&map->refcnt)) {
142 INIT_WORK(&map->work, bpf_map_free_deferred);
143 schedule_work(&map->work);
147 void bpf_map_put_with_uref(struct bpf_map *map)
149 bpf_map_put_uref(map);
153 static int bpf_map_release(struct inode *inode, struct file *filp)
155 struct bpf_map *map = filp->private_data;
157 if (map->ops->map_release)
158 map->ops->map_release(map, filp);
160 bpf_map_put_with_uref(map);
164 #ifdef CONFIG_PROC_FS
165 static void bpf_map_show_fdinfo(struct seq_file *m, struct file *filp)
167 const struct bpf_map *map = filp->private_data;
168 const struct bpf_array *array;
169 u32 owner_prog_type = 0;
171 if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY) {
172 array = container_of(map, struct bpf_array, map);
173 owner_prog_type = array->owner_prog_type;
188 map->pages * 1ULL << PAGE_SHIFT);
191 seq_printf(m, "owner_prog_type:\t%u\n",
196 static const struct file_operations bpf_map_fops = {
197 #ifdef CONFIG_PROC_FS
198 .show_fdinfo = bpf_map_show_fdinfo,
200 .release = bpf_map_release,
203 int bpf_map_new_fd(struct bpf_map *map)
205 return anon_inode_getfd("bpf-map", &bpf_map_fops, map,
209 /* helper macro to check that unused fields 'union bpf_attr' are zero */
210 #define CHECK_ATTR(CMD) \
211 memchr_inv((void *) &attr->CMD##_LAST_FIELD + \
212 sizeof(attr->CMD##_LAST_FIELD), 0, \
214 offsetof(union bpf_attr, CMD##_LAST_FIELD) - \
215 sizeof(attr->CMD##_LAST_FIELD)) != NULL
217 #define BPF_MAP_CREATE_LAST_FIELD map_flags
218 /* called via syscall */
219 static int map_create(union bpf_attr *attr)
224 err = CHECK_ATTR(BPF_MAP_CREATE);
228 /* find map type and init map: hashtable vs rbtree vs bloom vs ... */
229 map = find_and_alloc_map(attr);
233 atomic_set(&map->refcnt, 1);
234 atomic_set(&map->usercnt, 1);
236 err = bpf_map_charge_memlock(map);
238 goto free_map_nouncharge;
240 err = bpf_map_new_fd(map);
242 /* failed to allocate fd */
245 trace_bpf_map_create(map, err);
249 bpf_map_uncharge_memlock(map);
251 map->ops->map_free(map);
255 /* if error is returned, fd is released.
256 * On success caller should complete fd access with matching fdput()
258 struct bpf_map *__bpf_map_get(struct fd f)
261 return ERR_PTR(-EBADF);
262 if (f.file->f_op != &bpf_map_fops) {
264 return ERR_PTR(-EINVAL);
267 return f.file->private_data;
270 /* prog's and map's refcnt limit */
271 #define BPF_MAX_REFCNT 32768
273 struct bpf_map *bpf_map_inc(struct bpf_map *map, bool uref)
275 if (atomic_inc_return(&map->refcnt) > BPF_MAX_REFCNT) {
276 atomic_dec(&map->refcnt);
277 return ERR_PTR(-EBUSY);
280 atomic_inc(&map->usercnt);
284 struct bpf_map *bpf_map_get_with_uref(u32 ufd)
286 struct fd f = fdget(ufd);
289 map = __bpf_map_get(f);
293 map = bpf_map_inc(map, true);
299 int __weak bpf_stackmap_copy(struct bpf_map *map, void *key, void *value)
304 /* last field in 'union bpf_attr' used by this command */
305 #define BPF_MAP_LOOKUP_ELEM_LAST_FIELD value
307 static int map_lookup_elem(union bpf_attr *attr)
309 void __user *ukey = u64_to_user_ptr(attr->key);
310 void __user *uvalue = u64_to_user_ptr(attr->value);
311 int ufd = attr->map_fd;
313 void *key, *value, *ptr;
318 if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM))
322 map = __bpf_map_get(f);
327 key = kmalloc(map->key_size, GFP_USER);
332 if (copy_from_user(key, ukey, map->key_size) != 0)
335 if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
336 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||
337 map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY)
338 value_size = round_up(map->value_size, 8) * num_possible_cpus();
340 value_size = map->value_size;
343 value = kmalloc(value_size, GFP_USER | __GFP_NOWARN);
347 if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
348 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
349 err = bpf_percpu_hash_copy(map, key, value);
350 } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
351 err = bpf_percpu_array_copy(map, key, value);
352 } else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) {
353 err = bpf_stackmap_copy(map, key, value);
356 ptr = map->ops->map_lookup_elem(map, key);
358 memcpy(value, ptr, value_size);
360 err = ptr ? 0 : -ENOENT;
367 if (copy_to_user(uvalue, value, value_size) != 0)
370 trace_bpf_map_lookup_elem(map, ufd, key, value);
382 #define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags
384 static int map_update_elem(union bpf_attr *attr)
386 void __user *ukey = u64_to_user_ptr(attr->key);
387 void __user *uvalue = u64_to_user_ptr(attr->value);
388 int ufd = attr->map_fd;
395 if (CHECK_ATTR(BPF_MAP_UPDATE_ELEM))
399 map = __bpf_map_get(f);
404 key = kmalloc(map->key_size, GFP_USER);
409 if (copy_from_user(key, ukey, map->key_size) != 0)
412 if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
413 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||
414 map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY)
415 value_size = round_up(map->value_size, 8) * num_possible_cpus();
417 value_size = map->value_size;
420 value = kmalloc(value_size, GFP_USER | __GFP_NOWARN);
425 if (copy_from_user(value, uvalue, value_size) != 0)
428 /* must increment bpf_prog_active to avoid kprobe+bpf triggering from
429 * inside bpf map update or delete otherwise deadlocks are possible
432 __this_cpu_inc(bpf_prog_active);
433 if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
434 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
435 err = bpf_percpu_hash_update(map, key, value, attr->flags);
436 } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
437 err = bpf_percpu_array_update(map, key, value, attr->flags);
438 } else if (map->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY ||
439 map->map_type == BPF_MAP_TYPE_PROG_ARRAY ||
440 map->map_type == BPF_MAP_TYPE_CGROUP_ARRAY) {
442 err = bpf_fd_array_map_update_elem(map, f.file, key, value,
447 err = map->ops->map_update_elem(map, key, value, attr->flags);
450 __this_cpu_dec(bpf_prog_active);
454 trace_bpf_map_update_elem(map, ufd, key, value);
464 #define BPF_MAP_DELETE_ELEM_LAST_FIELD key
466 static int map_delete_elem(union bpf_attr *attr)
468 void __user *ukey = u64_to_user_ptr(attr->key);
469 int ufd = attr->map_fd;
475 if (CHECK_ATTR(BPF_MAP_DELETE_ELEM))
479 map = __bpf_map_get(f);
484 key = kmalloc(map->key_size, GFP_USER);
489 if (copy_from_user(key, ukey, map->key_size) != 0)
493 __this_cpu_inc(bpf_prog_active);
495 err = map->ops->map_delete_elem(map, key);
497 __this_cpu_dec(bpf_prog_active);
501 trace_bpf_map_delete_elem(map, ufd, key);
509 /* last field in 'union bpf_attr' used by this command */
510 #define BPF_MAP_GET_NEXT_KEY_LAST_FIELD next_key
512 static int map_get_next_key(union bpf_attr *attr)
514 void __user *ukey = u64_to_user_ptr(attr->key);
515 void __user *unext_key = u64_to_user_ptr(attr->next_key);
516 int ufd = attr->map_fd;
518 void *key, *next_key;
522 if (CHECK_ATTR(BPF_MAP_GET_NEXT_KEY))
526 map = __bpf_map_get(f);
531 key = kmalloc(map->key_size, GFP_USER);
536 if (copy_from_user(key, ukey, map->key_size) != 0)
540 next_key = kmalloc(map->key_size, GFP_USER);
545 err = map->ops->map_get_next_key(map, key, next_key);
551 if (copy_to_user(unext_key, next_key, map->key_size) != 0)
554 trace_bpf_map_next_key(map, ufd, key, next_key);
566 static LIST_HEAD(bpf_prog_types);
568 static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog)
570 struct bpf_prog_type_list *tl;
572 list_for_each_entry(tl, &bpf_prog_types, list_node) {
573 if (tl->type == type) {
574 prog->aux->ops = tl->ops;
583 void bpf_register_prog_type(struct bpf_prog_type_list *tl)
585 list_add(&tl->list_node, &bpf_prog_types);
588 /* fixup insn->imm field of bpf_call instructions:
589 * if (insn->imm == BPF_FUNC_map_lookup_elem)
590 * insn->imm = bpf_map_lookup_elem - __bpf_call_base;
591 * else if (insn->imm == BPF_FUNC_map_update_elem)
592 * insn->imm = bpf_map_update_elem - __bpf_call_base;
595 * this function is called after eBPF program passed verification
597 static void fixup_bpf_calls(struct bpf_prog *prog)
599 const struct bpf_func_proto *fn;
602 for (i = 0; i < prog->len; i++) {
603 struct bpf_insn *insn = &prog->insnsi[i];
605 if (insn->code == (BPF_JMP | BPF_CALL)) {
606 /* we reach here when program has bpf_call instructions
607 * and it passed bpf_check(), means that
608 * ops->get_func_proto must have been supplied, check it
610 BUG_ON(!prog->aux->ops->get_func_proto);
612 if (insn->imm == BPF_FUNC_get_route_realm)
613 prog->dst_needed = 1;
614 if (insn->imm == BPF_FUNC_get_prandom_u32)
615 bpf_user_rnd_init_once();
616 if (insn->imm == BPF_FUNC_xdp_adjust_head)
617 prog->xdp_adjust_head = 1;
618 if (insn->imm == BPF_FUNC_tail_call) {
619 /* mark bpf_tail_call as different opcode
620 * to avoid conditional branch in
621 * interpeter for every normal call
622 * and to prevent accidental JITing by
623 * JIT compiler that doesn't support
631 fn = prog->aux->ops->get_func_proto(insn->imm);
632 /* all functions that have prototype and verifier allowed
633 * programs to call them, must be real in-kernel functions
636 insn->imm = fn->func - __bpf_call_base;
641 /* drop refcnt on maps used by eBPF program and free auxilary data */
642 static void free_used_maps(struct bpf_prog_aux *aux)
646 for (i = 0; i < aux->used_map_cnt; i++)
647 bpf_map_put(aux->used_maps[i]);
649 kfree(aux->used_maps);
652 int __bpf_prog_charge(struct user_struct *user, u32 pages)
654 unsigned long memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
655 unsigned long user_bufs;
658 user_bufs = atomic_long_add_return(pages, &user->locked_vm);
659 if (user_bufs > memlock_limit) {
660 atomic_long_sub(pages, &user->locked_vm);
668 void __bpf_prog_uncharge(struct user_struct *user, u32 pages)
671 atomic_long_sub(pages, &user->locked_vm);
674 static int bpf_prog_charge_memlock(struct bpf_prog *prog)
676 struct user_struct *user = get_current_user();
679 ret = __bpf_prog_charge(user, prog->pages);
685 prog->aux->user = user;
689 static void bpf_prog_uncharge_memlock(struct bpf_prog *prog)
691 struct user_struct *user = prog->aux->user;
693 __bpf_prog_uncharge(user, prog->pages);
697 static void __bpf_prog_put_rcu(struct rcu_head *rcu)
699 struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu);
702 bpf_prog_uncharge_memlock(aux->prog);
703 bpf_prog_free(aux->prog);
706 void bpf_prog_put(struct bpf_prog *prog)
708 if (atomic_dec_and_test(&prog->aux->refcnt)) {
709 trace_bpf_prog_put_rcu(prog);
710 bpf_prog_kallsyms_del(prog);
711 call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu);
714 EXPORT_SYMBOL_GPL(bpf_prog_put);
716 static int bpf_prog_release(struct inode *inode, struct file *filp)
718 struct bpf_prog *prog = filp->private_data;
724 #ifdef CONFIG_PROC_FS
725 static void bpf_prog_show_fdinfo(struct seq_file *m, struct file *filp)
727 const struct bpf_prog *prog = filp->private_data;
728 char prog_tag[sizeof(prog->tag) * 2 + 1] = { };
730 bin2hex(prog_tag, prog->tag, sizeof(prog->tag));
739 prog->pages * 1ULL << PAGE_SHIFT);
743 static const struct file_operations bpf_prog_fops = {
744 #ifdef CONFIG_PROC_FS
745 .show_fdinfo = bpf_prog_show_fdinfo,
747 .release = bpf_prog_release,
750 int bpf_prog_new_fd(struct bpf_prog *prog)
752 return anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog,
756 static struct bpf_prog *____bpf_prog_get(struct fd f)
759 return ERR_PTR(-EBADF);
760 if (f.file->f_op != &bpf_prog_fops) {
762 return ERR_PTR(-EINVAL);
765 return f.file->private_data;
768 struct bpf_prog *bpf_prog_add(struct bpf_prog *prog, int i)
770 if (atomic_add_return(i, &prog->aux->refcnt) > BPF_MAX_REFCNT) {
771 atomic_sub(i, &prog->aux->refcnt);
772 return ERR_PTR(-EBUSY);
776 EXPORT_SYMBOL_GPL(bpf_prog_add);
778 void bpf_prog_sub(struct bpf_prog *prog, int i)
780 /* Only to be used for undoing previous bpf_prog_add() in some
781 * error path. We still know that another entity in our call
782 * path holds a reference to the program, thus atomic_sub() can
783 * be safely used in such cases!
785 WARN_ON(atomic_sub_return(i, &prog->aux->refcnt) == 0);
787 EXPORT_SYMBOL_GPL(bpf_prog_sub);
789 struct bpf_prog *bpf_prog_inc(struct bpf_prog *prog)
791 return bpf_prog_add(prog, 1);
793 EXPORT_SYMBOL_GPL(bpf_prog_inc);
795 static struct bpf_prog *__bpf_prog_get(u32 ufd, enum bpf_prog_type *type)
797 struct fd f = fdget(ufd);
798 struct bpf_prog *prog;
800 prog = ____bpf_prog_get(f);
803 if (type && prog->type != *type) {
804 prog = ERR_PTR(-EINVAL);
808 prog = bpf_prog_inc(prog);
814 struct bpf_prog *bpf_prog_get(u32 ufd)
816 return __bpf_prog_get(ufd, NULL);
819 struct bpf_prog *bpf_prog_get_type(u32 ufd, enum bpf_prog_type type)
821 struct bpf_prog *prog = __bpf_prog_get(ufd, &type);
824 trace_bpf_prog_get_type(prog);
827 EXPORT_SYMBOL_GPL(bpf_prog_get_type);
829 /* last field in 'union bpf_attr' used by this command */
830 #define BPF_PROG_LOAD_LAST_FIELD kern_version
832 static int bpf_prog_load(union bpf_attr *attr)
834 enum bpf_prog_type type = attr->prog_type;
835 struct bpf_prog *prog;
840 if (CHECK_ATTR(BPF_PROG_LOAD))
843 /* copy eBPF program license from user space */
844 if (strncpy_from_user(license, u64_to_user_ptr(attr->license),
845 sizeof(license) - 1) < 0)
847 license[sizeof(license) - 1] = 0;
849 /* eBPF programs must be GPL compatible to use GPL-ed functions */
850 is_gpl = license_is_gpl_compatible(license);
852 if (attr->insn_cnt == 0 || attr->insn_cnt > BPF_MAXINSNS)
855 if (type == BPF_PROG_TYPE_KPROBE &&
856 attr->kern_version != LINUX_VERSION_CODE)
859 if (type != BPF_PROG_TYPE_SOCKET_FILTER && !capable(CAP_SYS_ADMIN))
862 /* plain bpf_prog allocation */
863 prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER);
867 err = bpf_prog_charge_memlock(prog);
869 goto free_prog_nouncharge;
871 prog->len = attr->insn_cnt;
874 if (copy_from_user(prog->insns, u64_to_user_ptr(attr->insns),
875 bpf_prog_insn_size(prog)) != 0)
878 prog->orig_prog = NULL;
881 atomic_set(&prog->aux->refcnt, 1);
882 prog->gpl_compatible = is_gpl ? 1 : 0;
884 /* find program type: socket_filter vs tracing_filter */
885 err = find_prog_type(type, prog);
889 /* run eBPF verifier */
890 err = bpf_check(&prog, attr);
894 /* fixup BPF_CALL->imm field */
895 fixup_bpf_calls(prog);
897 /* eBPF program is ready to be JITed */
898 prog = bpf_prog_select_runtime(prog, &err);
902 err = bpf_prog_new_fd(prog);
904 /* failed to allocate fd */
907 bpf_prog_kallsyms_add(prog);
908 trace_bpf_prog_load(prog, err);
912 free_used_maps(prog->aux);
914 bpf_prog_uncharge_memlock(prog);
915 free_prog_nouncharge:
920 #define BPF_OBJ_LAST_FIELD bpf_fd
922 static int bpf_obj_pin(const union bpf_attr *attr)
924 if (CHECK_ATTR(BPF_OBJ))
927 return bpf_obj_pin_user(attr->bpf_fd, u64_to_user_ptr(attr->pathname));
930 static int bpf_obj_get(const union bpf_attr *attr)
932 if (CHECK_ATTR(BPF_OBJ) || attr->bpf_fd != 0)
935 return bpf_obj_get_user(u64_to_user_ptr(attr->pathname));
938 #ifdef CONFIG_CGROUP_BPF
940 #define BPF_PROG_ATTACH_LAST_FIELD attach_flags
942 static int bpf_prog_attach(const union bpf_attr *attr)
944 enum bpf_prog_type ptype;
945 struct bpf_prog *prog;
949 if (!capable(CAP_NET_ADMIN))
952 if (CHECK_ATTR(BPF_PROG_ATTACH))
955 if (attr->attach_flags & ~BPF_F_ALLOW_OVERRIDE)
958 switch (attr->attach_type) {
959 case BPF_CGROUP_INET_INGRESS:
960 case BPF_CGROUP_INET_EGRESS:
961 ptype = BPF_PROG_TYPE_CGROUP_SKB;
963 case BPF_CGROUP_INET_SOCK_CREATE:
964 ptype = BPF_PROG_TYPE_CGROUP_SOCK;
970 prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype);
972 return PTR_ERR(prog);
974 cgrp = cgroup_get_from_fd(attr->target_fd);
977 return PTR_ERR(cgrp);
980 ret = cgroup_bpf_update(cgrp, prog, attr->attach_type,
981 attr->attach_flags & BPF_F_ALLOW_OVERRIDE);
989 #define BPF_PROG_DETACH_LAST_FIELD attach_type
991 static int bpf_prog_detach(const union bpf_attr *attr)
996 if (!capable(CAP_NET_ADMIN))
999 if (CHECK_ATTR(BPF_PROG_DETACH))
1002 switch (attr->attach_type) {
1003 case BPF_CGROUP_INET_INGRESS:
1004 case BPF_CGROUP_INET_EGRESS:
1005 case BPF_CGROUP_INET_SOCK_CREATE:
1006 cgrp = cgroup_get_from_fd(attr->target_fd);
1008 return PTR_ERR(cgrp);
1010 ret = cgroup_bpf_update(cgrp, NULL, attr->attach_type, false);
1020 #endif /* CONFIG_CGROUP_BPF */
1022 SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size)
1024 union bpf_attr attr = {};
1027 if (!capable(CAP_SYS_ADMIN) && sysctl_unprivileged_bpf_disabled)
1030 if (!access_ok(VERIFY_READ, uattr, 1))
1033 if (size > PAGE_SIZE) /* silly large */
1036 /* If we're handed a bigger struct than we know of,
1037 * ensure all the unknown bits are 0 - i.e. new
1038 * user-space does not rely on any kernel feature
1039 * extensions we dont know about yet.
1041 if (size > sizeof(attr)) {
1042 unsigned char __user *addr;
1043 unsigned char __user *end;
1046 addr = (void __user *)uattr + sizeof(attr);
1047 end = (void __user *)uattr + size;
1049 for (; addr < end; addr++) {
1050 err = get_user(val, addr);
1056 size = sizeof(attr);
1059 /* copy attributes from user space, may be less than sizeof(bpf_attr) */
1060 if (copy_from_user(&attr, uattr, size) != 0)
1064 case BPF_MAP_CREATE:
1065 err = map_create(&attr);
1067 case BPF_MAP_LOOKUP_ELEM:
1068 err = map_lookup_elem(&attr);
1070 case BPF_MAP_UPDATE_ELEM:
1071 err = map_update_elem(&attr);
1073 case BPF_MAP_DELETE_ELEM:
1074 err = map_delete_elem(&attr);
1076 case BPF_MAP_GET_NEXT_KEY:
1077 err = map_get_next_key(&attr);
1080 err = bpf_prog_load(&attr);
1083 err = bpf_obj_pin(&attr);
1086 err = bpf_obj_get(&attr);
1089 #ifdef CONFIG_CGROUP_BPF
1090 case BPF_PROG_ATTACH:
1091 err = bpf_prog_attach(&attr);
1093 case BPF_PROG_DETACH:
1094 err = bpf_prog_detach(&attr);