Merge branch 'perf-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git...

[linux-2.6-microblaze.git] / kernel / bpf / syscall.c

diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index d7185c8..62f6bce 100644 (file)
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -463,7 +463,7 @@ int map_check_no_btf(const struct bpf_map *map,
        return -ENOTSUPP;
 }
 
-static int map_check_btf(const struct bpf_map *map, const struct btf *btf,
+static int map_check_btf(struct bpf_map *map, const struct btf *btf,
                         u32 btf_key_id, u32 btf_value_id)
 {
        const struct btf_type *key_type, *value_type;
@@ -478,6 +478,22 @@ static int map_check_btf(const struct bpf_map *map, const struct btf *btf,
        if (!value_type || value_size != map->value_size)
                return -EINVAL;
 
+       map->spin_lock_off = btf_find_spin_lock(btf, value_type);
+
+       if (map_value_has_spin_lock(map)) {
+               if (map->map_type != BPF_MAP_TYPE_HASH &&
+                   map->map_type != BPF_MAP_TYPE_ARRAY &&
+                   map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE)
+                       return -ENOTSUPP;
+               if (map->spin_lock_off + sizeof(struct bpf_spin_lock) >
+                   map->value_size) {
+                       WARN_ONCE(1,
+                                 "verifier bug spin_lock_off %d value_size %d\n",
+                                 map->spin_lock_off, map->value_size);
+                       return -EFAULT;
+               }
+       }
+
        if (map->ops->map_check_btf)
                ret = map->ops->map_check_btf(map, btf, key_type, value_type);
 
@@ -542,6 +558,8 @@ static int map_create(union bpf_attr *attr)
                map->btf = btf;
                map->btf_key_type_id = attr->btf_key_type_id;
                map->btf_value_type_id = attr->btf_value_type_id;
+       } else {
+               map->spin_lock_off = -EINVAL;
        }
 
        err = security_bpf_map_alloc(map);
@@ -559,12 +577,12 @@ static int map_create(union bpf_attr *attr)
        err = bpf_map_new_fd(map, f_flags);
        if (err < 0) {
                /* failed to allocate fd.
-                * bpf_map_put() is needed because the above
+                * bpf_map_put_with_uref() is needed because the above
                 * bpf_map_alloc_id() has published the map
                 * to the userspace and the userspace may
                 * have refcnt-ed it through BPF_MAP_GET_FD_BY_ID.
                 */
-               bpf_map_put(map);
+               bpf_map_put_with_uref(map);
                return err;
        }
 
@@ -664,7 +682,7 @@ static void *__bpf_copy_key(void __user *ukey, u64 key_size)
 }
 
 /* last field in 'union bpf_attr' used by this command */
-#define BPF_MAP_LOOKUP_ELEM_LAST_FIELD value
+#define BPF_MAP_LOOKUP_ELEM_LAST_FIELD flags
 
 static int map_lookup_elem(union bpf_attr *attr)
 {
@@ -680,6 +698,9 @@ static int map_lookup_elem(union bpf_attr *attr)
        if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM))
                return -EINVAL;
 
+       if (attr->flags & ~BPF_F_LOCK)
+               return -EINVAL;
+
        f = fdget(ufd);
        map = __bpf_map_get(f);
        if (IS_ERR(map))
@@ -690,6 +711,12 @@ static int map_lookup_elem(union bpf_attr *attr)
                goto err_put;
        }
 
+       if ((attr->flags & BPF_F_LOCK) &&
+           !map_value_has_spin_lock(map)) {
+               err = -EINVAL;
+               goto err_put;
+       }
+
        key = __bpf_copy_key(ukey, map->key_size);
        if (IS_ERR(key)) {
                err = PTR_ERR(key);
@@ -745,7 +772,13 @@ static int map_lookup_elem(union bpf_attr *attr)
                        err = -ENOENT;
                } else {
                        err = 0;
-                       memcpy(value, ptr, value_size);
+                       if (attr->flags & BPF_F_LOCK)
+                               /* lock 'ptr' and copy everything but lock */
+                               copy_map_value_locked(map, value, ptr, true);
+                       else
+                               copy_map_value(map, value, ptr);
+                       /* mask lock, since value wasn't zero inited */
+                       check_and_init_map_lock(map, value);
                }
                rcu_read_unlock();
        }
@@ -808,6 +841,12 @@ static int map_update_elem(union bpf_attr *attr)
                goto err_put;
        }
 
+       if ((attr->flags & BPF_F_LOCK) &&
+           !map_value_has_spin_lock(map)) {
+               err = -EINVAL;
+               goto err_put;
+       }
+
        key = __bpf_copy_key(ukey, map->key_size);
        if (IS_ERR(key)) {
                err = PTR_ERR(key);
@@ -1245,24 +1284,54 @@ static int bpf_prog_release(struct inode *inode, struct file *filp)
        return 0;
 }
 
+static void bpf_prog_get_stats(const struct bpf_prog *prog,
+                              struct bpf_prog_stats *stats)
+{
+       u64 nsecs = 0, cnt = 0;
+       int cpu;
+
+       for_each_possible_cpu(cpu) {
+               const struct bpf_prog_stats *st;
+               unsigned int start;
+               u64 tnsecs, tcnt;
+
+               st = per_cpu_ptr(prog->aux->stats, cpu);
+               do {
+                       start = u64_stats_fetch_begin_irq(&st->syncp);
+                       tnsecs = st->nsecs;
+                       tcnt = st->cnt;
+               } while (u64_stats_fetch_retry_irq(&st->syncp, start));
+               nsecs += tnsecs;
+               cnt += tcnt;
+       }
+       stats->nsecs = nsecs;
+       stats->cnt = cnt;
+}
+
 #ifdef CONFIG_PROC_FS
 static void bpf_prog_show_fdinfo(struct seq_file *m, struct file *filp)
 {
        const struct bpf_prog *prog = filp->private_data;
        char prog_tag[sizeof(prog->tag) * 2 + 1] = { };
+       struct bpf_prog_stats stats;
 
+       bpf_prog_get_stats(prog, &stats);
        bin2hex(prog_tag, prog->tag, sizeof(prog->tag));
        seq_printf(m,
                   "prog_type:\t%u\n"
                   "prog_jited:\t%u\n"
                   "prog_tag:\t%s\n"
                   "memlock:\t%llu\n"
-                  "prog_id:\t%u\n",
+                  "prog_id:\t%u\n"
+                  "run_time_ns:\t%llu\n"
+                  "run_cnt:\t%llu\n",
                   prog->type,
                   prog->jited,
                   prog_tag,
                   prog->pages * 1ULL << PAGE_SHIFT,
-                  prog->aux->id);
+                  prog->aux->id,
+                  stats.nsecs,
+                  stats.cnt);
 }
 #endif
 
@@ -1988,7 +2057,7 @@ static int bpf_map_get_fd_by_id(const union bpf_attr *attr)
 
        fd = bpf_map_new_fd(map, f_flags);
        if (fd < 0)
-               bpf_map_put(map);
+               bpf_map_put_with_uref(map);
 
        return fd;
 }
@@ -2085,6 +2154,7 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog,
        struct bpf_prog_info __user *uinfo = u64_to_user_ptr(attr->info.info);
        struct bpf_prog_info info = {};
        u32 info_len = attr->info.info_len;
+       struct bpf_prog_stats stats;
        char __user *uinsns;
        u32 ulen;
        int err;
@@ -2124,6 +2194,10 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog,
        if (err)
                return err;
 
+       bpf_prog_get_stats(prog, &stats);
+       info.run_time_ns = stats.nsecs;
+       info.run_cnt = stats.cnt;
+
        if (!capable(CAP_SYS_ADMIN)) {
                info.jited_prog_len = 0;
                info.xlated_prog_len = 0;