Merge tag 'perf-core-2021-10-31' of git://git.kernel.org/pub/scm/linux/kernel/git...

[linux-2.6-microblaze.git] / kernel / events / core.c

diff --git a/kernel/events/core.c b/kernel/events/core.c
index 0e90a50..a89b01b 100644 (file)
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -3707,6 +3707,29 @@ static noinline int visit_groups_merge(struct perf_cpu_context *cpuctx,
        return 0;
 }
 
+static inline bool event_update_userpage(struct perf_event *event)
+{
+       if (likely(!atomic_read(&event->mmap_count)))
+               return false;
+
+       perf_event_update_time(event);
+       perf_set_shadow_time(event, event->ctx);
+       perf_event_update_userpage(event);
+
+       return true;
+}
+
+static inline void group_update_userpage(struct perf_event *group_event)
+{
+       struct perf_event *event;
+
+       if (!event_update_userpage(group_event))
+               return;
+
+       for_each_sibling_event(event, group_event)
+               event_update_userpage(event);
+}
+
 static int merge_sched_in(struct perf_event *event, void *data)
 {
        struct perf_event_context *ctx = event->ctx;
@@ -3725,14 +3748,15 @@ static int merge_sched_in(struct perf_event *event, void *data)
        }
 
        if (event->state == PERF_EVENT_STATE_INACTIVE) {
+               *can_add_hw = 0;
                if (event->attr.pinned) {
                        perf_cgroup_event_disable(event, ctx);
                        perf_event_set_state(event, PERF_EVENT_STATE_ERROR);
+               } else {
+                       ctx->rotate_necessary = 1;
+                       perf_mux_hrtimer_restart(cpuctx);
+                       group_update_userpage(event);
                }
-
-               *can_add_hw = 0;
-               ctx->rotate_necessary = 1;
-               perf_mux_hrtimer_restart(cpuctx);
        }
 
        return 0;
@@ -4697,7 +4721,6 @@ errout:
 }
 
 static void perf_event_free_filter(struct perf_event *event);
-static void perf_event_free_bpf_prog(struct perf_event *event);
 
 static void free_event_rcu(struct rcu_head *head)
 {
@@ -5574,7 +5597,6 @@ static inline int perf_fget_light(int fd, struct fd *p)
 static int perf_event_set_output(struct perf_event *event,
                                 struct perf_event *output_event);
 static int perf_event_set_filter(struct perf_event *event, void __user *arg);
-static int perf_event_set_bpf_prog(struct perf_event *event, u32 prog_fd);
 static int perf_copy_attr(struct perf_event_attr __user *uattr,
                          struct perf_event_attr *attr);
 
@@ -5637,7 +5659,22 @@ static long _perf_ioctl(struct perf_event *event, unsigned int cmd, unsigned lon
                return perf_event_set_filter(event, (void __user *)arg);
 
        case PERF_EVENT_IOC_SET_BPF:
-               return perf_event_set_bpf_prog(event, arg);
+       {
+               struct bpf_prog *prog;
+               int err;
+
+               prog = bpf_prog_get(arg);
+               if (IS_ERR(prog))
+                       return PTR_ERR(prog);
+
+               err = perf_event_set_bpf_prog(event, prog, 0);
+               if (err) {
+                       bpf_prog_put(prog);
+                       return err;
+               }
+
+               return 0;
+       }
 
        case PERF_EVENT_IOC_PAUSE_OUTPUT: {
                struct perf_buffer *rb;
@@ -6311,6 +6348,8 @@ accounting:
 
                ring_buffer_attach(event, rb);
 
+               perf_event_update_time(event);
+               perf_set_shadow_time(event, event->ctx);
                perf_event_init_userpage(event);
                perf_event_update_userpage(event);
        } else {
@@ -8307,8 +8346,6 @@ static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
        else
                flags = MAP_PRIVATE;
 
-       if (vma->vm_flags & VM_DENYWRITE)
-               flags |= MAP_DENYWRITE;
        if (vma->vm_flags & VM_LOCKED)
                flags |= MAP_LOCKED;
        if (is_vm_hugetlb_page(vma))
@@ -9937,13 +9974,16 @@ static void bpf_overflow_handler(struct perf_event *event,
                .data = data,
                .event = event,
        };
+       struct bpf_prog *prog;
        int ret = 0;
 
        ctx.regs = perf_arch_bpf_user_pt_regs(regs);
        if (unlikely(__this_cpu_inc_return(bpf_prog_active) != 1))
                goto out;
        rcu_read_lock();
-       ret = BPF_PROG_RUN(event->prog, &ctx);
+       prog = READ_ONCE(event->prog);
+       if (prog)
+               ret = bpf_prog_run(prog, &ctx);
        rcu_read_unlock();
 out:
        __this_cpu_dec(bpf_prog_active);
@@ -9953,10 +9993,10 @@ out:
        event->orig_overflow_handler(event, data, regs);
 }
 
-static int perf_event_set_bpf_handler(struct perf_event *event, u32 prog_fd)
+static int perf_event_set_bpf_handler(struct perf_event *event,
+                                     struct bpf_prog *prog,
+                                     u64 bpf_cookie)
 {
-       struct bpf_prog *prog;
-
        if (event->overflow_handler_context)
                /* hw breakpoint or kernel counter */
                return -EINVAL;
@@ -9964,9 +10004,8 @@ static int perf_event_set_bpf_handler(struct perf_event *event, u32 prog_fd)
        if (event->prog)
                return -EEXIST;
 
-       prog = bpf_prog_get_type(prog_fd, BPF_PROG_TYPE_PERF_EVENT);
-       if (IS_ERR(prog))
-               return PTR_ERR(prog);
+       if (prog->type != BPF_PROG_TYPE_PERF_EVENT)
+               return -EINVAL;
 
        if (event->attr.precise_ip &&
            prog->call_get_stack &&
@@ -9982,11 +10021,11 @@ static int perf_event_set_bpf_handler(struct perf_event *event, u32 prog_fd)
                 * attached to perf_sample_data, do not allow attaching BPF
                 * program that calls bpf_get_[stack|stackid].
                 */
-               bpf_prog_put(prog);
                return -EPROTO;
        }
 
        event->prog = prog;
+       event->bpf_cookie = bpf_cookie;
        event->orig_overflow_handler = READ_ONCE(event->overflow_handler);
        WRITE_ONCE(event->overflow_handler, bpf_overflow_handler);
        return 0;
@@ -10004,7 +10043,9 @@ static void perf_event_free_bpf_handler(struct perf_event *event)
        bpf_prog_put(prog);
 }
 #else
-static int perf_event_set_bpf_handler(struct perf_event *event, u32 prog_fd)
+static int perf_event_set_bpf_handler(struct perf_event *event,
+                                     struct bpf_prog *prog,
+                                     u64 bpf_cookie)
 {
        return -EOPNOTSUPP;
 }
@@ -10032,14 +10073,13 @@ static inline bool perf_event_is_tracing(struct perf_event *event)
        return false;
 }
 
-static int perf_event_set_bpf_prog(struct perf_event *event, u32 prog_fd)
+int perf_event_set_bpf_prog(struct perf_event *event, struct bpf_prog *prog,
+                           u64 bpf_cookie)
 {
        bool is_kprobe, is_tracepoint, is_syscall_tp;
-       struct bpf_prog *prog;
-       int ret;
 
        if (!perf_event_is_tracing(event))
-               return perf_event_set_bpf_handler(event, prog_fd);
+               return perf_event_set_bpf_handler(event, prog, bpf_cookie);
 
        is_kprobe = event->tp_event->flags & TRACE_EVENT_FL_UKPROBE;
        is_tracepoint = event->tp_event->flags & TRACE_EVENT_FL_TRACEPOINT;
@@ -10048,41 +10088,27 @@ static int perf_event_set_bpf_prog(struct perf_event *event, u32 prog_fd)
                /* bpf programs can only be attached to u/kprobe or tracepoint */
                return -EINVAL;
 
-       prog = bpf_prog_get(prog_fd);
-       if (IS_ERR(prog))
-               return PTR_ERR(prog);
-
        if ((is_kprobe && prog->type != BPF_PROG_TYPE_KPROBE) ||
            (is_tracepoint && prog->type != BPF_PROG_TYPE_TRACEPOINT) ||
-           (is_syscall_tp && prog->type != BPF_PROG_TYPE_TRACEPOINT)) {
-               /* valid fd, but invalid bpf program type */
-               bpf_prog_put(prog);
+           (is_syscall_tp && prog->type != BPF_PROG_TYPE_TRACEPOINT))
                return -EINVAL;
-       }
 
        /* Kprobe override only works for kprobes, not uprobes. */
        if (prog->kprobe_override &&
-           !(event->tp_event->flags & TRACE_EVENT_FL_KPROBE)) {
-               bpf_prog_put(prog);
+           !(event->tp_event->flags & TRACE_EVENT_FL_KPROBE))
                return -EINVAL;
-       }
 
        if (is_tracepoint || is_syscall_tp) {
                int off = trace_event_get_offsets(event->tp_event);
 
-               if (prog->aux->max_ctx_offset > off) {
-                       bpf_prog_put(prog);
+               if (prog->aux->max_ctx_offset > off)
                        return -EACCES;
-               }
        }
 
-       ret = perf_event_attach_bpf_prog(event, prog);
-       if (ret)
-               bpf_prog_put(prog);
-       return ret;
+       return perf_event_attach_bpf_prog(event, prog, bpf_cookie);
 }
 
-static void perf_event_free_bpf_prog(struct perf_event *event)
+void perf_event_free_bpf_prog(struct perf_event *event)
 {
        if (!perf_event_is_tracing(event)) {
                perf_event_free_bpf_handler(event);
@@ -10101,12 +10127,13 @@ static void perf_event_free_filter(struct perf_event *event)
 {
 }
 
-static int perf_event_set_bpf_prog(struct perf_event *event, u32 prog_fd)
+int perf_event_set_bpf_prog(struct perf_event *event, struct bpf_prog *prog,
+                           u64 bpf_cookie)
 {
        return -ENOENT;
 }
 
-static void perf_event_free_bpf_prog(struct perf_event *event)
+void perf_event_free_bpf_prog(struct perf_event *event)
 {
 }
 #endif /* CONFIG_EVENT_TRACING */
@@ -10222,7 +10249,7 @@ static void perf_event_addr_filters_apply(struct perf_event *event)
                return;
 
        if (ifh->nr_file_filters) {
-               mm = get_task_mm(event->ctx->task);
+               mm = get_task_mm(task);
                if (!mm)
                        goto restart;