kprobes: Free kretprobe_instance with RCU callback

Free kretprobe_instance with RCU callback instead of directly
freeing the object in the kretprobe handler context.

This will make kretprobe run safer in NMI context.

Signed-off-by: Masami Hiramatsu <mhiramat@kernel.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Link: https://lore.kernel.org/r/159870616685.1229682.11978742048709542226.stgit@devnote2

diff --git a/include/linux/kprobes.h b/include/linux/kprobes.h
index 72142ae..3389067 100644 (file)
--- a/include/linux/kprobes.h
+++ b/include/linux/kprobes.h
@@ -156,7 +156,10 @@ struct kretprobe {
 };
 
 struct kretprobe_instance {
-       struct hlist_node hlist;
+       union {
+               struct hlist_node hlist;
+               struct rcu_head rcu;
+       };
        struct kretprobe *rp;
        kprobe_opcode_t *ret_addr;
        struct task_struct *task;
@@ -395,7 +398,6 @@ int register_kretprobes(struct kretprobe **rps, int num);
 void unregister_kretprobes(struct kretprobe **rps, int num);
 
 void kprobe_flush_task(struct task_struct *tk);
-void recycle_rp_inst(struct kretprobe_instance *ri, struct hlist_head *head);
 
 int disable_kprobe(struct kprobe *kp);
 int enable_kprobe(struct kprobe *kp);

diff --git a/kernel/kprobes.c b/kernel/kprobes.c
index 2111382..0676868 100644 (file)
--- a/kernel/kprobes.c
+++ b/kernel/kprobes.c
@@ -1223,8 +1223,7 @@ void kprobes_inc_nmissed_count(struct kprobe *p)
 }
 NOKPROBE_SYMBOL(kprobes_inc_nmissed_count);
 
-void recycle_rp_inst(struct kretprobe_instance *ri,
-                    struct hlist_head *head)
+static void recycle_rp_inst(struct kretprobe_instance *ri)
 {
        struct kretprobe *rp = ri->rp;
 
@@ -1236,8 +1235,7 @@ void recycle_rp_inst(struct kretprobe_instance *ri,
                hlist_add_head(&ri->hlist, &rp->free_instances);
                raw_spin_unlock(&rp->lock);
        } else
-               /* Unregistering */
-               hlist_add_head(&ri->hlist, head);
+               kfree_rcu(ri, rcu);
 }
 NOKPROBE_SYMBOL(recycle_rp_inst);
 
@@ -1313,7 +1311,7 @@ void kprobe_busy_end(void)
 void kprobe_flush_task(struct task_struct *tk)
 {
        struct kretprobe_instance *ri;
-       struct hlist_head *head, empty_rp;
+       struct hlist_head *head;
        struct hlist_node *tmp;
        unsigned long hash, flags = 0;
 
@@ -1323,19 +1321,14 @@ void kprobe_flush_task(struct task_struct *tk)
 
        kprobe_busy_begin();
 
-       INIT_HLIST_HEAD(&empty_rp);
        hash = hash_ptr(tk, KPROBE_HASH_BITS);
        head = &kretprobe_inst_table[hash];
        kretprobe_table_lock(hash, &flags);
        hlist_for_each_entry_safe(ri, tmp, head, hlist) {
                if (ri->task == tk)
-                       recycle_rp_inst(ri, &empty_rp);
+                       recycle_rp_inst(ri);
        }
        kretprobe_table_unlock(hash, &flags);
-       hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) {
-               hlist_del(&ri->hlist);
-               kfree(ri);
-       }
 
        kprobe_busy_end();
 }
@@ -1936,13 +1929,12 @@ unsigned long __kretprobe_trampoline_handler(struct pt_regs *regs,
                                             void *frame_pointer)
 {
        struct kretprobe_instance *ri = NULL, *last = NULL;
-       struct hlist_head *head, empty_rp;
+       struct hlist_head *head;
        struct hlist_node *tmp;
        unsigned long flags;
        kprobe_opcode_t *correct_ret_addr = NULL;
        bool skipped = false;
 
-       INIT_HLIST_HEAD(&empty_rp);
        kretprobe_hash_lock(current, &head, &flags);
 
        /*
@@ -2011,7 +2003,7 @@ unsigned long __kretprobe_trampoline_handler(struct pt_regs *regs,
                        __this_cpu_write(current_kprobe, prev);
                }
 
-               recycle_rp_inst(ri, &empty_rp);
+               recycle_rp_inst(ri);
 
                if (ri == last)
                        break;
@@ -2019,11 +2011,6 @@ unsigned long __kretprobe_trampoline_handler(struct pt_regs *regs,
 
        kretprobe_hash_unlock(current, &flags);
 
-       hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) {
-               hlist_del(&ri->hlist);
-               kfree(ri);
-       }
-
        return (unsigned long)correct_ret_addr;
 }
 NOKPROBE_SYMBOL(__kretprobe_trampoline_handler)